// Agent observability Dive: OpenTelemetry GenAI traces of a multi-agent system, from runs to // tool calls to business outcomes. Adapted from the /try LLM observability Dive // (motherduck-website, packages/try-motherduck/observability/dive.tsx). Reads DuckGen's // insurance-brightholm-agent-ops release: synthetic data, Oct 2025 to Jun 2026. import { createContext, useContext, useEffect, useState, type ReactNode } from "react"; import { useDiveState, useSQLQuery } from "@motherduck/react-sql-query"; import { Activity, ArrowLeft, Bot, Briefcase, Check, Clock, Coins, EyeOff, Gauge, LayoutDashboard, ListTree, Moon, Search, Sparkles, Sun, Target, Users, Wrench, X, } from "lucide-react"; import { Area, AreaChart, Bar, BarChart, CartesianGrid, Line, LineChart, ResponsiveContainer, Tooltip, XAxis, YAxis, } from "recharts"; // DuckGen release 20261103-4637fcbb-m-r1 (insurance-brightholm-agent-ops). Point this at another // database or share with the same tables (agent_traces, spans, cases, human_handoffs, tools) to // reuse the Dive; keep the alias. export const REQUIRED_DATABASES = [{ type: "share", path: "md:_share/dg_insurance_brightholm_agent_ops_m_1e02507e_r1/838d91b9-658d-4f11-ae51-d5beddb91950", alias: "insurance_brightholm_agent_ops" }]; // --------------------------------------------------------------------------------------- // SQL. Every query is built here from the global filters. const DB = "insurance_brightholm_agent_ops"; const T = `"${DB}"."main"`; type Range = "7d" | "30d" | "90d" | "all"; type Sort = "newest" | "slowest" | "costliest" | "failed"; type Facets = { outcome: string[]; model: string[]; prompt: string[]; workflow: string[] }; type Filters = { range: Range; agent: string }; type Params = Filters & { facets?: Facets; search?: string; sort?: Sort; traceId?: string; caseId?: string; tool?: string; metric?: string }; const q = (v: string) => `'${v.replace(/'/g, "''")}'`; const inList = (col: string, values: string[]) => (values.length ? `${col} IN (${values.map(q).join(", ")})` : null); const and = (parts: (string | null | false | undefined)[]) => parts.filter(Boolean).join(" AND ") || "TRUE"; // Ranges end at the last trace in the data, not at now(): a recorded or synthetic dataset stops // at a fixed date, and a now()-relative window would show nothing. const DAYS: Record = { "7d": 7, "30d": 30, "90d": 90, all: null }; const ANCHOR = `(SELECT max(started_at) FROM ${T}."agent_traces")`; const WIDTH: Record = { "7d": "INTERVAL 6 HOUR", "30d": "INTERVAL 1 DAY", "90d": "INTERVAL 1 DAY", all: "INTERVAL 7 DAY" }; const LABEL: Record = { "7d": "%b %d %Hh", "30d": "%b %d", "90d": "%b %d", all: "%b %d" }; function traceWhere(p: Filters, alias = ""): string { const a = alias ? `${alias}.` : ""; const days = DAYS[p.range]; return and([days ? `${a}started_at > ${ANCHOR} - INTERVAL ${days} DAY` : null, p.agent !== "all" && `${a}agent_name = ${q(p.agent)}`]); } // Spans carry their agent and start time, so span queries filter without joining traces. function spanWhere(p: Filters): string { const days = DAYS[p.range]; return and([days ? `start_time > ${ANCHOR} - INTERVAL ${days} DAY` : null, p.agent !== "all" && `gen_ai_agent_name = ${q(p.agent)}`]); } // One label per agent run. The root span's status, then the business outcome, then tool errors // the run recovered from. const OUTCOME = `CASE WHEN root_status_code = 'ERROR' THEN 'error' WHEN outcome = 'handed_off' THEN 'handed off' WHEN outcome = 'abandoned' THEN 'abandoned' WHEN tool_error_count > 0 THEN 'tool errors' ELSE 'completed' END`; const AGENT_COST = `(llm_cost_micros + tool_cost_micros) / 1e6`; // A tool call that failed without an error: the span is OK, the result is not a success. const SILENT = `(status_code <> 'ERROR' AND tool_result_status NOT IN ('success', 'none'))`; // A gapless time axis over the filtered traces in CTE t: adds `spine(b)`. const spine = (p: Filters) => ` w AS (SELECT min(started_at) AS lo, max(started_at) AS hi FROM t), spine AS (SELECT unnest(generate_series(time_bucket(${WIDTH[p.range]}, lo), time_bucket(${WIDTH[p.range]}, hi), ${WIDTH[p.range]})) AS b FROM w)`; function traceFilter(p: Params, skip?: keyof Facets): string { const f = p.facets ?? { outcome: [], model: [], prompt: [], workflow: [] }; const s = (p.search ?? "").trim(); return and([ traceWhere(p), skip !== "outcome" && inList(OUTCOME, f.outcome), skip !== "model" && inList("model", f.model), skip !== "prompt" && inList("prompt_version", f.prompt), skip !== "workflow" && inList("workflow", f.workflow), s ? `(trace_id::VARCHAR LIKE ${q(`${s}%`)} OR case_id::VARCHAR = ${q(s)})` : null, ]); } const SORT: Record = { newest: "started_at DESC", slowest: "duration_ms DESC", costliest: "llm_cost_micros + tool_cost_micros DESC", failed: "(root_status_code = 'ERROR') DESC, tool_error_count DESC, started_at DESC", }; const queryNames = [ "meta", "agents", "models", "overview.kpis", "overview.spark", "overview.series", "overview.models", "overview.observations", "overview.tools", "overview.failures", "overview.handoffs", "traces.facets", "traces.activity", "traces.histogram", "traces.list", "trace.detail", "trace.spans", "trace.journey", "tools.series", "tools.table", "tools.calls", "outcomes.kpis", "outcomes.stp", "outcomes.cost", "outcomes.handoffs", ] as const; type QueryName = (typeof queryNames)[number]; function sql(name: QueryName, p: Params): string { const tw = traceWhere(p); const sw = spanWhere(p); const fmt = LABEL[p.range]; switch (name) { // The data window, so the header can say which dates a range covers. case "meta": return `SELECT strftime(min(started_at), '%b %d, %Y') AS first_day, strftime(max(started_at), '%b %d, %Y') AS last_day, strftime(${p.range === "all" ? "min(started_at)" : `max(started_at) - INTERVAL ${DAYS[p.range]} DAY`}, '%b %d, %Y') AS range_start FROM ${T}."agent_traces"`; case "agents": return `SELECT agent_name AS agent, count(*) AS n FROM ${T}."agent_traces" GROUP BY ALL ORDER BY n DESC`; // Fixed model order, so a model keeps its color across tabs and filters. case "models": return `SELECT model AS m FROM ${T}."agent_traces" GROUP BY ALL ORDER BY count(*) DESC`; case "overview.kpis": return `SELECT count(*) AS traces, sum(tool_call_count) AS tool_calls, avg(span_count) AS avg_spans, sum(${AGENT_COST}) AS cost, avg(${AGENT_COST}) AS avg_cost, sum(human_cost_cents) / 100 AS human_cost, quantile_cont(duration_ms, 0.5) AS p50, quantile_cont(duration_ms, 0.95) AS p95, avg((outcome = 'handed_off')::INT) AS handoff_rate, count(*) FILTER (WHERE outcome = 'handed_off') AS handoffs, avg((tool_error_count > 0 OR root_status_code = 'ERROR')::INT) AS error_rate, sum(tool_error_count) AS tool_errors, count(*) FILTER (WHERE root_status_code = 'ERROR') AS failed_traces FROM ${T}."agent_traces" WHERE ${tw}`; case "overview.spark": return `WITH t AS (SELECT * FROM ${T}."agent_traces" WHERE ${tw}), ${spine(p)}, agg AS (SELECT time_bucket(${WIDTH[p.range]}, started_at) AS b, count(*) AS traces, sum(${AGENT_COST}) AS cost, quantile_cont(duration_ms, 0.5) AS p50, avg((outcome = 'handed_off')::INT) AS handoff_rate, avg((tool_error_count > 0 OR root_status_code = 'ERROR')::INT) AS error_rate FROM t GROUP BY ALL) SELECT strftime(s.b, '${fmt}') AS label, coalesce(a.traces, 0) AS traces, coalesce(a.cost, 0) AS cost, a.p50, a.handoff_rate, a.error_rate FROM spine s LEFT JOIN agg a USING (b) ORDER BY s.b`; case "overview.series": return `WITH t AS (SELECT * FROM ${T}."agent_traces" WHERE ${tw}), ${spine(p)}, m AS (SELECT DISTINCT model FROM t), agg AS (SELECT time_bucket(${WIDTH[p.range]}, started_at) AS b, model, count(*) AS traces, sum(${AGENT_COST}) AS cost, sum(total_input_tokens + total_output_tokens) AS tokens, quantile_cont(duration_ms, 0.5) AS p50, quantile_cont(duration_ms, 0.95) AS p95 FROM t GROUP BY ALL) SELECT strftime(s.b, '${fmt}') AS label, m.model, coalesce(a.traces, 0) AS traces, coalesce(a.cost, 0) AS cost, coalesce(a.tokens, 0) AS tokens, a.p50, a.p95 FROM spine s CROSS JOIN m LEFT JOIN agg a ON a.b = s.b AND a.model = m.model ORDER BY s.b, m.model`; // Per model: usage and cost from traces, and where a run's time goes. A root span lasts // exactly as long as its children, so tool time is run time minus model (chat) time. case "overview.models": return `WITH t AS ( SELECT model, count(*) AS traces, sum(total_input_tokens) AS tin, sum(total_output_tokens) AS tout, sum(${AGENT_COST}) AS cost, avg(${AGENT_COST}) AS avg_cost, avg(duration_ms) AS run_ms, avg(retry_count) AS retries FROM ${T}."agent_traces" WHERE ${tw} GROUP BY ALL), c AS (SELECT gen_ai_request_model AS model, sum(duration_ms) AS chat_ms, count(*) AS steps FROM ${T}."spans" WHERE ${sw} AND gen_ai_operation_name = 'chat' GROUP BY ALL) SELECT t.*, c.steps / t.traces AS steps_per_trace, c.chat_ms / t.traces AS model_ms, greatest(t.run_ms - c.chat_ms / t.traces, 0) AS tool_ms FROM t LEFT JOIN c USING (model) ORDER BY t.traces DESC`; case "overview.observations": return `WITH t AS (SELECT * FROM ${T}."agent_traces" WHERE ${tw}), ${spine(p)}, agg AS (SELECT time_bucket(${WIDTH[p.range]}, start_time) AS b, count(*) FILTER (WHERE gen_ai_operation_name = 'chat') AS chats, count(*) FILTER (WHERE gen_ai_operation_name = 'execute_tool' AND status_code <> 'ERROR') AS tools, count(*) FILTER (WHERE gen_ai_operation_name = 'execute_tool' AND status_code = 'ERROR') AS errors FROM ${T}."spans" WHERE ${sw} AND parent_span_id <> 0 GROUP BY ALL) SELECT strftime(s.b, '${fmt}') AS label, coalesce(a.chats, 0) AS chats, coalesce(a.tools, 0) AS tools, coalesce(a.errors, 0) AS errors FROM spine s LEFT JOIN agg a USING (b) ORDER BY s.b`; case "overview.tools": return `SELECT gen_ai_tool_name AS tool, count(*) AS calls, avg((status_code = 'ERROR')::INT) AS error_rate, avg(${SILENT}::INT) AS silent_rate, quantile_cont(duration_ms, 0.95) AS p95 FROM ${T}."spans" WHERE ${sw} AND gen_ai_operation_name = 'execute_tool' GROUP BY ALL ORDER BY error_rate + silent_rate DESC, calls DESC`; // Thrown errors (status ERROR, by error.type) next to silent failures (status OK, by result). case "overview.failures": return `SELECT CASE WHEN status_code = 'ERROR' THEN 'error' ELSE 'silent' END AS kind, CASE WHEN status_code = 'ERROR' THEN error_type ELSE tool_result_status END AS label, count(*) AS n, mode(gen_ai_tool_name) AS top_tool, count(DISTINCT gen_ai_tool_name) AS tools FROM ${T}."spans" WHERE ${sw} AND gen_ai_operation_name = 'execute_tool' AND (status_code = 'ERROR' OR ${SILENT}) GROUP BY ALL ORDER BY n DESC`; case "overview.handoffs": return `SELECT h.reason, count(*) AS n, quantile_cont(h.wait_minutes, 0.5) AS wait_p50, sum(h.loaded_cost_cents) / 100 AS human_cost FROM ${T}."human_handoffs" h JOIN ${T}."agent_traces" t USING (trace_id) WHERE ${traceWhere(p, "t")} GROUP BY ALL ORDER BY n DESC`; // Facet counts ignore their own facet's selection, so a picked value never hides the others. case "traces.facets": { const facet = (key: keyof Facets, col: string) => `SELECT '${key}' AS facet, ${col} AS value, count(*) AS n FROM ${T}."agent_traces" WHERE ${traceFilter(p, key)} GROUP BY ALL`; return `${facet("outcome", OUTCOME)} UNION ALL ${facet("model", "model")} UNION ALL ${facet("prompt", "prompt_version")} UNION ALL ${facet("workflow", "workflow")} ORDER BY facet, n DESC`; } case "traces.activity": return `WITH t AS (SELECT * FROM ${T}."agent_traces" WHERE ${traceFilter(p)}), ${spine(p)}, agg AS (SELECT time_bucket(${WIDTH[p.range]}, started_at) AS b, count(*) AS traces, count(*) FILTER (WHERE tool_error_count > 0 OR root_status_code = 'ERROR') AS errors, count(*) FILTER (WHERE outcome = 'handed_off') AS handoffs, sum(${AGENT_COST}) AS cost FROM t GROUP BY ALL) SELECT strftime(s.b, '${fmt}') AS label, coalesce(a.traces, 0) AS traces, coalesce(a.errors, 0) AS errors, coalesce(a.handoffs, 0) AS handoffs, coalesce(a.cost, 0) AS cost FROM spine s LEFT JOIN agg a USING (b) ORDER BY s.b`; case "traces.histogram": return `WITH bins AS (SELECT unnest(range(0, 46, 2)) AS lo), t AS (SELECT least(floor(duration_ms / 2000) * 2, 44)::INT AS lo, count(*) AS n FROM ${T}."agent_traces" WHERE ${traceFilter(p)} GROUP BY ALL) SELECT bins.lo, coalesce(t.n, 0) AS n FROM bins LEFT JOIN t USING (lo) ORDER BY bins.lo`; case "traces.list": { const order = SORT[p.sort ?? "newest"]; return `WITH t AS (SELECT *, ${OUTCOME} AS status FROM ${T}."agent_traces" WHERE ${traceFilter(p)} ORDER BY ${order} LIMIT 100), f AS (SELECT trace_id, count(*) FILTER (WHERE status_code = 'ERROR') AS failed_calls, arg_min(gen_ai_tool_name, start_time) FILTER (WHERE status_code = 'ERROR') AS error_tool, arg_min(error_type, start_time) FILTER (WHERE status_code = 'ERROR') AS error_type, arg_min(gen_ai_tool_name || ' → ' || tool_result_status, start_time) FILTER (WHERE ${SILENT}) AS silent FROM ${T}."spans" WHERE trace_id IN (SELECT trace_id FROM t) AND gen_ai_operation_name = 'execute_tool' GROUP BY ALL), h AS (SELECT trace_id, reason, queue FROM ${T}."human_handoffs" WHERE trace_id IN (SELECT trace_id FROM t)) SELECT t.trace_id::VARCHAR AS trace_id, t.case_id::VARCHAR AS case_id, strftime(t.started_at, '%b %d, %H:%M:%S') AS ts, t.agent_name, t.workflow, t.model, t.prompt_version, t.duration_ms, t.span_count, t.tool_call_count, t.tool_error_count, t.retry_count, t.total_input_tokens, t.total_output_tokens, (t.llm_cost_micros + t.tool_cost_micros) / 1e6 AS cost_usd, t.human_cost_cents / 100 AS human_usd, t.status, f.failed_calls, f.error_tool, f.error_type, f.silent, h.reason AS handoff_reason, h.queue AS handoff_queue FROM t LEFT JOIN f USING (trace_id) LEFT JOIN h USING (trace_id) ORDER BY ${order}`; } case "trace.detail": return `SELECT t.trace_id::VARCHAR AS trace_id, t.case_id::VARCHAR AS case_id, strftime(t.started_at, '%Y-%m-%d %H:%M:%S UTC') AS ts, t.agent_name, t.workflow, t.model, t.prompt_version, t.duration_ms, t.span_count, t.tool_call_count, t.tool_error_count, t.retry_count, t.total_input_tokens, t.total_output_tokens, t.llm_cost_micros / 1e6 AS llm_usd, t.tool_cost_micros / 1e6 AS tool_usd, t.human_cost_cents / 100 AS human_usd, t.root_status_code, t.outcome, ${OUTCOME.replace(/root_status_code|outcome|tool_error_count/g, (m) => `t.${m}`)} AS status, c.case_type, c.line, c.loss_type, c.multi_vehicle, c.intake_channel, c.region_id, c.segment, strftime(c.opened_at, '%Y-%m-%d %H:%M') AS opened_at, CASE WHEN year(c.closed_at) = 9999 THEN NULL ELSE strftime(c.closed_at, '%Y-%m-%d %H:%M') END AS closed_at, c.final_status, c.siu_referred, c.reopened, c.complaint_filed, c.claim_estimate_cents / 100 AS claim_estimate_usd, c.claim_paid_cents / 100 AS claim_paid_usd, c.quoted_premium_cents / 100 AS quoted_premium_usd, c.bound, h.queue AS handoff_queue, h.reason AS handoff_reason, h.wait_minutes AS handoff_wait_minutes, h.handling_minutes AS handoff_handling_minutes, h.resolution AS handoff_resolution FROM ${T}."agent_traces" t LEFT JOIN ${T}."cases" c USING (case_id) LEFT JOIN ${T}."human_handoffs" h ON h.trace_id = t.trace_id WHERE t.trace_id = ${Number(p.traceId) || 0}`; case "trace.spans": return `WITH s AS (SELECT * FROM ${T}."spans" WHERE trace_id = ${Number(p.traceId) || 0}), r AS (SELECT min(start_time) AS t0 FROM s) SELECT span_id::VARCHAR AS span_id, parent_span_id::VARCHAR AS parent_id, span_name, gen_ai_operation_name AS op, gen_ai_agent_name AS agent, gen_ai_request_model AS model, gen_ai_tool_name AS tool, gen_ai_tool_call_id AS call_id, epoch_ms(start_time) - epoch_ms(r.t0) AS offset_ms, duration_ms, strftime(start_time, '%H:%M:%S.%g') AS started, status_code, error_type, tool_result_status, input_tokens, output_tokens, attempt_number, llm_cost_micros / 1e6 AS llm_usd, tool_cost_micros / 1e6 AS tool_usd FROM s, r ORDER BY start_time, span_id`; // Every agent run of the same business case, in order: intake, triage, settlement, ... case "trace.journey": return `SELECT trace_id::VARCHAR AS trace_id, agent_name, model, strftime(started_at, '%b %d, %H:%M') AS ts, duration_ms, ${OUTCOME} AS status, (llm_cost_micros + tool_cost_micros) / 1e6 AS cost_usd, human_cost_cents / 100 AS human_usd FROM ${T}."agent_traces" WHERE case_id = ${Number(p.caseId) || 0} ORDER BY started_at`; // Tool health over time: the six tools that score worst on the chosen metric in this range. case "tools.series": { const metric = p.metric === "silent" ? `avg(${SILENT}::INT)` : p.metric === "p95" ? "quantile_cont(duration_ms, 0.95) / 1000" : "avg((status_code = 'ERROR')::INT)"; return `WITH t AS (SELECT * FROM ${T}."agent_traces" WHERE ${tw}), ${spine(p)}, s AS (SELECT * FROM ${T}."spans" WHERE ${sw} AND gen_ai_operation_name = 'execute_tool'), top AS (SELECT gen_ai_tool_name AS tool FROM s GROUP BY ALL ORDER BY ${metric} DESC, count(*) DESC LIMIT 6), agg AS (SELECT time_bucket(${WIDTH[p.range]}, start_time) AS b, gen_ai_tool_name AS tool, ${metric} AS v FROM s WHERE gen_ai_tool_name IN (SELECT tool FROM top) GROUP BY ALL) SELECT strftime(sp.b, '${fmt}') AS label, top.tool, a.v FROM spine sp CROSS JOIN top LEFT JOIN agg a ON a.b = sp.b AND a.tool = top.tool ORDER BY sp.b, top.tool`; } case "tools.table": return `SELECT s.gen_ai_tool_name AS tool, any_value(k.backing_system) AS backing_system, any_value(k.is_third_party) AS third_party, any_value(k.owner_team) AS owner_team, any_value(k.timeout_ms) AS timeout_ms, count(*) AS calls, avg((s.status_code = 'ERROR')::INT) AS error_rate, avg(${SILENT.replace(/status_code|tool_result_status/g, (m) => `s.${m}`)}::INT) AS silent_rate, count(*) FILTER (WHERE s.attempt_number > 1) AS retries, quantile_cont(s.duration_ms, 0.5) AS p50, quantile_cont(s.duration_ms, 0.95) AS p95, sum(s.tool_cost_micros) / 1e6 AS cost, mode(s.error_type) FILTER (WHERE s.status_code = 'ERROR') AS top_error FROM ${T}."spans" s LEFT JOIN ${T}."tools" k ON k.tool_name = s.gen_ai_tool_name WHERE ${sw.replace(/start_time|gen_ai_agent_name/g, (m) => `s.${m}`)} AND s.gen_ai_operation_name = 'execute_tool' GROUP BY ALL ORDER BY calls DESC`; // The latest failed or silently failed calls of one tool, to open their traces. case "tools.calls": return `SELECT trace_id::VARCHAR AS trace_id, case_id::VARCHAR AS case_id, gen_ai_agent_name AS agent, strftime(start_time, '%b %d, %H:%M:%S') AS ts, duration_ms, status_code, error_type, tool_result_status, attempt_number FROM ${T}."spans" WHERE ${sw} AND gen_ai_tool_name = ${q(p.tool ?? "")} AND (status_code = 'ERROR' OR ${SILENT}) ORDER BY start_time DESC LIMIT 25`; // Business outcomes of the cases these runs worked on (cases opened in the range). The agent // filter keeps the cases that agent touched. case "outcomes.kpis": return `WITH c AS (${outcomeCases(p)}), cost AS (SELECT sum(t.llm_cost_micros) / 1e6 AS llm, sum(t.tool_cost_micros) / 1e6 AS tool, sum(t.human_cost_cents) / 100 AS human FROM ${T}."agent_traces" t WHERE t.case_id IN (SELECT case_id FROM c)) SELECT count(*) AS cases, avg((final_status = 'resolved_stp')::INT) FILTER (WHERE final_status <> 'open') AS stp_rate, count(*) FILTER (WHERE final_status IN ('resolved_stp', 'resolved_with_human')) AS resolved, avg((handoff_queue <> 'none')::INT) AS handoff_rate, any_value(cost.llm) AS llm, any_value(cost.tool) AS tool, any_value(cost.human) AS human FROM c, cost`; case "outcomes.stp": return `WITH c AS (${outcomeCases(p)}), t AS (SELECT opened_at AS started_at FROM c), ${spine(p)}, agg AS (SELECT time_bucket(${WIDTH[p.range]}, opened_at) AS b, case_type, avg((final_status = 'resolved_stp')::INT) FILTER (WHERE final_status <> 'open') AS stp FROM c GROUP BY ALL), k AS (SELECT DISTINCT case_type FROM c) SELECT strftime(s.b, '${fmt}') AS label, k.case_type, a.stp FROM spine s CROSS JOIN k LEFT JOIN agg a ON a.b = s.b AND a.case_type = k.case_type ORDER BY s.b, k.case_type`; case "outcomes.cost": return `WITH c AS (${outcomeCases(p)}), t AS (SELECT opened_at AS started_at FROM c), ${spine(p)}, x AS (SELECT time_bucket(${WIDTH[p.range]}, c.opened_at) AS b, sum(t.llm_cost_micros) / 1e6 AS llm, sum(t.tool_cost_micros) / 1e6 AS tool, sum(t.human_cost_cents) / 100 AS human FROM c JOIN ${T}."agent_traces" t USING (case_id) GROUP BY ALL), r AS (SELECT time_bucket(${WIDTH[p.range]}, opened_at) AS b, count(*) FILTER (WHERE final_status IN ('resolved_stp', 'resolved_with_human')) AS resolved FROM c GROUP BY ALL) SELECT strftime(s.b, '${fmt}') AS label, x.llm / nullif(r.resolved, 0) AS llm, x.tool / nullif(r.resolved, 0) AS tool, x.human / nullif(r.resolved, 0) AS human FROM spine s LEFT JOIN x USING (b) LEFT JOIN r USING (b) ORDER BY s.b`; case "outcomes.handoffs": return `WITH c AS (${outcomeCases(p)}) SELECT handoff_reason AS reason, handoff_queue AS queue, count(*) AS n, quantile_cont(handoff_wait_minutes, 0.5) AS wait_p50, quantile_cont(handoff_wait_minutes, 0.9) AS wait_p90, avg((final_status = 'resolved_with_human')::INT) AS resolved_rate, avg(complaint_filed::INT) AS complaint_rate FROM c WHERE handoff_queue <> 'none' GROUP BY ALL ORDER BY n DESC LIMIT 12`; } } function outcomeCases(p: Filters): string { const days = DAYS[p.range]; return `SELECT * FROM ${T}."cases" WHERE ${and([ days ? `opened_at > ${ANCHOR} - INTERVAL ${days} DAY` : null, p.agent !== "all" && `case_id IN (SELECT case_id FROM ${T}."agent_traces" WHERE agent_name = ${q(p.agent)})`, ])}`; } // --------------------------------------------------------------------------------------- // Theme. Colors are inline styles (the Dive runtime has no arbitrary Tailwind values). const LIGHT = { dark: false, bg: "#f8fafc", surface: "#ffffff", subtle: "#f1f5f9", hover: "#f8fafc", border: "#e3e8ef", strong: "#cbd5e1", text: "#0f172a", muted: "#64748b", faint: "#94a3b8", accent: "#0777b3", accentSoft: "rgba(7,119,179,0.09)", ok: "#15803d", okSoft: "rgba(21,128,61,0.10)", err: "#c81e1e", errSoft: "rgba(200,30,30,0.09)", warn: "#b45309", warnSoft: "rgba(180,83,9,0.10)", gen: "#6d3fd6", genSoft: "rgba(109,63,214,0.10)", tool: "#c2610c", toolSoft: "rgba(194,97,12,0.10)", turn: "#0777b3", turnSoft: "rgba(7,119,179,0.10)", human: "#0e9384", grid: "#eef2f6", code: "#f8fafc", shadow: "none", series: ["#0777b3", "#7c4ddb", "#0e9384", "#e18727", "#d6457a", "#64748b", "#65a30d", "#0891b2"], }; type Theme = typeof LIGHT; const DARK: Theme = { dark: true, bg: "#0c0e13", surface: "#13161d", subtle: "#191d26", hover: "#1a1e28", border: "#252a35", strong: "#323846", text: "#e8eaf0", muted: "#8e95a6", faint: "#5d6475", accent: "#4fb0e8", accentSoft: "rgba(79,176,232,0.13)", ok: "#4ade80", okSoft: "rgba(74,222,128,0.12)", err: "#f87171", errSoft: "rgba(248,113,113,0.13)", warn: "#fbbf24", warnSoft: "rgba(251,191,36,0.12)", gen: "#a78bfa", genSoft: "rgba(167,139,250,0.14)", tool: "#f59e0b", toolSoft: "rgba(245,158,11,0.13)", turn: "#4fb0e8", turnSoft: "rgba(79,176,232,0.13)", human: "#2dd4bf", grid: "#1f2430", code: "#0f1218", shadow: "none", series: ["#4fb0e8", "#a78bfa", "#2dd4bf", "#fbbf24", "#f472b6", "#94a3b8", "#a3e635", "#22d3ee"], }; const ThemeContext = createContext(LIGHT); const useTheme = () => useContext(ThemeContext); const FONT = 'Inter, ui-sans-serif, system-ui, -apple-system, "Segoe UI", Roboto, sans-serif'; const MONO = 'ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace'; const NUM = { fontVariantNumeric: "tabular-nums" as const }; // --------------------------------------------------------------------------------------- // Formatting const N = (v: unknown): number => (v == null ? 0 : Number(v)); const S = (v: unknown): string => (v == null ? "" : String(v)); const usd = (v: number): string => v === 0 ? "$0" : v < 0.01 ? `$${v.toFixed(4)}` : v < 1 ? `$${v.toFixed(3)}` : v < 1000 ? `$${v.toFixed(2)}` : usdShort(v); const usdShort = (v: number): string => v >= 1e6 ? `$${(v / 1e6).toFixed(1)}M` : v >= 1e4 ? `$${(v / 1e3).toFixed(1)}k` : v >= 100 ? `$${Math.round(v).toLocaleString()}` : usd(v); const pct = (v: number) => `${(v * 100).toFixed(v < 0.1 ? 1 : 0)}%`; const secs = (ms: number) => (ms < 1000 ? `${Math.round(ms)}ms` : `${(ms / 1000).toFixed(1)}s`); const num = (v: number) => v >= 1e6 ? `${(v / 1e6).toFixed(2)}M` : v >= 1e4 ? `${(v / 1e3).toFixed(1)}k` : Math.round(v).toLocaleString(); const tokens = (v: number) => v >= 1e9 ? `${(v / 1e9).toFixed(1)}B` : v >= 1e6 ? `${(v / 1e6).toFixed(1)}M` : v >= 1000 ? `${(v / 1000).toFixed(1)}k` : String(Math.round(v)); const minutes = (m: number) => (m < 60 ? `${Math.round(m)}m` : m < 2880 ? `${(m / 60).toFixed(1)}h` : `${(m / 1440).toFixed(1)}d`); const agentLabel = (a: string) => a.replace(/_agent$/, "").replace(/_/g, " "); const rows = (query: { data?: unknown }) => (Array.isArray(query.data) ? (query.data as Record[]) : []); const useQ = (name: QueryName, p: Params, enabled = true) => useSQLQuery(sql(name, p), { enabled }); type Tab = "overview" | "traces" | "tools" | "outcomes"; const EMPTY_FACETS: Facets = { outcome: [], model: [], prompt: [], workflow: [] }; // --------------------------------------------------------------------------------------- // Primitives function Skeleton({ h, w }: { h: number; w?: number | string }) { const c = useTheme(); return

; } function ErrorText({ error }: { error: unknown }) { const c = useTheme(); return (

{String(error)}

); } function Card({ title, icon, right, children, flush, note }: { title?: string; icon?: ReactNode; right?: ReactNode; children: ReactNode; flush?: boolean; note?: string }) { const c = useTheme(); return (

{title ? (

{icon ? {icon} : null} {title}

{note ?

{note}

: null}

{right}

) : null}

{children}

); } type Tone = "ok" | "err" | "warn" | "gen" | "tool" | "turn" | "accent" | "neutral"; function Badge({ tone = "neutral", children, mono }: { tone?: Tone; children: ReactNode; mono?: boolean }) { const c = useTheme(); const colors: Record = { ok: [c.ok, c.okSoft], err: [c.err, c.errSoft], warn: [c.warn, c.warnSoft], gen: [c.gen, c.genSoft], tool: [c.tool, c.toolSoft], turn: [c.turn, c.turnSoft], accent: [c.accent, c.accentSoft], neutral: [c.muted, c.subtle], }; const [fg, bg] = colors[tone]; return ( {children} ); } function Dot({ color, size = 8 }: { color: string; size?: number }) { return ; } function Segmented({ value, options, onChange }: { value: V; options: { value: V; label: string; icon?: ReactNode }[]; onChange: (v: V) => void }) { const c = useTheme(); return (

{options.map((o) => { const active = o.value === value; return ( ); })}

); } // Latency chip, colored by how slow it is for its kind of span. function LatencyChip({ ms, kind }: { ms: number; kind: "run" | "chat" | "tool" }) { const [slow, verySlow] = kind === "run" ? [15_000, 25_000] : kind === "chat" ? [5_000, 10_000] : [2_000, 8_000]; const tone: Tone = ms >= verySlow ? "err" : ms >= slow ? "warn" : "ok"; return ( {secs(ms)} ); } // A number with a proportional bar behind it (table cells). function BarCell({ value, max, label, color }: { value: number; max: number; label: string; color?: string }) { const c = useTheme(); const width = max > 0 ? Math.max(2, (value / max) * 100) : 0; return (

{label}

); } // completed · tool errors (recovered) · handed off · abandoned · error (root span ERROR). function StatusDot({ status }: { status: string }) { const c = useTheme(); const failed = status === "error"; const warn = status === "tool errors" || status === "handed off"; const neutral = status === "abandoned"; const color = failed ? c.err : warn ? c.warn : neutral ? c.muted : c.ok; const bg = failed ? c.errSoft : warn ? c.warnSoft : neutral ? c.subtle : c.okSoft; return ( {failed ? : warn || neutral ? : } ); } const statusTone = (s: string): Tone => (s === "error" ? "err" : s === "handed off" || s === "tool errors" ? "warn" : s === "abandoned" ? "neutral" : "ok"); const th = (c: Theme, align: "left" | "right" = "left") => ({ textAlign: align, padding: "8px 12px", fontSize: 11, fontWeight: 500, color: c.muted, background: c.subtle, borderBottom: `1px solid ${c.border}`, whiteSpace: "nowrap" as const, }); const td = (c: Theme, align: "left" | "right" = "left") => ({ textAlign: align, padding: "9px 12px", whiteSpace: "nowrap" as const, fontSize: 12, color: c.text, borderBottom: `1px solid ${c.border}`, verticalAlign: "middle" as const, ...NUM, }); function ChartTooltip({ formatter }: { formatter?: (v: number, name: string) => string }) { const c = useTheme(); return ( (v == null ? "-" : formatter ? formatter(Number(v), String(name)) : String(v))} /> ); } const axis = (c: Theme) => ({ axisLine: false, tickLine: false, fontSize: 11, tick: { fill: c.faint } }); function Legend({ items }: { items: { label: string; color: string }[] }) { const c = useTheme(); return (

{items.map((i) => ( {i.label} ))}

); } function Loading({ q, h, children }: { q: { isLoading?: boolean; isError?: boolean; error?: unknown; data?: unknown }; h: number; children: ReactNode }) { if (q.isLoading) return ; if (q.isError && !q.data) return ; return <>{children}; } // Rows of (label, series key, value) into one chart row per label. function pivot(list: Record[], key: string, value: string, scale = 1) { const keys: string[] = []; const byLabel = new Map>(); for (const r of list) { const label = S(r.label); const k = S(r[key]); if (!keys.includes(k)) keys.push(k); const row = byLabel.get(label) ?? { label }; row[k] = r[value] == null ? null : N(r[value]) * scale; byLabel.set(label, row); } return { keys, data: [...byLabel.values()] }; } // --------------------------------------------------------------------------------------- // Root: header with global filters, tabs; a selected trace replaces the tab content. export default function AgentObservability() { const [themeName, setThemeName] = useDiveState<"light" | "dark">("theme", "light"); const [tab, setTab] = useDiveState("tab", "overview"); // "all" by default: the data is a fixed window, and its stories span months. const [range, setRange] = useDiveState("range", "all"); const [agent, setAgent] = useDiveState("agent", "all"); const [traceId, setTraceId] = useDiveState("trace", null); const c = themeName === "dark" ? DARK : LIGHT; const where: Params = { range, agent }; const modelsQuery = useQ("models", where); const models = rows(modelsQuery).map((r) => S(r.m)); const colorOf = (model: string) => { const i = models.indexOf(model); return c.series[(i < 0 ? models.length : i) % c.series.length]; }; const agents = rows(useQ("agents", where)).map((r) => S(r.agent)); const meta = rows(useQ("meta", where))[0]; const tabs: { value: Tab; label: string; icon: ReactNode }[] = [ { value: "overview", label: "Overview", icon: }, { value: "traces", label: "Traces", icon: }, { value: "tools", label: "Tools", icon: }, { value: "outcomes", label: "Outcomes", icon: }, ]; return (

Agent observability

Brightholm Mutual claims and policy agents · OpenTelemetry GenAI spans · synthetic DuckGen data

{tabs.map((t) => { const active = tab === t.value && !traceId; return ( ); })} {traceId ? ( Trace {traceId} ) : null}

{/* Ranges end at the last trace in the data, so every range has data. */}

{meta ? `${S(meta.range_start)} to ${S(meta.last_day)} · ranges end at the last trace (data: ${S(meta.first_day)} to ${S(meta.last_day)})` : ""}

{traceId ? ( setTraceId(null)} onOpen={setTraceId} colorOf={colorOf} /> ) : tab === "overview" ? ( ) : tab === "traces" ? ( ) : tab === "tools" ? ( ) : ( )}

); } // --------------------------------------------------------------------------------------- // Overview: KPIs with trends, cost and latency by model over time, model table, tools, // failures and handoffs. function Overview({ where, colorOf, onTab }: { where: Params; colorOf: (m: string) => string; onTab: (t: Tab) => void }) { const c = useTheme(); const [costMetric, setCostMetric] = useDiveState<"cost" | "tokens" | "traces">("costMetric", "cost"); const [latencyMetric, setLatencyMetric] = useDiveState<"p50" | "p95">("latencyMetric", "p50"); const kpiQuery = useQ("overview.kpis", where); const k = rows(kpiQuery)[0] ?? {}; const spark = rows(useQ("overview.spark", where)).map((r) => ({ label: S(r.label), traces: N(r.traces), cost: N(r.cost), latency: r.p50 == null ? null : N(r.p50) / 1000, handoffs: r.handoff_rate == null ? null : N(r.handoff_rate), errors: r.error_rate == null ? null : N(r.error_rate), })); const seriesQuery = useQ("overview.series", where); const usage = pivot(rows(seriesQuery), "model", costMetric); const latency = pivot(rows(seriesQuery), "model", latencyMetric, 1 / 1000); const modelQuery = useQ("overview.models", where); const byModel = rows(modelQuery); const maxCost = Math.max(0, ...byModel.map((r) => N(r.cost))); const maxTime = Math.max(0, ...byModel.map((r) => N(r.model_ms) + N(r.tool_ms))); const obsQuery = useQ("overview.observations", where); const obs = rows(obsQuery).map((r) => ({ label: S(r.label), chats: N(r.chats), tools: N(r.tools), errors: N(r.errors) })); const toolsQuery = useQ("overview.tools", where); const failuresQuery = useQ("overview.failures", where); const handoffsQuery = useQ("overview.handoffs", where); const handoffs = rows(handoffsQuery); const handoffTotal = handoffs.reduce((a, r) => a + N(r.n), 0); const kpis: { label: string; icon: ReactNode; value: string; sub: string; key: "traces" | "cost" | "latency" | "handoffs" | "errors"; color: string }[] = [ { label: "Agent runs", icon: , value: num(N(k.traces)), sub: `${num(N(k.tool_calls))} tool calls · ${N(k.avg_spans).toFixed(1)} spans per run`, key: "traces", color: c.accent }, { label: "Agent cost", icon: , value: usdShort(N(k.cost)), sub: `${usd(N(k.avg_cost))} per run · LLM + tools`, key: "cost", color: c.series[1] }, { label: "Run latency · p50", icon: , value: k.p50 == null ? "-" : secs(N(k.p50)), sub: `p95 ${k.p95 == null ? "-" : secs(N(k.p95))}`, key: "latency", color: c.series[2] }, { label: "Handed to a human", icon: , value: pct(N(k.handoff_rate)), sub: `${num(N(k.handoffs))} runs · ${usdShort(N(k.human_cost))} human cost`, key: "handoffs", color: c.series[3] }, { label: "Runs with errors", icon: , value: pct(N(k.error_rate)), sub: `${num(N(k.failed_traces))} failed · ${num(N(k.tool_errors))} failed tool calls`, key: "errors", color: c.err }, ]; const miniTable = (head: string[], body: ReactNode) => ( {head.map((h, i) => ( ))} {body}

{h}

); return (

{kpis.map((kpi) => (

{kpi.icon} {kpi.label}

{kpiQuery.isLoading ? (

) : (

{kpi.value}

)}

{kpi.sub}

{spark.length > 1 ? ( ) : null}

))}

} right={ } > (costMetric === "cost" ? usdShort(Number(v)) : tokens(Number(v)))} /> (costMetric === "cost" ? usd(v) : costMetric === "tokens" ? tokens(v) : num(v))} /> {usage.keys.map((m, i) => ( ))} ({ label: m, color: colorOf(m) }))} /> } right={ } > `${v}s`} /> `${v.toFixed(1)}s`} /> {latency.keys.map((m) => ( ))} ({ label: m, color: colorOf(m) }))} />

} flush> {modelQuery.isLoading ? (

) : modelQuery.isError && !modelQuery.data ? (

) : (

{byModel.map((r) => { const total = N(r.model_ms) + N(r.tool_ms); const part = (v: number) => `${maxTime > 0 ? (v / maxTime) * 100 : 0}%`; return ( ); })}

Model	Runs	Tokens in / out	Steps	$ / run	Total cost	Where a run's time goes
{S(r.model)}	{num(N(r.traces))}	{tokens(N(r.tin))} / {tokens(N(r.tout))}	{N(r.steps_per_trace).toFixed(1)}	{usd(N(r.avg_cost))}		{secs(total)}

)}

model (chat spans) tools (execute_tool spans)

}> tokens(Number(v))} /> num(v)} />

} flush right={ } > {miniTable( ["Tool", "Calls", "Errors", "Silent", "p95"], rows(toolsQuery) .slice(0, 6) .map((r) => ( {S(r.tool)} {num(N(r.calls))} = 0.05 ? c.err : c.text }}>{pct(N(r.error_rate))} = 0.05 ? c.warn : c.text }}>{pct(N(r.silent_rate))} {secs(N(r.p95))} )), )} } flush note="Errors set the span status. Silent failures are OK spans whose result is not a success."> {miniTable( ["Failure", "Mostly", "Calls"], rows(failuresQuery).map((r) => ( {S(r.kind)} {S(r.label)} {S(r.top_tool)} {N(r.tools) > 1 ? ` +${N(r.tools) - 1}` : ""} {num(N(r.n))} )), )} } flush> {miniTable( ["Reason", "Share", "Wait p50", "Human cost"], handoffs.map((r) => ( {S(r.reason).replace(/_/g, " ")} {pct(handoffTotal ? N(r.n) / handoffTotal : 0)} {minutes(N(r.wait_p50))} {usdShort(N(r.human_cost))} )), )}

); } // --------------------------------------------------------------------------------------- // Traces: facet sidebar, activity strip, searchable list. Click a row to open it. const FACET_LABEL: Record = { outcome: "Outcome", model: "Model", prompt: "Prompt version", workflow: "Workflow" }; function Traces({ where, onOpen, colorOf }: { where: Params; onOpen: (id: string) => void; colorOf: (m: string) => string }) { const c = useTheme(); const [sort, setSort] = useDiveState("sort", "newest"); const [facets, setFacets] = useDiveState("facets", EMPTY_FACETS); const [search, setSearch] = useDiveState("search", ""); const [draft, setDraft] = useState(search); useEffect(() => { const timer = setTimeout(() => setSearch(draft.trim()), 300); return () => clearTimeout(timer); }, [draft, setSearch]); const p: Params = { ...where, facets, search, sort }; const facetQuery = useQ("traces.facets", p); const facetRows = rows(facetQuery); const strip = rows(useQ("traces.activity", p)).map((r) => ({ label: S(r.label), traces: N(r.traces), errors: N(r.errors), handoffs: N(r.handoffs), cost: N(r.cost) })); const hist = rows(useQ("traces.histogram", p)).map((r) => ({ label: N(r.lo) >= 44 ? "44s+" : `${N(r.lo)}-${N(r.lo) + 2}s`, n: N(r.n) })); const tracesQuery = useQ("traces.list", p); const list = rows(tracesQuery); const toggle = (facet: keyof Facets, value: string) => setFacets({ ...facets, [facet]: (facets[facet] ?? []).includes(value) ? facets[facet].filter((v) => v !== value) : [...(facets[facet] ?? []), value] }); const activeFilters = Object.values(facets).reduce((a, v) => a + (v ?? []).length, 0); const totals = { traces: strip.reduce((a, r) => a + r.traces, 0), errors: strip.reduce((a, r) => a + r.errors, 0), handoffs: strip.reduce((a, r) => a + r.handoffs, 0), cost: strip.reduce((a, r) => a + r.cost, 0) }; const stripCard = (title: string, total: string, children: ReactNode) => (

{title} {total}

{children}

); const miniBars = (key: string, color: string, fmt?: (v: number) => string, data: Record[] = strip) => ( ); return (

setDraft(e.target.value)} placeholder="Trace id or case id" style={{ background: "transparent", outline: "none", fontSize: 12, color: c.text, width: "100%" }} />

{(Object.keys(FACET_LABEL) as (keyof Facets)[]).map((facet) => { const values = facetRows.filter((r) => S(r.facet) === facet); const max = Math.max(1, ...values.map((r) => N(r.n))); return (

{FACET_LABEL[facet]}

{facetQuery.isLoading ? ( ) : ( values.map((r) => { const value = S(r.value); const checked = (facets[facet] ?? []).includes(value); return ( ); }) )}

); })} {activeFilters ? ( ) : null}

{stripCard("Runs", num(totals.traces), miniBars("traces", c.accent, num))} {stripCard("With errors", num(totals.errors), miniBars("errors", c.err, num))} {stripCard("Duration (2s bins)", "", miniBars("n", c.series[3], (v) => `${num(v)} runs`, hist))} {stripCard( "Agent cost", usdShort(totals.cost), usd(v)} /> , )}

Traces {tracesQuery.isLoading ? "loading..." : `${list.length === 100 ? "top 100" : list.length} of ${num(totals.traces)}`}

{tracesQuery.isLoading ? (

) : tracesQuery.isError && !tracesQuery.data ? (

) : list.length === 0 ? (

No traces match these filters.

) : (

{list.map((r) => { const id = S(r.trace_id); return ( onOpen(id)} className="cursor-pointer" onMouseEnter={(e) => (e.currentTarget.style.background = c.hover)} onMouseLeave={(e) => (e.currentTarget.style.background = "transparent")} > ); })}

	Run	Workflow · what went wrong	Model	Latency	Tokens	Cost	Outcome
	{S(r.ts)} {agentLabel(S(r.agent_name))}	{S(r.workflow).replace(/_/g, " ")} · case {S(r.case_id)}	{S(r.model)} {S(r.prompt_version)} {N(r.span_count)} spans · {N(r.tool_call_count)} tools {N(r.tool_error_count) ? · {N(r.tool_error_count)} failed : null}		{tokens(N(r.total_input_tokens))} → {tokens(N(r.total_output_tokens))}	{usd(N(r.cost_usd))}	{S(r.status)}

)}

); } // Traces table: the run's first failed tool call, its silent failure, or why it was handed off. function ProblemLine({ r }: { r: Record }) { const c = useTheme(); const failed = N(r.failed_calls); const lines: { text: string; color: string; icon: ReactNode }[] = []; if (failed) lines.push({ text: `${S(r.error_tool)}: ${S(r.error_type)}${failed > 1 ? ` · +${failed - 1} more` : ""}`, color: c.err, icon: }); if (S(r.silent)) lines.push({ text: `${S(r.silent)} (status OK)`, color: c.warn, icon: }); if (S(r.handoff_reason)) lines.push({ text: `handed off: ${S(r.handoff_reason).replace(/_/g, " ")} → ${S(r.handoff_queue).replace(/_/g, " ")}`, color: c.muted, icon: }); return ( <> {lines.map((l) => ( {l.icon} {l.text} ))} ); } // --------------------------------------------------------------------------------------- // Trace detail: header, run tree with a latency waterfall, span attributes, the business case // behind the run, and the case's other agent runs. function SpanRow({ c, selected, traceMs, onSelect, id, depth, icon, tone, name, meta, offset, ms, failed, silent, }: { c: Theme; selected: string; traceMs: number; onSelect: (id: string) => void; id: string; depth: number; icon: ReactNode; tone: string; name: ReactNode; meta: ReactNode; offset: number; ms: number; failed?: boolean; silent?: boolean; }) { const active = selected === id; const left = Math.min(100, (Math.max(0, offset) / traceMs) * 100); const width = Math.max(0.8, Math.min(100 - left, (ms / traceMs) * 100)); const bar = failed ? c.err : silent ? c.warn : tone; return (

onSelect(id)} className="flex cursor-pointer items-center" style={{ minHeight: 40, borderBottom: `1px solid ${c.border}`, background: active ? c.accentSoft : undefined, boxShadow: active ? `inset 2px 0 0 ${c.accent}` : undefined }} >

{Array.from({ length: depth }, (_, i) => ( ))} {icon}

{name}

{meta}

{[0.25, 0.5, 0.75].map((f) => (

))}

{secs(ms)}

); } const isSilent = (o: Record) => S(o.op) === "execute_tool" && S(o.status_code) !== "ERROR" && !["success", "none"].includes(S(o.tool_result_status)); function TraceDetail({ traceId, onBack, onOpen, colorOf }: { traceId: string; onBack: () => void; onOpen: (id: string) => void; colorOf: (m: string) => string }) { const c = useTheme(); const [selected, setSelected] = useState("root"); const [panelTab, setPanelTab] = useState<"attributes" | "case">("attributes"); useEffect(() => setSelected("root"), [traceId]); const traceQuery = useQ("trace.detail", { range: "all", agent: "all", traceId }); const t = rows(traceQuery)[0]; const spansQuery = useQ("trace.spans", { range: "all", agent: "all", traceId }); const spans = rows(spansQuery); const root = spans.find((s) => S(s.parent_id) === "0"); const children = spans.filter((s) => S(s.parent_id) !== "0"); const traceMs = Math.max(1, N(root?.duration_ms) || N(t?.duration_ms)); const journeyQuery = useQ("trace.journey", { range: "all", agent: "all", caseId: S(t?.case_id) }, Boolean(t)); const journey = rows(journeyQuery); if (traceQuery.isLoading) return ; if (traceQuery.isError && !traceQuery.data) return ; if (!t) return

Trace not found.

; const current = selected === "root" ? root : (spans.find((s) => S(s.span_id) === selected) ?? root); const failures = children.filter((s) => S(s.status_code) === "ERROR" || isSilent(s)); const stat = (icon: ReactNode, label: string, value: string) => (

{icon} {label} {value}

); return (

invoke_agent {S(t.agent_name)} {S(t.trace_id)} {S(t.status)} status {S(t.root_status_code)} {S(t.workflow).replace(/_/g, " ")} {S(t.ts)}

{stat(, "Model · prompt", `${S(t.model)} · ${S(t.prompt_version)}`)} {stat(, "Latency", secs(N(t.duration_ms)))} {stat(, "Tool calls", N(t.tool_error_count) ? `${N(t.tool_call_count)} (${N(t.tool_error_count)} failed)` : String(N(t.tool_call_count)))} {stat(, "Spans · retries", `${N(t.span_count)} · ${N(t.retry_count)}`)} {stat(, "Tokens in / out", `${tokens(N(t.total_input_tokens))} / ${tokens(N(t.total_output_tokens))}`)} {stat(, "Agent cost", usd(N(t.llm_usd) + N(t.tool_usd)))} {stat(, "Human cost", N(t.human_usd) ? usd(N(t.human_usd)) : "-")}

{failures.length || S(t.handoff_reason) ? (

S(f.status_code) === "ERROR") ? c.errSoft : c.warnSoft, border: `1px solid ${c.border}`, borderRadius: 10, padding: "10px 12px" }}>

{[ failures.filter((f) => S(f.status_code) === "ERROR").length ? `${failures.filter((f) => S(f.status_code) === "ERROR").length} failed tool call(s)` : null, failures.filter(isSilent).length ? `${failures.filter(isSilent).length} silent failure(s)` : null, S(t.handoff_reason) ? `handed off to ${S(t.handoff_queue).replace(/_/g, " ")} (${S(t.handoff_reason).replace(/_/g, " ")}), picked up after ${minutes(N(t.handoff_wait_minutes))}` : null, ] .filter(Boolean) .join(" · ")}

{failures.map((o) => ( ))}

) : null}

Run tree {[0, 0.5, 1].map((f) => ( {secs(traceMs * f)} ))}

{spansQuery.isLoading ? (

) : ( <> {root ? ( } tone={c.turn} name={S(root.span_name)} meta={ <> invoke_agent {tokens(N(t.total_input_tokens) + N(t.total_output_tokens))} tok {usd(N(t.llm_usd) + N(t.tool_usd))} } offset={0} ms={N(root.duration_ms)} failed={S(root.status_code) === "ERROR"} /> ) : null} {children.map((o) => { const isTool = S(o.op) === "execute_tool"; return ( : } tone={isTool ? c.tool : c.gen} failed={S(o.status_code) === "ERROR"} silent={isSilent(o)} name={S(o.span_name)} meta={ isTool ? ( <> {S(o.status_code) === "ERROR" ? S(o.error_type) : S(o.tool_result_status)} {N(o.attempt_number) > 1 ? attempt {N(o.attempt_number)} : null} ) : ( <> chat {tokens(N(o.input_tokens))} → {tokens(N(o.output_tokens))} {usd(N(o.llm_usd))} ) } offset={N(o.offset_ms)} ms={N(o.duration_ms)} /> ); })} )}

{current && S(current.op) === "execute_tool" ? : current && S(current.op) === "chat" ? : } {current ? S(current.span_name) : "-"} {current && S(current.op) === "chat" ? ( {S(current.model)} ) : null}

{panelTab === "attributes" ? ( current ? ( ) : ( ) ) : ( )}

Synthetic traces carry no prompt or completion content, only GenAI semantic-convention attributes.

} flush note={`Every agent run on case ${S(t.case_id)}, in order. Click one to open it.`}> {journeyQuery.isLoading ? (

) : ( {journey.map((j, i) => { const here = S(j.trace_id) === S(t.trace_id); return ( (here ? null : onOpen(S(j.trace_id)))} className={here ? "" : "cursor-pointer"} style={{ background: here ? c.accentSoft : undefined }} onMouseEnter={(e) => (here ? null : (e.currentTarget.style.background = c.hover))} onMouseLeave={(e) => (here ? null : (e.currentTarget.style.background = "transparent"))} > ); })}

{i + 1}

{agentLabel(S(j.agent_name))} {S(j.trace_id)}

{S(j.ts)}

{S(j.model)}

{secs(N(j.duration_ms))}

{usd(N(j.cost_usd))}

{N(j.human_usd) ? +{usd(N(j.human_usd))} human : null}

{S(j.status)}

)}

); } // OpenTelemetry GenAI attribute names (semantic conventions) for one span; sentinels are dropped. function spanAttributes(o: Record): [string, string][] { const op = S(o.op); const entries: [string, string][] = [ ["span.name", S(o.span_name)], ["span.id", S(o.span_id)], ["parent.span.id", S(o.parent_id) === "0" ? "" : S(o.parent_id)], ["start", S(o.started)], ["duration", secs(N(o.duration_ms))], ["otel.status_code", S(o.status_code)], ["gen_ai.operation.name", op], ["gen_ai.agent.name", S(o.agent)], ["gen_ai.request.model", S(o.model)], ["gen_ai.tool.name", S(o.tool)], ["gen_ai.tool.call.id", S(o.call_id)], ["gen_ai.usage.input_tokens", op === "chat" ? String(N(o.input_tokens)) : ""], ["gen_ai.usage.output_tokens", op === "chat" ? String(N(o.output_tokens)) : ""], ["error.type", S(o.error_type)], ["tool.result.status", S(o.tool_result_status)], ["attempt", op === "execute_tool" ? String(N(o.attempt_number)) : ""], ["cost.llm_usd", N(o.llm_usd) ? usd(N(o.llm_usd)) : ""], ["cost.tool_usd", N(o.tool_usd) ? usd(N(o.tool_usd)) : ""], ]; return entries.filter(([, v]) => v !== "" && v !== "none"); } function caseAttributes(t: Record): [string, string][] { const money = (v: unknown) => (N(v) ? usd(N(v)) : ""); const entries: [string, string][] = [ ["case.id", S(t.case_id)], ["case.type", S(t.case_type)], ["line", S(t.line)], ["loss.type", S(t.loss_type)], ["multi_vehicle", t.multi_vehicle ? "true" : ""], ["intake.channel", S(t.intake_channel)], ["region", S(t.region_id)], ["segment", S(t.segment)], ["opened", S(t.opened_at)], ["closed", S(t.closed_at) || "open"], ["final.status", S(t.final_status)], ["claim.estimate", money(t.claim_estimate_usd)], ["claim.paid", money(t.claim_paid_usd)], ["quote.premium", money(t.quoted_premium_usd)], ["bound", t.bound ? "true" : ""], ["siu.referred", t.siu_referred ? "true" : ""], ["reopened", t.reopened ? "true" : ""], ["complaint.filed", t.complaint_filed ? "true" : ""], ["handoff.queue", S(t.handoff_queue)], ["handoff.reason", S(t.handoff_reason)], ["handoff.wait", t.handoff_wait_minutes == null ? "" : minutes(N(t.handoff_wait_minutes))], ["handoff.handling", t.handoff_handling_minutes == null ? "" : minutes(N(t.handoff_handling_minutes))], ["handoff.resolution", S(t.handoff_resolution)], ]; return entries.filter(([, v]) => v !== "" && v !== "none"); } function KeyValues({ entries }: { entries: [string, string][] }) { const c = useTheme(); return (

{entries.map(([k, v]) => (

{k} {v}

))}

); } // --------------------------------------------------------------------------------------- // Tools: failure and latency trends per tool, the catalog with health, and failing calls. function Tools({ where, onOpen }: { where: Params; onOpen: (id: string) => void }) { const c = useTheme(); const [metric, setMetric] = useDiveState<"error" | "silent" | "p95">("toolMetric", "error"); const [tool, setTool] = useDiveState("tool", "vehicle_valuation"); const seriesQuery = useQ("tools.series", { ...where, metric }); const series = pivot(rows(seriesQuery), "tool", "v"); const tableQuery = useQ("tools.table", where); const table = rows(tableQuery); const callsQuery = useQ("tools.calls", { ...where, tool }); const calls = rows(callsQuery); const fmt = (v: number) => (metric === "p95" ? `${v.toFixed(1)}s` : pct(v)); return (

} note={ metric === "silent" ? "Share of calls whose span is OK but whose result is not a success. Status codes alone miss these." : "The six tools that score worst on this metric in the range." } right={ } > fmt(Number(v))} /> fmt(v)} /> {series.keys.map((k, i) => ( ))} ({ label: k, color: c.series[i % c.series.length] }))} />

} flush note="Click a tool to list its latest failing calls.">

{table.map((r) => { const active = S(r.tool) === tool; return ( setTool(S(r.tool))} className="cursor-pointer" style={{ background: active ? c.accentSoft : undefined, boxShadow: active ? `inset 2px 0 0 ${c.accent}` : undefined }} > ); })}

Tool	Calls	Errors	Silent	Retries	p50 / p95	Timeout	Cost
{S(r.tool)} {S(r.backing_system).replace(/_/g, " ")} {r.third_party ? " · third party" : ""}	{num(N(r.calls))}	= 0.05 ? c.err : c.text }} title={S(r.top_error)}> {pct(N(r.error_rate))}	= 0.05 ? c.warn : c.text }}>{pct(N(r.silent_rate))}	{num(N(r.retries))}	{secs(N(r.p50))} / = N(r.timeout_ms) * 0.8 ? c.err : c.text }}>{secs(N(r.p95))}	{secs(N(r.timeout_ms))}	{usdShort(N(r.cost))}

} flush note="Thrown errors and silent failures, newest first. Click one to open its trace."> {calls.length === 0 ? (

No failing calls for this tool in the range.

) : ( {calls.map((r, i) => ( onOpen(S(r.trace_id))} className="cursor-pointer" onMouseEnter={(e) => (e.currentTarget.style.background = c.hover)} onMouseLeave={(e) => (e.currentTarget.style.background = "transparent")} > ))}

{S(r.ts)}

{agentLabel(S(r.agent))}

{S(r.status_code) === "ERROR" ? S(r.error_type) : S(r.tool_result_status)}

try {N(r.attempt_number)}

Open →

)}

); } // --------------------------------------------------------------------------------------- // Outcomes: what the runs did for the business. Straight-through processing, cost per resolved // case with human handling counted, and where handoffs go. const CASE_TYPES: Record = { claim_fnol: "claim (FNOL)", new_policy_quote: "new policy quote", endorsement: "endorsement", billing_inquiry: "billing inquiry", cancellation: "cancellation", }; function Outcomes({ where }: { where: Params }) { const c = useTheme(); const [costView, setCostView] = useDiveState<"all" | "agent">("costView", "all"); const kpiQuery = useQ("outcomes.kpis", where); const k = rows(kpiQuery)[0] ?? {}; const stpQuery = useQ("outcomes.stp", where); const stp = pivot(rows(stpQuery), "case_type", "stp"); const costQuery = useQ("outcomes.cost", where); const cost = rows(costQuery).map((r) => ({ label: S(r.label), llm: r.llm == null ? null : N(r.llm), tool: r.tool == null ? null : N(r.tool), human: r.human == null ? null : N(r.human) })); const handoffsQuery = useQ("outcomes.handoffs", where); const resolved = N(k.resolved); const agentCost = N(k.llm) + N(k.tool); const kpis = [ { label: "Cases", value: num(N(k.cases)), sub: `${num(resolved)} resolved` }, { label: "Straight-through", value: pct(N(k.stp_rate)), sub: "of closed cases, no human touch" }, { label: "Handed to a human", value: pct(N(k.handoff_rate)), sub: "of cases" }, { label: "Cost per resolved case", value: usd(resolved ? (agentCost + N(k.human)) / resolved : 0), sub: `agent ${usd(resolved ? agentCost / resolved : 0)} · human ${usd(resolved ? N(k.human) / resolved : 0)}` }, ]; return (

{kpis.map((kpi) => (

{kpiQuery.isLoading ? :

{kpi.value}

}

{kpi.label}

{kpi.sub}

))}

} note="Closed cases resolved with no handoff, by the week or day they opened."> `${Math.round(Number(v) * 100)}%`} /> pct(v)} /> {stp.keys.map((key, i) => ( ))} ({ label: CASE_TYPES[key] ?? key, color: c.series[i % c.series.length] }))} /> } note={costView === "all" ? "Human handling dominates: agent cost is cents, a handoff is dollars." : "LLM and tool spend only, per resolved case."} right={ } > usd(Number(v))} /> usd(v)} /> {costView === "all" ? : null}

} flush note="Cases handed to a human, by reason and queue. Before/after comparisons in this data are observational.">

{rows(handoffsQuery).map((r) => ( ))}

Reason	Queue	Cases	Wait p50 / p90	Resolved by human	Complaints
{S(r.reason).replace(/_/g, " ")}	{S(r.queue).replace(/_/g, " ")}	{num(N(r.n))}	{minutes(N(r.wait_p50))} / {minutes(N(r.wait_p90))}	{pct(N(r.resolved_rate))}	= 0.05 ? c.warn : c.text }}>{pct(N(r.complaint_rate))}

); }

Agent observability

Comments