import { useState, useMemo, useEffect, useRef } from "react"; import { useSQLQuery, useDiveState } from "@motherduck/react-sql-query"; import { AreaChart, Area, BarChart, Bar, XAxis, YAxis, ReferenceArea, CartesianGrid, Tooltip, Legend, ResponsiveContainer } from "recharts"; const N = (v) => (v != null ? Number(v) : 0); const MD = { bg: "#0d0d1a", surface: "#1a1a2e", surface2: "#22223a", border: "#2d2d4a", yellow: "#FFC907", text: "#ffffff", textMuted: "#8b8ba8", textDim: "#5a5a7a", }; const PALETTE = [ "#FFC907","#7c6ff7","#2dd4bf","#f97316", "#e879f9","#4ade80","#fb7185","#60a5fa", "#facc15","#a78bfa","#34d399","#fb923c", "#f0abfc","#86efac","#fda4af","#93c5fd", ]; const CAT_LABELS = { "cs":"Computer Science","math":"Mathematics","physics":"Physics", "astro-ph":"Astrophysics","cond-mat":"Condensed Matter","quant-ph":"Quantum Physics", "hep-ph":"High Energy Physics","hep-th":"HEP – Theory","stat":"Statistics", "q-bio":"Quantitative Biology","math-ph":"Math Physics","gr-qc":"General Relativity", "eess":"Elec. Eng. & Sys. Sci.","econ":"Economics","hep-ex":"HEP – Experiment", "hep-lat":"HEP – Lattice","nlin":"Nonlinear Sciences","nucl-th":"Nuclear Theory", "nucl-ex":"Nuclear Experiment","q-fin":"Quantitative Finance", }; const catLabel = (k) => CAT_LABELS[k] || k; const TABLE = '"arXiv_metadata".main.trainof'; const MIN_YEAR = 2007; const MAX_YEAR = 2023; const MONTHS = ["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"]; const DAYS = ["Sun","Mon","Tue","Wed","Thu","Fri","Sat"]; const shimmerCSS = `@keyframes shimmer{0%{background-position:200% 0}100%{background-position:-200% 0}}`; function Skeleton({ height=240, width="100%" }) { return

; } function Pill({ active, onClick, children, accent }) { return ( ); } function SectionHeader({ title, subtitle }) { return (

{title}

{subtitle &&

{subtitle}

}

); } function CustomTooltip({ active, payload, label:yr }) { if (!active||!payload?.length) return null; const sorted=[...payload].filter(p=>N(p.value)>0).sort((a,b)=>N(b.value)-N(a.value)).slice(0,10); return (

{yr}

{sorted.map(p=>(

{p.dataKey} {N(p.value).toLocaleString()}

))}

); } // ── Co-occurrence Network (pure SVG, no extra lib) ─────────────────────────── function CooccurrenceNetwork({ data, loading }) { const [hovered, setHovered] = useState(null); const [tooltip, setTooltip] = useState(null); const { nodes, edges } = useMemo(() => { if (!data?.length) return { nodes:[], edges:[] }; // Count node totals const nodeTotals = {}; for (const r of data) { nodeTotals[r.group_a] = (nodeTotals[r.group_a]||0) + N(r.co_count); nodeTotals[r.group_b] = (nodeTotals[r.group_b]||0) + N(r.co_count); } const nodeKeys = Object.keys(nodeTotals).sort((a,b)=>nodeTotals[b]-nodeTotals[a]); const maxTotal = Math.max(...Object.values(nodeTotals)); // Place nodes in a circle const cx=300, cy=220, r=160; const nodes = nodeKeys.map((key,i) => { const angle = (i/nodeKeys.length)*2*Math.PI - Math.PI/2; return { key, x: cx + r*Math.cos(angle), y: cy + r*Math.sin(angle), size: 4 + 14*(nodeTotals[key]/maxTotal), color: PALETTE[nodeKeys.indexOf(key)%PALETTE.length], total: nodeTotals[key], }; }); const nodeMap = Object.fromEntries(nodes.map(n=>[n.key,n])); const maxEdge = Math.max(...data.map(r=>N(r.co_count))); const edges = data.map(r=>({ a: nodeMap[r.group_a], b: nodeMap[r.group_b], count: N(r.co_count), width: 0.5 + 4*(N(r.co_count)/maxEdge), opacity: 0.15 + 0.6*(N(r.co_count)/maxEdge), })).filter(e=>e.a&&e.b); return { nodes, edges }; }, [data]); if (loading) return ; if (!nodes.length) return null; return ( ); } // ── Publication Heatmap ────────────────────────────────────────────────────── function PublicationHeatmap({ data, loading, year }) { const grid = useMemo(() => { // grid[month][dow] = count const g = Array.from({length:12},()=>Array(7).fill(0)); for (const r of (data||[])) { const m = N(r.month)-1; const d = N(r.dow); if (m>=0&&m<12&&d>=0&&d<7) g[m][d]=N(r.papers); } return g; }, [data]); const maxVal = useMemo(()=>Math.max(...(data||[]).map(r=>N(r.papers)),1),[data]); const [hov, setHov] = useState(null); if (loading) return ; const cellW=38, cellH=28, padL=36, padT=28; return ( ); } // ── Main component ─────────────────────────────────────────────────────────── export default function ArxivDive() { const [chartType, setChartType] = useDiveState("chartType","area"); const [normalize, setNormalize] = useDiveState("normalize",false); const [selectedGroup, setSelectedGroup] = useDiveState("selectedGroup",null); const [yearFrom, setYearFrom] = useDiveState("yearFrom",MIN_YEAR); const [yearTo, setYearTo] = useDiveState("yearTo",MAX_YEAR); const [tableSortCol, setTableSortCol] = useDiveState("tableSortCol","total_papers"); const [tableSortDir, setTableSortDir] = useDiveState("tableSortDir","desc"); const [brushStart, setBrushStart] = useState(null); const [brushEnd, setBrushEnd] = useState(null); const [isDragging, setIsDragging] = useState(false); const yFrom = N(yearFrom)||MIN_YEAR; const yTo = N(yearTo)||MAX_YEAR; const isSingleYear = yFrom === yTo; const isZoomed = yFrom!==MIN_YEAR||yTo!==MAX_YEAR; // ── Queries ───────────────────────────────────────────────────────────────── const kpiQ = useSQLQuery(` SELECT COUNT(*) AS total_papers, COUNT(DISTINCT SPLIT_PART(TRIM(categories),' ',1)) AS unique_subcategories, COUNT(DISTINCT REGEXP_REPLACE(SPLIT_PART(TRIM(categories),' ',1),'\\..+','')) AS unique_groups, MIN(YEAR(update_date)) AS first_year, MAX(YEAR(update_date)) AS last_year FROM ${TABLE} WHERE categories IS NOT NULL AND YEAR(update_date) BETWEEN ${yFrom} AND ${yTo} `); const trendQ = useSQLQuery(` SELECT YEAR(update_date)::VARCHAR AS year, REGEXP_REPLACE(SPLIT_PART(TRIM(categories),' ',1),'\\..+','') AS cat_group, COUNT(*) AS papers FROM ${TABLE} WHERE YEAR(update_date) BETWEEN ${MIN_YEAR} AND ${MAX_YEAR} AND categories IS NOT NULL GROUP BY year, cat_group ORDER BY year, papers DESC `); const allSubQ = useSQLQuery(` SELECT SPLIT_PART(TRIM(categories),' ',1) AS subcategory, REGEXP_REPLACE(SPLIT_PART(TRIM(categories),' ',1),'\\..+','') AS cat_group, COUNT(*) AS total_papers, MIN(YEAR(update_date)) AS first_year, MAX(YEAR(update_date)) AS last_year FROM ${TABLE} WHERE categories IS NOT NULL AND YEAR(update_date) BETWEEN ${yFrom} AND ${yTo} GROUP BY subcategory, cat_group ORDER BY total_papers DESC `); // Co-occurrence — filtered by zoom range const coocQ = useSQLQuery(` WITH exploded AS ( SELECT id, REGEXP_REPLACE(UNNEST(STRING_SPLIT(TRIM(categories),' ')), '\\..+','') AS grp FROM ${TABLE} WHERE categories IS NOT NULL AND ARRAY_LENGTH(STRING_SPLIT(TRIM(categories),' '))>1 AND YEAR(update_date) BETWEEN ${yFrom} AND ${yTo} ) SELECT a.grp AS group_a, b.grp AS group_b, COUNT(DISTINCT a.id) AS co_count FROM exploded a JOIN exploded b ON a.id=b.id AND a.grp[...new Set(trendRows.map(r=>r.cat_group))].sort(),[trendRows]); const byYear = useMemo(()=>{ const m={}; for (const r of trendRows){ if (!m[r.year]) m[r.year]={year:r.year}; m[r.year][r.cat_group]=N(r.papers); } return Object.values(m).sort((a,b)=>a.year.localeCompare(b.year)); },[trendRows]); const chartData = useMemo(()=>{ if (!normalize) return byYear; return byYear.map(row=>{ const total=allGroups.reduce((s,g)=>s+N(row[g]),0)||1; const out={year:row.year}; allGroups.forEach(g=>{out[g]=+((N(row[g])/total)*100).toFixed(2);}); return out; }); },[byYear,allGroups,normalize]); const allSubs = Array.isArray(allSubQ.data)?allSubQ.data:[]; const sortedSubs = useMemo(()=>[...allSubs].sort((a,b)=>{ const dir=tableSortDir==="asc"?1:-1; if (tableSortCol==="subcategory") return dir*a.subcategory.localeCompare(b.subcategory); if (tableSortCol==="total_papers") return dir*(N(a.total_papers)-N(b.total_papers)); if (tableSortCol==="cat_group") return dir*a.cat_group.localeCompare(b.cat_group); return 0; }),[allSubs,tableSortCol,tableSortDir]); const kpi = Array.isArray(kpiQ.data)?kpiQ.data[0]:null; const yFmt = v=>normalize?`${v}%`:v>=1000?`${(v/1000).toFixed(0)}k`:v; // ── Brush ───────────────────────────────────────────────────────────────── const handleMouseDown = e=>{ if(e?.activeLabel){setBrushStart(e.activeLabel);setBrushEnd(null);setIsDragging(true);} }; const handleMouseMove = e=>{ if(isDragging&&e?.activeLabel) setBrushEnd(e.activeLabel); }; const handleMouseUp = ()=>{ if(isDragging&&brushStart&&brushEnd){ const a=Math.min(N(brushStart),N(brushEnd)), b=Math.max(N(brushStart),N(brushEnd)); if(a!==b){setYearFrom(a);setYearTo(b);} } setIsDragging(false);setBrushStart(null);setBrushEnd(null); }; const resetZoom=()=>{setYearFrom(MIN_YEAR);setYearTo(MAX_YEAR);}; const SortTh=({col,children,align="left"})=>{ const active=tableSortCol===col; return ( { if(active) setTableSortDir(tableSortDir==="asc"?"desc":"asc"); else{setTableSortCol(col);setTableSortDir("desc");} }} style={{textAlign:align,padding:"6px 10px",color:active?MD.yellow:MD.textMuted,fontWeight:600, fontSize:11,textTransform:"uppercase",letterSpacing:".05em",cursor:"pointer",userSelect:"none",whiteSpace:"nowrap"}}> {children} {active?(tableSortDir==="asc"?"↑":"↓"):↕} ); }; return (

{/* Header */}

arXiv

The Shape of Science

{/* KPIs */}

{[ {label:"Total Papers", val:kpi?N(kpi.total_papers).toLocaleString():null}, {label:"Sub-categories", val:kpi?N(kpi.unique_subcategories).toLocaleString():null}, {label:"Discipline Groups", val:kpi?N(kpi.unique_groups).toLocaleString():null}, {label:"Period", val:kpi?`${N(kpi.first_year)} – ${N(kpi.last_year)}`:null}, ].map(({label,val},i)=>(

{kpiQ.isLoading? :

{val}

}

{label}{isZoomed&&i===3? ●:""}

))}

{/* Trend chart */}

Papers per year by discipline

{isZoomed ? <>Zoomed: {yFrom}{isSingleYear?"":` – ${yTo}`} · Reset : "Click & drag on chart to zoom · updates all panels below"}

{isZoomed&&✕ Reset zoom} setChartType("area")}>Stacked setChartType("bar")}>Grouped setNormalize(!normalize)} accent="#7c6ff7">% Share

{trendQ.isLoading?:(

{chartType==="area"?( }/> {v}} wrapperStyle={{cursor:"pointer",fontSize:11}} onClick={e=>setSelectedGroup(e.dataKey===selectedGroup?null:e.dataKey)}/> {allGroups.map((g,i)=>( ))} {isDragging&&brushStart&&brushEnd&&( )} {isZoomed&&!isDragging&&( )} ):( }/> {v}} wrapperStyle={{cursor:"pointer",fontSize:11}} onClick={e=>setSelectedGroup(e.dataKey===selectedGroup?null:e.dataKey)}/> {allGroups.map((g,i)=>( ))} {isDragging&&brushStart&&brushEnd&&( )} {isZoomed&&!isDragging&&( )} )}

)}

{/* ── Co-occurrence network ── */}

{/* ── Publication heatmap ── */}

{isSingleYear?( <> ):(

📅

Submission Heatmap

Zoom to a single year on the chart above to see day-of-week × month submission patterns

)}

{/* ── Subcategories table ── */}

All subcategories {isZoomed&& · {yFrom}{isSingleYear?"":` – ${yTo}`}} · {sortedSubs.length} total

Click column headers to sort

{allSubQ.isLoading?(

{[...Array(8)].map((_,i)=>)}

):(

SubcategoryDisciplinePapers {sortedSubs.map((row,i)=>{ const active=selectedGroup===row.cat_group; const gi=allGroups.indexOf(row.cat_group); const rc=gi>=0?PALETTE[gi%PALETTE.length]:MD.yellow; return ( setSelectedGroup(row.cat_group===selectedGroup?null:row.cat_group)} style={{borderBottom:`1px solid ${MD.border}`,background:active?`${rc}18`:"transparent",cursor:"pointer"}} > ); })}

#	Active
{i+1}	e.stopPropagation()}> {row.subcategory}	{catLabel(row.cat_group)}	{N(row.total_papers).toLocaleString()}	{N(row.first_year)} – {N(row.last_year)}

)}

Click any row to highlight that discipline · Subcategory names link to arXiv listings

); }

arXiv: The Shape of Science

Comments