diff --git a/app/(dashboard)/DashboardLayoutClient.tsx b/app/(dashboard)/DashboardLayoutClient.tsx index bb399379..7cac90e3 100644 --- a/app/(dashboard)/DashboardLayoutClient.tsx +++ b/app/(dashboard)/DashboardLayoutClient.tsx @@ -27,6 +27,7 @@ import ShieldIcon from "@mui/icons-material/Shield"; import SettingsIcon from "@mui/icons-material/Settings"; import HistoryIcon from "@mui/icons-material/History"; import LogoutIcon from "@mui/icons-material/Logout"; +import BarChartIcon from "@mui/icons-material/BarChart"; type User = { id: string; @@ -37,6 +38,7 @@ type User = { const NAV_ITEMS = [ { href: "/", label: "Overview", icon: DashboardIcon }, + { href: "/analytics", label: "Analytics", icon: BarChartIcon }, { href: "/proxy-hosts", label: "Proxy Hosts", icon: DnsIcon }, { href: "/access-lists", label: "Access Lists", icon: SecurityIcon }, { href: "/certificates", label: "Certificates", icon: ShieldIcon }, diff --git a/app/(dashboard)/OverviewClient.tsx b/app/(dashboard)/OverviewClient.tsx index 1dd8bcdd..98def3b6 100644 --- a/app/(dashboard)/OverviewClient.tsx +++ b/app/(dashboard)/OverviewClient.tsx @@ -17,13 +17,20 @@ type RecentEvent = { created_at: string; }; +type TrafficSummary = { + totalRequests: number; + blockedPercent: number; +} | null; + export default function OverviewClient({ userName, stats, + trafficSummary, recentEvents }: { userName: string; stats: StatCard[]; + trafficSummary: TrafficSummary; recentEvents: RecentEvent[]; }) { return ( @@ -90,6 +97,51 @@ export default function OverviewClient({ ))} + + {/* Traffic (24h) card */} + + + + + + + + + + {trafficSummary ? ( + <> + + {trafficSummary.totalRequests.toLocaleString()} + + + Traffic (24h) + {trafficSummary.totalRequests > 0 && ( + 0 ? "error.light" : "text.secondary", fontSize: "0.8em" }}> + · {trafficSummary.blockedPercent}% blocked + + )} + + + ) : ( + <> + — + Traffic (24h) + + )} + + + + diff --git a/app/(dashboard)/analytics/AnalyticsClient.tsx b/app/(dashboard)/analytics/AnalyticsClient.tsx new file mode 100644 index 00000000..1ad7b23f --- /dev/null +++ b/app/(dashboard)/analytics/AnalyticsClient.tsx @@ -0,0 +1,488 @@ +'use client'; + +import { useState, useEffect, useCallback } from 'react'; +import dynamic from 'next/dynamic'; +import Link from 'next/link'; +import { + Alert, + Box, + Card, + CardContent, + CircularProgress, + FormControl, + Grid, + MenuItem, + Pagination, + Paper, + Select, + SelectChangeEvent, + Stack, + Table, + TableBody, + TableCell, + TableHead, + TableRow, + ToggleButton, + ToggleButtonGroup, + Typography, +} from '@mui/material'; +import type { ApexOptions } from 'apexcharts'; + +// ── Dynamic imports (browser-only) ──────────────────────────────────────────── + +const ReactApexChart = dynamic(() => import('react-apexcharts'), { ssr: false }); + +const WorldMap = dynamic(() => import('./WorldMapInner'), { + ssr: false, + loading: () => ( + + + + ), +}); + +// ── Types (mirrored from analytics-db — can't import server-only code) ──────── + +type Interval = '24h' | '7d' | '30d'; + +interface AnalyticsSummary { + totalRequests: number; + uniqueIps: number; + blockedRequests: number; + blockedPercent: number; + bytesServed: number; + loggingDisabled: boolean; +} + +interface TimelineBucket { ts: number; total: number; blocked: number; } +interface CountryStats { countryCode: string; total: number; blocked: number; } +interface ProtoStats { proto: string; count: number; percent: number; } +interface UAStats { userAgent: string; count: number; percent: number; } + +interface BlockedEvent { + id: number; ts: number; clientIp: string; countryCode: string | null; + method: string; uri: string; status: number; host: string; +} +interface BlockedPage { events: BlockedEvent[]; total: number; page: number; pages: number; } + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +function countryFlag(code: string): string { + if (!code || code.length !== 2) return '🌐'; + return String.fromCodePoint(...[...code.toUpperCase()].map(c => 0x1F1E6 + c.charCodeAt(0) - 65)); +} + +function parseUA(ua: string): string { + if (!ua) return 'Unknown'; + if (/Googlebot/i.test(ua)) return 'Googlebot'; + if (/bingbot/i.test(ua)) return 'Bingbot'; + if (/DuckDuckBot/i.test(ua)) return 'DuckDuckBot'; + if (/curl/i.test(ua)) return 'curl'; + if (/python-requests|Python\//i.test(ua)) return 'Python'; + if (/Go-http-client/i.test(ua)) return 'Go'; + if (/wget/i.test(ua)) return 'wget'; + if (/Edg\//i.test(ua)) return 'Edge'; + if (/OPR\//i.test(ua)) return 'Opera'; + if (/SamsungBrowser/i.test(ua)) return 'Samsung Browser'; + if (/Chrome\//i.test(ua)) return 'Chrome'; + if (/Firefox\//i.test(ua)) return 'Firefox'; + if (/Safari\//i.test(ua)) return 'Safari'; + return ua.substring(0, 32); +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(1)} MB`; + return `${(bytes / 1024 / 1024 / 1024).toFixed(2)} GB`; +} + +function formatTs(ts: number, interval: Interval): string { + const d = new Date(ts * 1000); + if (interval === '24h') return d.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }); + if (interval === '7d') return d.toLocaleDateString([], { weekday: 'short' }) + ' ' + d.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }); + return d.toLocaleDateString([], { month: 'short', day: 'numeric' }); +} + +const DARK_CHART: ApexOptions = { + chart: { background: 'transparent', toolbar: { show: false }, animations: { enabled: false } }, + theme: { mode: 'dark' }, + grid: { borderColor: 'rgba(255,255,255,0.06)' }, + tooltip: { theme: 'dark' }, +}; + +// ── Stat card ───────────────────────────────────────────────────────────────── + +function StatCard({ label, value, sub, color }: { label: string; value: string; sub?: string; color?: string }) { + return ( + + + + {label} + + + {value} + + {sub && {sub}} + + + ); +} + +// ── Main component ──────────────────────────────────────────────────────────── + +export default function AnalyticsClient() { + const [interval, setIntervalVal] = useState('24h'); + const [host, setHost] = useState('all'); + const [hosts, setHosts] = useState([]); + + const [summary, setSummary] = useState(null); + const [timeline, setTimeline] = useState([]); + const [countries, setCountries] = useState([]); + const [protocols, setProtocols] = useState([]); + const [userAgents, setUserAgents] = useState([]); + const [blocked, setBlocked] = useState(null); + const [loading, setLoading] = useState(true); + + // Fetch hosts once + useEffect(() => { + fetch('/api/analytics/hosts').then(r => r.json()).then(setHosts).catch(() => {}); + }, []); + + // Fetch all analytics data when interval or host changes + useEffect(() => { + setLoading(true); + const params = `?interval=${interval}&host=${encodeURIComponent(host)}`; + Promise.all([ + fetch(`/api/analytics/summary${params}`).then(r => r.json()), + fetch(`/api/analytics/timeline${params}`).then(r => r.json()), + fetch(`/api/analytics/countries${params}`).then(r => r.json()), + fetch(`/api/analytics/protocols${params}`).then(r => r.json()), + fetch(`/api/analytics/user-agents${params}`).then(r => r.json()), + fetch(`/api/analytics/blocked${params}&page=1`).then(r => r.json()), + ]).then(([s, t, c, p, u, b]) => { + setSummary(s); + setTimeline(t); + setCountries(c); + setProtocols(p); + setUserAgents(u); + setBlocked(b); + }).catch(() => {}).finally(() => setLoading(false)); + }, [interval, host]); + + const fetchBlockedPage = useCallback((page: number) => { + const params = `?interval=${interval}&host=${encodeURIComponent(host)}&page=${page}`; + fetch(`/api/analytics/blocked${params}`).then(r => r.json()).then(setBlocked).catch(() => {}); + }, [interval, host]); + + // ── Chart configs ───────────────────────────────────────────────────────── + + const timelineLabels = timeline.map(b => formatTs(b.ts, interval)); + const timelineOptions: ApexOptions = { + ...DARK_CHART, + chart: { ...DARK_CHART.chart, type: 'area', stacked: true, id: 'timeline' }, + colors: ['#3b82f6', '#ef4444'], + fill: { type: 'gradient', gradient: { shadeIntensity: 1, opacityFrom: 0.45, opacityTo: 0.05 } }, + stroke: { curve: 'smooth', width: 2 }, + dataLabels: { enabled: false }, + xaxis: { categories: timelineLabels, labels: { rotate: 0, style: { colors: '#94a3b8', fontSize: '11px' } }, axisBorder: { show: false }, axisTicks: { show: false } }, + yaxis: { labels: { style: { colors: '#94a3b8' } } }, + legend: { labels: { colors: '#94a3b8' } }, + tooltip: { theme: 'dark', shared: true, intersect: false }, + }; + const timelineSeries = [ + { name: 'Allowed', data: timeline.map(b => b.total - b.blocked) }, + { name: 'Blocked', data: timeline.map(b => b.blocked) }, + ]; + + const donutOptions: ApexOptions = { + ...DARK_CHART, + chart: { ...DARK_CHART.chart, type: 'donut', id: 'protocols' }, + colors: ['#3b82f6', '#8b5cf6', '#06b6d4', '#f59e0b'], + labels: protocols.map(p => p.proto), + legend: { position: 'bottom', labels: { colors: '#94a3b8' } }, + dataLabels: { style: { colors: ['#fff'] } }, + plotOptions: { pie: { donut: { size: '65%' } } }, + }; + const donutSeries = protocols.map(p => p.count); + + const uaNames = userAgents.map(u => parseUA(u.userAgent)); + const barOptions: ApexOptions = { + ...DARK_CHART, + chart: { ...DARK_CHART.chart, type: 'bar', id: 'ua' }, + colors: ['#7f5bff'], + plotOptions: { bar: { horizontal: true, borderRadius: 4 } }, + dataLabels: { enabled: false }, + xaxis: { categories: uaNames, labels: { style: { colors: '#94a3b8', fontSize: '12px' } } }, + yaxis: { labels: { style: { colors: '#94a3b8', fontSize: '12px' } } }, + }; + const barSeries = [{ name: 'Requests', data: userAgents.map(u => u.count) }]; + + // ── Render ──────────────────────────────────────────────────────────────── + + return ( + + {/* Header */} + + + + Traffic Intelligence + + + Analytics + + + + { if (v) setIntervalVal(v); }} + > + 24h + 7d + 30d + + + + + + + + {/* Logging disabled alert */} + {summary?.loggingDisabled && ( + + Caddy access logging is not enabled — no traffic data is being collected.{' '} + Enable logging in Settings. + + )} + + {/* Loading overlay */} + {loading && ( + + + + )} + + {!loading && summary && ( + <> + {/* Stats row */} + + + + + + + + + 0 ? '#ef4444' : undefined} + /> + + + 10 ? '#f59e0b' : undefined} + /> + + + + {/* Timeline */} + + + + Requests Over Time + + {timeline.length === 0 ? ( + No data for this period + ) : ( + + )} + + + + {/* World map + Countries */} + + + + + + Traffic by Country + + + + + + + + + + Top Countries + + {countries.length === 0 ? ( + No geo data available + ) : ( + + + + Country + Requests + Blocked + + + + {countries.slice(0, 10).map(c => ( + + + + {countryFlag(c.countryCode)} + {c.countryCode} + + + + {c.total.toLocaleString()} + + + 0 ? 'error.light' : 'text.secondary'}> + {c.blocked.toLocaleString()} + + + + ))} + +
+ )} + + + + + + {/* Protocols + User Agents */} + + + + + + HTTP Protocols + + {protocols.length === 0 ? ( + No data + ) : ( + <> + + + + {protocols.map(p => ( + + {p.proto} + {p.count.toLocaleString()} + {p.percent}% + + ))} + +
+ + )} + + + + + + + + Top User Agents + + {userAgents.length === 0 ? ( + No data + ) : ( + + )} + + + + + + {/* Recent Blocked Requests */} + + + + Recent Blocked Requests + + {!blocked || blocked.events.length === 0 ? ( + + No blocked requests in this period + + ) : ( + <> + + + + {['Time', 'IP', 'Country', 'Host', 'Method', 'URI', 'Status'].map(h => ( + {h} + ))} + + + + {blocked.events.map(ev => ( + + + + {new Date(ev.ts * 1000).toLocaleString()} + + + {ev.clientIp} + + + {ev.countryCode ? `${countryFlag(ev.countryCode)} ${ev.countryCode}` : '—'} + + + + {ev.host || '—'} + + {ev.method} + + {ev.uri} + + + {ev.status} + + + ))} + +
+ {blocked.pages > 1 && ( + + fetchBlockedPage(p)} + color="primary" + size="small" + /> + + )} + + )} + + + + )} + + ); +} diff --git a/app/(dashboard)/analytics/WorldMapInner.tsx b/app/(dashboard)/analytics/WorldMapInner.tsx new file mode 100644 index 00000000..3a37f67a --- /dev/null +++ b/app/(dashboard)/analytics/WorldMapInner.tsx @@ -0,0 +1,117 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { geoNaturalEarth1, geoPath } from 'd3-geo'; +import { feature } from 'topojson-client'; +import type { Topology, GeometryCollection } from 'topojson-specification'; +import { Box, CircularProgress } from '@mui/material'; + +const WORLD_ATLAS = 'https://cdn.jsdelivr.net/npm/world-atlas@2/countries-110m.json'; + +// ISO 3166-1 alpha-2 → numeric +const A2N: Record = { + AF:'4',AL:'8',DZ:'12',AD:'20',AO:'24',AG:'28',AR:'32',AM:'51', + AU:'36',AT:'40',AZ:'31',BS:'44',BH:'48',BD:'50',BB:'52',BY:'112', + BE:'56',BZ:'84',BJ:'204',BT:'64',BO:'68',BA:'70',BW:'72',BR:'76', + BN:'96',BG:'100',BF:'854',BI:'108',CV:'132',KH:'116',CM:'120', + CA:'124',CF:'140',TD:'148',CL:'152',CN:'156',CO:'170',KM:'174', + CG:'178',CD:'180',CR:'188',CI:'384',HR:'191',CU:'192',CY:'196', + CZ:'203',DK:'208',DJ:'262',DM:'212',DO:'214',EC:'218',EG:'818', + SV:'222',GQ:'226',ER:'232',EE:'233',SZ:'748',ET:'231',FJ:'242', + FI:'246',FR:'250',GA:'266',GM:'270',GE:'268',DE:'276',GH:'288', + GR:'300',GD:'308',GT:'320',GN:'324',GW:'624',GY:'328',HT:'332', + HN:'340',HU:'348',IS:'352',IN:'356',ID:'360',IR:'364',IQ:'368', + IE:'372',IL:'376',IT:'380',JM:'388',JP:'392',JO:'400',KZ:'398', + KE:'404',KI:'296',KP:'408',KR:'410',KW:'414',KG:'417',LA:'418', + LV:'428',LB:'422',LS:'426',LR:'430',LY:'434',LI:'438',LT:'440', + LU:'442',MG:'450',MW:'454',MY:'458',MV:'462',ML:'466',MT:'470', + MH:'584',MR:'478',MU:'480',MX:'484',FM:'583',MD:'498',MC:'492', + MN:'496',ME:'499',MA:'504',MZ:'508',MM:'104',NA:'516',NR:'520', + NP:'524',NL:'528',NZ:'554',NI:'558',NE:'562',NG:'566',NO:'578', + OM:'512',PK:'586',PW:'585',PA:'591',PG:'598',PY:'600',PE:'604', + PH:'608',PL:'616',PT:'620',QA:'634',RO:'642',RU:'643',RW:'646', + KN:'659',LC:'662',VC:'670',WS:'882',SM:'674',ST:'678',SA:'682', + SN:'686',RS:'688',SC:'690',SL:'694',SG:'702',SK:'703',SI:'705', + SB:'90',SO:'706',ZA:'710',SS:'728',ES:'724',LK:'144',SD:'729', + SR:'740',SE:'752',CH:'756',SY:'760',TW:'158',TJ:'762',TZ:'834', + TH:'764',TL:'626',TG:'768',TO:'776',TT:'780',TN:'788',TR:'792', + TM:'795',TV:'798',UG:'800',UA:'804',AE:'784',GB:'826',US:'840', + UY:'858',UZ:'860',VU:'548',VE:'862',VN:'704',YE:'887',ZM:'894', + ZW:'716',PS:'275', +}; + +const N2A: Record = Object.fromEntries( + Object.entries(A2N).map(([a, n]) => [n, a]) +); + +function colorForCount(count: number, max: number): string { + if (!count) return '#22263a'; + const t = Math.sqrt(Math.min(count / Math.max(max, 1), 1)); + const r = Math.round(0x1a + (0x3b - 0x1a) * t); + const g = Math.round(0x3a + (0x82 - 0x3a) * t); + const b = Math.round(0x5c + (0xf6 - 0x5c) * t); + return `rgb(${r},${g},${b})`; +} + +export interface CountryStats { countryCode: string; total: number; blocked: number; } + +interface PathEntry { id: string; d: string; } + +export default function WorldMapInner({ data }: { data: CountryStats[] }) { + const [paths, setPaths] = useState(null); + + useEffect(() => { + fetch(WORLD_ATLAS) + .then(r => r.json()) + .then((topology: Topology) => { + const countries = feature( + topology, + topology.objects.countries as GeometryCollection + ); + const projection = geoNaturalEarth1().scale(153).translate([480, 250]); + const pathGen = geoPath(projection); + const result: PathEntry[] = []; + for (const f of countries.features) { + const d = pathGen(f); + if (d) result.push({ id: String(f.id ?? ''), d }); + } + setPaths(result); + }) + .catch(() => setPaths([])); + }, []); + + if (paths === null) { + return ( + + + + ); + } + + const countMap = new Map(data.map(d => [d.countryCode, d.total])); + const max = data.reduce((m, d) => Math.max(m, d.total), 0); + + return ( + + + {paths.map(({ id, d }) => { + const alpha2 = N2A[id] ?? null; + const count = alpha2 ? (countMap.get(alpha2) ?? 0) : 0; + return ( + + {alpha2 && count > 0 && ( + {alpha2}: {count.toLocaleString()} requests + )} + + ); + })} + + + ); +} diff --git a/app/(dashboard)/analytics/page.tsx b/app/(dashboard)/analytics/page.tsx new file mode 100644 index 00000000..79b163c6 --- /dev/null +++ b/app/(dashboard)/analytics/page.tsx @@ -0,0 +1,7 @@ +import { requireUser } from '@/src/lib/auth'; +import AnalyticsClient from './AnalyticsClient'; + +export default async function AnalyticsPage() { + await requireUser(); + return ; +} diff --git a/app/(dashboard)/page.tsx b/app/(dashboard)/page.tsx index 9ed3a120..63a6dfae 100644 --- a/app/(dashboard)/page.tsx +++ b/app/(dashboard)/page.tsx @@ -12,6 +12,7 @@ import SwapHorizIcon from "@mui/icons-material/SwapHoriz"; import SecurityIcon from "@mui/icons-material/Security"; import VpnKeyIcon from "@mui/icons-material/VpnKey"; import { ReactNode } from "react"; +import { getAnalyticsSummary } from "@/src/lib/analytics-db"; type StatCard = { label: string; @@ -40,23 +41,27 @@ async function loadStats(): Promise { export default async function OverviewPage() { const session = await requireAdmin(); - const stats = await loadStats(); - const recentEvents = await db - .select({ - action: auditEvents.action, - entityType: auditEvents.entityType, - summary: auditEvents.summary, - createdAt: auditEvents.createdAt - }) - .from(auditEvents) - .orderBy(desc(auditEvents.createdAt)) - .limit(8); + const [stats, trafficSummary, recentEventsRaw] = await Promise.all([ + loadStats(), + getAnalyticsSummary('24h', 'all').catch(() => null), + db + .select({ + action: auditEvents.action, + entityType: auditEvents.entityType, + summary: auditEvents.summary, + createdAt: auditEvents.createdAt + }) + .from(auditEvents) + .orderBy(desc(auditEvents.createdAt)) + .limit(8), + ]); return ( ({ + trafficSummary={trafficSummary} + recentEvents={recentEventsRaw.map((event) => ({ summary: event.summary ?? `${event.action} on ${event.entityType}`, created_at: toIso(event.createdAt)! }))} diff --git a/app/api/analytics/blocked/route.ts b/app/api/analytics/blocked/route.ts new file mode 100644 index 00000000..e75725ea --- /dev/null +++ b/app/api/analytics/blocked/route.ts @@ -0,0 +1,13 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { requireUser } from '@/src/lib/auth'; +import { getAnalyticsBlocked, type Interval } from '@/src/lib/analytics-db'; + +export async function GET(req: NextRequest) { + await requireUser(); + const { searchParams } = req.nextUrl; + const interval = (searchParams.get('interval') ?? '24h') as Interval; + const host = searchParams.get('host') ?? 'all'; + const page = parseInt(searchParams.get('page') ?? '1', 10); + const data = await getAnalyticsBlocked(interval, host, page); + return NextResponse.json(data); +} diff --git a/app/api/analytics/countries/route.ts b/app/api/analytics/countries/route.ts new file mode 100644 index 00000000..9606485e --- /dev/null +++ b/app/api/analytics/countries/route.ts @@ -0,0 +1,12 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { requireUser } from '@/src/lib/auth'; +import { getAnalyticsCountries, type Interval } from '@/src/lib/analytics-db'; + +export async function GET(req: NextRequest) { + await requireUser(); + const { searchParams } = req.nextUrl; + const interval = (searchParams.get('interval') ?? '24h') as Interval; + const host = searchParams.get('host') ?? 'all'; + const data = await getAnalyticsCountries(interval, host); + return NextResponse.json(data); +} diff --git a/app/api/analytics/hosts/route.ts b/app/api/analytics/hosts/route.ts new file mode 100644 index 00000000..2cb3bd1e --- /dev/null +++ b/app/api/analytics/hosts/route.ts @@ -0,0 +1,9 @@ +import { NextResponse } from 'next/server'; +import { requireUser } from '@/src/lib/auth'; +import { getAnalyticsHosts } from '@/src/lib/analytics-db'; + +export async function GET() { + await requireUser(); + const hosts = await getAnalyticsHosts(); + return NextResponse.json(hosts); +} diff --git a/app/api/analytics/protocols/route.ts b/app/api/analytics/protocols/route.ts new file mode 100644 index 00000000..77fcfc15 --- /dev/null +++ b/app/api/analytics/protocols/route.ts @@ -0,0 +1,12 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { requireUser } from '@/src/lib/auth'; +import { getAnalyticsProtocols, type Interval } from '@/src/lib/analytics-db'; + +export async function GET(req: NextRequest) { + await requireUser(); + const { searchParams } = req.nextUrl; + const interval = (searchParams.get('interval') ?? '24h') as Interval; + const host = searchParams.get('host') ?? 'all'; + const data = await getAnalyticsProtocols(interval, host); + return NextResponse.json(data); +} diff --git a/app/api/analytics/summary/route.ts b/app/api/analytics/summary/route.ts new file mode 100644 index 00000000..13515430 --- /dev/null +++ b/app/api/analytics/summary/route.ts @@ -0,0 +1,12 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { requireUser } from '@/src/lib/auth'; +import { getAnalyticsSummary, type Interval } from '@/src/lib/analytics-db'; + +export async function GET(req: NextRequest) { + await requireUser(); + const { searchParams } = req.nextUrl; + const interval = (searchParams.get('interval') ?? '24h') as Interval; + const host = searchParams.get('host') ?? 'all'; + const data = await getAnalyticsSummary(interval, host); + return NextResponse.json(data); +} diff --git a/app/api/analytics/timeline/route.ts b/app/api/analytics/timeline/route.ts new file mode 100644 index 00000000..10008d57 --- /dev/null +++ b/app/api/analytics/timeline/route.ts @@ -0,0 +1,12 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { requireUser } from '@/src/lib/auth'; +import { getAnalyticsTimeline, type Interval } from '@/src/lib/analytics-db'; + +export async function GET(req: NextRequest) { + await requireUser(); + const { searchParams } = req.nextUrl; + const interval = (searchParams.get('interval') ?? '24h') as Interval; + const host = searchParams.get('host') ?? 'all'; + const data = await getAnalyticsTimeline(interval, host); + return NextResponse.json(data); +} diff --git a/app/api/analytics/user-agents/route.ts b/app/api/analytics/user-agents/route.ts new file mode 100644 index 00000000..57cd87f8 --- /dev/null +++ b/app/api/analytics/user-agents/route.ts @@ -0,0 +1,12 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { requireUser } from '@/src/lib/auth'; +import { getAnalyticsUserAgents, type Interval } from '@/src/lib/analytics-db'; + +export async function GET(req: NextRequest) { + await requireUser(); + const { searchParams } = req.nextUrl; + const interval = (searchParams.get('interval') ?? '24h') as Interval; + const host = searchParams.get('host') ?? 'all'; + const data = await getAnalyticsUserAgents(interval, host); + return NextResponse.json(data); +} diff --git a/docker-compose.yml b/docker-compose.yml index 90d14275..30ff4ee9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,9 +52,12 @@ services: OAUTH_TOKEN_URL: ${OAUTH_TOKEN_URL:-} OAUTH_USERINFO_URL: ${OAUTH_USERINFO_URL:-} OAUTH_ALLOW_AUTO_LINKING: ${OAUTH_ALLOW_AUTO_LINKING:-false} + group_add: + - "${CADDY_GID:-10000}" # caddy's GID — lets the web user read /logs/access.log volumes: - caddy-manager-data:/app/data - geoip-data:/usr/share/GeoIP:ro,z + - caddy-logs:/logs:ro depends_on: caddy: condition: service_healthy diff --git a/drizzle/0009_watery_bill_hollister.sql b/drizzle/0009_watery_bill_hollister.sql new file mode 100644 index 00000000..438c1e72 --- /dev/null +++ b/drizzle/0009_watery_bill_hollister.sql @@ -0,0 +1,24 @@ +-- Custom SQL migration file, put your code below! -- +CREATE TABLE `traffic_events` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `ts` integer NOT NULL, + `client_ip` text NOT NULL, + `country_code` text, + `host` text NOT NULL DEFAULT '', + `method` text NOT NULL DEFAULT '', + `uri` text NOT NULL DEFAULT '', + `status` integer NOT NULL DEFAULT 0, + `proto` text NOT NULL DEFAULT '', + `bytes_sent` integer NOT NULL DEFAULT 0, + `user_agent` text NOT NULL DEFAULT '', + `is_blocked` integer NOT NULL DEFAULT false +); +--> statement-breakpoint +CREATE INDEX `idx_traffic_events_ts` ON `traffic_events` (`ts`); +--> statement-breakpoint +CREATE INDEX `idx_traffic_events_host_ts` ON `traffic_events` (`host`, `ts`); +--> statement-breakpoint +CREATE TABLE `log_parse_state` ( + `key` text PRIMARY KEY NOT NULL, + `value` text NOT NULL +); diff --git a/drizzle/meta/0009_snapshot.json b/drizzle/meta/0009_snapshot.json new file mode 100644 index 00000000..39ad5af1 --- /dev/null +++ b/drizzle/meta/0009_snapshot.json @@ -0,0 +1,1137 @@ +{ + "id": "9f96cdcc-3cae-4476-a8d6-73b16bb0417c", + "prevId": "7e98b252-3ad2-4fd7-b8b1-b6e71a546310", + "version": "6", + "dialect": "sqlite", + "tables": { + "access_list_entries": { + "name": "access_list_entries", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "access_list_id": { + "name": "access_list_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "access_list_entries_list_idx": { + "name": "access_list_entries_list_idx", + "columns": [ + "access_list_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "access_list_entries_access_list_id_access_lists_id_fk": { + "name": "access_list_entries_access_list_id_access_lists_id_fk", + "tableFrom": "access_list_entries", + "columnsFrom": [ + "access_list_id" + ], + "tableTo": "access_lists", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "access_lists": { + "name": "access_lists", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "access_lists_created_by_users_id_fk": { + "name": "access_lists_created_by_users_id_fk", + "tableFrom": "access_lists", + "columnsFrom": [ + "created_by" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "api_tokens": { + "name": "api_tokens", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "api_tokens_token_hash_unique": { + "name": "api_tokens_token_hash_unique", + "columns": [ + "token_hash" + ], + "isUnique": true + } + }, + "foreignKeys": { + "api_tokens_created_by_users_id_fk": { + "name": "api_tokens_created_by_users_id_fk", + "tableFrom": "api_tokens", + "columnsFrom": [ + "created_by" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "audit_events": { + "name": "audit_events", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entity_type": { + "name": "entity_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entity_id": { + "name": "entity_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "audit_events_user_id_users_id_fk": { + "name": "audit_events_user_id_users_id_fk", + "tableFrom": "audit_events", + "columnsFrom": [ + "user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "certificates": { + "name": "certificates", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domain_names": { + "name": "domain_names", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "auto_renew": { + "name": "auto_renew", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "provider_options": { + "name": "provider_options", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "certificate_pem": { + "name": "certificate_pem", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_key_pem": { + "name": "private_key_pem", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "certificates_created_by_users_id_fk": { + "name": "certificates_created_by_users_id_fk", + "tableFrom": "certificates", + "columnsFrom": [ + "created_by" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instances": { + "name": "instances", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_token": { + "name": "api_token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "last_sync_at": { + "name": "last_sync_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "instances_base_url_unique": { + "name": "instances_base_url_unique", + "columns": [ + "base_url" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "oauth_states": { + "name": "oauth_states", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "code_verifier": { + "name": "code_verifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_to": { + "name": "redirect_to", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "oauth_state_unique": { + "name": "oauth_state_unique", + "columns": [ + "state" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "pending_oauth_links": { + "name": "pending_oauth_links", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text(50)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_email": { + "name": "user_email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "pending_oauth_user_provider_unique": { + "name": "pending_oauth_user_provider_unique", + "columns": [ + "user_id", + "provider" + ], + "isUnique": true + } + }, + "foreignKeys": { + "pending_oauth_links_user_id_users_id_fk": { + "name": "pending_oauth_links_user_id_users_id_fk", + "tableFrom": "pending_oauth_links", + "columnsFrom": [ + "user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "proxy_hosts": { + "name": "proxy_hosts", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domains": { + "name": "domains", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "upstreams": { + "name": "upstreams", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "certificate_id": { + "name": "certificate_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_list_id": { + "name": "access_list_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ssl_forced": { + "name": "ssl_forced", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "hsts_enabled": { + "name": "hsts_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "hsts_subdomains": { + "name": "hsts_subdomains", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "allow_websocket": { + "name": "allow_websocket", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "preserve_host_header": { + "name": "preserve_host_header", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "meta": { + "name": "meta", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "skip_https_hostname_validation": { + "name": "skip_https_hostname_validation", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "response_mode": { + "name": "response_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'proxy'" + }, + "static_status_code": { + "name": "static_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 503 + }, + "static_response_body": { + "name": "static_response_body", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "proxy_hosts_certificate_id_certificates_id_fk": { + "name": "proxy_hosts_certificate_id_certificates_id_fk", + "tableFrom": "proxy_hosts", + "columnsFrom": [ + "certificate_id" + ], + "tableTo": "certificates", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + }, + "proxy_hosts_access_list_id_access_lists_id_fk": { + "name": "proxy_hosts_access_list_id_access_lists_id_fk", + "tableFrom": "proxy_hosts", + "columnsFrom": [ + "access_list_id" + ], + "tableTo": "access_lists", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + }, + "proxy_hosts_owner_user_id_users_id_fk": { + "name": "proxy_hosts_owner_user_id_users_id_fk", + "tableFrom": "proxy_hosts", + "columnsFrom": [ + "owner_user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "redirect_hosts": { + "name": "redirect_hosts", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domains": { + "name": "domains", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "destination": { + "name": "destination", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 302 + }, + "preserve_query": { + "name": "preserve_query", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "redirect_hosts_created_by_users_id_fk": { + "name": "redirect_hosts_created_by_users_id_fk", + "tableFrom": "redirect_hosts", + "columnsFrom": [ + "created_by" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "set null" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "sessions_token_unique": { + "name": "sessions_token_unique", + "columns": [ + "token" + ], + "isUnique": true + } + }, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "columnsFrom": [ + "user_id" + ], + "tableTo": "users", + "columnsTo": [ + "id" + ], + "onUpdate": "no action", + "onDelete": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'user'" + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "users_email_unique": { + "name": "users_email_unique", + "columns": [ + "email" + ], + "isUnique": true + }, + "users_provider_subject_idx": { + "name": "users_provider_subject_idx", + "columns": [ + "provider", + "subject" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 059f3376..e33ce34e 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -64,6 +64,13 @@ "when": 1740960000000, "tag": "0008_unique_provider_subject", "breakpoints": true + }, + { + "idx": 9, + "version": "6", + "when": 1772129593846, + "tag": "0009_watery_bill_hollister", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index caf833bf..ecaae8b3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,18 +13,25 @@ "@mui/icons-material": "^7.3.8", "@mui/material": "^7.3.6", "@types/better-sqlite3": "^7.6.13", + "apexcharts": "^5.6.0", "bcryptjs": "^3.0.3", "better-sqlite3": "^12.6.2", + "d3-geo": "^3.1.1", "drizzle-orm": "^0.45.1", + "maxmind": "^5.0.5", "next": "^16.1.3", "next-auth": "^5.0.0-beta.30", "react": "^19.2.3", - "react-dom": "^19.2.3" + "react-apexcharts": "^2.0.1", + "react-dom": "^19.2.3", + "topojson-client": "^3.1.0" }, "devDependencies": { + "@types/d3-geo": "^3.1.0", "@types/node": "^25.2.3", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", + "@types/topojson-client": "^3.1.5", "drizzle-kit": "^0.31.9", "eslint": "^9.39.2", "eslint-config-next": "^16.1.3", @@ -2571,6 +2578,16 @@ "@types/node": "*" } }, + "node_modules/@types/d3-geo": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-geo/-/d3-geo-3.1.0.tgz", + "integrity": "sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -2578,6 +2595,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -2641,6 +2665,27 @@ "@types/react": "*" } }, + "node_modules/@types/topojson-client": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@types/topojson-client/-/topojson-client-3.1.5.tgz", + "integrity": "sha512-C79rySTyPxnQNNguTZNI1Ct4D7IXgvyAs3p9HPecnl6mNrJ5+UhvGNYcZfpROYV2lMHI48kJPxwR+F9C6c7nmw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/geojson": "*", + "@types/topojson-specification": "*" + } + }, + "node_modules/@types/topojson-specification": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/topojson-specification/-/topojson-specification-1.0.5.tgz", + "integrity": "sha512-C7KvcQh+C2nr6Y2Ub4YfgvWvWCgP2nOQMtfhlnwsRL4pYmmwzBS7HclGiS87eQfDOU/DLQpX6GEscviaz4yLIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.54.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.54.0.tgz", @@ -3179,6 +3224,12 @@ "win32" ] }, + "node_modules/@yr/monotone-cubic-spline": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@yr/monotone-cubic-spline/-/monotone-cubic-spline-1.0.3.tgz", + "integrity": "sha512-FQXkOta0XBSUPHndIKON2Y9JeQz5ZeMqLYZVVK93FliNBFm7LNMIZmY6FrMEB9XPcDbE2bekMbZD6kzDkxwYjA==", + "license": "MIT" + }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", @@ -3235,6 +3286,15 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/apexcharts": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-5.6.0.tgz", + "integrity": "sha512-BZua59yedRsaDfnxkzNrkyLCvluq2c3ZDBIz4joxSKtgr0xDQXQ5dzceMhf/TpTbAjaF+2NYIpLP3BEEIG2s/w==", + "license": "SEE LICENSE IN LICENSE", + "dependencies": { + "@yr/monotone-cubic-spline": "^1.0.3" + } + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -3785,6 +3845,12 @@ "dev": true, "license": "MIT" }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "license": "MIT" + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -3835,6 +3901,30 @@ "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", "license": "MIT" }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-geo": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.1.tgz", + "integrity": "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2.5.0 - 3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/damerau-levenshtein": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz", @@ -5446,6 +5536,15 @@ "node": ">= 0.4" } }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/is-array-buffer": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz", @@ -6099,6 +6198,20 @@ "node": ">= 0.4" } }, + "node_modules/maxmind": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/maxmind/-/maxmind-5.0.5.tgz", + "integrity": "sha512-1lcH2kMjbBpCFhuHaMU32vz8CuOsKttRcWMQyXvtlklopCzN7NNHSVR/h9RYa8JPuFTGmkn2vYARm+7cIGuqDw==", + "license": "MIT", + "dependencies": { + "mmdb-lib": "3.0.2", + "tiny-lru": "11.4.7" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -6163,6 +6276,16 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", "license": "MIT" }, + "node_modules/mmdb-lib": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mmdb-lib/-/mmdb-lib-3.0.2.tgz", + "integrity": "sha512-7e87vk0DdWT647wjcfEtWeMtjm+zVGqNohN/aeIymbUfjHQ2T4Sx5kM+1irVDBSloNC3CkGKxswdMoo8yhqTDg==", + "license": "MIT", + "engines": { + "node": ">=10", + "npm": ">=6" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -6803,6 +6926,19 @@ "node": ">=0.10.0" } }, + "node_modules/react-apexcharts": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/react-apexcharts/-/react-apexcharts-2.0.1.tgz", + "integrity": "sha512-AN9u5C0kDAkQyMyJP6yXORexiuiaTa0GaxR5dOaF075vWLTTbQxxtYF/kXyvUo4jjtBEIe4lLoCZTo/kB3bRoA==", + "license": "SEE LICENSE IN LICENSE", + "dependencies": { + "prop-types": "^15.8.1" + }, + "peerDependencies": { + "apexcharts": ">=5.6.0", + "react": ">=16.8.0" + } + }, "node_modules/react-dom": { "version": "19.2.4", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz", @@ -7598,6 +7734,15 @@ "node": ">=6" } }, + "node_modules/tiny-lru": { + "version": "11.4.7", + "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-11.4.7.tgz", + "integrity": "sha512-w/Te7uMUVeH0CR8vZIjr+XiN41V+30lkDdK+NRIDCUYKKuL9VcmaUEmaPISuwGhLlrTGh5yu18lENtR9axSxYw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=12" + } + }, "node_modules/tinyglobby": { "version": "0.2.15", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", @@ -7659,6 +7804,20 @@ "node": ">=8.0" } }, + "node_modules/topojson-client": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/topojson-client/-/topojson-client-3.1.0.tgz", + "integrity": "sha512-605uxS6bcYxGXw9qi62XyrV6Q3xwbndjachmNxu8HWTtVPxZfEJN9fd/SZS1Q54Sn2y0TMyMxFj/cJINqGHrKw==", + "license": "ISC", + "dependencies": { + "commander": "2" + }, + "bin": { + "topo2geo": "bin/topo2geo", + "topomerge": "bin/topomerge", + "topoquantize": "bin/topoquantize" + } + }, "node_modules/ts-api-utils": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.4.0.tgz", diff --git a/package.json b/package.json index 04a3befe..2a20540c 100644 --- a/package.json +++ b/package.json @@ -18,19 +18,26 @@ "@mui/icons-material": "^7.3.8", "@mui/material": "^7.3.6", "@types/better-sqlite3": "^7.6.13", + "apexcharts": "^5.6.0", "bcryptjs": "^3.0.3", "better-sqlite3": "^12.6.2", + "d3-geo": "^3.1.1", "drizzle-orm": "^0.45.1", + "maxmind": "^5.0.5", "next": "^16.1.3", "next-auth": "^5.0.0-beta.30", "react": "^19.2.3", - "react-dom": "^19.2.3" + "react-apexcharts": "^2.0.1", + "react-dom": "^19.2.3", + "topojson-client": "^3.1.0" }, "devDependencies": { - "drizzle-kit": "^0.31.9", + "@types/d3-geo": "^3.1.0", "@types/node": "^25.2.3", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", + "@types/topojson-client": "^3.1.5", + "drizzle-kit": "^0.31.9", "eslint": "^9.39.2", "eslint-config-next": "^16.1.3", "typescript": "^5.9.3" diff --git a/src/instrumentation.ts b/src/instrumentation.ts index e86b2e09..9068767c 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -48,6 +48,26 @@ export async function register() { // Don't throw - monitoring is a nice-to-have feature } + // Start log parser for analytics + const { initLogParser, parseNewLogEntries, stopLogParser } = await import("./lib/log-parser"); + try { + await initLogParser(); + const logParserInterval = setInterval(async () => { + try { + await parseNewLogEntries(); + } catch (err) { + console.error("Log parser interval error:", err); + } + }, 30_000); + process.on("SIGTERM", () => { + stopLogParser(); + clearInterval(logParserInterval); + }); + console.log("Log parser started"); + } catch (error) { + console.error("Failed to start log parser:", error); + } + // Start periodic instance sync if configured (master mode only) const { getInstanceMode, getSyncIntervalMs, syncInstances } = await import("./lib/instance-sync"); try { diff --git a/src/lib/analytics-db.ts b/src/lib/analytics-db.ts new file mode 100644 index 00000000..48d8a181 --- /dev/null +++ b/src/lib/analytics-db.ts @@ -0,0 +1,244 @@ +import { sql, and, gte, eq } from 'drizzle-orm'; +import db from './db'; +import { trafficEvents } from './db/schema'; +import { existsSync } from 'node:fs'; + +export type Interval = '24h' | '7d' | '30d'; + +const LOG_FILE = '/logs/access.log'; + +function getIntervalStart(interval: Interval): number { + const seconds = interval === '24h' ? 86400 : interval === '7d' ? 7 * 86400 : 30 * 86400; + return Math.floor(Date.now() / 1000) - seconds; +} + +function buildWhere(interval: Interval, host: string) { + const since = getIntervalStart(interval); + const conditions = [gte(trafficEvents.ts, since)]; + if (host !== 'all' && host !== '') conditions.push(eq(trafficEvents.host, host)); + return and(...conditions); +} + +// ── Summary ────────────────────────────────────────────────────────────────── + +export interface AnalyticsSummary { + totalRequests: number; + uniqueIps: number; + blockedRequests: number; + blockedPercent: number; + bytesServed: number; + loggingDisabled: boolean; +} + +export async function getAnalyticsSummary(interval: Interval, host: string): Promise { + const loggingDisabled = !existsSync(LOG_FILE); + const where = buildWhere(interval, host); + + const row = db + .select({ + total: sql`count(*)`, + uniqueIps: sql`count(distinct ${trafficEvents.clientIp})`, + blocked: sql`sum(case when ${trafficEvents.isBlocked} then 1 else 0 end)`, + bytes: sql`sum(${trafficEvents.bytesSent})`, + }) + .from(trafficEvents) + .where(where) + .get(); + + const total = row?.total ?? 0; + const blocked = row?.blocked ?? 0; + + return { + totalRequests: total, + uniqueIps: row?.uniqueIps ?? 0, + blockedRequests: blocked, + blockedPercent: total > 0 ? Math.round((blocked / total) * 1000) / 10 : 0, + bytesServed: row?.bytes ?? 0, + loggingDisabled, + }; +} + +// ── Timeline ───────────────────────────────────────────────────────────────── + +export interface TimelineBucket { + ts: number; + total: number; + blocked: number; +} + +export async function getAnalyticsTimeline(interval: Interval, host: string): Promise { + const bucketSize = interval === '24h' ? 3600 : interval === '7d' ? 21600 : 86400; + const where = buildWhere(interval, host); + + const rows = db + .select({ + bucket: sql`(${trafficEvents.ts} / ${sql.raw(String(bucketSize))})`, + total: sql`count(*)`, + blocked: sql`sum(case when ${trafficEvents.isBlocked} then 1 else 0 end)`, + }) + .from(trafficEvents) + .where(where) + .groupBy(sql`(${trafficEvents.ts} / ${sql.raw(String(bucketSize))})`) + .orderBy(sql`(${trafficEvents.ts} / ${sql.raw(String(bucketSize))})`) + .all(); + + return rows.map((r) => ({ + ts: r.bucket * bucketSize, + total: r.total, + blocked: r.blocked ?? 0, + })); +} + +// ── Countries ──────────────────────────────────────────────────────────────── + +export interface CountryStats { + countryCode: string; + total: number; + blocked: number; +} + +export async function getAnalyticsCountries(interval: Interval, host: string): Promise { + const where = buildWhere(interval, host); + + const rows = db + .select({ + countryCode: trafficEvents.countryCode, + total: sql`count(*)`, + blocked: sql`sum(case when ${trafficEvents.isBlocked} then 1 else 0 end)`, + }) + .from(trafficEvents) + .where(where) + .groupBy(trafficEvents.countryCode) + .orderBy(sql`count(*) desc`) + .all(); + + return rows.map((r) => ({ + countryCode: r.countryCode ?? 'XX', + total: r.total, + blocked: r.blocked ?? 0, + })); +} + +// ── Protocols ──────────────────────────────────────────────────────────────── + +export interface ProtoStats { + proto: string; + count: number; + percent: number; +} + +export async function getAnalyticsProtocols(interval: Interval, host: string): Promise { + const where = buildWhere(interval, host); + + const rows = db + .select({ + proto: trafficEvents.proto, + count: sql`count(*)`, + }) + .from(trafficEvents) + .where(where) + .groupBy(trafficEvents.proto) + .orderBy(sql`count(*) desc`) + .all(); + + const total = rows.reduce((s, r) => s + r.count, 0); + + return rows.map((r) => ({ + proto: r.proto || 'Unknown', + count: r.count, + percent: total > 0 ? Math.round((r.count / total) * 1000) / 10 : 0, + })); +} + +// ── User Agents ────────────────────────────────────────────────────────────── + +export interface UAStats { + userAgent: string; + count: number; + percent: number; +} + +export async function getAnalyticsUserAgents(interval: Interval, host: string): Promise { + const where = buildWhere(interval, host); + + const rows = db + .select({ + userAgent: trafficEvents.userAgent, + count: sql`count(*)`, + }) + .from(trafficEvents) + .where(where) + .groupBy(trafficEvents.userAgent) + .orderBy(sql`count(*) desc`) + .limit(10) + .all(); + + const total = rows.reduce((s, r) => s + r.count, 0); + + return rows.map((r) => ({ + userAgent: r.userAgent || 'Unknown', + count: r.count, + percent: total > 0 ? Math.round((r.count / total) * 1000) / 10 : 0, + })); +} + +// ── Blocked events ─────────────────────────────────────────────────────────── + +export interface BlockedEvent { + id: number; + ts: number; + clientIp: string; + countryCode: string | null; + method: string; + uri: string; + status: number; + host: string; +} + +export interface BlockedPage { + events: BlockedEvent[]; + total: number; + page: number; + pages: number; +} + +export async function getAnalyticsBlocked(interval: Interval, host: string, page: number): Promise { + const pageSize = 10; + const where = and(buildWhere(interval, host), eq(trafficEvents.isBlocked, true)); + + const totalRow = db.select({ total: sql`count(*)` }).from(trafficEvents).where(where).get(); + const total = totalRow?.total ?? 0; + const pages = Math.max(1, Math.ceil(total / pageSize)); + const safePage = Math.min(Math.max(1, page), pages); + + const rows = db + .select({ + id: trafficEvents.id, + ts: trafficEvents.ts, + clientIp: trafficEvents.clientIp, + countryCode: trafficEvents.countryCode, + method: trafficEvents.method, + uri: trafficEvents.uri, + status: trafficEvents.status, + host: trafficEvents.host, + }) + .from(trafficEvents) + .where(where) + .orderBy(sql`${trafficEvents.ts} desc`) + .limit(pageSize) + .offset((safePage - 1) * pageSize) + .all(); + + return { events: rows, total, page: safePage, pages }; +} + +// ── Hosts ──────────────────────────────────────────────────────────────────── + +export async function getAnalyticsHosts(): Promise { + const rows = db + .selectDistinct({ host: trafficEvents.host }) + .from(trafficEvents) + .orderBy(trafficEvents.host) + .all(); + return rows.map((r) => r.host).filter(Boolean); +} diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index edd26687..54495902 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -185,3 +185,30 @@ export const linkingTokens = sqliteTable("linking_tokens", { createdAt: text("created_at").notNull(), expiresAt: text("expires_at").notNull() }); + +export const trafficEvents = sqliteTable( + 'traffic_events', + { + id: integer('id').primaryKey({ autoIncrement: true }), + ts: integer('ts').notNull(), + clientIp: text('client_ip').notNull(), + countryCode: text('country_code'), + host: text('host').notNull().default(''), + method: text('method').notNull().default(''), + uri: text('uri').notNull().default(''), + status: integer('status').notNull().default(0), + proto: text('proto').notNull().default(''), + bytesSent: integer('bytes_sent').notNull().default(0), + userAgent: text('user_agent').notNull().default(''), + isBlocked: integer('is_blocked', { mode: 'boolean' }).notNull().default(false), + }, + (table) => ({ + tsIdx: index('idx_traffic_events_ts').on(table.ts), + hostTsIdx: index('idx_traffic_events_host_ts').on(table.host, table.ts), + }) +); + +export const logParseState = sqliteTable('log_parse_state', { + key: text('key').primaryKey(), + value: text('value').notNull(), +}); diff --git a/src/lib/log-parser.ts b/src/lib/log-parser.ts new file mode 100644 index 00000000..58538f38 --- /dev/null +++ b/src/lib/log-parser.ts @@ -0,0 +1,187 @@ +import { createReadStream, existsSync, statSync } from 'node:fs'; +import { createInterface } from 'node:readline'; +import maxmind, { CountryResponse } from 'maxmind'; +import db from './db'; +import { trafficEvents, logParseState } from './db/schema'; +import { eq } from 'drizzle-orm'; + +const LOG_FILE = '/logs/access.log'; +const GEOIP_DB = '/usr/share/GeoIP/GeoLite2-Country.mmdb'; +const BATCH_SIZE = 500; +const RETENTION_DAYS = 90; + +// GeoIP reader — null if mmdb not available +let geoReader: Awaited>> | null = null; +const geoCache = new Map(); + +let stopped = false; + +// ── state helpers ──────────────────────────────────────────────────────────── + +function getState(key: string): string | null { + const row = db.select({ value: logParseState.value }).from(logParseState).where(eq(logParseState.key, key)).get(); + return row?.value ?? null; +} + +function setState(key: string, value: string): void { + db.insert(logParseState).values({ key, value }).onConflictDoUpdate({ target: logParseState.key, set: { value } }).run(); +} + +// ── GeoIP ──────────────────────────────────────────────────────────────────── + +async function initGeoIP(): Promise { + if (!existsSync(GEOIP_DB)) { + console.log('[log-parser] GeoIP database not found, country codes will be null'); + return; + } + try { + geoReader = await maxmind.open(GEOIP_DB); + console.log('[log-parser] GeoIP database loaded'); + } catch (err) { + console.warn('[log-parser] Failed to load GeoIP database:', err); + } +} + +function lookupCountry(ip: string): string | null { + if (!geoReader) return null; + if (geoCache.has(ip)) return geoCache.get(ip)!; + if (geoCache.size > 10_000) geoCache.clear(); + try { + const result = geoReader.get(ip); + const code = result?.country?.iso_code ?? null; + geoCache.set(ip, code); + return code; + } catch { + geoCache.set(ip, null); + return null; + } +} + +// ── log parsing ────────────────────────────────────────────────────────────── + +interface CaddyLogEntry { + ts?: number; + msg?: string; + status?: number; + size?: number; + request?: { + client_ip?: string; + remote_ip?: string; + host?: string; + method?: string; + uri?: string; + proto?: string; + headers?: Record; + }; +} + +function parseLine(line: string): typeof trafficEvents.$inferInsert | null { + let entry: CaddyLogEntry; + try { + entry = JSON.parse(line); + } catch { + return null; + } + + // Only process "handled request" log entries + if (entry.msg !== 'handled request') return null; + + const req = entry.request ?? {}; + const clientIp = req.client_ip || req.remote_ip || ''; + const status = entry.status ?? 0; + + return { + ts: Math.floor(entry.ts ?? Date.now() / 1000), + clientIp, + countryCode: clientIp ? lookupCountry(clientIp) : null, + host: req.host ?? '', + method: req.method ?? '', + uri: req.uri ?? '', + status, + proto: req.proto ?? '', + bytesSent: entry.size ?? 0, + userAgent: req.headers?.['User-Agent']?.[0] ?? '', + isBlocked: status === 403, + }; +} + +async function readLines(startOffset: number): Promise<{ rows: typeof trafficEvents.$inferInsert[]; newOffset: number }> { + return new Promise((resolve, reject) => { + const rows: typeof trafficEvents.$inferInsert[] = []; + let bytesRead = 0; + + const stream = createReadStream(LOG_FILE, { start: startOffset, encoding: 'utf8' }); + stream.on('error', (err: NodeJS.ErrnoException) => { + if (err.code === 'ENOENT') resolve({ rows: [], newOffset: startOffset }); + else reject(err); + }); + + const rl = createInterface({ input: stream, crlfDelay: Infinity }); + rl.on('line', (line) => { + bytesRead += Buffer.byteLength(line, 'utf8') + 1; // +1 for newline + const row = parseLine(line.trim()); + if (row) rows.push(row); + }); + rl.on('close', () => resolve({ rows, newOffset: startOffset + bytesRead })); + rl.on('error', reject); + }); +} + +function insertBatch(rows: typeof trafficEvents.$inferInsert[]): void { + for (let i = 0; i < rows.length; i += BATCH_SIZE) { + db.insert(trafficEvents).values(rows.slice(i, i + BATCH_SIZE)).run(); + } +} + +function purgeOldEntries(): void { + const cutoff = Math.floor(Date.now() / 1000) - RETENTION_DAYS * 86400; + db.delete(trafficEvents).where(eq(trafficEvents.ts, cutoff)).run(); + // Use raw sql for < comparison + db.run(`DELETE FROM traffic_events WHERE ts < ${cutoff}`); +} + +// ── public API ─────────────────────────────────────────────────────────────── + +export async function initLogParser(): Promise { + await initGeoIP(); + console.log('[log-parser] initialized'); +} + +export async function parseNewLogEntries(): Promise { + if (stopped) return; + if (!existsSync(LOG_FILE)) return; + + try { + const storedOffset = parseInt(getState('access_log_offset') ?? '0', 10); + const storedSize = parseInt(getState('access_log_size') ?? '0', 10); + + let currentSize: number; + try { + currentSize = statSync(LOG_FILE).size; + } catch { + return; + } + + // Detect log rotation: file shrank + const startOffset = currentSize < storedSize ? 0 : storedOffset; + + const { rows, newOffset } = await readLines(startOffset); + + if (rows.length > 0) { + insertBatch(rows); + console.log(`[log-parser] inserted ${rows.length} traffic events`); + } + + setState('access_log_offset', String(newOffset)); + setState('access_log_size', String(currentSize)); + + // Purge old entries once per run (cheap since it's indexed) + purgeOldEntries(); + } catch (err) { + console.error('[log-parser] error during parse:', err); + } +} + +export function stopLogParser(): void { + stopped = true; +}