akanealw/caddy-proxy-manager
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Migrate analytics from SQLite to ClickHouse

Browse Source 
SQLite was too slow for analytical aggregations on traffic_events and
waf_events (millions of rows, GROUP BY, COUNT DISTINCT). ClickHouse is
a columnar OLAP database purpose-built for this workload.

- Add ClickHouse container to Docker Compose with health check
- Create src/lib/clickhouse/client.ts with singleton client, table DDL,
  insert helpers, and all analytics query functions
- Update log-parser.ts and waf-log-parser.ts to write to ClickHouse
- Remove purgeOldEntries — ClickHouse TTL handles 90-day retention
- Rewrite analytics-db.ts and waf-events.ts to query ClickHouse
- Remove trafficEvents/wafEvents from SQLite schema, add migration
- CLICKHOUSE_PASSWORD is required (no hardcoded default)
- Update .env.example, README, and test infrastructure

API response shapes are unchanged — no frontend modifications needed.
Parse state (file offsets) remains in SQLite.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
fuomag9
2026-04-10 00:05:38 +02:00
parent 833284efb1
commit e1c97038d4
21 changed files with 819 additions and 763 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tests/docker-compose.test.yml
View File

@@ -4,6 +4,7 @@ services:
SESSION_SECRET: "test-session-secret-32chars!xxxY"
ADMIN_USERNAME: testadmin
ADMIN_PASSWORD: "TestPassword2026!"
CLICKHOUSE_PASSWORD: "test-clickhouse-password-2026"
BASE_URL: http://localhost:3000
NEXTAUTH_URL: http://localhost:3000
# OAuth via Dex OIDC provider
@@ -16,6 +17,9 @@ services:
OAUTH_TOKEN_URL: http://dex:5556/dex/token
OAUTH_USERINFO_URL: http://dex:5556/dex/userinfo
OAUTH_ALLOW_AUTO_LINKING: "true"
clickhouse:
environment:
CLICKHOUSE_PASSWORD: "test-clickhouse-password-2026"
caddy:
ports:
- "80:80"
@@ -84,3 +88,5 @@ volumes:
name: caddy-logs-test
geoip-data:
name: geoip-data-test
clickhouse-data:
name: clickhouse-data-test

3
tests/global-setup.ts
View File

@@ -35,7 +35,7 @@ async function waitForHealth(): Promise<void> {
console.error('[global-setup] Health check timed out. Container logs:');
try {
execFileSync('docker', [...COMPOSE_ARGS, 'logs', '--tail=50'], { stdio: 'inherit', cwd: process.cwd() });
execFileSync('docker', [...COMPOSE_ARGS, 'logs', '--tail=50'], { stdio: 'inherit', cwd: process.cwd(), env: { ...process.env, CLICKHOUSE_PASSWORD: 'test-clickhouse-password-2026' } });
} catch { /* ignore */ }
throw new Error(`App did not become healthy within ${MAX_WAIT_MS}ms`);
@@ -74,6 +74,7 @@ export default async function globalSetup() {
], {
stdio: 'inherit',
cwd: process.cwd(),
env: { ...process.env, CLICKHOUSE_PASSWORD: 'test-clickhouse-password-2026' },
});
console.log('[global-setup] Containers up. Waiting for /api/health...');

1
tests/global-teardown.ts
View File

@@ -14,6 +14,7 @@ export default async function globalTeardown() {
execFileSync('docker', [...COMPOSE_ARGS, 'down', '-v', '--remove-orphans'], {
stdio: 'inherit',
cwd: process.cwd(),
env: { ...process.env, CLICKHOUSE_PASSWORD: 'test-clickhouse-password-2026' },
});
} catch (err) {
console.warn('[global-teardown] docker compose down failed:', err);

384
tests/integration/analytics-blocked.test.ts
View File

@@ -1,10 +1,6 @@
import { describe, it, expect, beforeEach, vi } from 'vitest';
import { createTestDb, type TestDb } from '../helpers/db';
import { trafficEvents } from '@/src/lib/db/schema';
import { sql, and, gte, lte, eq, inArray } from 'drizzle-orm';
import { describe, it, expect, vi } from 'vitest';
// Mock dependencies so we can import collectBlockedSignatures and parseLine.
// These run in the log-parser module scope on import.
vi.mock('@/src/lib/db', () => ({
default: {
select: vi.fn().mockReturnValue({ from: vi.fn().mockReturnValue({ where: vi.fn().mockReturnValue({ get: vi.fn().mockReturnValue(null) }) }) }),
@@ -19,351 +15,81 @@ vi.mock('node:fs', () => ({
statSync: vi.fn().mockReturnValue({ size: 0 }),
createReadStream: vi.fn(),
}));
vi.mock('@/src/lib/clickhouse/client', () => ({
insertTrafficEvents: vi.fn().mockResolvedValue(undefined),
}));
import { collectBlockedSignatures, parseLine } from '@/src/lib/log-parser';
let db: TestDb;
const NOW = Math.floor(Date.now() / 1000);
beforeEach(() => {
db = createTestDb();
});
/** Insert a traffic event row with sensible defaults. */
function insertEvent(overrides: Partial<typeof trafficEvents.$inferInsert> = {}) {
db.insert(trafficEvents).values({
ts: NOW,
clientIp: '1.2.3.4',
countryCode: 'DE',
host: 'example.com',
method: 'GET',
uri: '/',
status: 200,
proto: 'HTTP/2.0',
bytesSent: 1024,
userAgent: 'Mozilla/5.0',
isBlocked: false,
...overrides,
}).run();
}
// ── Helpers that mirror analytics-db.ts queries ─────────────────────────────
// We duplicate the SQL here intentionally — if the production queries ever
// drift from what the schema supports, these tests will catch it.
function buildWhere(from: number, to: number, hosts: string[]) {
const conditions = [gte(trafficEvents.ts, from), lte(trafficEvents.ts, to)];
if (hosts.length === 1) {
conditions.push(eq(trafficEvents.host, hosts[0]));
} else if (hosts.length > 1) {
conditions.push(inArray(trafficEvents.host, hosts));
}
return and(...conditions);
}
function getSummary(from: number, to: number, hosts: string[] = []) {
const where = buildWhere(from, to, hosts);
return db
.select({
total: sql<number>`count(*)`,
uniqueIps: sql<number>`count(distinct ${trafficEvents.clientIp})`,
blocked: sql<number>`sum(case when ${trafficEvents.isBlocked} then 1 else 0 end)`,
bytes: sql<number>`sum(${trafficEvents.bytesSent})`,
})
.from(trafficEvents)
.where(where)
.get();
}
function getCountries(from: number, to: number, hosts: string[] = []) {
const where = buildWhere(from, to, hosts);
return db
.select({
countryCode: trafficEvents.countryCode,
total: sql<number>`count(*)`,
blocked: sql<number>`sum(case when ${trafficEvents.isBlocked} then 1 else 0 end)`,
})
.from(trafficEvents)
.where(where)
.groupBy(trafficEvents.countryCode)
.orderBy(sql`count(*) desc`)
.all();
}
function getTimeline(from: number, to: number, bucketSize: number, hosts: string[] = []) {
const where = buildWhere(from, to, hosts);
return db
.select({
bucket: sql<number>`(${trafficEvents.ts} / ${sql.raw(String(bucketSize))})`,
total: sql<number>`count(*)`,
blocked: sql<number>`sum(case when ${trafficEvents.isBlocked} then 1 else 0 end)`,
})
.from(trafficEvents)
.where(where)
.groupBy(sql`(${trafficEvents.ts} / ${sql.raw(String(bucketSize))})`)
.orderBy(sql`(${trafficEvents.ts} / ${sql.raw(String(bucketSize))})`)
.all();
}
function getBlockedEvents(from: number, to: number, hosts: string[] = []) {
const where = and(buildWhere(from, to, hosts), eq(trafficEvents.isBlocked, true));
return db
.select({
id: trafficEvents.id,
ts: trafficEvents.ts,
clientIp: trafficEvents.clientIp,
countryCode: trafficEvents.countryCode,
host: trafficEvents.host,
status: trafficEvents.status,
})
.from(trafficEvents)
.where(where)
.orderBy(sql`${trafficEvents.ts} desc`)
.all();
}
// ── Tests ────────────────────────────────────────────────────────────────────
describe('analytics blocked counting', () => {
const from = NOW - 3600;
const to = NOW + 3600;
describe('summary', () => {
it('counts zero blocked when no events are blocked', () => {
insertEvent({ isBlocked: false });
insertEvent({ isBlocked: false });
const row = getSummary(from, to);
expect(row!.total).toBe(2);
expect(row!.blocked).toBe(0);
describe('log-parser blocked detection', () => {
describe('collectBlockedSignatures', () => {
it('collects signatures from caddy-blocker "request blocked" entries', () => {
const lines = [
JSON.stringify({ ts: NOW + 0.01, msg: 'request blocked', plugin: 'caddy-blocker', client_ip: '1.2.3.4', method: 'GET', uri: '/secret' }),
JSON.stringify({ ts: NOW + 0.5, msg: 'handled request', status: 200, request: { client_ip: '5.6.7.8' } }),
];
const set = collectBlockedSignatures(lines);
expect(set.size).toBe(1);
});
it('counts geo-blocked requests correctly', () => {
insertEvent({ isBlocked: true, status: 403, clientIp: '5.6.7.8', countryCode: 'CN' });
insertEvent({ isBlocked: true, status: 403, clientIp: '9.10.11.12', countryCode: 'RU' });
insertEvent({ isBlocked: false, status: 200 });
const row = getSummary(from, to);
expect(row!.total).toBe(3);
expect(row!.blocked).toBe(2);
it('returns empty set when no blocked entries', () => {
const lines = [
JSON.stringify({ ts: NOW, msg: 'handled request', status: 200, request: { client_ip: '1.2.3.4' } }),
];
expect(collectBlockedSignatures(lines).size).toBe(0);
});
it('filters by host', () => {
insertEvent({ isBlocked: true, host: 'blocked.com' });
insertEvent({ isBlocked: false, host: 'blocked.com' });
insertEvent({ isBlocked: true, host: 'other.com' });
const row = getSummary(from, to, ['blocked.com']);
expect(row!.total).toBe(2);
expect(row!.blocked).toBe(1);
it('ignores non-caddy-blocker entries', () => {
const lines = [
JSON.stringify({ ts: NOW, msg: 'request blocked', plugin: 'other-plugin', client_ip: '1.2.3.4', method: 'GET', uri: '/' }),
];
expect(collectBlockedSignatures(lines).size).toBe(0);
});
});
describe('countries', () => {
it('shows blocked count per country', () => {
insertEvent({ isBlocked: true, countryCode: 'CN' });
insertEvent({ isBlocked: true, countryCode: 'CN' });
insertEvent({ isBlocked: false, countryCode: 'CN' });
insertEvent({ isBlocked: true, countryCode: 'RU' });
insertEvent({ isBlocked: false, countryCode: 'US' });
const rows = getCountries(from, to);
const cn = rows.find(r => r.countryCode === 'CN');
const ru = rows.find(r => r.countryCode === 'RU');
const us = rows.find(r => r.countryCode === 'US');
expect(cn!.total).toBe(3);
expect(cn!.blocked).toBe(2);
expect(ru!.total).toBe(1);
expect(ru!.blocked).toBe(1);
expect(us!.total).toBe(1);
expect(us!.blocked).toBe(0);
});
});
describe('timeline', () => {
it('shows blocked count per time bucket', () => {
const bucketSize = 3600;
const bucket1Ts = NOW;
const bucket2Ts = NOW + 3600;
insertEvent({ ts: bucket1Ts, isBlocked: true });
insertEvent({ ts: bucket1Ts, isBlocked: true });
insertEvent({ ts: bucket1Ts, isBlocked: false });
insertEvent({ ts: bucket2Ts, isBlocked: true });
insertEvent({ ts: bucket2Ts, isBlocked: false });
insertEvent({ ts: bucket2Ts, isBlocked: false });
const rows = getTimeline(from, to + 7200, bucketSize);
expect(rows.length).toBe(2);
const b1 = rows[0];
expect(b1.total).toBe(3);
expect(b1.blocked).toBe(2);
const b2 = rows[1];
expect(b2.total).toBe(3);
expect(b2.blocked).toBe(1);
});
});
describe('blocked events list', () => {
it('returns only blocked events', () => {
insertEvent({ isBlocked: true, clientIp: '5.6.7.8', countryCode: 'CN', status: 403 });
insertEvent({ isBlocked: false, clientIp: '1.2.3.4', countryCode: 'US', status: 200 });
insertEvent({ isBlocked: true, clientIp: '9.10.11.12', countryCode: 'RU', status: 403 });
const rows = getBlockedEvents(from, to);
expect(rows.length).toBe(2);
expect(rows.every(r => r.status === 403)).toBe(true);
const ips = rows.map(r => r.clientIp).sort();
expect(ips).toEqual(['5.6.7.8', '9.10.11.12']);
});
it('returns empty list when nothing is blocked', () => {
insertEvent({ isBlocked: false });
insertEvent({ isBlocked: false });
const rows = getBlockedEvents(from, to);
expect(rows.length).toBe(0);
});
it('filters blocked events by host', () => {
insertEvent({ isBlocked: true, host: 'target.com' });
insertEvent({ isBlocked: true, host: 'other.com' });
const rows = getBlockedEvents(from, to, ['target.com']);
expect(rows.length).toBe(1);
expect(rows[0].host).toBe('target.com');
});
});
describe('full pipeline: raw log lines → parseLine → DB → analytics queries', () => {
it('geo-blocked request flows through the entire pipeline', () => {
const ts = NOW;
// Simulate the two log entries that Caddy writes to access.log for a
// geo-blocked request: the blocker's "request blocked" entry followed
// by the standard "handled request" entry.
describe('parseLine', () => {
it('marks blocked request as is_blocked=true', () => {
const blockedLogLine = JSON.stringify({
ts: ts + 0.01,
msg: 'request blocked',
plugin: 'caddy-blocker',
client_ip: '203.0.113.5',
method: 'GET',
uri: '/secret',
ts: NOW + 0.01, msg: 'request blocked', plugin: 'caddy-blocker',
client_ip: '203.0.113.5', method: 'GET', uri: '/secret',
});
const handledBlockedLine = JSON.stringify({
ts: ts + 0.99,
msg: 'handled request',
status: 403,
size: 9,
request: {
client_ip: '203.0.113.5',
host: 'secure.example.com',
method: 'GET',
uri: '/secret',
proto: 'HTTP/2.0',
headers: { 'User-Agent': ['BlockedBot/1.0'] },
},
const handledLine = JSON.stringify({
ts: NOW + 0.99, msg: 'handled request', status: 403, size: 9,
request: { client_ip: '203.0.113.5', host: 'example.com', method: 'GET', uri: '/secret', proto: 'HTTP/2.0' },
});
// A normal allowed request in the same log batch.
const allowedLine = JSON.stringify({
ts: ts + 1.5,
msg: 'handled request',
status: 200,
size: 4096,
request: {
client_ip: '198.51.100.1',
host: 'secure.example.com',
method: 'GET',
uri: '/',
proto: 'HTTP/2.0',
headers: { 'User-Agent': ['GoodBot/2.0'] },
},
});
// Step 1: collectBlockedSignatures builds the blocked set from all lines
const lines = [blockedLogLine, handledBlockedLine, allowedLine];
const blockedSet = collectBlockedSignatures(lines);
expect(blockedSet.size).toBe(1);
// Step 2: parseLine processes each "handled request" line
const blockedRow = parseLine(handledBlockedLine, blockedSet);
const allowedRow = parseLine(allowedLine, blockedSet);
expect(blockedRow).not.toBeNull();
expect(allowedRow).not.toBeNull();
expect(blockedRow!.isBlocked).toBe(true);
expect(allowedRow!.isBlocked).toBe(false);
// Step 3: Insert into DB (as the real log parser does)
db.insert(trafficEvents).values(blockedRow!).run();
db.insert(trafficEvents).values(allowedRow!).run();
// Step 4: Verify all analytics queries reflect the blocked request
// Summary
const summary = getSummary(from, to);
expect(summary!.total).toBe(2);
expect(summary!.blocked).toBe(1);
// Countries (GeoIP is mocked so countryCode is null → grouped together)
const countries = getCountries(from, to);
const group = countries[0];
expect(group.total).toBe(2);
expect(group.blocked).toBe(1);
// Timeline
const timeline = getTimeline(from, to, 3600);
expect(timeline.length).toBe(1);
expect(timeline[0].total).toBe(2);
expect(timeline[0].blocked).toBe(1);
// Blocked events list
const blocked = getBlockedEvents(from, to);
expect(blocked.length).toBe(1);
expect(blocked[0].clientIp).toBe('203.0.113.5');
expect(blocked[0].status).toBe(403);
// Filtered by host
const filteredSummary = getSummary(from, to, ['secure.example.com']);
expect(filteredSummary!.blocked).toBe(1);
const wrongHost = getSummary(from, to, ['other.com']);
expect(wrongHost!.total).toBe(0);
const blockedSet = collectBlockedSignatures([blockedLogLine, handledLine]);
const row = parseLine(handledLine, blockedSet);
expect(row).not.toBeNull();
expect(row!.is_blocked).toBe(true);
expect(row!.client_ip).toBe('203.0.113.5');
expect(row!.host).toBe('example.com');
});
it('non-blocked request does not appear in blocked stats', () => {
const ts = NOW;
// Only a normal "handled request" — no "request blocked" entry
const normalLine = JSON.stringify({
ts: ts + 0.5,
msg: 'handled request',
status: 200,
size: 2048,
request: {
client_ip: '198.51.100.1',
host: 'open.example.com',
method: 'GET',
uri: '/public',
proto: 'HTTP/2.0',
},
it('marks normal request as is_blocked=false', () => {
const line = JSON.stringify({
ts: NOW, msg: 'handled request', status: 200, size: 1024,
request: { client_ip: '1.2.3.4', host: 'example.com', method: 'GET', uri: '/', proto: 'HTTP/2.0' },
});
const row = parseLine(line, new Set());
expect(row).not.toBeNull();
expect(row!.is_blocked).toBe(false);
});
const lines = [normalLine];
const blockedSet = collectBlockedSignatures(lines);
expect(blockedSet.size).toBe(0);
it('skips non-handled-request entries', () => {
const line = JSON.stringify({ ts: NOW, msg: 'request blocked', plugin: 'caddy-blocker' });
expect(parseLine(line, new Set())).toBeNull();
});
const row = parseLine(normalLine, blockedSet);
expect(row!.isBlocked).toBe(false);
db.insert(trafficEvents).values(row!).run();
const summary = getSummary(from, to);
expect(summary!.total).toBe(1);
expect(summary!.blocked).toBe(0);
const blocked = getBlockedEvents(from, to);
expect(blocked.length).toBe(0);
it('extracts user agent from headers', () => {
const line = JSON.stringify({
ts: NOW, msg: 'handled request', status: 200, size: 0,
request: { client_ip: '1.2.3.4', host: 'example.com', method: 'GET', uri: '/', proto: 'HTTP/1.1', headers: { 'User-Agent': ['TestBot/1.0'] } },
});
const row = parseLine(line, new Set());
expect(row!.user_agent).toBe('TestBot/1.0');
});
});
});

22
tests/unit/log-parser.test.ts
View File

@@ -24,6 +24,10 @@ vi.mock('node:fs', () => ({
createReadStream: vi.fn(),
}));
vi.mock('@/src/lib/clickhouse/client', () => ({
insertTrafficEvents: vi.fn().mockResolvedValue(undefined),
}));
import { parseLine, collectBlockedSignatures } from '@/src/lib/log-parser';
describe('log-parser', () => {
@@ -112,15 +116,15 @@ describe('log-parser', () => {
const result = parseLine(entry, emptyBlocked);
expect(result).not.toBeNull();
expect(result!.ts).toBe(1700000100);
expect(result!.clientIp).toBe('10.0.0.1');
expect(result!.client_ip).toBe('10.0.0.1');
expect(result!.host).toBe('example.com');
expect(result!.method).toBe('GET');
expect(result!.uri).toBe('/path');
expect(result!.status).toBe(200);
expect(result!.proto).toBe('HTTP/1.1');
expect(result!.bytesSent).toBe(1234);
expect(result!.userAgent).toBe('Mozilla/5.0');
expect(result!.isBlocked).toBe(false);
expect(result!.bytes_sent).toBe(1234);
expect(result!.user_agent).toBe('Mozilla/5.0');
expect(result!.is_blocked).toBe(false);
});
it('returns null for entries with wrong msg field', () => {
@@ -136,11 +140,11 @@ describe('log-parser', () => {
const entry = JSON.stringify({ ts: 1700000100, msg: 'handled request', status: 200 });
const result = parseLine(entry, emptyBlocked);
expect(result).not.toBeNull();
expect(result!.clientIp).toBe('');
expect(result!.client_ip).toBe('');
expect(result!.host).toBe('');
expect(result!.method).toBe('');
expect(result!.uri).toBe('');
expect(result!.userAgent).toBe('');
expect(result!.user_agent).toBe('');
});
it('marks isBlocked true when signature matches blocked set', () => {
@@ -153,7 +157,7 @@ describe('log-parser', () => {
});
const blocked = new Set([`${ts}|1.2.3.4|GET|/evil`]);
const result = parseLine(entry, blocked);
expect(result!.isBlocked).toBe(true);
expect(result!.is_blocked).toBe(true);
});
it('uses remote_ip as fallback when client_ip is missing', () => {
@@ -164,7 +168,7 @@ describe('log-parser', () => {
request: { remote_ip: '9.8.7.6', host: 'test.com', method: 'GET', uri: '/' },
});
const result = parseLine(entry, emptyBlocked);
expect(result!.clientIp).toBe('9.8.7.6');
expect(result!.client_ip).toBe('9.8.7.6');
});
it('countryCode is null when GeoIP reader is not initialized', () => {
@@ -175,7 +179,7 @@ describe('log-parser', () => {
request: { client_ip: '8.8.8.8', host: 'test.com', method: 'GET', uri: '/' },
});
const result = parseLine(entry, emptyBlocked);
expect(result!.countryCode).toBeNull();
expect(result!.country_code).toBeNull();
});
});
});