Migrate analytics from SQLite to ClickHouse

SQLite was too slow for analytical aggregations on traffic_events and
waf_events (millions of rows, GROUP BY, COUNT DISTINCT). ClickHouse is
a columnar OLAP database purpose-built for this workload.

- Add ClickHouse container to Docker Compose with health check
- Create src/lib/clickhouse/client.ts with singleton client, table DDL,
  insert helpers, and all analytics query functions
- Update log-parser.ts and waf-log-parser.ts to write to ClickHouse
- Remove purgeOldEntries — ClickHouse TTL handles 90-day retention
- Rewrite analytics-db.ts and waf-events.ts to query ClickHouse
- Remove trafficEvents/wafEvents from SQLite schema, add migration
- CLICKHOUSE_PASSWORD is required (no hardcoded default)
- Update .env.example, README, and test infrastructure

API response shapes are unchanged — no frontend modifications needed.
Parse state (file offsets) remains in SQLite.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
fuomag9
2026-04-10 00:05:38 +02:00
parent 833284efb1
commit e1c97038d4
21 changed files with 819 additions and 763 deletions

View File

@@ -220,55 +220,14 @@ export const linkingTokens = sqliteTable("linking_tokens", {
expiresAt: text("expires_at").notNull()
});
export const trafficEvents = sqliteTable(
'traffic_events',
{
id: integer('id').primaryKey({ autoIncrement: true }),
ts: integer('ts').notNull(),
clientIp: text('client_ip').notNull(),
countryCode: text('country_code'),
host: text('host').notNull().default(''),
method: text('method').notNull().default(''),
uri: text('uri').notNull().default(''),
status: integer('status').notNull().default(0),
proto: text('proto').notNull().default(''),
bytesSent: integer('bytes_sent').notNull().default(0),
userAgent: text('user_agent').notNull().default(''),
isBlocked: integer('is_blocked', { mode: 'boolean' }).notNull().default(false),
},
(table) => ({
tsIdx: index('idx_traffic_events_ts').on(table.ts),
hostTsIdx: index('idx_traffic_events_host_ts').on(table.host, table.ts),
})
);
// traffic_events and waf_events have been migrated to ClickHouse.
// See src/lib/clickhouse/client.ts for the ClickHouse schema.
export const logParseState = sqliteTable('log_parse_state', {
key: text('key').primaryKey(),
value: text('value').notNull(),
});
export const wafEvents = sqliteTable(
'waf_events',
{
id: integer('id').primaryKey({ autoIncrement: true }),
ts: integer('ts').notNull(),
host: text('host').notNull().default(''),
clientIp: text('client_ip').notNull(),
countryCode: text('country_code'),
method: text('method').notNull().default(''),
uri: text('uri').notNull().default(''),
ruleId: integer('rule_id'),
ruleMessage: text('rule_message'),
severity: text('severity'),
rawData: text('raw_data'),
blocked: integer('blocked', { mode: 'boolean' }).notNull().default(true),
},
(table) => ({
tsIdx: index('idx_waf_events_ts').on(table.ts),
hostTsIdx: index('idx_waf_events_host_ts').on(table.host, table.ts),
})
);
export const wafLogParseState = sqliteTable('waf_log_parse_state', {
key: text('key').primaryKey(),
value: text('value').notNull(),