Migrate analytics from SQLite to ClickHouse
SQLite was too slow for analytical aggregations on traffic_events and waf_events (millions of rows, GROUP BY, COUNT DISTINCT). ClickHouse is a columnar OLAP database purpose-built for this workload. - Add ClickHouse container to Docker Compose with health check - Create src/lib/clickhouse/client.ts with singleton client, table DDL, insert helpers, and all analytics query functions - Update log-parser.ts and waf-log-parser.ts to write to ClickHouse - Remove purgeOldEntries — ClickHouse TTL handles 90-day retention - Rewrite analytics-db.ts and waf-events.ts to query ClickHouse - Remove trafficEvents/wafEvents from SQLite schema, add migration - CLICKHOUSE_PASSWORD is required (no hardcoded default) - Update .env.example, README, and test infrastructure API response shapes are unchanged — no frontend modifications needed. Parse state (file offsets) remains in SQLite. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -220,55 +220,14 @@ export const linkingTokens = sqliteTable("linking_tokens", {
|
||||
expiresAt: text("expires_at").notNull()
|
||||
});
|
||||
|
||||
export const trafficEvents = sqliteTable(
|
||||
'traffic_events',
|
||||
{
|
||||
id: integer('id').primaryKey({ autoIncrement: true }),
|
||||
ts: integer('ts').notNull(),
|
||||
clientIp: text('client_ip').notNull(),
|
||||
countryCode: text('country_code'),
|
||||
host: text('host').notNull().default(''),
|
||||
method: text('method').notNull().default(''),
|
||||
uri: text('uri').notNull().default(''),
|
||||
status: integer('status').notNull().default(0),
|
||||
proto: text('proto').notNull().default(''),
|
||||
bytesSent: integer('bytes_sent').notNull().default(0),
|
||||
userAgent: text('user_agent').notNull().default(''),
|
||||
isBlocked: integer('is_blocked', { mode: 'boolean' }).notNull().default(false),
|
||||
},
|
||||
(table) => ({
|
||||
tsIdx: index('idx_traffic_events_ts').on(table.ts),
|
||||
hostTsIdx: index('idx_traffic_events_host_ts').on(table.host, table.ts),
|
||||
})
|
||||
);
|
||||
// traffic_events and waf_events have been migrated to ClickHouse.
|
||||
// See src/lib/clickhouse/client.ts for the ClickHouse schema.
|
||||
|
||||
export const logParseState = sqliteTable('log_parse_state', {
|
||||
key: text('key').primaryKey(),
|
||||
value: text('value').notNull(),
|
||||
});
|
||||
|
||||
export const wafEvents = sqliteTable(
|
||||
'waf_events',
|
||||
{
|
||||
id: integer('id').primaryKey({ autoIncrement: true }),
|
||||
ts: integer('ts').notNull(),
|
||||
host: text('host').notNull().default(''),
|
||||
clientIp: text('client_ip').notNull(),
|
||||
countryCode: text('country_code'),
|
||||
method: text('method').notNull().default(''),
|
||||
uri: text('uri').notNull().default(''),
|
||||
ruleId: integer('rule_id'),
|
||||
ruleMessage: text('rule_message'),
|
||||
severity: text('severity'),
|
||||
rawData: text('raw_data'),
|
||||
blocked: integer('blocked', { mode: 'boolean' }).notNull().default(true),
|
||||
},
|
||||
(table) => ({
|
||||
tsIdx: index('idx_waf_events_ts').on(table.ts),
|
||||
hostTsIdx: index('idx_waf_events_host_ts').on(table.host, table.ts),
|
||||
})
|
||||
);
|
||||
|
||||
export const wafLogParseState = sqliteTable('waf_log_parse_state', {
|
||||
key: text('key').primaryKey(),
|
||||
value: text('value').notNull(),
|
||||
|
||||
Reference in New Issue
Block a user