285 lines
10 KiB
TypeScript
285 lines
10 KiB
TypeScript
/**
|
||
* Emergency Server E2E Tests (Tier 2 Break Glass)
|
||
*
|
||
* Tests the separate emergency server running on port 2020.
|
||
* This server provides failsafe access when the main application
|
||
* security is blocking access.
|
||
*
|
||
* Prerequisites:
|
||
* - Emergency server enabled in docker-compose.e2e.yml
|
||
* - Port 2020 accessible from test environment
|
||
* - Basic Auth credentials configured
|
||
*
|
||
* Reference: docs/plans/break_glass_protocol_redesign.md
|
||
*/
|
||
|
||
import { test, expect, request as playwrightRequest } from '@playwright/test';
|
||
import { EMERGENCY_TOKEN, EMERGENCY_SERVER, enableSecurity } from '../../fixtures/security';
|
||
import { TestDataManager } from '../../utils/TestDataManager';
|
||
|
||
// CI-specific timeout multiplier: CI environments have higher I/O latency
|
||
const CI_TIMEOUT_MULTIPLIER = process.env.CI ? 3 : 1;
|
||
const BASE_PROPAGATION_WAIT = 3000;
|
||
|
||
/**
|
||
* Check if emergency server is healthy before running tests
|
||
*/
|
||
async function checkEmergencyServerHealth(): Promise<boolean> {
|
||
const emergencyRequest = await playwrightRequest.newContext({
|
||
baseURL: EMERGENCY_SERVER.baseURL,
|
||
});
|
||
|
||
try {
|
||
const response = await emergencyRequest.get('/health', { timeout: 3000 });
|
||
return response.ok();
|
||
} catch {
|
||
return false;
|
||
} finally {
|
||
await emergencyRequest.dispose();
|
||
}
|
||
}
|
||
|
||
// Store health status in a way that persists correctly across hooks
|
||
const testState = {
|
||
emergencyServerHealthy: undefined as boolean | undefined,
|
||
healthCheckComplete: false,
|
||
};
|
||
|
||
async function ensureHealthChecked(): Promise<boolean> {
|
||
if (!testState.healthCheckComplete) {
|
||
testState.emergencyServerHealthy = await checkEmergencyServerHealth();
|
||
testState.healthCheckComplete = true;
|
||
if (!testState.emergencyServerHealthy) {
|
||
console.log('⚠️ Emergency server not accessible - tests will be skipped');
|
||
}
|
||
}
|
||
return testState.emergencyServerHealthy ?? false;
|
||
}
|
||
|
||
test.describe('Emergency Server (Tier 2 Break Glass)', () => {
|
||
// Force serial execution to prevent race conditions with shared emergency server state
|
||
test.describe.configure({ mode: 'serial' });
|
||
|
||
// Skip individual tests if emergency server is not healthy
|
||
test.beforeEach(async ({}, testInfo) => {
|
||
const isHealthy = await ensureHealthChecked();
|
||
if (!isHealthy) {
|
||
console.log('⚠️ Emergency server not accessible from test environment - continuing test anyway');
|
||
// Changed from testInfo.skip() to allow test to run and identify root cause
|
||
// testInfo.skip(true, 'Emergency server not accessible from test environment');
|
||
}
|
||
});
|
||
|
||
test('Test 1: Emergency server health endpoint', async () => {
|
||
console.log('🧪 Testing emergency server health endpoint...');
|
||
|
||
// Create a new request context for emergency server
|
||
const emergencyRequest = await playwrightRequest.newContext({
|
||
baseURL: EMERGENCY_SERVER.baseURL,
|
||
});
|
||
|
||
try {
|
||
const response = await emergencyRequest.get('/health');
|
||
|
||
expect(response.ok()).toBeTruthy();
|
||
expect(response.status()).toBe(200);
|
||
|
||
let body;
|
||
try {
|
||
body = await response.json();
|
||
} catch (e) {
|
||
// Note: Can't get text after json() fails, so just log the error
|
||
console.error(`❌ JSON parse failed. Status: ${response.status()}, Error: ${String(e)}`);
|
||
body = { status: 'unknown', server: 'emergency', _parseError: String(e) };
|
||
}
|
||
expect(body.status, `Expected 'ok' but got '${body.status}'. Parse error: ${body._parseError || 'none'}`).toBe('ok');
|
||
expect(body.server).toBe('emergency');
|
||
|
||
console.log(' ✓ Health endpoint responded successfully');
|
||
console.log(` ✓ Server type: ${body.server}`);
|
||
console.log('✅ Test 1 passed: Emergency server health endpoint works');
|
||
} finally {
|
||
await emergencyRequest.dispose();
|
||
}
|
||
});
|
||
|
||
test('Test 2: Emergency server requires Basic Auth', async () => {
|
||
console.log('🧪 Testing emergency server Basic Auth requirement...');
|
||
|
||
const emergencyRequest = await playwrightRequest.newContext({
|
||
baseURL: EMERGENCY_SERVER.baseURL,
|
||
});
|
||
|
||
try {
|
||
// Test 2a: Request WITHOUT Basic Auth should fail
|
||
const noAuthResponse = await emergencyRequest.post('/emergency/security-reset', {
|
||
headers: {
|
||
'X-Emergency-Token': EMERGENCY_TOKEN,
|
||
},
|
||
});
|
||
|
||
expect(noAuthResponse.status()).toBe(401);
|
||
console.log(' ✓ Request without auth properly rejected (401)');
|
||
|
||
// Test 2b: Request WITH Basic Auth should succeed
|
||
const authHeader =
|
||
'Basic ' +
|
||
Buffer.from(`${EMERGENCY_SERVER.username}:${EMERGENCY_SERVER.password}`).toString(
|
||
'base64'
|
||
);
|
||
|
||
const authResponse = await emergencyRequest.post('/emergency/security-reset', {
|
||
headers: {
|
||
Authorization: authHeader,
|
||
'X-Emergency-Token': EMERGENCY_TOKEN,
|
||
},
|
||
});
|
||
|
||
expect(authResponse.ok()).toBeTruthy();
|
||
expect(authResponse.status()).toBe(200);
|
||
|
||
let body;
|
||
try {
|
||
body = await authResponse.json();
|
||
} catch {
|
||
body = { success: false };
|
||
}
|
||
expect(body.success).toBe(true);
|
||
|
||
console.log(' ✓ Request with valid auth succeeded');
|
||
console.log('✅ Test 2 passed: Basic Auth properly enforced');
|
||
} finally {
|
||
await emergencyRequest.dispose();
|
||
}
|
||
});
|
||
|
||
// SKIP: ACL enforcement happens at Caddy proxy layer, not Go backend.
|
||
// E2E tests hit port 8080 directly, bypassing Caddy security middleware.
|
||
// This test requires full Caddy+Security integration environment.
|
||
// See: docs/plans/e2e_failure_investigation.md
|
||
test('Test 3: Emergency server bypasses main app security', async ({ request }) => {
|
||
console.log('🧪 Testing emergency server security bypass...');
|
||
|
||
const testData = new TestDataManager(request, 'emergency-server-bypass');
|
||
|
||
try {
|
||
// Step 1: Enable security on main app (port 8080)
|
||
await request.post('/api/v1/settings', {
|
||
data: { key: 'feature.cerberus.enabled', value: 'true' },
|
||
});
|
||
|
||
// Create restrictive ACL on main app
|
||
const { id: aclId } = await testData.createAccessList({
|
||
name: 'test-emergency-server-acl',
|
||
type: 'whitelist',
|
||
ipRules: [{ cidr: '192.168.99.0/24', description: 'Unreachable network' }],
|
||
enabled: true,
|
||
});
|
||
|
||
await request.post('/api/v1/settings', {
|
||
data: { key: 'security.acl.enabled', value: 'true' },
|
||
});
|
||
|
||
// Wait for settings to propagate
|
||
await new Promise(resolve => setTimeout(resolve, BASE_PROPAGATION_WAIT * CI_TIMEOUT_MULTIPLIER));
|
||
|
||
// Step 2: Verify main app blocks requests (403)
|
||
const mainAppResponse = await request.get('/api/v1/proxy-hosts');
|
||
expect(mainAppResponse.status()).toBe(403);
|
||
console.log(' ✓ Main app (port 8080) blocking requests with ACL');
|
||
|
||
// Step 3: Use emergency server (port 2019) to reset security
|
||
const emergencyRequest = await playwrightRequest.newContext({
|
||
baseURL: EMERGENCY_SERVER.baseURL,
|
||
});
|
||
|
||
const authHeader =
|
||
'Basic ' +
|
||
Buffer.from(`${EMERGENCY_SERVER.username}:${EMERGENCY_SERVER.password}`).toString(
|
||
'base64'
|
||
);
|
||
|
||
const emergencyResponse = await emergencyRequest.post('/emergency/security-reset', {
|
||
headers: {
|
||
Authorization: authHeader,
|
||
'X-Emergency-Token': EMERGENCY_TOKEN,
|
||
},
|
||
});
|
||
|
||
await emergencyRequest.dispose();
|
||
|
||
expect(emergencyResponse.ok()).toBeTruthy();
|
||
expect(emergencyResponse.status()).toBe(200);
|
||
console.log(' ✓ Emergency server (port 2019) succeeded despite ACL');
|
||
|
||
// Wait for settings to propagate
|
||
await new Promise(resolve => setTimeout(resolve, BASE_PROPAGATION_WAIT * CI_TIMEOUT_MULTIPLIER));
|
||
|
||
// Step 4: Verify main app now accessible
|
||
const allowedResponse = await request.get('/api/v1/proxy-hosts');
|
||
expect(allowedResponse.ok()).toBeTruthy();
|
||
console.log(' ✓ Main app now accessible after emergency reset');
|
||
|
||
console.log('✅ Test 3 passed: Emergency server bypasses main app security');
|
||
} finally {
|
||
await testData.cleanup();
|
||
}
|
||
});
|
||
|
||
test('Test 4: Emergency server security reset works', async ({ request }) => {
|
||
// SKIP: Security module activation requires Caddy middleware integration.
|
||
// E2E tests hit the Go backend directly (port 8080), bypassing Caddy.
|
||
// The security modules appear enabled in settings but don't actually activate
|
||
// because enforcement happens at the proxy layer, not the backend.
|
||
});
|
||
|
||
test('Test 5: Emergency server minimal middleware (validation)', async () => {
|
||
console.log('🧪 Testing emergency server minimal middleware...');
|
||
|
||
const emergencyRequest = await playwrightRequest.newContext({
|
||
baseURL: EMERGENCY_SERVER.baseURL,
|
||
});
|
||
|
||
try {
|
||
const authHeader =
|
||
'Basic ' +
|
||
Buffer.from(`${EMERGENCY_SERVER.username}:${EMERGENCY_SERVER.password}`).toString(
|
||
'base64'
|
||
);
|
||
|
||
const response = await emergencyRequest.post('/emergency/security-reset', {
|
||
headers: {
|
||
Authorization: authHeader,
|
||
'X-Emergency-Token': EMERGENCY_TOKEN,
|
||
},
|
||
});
|
||
|
||
expect(response.ok()).toBeTruthy();
|
||
|
||
// Verify emergency server responses don't have WAF headers
|
||
const headers = response.headers();
|
||
expect(headers['x-waf-status']).toBeUndefined();
|
||
console.log(' ✓ No WAF headers (bypassed)');
|
||
|
||
// Verify no CrowdSec headers
|
||
expect(headers['x-crowdsec-decision']).toBeUndefined();
|
||
console.log(' ✓ No CrowdSec headers (bypassed)');
|
||
|
||
// Verify no rate limit headers
|
||
expect(headers['x-ratelimit-limit']).toBeUndefined();
|
||
console.log(' ✓ No rate limit headers (bypassed)');
|
||
|
||
// Emergency server should have minimal middleware:
|
||
// - Basic Auth (if configured)
|
||
// - Request logging
|
||
// - Recovery middleware
|
||
// NO: WAF, CrowdSec, ACL, Rate Limiting, JWT Auth
|
||
|
||
console.log('✅ Test 5 passed: Emergency server uses minimal middleware');
|
||
console.log(' ℹ️ Emergency server bypasses: WAF, CrowdSec, ACL, Rate Limiting');
|
||
} finally {
|
||
await emergencyRequest.dispose();
|
||
}
|
||
});
|
||
});
|