fix(tests): enhance system settings tests with feature flag propagation and retry logic
- Added initial feature flag state verification before tests to ensure a stable starting point. - Implemented retry logic with exponential backoff for toggling feature flags, improving resilience against transient failures. - Introduced `waitForFeatureFlagPropagation` utility to replace hard-coded waits with condition-based verification for feature flag states. - Added advanced test scenarios for handling concurrent toggle operations and retrying on network failures. - Updated existing tests to utilize the new retry and propagation utilities for better reliability and maintainability.
This commit is contained in:
GitHub Actions
@@ -13,7 +13,14 @@
|
||||
*/
|
||||
|
||||
import { test, expect, loginUser } from '../fixtures/auth-fixtures';
|
||||
import { waitForLoadingComplete, waitForToast, waitForAPIResponse, clickAndWaitForResponse } from '../utils/wait-helpers';
|
||||
import {
|
||||
waitForLoadingComplete,
|
||||
waitForToast,
|
||||
waitForAPIResponse,
|
||||
clickAndWaitForResponse,
|
||||
waitForFeatureFlagPropagation,
|
||||
retryAction,
|
||||
} from '../utils/wait-helpers';
|
||||
import { getToastLocator } from '../utils/ui-helpers';
|
||||
|
||||
test.describe('System Settings', () => {
|
||||
@@ -22,6 +29,22 @@ test.describe('System Settings', () => {
|
||||
await waitForLoadingComplete(page);
|
||||
await page.goto('/settings/system');
|
||||
await waitForLoadingComplete(page);
|
||||
|
||||
// Phase 4: Verify initial feature flag state before tests start
|
||||
// This ensures tests start with a stable, known state
|
||||
await waitForFeatureFlagPropagation(
|
||||
page,
|
||||
{
|
||||
'cerberus.enabled': true, // Default: enabled
|
||||
'crowdsec.console_enrollment': false, // Default: disabled
|
||||
'uptime.enabled': false, // Default: disabled
|
||||
},
|
||||
{ timeout: 10000 } // Shorter timeout for initial check
|
||||
).catch(() => {
|
||||
// Initial state verification is best-effort
|
||||
// Some tests may have left toggles in different states
|
||||
console.log('[WARN] Initial state verification skipped - flags may be in non-default state');
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('Navigation & Page Load', () => {
|
||||
@@ -146,26 +169,27 @@ test.describe('System Settings', () => {
|
||||
const toggle = cerberusToggle.first();
|
||||
|
||||
const initialState = await toggle.isChecked().catch(() => false);
|
||||
const expectedState = !initialState;
|
||||
|
||||
// Step 1: Click toggle and wait for PUT request (atomic operation)
|
||||
const putResponse = await clickAndWaitForResponse(
|
||||
page,
|
||||
toggle,
|
||||
/\/feature-flags/,
|
||||
{ status: 200, timeout: 15000 } // 15s for CI safety
|
||||
);
|
||||
expect(putResponse.ok()).toBeTruthy();
|
||||
// Use retry logic with exponential backoff
|
||||
await retryAction(async () => {
|
||||
// Click toggle and wait for PUT request
|
||||
const putResponse = await clickAndWaitForResponse(
|
||||
page,
|
||||
toggle,
|
||||
/\/feature-flags/
|
||||
);
|
||||
expect(putResponse.ok()).toBeTruthy();
|
||||
|
||||
// Step 2: Wait for subsequent GET request to refresh state
|
||||
const getResponse = await waitForAPIResponse(
|
||||
page,
|
||||
/\/feature-flags/,
|
||||
{ status: 200, timeout: 10000 } // 10s for CI safety
|
||||
);
|
||||
expect(getResponse.ok()).toBeTruthy();
|
||||
// Verify state propagated with condition-based polling
|
||||
await waitForFeatureFlagPropagation(page, {
|
||||
'cerberus.enabled': expectedState,
|
||||
});
|
||||
|
||||
const newState = await toggle.isChecked().catch(() => !initialState);
|
||||
expect(newState).not.toBe(initialState);
|
||||
// Verify UI reflects the change
|
||||
const newState = await toggle.isChecked().catch(() => initialState);
|
||||
expect(newState).toBe(expectedState);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -190,26 +214,27 @@ test.describe('System Settings', () => {
|
||||
const toggle = crowdsecToggle.first();
|
||||
|
||||
const initialState = await toggle.isChecked().catch(() => false);
|
||||
const expectedState = !initialState;
|
||||
|
||||
// Step 1: Click toggle and wait for PUT request (atomic operation)
|
||||
const putResponse = await clickAndWaitForResponse(
|
||||
page,
|
||||
toggle,
|
||||
/\/feature-flags/,
|
||||
{ status: 200, timeout: 15000 } // 15s for CI safety
|
||||
);
|
||||
expect(putResponse.ok()).toBeTruthy();
|
||||
// Use retry logic with exponential backoff
|
||||
await retryAction(async () => {
|
||||
// Click toggle and wait for PUT request
|
||||
const putResponse = await clickAndWaitForResponse(
|
||||
page,
|
||||
toggle,
|
||||
/\/feature-flags/
|
||||
);
|
||||
expect(putResponse.ok()).toBeTruthy();
|
||||
|
||||
// Step 2: Wait for subsequent GET request to refresh state
|
||||
const getResponse = await waitForAPIResponse(
|
||||
page,
|
||||
/\/feature-flags/,
|
||||
{ status: 200, timeout: 10000 } // 10s for CI safety
|
||||
);
|
||||
expect(getResponse.ok()).toBeTruthy();
|
||||
// Verify state propagated with condition-based polling
|
||||
await waitForFeatureFlagPropagation(page, {
|
||||
'crowdsec.console_enrollment': expectedState,
|
||||
});
|
||||
|
||||
const newState = await toggle.isChecked().catch(() => !initialState);
|
||||
expect(newState).not.toBe(initialState);
|
||||
// Verify UI reflects the change
|
||||
const newState = await toggle.isChecked().catch(() => initialState);
|
||||
expect(newState).toBe(expectedState);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -234,26 +259,27 @@ test.describe('System Settings', () => {
|
||||
const toggle = uptimeToggle.first();
|
||||
|
||||
const initialState = await toggle.isChecked().catch(() => false);
|
||||
const expectedState = !initialState;
|
||||
|
||||
// Step 1: Click toggle and wait for PUT request (atomic operation)
|
||||
const putResponse = await clickAndWaitForResponse(
|
||||
page,
|
||||
toggle,
|
||||
/\/feature-flags/,
|
||||
{ status: 200, timeout: 15000 } // 15s for CI safety
|
||||
);
|
||||
expect(putResponse.ok()).toBeTruthy();
|
||||
// Use retry logic with exponential backoff
|
||||
await retryAction(async () => {
|
||||
// Click toggle and wait for PUT request
|
||||
const putResponse = await clickAndWaitForResponse(
|
||||
page,
|
||||
toggle,
|
||||
/\/feature-flags/
|
||||
);
|
||||
expect(putResponse.ok()).toBeTruthy();
|
||||
|
||||
// Step 2: Wait for subsequent GET request to refresh state
|
||||
const getResponse = await waitForAPIResponse(
|
||||
page,
|
||||
/\/feature-flags/,
|
||||
{ status: 200, timeout: 10000 } // 10s for CI safety
|
||||
);
|
||||
expect(getResponse.ok()).toBeTruthy();
|
||||
// Verify state propagated with condition-based polling
|
||||
await waitForFeatureFlagPropagation(page, {
|
||||
'uptime.enabled': expectedState,
|
||||
});
|
||||
|
||||
const newState = await toggle.isChecked().catch(() => !initialState);
|
||||
expect(newState).not.toBe(initialState);
|
||||
// Verify UI reflects the change
|
||||
const newState = await toggle.isChecked().catch(() => initialState);
|
||||
expect(newState).toBe(expectedState);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -275,49 +301,54 @@ test.describe('System Settings', () => {
|
||||
});
|
||||
|
||||
await test.step('Toggle the feature', async () => {
|
||||
// Step 1: Click toggle and wait for PUT request (atomic operation)
|
||||
const putResponse = await clickAndWaitForResponse(
|
||||
page,
|
||||
toggle,
|
||||
/\/feature-flags/,
|
||||
{ status: 200, timeout: 15000 } // 15s for CI safety
|
||||
);
|
||||
expect(putResponse.ok()).toBeTruthy();
|
||||
const expectedState = !initialState;
|
||||
|
||||
// Step 2: Wait for subsequent GET request to refresh state
|
||||
const getResponse = await waitForAPIResponse(
|
||||
page,
|
||||
/\/feature-flags/,
|
||||
{ status: 200, timeout: 10000 } // 10s for CI safety
|
||||
);
|
||||
expect(getResponse.ok()).toBeTruthy();
|
||||
// Use retry logic with exponential backoff
|
||||
await retryAction(async () => {
|
||||
// Click toggle and wait for PUT request
|
||||
const putResponse = await clickAndWaitForResponse(
|
||||
page,
|
||||
toggle,
|
||||
/\/feature-flags/
|
||||
);
|
||||
expect(putResponse.ok()).toBeTruthy();
|
||||
|
||||
// Verify state propagated with condition-based polling
|
||||
await waitForFeatureFlagPropagation(page, {
|
||||
'uptime.enabled': expectedState,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
await test.step('Reload page and verify persistence', async () => {
|
||||
await page.reload();
|
||||
await waitForLoadingComplete(page);
|
||||
|
||||
// Verify state persisted after reload
|
||||
await waitForFeatureFlagPropagation(page, {
|
||||
'uptime.enabled': !initialState,
|
||||
});
|
||||
|
||||
const newState = await toggle.isChecked().catch(() => initialState);
|
||||
expect(newState).not.toBe(initialState);
|
||||
});
|
||||
|
||||
await test.step('Restore original state', async () => {
|
||||
// Step 1: Click toggle and wait for PUT request (atomic operation)
|
||||
const putResponse = await clickAndWaitForResponse(
|
||||
page,
|
||||
toggle,
|
||||
/\/feature-flags/,
|
||||
{ status: 200, timeout: 15000 } // 15s for CI safety
|
||||
);
|
||||
expect(putResponse.ok()).toBeTruthy();
|
||||
// Use retry logic with exponential backoff
|
||||
await retryAction(async () => {
|
||||
// Click toggle and wait for PUT request
|
||||
const putResponse = await clickAndWaitForResponse(
|
||||
page,
|
||||
toggle,
|
||||
/\/feature-flags/
|
||||
);
|
||||
expect(putResponse.ok()).toBeTruthy();
|
||||
|
||||
// Step 2: Wait for subsequent GET request to refresh state
|
||||
const getResponse = await waitForAPIResponse(
|
||||
page,
|
||||
/\/feature-flags/,
|
||||
{ status: 200, timeout: 10000 } // 10s for CI safety
|
||||
);
|
||||
expect(getResponse.ok()).toBeTruthy();
|
||||
// Verify state propagated with condition-based polling
|
||||
await waitForFeatureFlagPropagation(page, {
|
||||
'uptime.enabled': initialState,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -362,6 +393,218 @@ test.describe('System Settings', () => {
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('Feature Toggles - Advanced Scenarios (Phase 4)', () => {
|
||||
/**
|
||||
* Test: Handle concurrent toggle operations
|
||||
* Priority: P1
|
||||
*/
|
||||
test('should handle concurrent toggle operations', async ({ page }) => {
|
||||
await test.step('Toggle three flags simultaneously', async () => {
|
||||
const cerberusToggle = page
|
||||
.getByRole('switch', { name: /cerberus.*toggle/i })
|
||||
.or(page.locator('[aria-label*="Cerberus"][aria-label*="toggle"]'))
|
||||
.first();
|
||||
|
||||
const crowdsecToggle = page
|
||||
.getByRole('switch', { name: /crowdsec.*toggle/i })
|
||||
.or(page.locator('[aria-label*="CrowdSec"][aria-label*="toggle"]'))
|
||||
.first();
|
||||
|
||||
const uptimeToggle = page
|
||||
.getByRole('switch', { name: /uptime.*toggle/i })
|
||||
.or(page.locator('[aria-label*="Uptime"][aria-label*="toggle"]'))
|
||||
.first();
|
||||
|
||||
// Get initial states
|
||||
const cerberusInitial = await cerberusToggle.isChecked().catch(() => false);
|
||||
const crowdsecInitial = await crowdsecToggle.isChecked().catch(() => false);
|
||||
const uptimeInitial = await uptimeToggle.isChecked().catch(() => false);
|
||||
|
||||
// Toggle all three simultaneously
|
||||
const togglePromises = [
|
||||
retryAction(async () => {
|
||||
const response = await clickAndWaitForResponse(
|
||||
page,
|
||||
cerberusToggle,
|
||||
/\/feature-flags/
|
||||
);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
}),
|
||||
retryAction(async () => {
|
||||
const response = await clickAndWaitForResponse(
|
||||
page,
|
||||
crowdsecToggle,
|
||||
/\/feature-flags/
|
||||
);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
}),
|
||||
retryAction(async () => {
|
||||
const response = await clickAndWaitForResponse(
|
||||
page,
|
||||
uptimeToggle,
|
||||
/\/feature-flags/
|
||||
);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
}),
|
||||
];
|
||||
|
||||
await Promise.all(togglePromises);
|
||||
|
||||
// Verify all flags propagated correctly
|
||||
await waitForFeatureFlagPropagation(page, {
|
||||
'cerberus.enabled': !cerberusInitial,
|
||||
'crowdsec.console_enrollment': !crowdsecInitial,
|
||||
'uptime.enabled': !uptimeInitial,
|
||||
});
|
||||
});
|
||||
|
||||
await test.step('Restore original states', async () => {
|
||||
// Reload to get fresh state
|
||||
await page.reload();
|
||||
await waitForLoadingComplete(page);
|
||||
|
||||
// Toggle all back (they're now in opposite state)
|
||||
const cerberusToggle = page
|
||||
.getByRole('switch', { name: /cerberus.*toggle/i })
|
||||
.first();
|
||||
const crowdsecToggle = page
|
||||
.getByRole('switch', { name: /crowdsec.*toggle/i })
|
||||
.first();
|
||||
const uptimeToggle = page
|
||||
.getByRole('switch', { name: /uptime.*toggle/i })
|
||||
.first();
|
||||
|
||||
await Promise.all([
|
||||
clickAndWaitForResponse(page, cerberusToggle, /\/feature-flags/),
|
||||
clickAndWaitForResponse(page, crowdsecToggle, /\/feature-flags/),
|
||||
clickAndWaitForResponse(page, uptimeToggle, /\/feature-flags/),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Test: Retry on network failure (500 error)
|
||||
* Priority: P1
|
||||
*/
|
||||
test('should retry on 500 Internal Server Error', async ({ page }) => {
|
||||
let attemptCount = 0;
|
||||
|
||||
await test.step('Simulate transient backend failure', async () => {
|
||||
// Intercept first PUT request and fail it
|
||||
await page.route('/api/v1/feature-flags', async (route) => {
|
||||
const request = route.request();
|
||||
if (request.method() === 'PUT') {
|
||||
attemptCount++;
|
||||
if (attemptCount === 1) {
|
||||
// First attempt: fail with 500
|
||||
await route.fulfill({
|
||||
status: 500,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ error: 'Database error' }),
|
||||
});
|
||||
} else {
|
||||
// Subsequent attempts: allow through
|
||||
await route.continue();
|
||||
}
|
||||
} else {
|
||||
// Allow GET requests
|
||||
await route.continue();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
await test.step('Toggle should succeed after retry', async () => {
|
||||
const uptimeToggle = page
|
||||
.getByRole('switch', { name: /uptime.*toggle/i })
|
||||
.first();
|
||||
|
||||
const initialState = await uptimeToggle.isChecked().catch(() => false);
|
||||
const expectedState = !initialState;
|
||||
|
||||
// Should retry and succeed on second attempt
|
||||
await retryAction(async () => {
|
||||
const response = await clickAndWaitForResponse(
|
||||
page,
|
||||
uptimeToggle,
|
||||
/\/feature-flags/
|
||||
);
|
||||
expect(response.ok()).toBeTruthy();
|
||||
|
||||
await waitForFeatureFlagPropagation(page, {
|
||||
'uptime.enabled': expectedState,
|
||||
});
|
||||
});
|
||||
|
||||
// Verify retry was attempted
|
||||
expect(attemptCount).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
await test.step('Cleanup route interception', async () => {
|
||||
await page.unroute('/api/v1/feature-flags');
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Test: Fail gracefully after max retries
|
||||
* Priority: P1
|
||||
*/
|
||||
test('should fail gracefully after max retries exceeded', async ({ page }) => {
|
||||
await test.step('Simulate persistent backend failure', async () => {
|
||||
// Intercept ALL requests and fail them
|
||||
await page.route('/api/v1/feature-flags', async (route) => {
|
||||
const request = route.request();
|
||||
if (request.method() === 'PUT') {
|
||||
await route.fulfill({
|
||||
status: 500,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ error: 'Database error' }),
|
||||
});
|
||||
} else {
|
||||
await route.continue();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
await test.step('Toggle should fail after 3 attempts', async () => {
|
||||
const uptimeToggle = page
|
||||
.getByRole('switch', { name: /uptime.*toggle/i })
|
||||
.first();
|
||||
|
||||
// Should throw after 3 attempts
|
||||
await expect(
|
||||
retryAction(async () => {
|
||||
await clickAndWaitForResponse(page, uptimeToggle, /\/feature-flags/);
|
||||
})
|
||||
).rejects.toThrow(/Action failed after 3 attempts/);
|
||||
});
|
||||
|
||||
await test.step('Cleanup route interception', async () => {
|
||||
await page.unroute('/api/v1/feature-flags');
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Test: Initial state verification in beforeEach
|
||||
* Priority: P0
|
||||
*/
|
||||
test('should verify initial feature flag state before tests', async ({ page }) => {
|
||||
await test.step('Verify expected initial state', async () => {
|
||||
// This demonstrates the pattern that should be in beforeEach
|
||||
// Verify all feature flags are in expected initial state
|
||||
const flags = await waitForFeatureFlagPropagation(page, {
|
||||
'cerberus.enabled': true, // Default: enabled
|
||||
'crowdsec.console_enrollment': false, // Default: disabled
|
||||
'uptime.enabled': false, // Default: disabled
|
||||
});
|
||||
|
||||
// Verify flags object contains expected keys
|
||||
expect(flags).toHaveProperty('cerberus.enabled');
|
||||
expect(flags).toHaveProperty('crowdsec.console_enrollment');
|
||||
expect(flags).toHaveProperty('uptime.enabled');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('General Configuration', () => {
|
||||
/**
|
||||
* Test: Update Caddy Admin API URL
|
||||
|
||||
Reference in New Issue
Block a user