diff --git a/.github/instructions/playwright-typescript.instructions.md b/.github/instructions/playwright-typescript.instructions.md index a0509765..ccb01b5b 100644 --- a/.github/instructions/playwright-typescript.instructions.md +++ b/.github/instructions/playwright-typescript.instructions.md @@ -9,7 +9,6 @@ applyTo: '**' - **Locators**: Prioritize user-facing, role-based locators (`getByRole`, `getByLabel`, `getByText`, etc.) for resilience and accessibility. Use `test.step()` to group interactions and improve test readability and reporting. - **Assertions**: Use auto-retrying web-first assertions. These assertions start with the `await` keyword (e.g., `await expect(locator).toHaveText()`). Avoid `expect(locator).toBeVisible()` unless specifically testing for visibility changes. - **Timeouts**: Rely on Playwright's built-in auto-waiting mechanisms. Avoid hard-coded waits or increased default timeouts. -- **Switch/Toggle Components**: Use helper functions from `tests/utils/ui-helpers.ts` (`clickSwitch`, `expectSwitchState`, `toggleSwitch`) for reliable interactions. Never use `{ force: true }` or direct clicks on hidden inputs. - **Clarity**: Use descriptive test and step titles that clearly state the intent. Add comments only to explain complex logic or non-obvious interactions. @@ -30,123 +29,6 @@ applyTo: '**' - **Element Counts**: Use `toHaveCount` to assert the number of elements found by a locator. - **Text Content**: Use `toHaveText` for exact text matches and `toContainText` for partial matches. - **Navigation**: Use `toHaveURL` to verify the page URL after an action. -- **Switch States**: Use `expectSwitchState(locator, boolean)` to verify toggle states. This is more reliable than `toBeChecked()` directly. - -### Switch/Toggle Interaction Patterns - -Switch components use a hidden `` with styled siblings, requiring special handling: - -```typescript -import { clickSwitch, expectSwitchState, toggleSwitch } from './utils/ui-helpers'; - -// ✅ RECOMMENDED: Click switch with helper -const aclSwitch = page.getByRole('switch', { name: /acl/i }); -await clickSwitch(aclSwitch); - -// ✅ RECOMMENDED: Assert switch state -await expectSwitchState(aclSwitch, true); // Checked - -// ✅ RECOMMENDED: Toggle and verify state change -const newState = await toggleSwitch(aclSwitch); -console.log(`Switch is now ${newState ? 'enabled' : 'disabled'}`); - -// ❌ AVOID: Direct click on hidden input -await aclSwitch.click(); // May fail in WebKit/Firefox - -// ❌ AVOID: Force clicking (anti-pattern) -await aclSwitch.click({ force: true }); // Bypasses real user behavior - -// ❌ AVOID: Hard-coded waits -await page.waitForTimeout(500); // Non-deterministic, slows tests -``` - -**When to Use**: -- Settings pages with enable/disable toggles -- Security dashboard module switches (CrowdSec, ACL, WAF, Rate Limiting) -- Access lists and configuration toggles -- Any UI component using the `Switch` primitive from shadcn/ui - -**References**: -- [Helper Implementation](../../tests/utils/ui-helpers.ts) -- [QA Report](../../docs/reports/qa_report.md) - -### Testing Scope: E2E vs Integration - -**CRITICAL:** Playwright E2E tests verify **UI/UX functionality** on the Charon management interface (port 8080). They should NOT test middleware enforcement behavior. - -#### What E2E Tests SHOULD Cover - -✅ **User Interface Interactions:** -- Form submissions and validation -- Navigation and routing -- Visual state changes (toggles, badges, status indicators) -- Authentication flows (login, logout, session management) -- CRUD operations via the management API -- Responsive design (mobile vs desktop layouts) -- Accessibility (ARIA labels, keyboard navigation) - -✅ **Example E2E Assertions:** -```typescript -// GOOD: Testing UI state -await expect(aclToggle).toBeChecked(); -await expect(statusBadge).toHaveText('Active'); -await expect(page).toHaveURL('/proxy-hosts'); - -// GOOD: Testing API responses in management interface -const response = await request.post('/api/v1/proxy-hosts', { data: hostConfig }); -expect(response.ok()).toBeTruthy(); -``` - -#### What E2E Tests should NOT Cover - -❌ **Middleware Enforcement Behavior:** -- Rate limiting blocking requests (429 responses) -- ACL denying access based on IP rules (403 responses) -- WAF blocking malicious payloads (SQL injection, XSS) -- CrowdSec IP bans - -❌ **Example Wrong E2E Assertions:** -```typescript -// BAD: Testing middleware behavior (rate limiting) -for (let i = 0; i < 6; i++) { - await request.post('/api/v1/emergency/reset'); -} -expect(response.status()).toBe(429); // ❌ This tests Caddy middleware - -// BAD: Testing WAF blocking -await request.post('/api/v1/data', { data: "'; DROP TABLE users--" }); -expect(response.status()).toBe(403); // ❌ This tests Coraza WAF -``` - -#### Integration Tests for Middleware - -Middleware enforcement is verified by **integration tests** in `backend/integration/`: - -- `cerberus_integration_test.go` - Overall security suite behavior -- `coraza_integration_test.go` - WAF blocking (SQL injection, XSS) -- `crowdsec_integration_test.go` - IP reputation and bans -- `rate_limit_integration_test.go` - Request throttling - -These tests run in Docker Compose with full Caddy+Cerberus stack and are executed in separate CI workflows. - -#### When to Skip Tests - -Use `test.skip()` for tests that require middleware enforcement: - -```typescript -test('should rate limit after 5 attempts', async ({ request }) => { - test.skip( - true, - 'Rate limiting enforced via Cerberus middleware (port 80). Verified in integration tests (backend/integration/).' - ); - // Test body... -}); -``` - -**Skip Reason Template:** -``` -"[Behavior] enforced via Cerberus middleware (port 80). Verified in integration tests (backend/integration/)." -``` ## Example Test Structure @@ -194,11 +76,6 @@ test.describe('Movie Search Feature', () => { 4. **Validate**: Ensure tests pass consistently and cover the intended functionality 5. **Report**: Provide feedback on test results and any issues discovered -### Execution Constraints - -- **No Truncation**: Never pipe Playwright test output through `head`, `tail`, or other truncating commands. Playwright runs interactively and requires user input to quit when piped, causing the command to hang indefinitely. -- **Full Output**: Always capture the complete test output to analyze failures accurately. - ## Quality Checklist Before finalizing tests, ensure: diff --git a/.github/workflows/e2e-tests-split.yml b/.github/workflows/e2e-tests-split.yml index 97a9b344..da55242b 100644 --- a/.github/workflows/e2e-tests-split.yml +++ b/.github/workflows/e2e-tests-split.yml @@ -46,8 +46,8 @@ env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository_owner }}/charon PLAYWRIGHT_COVERAGE: ${{ vars.PLAYWRIGHT_COVERAGE || '0' }} - DEBUG: 'charon:*,charon-test:*' - PLAYWRIGHT_DEBUG: '1' + # Standard Playwright runner debugging - shows test execution flow + DEBUG: 'pw:test' CI_LOG_LEVEL: 'verbose' concurrency: @@ -215,36 +215,19 @@ jobs: - name: Run Chromium tests timeout-minutes: 4 run: | - echo "═══════════════════════════════════════════=" + echo "════════════════════════════════════════════" echo "Chromium E2E Tests" echo "Start Time: $(date -u +'%Y-%m-%dT%H:%M:%SZ')" echo "════════════════════════════════════════════" echo "Node: $(node -v)" echo "NPM: $(npm -v)" - echo "Kernel: $(uname -a)" - echo "Memory (free -m):" - free -m || true SHARD_START=$(date +%s) echo "SHARD_START=$SHARD_START" >> $GITHUB_ENV - # Enable verbose Playwright output to diagnose browser launch hang - # pw:browser* = browser connection and launch - # pw:protocol = CDP protocol messages - # pw:channel = IPC between Playwright and browser - # pw:api = Playwright runner API calls (helps when hang occurs before browser launch) - export DEBUG=pw:api,pw:browser*,pw:protocol,pw:channel - - echo "════════════════════════════════════════════" - echo "Preflight: listing tests (discovery only)" - echo "Time: $(date -u +'%Y-%m-%dT%H:%M:%SZ')" - npx playwright test --project=chromium --list - echo "════════════════════════════════════════════" - echo "Running tests" - echo "Time: $(date -u +'%Y-%m-%dT%H:%M:%SZ')" - npx playwright test --project=chromium + echo "════════════════════════════════════════════" env: PLAYWRIGHT_BASE_URL: http://127.0.0.1:8080 @@ -410,12 +393,6 @@ jobs: SHARD_START=$(date +%s) echo "SHARD_START=$SHARD_START" >> $GITHUB_ENV - # Enable verbose Playwright output to diagnose browser launch hang - # pw:browser* = browser connection and launch - # pw:protocol = CDP protocol messages - # pw:channel = IPC between Playwright and browser - export DEBUG=pw:browser*,pw:protocol,pw:channel - npx playwright test --project=firefox SHARD_END=$(date +%s) @@ -424,7 +401,6 @@ jobs: echo "════════════════════════════════════════════" echo "Firefox Tests Complete | Duration: ${SHARD_DURATION}s" echo "════════════════════════════════════════════" - echo "════════════════════════════════════════════" env: PLAYWRIGHT_BASE_URL: http://127.0.0.1:8080 CI: true @@ -589,12 +565,6 @@ jobs: SHARD_START=$(date +%s) echo "SHARD_START=$SHARD_START" >> $GITHUB_ENV - # Enable verbose Playwright output to diagnose browser launch hang - # pw:browser* = browser connection and launch - # pw:protocol = CDP protocol messages - # pw:channel = IPC between Playwright and browser - export DEBUG=pw:browser*,pw:protocol,pw:channel - npx playwright test --project=webkit SHARD_END=$(date +%s) diff --git a/playwright.config.js b/playwright.config.js index c16cd16d..225576f9 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -20,13 +20,12 @@ const STORAGE_STATE = join(__dirname, 'playwright/.auth/user.json'); /** * Coverage reporter configuration for E2E tests - * Tracks V8 coverage during Playwright test execution + * Only loaded when PLAYWRIGHT_COVERAGE=1 */ -const coverageReporterConfig = defineCoverageReporterConfig({ - // Root directory for source file resolution - sourceRoot: __dirname, +const enableCoverage = process.env.PLAYWRIGHT_COVERAGE === '1'; - // Exclude non-application code from coverage +const coverageReporterConfig = enableCoverage ? defineCoverageReporterConfig({ + sourceRoot: __dirname, exclude: [ '**/node_modules/**', '**/playwright/**', @@ -38,86 +37,60 @@ const coverageReporterConfig = defineCoverageReporterConfig({ '**/dist/**', '**/build/**', ], - - // Output directory for coverage reports resultDir: join(__dirname, 'coverage/e2e'), - - // Generate multiple report formats reports: [ - // HTML report for visual inspection ['html'], - // LCOV for Codecov upload ['lcovonly', { file: 'lcov.info' }], - // JSON for programmatic access ['json', { file: 'coverage.json' }], - // Text summary in console ['text-summary', { file: null }], ], - - // Coverage watermarks (visual thresholds in HTML report) watermarks: { statements: [50, 80], branches: [50, 80], functions: [50, 80], lines: [50, 80], }, - // Path rewriting for source file resolution - rewritePath: ({ absolutePath, relativePath }) => { - // Handle paths from Docker container + rewritePath: ({ absolutePath }) => { if (absolutePath.startsWith('/app/')) { return absolutePath.replace('/app/', `${__dirname}/`); } - - // Handle Vite dev server paths (relative to frontend/src) - // Vite serves files like "/src/components/Button.tsx" if (absolutePath.startsWith('/src/')) { return join(__dirname, 'frontend', absolutePath); } - - // If path doesn't start with /, prepend frontend/src if (!absolutePath.startsWith('/') && !absolutePath.includes('/')) { - // Bare filenames like "Button.tsx" - try to resolve to frontend/src return join(__dirname, 'frontend/src', absolutePath); } - return absolutePath; }, -}); - -const enableCoverage = process.env.PLAYWRIGHT_COVERAGE === '1'; +}) : null; /** * @see https://playwright.dev/docs/test-configuration */ export default defineConfig({ testDir: './tests', - /* Ignore old/deprecated test directories */ testIgnore: ['**/frontend/**', '**/node_modules/**', '**/backend/**'], - /* Global timeout for each test - increased to 90s for feature flag propagation - * CI uses 60s to fail fast in resource-constrained environment (2-core runners) - */ + + /* Standard globalSetup - runs once before all tests */ + globalSetup: './tests/global-setup.ts', + + /* Timeouts */ timeout: process.env.CI ? 60000 : 90000, - /* Timeout for expect() assertions */ - expect: { - timeout: 5000, - }, - /* Run tests in files in parallel */ + expect: { timeout: 5000 }, + + /* Parallelization */ fullyParallel: true, - /* Fail the build on CI if you accidentally left test.only in the source code. */ - forbidOnly: !!process.env.CI, - /* Retry on CI only */ - retries: process.env.CI ? 2 : 0, - /* Opt out of parallel tests on CI - single worker to avoid resource starvation */ workers: process.env.CI ? 1 : undefined, - /* Reporter to use. See https://playwright.dev/docs/test-reporters - * CI uses per-shard HTML reports (no blob merging needed). - * Each shard uploads its own HTML report for easier debugging. - */ + + /* CI settings */ + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + + /* Reporters - simplified for CI */ reporter: [ - ...(process.env.CI ? [['github']] : [['list']]), + process.env.CI ? ['github'] : ['list'], ['html', { open: process.env.CI ? 'never' : 'on-failure' }], ...(enableCoverage ? [['@bgotink/playwright-coverage', coverageReporterConfig]] : []), - ['./tests/reporters/debug-reporter.ts'], ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { @@ -168,25 +141,13 @@ export default defineConfig({ /* Configure projects for major browsers */ projects: [ - // 1. Setup project - authentication (runs FIRST) + // Setup project - authentication (runs FIRST) { name: 'setup', testMatch: /auth\.setup\.ts/, }, - // 2. Preflight setup - runs AFTER auth.setup.ts to ensure storage state exists - // This replaces Playwright globalSetup so authenticated setup work can run - // deterministically in fresh CI workspaces. - { - name: 'preflight', - testMatch: /preflight\.setup\.ts/, - dependencies: ['setup'], - fullyParallel: false, - workers: 1, - }, - - // 2. Security Tests - Run WITH security enabled (SEQUENTIAL, headless Chromium) - // These tests enable security modules, verify enforcement, then teardown disables all. + // Security Tests - Run WITH security enabled (SEQUENTIAL, Chromium only) { name: 'security-tests', testDir: './tests', @@ -196,33 +157,29 @@ export default defineConfig({ ], dependencies: ['setup'], teardown: 'security-teardown', - fullyParallel: false, // Force sequential - modules share state - workers: 1, // Force single worker to prevent race conditions on security settings + fullyParallel: false, + workers: 1, use: { ...devices['Desktop Chrome'], - headless: true, // Security tests are API-level, don't need headed + headless: true, storageState: STORAGE_STATE, }, }, - // 3. Security Teardown - Disable ALL security modules after security-tests + // Security Teardown - Disable ALL security modules { name: 'security-teardown', testMatch: /security-teardown\.setup\.ts/, }, - // 4. Browser projects - Depend on setup and security-tests (with teardown) for order - // Note: Security modules are re-disabled by teardown before these projects execute - // TEMPORARY CI FIX: Skip security-tests dependency to unblock pipeline - // Re-enable after fixing hanging security test + // Browser projects - standard Playwright pattern { name: 'chromium', use: { ...devices['Desktop Chrome'], - // Use stored authentication state storageState: STORAGE_STATE, }, - dependencies: ['preflight'], // Temporarily removed 'security-tests' + dependencies: ['setup', 'security-tests'], }, { @@ -231,7 +188,7 @@ export default defineConfig({ ...devices['Desktop Firefox'], storageState: STORAGE_STATE, }, - dependencies: ['preflight'], // Temporarily removed 'security-tests' + dependencies: ['setup', 'security-tests'], }, { @@ -240,7 +197,7 @@ export default defineConfig({ ...devices['Desktop Safari'], storageState: STORAGE_STATE, }, - dependencies: ['preflight'], // Temporarily removed 'security-tests' + dependencies: ['setup', 'security-tests'], }, /* Test against mobile viewports. */ diff --git a/tests/preflight.setup.ts b/tests/preflight.setup.ts deleted file mode 100644 index 692e384c..00000000 --- a/tests/preflight.setup.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { test } from './fixtures/test'; -import globalSetup from './global-setup'; - -test('preflight', async () => { - await globalSetup(); -});