# E2E Tests Workflow # Runs Playwright E2E tests with sharding for faster execution # and collects frontend code coverage via @bgotink/playwright-coverage # # Test Execution Architecture: # - Parallel Sharding: Tests split across 4 shards for speed # - Per-Shard HTML Reports: Each shard generates its own HTML report # - No Merging Needed: Smaller reports are easier to debug # - Trace Collection: Failure traces captured for debugging # # Coverage Architecture: # - Backend: Docker container at localhost:8080 (API) # - Frontend: Vite dev server at localhost:3000 (serves source files) # - Tests hit Vite, which proxies API calls to Docker # - V8 coverage maps directly to source files for accurate reporting # - Coverage disabled by default (requires PLAYWRIGHT_COVERAGE=1) # # Triggers: # - Pull requests to main/develop (with path filters) # - Push to main branch # - Manual dispatch with browser selection # # Jobs: # 1. build: Build Docker image and upload as artifact # 2. e2e-tests: Run tests in parallel shards, upload per-shard HTML reports # 3. test-summary: Generate summary with links to shard reports # 4. comment-results: Post test results as PR comment # 5. upload-coverage: Merge and upload E2E coverage to Codecov (if enabled) # 6. e2e-results: Status check to block merge on failure name: E2E Tests on: pull_request: branches: - main - development - 'feature/**' paths: - 'frontend/**' - 'backend/**' - 'tests/**' - 'playwright.config.js' - '.github/workflows/e2e-tests.yml' workflow_dispatch: inputs: browser: description: 'Browser to test' required: false default: 'chromium' type: choice options: - chromium - firefox - webkit - all env: NODE_VERSION: '20' GO_VERSION: '1.25.6' GOTOOLCHAIN: auto REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository_owner }}/charon PLAYWRIGHT_COVERAGE: ${{ vars.PLAYWRIGHT_COVERAGE || '0' }} # Enhanced debugging environment variables DEBUG: 'charon:*,charon-test:*' PLAYWRIGHT_DEBUG: '1' CI_LOG_LEVEL: 'verbose' concurrency: group: e2e-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: # Build application once, share across test shards build: name: Build Application runs-on: ubuntu-latest outputs: image_digest: ${{ steps.build-image.outputs.digest }} steps: - name: Checkout repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 - name: Set up Go uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6 with: go-version: ${{ env.GO_VERSION }} cache: true cache-dependency-path: backend/go.sum - name: Set up Node.js uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 with: node-version: ${{ env.NODE_VERSION }} cache: 'npm' - name: Cache npm dependencies uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: path: ~/.npm key: npm-${{ hashFiles('package-lock.json') }} restore-keys: npm- - name: Install dependencies run: npm ci - name: Set up Docker Buildx uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Build Docker image id: build-image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6 with: context: . file: ./Dockerfile push: false load: true tags: charon:e2e-test cache-from: type=gha cache-to: type=gha,mode=max - name: Save Docker image run: docker save charon:e2e-test -o charon-e2e-image.tar - name: Upload Docker image artifact uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: docker-image path: charon-e2e-image.tar retention-days: 1 # Run tests in parallel shards e2e-tests: name: E2E ${{ matrix.browser }} (Shard ${{ matrix.shard }}/${{ matrix.total-shards }}) runs-on: ubuntu-latest needs: build timeout-minutes: 30 env: # Required for security teardown (emergency reset fallback when ACL blocks API) CHARON_EMERGENCY_TOKEN: ${{ secrets.CHARON_EMERGENCY_TOKEN }} # Enable security-focused endpoints and test gating CHARON_EMERGENCY_SERVER_ENABLED: "true" CHARON_SECURITY_TESTS_ENABLED: "true" CHARON_E2E_IMAGE_TAG: charon:e2e-test strategy: fail-fast: false matrix: shard: [1, 2, 3, 4] total-shards: [4] browser: [chromium, firefox, webkit] steps: - name: Checkout repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 - name: Set up Node.js uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 with: node-version: ${{ env.NODE_VERSION }} cache: 'npm' - name: Download Docker image uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: docker-image - name: Validate Emergency Token Configuration run: | echo "🔐 Validating emergency token configuration..." if [ -z "$CHARON_EMERGENCY_TOKEN" ]; then echo "::error title=Missing Secret::CHARON_EMERGENCY_TOKEN secret not configured in repository settings" echo "::error::Navigate to: Repository Settings → Secrets and Variables → Actions" echo "::error::Create secret: CHARON_EMERGENCY_TOKEN" echo "::error::Generate value with: openssl rand -hex 32" echo "::error::See docs/github-setup.md for detailed instructions" exit 1 fi TOKEN_LENGTH=${#CHARON_EMERGENCY_TOKEN} if [ $TOKEN_LENGTH -lt 64 ]; then echo "::error title=Invalid Token Length::CHARON_EMERGENCY_TOKEN must be at least 64 characters (current: $TOKEN_LENGTH)" echo "::error::Generate new token with: openssl rand -hex 32" exit 1 fi # Mask token in output (show first 8 chars only) MASKED_TOKEN="${CHARON_EMERGENCY_TOKEN:0:8}...${CHARON_EMERGENCY_TOKEN: -4}" echo "::notice::Emergency token validated (length: $TOKEN_LENGTH, preview: $MASKED_TOKEN)" env: CHARON_EMERGENCY_TOKEN: ${{ secrets.CHARON_EMERGENCY_TOKEN }} - name: Load Docker image run: | docker load -i charon-e2e-image.tar docker images | grep charon - name: Generate ephemeral encryption key run: | # Generate a unique, ephemeral encryption key for this CI run # Key is 32 bytes, base64-encoded as required by CHARON_ENCRYPTION_KEY echo "CHARON_ENCRYPTION_KEY=$(openssl rand -base64 32)" >> $GITHUB_ENV echo "✅ Generated ephemeral encryption key for E2E tests" - name: Start test environment run: | # Use docker-compose.playwright-ci.yml for CI (no .env file, uses GitHub Secrets) # Note: Using pre-built image loaded from artifact - no rebuild needed docker compose -f .docker/compose/docker-compose.playwright-ci.yml --profile security-tests up -d echo "✅ Container started via docker-compose.playwright-ci.yml" - name: Wait for service health run: | echo "⏳ Waiting for Charon to be healthy..." MAX_ATTEMPTS=30 ATTEMPT=0 while [[ ${ATTEMPT} -lt ${MAX_ATTEMPTS} ]]; do ATTEMPT=$((ATTEMPT + 1)) echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}..." if curl -sf http://localhost:8080/api/v1/health > /dev/null 2>&1; then echo "✅ Charon is healthy!" curl -s http://localhost:8080/api/v1/health | jq . exit 0 fi sleep 2 done echo "❌ Health check failed" docker compose -f .docker/compose/docker-compose.playwright-ci.yml logs exit 1 - name: Install dependencies run: npm ci - name: Clean Playwright browser cache run: rm -rf ~/.cache/ms-playwright - name: Cache Playwright browsers id: playwright-cache uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: path: ~/.cache/ms-playwright # Use exact match only - no restore-keys fallback # This ensures we don't restore stale browsers when Playwright version changes key: playwright-${{ matrix.browser }}-${{ hashFiles('package-lock.json') }} - name: Install & verify Playwright browsers run: | npx playwright install --with-deps --force set -euo pipefail echo "🎯 Playwright CLI version" npx playwright --version || true echo "🔍 Showing Playwright cache root (if present)" ls -la ~/.cache/ms-playwright || true echo "📥 Install or verify browser: ${{ matrix.browser }}" # Install when cache miss, otherwise verify the expected executables exist if [[ "${{ steps.playwright-cache.outputs.cache-hit }}" != "true" ]]; then echo "📥 Cache miss - downloading ${{ matrix.browser }} browser..." npx playwright install --with-deps ${{ matrix.browser }} else echo "✅ Cache hit - verifying ${{ matrix.browser }} browser files..." fi # Look for the browser-specific headless shell executable(s) case "${{ matrix.browser }}" in chromium) EXPECTED_PATTERN="chrome-headless-shell*" ;; firefox) EXPECTED_PATTERN="firefox*" ;; webkit) EXPECTED_PATTERN="webkit*" ;; *) EXPECTED_PATTERN="*" ;; esac echo "Searching for expected files (pattern=$EXPECTED_PATTERN)..." find ~/.cache/ms-playwright -maxdepth 4 -type f -name "$EXPECTED_PATTERN" -print || true # Attempt to derive the exact executable path Playwright will use echo "Attempting to resolve Playwright's executable path via Node API (best-effort)" node -e "try{ const pw = require('playwright'); const b = pw['${{ matrix.browser }}']; console.log('exePath:', b.executablePath ? b.executablePath() : 'n/a'); }catch(e){ console.error('node-check-failed', e.message); process.exit(0); }" || true # If the expected binary is missing, force reinstall MISSING_COUNT=$(find ~/.cache/ms-playwright -maxdepth 4 -type f -name "$EXPECTED_PATTERN" | wc -l || true) if [[ "$MISSING_COUNT" -lt 1 ]]; then echo "⚠️ Expected Playwright browser executable not found (count=$MISSING_COUNT). Forcing reinstall..." npx playwright install --with-deps ${{ matrix.browser }} --force fi echo "Post-install: show cache contents (top 5 lines)" find ~/.cache/ms-playwright -maxdepth 3 -printf '%p\n' | head -40 || true # Final sanity check: try a headless launch via a tiny Node script (browser-specific args, retry without args) echo "🔁 Verifying browser can be launched (headless)" node -e "(async()=>{ try{ const pw=require('playwright'); const name='${{ matrix.browser }}'; const browser = pw[name]; const argsMap = { chromium: ['--no-sandbox'], firefox: ['--no-sandbox'], webkit: [] }; const args = argsMap[name] || []; // First attempt: launch with recommended args for this browser try { console.log('attempt-launch', name, 'args', JSON.stringify(args)); const b = await browser.launch({ headless: true, args }); await b.close(); console.log('launch-ok', 'argsUsed', JSON.stringify(args)); process.exit(0); } catch (err) { console.warn('launch-with-args-failed', err && err.message); if (args.length) { // Retry without args (some browsers reject unknown flags) console.log('retrying-without-args'); const b2 = await browser.launch({ headless: true }); await b2.close(); console.log('launch-ok-no-args'); process.exit(0); } throw err; } } catch (e) { console.error('launch-failed', e && e.message); process.exit(2); } })()" || (echo '❌ Browser launch verification failed' && exit 1) echo "✅ Playwright ${{ matrix.browser }} ready and verified" - name: Run E2E tests (Shard ${{ matrix.shard }}/${{ matrix.total-shards }}) run: | echo "════════════════════════════════════════════════════════════" echo "E2E Test Shard ${{ matrix.shard }}/${{ matrix.total-shards }}" echo "Browser: ${{ matrix.browser }}" echo "Start Time: $(date -u +'%Y-%m-%dT%H:%M:%SZ')" echo "" echo "Reporter: HTML (per-shard reports)" echo "Output: playwright-report/ directory" echo "════════════════════════════════════════════════════════════" # Capture start time for performance budget tracking SHARD_START=$(date +%s) echo "SHARD_START=$SHARD_START" >> $GITHUB_ENV npx playwright test \ --project=${{ matrix.browser }} \ --shard=${{ matrix.shard }}/${{ matrix.total-shards }} # Capture end time for performance budget tracking SHARD_END=$(date +%s) echo "SHARD_END=$SHARD_END" >> $GITHUB_ENV SHARD_DURATION=$((SHARD_END - SHARD_START)) echo "" echo "════════════════════════════════════════════════════════════" echo "Shard ${{ matrix.shard }} Complete | Duration: ${SHARD_DURATION}s" echo "════════════════════════════════════════════════════════════" env: # Test directly against Docker container (no coverage) PLAYWRIGHT_BASE_URL: http://localhost:8080 CI: true TEST_WORKER_INDEX: ${{ matrix.shard }} - name: Verify shard performance budget if: always() run: | # Calculate shard execution time SHARD_DURATION=$((SHARD_END - SHARD_START)) MAX_DURATION=900 # 15 minutes echo "📊 Performance Budget Check" echo " Shard Duration: ${SHARD_DURATION}s" echo " Budget Limit: ${MAX_DURATION}s" echo " Utilization: $((SHARD_DURATION * 100 / MAX_DURATION))%" # Fail if shard exceeded performance budget if [[ $SHARD_DURATION -gt $MAX_DURATION ]]; then echo "::error::Shard exceeded performance budget: ${SHARD_DURATION}s > ${MAX_DURATION}s" echo "::error::This likely indicates feature flag polling regression or API bottleneck" echo "::error::Review test logs and consider optimizing wait helpers or API calls" exit 1 fi echo "✅ Shard completed within budget: ${SHARD_DURATION}s" - name: Upload HTML report (per-shard) if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: playwright-report-${{ matrix.browser }}-shard-${{ matrix.shard }} path: playwright-report/ retention-days: 14 - name: Upload test traces on failure if: failure() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: traces-${{ matrix.browser }}-shard-${{ matrix.shard }} path: test-results/**/*.zip retention-days: 7 - name: Collect Docker logs on failure if: failure() run: | echo "📋 Container logs:" docker compose -f .docker/compose/docker-compose.playwright-ci.yml logs > docker-logs-${{ matrix.browser }}-shard-${{ matrix.shard }}.txt 2>&1 - name: Upload Docker logs on failure if: failure() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: docker-logs-${{ matrix.browser }}-shard-${{ matrix.shard }} path: docker-logs-${{ matrix.browser }}-shard-${{ matrix.shard }}.txt retention-days: 7 - name: Cleanup if: always() run: | docker compose -f .docker/compose/docker-compose.playwright-ci.yml down -v 2>/dev/null || true # Summarize test results from all shards (no merging needed) test-summary: name: E2E Test Summary runs-on: ubuntu-latest needs: e2e-tests if: always() steps: - name: Generate job summary with per-shard links run: | echo "## 📊 E2E Test Results" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "### Per-Shard HTML Reports" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "Each shard generates its own HTML report for easier debugging:" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "| Browser | Shards | HTML Reports | Traces (on failure) |" >> $GITHUB_STEP_SUMMARY echo "|---------|--------|--------------|---------------------|" >> $GITHUB_STEP_SUMMARY echo "| Chromium | 1-4 | \`playwright-report-chromium-shard-{1..4}\` | \`traces-chromium-shard-{1..4}\` |" >> $GITHUB_STEP_SUMMARY echo "| Firefox | 1-4 | \`playwright-report-firefox-shard-{1..4}\` | \`traces-firefox-shard-{1..4}\` |" >> $GITHUB_STEP_SUMMARY echo "| WebKit | 1-4 | \`playwright-report-webkit-shard-{1..4}\` | \`traces-webkit-shard-{1..4}\` |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "### How to View Reports" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "1. Download the shard HTML report artifact (zip file)" >> $GITHUB_STEP_SUMMARY echo "2. Extract and open \`index.html\` in your browser" >> $GITHUB_STEP_SUMMARY echo "3. Or run: \`npx playwright show-report path/to/extracted-folder\`" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "### Debugging Tips" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "- **Failed tests?** Download the shard report that failed. Each shard has a focused subset of tests." >> $GITHUB_STEP_SUMMARY echo "- **Traces**: Available in trace artifacts (only on failure)" >> $GITHUB_STEP_SUMMARY echo "- **Docker Logs**: Backend errors available in docker-logs-shard-N artifacts" >> $GITHUB_STEP_SUMMARY echo "- **Local repro**: \`npx playwright test --grep=\"test name\"\`" >> $GITHUB_STEP_SUMMARY # Comment on PR with results comment-results: name: Comment Test Results runs-on: ubuntu-latest needs: [e2e-tests, test-summary] if: github.event_name == 'pull_request' && always() permissions: pull-requests: write steps: - name: Determine test status id: status run: | if [[ "${{ needs.e2e-tests.result }}" == "success" ]]; then echo "emoji=✅" >> $GITHUB_OUTPUT echo "status=PASSED" >> $GITHUB_OUTPUT echo "message=All E2E tests passed!" >> $GITHUB_OUTPUT elif [[ "${{ needs.e2e-tests.result }}" == "failure" ]]; then echo "emoji=❌" >> $GITHUB_OUTPUT echo "status=FAILED" >> $GITHUB_OUTPUT echo "message=Some E2E tests failed. Check artifacts for per-shard reports." >> $GITHUB_OUTPUT else echo "emoji=⚠️" >> $GITHUB_OUTPUT echo "status=UNKNOWN" >> $GITHUB_OUTPUT echo "message=E2E tests did not complete successfully." >> $GITHUB_OUTPUT fi - name: Comment on PR uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 with: script: | const emoji = '${{ steps.status.outputs.emoji }}'; const status = '${{ steps.status.outputs.status }}'; const message = '${{ steps.status.outputs.message }}'; const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; const body = `## ${emoji} E2E Test Results: ${status} ${message} | Metric | Result | |--------|--------| | Browsers | Chromium, Firefox, WebKit | | Shards per Browser | 4 | | Total Jobs | 12 | | Status | ${status} | **Per-Shard HTML Reports** (easier to debug): - \`playwright-report-{browser}-shard-{1..4}\` (12 total artifacts) - Trace artifacts: \`traces-{browser}-shard-{N}\` [📊 View workflow run & download reports](${runUrl}) --- 🤖 This comment was automatically generated by the E2E Tests workflow.`; // Find existing comment const { data: comments } = await github.rest.issues.listComments({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, }); const botComment = comments.find(comment => comment.user.type === 'Bot' && comment.body.includes('E2E Test Results') ); if (botComment) { await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: botComment.id, body: body }); } else { await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, body: body }); } # Upload merged E2E coverage to Codecov upload-coverage: name: Upload E2E Coverage runs-on: ubuntu-latest needs: e2e-tests # Coverage is only produced when PLAYWRIGHT_COVERAGE=1 (requires Vite dev server) if: vars.PLAYWRIGHT_COVERAGE == '1' steps: - name: Checkout repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 - name: Set up Node.js uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 with: node-version: ${{ env.NODE_VERSION }} cache: 'npm' - name: Download all coverage artifacts uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: pattern: e2e-coverage-* path: all-coverage merge-multiple: false - name: Merge LCOV coverage files run: | # Install lcov for merging sudo apt-get update && sudo apt-get install -y lcov # Create merged coverage directory mkdir -p coverage/e2e-merged # Find all lcov.info files and merge them LCOV_FILES=$(find all-coverage -name "lcov.info" -type f) if [[ -n "$LCOV_FILES" ]]; then # Build merge command MERGE_ARGS="" for file in $LCOV_FILES; do MERGE_ARGS="$MERGE_ARGS -a $file" done lcov $MERGE_ARGS -o coverage/e2e-merged/lcov.info echo "✅ Merged $(echo "$LCOV_FILES" | wc -w) coverage files" else echo "⚠️ No coverage files found to merge" exit 0 fi - name: Upload E2E coverage to Codecov uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage/e2e-merged/lcov.info flags: e2e name: e2e-coverage fail_ci_if_error: false - name: Upload merged coverage artifact uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: e2e-coverage-merged path: coverage/e2e-merged/ retention-days: 30 # Final status check - blocks merge if tests fail e2e-results: name: E2E Test Results runs-on: ubuntu-latest needs: e2e-tests if: always() steps: - name: Check test results run: | if [[ "${{ needs.e2e-tests.result }}" == "success" ]]; then echo "✅ All E2E tests passed" exit 0 elif [[ "${{ needs.e2e-tests.result }}" == "skipped" ]]; then echo "⏭️ E2E tests were skipped" exit 0 else echo "❌ E2E tests failed or were cancelled" echo "Result: ${{ needs.e2e-tests.result }}" exit 1 fi