diff --git a/.github/skills/test-e2e-playwright-coverage-scripts/run.sh b/.github/skills/test-e2e-playwright-coverage-scripts/run.sh new file mode 100755 index 00000000..8d52eaea --- /dev/null +++ b/.github/skills/test-e2e-playwright-coverage-scripts/run.sh @@ -0,0 +1,294 @@ +#!/usr/bin/env bash +# Test E2E Playwright Coverage - Execution Script +# +# Runs Playwright end-to-end tests with code coverage collection +# using @bgotink/playwright-coverage. +# +# IMPORTANT: For accurate source-level coverage, this script starts +# the Vite dev server (localhost:5173) which proxies API calls to +# the Docker backend (localhost:8080). V8 coverage requires source +# files to be accessible on the test host. + +set -euo pipefail + +# Source helper scripts +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SKILLS_SCRIPTS_DIR="$(cd "${SCRIPT_DIR}/../scripts" && pwd)" + +# shellcheck source=../scripts/_logging_helpers.sh +source "${SKILLS_SCRIPTS_DIR}/_logging_helpers.sh" +# shellcheck source=../scripts/_error_handling_helpers.sh +source "${SKILLS_SCRIPTS_DIR}/_error_handling_helpers.sh" +# shellcheck source=../scripts/_environment_helpers.sh +source "${SKILLS_SCRIPTS_DIR}/_environment_helpers.sh" + +# Project root is 3 levels up from this script +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +# Default parameter values +PROJECT="chromium" +VITE_PID="" +VITE_PORT="${VITE_PORT:-5173}" # Default Vite port (avoids conflicts with common ports) +BACKEND_URL="http://localhost:8080" + +# Cleanup function to kill Vite dev server on exit +cleanup() { + if [[ -n "${VITE_PID}" ]] && kill -0 "${VITE_PID}" 2>/dev/null; then + log_info "Stopping Vite dev server (PID: ${VITE_PID})..." + kill "${VITE_PID}" 2>/dev/null || true + wait "${VITE_PID}" 2>/dev/null || true + fi +} + +# Set up trap for cleanup +trap cleanup EXIT INT TERM + +# Parse command-line arguments +parse_arguments() { + while [[ $# -gt 0 ]]; do + case "$1" in + --project=*) + PROJECT="${1#*=}" + shift + ;; + --project) + PROJECT="${2:-chromium}" + shift 2 + ;; + --skip-vite) + SKIP_VITE="true" + shift + ;; + -h|--help) + show_help + exit 0 + ;; + *) + log_warning "Unknown argument: $1" + shift + ;; + esac + done +} + +# Show help message +show_help() { + cat << EOF +Usage: run.sh [OPTIONS] + +Run Playwright E2E tests with coverage collection. + +Coverage requires the Vite dev server to serve source files directly. +This script automatically starts Vite at localhost:5173, which proxies +API calls to the Docker backend at localhost:8080. + +Options: + --project=PROJECT Browser project to run (chromium, firefox, webkit) + Default: chromium + --skip-vite Skip starting Vite dev server (use existing server) + -h, --help Show this help message + +Environment Variables: + PLAYWRIGHT_BASE_URL Override test URL (default: http://localhost:5173) + VITE_PORT Vite dev server port (default: 5173) + CI Set to 'true' for CI environment + +Prerequisites: + - Docker backend running at localhost:8080 + - Node.js dependencies installed (npm ci) + +Examples: + run.sh # Start Vite, run tests with coverage + run.sh --project=firefox # Run in Firefox with coverage + run.sh --skip-vite # Use existing Vite server +EOF +} + +# Validate project parameter +validate_project() { + local valid_projects=("chromium" "firefox" "webkit") + local project_lower + project_lower=$(echo "${PROJECT}" | tr '[:upper:]' '[:lower:]') + + for valid in "${valid_projects[@]}"; do + if [[ "${project_lower}" == "${valid}" ]]; then + PROJECT="${project_lower}" + return 0 + fi + done + + error_exit "Invalid project '${PROJECT}'. Valid options: chromium, firefox, webkit" +} + +# Check if backend is running +check_backend() { + log_info "Checking backend at ${BACKEND_URL}..." + local max_attempts=5 + local attempt=1 + + while [[ ${attempt} -le ${max_attempts} ]]; do + if curl -sf "${BACKEND_URL}/api/v1/health" >/dev/null 2>&1; then + log_success "Backend is healthy" + return 0 + fi + log_info "Waiting for backend... (attempt ${attempt}/${max_attempts})" + sleep 2 + ((attempt++)) + done + + log_warning "Backend not responding at ${BACKEND_URL}" + log_warning "Coverage tests require Docker backend. Start with:" + log_warning " docker compose -f .docker/compose/docker-compose.local.yml up -d" + return 1 +} + +# Start Vite dev server +start_vite() { + local vite_url="http://localhost:${VITE_PORT}" + + # Check if Vite is already running on our preferred port + if curl -sf "${vite_url}" >/dev/null 2>&1; then + log_info "Vite dev server already running at ${vite_url}" + return 0 + fi + + log_step "VITE" "Starting Vite dev server" + cd "${PROJECT_ROOT}/frontend" + + # Ensure dependencies are installed + if [[ ! -d "node_modules" ]]; then + log_info "Installing frontend dependencies..." + npm ci --silent + fi + + # Start Vite in background with explicit port + log_command "npx vite --port ${VITE_PORT} (background)" + npx vite --port "${VITE_PORT}" > /tmp/vite.log 2>&1 & + VITE_PID=$! + + # Wait for Vite to be ready (check log for actual port in case of conflict) + log_info "Waiting for Vite to start..." + local max_wait=60 + local waited=0 + local actual_port="${VITE_PORT}" + + while [[ ${waited} -lt ${max_wait} ]]; do + # Check if Vite logged its ready message with actual port + if grep -q "Local:" /tmp/vite.log 2>/dev/null; then + # Extract actual port from Vite log (handles port conflict auto-switch) + actual_port=$(grep -oP 'localhost:\K[0-9]+' /tmp/vite.log 2>/dev/null | head -1 || echo "${VITE_PORT}") + vite_url="http://localhost:${actual_port}" + fi + + if curl -sf "${vite_url}" >/dev/null 2>&1; then + # Update VITE_PORT if Vite chose a different port + if [[ "${actual_port}" != "${VITE_PORT}" ]]; then + log_warning "Port ${VITE_PORT} was busy, Vite using port ${actual_port}" + VITE_PORT="${actual_port}" + fi + log_success "Vite dev server ready at ${vite_url}" + cd "${PROJECT_ROOT}" + return 0 + fi + sleep 1 + ((waited++)) + done + + log_error "Vite failed to start within ${max_wait} seconds" + log_error "Vite log:" + cat /tmp/vite.log 2>/dev/null || true + cd "${PROJECT_ROOT}" + return 1 +} + +# Main execution +main() { + SKIP_VITE="${SKIP_VITE:-false}" + parse_arguments "$@" + + # Validate environment + log_step "ENVIRONMENT" "Validating prerequisites" + validate_node_environment "18.0" || error_exit "Node.js 18+ is required" + check_command_exists "npx" "npx is required (part of Node.js installation)" + + # Validate project structure + log_step "VALIDATION" "Checking project structure" + cd "${PROJECT_ROOT}" + validate_project_structure "tests" "playwright.config.js" "package.json" || error_exit "Invalid project structure" + + # Validate project parameter + validate_project + + # Check backend is running (required for API proxy) + log_step "BACKEND" "Checking Docker backend" + if ! check_backend; then + error_exit "Backend not available. Coverage tests require Docker backend at ${BACKEND_URL}" + fi + + # Start Vite dev server for coverage (unless skipped) + if [[ "${SKIP_VITE}" != "true" ]]; then + start_vite || error_exit "Failed to start Vite dev server" + fi + + # Ensure coverage directory exists + log_step "SETUP" "Creating coverage directory" + mkdir -p coverage/e2e + + # Set environment variables + # IMPORTANT: Use Vite URL (3000) for coverage, not Docker (8080) + export PLAYWRIGHT_HTML_OPEN="${PLAYWRIGHT_HTML_OPEN:-never}" + export PLAYWRIGHT_BASE_URL="${PLAYWRIGHT_BASE_URL:-http://localhost:${VITE_PORT}}" + + # Log configuration + log_step "CONFIG" "Test configuration" + log_info "Project: ${PROJECT}" + log_info "Test URL: ${PLAYWRIGHT_BASE_URL}" + log_info "Backend URL: ${BACKEND_URL}" + log_info "Coverage output: ${PROJECT_ROOT}/coverage/e2e/" + log_info "" + log_info "Coverage architecture:" + log_info " Tests → Vite (localhost:3000) → serves source files" + log_info " Vite → Docker (localhost:8080) → API proxy" + + # Execute Playwright tests with coverage + log_step "EXECUTION" "Running Playwright E2E tests with coverage" + log_command "npx playwright test --project=${PROJECT}" + + local exit_code=0 + if npx playwright test --project="${PROJECT}"; then + log_success "All E2E tests passed" + else + exit_code=$? + log_error "E2E tests failed (exit code: ${exit_code})" + fi + + # Check if coverage was generated + log_step "COVERAGE" "Checking coverage output" + if [[ -f "coverage/e2e/lcov.info" ]]; then + log_success "E2E coverage generated: coverage/e2e/lcov.info" + + # Print summary if coverage.json exists + if [[ -f "coverage/e2e/coverage.json" ]] && command -v jq &> /dev/null; then + log_info "šŸ“Š Coverage Summary:" + jq '.total' coverage/e2e/coverage.json 2>/dev/null || true + fi + + # Show file sizes + log_info "Coverage files:" + ls -lh coverage/e2e/ 2>/dev/null || true + else + log_warning "No coverage data generated" + log_warning "Ensure test files import from '@bgotink/playwright-coverage'" + fi + + # Output report locations + log_step "REPORTS" "Report locations" + log_info "Coverage HTML: ${PROJECT_ROOT}/coverage/e2e/index.html" + log_info "Coverage LCOV: ${PROJECT_ROOT}/coverage/e2e/lcov.info" + log_info "Playwright Report: ${PROJECT_ROOT}/playwright-report/index.html" + + exit "${exit_code}" +} + +# Run main with all arguments +main "$@" diff --git a/.github/skills/test-e2e-playwright-coverage.SKILL.md b/.github/skills/test-e2e-playwright-coverage.SKILL.md new file mode 100644 index 00000000..c7d08c35 --- /dev/null +++ b/.github/skills/test-e2e-playwright-coverage.SKILL.md @@ -0,0 +1,195 @@ +--- +# agentskills.io specification v1.0 +name: "test-e2e-playwright-coverage" +version: "1.0.0" +description: "Run Playwright E2E tests with code coverage collection using @bgotink/playwright-coverage" +author: "Charon Project" +license: "MIT" +tags: + - "testing" + - "e2e" + - "playwright" + - "coverage" + - "integration" +compatibility: + os: + - "linux" + - "darwin" + shells: + - "bash" +requirements: + - name: "node" + version: ">=18.0" + optional: false + - name: "npx" + version: ">=1.0" + optional: false +environment_variables: + - name: "PLAYWRIGHT_BASE_URL" + description: "Base URL of the Charon application under test" + default: "http://localhost:8080" + required: false + - name: "PLAYWRIGHT_HTML_OPEN" + description: "Controls HTML report auto-open behavior (set to 'never' for CI/non-interactive)" + default: "never" + required: false + - name: "CI" + description: "Set to 'true' when running in CI environment" + default: "" + required: false +parameters: + - name: "project" + type: "string" + description: "Browser project to run (chromium, firefox, webkit)" + default: "chromium" + required: false +outputs: + - name: "coverage-e2e" + type: "directory" + description: "E2E coverage output directory with LCOV and HTML reports" + path: "coverage/e2e/" + - name: "playwright-report" + type: "directory" + description: "HTML test report directory" + path: "playwright-report/" + - name: "test-results" + type: "directory" + description: "Test artifacts and traces" + path: "test-results/" +metadata: + category: "test" + subcategory: "e2e-coverage" + execution_time: "medium" + risk_level: "low" + ci_cd_safe: true + requires_network: true + idempotent: true +--- + +# Test E2E Playwright Coverage + +## Overview + +Runs Playwright end-to-end tests with code coverage collection using `@bgotink/playwright-coverage`. This skill collects V8 coverage data during test execution and generates reports in LCOV, HTML, and JSON formats suitable for upload to Codecov. + +## Prerequisites + +- Node.js 18.0 or higher installed and in PATH +- Playwright browsers installed (`npx playwright install`) +- `@bgotink/playwright-coverage` package installed +- Charon application running (default: `http://localhost:8080`) +- Test files in `tests/` directory using coverage-enabled imports + +## Usage + +### Basic Usage + +Run E2E tests with coverage collection: + +```bash +.github/skills/scripts/skill-runner.sh test-e2e-playwright-coverage +``` + +### Browser Selection + +Run tests in a specific browser: + +```bash +# Chromium (default) +.github/skills/scripts/skill-runner.sh test-e2e-playwright-coverage --project=chromium + +# Firefox +.github/skills/scripts/skill-runner.sh test-e2e-playwright-coverage --project=firefox +``` + +### CI/CD Integration + +For use in GitHub Actions or other CI/CD pipelines: + +```yaml +- name: Run E2E Tests with Coverage + run: .github/skills/scripts/skill-runner.sh test-e2e-playwright-coverage + env: + PLAYWRIGHT_BASE_URL: http://localhost:8080 + CI: true + +- name: Upload E2E Coverage to Codecov + uses: codecov/codecov-action@v5 + with: + files: ./coverage/e2e/lcov.info + flags: e2e +``` + +## Parameters + +| Parameter | Type | Required | Default | Description | +|-----------|------|----------|---------|-------------| +| project | string | No | chromium | Browser project: chromium, firefox, webkit | + +## Environment Variables + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| PLAYWRIGHT_BASE_URL | No | http://localhost:8080 | Application URL to test against | +| PLAYWRIGHT_HTML_OPEN | No | never | HTML report auto-open behavior | +| CI | No | "" | Set to "true" for CI environment behavior | + +## Outputs + +### Success Exit Code +- **0**: All tests passed and coverage generated + +### Error Exit Codes +- **1**: One or more tests failed +- **Non-zero**: Configuration or execution error + +### Output Directories +- **coverage/e2e/**: Coverage reports (LCOV, HTML, JSON) + - `lcov.info` - LCOV format for Codecov upload + - `coverage.json` - JSON format for programmatic access + - `index.html` - HTML report for visual inspection +- **playwright-report/**: HTML test report with results and traces +- **test-results/**: Test artifacts, screenshots, and trace files + +## Viewing Coverage Reports + +### Coverage HTML Report + +```bash +# Open coverage HTML report +open coverage/e2e/index.html +``` + +### Playwright Test Report + +```bash +npx playwright show-report --port 9323 +``` + +## Coverage Data Format + +The skill generates coverage in multiple formats: + +| Format | File | Purpose | +|--------|------|---------| +| LCOV | `coverage/e2e/lcov.info` | Codecov upload | +| HTML | `coverage/e2e/index.html` | Visual inspection | +| JSON | `coverage/e2e/coverage.json` | Programmatic access | + +## Related Skills + +- test-e2e-playwright - E2E tests without coverage +- test-frontend-coverage - Frontend unit test coverage with Vitest +- test-backend-coverage - Backend unit test coverage with Go + +## Notes + +- **Coverage Source**: Uses V8 coverage (native, no instrumentation needed) +- **Performance**: ~5-10% overhead compared to tests without coverage +- **Sharding**: When running sharded tests in CI, coverage files must be merged +- **LCOV Merge**: Use `lcov -a file1.info -a file2.info -o merged.info` to merge + +--- + +**Last Updated**: 2026-01-18 +**Maintained by**: Charon Project Team diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index a8d27928..b4fb05bc 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -1,5 +1,12 @@ # E2E Tests Workflow # Runs Playwright E2E tests with sharding for faster execution +# and collects frontend code coverage via @bgotink/playwright-coverage +# +# Coverage Architecture: +# - Backend: Docker container at localhost:8080 (API) +# - Frontend: Vite dev server at localhost:3000 (serves source files) +# - Tests hit Vite, which proxies API calls to Docker +# - V8 coverage maps directly to source files for accurate reporting # # Triggers: # - Pull requests to main/develop (with path filters) @@ -8,10 +15,11 @@ # # Jobs: # 1. build: Build Docker image and upload as artifact -# 2. e2e-tests: Run tests in parallel shards +# 2. e2e-tests: Run tests in parallel shards with coverage # 3. merge-reports: Combine HTML reports from all shards # 4. comment-results: Post test results as PR comment -# 5. e2e-results: Status check to block merge on failure +# 5. upload-coverage: Merge and upload E2E coverage to Codecov +# 6. e2e-results: Status check to block merge on failure name: E2E Tests @@ -197,6 +205,35 @@ jobs: - name: Install Playwright browsers run: npx playwright install --with-deps ${{ matrix.browser }} + - name: Start Vite dev server for coverage + run: | + echo "šŸš€ Starting Vite dev server for E2E coverage..." + cd frontend + + # Use port 5173 (Vite default) with --strictPort to fail if busy + VITE_PORT=5173 + npx vite --port ${VITE_PORT} --strictPort > /tmp/vite.log 2>&1 & + VITE_PID=$! + echo "VITE_PID=${VITE_PID}" >> $GITHUB_ENV + echo "VITE_PORT=${VITE_PORT}" >> $GITHUB_ENV + + # Wait for Vite to be ready + echo "ā³ Waiting for Vite to start on port ${VITE_PORT}..." + MAX_WAIT=60 + WAITED=0 + while [[ ${WAITED} -lt ${MAX_WAIT} ]]; do + if curl -sf http://localhost:${VITE_PORT} > /dev/null 2>&1; then + echo "āœ… Vite dev server ready at http://localhost:${VITE_PORT}" + exit 0 + fi + sleep 1 + WAITED=$((WAITED + 1)) + done + + echo "āŒ Vite failed to start" + cat /tmp/vite.log + exit 1 + - name: Run E2E tests (Shard ${{ matrix.shard }}/${{ matrix.total-shards }}) run: | npx playwright test \ @@ -204,10 +241,18 @@ jobs: --shard=${{ matrix.shard }}/${{ matrix.total-shards }} \ --reporter=html,json,github env: - PLAYWRIGHT_BASE_URL: http://localhost:8080 + # Use Vite dev server for coverage (proxies API to Docker at 8080) + PLAYWRIGHT_BASE_URL: http://localhost:${{ env.VITE_PORT }} CI: true TEST_WORKER_INDEX: ${{ matrix.shard }} + - name: Stop Vite dev server + if: always() + run: | + if [[ -n "${VITE_PID}" ]]; then + kill ${VITE_PID} 2>/dev/null || true + fi + - name: Upload test results if: always() uses: actions/upload-artifact@v4 @@ -218,6 +263,14 @@ jobs: test-results/ retention-days: 7 + - name: Upload E2E coverage artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-coverage-shard-${{ matrix.shard }} + path: coverage/e2e/ + retention-days: 7 + - name: Upload test traces on failure if: failure() uses: actions/upload-artifact@v4 @@ -412,6 +465,71 @@ jobs: }); } + # Upload merged E2E coverage to Codecov + upload-coverage: + name: Upload E2E Coverage + runs-on: ubuntu-latest + needs: e2e-tests + if: always() && needs.e2e-tests.result == 'success' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + + - name: Download all coverage artifacts + uses: actions/download-artifact@v4 + with: + pattern: e2e-coverage-* + path: all-coverage + merge-multiple: false + + - name: Merge LCOV coverage files + run: | + # Install lcov for merging + sudo apt-get update && sudo apt-get install -y lcov + + # Create merged coverage directory + mkdir -p coverage/e2e-merged + + # Find all lcov.info files and merge them + LCOV_FILES=$(find all-coverage -name "lcov.info" -type f) + + if [[ -n "$LCOV_FILES" ]]; then + # Build merge command + MERGE_ARGS="" + for file in $LCOV_FILES; do + MERGE_ARGS="$MERGE_ARGS -a $file" + done + + lcov $MERGE_ARGS -o coverage/e2e-merged/lcov.info + echo "āœ… Merged $(echo "$LCOV_FILES" | wc -w) coverage files" + else + echo "āš ļø No coverage files found to merge" + exit 0 + fi + + - name: Upload E2E coverage to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage/e2e-merged/lcov.info + flags: e2e + name: e2e-coverage + fail_ci_if_error: false + + - name: Upload merged coverage artifact + uses: actions/upload-artifact@v4 + with: + name: e2e-coverage-merged + path: coverage/e2e-merged/ + retention-days: 30 + # Final status check - blocks merge if tests fail e2e-results: name: E2E Test Results diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 798a445f..1cd7c250 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -416,6 +416,18 @@ "close": false } }, + { + "label": "Test: E2E Playwright with Coverage", + "type": "shell", + "command": ".github/skills/scripts/skill-runner.sh test-e2e-playwright-coverage", + "group": "test", + "problemMatcher": [], + "presentation": { + "reveal": "always", + "panel": "dedicated", + "close": false + } + }, { "label": "Test: E2E Playwright - View Report", "type": "shell", diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..b871769f --- /dev/null +++ b/codecov.yml @@ -0,0 +1,41 @@ +# Codecov Configuration +# https://docs.codecov.com/docs/codecov-yaml + +coverage: + status: + project: + default: + target: auto + threshold: 1% + patch: + default: + target: 100% + +flags: + backend: + paths: + - backend/ + carryforward: true + + frontend: + paths: + - frontend/ + carryforward: true + + # E2E coverage flag - tracks frontend code exercised by Playwright tests + e2e: + paths: + - frontend/ + carryforward: true + +component_management: + individual_components: + - component_id: backend + paths: + - backend/** + - component_id: frontend + paths: + - frontend/** + - component_id: e2e + paths: + - frontend/** diff --git a/docs/plans/playwright-coverage-plan.md b/docs/plans/playwright-coverage-plan.md new file mode 100644 index 00000000..06466de1 --- /dev/null +++ b/docs/plans/playwright-coverage-plan.md @@ -0,0 +1,795 @@ +# Playwright E2E Coverage Integration Plan + +**Date:** January 18, 2026 +**Status:** šŸ”„ In Progress - Requires Vite Dev Server for source coverage +**Priority:** High - Enables coverage visibility for E2E tests +**Objective:** Integrate `@bgotink/playwright-coverage` to track frontend code coverage during E2E tests + +--- + +## āš ļø CRITICAL ARCHITECTURE NOTE + +**The original plan assumed V8 coverage would work against Docker production builds. This is INCORRECT.** + +### Why Docker Build Coverage Doesn't Work + +1. **V8 coverage** captures execution data for the bundled/minified JS served by Docker +2. **Source maps** in Docker container map to paths like `/app/frontend/src/...` +3. These source files **don't exist on the test host** (they're inside the container) +4. The coverage reporter can't resolve paths → 0/0 coverage + +### Correct Approach: Vite Dev Server + +For accurate source-level coverage: + +1. **Backend**: Docker container at `localhost:8080` (unchanged) +2. **Frontend**: Vite dev server at `localhost:3000` (`npm run dev`) +3. **Coverage tests**: Hit `localhost:3000` (Vite proxies API to Docker) + +**Why this works:** +- Vite serves actual `.tsx`/`.ts` files (not bundled) +- Source files exist on disk at project root +- V8 coverage maps directly to source without path rewriting +- Accurate line-level coverage for Codecov + +--- + +## Table of Contents + +1. [Overview](#1-overview) +2. [Current State Analysis](#2-current-state-analysis) +3. [Installation Steps](#3-installation-steps) +4. [Configuration Changes](#4-configuration-changes) +5. [Test File Modifications](#5-test-file-modifications) +6. [CI/CD Integration](#6-cicd-integration) +7. [Coverage Threshold Enforcement](#7-coverage-threshold-enforcement) +8. [Implementation Checklist](#8-implementation-checklist) +9. [Risks and Mitigations](#9-risks-and-mitigations) +10. [References](#10-references) + +--- + +## 1. Overview + +### What is `@bgotink/playwright-coverage`? + +`@bgotink/playwright-coverage` is a Playwright reporter that tracks JavaScript code coverage using V8 coverage without requiring any instrumentation. It: + +- Hooks into Playwright's Page objects to track V8 coverage +- Collects coverage data as test attachments +- Merges coverage from all tests into Istanbul format +- Generates reports in HTML, LCOV, JSON, and other Istanbul formats + +### Why Integrate E2E Coverage? + +- **Visibility**: Understand which frontend code paths are exercised by E2E tests +- **Gap Analysis**: Identify untested user journeys and critical paths +- **Quality Gate**: Ensure new features have E2E test coverage +- **Codecov Integration**: Unified coverage view across unit and E2E tests + +### Package Details + +| Property | Value | +|----------|-------| +| Package | `@bgotink/playwright-coverage` | +| Version | `0.3.2` (latest) | +| License | MIT | +| NPM Weekly Downloads | ~5,300 | +| Playwright Compatibility | ≄1.40.0 | + +--- + +## 2. Current State Analysis + +### Existing Playwright Setup + +**Configuration File:** [playwright.config.js](../../playwright.config.js) + +```javascript +// Current reporter configuration +reporter: process.env.CI + ? [['github'], ['html', { open: 'never' }]] + : [['list'], ['html', { open: 'on-failure' }]], +``` + +**Current Test Imports:** +```typescript +// Tests currently import directly from @playwright/test +import { test, expect } from '@playwright/test'; +``` + +**package.json Dependencies:** +```json +{ + "devDependencies": { + "@playwright/test": "^1.57.0", + "@types/node": "^25.0.9" + } +} +``` + +### Existing CI Workflows + +Two workflows handle Playwright tests: + +1. **[playwright.yml](../../.github/workflows/playwright.yml)** - Runs on workflow_run after Docker build +2. **[e2e-tests.yml](../../.github/workflows/e2e-tests.yml)** - Runs with sharding on PR/push + +### Existing Test Structure + +``` +tests/ +ā”œā”€ā”€ auth.setup.ts # Uses @playwright/test +ā”œā”€ā”€ core/ # Feature tests +ā”œā”€ā”€ dns-provider-*.spec.ts # Use @playwright/test +ā”œā”€ā”€ manual-dns-provider.spec.ts # Uses @playwright/test +ā”œā”€ā”€ fixtures/ # Shared fixtures +└── utils/ # Helper utilities +``` + +### Codecov Integration + +Current coverage uploads exist in [codecov-upload.yml](../../.github/workflows/codecov-upload.yml): +- `backend` flag for Go coverage +- `frontend` flag for Vitest/unit test coverage +- **Missing:** E2E coverage flag + +--- + +## 3. Installation Steps + +### Step 1: Install the Package + +```bash +npm install -D @bgotink/playwright-coverage +``` + +### Step 2: Verify Dependencies + +The package requires: +- Playwright ≄1.40.0 āœ… (Current: 1.57.0) +- Node.js ≄18 āœ… (Current: using LTS) + +### Step 3: Update package.json + +After installation, `package.json` should have: + +```json +{ + "devDependencies": { + "@bgotink/playwright-coverage": "^0.3.2", + "@playwright/test": "^1.57.0", + "@types/node": "^25.0.9", + "markdownlint-cli2": "^0.20.0" + } +} +``` + +--- + +## 4. Configuration Changes + +### 4.1 playwright.config.js Modifications + +Replace the current configuration with coverage-enabled version: + +```javascript +// @ts-check +import { defineConfig, devices } from '@playwright/test'; +import { defineCoverageReporterConfig } from '@bgotink/playwright-coverage'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const STORAGE_STATE = join(__dirname, 'playwright/.auth/user.json'); + +// Coverage reporter configuration +const coverageReporterConfig = defineCoverageReporterConfig({ + // Root directory for source file resolution + sourceRoot: __dirname, + + // Exclude non-application code from coverage + exclude: [ + '**/node_modules/**', + '**/playwright/**', + '**/tests/**', + '**/*.spec.ts', + '**/*.test.ts', + '**/coverage/**', + '**/dist/**', + '**/build/**', + ], + + // Output directory for coverage reports + resultDir: join(__dirname, 'coverage/e2e'), + + // Generate multiple report formats + reports: [ + // HTML report for visual inspection + ['html'], + // LCOV for Codecov upload + ['lcovonly', { file: 'lcov.info' }], + // JSON for programmatic access + ['json', { file: 'coverage.json' }], + // Text summary in console + ['text-summary', { file: null }], + ], + + // Coverage watermarks (visual thresholds in HTML report) + watermarks: { + statements: [50, 80], + branches: [50, 80], + functions: [50, 80], + lines: [50, 80], + }, + + // Path rewriting for Docker/CI environments + rewritePath: ({ absolutePath, relativePath }) => { + // Handle paths from Docker container + if (absolutePath.startsWith('/app/')) { + return absolutePath.replace('/app/', `${__dirname}/`); + } + return absolutePath; + }, +}); + +/** + * @see https://playwright.dev/docs/test-configuration + */ +export default defineConfig({ + testDir: './tests', + timeout: 30000, + expect: { + timeout: 5000, + }, + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 1 : undefined, + + // Updated reporter configuration with coverage + reporter: process.env.CI + ? [ + ['github'], + ['html', { open: 'never' }], + ['@bgotink/playwright-coverage', coverageReporterConfig], + ] + : [ + ['list'], + ['html', { open: 'on-failure' }], + ['@bgotink/playwright-coverage', coverageReporterConfig], + ], + + use: { + baseURL: process.env.PLAYWRIGHT_BASE_URL || 'http://localhost:8080', + trace: 'on-first-retry', + }, + + projects: [ + { + name: 'setup', + testMatch: /auth\.setup\.ts/, + }, + { + name: 'chromium', + use: { + ...devices['Desktop Chrome'], + storageState: STORAGE_STATE, + }, + dependencies: ['setup'], + }, + { + name: 'firefox', + use: { + ...devices['Desktop Firefox'], + storageState: STORAGE_STATE, + }, + dependencies: ['setup'], + }, + { + name: 'webkit', + use: { + ...devices['Desktop Safari'], + storageState: STORAGE_STATE, + }, + dependencies: ['setup'], + }, + ], +}); +``` + +### 4.2 Add Coverage Directory to .gitignore + +Ensure coverage outputs are not committed: + +```gitignore +# E2E Coverage +coverage/e2e/ +``` + +--- + +## 5. Test File Modifications + +### 5.1 Create Shared Test Fixture with Coverage + +Create a base test fixture that wraps `@bgotink/playwright-coverage`: + +```typescript +// tests/fixtures/coverage-test.ts + +import { test as coverageTest, expect } from '@bgotink/playwright-coverage'; +import { mergeTests } from '@playwright/test'; +import { test as authTest } from './auth-fixtures'; + +// Merge coverage tracking with auth fixtures +export const test = mergeTests(coverageTest, authTest); +export { expect }; +``` + +### 5.2 Update Test Files to Use Coverage-Enabled Test + +**Option A: Update Each Test File (Recommended for gradual rollout)** + +```typescript +// Before +import { test, expect } from '@playwright/test'; + +// After +import { test, expect } from '@bgotink/playwright-coverage'; +``` + +**Option B: Use Merged Fixture (Recommended for projects with custom fixtures)** + +```typescript +// Before +import { test, expect } from '../fixtures/auth-fixtures'; + +// After +import { test, expect } from '../fixtures/coverage-test'; +``` + +### 5.3 Update auth.setup.ts + +```typescript +// tests/auth.setup.ts + +// Use coverage-enabled test for setup +import { test as setup, expect } from '@bgotink/playwright-coverage'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +// ... rest of the file remains the same +``` + +### 5.4 Files Requiring Updates + +The following test files need import changes: + +| File | Current Import | New Import | +|------|----------------|------------| +| `tests/auth.setup.ts` | `@playwright/test` | `@bgotink/playwright-coverage` | +| `tests/manual-dns-provider.spec.ts` | `@playwright/test` | `@bgotink/playwright-coverage` | +| `tests/dns-provider-crud.spec.ts` | `@playwright/test` | `@bgotink/playwright-coverage` | +| `tests/dns-provider-types.spec.ts` | `@playwright/test` | `@bgotink/playwright-coverage` | +| `tests/example.spec.js` | `@playwright/test` | `@bgotink/playwright-coverage` | +| All files in `tests/core/` | `@playwright/test` | `@bgotink/playwright-coverage` | +| All files in `tests/fixtures/` | `@playwright/test` | `@bgotink/playwright-coverage` | + +--- + +## 6. CI/CD Integration + +### 6.1 Update Skill Script + +Create or update the skill script for E2E tests with coverage: + +**File:** `.github/skills/scripts/test-e2e-playwright-coverage.sh` + +```bash +#!/usr/bin/env bash +# test-e2e-playwright-coverage.sh +# Run Playwright E2E tests with coverage collection + +set -euo pipefail + +# Source helpers +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/_logging_helpers.sh" +source "${SCRIPT_DIR}/_error_handling_helpers.sh" + +log_info "šŸŽ­ Running Playwright E2E tests with coverage..." + +# Ensure coverage directory exists +mkdir -p coverage/e2e + +# Run Playwright tests +PLAYWRIGHT_HTML_OPEN=never npx playwright test --project=chromium + +# Check if coverage was generated +if [[ -f "coverage/e2e/lcov.info" ]]; then + log_success "āœ… E2E coverage generated: coverage/e2e/lcov.info" + + # Print summary + if [[ -f "coverage/e2e/coverage.json" ]]; then + log_info "šŸ“Š Coverage Summary:" + cat coverage/e2e/coverage.json | jq '.total' + fi +else + log_warning "āš ļø No coverage data generated (tests may have failed)" +fi +``` + +### 6.2 Update e2e-tests.yml Workflow + +Add coverage upload step to the existing workflow: + +```yaml +# .github/workflows/e2e-tests.yml - Add after test execution + + - name: Run E2E tests (Shard ${{ matrix.shard }}/${{ matrix.total-shards }}) + run: | + npx playwright test \ + --project=${{ matrix.browser }} \ + --shard=${{ matrix.shard }}/${{ matrix.total-shards }} \ + --reporter=html,json,github + env: + PLAYWRIGHT_BASE_URL: http://localhost:8080 + CI: true + TEST_WORKER_INDEX: ${{ matrix.shard }} + + # NEW: Upload E2E coverage + - name: Upload E2E coverage artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-coverage-shard-${{ matrix.shard }} + path: coverage/e2e/ + retention-days: 7 +``` + +### 6.3 Add Coverage Merge Job + +Add a job to merge sharded coverage and upload to Codecov: + +```yaml + # Add after merge-reports job + upload-coverage: + name: Upload E2E Coverage + runs-on: ubuntu-latest + needs: e2e-tests + if: always() && needs.e2e-tests.result == 'success' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + + - name: Download all coverage artifacts + uses: actions/download-artifact@v4 + with: + pattern: e2e-coverage-* + path: all-coverage + merge-multiple: false + + - name: Merge LCOV coverage files + run: | + # Install lcov for merging + sudo apt-get update && sudo apt-get install -y lcov + + # Create merged coverage directory + mkdir -p coverage/e2e-merged + + # Find all lcov.info files and merge them + LCOV_FILES=$(find all-coverage -name "lcov.info" -type f) + + if [[ -n "$LCOV_FILES" ]]; then + # Build merge command + MERGE_ARGS="" + for file in $LCOV_FILES; do + MERGE_ARGS="$MERGE_ARGS -a $file" + done + + lcov $MERGE_ARGS -o coverage/e2e-merged/lcov.info + echo "āœ… Merged $(echo "$LCOV_FILES" | wc -w) coverage files" + else + echo "āš ļø No coverage files found to merge" + exit 0 + fi + + - name: Upload E2E coverage to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage/e2e-merged/lcov.info + flags: e2e + name: e2e-coverage + fail_ci_if_error: false # Don't fail build on upload error + + - name: Upload merged coverage artifact + uses: actions/upload-artifact@v4 + with: + name: e2e-coverage-merged + path: coverage/e2e-merged/ + retention-days: 30 +``` + +### 6.4 Update codecov.yml Configuration + +Add E2E flag configuration: + +```yaml +# codecov.yml + +coverage: + status: + project: + default: + target: auto + threshold: 1% + patch: + default: + target: 100% + +flags: + backend: + paths: + - backend/ + carryforward: true + + frontend: + paths: + - frontend/ + carryforward: true + + # NEW: E2E coverage flag + e2e: + paths: + - frontend/ # E2E tests cover frontend code + carryforward: true + +component_management: + individual_components: + - component_id: backend + paths: + - backend/** + - component_id: frontend + paths: + - frontend/** + - component_id: e2e + paths: + - frontend/** +``` + +--- + +## 7. Coverage Threshold Enforcement + +### 7.1 Phase 1: Visibility Only (Current) + +Initially, collect coverage without failing builds: + +```javascript +// playwright.config.js - No threshold enforcement +// Coverage reporter only generates reports +``` + +### 7.2 Phase 2: Add Thresholds (After Baseline Established) + +Once baseline coverage is established (after ~2 weeks of data), add thresholds: + +```javascript +// Update playwright.config.js + +import { execSync } from 'child_process'; + +// Optional: Fail if coverage drops below threshold +const coverageReporterConfig = defineCoverageReporterConfig({ + // ... existing config ... + + // Add coverage check (Phase 2) + onEnd: async (coverageResults) => { + const summary = coverageResults.total; + + // Define minimum thresholds + const thresholds = { + statements: 40, + branches: 30, + functions: 40, + lines: 40, + }; + + let failed = false; + const failures = []; + + for (const [metric, threshold] of Object.entries(thresholds)) { + const actual = summary[metric].pct; + if (actual < threshold) { + failed = true; + failures.push(`${metric}: ${actual.toFixed(1)}% < ${threshold}%`); + } + } + + if (failed && process.env.ENFORCE_COVERAGE === 'true') { + console.error('\nāŒ E2E Coverage thresholds not met:'); + failures.forEach(f => console.error(` - ${f}`)); + process.exit(1); + } + }, +}); +``` + +### 7.3 Phase 3: Strict Enforcement (Future) + +After comprehensive E2E coverage is achieved: + +```yaml +# .github/workflows/e2e-tests.yml + + - name: Run E2E tests with coverage enforcement + run: npx playwright test --project=chromium + env: + ENFORCE_COVERAGE: 'true' # Enable threshold enforcement +``` + +--- + +## 8. Implementation Checklist + +### Phase 1: Installation and Configuration + +- [ ] Install `@bgotink/playwright-coverage` package +- [ ] Update `playwright.config.js` with coverage reporter +- [ ] Add `coverage/e2e/` to `.gitignore` +- [ ] Create coverage output directory structure + +### Phase 2: Test File Updates + +- [ ] Update `tests/auth.setup.ts` to use coverage-enabled test +- [ ] Update `tests/manual-dns-provider.spec.ts` +- [ ] Update `tests/dns-provider-crud.spec.ts` +- [ ] Update `tests/dns-provider-types.spec.ts` +- [ ] Update all files in `tests/core/` +- [ ] Update `tests/fixtures/auth-fixtures.ts` +- [ ] Create `tests/fixtures/coverage-test.ts` (merged fixture) + +### Phase 3: CI/CD Integration + +- [ ] Create `test-e2e-playwright-coverage.sh` skill script +- [ ] Update `.github/workflows/e2e-tests.yml` with coverage upload +- [ ] Add coverage merge job to workflow +- [ ] Update `codecov.yml` with `e2e` flag +- [ ] Test workflow on feature branch + +### Phase 4: Validation + +- [ ] Run tests locally and verify coverage generated +- [ ] Verify coverage appears in Codecov dashboard +- [ ] Verify HTML report is viewable +- [ ] Verify LCOV format is valid + +### Phase 5: Documentation + +- [ ] Update `docs/plans/current_spec.md` with coverage integration +- [ ] Add coverage information to `CONTRIBUTING.md` +- [ ] Document how to view local coverage reports + +--- + +## 9. Risks and Mitigations + +### Risk 1: Performance Impact + +**Risk:** Coverage collection may slow down test execution. + +**Mitigation:** +- V8 coverage is native and has minimal overhead (~5-10%) +- Coverage is only collected in CI, not blocking local development +- Monitor CI run times after integration + +### Risk 2: Incomplete Coverage Data + +**Risk:** Coverage may not capture all executed code paths. + +**Mitigation:** +- Ensure all test files use the coverage-enabled `test` function +- Verify source maps are correctly configured in frontend build +- Use `rewritePath` option to handle Docker path differences + +### Risk 3: Sharding Coverage Merge Issues + +**Risk:** Sharded test runs may produce incomplete merged coverage. + +**Mitigation:** +- Use `lcov` tool for reliable LCOV merging +- Verify merged coverage includes all shards +- Add validation step to check coverage completeness + +### Risk 4: Codecov Upload Failures + +**Risk:** Coverage uploads may fail intermittently. + +**Mitigation:** +- Set `fail_ci_if_error: false` initially +- Archive coverage artifacts for manual inspection +- Add retry logic if needed + +### Risk 5: Package Stability + +**Risk:** `@bgotink/playwright-coverage` is marked as "experimental". + +**Mitigation:** +- Package has been stable since 2019 with regular updates +- Pin to specific version in `package.json` +- Have fallback plan to remove if issues arise + +--- + +## 10. References + +- [bgotink/playwright-coverage GitHub](https://github.com/bgotink/playwright-coverage) +- [NPM Package](https://www.npmjs.com/package/@bgotink/playwright-coverage) +- [Playwright Test Configuration](https://playwright.dev/docs/test-configuration) +- [Istanbul Report Formats](https://istanbul.js.org/docs/advanced/alternative-reporters/) +- [Codecov Flags Documentation](https://docs.codecov.com/docs/flags) +- [LCOV Format Specification](https://ltp.sourceforge.net/coverage/lcov/geninfo.1.php) + +--- + +## Appendix A: Quick Start Commands + +```bash +# Install package +npm install -D @bgotink/playwright-coverage + +# Run tests locally with coverage +npx playwright test --project=chromium + +# View coverage HTML report +open coverage/e2e/index.html + +# Run specific test file with coverage +npx playwright test tests/manual-dns-provider.spec.ts + +# Generate only LCOV (for CI) +npx playwright test --project=chromium --reporter=@bgotink/playwright-coverage +``` + +## Appendix B: Troubleshooting + +### Coverage is Empty + +**Cause:** Tests are using `@playwright/test` instead of `@bgotink/playwright-coverage`. + +**Solution:** Update imports in all test files: +```typescript +import { test, expect } from '@bgotink/playwright-coverage'; +``` + +### Source Files Not Found in Report + +**Cause:** Path mismatch between test environment and source files. + +**Solution:** Configure `rewritePath` in coverage reporter config: +```javascript +rewritePath: ({ absolutePath }) => { + return absolutePath.replace('/app/', process.cwd() + '/'); +}, +``` + +### LCOV Merge Fails + +**Cause:** Missing `lcov` tool or malformed LCOV files. + +**Solution:** Install lcov and validate files: +```bash +sudo apt-get install lcov +lcov --version +head -20 coverage/e2e/lcov.info +``` diff --git a/frontend/e2e/tests/security-mobile.spec.ts b/frontend/e2e/tests/security-mobile.spec.ts index 50d32c71..f31660e4 100644 --- a/frontend/e2e/tests/security-mobile.spec.ts +++ b/frontend/e2e/tests/security-mobile.spec.ts @@ -5,7 +5,7 @@ * Tests mobile viewport (375x667), tablet viewport (768x1024), * touch targets, scrolling, and layout responsiveness. */ -import { test, expect } from '@playwright/test' +import { test, expect } from '@bgotink/playwright-coverage' const base = process.env.CHARON_BASE_URL || 'http://localhost:8080' diff --git a/frontend/e2e/tests/waf.spec.ts b/frontend/e2e/tests/waf.spec.ts index 31d6989a..8cf53121 100644 --- a/frontend/e2e/tests/waf.spec.ts +++ b/frontend/e2e/tests/waf.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test' +import { test, expect } from '@bgotink/playwright-coverage' const base = process.env.CHARON_BASE_URL || 'http://localhost:8080' diff --git a/frontend/tests/login.smoke.spec.ts b/frontend/tests/login.smoke.spec.ts index f200f813..960270f4 100644 --- a/frontend/tests/login.smoke.spec.ts +++ b/frontend/tests/login.smoke.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test' +import { test, expect } from '@bgotink/playwright-coverage' test.describe('Login - smoke', () => { test('renders and has no console errors on load', async ({ page }) => { diff --git a/package-lock.json b/package-lock.json index 55a15afa..4c69a8e6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,11 +8,68 @@ "tldts": "^7.0.19" }, "devDependencies": { + "@bgotink/playwright-coverage": "^0.3.2", "@playwright/test": "^1.57.0", "@types/node": "^25.0.9", "markdownlint-cli2": "^0.20.0" } }, + "node_modules/@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@bgotink/playwright-coverage": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@bgotink/playwright-coverage/-/playwright-coverage-0.3.2.tgz", + "integrity": "sha512-F6ow6TD2LpELb+qD4MrmUj4TyP48JByQ/PNu6gehRLRtnU1mwXCnqfpT8AQ0bGiqS73EEg6Ifa5ts5DPSYYU8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^0.2.3", + "comlink": "^4.4.2", + "convert-source-map": "^2.0.0", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.1.7", + "micromatch": "^4.0.8", + "node-fetch": "^3.2.0", + "v8-to-istanbul": "^9.3.0" + }, + "peerDependencies": { + "@playwright/test": "^1.14.1" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -90,6 +147,13 @@ "@types/ms": "*" } }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/katex": { "version": "0.16.7", "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.7.tgz", @@ -187,6 +251,13 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/comlink": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/comlink/-/comlink-4.4.2.tgz", + "integrity": "sha512-OxGdvBmJuNKSCMO4NTl1L47VRp6xn2wG4F/2hYzB6tiCb709otOxtEYCSvK80PtjODfXXZu8ds+Nw5kVCjqd2g==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/commander": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", @@ -197,6 +268,23 @@ "node": ">= 12" } }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -293,6 +381,30 @@ "reusify": "^1.0.4" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -306,6 +418,19 @@ "node": ">=8" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -368,6 +493,23 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, "node_modules/ignore": { "version": "7.0.5", "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", @@ -459,6 +601,45 @@ "node": ">=0.12.0" } }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", @@ -506,6 +687,22 @@ "uc.micro": "^2.0.0" } }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/markdown-it": { "version": "14.1.0", "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", @@ -554,7 +751,6 @@ "integrity": "sha512-esPk+8Qvx/f0bzI7YelUeZp+jCtFOk3KjZ7s9iBQZ6HlymSXoTtWGiIRZP05/9Oy2ehIoIjenVwndxGtxOIJYQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "globby": "15.0.0", "js-yaml": "4.1.1", @@ -1161,6 +1357,46 @@ "dev": true, "license": "MIT" }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dev": true, + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "node_modules/parse-entities": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", @@ -1305,6 +1541,19 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/slash": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", @@ -1351,6 +1600,19 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/tldts": { "version": "7.0.19", "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.19.tgz", @@ -1408,6 +1670,31 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/v8-to-istanbul": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", + "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", + "dev": true, + "license": "ISC", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } } } } diff --git a/package.json b/package.json index 27120be0..898be60d 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "tldts": "^7.0.19" }, "devDependencies": { + "@bgotink/playwright-coverage": "^0.3.2", "@playwright/test": "^1.57.0", "@types/node": "^25.0.9", "markdownlint-cli2": "^0.20.0" diff --git a/playwright.config.js b/playwright.config.js index 20b08be8..43cd515c 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -1,5 +1,6 @@ // @ts-check import { defineConfig, devices } from '@playwright/test'; +import { defineCoverageReporterConfig } from '@bgotink/playwright-coverage'; import { fileURLToPath } from 'url'; import { dirname, join } from 'path'; @@ -17,6 +18,73 @@ const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const STORAGE_STATE = join(__dirname, 'playwright/.auth/user.json'); +/** + * Coverage reporter configuration for E2E tests + * Tracks V8 coverage during Playwright test execution + */ +const coverageReporterConfig = defineCoverageReporterConfig({ + // Root directory for source file resolution + sourceRoot: __dirname, + + // Exclude non-application code from coverage + exclude: [ + '**/node_modules/**', + '**/playwright/**', + '**/tests/**', + '**/*.spec.ts', + '**/*.spec.js', + '**/*.test.ts', + '**/coverage/**', + '**/dist/**', + '**/build/**', + ], + + // Output directory for coverage reports + resultDir: join(__dirname, 'coverage/e2e'), + + // Generate multiple report formats + reports: [ + // HTML report for visual inspection + ['html'], + // LCOV for Codecov upload + ['lcovonly', { file: 'lcov.info' }], + // JSON for programmatic access + ['json', { file: 'coverage.json' }], + // Text summary in console + ['text-summary', { file: null }], + ], + + // Coverage watermarks (visual thresholds in HTML report) + watermarks: { + statements: [50, 80], + branches: [50, 80], + functions: [50, 80], + lines: [50, 80], + }, + + // Path rewriting for source file resolution + rewritePath: ({ absolutePath, relativePath }) => { + // Handle paths from Docker container + if (absolutePath.startsWith('/app/')) { + return absolutePath.replace('/app/', `${__dirname}/`); + } + + // Handle Vite dev server paths (relative to frontend/src) + // Vite serves files like "/src/components/Button.tsx" + if (absolutePath.startsWith('/src/')) { + return join(__dirname, 'frontend', absolutePath); + } + + // If path doesn't start with /, prepend frontend/src + if (!absolutePath.startsWith('/') && !absolutePath.includes('/')) { + // Bare filenames like "Button.tsx" - try to resolve to frontend/src + return join(__dirname, 'frontend/src', absolutePath); + } + + return absolutePath; + }, +}); + /** * @see https://playwright.dev/docs/test-configuration */ @@ -38,8 +106,16 @@ export default defineConfig({ workers: process.env.CI ? 1 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: process.env.CI - ? [['github'], ['html', { open: 'never' }]] - : [['list'], ['html', { open: 'on-failure' }]], + ? [ + ['github'], + ['html', { open: 'never' }], + ['@bgotink/playwright-coverage', coverageReporterConfig], + ] + : [ + ['list'], + ['html', { open: 'on-failure' }], + ['@bgotink/playwright-coverage', coverageReporterConfig], + ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('')`. */ diff --git a/tests/auth.setup.ts b/tests/auth.setup.ts index d509a0ad..75220fd4 100644 --- a/tests/auth.setup.ts +++ b/tests/auth.setup.ts @@ -1,4 +1,4 @@ -import { test as setup, expect } from '@playwright/test'; +import { test as setup, expect } from '@bgotink/playwright-coverage'; import { fileURLToPath } from 'url'; import { dirname, join } from 'path'; diff --git a/tests/core/proxy-hosts.spec.ts b/tests/core/proxy-hosts.spec.ts index 50f64dfc..f9eb2bd9 100644 --- a/tests/core/proxy-hosts.spec.ts +++ b/tests/core/proxy-hosts.spec.ts @@ -301,14 +301,50 @@ test.describe('Proxy Hosts - CRUD Operations', () => { }); await test.step('Verify host was created', async () => { - // Either toast notification or host appears in list + // Wait for either success toast OR host appearing in list + // Use Promise.race with proper Playwright auto-waiting for reliability const successToast = page.getByText(/success|created|saved/i); const hostInList = page.getByText(hostConfig.domain); - const hasSuccess = await successToast.isVisible({ timeout: 5000 }).catch(() => false); - const hasHostInList = await hostInList.isVisible({ timeout: 5000 }).catch(() => false); + // First, wait for any modal to close (form submission complete) + await page.waitForTimeout(1000); - expect(hasSuccess || hasHostInList).toBeTruthy(); + // Try waiting for success indicators with proper retry logic + let verified = false; + + // Check 1: Wait for toast (may have already disappeared) + const hasSuccess = await successToast.isVisible({ timeout: 2000 }).catch(() => false); + if (hasSuccess) { + verified = true; + } + + // Check 2: If no toast, check if we're back on list page with the host visible + if (!verified) { + // Wait for navigation back to list + await page.waitForURL(/\/proxy-hosts(?!\/)/, { timeout: 5000 }).catch(() => {}); + await waitForLoadingComplete(page); + + // Now check for the host in the list + const hasHostInList = await hostInList.isVisible({ timeout: 5000 }).catch(() => false); + if (hasHostInList) { + verified = true; + } + } + + // Check 3: If still not verified, the form might still be open - check for no error + if (!verified) { + const errorMessage = page.getByText(/error|failed|invalid/i); + const hasError = await errorMessage.isVisible({ timeout: 1000 }).catch(() => false); + // If no error is shown and we're past the form, consider it a pass + if (!hasError) { + // Refresh and check list + await page.goto('/proxy-hosts'); + await waitForLoadingComplete(page); + verified = await hostInList.isVisible({ timeout: 5000 }).catch(() => false); + } + } + + expect(verified).toBeTruthy(); }); }); diff --git a/tests/dns-provider-crud.spec.ts b/tests/dns-provider-crud.spec.ts index 8fccea35..aff67c68 100644 --- a/tests/dns-provider-crud.spec.ts +++ b/tests/dns-provider-crud.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '@bgotink/playwright-coverage'; /** * DNS Provider CRUD Operations E2E Tests diff --git a/tests/dns-provider-types.spec.ts b/tests/dns-provider-types.spec.ts index 7c3e040a..5b8bd666 100644 --- a/tests/dns-provider-types.spec.ts +++ b/tests/dns-provider-types.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from '@playwright/test'; +import { test, expect } from '@bgotink/playwright-coverage'; /** * DNS Provider Types E2E Tests diff --git a/tests/example.spec.js b/tests/example.spec.js index 26ed2060..9a4cd5dc 100644 --- a/tests/example.spec.js +++ b/tests/example.spec.js @@ -1,5 +1,5 @@ // @ts-check -import { test, expect } from '@playwright/test'; +import { test, expect } from '@bgotink/playwright-coverage'; test('has title', async ({ page }) => { await page.goto('https://playwright.dev/'); diff --git a/tests/fixtures/auth-fixtures.ts b/tests/fixtures/auth-fixtures.ts index 4ef5bcb1..682f6151 100644 --- a/tests/fixtures/auth-fixtures.ts +++ b/tests/fixtures/auth-fixtures.ts @@ -22,7 +22,7 @@ * ``` */ -import { test as base, expect } from '@playwright/test'; +import { test as base, expect } from '@bgotink/playwright-coverage'; import { TestDataManager } from '../utils/TestDataManager'; /** @@ -180,7 +180,7 @@ export async function logoutUser(page: import('@playwright/test').Page): Promise /** * Re-export expect from @playwright/test for convenience */ -export { expect } from '@playwright/test'; +export { expect } from '@bgotink/playwright-coverage'; /** * Re-export the default test password for use in tests diff --git a/tests/manual-dns-provider.spec.ts b/tests/manual-dns-provider.spec.ts index ad37aaba..43ed8e33 100644 --- a/tests/manual-dns-provider.spec.ts +++ b/tests/manual-dns-provider.spec.ts @@ -1,4 +1,5 @@ -import { test, expect, type Page } from '@playwright/test'; +import { test, expect } from '@bgotink/playwright-coverage'; +import type { Page } from '@playwright/test'; /** * Manual DNS Provider E2E Tests diff --git a/tests/utils/wait-helpers.ts b/tests/utils/wait-helpers.ts index 20d09771..bcf3668b 100644 --- a/tests/utils/wait-helpers.ts +++ b/tests/utils/wait-helpers.ts @@ -15,7 +15,8 @@ * ``` */ -import { Page, Locator, expect, Response } from '@playwright/test'; +import { expect } from '@bgotink/playwright-coverage'; +import type { Page, Locator, Response } from '@playwright/test'; /** * Options for waitForToast