diff --git a/.docker/compose/docker-compose.playwright-ci.yml b/.docker/compose/docker-compose.playwright-ci.yml index 79006f41..0a0e4606 100644 --- a/.docker/compose/docker-compose.playwright-ci.yml +++ b/.docker/compose/docker-compose.playwright-ci.yml @@ -27,7 +27,7 @@ services: # Charon Application - Core E2E Testing Service # ============================================================================= charon-app: - # CI provides CHARON_E2E_IMAGE_TAG=charon:e2e-test (locally built image) + # CI provides CHARON_E2E_IMAGE_TAG=charon:e2e-test (retagged from shared digest) # Local development uses the default fallback value image: ${CHARON_E2E_IMAGE_TAG:-charon:e2e-test} container_name: charon-playwright diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index dbd071f7..5228bd98 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,9 +1,6 @@ name: Go Benchmark on: - workflow_run: - workflows: ["Docker Build, Publish & Test"] - types: [completed] workflow_dispatch: concurrency: diff --git a/.github/workflows/cerberus-integration.yml b/.github/workflows/cerberus-integration.yml index 2dad8984..d22e021a 100644 --- a/.github/workflows/cerberus-integration.yml +++ b/.github/workflows/cerberus-integration.yml @@ -3,11 +3,6 @@ name: Cerberus Integration # Phase 2-3: Build Once, Test Many - Use registry image instead of building # This workflow now waits for docker-build.yml to complete and pulls the built image on: - workflow_run: - workflows: ["Docker Build, Publish & Test"] - types: [completed] - branches: [main, development, 'feature/**', 'hotfix/**'] - # Allow manual trigger for debugging workflow_dispatch: inputs: image_tag: @@ -27,7 +22,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 # Only run if docker-build.yml succeeded, or if manually triggered - if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') }} + if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && (github.event.workflow_run.status != 'completed' || github.event.workflow_run.conclusion == 'success')) }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -57,9 +52,10 @@ jobs: # Extract 7-character short SHA SHORT_SHA=$(echo "$SHA" | cut -c1-7) - if [[ "$EVENT" == "pull_request" ]]; then - # Use native pull_requests array (no API calls needed) - PR_NUM=$(echo '${{ toJson(github.event.workflow_run.pull_requests) }}' | jq -r '.[0].number') + # Use native pull_requests array (no API calls needed) + PR_NUM=$(echo '${{ toJson(github.event.workflow_run.pull_requests) }}' | jq -r '.[0].number // empty') + + if [[ "$EVENT" == "pull_request" || -n "$PR_NUM" ]]; then # Fallback for direct PR trigger if [[ -z "$PR_NUM" || "$PR_NUM" == "null" ]]; then diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml new file mode 100644 index 00000000..b96860cb --- /dev/null +++ b/.github/workflows/ci-pipeline.yml @@ -0,0 +1,697 @@ +name: CI Pipeline + +on: + pull_request: + workflow_dispatch: + inputs: + image_tag_override: + description: 'Optional image tag to use for build outputs' + required: false + type: string + run_coverage: + description: 'Run backend/frontend coverage jobs' + required: false + default: true + type: boolean + run_security_scans: + description: 'Run CodeQL, Trivy, and supply-chain checks' + required: false + default: true + type: boolean + run_integration: + description: 'Run integration test jobs' + required: false + default: true + type: boolean + run_e2e: + description: 'Run Playwright E2E tests' + required: false + default: true + type: boolean + +concurrency: + group: ci-manual-pipeline-${{ github.ref_name }}-${{ github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + GO_VERSION: '1.25.7' + NODE_VERSION: '24.12.0' + GOTOOLCHAIN: auto + GHCR_REGISTRY: ghcr.io + DOCKERHUB_REGISTRY: docker.io + IMAGE_NAME: wikid82/charon + IS_FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == true && github.repository != github.event.pull_request.head.repo.full_name }} + +jobs: + lint: + name: Lint and Repo Health + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Repo health check + run: bash scripts/repo_health_check.sh + + - name: Run Hadolint + uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 + with: + dockerfile: Dockerfile + config: .hadolint.yaml + failure-threshold: warning + + - name: Set up Go + uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: backend/go.sum + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20 # v9.2.0 + with: + version: latest + working-directory: backend + args: --timeout=5m + continue-on-error: true + + - name: GORM Security Scanner + run: | + chmod +x scripts/scan-gorm-security.sh + ./scripts/scan-gorm-security.sh --check + + - name: Set up Node.js + uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: frontend/package-lock.json + + - name: Install frontend dependencies + working-directory: frontend + run: npm ci + + - name: Run frontend lint + working-directory: frontend + run: npm run lint + continue-on-error: true + + build-image: + name: Build and Publish Image + runs-on: ubuntu-latest + needs: lint + permissions: + contents: read + packages: write + outputs: + image_digest: ${{ steps.build.outputs.digest }} + image_ref: ${{ steps.outputs.outputs.image_ref_dockerhub }} + image_ref_dockerhub: ${{ steps.outputs.outputs.image_ref_dockerhub }} + image_ref_ghcr: ${{ steps.outputs.outputs.image_ref_ghcr }} + image_tag: ${{ steps.outputs.outputs.image_tag }} + push_image: ${{ steps.image-policy.outputs.push }} + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Normalize image name + run: | + IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') + echo "IMAGE_NAME=${IMAGE_NAME}" >> $GITHUB_ENV + + - name: Determine image push policy + id: image-policy + run: | + PUSH_IMAGE=true + if [ "${{ github.event_name }}" = "pull_request" ]; then + if [ "${{ github.event.pull_request.head.repo.fork }}" = "true" ] && \ + [ "${{ github.repository }}" != "${{ github.event.pull_request.head.repo.full_name }}" ]; then + PUSH_IMAGE=false + fi + fi + echo "push=${PUSH_IMAGE}" >> "$GITHUB_OUTPUT" + + - name: Compute image tags + id: tags + run: | + SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) + DEFAULT_TAG="sha-${SHORT_SHA}" + if [ -n "${{ inputs.image_tag_override }}" ]; then + DEFAULT_TAG="${{ inputs.image_tag_override }}" + elif [ "${{ github.event_name }}" = "pull_request" ]; then + PR_NUMBER="${{ github.event.pull_request.number }}" + if [ -n "${PR_NUMBER}" ]; then + DEFAULT_TAG="pr-${PR_NUMBER}-${SHORT_SHA}" + fi + fi + + TAGS=() + TAGS+=("${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:${DEFAULT_TAG}") + TAGS+=("${{ env.DOCKERHUB_REGISTRY }}/${{ env.IMAGE_NAME }}:${DEFAULT_TAG}") + + if [ "${{ github.ref_name }}" = "main" ]; then + TAGS+=("${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest") + TAGS+=("${{ env.DOCKERHUB_REGISTRY }}/${{ env.IMAGE_NAME }}:latest") + fi + + if [ "${{ github.ref_name }}" = "development" ]; then + TAGS+=("${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:dev") + TAGS+=("${{ env.DOCKERHUB_REGISTRY }}/${{ env.IMAGE_NAME }}:dev") + fi + + if [ "${{ github.ref_name }}" = "nightly" ]; then + TAGS+=("${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:nightly") + TAGS+=("${{ env.DOCKERHUB_REGISTRY }}/${{ env.IMAGE_NAME }}:nightly") + fi + + { + echo "tags<> "$GITHUB_OUTPUT" + echo "image_tag=${DEFAULT_TAG}" >> "$GITHUB_OUTPUT" + + - name: Set up QEMU + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + + - name: Log in to GitHub Container Registry + if: ${{ steps.image-policy.outputs.push == 'true' }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.GHCR_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to Docker Hub + if: ${{ steps.image-policy.outputs.push == 'true' && secrets.DOCKERHUB_TOKEN != '' }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push Docker image + id: build + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6 + with: + context: . + file: ./Dockerfile + push: ${{ steps.image-policy.outputs.push == 'true' }} + load: ${{ steps.image-policy.outputs.push != 'true' }} + tags: ${{ steps.tags.outputs.tags }} + labels: | + org.opencontainers.image.revision=${{ github.sha }} + + - name: Emit image outputs + id: outputs + run: | + DIGEST="${{ steps.build.outputs.digest }}" + if [ -z "${DIGEST}" ]; then + echo "image_ref_dockerhub=" >> $GITHUB_OUTPUT + echo "image_ref_ghcr=" >> $GITHUB_OUTPUT + else + IMAGE_REF_DOCKERHUB="${{ env.DOCKERHUB_REGISTRY }}/${{ env.IMAGE_NAME }}@${DIGEST}" + IMAGE_REF_GHCR="${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}@${DIGEST}" + echo "image_ref_dockerhub=${IMAGE_REF_DOCKERHUB}" >> $GITHUB_OUTPUT + echo "image_ref_ghcr=${IMAGE_REF_GHCR}" >> $GITHUB_OUTPUT + fi + echo "image_tag=${{ steps.tags.outputs.image_tag }}" >> $GITHUB_OUTPUT + + integration-cerberus: + name: Integration - Cerberus + runs-on: ubuntu-latest + needs: build-image + if: inputs.run_integration != false && needs.build-image.outputs.push_image == 'true' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Log in to Docker Hub + if: ${{ secrets.DOCKERHUB_TOKEN != '' }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull shared image + run: | + docker pull "${{ needs.build-image.outputs.image_ref_dockerhub }}" + docker tag "${{ needs.build-image.outputs.image_ref_dockerhub }}" charon:local + + - name: Run Cerberus integration tests + run: | + chmod +x scripts/cerberus_integration.sh + scripts/cerberus_integration.sh + + integration-crowdsec: + name: Integration - CrowdSec + runs-on: ubuntu-latest + needs: build-image + if: inputs.run_integration != false && needs.build-image.outputs.push_image == 'true' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Log in to Docker Hub + if: ${{ secrets.DOCKERHUB_TOKEN != '' }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull shared image + run: | + docker pull "${{ needs.build-image.outputs.image_ref_dockerhub }}" + docker tag "${{ needs.build-image.outputs.image_ref_dockerhub }}" charon:local + + - name: Run CrowdSec integration tests + run: | + chmod +x .github/skills/scripts/skill-runner.sh + .github/skills/scripts/skill-runner.sh integration-test-crowdsec + .github/skills/scripts/skill-runner.sh integration-test-crowdsec-startup + + integration-waf: + name: Integration - WAF + runs-on: ubuntu-latest + needs: build-image + if: inputs.run_integration != false && needs.build-image.outputs.push_image == 'true' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Log in to Docker Hub + if: ${{ secrets.DOCKERHUB_TOKEN != '' }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull shared image + run: | + docker pull "${{ needs.build-image.outputs.image_ref_dockerhub }}" + docker tag "${{ needs.build-image.outputs.image_ref_dockerhub }}" charon:local + + - name: Run WAF integration tests + run: | + chmod +x scripts/coraza_integration.sh + scripts/coraza_integration.sh + + integration-ratelimit: + name: Integration - Rate Limit + runs-on: ubuntu-latest + needs: build-image + if: inputs.run_integration != false && needs.build-image.outputs.push_image == 'true' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Log in to Docker Hub + if: ${{ secrets.DOCKERHUB_TOKEN != '' }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull shared image + run: | + docker pull "${{ needs.build-image.outputs.image_ref_dockerhub }}" + docker tag "${{ needs.build-image.outputs.image_ref_dockerhub }}" charon:local + + - name: Run rate limit integration tests + run: | + chmod +x scripts/rate_limit_integration.sh + scripts/rate_limit_integration.sh + + integration-gate: + name: Integration Gate + runs-on: ubuntu-latest + needs: + - integration-cerberus + - integration-crowdsec + - integration-waf + - integration-ratelimit + if: always() + steps: + - name: Evaluate integration results + run: | + if [ "${{ inputs.run_integration }}" = "false" ]; then + echo "Integration stage skipped." + exit 0 + fi + + RESULTS=( + "${{ needs.integration-cerberus.result }}" + "${{ needs.integration-crowdsec.result }}" + "${{ needs.integration-waf.result }}" + "${{ needs.integration-ratelimit.result }}" + ) + + for RESULT in "${RESULTS[@]}"; do + if [ "$RESULT" = "failure" ] || [ "$RESULT" = "cancelled" ]; then + echo "Integration stage failed: $RESULT" + exit 1 + fi + done + + e2e: + name: E2E Tests with Coverage + needs: + - build-image + - integration-gate + if: inputs.run_e2e != false && needs.build-image.outputs.push_image == 'true' + uses: ./.github/workflows/e2e-tests-split.yml + with: + browser: all + test_category: all + image_ref: ${{ needs.build-image.outputs.image_ref_dockerhub }} + image_tag: charon:e2e-test + playwright_coverage: true + secrets: inherit + + coverage-backend: + name: Coverage - Backend + runs-on: ubuntu-latest + needs: + - build-image + - integration-gate + if: inputs.run_coverage != false + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: backend/go.sum + + - name: Run Go tests with coverage + env: + CGO_ENABLED: 1 + run: | + bash scripts/go-test-coverage.sh 2>&1 | tee backend/test-output.txt + exit ${PIPESTATUS[0]} + + - name: Upload coverage artifact + uses: actions/upload-artifact@ea165f2524e81b1a7f1f18e1bdb77f0840c18dd9 # v4 + with: + name: backend-coverage + path: backend/coverage.txt + retention-days: 1 + + coverage-frontend: + name: Coverage - Frontend + runs-on: ubuntu-latest + needs: + - build-image + - integration-gate + if: inputs.run_coverage != false + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + + - name: Set up Node.js + uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + working-directory: frontend + run: npm ci + + - name: Run frontend tests and coverage + run: | + bash scripts/frontend-test-coverage.sh 2>&1 | tee frontend/test-output.txt + exit ${PIPESTATUS[0]} + + - name: Upload coverage artifact + uses: actions/upload-artifact@ea165f2524e81b1a7f1f18e1bdb77f0840c18dd9 # v4 + with: + name: frontend-coverage + path: frontend/coverage + retention-days: 1 + + coverage-gate: + name: Coverage Gate + runs-on: ubuntu-latest + needs: + - coverage-backend + - coverage-frontend + - e2e + if: always() + steps: + - name: Evaluate coverage results + run: | + if [ "${{ inputs.run_coverage }}" = "false" ]; then + echo "Coverage stage skipped." + exit 0 + fi + + RESULTS=( + "${{ needs.coverage-backend.result }}" + "${{ needs.coverage-frontend.result }}" + "${{ needs.e2e.result }}" + ) + + for RESULT in "${RESULTS[@]}"; do + if [ "$RESULT" = "failure" ] || [ "$RESULT" = "cancelled" ]; then + echo "Coverage stage failed: $RESULT" + exit 1 + fi + done + + codecov-upload: + name: Codecov Upload + runs-on: ubuntu-latest + needs: + - coverage-gate + if: inputs.run_coverage != false + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Download backend coverage artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: backend-coverage + path: backend/ + + - name: Download frontend coverage artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: frontend-coverage + path: frontend/coverage + + - name: Download E2E coverage artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: e2e-coverage + path: coverage/e2e + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@7f9fc5e3cf521e84e0c9a667b0f6c6ad08c94b82 # v5.1.3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: backend + files: backend/coverage.txt + fail_ci_if_error: false + + - name: Upload frontend coverage to Codecov + uses: codecov/codecov-action@7f9fc5e3cf521e84e0c9a667b0f6c6ad08c94b82 # v5.1.3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: frontend + files: frontend/coverage/lcov.info + fail_ci_if_error: false + + - name: Upload E2E coverage to Codecov + uses: codecov/codecov-action@7f9fc5e3cf521e84e0c9a667b0f6c6ad08c94b82 # v5.1.3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: e2e + files: coverage/e2e/lcov.info + fail_ci_if_error: false + + codecov-gate: + name: Codecov Gate + runs-on: ubuntu-latest + needs: + - codecov-upload + if: always() + steps: + - name: Evaluate Codecov upload results + run: | + if [ "${{ inputs.run_coverage }}" = "false" ]; then + echo "Codecov upload stage skipped." + exit 0 + fi + + if [ "${{ needs.codecov-upload.result }}" = "failure" ] || [ "${{ needs.codecov-upload.result }}" = "cancelled" ]; then + echo "Codecov upload failed: ${{ needs.codecov-upload.result }}" + exit 1 + fi + + security-codeql: + name: Security - CodeQL + runs-on: ubuntu-latest + needs: + - codecov-gate + if: inputs.run_security_scans != false && env.IS_FORK != 'true' + permissions: + contents: read + security-events: write + actions: read + pull-requests: read + strategy: + fail-fast: false + matrix: + language: ['go', 'javascript-typescript'] + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Initialize CodeQL + uses: github/codeql-action/init@45cbd0c69e560cd9e7cd7f8c32362050c9b7ded2 # v4 + with: + languages: ${{ matrix.language }} + config-file: ./.github/codeql/codeql-config.yml + + - name: Setup Go + if: matrix.language == 'go' + uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: backend/go.sum + + - name: Autobuild + uses: github/codeql-action/autobuild@45cbd0c69e560cd9e7cd7f8c32362050c9b7ded2 # v4 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@45cbd0c69e560cd9e7cd7f8c32362050c9b7ded2 # v4 + with: + category: "/language:${{ matrix.language }}" + + security-trivy: + name: Security - Trivy Image Scan + runs-on: ubuntu-latest + needs: + - build-image + - codecov-gate + if: inputs.run_security_scans != false && needs.build-image.outputs.push_image == 'true' + permissions: + contents: read + security-events: write + steps: + - name: Log in to Docker Hub + if: ${{ secrets.DOCKERHUB_TOKEN != '' }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Run Trivy image scan (SARIF) + uses: aquasecurity/trivy-action@22438a435773de8c97dc0958cc0b823c45b064ac + with: + scan-type: image + image-ref: ${{ needs.build-image.outputs.image_ref_dockerhub }} + format: sarif + output: trivy-image-results.sarif + severity: 'CRITICAL,HIGH,MEDIUM' + continue-on-error: true + + - name: Upload Trivy SARIF to GitHub Security + uses: github/codeql-action/upload-sarif@b13d724d35ff0a814e21683638ed68ed34cf53d1 + with: + sarif_file: trivy-image-results.sarif + category: trivy-image + continue-on-error: true + + - name: Run Trivy image scan (fail on CRITICAL/HIGH) + uses: aquasecurity/trivy-action@22438a435773de8c97dc0958cc0b823c45b064ac + with: + scan-type: image + image-ref: ${{ needs.build-image.outputs.image_ref_dockerhub }} + format: table + severity: 'CRITICAL,HIGH' + exit-code: '1' + + security-supply-chain: + name: Security - Supply Chain + runs-on: ubuntu-latest + needs: + - build-image + - codecov-gate + if: inputs.run_security_scans != false && needs.build-image.outputs.push_image == 'true' + permissions: + contents: read + security-events: write + steps: + - name: Log in to Docker Hub + if: ${{ secrets.DOCKERHUB_TOKEN != '' }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Generate SBOM + uses: anchore/sbom-action@28d71544de8eaf1b958d335707167c5f783590ad # v0.22.2 + with: + image: ${{ needs.build-image.outputs.image_ref_dockerhub }} + format: cyclonedx-json + output-file: sbom.cyclonedx.json + + - name: Scan SBOM for vulnerabilities + uses: anchore/scan-action@7037fa011853d5a11690026fb85feee79f4c946c # v7.3.2 + with: + sbom: sbom.cyclonedx.json + fail-build: false + output-format: json + + pipeline-gate: + name: Pipeline Gate + runs-on: ubuntu-latest + needs: + - lint + - build-image + - integration-gate + - coverage-gate + - codecov-gate + - security-codeql + - security-trivy + - security-supply-chain + if: always() + steps: + - name: Evaluate pipeline results + run: | + RESULTS=( + "${{ needs.lint.result }}" + "${{ needs.build-image.result }}" + "${{ needs.integration-gate.result }}" + "${{ needs.coverage-gate.result }}" + "${{ needs.codecov-gate.result }}" + "${{ needs.security-codeql.result }}" + "${{ needs.security-trivy.result }}" + "${{ needs.security-supply-chain.result }}" + ) + + for RESULT in "${RESULTS[@]}"; do + if [ "$RESULT" = "failure" ] || [ "$RESULT" = "cancelled" ]; then + echo "Pipeline failed: $RESULT" + exit 1 + fi + done diff --git a/.github/workflows/codecov-upload.yml b/.github/workflows/codecov-upload.yml index 705d162c..b39739ef 100644 --- a/.github/workflows/codecov-upload.yml +++ b/.github/workflows/codecov-upload.yml @@ -1,12 +1,21 @@ name: Upload Coverage to Codecov on: - workflow_run: - workflows: ["Docker Build, Publish & Test"] - types: [completed] + workflow_dispatch: + inputs: + run_backend: + description: 'Run backend coverage upload' + required: false + default: true + type: boolean + run_frontend: + description: 'Run frontend coverage upload' + required: false + default: true + type: boolean concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.workflow_run.head_branch || github.head_ref || github.ref_name }} + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.run_id }} cancel-in-progress: true env: @@ -22,13 +31,13 @@ jobs: name: Backend Codecov Upload runs-on: ubuntu-latest timeout-minutes: 15 - if: ${{ github.event.workflow_run.conclusion == 'success' }} + if: ${{ inputs.run_backend != false }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 - ref: ${{ github.event.workflow_run.head_sha || github.sha }} + ref: ${{ github.sha }} - name: Set up Go uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6 @@ -56,13 +65,13 @@ jobs: name: Frontend Codecov Upload runs-on: ubuntu-latest timeout-minutes: 15 - if: ${{ github.event.workflow_run.conclusion == 'success' }} + if: ${{ inputs.run_frontend != false }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 - ref: ${{ github.event.workflow_run.head_sha || github.sha }} + ref: ${{ github.sha }} - name: Set up Node.js uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 6fa77034..d2149228 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,11 +1,9 @@ name: CodeQL - Analyze on: - workflow_run: - workflows: ["Docker Build, Publish & Test"] - types: [completed] + workflow_dispatch: schedule: - - cron: '0 3 * * 1' + - cron: '0 3 * * 1' # Mondays 03:00 UTC concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.workflow_run.head_branch || github.head_ref || github.ref_name }} @@ -27,7 +25,7 @@ jobs: runs-on: ubuntu-latest # Skip forked PRs where CHARON_TOKEN lacks security-events permissions if: >- - (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success') + (github.event_name != 'workflow_run' || github.event.workflow_run.status != 'completed' || github.event.workflow_run.conclusion == 'success') permissions: contents: read security-events: write diff --git a/.github/workflows/crowdsec-integration.yml b/.github/workflows/crowdsec-integration.yml index b56c2ec3..95e4b708 100644 --- a/.github/workflows/crowdsec-integration.yml +++ b/.github/workflows/crowdsec-integration.yml @@ -3,11 +3,6 @@ name: CrowdSec Integration # Phase 2-3: Build Once, Test Many - Use registry image instead of building # This workflow now waits for docker-build.yml to complete and pulls the built image on: - workflow_run: - workflows: ["Docker Build, Publish & Test"] - types: [completed] - branches: [main, development, 'feature/**', 'hotfix/**'] - # Allow manual trigger for debugging workflow_dispatch: inputs: image_tag: @@ -27,7 +22,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 # Only run if docker-build.yml succeeded, or if manually triggered - if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') }} + if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && (github.event.workflow_run.status != 'completed' || github.event.workflow_run.conclusion == 'success')) }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -57,9 +52,10 @@ jobs: # Extract 7-character short SHA SHORT_SHA=$(echo "$SHA" | cut -c1-7) - if [[ "$EVENT" == "pull_request" ]]; then - # Use native pull_requests array (no API calls needed) - PR_NUM=$(echo '${{ toJson(github.event.workflow_run.pull_requests) }}' | jq -r '.[0].number') + # Use native pull_requests array (no API calls needed) + PR_NUM=$(echo '${{ toJson(github.event.workflow_run.pull_requests) }}' | jq -r '.[0].number // empty') + + if [[ "$EVENT" == "pull_request" || -n "$PR_NUM" ]]; then # Fallback for direct PR trigger if [[ -z "$PR_NUM" || "$PR_NUM" == "null" ]]; then diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 9bb2f792..760de655 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -21,17 +21,7 @@ name: Docker Build, Publish & Test # See: docs/plans/current_spec.md (Section 4.1 - docker-build.yml changes) on: - workflow_run: - workflows: [Docker Lint] - types: - - completed - branches: - - main - - development - - 'feature/**' - - 'hotfix/**' workflow_dispatch: - workflow_call: concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch || github.head_ref || github.ref_name }} diff --git a/.github/workflows/docker-lint.yml b/.github/workflows/docker-lint.yml index 6e4ca354..4186387f 100644 --- a/.github/workflows/docker-lint.yml +++ b/.github/workflows/docker-lint.yml @@ -1,10 +1,7 @@ name: Docker Lint on: - push: - branches: [ main, development, 'feature/**', 'hotfix/**' ] - pull_request: - branches: [ main, development, 'feature/**', 'hotfix/**' ] + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref_name }} diff --git a/.github/workflows/e2e-tests-split.yml b/.github/workflows/e2e-tests-split.yml index 1dc4a191..d869d7ed 100644 --- a/.github/workflows/e2e-tests-split.yml +++ b/.github/workflows/e2e-tests-split.yml @@ -13,9 +13,38 @@ name: 'E2E Tests' on: - workflow_run: - workflows: ["Docker Build, Publish & Test"] - types: [completed] + workflow_call: + inputs: + browser: + description: 'Browser to test' + required: false + default: 'all' + type: string + test_category: + description: 'Test category' + required: false + default: 'all' + type: string + image_ref: + description: 'Image reference (digest) to test, e.g. docker.io/wikid82/charon@sha256:...' + required: false + type: string + image_tag: + description: 'Local image tag for compose usage (default: charon:e2e-test)' + required: false + type: string + playwright_coverage: + description: 'Enable Playwright coverage (V8)' + required: false + default: false + type: boolean + secrets: + CHARON_EMERGENCY_TOKEN: + required: false + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false workflow_dispatch: inputs: browser: @@ -37,37 +66,75 @@ on: - all - security - non-security + image_ref: + description: 'Image reference (digest) to test, e.g. docker.io/wikid82/charon@sha256:...' + required: false + type: string + image_tag: + description: 'Local image tag for compose usage (default: charon:e2e-test)' + required: false + type: string + playwright_coverage: + description: 'Enable Playwright coverage (V8)' + required: false + default: false + type: boolean env: NODE_VERSION: '20' GO_VERSION: '1.25.7' GOTOOLCHAIN: auto - REGISTRY: ghcr.io + DOCKERHUB_REGISTRY: docker.io IMAGE_NAME: ${{ github.repository_owner }}/charon - PLAYWRIGHT_COVERAGE: ${{ vars.PLAYWRIGHT_COVERAGE || '0' }} + PLAYWRIGHT_COVERAGE: ${{ (inputs.playwright_coverage && '1') || (vars.PLAYWRIGHT_COVERAGE || '0') }} DEBUG: 'charon:*,charon-test:*' PLAYWRIGHT_DEBUG: '1' CI_LOG_LEVEL: 'verbose' concurrency: - group: e2e-split-${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || github.event.pull_request.number || github.event.workflow_run.head_branch || github.head_ref || github.ref_name }} + group: e2e-split-${{ github.workflow }}-${{ github.ref_name }}-${{ github.run_id }} cancel-in-progress: true jobs: - # Build application once, share across all browser jobs + # Prepare application image once, share across all browser jobs build: - name: Build Application + name: Prepare Application Image runs-on: ubuntu-latest - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} outputs: - image_digest: ${{ steps.build-image.outputs.digest }} + image_source: ${{ steps.resolve-image.outputs.image_source }} + image_ref: ${{ steps.resolve-image.outputs.image_ref }} + image_tag: ${{ steps.resolve-image.outputs.image_tag }} + image_digest: ${{ steps.resolve-image.outputs.image_digest != '' && steps.resolve-image.outputs.image_digest || steps.build-image.outputs.digest }} steps: + - name: Resolve image inputs + id: resolve-image + run: | + IMAGE_REF="${{ inputs.image_ref }}" + IMAGE_TAG="${{ inputs.image_tag || 'charon:e2e-test' }}" + if [ -n "$IMAGE_REF" ]; then + echo "image_source=registry" >> "$GITHUB_OUTPUT" + echo "image_ref=$IMAGE_REF" >> "$GITHUB_OUTPUT" + echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT" + if [[ "$IMAGE_REF" == *@* ]]; then + echo "image_digest=${IMAGE_REF#*@}" >> "$GITHUB_OUTPUT" + else + echo "image_digest=" >> "$GITHUB_OUTPUT" + fi + exit 0 + fi + echo "image_source=build" >> "$GITHUB_OUTPUT" + echo "image_ref=" >> "$GITHUB_OUTPUT" + echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT" + echo "image_digest=" >> "$GITHUB_OUTPUT" + - name: Checkout repository + if: steps.resolve-image.outputs.image_source == 'build' uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: - ref: ${{ github.event.workflow_run.head_sha || github.sha }} + ref: ${{ github.sha }} - name: Set up Go + if: steps.resolve-image.outputs.image_source == 'build' uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6 with: go-version: ${{ env.GO_VERSION }} @@ -75,12 +142,14 @@ jobs: cache-dependency-path: backend/go.sum - name: Set up Node.js + if: steps.resolve-image.outputs.image_source == 'build' uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 with: node-version: ${{ env.NODE_VERSION }} cache: 'npm' - name: Cache npm dependencies + if: steps.resolve-image.outputs.image_source == 'build' uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 with: path: ~/.npm @@ -88,27 +157,32 @@ jobs: restore-keys: npm- - name: Install dependencies + if: steps.resolve-image.outputs.image_source == 'build' run: npm ci - name: Set up Docker Buildx + if: steps.resolve-image.outputs.image_source == 'build' uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Build Docker image id: build-image + if: steps.resolve-image.outputs.image_source == 'build' uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6 with: context: . file: ./Dockerfile push: false load: true - tags: charon:e2e-test + tags: ${{ steps.resolve-image.outputs.image_tag }} cache-from: type=gha cache-to: type=gha,mode=max - name: Save Docker image - run: docker save charon:e2e-test -o charon-e2e-image.tar + if: steps.resolve-image.outputs.image_source == 'build' + run: docker save ${{ steps.resolve-image.outputs.image_tag }} -o charon-e2e-image.tar - name: Upload Docker image artifact + if: steps.resolve-image.outputs.image_source == 'build' uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: docker-image @@ -127,21 +201,20 @@ jobs: runs-on: ubuntu-latest needs: build if: | - (github.event_name != 'workflow_dispatch') || - (github.event.inputs.browser == 'chromium' || github.event.inputs.browser == 'all') && - (github.event.inputs.test_category == 'security' || github.event.inputs.test_category == 'all') + (inputs.browser == 'chromium' || inputs.browser == 'all') && + (inputs.test_category == 'security' || inputs.test_category == 'all') timeout-minutes: 30 env: CHARON_EMERGENCY_TOKEN: ${{ secrets.CHARON_EMERGENCY_TOKEN }} CHARON_EMERGENCY_SERVER_ENABLED: "true" CHARON_SECURITY_TESTS_ENABLED: "true" # Cerberus ON for enforcement tests - CHARON_E2E_IMAGE_TAG: charon:e2e-test + CHARON_E2E_IMAGE_TAG: ${{ needs.build.outputs.image_tag }} steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: - ref: ${{ github.event.workflow_run.head_sha || github.sha }} + ref: ${{ github.sha }} - name: Set up Node.js uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 @@ -149,7 +222,23 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' - - name: Download Docker image + - name: Log in to Docker Hub + if: needs.build.outputs.image_source == 'registry' && secrets.DOCKERHUB_TOKEN != '' + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull shared Docker image + if: needs.build.outputs.image_source == 'registry' + run: | + docker pull "${{ needs.build.outputs.image_ref }}" + docker tag "${{ needs.build.outputs.image_ref }}" "${{ needs.build.outputs.image_tag }}" + docker images | grep charon + + - name: Download Docker image artifact + if: needs.build.outputs.image_source == 'build' uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: docker-image @@ -171,7 +260,8 @@ jobs: env: CHARON_EMERGENCY_TOKEN: ${{ secrets.CHARON_EMERGENCY_TOKEN }} - - name: Load Docker image + - name: Load Docker image artifact + if: needs.build.outputs.image_source == 'build' run: | docker load -i charon-e2e-image.tar docker images | grep charon @@ -287,21 +377,20 @@ jobs: runs-on: ubuntu-latest needs: build if: | - (github.event_name != 'workflow_dispatch') || - (github.event.inputs.browser == 'firefox' || github.event.inputs.browser == 'all') && - (github.event.inputs.test_category == 'security' || github.event.inputs.test_category == 'all') + (inputs.browser == 'firefox' || inputs.browser == 'all') && + (inputs.test_category == 'security' || inputs.test_category == 'all') timeout-minutes: 30 env: CHARON_EMERGENCY_TOKEN: ${{ secrets.CHARON_EMERGENCY_TOKEN }} CHARON_EMERGENCY_SERVER_ENABLED: "true" CHARON_SECURITY_TESTS_ENABLED: "true" # Cerberus ON for enforcement tests - CHARON_E2E_IMAGE_TAG: charon:e2e-test + CHARON_E2E_IMAGE_TAG: ${{ needs.build.outputs.image_tag }} steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: - ref: ${{ github.event.workflow_run.head_sha || github.sha }} + ref: ${{ github.sha }} - name: Set up Node.js uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 @@ -309,7 +398,23 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' - - name: Download Docker image + - name: Log in to Docker Hub + if: needs.build.outputs.image_source == 'registry' && secrets.DOCKERHUB_TOKEN != '' + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull shared Docker image + if: needs.build.outputs.image_source == 'registry' + run: | + docker pull "${{ needs.build.outputs.image_ref }}" + docker tag "${{ needs.build.outputs.image_ref }}" "${{ needs.build.outputs.image_tag }}" + docker images | grep charon + + - name: Download Docker image artifact + if: needs.build.outputs.image_source == 'build' uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: docker-image @@ -331,7 +436,8 @@ jobs: env: CHARON_EMERGENCY_TOKEN: ${{ secrets.CHARON_EMERGENCY_TOKEN }} - - name: Load Docker image + - name: Load Docker image artifact + if: needs.build.outputs.image_source == 'build' run: | docker load -i charon-e2e-image.tar docker images | grep charon @@ -455,21 +561,20 @@ jobs: runs-on: ubuntu-latest needs: build if: | - (github.event_name != 'workflow_dispatch') || - (github.event.inputs.browser == 'webkit' || github.event.inputs.browser == 'all') && - (github.event.inputs.test_category == 'security' || github.event.inputs.test_category == 'all') + (inputs.browser == 'webkit' || inputs.browser == 'all') && + (inputs.test_category == 'security' || inputs.test_category == 'all') timeout-minutes: 30 env: CHARON_EMERGENCY_TOKEN: ${{ secrets.CHARON_EMERGENCY_TOKEN }} CHARON_EMERGENCY_SERVER_ENABLED: "true" CHARON_SECURITY_TESTS_ENABLED: "true" # Cerberus ON for enforcement tests - CHARON_E2E_IMAGE_TAG: charon:e2e-test + CHARON_E2E_IMAGE_TAG: ${{ needs.build.outputs.image_tag }} steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: - ref: ${{ github.event.workflow_run.head_sha || github.sha }} + ref: ${{ github.sha }} - name: Set up Node.js uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 @@ -477,7 +582,23 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' - - name: Download Docker image + - name: Log in to Docker Hub + if: needs.build.outputs.image_source == 'registry' && secrets.DOCKERHUB_TOKEN != '' + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull shared Docker image + if: needs.build.outputs.image_source == 'registry' + run: | + docker pull "${{ needs.build.outputs.image_ref }}" + docker tag "${{ needs.build.outputs.image_ref }}" "${{ needs.build.outputs.image_tag }}" + docker images | grep charon + + - name: Download Docker image artifact + if: needs.build.outputs.image_source == 'build' uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: docker-image @@ -499,7 +620,8 @@ jobs: env: CHARON_EMERGENCY_TOKEN: ${{ secrets.CHARON_EMERGENCY_TOKEN }} - - name: Load Docker image + - name: Load Docker image artifact + if: needs.build.outputs.image_source == 'build' run: | docker load -i charon-e2e-image.tar docker images | grep charon @@ -630,15 +752,14 @@ jobs: runs-on: ubuntu-latest needs: build if: | - (github.event_name != 'workflow_dispatch') || - (github.event.inputs.browser == 'chromium' || github.event.inputs.browser == 'all') && - (github.event.inputs.test_category == 'non-security' || github.event.inputs.test_category == 'all') + (inputs.browser == 'chromium' || inputs.browser == 'all') && + (inputs.test_category == 'non-security' || inputs.test_category == 'all') timeout-minutes: 20 env: CHARON_EMERGENCY_TOKEN: ${{ secrets.CHARON_EMERGENCY_TOKEN }} CHARON_EMERGENCY_SERVER_ENABLED: "true" CHARON_SECURITY_TESTS_ENABLED: "false" # Cerberus OFF for non-security tests - CHARON_E2E_IMAGE_TAG: charon:e2e-test + CHARON_E2E_IMAGE_TAG: ${{ needs.build.outputs.image_tag }} strategy: fail-fast: false matrix: @@ -649,7 +770,7 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: - ref: ${{ github.event.workflow_run.head_sha || github.sha }} + ref: ${{ github.sha }} - name: Set up Node.js uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 @@ -657,12 +778,29 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' - - name: Download Docker image + - name: Log in to Docker Hub + if: needs.build.outputs.image_source == 'registry' && secrets.DOCKERHUB_TOKEN != '' + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull shared Docker image + if: needs.build.outputs.image_source == 'registry' + run: | + docker pull "${{ needs.build.outputs.image_ref }}" + docker tag "${{ needs.build.outputs.image_ref }}" "${{ needs.build.outputs.image_tag }}" + docker images | grep charon + + - name: Download Docker image artifact + if: needs.build.outputs.image_source == 'build' uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: docker-image - - name: Load Docker image + - name: Load Docker image artifact + if: needs.build.outputs.image_source == 'build' run: | docker load -i charon-e2e-image.tar docker images | grep charon @@ -787,15 +925,14 @@ jobs: runs-on: ubuntu-latest needs: build if: | - (github.event_name != 'workflow_dispatch') || - (github.event.inputs.browser == 'firefox' || github.event.inputs.browser == 'all') && - (github.event.inputs.test_category == 'non-security' || github.event.inputs.test_category == 'all') + (inputs.browser == 'firefox' || inputs.browser == 'all') && + (inputs.test_category == 'non-security' || inputs.test_category == 'all') timeout-minutes: 20 env: CHARON_EMERGENCY_TOKEN: ${{ secrets.CHARON_EMERGENCY_TOKEN }} CHARON_EMERGENCY_SERVER_ENABLED: "true" CHARON_SECURITY_TESTS_ENABLED: "false" # Cerberus OFF for non-security tests - CHARON_E2E_IMAGE_TAG: charon:e2e-test + CHARON_E2E_IMAGE_TAG: ${{ needs.build.outputs.image_tag }} strategy: fail-fast: false matrix: @@ -806,7 +943,7 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: - ref: ${{ github.event.workflow_run.head_sha || github.sha }} + ref: ${{ github.sha }} - name: Set up Node.js uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 @@ -814,12 +951,29 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' - - name: Download Docker image + - name: Log in to Docker Hub + if: needs.build.outputs.image_source == 'registry' && secrets.DOCKERHUB_TOKEN != '' + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull shared Docker image + if: needs.build.outputs.image_source == 'registry' + run: | + docker pull "${{ needs.build.outputs.image_ref }}" + docker tag "${{ needs.build.outputs.image_ref }}" "${{ needs.build.outputs.image_tag }}" + docker images | grep charon + + - name: Download Docker image artifact + if: needs.build.outputs.image_source == 'build' uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: docker-image - - name: Load Docker image + - name: Load Docker image artifact + if: needs.build.outputs.image_source == 'build' run: | docker load -i charon-e2e-image.tar docker images | grep charon @@ -952,15 +1106,14 @@ jobs: runs-on: ubuntu-latest needs: build if: | - (github.event_name != 'workflow_dispatch') || - (github.event.inputs.browser == 'webkit' || github.event.inputs.browser == 'all') && - (github.event.inputs.test_category == 'non-security' || github.event.inputs.test_category == 'all') + (inputs.browser == 'webkit' || inputs.browser == 'all') && + (inputs.test_category == 'non-security' || inputs.test_category == 'all') timeout-minutes: 20 env: CHARON_EMERGENCY_TOKEN: ${{ secrets.CHARON_EMERGENCY_TOKEN }} CHARON_EMERGENCY_SERVER_ENABLED: "true" CHARON_SECURITY_TESTS_ENABLED: "false" # Cerberus OFF for non-security tests - CHARON_E2E_IMAGE_TAG: charon:e2e-test + CHARON_E2E_IMAGE_TAG: ${{ needs.build.outputs.image_tag }} strategy: fail-fast: false matrix: @@ -971,7 +1124,7 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: - ref: ${{ github.event.workflow_run.head_sha || github.sha }} + ref: ${{ github.sha }} - name: Set up Node.js uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6 @@ -979,12 +1132,29 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' - - name: Download Docker image + - name: Log in to Docker Hub + if: needs.build.outputs.image_source == 'registry' && secrets.DOCKERHUB_TOKEN != '' + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.DOCKERHUB_REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull shared Docker image + if: needs.build.outputs.image_source == 'registry' + run: | + docker pull "${{ needs.build.outputs.image_ref }}" + docker tag "${{ needs.build.outputs.image_ref }}" "${{ needs.build.outputs.image_tag }}" + docker images | grep charon + + - name: Download Docker image artifact + if: needs.build.outputs.image_source == 'build' uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 with: name: docker-image - - name: Load Docker image + - name: Load Docker image artifact + if: needs.build.outputs.image_source == 'build' run: | docker load -i charon-e2e-image.tar docker images | grep charon diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index ed8b009a..652aa090 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -1,12 +1,16 @@ name: Quality Checks on: - workflow_run: - workflows: ["Docker Build, Publish & Test"] - types: [completed] + workflow_dispatch: + inputs: + run_frontend: + description: 'Run frontend checks' + required: false + default: true + type: boolean concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.workflow_run.head_branch || github.head_ref || github.ref_name }} + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.run_id }} cancel-in-progress: true permissions: @@ -22,11 +26,10 @@ jobs: backend-quality: name: Backend (Go) runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: - ref: ${{ github.event.workflow_run.head_sha || github.sha }} + ref: ${{ github.sha }} - name: Set up Go uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 @@ -126,12 +129,12 @@ jobs: frontend-quality: name: Frontend (React) runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} + if: ${{ inputs.run_frontend != false }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 - ref: ${{ github.event.workflow_run.head_sha || github.sha }} + ref: ${{ github.sha }} - name: Repo health check run: | @@ -147,8 +150,8 @@ jobs: - name: Check if frontend was modified in PR id: check-frontend run: | - EVENT_NAME="${{ github.event.workflow_run.event || github.event_name }}" - BASE_REF="${{ github.event.workflow_run.pull_requests[0].base.ref || github.event.pull_request.base.ref }}" + EVENT_NAME="${{ github.event_name }}" + BASE_REF="${{ github.event.pull_request.base.ref }}" if [ "$EVENT_NAME" = "push" ]; then echo "frontend_changed=true" >> $GITHUB_OUTPUT @@ -188,13 +191,13 @@ jobs: - name: Install dependencies working-directory: frontend - if: ${{ github.event.workflow_run.event == 'push' || steps.check-frontend.outputs.frontend_changed == 'true' }} + if: ${{ inputs.run_frontend != false && (github.event_name == 'workflow_dispatch' || steps.check-frontend.outputs.frontend_changed == 'true') }} run: npm ci - name: Run frontend tests and coverage id: frontend-tests working-directory: ${{ github.workspace }} - if: ${{ github.event.workflow_run.event == 'push' || steps.check-frontend.outputs.frontend_changed == 'true' }} + if: ${{ inputs.run_frontend != false && (github.event_name == 'workflow_dispatch' || steps.check-frontend.outputs.frontend_changed == 'true') }} run: | bash scripts/frontend-test-coverage.sh 2>&1 | tee frontend/test-output.txt exit ${PIPESTATUS[0]} diff --git a/.github/workflows/rate-limit-integration.yml b/.github/workflows/rate-limit-integration.yml index c74c3e32..8dd24650 100644 --- a/.github/workflows/rate-limit-integration.yml +++ b/.github/workflows/rate-limit-integration.yml @@ -3,11 +3,6 @@ name: Rate Limit integration # Phase 2-3: Build Once, Test Many - Use registry image instead of building # This workflow now waits for docker-build.yml to complete and pulls the built image on: - workflow_run: - workflows: ["Docker Build, Publish & Test"] - types: [completed] - branches: [main, development, 'feature/**', 'hotfix/**'] - # Allow manual trigger for debugging workflow_dispatch: inputs: image_tag: @@ -27,7 +22,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 # Only run if docker-build.yml succeeded, or if manually triggered - if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') }} + if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && (github.event.workflow_run.status != 'completed' || github.event.workflow_run.conclusion == 'success')) }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -57,9 +52,10 @@ jobs: # Extract 7-character short SHA SHORT_SHA=$(echo "$SHA" | cut -c1-7) - if [[ "$EVENT" == "pull_request" ]]; then - # Use native pull_requests array (no API calls needed) - PR_NUM=$(echo '${{ toJson(github.event.workflow_run.pull_requests) }}' | jq -r '.[0].number') + # Use native pull_requests array (no API calls needed) + PR_NUM=$(echo '${{ toJson(github.event.workflow_run.pull_requests) }}' | jq -r '.[0].number // empty') + + if [[ "$EVENT" == "pull_request" || -n "$PR_NUM" ]]; then # Fallback for direct PR trigger if [[ -z "$PR_NUM" || "$PR_NUM" == "null" ]]; then diff --git a/.github/workflows/repo-health.yml b/.github/workflows/repo-health.yml index 84401601..a41db062 100644 --- a/.github/workflows/repo-health.yml +++ b/.github/workflows/repo-health.yml @@ -3,8 +3,6 @@ name: Repo Health Check on: schedule: - cron: '0 0 * * *' - pull_request: - types: [opened, synchronize, reopened] workflow_dispatch: {} concurrency: diff --git a/.github/workflows/security-pr.yml b/.github/workflows/security-pr.yml index d3acad45..f1f27539 100644 --- a/.github/workflows/security-pr.yml +++ b/.github/workflows/security-pr.yml @@ -4,12 +4,6 @@ name: Security Scan (PR) on: - workflow_run: - workflows: ["Docker Build, Publish & Test"] - types: - - completed - branches: [main, development, 'feature/**', 'hotfix/**'] - workflow_dispatch: inputs: pr_number: @@ -29,8 +23,8 @@ jobs: # Run for: manual dispatch, PR builds, or any push builds from docker-build if: >- github.event_name == 'workflow_dispatch' || - ((github.event.workflow_run.event == 'pull_request' || github.event.workflow_run.event == 'push') && - github.event.workflow_run.conclusion == 'success') + ((github.event.workflow_run.event == 'push' || github.event.workflow_run.pull_requests[0].number != null) && + (github.event.workflow_run.status != 'completed' || github.event.workflow_run.conclusion == 'success')) permissions: contents: read @@ -82,7 +76,7 @@ jobs: fi # Check if this is a push event (not a PR) - if [[ "${{ github.event.workflow_run.event }}" == "push" || "${{ github.event_name }}" == "push" ]]; then + if [[ "${{ github.event_name }}" == "push" || "${{ github.event.workflow_run.event }}" == "push" || -z "${PR_NUMBER}" ]]; then HEAD_BRANCH="${{ github.event.workflow_run.head_branch || github.ref_name }}" echo "is_push=true" >> "$GITHUB_OUTPUT" echo "✅ Detected push build from branch: ${HEAD_BRANCH}" diff --git a/.github/workflows/supply-chain-pr.yml b/.github/workflows/supply-chain-pr.yml index 565c290d..e5b4d66e 100644 --- a/.github/workflows/supply-chain-pr.yml +++ b/.github/workflows/supply-chain-pr.yml @@ -3,12 +3,6 @@ name: Supply Chain Verification (PR) on: - workflow_run: - workflows: ["Docker Build, Publish & Test"] - types: - - completed - branches: [main, development, 'feature/**', 'hotfix/**'] - workflow_dispatch: inputs: pr_number: @@ -35,8 +29,8 @@ jobs: if: > github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && - (github.event.workflow_run.event == 'pull_request' || github.event.workflow_run.event == 'push') && - github.event.workflow_run.conclusion == 'success') + (github.event.workflow_run.event == 'push' || github.event.workflow_run.pull_requests[0].number != null) && + (github.event.workflow_run.status != 'completed' || github.event.workflow_run.conclusion == 'success')) steps: - name: Checkout repository @@ -95,7 +89,7 @@ jobs: fi # Check if this is a push event (not a PR) - if [[ "${WORKFLOW_RUN_EVENT}" == "push" || "${EVENT_NAME}" == "push" ]]; then + if [[ "${WORKFLOW_RUN_EVENT}" == "push" || "${EVENT_NAME}" == "push" || -z "${PR_NUMBER}" ]]; then echo "is_push=true" >> "$GITHUB_OUTPUT" echo "✅ Detected push build from branch: ${HEAD_BRANCH}" else diff --git a/.github/workflows/supply-chain-verify.yml b/.github/workflows/supply-chain-verify.yml index 9b12d6e4..024f4f28 100644 --- a/.github/workflows/supply-chain-verify.yml +++ b/.github/workflows/supply-chain-verify.yml @@ -1,26 +1,9 @@ name: Supply Chain Verification on: - release: - types: [published] - - # Triggered after docker-build workflow completes - # Note: workflow_run can only chain 3 levels deep; we're at level 2 (safe) - # - # IMPORTANT: No branches filter here by design - # GitHub Actions limitation: branches filter in workflow_run only matches the default branch. - # Without a filter, this workflow triggers for ALL branches where docker-build completes, - # providing proper supply chain verification coverage for feature branches and PRs. - # Security: The workflow file must exist on the branch to execute, preventing untrusted code. - workflow_run: - workflows: ["Docker Build, Publish & Test"] - types: [completed] - - schedule: - # Run weekly on Mondays at 00:00 UTC - - cron: '0 0 * * 1' - workflow_dispatch: + schedule: + - cron: '0 0 * * 1' # Mondays 00:00 UTC permissions: contents: read @@ -39,8 +22,8 @@ jobs: if: | (github.event_name != 'schedule' || github.ref == 'refs/heads/main') && (github.event_name != 'workflow_run' || - (github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.event != 'pull_request')) + (github.event.workflow_run.event != 'pull_request' && + (github.event.workflow_run.status != 'completed' || github.event.workflow_run.conclusion == 'success'))) steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.github/workflows/waf-integration.yml b/.github/workflows/waf-integration.yml index 9ce53fd6..11a9ba62 100644 --- a/.github/workflows/waf-integration.yml +++ b/.github/workflows/waf-integration.yml @@ -3,11 +3,6 @@ name: WAF integration # Phase 2-3: Build Once, Test Many - Use registry image instead of building # This workflow now waits for docker-build.yml to complete and pulls the built image on: - workflow_run: - workflows: ["Docker Build, Publish & Test"] - types: [completed] - branches: [main, development, 'feature/**', 'hotfix/**'] - # Allow manual trigger for debugging workflow_dispatch: inputs: image_tag: diff --git a/docs/plans/current_spec.md b/docs/plans/current_spec.md index a1322a31..89392c16 100644 --- a/docs/plans/current_spec.md +++ b/docs/plans/current_spec.md @@ -1,282 +1,302 @@ --- -title: "Migration to Alpine (Issue #631)" +title: "CI Pipeline Consolidation" status: "draft" -scope: "docker/alpine-migration" -notes: This plan has yet to be finished. You may add to but, ** DO NOT ** overwrite until completion of PR #666. +scope: "ci/pipeline" +notes: This plan replaces the current CI workflow chain with a single pipeline that supports PR triggers while keeping maintenance workflows scheduled. --- ## 1. Introduction -This plan defines the migration of the Charon Docker image base from -Debian Trixie Slim to Alpine Linux to address inherited glibc CVEs and -reduce image size (Issue #631). The plan consolidates the prior Alpine -migration research and translates it into a minimal-change, test-first -implementation path aligned with current CI and container workflows. +This plan consolidates the existing CI workflows into one pipeline +workflow that can trigger on pull requests across branches (in addition +to manual dispatch). The pipeline will run in a strict order defined by +the user: +lint, build, parallel integration prerequisites, E2E, parallel +coverage, then security scans. All stages will consume the same built +Docker image to ensure consistent test results. + +Maintenance workflows remain scheduled (nightly/weekly/Renovate/repo +health) and are explicitly out of scope for trigger changes. + +Out of scope: Alpine migration. Any base-image migration work must be +captured in a separate plan/spec. Objectives: -- Replace Debian-based runtime with Alpine 3.23.x while maintaining - feature parity. -- Eliminate Debian glibc HIGH CVEs in the runtime image. -- Keep build stages compatible with multi-arch Buildx and existing - supply chain checks. -- Validate DNS resolution, SQLite (CGO) behavior, and security suite - functionality under musl. -- Review and update .gitignore, codecov.yml, .dockerignore, and - Dockerfile as needed. +- Enable the pipeline to run on pull requests across branches in + addition to manual dispatch. +- Create one pipeline workflow that sequences jobs in the requested + order with explicit dependencies. +- Ensure all integration, E2E, coverage, and security checks use the + same image digest produced by the pipeline build job. +- Push the pipeline image to Docker Hub and GHCR, but use Docker Hub as + the test image source. +- Keep the E2E image tag unchanged from the current convention. +- Align the pipeline with the current Definition of Done (DoD) by + mapping required checks into pipeline stages. +- Preserve scheduled maintenance workflows and do not convert them to + manual-only triggers. ## 2. Research Findings -### 2.1 Existing Plans and Security Context +### 2.1 Current Workflow Topology -- Alpine migration specification already exists and is comprehensive: - docs/plans/alpine_migration_spec.md. -- Debian CVE acceptance is temporary and explicitly tied to Alpine - migration: - docs/security/VULNERABILITY_ACCEPTANCE.md. -- Past Alpine-related issues and trade-offs are documented, including - musl DNS differences: - docs/analysis/crowdsec_integration_failure_analysis.md. +The CI chain is currently split across multiple workflows linked by +workflow_run triggers. The core files in scope are: -### 2.2 Current Docker and CI Touchpoints +- .github/workflows/docker-build.yml +- .github/workflows/docker-lint.yml +- .github/workflows/e2e-tests-split.yml +- .github/workflows/quality-checks.yml +- .github/workflows/codecov-upload.yml +- .github/workflows/codeql.yml +- .github/workflows/security-pr.yml +- .github/workflows/supply-chain-pr.yml +- .github/workflows/cerberus-integration.yml +- .github/workflows/crowdsec-integration.yml +- .github/workflows/waf-integration.yml +- .github/workflows/rate-limit-integration.yml +- .github/workflows/benchmark.yml +- .github/workflows/supply-chain-verify.yml -Primary files that must be considered for the migration: +Several maintenance workflows also exist (nightly builds, weekly +security rebuilds, repository health, Renovate automation). They are +not part of the requested pipeline order and will remain scheduled +with their existing triggers. -- Dockerfile (multi-stage build with Debian runtime base). -- .docker/docker-entrypoint.sh (uses user/group management and tools - that differ on Alpine). -- .docker/compose/docker-compose.yml (image tag references). -- .github/workflows/docker-build.yml (base image digest resolution and - build args). -- .github/workflows/security-pr.yml and supply-chain-pr.yml (build and - scan behaviors depend on the container layout). -- tools/dockerfile_check.sh (package manager validation). +### 2.2 Current Image Tagging and Digest Sources -### 2.3 Compatibility Summary (musl vs glibc) +- docker-build.yml outputs a build digest from the buildx iidfile and + pushes images to GHCR and Docker Hub. +- Tags currently include: + - pr-{number}-{short-sha} for PRs + - {sanitized-branch}-{short-sha} for feature branches + - latest/dev/nightly for main/development/nightly builds + - sha-{short-sha} for non-PR builds + - nightly branch tag (per user request) for nightly branch builds -Based on alpine_migration_spec.md and current runtime behavior: +### 2.3 Definition of Done (DoD) Alignment -- Go services and Caddy/CrowdSec are Go binaries and compatible with - musl. -- SQLite is CGO-backed; ensure CGO remains enabled and libsqlite3 is - available under musl, then validate runtime CRUD behavior. -- DNS resolution differences are the primary operational risk; - mitigation is available via $GODEBUG=netdns=go. -- Entrypoint uses Debian-specific user/group tools; Alpine requires - adduser/addgroup or the shadow package. +The DoD requires E2E tests to run first, then security scans, pre-commit +checks, static analysis, coverage gates, type checks, and build +verification. The requested pipeline order differs by placing E2E after +integration prerequisites and before coverage and security scans. + +Decision: the pipeline order is authoritative for CI. The DoD +order remains guidance for local workflows, but CI ordering will follow +the requested pipeline sequence and map required checks into stages. ## 3. Technical Specifications -### 3.1 Target Base Image +### 3.1 Workflow Trigger Strategy -- Runtime base: alpine:3.23.x pinned by digest (Renovate-managed). -- Build stages: switch to alpine-based golang/node images where required - to use apk/xx-apk consistently. -- Build-stage images should be digest-pinned when feasible. If a digest - pin is not practical (e.g., multi-arch tag compatibility), document - the reason and keep the tag Renovate-managed. +The new pipeline workflow will trigger on pull_request across branches +and workflow_dispatch. Existing CI workflows listed in Section 2.1 will +be converted to workflow_dispatch only (no PR triggers). Existing +workflow_run triggers will be removed. Scheduled maintenance workflows +will keep their schedules intact. -### 3.2 Dockerfile Changes (Stage-by-Stage) +### 3.2 New Pipeline Workflow -Stages and expected changes (paths and stage names are current): +Create a new workflow file that runs the entire pipeline in one run: -1) gosu-builder (Dockerfile): - - Replace apt-get with apk. - - Replace xx-apt with xx-apk. - - Expected packages: git, clang, lld, gcc, musl-dev. +- File: .github/workflows/ci-pipeline.yml +- Trigger: workflow_dispatch and pull_request across branches +- Inputs: + - image_tag_override (optional) + - run_coverage (boolean) + - run_security_scans (boolean) + - run_integration (boolean) + - run_e2e (boolean) -2) frontend-builder (Dockerfile): - - Use node:24.x-alpine. - - Keep npm_config_rollup_skip_nodejs_native settings for cross-arch - builds. +### 3.3 Job Order and Dependencies -3) backend-builder (Dockerfile): - - Replace apt-get with apk. - - Replace xx-apt with xx-apk. - - Expected packages: clang, lld, gcc, musl-dev, sqlite-dev. +The pipeline job graph will enforce the requested order. -4) caddy-builder (Dockerfile): - - Replace apt-get with apk. - - Expected packages: git. +Job dependency table: -5) crowdsec-builder (Dockerfile): - - Replace apt-get with apk. - - Replace xx-apt with xx-apk. - - Expected packages: git, clang, lld, gcc, musl-dev. +| Job | Purpose | Needs | +| --- | --- | --- | +| lint | Dockerfile lint, Go lint, frontend lint, repo health | none | +| build-image | Build and push Docker image, emit digest | lint | +| integration-cerberus | Cerberus integration tests | build-image | +| integration-crowdsec | CrowdSec integration tests | build-image | +| integration-waf | WAF integration tests | build-image | +| integration-ratelimit | Rate limit integration tests | build-image | +| e2e | Playwright E2E split workflow equivalent | integration-* | +| coverage-backend | Go tests with coverage and Codecov upload | e2e | +| coverage-frontend | Frontend tests with coverage and Codecov upload | e2e | +| coverage-e2e | Optional E2E coverage job | e2e | +| security-codeql | CodeQL Go and JS scans | coverage-* | +| security-trivy | Trivy image scan | coverage-* | +| security-supply-chain | SBOM generation and attestation | coverage-* | -6) crowdsec-fallback (Dockerfile): - - Replace debian:trixie-slim with alpine:3.23.x. - - Use apk add curl ca-certificates (tar is provided by busybox). +Integration jobs should run in parallel. Coverage and security jobs +should run in parallel within their stages. -7) final runtime stage (Dockerfile): - - Replace CADDY_IMAGE base from Debian to Alpine. - - Replace apt-get with apk add. - - Runtime packages: bash, ca-certificates, sqlite-libs, sqlite, - tzdata, curl, gettext, libcap, c-ares, binutils, libc-utils - (for getent), busybox-extras or coreutils (for timeout), - libcap-utils (for setcap). - - Add ENV GODEBUG=netdns=go to mitigate musl DNS edge cases. +### 3.4 Shared Image Strategy -### 3.3 Entrypoint Adjustments +All downstream jobs must use the same image digest produced by the +build-image job. The build-image job will output: -File: .docker/docker-entrypoint.sh +- image_digest: from docker/build-push-action or iidfile +- image_ref: docker.io/wikid82/charon@sha256:... +- image_ref_ghcr: ghcr.io/wikid82/charon@sha256:... +- image_tag: pr-{number}-{short-sha} or sha-{short-sha} -Functions and command usage that must be Alpine-safe: +Downstream jobs will pull the image by digest to ensure immutability and +retag it locally as charon:e2e-test for docker compose usage. For test +stages, the image source registry must be Docker Hub even though GHCR is +also pushed. The E2E image tag must remain unchanged from the current +convention. -- is_root(): no change. -- run_as_charon(): no change. -- Docker socket group handling: - - Replace groupadd/usermod with addgroup/adduser if shadow tools are - not installed. - - If using getent, ensure libc-utils is installed or implement a - /etc/group parsing fallback. -- CrowdSec initialization: - - Ensure sed -i usage is compatible with busybox sed. - - Verify timeout is available (busybox provides timeout). +### 3.5 Required File Updates -### 3.4 CI and Workflow Updates +Workflow updates to manual-only triggers: -File: .github/workflows/docker-build.yml +- .github/workflows/docker-build.yml +- .github/workflows/docker-lint.yml +- .github/workflows/e2e-tests-split.yml +- .github/workflows/quality-checks.yml +- .github/workflows/codecov-upload.yml +- .github/workflows/codeql.yml +- .github/workflows/security-pr.yml +- .github/workflows/supply-chain-pr.yml +- .github/workflows/cerberus-integration.yml +- .github/workflows/crowdsec-integration.yml +- .github/workflows/waf-integration.yml +- .github/workflows/rate-limit-integration.yml +- .github/workflows/benchmark.yml +- .github/workflows/supply-chain-verify.yml -- Replace "Resolve Debian base image digest" step to pull and resolve - alpine:3.23.x digest. -- Update CADDY_IMAGE build-arg to use the Alpine digest. -- Ensure buildx cache and tag logic remain unchanged. +Workflow additions (PR + manual triggers): -No changes are expected to security-pr.yml and supply-chain-pr.yml -unless the container layout changes (paths used for binary extraction -and SBOM remain consistent). +- .github/workflows/ci-pipeline.yml -### 3.5 Data Flow and Runtime Behavior +Optional configuration updates if required for image reuse: -```mermaid -flowchart LR - A[Docker Build] --> B[Multi-stage build on Alpine] - B --> C[Runtime: alpine base + charon + caddy + crowdsec] - C --> D[Entrypoint initializes volumes, CrowdSec, Caddy] - D --> E[Charon API + UI] -``` +- .docker/compose/docker-compose.playwright-ci.yml (use image ref or + tag via environment variable) +- scripts/*.sh or .github/skills/scripts/skill-runner.sh, only if + necessary to accept image ref overrides -### 3.6 Requirements (EARS Notation) +### 3.6 Error Handling and Gates -- WHEN the Docker image is built, THE SYSTEM SHALL use Alpine 3.23.x - as the runtime base image. -- WHEN the container starts, THE SYSTEM SHALL create the charon user - and groups using Alpine-compatible tools. -- WHEN DNS resolution is performed, THE SYSTEM SHALL use the Go DNS - resolver to avoid musl NSS limitations. -- WHEN SQLite-backed operations run, THE SYSTEM SHALL read and write - data with CGO enabled and no schema errors under musl. -- IF Alpine package CVEs reappear at HIGH or CRITICAL, THEN THE SYSTEM - SHALL fail the security gate and block release. +- Fail fast in lint and build stages. +- Integration, E2E, coverage, and security stages should fail the + pipeline if any job fails. +- Preserve existing retry behavior for registry pushes and pulls. -## 4. Implementation Plan (Minimal-Request Phases) +### 3.7 Required Checks and Branch Protection + +- Add a pipeline summary job (e.g., pipeline-gate) that depends on all + pipeline jobs and fails if any required job fails. +- Require the pipeline-gate status check in branch protection/rulesets + for main and release branches. +- Pipeline workflows remain required by enforcing that the pipeline is + run against the merge commit or branch HEAD before merge. +- Keep admin bypass disabled for protected branches unless explicitly + approved. + +### 3.7 Requirements (EARS Notation) + +- WHEN a user manually dispatches the pipeline or opens a pull request, + THE SYSTEM SHALL run the lint stage before any build or test jobs. +- WHEN the build stage completes, THE SYSTEM SHALL publish a single + image digest that all later jobs consume. +- WHEN any integration test fails, THE SYSTEM SHALL stop the pipeline + before E2E execution. +- WHEN E2E completes, THE SYSTEM SHALL run coverage jobs in parallel. +- WHEN coverage completes, THE SYSTEM SHALL run security scans in + parallel using the same image digest. +- WHEN the pipeline pushes images, THE SYSTEM SHALL push to Docker Hub + and GHCR but use Docker Hub as the test image source. +- WHEN E2E runs, THE SYSTEM SHALL keep the existing E2E image tag and + preserve the security shard as a separate shard with the current + timeout-safe layout. +- IF any required DoD check fails, THEN THE SYSTEM SHALL fail the + pipeline and report the failing stage. + +## 4. Implementation Plan ### Phase 1: Playwright Tests (Behavior Baseline) -- Rebuild the E2E container when Docker build inputs change, then run - E2E smoke tests before any unit or integration tests to establish the - UI baseline (tests/). Focus on login, proxy host CRUD, security - toggles. -- Record baseline timings for key flows to compare after migration. +- Validate the existing Playwright suites used by e2e-tests-split.yml + can run under the new pipeline using the shared image digest. +- Confirm the E2E stage still honors security and non-security shards + and that Cerberus toggle logic is preserved. -### Phase 2: Backend Implementation (Runtime and Container) +### Phase 2: Backend and CI Workflow Refactor -- Update Dockerfile stages to Alpine equivalents (see Section 3.2). -- Update .docker/docker-entrypoint.sh for Alpine user/group commands and - tool availability (see Section 3.3). -- Add ENV GODEBUG=netdns=go to Dockerfile runtime stage. -- Update tools/dockerfile_check.sh to validate apk and xx-apk usage in - Alpine-based stages, replacing any Debian-specific checks. -- Run tools/dockerfile_check.sh and capture results for apk/xx-apk - verification. -- Validate crowdsec and caddy binaries remain in the same paths: - /usr/bin/caddy, /usr/local/bin/crowdsec, /usr/local/bin/cscli. +- Add the new pipeline workflow file. +- Modify existing CI workflows in Section 3.5 to use workflow_dispatch + only (no pull_request triggers). +- Move the docker-build logic into the pipeline build-image job and + export digest and tag outputs. +- Update integration job steps to consume the digest and retag locally + as needed for existing scripts. -### Phase 3: Frontend Implementation +### Phase 3: Frontend and E2E Workflow Refactor -- No application-level frontend changes expected. -- Ensure frontend build stage uses node:24.x-alpine in Dockerfile. +- Update the E2E steps to pull the Docker Hub digest and retag to + charon:e2e-test before docker compose starts. +- Ensure environment variables or compose overrides reference the + shared image and keep the E2E tag unchanged. +- Preserve E2E sharding so the security shard remains separate and the + shard layout avoids timeouts. -### Phase 4: Integration and Testing +### Phase 4: Coverage and Security Stage Consolidation -- Rebuild E2E container and run Playwright suite (Docker mode). -- Run targeted integration tests: - - CrowdSec integration workflows. - - WAF and rate-limit workflows. -- Validate DNS challenges for at least one provider (Cloudflare). -- Validate SQLite CGO operations using health endpoints and basic CRUD. -- Validate multi-arch Buildx output and supply-chain workflows for the - Docker image: - - .github/workflows/docker-build.yml - - .github/workflows/security-pr.yml - - .github/workflows/supply-chain-pr.yml -- Run Trivy image scan and verify no HIGH/CRITICAL findings. +- Replace codecov-upload.yml and codeql.yml with pipeline jobs that run + after E2E completion. +- Ensure Codecov uploads and CodeQL scans run with the same code + checkout and digest metadata for traceability. -### Phase 5: Documentation and Deployment +### Phase 5: Documentation and DoD Alignment -- Update ARCHITECTURE.md to reflect Alpine base image. -- Update docs/security/VULNERABILITY_ACCEPTANCE.md to close the Debian - CVE acceptance and note Alpine status. -- Update any Docker guidance in README or .docker/README.md if it - references Debian. +- Update docs/plans/current_spec.md with the final pipeline plan. +- Document the DoD ordering impact and confirm whether the DoD should + be updated to match the new pipeline order or the pipeline should + adapt to the DoD ordering. -## 5. Config Hygiene Review (Requested Files) +### Phase 6: Branch Protection Updates -### 5.1 .gitignore +- Update branch protection/rulesets to require the pipeline-gate check. +- Document the manual pipeline run requirement for PR validation. -- No new ignore patterns required for Alpine migration. -- Verify no new build artifacts are introduced (apk cache is in-image - only). +## 5. Acceptance Criteria -### 5.2 .dockerignore +- The pipeline workflow triggers via pull_request across branches and + workflow_dispatch. +- All CI workflows listed in Section 3.5 trigger via + workflow_dispatch only and no longer use workflow_run or + pull_request. +- Maintenance workflows (nightly/weekly/Renovate/repo health) retain + their scheduled triggers and are not changed to PR/manual-only. +- The new pipeline workflow runs lint, build, integration, E2E, + coverage, and security stages in the requested order. +- Integration, E2E, coverage, and security jobs consume the same image + digest produced by the build stage. +- The pipeline exposes image_digest and image_ref outputs for audit + and debugging. +- All DoD-required checks are represented in the pipeline and fail the + run when they do not pass. +- The pipeline pushes images to Docker Hub and GHCR, and test stages + pull from Docker Hub. +- E2E sharding keeps the security shard separate and retains the + timeout-safe shard layout. +- The nightly branch tag remains part of the image tagging scheme. +## 6. Risks and Mitigations -- No changes required; keep excluding docs and CI artifacts to minimize - build context size. + - Risk: PR-triggered pipeline increases CI load and could cause noisy + failures on draft or experimental branches. + - Mitigation: keep legacy workflows manual-only, enforce the + pipeline-gate required check, and allow maintainers to re-run the + pipeline as needed. -### 5.3 codecov.yml +## 7. Confidence Score -- No changes required; migration does not add new code paths that should - be excluded from coverage. +Confidence: 86 percent -### 5.4 Dockerfile (Required) - -- Update base images and package manager usage per Section 3.2. -- Add GODEBUG=netdns=go in runtime stage. -- Replace useradd/groupadd with adduser/addgroup or add shadow tools if - preferred. - -## 6. Acceptance Criteria - -- The Docker image builds on Alpine with no build-stage failures. -- Runtime container starts with non-root user and no permission errors. -- All Playwright E2E tests pass against the Alpine-based container. -- Integration tests (CrowdSec, WAF, Rate Limit) pass without regressions. -- Trivy image scan reports zero HIGH/CRITICAL CVEs in the runtime image. -- tools/dockerfile_check.sh passes with apk and xx-apk checks for all - Alpine-based stages. -- Multi-arch Buildx validation succeeds and supply-chain workflows - (docker-build.yml, security-pr.yml, supply-chain-pr.yml) complete with - no regressions. -- ARCHITECTURE.md and security acceptance docs reflect Alpine as the - runtime base. - -## 7. Risks and Mitigations - -- Risk: musl DNS resolver differences cause ACME or webhook failures. - - Mitigation: set GODEBUG=netdns=go and run DNS provider tests. - -- Risk: Alpine user/group tooling mismatch breaks Docker socket handling. - - Mitigation: adjust entrypoint to use adduser/addgroup or install - shadow tools and libc-utils for getent. - -- Risk: SQLite CGO compatibility issues. - - Mitigation: run database integrity checks and CRUD tests. - -## 8. Confidence Score - -Confidence: 84 percent - -Rationale: Alpine migration has a detailed existing spec and low code -surface change, but runtime differences (musl DNS, user/group tooling) -require careful validation. +Rationale: Manual pipeline consolidation is well scoped, but requires +careful coordination with branch protection and required checks.