diff --git a/.docker/docker-entrypoint.sh b/.docker/docker-entrypoint.sh index 82429cac..ac6571b2 100755 --- a/.docker/docker-entrypoint.sh +++ b/.docker/docker-entrypoint.sh @@ -284,19 +284,34 @@ done # Note: When running as root, we use gosu; otherwise we run directly. echo "Starting Charon management application..." DEBUG_FLAG=${CHARON_DEBUG:-$CPMP_DEBUG} -DEBUG_PORT=${CHARON_DEBUG_PORT:-$CPMP_DEBUG_PORT} +DEBUG_PORT=${CHARON_DEBUG_PORT:-${CPMP_DEBUG_PORT:-2345}} + +# Determine binary path +bin_path=/app/charon +if [ ! -f "$bin_path" ]; then + bin_path=/app/cpmp +fi + if [ "$DEBUG_FLAG" = "1" ]; then - echo "Running Charon under Delve (port $DEBUG_PORT)" - bin_path=/app/charon - if [ ! -f "$bin_path" ]; then - bin_path=/app/cpmp + # Check if binary has debug symbols (required for Delve) + # objdump -h lists section headers; .debug_info is present if DWARF symbols exist + if command -v objdump >/dev/null 2>&1; then + if ! objdump -h "$bin_path" 2>/dev/null | grep -q '\.debug_info'; then + echo "⚠️ WARNING: Binary lacks debug symbols (DWARF info stripped)." + echo " Delve debugging will NOT work with this binary." + echo " To fix, rebuild with: docker build --build-arg BUILD_DEBUG=1 ..." + echo " Falling back to normal execution (without debugger)." + run_as_charon "$bin_path" & + else + echo "✓ Debug symbols detected. Running Charon under Delve (port $DEBUG_PORT)" + run_as_charon /usr/local/bin/dlv exec "$bin_path" --headless --listen=":$DEBUG_PORT" --api-version=2 --accept-multiclient --continue --log -- & + fi + else + # objdump not available, try to run Delve anyway with a warning + echo "Note: Cannot verify debug symbols (objdump not found). Attempting Delve..." + run_as_charon /usr/local/bin/dlv exec "$bin_path" --headless --listen=":$DEBUG_PORT" --api-version=2 --accept-multiclient --continue --log -- & fi - run_as_charon /usr/local/bin/dlv exec "$bin_path" --headless --listen=":$DEBUG_PORT" --api-version=2 --accept-multiclient --continue --log -- & else - bin_path=/app/charon - if [ ! -f "$bin_path" ]; then - bin_path=/app/cpmp - fi run_as_charon "$bin_path" & fi APP_PID=$! diff --git a/.github/workflows/cerberus-integration.yml b/.github/workflows/cerberus-integration.yml new file mode 100644 index 00000000..899e839f --- /dev/null +++ b/.github/workflows/cerberus-integration.yml @@ -0,0 +1,119 @@ +name: Cerberus Integration Tests + +on: + push: + branches: [ main, development, 'feature/**' ] + paths: + - 'backend/internal/caddy/**' + - 'backend/internal/security/**' + - 'backend/internal/handlers/security*.go' + - 'backend/internal/models/security*.go' + - 'scripts/cerberus_integration.sh' + - 'Dockerfile' + - '.github/workflows/cerberus-integration.yml' + pull_request: + branches: [ main, development ] + paths: + - 'backend/internal/caddy/**' + - 'backend/internal/security/**' + - 'backend/internal/handlers/security*.go' + - 'backend/internal/models/security*.go' + - 'scripts/cerberus_integration.sh' + - 'Dockerfile' + - '.github/workflows/cerberus-integration.yml' + # Allow manual trigger + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + cerberus-integration: + name: Cerberus Security Stack Integration + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + + - name: Build Docker image + run: | + docker build \ + --no-cache \ + --build-arg VCS_REF=${{ github.sha }} \ + -t charon:local . + + - name: Run Cerberus integration tests + id: cerberus-test + run: | + chmod +x scripts/cerberus_integration.sh + scripts/cerberus_integration.sh 2>&1 | tee cerberus-test-output.txt + exit ${PIPESTATUS[0]} + + - name: Dump Debug Info on Failure + if: failure() + run: | + echo "## 🔍 Debug Information" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### Container Status" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + docker ps -a --filter "name=charon" --filter "name=cerberus" --filter "name=backend" >> $GITHUB_STEP_SUMMARY 2>&1 || true + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### Security Status API" >> $GITHUB_STEP_SUMMARY + echo '```json' >> $GITHUB_STEP_SUMMARY + curl -s http://localhost:8480/api/v1/security/status 2>/dev/null | head -100 >> $GITHUB_STEP_SUMMARY || echo "Could not retrieve security status" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### Caddy Admin Config" >> $GITHUB_STEP_SUMMARY + echo '```json' >> $GITHUB_STEP_SUMMARY + curl -s http://localhost:2319/config 2>/dev/null | head -200 >> $GITHUB_STEP_SUMMARY || echo "Could not retrieve Caddy config" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### Charon Container Logs (last 100 lines)" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + docker logs charon-cerberus-test 2>&1 | tail -100 >> $GITHUB_STEP_SUMMARY || echo "No container logs available" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + - name: Cerberus Integration Summary + if: always() + run: | + echo "## 🔱 Cerberus Integration Test Results" >> $GITHUB_STEP_SUMMARY + if [ "${{ steps.cerberus-test.outcome }}" == "success" ]; then + echo "✅ **All Cerberus tests passed**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Test Results:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep -E "✓|PASS|TC-[0-9]|=== ALL" cerberus-test-output.txt || echo "See logs for details" + grep -E "✓|PASS|TC-[0-9]|=== ALL" cerberus-test-output.txt >> $GITHUB_STEP_SUMMARY || echo "See logs for details" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Features Tested:" >> $GITHUB_STEP_SUMMARY + echo "- WAF (Coraza) payload inspection" >> $GITHUB_STEP_SUMMARY + echo "- Rate limiting enforcement" >> $GITHUB_STEP_SUMMARY + echo "- Security handler ordering" >> $GITHUB_STEP_SUMMARY + echo "- Legitimate traffic flow" >> $GITHUB_STEP_SUMMARY + else + echo "❌ **Cerberus tests failed**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Failure Details:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep -E "✗|FAIL|Error|failed" cerberus-test-output.txt | head -30 >> $GITHUB_STEP_SUMMARY || echo "See logs for details" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi + + - name: Cleanup + if: always() + run: | + docker rm -f charon-cerberus-test || true + docker rm -f cerberus-backend || true + docker volume rm charon_cerberus_test_data caddy_cerberus_test_data caddy_cerberus_test_config 2>/dev/null || true + docker network rm containers_default || true diff --git a/.github/workflows/crowdsec-integration.yml b/.github/workflows/crowdsec-integration.yml new file mode 100644 index 00000000..dbed06fc --- /dev/null +++ b/.github/workflows/crowdsec-integration.yml @@ -0,0 +1,122 @@ +name: CrowdSec Integration Tests + +on: + push: + branches: [ main, development, 'feature/**' ] + paths: + - 'backend/internal/crowdsec/**' + - 'backend/internal/models/crowdsec*.go' + - 'configs/crowdsec/**' + - 'scripts/crowdsec_integration.sh' + - 'scripts/crowdsec_decision_integration.sh' + - 'scripts/crowdsec_startup_test.sh' + - '.github/skills/integration-test-crowdsec*/**' + - 'Dockerfile' + - '.github/workflows/crowdsec-integration.yml' + pull_request: + branches: [ main, development ] + paths: + - 'backend/internal/crowdsec/**' + - 'backend/internal/models/crowdsec*.go' + - 'configs/crowdsec/**' + - 'scripts/crowdsec_integration.sh' + - 'scripts/crowdsec_decision_integration.sh' + - 'scripts/crowdsec_startup_test.sh' + - '.github/skills/integration-test-crowdsec*/**' + - 'Dockerfile' + - '.github/workflows/crowdsec-integration.yml' + # Allow manual trigger + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + crowdsec-integration: + name: CrowdSec Bouncer Integration + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + + - name: Build Docker image + run: | + docker build \ + --no-cache \ + --build-arg VCS_REF=${{ github.sha }} \ + -t charon:local . + + - name: Run CrowdSec integration tests + id: crowdsec-test + run: | + chmod +x .github/skills/scripts/skill-runner.sh + .github/skills/scripts/skill-runner.sh integration-test-crowdsec 2>&1 | tee crowdsec-test-output.txt + exit ${PIPESTATUS[0]} + + - name: Dump Debug Info on Failure + if: failure() + run: | + echo "## 🔍 Debug Information" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### Container Status" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + docker ps -a --filter "name=charon" --filter "name=crowdsec" >> $GITHUB_STEP_SUMMARY 2>&1 || true + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### CrowdSec LAPI Status" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + docker exec crowdsec cscli bouncers list 2>/dev/null >> $GITHUB_STEP_SUMMARY || echo "Could not retrieve bouncer list" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### CrowdSec Decisions" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + docker exec crowdsec cscli decisions list 2>/dev/null >> $GITHUB_STEP_SUMMARY || echo "Could not retrieve decisions" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### Charon Container Logs (last 100 lines)" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + docker logs charon-debug 2>&1 | tail -100 >> $GITHUB_STEP_SUMMARY || echo "No container logs available" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### CrowdSec Container Logs (last 50 lines)" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + docker logs crowdsec 2>&1 | tail -50 >> $GITHUB_STEP_SUMMARY || echo "No CrowdSec logs available" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + - name: CrowdSec Integration Summary + if: always() + run: | + echo "## 🛡️ CrowdSec Integration Test Results" >> $GITHUB_STEP_SUMMARY + if [ "${{ steps.crowdsec-test.outcome }}" == "success" ]; then + echo "✅ **All CrowdSec tests passed**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Test Results:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep -E "^✓|^===|^Pull|^Apply" crowdsec-test-output.txt || echo "See logs for details" + grep -E "^✓|^===|^Pull|^Apply" crowdsec-test-output.txt >> $GITHUB_STEP_SUMMARY || echo "See logs for details" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + else + echo "❌ **CrowdSec tests failed**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Failure Details:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep -E "^✗|Unexpected|Error|failed|FAIL" crowdsec-test-output.txt | head -20 >> $GITHUB_STEP_SUMMARY || echo "See logs for details" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi + + - name: Cleanup + if: always() + run: | + docker rm -f charon-debug || true + docker rm -f crowdsec || true + docker network rm containers_default || true diff --git a/.github/workflows/rate-limit-integration.yml b/.github/workflows/rate-limit-integration.yml new file mode 100644 index 00000000..8625c1ad --- /dev/null +++ b/.github/workflows/rate-limit-integration.yml @@ -0,0 +1,125 @@ +name: Rate Limit Integration Tests + +on: + push: + branches: [ main, development, 'feature/**' ] + paths: + - 'backend/internal/caddy/**' + - 'backend/internal/security/**' + - 'backend/internal/handlers/security*.go' + - 'backend/internal/models/security*.go' + - 'scripts/rate_limit_integration.sh' + - 'Dockerfile' + - '.github/workflows/rate-limit-integration.yml' + pull_request: + branches: [ main, development ] + paths: + - 'backend/internal/caddy/**' + - 'backend/internal/security/**' + - 'backend/internal/handlers/security*.go' + - 'backend/internal/models/security*.go' + - 'scripts/rate_limit_integration.sh' + - 'Dockerfile' + - '.github/workflows/rate-limit-integration.yml' + # Allow manual trigger + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + rate-limit-integration: + name: Rate Limiting Integration + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + + - name: Build Docker image + run: | + docker build \ + --no-cache \ + --build-arg VCS_REF=${{ github.sha }} \ + -t charon:local . + + - name: Run rate limit integration tests + id: ratelimit-test + run: | + chmod +x scripts/rate_limit_integration.sh + scripts/rate_limit_integration.sh 2>&1 | tee ratelimit-test-output.txt + exit ${PIPESTATUS[0]} + + - name: Dump Debug Info on Failure + if: failure() + run: | + echo "## 🔍 Debug Information" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### Container Status" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + docker ps -a --filter "name=charon" --filter "name=ratelimit" --filter "name=backend" >> $GITHUB_STEP_SUMMARY 2>&1 || true + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### Security Config API" >> $GITHUB_STEP_SUMMARY + echo '```json' >> $GITHUB_STEP_SUMMARY + curl -s http://localhost:8280/api/v1/security/config 2>/dev/null | head -100 >> $GITHUB_STEP_SUMMARY || echo "Could not retrieve security config" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### Security Status API" >> $GITHUB_STEP_SUMMARY + echo '```json' >> $GITHUB_STEP_SUMMARY + curl -s http://localhost:8280/api/v1/security/status 2>/dev/null | head -100 >> $GITHUB_STEP_SUMMARY || echo "Could not retrieve security status" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### Caddy Admin Config (rate_limit handlers)" >> $GITHUB_STEP_SUMMARY + echo '```json' >> $GITHUB_STEP_SUMMARY + curl -s http://localhost:2119/config 2>/dev/null | grep -A 20 '"handler":"rate_limit"' | head -30 >> $GITHUB_STEP_SUMMARY || echo "Could not retrieve Caddy config" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "### Charon Container Logs (last 100 lines)" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + docker logs charon-ratelimit-test 2>&1 | tail -100 >> $GITHUB_STEP_SUMMARY || echo "No container logs available" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + - name: Rate Limit Integration Summary + if: always() + run: | + echo "## ⏱️ Rate Limit Integration Test Results" >> $GITHUB_STEP_SUMMARY + if [ "${{ steps.ratelimit-test.outcome }}" == "success" ]; then + echo "✅ **All rate limit tests passed**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Test Results:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep -E "✓|=== ALL|HTTP 429|HTTP 200" ratelimit-test-output.txt | head -30 || echo "See logs for details" + grep -E "✓|=== ALL|HTTP 429|HTTP 200" ratelimit-test-output.txt | head -30 >> $GITHUB_STEP_SUMMARY || echo "See logs for details" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Verified Behaviors:" >> $GITHUB_STEP_SUMMARY + echo "- Requests within limit return HTTP 200" >> $GITHUB_STEP_SUMMARY + echo "- Requests exceeding limit return HTTP 429" >> $GITHUB_STEP_SUMMARY + echo "- Retry-After header present on blocked responses" >> $GITHUB_STEP_SUMMARY + echo "- Rate limit window resets correctly" >> $GITHUB_STEP_SUMMARY + else + echo "❌ **Rate limit tests failed**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Failure Details:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + grep -E "✗|FAIL|Error|failed|expected" ratelimit-test-output.txt | head -30 >> $GITHUB_STEP_SUMMARY || echo "See logs for details" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi + + - name: Cleanup + if: always() + run: | + docker rm -f charon-ratelimit-test || true + docker rm -f ratelimit-backend || true + docker volume rm charon_ratelimit_data caddy_ratelimit_data caddy_ratelimit_config 2>/dev/null || true + docker network rm containers_default || true diff --git a/Dockerfile b/Dockerfile index dd6b18df..c4e55342 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,8 @@ ARG VERSION=dev ARG BUILD_DATE ARG VCS_REF +# Set BUILD_DEBUG=1 to build with debug symbols (required for Delve debugging) +ARG BUILD_DEBUG=0 # Allow pinning Caddy version - Renovate will update this # Build the most recent Caddy 2.x release (keeps major pinned under v3). @@ -121,18 +123,32 @@ COPY backend/ ./ ARG VERSION=dev ARG VCS_REF=unknown ARG BUILD_DATE=unknown +ARG BUILD_DEBUG=0 # Build the Go binary with version information injected via ldflags # xx-go handles CGO and cross-compilation flags automatically # Note: Go 1.25 uses gold linker for ARM64; binutils-gold is installed above +# When BUILD_DEBUG=1, we preserve debug symbols (no -s -w) and disable optimizations +# for Delve debugging. Otherwise, strip symbols for smaller production binaries. RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ - CGO_ENABLED=1 xx-go build \ - -ldflags "-s -w \ - -X github.com/Wikid82/charon/backend/internal/version.Version=${VERSION} \ - -X github.com/Wikid82/charon/backend/internal/version.GitCommit=${VCS_REF} \ - -X github.com/Wikid82/charon/backend/internal/version.BuildTime=${BUILD_DATE}" \ - -o charon ./cmd/api + if [ "$BUILD_DEBUG" = "1" ]; then \ + echo "Building with debug symbols for Delve..."; \ + CGO_ENABLED=1 xx-go build \ + -gcflags="all=-N -l" \ + -ldflags "-X github.com/Wikid82/charon/backend/internal/version.Version=${VERSION} \ + -X github.com/Wikid82/charon/backend/internal/version.GitCommit=${VCS_REF} \ + -X github.com/Wikid82/charon/backend/internal/version.BuildTime=${BUILD_DATE}" \ + -o charon ./cmd/api; \ + else \ + echo "Building optimized production binary..."; \ + CGO_ENABLED=1 xx-go build \ + -ldflags "-s -w \ + -X github.com/Wikid82/charon/backend/internal/version.Version=${VERSION} \ + -X github.com/Wikid82/charon/backend/internal/version.GitCommit=${VCS_REF} \ + -X github.com/Wikid82/charon/backend/internal/version.BuildTime=${BUILD_DATE}" \ + -o charon ./cmd/api; \ + fi # ---- Caddy Builder ---- # Build Caddy from source to ensure we use the latest Go version and dependencies @@ -300,8 +316,9 @@ WORKDIR /app # Install runtime dependencies for Charon, including bash for maintenance scripts # Note: gosu is now built from source (see gosu-builder stage) to avoid CVEs from Debian's pre-compiled version # Explicitly upgrade packages to fix security vulnerabilities +# binutils provides objdump for debug symbol detection in docker-entrypoint.sh RUN apt-get update && apt-get install -y --no-install-recommends \ - bash ca-certificates libsqlite3-0 sqlite3 tzdata curl gettext-base libcap2-bin libc-ares2 \ + bash ca-certificates libsqlite3-0 sqlite3 tzdata curl gettext-base libcap2-bin libc-ares2 binutils \ && apt-get upgrade -y \ && rm -rf /var/lib/apt/lists/* diff --git a/docs/plans/current_spec.md b/docs/plans/current_spec.md index 6f90d846..b02ddcfe 100644 --- a/docs/plans/current_spec.md +++ b/docs/plans/current_spec.md @@ -1,7 +1,215 @@ -# Git & Workflow Recovery Plan +# WAF Integration Workflow Fix: wget-style curl Syntax Migration + +**Plan ID**: WAF-2026-001 +**Status**: 📋 PENDING +**Priority**: High +**Created**: 2026-01-25 +**Scope**: Fix integration test scripts using incorrect wget-style curl syntax + +--- + +## Problem Summary + +After migrating the Docker base image from Alpine to Debian Trixie (PR #550), the WAF integration workflow is failing. The root cause is **not** a missing `wget` command, but rather several integration test scripts using **wget-style options with curl** that don't work correctly. + +### Root Cause + +Multiple scripts use `curl -q -O-` which is **wget syntax, not curl syntax**: + +| Syntax | Tool | Meaning | +|--------|------|---------| +| `-q` | **wget** | Quiet mode | +| `-q` | **curl** | **Invalid** - does nothing useful | +| `-O-` | **wget** | Output to stdout | +| `-O-` | **curl** | **Wrong** - `-O` means "save with remote filename", `-` is treated as a separate URL | + +The correct curl equivalents are: +| wget | curl | Notes | +|------|------|-------| +| `wget -q` | `curl -s` | Silent mode | +| `wget -O-` | `curl -s` | stdout is curl's default output | +| `wget -q -O- URL` | `curl -s URL` | Full equivalent | +| `wget -O filename` | `curl -o filename` | Note: lowercase `-o` in curl | + +--- + +## Files Requiring Changes + +### Priority 1: Integration Test Scripts (Blocking WAF Workflow) + +| File | Line | Current Code | Issue | +|------|------|--------------|-------| +| [scripts/waf_integration.sh](../../scripts/waf_integration.sh#L205) | 205 | `curl -q -O- http://${BACKEND_CONTAINER}/get` | wget syntax | +| [scripts/cerberus_integration.sh](../../scripts/cerberus_integration.sh#L214) | 214 | `curl -q -O- http://${BACKEND_CONTAINER}/get` | wget syntax | +| [scripts/rate_limit_integration.sh](../../scripts/rate_limit_integration.sh#L190) | 190 | `curl -q -O- http://${BACKEND_CONTAINER}/get` | wget syntax | +| [scripts/crowdsec_startup_test.sh](../../scripts/crowdsec_startup_test.sh#L178) | 178 | `curl -q -O- http://127.0.0.1:8085/health` | wget syntax | + +### Priority 2: Utility Scripts + +| File | Line | Current Code | Issue | +|------|------|--------------|-------| +| [scripts/install-go-1.25.5.sh](../../scripts/install-go-1.25.5.sh#L18) | 18 | `curl -q -O "$TMPFILE" "URL"` | Wrong syntax - `-O` doesn't take an argument in curl | + +--- + +## Detailed Fixes + +### Fix 1: scripts/waf_integration.sh (Line 205) + +**Current (broken):** +```bash +if docker exec ${CONTAINER_NAME} sh -c "curl -q -O- http://${BACKEND_CONTAINER}/get 2>/dev/null || curl -s http://${BACKEND_CONTAINER}/get" >/dev/null 2>&1; then +``` + +**Fixed:** +```bash +if docker exec ${CONTAINER_NAME} sh -c "curl -sf http://${BACKEND_CONTAINER}/get" >/dev/null 2>&1; then +``` + +**Notes:** +- `-s` = silent (no progress meter) +- `-f` = fail silently on HTTP errors (returns non-zero exit code) +- Removed redundant fallback since the fix makes the command work correctly + +--- + +### Fix 2: scripts/cerberus_integration.sh (Line 214) + +**Current (broken):** +```bash +if docker exec ${CONTAINER_NAME} sh -c "curl -q -O- http://${BACKEND_CONTAINER}/get 2>/dev/null || curl -s http://${BACKEND_CONTAINER}/get" >/dev/null 2>&1; then +``` + +**Fixed:** +```bash +if docker exec ${CONTAINER_NAME} sh -c "curl -sf http://${BACKEND_CONTAINER}/get" >/dev/null 2>&1; then +``` + +--- + +### Fix 3: scripts/rate_limit_integration.sh (Line 190) + +**Current (broken):** +```bash +if docker exec ${CONTAINER_NAME} sh -c "curl -q -O- http://${BACKEND_CONTAINER}/get 2>/dev/null || curl -s http://${BACKEND_CONTAINER}/get" >/dev/null 2>&1; then +``` + +**Fixed:** +```bash +if docker exec ${CONTAINER_NAME} sh -c "curl -sf http://${BACKEND_CONTAINER}/get" >/dev/null 2>&1; then +``` + +--- + +### Fix 4: scripts/crowdsec_startup_test.sh (Line 178) + +**Current (broken):** +```bash +LAPI_HEALTH=$(docker exec ${CONTAINER_NAME} curl -q -O- http://127.0.0.1:8085/health 2>/dev/null || echo "FAILED") +``` + +**Fixed:** +```bash +LAPI_HEALTH=$(docker exec ${CONTAINER_NAME} curl -sf http://127.0.0.1:8085/health 2>/dev/null || echo "FAILED") +``` + +--- + +### Fix 5: scripts/install-go-1.25.5.sh (Line 18) + +**Current (broken):** +```bash +curl -q -O "$TMPFILE" "https://go.dev/dl/${TARFILE}" +``` + +**Fixed:** +```bash +curl -sSfL -o "$TMPFILE" "https://go.dev/dl/${TARFILE}" +``` + +**Notes:** +- `-s` = silent +- `-S` = show errors even in silent mode +- `-f` = fail on HTTP errors +- `-L` = follow redirects (important for go.dev downloads) +- `-o filename` = output to specified file (lowercase `-o`) + +--- + +## Verification Commands + +After applying fixes, verify each script works: + +```bash +# Test WAF integration +./scripts/waf_integration.sh + +# Test Cerberus integration +./scripts/cerberus_integration.sh + +# Test Rate Limit integration +./scripts/rate_limit_integration.sh + +# Test CrowdSec startup +./scripts/crowdsec_startup_test.sh + +# Verify Go install script syntax +bash -n ./scripts/install-go-1.25.5.sh +``` + +--- + +## Behavior Differences: wget vs curl + +When migrating from wget to curl, be aware of these differences: + +| Behavior | wget | curl | +|----------|------|------| +| Output destination | File by default | stdout by default | +| Follow redirects | Yes by default | Requires `-L` flag | +| Retry on failure | Built-in retry | Requires `--retry N` | +| Progress display | Text progress bar | Progress meter (use `-s` to hide) | +| HTTP error handling | Non-zero exit on 404 | Requires `-f` for non-zero exit on HTTP errors | +| Quiet mode | `-q` | `-s` (silent) | +| Output to file | `-O filename` (uppercase) | `-o filename` (lowercase) | +| Save with remote name | `-O` (no arg) | `-O` (uppercase, no arg) | + +--- + +## Execution Checklist + +- [ ] **Fix 1**: Update `scripts/waf_integration.sh` line 205 +- [ ] **Fix 2**: Update `scripts/cerberus_integration.sh` line 214 +- [ ] **Fix 3**: Update `scripts/rate_limit_integration.sh` line 190 +- [ ] **Fix 4**: Update `scripts/crowdsec_startup_test.sh` line 178 +- [ ] **Fix 5**: Update `scripts/install-go-1.25.5.sh` line 18 +- [ ] **Verify**: Run each integration test locally +- [ ] **CI**: Confirm WAF integration workflow passes + +--- + +## Notes + +1. **Deprecated Scripts**: Several affected scripts are marked deprecated (will be removed in v2.0.0). However, they are still used by CI workflows, so fixes are required. + +2. **Skill-Based Replacements**: The `.github/skills/scripts/` directory was checked and contains no wget usage - those scripts already use correct curl syntax. + +3. **Docker Compose Files**: All health checks in docker-compose files already use correct curl syntax (`curl -f`, `curl -fsS`). + +4. **Dockerfile**: The main Dockerfile correctly installs `curl` and uses correct curl syntax in the HEALTHCHECK instruction. + +--- + +# Previous Plan (Archived) + +The previous Git & Workflow Recovery Plan has been archived below. + +--- + +# Git & Workflow Recovery Plan (ARCHIVED) **Plan ID**: GIT-2026-001 -**Status**: 📋 PENDING +**Status**: ✅ ARCHIVED **Priority**: High **Created**: 2026-01-25 **Scope**: Git recovery, Renovate fix, Workflow simplification @@ -366,3 +574,79 @@ docker compose -f .docker/compose/docker-compose.playwright.yml config ### References - **OWASP**: [A02:2021 – Cryptographic Failures](https://owasp.org/Top10/A02_2021-Cryptographic_Failures/) + +--- + +# Future Phase: Playwright Security Test Helpers + +**Plan ID**: E2E-SEC-001 +**Status**: 📋 TODO (Follow-up Task) +**Priority**: Medium +**Created**: 2026-01-25 +**Scope**: Add security test helpers to prevent ACL deadlock in E2E tests + +--- + +## Problem Summary + +During E2E testing, if ACL is left enabled from a previous test run (e.g., due to test failure), it can create a **deadlock**: +1. ACL blocks API requests → returns 403 Forbidden +2. Global cleanup can't run → API blocked +3. Auth setup fails → tests skip +4. Manual intervention required to reset volumes + +## Solution: Security Test Helpers + +Create `tests/utils/security-helpers.ts` with API helpers for: + +### 1. Get Security Status + +```typescript +export async function getSecurityStatus(request: APIRequestContext): Promise +``` + +### 2. Toggle ACL via API + +```typescript +export async function setAclEnabled(request: APIRequestContext, enabled: boolean): Promise +``` + +### 3. Ensure ACL Enabled with Cleanup + +```typescript +export async function ensureAclEnabled(request: APIRequestContext): Promise +export async function restoreSecurityState(request: APIRequestContext, originalState: OriginalState): Promise +``` + +## Usage Pattern + +```typescript +test.describe('ACL Enforcement Tests', () => { + let originalState: OriginalState; + + test.beforeAll(async ({ request }) => { + originalState = await ensureAclEnabled(request); + }); + + test.afterAll(async ({ request }) => { + await restoreSecurityState(request, originalState); // Runs even on failure + }); + + // ACL tests here... +}); +``` + +## Benefits + +1. **No deadlock**: Tests can safely enable/disable ACL +2. **Cleanup guaranteed**: `test.afterAll` runs even on failure +3. **Realistic testing**: ACL tests use the same toggle mechanism as users +4. **Isolation**: Other tests unaffected by ACL state + +## Files to Create/Modify + +| File | Action | +|------|--------| +| `tests/utils/security-helpers.ts` | **Create** - New helper module | +| `tests/security/security-dashboard.spec.ts` | **Modify** - Use new helpers | +| `tests/integration/proxy-acl-integration.spec.ts` | **Modify** - Use new helpers | diff --git a/docs/reports/qa_report_waf_integration_fix_2026-01-25.md b/docs/reports/qa_report_waf_integration_fix_2026-01-25.md new file mode 100644 index 00000000..0f32335d --- /dev/null +++ b/docs/reports/qa_report_waf_integration_fix_2026-01-25.md @@ -0,0 +1,157 @@ +# QA Report: WAF Integration Fix + +**Date**: 2026-01-25 +**Branch**: feature/beta-release (merged from development) +**Context**: Fixed integration scripts using wget-style curl syntax + +--- + +## Definition of Done Audit Summary + +| Check | Status | Details | +|-------|--------|---------| +| Playwright E2E Tests | ❌ **FAILED** | 230/707 tests failed - ACL blocking test user creation | +| Backend Coverage | ✅ **PASS** | 86.5% (minimum: 85%) | +| Frontend Coverage | ⚠️ **1 FAILURE** | 1499/1500 passed, 1 failed test | +| TypeScript Type Check | ✅ **PASS** | No errors | +| Pre-commit Hooks | ✅ **PASS** | All hooks passed | +| Trivy Security Scan | ⚠️ **WARNING** | 2 HIGH (OS-level, no fix available) | +| Grype Security Scan | ✅ **PASS** | No fixable HIGH/CRITICAL issues | + +--- + +## Detailed Results + +### 1. Playwright E2E Tests ❌ FAILED + +**Result**: 230 tests failed, 472 passed, 39 skipped + +**Root Cause**: All failures show identical error: +``` +Error: Failed to create user: {"error":"Blocked by access control list"} +``` + +**Analysis**: The WAF/ACL configuration is blocking the test fixture's ability to create test users via the API. This is a configuration issue in the Docker container's Cerberus security layer, not a code defect. + +**Affected Test Suites**: +- `tests/security/` - Security dashboard, WAF config, rate limiting +- `tests/settings/` - Account settings, user management, notifications, SMTP +- `tests/tasks/` - Backups, imports, logs viewing + +**Remediation Required**: +1. Review Cerberus ACL whitelist configuration for test environment +2. Ensure test API endpoints or test user IPs are whitelisted +3. Check if `DISABLE_ACL_FOR_TESTS` environment variable is needed + +--- + +### 2. Backend Coverage ✅ PASS + +**Result**: 86.5% statement coverage + +- Minimum required: 85% +- All test suites passed +- No test failures + +--- + +### 3. Frontend Coverage ⚠️ 1 FAILURE + +**Result**: 1499 passed, 1 failed, 2 skipped + +**Failed Test**: +``` +FAIL src/components/__tests__/SecurityNotificationSettingsModal.test.tsx + > loads and displays existing settings + +AssertionError: expected false to be true + - enableSwitch.checked expected to be true +``` + +**Analysis**: This appears to be a timing/async issue in the test where the modal's settings aren't loaded before the assertion runs. This is a **test flakiness issue**, not a production bug. + +**Remediation**: Add `waitFor` or increase timeout for settings load in the test. + +--- + +### 4. TypeScript Type Check ✅ PASS + +**Result**: `tsc --noEmit` completed with zero errors + +--- + +### 5. Pre-commit Hooks ✅ PASS + +All hooks passed: +- fix end of files +- trim trailing whitespace +- check yaml +- check for added large files +- dockerfile validation +- Go Vet +- golangci-lint +- .version tag match +- LFS checks +- CodeQL DB artifact prevention +- Frontend TypeScript Check +- Frontend Lint + +--- + +### 6. Security Scans + +#### Trivy ⚠️ WARNING + +**Findings**: 2 HIGH, 0 CRITICAL + +| Library | CVE | Severity | Status | Notes | +|---------|-----|----------|--------|-------| +| libc-bin | CVE-2026-0861 | HIGH | affected | glibc heap corruption - no fix available | +| libc6 | CVE-2026-0861 | HIGH | affected | Same as above | + +**Assessment**: These are base OS (Debian 13.3) vulnerabilities in glibc with no upstream fix available. The application code (Go binaries, Caddy, CrowdSec) has **zero vulnerabilities**. + +#### Grype ✅ PASS + +**Result**: No fixable vulnerabilities found + +--- + +## Issues Blocking Merge + +### Critical (Must Fix Before Merge) + +1. **Playwright E2E Test ACL Blocking** + - **Issue**: Cerberus ACL blocks test user creation + - **Impact**: 230 E2E tests cannot run + - **Owner**: DevOps/Security configuration + - **Fix**: Whitelist test API or add test environment bypass + +### Minor (Can Be Fixed Post-Merge) + +2. **Flaky Frontend Test** + - **Issue**: `SecurityNotificationSettingsModal` test timing issue + - **Impact**: 1 test failure + - **Owner**: Frontend team + - **Fix**: Add proper async waiting in test + +--- + +## Recommendations + +1. **Immediate**: Investigate and fix the ACL configuration blocking E2E tests +2. **Before Merge**: Re-run full E2E suite after ACL fix +3. **Post-Merge**: Fix the flaky frontend test +4. **Ongoing**: Monitor CVE-2026-0861 for upstream glibc fix + +--- + +## Sign-off + +- [ ] ACL blocking issue resolved +- [ ] E2E tests passing (aim: >95%) +- [ ] Frontend flaky test fixed or documented +- [ ] Security scan reviewed and accepted + +**Auditor**: GitHub Copilot (Claude Opus 4.5) +**Audit Date**: 2026-01-25 diff --git a/scripts/cerberus_integration.sh b/scripts/cerberus_integration.sh index 6f1840f2..8e67efad 100755 --- a/scripts/cerberus_integration.sh +++ b/scripts/cerberus_integration.sh @@ -196,7 +196,7 @@ docker run -d --name ${CONTAINER_NAME} \ log_info "Waiting for Charon API to be ready..." for i in {1..30}; do - if curl -s -f "http://localhost:${API_PORT}/api/v1/" >/dev/null 2>&1; then + if curl -s -f "http://localhost:${API_PORT}/api/v1/health" >/dev/null 2>&1; then log_info "Charon API is ready" break fi @@ -211,7 +211,7 @@ echo "" log_info "Waiting for httpbin backend to be ready..." for i in {1..20}; do - if docker exec ${CONTAINER_NAME} sh -c "curl -q -O- http://${BACKEND_CONTAINER}/get 2>/dev/null || curl -s http://${BACKEND_CONTAINER}/get" >/dev/null 2>&1; then + if docker exec ${CONTAINER_NAME} sh -c "curl -sf http://${BACKEND_CONTAINER}/get" >/dev/null 2>&1; then log_info "httpbin backend is ready" break fi diff --git a/scripts/crowdsec_startup_test.sh b/scripts/crowdsec_startup_test.sh index f7da0223..a82ea7f8 100755 --- a/scripts/crowdsec_startup_test.sh +++ b/scripts/crowdsec_startup_test.sh @@ -175,7 +175,7 @@ fi log_test "Check 2: CrowdSec LAPI health (127.0.0.1:8085/health)" # Use docker exec to check LAPI health from inside the container -LAPI_HEALTH=$(docker exec ${CONTAINER_NAME} curl -q -O- http://127.0.0.1:8085/health 2>/dev/null || echo "FAILED") +LAPI_HEALTH=$(docker exec ${CONTAINER_NAME} curl -sf http://127.0.0.1:8085/health 2>/dev/null || echo "FAILED") if [ "$LAPI_HEALTH" != "FAILED" ] && [ -n "$LAPI_HEALTH" ]; then log_info " LAPI is healthy" diff --git a/scripts/install-go-1.25.5.sh b/scripts/install-go-1.25.5.sh index 9a93f0e7..49e1f124 100755 --- a/scripts/install-go-1.25.5.sh +++ b/scripts/install-go-1.25.5.sh @@ -15,7 +15,7 @@ TMPFILE="/tmp/${TARFILE}" # Download if [ ! -f "$TMPFILE" ]; then echo "Downloading go${GO_VERSION}..." - curl -q -O "$TMPFILE" "https://go.dev/dl/${TARFILE}" + curl -sSfL -o "$TMPFILE" "https://go.dev/dl/${TARFILE}" fi # Remove existing installation diff --git a/scripts/rate_limit_integration.sh b/scripts/rate_limit_integration.sh index a3e1bc0a..1622adcb 100755 --- a/scripts/rate_limit_integration.sh +++ b/scripts/rate_limit_integration.sh @@ -166,7 +166,7 @@ docker run -d --name ${CONTAINER_NAME} \ echo "Waiting for Charon API to be ready..." for i in {1..30}; do - if curl -s -f http://localhost:8280/api/v1/ >/dev/null 2>&1; then + if curl -s -f http://localhost:8280/api/v1/health >/dev/null 2>&1; then echo "✓ Charon API is ready" break fi @@ -187,7 +187,7 @@ docker run -d --name ${BACKEND_CONTAINER} --network containers_default kennethre echo "Waiting for httpbin backend to be ready..." for i in {1..20}; do - if docker exec ${CONTAINER_NAME} sh -c "curl -q -O- http://${BACKEND_CONTAINER}/get 2>/dev/null || curl -s http://${BACKEND_CONTAINER}/get" >/dev/null 2>&1; then + if docker exec ${CONTAINER_NAME} sh -c "curl -sf http://${BACKEND_CONTAINER}/get" >/dev/null 2>&1; then echo "✓ httpbin backend is ready" break fi diff --git a/scripts/waf_integration.sh b/scripts/waf_integration.sh index f0f53b20..76d9e8ed 100755 --- a/scripts/waf_integration.sh +++ b/scripts/waf_integration.sh @@ -187,7 +187,7 @@ docker run -d --name ${CONTAINER_NAME} \ log_info "Waiting for Charon API to be ready..." for i in {1..30}; do - if curl -s -f "http://localhost:${API_PORT}/api/v1/" >/dev/null 2>&1; then + if curl -s -f "http://localhost:${API_PORT}/api/v1/health" >/dev/null 2>&1; then log_info "Charon API is ready" break fi @@ -202,7 +202,7 @@ echo "" log_info "Waiting for httpbin backend to be ready..." for i in {1..20}; do - if docker exec ${CONTAINER_NAME} sh -c "curl -q -O- http://${BACKEND_CONTAINER}/get 2>/dev/null || curl -s http://${BACKEND_CONTAINER}/get" >/dev/null 2>&1; then + if docker exec ${CONTAINER_NAME} sh -c "curl -sf http://${BACKEND_CONTAINER}/get" >/dev/null 2>&1; then log_info "httpbin backend is ready" break fi