#!/usr/bin/env bash set -euo pipefail readonly DEFAULT_CANDIDATE_VERSION="2.11.2" readonly DEFAULT_PATCH_SCENARIOS="A,B,C" readonly DEFAULT_PLATFORMS="linux/amd64,linux/arm64" readonly DEFAULT_PLUGIN_SET="caddy-security,coraza-caddy,caddy-crowdsec-bouncer,caddy-geoip2,caddy-ratelimit" readonly DEFAULT_SMOKE_SET="boot_caddy,plugin_modules,config_validate,admin_api_health" OUTPUT_DIR="test-results/caddy-compat" DOCS_REPORT="docs/reports/caddy-compatibility-matrix.md" CANDIDATE_VERSION="$DEFAULT_CANDIDATE_VERSION" PATCH_SCENARIOS="$DEFAULT_PATCH_SCENARIOS" PLATFORMS="$DEFAULT_PLATFORMS" PLUGIN_SET="$DEFAULT_PLUGIN_SET" SMOKE_SET="$DEFAULT_SMOKE_SET" BASE_IMAGE_TAG="charon" KEEP_IMAGES="0" REQUIRED_MODULES=( "http.handlers.auth_portal" "http.handlers.waf" "http.handlers.crowdsec" "http.handlers.geoip2" "http.handlers.rate_limit" ) usage() { cat <<'EOF' Usage: scripts/caddy-compat-matrix.sh [options] Options: --output-dir Output directory (default: test-results/caddy-compat) --docs-report Markdown report path (default: docs/reports/caddy-compatibility-matrix.md) --candidate-version Candidate Caddy version (default: 2.11.2) --patch-scenarios Patch scenarios CSV (default: A,B,C) --platforms Platforms CSV (default: linux/amd64,linux/arm64) --plugin-set Plugin set descriptor for report metadata --smoke-set Smoke set descriptor for report metadata --base-image-tag Base image tag prefix (default: charon) --keep-images Keep generated local images -h, --help Show this help Deterministic pass/fail: Promotion gate PASS only if Scenario A passes on linux/amd64 and linux/arm64. Scenario B/C are evidence-only and do not fail the promotion gate. EOF } require_cmd() { local cmd="$1" if ! command -v "$cmd" >/dev/null 2>&1; then echo "ERROR: Required command not found: $cmd" >&2 exit 1 fi } parse_args() { while [[ $# -gt 0 ]]; do case "$1" in --output-dir) OUTPUT_DIR="$2" shift 2 ;; --docs-report) DOCS_REPORT="$2" shift 2 ;; --candidate-version) CANDIDATE_VERSION="$2" shift 2 ;; --patch-scenarios) PATCH_SCENARIOS="$2" shift 2 ;; --platforms) PLATFORMS="$2" shift 2 ;; --plugin-set) PLUGIN_SET="$2" shift 2 ;; --smoke-set) SMOKE_SET="$2" shift 2 ;; --base-image-tag) BASE_IMAGE_TAG="$2" shift 2 ;; --keep-images) KEEP_IMAGES="1" shift ;; -h|--help) usage exit 0 ;; *) echo "Unknown option: $1" >&2 usage exit 1 ;; esac done } prepare_dirs() { mkdir -p "$OUTPUT_DIR" mkdir -p "$(dirname "$DOCS_REPORT")" } write_reports_header() { local metadata_file="$OUTPUT_DIR/metadata.env" local summary_csv="$OUTPUT_DIR/matrix-summary.csv" cat > "$metadata_file" < "$summary_csv" } contains_value() { local needle="$1" shift local value for value in "$@"; do if [[ "$value" == "$needle" ]]; then return 0 fi done return 1 } enforce_required_gate_dimensions() { local -n scenario_ref=$1 local -n platform_ref=$2 if ! contains_value "A" "${scenario_ref[@]}"; then echo "[compat] ERROR: Scenario A is required for PR-1 promotion gate" >&2 return 1 fi if ! contains_value "linux/amd64" "${platform_ref[@]}"; then echo "[compat] ERROR: linux/amd64 is required for PR-1 promotion gate" >&2 return 1 fi if ! contains_value "linux/arm64" "${platform_ref[@]}"; then echo "[compat] ERROR: linux/arm64 is required for PR-1 promotion gate" >&2 return 1 fi } validate_matrix_completeness() { local summary_csv="$1" local -n scenario_ref=$2 local -n platform_ref=$3 local expected_rows expected_rows=$(( ${#scenario_ref[@]} * ${#platform_ref[@]} )) local actual_rows actual_rows="$(tail -n +2 "$summary_csv" | sed '/^\s*$/d' | wc -l | tr -d '[:space:]')" if [[ "$actual_rows" != "$expected_rows" ]]; then echo "[compat] ERROR: matrix completeness failed (expected ${expected_rows} rows, found ${actual_rows})" >&2 return 1 fi local scenario local platform for scenario in "${scenario_ref[@]}"; do for platform in "${platform_ref[@]}"; do if ! grep -q "^${scenario},${platform}," "$summary_csv"; then echo "[compat] ERROR: missing matrix cell scenario=${scenario} platform=${platform}" >&2 return 1 fi done done } evaluate_promotion_gate() { local summary_csv="$1" local scenario_a_failures scenario_a_failures="$(tail -n +2 "$summary_csv" | awk -F',' '$1=="A" && $10=="FAIL" {count++} END {print count+0}')" local evidence_failures evidence_failures="$(tail -n +2 "$summary_csv" | awk -F',' '$1!="A" && $10=="FAIL" {count++} END {print count+0}')" if [[ "$evidence_failures" -gt 0 ]]; then echo "[compat] Evidence-only failures (Scenario B/C): ${evidence_failures}" fi if [[ "$scenario_a_failures" -gt 0 ]]; then echo "[compat] Promotion gate result: FAIL (Scenario A failures: ${scenario_a_failures})" return 1 fi echo "[compat] Promotion gate result: PASS (Scenario A on both required architectures)" } build_image_for_cell() { local scenario="$1" local platform="$2" local image_tag="$3" docker buildx build \ --platform "$platform" \ --load \ --pull \ --build-arg CADDY_USE_CANDIDATE=1 \ --build-arg CADDY_CANDIDATE_VERSION="$CANDIDATE_VERSION" \ --build-arg CADDY_PATCH_SCENARIO="$scenario" \ -t "$image_tag" \ . >/dev/null } smoke_boot_caddy() { local image_tag="$1" docker run --rm --pull=never --entrypoint caddy "$image_tag" version >/dev/null } smoke_plugin_modules() { local image_tag="$1" local output_file="$2" docker run --rm --pull=never --entrypoint caddy "$image_tag" list-modules > "$output_file" local module for module in "${REQUIRED_MODULES[@]}"; do grep -q "^${module}$" "$output_file" done } smoke_config_validate() { local image_tag="$1" docker run --rm --pull=never --entrypoint sh "$image_tag" -lc ' cat > /tmp/compat-config.json <<"JSON" { "admin": {"listen": ":2019"}, "apps": { "http": { "servers": { "compat": { "listen": [":2080"], "routes": [ { "handle": [ { "handler": "static_response", "body": "compat-ok", "status_code": 200 } ] } ] } } } } } JSON caddy validate --config /tmp/compat-config.json >/dev/null ' } smoke_admin_api_health() { local image_tag="$1" local admin_port="$2" local run_id="compat-${admin_port}" docker run -d --name "$run_id" --pull=never --entrypoint sh -p "${admin_port}:2019" "$image_tag" -lc ' cat > /tmp/admin-config.json <<"JSON" { "admin": {"listen": ":2019"}, "apps": { "http": { "servers": { "admin": { "listen": [":2081"], "routes": [ { "handle": [ { "handler": "static_response", "body": "admin-ok", "status_code": 200 } ] } ] } } } } } JSON caddy run --config /tmp/admin-config.json ' >/dev/null local attempts=0 until curl -sS "http://127.0.0.1:${admin_port}/config/" >/dev/null 2>&1; do attempts=$((attempts + 1)) if [[ $attempts -ge 30 ]]; then docker logs "$run_id" || true docker rm -f "$run_id" >/dev/null 2>&1 || true return 1 fi sleep 1 done docker rm -f "$run_id" >/dev/null 2>&1 || true } extract_module_inventory() { local image_tag="$1" local output_prefix="$2" local container_id container_id="$(docker create --pull=never "$image_tag")" docker cp "${container_id}:/usr/bin/caddy" "${output_prefix}-caddy" docker rm "$container_id" >/dev/null if command -v go >/dev/null 2>&1; then go version -m "${output_prefix}-caddy" > "${output_prefix}-go-version-m.txt" || true else echo "go toolchain not available; module inventory skipped" > "${output_prefix}-go-version-m.txt" fi docker run --rm --pull=never --entrypoint caddy "$image_tag" list-modules > "${output_prefix}-modules.txt" } run_cell() { local scenario="$1" local platform="$2" local cell_index="$3" local summary_csv="$OUTPUT_DIR/matrix-summary.csv" local safe_platform safe_platform="${platform//\//-}" local image_tag="${BASE_IMAGE_TAG}:caddy-${CANDIDATE_VERSION}-candidate-${scenario}-${safe_platform}" local module_prefix="$OUTPUT_DIR/module-inventory-${scenario}-${safe_platform}" local modules_list_file="$OUTPUT_DIR/modules-${scenario}-${safe_platform}.txt" local admin_port=$((22019 + cell_index)) local checked_plugins checked_plugins="${REQUIRED_MODULES[*]}" checked_plugins="${checked_plugins// /;}" echo "[compat] building cell scenario=${scenario} platform=${platform}" local boot_status="FAIL" local modules_status="FAIL" local validate_status="FAIL" local admin_status="FAIL" local inventory_status="FAIL" local cell_status="FAIL" if build_image_for_cell "$scenario" "$platform" "$image_tag"; then smoke_boot_caddy "$image_tag" && boot_status="PASS" || boot_status="FAIL" smoke_plugin_modules "$image_tag" "$modules_list_file" && modules_status="PASS" || modules_status="FAIL" smoke_config_validate "$image_tag" && validate_status="PASS" || validate_status="FAIL" smoke_admin_api_health "$image_tag" "$admin_port" && admin_status="PASS" || admin_status="FAIL" if extract_module_inventory "$image_tag" "$module_prefix"; then inventory_status="PASS" fi fi if [[ "$boot_status" == "PASS" && "$modules_status" == "PASS" && "$validate_status" == "PASS" && "$admin_status" == "PASS" && "$inventory_status" == "PASS" ]]; then cell_status="PASS" fi echo "${scenario},${platform},${image_tag},${checked_plugins},${boot_status},${modules_status},${validate_status},${admin_status},${inventory_status},${cell_status}" >> "$summary_csv" echo "[compat] RESULT scenario=${scenario} platform=${platform} status=${cell_status}" if [[ "$KEEP_IMAGES" != "1" ]]; then docker image rm "$image_tag" >/dev/null 2>&1 || true fi } write_docs_report() { local summary_csv="$OUTPUT_DIR/matrix-summary.csv" local generated_at generated_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" { echo "# PR-1 Caddy Compatibility Matrix Report" echo echo "- Generated at: ${generated_at}" echo "- Candidate Caddy version: ${CANDIDATE_VERSION}" echo "- Plugin set: ${PLUGIN_SET}" echo "- Smoke set: ${SMOKE_SET}" echo "- Matrix dimensions: patch scenario × platform/arch × checked plugin modules" echo echo "## Deterministic Pass/Fail" echo echo "A matrix cell is PASS only when every smoke check and module inventory extraction passes." echo echo "Promotion gate semantics (spec-aligned):" echo "- Scenario A on linux/amd64 and linux/arm64 is promotion-gating." echo "- Scenario B/C are evidence-only; failures in B/C do not fail the PR-1 promotion gate." echo echo "## Matrix Output" echo echo "| Scenario | Platform | Plugins Checked | boot_caddy | plugin_modules | config_validate | admin_api_health | module_inventory | Status |" echo "| --- | --- | --- | --- | --- | --- | --- | --- | --- |" tail -n +2 "$summary_csv" | while IFS=',' read -r scenario platform _image checked_plugins boot modules validate admin inventory status; do local plugins_display plugins_display="${checked_plugins//;/, }" echo "| ${scenario} | ${platform} | ${plugins_display} | ${boot} | ${modules} | ${validate} | ${admin} | ${inventory} | ${status} |" done echo echo "## Artifacts" echo echo "- Matrix CSV: ${OUTPUT_DIR}/matrix-summary.csv" echo "- Per-cell module inventories: ${OUTPUT_DIR}/module-inventory-*-go-version-m.txt" echo "- Per-cell Caddy module listings: ${OUTPUT_DIR}/module-inventory-*-modules.txt" } > "$DOCS_REPORT" } main() { parse_args "$@" require_cmd docker require_cmd curl prepare_dirs write_reports_header local -a scenario_list local -a platform_list IFS=',' read -r -a scenario_list <<< "$PATCH_SCENARIOS" IFS=',' read -r -a platform_list <<< "$PLATFORMS" enforce_required_gate_dimensions scenario_list platform_list local cell_index=0 local scenario local platform for scenario in "${scenario_list[@]}"; do for platform in "${platform_list[@]}"; do run_cell "$scenario" "$platform" "$cell_index" cell_index=$((cell_index + 1)) done done write_docs_report local summary_csv="$OUTPUT_DIR/matrix-summary.csv" validate_matrix_completeness "$summary_csv" scenario_list platform_list evaluate_promotion_gate "$summary_csv" } main "$@"