From 6675f2a169408ac938c8782b71a967fdebb383a3 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Fri, 30 Jan 2026 06:38:56 +0000 Subject: [PATCH] fix: Implement dependency digest tracking for nightly builds - Updated Docker Compose files to use digest-pinned images for CI contexts. - Enhanced Dockerfile to pin Go tool installations and verify external downloads with SHA256 checksums. - Added Renovate configuration for tracking Go tool versions and digest updates. - Introduced a new design document outlining the architecture and data flow for dependency tracking. - Created tasks and requirements documentation to ensure compliance with the new digest pinning policy. - Updated security documentation to reflect the new digest pinning policy and exceptions. --- .docker/compose/docker-compose.dev.yml | 4 +- .../compose/docker-compose.playwright-ci.yml | 10 +- .docker/compose/docker-compose.remote.yml | 2 +- .docker/compose/docker-compose.yml | 4 +- .github/renovate.json | 55 +++ .../utility-update-go-version-scripts/run.sh | 3 + .github/workflows/docker-build.yml | 2 +- .github/workflows/e2e-tests.yml | 4 + .github/workflows/nightly-build.yml | 13 +- ARCHITECTURE.md | 6 +- Dockerfile | 18 +- SECURITY.md | 35 +- categories.txt | 4 + docs/plans/current_spec.md | 386 +++++++++++++++--- docs/plans/design.md | 32 ++ docs/plans/requirements.md | 13 + docs/plans/tasks.md | 18 + scripts/install-go-1.25.6.sh | 3 +- scripts/security-scan.sh | 3 +- 19 files changed, 545 insertions(+), 70 deletions(-) create mode 100644 categories.txt create mode 100644 docs/plans/design.md create mode 100644 docs/plans/requirements.md create mode 100644 docs/plans/tasks.md diff --git a/.docker/compose/docker-compose.dev.yml b/.docker/compose/docker-compose.dev.yml index 7c4a8261..8d9a3150 100644 --- a/.docker/compose/docker-compose.dev.yml +++ b/.docker/compose/docker-compose.dev.yml @@ -2,7 +2,9 @@ services: app: - image: ghcr.io/wikid82/charon:dev + # Override for local testing: + # CHARON_DEV_IMAGE=ghcr.io/wikid82/charon:dev + image: ${CHARON_DEV_IMAGE:-ghcr.io/wikid82/charon:dev@sha256:8ed38f884c217ee09da02d5b7ba990fa22ccdd4fb0d2e01a4da1b5963301104f} # Development: expose Caddy admin API externally for debugging ports: - "80:80" diff --git a/.docker/compose/docker-compose.playwright-ci.yml b/.docker/compose/docker-compose.playwright-ci.yml index add65361..0e4c6b64 100644 --- a/.docker/compose/docker-compose.playwright-ci.yml +++ b/.docker/compose/docker-compose.playwright-ci.yml @@ -27,7 +27,11 @@ services: # Charon Application - Core E2E Testing Service # ============================================================================= charon-app: - image: ${CHARON_E2E_IMAGE:-charon:e2e-test} + # CI default (digest-pinned via workflow output): + # CHARON_E2E_IMAGE_DIGEST=ghcr.io/wikid82/charon:nightly@sha256: + # Local override (tag-based): + # CHARON_E2E_IMAGE=charon:e2e-test + image: ${CHARON_E2E_IMAGE_DIGEST:-${CHARON_E2E_IMAGE:-charon:e2e-test}} container_name: charon-playwright restart: "no" # CI generates CHARON_ENCRYPTION_KEY dynamically in GitHub Actions workflow @@ -96,7 +100,7 @@ services: # CrowdSec - Security Testing Service (Optional Profile) # ============================================================================= crowdsec: - image: crowdsecurity/crowdsec:latest + image: crowdsecurity/crowdsec:latest@sha256:63b595fef92de1778573b375897a45dd226637ee9a3d3db9f57ac7355c369493 container_name: charon-playwright-crowdsec profiles: - security-tests @@ -122,7 +126,7 @@ services: # MailHog - Email Testing Service (Optional Profile) # ============================================================================= mailhog: - image: mailhog/mailhog:latest + image: mailhog/mailhog:latest@sha256:8d76a3d4ffa32a3661311944007a415332c4bb855657f4f6c57996405c009bea container_name: charon-playwright-mailhog profiles: - notification-tests diff --git a/.docker/compose/docker-compose.remote.yml b/.docker/compose/docker-compose.remote.yml index 0ab6f481..a65d619e 100644 --- a/.docker/compose/docker-compose.remote.yml +++ b/.docker/compose/docker-compose.remote.yml @@ -4,7 +4,7 @@ services: # Run this service on your REMOTE servers (not the one running Charon) # to allow Charon to discover containers running there (legacy: CPMP). docker-socket-proxy: - image: alpine/socat + image: alpine/socat:latest@sha256:bd8d6a251eb7d1b8c08f7117e3e583e14ec86f43f25d2bf31a6e16ff5dc15f58 container_name: docker-socket-proxy restart: unless-stopped ports: diff --git a/.docker/compose/docker-compose.yml b/.docker/compose/docker-compose.yml index a645752c..4dc6da9b 100644 --- a/.docker/compose/docker-compose.yml +++ b/.docker/compose/docker-compose.yml @@ -1,6 +1,8 @@ services: charon: - image: ghcr.io/wikid82/charon:latest + # Override for local testing: + # CHARON_IMAGE=ghcr.io/wikid82/charon:latest + image: ${CHARON_IMAGE:-ghcr.io/wikid82/charon:latest@sha256:371a3fdabc7f52da65a4ac888531a413b6a56294f65041a42fdc0c407e8454c4} container_name: charon restart: unless-stopped ports: diff --git a/.github/renovate.json b/.github/renovate.json index 27f6939f..1b636d28 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -55,6 +55,61 @@ "depNameTemplate": "debian", "datasourceTemplate": "docker", "versioningTemplate": "docker" + }, + { + "customType": "regex", + "description": "Track Delve version in Dockerfile", + "managerFilePatterns": ["/^Dockerfile$/"], + "matchStrings": [ + "ARG DLV_VERSION=(?[^\\s]+)" + ], + "depNameTemplate": "github.com/go-delve/delve", + "datasourceTemplate": "go", + "versioningTemplate": "semver" + }, + { + "customType": "regex", + "description": "Track xcaddy version in Dockerfile", + "managerFilePatterns": ["/^Dockerfile$/"], + "matchStrings": [ + "ARG XCADDY_VERSION=(?[^\\s]+)" + ], + "depNameTemplate": "github.com/caddyserver/xcaddy", + "datasourceTemplate": "go", + "versioningTemplate": "semver" + }, + { + "customType": "regex", + "description": "Track govulncheck version in scripts", + "managerFilePatterns": ["/^scripts\\/security-scan\\.sh$/"], + "matchStrings": [ + "govulncheck@v(?[^\\s]+)" + ], + "depNameTemplate": "golang.org/x/vuln", + "datasourceTemplate": "go", + "versioningTemplate": "semver" + }, + { + "customType": "regex", + "description": "Track gopls version in Go install script", + "managerFilePatterns": ["/^scripts\\/install-go-1\\.25\\.6\\.sh$/"], + "matchStrings": [ + "gopls@v(?[^\\s]+)" + ], + "depNameTemplate": "golang.org/x/tools", + "datasourceTemplate": "go", + "versioningTemplate": "semver" + }, + { + "customType": "regex", + "description": "Track Go toolchain version in go.work for the dl shim", + "managerFilePatterns": ["/^go\\.work$/"], + "matchStrings": [ + "^go (?\\d+\\.\\d+\\.\\d+)$" + ], + "depNameTemplate": "golang/go", + "datasourceTemplate": "golang-version", + "versioningTemplate": "semver" } ], diff --git a/.github/skills/utility-update-go-version-scripts/run.sh b/.github/skills/utility-update-go-version-scripts/run.sh index 92840ea1..178acf49 100755 --- a/.github/skills/utility-update-go-version-scripts/run.sh +++ b/.github/skills/utility-update-go-version-scripts/run.sh @@ -37,6 +37,9 @@ echo "๐Ÿ”„ Updating Go from $CURRENT_VERSION to $REQUIRED_VERSION..." # Download the new Go version using the official dl tool echo "๐Ÿ“ฅ Downloading Go $REQUIRED_VERSION..." +# Exception: golang.org/dl requires @latest to resolve the versioned shim. +# Compensating controls: REQUIRED_VERSION is pinned in go.work, and the dl tool +# downloads the official Go release for that exact version. go install "golang.org/dl/go${REQUIRED_VERSION}@latest" # Download the SDK diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 5d1bc8a2..2951ef4f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -509,7 +509,7 @@ jobs: docker run -d \ --name whoami \ --network charon-test-net \ - traefik/whoami + traefik/whoami:latest@sha256:200689790a0a0ea48ca45992e0450bc26ccab5307375b41c84dfc4f2475937ab - name: Run Charon Container timeout-minutes: 3 diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index b2c34274..02c16518 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -89,6 +89,8 @@ jobs: build: name: Build Application runs-on: ubuntu-latest + outputs: + image_digest: ${{ steps.build-image.outputs.digest }} steps: - name: Checkout repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 @@ -120,6 +122,7 @@ jobs: uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Build Docker image + id: build-image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6 with: context: . @@ -152,6 +155,7 @@ jobs: # Enable security-focused endpoints and test gating CHARON_EMERGENCY_SERVER_ENABLED: "true" CHARON_SECURITY_TESTS_ENABLED: "true" + CHARON_E2E_IMAGE_DIGEST: ${{ needs.build.outputs.image_digest }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index c1281614..37f9153f 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -141,10 +141,15 @@ jobs: provenance: true sbom: true + - name: Record nightly image digest + run: | + echo "## ๐Ÿงพ Nightly Image Digest" >> $GITHUB_STEP_SUMMARY + echo "- ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:nightly@${{ steps.build.outputs.digest }}" >> $GITHUB_STEP_SUMMARY + - name: Generate SBOM uses: anchore/sbom-action@deef08a0db64bfad603422135db61477b16cef56 # v0.22.1 with: - image: ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:nightly + image: ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:nightly@${{ steps.build.outputs.digest }} format: cyclonedx-json output-file: sbom-nightly.json @@ -206,13 +211,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Pull nightly image - run: docker pull ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:nightly + run: docker pull ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:nightly@${{ needs.build-and-push-nightly.outputs.digest }} - name: Run container smoke test run: | docker run --name charon-nightly -d \ -p 8080:8080 \ - ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:nightly + ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:nightly@${{ needs.build-and-push-nightly.outputs.digest }} # Wait for container to start sleep 10 @@ -309,7 +314,7 @@ jobs: - name: Scan with Trivy uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # 0.33.1 with: - image-ref: ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}:nightly + image-ref: ${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build-and-push-nightly.outputs.digest }} format: 'sarif' output: 'trivy-nightly.sarif' diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 30d310b2..da89b729 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,7 +1,7 @@ # Charon System Architecture **Version:** 1.0 -**Last Updated:** January 28, 2026 +**Last Updated:** 2026-01-30 **Status:** Living Document --- @@ -1389,8 +1389,8 @@ docker exec charon /app/scripts/restore-backup.sh \ ### Known Issues 1. **GORM Struct Reuse:** - - Fixed in v1.2.0 (see `docs/plans/current_spec.md`) - - Prior versions had ID leakage in Settings queries + - Fixed in v1.2.0 (see [docs/implementation/gorm_security_scanner_complete.md](docs/implementation/gorm_security_scanner_complete.md)) + - Prior versions had ID leakage in Settings queries 2. **Docker Discovery:** - Requires `docker.sock` mount (security trade-off) diff --git a/Dockerfile b/Dockerfile index 6c850d40..c5132d24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,8 +108,10 @@ RUN xx-apt install -y gcc libc6-dev libsqlite3-dev # Install Delve (cross-compile for target) # Note: xx-go install puts binaries in /go/bin/TARGETOS_TARGETARCH/dlv if cross-compiling. # We find it and move it to /go/bin/dlv so it's in a consistent location for the next stage. +# renovate: datasource=go depName=github.com/go-delve/delve +ARG DLV_VERSION=1.26.0 # hadolint ignore=DL3059,DL4006 -RUN CGO_ENABLED=0 xx-go install github.com/go-delve/delve/cmd/dlv@latest && \ +RUN CGO_ENABLED=0 xx-go install github.com/go-delve/delve/cmd/dlv@v${DLV_VERSION} && \ DLV_PATH=$(find /go/bin -name dlv -type f | head -n 1) && \ if [ -n "$DLV_PATH" ] && [ "$DLV_PATH" != "/go/bin/dlv" ]; then \ mv "$DLV_PATH" /go/bin/dlv; \ @@ -164,12 +166,14 @@ FROM --platform=$BUILDPLATFORM golang:1.25-trixie@sha256:fb4b74a39c7318d53539ebd ARG TARGETOS ARG TARGETARCH ARG CADDY_VERSION +# renovate: datasource=go depName=github.com/caddyserver/xcaddy +ARG XCADDY_VERSION=0.4.5 RUN apt-get update && apt-get install -y --no-install-recommends git \ && rm -rf /var/lib/apt/lists/* # hadolint ignore=DL3062 RUN --mount=type=cache,target=/go/pkg/mod \ - go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest + go install github.com/caddyserver/xcaddy/cmd/xcaddy@v${XCADDY_VERSION} # Build Caddy for the target architecture with security plugins. # Two-stage approach: xcaddy generates go.mod, we patch it, then build from scratch. @@ -234,6 +238,8 @@ ARG TARGETARCH # CrowdSec version - Renovate can update this # renovate: datasource=github-releases depName=crowdsecurity/crowdsec ARG CROWDSEC_VERSION=1.7.6 +# CrowdSec fallback tarball checksum (v${CROWDSEC_VERSION}) +ARG CROWDSEC_RELEASE_SHA256=704e37121e7ac215991441cef0d8732e33fa3b1a2b2b88b53a0bfe5e38f863bd RUN apt-get update && apt-get install -y --no-install-recommends \ git clang lld \ @@ -288,6 +294,7 @@ ARG TARGETARCH # CrowdSec version - Renovate can update this # renovate: datasource=github-releases depName=crowdsecurity/crowdsec ARG CROWDSEC_VERSION=1.7.6 +ARG CROWDSEC_RELEASE_SHA256=704e37121e7ac215991441cef0d8732e33fa3b1a2b2b88b53a0bfe5e38f863bd # Note: Debian slim does NOT include tar by default - must be explicitly installed RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -303,6 +310,7 @@ RUN set -eux; \ echo "Downloading CrowdSec binaries for amd64 (fallback)..."; \ curl -fSL "https://github.com/crowdsecurity/crowdsec/releases/download/v${CROWDSEC_VERSION}/crowdsec-release.tgz" \ -o /tmp/crowdsec.tar.gz && \ + echo "${CROWDSEC_RELEASE_SHA256} /tmp/crowdsec.tar.gz" | sha256sum -c - && \ tar -xzf /tmp/crowdsec.tar.gz -C /tmp && \ cp "/tmp/crowdsec-v${CROWDSEC_VERSION}/cmd/crowdsec-cli/cscli" /crowdsec-out/bin/ && \ cp "/tmp/crowdsec-v${CROWDSEC_VERSION}/cmd/crowdsec/crowdsec" /crowdsec-out/bin/ && \ @@ -341,9 +349,11 @@ RUN groupadd -g 1000 charon && \ # Download MaxMind GeoLite2 Country database # Note: In production, users should provide their own MaxMind license key # This uses the publicly available GeoLite2 database +ARG GEOLITE2_COUNTRY_SHA256=6b778471c086c44d15bd4df954661d441a5513ec48f1af5545cb05af8f2e15b9 RUN mkdir -p /app/data/geoip && \ - curl -L "https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-Country.mmdb" \ - -o /app/data/geoip/GeoLite2-Country.mmdb + curl -fSL "https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-Country.mmdb" \ + -o /app/data/geoip/GeoLite2-Country.mmdb && \ + echo "${GEOLITE2_COUNTRY_SHA256} /app/data/geoip/GeoLite2-Country.mmdb" | sha256sum -c - # Copy Caddy binary from caddy-builder (overwriting the one from base image) COPY --from=caddy-builder /usr/bin/caddy /usr/bin/caddy diff --git a/SECURITY.md b/SECURITY.md index b83139cc..aacabdf3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -279,6 +279,39 @@ Integrate supply chain verification into your deployment pipeline: - **Build Process**: SLSA Level 3 compliant build provenance - **Dependencies**: Complete SBOM including all direct and transitive dependencies +### Digest Pinning Policy + +Charon uses digest pinning to reduce supply chain risk and ensure CI runs against immutable artifacts. + +**Scope (Required):** + +- **CI workflows**: `.github/workflows/*.yml`, `.github/workflows/*.yaml` +- **CI compose files**: `.docker/compose/*.yml`, `.docker/compose/*.yaml`, `.docker/compose/docker-compose*.yml`, `.docker/compose/docker-compose*.yaml` +- **CI helper actions with container refs**: `.github/actions/**/*.yml`, `.github/actions/**/*.yaml` +- CI workflows and CI compose files MUST use digest-pinned images for third-party services. +- Tag+digest pairs are preferred for human-readable references with immutable resolution. +- Self-built images MUST propagate digests to downstream jobs and tests. + +**Rationale:** + +- Prevent tag drift and supply chain substitution in automated runs. +- Ensure deterministic builds, reproducible scans, and stable SBOM generation. +- Reduce rollback risk by guaranteeing CI uses immutable artifacts. + +**Local Development Exceptions:** + +- Local-only overrides (e.g., `CHARON_E2E_IMAGE`, `CHARON_IMAGE`, `CHARON_DEV_IMAGE`) MAY use tags for developer iteration. +- Tag-only overrides MUST NOT be used in CI contexts. + +**Documented Exceptions & Compensating Controls:** + +1. **Go toolchain shim** (`golang.org/dl/goX.Y.Z@latest`) + - **Exception:** Uses `@latest` to install the shim. + - **Compensating controls:** The target toolchain version is pinned in `go.work`, and Renovate tracks the required version for updates. +2. **Unpinnable dependencies** (no stable digest or checksum source) + - **Exception:** Dependency cannot be pinned by digest. + - **Compensating controls:** Require documented justification, prefer vendor-provided checksums or signed releases when available, and keep SBOM/vulnerability scans in CI. + ### Learn More - **[User Guide](docs/guides/supply-chain-security-user-guide.md)**: Step-by-step verification instructions @@ -477,5 +510,5 @@ This security policy is part of the Charon project, licensed under the MIT Licen --- -**Last Updated**: December 31, 2025 +**Last Updated**: January 30, 2026 **Version**: 1.2 diff --git a/categories.txt b/categories.txt new file mode 100644 index 00000000..cf4ffc46 --- /dev/null +++ b/categories.txt @@ -0,0 +1,4 @@ +actions +ci +security +testing diff --git a/docs/plans/current_spec.md b/docs/plans/current_spec.md index 5e8d763f..9907cb84 100644 --- a/docs/plans/current_spec.md +++ b/docs/plans/current_spec.md @@ -1,60 +1,348 @@ -# Reddit Feedback Implementation Plan: Logs UI, Caddy Import, Settings 400 Errors +# Dependency Digest Tracking Plan: Nightly Build Supply-Chain Hardening **Version:** 1.0 -**Status:** Research Complete - Ready for Implementation +**Status:** Research Complete - Phase 2 In Progress **Priority:** HIGH -**Created:** 2026-01-29 -**Source:** Reddit user feedback - -> **Note:** Previous active plan (E2E Test Architecture Fix) archived to [e2e_architecture_port80_spec.md](./e2e_architecture_port80_spec.md) - ---- - -## Active Plan - -See **[reddit_feedback_spec.md](./reddit_feedback_spec.md)** for the complete specification. - ---- - -## Quick Reference - -### Three Issues Addressed - -1. **Logs UI on widescreen** - Fixed `h-96` height, multi-span entries -2. **Caddy import not working** - Silent skipping, cryptic errors -3. **Settings 400 errors** - CIDR/URL validation, unfriendly messages - -### Key Files - -| Issue | Primary File | Line | -|-------|-------------|------| -| Logs UI | `frontend/src/components/LiveLogViewer.tsx` | 435 | -| Import | `backend/internal/api/handlers/import_handler.go` | 297 | -| Settings | `backend/internal/api/handlers/settings_handler.go` | 84 | - -### Implementation Timeline - -- **Day 1:** Quick wins (responsive height, error messages, normalization) -- **Day 2:** Core features (compact mode, skipped hosts, validation) -- **Day 3:** Polish (density control, import directive UI, inline validation) +**Created:** 2026-01-30 +**Source:** Nightly build readiness review --- ## Executive Summary -Three user-reported issues from Reddit: -1. **Logs UI** - Fixed height wastes screen space, entries wrap across multiple lines -2. **Caddy Import** - Silent failures, cryptic errors, missing feedback on skipped sites -3. **Settings 400** - Validation errors not user-friendly, missing auto-correction - -**Root Causes Identified:** -- LiveLogViewer uses `h-96` fixed height, multi-span entries -- Import handler silently skips hosts without `reverse_proxy` -- Settings handler returns raw Go validation errors - -**Solution:** Responsive UI, enhanced error messages, input normalization +The nightly build pipeline is wired and waiting; now the supply chain needs a sharper edge. This plan catalogs every dependency used by the nightly workflow and its supporting build paths, highlights those not tracked by digest or checksum, and lays out a phased strategy to lock them down. The objective is simple: when the nightly build wakes up, it should pull only what we intendedโ€”no silent drift, no invisible updates, and no mystery bytes. --- -*For full specification, see [reddit_feedback_spec.md](./reddit_feedback_spec.md)* -*Previous E2E plan archived to [e2e_architecture_port80_spec.md](./e2e_architecture_port80_spec.md)* +## Goals + +1. **Digest-Tracked Dependencies**: Ensure all container images and external artifacts used in nightly build paths are pinned by digest or verified by checksum. +2. **Repeatable Nightly Builds**: Make the nightly build reproducible by eliminating unpinned tags and `@latest` installs. +3. **Clear Ownership**: Centralize digest updates via Renovate where feasible. +4. **Minimal Change Surface**: Only adjust files necessary for dependency integrity. + +## Non-Goals + +- Redesigning the nightly workflow logic. +- Changing release tagging or publishing conventions. +- Reworking the Docker build pipeline beyond dependency pinning. + +--- + +## Research Inventory (Current State) + +### Workflows + +- Nightly workflow: [.github/workflows/nightly-build.yml](.github/workflows/nightly-build.yml) +- Docker build workflow: [.github/workflows/docker-build.yml](.github/workflows/docker-build.yml) +- Playwright workflow (nightly test support): [.github/workflows/playwright.yml](.github/workflows/playwright.yml) + +### Docker & Compose + +- Runtime image build: [Dockerfile](Dockerfile) +- Compose (E2E CI): [.docker/compose/docker-compose.playwright-ci.yml](.docker/compose/docker-compose.playwright-ci.yml) +- Compose (primary): [.docker/compose/docker-compose.yml](.docker/compose/docker-compose.yml) +- Compose (dev): [.docker/compose/docker-compose.dev.yml](.docker/compose/docker-compose.dev.yml) +- Compose (remote): [.docker/compose/docker-compose.remote.yml](.docker/compose/docker-compose.remote.yml) + +### Scripts & Tooling + +- Security scan helper: [scripts/security-scan.sh](scripts/security-scan.sh) +- Local Go installer: [scripts/install-go-1.25.6.sh](scripts/install-go-1.25.6.sh) +- Go version updater skill: [.github/skills/utility-update-go-version-scripts/run.sh](.github/skills/utility-update-go-version-scripts/run.sh) +- Renovate rules: [.github/renovate.json](.github/renovate.json) + +--- + +## Findings: Dependencies Not Yet Tracked by Digest/Checksum + +### Dependency Table (Phase 1 Requirement) + +| File path | Dependency | Current pin state | Target pin method | +| --- | --- | --- | --- | +| .docker/compose/docker-compose.playwright-ci.yml | crowdsecurity/crowdsec:latest | Tag `latest` | Tag + digest (Renovate-managed) | +| .docker/compose/docker-compose.playwright-ci.yml | mailhog/mailhog:latest | Tag `latest` | Tag + digest (Renovate-managed) | +| .docker/compose/docker-compose.playwright-ci.yml | CHARON_E2E_IMAGE (charon:e2e-test) | Tag only | Default to workflow digest output; allow tag override | +| .docker/compose/docker-compose.remote.yml | alpine/socat | Tagless (defaults to latest) | Tag + digest (Renovate-managed) | +| .docker/compose/docker-compose.yml | ghcr.io/wikid82/charon:latest | Tag `latest` | Tag + digest, allow local override | +| .docker/compose/docker-compose.dev.yml | ghcr.io/wikid82/charon:dev | Tag only | Tag + digest, allow local override | +| .github/workflows/docker-build.yml | traefik/whoami | Tagless (defaults to latest) | Tag + digest (Renovate-managed) | +| Dockerfile (backend-builder) | dlv@latest | Go tool `@latest` | Pinned version (Renovate-managed) | +| Dockerfile (caddy-builder) | xcaddy@latest | Go tool `@latest` | Pinned version (Renovate-managed) | +| Dockerfile (crowdsec-fallback) | crowdsec-release.tgz | No checksum | SHA256 verification | +| Dockerfile (final runtime) | GeoLite2-Country.mmdb | No checksum | SHA256 verification | +| scripts/security-scan.sh | govulncheck@latest | Go tool `@latest` | Pinned version (Renovate-managed) | +| scripts/install-go-1.25.6.sh | gopls@latest | Go tool `@latest` | Pinned version (Renovate-managed) | +| .github/skills/utility-update-go-version-scripts/run.sh | golang.org/dl/go${REQUIRED_VERSION}@latest | Allowed exception | Exception + compensating controls | + +### A. Container Images (Compose & Workflows) + +1. **E2E Playwright Compose** + - File: [.docker/compose/docker-compose.playwright-ci.yml](.docker/compose/docker-compose.playwright-ci.yml) + - Images: + - `crowdsecurity/crowdsec:latest` + - `mailhog/mailhog:latest` + - `CHARON_E2E_IMAGE_DIGEST` from workflow output (default) + - `CHARON_E2E_IMAGE` tag override for local runs +2. **Remote Docker socket proxy** + - File: [.docker/compose/docker-compose.remote.yml](.docker/compose/docker-compose.remote.yml) + - Image: `alpine/socat` +3. **Dev and prod compose images** + - File: [.docker/compose/docker-compose.yml](.docker/compose/docker-compose.yml) + - Image: `ghcr.io/wikid82/charon:latest` + - File: [.docker/compose/docker-compose.dev.yml](.docker/compose/docker-compose.dev.yml) + - Image: `ghcr.io/wikid82/charon:dev` +4. **Workflow test service image** + - File: [.github/workflows/docker-build.yml](.github/workflows/docker-build.yml) + - Image: `traefik/whoami` (tagless, latest by default) + +### B. Dockerfile External Downloads & Unpinned Go Installs + +1. **Go tools installed with @latest** + - Stage: `backend-builder` + - File: [Dockerfile](Dockerfile) + - Tool: `github.com/go-delve/delve/cmd/dlv@latest` +2. **Caddy builder uses @latest for xcaddy** + - Stage: `caddy-builder` + - File: [Dockerfile](Dockerfile) + - Tool: `github.com/caddyserver/xcaddy/cmd/xcaddy@latest` +3. **CrowdSec fallback download without checksum** + - Stage: `crowdsec-fallback` + - File: [Dockerfile](Dockerfile) + - Artifact: `crowdsec-release.tgz` (no sha256 verification) +4. **GeoLite2 database download without checksum** + - Stage: final runtime + - File: [Dockerfile](Dockerfile) + - Artifact: `GeoLite2-Country.mmdb` (raw GitHub download) + +### C. Scripts Installing Go Tools with @latest + +1. [scripts/security-scan.sh](scripts/security-scan.sh) + - `golang.org/x/vuln/cmd/govulncheck@latest` +2. [scripts/install-go-1.25.6.sh](scripts/install-go-1.25.6.sh) + - `golang.org/x/tools/gopls@latest` +3. [.github/skills/utility-update-go-version-scripts/run.sh](.github/skills/utility-update-go-version-scripts/run.sh) + - `golang.org/dl/go${REQUIRED_VERSION}@latest` + - **Exception candidate:** Go toolchain installer (requires `@latest` for versioned shim) + +--- + +## Requirements (EARS Notation) + +1. WHEN the nightly workflow executes, THE SYSTEM SHALL use container images pinned by digest for any external service images it runs (e.g., `traefik/whoami`). +2. WHEN a Docker Compose file is used in CI contexts, THE SYSTEM SHALL pin all third-party images by digest or provide a checksum verification step. +3. WHEN the Dockerfile downloads external artifacts, THE SYSTEM SHALL verify them with checksums or pinned release asset digests. +4. WHEN Go tools are installed in build stages or scripts, THE SYSTEM SHALL pin a specific semantic version instead of `@latest`. +5. WHEN Renovate is configured, THE SYSTEM SHALL be able to update pinned digests and versioned tool installs without manual drift. +6. IF a dependency cannot be pinned by digest (e.g., variable build outputs), THEN THE SYSTEM SHALL document the exception and the compensating control (checksum, SBOM, or provenance). +7. WHEN the Go toolchain shim is installed via `golang.org/dl/goX.Y.Z@latest`, THE SYSTEM SHALL allow this as an explicit exception and SHALL enforce compensating controls (pinned `goX.Y.Z`, checksum or provenance validation for the installed toolchain, and Renovate visibility). +8. WHEN CI builds a self-hosted image, THE SYSTEM SHALL capture the resulting digest and propagate it to downstream jobs and tests as an immutable reference. + +--- + +## Design Decisions (Draft) + +1. **Digest Pinning Strategy** + - Use `image: name:tag@sha256:...` for compose and workflow `docker run` usage when possible. + - For the self-built nightly image, keep the tag for readability but capture and propagate the digest to downstream verification steps. + - Use tag+digest pairs consistently to preserve human-readable tags while enforcing immutability. +2. **Checksum Verification for Artifacts** + - Add `ARG` + `SHA256` environment variables for CrowdSec tarball and GeoLite2 DB. + - Verify downloads in Dockerfile with `sha256sum -c`. + - GeoLite2 checksum provenance: prefer MaxMind-provided SHA256 from the official GeoLite2 download API (license-key gated) and document the applicable GeoLite2 EULA/licensing source. +3. **Version Pinning for Go Tools** + - Replace `@latest` installs with pinned versions and Renovate annotations. +4. **Exception: `golang.org/dl/goX.Y.Z@latest`** + - Allow the go toolchain shim to use `@latest` for the specific `goX.Y.Z` target version. + - Compensating controls: ensure `REQUIRED_VERSION` is pinned, verify the resulting toolchain provenance (Go checksum database or release manifest), and add Renovate monitoring for `REQUIRED_VERSION` updates. + +--- + +## Planned Updates (Files & Components) + +### Workflows + +1. **Nightly Build** + - File: [.github/workflows/nightly-build.yml](.github/workflows/nightly-build.yml) + - Component: `test-nightly-image` job + - Capture the nightly image digest from the build step and export it as a job output (e.g., `nightly_image_digest`). + - Propagate the digest to downstream jobs via `needs..outputs.nightly_image_digest` and use `image: tag@sha256:...` where possible. + - Record the tag+digest pair in job summary for auditability. + +2. **Docker Build Workflow** + - File: [.github/workflows/docker-build.yml](.github/workflows/docker-build.yml) + - Component: `Run Upstream Service (whoami)` step + - Replace `traefik/whoami` with `traefik/whoami:tag@sha256:...` and document digest ownership. + - Capture the built image digest from buildx output (or `docker buildx imagetools inspect`) and expose it as a workflow output for reuse in later jobs. + +### Dockerfile + +1. **Stage: backend-builder** + - Replace `dlv@latest` with a pinned version (e.g., `@v1.x.y`) tracked by Renovate. +2. **Stage: caddy-builder** + - Replace `xcaddy@latest` with pinned version; add Renovate directive. +3. **Stage: crowdsec-fallback** + - Add checksum verification for `crowdsec-release.tgz` using `sha256sum`. +4. **Stage: final runtime** + - Add checksum verification for GeoLite2 DB, preferably from a fixed release artifact or vendor checksum list. + - Document GeoLite2 checksum provenance in the Dockerfile or plan (MaxMind GeoLite2 download API + EULA source). + +### Compose Files + +1. **E2E CI Compose** + - File: [.docker/compose/docker-compose.playwright-ci.yml](.docker/compose/docker-compose.playwright-ci.yml) + - Pin `crowdsecurity/crowdsec`, `mailhog/mailhog` by digest. + - Default to `CHARON_E2E_IMAGE_DIGEST` from workflow outputs with `CHARON_E2E_IMAGE` tag override for local runs. +2. **Remote Socket Proxy** + - File: [.docker/compose/docker-compose.remote.yml](.docker/compose/docker-compose.remote.yml) + - Pin `alpine/socat` by digest. +3. **Dev & Prod Compose** + - File: [.docker/compose/docker-compose.yml](.docker/compose/docker-compose.yml) + - File: [.docker/compose/docker-compose.dev.yml](.docker/compose/docker-compose.dev.yml) + - Decide whether to: + - Keep tags for local convenience, OR + - Provide commented tag+digest options and Renovate-managed examples. + +### Renovate Configuration + +1. **Enable Digest Pinning for Docker Compose** + - File: [.github/renovate.json](.github/renovate.json) + - Ensure docker digest pinning is enabled for compose images and tag+digest pairs are preserved. +2. **Add Custom Managers for Go Tools** + - Track pinned versions for `dlv` and `xcaddy` in Dockerfile. + - Track `REQUIRED_VERSION` for `golang.org/dl/goX.Y.Z@latest` exception to keep the target version current. + +--- + +## Review Notes for Supporting Files + +1. **.gitignore** + - No immediate changes required. If a new dependency lock manifest is introduced (e.g., `dependency-digests.json`), ensure it is not ignored. +2. **.dockerignore** + - No blocking issues found. Consider excluding any new digest manifest artifacts only if they are not required in image builds. +3. **codecov.yml** + - No changes required for dependency tracking. Coverage ignore patterns are acceptable for this effort. +4. **Dockerfile** + - Changes required (pin `@latest` tools, verify external downloads with checksums). + +--- + +## Risks & Mitigations + +1. **Digest Rotation** + - Risk: pinned digests require updates. + - Mitigation: Renovate updates digests on schedule. +2. **Checksum Source Reliability** + - Risk: upstream artifacts lack stable checksum URLs. + - Mitigation: use release checksums or vendor-provided signed assets; document exceptions. +3. **Local Developer Friction** + - Risk: digest pinning may slow dev iteration. + - Mitigation: keep optional tag paths or override vars for local use. + +--- + +## Implementation Plan (Phased, Minimal Requests) + +### Phase 1 โ€” Inventory & Decision Map (Single Request) + +**Objective:** Establish the canonical list of digest-tracked dependencies and confirm which files will be modified. + +**Status:** Complete (dependency table added; dev/prod compose pinning decision set) + +**Actions:** +- Create a dependency table in `docs/plans/current_spec.md` (this file) with: + - File path + - Dependency name + - Current pin state (tag, digest, checksum, latest) + - Target pin method +- Decide whether dev compose files are pinned or left flexible with documented overrides. + - **Owner:** DevOps + - **Decision Date:** 2026-01-30 + - **Decision:** Pin dev/prod compose images with tag+digest defaults while allowing local overrides via env vars. + +**Deliverables:** +- Finalized dependency inventory and pinning policy. + +### Phase 2 โ€” Pinning & Verification Updates (Single Request) + +**Objective:** Apply digest pinning, version pinning, and checksum verification changes across build and CI surfaces. + +**Actions:** +- Update Dockerfile stages: + - Pin `dlv` and `xcaddy` versions. + - Add checksum verification for GeoLite2 and CrowdSec tarball. +- Update compose images to digest form where required. +- Update workflow `docker run` test image to digest form. +- Update Renovate config to keep digests and Go tool versions fresh. + +**Deliverables:** +- All dependencies in nightly path pinned or checksum-verified. + +### Phase 3 โ€” Validation & Guardrails (Single Request) + +**Objective:** Ensure policy compliance and prevent regression. + +**Actions:** +- Add documentation in `docs/` or `SECURITY.md` describing digest policy. +- Verify SBOM generation still succeeds with pinned dependencies. + - Add a lint check (required) to detect unpinned tags and `@latest` in CI-critical files. + - Scope files: + - `.github/workflows/*.yml` + - `.docker/compose/*.yml` + - `Dockerfile` + - `scripts/*.sh` + - Patterns to flag (non-exhaustive): + - `:latest` image tags (except explicitly documented local-only compose examples) + - `@latest` in Go tool installs (except `golang.org/dl/goX.Y.Z@latest`) + - Docker image references lacking `@sha256:` in CI/test contexts + +**Deliverables:** +- Policy documentation and validation evidence. + +--- + +## Acceptance Criteria + +1. All external images referenced by CI workflows or CI compose files are pinned by digest. +2. All Dockerfile external downloads are checksum-verified. +3. No `@latest` installs remain in Dockerfile or CI-critical scripts without explicit exception. +4. The Go toolchain shim exception is documented with compensating controls and Renovate visibility. +5. CI workflows capture and propagate self-built image digests for downstream usage. +6. Renovate can update digests and pinned tool versions automatically. +7. Documentation clearly states which files must use digests and why. + +--- + +## Handoff Contract (JSON) + +```json +{ + "plan": "Dependency Digest Tracking Plan: Nightly Build Supply-Chain Hardening", + "phase": "Phase 1 โ€” Inventory & Decision Map", + "status": "In Progress", + "owner": "DevOps", + "handoffTargets": ["Backend_Dev", "DevOps", "QA_Security"], + "decisionRequired": "Dev compose pinning policy", + "decisionDate": "2026-01-30", + "dependencies": [ + ".github/workflows/nightly-build.yml", + ".github/workflows/docker-build.yml", + ".docker/compose/docker-compose.playwright-ci.yml", + ".docker/compose/docker-compose.yml", + ".docker/compose/docker-compose.dev.yml", + ".docker/compose/docker-compose.remote.yml", + "Dockerfile", + ".github/renovate.json", + "scripts/security-scan.sh", + "scripts/install-go-1.25.6.sh", + ".github/skills/utility-update-go-version-scripts/run.sh" + ], + "notes": "Digest pinning and checksum verification must align with Acceptance Criteria and Renovate ownership." +} +``` + +--- + +## Handoff Notes + +Once this plan is accepted, delegate implementation to `DevOps` and `Backend_Dev` for Dockerfile and workflow changes, and `QA_Security` for validation and policy checks. diff --git a/docs/plans/design.md b/docs/plans/design.md new file mode 100644 index 00000000..1113a5dd --- /dev/null +++ b/docs/plans/design.md @@ -0,0 +1,32 @@ +# Design - Dependency Digest Tracking Plan + +## Architecture Overview + +This change set hardens the nightly build and CI surfaces by pinning container images to digests, pinning Go tool installs to fixed versions, and verifying external artifact downloads with SHA256 checksums. + +## Data Flow + +1. Build workflows produce an image digest via buildx and expose it as a job output. +2. Downstream jobs and tests consume the digest to pull and run immutable images. +3. CI compose files reference third-party images as `name:tag@sha256:digest`. +4. Dockerfile download steps verify artifacts using SHA256 checksums before extraction. + +## Interfaces + +- GitHub Actions job outputs: + - `build-and-push-nightly.outputs.digest` +- Compose overrides: + - `CHARON_E2E_IMAGE_DIGEST` (preferred, digest-pinned from workflow output) + - `CHARON_E2E_IMAGE` (tag-based local override) + - `CHARON_IMAGE`, `CHARON_DEV_IMAGE` (local override for tag-only usage) + +## Error Handling + +- Dockerfile checksum verification uses `sha256sum -c` to fail fast on mismatches. +- CI workflows rely on digest references; failure to resolve a digest fails the job early. + +## Implementation Considerations + +- Tag+digest pairs preserve human-readable tags while enforcing immutability. +- Renovate regex managers track pinned versions for Go tools and go.work toolchain version. +- The Go toolchain shim uses `@latest` by exception and reads the pinned version from go.work. diff --git a/docs/plans/requirements.md b/docs/plans/requirements.md new file mode 100644 index 00000000..c03204b9 --- /dev/null +++ b/docs/plans/requirements.md @@ -0,0 +1,13 @@ +# Requirements - Dependency Digest Tracking Plan + +## EARS Requirements + +1. WHEN the nightly workflow executes, THE SYSTEM SHALL use container images pinned by digest for any external service images it runs. +2. WHEN a Docker Compose file is used in CI contexts, THE SYSTEM SHALL pin all third-party images by digest or provide a checksum verification step. +3. WHEN the Dockerfile downloads external artifacts, THE SYSTEM SHALL verify them with checksums. +4. WHEN Go tools are installed in build stages or scripts, THE SYSTEM SHALL pin a specific semantic version instead of `@latest`. +5. WHEN Renovate is configured, THE SYSTEM SHALL be able to update pinned digests and versioned tool installs without manual drift. +6. IF a dependency cannot be pinned by digest, THEN THE SYSTEM SHALL document the exception and compensating controls. +7. WHEN the Go toolchain shim is installed via `golang.org/dl/goX.Y.Z@latest`, THE SYSTEM SHALL allow this as an explicit exception and SHALL enforce compensating controls. +8. WHEN CI builds a self-hosted image, THE SYSTEM SHALL capture the resulting digest and propagate it to downstream jobs and tests. +9. WHEN CI starts the E2E compose stack, THE SYSTEM SHALL default to a digest-pinned image from workflow outputs while allowing a tag override for local runs. diff --git a/docs/plans/tasks.md b/docs/plans/tasks.md new file mode 100644 index 00000000..176e4da8 --- /dev/null +++ b/docs/plans/tasks.md @@ -0,0 +1,18 @@ +# Tasks - Dependency Digest Tracking Plan + +## Phase 2 - Pinning & Verification Updates + +- [x] Pin `dlv` and `xcaddy` versions in Dockerfile. +- [x] Add checksum verification for CrowdSec fallback tarball. +- [x] Add checksum verification for GeoLite2 database download. +- [x] Pin CI compose images by digest. +- [x] Default Playwright CI compose to workflow digest output with tag override for local runs. +- [x] Pin whoami test service image by digest in docker-build workflow. +- [x] Propagate nightly image digest to smoke tests and scans. +- [x] Pin `govulncheck` and `gopls` versions in scripts. +- [x] Add Renovate regex managers for pinned tool versions and go.work. + +## Follow-ups + +- [ ] Add policy linting to detect unpinned tags in CI-critical files. +- [ ] Update security documentation for digest policy and exceptions. diff --git a/scripts/install-go-1.25.6.sh b/scripts/install-go-1.25.6.sh index b075d366..c9c467b7 100755 --- a/scripts/install-go-1.25.6.sh +++ b/scripts/install-go-1.25.6.sh @@ -43,7 +43,8 @@ echo "Installed go: $(go version)" # Optionally install gopls echo "Installing gopls..." -go install golang.org/x/tools/gopls@latest +# renovate: datasource=go depName=golang.org/x/tools +go install golang.org/x/tools/gopls@v0.41.0 GOPLS_PATH="$GOPATH/bin/gopls" if [ -f "$GOPLS_PATH" ]; then diff --git a/scripts/security-scan.sh b/scripts/security-scan.sh index ccb928e7..046abdaf 100755 --- a/scripts/security-scan.sh +++ b/scripts/security-scan.sh @@ -19,7 +19,8 @@ echo "๐Ÿ”’ Running local security scan..." # Check if govulncheck is installed if ! command -v govulncheck &> /dev/null; then echo -e "${YELLOW}Installing govulncheck...${NC}" - go install golang.org/x/vuln/cmd/govulncheck@latest + # renovate: datasource=go depName=golang.org/x/vuln + go install golang.org/x/vuln/cmd/govulncheck@v1.1.4 fi # Run govulncheck on backend Go code