feat: add HUB_BASE_URL configuration and enhance CrowdSec hub sync functionality with error handling and tests

backend/.env.example

@@ -3,6 +3,8 @@ CHARON_HTTP_PORT=8080
 CHARON_DB_PATH=./data/charon.db
 CHARON_CADDY_ADMIN_API=http://localhost:2019
 CHARON_CADDY_CONFIG_DIR=./data/caddy
+# HUB_BASE_URL overrides the CrowdSec hub endpoint used when cscli is unavailable (defaults to https://hub-data.crowdsec.net)
+# HUB_BASE_URL=https://hub-data.crowdsec.net
 CERBERUS_SECURITY_CERBERUS_ENABLED=false
 CHARON_SECURITY_CERBERUS_ENABLED=false
 CPM_SECURITY_CERBERUS_ENABLED=false

backend/internal/api/handlers/crowdsec_presets_handler_test.go

@@ -273,12 +273,13 @@ func TestApplyPresetHandlerBackupFailure(t *testing.T) {
 	r.ServeHTTP(w, req)
 
 	require.Equal(t, http.StatusInternalServerError, w.Code)
+	require.Contains(t, w.Body.String(), "cscli unavailable")
 
 	var events []models.CrowdsecPresetEvent
 	require.NoError(t, db.Find(&events).Error)
 	require.Len(t, events, 1)
 	require.Equal(t, "failed", events[0].Status)
-	require.NotEmpty(t, events[0].BackupPath)
+	require.Empty(t, events[0].BackupPath)
 
 	content, readErr := os.ReadFile(filepath.Join(dataDir, "keep.txt"))
 	require.NoError(t, readErr)

backend/internal/api/middleware/sanitize_test.go

@@ -0,0 +1,55 @@
+package middleware
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSanitizeHeaders(t *testing.T) {
+	t.Run("nil headers", func(t *testing.T) {
+		require.Nil(t, SanitizeHeaders(nil))
+	})
+
+	t.Run("redacts sensitive headers", func(t *testing.T) {
+		headers := http.Header{}
+		headers.Set("Authorization", "secret")
+		headers.Set("X-Api-Key", "token")
+		headers.Set("Cookie", "sessionid=abc")
+
+		sanitized := SanitizeHeaders(headers)
+
+		require.Equal(t, []string{"<redacted>"}, sanitized["Authorization"])
+		require.Equal(t, []string{"<redacted>"}, sanitized["X-Api-Key"])
+		require.Equal(t, []string{"<redacted>"}, sanitized["Cookie"])
+	})
+
+	t.Run("sanitizes and truncates values", func(t *testing.T) {
+		headers := http.Header{}
+		headers.Add("X-Trace", "line1\nline2\r\t")
+		headers.Add("X-Custom", strings.Repeat("a", 210))
+
+		sanitized := SanitizeHeaders(headers)
+
+		traceValue := sanitized["X-Trace"][0]
+		require.NotContains(t, traceValue, "\n")
+		require.NotContains(t, traceValue, "\r")
+		require.NotContains(t, traceValue, "\t")
+
+		customValue := sanitized["X-Custom"][0]
+		require.Equal(t, 200, len(customValue))
+		require.True(t, strings.HasPrefix(customValue, strings.Repeat("a", 200)))
+	})
+}
+
+func TestSanitizePath(t *testing.T) {
+	paddedPath := "/api/v1/resource/" + strings.Repeat("x", 210) + "?token=secret"
+
+	sanitized := SanitizePath(paddedPath)
+
+	require.NotContains(t, sanitized, "?")
+	require.False(t, strings.ContainsAny(sanitized, "\n\r\t"))
+	require.Equal(t, 200, len(sanitized))
+}

backend/internal/crowdsec/hub_sync.go

@@ -24,6 +24,7 @@ type CommandExecutor interface {
 }
 
 const (
+	defaultHubBaseURL     = "https://hub-data.crowdsec.net"
 	defaultHubIndexPath   = "/api/index.json"
 	defaultHubArchivePath = "/%s.tgz"
 	defaultHubPreviewPath = "/%s.yaml"
@@ -77,17 +78,35 @@ type HubService struct {
 
 // NewHubService constructs a HubService with sane defaults.
 func NewHubService(exec CommandExecutor, cache *HubCache, dataDir string) *HubService {
+	clientTimeout := 10 * time.Second
 	return &HubService{
 		Exec:         exec,
 		Cache:        cache,
 		DataDir:      dataDir,
-		HTTPClient:   &http.Client{Timeout: 10 * time.Second},
-		HubBaseURL:   "https://hub.crowdsec.net",
-		PullTimeout:  10 * time.Second,
+		HTTPClient:   newHubHTTPClient(clientTimeout),
+		HubBaseURL:   normalizeHubBaseURL(os.Getenv("HUB_BASE_URL")),
+		PullTimeout:  clientTimeout,
 		ApplyTimeout: 15 * time.Second,
 	}
 }
 
+func newHubHTTPClient(timeout time.Duration) *http.Client {
+	return &http.Client{
+		Timeout: timeout,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+}
+
+func normalizeHubBaseURL(raw string) string {
+	trimmed := strings.TrimSpace(raw)
+	if trimmed == "" {
+		return defaultHubBaseURL
+	}
+	return strings.TrimRight(trimmed, "/")
+}
+
 // FetchIndex downloads the hub index. If the hub is unreachable, returns ErrCacheMiss.
 func (s *HubService) FetchIndex(ctx context.Context) (HubIndex, error) {
 	if s.Exec != nil {
@@ -178,20 +197,39 @@ func (s *HubService) fetchIndexHTTP(ctx context.Context) (HubIndex, error) {
 	if s.HTTPClient == nil {
 		return HubIndex{}, fmt.Errorf("http client missing")
 	}
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.HubBaseURL+defaultHubIndexPath, nil)
+	target := strings.TrimRight(s.HubBaseURL, "/") + defaultHubIndexPath
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
 	if err != nil {
 		return HubIndex{}, err
 	}
 	resp, err := s.HTTPClient.Do(req)
 	if err != nil {
-		return HubIndex{}, err
+		return HubIndex{}, fmt.Errorf("fetch hub index: %w", err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return HubIndex{}, fmt.Errorf("hub index status %d", resp.StatusCode)
+		if resp.StatusCode >= 300 && resp.StatusCode < 400 {
+			loc := resp.Header.Get("Location")
+			return HubIndex{}, fmt.Errorf("hub index redirect (%d) to %s; install cscli or set HUB_BASE_URL to a JSON hub endpoint", resp.StatusCode, firstNonEmpty(loc, target))
+		}
+		return HubIndex{}, fmt.Errorf("hub index status %d from %s", resp.StatusCode, target)
+	}
+	data, err := io.ReadAll(io.LimitReader(resp.Body, maxArchiveSize))
+	if err != nil {
+		return HubIndex{}, fmt.Errorf("read hub index: %w", err)
+	}
+	ct := strings.ToLower(resp.Header.Get("Content-Type"))
+	if ct != "" && !strings.Contains(ct, "application/json") {
+		if isLikelyHTML(data) {
+			return HubIndex{}, fmt.Errorf("hub index responded with HTML; install cscli or set HUB_BASE_URL to a JSON hub endpoint")
+		}
+		return HubIndex{}, fmt.Errorf("unexpected hub content-type %s; install cscli or set HUB_BASE_URL to a JSON hub endpoint", ct)
 	}
 	var idx HubIndex
-	if err := json.NewDecoder(resp.Body).Decode(&idx); err != nil {
+	if err := json.Unmarshal(data, &idx); err != nil {
+		if isLikelyHTML(data) {
+			return HubIndex{}, fmt.Errorf("hub index responded with HTML; install cscli or set HUB_BASE_URL to a JSON hub endpoint")
+		}
 		return HubIndex{}, fmt.Errorf("decode hub index: %w", err)
 	}
 	return idx, nil
@@ -234,11 +272,11 @@ func (s *HubService) Pull(ctx context.Context, slug string) (PullResult, error)
 
 	archiveURL := entry.DownloadURL
 	if archiveURL == "" {
-		archiveURL = fmt.Sprintf(s.HubBaseURL+defaultHubArchivePath, cleanSlug)
+		archiveURL = fmt.Sprintf(strings.TrimRight(s.HubBaseURL, "/")+defaultHubArchivePath, cleanSlug)
 	}
 	previewURL := entry.PreviewURL
 	if previewURL == "" {
-		previewURL = fmt.Sprintf(s.HubBaseURL+defaultHubPreviewPath, cleanSlug)
+		previewURL = fmt.Sprintf(strings.TrimRight(s.HubBaseURL, "/")+defaultHubPreviewPath, cleanSlug)
 	}
 
 	archiveBytes, err := s.fetchWithLimit(pullCtx, archiveURL)
@@ -266,37 +304,46 @@ func (s *HubService) Apply(ctx context.Context, slug string) (ApplyResult, error
 	if cleanSlug == "" {
 		return ApplyResult{}, fmt.Errorf("invalid slug")
 	}
+	applyCtx, cancel := context.WithTimeout(ctx, s.ApplyTimeout)
+	defer cancel()
+
+	result := ApplyResult{AppliedPreset: cleanSlug, Status: "failed"}
+	meta, metaErr := s.loadCacheMeta(applyCtx, cleanSlug)
+	if metaErr == nil {
+		result.CacheKey = meta.CacheKey
+	}
+	hasCS := s.hasCSCLI(applyCtx)
+	if !hasCS && metaErr != nil {
+		msg := "cscli unavailable and no cached preset; pull the preset or install cscli"
+		result.ErrorMessage = msg
+		return result, errors.New(msg)
+	}
 
 	backupPath := filepath.Clean(s.DataDir) + ".backup." + time.Now().Format("20060102-150405")
-	result := ApplyResult{BackupPath: backupPath, AppliedPreset: cleanSlug, Status: "failed"}
+	result.BackupPath = backupPath
 	if err := s.backupExisting(backupPath); err != nil {
 		return result, fmt.Errorf("backup: %w", err)
 	}
 
-	applyCtx, cancel := context.WithTimeout(ctx, s.ApplyTimeout)
-	defer cancel()
-	if meta, err := s.loadCacheMeta(applyCtx, cleanSlug); err == nil {
-		result.CacheKey = meta.CacheKey
-	}
-
 	// Try cscli first
-	if s.hasCSCLI(applyCtx) {
-		if err := s.runCSCLI(applyCtx, cleanSlug); err == nil {
+	if hasCS {
+		cscliErr := s.runCSCLI(applyCtx, cleanSlug)
+		if cscliErr == nil {
 			result.Status = "applied"
 			result.ReloadHint = true
 			result.UsedCSCLI = true
 			return result, nil
-		} else {
-			logger.Log().WithError(err).WithField("slug", cleanSlug).Warn("cscli install failed; attempting cache fallback")
 		}
+		logger.Log().WithField("slug", cleanSlug).WithError(cscliErr).Warn("cscli install failed; attempting cache fallback")
 	}
 
-	meta, err := s.loadCacheMeta(applyCtx, cleanSlug)
-	if err != nil {
+	if metaErr != nil {
 		_ = s.rollback(backupPath)
-		return result, fmt.Errorf("load cache: %w", err)
+		msg := fmt.Sprintf("load cache: %v", metaErr)
+		result.ErrorMessage = msg
+		return result, errors.New(msg)
 	}
-	result.CacheKey = meta.CacheKey
+
 	archive, err := os.ReadFile(meta.ArchivePath)
 	if err != nil {
 		_ = s.rollback(backupPath)
@@ -398,6 +445,18 @@ func findIndexEntry(idx HubIndex, slug string) (HubIndexEntry, bool) {
 	return HubIndexEntry{}, false
 }
 
+func isLikelyHTML(data []byte) bool {
+	trimmed := bytes.TrimSpace(data)
+	if len(trimmed) == 0 {
+		return false
+	}
+	lower := bytes.ToLower(trimmed)
+	if bytes.HasPrefix(lower, []byte("<!doctype")) || bytes.HasPrefix(lower, []byte("<html")) {
+		return true
+	}
+	return bytes.Contains(lower, []byte("<html"))
+}
+
 func (s *HubService) hasCSCLI(ctx context.Context) bool {
 	if s.Exec == nil {
 		return false

backend/internal/crowdsec/hub_sync_test.go

@@ -61,6 +61,13 @@ func makeTarGz(t *testing.T, files map[string]string) []byte {
 	return buf.Bytes()
 }
 
+func readFixture(t *testing.T, name string) string {
+	t.Helper()
+	data, err := os.ReadFile(filepath.Join("testdata", name))
+	require.NoError(t, err)
+	return string(data)
+}
+
 func TestFetchIndexPrefersCSCLI(t *testing.T) {
 	exec := &recordingExec{outputs: map[string][]byte{"cscli hub list -o json": []byte(`{"collections":[{"name":"crowdsecurity/test","description":"desc","version":"1.0"}]}`)}}
 	svc := NewHubService(exec, nil, t.TempDir())
@@ -77,9 +84,13 @@ func TestFetchIndexFallbackHTTP(t *testing.T) {
 	exec := &recordingExec{errors: map[string]error{"cscli hub list -o json": fmt.Errorf("boom")}}
 	cacheDir := t.TempDir()
 	svc := NewHubService(exec, nil, cacheDir)
+	svc.HubBaseURL = "http://example.com"
+	indexBody := readFixture(t, "hub_index.json")
 	svc.HTTPClient = &http.Client{Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
-		if req.URL.Path == defaultHubIndexPath {
-			return newResponse(http.StatusOK, `{"items":[{"name":"crowdsecurity/demo","title":"Demo","description":"desc","type":"collection"}]}`), nil
+		if req.URL.String() == "http://example.com"+defaultHubIndexPath {
+			resp := newResponse(http.StatusOK, indexBody)
+			resp.Header.Set("Content-Type", "application/json")
+			return resp, nil
 		}
 		return newResponse(http.StatusNotFound, ""), nil
 	})}
@@ -90,6 +101,34 @@ func TestFetchIndexFallbackHTTP(t *testing.T) {
 	require.Equal(t, "crowdsecurity/demo", idx.Items[0].Name)
 }
 
+func TestFetchIndexHTTPRejectsRedirect(t *testing.T) {
+	svc := NewHubService(nil, nil, t.TempDir())
+	svc.HubBaseURL = "http://hub.example"
+	svc.HTTPClient = &http.Client{Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
+		resp := newResponse(http.StatusMovedPermanently, "")
+		resp.Header.Set("Location", "https://hub.crowdsec.net/")
+		return resp, nil
+	})}
+
+	_, err := svc.fetchIndexHTTP(context.Background())
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "redirect")
+}
+
+func TestFetchIndexHTTPRejectsHTML(t *testing.T) {
+	svc := NewHubService(nil, nil, t.TempDir())
+	htmlBody := readFixture(t, "hub_index_html.html")
+	svc.HTTPClient = &http.Client{Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
+		resp := newResponse(http.StatusOK, htmlBody)
+		resp.Header.Set("Content-Type", "text/html")
+		return resp, nil
+	})}
+
+	_, err := svc.fetchIndexHTTP(context.Background())
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "HTML")
+}
+
 func TestPullCachesPreview(t *testing.T) {
 	cacheDir := t.TempDir()
 	dataDir := filepath.Join(t.TempDir(), "crowdsec")
@@ -398,8 +437,11 @@ func TestApplyRollsBackWhenCacheMissing(t *testing.T) {
 	require.NoError(t, os.WriteFile(filepath.Join(dataDir, "keep.txt"), []byte("before"), 0o644))
 
 	svc := NewHubService(nil, nil, dataDir)
-	_, err := svc.Apply(context.Background(), "crowdsecurity/demo")
+	res, err := svc.Apply(context.Background(), "crowdsecurity/demo")
 	require.Error(t, err)
+	require.Contains(t, err.Error(), "cscli unavailable")
+	require.Empty(t, res.BackupPath)
+	require.Equal(t, "failed", res.Status)
 
 	content, readErr := os.ReadFile(filepath.Join(dataDir, "keep.txt"))
 	require.NoError(t, readErr)

backend/internal/crowdsec/testdata/hub_index.json

@@ -0,0 +1 @@
+{"items":[{"name":"crowdsecurity/demo","title":"Demo","description":"desc","type":"collection","etag":"etag1","download_url":"http://example.com/demo.tgz","preview_url":"http://example.com/demo.yaml"}]}

backend/internal/crowdsec/testdata/hub_index_html.html

@@ -0,0 +1,5 @@
+<!DOCTYPE html>
+<html>
+<head><title>Moved</title></head>
+<body><h1>Moved</h1><p>Resource moved.</p></body>
+</html>

docs/plans/current_spec.md

@@ -6,6 +6,14 @@
 - Frontend: [frontend/src/pages/CrowdSecConfig.tsx](frontend/src/pages/CrowdSecConfig.tsx) calls `pullAndApplyCrowdsecPreset` then falls back to local `writeCrowdsecFile` apply. Preset catalog merges backend list with [frontend/src/data/crowdsecPresets.ts](frontend/src/data/crowdsecPresets.ts). Errors 501/404 are surfaced as info to keep local apply working. Overview toggle/start/stop already wired to `startCrowdsec`/`stopCrowdsec`.
 - Docs: [docs/cerberus.md](docs/cerberus.md) still notes CrowdSec integration is a placeholder; no hub sync described.
 
+## Incident Triage: CrowdSec preset pull/apply 502/500 (feature/beta-release)
+- Logs to pull first: backend app/GIN logs under `/app/data/logs/charon.log` (or `data/logs/charon.log` in dev) via [backend/cmd/api/main.go](backend/cmd/api/main.go); look for warnings "crowdsec preset pull failed" / "crowdsec preset apply failed" emitted in [backend/internal/api/handlers/crowdsec_handler.go](backend/internal/api/handlers/crowdsec_handler.go). Access logs will also show 502/500 for POST `/api/v1/admin/crowdsec/presets/pull` and `/apply`.
+- Routes and code paths: handlers `PullPreset` and `ApplyPreset` live in [backend/internal/api/handlers/crowdsec_handler.go](backend/internal/api/handlers/crowdsec_handler.go) and delegate to `HubService.Pull/Apply` in [backend/internal/crowdsec/hub_sync.go](backend/internal/crowdsec/hub_sync.go) with cache helpers in [backend/internal/crowdsec/hub_cache.go](backend/internal/crowdsec/hub_cache.go). Data dir used is `data/crowdsec` with cache under `data/crowdsec/hub_cache` from [backend/internal/api/routes/routes.go](backend/internal/api/routes/routes.go).
+- Quick checks before repro: (1) Cerberus enabled (`feature.cerberus.enabled` setting or `FEATURE_CERBERUS_ENABLED`/`CERBERUS_ENABLED` env) or handler returns 404 early; (2) `cscli` on PATH and executable (`HubService` uses real executor and calls `cscli version`/`cscli hub install`); (3) outbound HTTPS to https://hub.crowdsec.net reachable (fallback after `cscli hub list`); (4) cache dir writable `data/crowdsec/hub_cache` and contains per-slug `metadata.json`, `bundle.tgz`, `preview.yaml`; (5) backup path writable (apply renames `data/crowdsec` to `data/crowdsec.backup.<ts>`).
+- Likely 502 on pull: hub cache unavailable or init failed (cache dir permission), invalid slug, hub index fetch errors (`cscli hub list -o json` or direct GET `/api/index.json`), download blocked/size >25MiB, preview/download HTTP non-200, or cache write errors. Handler logs warning and returns 502 with error string.
+- Likely 500 on apply: backup rename fails, `cscli` install fails with no cache fallback (if pull never succeeded or cache expired/missing), cache read errors (`metadata.json`/`bundle.tgz` unreadable), tar extraction rejects symlinks/unsafe paths, or rollback after extract failure. Handler writes `CrowdsecPresetEvent` (if DB reachable) with backup path and returns 500 with `backup` hint.
+- Validation steps during triage: verify cache entry freshness (TTL 24h) via `metadata.json` timestamps; confirm `cscli hub install <slug>` succeeds manually; if cscli missing, ensure prior pull populated cache; test hub egress with curl to hub index and archive URLs; check file ownership/permissions on `data/crowdsec` and `data/crowdsec/hub_cache`; confirm log lines around warnings for exact error message; inspect backup directory to restore if partial apply.
+
 ## Goal
 Implement real CrowdSec Hub preset sync + apply on backend (using cscli or direct hub index) with caching, validation, backups, rollback, and wire the UI to new endpoints so operators can preview/apply hub items with clear status/errors.
 

docs/reports/qa_report.md

@@ -9,7 +9,7 @@
 
 **Final Verdict:** ✅ PASS (coverage gate met)
 
-- `pre-commit run --all-files` passes; coverage hook reports 85.0% vs required 85% (gate met) with hooks including Go vet, version check, frontend type-check, and lint fix.
+- `pre-commit run --all-files` passes; coverage hook reports 85.0% vs required 85% (gate met) after adding middleware sanitize tests. Hooks include Go vet, version check, frontend type-check, and lint fix.
 - `go test ./...` (backend) passes via task `Go: Test Backend`.
 - `npm run test:ci` passes (Vitest, 70 files / 598 tests). React Query undefined-data warnings and jsdom navigation warnings appear but suites stay green.
 
@@ -17,7 +17,7 @@
 
 | Area | Status | Notes |
 | --- | --- | --- |
-| Pre-commit | ✅ PASS | Coverage gate satisfied at 85.0%; all hooks succeeded. |
+| Pre-commit | ✅ PASS | Coverage gate satisfied at 85.0% (minimum 85%) after middleware sanitize tests; all hooks succeeded. |
 | Backend Unit Tests | ✅ PASS | `cd backend && go test ./...` (task: Go: Test Backend). |
 | Frontend Unit Tests | ✅ PASS* | `npm run test:ci` (Vitest, 70 files / 598 tests). Warnings: React Query "query data cannot be undefined" for `securityConfig`/`securityRulesets`/`feature-flags`; jsdom "navigation to another Document". |