diff --git a/backend/.env.example b/backend/.env.example index 7b6f098e..a2559f92 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -3,6 +3,8 @@ CHARON_HTTP_PORT=8080 CHARON_DB_PATH=./data/charon.db CHARON_CADDY_ADMIN_API=http://localhost:2019 CHARON_CADDY_CONFIG_DIR=./data/caddy +# HUB_BASE_URL overrides the CrowdSec hub endpoint used when cscli is unavailable (defaults to https://hub-data.crowdsec.net) +# HUB_BASE_URL=https://hub-data.crowdsec.net CERBERUS_SECURITY_CERBERUS_ENABLED=false CHARON_SECURITY_CERBERUS_ENABLED=false CPM_SECURITY_CERBERUS_ENABLED=false diff --git a/backend/internal/api/handlers/crowdsec_presets_handler_test.go b/backend/internal/api/handlers/crowdsec_presets_handler_test.go index 754fc68e..d7ecc2fd 100644 --- a/backend/internal/api/handlers/crowdsec_presets_handler_test.go +++ b/backend/internal/api/handlers/crowdsec_presets_handler_test.go @@ -273,12 +273,13 @@ func TestApplyPresetHandlerBackupFailure(t *testing.T) { r.ServeHTTP(w, req) require.Equal(t, http.StatusInternalServerError, w.Code) + require.Contains(t, w.Body.String(), "cscli unavailable") var events []models.CrowdsecPresetEvent require.NoError(t, db.Find(&events).Error) require.Len(t, events, 1) require.Equal(t, "failed", events[0].Status) - require.NotEmpty(t, events[0].BackupPath) + require.Empty(t, events[0].BackupPath) content, readErr := os.ReadFile(filepath.Join(dataDir, "keep.txt")) require.NoError(t, readErr) diff --git a/backend/internal/api/middleware/sanitize_test.go b/backend/internal/api/middleware/sanitize_test.go new file mode 100644 index 00000000..dc581479 --- /dev/null +++ b/backend/internal/api/middleware/sanitize_test.go @@ -0,0 +1,55 @@ +package middleware + +import ( + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSanitizeHeaders(t *testing.T) { + t.Run("nil headers", func(t *testing.T) { + require.Nil(t, SanitizeHeaders(nil)) + }) + + t.Run("redacts sensitive headers", func(t *testing.T) { + headers := http.Header{} + headers.Set("Authorization", "secret") + headers.Set("X-Api-Key", "token") + headers.Set("Cookie", "sessionid=abc") + + sanitized := SanitizeHeaders(headers) + + require.Equal(t, []string{""}, sanitized["Authorization"]) + require.Equal(t, []string{""}, sanitized["X-Api-Key"]) + require.Equal(t, []string{""}, sanitized["Cookie"]) + }) + + t.Run("sanitizes and truncates values", func(t *testing.T) { + headers := http.Header{} + headers.Add("X-Trace", "line1\nline2\r\t") + headers.Add("X-Custom", strings.Repeat("a", 210)) + + sanitized := SanitizeHeaders(headers) + + traceValue := sanitized["X-Trace"][0] + require.NotContains(t, traceValue, "\n") + require.NotContains(t, traceValue, "\r") + require.NotContains(t, traceValue, "\t") + + customValue := sanitized["X-Custom"][0] + require.Equal(t, 200, len(customValue)) + require.True(t, strings.HasPrefix(customValue, strings.Repeat("a", 200))) + }) +} + +func TestSanitizePath(t *testing.T) { + paddedPath := "/api/v1/resource/" + strings.Repeat("x", 210) + "?token=secret" + + sanitized := SanitizePath(paddedPath) + + require.NotContains(t, sanitized, "?") + require.False(t, strings.ContainsAny(sanitized, "\n\r\t")) + require.Equal(t, 200, len(sanitized)) +} diff --git a/backend/internal/crowdsec/hub_sync.go b/backend/internal/crowdsec/hub_sync.go index effbc466..61e0667b 100644 --- a/backend/internal/crowdsec/hub_sync.go +++ b/backend/internal/crowdsec/hub_sync.go @@ -24,6 +24,7 @@ type CommandExecutor interface { } const ( + defaultHubBaseURL = "https://hub-data.crowdsec.net" defaultHubIndexPath = "/api/index.json" defaultHubArchivePath = "/%s.tgz" defaultHubPreviewPath = "/%s.yaml" @@ -77,17 +78,35 @@ type HubService struct { // NewHubService constructs a HubService with sane defaults. func NewHubService(exec CommandExecutor, cache *HubCache, dataDir string) *HubService { + clientTimeout := 10 * time.Second return &HubService{ Exec: exec, Cache: cache, DataDir: dataDir, - HTTPClient: &http.Client{Timeout: 10 * time.Second}, - HubBaseURL: "https://hub.crowdsec.net", - PullTimeout: 10 * time.Second, + HTTPClient: newHubHTTPClient(clientTimeout), + HubBaseURL: normalizeHubBaseURL(os.Getenv("HUB_BASE_URL")), + PullTimeout: clientTimeout, ApplyTimeout: 15 * time.Second, } } +func newHubHTTPClient(timeout time.Duration) *http.Client { + return &http.Client{ + Timeout: timeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + +func normalizeHubBaseURL(raw string) string { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return defaultHubBaseURL + } + return strings.TrimRight(trimmed, "/") +} + // FetchIndex downloads the hub index. If the hub is unreachable, returns ErrCacheMiss. func (s *HubService) FetchIndex(ctx context.Context) (HubIndex, error) { if s.Exec != nil { @@ -178,20 +197,39 @@ func (s *HubService) fetchIndexHTTP(ctx context.Context) (HubIndex, error) { if s.HTTPClient == nil { return HubIndex{}, fmt.Errorf("http client missing") } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.HubBaseURL+defaultHubIndexPath, nil) + target := strings.TrimRight(s.HubBaseURL, "/") + defaultHubIndexPath + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) if err != nil { return HubIndex{}, err } resp, err := s.HTTPClient.Do(req) if err != nil { - return HubIndex{}, err + return HubIndex{}, fmt.Errorf("fetch hub index: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return HubIndex{}, fmt.Errorf("hub index status %d", resp.StatusCode) + if resp.StatusCode >= 300 && resp.StatusCode < 400 { + loc := resp.Header.Get("Location") + return HubIndex{}, fmt.Errorf("hub index redirect (%d) to %s; install cscli or set HUB_BASE_URL to a JSON hub endpoint", resp.StatusCode, firstNonEmpty(loc, target)) + } + return HubIndex{}, fmt.Errorf("hub index status %d from %s", resp.StatusCode, target) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, maxArchiveSize)) + if err != nil { + return HubIndex{}, fmt.Errorf("read hub index: %w", err) + } + ct := strings.ToLower(resp.Header.Get("Content-Type")) + if ct != "" && !strings.Contains(ct, "application/json") { + if isLikelyHTML(data) { + return HubIndex{}, fmt.Errorf("hub index responded with HTML; install cscli or set HUB_BASE_URL to a JSON hub endpoint") + } + return HubIndex{}, fmt.Errorf("unexpected hub content-type %s; install cscli or set HUB_BASE_URL to a JSON hub endpoint", ct) } var idx HubIndex - if err := json.NewDecoder(resp.Body).Decode(&idx); err != nil { + if err := json.Unmarshal(data, &idx); err != nil { + if isLikelyHTML(data) { + return HubIndex{}, fmt.Errorf("hub index responded with HTML; install cscli or set HUB_BASE_URL to a JSON hub endpoint") + } return HubIndex{}, fmt.Errorf("decode hub index: %w", err) } return idx, nil @@ -234,11 +272,11 @@ func (s *HubService) Pull(ctx context.Context, slug string) (PullResult, error) archiveURL := entry.DownloadURL if archiveURL == "" { - archiveURL = fmt.Sprintf(s.HubBaseURL+defaultHubArchivePath, cleanSlug) + archiveURL = fmt.Sprintf(strings.TrimRight(s.HubBaseURL, "/")+defaultHubArchivePath, cleanSlug) } previewURL := entry.PreviewURL if previewURL == "" { - previewURL = fmt.Sprintf(s.HubBaseURL+defaultHubPreviewPath, cleanSlug) + previewURL = fmt.Sprintf(strings.TrimRight(s.HubBaseURL, "/")+defaultHubPreviewPath, cleanSlug) } archiveBytes, err := s.fetchWithLimit(pullCtx, archiveURL) @@ -266,37 +304,46 @@ func (s *HubService) Apply(ctx context.Context, slug string) (ApplyResult, error if cleanSlug == "" { return ApplyResult{}, fmt.Errorf("invalid slug") } + applyCtx, cancel := context.WithTimeout(ctx, s.ApplyTimeout) + defer cancel() + + result := ApplyResult{AppliedPreset: cleanSlug, Status: "failed"} + meta, metaErr := s.loadCacheMeta(applyCtx, cleanSlug) + if metaErr == nil { + result.CacheKey = meta.CacheKey + } + hasCS := s.hasCSCLI(applyCtx) + if !hasCS && metaErr != nil { + msg := "cscli unavailable and no cached preset; pull the preset or install cscli" + result.ErrorMessage = msg + return result, errors.New(msg) + } backupPath := filepath.Clean(s.DataDir) + ".backup." + time.Now().Format("20060102-150405") - result := ApplyResult{BackupPath: backupPath, AppliedPreset: cleanSlug, Status: "failed"} + result.BackupPath = backupPath if err := s.backupExisting(backupPath); err != nil { return result, fmt.Errorf("backup: %w", err) } - applyCtx, cancel := context.WithTimeout(ctx, s.ApplyTimeout) - defer cancel() - if meta, err := s.loadCacheMeta(applyCtx, cleanSlug); err == nil { - result.CacheKey = meta.CacheKey - } - // Try cscli first - if s.hasCSCLI(applyCtx) { - if err := s.runCSCLI(applyCtx, cleanSlug); err == nil { + if hasCS { + cscliErr := s.runCSCLI(applyCtx, cleanSlug) + if cscliErr == nil { result.Status = "applied" result.ReloadHint = true result.UsedCSCLI = true return result, nil - } else { - logger.Log().WithError(err).WithField("slug", cleanSlug).Warn("cscli install failed; attempting cache fallback") } + logger.Log().WithField("slug", cleanSlug).WithError(cscliErr).Warn("cscli install failed; attempting cache fallback") } - meta, err := s.loadCacheMeta(applyCtx, cleanSlug) - if err != nil { + if metaErr != nil { _ = s.rollback(backupPath) - return result, fmt.Errorf("load cache: %w", err) + msg := fmt.Sprintf("load cache: %v", metaErr) + result.ErrorMessage = msg + return result, errors.New(msg) } - result.CacheKey = meta.CacheKey + archive, err := os.ReadFile(meta.ArchivePath) if err != nil { _ = s.rollback(backupPath) @@ -398,6 +445,18 @@ func findIndexEntry(idx HubIndex, slug string) (HubIndexEntry, bool) { return HubIndexEntry{}, false } +func isLikelyHTML(data []byte) bool { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 { + return false + } + lower := bytes.ToLower(trimmed) + if bytes.HasPrefix(lower, []byte(" + +Moved +

Moved

Resource moved.

+ diff --git a/docs/plans/current_spec.md b/docs/plans/current_spec.md index 09c5775d..e03b2228 100644 --- a/docs/plans/current_spec.md +++ b/docs/plans/current_spec.md @@ -6,6 +6,14 @@ - Frontend: [frontend/src/pages/CrowdSecConfig.tsx](frontend/src/pages/CrowdSecConfig.tsx) calls `pullAndApplyCrowdsecPreset` then falls back to local `writeCrowdsecFile` apply. Preset catalog merges backend list with [frontend/src/data/crowdsecPresets.ts](frontend/src/data/crowdsecPresets.ts). Errors 501/404 are surfaced as info to keep local apply working. Overview toggle/start/stop already wired to `startCrowdsec`/`stopCrowdsec`. - Docs: [docs/cerberus.md](docs/cerberus.md) still notes CrowdSec integration is a placeholder; no hub sync described. +## Incident Triage: CrowdSec preset pull/apply 502/500 (feature/beta-release) +- Logs to pull first: backend app/GIN logs under `/app/data/logs/charon.log` (or `data/logs/charon.log` in dev) via [backend/cmd/api/main.go](backend/cmd/api/main.go); look for warnings "crowdsec preset pull failed" / "crowdsec preset apply failed" emitted in [backend/internal/api/handlers/crowdsec_handler.go](backend/internal/api/handlers/crowdsec_handler.go). Access logs will also show 502/500 for POST `/api/v1/admin/crowdsec/presets/pull` and `/apply`. +- Routes and code paths: handlers `PullPreset` and `ApplyPreset` live in [backend/internal/api/handlers/crowdsec_handler.go](backend/internal/api/handlers/crowdsec_handler.go) and delegate to `HubService.Pull/Apply` in [backend/internal/crowdsec/hub_sync.go](backend/internal/crowdsec/hub_sync.go) with cache helpers in [backend/internal/crowdsec/hub_cache.go](backend/internal/crowdsec/hub_cache.go). Data dir used is `data/crowdsec` with cache under `data/crowdsec/hub_cache` from [backend/internal/api/routes/routes.go](backend/internal/api/routes/routes.go). +- Quick checks before repro: (1) Cerberus enabled (`feature.cerberus.enabled` setting or `FEATURE_CERBERUS_ENABLED`/`CERBERUS_ENABLED` env) or handler returns 404 early; (2) `cscli` on PATH and executable (`HubService` uses real executor and calls `cscli version`/`cscli hub install`); (3) outbound HTTPS to https://hub.crowdsec.net reachable (fallback after `cscli hub list`); (4) cache dir writable `data/crowdsec/hub_cache` and contains per-slug `metadata.json`, `bundle.tgz`, `preview.yaml`; (5) backup path writable (apply renames `data/crowdsec` to `data/crowdsec.backup.`). +- Likely 502 on pull: hub cache unavailable or init failed (cache dir permission), invalid slug, hub index fetch errors (`cscli hub list -o json` or direct GET `/api/index.json`), download blocked/size >25MiB, preview/download HTTP non-200, or cache write errors. Handler logs warning and returns 502 with error string. +- Likely 500 on apply: backup rename fails, `cscli` install fails with no cache fallback (if pull never succeeded or cache expired/missing), cache read errors (`metadata.json`/`bundle.tgz` unreadable), tar extraction rejects symlinks/unsafe paths, or rollback after extract failure. Handler writes `CrowdsecPresetEvent` (if DB reachable) with backup path and returns 500 with `backup` hint. +- Validation steps during triage: verify cache entry freshness (TTL 24h) via `metadata.json` timestamps; confirm `cscli hub install ` succeeds manually; if cscli missing, ensure prior pull populated cache; test hub egress with curl to hub index and archive URLs; check file ownership/permissions on `data/crowdsec` and `data/crowdsec/hub_cache`; confirm log lines around warnings for exact error message; inspect backup directory to restore if partial apply. + ## Goal Implement real CrowdSec Hub preset sync + apply on backend (using cscli or direct hub index) with caching, validation, backups, rollback, and wire the UI to new endpoints so operators can preview/apply hub items with clear status/errors. diff --git a/docs/reports/qa_report.md b/docs/reports/qa_report.md index fd16c0dd..04ebfffe 100644 --- a/docs/reports/qa_report.md +++ b/docs/reports/qa_report.md @@ -9,7 +9,7 @@ **Final Verdict:** ✅ PASS (coverage gate met) -- `pre-commit run --all-files` passes; coverage hook reports 85.0% vs required 85% (gate met) with hooks including Go vet, version check, frontend type-check, and lint fix. +- `pre-commit run --all-files` passes; coverage hook reports 85.0% vs required 85% (gate met) after adding middleware sanitize tests. Hooks include Go vet, version check, frontend type-check, and lint fix. - `go test ./...` (backend) passes via task `Go: Test Backend`. - `npm run test:ci` passes (Vitest, 70 files / 598 tests). React Query undefined-data warnings and jsdom navigation warnings appear but suites stay green. @@ -17,7 +17,7 @@ | Area | Status | Notes | | --- | --- | --- | -| Pre-commit | ✅ PASS | Coverage gate satisfied at 85.0%; all hooks succeeded. | +| Pre-commit | ✅ PASS | Coverage gate satisfied at 85.0% (minimum 85%) after middleware sanitize tests; all hooks succeeded. | | Backend Unit Tests | ✅ PASS | `cd backend && go test ./...` (task: Go: Test Backend). | | Frontend Unit Tests | ✅ PASS* | `npm run test:ci` (Vitest, 70 files / 598 tests). Warnings: React Query "query data cannot be undefined" for `securityConfig`/`securityRulesets`/`feature-flags`; jsdom "navigation to another Document". |