Fixes CrowdSec not starting automatically on container boot and LAPI binding failures due to permission issues. Changes: - Fix Dockerfile: Add charon:charon ownership for CrowdSec directories - Move reconciliation from routes.go goroutine to main.go initialization - Add mutex protection to prevent concurrent reconciliation - Increase LAPI startup timeout from 30s to 60s - Add config validation in entrypoint script Testing: - Backend coverage: 85.4% (✅ meets requirement) - Frontend coverage: 87.01% (✅ exceeds requirement) - Security: 0 Critical/High vulnerabilities (✅ Trivy + Go scans) - All CrowdSec-specific tests passing (✅ 100%) Technical Details: - Reconciliation now runs synchronously during app initialization (after DB migrations, before HTTP server starts) - Maintains "GUI-controlled" design philosophy per entrypoint docs - Follows principle of least privilege (charon user, not root) - No breaking changes to API or behavior Documentation: - Implementation guide: docs/implementation/crowdsec_startup_fix_COMPLETE.md - Migration guide: docs/implementation/crowdsec_startup_fix_MIGRATION.md - QA report: docs/reports/qa_report_crowdsec_startup_fix.md Related: #crowdsec-startup-timeout
232 lines
8.7 KiB
Go
232 lines
8.7 KiB
Go
package services
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/Wikid82/charon/backend/internal/logger"
|
|
"github.com/Wikid82/charon/backend/internal/models"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// reconcileLock prevents concurrent reconciliation calls.
|
|
// This mutex is necessary because reconciliation can be triggered from multiple sources:
|
|
// 1. Container startup (main.go calls synchronously during boot)
|
|
// 2. Manual GUI toggle (user clicks Start/Stop in Security dashboard)
|
|
// 3. Future auto-restart (watchdog could trigger on crash)
|
|
// Without this mutex, race conditions could occur:
|
|
// - Multiple goroutines starting CrowdSec simultaneously
|
|
// - Database race conditions on SecurityConfig table
|
|
// - Duplicate process spawning
|
|
// - Corrupted state in executor
|
|
var reconcileLock sync.Mutex
|
|
|
|
// CrowdsecProcessManager abstracts starting/stopping/status of CrowdSec process.
|
|
// This interface is structurally compatible with handlers.CrowdsecExecutor.
|
|
type CrowdsecProcessManager interface {
|
|
Start(ctx context.Context, binPath, configDir string) (int, error)
|
|
Stop(ctx context.Context, configDir string) error
|
|
Status(ctx context.Context, configDir string) (running bool, pid int, err error)
|
|
}
|
|
|
|
// ReconcileCrowdSecOnStartup checks if CrowdSec should be running based on DB settings
|
|
// and starts it if necessary. This handles container restart scenarios where the
|
|
// user's preference was to have CrowdSec enabled.
|
|
//
|
|
// This function is called during container startup (before HTTP server starts) and
|
|
// ensures CrowdSec automatically resumes if it was previously enabled. It checks both
|
|
// the SecurityConfig table (primary source) and Settings table (fallback/legacy support).
|
|
//
|
|
// Mutex Protection: This function uses a global mutex to prevent concurrent execution,
|
|
// which could occur if multiple startup routines or manual toggles happen simultaneously.
|
|
//
|
|
// Initialization Order:
|
|
// 1. Container boot
|
|
// 2. Database migrations (ensures SecurityConfig table exists)
|
|
// 3. ReconcileCrowdSecOnStartup (this function) ← YOU ARE HERE
|
|
// 4. HTTP server starts
|
|
// 5. Routes registered
|
|
//
|
|
// Auto-start conditions (if ANY true, CrowdSec starts):
|
|
// - SecurityConfig.crowdsec_mode == "local"
|
|
// - Settings["security.crowdsec.enabled"] == "true"
|
|
func ReconcileCrowdSecOnStartup(db *gorm.DB, executor CrowdsecProcessManager, binPath, dataDir string) {
|
|
// Prevent concurrent reconciliation calls
|
|
reconcileLock.Lock()
|
|
defer reconcileLock.Unlock()
|
|
|
|
logger.Log().WithFields(map[string]any{
|
|
"bin_path": binPath,
|
|
"data_dir": dataDir,
|
|
}).Info("CrowdSec reconciliation: starting startup check")
|
|
|
|
if db == nil || executor == nil {
|
|
logger.Log().Debug("CrowdSec reconciliation skipped: nil db or executor")
|
|
return
|
|
}
|
|
|
|
// Check if SecurityConfig table exists and has a record with CrowdSecMode = "local"
|
|
if !db.Migrator().HasTable(&models.SecurityConfig{}) {
|
|
logger.Log().Warn("CrowdSec reconciliation skipped: SecurityConfig table not found - run 'charon migrate' to fix")
|
|
return
|
|
}
|
|
|
|
var cfg models.SecurityConfig
|
|
if err := db.First(&cfg).Error; err != nil {
|
|
if err == gorm.ErrRecordNotFound {
|
|
// AUTO-INITIALIZE: Create default SecurityConfig by checking Settings table
|
|
logger.Log().Info("CrowdSec reconciliation: no SecurityConfig found, checking Settings table for user preference")
|
|
|
|
// Check if user has already enabled CrowdSec via Settings table (from toggle or legacy config)
|
|
var settingOverride struct{ Value string }
|
|
crowdSecEnabledInSettings := false
|
|
if err := db.Raw("SELECT value FROM settings WHERE key = ? LIMIT 1", "security.crowdsec.enabled").Scan(&settingOverride).Error; err == nil && settingOverride.Value != "" {
|
|
crowdSecEnabledInSettings = strings.EqualFold(settingOverride.Value, "true")
|
|
logger.Log().WithFields(map[string]any{
|
|
"setting_value": settingOverride.Value,
|
|
"enabled": crowdSecEnabledInSettings,
|
|
}).Info("CrowdSec reconciliation: found existing Settings table preference")
|
|
}
|
|
|
|
// Create SecurityConfig that matches Settings table state
|
|
crowdSecMode := "disabled"
|
|
if crowdSecEnabledInSettings {
|
|
crowdSecMode = "local"
|
|
}
|
|
|
|
defaultCfg := models.SecurityConfig{
|
|
UUID: "default",
|
|
Name: "Default Security Config",
|
|
Enabled: crowdSecEnabledInSettings,
|
|
CrowdSecMode: crowdSecMode,
|
|
WAFMode: "disabled",
|
|
WAFParanoiaLevel: 1,
|
|
RateLimitMode: "disabled",
|
|
RateLimitBurst: 10,
|
|
RateLimitRequests: 100,
|
|
RateLimitWindowSec: 60,
|
|
}
|
|
|
|
if err := db.Create(&defaultCfg).Error; err != nil {
|
|
logger.Log().WithError(err).Error("CrowdSec reconciliation: failed to create default SecurityConfig")
|
|
return
|
|
}
|
|
|
|
logger.Log().WithFields(map[string]any{
|
|
"crowdsec_mode": defaultCfg.CrowdSecMode,
|
|
"enabled": defaultCfg.Enabled,
|
|
"source": "settings_table",
|
|
}).Info("CrowdSec reconciliation: default SecurityConfig created from Settings preference")
|
|
|
|
// Continue to process the config (DON'T return early)
|
|
cfg = defaultCfg
|
|
} else {
|
|
logger.Log().WithError(err).Warn("CrowdSec reconciliation: failed to read SecurityConfig")
|
|
return
|
|
}
|
|
}
|
|
|
|
// Also check for runtime setting override in settings table
|
|
var settingOverride struct{ Value string }
|
|
crowdSecEnabled := false
|
|
if err := db.Raw("SELECT value FROM settings WHERE key = ? LIMIT 1", "security.crowdsec.enabled").Scan(&settingOverride).Error; err == nil && settingOverride.Value != "" {
|
|
crowdSecEnabled = strings.EqualFold(settingOverride.Value, "true")
|
|
logger.Log().WithFields(map[string]any{
|
|
"setting_value": settingOverride.Value,
|
|
"crowdsec_enabled": crowdSecEnabled,
|
|
}).Debug("CrowdSec reconciliation: found runtime setting override")
|
|
}
|
|
|
|
// Only auto-start if CrowdSecMode is "local" OR runtime setting is enabled
|
|
if cfg.CrowdSecMode != "local" && !crowdSecEnabled {
|
|
logger.Log().WithFields(map[string]any{
|
|
"db_mode": cfg.CrowdSecMode,
|
|
"setting_enabled": crowdSecEnabled,
|
|
}).Info("CrowdSec reconciliation skipped: both SecurityConfig and Settings indicate disabled")
|
|
return
|
|
}
|
|
|
|
// Log which source triggered the start
|
|
if cfg.CrowdSecMode == "local" {
|
|
logger.Log().WithField("mode", cfg.CrowdSecMode).Info("CrowdSec reconciliation: starting based on SecurityConfig mode='local'")
|
|
} else if crowdSecEnabled {
|
|
logger.Log().WithField("setting", "true").Info("CrowdSec reconciliation: starting based on Settings table override")
|
|
}
|
|
|
|
// VALIDATE: Ensure binary exists
|
|
if _, err := os.Stat(binPath); os.IsNotExist(err) {
|
|
logger.Log().WithField("path", binPath).Error("CrowdSec reconciliation: binary not found, cannot start")
|
|
return
|
|
}
|
|
|
|
// VALIDATE: Ensure config directory exists
|
|
configPath := filepath.Join(dataDir, "config")
|
|
if _, err := os.Stat(configPath); os.IsNotExist(err) {
|
|
logger.Log().WithField("path", configPath).Error("CrowdSec reconciliation: config directory not found, cannot start")
|
|
return
|
|
}
|
|
|
|
// Check if CrowdSec is already running
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
|
|
running, pid, err := executor.Status(ctx, dataDir)
|
|
if err != nil {
|
|
logger.Log().WithError(err).Warn("CrowdSec reconciliation: failed to check status")
|
|
return
|
|
}
|
|
|
|
if running {
|
|
logger.Log().WithField("pid", pid).Info("CrowdSec reconciliation: already running")
|
|
return
|
|
}
|
|
|
|
// CrowdSec should be running but isn't - start it
|
|
logger.Log().WithFields(map[string]any{
|
|
"bin_path": binPath,
|
|
"data_dir": dataDir,
|
|
}).Info("CrowdSec reconciliation: starting CrowdSec (mode=local, not currently running)")
|
|
|
|
startCtx, startCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer startCancel()
|
|
|
|
newPid, err := executor.Start(startCtx, binPath, dataDir)
|
|
if err != nil {
|
|
logger.Log().WithError(err).WithFields(map[string]any{
|
|
"bin_path": binPath,
|
|
"data_dir": dataDir,
|
|
}).Error("CrowdSec reconciliation: FAILED to start CrowdSec - check binary and config")
|
|
return
|
|
}
|
|
|
|
// VERIFY: Wait briefly and confirm process is actually running
|
|
time.Sleep(2 * time.Second)
|
|
|
|
verifyCtx, verifyCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer verifyCancel()
|
|
|
|
verifyRunning, verifyPid, verifyErr := executor.Status(verifyCtx, dataDir)
|
|
if verifyErr != nil {
|
|
logger.Log().WithError(verifyErr).WithField("expected_pid", newPid).Warn("CrowdSec reconciliation: started but failed to verify status")
|
|
return
|
|
}
|
|
|
|
if !verifyRunning {
|
|
logger.Log().WithFields(map[string]any{
|
|
"expected_pid": newPid,
|
|
"actual_pid": verifyPid,
|
|
"running": verifyRunning,
|
|
}).Error("CrowdSec reconciliation: process started but is no longer running - may have crashed")
|
|
return
|
|
}
|
|
|
|
logger.Log().WithFields(map[string]any{
|
|
"pid": newPid,
|
|
"verified": true,
|
|
}).Info("CrowdSec reconciliation: successfully started and verified CrowdSec")
|
|
}
|