feat: Implement database migration command and enhance CrowdSec startup verification
- Added TestMigrateCommand_Succeeds to validate migration functionality. - Introduced TestStartupVerification_MissingTables to ensure proper handling of missing security tables. - Updated crowdsec_startup.go to log warnings for missing SecurityConfig table. - Enhanced documentation for database migrations during upgrades, including steps and expected outputs. - Created a detailed migration QA report outlining testing results and recommendations. - Added troubleshooting guidance for CrowdSec not starting after upgrades due to missing tables. - Established a new plan for addressing CrowdSec reconciliation failures, including root cause analysis and proposed fixes.
This commit is contained in:
GitHub Actions
+91
-35
@@ -53,42 +53,71 @@ func main() {
|
||||
logger.Init(false, mw)
|
||||
|
||||
// Handle CLI commands
|
||||
if len(os.Args) > 1 && os.Args[1] == "reset-password" {
|
||||
if len(os.Args) != 4 {
|
||||
log.Fatalf("Usage: %s reset-password <email> <new-password>", os.Args[0])
|
||||
if len(os.Args) > 1 {
|
||||
switch os.Args[1] {
|
||||
case "migrate":
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("load config: %v", err)
|
||||
}
|
||||
|
||||
db, err := database.Connect(cfg.DatabasePath)
|
||||
if err != nil {
|
||||
log.Fatalf("connect database: %v", err)
|
||||
}
|
||||
|
||||
logger.Log().Info("Running database migrations for security tables...")
|
||||
if err := db.AutoMigrate(
|
||||
&models.SecurityConfig{},
|
||||
&models.SecurityDecision{},
|
||||
&models.SecurityAudit{},
|
||||
&models.SecurityRuleSet{},
|
||||
&models.CrowdsecPresetEvent{},
|
||||
&models.CrowdsecConsoleEnrollment{},
|
||||
); err != nil {
|
||||
log.Fatalf("migration failed: %v", err)
|
||||
}
|
||||
|
||||
logger.Log().Info("Migration completed successfully")
|
||||
return
|
||||
|
||||
case "reset-password":
|
||||
if len(os.Args) != 4 {
|
||||
log.Fatalf("Usage: %s reset-password <email> <new-password>", os.Args[0])
|
||||
}
|
||||
email := os.Args[2]
|
||||
newPassword := os.Args[3]
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("load config: %v", err)
|
||||
}
|
||||
|
||||
db, err := database.Connect(cfg.DatabasePath)
|
||||
if err != nil {
|
||||
log.Fatalf("connect database: %v", err)
|
||||
}
|
||||
|
||||
var user models.User
|
||||
if err := db.Where("email = ?", email).First(&user).Error; err != nil {
|
||||
log.Fatalf("user not found: %v", err)
|
||||
}
|
||||
|
||||
if err := user.SetPassword(newPassword); err != nil {
|
||||
log.Fatalf("failed to hash password: %v", err)
|
||||
}
|
||||
|
||||
// Unlock account if locked
|
||||
user.LockedUntil = nil
|
||||
user.FailedLoginAttempts = 0
|
||||
|
||||
if err := db.Save(&user).Error; err != nil {
|
||||
log.Fatalf("failed to save user: %v", err)
|
||||
}
|
||||
|
||||
logger.Log().Infof("Password updated successfully for user %s", email)
|
||||
return
|
||||
}
|
||||
email := os.Args[2]
|
||||
newPassword := os.Args[3]
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("load config: %v", err)
|
||||
}
|
||||
|
||||
db, err := database.Connect(cfg.DatabasePath)
|
||||
if err != nil {
|
||||
log.Fatalf("connect database: %v", err)
|
||||
}
|
||||
|
||||
var user models.User
|
||||
if err := db.Where("email = ?", email).First(&user).Error; err != nil {
|
||||
log.Fatalf("user not found: %v", err)
|
||||
}
|
||||
|
||||
if err := user.SetPassword(newPassword); err != nil {
|
||||
log.Fatalf("failed to hash password: %v", err)
|
||||
}
|
||||
|
||||
// Unlock account if locked
|
||||
user.LockedUntil = nil
|
||||
user.FailedLoginAttempts = 0
|
||||
|
||||
if err := db.Save(&user).Error; err != nil {
|
||||
log.Fatalf("failed to save user: %v", err)
|
||||
}
|
||||
|
||||
logger.Log().Infof("Password updated successfully for user %s", email)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Log().Infof("starting %s backend on version %s", version.Name, version.Full())
|
||||
@@ -103,6 +132,33 @@ func main() {
|
||||
log.Fatalf("connect database: %v", err)
|
||||
}
|
||||
|
||||
// Verify critical security tables exist before starting server
|
||||
// This prevents silent failures in CrowdSec reconciliation
|
||||
securityModels := []interface{}{
|
||||
&models.SecurityConfig{},
|
||||
&models.SecurityDecision{},
|
||||
&models.SecurityAudit{},
|
||||
&models.SecurityRuleSet{},
|
||||
&models.CrowdsecPresetEvent{},
|
||||
&models.CrowdsecConsoleEnrollment{},
|
||||
}
|
||||
|
||||
missingTables := false
|
||||
for _, model := range securityModels {
|
||||
if !db.Migrator().HasTable(model) {
|
||||
missingTables = true
|
||||
logger.Log().Warnf("Missing security table for model %T - running migration", model)
|
||||
}
|
||||
}
|
||||
|
||||
if missingTables {
|
||||
logger.Log().Warn("Security tables missing - running auto-migration")
|
||||
if err := db.AutoMigrate(securityModels...); err != nil {
|
||||
log.Fatalf("failed to migrate security tables: %v", err)
|
||||
}
|
||||
logger.Log().Info("Security tables migrated successfully")
|
||||
}
|
||||
|
||||
router := server.NewRouter(cfg.FrontendDir)
|
||||
// Initialize structured logger with same writer as stdlib log so both capture logs
|
||||
logger.Init(cfg.Debug, mw)
|
||||
|
||||
@@ -57,3 +57,134 @@ func TestResetPasswordCommand_Succeeds(t *testing.T) {
|
||||
t.Fatalf("expected exit 0; err=%v; output=%s", err, string(out))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateCommand_Succeeds(t *testing.T) {
|
||||
if os.Getenv("CHARON_TEST_RUN_MAIN") == "1" {
|
||||
// Child process: emulate CLI args and run main().
|
||||
os.Args = []string{"charon", "migrate"}
|
||||
main()
|
||||
return
|
||||
}
|
||||
|
||||
tmp := t.TempDir()
|
||||
dbPath := filepath.Join(tmp, "data", "test.db")
|
||||
if err := os.MkdirAll(filepath.Dir(dbPath), 0o755); err != nil {
|
||||
t.Fatalf("mkdir db dir: %v", err)
|
||||
}
|
||||
|
||||
// Create database without security tables
|
||||
db, err := database.Connect(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("connect db: %v", err)
|
||||
}
|
||||
// Only migrate User table to simulate old database
|
||||
if err := db.AutoMigrate(&models.User{}); err != nil {
|
||||
t.Fatalf("automigrate user: %v", err)
|
||||
}
|
||||
|
||||
// Verify security tables don't exist
|
||||
if db.Migrator().HasTable(&models.SecurityConfig{}) {
|
||||
t.Fatal("SecurityConfig table should not exist yet")
|
||||
}
|
||||
|
||||
cmd := exec.Command(os.Args[0], "-test.run=TestMigrateCommand_Succeeds")
|
||||
cmd.Dir = tmp
|
||||
cmd.Env = append(os.Environ(),
|
||||
"CHARON_TEST_RUN_MAIN=1",
|
||||
"CHARON_DB_PATH="+dbPath,
|
||||
"CHARON_CADDY_CONFIG_DIR="+filepath.Join(tmp, "caddy"),
|
||||
"CHARON_IMPORT_DIR="+filepath.Join(tmp, "imports"),
|
||||
)
|
||||
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("expected exit 0; err=%v; output=%s", err, string(out))
|
||||
}
|
||||
|
||||
// Reconnect and verify security tables were created
|
||||
db2, err := database.Connect(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("reconnect db: %v", err)
|
||||
}
|
||||
|
||||
securityModels := []interface{}{
|
||||
&models.SecurityConfig{},
|
||||
&models.SecurityDecision{},
|
||||
&models.SecurityAudit{},
|
||||
&models.SecurityRuleSet{},
|
||||
&models.CrowdsecPresetEvent{},
|
||||
&models.CrowdsecConsoleEnrollment{},
|
||||
}
|
||||
|
||||
for _, model := range securityModels {
|
||||
if !db2.Migrator().HasTable(model) {
|
||||
t.Errorf("Table for %T was not created by migrate command", model)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartupVerification_MissingTables(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
dbPath := filepath.Join(tmp, "data", "test.db")
|
||||
if err := os.MkdirAll(filepath.Dir(dbPath), 0o755); err != nil {
|
||||
t.Fatalf("mkdir db dir: %v", err)
|
||||
}
|
||||
|
||||
// Create database without security tables
|
||||
db, err := database.Connect(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("connect db: %v", err)
|
||||
}
|
||||
// Only migrate User table to simulate old database
|
||||
if err := db.AutoMigrate(&models.User{}); err != nil {
|
||||
t.Fatalf("automigrate user: %v", err)
|
||||
}
|
||||
|
||||
// Verify security tables don't exist
|
||||
if db.Migrator().HasTable(&models.SecurityConfig{}) {
|
||||
t.Fatal("SecurityConfig table should not exist yet")
|
||||
}
|
||||
|
||||
// Close and reopen to simulate startup scenario
|
||||
sqlDB, _ := db.DB()
|
||||
sqlDB.Close()
|
||||
|
||||
db, err = database.Connect(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("reconnect db: %v", err)
|
||||
}
|
||||
|
||||
// Simulate startup verification logic from main.go
|
||||
securityModels := []interface{}{
|
||||
&models.SecurityConfig{},
|
||||
&models.SecurityDecision{},
|
||||
&models.SecurityAudit{},
|
||||
&models.SecurityRuleSet{},
|
||||
&models.CrowdsecPresetEvent{},
|
||||
&models.CrowdsecConsoleEnrollment{},
|
||||
}
|
||||
|
||||
missingTables := false
|
||||
for _, model := range securityModels {
|
||||
if !db.Migrator().HasTable(model) {
|
||||
missingTables = true
|
||||
t.Logf("Missing table for model %T", model)
|
||||
}
|
||||
}
|
||||
|
||||
if !missingTables {
|
||||
t.Fatal("Expected to find missing tables but all were present")
|
||||
}
|
||||
|
||||
// Run auto-migration (simulating startup verification logic)
|
||||
if err := db.AutoMigrate(securityModels...); err != nil {
|
||||
t.Fatalf("failed to migrate security tables: %v", err)
|
||||
}
|
||||
|
||||
// Verify all tables now exist
|
||||
for _, model := range securityModels {
|
||||
if !db.Migrator().HasTable(model) {
|
||||
t.Errorf("Table for %T was not created by auto-migration", model)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user