Files
Charon/backend/internal/crowdsec/hub_pull_apply_test.go

486 lines
19 KiB
Go

package crowdsec
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
)
type stubExec struct {
responses map[string]error
calls []string
}
func (s *stubExec) Execute(ctx context.Context, name string, args ...string) ([]byte, error) {
cmd := strings.Join(append([]string{name}, args...), " ")
s.calls = append(s.calls, cmd)
for key, err := range s.responses {
if strings.Contains(cmd, key) {
return nil, err
}
}
return []byte("ok"), nil
}
// TestPullThenApplyFlow verifies that pulling a preset and then applying it works correctly.
func TestPullThenApplyFlow(t *testing.T) {
// Create temp directories for cache and data
cacheDir := t.TempDir()
dataDir := t.TempDir()
// Create cache with 1 hour TTL
cache, err := NewHubCache(cacheDir, time.Hour)
require.NoError(t, err)
// Create a test archive
archive := makeTestArchive(t, map[string]string{
"config.yaml": "test: config\nvalue: 123",
"profiles.yaml": "name: test",
})
// Create hub service with mock HTTP client
hub := NewHubService(nil, cache, dataDir)
hub.HubBaseURL = "http://test.example.com"
hub.HTTPClient = &http.Client{
Transport: mockTransport(func(req *http.Request) (*http.Response, error) {
switch req.URL.String() {
case "http://test.example.com/api/index.json":
body := `{"items":[{"name":"test/preset","title":"Test Preset","description":"Test","etag":"etag123","download_url":"http://test.example.com/test.tgz","preview_url":"http://test.example.com/test.yaml"}]}`
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(strings.NewReader(body)),
Header: make(http.Header),
}, nil
case "http://test.example.com/test.yaml":
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(strings.NewReader("test: preview\nkey: value")),
Header: make(http.Header),
}, nil
case "http://test.example.com/test.tgz":
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(bytes.NewReader(archive)),
Header: make(http.Header),
}, nil
default:
return &http.Response{
StatusCode: http.StatusNotFound,
Body: io.NopCloser(strings.NewReader("")),
Header: make(http.Header),
}, nil
}
}),
}
ctx := context.Background()
// Step 1: Pull the preset
t.Log("Step 1: Pulling preset")
pullResult, err := hub.Pull(ctx, "test/preset")
require.NoError(t, err, "Pull should succeed")
require.Equal(t, "test/preset", pullResult.Meta.Slug)
require.NotEmpty(t, pullResult.Meta.CacheKey)
require.NotEmpty(t, pullResult.Preview)
// Verify cache files exist
require.FileExists(t, pullResult.Meta.ArchivePath, "Archive should be cached")
require.FileExists(t, pullResult.Meta.PreviewPath, "Preview should be cached")
// Read the cached files to verify content
cachedArchive, err := os.ReadFile(pullResult.Meta.ArchivePath)
require.NoError(t, err)
require.Equal(t, archive, cachedArchive, "Cached archive should match original")
cachedPreview, err := os.ReadFile(pullResult.Meta.PreviewPath)
require.NoError(t, err)
require.Contains(t, string(cachedPreview), "preview", "Cached preview should contain expected content")
t.Log("Step 2: Verifying cache can be loaded")
// Verify we can load from cache
loaded, err := cache.Load(ctx, "test/preset")
require.NoError(t, err, "Should be able to load cached preset")
require.Equal(t, pullResult.Meta.Slug, loaded.Slug)
require.Equal(t, pullResult.Meta.CacheKey, loaded.CacheKey)
t.Log("Step 3: Applying preset from cache")
// Step 2: Apply the preset (should use cached version)
applyResult, err := hub.Apply(ctx, "test/preset")
require.NoError(t, err, "Apply should succeed after pull")
require.Equal(t, "applied", applyResult.Status)
require.NotEmpty(t, applyResult.BackupPath)
require.Equal(t, "test/preset", applyResult.AppliedPreset)
// Verify files were extracted to dataDir
extractedConfig := filepath.Join(dataDir, "config.yaml")
require.FileExists(t, extractedConfig, "Config should be extracted")
content, err := os.ReadFile(extractedConfig)
require.NoError(t, err)
require.Contains(t, string(content), "test: config")
}
func TestApplyRepullsOnCacheMissAfterCSCLIFailure(t *testing.T) {
cacheDir := t.TempDir()
dataDir := filepath.Join(t.TempDir(), "data")
cache, err := NewHubCache(cacheDir, time.Hour)
require.NoError(t, err)
archive := makeTestArchive(t, map[string]string{"config.yaml": "test: repull"})
exec := &stubExec{responses: map[string]error{"install": fmt.Errorf("install failed")}}
hub := NewHubService(exec, cache, dataDir)
hub.HubBaseURL = "http://test.example.com"
hub.HTTPClient = &http.Client{Transport: mockTransport(func(req *http.Request) (*http.Response, error) {
switch req.URL.String() {
case "http://test.example.com/api/index.json":
body := `{"items":[{"name":"test/preset","title":"Test","etag":"e1"}]}`
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil
case "http://test.example.com/test/preset.yaml":
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("preview")), Header: make(http.Header)}, nil
case "http://test.example.com/test/preset.tgz":
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(archive)), Header: make(http.Header)}, nil
default:
return &http.Response{StatusCode: http.StatusNotFound, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil
}
})}
ctx := context.Background()
res, err := hub.Apply(ctx, "test/preset")
require.NoError(t, err)
require.Equal(t, "applied", res.Status)
require.False(t, res.UsedCSCLI)
require.NotEmpty(t, res.CacheKey)
meta, loadErr := cache.Load(ctx, "test/preset")
require.NoError(t, loadErr)
require.Equal(t, res.CacheKey, meta.CacheKey)
require.FileExists(t, filepath.Join(dataDir, "config.yaml"))
}
func TestApplyRepullsOnCacheExpired(t *testing.T) {
cacheDir := t.TempDir()
dataDir := filepath.Join(t.TempDir(), "data")
cache, err := NewHubCache(cacheDir, 5*time.Millisecond)
require.NoError(t, err)
archive := makeTestArchive(t, map[string]string{"config.yaml": "test: expired"})
ctx := context.Background()
_, err = cache.Store(ctx, "expired/preset", "etag-old", "hub", "old", archive)
require.NoError(t, err)
// wait for expiration
time.Sleep(10 * time.Millisecond)
hub := NewHubService(nil, cache, dataDir)
hub.HubBaseURL = "http://test.example.com"
hub.HTTPClient = &http.Client{Transport: mockTransport(func(req *http.Request) (*http.Response, error) {
switch req.URL.String() {
case "http://test.example.com/api/index.json":
body := `{"items":[{"name":"expired/preset","title":"Expired","etag":"e2"}]}`
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil
case "http://test.example.com/expired/preset.yaml":
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("preview new")), Header: make(http.Header)}, nil
case "http://test.example.com/expired/preset.tgz":
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(archive)), Header: make(http.Header)}, nil
default:
return &http.Response{StatusCode: http.StatusNotFound, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil
}
})}
res, err := hub.Apply(ctx, "expired/preset")
require.NoError(t, err)
require.Equal(t, "applied", res.Status)
require.False(t, res.UsedCSCLI)
meta, loadErr := cache.Load(ctx, "expired/preset")
require.NoError(t, loadErr)
require.Equal(t, "e2", meta.Etag)
require.FileExists(t, filepath.Join(dataDir, "config.yaml"))
}
func TestPullAcceptsNamespacedIndexEntry(t *testing.T) {
cacheDir := t.TempDir()
dataDir := filepath.Join(t.TempDir(), "data")
cache, err := NewHubCache(cacheDir, time.Hour)
require.NoError(t, err)
archive := makeTestArchive(t, map[string]string{"config.yaml": "test: namespaced"})
hub := NewHubService(nil, cache, dataDir)
hub.HubBaseURL = "http://test.example.com"
hub.HTTPClient = &http.Client{Transport: mockTransport(func(req *http.Request) (*http.Response, error) {
switch req.URL.String() {
case "http://test.example.com/api/index.json":
body := `{"items":[{"name":"crowdsecurity/bot-mitigation-essentials","title":"Bot Mitigation Essentials","etag":"etag-bme"}]}`
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil
case "http://test.example.com/crowdsecurity/bot-mitigation-essentials.yaml":
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("namespaced preview")), Header: make(http.Header)}, nil
case "http://test.example.com/crowdsecurity/bot-mitigation-essentials.tgz":
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(archive)), Header: make(http.Header)}, nil
default:
return &http.Response{StatusCode: http.StatusNotFound, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil
}
})}
ctx := context.Background()
res, err := hub.Pull(ctx, "bot-mitigation-essentials")
require.NoError(t, err)
require.Equal(t, "bot-mitigation-essentials", res.Meta.Slug)
require.Equal(t, "etag-bme", res.Meta.Etag)
require.Contains(t, res.Preview, "namespaced preview")
}
func TestHubFallbackToMirrorOnForbidden(t *testing.T) {
cacheDir := t.TempDir()
dataDir := t.TempDir()
cache, err := NewHubCache(cacheDir, time.Hour)
require.NoError(t, err)
archive := makeTestArchive(t, map[string]string{"config.yaml": "mirror"})
hub := NewHubService(nil, cache, dataDir)
hub.HubBaseURL = "http://primary.example.com"
hub.MirrorBaseURL = "http://mirror.example.com"
hub.HTTPClient = &http.Client{Transport: mockTransport(func(req *http.Request) (*http.Response, error) {
switch req.URL.String() {
case "http://primary.example.com/api/index.json":
return &http.Response{StatusCode: http.StatusForbidden, Body: io.NopCloser(strings.NewReader("blocked")), Header: make(http.Header)}, nil
case "http://mirror.example.com/api/index.json":
body := `{"items":[{"name":"fallback/preset","title":"Fallback","etag":"etag-mirror"}]}`
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil
case "http://primary.example.com/fallback/preset.yaml":
return &http.Response{StatusCode: http.StatusForbidden, Body: io.NopCloser(strings.NewReader("blocked")), Header: make(http.Header)}, nil
case "http://mirror.example.com/fallback/preset.yaml":
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("mirror preview")), Header: make(http.Header)}, nil
case "http://primary.example.com/fallback/preset.tgz":
return &http.Response{StatusCode: http.StatusForbidden, Body: io.NopCloser(strings.NewReader("blocked")), Header: make(http.Header)}, nil
case "http://mirror.example.com/fallback/preset.tgz":
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(archive)), Header: make(http.Header)}, nil
default:
return &http.Response{StatusCode: http.StatusNotFound, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil
}
})}
ctx := context.Background()
res, err := hub.Pull(ctx, "fallback/preset")
require.NoError(t, err)
require.Equal(t, "etag-mirror", res.Meta.Etag)
require.Contains(t, res.Preview, "mirror preview")
}
// TestApplyWithoutPullFails verifies that applying without pulling first fails with proper error.
func TestApplyWithoutPullFails(t *testing.T) {
cacheDir := t.TempDir()
dataDir := t.TempDir()
cache, err := NewHubCache(cacheDir, time.Hour)
require.NoError(t, err)
// Create hub service without cscli (nil executor) and empty cache
hub := NewHubService(nil, cache, dataDir)
hub.HubBaseURL = "http://test.example.com"
hub.HTTPClient = &http.Client{Transport: mockTransport(func(req *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusInternalServerError, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil
})}
ctx := context.Background()
// Try to apply without pulling first
_, err = hub.Apply(ctx, "nonexistent/preset")
require.Error(t, err, "Apply should fail without cache and without cscli")
require.ErrorIs(t, err, ErrCacheMiss, "Error should expose cache miss for guidance")
require.Contains(t, err.Error(), "refresh cache", "Error should surface repull failure context")
}
// TestCacheExpiration verifies that expired cache is not used.
func TestCacheExpiration(t *testing.T) {
cacheDir := t.TempDir()
// Create cache with very short TTL
cache, err := NewHubCache(cacheDir, 1*time.Millisecond)
require.NoError(t, err)
// Store a preset
archive := makeTestArchive(t, map[string]string{"test.yaml": "content"})
ctx := context.Background()
cached, err := cache.Store(ctx, "test/preset", "etag1", "hub", "preview", archive)
require.NoError(t, err)
// Wait for expiration
time.Sleep(10 * time.Millisecond)
// Try to load - should get ErrCacheExpired
_, err = cache.Load(ctx, "test/preset")
require.ErrorIs(t, err, ErrCacheExpired, "Should get cache expired error")
// Verify the cache files still exist on disk (not deleted)
require.FileExists(t, cached.ArchivePath, "Archive file should still exist")
require.FileExists(t, cached.PreviewPath, "Preview file should still exist")
}
// TestCacheListAfterPull verifies that pulled presets appear in cache list.
func TestCacheListAfterPull(t *testing.T) {
cacheDir := t.TempDir()
dataDir := t.TempDir()
cache, err := NewHubCache(cacheDir, time.Hour)
require.NoError(t, err)
archive := makeTestArchive(t, map[string]string{"test.yaml": "content"})
hub := NewHubService(nil, cache, dataDir)
hub.HubBaseURL = "http://test.example.com"
hub.HTTPClient = &http.Client{
Transport: mockTransport(func(req *http.Request) (*http.Response, error) {
switch req.URL.String() {
case "http://test.example.com/api/index.json":
body := `{"items":[{"name":"preset1","title":"Preset 1","etag":"e1"}]}`
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body)), Header: make(http.Header)}, nil
case "http://test.example.com/preset1.yaml":
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("preview1")), Header: make(http.Header)}, nil
case "http://test.example.com/preset1.tgz":
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(archive)), Header: make(http.Header)}, nil
default:
return &http.Response{StatusCode: http.StatusNotFound, Body: io.NopCloser(strings.NewReader("")), Header: make(http.Header)}, nil
}
}),
}
ctx := context.Background()
// Pull preset
_, err = hub.Pull(ctx, "preset1")
require.NoError(t, err)
// List cache contents
cached, err := cache.List(ctx)
require.NoError(t, err)
require.Len(t, cached, 1, "Should have one cached preset")
require.Equal(t, "preset1", cached[0].Slug)
}
// makeTestArchive creates a test tar.gz archive.
func makeTestArchive(t *testing.T, files map[string]string) []byte {
t.Helper()
buf := &bytes.Buffer{}
gw := gzip.NewWriter(buf)
tw := tar.NewWriter(gw)
for name, content := range files {
hdr := &tar.Header{
Name: name,
Mode: 0o644,
Size: int64(len(content)),
}
require.NoError(t, tw.WriteHeader(hdr))
_, err := tw.Write([]byte(content))
require.NoError(t, err)
}
require.NoError(t, tw.Close())
require.NoError(t, gw.Close())
return buf.Bytes()
}
// mockTransport is a mock http.RoundTripper for testing.
type mockTransport func(*http.Request) (*http.Response, error)
func (m mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return m(req)
}
// TestApplyReadsArchiveBeforeBackup verifies the fix for the bug where Apply() would:
// 1. Load cache metadata (getting archive path inside DataDir/hub_cache)
// 2. Backup DataDir (moving the cache including the archive!)
// 3. Try to read archive from original path (FAIL - file no longer exists!)
//
// The fix reads the archive into memory BEFORE creating the backup, so the
// archive data is preserved even after the backup operation moves the files.
func TestApplyReadsArchiveBeforeBackup(t *testing.T) {
// Create base directory structure that mirrors production:
// baseDir/
// └── crowdsec/ <- DataDir (gets backed up)
// └── hub_cache/ <- Cache lives INSIDE DataDir (the bug!)
// └── test/preset/
// ├── bundle.tgz
// ├── preview.yaml
// └── metadata.json
baseDir := t.TempDir()
dataDir := filepath.Join(baseDir, "crowdsec")
cacheDir := filepath.Join(dataDir, "hub_cache") // Cache INSIDE DataDir - this is key!
// Create DataDir with some existing config to make backup realistic
require.NoError(t, os.MkdirAll(dataDir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dataDir, "config.yaml"), []byte("existing: config"), 0o644))
// Create cache inside DataDir
cache, err := NewHubCache(cacheDir, time.Hour)
require.NoError(t, err)
// Create test archive
archive := makeTestArchive(t, map[string]string{
"config.yaml": "test: applied_config\nvalue: 123",
"profiles.yaml": "name: test_profile",
})
// Pre-populate cache (simulating a prior Pull operation)
ctx := context.Background()
cachedMeta, err := cache.Store(ctx, "test/preset", "etag-pre", "hub", "preview: content", archive)
require.NoError(t, err)
require.FileExists(t, cachedMeta.ArchivePath, "Archive should exist in cache")
// Verify cache is inside DataDir (the scenario that triggers the bug)
require.True(t, strings.HasPrefix(cachedMeta.ArchivePath, dataDir),
"Cache archive must be inside DataDir for this test to be valid")
// Create hub service WITHOUT cscli (nil executor) to force cache fallback path
hub := NewHubService(nil, cache, dataDir)
hub.HubBaseURL = "http://test.example.com"
// HTTP client that fails everything - we don't want to hit network
hub.HTTPClient = &http.Client{
Transport: mockTransport(func(req *http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: http.StatusInternalServerError,
Body: io.NopCloser(strings.NewReader("intentionally failing")),
Header: make(http.Header),
}, nil
}),
}
// Apply - this SHOULD succeed because:
// 1. Archive is read into memory BEFORE backup
// 2. Backup moves DataDir (including cache) but we already have archive bytes
// 3. Extract uses the in-memory archive bytes
//
// BEFORE THE FIX, this would fail with:
// "read archive: open /tmp/.../crowdsec/hub_cache/.../bundle.tgz: no such file or directory"
result, err := hub.Apply(ctx, "test/preset")
require.NoError(t, err, "Apply should succeed - archive must be read before backup")
require.Equal(t, "applied", result.Status)
require.NotEmpty(t, result.BackupPath, "Backup should have been created")
require.False(t, result.UsedCSCLI, "Should have used cache fallback, not cscli")
// Verify backup was created
_, statErr := os.Stat(result.BackupPath)
require.NoError(t, statErr, "Backup directory should exist")
// Verify files were extracted to DataDir
extractedConfig := filepath.Join(dataDir, "config.yaml")
require.FileExists(t, extractedConfig, "Config should be extracted")
content, err := os.ReadFile(extractedConfig)
require.NoError(t, err)
require.Contains(t, string(content), "test: applied_config",
"Extracted config should contain content from archive, not original")
}