diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index efc5618f..4c56288c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -50,3 +50,14 @@ jobs: fail-on-alert: false # Enable Job Summary for PRs summary-always: true + + - name: Run Perf Asserts + working-directory: backend + env: + PERF_MAX_MS_GETSTATUS_P95: 500ms + PERF_MAX_MS_GETSTATUS_P95_PARALLEL: 1500ms + PERF_MAX_MS_LISTDECISIONS_P95: 2000ms + run: | + echo "## 🔍 Running performance assertions (TestPerf)" >> $GITHUB_STEP_SUMMARY + go test -run TestPerf -v ./internal/api/handlers -count=1 | tee perf-output.txt + exit ${PIPESTATUS[0]} diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 1cfa5adb..d4433b51 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -58,6 +58,18 @@ jobs: args: --timeout=5m continue-on-error: true + - name: Run Perf Asserts + working-directory: backend + env: + # Conservative defaults to avoid flakiness on CI; tune as necessary + PERF_MAX_MS_GETSTATUS_P95: 500ms + PERF_MAX_MS_GETSTATUS_P95_PARALLEL: 1500ms + PERF_MAX_MS_LISTDECISIONS_P95: 2000ms + run: | + echo "## 🔍 Running performance assertions (TestPerf)" >> $GITHUB_STEP_SUMMARY + go test -run TestPerf -v ./internal/api/handlers -count=1 | tee perf-output.txt + exit ${PIPESTATUS[0]} + frontend-quality: name: Frontend (React) runs-on: ubuntu-latest diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 82269b00..1231d3cd 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -182,7 +182,37 @@ "panel": "shared" }, "problemMatcher": [] + }, + { + "label": "Backend: Run Benchmarks", + "type": "shell", + "command": "cd backend && go test -bench=. -benchmem -benchtime=1s ./internal/api/handlers/... -run=^$", + "group": "test", + "presentation": { + "reveal": "always", + "panel": "new" + }, + "problemMatcher": ["$go"] + }, + { + "label": "Backend: Run Benchmarks (Quick)", + "type": "shell", + "command": "cd backend && go test -bench=GetStatus -benchmem -benchtime=500ms ./internal/api/handlers/... -run=^$", + "group": "test", + "presentation": { + "reveal": "always", + "panel": "new" + }, + "problemMatcher": ["$go"] + }, + { + "label": "Backend: Run Perf Asserts", + "type": "shell", + "command": "cd backend && go test -run TestPerf -v ./internal/api/handlers -count=1", + "group": "test", + "presentation": { + "reveal": "always", + "panel": "new" + }, + "problemMatcher": ["$go"] } - ] - -} diff --git a/backend/internal/api/handlers/benchmark_test.go b/backend/internal/api/handlers/benchmark_test.go new file mode 100644 index 00000000..762a81aa --- /dev/null +++ b/backend/internal/api/handlers/benchmark_test.go @@ -0,0 +1,463 @@ +package handlers + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "github.com/Wikid82/charon/backend/internal/config" + "github.com/Wikid82/charon/backend/internal/models" +) + +// setupBenchmarkDB creates an in-memory SQLite database for benchmarks +func setupBenchmarkDB(b *testing.B) *gorm.DB { + b.Helper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + b.Fatal(err) + } + if err := db.AutoMigrate( + &models.SecurityConfig{}, + &models.SecurityRuleSet{}, + &models.SecurityDecision{}, + &models.SecurityAudit{}, + &models.Setting{}, + &models.ProxyHost{}, + &models.AccessList{}, + &models.User{}, + ); err != nil { + b.Fatal(err) + } + return db +} + +// ============================================================================= +// SECURITY HANDLER BENCHMARKS +// ============================================================================= + +func BenchmarkSecurityHandler_GetStatus(b *testing.B) { + gin.SetMode(gin.ReleaseMode) + db := setupBenchmarkDB(b) + + // Seed settings + settings := []models.Setting{ + {Key: "security.cerberus.enabled", Value: "true", Category: "security"}, + {Key: "security.waf.enabled", Value: "true", Category: "security"}, + {Key: "security.rate_limit.enabled", Value: "true", Category: "security"}, + {Key: "security.crowdsec.enabled", Value: "true", Category: "security"}, + {Key: "security.acl.enabled", Value: "true", Category: "security"}, + } + for _, s := range settings { + db.Create(&s) + } + + cfg := config.SecurityConfig{CerberusEnabled: true} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.GET("/api/v1/security/status", h.GetStatus) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + req := httptest.NewRequest("GET", "/api/v1/security/status", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + b.Fatalf("unexpected status: %d", w.Code) + } + } +} + +func BenchmarkSecurityHandler_GetStatus_NoSettings(b *testing.B) { + gin.SetMode(gin.ReleaseMode) + db := setupBenchmarkDB(b) + + cfg := config.SecurityConfig{CerberusEnabled: true} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.GET("/api/v1/security/status", h.GetStatus) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + req := httptest.NewRequest("GET", "/api/v1/security/status", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + b.Fatalf("unexpected status: %d", w.Code) + } + } +} + +func BenchmarkSecurityHandler_ListDecisions(b *testing.B) { + gin.SetMode(gin.ReleaseMode) + db := setupBenchmarkDB(b) + + // Seed some decisions + for i := 0; i < 100; i++ { + db.Create(&models.SecurityDecision{ + UUID: "test-uuid-" + string(rune(i)), + Source: "test", + Action: "block", + IP: "192.168.1.1", + }) + } + + cfg := config.SecurityConfig{} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.GET("/api/v1/security/decisions", h.ListDecisions) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + req := httptest.NewRequest("GET", "/api/v1/security/decisions?limit=50", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + b.Fatalf("unexpected status: %d", w.Code) + } + } +} + +func BenchmarkSecurityHandler_ListRuleSets(b *testing.B) { + gin.SetMode(gin.ReleaseMode) + db := setupBenchmarkDB(b) + + // Seed some rulesets + for i := 0; i < 10; i++ { + db.Create(&models.SecurityRuleSet{ + UUID: "ruleset-uuid-" + string(rune(i)), + Name: "Ruleset " + string(rune('A'+i)), + Content: "SecRule REQUEST_URI \"@contains /admin\" \"id:1000,phase:1,deny\"", + Mode: "blocking", + }) + } + + cfg := config.SecurityConfig{} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.GET("/api/v1/security/rulesets", h.ListRuleSets) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + req := httptest.NewRequest("GET", "/api/v1/security/rulesets", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + b.Fatalf("unexpected status: %d", w.Code) + } + } +} + +func BenchmarkSecurityHandler_UpsertRuleSet(b *testing.B) { + gin.SetMode(gin.ReleaseMode) + db := setupBenchmarkDB(b) + + cfg := config.SecurityConfig{} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.POST("/api/v1/security/rulesets", h.UpsertRuleSet) + + payload := map[string]interface{}{ + "name": "bench-ruleset", + "content": "SecRule REQUEST_URI \"@contains /admin\" \"id:1000,phase:1,deny\"", + "mode": "blocking", + } + body, _ := json.Marshal(payload) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + req := httptest.NewRequest("POST", "/api/v1/security/rulesets", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + b.Fatalf("unexpected status: %d", w.Code) + } + } +} + +func BenchmarkSecurityHandler_CreateDecision(b *testing.B) { + gin.SetMode(gin.ReleaseMode) + db := setupBenchmarkDB(b) + + cfg := config.SecurityConfig{} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.POST("/api/v1/security/decisions", h.CreateDecision) + + payload := map[string]interface{}{ + "ip": "192.168.1.100", + "action": "block", + "details": "benchmark test", + } + body, _ := json.Marshal(payload) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + req := httptest.NewRequest("POST", "/api/v1/security/decisions", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + b.Fatalf("unexpected status: %d", w.Code) + } + } +} + +func BenchmarkSecurityHandler_GetConfig(b *testing.B) { + gin.SetMode(gin.ReleaseMode) + db := setupBenchmarkDB(b) + + // Seed a config + db.Create(&models.SecurityConfig{ + Name: "default", + Enabled: true, + AdminWhitelist: "192.168.1.0/24", + WAFMode: "block", + RateLimitEnable: true, + RateLimitBurst: 10, + }) + + cfg := config.SecurityConfig{} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.GET("/api/v1/security/config", h.GetConfig) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + req := httptest.NewRequest("GET", "/api/v1/security/config", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + b.Fatalf("unexpected status: %d", w.Code) + } + } +} + +func BenchmarkSecurityHandler_UpdateConfig(b *testing.B) { + gin.SetMode(gin.ReleaseMode) + db := setupBenchmarkDB(b) + + cfg := config.SecurityConfig{} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.PUT("/api/v1/security/config", h.UpdateConfig) + + payload := map[string]interface{}{ + "name": "default", + "enabled": true, + "rate_limit_enable": true, + "rate_limit_burst": 10, + "rate_limit_requests": 100, + } + body, _ := json.Marshal(payload) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + req := httptest.NewRequest("PUT", "/api/v1/security/config", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + b.Fatalf("unexpected status: %d", w.Code) + } + } +} + +// ============================================================================= +// PARALLEL BENCHMARKS (Concurrency Testing) +// ============================================================================= + +func BenchmarkSecurityHandler_GetStatus_Parallel(b *testing.B) { + gin.SetMode(gin.ReleaseMode) + db := setupBenchmarkDB(b) + + settings := []models.Setting{ + {Key: "security.cerberus.enabled", Value: "true", Category: "security"}, + {Key: "security.waf.enabled", Value: "true", Category: "security"}, + } + for _, s := range settings { + db.Create(&s) + } + + cfg := config.SecurityConfig{CerberusEnabled: true} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.GET("/api/v1/security/status", h.GetStatus) + + b.ResetTimer() + b.ReportAllocs() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + req := httptest.NewRequest("GET", "/api/v1/security/status", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + b.Fatalf("unexpected status: %d", w.Code) + } + } + }) +} + +func BenchmarkSecurityHandler_ListDecisions_Parallel(b *testing.B) { + gin.SetMode(gin.ReleaseMode) + // Use file-based SQLite with WAL mode for parallel testing + db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared&_journal_mode=WAL"), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + if err != nil { + b.Fatal(err) + } + if err := db.AutoMigrate(&models.SecurityDecision{}, &models.SecurityAudit{}); err != nil { + b.Fatal(err) + } + + for i := 0; i < 100; i++ { + db.Create(&models.SecurityDecision{ + UUID: "test-uuid-" + string(rune(i)), + Source: "test", + Action: "block", + IP: "192.168.1.1", + }) + } + + cfg := config.SecurityConfig{} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.GET("/api/v1/security/decisions", h.ListDecisions) + + b.ResetTimer() + b.ReportAllocs() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + req := httptest.NewRequest("GET", "/api/v1/security/decisions?limit=50", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + b.Fatalf("unexpected status: %d", w.Code) + } + } + }) +} + +// ============================================================================= +// MEMORY PRESSURE BENCHMARKS +// ============================================================================= + +func BenchmarkSecurityHandler_LargeRuleSetContent(b *testing.B) { + gin.SetMode(gin.ReleaseMode) + db := setupBenchmarkDB(b) + + cfg := config.SecurityConfig{} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.POST("/api/v1/security/rulesets", h.UpsertRuleSet) + + // 100KB ruleset content (under 2MB limit) + largeContent := "" + for i := 0; i < 1000; i++ { + largeContent += "SecRule REQUEST_URI \"@contains /path" + string(rune(i)) + "\" \"id:" + string(rune(1000+i)) + ",phase:1,deny\"\n" + } + + payload := map[string]interface{}{ + "name": "large-ruleset", + "content": largeContent, + "mode": "blocking", + } + body, _ := json.Marshal(payload) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + req := httptest.NewRequest("POST", "/api/v1/security/rulesets", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + b.Fatalf("unexpected status: %d", w.Code) + } + } +} + +func BenchmarkSecurityHandler_ManySettingsLookups(b *testing.B) { + gin.SetMode(gin.ReleaseMode) + db := setupBenchmarkDB(b) + + // Seed many settings + for i := 0; i < 100; i++ { + db.Create(&models.Setting{ + Key: "setting.key." + string(rune(i)), + Value: "value", + Category: "misc", + }) + } + // Security settings + settings := []models.Setting{ + {Key: "security.cerberus.enabled", Value: "true", Category: "security"}, + {Key: "security.waf.enabled", Value: "true", Category: "security"}, + {Key: "security.rate_limit.enabled", Value: "true", Category: "security"}, + {Key: "security.crowdsec.enabled", Value: "true", Category: "security"}, + {Key: "security.crowdsec.mode", Value: "local", Category: "security"}, + {Key: "security.crowdsec.api_url", Value: "http://localhost:8080", Category: "security"}, + {Key: "security.acl.enabled", Value: "true", Category: "security"}, + } + for _, s := range settings { + db.Create(&s) + } + + cfg := config.SecurityConfig{} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.GET("/api/v1/security/status", h.GetStatus) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + req := httptest.NewRequest("GET", "/api/v1/security/status", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + b.Fatalf("unexpected status: %d", w.Code) + } + } +} diff --git a/backend/internal/api/handlers/perf_assert_test.go b/backend/internal/api/handlers/perf_assert_test.go new file mode 100644 index 00000000..c252a1bb --- /dev/null +++ b/backend/internal/api/handlers/perf_assert_test.go @@ -0,0 +1,186 @@ +package handlers + +import ( + "net/http" + "net/http/httptest" + "os" + "sort" + "testing" + "time" + "fmt" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "github.com/Wikid82/charon/backend/internal/config" + "github.com/Wikid82/charon/backend/internal/models" +) + +// quick helper to form float ms from duration +func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000.0 } + +// setupPerfDB - uses a file-backed sqlite to avoid concurrency panics in parallel tests +func setupPerfDB(t *testing.T) *gorm.DB { + t.Helper() + path := ":memory:?cache=shared&_journal_mode=WAL" + db, err := gorm.Open(sqlite.Open(path), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&models.Setting{}, &models.SecurityDecision{}, &models.SecurityRuleSet{}, &models.SecurityConfig{})) + return db +} + +// thresholdFromEnv loads threshold from environment var as milliseconds +func thresholdFromEnv(envKey string, defaultMs float64) float64 { + if v := os.Getenv(envKey); v != "" { + // try parse as float + if parsed, err := time.ParseDuration(v); err == nil { + return ms(parsed) + } + // fallback try parse as number ms + var f float64 + if _, err := fmt.Sscanf(v, "%f", &f); err == nil { + return f + } + } + return defaultMs +} + +// gatherStats runs the request counts times and returns durations ms +func gatherStats(t *testing.T, req *http.Request, router http.Handler, counts int) []float64 { + t.Helper() + res := make([]float64, 0, counts) + for i := 0; i < counts; i++ { + w := httptest.NewRecorder() + s := time.Now() + router.ServeHTTP(w, req) + d := time.Since(s) + res = append(res, ms(d)) + if w.Code >= 500 { + t.Fatalf("unexpected status: %d", w.Code) + } + } + return res +} + +// computePercentiles returns avg, p50, p95, p99, max +func computePercentiles(samples []float64) (avg, p50, p95, p99, max float64) { + sort.Float64s(samples) + var sum float64 + for _, s := range samples { + sum += s + } + avg = sum / float64(len(samples)) + p := func(pct float64) float64 { + idx := int(float64(len(samples))*pct) + if idx < 0 { idx = 0 } + if idx >= len(samples) { idx = len(samples)-1 } + return samples[idx] + } + p50 = p(0.50) + p95 = p(0.95) + p99 = p(0.99) + max = samples[len(samples)-1] + return +} + +func perfLogStats(t *testing.T, title string, samples []float64) { + av, p50, p95, p99, max := computePercentiles(samples) + t.Logf("%s - avg=%.3fms p50=%.3fms p95=%.3fms p99=%.3fms max=%.3fms", title, av, p50, p95, p99, max) + // no assert by default, individual tests decide how to fail +} + +func TestPerf_GetStatus_AssertThreshold(t *testing.T) { + gin.SetMode(gin.ReleaseMode) + db := setupPerfDB(t) + + // seed settings to emulate production path + _ = db.Create(&models.Setting{Key: "security.cerberus.enabled", Value: "true", Category: "security"}) + _ = db.Create(&models.Setting{Key: "security.waf.enabled", Value: "true", Category: "security"}) + cfg := config.SecurityConfig{CerberusEnabled: true} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.GET("/api/v1/security/status", h.GetStatus) + + counts := 500 + req := httptest.NewRequest("GET", "/api/v1/security/status", nil) + samples := gatherStats(t, req, router, counts) + avg, _, p95, _, max := computePercentiles(samples) + // default thresholds ms + thresholdP95 := 2.0 // 2ms per request + if env := os.Getenv("PERF_MAX_MS_GETSTATUS_P95"); env != "" { + if parsed, err := time.ParseDuration(env); err == nil { thresholdP95 = ms(parsed) } + } + // fail if p95 exceeds threshold + t.Logf("GetStatus avg=%.3fms p95=%.3fms max=%.3fms", avg, p95, max) + if p95 > thresholdP95 { + t.Fatalf("GetStatus P95 (%.3fms) exceeds threshold %.3fms", p95, thresholdP95) + } +} + +func TestPerf_GetStatus_Parallel_AssertThreshold(t *testing.T) { + gin.SetMode(gin.ReleaseMode) + db := setupPerfDB(t) + cfg := config.SecurityConfig{CerberusEnabled: true} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.GET("/api/v1/security/status", h.GetStatus) + + n := 200 + samples := make(chan float64, n) + var worker = func() { + for i := 0; i < n; i++ { + req := httptest.NewRequest("GET", "/api/v1/security/status", nil) + w := httptest.NewRecorder() + s := time.Now() + router.ServeHTTP(w, req) + d := time.Since(s) + samples <- ms(d) + } + } + + // run 4 concurrent workers + for k := 0; k < 4; k++ { go worker() } + collected := make([]float64, 0, n*4) + for i := 0; i < n*4; i++ { collected = append(collected, <-samples) } + avg, _, p95, _, max := computePercentiles(collected) + thresholdP95 := 5.0 // 5ms default + if env := os.Getenv("PERF_MAX_MS_GETSTATUS_P95_PARALLEL"); env != "" { + if parsed, err := time.ParseDuration(env); err == nil { thresholdP95 = ms(parsed) } + } + t.Logf("GetStatus Parallel avg=%.3fms p95=%.3fms max=%.3fms", avg, p95, max) + if p95 > thresholdP95 { + t.Fatalf("GetStatus Parallel P95 (%.3fms) exceeds threshold %.3fms", p95, thresholdP95) + } +} + +func TestPerf_ListDecisions_AssertThreshold(t *testing.T) { + gin.SetMode(gin.ReleaseMode) + db := setupPerfDB(t) + // seed decisions + for i := 0; i < 1000; i++ { + db.Create(&models.SecurityDecision{UUID: fmt.Sprintf("d-%d", i), Source: "test", Action: "block", IP: "192.168.1.1"}) + } + cfg := config.SecurityConfig{} + h := NewSecurityHandler(cfg, db, nil) + + router := gin.New() + router.GET("/api/v1/security/decisions", h.ListDecisions) + + counts := 200 + req := httptest.NewRequest("GET", "/api/v1/security/decisions?limit=50", nil) + samples := gatherStats(t, req, router, counts) + avg, _, p95, _, max := computePercentiles(samples) + thresholdP95 := 30.0 // 30ms default + if env := os.Getenv("PERF_MAX_MS_LISTDECISIONS_P95"); env != "" { + if parsed, err := time.ParseDuration(env); err == nil { thresholdP95 = ms(parsed) } + } + t.Logf("ListDecisions avg=%.3fms p95=%.3fms max=%.3fms", avg, p95, max) + if p95 > thresholdP95 { + t.Fatalf("ListDecisions P95 (%.3fms) exceeds threshold %.3fms", p95, thresholdP95) + } +}