Add TTL cache, stream logs, and refactor handlers

Introduce a simple in-memory TTL cache and integrate it across API handlers to reduce repeated cscli/docker calls and improve performance. Key changes:

- Add internal/cache: a concurrency-safe TTLCache with Get/Set.
- Instantiate cache in cmd/server and pass into RegisterServicesRoutes; handlers accept optional TTLCache.
- Cache decisions (including a lightweight summary), metrics, and alerts analysis with short TTLs.
- Add CLIFlag and appendCLIFlags helpers in handlers/common.go and refactor decision add/delete handlers to use them.
- Replace file copy helper in ImportDecisions with writing file bytes into the container via WriteFileToContainer.
- Add getExternalIP helper to try multiple external IP services; use it in GetPublicIP and WhitelistCurrentIP.
- Add FollowContainerLogs to the Docker client and update StreamLogs to follow container logs natively (streaming ReadCloser), simplifying polling logic and reducing duplicate-send logic.
- Web UI: add getDecisionsSummary API call and update Dashboard to use the summary endpoint and reduce auto-refresh frequency/labels.

These changes aim to reduce load, improve responsiveness, simplify log streaming, and centralize common helpers.

Author: hhftechnology

cmd/server/main.go

@@ -18,6 +18,7 @@ import (
 	"crowdsec-manager/internal/api/handlers"
 	"crowdsec-manager/internal/api/middleware"
 	"crowdsec-manager/internal/backup"
+	"crowdsec-manager/internal/cache"
 	"crowdsec-manager/internal/config"
 	"crowdsec-manager/internal/configvalidator"
 	"crowdsec-manager/internal/cron"
@@ -132,7 +133,8 @@ func main() {
 		api.RegisterBackupRoutes(apiGroup, backupManager, dockerClient)
 		api.RegisterUpdateRoutes(apiGroup, dockerClient, cfg)
 		api.RegisterCronRoutes(apiGroup, cronScheduler)
-		api.RegisterServicesRoutes(apiGroup, dockerClient, db, cfg)
+		ttlCache := cache.New()
+		api.RegisterServicesRoutes(apiGroup, dockerClient, db, cfg, ttlCache)
 		api.RegisterNotificationRoutes(apiGroup, dockerClient, db, cfg)
 		api.RegisterProfileRoutes(apiGroup, db, cfg, dockerClient)
 		api.RegisterHostRoutes(apiGroup, multiHost)

internal/api/handlers/common.go

@@ -2,9 +2,12 @@ package handlers
 
 import (
 	"encoding/json"
+	"fmt"
+	"io"
 	"strings"
 	"time"
 
+	"crowdsec-manager/internal/constants"
 	"crowdsec-manager/internal/docker"
 	"crowdsec-manager/internal/logger"
 	"crowdsec-manager/internal/models"
@@ -195,3 +198,46 @@ func errString(err error) string {
 	}
 	return err.Error()
 }
+
+// CLIFlag represents a CLI flag and its value for building cscli commands.
+type CLIFlag struct {
+	Flag  string
+	Value string
+}
+
+// appendCLIFlags appends non-empty flag/value pairs to a command slice.
+// Returns the extended command and the number of flags added.
+func appendCLIFlags(cmd []string, flags []CLIFlag) ([]string, int) {
+	count := 0
+	for _, f := range flags {
+		if f.Value != "" {
+			cmd = append(cmd, f.Flag, f.Value)
+			count++
+		}
+	}
+	return cmd, count
+}
+
+// getExternalIP tries each external IP service in order and returns the first
+// successful result. Used by both GetPublicIP and WhitelistCurrentIP.
+func getExternalIP() (string, error) {
+	var lastErr error
+	for _, service := range constants.ExternalIPServices {
+		resp, err := constants.ExternalHTTPClient.Get(service)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		body, err := io.ReadAll(resp.Body)
+		resp.Body.Close()
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		ip := strings.TrimSpace(string(body))
+		if ip != "" {
+			return ip, nil
+		}
+	}
+	return "", fmt.Errorf("all IP services failed: %w", lastErr)
+}

internal/api/handlers/dashboard.go

@@ -1,17 +1,19 @@
 package handlers
 
 import (
-	"crowdsec-manager/internal/config"
-	"crowdsec-manager/internal/database"
-	"crowdsec-manager/internal/docker"
-	"crowdsec-manager/internal/logger"
-	"crowdsec-manager/internal/models"
 	"encoding/json"
 	"fmt"
 	"net/http"
 	"strings"
 	"time"
 
+	"crowdsec-manager/internal/cache"
+	"crowdsec-manager/internal/config"
+	"crowdsec-manager/internal/database"
+	"crowdsec-manager/internal/docker"
+	"crowdsec-manager/internal/logger"
+	"crowdsec-manager/internal/models"
+
 	"github.com/buger/jsonparser"
 	"github.com/gin-gonic/gin"
 )
@@ -21,10 +23,26 @@ import (
 // =============================================================================
 
 // GetDecisions retrieves CrowdSec decisions
-// GetDecisions retrieves CrowdSec decisions
-func GetDecisions(dockerClient *docker.Client, cfg *config.Config) gin.HandlerFunc {
+func GetDecisions(dockerClient *docker.Client, cfg *config.Config, ttlCache ...*cache.TTLCache) gin.HandlerFunc {
 	return func(c *gin.Context) {
 		dockerClient = resolveDockerClient(c, dockerClient)
+
+		// Check cache first
+		summary := c.Query("summary") == "true"
+		cacheKey := "decisions"
+		if summary {
+			cacheKey = "decisions-summary"
+		}
+		if len(ttlCache) > 0 && ttlCache[0] != nil {
+			if cached, ok := ttlCache[0].Get(cacheKey); ok {
+				c.JSON(http.StatusOK, models.Response{
+					Success: true,
+					Data:    cached,
+				})
+				return
+			}
+		}
+
 		logger.Info("Getting CrowdSec decisions via cscli")
 
 		output, err := dockerClient.ExecCommand(cfg.CrowdsecContainerName, []string{
@@ -120,18 +138,61 @@ func GetDecisions(dockerClient *docker.Client, cfg *config.Config) gin.HandlerFu
 
 		logger.Debug("Decisions retrieved successfully", "count", len(decisions))
 
+		// Summary mode: return only count and lightweight aggregations
+		if summary {
+			typeDistribution := make(map[string]int)
+			topScenarios := make(map[string]int)
+			for _, d := range decisions {
+				if d.Type != "" {
+					typeDistribution[d.Type]++
+				}
+				if d.Scenario != "" {
+					topScenarios[d.Scenario]++
+				}
+			}
+			result := gin.H{
+				"count":     len(decisions),
+				"types":     typeDistribution,
+				"scenarios": topScenarios,
+			}
+			if len(ttlCache) > 0 && ttlCache[0] != nil {
+				ttlCache[0].Set(cacheKey, result, 15*time.Second)
+			}
+			c.JSON(http.StatusOK, models.Response{
+				Success: true,
+				Data:    result,
+			})
+			return
+		}
+
+		result := gin.H{"decisions": decisions, "count": len(decisions)}
+		if len(ttlCache) > 0 && ttlCache[0] != nil {
+			ttlCache[0].Set(cacheKey, result, 15*time.Second)
+		}
+
 		c.JSON(http.StatusOK, models.Response{
 			Success: true,
-			Data:    gin.H{"decisions": decisions, "count": len(decisions)},
+			Data:    result,
 		})
 	}
 }
 
 // GetMetrics retrieves CrowdSec metrics
-// GetMetrics retrieves CrowdSec metrics
-func GetMetrics(dockerClient *docker.Client, cfg *config.Config) gin.HandlerFunc {
+func GetMetrics(dockerClient *docker.Client, cfg *config.Config, ttlCache ...*cache.TTLCache) gin.HandlerFunc {
 	return func(c *gin.Context) {
 		dockerClient = resolveDockerClient(c, dockerClient)
+
+		cacheKey := "metrics"
+		if len(ttlCache) > 0 && ttlCache[0] != nil {
+			if cached, ok := ttlCache[0].Get(cacheKey); ok {
+				c.JSON(http.StatusOK, models.Response{
+					Success: true,
+					Data:    cached,
+				})
+				return
+			}
+		}
+
 		logger.Info("Getting CrowdSec metrics")
 
 		output, err := dockerClient.ExecCommand(cfg.CrowdsecContainerName, []string{
@@ -145,7 +206,6 @@ func GetMetrics(dockerClient *docker.Client, cfg *config.Config) gin.HandlerFunc
 			return
 		}
 
-		// Parse as raw JSON
 		var metrics interface{}
 		if err := json.Unmarshal([]byte(output), &metrics); err != nil {
 			logger.Warn("Failed to parse metrics JSON", "error", err)
@@ -156,6 +216,10 @@ func GetMetrics(dockerClient *docker.Client, cfg *config.Config) gin.HandlerFunc
 			return
 		}
 
+		if len(ttlCache) > 0 && ttlCache[0] != nil {
+			ttlCache[0].Set(cacheKey, metrics, 30*time.Second)
+		}
+
 		c.JSON(http.StatusOK, models.Response{
 			Success: true,
 			Data:    metrics,

internal/api/handlers/dashboard_analysis.go

@@ -4,7 +4,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"time"
 
+	"crowdsec-manager/internal/cache"
 	"crowdsec-manager/internal/config"
 	"crowdsec-manager/internal/docker"
 	"crowdsec-manager/internal/logger"
@@ -152,9 +154,22 @@ func GetDecisionsAnalysis(dockerClient *docker.Client, cfg *config.Config) gin.H
 }
 
 // GetAlertsAnalysis retrieves CrowdSec alerts with advanced filtering
-func GetAlertsAnalysis(dockerClient *docker.Client, cfg *config.Config) gin.HandlerFunc {
+func GetAlertsAnalysis(dockerClient *docker.Client, cfg *config.Config, ttlCache ...*cache.TTLCache) gin.HandlerFunc {
 	return func(c *gin.Context) {
 		dockerClient = resolveDockerClient(c, dockerClient)
+
+		// Cache key includes the "since" param to differentiate dashboard vs analysis queries
+		cacheKey := "alerts-analysis-" + c.Query("since")
+		if len(ttlCache) > 0 && ttlCache[0] != nil {
+			if cached, ok := ttlCache[0].Get(cacheKey); ok {
+				c.JSON(http.StatusOK, models.Response{
+					Success: true,
+					Data:    cached,
+				})
+				return
+			}
+		}
+
 		logger.Info("Getting CrowdSec alerts analysis via cscli")
 
 		var cmd []string
@@ -233,9 +248,14 @@ func GetAlertsAnalysis(dockerClient *docker.Client, cfg *config.Config) gin.Hand
 
 		logger.Info("Alerts analysis retrieved successfully", "count", len(alerts))
 
+		result := gin.H{"alerts": alerts, "count": len(alerts)}
+		if len(ttlCache) > 0 && ttlCache[0] != nil {
+			ttlCache[0].Set(cacheKey, result, 30*time.Second)
+		}
+
 		c.JSON(http.StatusOK, models.Response{
 			Success: true,
-			Data:    gin.H{"alerts": alerts, "count": len(alerts)},
+			Data:    result,
 		})
 	}
 }

internal/api/handlers/decisions.go

@@ -37,21 +37,15 @@ func AddDecision(dockerClient *docker.Client, cfg *config.Config) gin.HandlerFun
 		}
 
 		cmd := []string{"cscli", "decisions", "add"}
-
-		// Helper to add flag if value is present
-		addFlag := func(flag, value string) {
-			if value != "" {
-				cmd = append(cmd, flag, value)
-			}
-		}
-
-		addFlag("--ip", req.IP)
-		addFlag("--range", req.Range)
-		addFlag("--duration", req.Duration)
-		addFlag("--type", req.Type)
-		addFlag("--scope", req.Scope)
-		addFlag("--value", req.Value)
-		addFlag("--reason", req.Reason)
+		cmd, _ = appendCLIFlags(cmd, []CLIFlag{
+			{"--ip", req.IP},
+			{"--range", req.Range},
+			{"--duration", req.Duration},
+			{"--type", req.Type},
+			{"--scope", req.Scope},
+			{"--value", req.Value},
+			{"--reason", req.Reason},
+		})
 
 		logger.Info("Adding decision", "command", cmd)
 
@@ -99,25 +93,19 @@ func DeleteDecision(dockerClient *docker.Client, cfg *config.Config) gin.Handler
 		}
 
 		cmd := []string{"cscli", "decisions", "delete"}
-		hasFilter := false
-
-		addFlag := func(flag, value string) {
-			if value != "" {
-				cmd = append(cmd, flag, value)
-				hasFilter = true
-			}
-		}
-
-		addFlag("--id", req.ID)
-		addFlag("--ip", req.IP)
-		addFlag("--range", req.Range)
-		addFlag("--type", req.Type)
-		addFlag("--scope", req.Scope)
-		addFlag("--value", req.Value)
-		addFlag("--scenario", req.Scenario)
-		addFlag("--origin", req.Origin)
+		var count int
+		cmd, count = appendCLIFlags(cmd, []CLIFlag{
+			{"--id", req.ID},
+			{"--ip", req.IP},
+			{"--range", req.Range},
+			{"--type", req.Type},
+			{"--scope", req.Scope},
+			{"--value", req.Value},
+			{"--scenario", req.Scenario},
+			{"--origin", req.Origin},
+		})
 
-		if !hasFilter {
+		if count == 0 {
 			c.JSON(http.StatusBadRequest, models.Response{
 				Success: false,
 				Error:   "At least one filter (id, ip, range, etc.) must be provided to delete decisions",

internal/api/handlers/decisions_import.go

@@ -117,7 +117,16 @@ func ImportDecisions(dockerClient *docker.Client, cfg *config.Config) gin.Handle
 		// Write to container
 		containerFile := "/tmp/decisions_import.csv"
 
-		err = copyToContainer(dockerClient, cfg.CrowdsecContainerName, tempFilePath, containerFile)
+		content, err := os.ReadFile(tempFilePath)
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, models.Response{
+				Success: false,
+				Error:   "Failed to read temp file: " + err.Error(),
+			})
+			return
+		}
+
+		err = dockerClient.WriteFileToContainer(cfg.CrowdsecContainerName, containerFile, content)
 		if err != nil {
 			c.JSON(http.StatusInternalServerError, models.Response{
 				Success: false,
@@ -148,14 +157,3 @@ func ImportDecisions(dockerClient *docker.Client, cfg *config.Config) gin.Handle
 		})
 	}
 }
-
-// copyToContainer is a helper to copy a file to the container
-func copyToContainer(client *docker.Client, containerName, srcPath, dstPath string) error {
-	f, err := os.Open(srcPath)
-	if err != nil {
-		return fmt.Errorf("failed to open source file: %w", err)
-	}
-	defer f.Close()
-
-	return client.CopyToContainer(containerName, dstPath, f)
-}