robust parse hub JSON output (backend+frontend)

Handle noisy cscli hub output by extracting the first JSON payload instead of assuming clean stdout. backend: add firstJSONStartIndex and parseHubJSONOutput to strip preamble and decode either object or array, replace ad-hoc slicing in hub handlers, and add unit tests. frontend: add parseHubJSONString and extractHubArray to robustly parse mixed text/JSON responses (including iterative trimming) and accept alternate keys for appsec configs/rules. This makes hub listing resilient to informational logs surrounding the JSON payload.

Author: hhftechnology

internal/api/handlers/hub.go

@@ -195,6 +195,41 @@ func upsertHubPreference(db *database.Database, category, mode, defaultPath, las
 	}
 }
 
+func firstJSONStartIndex(output string) int {
+	objectStart := strings.Index(output, "{")
+	arrayStart := strings.Index(output, "[")
+
+	switch {
+	case objectStart == -1:
+		return arrayStart
+	case arrayStart == -1:
+		return objectStart
+	case objectStart < arrayStart:
+		return objectStart
+	default:
+		return arrayStart
+	}
+}
+
+// parseHubJSONOutput extracts the first JSON value from mixed CLI output.
+// cscli may print informational preamble/trailing lines around the JSON body.
+func parseHubJSONOutput(output string) (interface{}, error) {
+	start := firstJSONStartIndex(output)
+	if start < 0 {
+		return nil, fmt.Errorf("no JSON payload found")
+	}
+
+	decoder := json.NewDecoder(strings.NewReader(output[start:]))
+	decoder.UseNumber()
+
+	var parsed interface{}
+	if err := decoder.Decode(&parsed); err != nil {
+		return nil, err
+	}
+
+	return parsed, nil
+}
+
 // ListHubCategories returns the supported category metadata.
 func ListHubCategories() gin.HandlerFunc {
 	return func(c *gin.Context) {
@@ -221,16 +256,8 @@ func ListHubItems(dockerClient *docker.Client, cfg *config.Config) gin.HandlerFu
 			return
 		}
 
-		// cscli hub list may output informational preamble lines (e.g.
-		// "Loaded: 161 parsers...") to stdout before the JSON object.
-		// Strip any text before the opening '{' to get clean JSON.
-		jsonOutput := output
-		if idx := strings.Index(output, "{"); idx > 0 {
-			jsonOutput = output[idx:]
-		}
-
-		var parsed interface{}
-		if err := json.Unmarshal([]byte(jsonOutput), &parsed); err != nil {
+		parsed, err := parseHubJSONOutput(output)
+		if err != nil {
 			logger.Warn("Failed to parse hub list JSON", "error", err, "output_preview", truncateString(output, 200))
 			c.JSON(http.StatusOK, models.Response{
 				Success: true,
@@ -265,16 +292,8 @@ func ListHubItemsByCategory(dockerClient *docker.Client, cfg *config.Config) gin
 			return
 		}
 
-		// Strip preamble text before JSON (cscli may output info lines to stdout)
-		jsonOutput := output
-		if idx := strings.Index(output, "{"); idx > 0 {
-			jsonOutput = output[idx:]
-		} else if idx := strings.Index(output, "["); idx > 0 {
-			jsonOutput = output[idx:]
-		}
-
-		var parsed interface{}
-		if err := json.Unmarshal([]byte(jsonOutput), &parsed); err != nil {
+		parsed, err := parseHubJSONOutput(output)
+		if err != nil {
 			c.JSON(http.StatusOK, models.Response{Success: true, Data: gin.H{"category": spec, "raw_output": output}})
 			return
 		}

internal/api/handlers/hub_parse_test.go

@@ -0,0 +1,102 @@
+package handlers
+
+import (
+	"testing"
+)
+
+func TestParseHubJSONOutput(t *testing.T) {
+	tests := []struct {
+		name      string
+		input     string
+		wantError bool
+		assert    func(t *testing.T, parsed interface{})
+	}{
+		{
+			name:  "parses clean object",
+			input: `{"collections":[{"name":"a"}]}`,
+			assert: func(t *testing.T, parsed interface{}) {
+				t.Helper()
+				record, ok := parsed.(map[string]interface{})
+				if !ok {
+					t.Fatalf("expected object, got %T", parsed)
+				}
+				items, ok := record["collections"].([]interface{})
+				if !ok || len(items) != 1 {
+					t.Fatalf("expected one collection item, got %#v", record["collections"])
+				}
+			},
+		},
+		{
+			name: "parses object with preamble and trailing logs",
+			input: "Loaded: 1 parser\\nUnmanaged items: 0\\n" +
+				`{"scenarios":[{"name":"test/scenario"}]}` +
+				"\\noperation completed",
+			assert: func(t *testing.T, parsed interface{}) {
+				t.Helper()
+				record, ok := parsed.(map[string]interface{})
+				if !ok {
+					t.Fatalf("expected object, got %T", parsed)
+				}
+				items, ok := record["scenarios"].([]interface{})
+				if !ok || len(items) != 1 {
+					t.Fatalf("expected one scenario item, got %#v", record["scenarios"])
+				}
+			},
+		},
+		{
+			name:  "parses list payload",
+			input: "Info line\\n" + `[{"name":"crowdsecurity/test"}]`,
+			assert: func(t *testing.T, parsed interface{}) {
+				t.Helper()
+				items, ok := parsed.([]interface{})
+				if !ok || len(items) != 1 {
+					t.Fatalf("expected one array item, got %#v", parsed)
+				}
+			},
+		},
+		{
+			name:      "fails without json",
+			input:     "Loaded: 1 parser\\nNo JSON output",
+			wantError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			parsed, err := parseHubJSONOutput(tt.input)
+			if tt.wantError {
+				if err == nil {
+					t.Fatalf("expected error, got parsed=%#v", parsed)
+				}
+				return
+			}
+
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if tt.assert != nil {
+				tt.assert(t, parsed)
+			}
+		})
+	}
+}
+
+func TestFirstJSONStartIndex(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  int
+	}{
+		{name: "object first", input: "prefix {\"k\":1}", want: 7},
+		{name: "array first", input: "prefix [1,2]", want: 7},
+		{name: "none", input: "no json", want: -1},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := firstJSONStartIndex(tt.input); got != tt.want {
+				t.Fatalf("got %d want %d", got, tt.want)
+			}
+		})
+	}
+}

web/src/pages/HubBrowser.tsx

@@ -59,33 +59,75 @@ type ParsedHubItems = {
   rawParseError: boolean
 }
 
+function extractHubArray(record: Record<string, unknown>, ...keys: string[]): HubBrowserItem[] {
+  for (const key of keys) {
+    if (Array.isArray(record[key])) return record[key] as HubBrowserItem[]
+  }
+  return []
+}
+
+function parseHubJSONString(raw: string): unknown {
+  const trimmed = raw.trim()
+  if (!trimmed) return null
+
+  try {
+    return JSON.parse(trimmed)
+  } catch {
+    // Continue with robust fallback below.
+  }
+
+  const firstObject = trimmed.indexOf('{')
+  const firstArray = trimmed.indexOf('[')
+
+  let start = -1
+  if (firstObject >= 0 && firstArray >= 0) start = Math.min(firstObject, firstArray)
+  else if (firstObject >= 0) start = firstObject
+  else if (firstArray >= 0) start = firstArray
+
+  if (start < 0) {
+    throw new Error('No JSON payload found in hub response')
+  }
+
+  const openChar = trimmed[start]
+  const closeChar = openChar === '{' ? '}' : ']'
+  for (let end = trimmed.length - 1; end > start; end -= 1) {
+    if (trimmed[end] !== closeChar) continue
+    const candidate = trimmed.slice(start, end + 1)
+    try {
+      return JSON.parse(candidate)
+    } catch {
+      // Keep shrinking until we find a parseable payload.
+    }
+  }
+
+  throw new Error('Unable to parse JSON payload from hub response')
+}
+
 function parseHubItems(data: unknown): ParsedHubItems {
   if (!data) return { items: EMPTY_HUB_ITEMS, rawParseError: false }
   if (Array.isArray(data)) {
     return { items: { ...EMPTY_HUB_ITEMS, scenarios: data as HubBrowserItem[] }, rawParseError: false }
   }
   if (typeof data === 'string') {
     try {
-      const parsed = JSON.parse(data)
+      const parsed = parseHubJSONString(data)
       return parseHubItems(parsed)
     } catch {
       return { items: EMPTY_HUB_ITEMS, rawParseError: true }
     }
   }
   if (typeof data === 'object') {
     const record = data as Record<string, unknown>
-    const extract = (key: string): HubBrowserItem[] =>
-      Array.isArray(record[key]) ? (record[key] as HubBrowserItem[]) : []
     return {
       rawParseError: false,
       items: {
-        collections: extract('collections'),
-        scenarios: extract('scenarios'),
-        parsers: extract('parsers'),
-        postoverflows: extract('postoverflows'),
-        'appsec-configs': extract('appsec-configs'),
-        'appsec-rules': extract('appsec-rules'),
-        contexts: extract('contexts'),
+        collections: extractHubArray(record, 'collections'),
+        scenarios: extractHubArray(record, 'scenarios'),
+        parsers: extractHubArray(record, 'parsers'),
+        postoverflows: extractHubArray(record, 'postoverflows'),
+        'appsec-configs': extractHubArray(record, 'appsec-configs', 'appsec_configs'),
+        'appsec-rules': extractHubArray(record, 'appsec-rules', 'appsec_rules'),
+        contexts: extractHubArray(record, 'contexts'),
       },
     }
   }