fix: improve aws and azure benchmark cases

cases/azure_blob_to_tcp_performance/case.yaml

@@ -30,6 +30,8 @@ requires: [azure_blob_source]
 
 subjects:
   - vmetric
+  - filebeat
+  - cribl-stream
 
 configurations:
   default:

cases/azure_blob_to_tcp_performance/configs/cribl-stream/cribl.yml

@@ -0,0 +1,13 @@
+api:
+  host: 0.0.0.0
+  port: 9999
+  disabled: false
+auth:
+  type: local
+system:
+  installType: standalone
+  intercom: false
+workers:
+  count: -2
+  minimum: 1
+  memory: 2048

cases/azure_blob_to_tcp_performance/configs/cribl-stream/inputs.yml

@@ -0,0 +1,12 @@
+inputs:
+  bench_azblob_in:
+    type: azure_blob
+    authType: manual
+    # Azurite's well-known development-storage account — public emulator
+    # constants, not secrets. Queue endpoint included: the source is
+    # driven by BlobCreated messages on the container-named storage
+    # queue (the bench generator enqueues them — Azurite never emits
+    # them itself).
+    connectionString: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;"
+    queueName: bench-in
+    sendToRoutes: true

cases/azure_blob_to_tcp_performance/configs/cribl-stream/messages.yml

@@ -0,0 +1 @@
+messages: {}

cases/azure_blob_to_tcp_performance/configs/cribl-stream/outputs.yml

@@ -0,0 +1,14 @@
+outputs:
+  default:
+    defaultId: bench_tcp_out
+    type: default
+  bench_tcp_out:
+    type: tcpjson
+    host: receiver
+    port: 9001
+    nestedFields: none
+    throttleRatePerSec: "0"
+    compression: none
+    tls:
+      disabled: true
+    authType: manual

cases/azure_blob_to_tcp_performance/configs/cribl-stream/pipelines/route.yml

@@ -0,0 +1,12 @@
+id: default
+groups: {}
+comments: []
+routes:
+  - id: bench_route
+    name: bench_passthru
+    final: true
+    disabled: false
+    pipeline: passthru
+    filter: "true"
+    output: bench_tcp_out
+    description: "Route all data through passthru to TCP output"

cases/azure_blob_to_tcp_performance/configs/filebeat.yml

@@ -0,0 +1,29 @@
+filebeat.inputs:
+  - type: azure-blob-storage
+    account_name: devstoreaccount1
+    # Azurite's well-known development-storage key — public emulator
+    # constant, not a secret.
+    auth.shared_credentials.account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+    # storage_url overrides the https://<account>.blob.core.windows.net
+    # default — this is what makes the input Azurite-compatible. The
+    # trailing slash matters: filebeat appends the container name with
+    # plain string concatenation.
+    storage_url: "http://azurite:10000/devstoreaccount1/"
+    containers:
+      - name: bench-in
+        # poll defaults to false (one-shot listing) — the bench needs
+        # continuous pickup of blobs the generator keeps uploading.
+        poll: true
+        poll_interval: 2s
+        max_workers: 4
+
+output.elasticsearch:
+  hosts: ["http://receiver:9002"]
+  allow_older_versions: true
+  compression_level: 0
+
+logging.level: warning
+queue.mem:
+  events: 4096
+  flush.min_events: 2048
+  flush.timeout: 1s

cases/azure_blob_to_tcp_performance/configs/vmetric.yml

@@ -3,17 +3,8 @@ devices:
     name: azblob-bench
     type: azblob
     properties:
-      # Azurite's well-known development-storage account — public emulator
-      # constants, not secrets.
       connection_string: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;"
-      # container_name doubles as the storage queue polled for BlobCreated
-      # events (the generator enqueues them).
       container_name: "bench-in"
-      # timeout is BOTH the dequeue poll interval and the queue visibility
-      # timeout. The 60s default would schedule the first poll after a short
-      # bench run already ended.
-      timeout: 2
-      batch_size: 32
 targets:
   - name: tcp-target-bench
     type: tcp

cases/kafka_batch_to_tcp_correctness/configs/vmetric.yml

@@ -1,9 +1,3 @@
-# Kafka consumer device that SPLITS JSON-array batch messages into individual
-# events. The generator packs `kafka_batch` JSON objects per Kafka message as a
-# JSON array ([{...},{...}]); split_mode: json_array makes the device emit one
-# event per array element, so the TCP receiver sees one record per original log
-# (count preserved 1:1). Without split_mode (default "none") the entire array
-# would be forwarded as a single event and the counts would not reconcile.
 devices:
   - id: 1
     name: kafka-bench

cases/kafka_crash_correctness/configs/vmetric.yml

@@ -1,10 +1,3 @@
-# Kafka consumer in MEMORY mode (no persistent_storage). Crash-safety for kafka
-# comes from Kafka itself + the listener's deferred offset commit: the offset is
-# committed only after the sender delivers the records to the target, so a
-# stop/kill before delivery leaves the offset uncommitted and Kafka re-delivers
-# the records on restart (at-least-once, no loss). The on-disk crash-resistance
-# queue is intentionally NOT enabled — pairing it with Kafka re-consumption
-# would recover from both and double-deliver.
 devices:
   - id: 1
     name: kafka-bench

cases/kafka_inflight_crash_correctness/configs/vmetric.yml

@@ -1,10 +1,3 @@
-# Kafka consumer in MEMORY mode (no persistent_storage). Crash-safety for kafka
-# comes from Kafka itself + the listener's deferred offset commit: the offset is
-# committed only after the sender delivers the records to the target, so a
-# stop/kill before delivery leaves the offset uncommitted and Kafka re-delivers
-# the records on restart (at-least-once, no loss). The on-disk crash-resistance
-# queue is intentionally NOT enabled — pairing it with Kafka re-consumption
-# would recover from both and double-deliver.
 devices:
   - id: 1
     name: kafka-bench

cases/kafka_restart_correctness/configs/vmetric.yml

@@ -1,10 +1,3 @@
-# Kafka consumer in MEMORY mode (no persistent_storage). Crash-safety for kafka
-# comes from Kafka itself + the listener's deferred offset commit: the offset is
-# committed only after the sender delivers the records to the target, so a
-# stop/kill before delivery leaves the offset uncommitted and Kafka re-delivers
-# the records on restart (at-least-once, no loss). The on-disk crash-resistance
-# queue is intentionally NOT enabled — pairing it with Kafka re-consumption
-# would recover from both and double-deliver.
 devices:
   - id: 1
     name: kafka-bench

cases/kafka_to_tcp_correctness/configs/vmetric.yml

@@ -1,8 +1,3 @@
-# Kafka consumer device per https://vdocs.virtualmetric.com/configuration/devices/mq/kafka
-# Consume topic `bench` from the redpanda broker and forward verbatim to the
-# TCP receiver. The director's kafka device has no offset-reset knob, so the
-# generator waits out its `warmup` before producing — giving this consumer time
-# to join the group and be positioned before the first record is sent.
 devices:
   - id: 1
     name: kafka-bench

cases/kafka_to_tcp_performance/configs/vmetric.yml

@@ -1,8 +1,3 @@
-# Kafka consumer device per https://vdocs.virtualmetric.com/configuration/devices/mq/kafka
-# Consume topic `bench` from the redpanda broker and forward verbatim to the
-# TCP receiver. The director's kafka device has no offset-reset knob, so the
-# generator waits out its `warmup` before producing — giving this consumer time
-# to join the group and be positioned before the first record is sent.
 devices:
   - id: 1
     name: kafka-bench
@@ -12,10 +7,6 @@ devices:
       port: 9092
       topic: "bench"
       group: "bench"
-      # 8 consumer workers — each joins the group as its own franz-go member, so
-      # the 8 topic partitions (kafka.partitions in case.yaml) are consumed in
-      # parallel. Default is NumCPU/4; pinning 8 matches the partition count.
-      workers: 8
 targets:
   - name: tcp-target-bench
     type: tcp

cases/s3_to_tcp_correctness/configs/vmetric.yml

@@ -4,15 +4,10 @@ devices:
     type: awss3
     properties:
       endpoint: "http://localstack:4566"
-      use_path_style: true
       region: "us-east-1"
-      access_key_id: "test"
-      secret_access_key: "test"
       bucket_name: "bench-in"
       queue_url: "http://localstack:4566/000000000000/bench-events"
       file_name_filter: ".*"
-      sqs_max_messages: 10
-      sqs_wait_time_seconds: 2
 targets:
   - name: tcp-target-bench
     type: tcp

cases/s3_to_tcp_performance/configs/vmetric.yml

@@ -4,15 +4,10 @@ devices:
     type: awss3
     properties:
       endpoint: "http://localstack:4566"
-      use_path_style: true
       region: "us-east-1"
-      access_key_id: "test"
-      secret_access_key: "test"
       bucket_name: "bench-in"
       queue_url: "http://localstack:4566/000000000000/bench-events"
       file_name_filter: ".*"
-      sqs_max_messages: 10
-      sqs_wait_time_seconds: 2
 targets:
   - name: tcp-target-bench
     type: tcp

cases/tcp_to_azure_blob_correctness/case.yaml

@@ -27,6 +27,7 @@ subjects:
   - vmetric
   - vector
   - fluent-bit
+  - cribl-stream
 
 correctness:
   expected_loss_pct: 0

cases/tcp_to_azure_blob_correctness/configs/cribl-stream/cribl.yml

@@ -0,0 +1,13 @@
+api:
+  host: 0.0.0.0
+  port: 9999
+  disabled: false
+auth:
+  type: local
+system:
+  installType: standalone
+  intercom: false
+workers:
+  count: -2
+  minimum: 1
+  memory: 2048

cases/tcp_to_azure_blob_correctness/configs/cribl-stream/inputs.yml

@@ -0,0 +1,16 @@
+inputs:
+  bench_tcp_in:
+    type: tcp
+    host: 0.0.0.0
+    port: 9000
+    disabled: false
+    tls:
+      disabled: true
+    enableProxyHeader: false
+    staleChannelFlushMs: 10000
+    enableHeader: false
+    preprocess:
+      disabled: true
+    sendToRoutes: true
+    pqEnabled: false
+    authType: manual

cases/tcp_to_azure_blob_correctness/configs/cribl-stream/messages.yml

@@ -0,0 +1 @@
+messages: {}

cases/tcp_to_azure_blob_correctness/configs/cribl-stream/outputs.yml

@@ -0,0 +1,23 @@
+outputs:
+  default:
+    defaultId: bench_azblob_out
+    type: default
+  bench_azblob_out:
+    type: azure_blob
+    authType: manual
+    # Azurite's well-known development-storage account — public emulator
+    # constants, not secrets.
+    connectionString: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;"
+    containerName: bench-out
+    createContainer: true
+    format: json
+    baseFileName: CriblOut
+    compress: none
+    # Schema minimums (open >= 10s, idle >= 5s) — the lowest Cribl accepts,
+    # so blobs land within the run + drain window instead of sitting in
+    # staging for the 300s/30s defaults.
+    maxFileSizeMB: 32
+    maxFileOpenTimeSec: 10
+    maxFileIdleTimeSec: 5
+    onBackpressure: block
+    stagePath: $CRIBL_HOME/state/outputs/staging

cases/tcp_to_azure_blob_correctness/configs/cribl-stream/pipelines/route.yml

@@ -0,0 +1,12 @@
+id: default
+groups: {}
+comments: []
+routes:
+  - id: bench_route
+    name: bench_passthru
+    final: true
+    disabled: false
+    pipeline: passthru
+    filter: "true"
+    output: bench_azblob_out
+    description: "Route all data through passthru to Azure Blob output"

cases/tcp_to_azure_blob_correctness/configs/fluent-bit.conf

@@ -16,6 +16,9 @@
     account_name          devstoreaccount1
     shared_key            Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
     endpoint              http://azurite:10000
+    # Azurite paths embed the account name — without emulator_mode the
+    # container probe 400s and nothing is ever uploaded.
+    emulator_mode         on
     container_name        bench-out
     auto_create_container on
     blob_type             blockblob

cases/tcp_to_azure_blob_correctness/configs/vmetric.yml

@@ -9,8 +9,6 @@ targets:
   - name: azblob-target-bench
     type: azblob
     properties:
-      # Azurite's well-known development-storage account — public emulator
-      # constants, not secrets.
       connection_string: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;"
       container: "bench-out"
       format: json

cases/tcp_to_azure_blob_performance/case.yaml

@@ -27,6 +27,7 @@ subjects:
   - vmetric
   - vector
   - fluent-bit
+  - cribl-stream
 
 configurations:
   default:

cases/tcp_to_azure_blob_performance/configs/cribl-stream/cribl.yml

@@ -0,0 +1,13 @@
+api:
+  host: 0.0.0.0
+  port: 9999
+  disabled: false
+auth:
+  type: local
+system:
+  installType: standalone
+  intercom: false
+workers:
+  count: -2
+  minimum: 1
+  memory: 2048

cases/tcp_to_azure_blob_performance/configs/cribl-stream/inputs.yml

@@ -0,0 +1,16 @@
+inputs:
+  bench_tcp_in:
+    type: tcp
+    host: 0.0.0.0
+    port: 9000
+    disabled: false
+    tls:
+      disabled: true
+    enableProxyHeader: false
+    staleChannelFlushMs: 10000
+    enableHeader: false
+    preprocess:
+      disabled: true
+    sendToRoutes: true
+    pqEnabled: false
+    authType: manual

cases/tcp_to_azure_blob_performance/configs/cribl-stream/messages.yml

@@ -0,0 +1 @@
+messages: {}

cases/tcp_to_azure_blob_performance/configs/cribl-stream/outputs.yml

@@ -0,0 +1,23 @@
+outputs:
+  default:
+    defaultId: bench_azblob_out
+    type: default
+  bench_azblob_out:
+    type: azure_blob
+    authType: manual
+    # Azurite's well-known development-storage account — public emulator
+    # constants, not secrets.
+    connectionString: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;"
+    containerName: bench-out
+    createContainer: true
+    format: json
+    baseFileName: CriblOut
+    compress: none
+    # Schema minimums (open >= 10s, idle >= 5s) — the lowest Cribl accepts,
+    # so blobs land within the run + drain window instead of sitting in
+    # staging for the 300s/30s defaults.
+    maxFileSizeMB: 32
+    maxFileOpenTimeSec: 10
+    maxFileIdleTimeSec: 5
+    onBackpressure: block
+    stagePath: $CRIBL_HOME/state/outputs/staging

cases/tcp_to_azure_blob_performance/configs/cribl-stream/pipelines/route.yml

@@ -0,0 +1,12 @@
+id: default
+groups: {}
+comments: []
+routes:
+  - id: bench_route
+    name: bench_passthru
+    final: true
+    disabled: false
+    pipeline: passthru
+    filter: "true"
+    output: bench_azblob_out
+    description: "Route all data through passthru to Azure Blob output"