diff --git a/cases/disk_buffer_crash_performance/configs/axosyslog.conf b/cases/disk_buffer_crash_performance/configs/axosyslog.conf index 341e925..292b4c1 100644 --- a/cases/disk_buffer_crash_performance/configs/axosyslog.conf +++ b/cases/disk_buffer_crash_performance/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -14,6 +17,7 @@ destination d_tcp { port(9001) disk-buffer( reliable(yes) + prealloc(yes) capacity-bytes(4294967296) dir("/var/lib/syslog-ng/disk-buffer/") ) @@ -23,4 +27,5 @@ destination d_tcp { log { source(s_tcp); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/disk_buffer_performance/configs/axosyslog.conf b/cases/disk_buffer_performance/configs/axosyslog.conf index 2301707..24d3cf1 100644 --- a/cases/disk_buffer_performance/configs/axosyslog.conf +++ b/cases/disk_buffer_performance/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -14,6 +17,8 @@ destination d_tcp { port(9001) disk-buffer( reliable(no) + prealloc(yes) + front-cache-size(10000) capacity-bytes(4294967296) dir("/var/lib/syslog-ng/disk-buffer/") ) @@ -23,4 +28,5 @@ destination d_tcp { log { source(s_tcp); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/disk_buffer_persistence_correctness/configs/axosyslog.conf b/cases/disk_buffer_persistence_correctness/configs/axosyslog.conf index e3e8f3f..e459a83 100644 --- a/cases/disk_buffer_persistence_correctness/configs/axosyslog.conf +++ b/cases/disk_buffer_persistence_correctness/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -15,6 +18,8 @@ destination d_tcp { template("${MESSAGE}\n") disk-buffer( reliable(no) + prealloc(yes) + front-cache-size(10000) capacity-bytes(4294967296) dir("/var/lib/syslog-ng/disk-buffer/") ) @@ -24,4 +29,5 @@ destination d_tcp { log { source(s_tcp); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/file_rotate_create_correctness/configs/axosyslog.conf b/cases/file_rotate_create_correctness/configs/axosyslog.conf index 6afebcf..38a806a 100644 --- a/cases/file_rotate_create_correctness/configs/axosyslog.conf +++ b/cases/file_rotate_create_correctness/configs/axosyslog.conf @@ -6,6 +6,8 @@ source s_file { filename-pattern("*.log*") flags(no-parse) follow-freq(0.1) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) ); }; @@ -19,4 +21,5 @@ destination d_tcp { log { source(s_file); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/file_rotate_restart_correctness/configs/axosyslog.conf b/cases/file_rotate_restart_correctness/configs/axosyslog.conf index a021293..0ad29e0 100644 --- a/cases/file_rotate_restart_correctness/configs/axosyslog.conf +++ b/cases/file_rotate_restart_correctness/configs/axosyslog.conf @@ -12,6 +12,8 @@ source s_file { filename-pattern("*.log*") flags(no-parse) follow-freq(0.1) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) recursive(no) ); }; @@ -27,4 +29,5 @@ destination d_tcp { log { source(s_file); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/file_rotate_truncate_correctness/configs/axosyslog.conf b/cases/file_rotate_truncate_correctness/configs/axosyslog.conf index 6afebcf..38a806a 100644 --- a/cases/file_rotate_truncate_correctness/configs/axosyslog.conf +++ b/cases/file_rotate_truncate_correctness/configs/axosyslog.conf @@ -6,6 +6,8 @@ source s_file { filename-pattern("*.log*") flags(no-parse) follow-freq(0.1) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) ); }; @@ -19,4 +21,5 @@ destination d_tcp { log { source(s_file); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/file_to_tcp_performance/configs/axosyslog.conf b/cases/file_to_tcp_performance/configs/axosyslog.conf index 6afebcf..38a806a 100644 --- a/cases/file_to_tcp_performance/configs/axosyslog.conf +++ b/cases/file_to_tcp_performance/configs/axosyslog.conf @@ -6,6 +6,8 @@ source s_file { filename-pattern("*.log*") flags(no-parse) follow-freq(0.1) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) ); }; @@ -19,4 +21,5 @@ destination d_tcp { log { source(s_file); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/file_truncate_correctness/configs/axosyslog.conf b/cases/file_truncate_correctness/configs/axosyslog.conf index 6e2f8b3..768b325 100644 --- a/cases/file_truncate_correctness/configs/axosyslog.conf +++ b/cases/file_truncate_correctness/configs/axosyslog.conf @@ -5,6 +5,8 @@ source s_file { "/data/input.log" flags(no-parse) follow-freq(0.1) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) ); }; @@ -18,4 +20,5 @@ destination d_tcp { log { source(s_file); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/otlp_grpc_to_otlp_grpc_correctness/configs/axosyslog.conf b/cases/otlp_grpc_to_otlp_grpc_correctness/configs/axosyslog.conf index 26b2896..0283d01 100644 --- a/cases/otlp_grpc_to_otlp_grpc_correctness/configs/axosyslog.conf +++ b/cases/otlp_grpc_to_otlp_grpc_correctness/configs/axosyslog.conf @@ -17,6 +17,8 @@ source s_otlp { opentelemetry( port(4317) + workers({{.CPUs}}) + concurrent-requests(4) ); }; @@ -29,4 +31,5 @@ destination d_otlp { log { source(s_otlp); destination(d_otlp); + flags(flow-control); }; diff --git a/cases/otlp_grpc_to_otlp_grpc_performance/configs/axosyslog.conf b/cases/otlp_grpc_to_otlp_grpc_performance/configs/axosyslog.conf index 26b2896..0283d01 100644 --- a/cases/otlp_grpc_to_otlp_grpc_performance/configs/axosyslog.conf +++ b/cases/otlp_grpc_to_otlp_grpc_performance/configs/axosyslog.conf @@ -17,6 +17,8 @@ source s_otlp { opentelemetry( port(4317) + workers({{.CPUs}}) + concurrent-requests(4) ); }; @@ -29,4 +31,5 @@ destination d_otlp { log { source(s_otlp); destination(d_otlp); + flags(flow-control); }; diff --git a/cases/real_world_1_performance/configs/axosyslog.conf b/cases/real_world_1_performance/configs/axosyslog.conf index 9a7ca6e..b47697d 100644 --- a/cases/real_world_1_performance/configs/axosyslog.conf +++ b/cases/real_world_1_performance/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(syslog-protocol) ); }; @@ -32,4 +35,5 @@ log { rewrite(r_mark_dropped); }; destination(d_tcp); + flags(flow-control); }; diff --git a/cases/regex_mask_performance/configs/axosyslog.conf b/cases/regex_mask_performance/configs/axosyslog.conf index 3958cbd..0167638 100644 --- a/cases/regex_mask_performance/configs/axosyslog.conf +++ b/cases/regex_mask_performance/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -23,4 +26,5 @@ log { source(s_tcp); rewrite(r_mask_conn); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/set_field_performance/configs/axosyslog.conf b/cases/set_field_performance/configs/axosyslog.conf index 068359a..9873890 100644 --- a/cases/set_field_performance/configs/axosyslog.conf +++ b/cases/set_field_performance/configs/axosyslog.conf @@ -7,6 +7,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -26,4 +29,5 @@ log { source(s_tcp); rewrite(r_set); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/sighup_correctness/configs/axosyslog.conf b/cases/sighup_correctness/configs/axosyslog.conf index b0bd6c7..2023e08 100644 --- a/cases/sighup_correctness/configs/axosyslog.conf +++ b/cases/sighup_correctness/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -19,4 +22,5 @@ destination d_tcp { log { source(s_tcp); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/syslog_parsing_performance/configs/axosyslog.conf b/cases/syslog_parsing_performance/configs/axosyslog.conf index 05714e5..d166a08 100644 --- a/cases/syslog_parsing_performance/configs/axosyslog.conf +++ b/cases/syslog_parsing_performance/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(syslog-protocol) ); }; @@ -18,4 +21,5 @@ destination d_tcp { log { source(s_tcp); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/tcp_to_blackhole_performance/configs/axosyslog.conf b/cases/tcp_to_blackhole_performance/configs/axosyslog.conf index 50b5cc3..63cccb2 100644 --- a/cases/tcp_to_blackhole_performance/configs/axosyslog.conf +++ b/cases/tcp_to_blackhole_performance/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -15,4 +18,5 @@ destination d_null { log { source(s_tcp); destination(d_null); + flags(flow-control); }; diff --git a/cases/tcp_to_http_5min_performance/configs/axosyslog.conf b/cases/tcp_to_http_5min_performance/configs/axosyslog.conf index 3ec10c2..d3e18ab 100644 --- a/cases/tcp_to_http_5min_performance/configs/axosyslog.conf +++ b/cases/tcp_to_http_5min_performance/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -13,7 +16,7 @@ destination d_http { url("http://receiver:9001/") method("POST") body("${MESSAGE}") - workers(4) + workers({{.CPUs}}) batch-lines(1000) batch-timeout(100) ); @@ -22,4 +25,5 @@ destination d_http { log { source(s_tcp); destination(d_http); + flags(flow-control); }; diff --git a/cases/tcp_to_http_performance/configs/axosyslog.conf b/cases/tcp_to_http_performance/configs/axosyslog.conf index 3ec10c2..d3e18ab 100644 --- a/cases/tcp_to_http_performance/configs/axosyslog.conf +++ b/cases/tcp_to_http_performance/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -13,7 +16,7 @@ destination d_http { url("http://receiver:9001/") method("POST") body("${MESSAGE}") - workers(4) + workers({{.CPUs}}) batch-lines(1000) batch-timeout(100) ); @@ -22,4 +25,5 @@ destination d_http { log { source(s_tcp); destination(d_http); + flags(flow-control); }; diff --git a/cases/tcp_to_http_persistent_correctness/configs/axosyslog.conf b/cases/tcp_to_http_persistent_correctness/configs/axosyslog.conf index 4e45500..c7e6398 100644 --- a/cases/tcp_to_http_persistent_correctness/configs/axosyslog.conf +++ b/cases/tcp_to_http_persistent_correctness/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -13,11 +16,13 @@ destination d_http { url("http://receiver:9001/") method("POST") body("${MESSAGE}") - workers(4) + workers({{.CPUs}}) batch-lines(1000) batch-timeout(100) disk-buffer( reliable(no) + prealloc(yes) + front-cache-size(10000) capacity-bytes(4294967296) dir("/var/lib/syslog-ng/disk-buffer/") ) @@ -27,4 +32,5 @@ destination d_http { log { source(s_tcp); destination(d_http); + flags(flow-control); }; diff --git a/cases/tcp_to_tcp_5min_performance/configs/axosyslog.conf b/cases/tcp_to_tcp_5min_performance/configs/axosyslog.conf index 5b0c799..2f78a92 100644 --- a/cases/tcp_to_tcp_5min_performance/configs/axosyslog.conf +++ b/cases/tcp_to_tcp_5min_performance/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -18,4 +21,5 @@ destination d_tcp { log { source(s_tcp); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/tcp_to_tcp_performance/configs/axosyslog.conf b/cases/tcp_to_tcp_performance/configs/axosyslog.conf index 5b0c799..2f78a92 100644 --- a/cases/tcp_to_tcp_performance/configs/axosyslog.conf +++ b/cases/tcp_to_tcp_performance/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -18,4 +21,5 @@ destination d_tcp { log { source(s_tcp); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/tcp_to_tcp_persistent_correctness/configs/axosyslog.conf b/cases/tcp_to_tcp_persistent_correctness/configs/axosyslog.conf index e3e8f3f..e459a83 100644 --- a/cases/tcp_to_tcp_persistent_correctness/configs/axosyslog.conf +++ b/cases/tcp_to_tcp_persistent_correctness/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -15,6 +18,8 @@ destination d_tcp { template("${MESSAGE}\n") disk-buffer( reliable(no) + prealloc(yes) + front-cache-size(10000) capacity-bytes(4294967296) dir("/var/lib/syslog-ng/disk-buffer/") ) @@ -24,4 +29,5 @@ destination d_tcp { log { source(s_tcp); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/tcp_to_tcp_persistent_crash_correctness/configs/axosyslog.conf b/cases/tcp_to_tcp_persistent_crash_correctness/configs/axosyslog.conf index abf9839..c06f060 100644 --- a/cases/tcp_to_tcp_persistent_crash_correctness/configs/axosyslog.conf +++ b/cases/tcp_to_tcp_persistent_crash_correctness/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -15,6 +18,7 @@ destination d_tcp { template("${MESSAGE}\n") disk-buffer( reliable(yes) + prealloc(yes) capacity-bytes(4294967296) dir("/var/lib/syslog-ng/disk-buffer/") ) @@ -24,4 +28,5 @@ destination d_tcp { log { source(s_tcp); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/tcp_to_tcp_persistent_performance/configs/axosyslog.conf b/cases/tcp_to_tcp_persistent_performance/configs/axosyslog.conf index 2301707..24d3cf1 100644 --- a/cases/tcp_to_tcp_persistent_performance/configs/axosyslog.conf +++ b/cases/tcp_to_tcp_persistent_performance/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -14,6 +17,8 @@ destination d_tcp { port(9001) disk-buffer( reliable(no) + prealloc(yes) + front-cache-size(10000) capacity-bytes(4294967296) dir("/var/lib/syslog-ng/disk-buffer/") ) @@ -23,4 +28,5 @@ destination d_tcp { log { source(s_tcp); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/tcp_to_tcp_persistent_restart_correctness/configs/axosyslog.conf b/cases/tcp_to_tcp_persistent_restart_correctness/configs/axosyslog.conf index e3e8f3f..e459a83 100644 --- a/cases/tcp_to_tcp_persistent_restart_correctness/configs/axosyslog.conf +++ b/cases/tcp_to_tcp_persistent_restart_correctness/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -15,6 +18,8 @@ destination d_tcp { template("${MESSAGE}\n") disk-buffer( reliable(no) + prealloc(yes) + front-cache-size(10000) capacity-bytes(4294967296) dir("/var/lib/syslog-ng/disk-buffer/") ) @@ -24,4 +29,5 @@ destination d_tcp { log { source(s_tcp); destination(d_tcp); + flags(flow-control); }; diff --git a/cases/wrapped_json_correctness/configs/axosyslog.conf b/cases/wrapped_json_correctness/configs/axosyslog.conf index ba37136..19c8109 100644 --- a/cases/wrapped_json_correctness/configs/axosyslog.conf +++ b/cases/wrapped_json_correctness/configs/axosyslog.conf @@ -4,6 +4,9 @@ source s_tcp { tcp( ip("0.0.0.0") port(9000) + max-connections({{.Connections}}) + log-iw-size({{.Connections}}000) + log-fetch-limit(1000) flags(no-parse) ); }; @@ -24,4 +27,5 @@ log { source(s_tcp); parser(p_json); destination(d_tcp); + flags(flow-control); }; diff --git a/internal/orchestrator/docker.go b/internal/orchestrator/docker.go index c421ec7..e5be767 100644 --- a/internal/orchestrator/docker.go +++ b/internal/orchestrator/docker.go @@ -36,6 +36,67 @@ func resolveGeneratorConnections(explicit int) int { return 3 } +// totalGeneratorConnections reports how many parallel connections the +// generator(s) will open to the subject this run, summing across all +// generators in multi-generator mode (they all fan in to the same subject). +func totalGeneratorConnections(tc *config.TestCase) int { + if tc.MultiGenerator() { + total := 0 + for _, g := range tc.Generators { + total += resolveGeneratorConnections(g.Connections) + } + return total + } + return resolveGeneratorConnections(tc.Generator.Connections) +} + +// configTemplateContext is the data exposed to a subject config rendered as a +// Go text/template. It carries only generic facts about the run — never a +// subject's tuning vocabulary — so a config adapts itself to the workload +// (e.g. AxoSyslog sizes max-connections() to the generator fan-out, or an +// OpenTelemetry source sets workers() to the CPU count). +type configTemplateContext struct { + // Connections is the total number of parallel connections the + // generator(s) will open to the subject this run. + Connections int + // CPUs is the host CPU count. + CPUs int +} + +// renderSubjectConfig renders the subject config at srcPath as a Go +// text/template when it contains template actions ("{{"), writing the result +// into tmpDir and returning the rendered file's path. Configs with no template +// actions, directory-form configs (e.g. cribl-stream), and missing paths are +// returned unchanged — so non-templated subjects, and the existing +// missing-path tolerance, are unaffected. +func renderSubjectConfig(srcPath, tmpDir string, ctx configTemplateContext) (string, error) { + info, err := os.Stat(srcPath) + if err != nil || info.IsDir() { + return srcPath, nil + } + raw, err := os.ReadFile(srcPath) + if err != nil { + return "", err + } + if !strings.Contains(string(raw), "{{") { + return srcPath, nil + } + tmpl, err := template.New(filepath.Base(srcPath)).Parse(string(raw)) + if err != nil { + return "", fmt.Errorf("parsing config template %s: %w", srcPath, err) + } + outPath := filepath.Join(tmpDir, filepath.Base(srcPath)) + f, err := os.Create(outPath) + if err != nil { + return "", err + } + defer f.Close() + if err := tmpl.Execute(f, ctx); err != nil { + return "", fmt.Errorf("rendering config template %s: %w", srcPath, err) + } + return outPath, nil +} + const composeTemplate = `# Auto-generated by harness — do not edit networks: bench: @@ -690,9 +751,19 @@ func writeCompose(path string, cfg RunConfig) error { tc := cfg.TestCase s := cfg.Subject + // Render the subject config as a template (opt-in via {{...}}) so it can + // adapt to the run's connection/CPU counts. + renderedConfigSrc, err := renderSubjectConfig(cfg.ConfigSrcPath, cfg.TmpDir, configTemplateContext{ + Connections: totalGeneratorConnections(tc), + CPUs: runtime.NumCPU(), + }) + if err != nil { + return err + } + // Docker Compose on Windows needs forward slashes in volume paths; // backslashes are interpreted as YAML escape sequences inside quoted strings. - configSrc := filepath.ToSlash(cfg.ConfigSrcPath) + configSrc := filepath.ToSlash(renderedConfigSrc) tmpDir := filepath.ToSlash(cfg.TmpDir) subjectContainer := "bench-subject-" + s.Name