From 320a5b0b930755bdd104007264805a79ec911afc Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sun, 19 Jul 2026 21:03:43 +0200 Subject: [PATCH 01/11] snailrace v0.0.5: fixed rendering progress for small window width Signed-off-by: Polina Simonenko --- internal/app/progress.go | 56 ++++++++++++++++++++++++++--- internal/app/progress_race_test.go | 58 ++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 4 deletions(-) diff --git a/internal/app/progress.go b/internal/app/progress.go index 527ba8c..7d51e18 100644 --- a/internal/app/progress.go +++ b/internal/app/progress.go @@ -23,7 +23,7 @@ func progressRenderer( return nil } var mutex sync.Mutex - renderedLines := 0 + var renderedLines []string cursorHidden := false return func(event runner.ProgressEvent) { mutex.Lock() @@ -32,20 +32,20 @@ func progressRenderer( if cursorHidden { fmt.Fprint(writer, "\x1b[?25h\r\n") } - renderedLines = 0 + renderedLines = nil return } if !cursorHidden { fmt.Fprint(writer, "\x1b[?25l") cursorHidden = true } - clearProgress(writer, renderedLines) width, _, _ := term.GetSize(int(file.Fd())) + clearProgress(writer, progressRenderedRows(renderedLines, width)) lines := append( []string{progressStatus(event, nameWidth)}, progressRaceLines(event, width)..., ) fmt.Fprint(writer, strings.Join(lines, "\n")) - renderedLines = len(lines) + renderedLines = lines } } @@ -121,6 +121,54 @@ func clearProgress(writer io.Writer, lines int) { } } +func progressRenderedRows(lines []string, terminalWidth int) int { + if terminalWidth <= 0 { + return len(lines) + } + rows := 0 + for _, line := range lines { + width := progressDisplayWidth(line) + lineRows := (width + terminalWidth - 1) / terminalWidth + if lineRows == 0 { + lineRows = 1 + } + rows += lineRows + } + return rows +} + +func progressDisplayWidth(value string) int { + width := 0 + for index := 0; index < len(value); { + if value[index] == '\x1b' { + next := progressANSIEscapeEnd(value[index:]) + if next > 0 { + index += next + continue + } + } + r, size := utf8.DecodeRuneInString(value[index:]) + index += size + width++ + if r == '🐌' || r == '🏁' { + width++ + } + } + return width +} + +func progressANSIEscapeEnd(value string) int { + if len(value) < 2 || value[0] != '\x1b' || value[1] != '[' { + return 0 + } + for index := 2; index < len(value); index++ { + if value[index] >= '@' && value[index] <= '~' { + return index + 1 + } + } + return 0 +} + func progressDuration(seconds float64) string { if seconds < 0.001 { return fmt.Sprintf("%.3g us", seconds*1e6) diff --git a/internal/app/progress_race_test.go b/internal/app/progress_race_test.go index 2b95f2b..c5022d5 100644 --- a/internal/app/progress_race_test.go +++ b/internal/app/progress_race_test.go @@ -1,12 +1,15 @@ package app import ( + "bytes" "math" "regexp" "strings" "testing" "time" + "github.com/creack/pty" + "github.com/shellcell/snailrace/internal/model" "github.com/shellcell/snailrace/internal/runner" ) @@ -135,6 +138,61 @@ func TestProgressStatusAlignsNumbersAndName(t *testing.T) { } } +func TestProgressRenderedRowsCountsWrappedLines(t *testing.T) { + lines := []string{"12345678901", "short"} + if got := progressRenderedRows(lines, 10); got != 3 { + t.Fatalf("rendered rows = %d, want 3", got) + } +} + +func TestProgressRenderedRowsIgnoresANSIAndCountsKnownEmojiWidth(t *testing.T) { + line := "\x1b[38;2;136;192;208m🐌\x1b[0m123456789" + if got := progressRenderedRows([]string{line}, 10); got != 2 { + t.Fatalf("rendered rows = %d, want 2", got) + } +} + +func TestProgressRendererRecalculatesRowsAfterResize(t *testing.T) { + primary, terminal, err := pty.Open() + if err != nil { + t.Fatal(err) + } + defer primary.Close() + defer terminal.Close() + if err := pty.Setsize(terminal, &pty.Winsize{Cols: 200, Rows: 24}); err != nil { + t.Fatal(err) + } + output := progressTerminalBuffer{fd: terminal.Fd()} + render := progressRenderer(&output, true, 0) + if render == nil { + t.Fatal("progress renderer disabled for PTY") + } + event := runner.ProgressEvent{ + ToolName: "first", Tool: 1, ToolCount: 1, + Iteration: 1, Iterations: 10, Total: 10, + } + render(event) + redrawStart := output.Len() + if err := pty.Setsize(terminal, &pty.Winsize{Cols: 40, Rows: 24}); err != nil { + t.Fatal(err) + } + render(event) + + previous := []string{progressStatus(event, 0)} + wantRows := progressRenderedRows(previous, 40) + redraw := output.String()[redrawStart:] + if got := strings.Count(redraw, "\x1b[1A") + 1; got != wantRows { + t.Fatalf("cleared rows after resize = %d, want %d", got, wantRows) + } +} + +type progressTerminalBuffer struct { + bytes.Buffer + fd uintptr +} + +func (buffer *progressTerminalBuffer) Fd() uintptr { return buffer.fd } + func progressEstimate(name string, scale float64) runner.ProgressEstimate { stats := model.Stats{Mean: scale} return runner.ProgressEstimate{ From 100ed50b4a857ce888134ddf00ef3f07ae32ec36 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sun, 19 Jul 2026 21:08:30 +0200 Subject: [PATCH 02/11] snailrace v0.0.5: continue if command exits with failure Signed-off-by: Polina Simonenko --- internal/app/app.go | 16 +--------------- internal/app/output_test.go | 13 +++++++++++++ internal/runner/benchmark_test.go | 25 +++++++++++++++++++++++++ internal/runner/run.go | 10 +++++++++- internal/runner/run_test.go | 13 ++++++++----- internal/runner/tui.go | 2 +- internal/runner/tui_test.go | 13 +++++++++++++ 7 files changed, 70 insertions(+), 22 deletions(-) diff --git a/internal/app/app.go b/internal/app/app.go index e4f4d88..60b83d0 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -139,7 +139,7 @@ func Run(arguments []string, stdout, stderr io.Writer) error { ); err != nil { return err } - return checkExitCodes(benchmarks) + return nil } func oneBasedOrder(order [][]int) [][]int { @@ -193,20 +193,6 @@ func writeResult( return saveReportFormats(stderr, directory, formats, result) } -func checkExitCodes(benchmarks []model.Benchmark) error { - for _, benchmark := range benchmarks { - for _, run := range benchmark.Runs { - if run.ExitCode != 0 && run.StopReason != "duration" { - return fmt.Errorf( - "%q exited unsuccessfully in one or more runs", - benchmark.Tool.Name, - ) - } - } - } - return nil -} - func modeName(tui bool) string { if tui { return "tui" diff --git a/internal/app/output_test.go b/internal/app/output_test.go index 0b12ed9..8157321 100644 --- a/internal/app/output_test.go +++ b/internal/app/output_test.go @@ -103,3 +103,16 @@ func TestMarkdownAndSVGSavedAsChartBundle(t *testing.T) { t.Fatalf("chart files = %v, error = %v", charts, err) } } + +func TestRunIgnoresNonZeroCommandExit(t *testing.T) { + var stdout, stderr bytes.Buffer + if err := Run( + []string{"-n", "2", "-warmups", "0", "-c", "exit 7"}, + &stdout, &stderr, + ); err != nil { + t.Fatal(err) + } + if !strings.Contains(stdout.String(), "TIME") { + t.Fatal("stdout does not contain the completed report") + } +} diff --git a/internal/runner/benchmark_test.go b/internal/runner/benchmark_test.go index 867125c..3ef0089 100644 --- a/internal/runner/benchmark_test.go +++ b/internal/runner/benchmark_test.go @@ -50,6 +50,31 @@ func TestBenchmarkInterruptionBeforeAnyRoundFails(t *testing.T) { } } +func TestBenchmarkContinuesAfterNonZeroExit(t *testing.T) { + benchmarks, err := Benchmark( + context.Background(), + []Spec{ + {Name: "exit", Shell: "exit 7"}, + {Name: "true", Args: []string{"/bin/true"}}, + }, + Config{Runs: 3, Warmups: 1, Interval: time.Millisecond}, + Options{}, + ) + if err != nil { + t.Fatal(err) + } + for index, benchmark := range benchmarks { + if len(benchmark.Runs) != 3 { + t.Fatalf("tool %d recorded %d runs, want 3", index, len(benchmark.Runs)) + } + } + for _, run := range benchmarks[0].Runs { + if run.ExitCode != 7 { + t.Fatalf("exit code = %d, want 7", run.ExitCode) + } + } +} + func completedRoundCount(estimates []ProgressEstimate) int { if len(estimates) == 0 { return 0 diff --git a/internal/runner/run.go b/internal/runner/run.go index 78fe32c..cdde499 100644 --- a/internal/runner/run.go +++ b/internal/runner/run.go @@ -2,7 +2,9 @@ package runner import ( "context" + "errors" "os" + "os/exec" "syscall" "time" @@ -48,7 +50,7 @@ func runOnce( if ctx.Err() != nil { return model.Run{}, ctx.Err() } - if waitErr != nil { + if waitErr != nil && !isNonZeroExit(cmd, waitErr) { return model.Run{}, waitErr } meanResident := sampledMeanResident(peak) @@ -70,6 +72,12 @@ func runOnce( }, nil } +func isNonZeroExit(cmd *exec.Cmd, err error) bool { + var exitError *exec.ExitError + return errors.As(err, &exitError) && cmd.ProcessState != nil && + cmd.ProcessState.ExitCode() > 0 +} + func monitor( pid int, interval time.Duration, diff --git a/internal/runner/run_test.go b/internal/runner/run_test.go index f63e9c2..d5ff32a 100644 --- a/internal/runner/run_test.go +++ b/internal/runner/run_test.go @@ -28,13 +28,16 @@ func TestRunOnceUsesInjectedOutputWriter(t *testing.T) { } } -func TestRunOnceRejectsFailedCommand(t *testing.T) { - _, err := runOnce( - context.Background(), Spec{Name: "false", Args: []string{"/bin/false"}}, +func TestRunOnceRecordsNonZeroExit(t *testing.T) { + run, err := runOnce( + context.Background(), Spec{Name: "exit", Shell: "exit 7"}, time.Millisecond, Options{}, ) - if err == nil { - t.Fatal("failed command should not become a benchmark observation") + if err != nil { + t.Fatal(err) + } + if run.ExitCode != 7 { + t.Fatalf("exit code = %d, want 7", run.ExitCode) } } diff --git a/internal/runner/tui.go b/internal/runner/tui.go index 2bf217f..1599656 100644 --- a/internal/runner/tui.go +++ b/internal/runner/tui.go @@ -80,7 +80,7 @@ func runTUIOnce( if options.Duration > 0 && reason == "exited" { return model.Run{}, errors.New("TUI exited before the fixed duration") } - if waitErr != nil && reason == "exited" { + if waitErr != nil && reason == "exited" && !isNonZeroExit(cmd, waitErr) { return model.Run{}, waitErr } select { diff --git a/internal/runner/tui_test.go b/internal/runner/tui_test.go index 3763e19..f5f42ab 100644 --- a/internal/runner/tui_test.go +++ b/internal/runner/tui_test.go @@ -35,6 +35,19 @@ func TestFixedDurationTUIRejectsEarlyExit(t *testing.T) { } } +func TestTUIRecordsNonZeroExit(t *testing.T) { + run, err := runTUIOnce( + context.Background(), Spec{Name: "exit", Shell: "exit 7"}, + time.Millisecond, Options{TUI: true}, + ) + if err != nil { + t.Fatal(err) + } + if run.ExitCode != 7 { + t.Fatalf("exit code = %d, want 7", run.ExitCode) + } +} + func TestTerminalSizeOverridesDimensions(t *testing.T) { width, height := ResolveTerminalSize(132, 43) if width != 132 || height != 43 { From 615f267ab3d1f7a3a58622e2ba938f2b8792820d Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sun, 19 Jul 2026 21:14:49 +0200 Subject: [PATCH 03/11] exex v0.0.5: updated deps Signed-off-by: Polina Simonenko --- .github/workflows/release.yml | 12 ++++++------ go.mod | 8 +++++--- go.sum | 8 ++++---- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 21aa86f..8a217e1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,8 +12,8 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 with: go-version-file: go.mod - run: go test ./... @@ -32,8 +32,8 @@ jobs: - { goos: darwin, goarch: amd64, cgo: '1', runner: macos-26-intel, label: darwin-amd64 } - { goos: darwin, goarch: arm64, cgo: '1', runner: macos-latest, label: darwin-arm64 } steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 with: go-version-file: go.mod - name: Test Darwin @@ -45,7 +45,7 @@ jobs: GOOS: ${{ matrix.goos }} GOARCH: ${{ matrix.goarch }} GOARM: ${{ matrix.goarm }} - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_target || '12.0' }} + MACOSX_DEPLOYMENT_TARGET: '12.0' run: | go build -trimpath \ -ldflags="-s -w -X github.com/shellcell/snailrace/internal/app.Version=${GITHUB_REF_NAME}" \ @@ -72,7 +72,7 @@ jobs: nfpm package -f /tmp/nfpm.yaml -p "$fmt" -t pkgs done - name: Upload to release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: files: | ${{ env.ARCHIVE }} diff --git a/go.mod b/go.mod index caa53ef..a942b0b 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,12 @@ module github.com/shellcell/snailrace -go 1.22 +go 1.25.0 + +toolchain go1.25.12 require ( github.com/creack/pty v1.1.24 - golang.org/x/term v0.21.0 + golang.org/x/term v0.45.0 ) -require golang.org/x/sys v0.21.0 // indirect +require golang.org/x/sys v0.47.0 // indirect diff --git a/go.sum b/go.sum index 7fb5fea..59d547f 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,6 @@ github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= From 958c523fa73d221c247e56f3b780886951441ee1 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sun, 19 Jul 2026 21:24:03 +0200 Subject: [PATCH 04/11] snailrace v0.0.5: report default dir, cirrect sorting for ranking Signed-off-by: Polina Simonenko --- README.md | 6 +++--- docs/snailrace.1 | 4 +++- internal/app/formats_test.go | 3 +++ internal/app/options.go | 6 +++--- internal/app/output_test.go | 23 ++++++++++++++++++++++ internal/report/chart_ranking.go | 7 ++++++- internal/report/chart_test.go | 33 ++++++++++++++++++++++++++++++++ 7 files changed, 74 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7e20701..59956b0 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Compare shell commands: ./snailrace -n 20 -c 'grep needle data.txt' -c 'rg needle data.txt' ``` -Save an HTML report: +An HTML report is saved to the current directory by default. Choose another directory: ```sh ./snailrace -format html -output ./reports -- ./my-program --flag @@ -85,8 +85,8 @@ Measure an interactive TUI: | `-interval` | Sampling interval; default 10 ms | | `-index` | Index dimensions: `time,cpu,ram,disk` | | `-baseline` | 1-based baseline; `0` selects the winner | -| `-f`, `-format` | `html`, `svg`, `markdown`, `json`, or `text` | -| `-o`, `-output` | Report directory | +| `-f`, `-format` | Saved format; default `html` | +| `-o`, `-output` | Report directory; default current directory | | `-verbose` | Full statistical tables on stdout | | `-show-output` | Forward command output to stderr | | `tui` | Run inside a pseudo-terminal | diff --git a/docs/snailrace.1 b/docs/snailrace.1 index 7b1a744..a06285c 100644 --- a/docs/snailrace.1 +++ b/docs/snailrace.1 @@ -71,8 +71,10 @@ Saved report format: .Ql markdown , .Ql json , .Ql text . +The default is +.Ql html . .It Fl o , Fl output Ar string -Directory for a saved report. +Directory for a saved report; defaults to the current directory. .It Fl verbose Print full statistical tables to stdout. .It Fl show-output diff --git a/internal/app/formats_test.go b/internal/app/formats_test.go index 9d334c1..75e6d18 100644 --- a/internal/app/formats_test.go +++ b/internal/app/formats_test.go @@ -30,4 +30,7 @@ func TestDefaultFormatIsHTML(t *testing.T) { if !reflect.DeepEqual(options.formats, []string{"html"}) { t.Fatalf("formats = %#v, want html", options.formats) } + if options.output != "." { + t.Fatalf("output = %q, want current directory", options.output) + } } diff --git a/internal/app/options.go b/internal/app/options.go index a640097..e2f6d11 100644 --- a/internal/app/options.go +++ b/internal/app/options.go @@ -80,8 +80,8 @@ func parseOptions(arguments []string, stderr io.Writer) (options, error) { "saved format; repeat or comma-separate: html, svg, markdown, json, text", ) flags.Var(&formats, "f", "saved format (shorthand)") - flags.StringVar(&result.output, "output", "", "directory for a saved report") - flags.StringVar(&result.output, "o", "", "directory for a saved report (shorthand)") + flags.StringVar(&result.output, "output", ".", "report directory; default current directory") + flags.StringVar(&result.output, "o", ".", "report directory (shorthand)") flags.BoolVar(&result.verbose, "verbose", false, "print full statistical tables to stdout") flags.BoolVar(&result.showOutput, "show-output", false, "show command output") // TUI. @@ -150,7 +150,7 @@ func printUsage(stderr io.Writer) { }}, {"Output", []string{ "-f, -format list saved format: html, svg, markdown, json, text", - "-o, -output string directory for a saved report", + "-o, -output string report directory; default current directory", "-verbose print full statistical tables to stdout", "-show-output forward command output to stderr", }}, diff --git a/internal/app/output_test.go b/internal/app/output_test.go index 8157321..6ca5e2e 100644 --- a/internal/app/output_test.go +++ b/internal/app/output_test.go @@ -105,6 +105,7 @@ func TestMarkdownAndSVGSavedAsChartBundle(t *testing.T) { } func TestRunIgnoresNonZeroCommandExit(t *testing.T) { + t.Chdir(t.TempDir()) var stdout, stderr bytes.Buffer if err := Run( []string{"-n", "2", "-warmups", "0", "-c", "exit 7"}, @@ -116,3 +117,25 @@ func TestRunIgnoresNonZeroCommandExit(t *testing.T) { t.Fatal("stdout does not contain the completed report") } } + +func TestRunSavesDefaultHTMLInCurrentDirectory(t *testing.T) { + directory := t.TempDir() + t.Chdir(directory) + var stdout, stderr bytes.Buffer + if err := Run( + []string{"-n", "1", "-warmups", "0", "--", "/bin/true"}, + &stdout, &stderr, + ); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || !strings.HasSuffix(entries[0].Name(), ".html") { + t.Fatalf("saved entries = %v, want one HTML report", entries) + } + if !strings.Contains(stderr.String(), "Report saved to") { + t.Fatal("stderr does not announce the default report path") + } +} diff --git a/internal/report/chart_ranking.go b/internal/report/chart_ranking.go index 61dc8d1..accec9a 100644 --- a/internal/report/chart_ranking.go +++ b/internal/report/chart_ranking.go @@ -4,6 +4,7 @@ import ( "fmt" "html" "math" + "sort" "strings" "github.com/shellcell/snailrace/internal/model" @@ -102,7 +103,11 @@ func rankingBarChart( `%s`, html.EscapeString(metric.name), html.EscapeString(subtitle), ) - for index, row := range ranking.rows { + rows := append([]rankingRow(nil), ranking.rows...) + sort.SliceStable(rows, func(left, right int) bool { + return metric.rank(rows[left]) < metric.rank(rows[right]) + }) + for index, row := range rows { y := 62 + index*rowHeight value := metric.value(row) width := value / maximum * plotWidth diff --git a/internal/report/chart_test.go b/internal/report/chart_test.go index 46c5b6b..abf479d 100644 --- a/internal/report/chart_test.go +++ b/internal/report/chart_test.go @@ -120,6 +120,39 @@ func TestChartLegendsUseToolIdentityColors(t *testing.T) { } } +func TestRankingChartsPutBestAtTopAndWorstAtBottom(t *testing.T) { + report := model.Report{ + Config: model.Config{ + Mode: "command", Baseline: 3, + IndexDimensions: []string{"time", "cpu"}, + }, + Benchmarks: []model.Benchmark{ + benchmarkWithResourceCosts("fast", 1, 9, 300, 300, 300), + benchmarkWithResourceCosts("middle", 2, 2, 200, 200, 200), + benchmarkWithResourceCosts("efficient", 3, 1, 100, 100, 100), + }, + } + expected := map[string][]string{ + "BALANCED INDEX": {"#1 efficient", "#2 middle", "#3 fast"}, + "TIME": {"#1 fast", "#2 middle", "#3 efficient"}, + "CPU COST": {"#1 efficient", "#2 middle", "#3 fast"}, + "RAM AGGREGATE": {"#1 efficient", "#2 middle", "#3 fast"}, + "LINKED SIZE": {"#1 efficient", "#2 middle", "#3 fast"}, + } + for _, chart := range rankingCharts(report) { + order, ok := expected[chart.title] + if !ok { + t.Fatalf("missing expected order for %s chart", chart.title) + } + best := strings.Index(chart.body, order[0]) + middle := strings.Index(chart.body, order[1]) + worst := strings.Index(chart.body, order[2]) + if best < 0 || middle <= best || worst <= middle { + t.Fatalf("%s chart does not put best at top and worst at bottom", chart.title) + } + } +} + func TestSingleRunDistributionOmitsUndefinedConfidenceWhisker(t *testing.T) { report := model.Report{ Config: model.Config{Mode: "command", Baseline: 1}, From dc528db38e3d83d72d3c5910950cfa94d37afc00 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sun, 19 Jul 2026 22:05:53 +0200 Subject: [PATCH 05/11] snailrace: v0.0.5 Validation for empty commands unavailable rankings without non-finite values exclude failed tools from rankings and report failures exclude TUI teardown from fixed-duration measurements clean up prepare process groups on cancellation propagate Markdown, HTML, and SVG writer errors make report saving collision-safe and atomic Signed-off-by: Polina Simonenko --- README.md | 5 + docs/snailrace.1 | 7 ++ internal/analysis/ranking.go | 129 ++++++++++++++++-------- internal/analysis/ranking_math.go | 49 +++++---- internal/analysis/ranking_test.go | 40 +++++++- internal/app/app.go | 36 +++++-- internal/app/filename.go | 6 +- internal/app/options_test.go | 8 ++ internal/app/output_bundle.go | 130 +++++++++++++++++++----- internal/app/output_test.go | 143 ++++++++++++++++++++++++++- internal/app/progress_race_test.go | 4 + internal/app/publish_darwin.go | 9 ++ internal/app/publish_linux.go | 11 +++ internal/app/validation.go | 6 ++ internal/model/model.go | 18 ++++ internal/report/chart_artifacts.go | 6 +- internal/report/chart_ranking.go | 16 +-- internal/report/chart_test.go | 9 +- internal/report/error_writer.go | 28 ++++++ internal/report/error_writer_test.go | 42 ++++++++ internal/report/failures.go | 125 +++++++++++++++++++++++ internal/report/html.go | 7 +- internal/report/markdown.go | 5 +- internal/report/ranking.go | 3 +- internal/report/ranking_html.go | 14 +++ internal/report/ranking_json.go | 55 ++++++++--- internal/report/ranking_markdown.go | 14 +++ internal/report/ranking_test.go | 2 +- internal/report/ranking_text.go | 8 ++ internal/report/ranking_types.go | 32 +++--- internal/report/ranking_winners.go | 28 +++--- internal/report/svg.go | 10 +- internal/report/text.go | 1 + internal/report/text_bars.go | 33 ++++--- internal/report/ui_test.go | 42 ++++++++ internal/runner/benchmark.go | 3 + internal/runner/benchmark_test.go | 31 ++++++ internal/runner/prepare.go | 31 ++++++ internal/runner/prepare_test.go | 70 +++++++++++++ internal/runner/tui.go | 44 +++++---- internal/runner/tui_test.go | 20 ++++ internal/runner/validation.go | 78 +++++++++++++++ 42 files changed, 1172 insertions(+), 186 deletions(-) create mode 100644 internal/app/publish_darwin.go create mode 100644 internal/app/publish_linux.go create mode 100644 internal/report/error_writer.go create mode 100644 internal/report/error_writer_test.go create mode 100644 internal/report/failures.go create mode 100644 internal/runner/prepare.go create mode 100644 internal/runner/prepare_test.go create mode 100644 internal/runner/validation.go diff --git a/README.md b/README.md index 59956b0..bb31012 100644 --- a/README.md +++ b/README.md @@ -113,9 +113,14 @@ sample standard deviation, median, p95, range, and a 95% Student's t interval. - The default index combines normalized time, CPU, and RAM costs. - RAM cost combines mean and peak RSS. - Sampling-limited RAM is excluded from the index. +- Commands with non-zero measured exits continue running but are excluded from rankings. +- If none of the selected dimensions is usable, reports mark the balanced ranking unavailable. - Ctrl+C keeps completed comparison rounds. - Command output is discarded unless `-show-output` is set. +Saved reports are staged before publication and receive a numeric suffix rather +than overwriting an existing report with the same timestamp. + ## Platform Notes - Linux reads `/proc` and follows each process's children. diff --git a/docs/snailrace.1 b/docs/snailrace.1 index a06285c..f682fad 100644 --- a/docs/snailrace.1 +++ b/docs/snailrace.1 @@ -29,6 +29,11 @@ Commands are given either after a separator or with repeated .Fl c flags. +.Pp +Commands with non-zero measured exits continue through all configured rounds, +but are reported as failures and excluded from rankings. +If none of the selected dimensions is usable, the balanced ranking is marked +unavailable instead of selecting a winner. The .Cm tui subcommand renders a live terminal view while measuring. @@ -75,6 +80,8 @@ The default is .Ql html . .It Fl o , Fl output Ar string Directory for a saved report; defaults to the current directory. +Reports are staged before publication and receive a numeric suffix instead of +overwriting an existing report with the same name. .It Fl verbose Print full statistical tables to stdout. .It Fl show-output diff --git a/internal/analysis/ranking.go b/internal/analysis/ranking.go index 1d90b9c..2f4648b 100644 --- a/internal/analysis/ranking.go +++ b/internal/analysis/ranking.go @@ -8,18 +8,20 @@ import ( ) type Ranking struct { - Rows []RankingRow - RAMAvailable bool - RAMPresent bool - BestOverall float64 - FixedTUI bool - PrimaryRatio bool - CPURatio bool - FootprintRatio bool - IndexPrimary bool - IndexCPU bool - IndexRAM bool - IndexFootprint bool + Rows []RankingRow + Available bool + UnavailableReason string + RAMAvailable bool + RAMPresent bool + BestOverall float64 + FixedTUI bool + PrimaryRatio bool + CPURatio bool + FootprintRatio bool + IndexPrimary bool + IndexCPU bool + IndexRAM bool + IndexFootprint bool } type RankingRow struct { @@ -42,16 +44,22 @@ type RankingRow struct { func Calculate(config model.Config, benchmarks []model.Benchmark) Ranking { count := len(benchmarks) - result := Ranking{Rows: make([]RankingRow, count)} + result := Ranking{} if count == 0 { + result.UnavailableReason = "no benchmarks" return result } result.FixedTUI = config.Mode == "tui" && config.DurationSeconds > 0 - result.RAMAvailable = samplesReliable(config, benchmarks) + eligible := make([]bool, count) + eligibleCount := 0 primary, cpu := make([]float64, count), make([]float64, count) meanRAM, peakRAM := make([]float64, count), make([]float64, count) footprint := make([]float64, count) for index, benchmark := range benchmarks { + eligible[index] = benchmark.EligibleForRanking() + if eligible[index] { + eligibleCount++ + } primary[index] = benchmark.Summary.WallSeconds.Mean cpu[index] = benchmark.Summary.CPUTotalSeconds.Mean if result.FixedTUI { @@ -62,24 +70,36 @@ func Calculate(config model.Config, benchmarks []model.Benchmark) Ranking { peakRAM[index] = benchmark.Summary.PeakResidentBytes.Mean footprint[index] = float64(benchmark.Tool.DiskFootprintBytes) } - if !hasPositive(meanRAM) || !hasPositive(peakRAM) { + result.RAMAvailable = samplesReliable(config, benchmarks, eligible) + if !hasPositive(meanRAM, eligible) || !hasPositive(peakRAM, eligible) { result.RAMAvailable = false } - result.RAMPresent = allPositive(meanRAM) && allPositive(peakRAM) + result.RAMPresent = allPositive(meanRAM, eligible) && allPositive(peakRAM, eligible) result.RAMAvailable = result.RAMAvailable && result.RAMPresent - result.PrimaryRatio = allPositive(primary) - result.CPURatio = allPositive(cpu) - result.FootprintRatio = allPositive(footprint) - primaryScore, cpuScore := normalized(primary), normalized(cpu) - ramScore := normalizedPair(meanRAM, peakRAM, result.RAMAvailable) - footprintScore := normalized(footprint) + result.PrimaryRatio = allPositive(primary, eligible) + result.CPURatio = allPositive(cpu, eligible) + result.FootprintRatio = allPositive(footprint, eligible) + primaryScore, cpuScore := normalized(primary, eligible), normalized(cpu, eligible) + ramScore := normalizedPair(meanRAM, peakRAM, eligible, result.RAMAvailable) + footprintScore := normalized(footprint, eligible) included := indexSet(config.IndexDimensions) result.IndexPrimary = included["time"] && !result.FixedTUI && result.PrimaryRatio result.IndexCPU = (included["cpu"] || (result.FixedTUI && included["time"])) && result.CPURatio result.IndexRAM = included["ram"] && result.RAMAvailable result.IndexFootprint = included["disk"] && result.FootprintRatio - for index := range result.Rows { + result.Available = eligibleCount > 0 && (result.IndexPrimary || result.IndexCPU || + result.IndexRAM || result.IndexFootprint) + if eligibleCount == 0 { + result.UnavailableReason = "no successful measured runs" + } else if !result.Available { + result.UnavailableReason = "none of the selected dimensions has usable positive values" + } + result.Rows = make([]RankingRow, 0, eligibleCount) + for index := range benchmarks { + if !eligible[index] { + continue + } var scores []float64 if result.IndexPrimary { scores = append(scores, primaryScore[index]) @@ -97,28 +117,37 @@ func Calculate(config model.Config, benchmarks []model.Benchmark) Ranking { if result.RAMPresent { ramValue = geometricMean([]float64{meanRAM[index], peakRAM[index]}) } - result.Rows[index] = RankingRow{ - Benchmark: index, OverallScore: geometricMean(scores), + overallScore := 0.0 + if result.Available { + overallScore = geometricMean(scores) + } + result.Rows = append(result.Rows, RankingRow{ + Benchmark: index, OverallScore: overallScore, PrimaryScore: primaryScore[index], CPUScore: cpuScore[index], RAMScore: ramScore[index], FootprintScore: footprintScore[index], PrimaryValue: primary[index], CPUValue: cpu[index], RAMValue: ramValue, FootprintValue: footprint[index], - } + }) } - applyRanks(result.Rows, primaryScore, cpuScore, ramScore, footprintScore) - result.BestOverall = math.Inf(1) - for _, row := range result.Rows { - result.BestOverall = math.Min(result.BestOverall, row.OverallScore) + applyRanks( + result.Rows, eligible, result.Available, + primaryScore, cpuScore, ramScore, footprintScore, + ) + if result.Available { + result.BestOverall = math.Inf(1) + for _, row := range result.Rows { + result.BestOverall = math.Min(result.BestOverall, row.OverallScore) + } } return result } -func AutomaticBaseline(config model.Config, benchmarks []model.Benchmark) int { +func AutomaticBaseline(config model.Config, benchmarks []model.Benchmark) (int, bool) { ranking := Calculate(config, benchmarks) - if len(ranking.Rows) == 0 { - return 1 + if !ranking.Available || len(ranking.Rows) == 0 { + return 0, false } - return ranking.Rows[0].Benchmark + 1 + return ranking.Rows[0].Benchmark + 1, true } // IndexDimensions are the cost categories that may compose the balanced index. @@ -157,19 +186,33 @@ func BalancedIndexes(config model.Config, benchmarks []model.Benchmark) []float6 return indexes } -func applyRanks(rows []RankingRow, primary, cpu, ram, footprint []float64) { - overall := make([]float64, len(rows)) - for index := range rows { - overall[index] = rows[index].OverallScore +func applyRanks( + rows []RankingRow, + eligible []bool, + overallAvailable bool, + primary, cpu, ram, footprint []float64, +) { + overall := make([]float64, len(eligible)) + for index := range overall { + overall[index] = math.Inf(1) + } + for _, row := range rows { + overall[row.Benchmark] = row.OverallScore } for index := range rows { - rows[index].OverallRank = rankOf(overall, index) - rows[index].PrimaryRank = rankOf(primary, index) - rows[index].CPURank = rankOf(cpu, index) - rows[index].RAMRank = rankOf(ram, index) - rows[index].FootprintRank = rankOf(footprint, index) + benchmark := rows[index].Benchmark + if overallAvailable { + rows[index].OverallRank = rankOf(overall, benchmark, eligible) + } + rows[index].PrimaryRank = rankOf(primary, benchmark, eligible) + rows[index].CPURank = rankOf(cpu, benchmark, eligible) + rows[index].RAMRank = rankOf(ram, benchmark, eligible) + rows[index].FootprintRank = rankOf(footprint, benchmark, eligible) } sort.SliceStable(rows, func(i, j int) bool { + if !overallAvailable { + return rows[i].Benchmark < rows[j].Benchmark + } return rows[i].OverallRank < rows[j].OverallRank }) } diff --git a/internal/analysis/ranking_math.go b/internal/analysis/ranking_math.go index c1a7a59..d2dbe9c 100644 --- a/internal/analysis/ranking_math.go +++ b/internal/analysis/ranking_math.go @@ -6,10 +6,13 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func normalized(values []float64) []float64 { +func normalized(values []float64, eligible []bool) []float64 { minimum := math.Inf(1) hasZero := false - for _, value := range values { + for index, value := range values { + if !eligible[index] { + continue + } hasZero = hasZero || value == 0 if value > 0 && value < minimum { minimum = value @@ -18,6 +21,8 @@ func normalized(values []float64) []float64 { result := make([]float64, len(values)) for index, value := range values { switch { + case !eligible[index]: + result[index] = math.Inf(1) case value == 0: result[index] = 1 case math.IsInf(minimum, 1) || value < 0: @@ -31,7 +36,7 @@ func normalized(values []float64) []float64 { return result } -func normalizedPair(first, second []float64, available bool) []float64 { +func normalizedPair(first, second []float64, eligible []bool, available bool) []float64 { result := make([]float64, len(first)) if !available { for index := range result { @@ -39,11 +44,11 @@ func normalizedPair(first, second []float64, available bool) []float64 { } return result } - one, two := normalized(first), normalized(second) + one, two := normalized(first, eligible), normalized(second, eligible) for index := range result { result[index] = geometricMean([]float64{one[index], two[index]}) } - return normalized(result) + return normalized(result, eligible) } func geometricMean(values []float64) float64 { @@ -60,12 +65,15 @@ func geometricMean(values []float64) float64 { return math.Exp(total / float64(count)) } -func samplesReliable(config model.Config, benchmarks []model.Benchmark) bool { +func samplesReliable(config model.Config, benchmarks []model.Benchmark, eligible []bool) bool { minimum := config.IntervalMS / 1000 * 2 if minimum <= 0 { return true } - for _, benchmark := range benchmarks { + for index, benchmark := range benchmarks { + if !eligible[index] { + continue + } for _, run := range benchmark.Runs { if run.WallSeconds < minimum || run.SampleCount < 2 || run.SampleCoverageSeconds < config.IntervalMS/1000 { @@ -76,31 +84,36 @@ func samplesReliable(config model.Config, benchmarks []model.Benchmark) bool { return true } -func hasPositive(values []float64) bool { - for _, value := range values { - if value > 0 { +func hasPositive(values []float64, eligible []bool) bool { + for index, value := range values { + if eligible[index] && value > 0 { return true } } return false } -func allPositive(values []float64) bool { - if len(values) == 0 { - return false - } - for _, value := range values { +func allPositive(values []float64, eligible []bool) bool { + found := false + for index, value := range values { + if !eligible[index] { + continue + } + found = true if value <= 0 || math.IsInf(value, 0) || math.IsNaN(value) { return false } } - return true + return found } -func rankOf(values []float64, target int) int { +func rankOf(values []float64, target int, eligible []bool) int { + if !eligible[target] || math.IsInf(values[target], 0) || math.IsNaN(values[target]) { + return 0 + } rank := 1 for index, value := range values { - if index != target && value < values[target] { + if index != target && eligible[index] && value < values[target] { rank++ } } diff --git a/internal/analysis/ranking_test.go b/internal/analysis/ranking_test.go index 2af36c8..07e31a4 100644 --- a/internal/analysis/ranking_test.go +++ b/internal/analysis/ranking_test.go @@ -1,6 +1,7 @@ package analysis import ( + "math" "testing" "github.com/shellcell/snailrace/internal/model" @@ -12,18 +13,53 @@ func TestDefaultIndexExcludesDiskFootprint(t *testing.T) { indexBenchmark("fast-but-fat", 1, 1, 100, 100, 1_000_000), } // Default index (time, cpu, ram) ranks the fast tool first despite its size. - if got := AutomaticBaseline(model.Config{Mode: "command"}, benchmarks); got != 2 { + if got, ok := AutomaticBaseline(model.Config{Mode: "command"}, benchmarks); !ok || got != 2 { t.Fatalf("default baseline = %d, want fast tool (2)", got) } // Including disk lets the huge footprint sink the fast tool. withDisk := model.Config{ Mode: "command", IndexDimensions: []string{"time", "cpu", "ram", "disk"}, } - if got := AutomaticBaseline(withDisk, benchmarks); got != 1 { + if got, ok := AutomaticBaseline(withDisk, benchmarks); !ok || got != 1 { t.Fatalf("disk baseline = %d, want compact tool (1)", got) } } +func TestUnavailableRankingUsesExplicitState(t *testing.T) { + benchmarks := []model.Benchmark{indexBenchmark("short", 1, 1, 100, 100, 10)} + config := model.Config{ + Mode: "command", IntervalMS: 10, IndexDimensions: []string{"ram"}, + } + ranking := Calculate(config, benchmarks) + if ranking.Available || ranking.UnavailableReason == "" { + t.Fatalf("ranking = %+v, want explicit unavailable state", ranking) + } + if len(ranking.Rows) != 1 || ranking.Rows[0].OverallRank != 0 || + math.IsInf(ranking.Rows[0].OverallScore, 0) || math.IsNaN(ranking.Rows[0].OverallScore) { + t.Fatalf("unavailable row contains invalid ranking values: %+v", ranking.Rows) + } + if _, ok := AutomaticBaseline(config, benchmarks); ok { + t.Fatal("unavailable ranking should not select an automatic baseline") + } +} + +func TestFailedBenchmarkIsExcludedFromRanking(t *testing.T) { + failed := indexBenchmark("failed", 0.001, 0.001, 100, 100, 10) + failed.Runs[0].ExitCode = 7 + failed.Summary = model.Summarize(failed.Runs) + success := indexBenchmark("success", 1, 1, 200, 200, 20) + config := model.Config{ + Mode: "command", IndexDimensions: []string{"time", "cpu"}, + } + ranking := Calculate(config, []model.Benchmark{failed, success}) + if !ranking.Available || len(ranking.Rows) != 1 || ranking.Rows[0].Benchmark != 1 { + t.Fatalf("failed tool was not excluded: %+v", ranking) + } + if baseline, ok := AutomaticBaseline(config, []model.Benchmark{failed, success}); !ok || baseline != 2 { + t.Fatalf("baseline = %d/%v, want successful tool 2", baseline, ok) + } +} + func indexBenchmark( name string, wall, cpu, meanRAM, peakRAM float64, diff --git a/internal/app/app.go b/internal/app/app.go index 60b83d0..bcbd908 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "os" - "os/exec" "os/signal" "runtime" "syscall" @@ -100,10 +99,29 @@ func Run(arguments []string, stdout, stderr io.Writer) error { if err != nil && !interrupted { return err } + rankingNote := "" + if config.Baseline == 0 { + ranking := analysis.Calculate(config, benchmarks) + if ranking.Available && len(ranking.Rows) > 0 { + config.Baseline = ranking.Rows[0].Benchmark + 1 + } else { + config.Baseline = firstEligibleBenchmark(benchmarks) + 1 + config.BaselineAutomatic = false + rankingNote = fmt.Sprintf( + "Balanced ranking unavailable: %s. %s is used only as the comparison baseline.", + ranking.UnavailableReason, + benchmarks[config.Baseline-1].Tool.Name, + ) + } + } notes := platformNotes( options.tui, options.duration, len(options.specs) > 1, config.BaselineAutomatic, config.OrderSeed, config.OutputMode, ) + if rankingNote != "" { + fmt.Fprintln(stderr, rankingNote) + notes = append([]string{rankingNote}, notes...) + } if interrupted { completed := 0 if len(benchmarks) > 0 { @@ -125,9 +143,6 @@ func Run(arguments []string, stdout, stderr io.Writer) error { fmt.Fprintln(stderr, hint) notes = append(notes, hint) } - if config.Baseline == 0 { - config.Baseline = analysis.AutomaticBaseline(config, benchmarks) - } _, host.MemoryAfterBytes = platform.Memory() result := model.Report{ MeasuredAt: measuredAt, Config: config, Host: host, @@ -142,6 +157,15 @@ func Run(arguments []string, stdout, stderr io.Writer) error { return nil } +func firstEligibleBenchmark(benchmarks []model.Benchmark) int { + for index, benchmark := range benchmarks { + if benchmark.EligibleForRanking() { + return index + } + } + return 0 +} + func oneBasedOrder(order [][]int) [][]int { result := make([][]int, len(order)) for round, tools := range order { @@ -170,9 +194,7 @@ func prepare(ctx context.Context, command string, output io.Writer) error { if command == "" { return nil } - cmd := exec.CommandContext(ctx, "/bin/sh", "-c", command) - cmd.Stdout, cmd.Stderr = output, output - if err := cmd.Run(); err != nil { + if err := runner.RunPreparation(ctx, command, output); err != nil { return fmt.Errorf("prepare command: %w", err) } return nil diff --git a/internal/app/filename.go b/internal/app/filename.go index c6b3b0e..da28df4 100644 --- a/internal/app/filename.go +++ b/internal/app/filename.go @@ -10,6 +10,10 @@ import ( ) func reportPath(directory, format string, report model.Report) string { + return reportPathWithStem(directory, format, reportStem(report)) +} + +func reportPathWithStem(directory, format, stem string) string { extension := strings.ToLower(format) switch extension { case "text": @@ -17,7 +21,7 @@ func reportPath(directory, format string, report model.Report) string { case "markdown": extension = "md" } - return filepath.Join(directory, reportStem(report)+"."+extension) + return filepath.Join(directory, stem+"."+extension) } func reportStem(report model.Report) string { diff --git a/internal/app/options_test.go b/internal/app/options_test.go index 252e076..d15d9d4 100644 --- a/internal/app/options_test.go +++ b/internal/app/options_test.go @@ -31,6 +31,14 @@ func TestRejectsLabelCountMismatch(t *testing.T) { } } +func TestRejectsEmptyShellCommand(t *testing.T) { + for _, command := range []string{"", " \t\n"} { + if _, err := parseOptions([]string{"-c", command}, io.Discard); err == nil { + t.Fatalf("command %q should be rejected", command) + } + } +} + func TestSingleLabelNamesPositionalCommand(t *testing.T) { options, err := parseOptions( []string{"-label", "build", "--", "sleep", "0.1"}, diff --git a/internal/app/output_bundle.go b/internal/app/output_bundle.go index f97f570..84306ed 100644 --- a/internal/app/output_bundle.go +++ b/internal/app/output_bundle.go @@ -1,10 +1,12 @@ package app import ( + "errors" "fmt" "io" "os" "path/filepath" + "strings" "github.com/shellcell/snailrace/internal/model" "github.com/shellcell/snailrace/internal/report" @@ -21,12 +23,23 @@ func saveReportFormats( } formats = uniqueFormats(formats) needsCharts := containsFormat(formats, "svg") || - containsFormat(formats, "markdown") || containsFormat(formats, "md") + containsFormat(formats, "markdown") + stem, release, err := reserveReportStem(directory, result) + if err != nil { + return err + } + defer release() + staging, err := os.MkdirTemp(directory, "."+stem+".tmp-") + if err != nil { + return err + } + defer os.RemoveAll(staging) + var charts []report.ChartArtifact - bundleDirectory := filepath.Join(directory, reportStem(result)) + stagedBundle := filepath.Join(staging, stem) + finalBundle := filepath.Join(directory, stem) if needsCharts { - chartDirectory := filepath.Join(bundleDirectory, "charts") - var err error + chartDirectory := filepath.Join(stagedBundle, "charts") charts, err = report.WriteChartFiles( chartDirectory, result, containsFormat(formats, "svg"), ) @@ -34,33 +47,93 @@ func saveReportFormats( return err } } + type stagedOutput struct{ staged, final string } + var outputs []stagedOutput + var announcements []string for _, format := range formats { switch format { case "svg": - if err := announce(stderr, filepath.Join(bundleDirectory, "charts")); err != nil { - return err - } - case "markdown", "md": - path := filepath.Join(bundleDirectory, "report.md") + announcements = append(announcements, filepath.Join(finalBundle, "charts")) + case "markdown": + path := filepath.Join(stagedBundle, "report.md") if err := writeMarkdownFile(path, result, charts); err != nil { return err } - if err := announce(stderr, path); err != nil { - return err - } + announcements = append(announcements, filepath.Join(finalBundle, "report.md")) default: - path := reportPath(directory, format, result) - if err := writeReportFile(path, format, result); err != nil { + stagedPath := reportPathWithStem(staging, format, stem) + if err := writeReportFile(stagedPath, format, result); err != nil { return err } - if err := announce(stderr, path); err != nil { - return err + finalPath := reportPathWithStem(directory, format, stem) + outputs = append(outputs, stagedOutput{staged: stagedPath, final: finalPath}) + announcements = append(announcements, finalPath) + } + } + if needsCharts { + outputs = append(outputs, stagedOutput{staged: stagedBundle, final: finalBundle}) + } + var published []string + for _, output := range outputs { + if err := renameNoReplace(output.staged, output.final); err != nil { + for _, path := range published { + os.RemoveAll(path) } + return err + } + published = append(published, output.final) + } + for _, path := range announcements { + if err := announce(stderr, path); err != nil { + return err } } return nil } +func reserveReportStem( + directory string, + result model.Report, +) (string, func(), error) { + base := reportStem(result) + for suffix := 1; ; suffix++ { + stem := base + if suffix > 1 { + stem = fmt.Sprintf("%s-%d", base, suffix) + } + lockPath := filepath.Join(directory, "."+stem+".lock") + lock, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if errors.Is(err, os.ErrExist) { + continue + } + if err != nil { + return "", nil, err + } + if err := lock.Close(); err != nil { + os.Remove(lockPath) + return "", nil, err + } + if reportStemExists(directory, stem) { + os.Remove(lockPath) + continue + } + return stem, func() { os.Remove(lockPath) }, nil + } +} + +func reportStemExists(directory, stem string) bool { + entries, err := os.ReadDir(directory) + if err != nil { + return true + } + for _, entry := range entries { + if entry.Name() == stem || strings.HasPrefix(entry.Name(), stem+".") { + return true + } + } + return false +} + func writeMarkdownFile( path string, result model.Report, @@ -74,11 +147,12 @@ func writeMarkdownFile( return err } writeErr := report.WriteMarkdownWithCharts(file, result, charts, "charts") - closeErr := file.Close() - if writeErr != nil { - return writeErr + syncErr := error(nil) + if writeErr == nil { + syncErr = file.Sync() } - return closeErr + closeErr := file.Close() + return errors.Join(writeErr, syncErr, closeErr) } func writeReportFile(path, format string, result model.Report) error { @@ -87,17 +161,25 @@ func writeReportFile(path, format string, result model.Report) error { return err } writeErr := report.Write(file, format, result) - closeErr := file.Close() - if writeErr != nil { - return writeErr + syncErr := error(nil) + if writeErr == nil { + syncErr = file.Sync() } - return closeErr + closeErr := file.Close() + return errors.Join(writeErr, syncErr, closeErr) } func uniqueFormats(formats []string) []string { seen := make(map[string]bool) result := make([]string, 0, len(formats)) for _, format := range formats { + format = strings.ToLower(format) + switch format { + case "txt": + format = "text" + case "md": + format = "markdown" + } if !seen[format] { seen[format] = true result = append(result, format) diff --git a/internal/app/output_test.go b/internal/app/output_test.go index 6ca5e2e..34bee06 100644 --- a/internal/app/output_test.go +++ b/internal/app/output_test.go @@ -2,8 +2,10 @@ package app import ( "bytes" + "io" "os" "strings" + "sync" "testing" "github.com/shellcell/snailrace/internal/model" @@ -113,8 +115,9 @@ func TestRunIgnoresNonZeroCommandExit(t *testing.T) { ); err != nil { t.Fatal(err) } - if !strings.Contains(stdout.String(), "TIME") { - t.Fatal("stdout does not contain the completed report") + if !strings.Contains(stdout.String(), "FAILED") || + !strings.Contains(stdout.String(), "excluded from rankings") { + t.Fatal("stdout does not report the failed command") } } @@ -139,3 +142,139 @@ func TestRunSavesDefaultHTMLInCurrentDirectory(t *testing.T) { t.Fatal("stderr does not announce the default report path") } } + +func TestSavingSameReportUsesUniqueNames(t *testing.T) { + directory := t.TempDir() + result := model.Report{ + Config: model.Config{Mode: "command", Baseline: 1}, + Benchmarks: []model.Benchmark{{ + Tool: model.ToolInfo{Name: "same"}, + }}, + } + const saves = 6 + errors := make(chan error, saves) + var wait sync.WaitGroup + for index := 0; index < saves; index++ { + wait.Add(1) + go func() { + defer wait.Done() + errors <- saveReportFormats(io.Discard, directory, []string{"html"}, result) + }() + } + wait.Wait() + close(errors) + for err := range errors { + if err != nil { + t.Fatal(err) + } + } + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + if len(entries) != saves { + t.Fatalf("saved entries = %d, want %d unique reports", len(entries), saves) + } +} + +func TestSavingDifferentFormatsDoesNotReuseExistingStem(t *testing.T) { + directory := t.TempDir() + result := model.Report{ + Config: model.Config{Mode: "command", Baseline: 1}, + Benchmarks: []model.Benchmark{{Tool: model.ToolInfo{Name: "same"}}}, + } + if err := saveReportFormats(io.Discard, directory, []string{"html"}, result); err != nil { + t.Fatal(err) + } + if err := saveReportFormats(io.Discard, directory, []string{"markdown"}, result); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + if len(entries) != 2 || entries[0].Name() == entries[1].Name() { + t.Fatalf("saved entries reused a report stem: %v", entries) + } +} + +func TestFailedSavePublishesNoPartialReports(t *testing.T) { + directory := t.TempDir() + result := model.Report{ + Config: model.Config{Mode: "command", Baseline: 1}, + Benchmarks: []model.Benchmark{{Tool: model.ToolInfo{Name: "tool"}}}, + } + if err := saveReportFormats( + io.Discard, directory, []string{"html", "unknown"}, result, + ); err == nil { + t.Fatal("invalid staged format should fail") + } + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("failed save left partial entries: %v", entries) + } +} + +func TestPublishingNeverReplacesExistingDestination(t *testing.T) { + directory := t.TempDir() + staged := directory + "/staged" + final := directory + "/final" + if err := os.WriteFile(staged, []byte("new"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(final, []byte("existing"), 0o600); err != nil { + t.Fatal(err) + } + if err := renameNoReplace(staged, final); err == nil { + t.Fatal("publication replaced an existing destination") + } + data, err := os.ReadFile(final) + if err != nil { + t.Fatal(err) + } + if string(data) != "existing" { + t.Fatalf("destination content = %q, want existing", data) + } +} + +func TestSavedSVGBundleIncludesCommandFailures(t *testing.T) { + directory := t.TempDir() + failedRun := model.Run{ExitCode: 7, StopReason: "exited", WallSeconds: 0.001} + result := model.Report{ + Config: model.Config{Mode: "command", Baseline: 1}, + Benchmarks: []model.Benchmark{{ + Tool: model.ToolInfo{Name: "failed"}, + Runs: []model.Run{failedRun}, Summary: model.Summarize([]model.Run{failedRun}), + }}, + } + if err := saveReportFormats(io.Discard, directory, []string{"svg"}, result); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(directory) + if err != nil || len(entries) != 1 { + t.Fatalf("bundle entries = %v, error = %v", entries, err) + } + charts, err := os.ReadDir(directory + "/" + entries[0].Name() + "/charts") + if err != nil { + t.Fatal(err) + } + found := false + for _, chart := range charts { + if strings.Contains(chart.Name(), "command-failures") { + found = true + } + } + if !found { + t.Fatalf("SVG charts do not include command failures: %v", charts) + } +} + +func TestFormatAliasesAreCanonicalizedBeforeSaving(t *testing.T) { + formats := uniqueFormats([]string{"text", "txt", "markdown", "md"}) + if len(formats) != 2 || formats[0] != "text" || formats[1] != "markdown" { + t.Fatalf("canonical formats = %v", formats) + } +} diff --git a/internal/app/progress_race_test.go b/internal/app/progress_race_test.go index c5022d5..7cdbd6f 100644 --- a/internal/app/progress_race_test.go +++ b/internal/app/progress_race_test.go @@ -197,6 +197,10 @@ func progressEstimate(name string, scale float64) runner.ProgressEstimate { stats := model.Stats{Mean: scale} return runner.ProgressEstimate{ ToolName: name, Completed: 1, Total: 2, HasEstimate: true, + Runs: []model.Run{{ + WallSeconds: scale, CPUUserSeconds: scale, + MeanResidentBytes: scale, PeakResidentBytes: scale, + }}, DiskFootprintBytes: int64(scale * 100), Estimate: model.Summary{ WallSeconds: stats, CPUTotalSeconds: stats, diff --git a/internal/app/publish_darwin.go b/internal/app/publish_darwin.go new file mode 100644 index 0000000..3f897dd --- /dev/null +++ b/internal/app/publish_darwin.go @@ -0,0 +1,9 @@ +//go:build darwin + +package app + +import "golang.org/x/sys/unix" + +func renameNoReplace(oldPath, newPath string) error { + return unix.RenamexNp(oldPath, newPath, unix.RENAME_EXCL) +} diff --git a/internal/app/publish_linux.go b/internal/app/publish_linux.go new file mode 100644 index 0000000..481c18d --- /dev/null +++ b/internal/app/publish_linux.go @@ -0,0 +1,11 @@ +//go:build linux + +package app + +import "golang.org/x/sys/unix" + +func renameNoReplace(oldPath, newPath string) error { + return unix.Renameat2( + unix.AT_FDCWD, oldPath, unix.AT_FDCWD, newPath, unix.RENAME_NOREPLACE, + ) +} diff --git a/internal/app/validation.go b/internal/app/validation.go index 271758a..ea339fa 100644 --- a/internal/app/validation.go +++ b/internal/app/validation.go @@ -3,6 +3,7 @@ package app import ( "errors" "fmt" + "strings" "time" "github.com/shellcell/snailrace/internal/report" @@ -41,6 +42,11 @@ func validateOptions(result options, commands []string, positional int) error { if len(commands) == 0 && positional == 0 { return errors.New("no command specified") } + for _, command := range commands { + if strings.TrimSpace(command) == "" { + return errors.New("command cannot be empty") + } + } commandCount := len(commands) if positional > 0 { commandCount = 1 diff --git a/internal/model/model.go b/internal/model/model.go index 167a18e..852f721 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -72,6 +72,10 @@ type Run struct { SampleCoverageSeconds float64 `json:"sample_coverage_seconds"` } +func (run Run) Failed() bool { + return run.StopReason != "duration" && run.ExitCode != 0 +} + type Stats struct { N int `json:"n"` Min float64 `json:"min"` @@ -116,6 +120,20 @@ type Benchmark struct { Summary Summary `json:"summary"` } +func (benchmark Benchmark) FailedRunCount() int { + count := 0 + for _, run := range benchmark.Runs { + if run.Failed() { + count++ + } + } + return count +} + +func (benchmark Benchmark) EligibleForRanking() bool { + return len(benchmark.Runs) > 0 && benchmark.FailedRunCount() == 0 +} + type Report struct { MeasuredAt time.Time `json:"measured_at"` Config Config `json:"config"` diff --git a/internal/report/chart_artifacts.go b/internal/report/chart_artifacts.go index fe822c9..4cc7b15 100644 --- a/internal/report/chart_artifacts.go +++ b/internal/report/chart_artifacts.go @@ -25,7 +25,11 @@ func WriteChartFiles( } charts := reportCharts(report) if includeCommands { - charts = append([]svgChart{commandLegendChart(report)}, charts...) + prefix := []svgChart{commandLegendChart(report)} + if failures := failureChart(report); failures.height > 0 { + prefix = append(prefix, failures) + } + charts = append(prefix, charts...) } artifacts := make([]ChartArtifact, 0, len(charts)) for index, chart := range charts { diff --git a/internal/report/chart_ranking.go b/internal/report/chart_ranking.go index accec9a..39ff07f 100644 --- a/internal/report/chart_ranking.go +++ b/internal/report/chart_ranking.go @@ -19,14 +19,18 @@ type rankingChartMetric struct { func rankingCharts(report model.Report) []svgChart { ranking := calculateRanking(report) - metrics := []rankingChartMetric{ - {"BALANCED INDEX", formatScore, + if len(ranking.rows) == 0 { + return nil + } + var metrics []rankingChartMetric + if ranking.available { + metrics = append(metrics, rankingChartMetric{"BALANCED INDEX", formatScore, func(row rankingRow) float64 { return row.overallScore }, - func(row rankingRow) int { return row.overallRank }}, - {ranking.primaryLabel, ranking.primaryUnit, - func(row rankingRow) float64 { return row.primaryValue }, - func(row rankingRow) int { return row.primaryRank }}, + func(row rankingRow) int { return row.overallRank }}) } + metrics = append(metrics, rankingChartMetric{ranking.primaryLabel, ranking.primaryUnit, + func(row rankingRow) float64 { return row.primaryValue }, + func(row rankingRow) int { return row.primaryRank }}) if !(report.Config.Mode == "tui" && report.Config.DurationSeconds > 0) { metrics = append(metrics, rankingChartMetric{ "CPU COST", formatDuration, diff --git a/internal/report/chart_test.go b/internal/report/chart_test.go index abf479d..b89208d 100644 --- a/internal/report/chart_test.go +++ b/internal/report/chart_test.go @@ -186,8 +186,13 @@ func TestFixedTUIRankingChartUsesCPUUnits(t *testing.T) { }, } charts := rankingCharts(report) - if len(charts) < 2 || charts[1].title != "CPU" || - !strings.Contains(charts[1].body, "%") { + found := false + for _, chart := range charts { + if chart.title == "CPU" && strings.Contains(chart.body, "%") { + found = true + } + } + if !found { t.Fatal("fixed TUI primary ranking chart should show average CPU percentage") } } diff --git a/internal/report/error_writer.go b/internal/report/error_writer.go new file mode 100644 index 0000000..d005df5 --- /dev/null +++ b/internal/report/error_writer.go @@ -0,0 +1,28 @@ +package report + +import "io" + +type errorWriter struct { + writer io.Writer + err error +} + +func newErrorWriter(writer io.Writer) *errorWriter { + if checked, ok := writer.(*errorWriter); ok { + return checked + } + return &errorWriter{writer: writer} +} + +func (writer *errorWriter) Write(value []byte) (int, error) { + if writer.err != nil { + return 0, writer.err + } + written, err := writer.writer.Write(value) + if err != nil { + writer.err = err + } + return written, err +} + +func (writer *errorWriter) Err() error { return writer.err } diff --git a/internal/report/error_writer_test.go b/internal/report/error_writer_test.go new file mode 100644 index 0000000..e6392fe --- /dev/null +++ b/internal/report/error_writer_test.go @@ -0,0 +1,42 @@ +package report + +import ( + "errors" + "testing" + + "github.com/shellcell/snailrace/internal/model" +) + +var errWriteLimit = errors.New("write limit reached") + +type limitedWriter struct { + remaining int +} + +func (writer *limitedWriter) Write(value []byte) (int, error) { + if writer.remaining <= 0 { + return 0, errWriteLimit + } + if len(value) > writer.remaining { + written := writer.remaining + writer.remaining = 0 + return written, errWriteLimit + } + writer.remaining -= len(value) + return len(value), nil +} + +func TestReportFormatsPropagateWriterErrors(t *testing.T) { + report := model.Report{ + Config: model.Config{Mode: "command", Baseline: 1}, + Benchmarks: []model.Benchmark{benchmarkWithWallTimes("tool", 1, 2)}, + } + for _, format := range []string{"text", "html", "markdown", "svg", "json"} { + t.Run(format, func(t *testing.T) { + err := Write(&limitedWriter{}, format, report) + if !errors.Is(err, errWriteLimit) { + t.Fatalf("error = %v, want write failure", err) + } + }) + } +} diff --git a/internal/report/failures.go b/internal/report/failures.go new file mode 100644 index 0000000..0ed9ade --- /dev/null +++ b/internal/report/failures.go @@ -0,0 +1,125 @@ +package report + +import ( + "fmt" + "html" + "io" + "sort" + "strconv" + "strings" + + "github.com/shellcell/snailrace/internal/model" +) + +type benchmarkFailure struct { + benchmark int + failed int + total int + exitCodes []int +} + +func benchmarkFailures(report model.Report) []benchmarkFailure { + var failures []benchmarkFailure + for index, benchmark := range report.Benchmarks { + failed := benchmark.FailedRunCount() + if failed == 0 { + continue + } + seen := make(map[int]bool) + for _, run := range benchmark.Runs { + if run.Failed() { + seen[run.ExitCode] = true + } + } + codes := make([]int, 0, len(seen)) + for code := range seen { + codes = append(codes, code) + } + sort.Ints(codes) + failures = append(failures, benchmarkFailure{ + benchmark: index, failed: failed, total: len(benchmark.Runs), exitCodes: codes, + }) + } + return failures +} + +func failureExitCodes(failure benchmarkFailure) string { + values := make([]string, len(failure.exitCodes)) + for index, code := range failure.exitCodes { + values[index] = strconv.Itoa(code) + } + return strings.Join(values, ", ") +} + +func writeTextFailures(writer io.Writer, report model.Report) { + failures := benchmarkFailures(report) + for _, failure := range failures { + fmt.Fprintf( + writer, "FAILED %s: %d/%d measured runs exited non-zero (codes %s); excluded from rankings\n", + report.Benchmarks[failure.benchmark].Tool.Name, + failure.failed, failure.total, failureExitCodes(failure), + ) + } + if len(failures) > 0 { + fmt.Fprintln(writer) + } +} + +func writeHTMLFailures(writer io.Writer, report model.Report) { + failures := benchmarkFailures(report) + if len(failures) == 0 { + return + } + fmt.Fprint(writer, `

Command failures

`) + for _, failure := range failures { + fmt.Fprintf( + writer, `

%s: %d/%d measured runs exited `+ + `non-zero (codes %s); excluded from rankings.

`, + htmlToolLabel(report, failure.benchmark, true), + failure.failed, failure.total, html.EscapeString(failureExitCodes(failure)), + ) + } + fmt.Fprint(writer, `
`) +} + +func writeMarkdownFailures(writer io.Writer, report model.Report) { + failures := benchmarkFailures(report) + if len(failures) == 0 { + return + } + fmt.Fprintln(writer, "## Command Failures") + for _, failure := range failures { + fmt.Fprintf( + writer, "\n- **%s**: %d/%d measured runs exited non-zero (codes %s); excluded from rankings.\n", + escapeMarkdown(report.Benchmarks[failure.benchmark].Tool.Name), + failure.failed, failure.total, failureExitCodes(failure), + ) + } + fmt.Fprintln(writer) +} + +func failureChart(report model.Report) svgChart { + failures := benchmarkFailures(report) + if len(failures) == 0 { + return svgChart{} + } + height := 54 + len(failures)*24 + var body strings.Builder + body.WriteString(svgChartStyle) + body.WriteString(``) + body.WriteString(`COMMAND FAILURES`) + for index, failure := range failures { + fmt.Fprintf( + &body, `%s: `+ + `%d/%d runs, codes %s; excluded from rankings`, + 50+index*24, + html.EscapeString(clip(report.Benchmarks[failure.benchmark].Tool.Name, 28)), + failure.failed, failure.total, html.EscapeString(failureExitCodes(failure)), + ) + } + return svgChart{ + kind: "failure", title: "Command failures", slug: "command-failures", + description: "Lists commands with non-zero measured runs that were excluded from rankings.", + body: body.String(), height: height, + } +} diff --git a/internal/report/html.go b/internal/report/html.go index 4c43814..1e5042a 100644 --- a/internal/report/html.go +++ b/internal/report/html.go @@ -59,6 +59,8 @@ th:first-child,td:first-child{text-align:left}code{color:var(--accent);white-spa
` func writeHTML(writer io.Writer, report model.Report) error { + checked := newErrorWriter(writer) + writer = checked fmt.Fprint(writer, htmlStart) fmt.Fprintf( writer, @@ -68,6 +70,7 @@ func writeHTML(writer io.Writer, report model.Report) error { ) writeCards(writer, report) writeHTMLCommandLegend(writer, report) + writeHTMLFailures(writer, report) if len(report.Benchmarks) > 1 { writeHTMLRanking(writer, report) } @@ -89,8 +92,8 @@ func writeHTML(writer io.Writer, report model.Report) error { for _, note := range report.Notes { fmt.Fprintf(writer, "

Note: %s

", html.EscapeString(note)) } - _, err := fmt.Fprint(writer, "
") - return err + fmt.Fprint(writer, "") + return checked.Err() } type chartSection struct { diff --git a/internal/report/markdown.go b/internal/report/markdown.go index 16154e4..f08c0b9 100644 --- a/internal/report/markdown.go +++ b/internal/report/markdown.go @@ -18,6 +18,8 @@ func WriteMarkdownWithCharts( charts []ChartArtifact, chartDirectory string, ) error { + checked := newErrorWriter(writer) + writer = checked fmt.Fprintf( writer, "# 🐌 Snailrace Report\n\nMeasured %s on `%s/%s`. "+ @@ -27,6 +29,7 @@ func WriteMarkdownWithCharts( report.Config.Runs, report.Config.Warmups, report.Config.Mode, ) writeMarkdownCommandLegend(writer, report) + writeMarkdownFailures(writer, report) if len(report.Benchmarks) > 1 { writeMarkdownRanking(writer, report) } @@ -56,7 +59,7 @@ func WriteMarkdownWithCharts( for _, note := range report.Notes { fmt.Fprintf(writer, "> %s\n\n", escapeMarkdown(note)) } - return nil + return checked.Err() } func writeMarkdownBenchmark( diff --git a/internal/report/ranking.go b/internal/report/ranking.go index 7d1b445..78acb63 100644 --- a/internal/report/ranking.go +++ b/internal/report/ranking.go @@ -8,7 +8,8 @@ import ( func calculateRanking(report model.Report) rankingData { numeric := analysis.Calculate(report.Config, report.Benchmarks) result := rankingData{ - rows: make([]rankingRow, len(numeric.Rows)), + rows: make([]rankingRow, len(numeric.Rows)), + available: numeric.Available, unavailableReason: numeric.UnavailableReason, ramAvailable: numeric.RAMAvailable, ramPresent: numeric.RAMPresent, bestOverall: numeric.BestOverall, primaryRatio: numeric.PrimaryRatio, cpuRatio: numeric.CPURatio, diff --git a/internal/report/ranking_html.go b/internal/report/ranking_html.go index 1314c5c..2461490 100644 --- a/internal/report/ranking_html.go +++ b/internal/report/ranking_html.go @@ -11,6 +11,13 @@ import ( func writeHTMLRanking(writer io.Writer, report model.Report) { ranking := calculateRanking(report) + if len(ranking.rows) == 0 { + fmt.Fprintf( + writer, `

Ranking unavailable

%s

`, + html.EscapeString(ranking.unavailableReason), + ) + return + } fmt.Fprint(writer, `

Category leaders (point estimates)

`) for _, winner := range ranking.winners { var names string @@ -28,6 +35,13 @@ func writeHTMLRanking(writer io.Writer, report model.Report) { html.EscapeString(winner.value), ) } + if !ranking.available { + fmt.Fprintf( + writer, `

Overall ranking unavailable

%s

`, + html.EscapeString(ranking.unavailableReason), + ) + return + } fmt.Fprintf( writer, `

Overall ranking (point estimates)

Comparison baseline: `+ diff --git a/internal/report/ranking_json.go b/internal/report/ranking_json.go index 2853a01..8db795d 100644 --- a/internal/report/ranking_json.go +++ b/internal/report/ranking_json.go @@ -13,11 +13,18 @@ type jsonWinner struct { Value string `json:"formatted_value"` } +type jsonFailure struct { + Tool string `json:"tool"` + Failed int `json:"failed_runs"` + Total int `json:"total_runs"` + ExitCodes []int `json:"exit_codes"` +} + type jsonRankingRow struct { - Rank int `json:"rank"` + Rank int `json:"rank,omitempty"` Tool string `json:"tool"` - BalancedIndex float64 `json:"balanced_index"` - BalancedDeltaPercent float64 `json:"balanced_delta_percent"` + BalancedIndex *float64 `json:"balanced_index,omitempty"` + BalancedDeltaPercent *float64 `json:"balanced_delta_percent,omitempty"` PrimaryValue float64 `json:"primary_value"` PrimaryRank int `json:"primary_rank"` PrimaryDeltaPercent *float64 `json:"primary_delta_percent,omitempty"` @@ -33,25 +40,42 @@ type jsonRankingRow struct { } type jsonRanking struct { - Method string `json:"method"` - PrimaryMetric string `json:"primary_metric"` - Winners []jsonWinner `json:"winners"` - Rows []jsonRankingRow `json:"rows"` + Available bool `json:"available"` + UnavailableReason string `json:"unavailable_reason,omitempty"` + Method string `json:"method"` + PrimaryMetric string `json:"primary_metric"` + Winners []jsonWinner `json:"winners"` + Rows []jsonRankingRow `json:"rows"` } func writeJSON(writer io.Writer, report model.Report) error { payload := struct { model.Report - Ranking jsonRanking `json:"ranking"` - }{Report: report, Ranking: makeJSONRanking(report)} + Ranking jsonRanking `json:"ranking"` + Failures []jsonFailure `json:"failures,omitempty"` + }{Report: report, Ranking: makeJSONRanking(report), Failures: makeJSONFailures(report)} encoder := json.NewEncoder(writer) encoder.SetIndent("", " ") return encoder.Encode(payload) } +func makeJSONFailures(report model.Report) []jsonFailure { + failures := benchmarkFailures(report) + result := make([]jsonFailure, 0, len(failures)) + for _, failure := range failures { + result = append(result, jsonFailure{ + Tool: report.Benchmarks[failure.benchmark].Tool.Name, + Failed: failure.failed, Total: failure.total, + ExitCodes: append([]int(nil), failure.exitCodes...), + }) + } + return result +} + func makeJSONRanking(report model.Report) jsonRanking { ranking := calculateRanking(report) result := jsonRanking{ + Available: ranking.available, UnavailableReason: ranking.unavailableReason, Method: "equal-weight geometric mean of normalized category costs; included: " + balancedIndexCategories(ranking), PrimaryMetric: ranking.primaryLabel, @@ -64,13 +88,18 @@ func makeJSONRanking(report model.Report) jsonRanking { } for _, row := range ranking.rows { item := jsonRankingRow{ - Rank: row.overallRank, Tool: report.Benchmarks[row.benchmark].Tool.Name, - BalancedIndex: row.overallScore, - BalancedDeltaPercent: (row.overallScore/ranking.bestOverall - 1) * 100, - PrimaryValue: row.primaryValue, PrimaryRank: row.primaryRank, + Tool: report.Benchmarks[row.benchmark].Tool.Name, + PrimaryValue: row.primaryValue, PrimaryRank: row.primaryRank, CPUValue: row.cpuValue, CPURank: row.cpuRank, FootprintBytes: row.footprintValue, FootprintRank: row.footprintRank, } + if ranking.available { + item.Rank = row.overallRank + item.BalancedIndex = floatPointer(row.overallScore) + item.BalancedDeltaPercent = floatPointer( + (row.overallScore/ranking.bestOverall - 1) * 100, + ) + } if ranking.primaryRatio { item.PrimaryDeltaPercent = floatPointer((row.primaryScore - 1) * 100) } diff --git a/internal/report/ranking_markdown.go b/internal/report/ranking_markdown.go index b078d1f..394111f 100644 --- a/internal/report/ranking_markdown.go +++ b/internal/report/ranking_markdown.go @@ -9,6 +9,13 @@ import ( func writeMarkdownRanking(writer io.Writer, report model.Report) { ranking := calculateRanking(report) + if len(ranking.rows) == 0 { + fmt.Fprintf( + writer, "## Ranking Unavailable\n\n%s.\n\n", + escapeMarkdown(ranking.unavailableReason), + ) + return + } fmt.Fprintln(writer, "## Category Leaders (Point Estimates)") fmt.Fprintln(writer, "\n| Category | Tool | Value |") fmt.Fprintln(writer, "|---|---|---:|") @@ -22,6 +29,13 @@ func writeMarkdownRanking(writer io.Writer, report model.Report) { writer, "\nComparison baseline: **%s**. Lower balanced index is better.\n", escapeMarkdown(report.Benchmarks[baselineIndex(report)].Tool.Name), ) + if !ranking.available { + fmt.Fprintf( + writer, "\n## Overall Ranking Unavailable\n\n%s.\n\n", + escapeMarkdown(ranking.unavailableReason), + ) + return + } fmt.Fprintln(writer, "\n## Overall Ranking (Point Estimates)") fmt.Fprintf( writer, diff --git a/internal/report/ranking_test.go b/internal/report/ranking_test.go index 6f37ab8..dee9845 100644 --- a/internal/report/ranking_test.go +++ b/internal/report/ranking_test.go @@ -14,7 +14,7 @@ func TestAutomaticBaselineUsesBalancedWinner(t *testing.T) { rankingBenchmark("efficient", 3, 1, 100, 100, 100), } config := model.Config{Mode: "command"} - if got := analysis.AutomaticBaseline(config, benchmarks); got != 3 { + if got, ok := analysis.AutomaticBaseline(config, benchmarks); !ok || got != 3 { t.Fatalf("baseline = %d, want efficient tool at index 3", got) } ranking := calculateRanking(model.Report{Config: config, Benchmarks: benchmarks}) diff --git a/internal/report/ranking_text.go b/internal/report/ranking_text.go index b65d4f0..e09a835 100644 --- a/internal/report/ranking_text.go +++ b/internal/report/ranking_text.go @@ -10,6 +10,10 @@ import ( func writeTextRanking(writer io.Writer, report model.Report) { ranking := calculateRanking(report) + if len(ranking.rows) == 0 { + fmt.Fprintf(writer, "Ranking unavailable\t%s\n\n", ranking.unavailableReason) + return + } fmt.Fprintln(writer, "Category leaders (point estimates)\tTool\tValue") for _, winner := range ranking.winners { fmt.Fprintf( @@ -22,6 +26,10 @@ func writeTextRanking(writer io.Writer, report model.Report) { writer, "Comparison baseline\t%s\n\n", report.Benchmarks[baselineIndex(report)].Tool.Name, ) + if !ranking.available { + fmt.Fprintf(writer, "Overall ranking unavailable\t%s\n\n", ranking.unavailableReason) + return + } fmt.Fprintf( writer, "Overall ranking (point estimates)\tTool\tBalanced\t%s\tCPU cost\tRAM aggregate\tLinked size\n", diff --git a/internal/report/ranking_types.go b/internal/report/ranking_types.go index 944fab7..9999145 100644 --- a/internal/report/ranking_types.go +++ b/internal/report/ranking_types.go @@ -25,19 +25,21 @@ type categoryWinner struct { } type rankingData struct { - rows []rankingRow - winners []categoryWinner - primaryLabel string - primaryUnit func(float64) string - ramAvailable bool - ramPresent bool - samplingInterval string - primaryRatio bool - cpuRatio bool - footprintRatio bool - indexPrimary bool - indexCPU bool - indexRAM bool - indexFootprint bool - bestOverall float64 + rows []rankingRow + winners []categoryWinner + available bool + unavailableReason string + primaryLabel string + primaryUnit func(float64) string + ramAvailable bool + ramPresent bool + samplingInterval string + primaryRatio bool + cpuRatio bool + footprintRatio bool + indexPrimary bool + indexCPU bool + indexRAM bool + indexFootprint bool + bestOverall float64 } diff --git a/internal/report/ranking_winners.go b/internal/report/ranking_winners.go index 8547bd0..d542ff0 100644 --- a/internal/report/ranking_winners.go +++ b/internal/report/ranking_winners.go @@ -11,20 +11,26 @@ func rankingWinners(report model.Report, ranking rankingData) []categoryWinner { if len(ranking.rows) == 0 { return nil } - winners := []categoryWinner{ - winnerForRank(report, ranking.rows, "OVERALL", func(row rankingRow) int { + var winners []categoryWinner + appendWinner := func(winner categoryWinner) { + if len(winner.benchmarks) > 0 { + winners = append(winners, winner) + } + } + if ranking.available { + appendWinner(winnerForRank(report, ranking.rows, "OVERALL", func(row rankingRow) int { return row.overallRank }, func(row rankingRow) string { return fmt.Sprintf("%.3fx balanced index", row.overallScore) - }), - winnerForRank(report, ranking.rows, ranking.primaryLabel, func(row rankingRow) int { - return row.primaryRank - }, func(row rankingRow) string { - return ranking.primaryUnit(row.primaryValue) - }), + })) } + appendWinner(winnerForRank(report, ranking.rows, ranking.primaryLabel, func(row rankingRow) int { + return row.primaryRank + }, func(row rankingRow) string { + return ranking.primaryUnit(row.primaryValue) + })) if !(report.Config.Mode == "tui" && report.Config.DurationSeconds > 0) { - winners = append(winners, winnerForRank( + appendWinner(winnerForRank( report, ranking.rows, "CPU COST", func(row rankingRow) int { return row.cpuRank }, func(row rankingRow) string { @@ -32,13 +38,13 @@ func rankingWinners(report model.Report, ranking rankingData) []categoryWinner { })) } if ranking.ramAvailable { - winners = append(winners, winnerForRank( + appendWinner(winnerForRank( report, ranking.rows, "RAM COST", func(row rankingRow) int { return row.ramRank }, func(row rankingRow) string { return formatBytes(row.ramValue) + " aggregate" }, )) } - winners = append(winners, winnerForRank( + appendWinner(winnerForRank( report, ranking.rows, "LINKED SIZE", func(row rankingRow) int { return row.footprintRank }, func(row rankingRow) string { return formatBytes(row.footprintValue) }, diff --git a/internal/report/svg.go b/internal/report/svg.go index 2e044a0..dc7ff8d 100644 --- a/internal/report/svg.go +++ b/internal/report/svg.go @@ -22,10 +22,16 @@ const svgReportStyle = `` func writeSVG(writer io.Writer, report model.Report) error { + checked := newErrorWriter(writer) + writer = checked var content strings.Builder y := svgHeader(&content, report) y = svgCommandLegend(&content, report, y) - y = svgCharts(&content, reportCharts(report), y) + charts := reportCharts(report) + if failures := failureChart(report); failures.height > 0 { + charts = append([]svgChart{failures}, charts...) + } + y = svgCharts(&content, charts, y) height := y + 36 fmt.Fprintf( writer, @@ -36,7 +42,7 @@ func writeSVG(writer io.Writer, report model.Report) error { `%s%s`, svgReportWidth, height, svgReportStyle, content.String(), ) - return nil + return checked.Err() } func svgHeader(output *strings.Builder, report model.Report) int { diff --git a/internal/report/text.go b/internal/report/text.go index 3332219..173f859 100644 --- a/internal/report/text.go +++ b/internal/report/text.go @@ -58,6 +58,7 @@ func writeVerboseText(writer io.Writer, report model.Report) error { } fmt.Fprintln(w) writeTextCommandLegend(w, report) + writeTextFailures(w, report) if len(report.Benchmarks) > 1 { writeTextRanking(w, report) } diff --git a/internal/report/text_bars.go b/internal/report/text_bars.go index 46d44ec..0339053 100644 --- a/internal/report/text_bars.go +++ b/internal/report/text_bars.go @@ -18,6 +18,7 @@ func writeCompactText(writer io.Writer, report model.Report) error { terminal := writerIsTerminal(writer) var body strings.Builder body.WriteString("\n") + writeTextFailures(&body, report) if len(report.Benchmarks) == 1 { writeCompactSingle(&body, report, terminal) } else if len(report.Benchmarks) > 1 { @@ -101,17 +102,25 @@ func cpuCompactUnit(fixedTUI bool, value float64) string { func writeCompactBars(body *strings.Builder, report model.Report, terminal bool) { ranking := calculateRanking(report) + if len(ranking.rows) == 0 { + fmt.Fprintf(body, "BALANCED unavailable: %s\n\n", ranking.unavailableReason) + return + } rows := rowsByBenchmark(ranking.rows) labelWidth := compactLabelWidth(report) - winner := report.Benchmarks[ranking.rows[0].benchmark].Tool.Name - fmt.Fprintf( - body, "BALANCED %s winner %s\n", - strings.Join(indexDimensionLabels(report.Config), ","), winner, - ) - writeBarGroup(body, report, terminal, labelWidth, rows, func(r rankingRow) float64 { - return r.overallScore - }, func(r rankingRow) string { return formatScore(r.overallScore) }) + if ranking.available { + winner := report.Benchmarks[ranking.rows[0].benchmark].Tool.Name + fmt.Fprintf( + body, "BALANCED %s winner %s\n", + balancedIndexCategories(ranking), winner, + ) + writeBarGroup(body, report, terminal, labelWidth, rows, func(r rankingRow) float64 { + return r.overallScore + }, func(r rankingRow) string { return formatScore(r.overallScore) }) + } else { + fmt.Fprintf(body, "BALANCED unavailable: %s\n\n", ranking.unavailableReason) + } for _, metric := range compactMetrics(report, ranking) { if !metric.available { @@ -231,11 +240,3 @@ func padLabel(label string, width int) string { } return label } - -func indexDimensionLabels(config model.Config) []string { - dimensions := config.IndexDimensions - if len(dimensions) == 0 { - dimensions = []string{"time", "cpu", "ram"} - } - return dimensions -} diff --git a/internal/report/ui_test.go b/internal/report/ui_test.go index ed3d843..fa21721 100644 --- a/internal/report/ui_test.go +++ b/internal/report/ui_test.go @@ -41,6 +41,48 @@ func TestJSONIncludesComputedRanking(t *testing.T) { } } +func TestUnavailableRankingSerializesWithoutNonFiniteValues(t *testing.T) { + report := model.Report{ + Config: model.Config{ + Mode: "command", Baseline: 1, IntervalMS: 10, + IndexDimensions: []string{"ram"}, + }, + Benchmarks: []model.Benchmark{benchmarkWithWallTimes("short", 0.001)}, + } + var output bytes.Buffer + if err := writeJSON(&output, report); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), `"available": false`) || + strings.Contains(output.String(), "+Inf") || strings.Contains(output.String(), "NaN") { + t.Fatalf("unexpected unavailable ranking JSON: %s", output.String()) + } +} + +func TestFailedCommandsAreProminentInEveryFormat(t *testing.T) { + failed := benchmarkWithWallTimes("failed", 0.001) + failed.Runs[0].ExitCode = 7 + failed.Summary = model.Summarize(failed.Runs) + report := model.Report{ + Config: model.Config{ + Mode: "command", Baseline: 2, IndexDimensions: []string{"time"}, + }, + Benchmarks: []model.Benchmark{failed, benchmarkWithWallTimes("success", 1)}, + } + for _, format := range []string{"text", "html", "markdown", "svg", "json"} { + t.Run(format, func(t *testing.T) { + var output bytes.Buffer + if err := Write(&output, format, report); err != nil { + t.Fatal(err) + } + value := strings.ToLower(output.String()) + if !strings.Contains(value, "failed") || !strings.Contains(value, "7") { + t.Fatalf("%s does not prominently report failure: %s", format, output.String()) + } + }) + } +} + func TestHTMLPreservesLongCommandInWrappingBlock(t *testing.T) { command := "tool --value=" + strings.Repeat("x", 300) report := model.Report{ diff --git a/internal/runner/benchmark.go b/internal/runner/benchmark.go index e76163d..d75cbda 100644 --- a/internal/runner/benchmark.go +++ b/internal/runner/benchmark.go @@ -19,6 +19,9 @@ func Benchmark( config Config, options Options, ) ([]model.Benchmark, error) { + if err := validateBenchmarkInputs(ctx, specs, config); err != nil { + return nil, err + } benchmarks := make([]model.Benchmark, len(specs)) progress := newProgressTracker(options, specs, config) defer progress.finish() diff --git a/internal/runner/benchmark_test.go b/internal/runner/benchmark_test.go index 3ef0089..926a9b6 100644 --- a/internal/runner/benchmark_test.go +++ b/internal/runner/benchmark_test.go @@ -75,6 +75,37 @@ func TestBenchmarkContinuesAfterNonZeroExit(t *testing.T) { } } +func TestBenchmarkRejectsInvalidInputs(t *testing.T) { + validSpec := Spec{Name: "true", Args: []string{"/bin/true"}} + validConfig := Config{Runs: 1, Interval: time.Millisecond} + tests := []struct { + name string + specs []Spec + config Config + }{ + {"no specs", nil, validConfig}, + {"empty spec", []Spec{{Name: "empty"}}, validConfig}, + {"both command forms", []Spec{{Name: "both", Shell: "true", Args: []string{"true"}}}, validConfig}, + {"zero runs", []Spec{validSpec}, Config{Interval: time.Millisecond}}, + {"zero interval", []Spec{validSpec}, Config{Runs: 1}}, + {"bad schedule index", []Spec{validSpec}, Config{ + Runs: 1, Interval: time.Millisecond, MeasurementOrder: [][]int{{1}}, + }}, + {"wrong schedule rounds", []Spec{validSpec}, Config{ + Runs: 2, Interval: time.Millisecond, MeasurementOrder: [][]int{{0}}, + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := Benchmark( + context.Background(), test.specs, test.config, Options{}, + ); err == nil { + t.Fatal("expected validation error") + } + }) + } +} + func completedRoundCount(estimates []ProgressEstimate) int { if len(estimates) == 0 { return 0 diff --git a/internal/runner/prepare.go b/internal/runner/prepare.go new file mode 100644 index 0000000..0773191 --- /dev/null +++ b/internal/runner/prepare.go @@ -0,0 +1,31 @@ +package runner + +import ( + "context" + "io" + "os/exec" + "syscall" +) + +func RunPreparation(ctx context.Context, shell string, output io.Writer) error { + if err := ctx.Err(); err != nil { + return err + } + cmd := exec.Command("/bin/sh", "-c", shell) + configureProcessGroup(cmd) + cmd.Stdout, cmd.Stderr = output, output + if err := cmd.Start(); err != nil { + return err + } + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + select { + case err := <-done: + signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) + return err + case <-ctx.Done(): + signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) + <-done + return ctx.Err() + } +} diff --git a/internal/runner/prepare_test.go b/internal/runner/prepare_test.go new file mode 100644 index 0000000..c320527 --- /dev/null +++ b/internal/runner/prepare_test.go @@ -0,0 +1,70 @@ +package runner + +import ( + "context" + "errors" + "fmt" + "os" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +func TestPreparationCancellationKillsDescendants(t *testing.T) { + pidPath := t.TempDir() + "/child.pid" + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- RunPreparation( + ctx, fmt.Sprintf("sleep 30 & echo $! > %q; wait", pidPath), os.Stderr, + ) + }() + var pid int + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(pidPath) + if err == nil { + pid, err = strconv.Atoi(strings.TrimSpace(string(data))) + if err == nil { + break + } + } + time.Sleep(5 * time.Millisecond) + } + if pid == 0 { + cancel() + <-done + t.Fatal("preparation did not start its child") + } + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("preparation error = %v, want cancellation", err) + } + deadline = time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) { + return + } + if state, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid)); err == nil && + strings.Contains(string(state), ") Z ") { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("preparation child %d survived cancellation", pid) +} + +func TestCancelledPreparationDoesNotStart(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + marker := t.TempDir() + "/started" + err := RunPreparation(ctx, fmt.Sprintf("touch %q", marker), os.Stderr) + if !errors.Is(err, context.Canceled) { + t.Fatalf("preparation error = %v, want cancellation", err) + } + if _, err := os.Stat(marker); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("cancelled preparation started: %v", err) + } +} diff --git a/internal/runner/tui.go b/internal/runner/tui.go index 1599656..7cb4cd5 100644 --- a/internal/runner/tui.go +++ b/internal/runner/tui.go @@ -65,7 +65,7 @@ func runTUIOnce( waitDone := make(chan error, 1) go func() { waitDone <- cmd.Wait() }() - waitErr, elapsed, reason := waitForTUI( + waitResult := waitForTUI( ctx, cmd, options.Duration, started, waitDone, ) signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) @@ -77,17 +77,26 @@ func runTUIOnce( if ctx.Err() != nil { return model.Run{}, ctx.Err() } - if options.Duration > 0 && reason == "exited" { + if options.Duration > 0 && waitResult.reason == "exited" { return model.Run{}, errors.New("TUI exited before the fixed duration") } - if waitErr != nil && reason == "exited" && !isNonZeroExit(cmd, waitErr) { - return model.Run{}, waitErr + if waitResult.err != nil && waitResult.reason == "exited" && + !isNonZeroExit(cmd, waitResult.err) { + return model.Run{}, waitResult.err } select { case <-outputDone: case <-time.After(100 * time.Millisecond): } - return makeTUIRun(cmd, elapsed, user, system, rusageRSS, peak, reason), nil + return makeTUIRun( + cmd, waitResult.elapsed, user, system, rusageRSS, peak, waitResult.reason, + ), nil +} + +type tuiWaitResult struct { + err error + elapsed time.Duration + reason string } func waitForTUI( @@ -96,14 +105,16 @@ func waitForTUI( duration time.Duration, started time.Time, waitDone <-chan error, -) (error, time.Duration, string) { +) tuiWaitResult { if duration == 0 { select { case err := <-waitDone: - return err, time.Since(started), "exited" + return tuiWaitResult{err: err, elapsed: time.Since(started), reason: "exited"} case <-ctx.Done(): signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) - return <-waitDone, time.Since(started), "interrupted" + return tuiWaitResult{ + err: <-waitDone, elapsed: time.Since(started), reason: "interrupted", + } } } remaining := time.Until(started.Add(duration)) @@ -114,18 +125,15 @@ func waitForTUI( defer timer.Stop() select { case err := <-waitDone: - return err, time.Since(started), "exited" + return tuiWaitResult{err: err, elapsed: time.Since(started), reason: "exited"} case <-ctx.Done(): signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) - return <-waitDone, time.Since(started), "interrupted" - case <-timer.C: - signalProcessGroup(cmd.Process.Pid, syscall.SIGTERM) - select { - case err := <-waitDone: - return err, time.Since(started), "duration" - case <-time.After(time.Second): - signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) - return <-waitDone, time.Since(started), "duration" + return tuiWaitResult{ + err: <-waitDone, elapsed: time.Since(started), reason: "interrupted", } + case <-timer.C: + elapsed := time.Since(started) + signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) + return tuiWaitResult{err: <-waitDone, elapsed: elapsed, reason: "duration"} } } diff --git a/internal/runner/tui_test.go b/internal/runner/tui_test.go index f5f42ab..d3ad235 100644 --- a/internal/runner/tui_test.go +++ b/internal/runner/tui_test.go @@ -35,6 +35,26 @@ func TestFixedDurationTUIRejectsEarlyExit(t *testing.T) { } } +func TestFixedDurationTUIExcludesTerminationDelay(t *testing.T) { + duration := 50 * time.Millisecond + started := time.Now() + run, err := runTUIOnce( + context.Background(), + Spec{Name: "ignore-term", Shell: "trap '' TERM; while :; do :; done"}, + 5*time.Millisecond, + Options{TUI: true, Duration: duration}, + ) + if err != nil { + t.Fatal(err) + } + if runtime := time.Since(started); runtime > 2*time.Second { + t.Fatalf("fixed-duration run took %s, teardown should be outside measurement", runtime) + } + if run.WallSeconds < duration.Seconds() || run.WallSeconds > 0.5 { + t.Fatalf("wall time = %.3fs, want approximately %.3fs", run.WallSeconds, duration.Seconds()) + } +} + func TestTUIRecordsNonZeroExit(t *testing.T) { run, err := runTUIOnce( context.Background(), Spec{Name: "exit", Shell: "exit 7"}, diff --git a/internal/runner/validation.go b/internal/runner/validation.go new file mode 100644 index 0000000..879e2fb --- /dev/null +++ b/internal/runner/validation.go @@ -0,0 +1,78 @@ +package runner + +import ( + "context" + "errors" + "fmt" + "strings" +) + +func validateBenchmarkInputs(ctx context.Context, specs []Spec, config Config) error { + if ctx == nil { + return errors.New("context cannot be nil") + } + if len(specs) == 0 { + return errors.New("at least one command is required") + } + if config.Runs < 1 { + return errors.New("runs must be at least 1") + } + if config.Warmups < 0 { + return errors.New("warmups cannot be negative") + } + if config.Interval <= 0 { + return errors.New("sampling interval must be positive") + } + for index, spec := range specs { + if spec.Shell != "" && strings.TrimSpace(spec.Shell) == "" { + return fmt.Errorf("command %d shell source cannot be blank", index+1) + } + hasShell := spec.Shell != "" + hasArgs := len(spec.Args) > 0 && strings.TrimSpace(spec.Args[0]) != "" + if hasShell == hasArgs { + return fmt.Errorf("command %d must define either shell source or argv", index+1) + } + } + if len(config.MeasurementOrder) > 0 && len(config.MeasurementOrder) != config.Runs { + return fmt.Errorf( + "measurement order has %d rounds, want %d", + len(config.MeasurementOrder), config.Runs, + ) + } + if len(config.MeasurementOrder) == config.Runs { + if err := validateSchedule(config.MeasurementOrder, len(specs)); err != nil { + return fmt.Errorf("measurement order: %w", err) + } + } + if len(config.WarmupOrder) > 0 && len(config.WarmupOrder) != config.Warmups { + return fmt.Errorf( + "warmup order has %d rounds, want %d", + len(config.WarmupOrder), config.Warmups, + ) + } + if len(config.WarmupOrder) == config.Warmups { + if err := validateSchedule(config.WarmupOrder, len(specs)); err != nil { + return fmt.Errorf("warmup order: %w", err) + } + } + return nil +} + +func validateSchedule(schedule [][]int, tools int) error { + for round, order := range schedule { + if len(order) != tools { + return fmt.Errorf("round %d has %d tools, want %d", round+1, len(order), tools) + } + seen := make([]bool, tools) + for _, index := range order { + if index < 0 || index >= tools { + return fmt.Errorf("round %d contains tool index %d", round+1, index) + } + if seen[index] { + return fmt.Errorf("round %d repeats tool index %d", round+1, index) + } + seen[index] = true + } + } + return nil +} From a1f1742ba40dee421159cb6bb55498cc6798c014 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sun, 19 Jul 2026 22:32:49 +0200 Subject: [PATCH 06/11] snailrace v0.0.5 save reports before stdout and add explicit no-save mode sample Linux process groups including reparented descendants make tool inspection cancellable harden statistics and JSON round trips make interactive TUI streams owned and cancellable fix help exit semantics unify cross-format reliability and included-dimension output Signed-off-by: Polina Simonenko --- .github/workflows/release.yml | 35 +++- README.md | 13 +- completions/_snailrace | 1 + completions/snailrace.bash | 2 +- completions/snailrace.fish | 1 + docs/snailrace.1 | 8 +- internal/app/app.go | 23 ++- internal/app/formats_test.go | 12 ++ internal/app/options.go | 3 + internal/app/options_test.go | 11 ++ internal/app/output_test.go | 39 +++++ internal/app/specs.go | 3 + internal/app/validation.go | 3 + internal/model/model.go | 4 + internal/model/model_json.go | 50 ++++++ internal/model/model_json_test.go | 50 ++++++ internal/model/stats.go | 90 +++++++--- internal/model/stats_json.go | 34 +++- internal/model/stats_test.go | 61 +++++++ internal/platform/dependencies_darwin.go | 35 ++-- internal/platform/dependencies_darwin_test.go | 3 + internal/platform/dependencies_linux.go | 15 +- internal/platform/sampler_darwin.go | 15 +- internal/platform/sampler_linux.go | 53 ++---- internal/platform/sampler_linux_test.go | 45 +++++ internal/platform/types.go | 24 +-- internal/report/chart_artifacts.go | 1 + internal/report/chart_delta.go | 16 +- internal/report/chart_distribution.go | 8 + internal/report/chart_metrics.go | 4 + internal/report/chart_trend.go | 38 ++++- internal/report/command_legend.go | 35 +++- internal/report/command_legend_test.go | 48 ++++++ internal/report/comparison.go | 37 +++- internal/report/comparison_html.go | 6 +- internal/report/comparison_markdown.go | 7 +- internal/report/comparison_test.go | 21 +++ internal/report/comparison_text.go | 7 +- internal/report/format.go | 17 ++ internal/report/html.go | 15 +- internal/report/markdown.go | 12 +- internal/report/ranking_json.go | 25 +-- internal/report/report.go | 14 ++ internal/report/semantics.go | 77 +++++++++ internal/report/svg.go | 22 ++- internal/report/text.go | 9 +- internal/report/text_bars.go | 7 +- internal/report/text_test.go | 38 +++++ internal/runner/benchmark.go | 47 +++++- internal/runner/benchmark_linux_test.go | 21 +++ internal/runner/benchmark_test.go | 54 ++++++ internal/runner/executable_darwin.go | 9 + internal/runner/executable_linux.go | 12 ++ internal/runner/options.go | 2 + internal/runner/run.go | 30 +++- internal/runner/run_test.go | 21 ++- internal/runner/spec.go | 27 ++- internal/runner/terminal.go | 120 ++++++++++++- internal/runner/tool.go | 158 ++++++++++++++++-- internal/runner/tool_test.go | 78 ++++++++- internal/runner/tui.go | 59 +++++-- internal/runner/tui_result.go | 1 + internal/runner/tui_test.go | 125 ++++++++++++++ internal/style/text.go | 20 +++ 64 files changed, 1666 insertions(+), 215 deletions(-) create mode 100644 internal/model/model_json.go create mode 100644 internal/model/model_json_test.go create mode 100644 internal/report/command_legend_test.go create mode 100644 internal/report/semantics.go create mode 100644 internal/runner/benchmark_linux_test.go create mode 100644 internal/runner/executable_darwin.go create mode 100644 internal/runner/executable_linux.go create mode 100644 internal/style/text.go diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a217e1..4948e6d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,11 +16,42 @@ jobs: - uses: actions/setup-go@v7 with: go-version-file: go.mod - - run: go test ./... + - run: go test -race ./... + - run: go vet ./... - release: + validate-release: needs: test runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - { goos: linux, goarch: amd64, cgo: '0', runner: ubuntu-latest } + - { goos: linux, goarch: arm64, cgo: '0', runner: ubuntu-latest } + - { goos: linux, goarch: arm, goarm: '7', cgo: '0', runner: ubuntu-latest } + - { goos: linux, goarch: '386', cgo: '0', runner: ubuntu-latest } + - { goos: darwin, goarch: amd64, cgo: '1', runner: macos-26-intel } + - { goos: darwin, goarch: arm64, cgo: '1', runner: macos-latest } + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + - name: Test native platform + if: matrix.goos == 'darwin' || matrix.goarch == 'amd64' + run: go test ./... + - name: Validate release build + env: + CGO_ENABLED: ${{ matrix.cgo }} + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + GOARM: ${{ matrix.goarm }} + MACOSX_DEPLOYMENT_TARGET: '12.0' + run: go build -trimpath -o /tmp/snailrace ./cmd/snailrace + + release: + needs: validate-release + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: diff --git a/README.md b/README.md index bb31012..a129be3 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,8 @@ An HTML report is saved to the current directory by default. Choose another dire ./snailrace -format html -output ./reports -- ./my-program --flag ``` +Use `-no-save` when only stdout output is wanted. + Measure an interactive TUI: ```sh @@ -87,6 +89,7 @@ Measure an interactive TUI: | `-baseline` | 1-based baseline; `0` selects the winner | | `-f`, `-format` | Saved format; default `html` | | `-o`, `-output` | Report directory; default current directory | +| `-no-save` | Do not save report files | | `-verbose` | Full statistical tables on stdout | | `-show-output` | Forward command output to stderr | | `tui` | Run inside a pseudo-terminal | @@ -106,6 +109,8 @@ Measure an interactive TUI: Raw runs and summary statistics are available in JSON. Reports include mean, sample standard deviation, median, p95, range, and a 95% Student's t interval. +Every format identifies the dimensions actually included in the balanced index +and records sampling or metric-availability caveats. ## Method @@ -117,22 +122,28 @@ sample standard deviation, median, p95, range, and a 95% Student's t interval. - If none of the selected dimensions is usable, reports mark the balanced ranking unavailable. - Ctrl+C keeps completed comparison rounds. - Command output is discarded unless `-show-output` is set. +- Native executables run through the inspected artifact and are identity/hash-checked after measurement. +- Scripts preserve their original invocation path so `$0`-relative behavior is unchanged. Saved reports are staged before publication and receive a numeric suffix rather than overwriting an existing report with the same timestamp. ## Platform Notes -- Linux reads `/proc` and follows each process's children. +- Linux reads `/proc` and measures every member of the command's process group, + including descendants reparented while the benchmark is running. - macOS uses cgo with `proc_listpgrppids`, `proc_pidinfo`, and `proc_pid_rusage`. - macOS physical footprint estimates memory pressure charged to each process. +- Physical-footprint validity is tracked per run; unavailable samples are not treated as zero. - Tree RSS can double-count shared pages. - Sampled peaks can miss activity shorter than the interval. - macOS does not report file descriptor counts. ## Disk Footprint +- For a simple external `-c` command, disk footprint describes its leading executable; + shell builtins describe the shell itself. - Disk-backed `@rpath`, `@loader_path`, and `@executable_path` libraries are resolved recursively. - macOS dyld shared-cache libraries are listed but excluded from byte totals. diff --git a/completions/_snailrace b/completions/_snailrace index fbfb435..a56dd9f 100644 --- a/completions/_snailrace +++ b/completions/_snailrace @@ -13,6 +13,7 @@ _snailrace() { '-baseline[set 1-based baseline]:index:' \ '*'{-f,-format}'[select report format]:format:(html svg markdown json text)' \ '(-o -output)'{-o,-output}'[set report directory]:directory:_directories' \ + '-no-save[do not save report files]' \ '-verbose[print full statistical tables]' \ '-show-output[forward command output]' \ '(-d -duration)'{-d,-duration}'[set fixed TUI duration]:duration:' \ diff --git a/completions/snailrace.bash b/completions/snailrace.bash index af2dbe5..f902e05 100644 --- a/completions/snailrace.bash +++ b/completions/snailrace.bash @@ -2,7 +2,7 @@ _snailrace() { local cur prev options command_seen expect_value after_separator word i cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" - options="-c -command -label -prepare -n -runs -warmups -interval -index -baseline -f -format -o -output -verbose -show-output -d -duration -width -height -v -version -h -help" + options="-c -command -label -prepare -n -runs -warmups -interval -index -baseline -f -format -o -output -no-save -verbose -show-output -d -duration -width -height -v -version -h -help" command_seen=0 expect_value=0 diff --git a/completions/snailrace.fish b/completions/snailrace.fish index 910554d..d44b441 100644 --- a/completions/snailrace.fish +++ b/completions/snailrace.fish @@ -8,6 +8,7 @@ complete -c snailrace -l index -r -a 'time cpu ram disk' -d 'Set index dimension complete -c snailrace -l baseline -r -d 'Set 1-based baseline' complete -c snailrace -s f -l format -r -a 'html svg markdown json text' -d 'Select report format' complete -c snailrace -s o -l output -r -a '(__fish_complete_directories)' -d 'Set report directory' +complete -c snailrace -l no-save -d 'Do not save report files' complete -c snailrace -l verbose -d 'Print full statistical tables' complete -c snailrace -l show-output -d 'Forward command output' complete -c snailrace -s d -l duration -r -d 'Set fixed TUI duration' diff --git a/docs/snailrace.1 b/docs/snailrace.1 index f682fad..a1c6c07 100644 --- a/docs/snailrace.1 +++ b/docs/snailrace.1 @@ -34,6 +34,10 @@ Commands with non-zero measured exits continue through all configured rounds, but are reported as failures and excluded from rankings. If none of the selected dimensions is usable, the balanced ranking is marked unavailable instead of selecting a winner. +.Pp +For simple external shell commands, executable metadata and disk footprint +describe the leading command. Shell builtins describe the shell itself. +Scripts retain their original invocation path. The .Cm tui subcommand renders a live terminal view while measuring. @@ -82,6 +86,8 @@ The default is Directory for a saved report; defaults to the current directory. Reports are staged before publication and receive a numeric suffix instead of overwriting an existing report with the same name. +.It Fl no-save +Do not save report files; write the text report to standard output only. .It Fl verbose Print full statistical tables to stdout. .It Fl show-output @@ -101,7 +107,7 @@ TUI rows; default inherits the terminal. .It Fl v , Fl version Print version and exit. .It Fl h , Fl help -Show help. +Show help and exit successfully. .El .Sh EXAMPLES Benchmark one command with warmups: diff --git a/internal/app/app.go b/internal/app/app.go index bcbd908..d358c94 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -3,6 +3,7 @@ package app import ( "context" "errors" + "flag" "fmt" "io" "os" @@ -25,6 +26,9 @@ var Version = "dev" func Run(arguments []string, stdout, stderr io.Writer) error { options, err := parseOptions(arguments, stderr) if err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } return err } if options.version { @@ -84,7 +88,7 @@ func Run(arguments []string, stdout, stderr io.Writer) error { WarmupOrder: warmupOrder, }, runner.Options{ ShowOutput: options.showOutput, TUI: options.tui, - Output: stderr, + Output: stderr, Input: os.Stdin, TerminalOut: os.Stdout, Duration: options.duration, Width: width, Height: height, Interactive: interactiveTUI, FollowResize: followResize, @@ -149,8 +153,12 @@ func Run(arguments []string, stdout, stderr io.Writer) error { Benchmarks: benchmarks, Verbose: options.verbose, Notes: notes, } + outputDirectory := options.output + if options.noSave { + outputDirectory = "" + } if err := writeResult( - stdout, stderr, options.output, options.formats, result, + stdout, stderr, outputDirectory, options.formats, result, ); err != nil { return err } @@ -206,13 +214,12 @@ func writeResult( formats []string, result model.Report, ) error { - if err := report.Write(stdout, "text", result); err != nil { - return err - } - if directory == "" { - return nil + var saveErr error + if directory != "" { + saveErr = saveReportFormats(stderr, directory, formats, result) } - return saveReportFormats(stderr, directory, formats, result) + textErr := report.Write(stdout, "text", result) + return errors.Join(saveErr, textErr) } func modeName(tui bool) string { diff --git a/internal/app/formats_test.go b/internal/app/formats_test.go index 75e6d18..f2dcd89 100644 --- a/internal/app/formats_test.go +++ b/internal/app/formats_test.go @@ -34,3 +34,15 @@ func TestDefaultFormatIsHTML(t *testing.T) { t.Fatalf("output = %q, want current directory", options.output) } } + +func TestEmptyOutputRequiresNoSave(t *testing.T) { + if _, err := parseOptions([]string{"-o", "", "--", "true"}, io.Discard); err == nil { + t.Fatal("empty output should require -no-save") + } + options, err := parseOptions( + []string{"-no-save", "-o", "", "--", "true"}, io.Discard, + ) + if err != nil || !options.noSave { + t.Fatalf("no-save options = %+v, error = %v", options, err) + } +} diff --git a/internal/app/options.go b/internal/app/options.go index e2f6d11..7e8937f 100644 --- a/internal/app/options.go +++ b/internal/app/options.go @@ -31,6 +31,7 @@ type options struct { index []string verbose bool showOutput bool + noSave bool version bool tui bool width, height uint16 @@ -82,6 +83,7 @@ func parseOptions(arguments []string, stderr io.Writer) (options, error) { flags.Var(&formats, "f", "saved format (shorthand)") flags.StringVar(&result.output, "output", ".", "report directory; default current directory") flags.StringVar(&result.output, "o", ".", "report directory (shorthand)") + flags.BoolVar(&result.noSave, "no-save", false, "do not save report files") flags.BoolVar(&result.verbose, "verbose", false, "print full statistical tables to stdout") flags.BoolVar(&result.showOutput, "show-output", false, "show command output") // TUI. @@ -151,6 +153,7 @@ func printUsage(stderr io.Writer) { {"Output", []string{ "-f, -format list saved format: html, svg, markdown, json, text", "-o, -output string report directory; default current directory", + "-no-save do not save report files", "-verbose print full statistical tables to stdout", "-show-output forward command output to stderr", }}, diff --git a/internal/app/options_test.go b/internal/app/options_test.go index d15d9d4..b5fbc87 100644 --- a/internal/app/options_test.go +++ b/internal/app/options_test.go @@ -2,6 +2,7 @@ package app import ( "io" + "strings" "testing" ) @@ -21,6 +22,16 @@ func TestParseDirectCommand(t *testing.T) { } } +func TestHelpReturnsSuccess(t *testing.T) { + var output strings.Builder + if err := Run([]string{"-h"}, io.Discard, &output); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), "Usage: snailrace") { + t.Fatal("help output is missing usage") + } +} + func TestRejectsLabelCountMismatch(t *testing.T) { _, err := parseOptions( []string{"-label", "x", "-c", "true", "-c", "false"}, diff --git a/internal/app/output_test.go b/internal/app/output_test.go index 34bee06..b4159b5 100644 --- a/internal/app/output_test.go +++ b/internal/app/output_test.go @@ -2,6 +2,7 @@ package app import ( "bytes" + "errors" "io" "os" "strings" @@ -41,6 +42,29 @@ func TestSavingAlsoWritesCompleteReportToStdout(t *testing.T) { } } +func TestStdoutFailureDoesNotPreventSaving(t *testing.T) { + directory := t.TempDir() + report := model.Report{ + Config: model.Config{Mode: "command", Baseline: 1}, + Benchmarks: []model.Benchmark{{Tool: model.ToolInfo{Name: "tool"}}}, + } + if err := writeResult( + failingWriter{}, io.Discard, directory, []string{"html"}, report, + ); err == nil { + t.Fatal("stdout failure should be returned") + } + entries, err := os.ReadDir(directory) + if err != nil || len(entries) != 1 { + t.Fatalf("saved entries = %v, error = %v", entries, err) + } +} + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { + return 0, errors.New("write failed") +} + func TestSavedFormatDoesNotReplaceTextStdout(t *testing.T) { directory := t.TempDir() var stdout, stderr bytes.Buffer @@ -143,6 +167,21 @@ func TestRunSavesDefaultHTMLInCurrentDirectory(t *testing.T) { } } +func TestRunNoSaveLeavesCurrentDirectoryEmpty(t *testing.T) { + directory := t.TempDir() + t.Chdir(directory) + if err := Run( + []string{"-no-save", "-n", "1", "-warmups", "0", "--", "/bin/true"}, + io.Discard, io.Discard, + ); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(directory) + if err != nil || len(entries) != 0 { + t.Fatalf("no-save entries = %v, error = %v", entries, err) + } +} + func TestSavingSameReportUsesUniqueNames(t *testing.T) { directory := t.TempDir() result := model.Report{ diff --git a/internal/app/specs.go b/internal/app/specs.go index d125890..a171cc8 100644 --- a/internal/app/specs.go +++ b/internal/app/specs.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/shellcell/snailrace/internal/runner" + "github.com/shellcell/snailrace/internal/style" ) func makeSpecs(commands, arguments []string, labels []string) []runner.Spec { @@ -14,6 +15,7 @@ func makeSpecs(commands, arguments []string, labels []string) []runner.Spec { if len(labels) > 0 { name = labels[0] } + name = style.SafeText(name) return []runner.Spec{{Name: name, Args: arguments}} } defaultLabels := make([]string, len(commands)) @@ -32,6 +34,7 @@ func makeSpecs(commands, arguments []string, labels []string) []runner.Spec { seen[label]++ label += " #" + fmt.Sprint(seen[label]) } + label = style.SafeText(label) result = append(result, runner.Spec{Name: label, Shell: command}) } return result diff --git a/internal/app/validation.go b/internal/app/validation.go index ea339fa..42f7008 100644 --- a/internal/app/validation.go +++ b/internal/app/validation.go @@ -28,6 +28,9 @@ func validateOptions(result options, commands []string, positional int) error { if len(result.formats) == 0 { return errors.New("at least one output format is required") } + if !result.noSave && strings.TrimSpace(result.output) == "" { + return errors.New("output directory cannot be empty; use -no-save") + } for _, format := range result.formats { if !report.ValidFormat(format) { return fmt.Errorf("unknown report format %q", format) diff --git a/internal/model/model.go b/internal/model/model.go index 852f721..1b15696 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -38,6 +38,9 @@ type HostInfo struct { type ToolInfo struct { Name string `json:"name"` Command []string `json:"command"` + ShellCommand bool `json:"shell_command"` + ShellTarget bool `json:"-"` + ProvenanceVerified bool `json:"provenance_verified"` Executable string `json:"executable,omitempty"` SizeBytes int64 `json:"size_bytes,omitempty"` SHA256 string `json:"sha256,omitempty"` @@ -61,6 +64,7 @@ type Run struct { AverageCPUPercent float64 `json:"average_cpu_percent"` PeakResidentBytes float64 `json:"peak_resident_bytes"` PeakPhysicalFootprintBytes float64 `json:"peak_physical_footprint_bytes,omitempty"` + PhysicalFootprintValid bool `json:"physical_footprint_valid"` OSMaxRSSBytes float64 `json:"os_max_rss_bytes"` MeanResidentBytes float64 `json:"mean_resident_bytes"` PeakVirtualBytes float64 `json:"peak_virtual_bytes"` diff --git a/internal/model/model_json.go b/internal/model/model_json.go new file mode 100644 index 0000000..965c6fa --- /dev/null +++ b/internal/model/model_json.go @@ -0,0 +1,50 @@ +package model + +import ( + "encoding/json" + "path/filepath" +) + +func (tool *ToolInfo) UnmarshalJSON(data []byte) error { + type plain ToolInfo + decoded := struct { + plain + ShellCommand *bool `json:"shell_command"` + ProvenanceVerified *bool `json:"provenance_verified"` + }{} + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + *tool = ToolInfo(decoded.plain) + if decoded.ShellCommand != nil { + tool.ShellCommand = *decoded.ShellCommand + } else { + tool.ShellCommand = len(tool.Command) == 1 && + tool.Command[0] != tool.Executable && + tool.Command[0] != filepath.Base(tool.Executable) + } + if decoded.ProvenanceVerified != nil { + tool.ProvenanceVerified = *decoded.ProvenanceVerified + } else { + tool.ProvenanceVerified = tool.SHA256 != "" + } + return nil +} + +func (run *Run) UnmarshalJSON(data []byte) error { + type plain Run + decoded := struct { + plain + PhysicalFootprintValid *bool `json:"physical_footprint_valid"` + }{} + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + *run = Run(decoded.plain) + if decoded.PhysicalFootprintValid != nil { + run.PhysicalFootprintValid = *decoded.PhysicalFootprintValid + } else { + run.PhysicalFootprintValid = run.PeakPhysicalFootprintBytes > 0 + } + return nil +} diff --git a/internal/model/model_json_test.go b/internal/model/model_json_test.go new file mode 100644 index 0000000..a803914 --- /dev/null +++ b/internal/model/model_json_test.go @@ -0,0 +1,50 @@ +package model + +import ( + "encoding/json" + "testing" +) + +func TestLegacyRunInfersPhysicalFootprintValidity(t *testing.T) { + var run Run + if err := json.Unmarshal( + []byte(`{"peak_physical_footprint_bytes":123}`), &run, + ); err != nil { + t.Fatal(err) + } + if !run.PhysicalFootprintValid { + t.Fatal("legacy positive physical footprint should remain valid") + } +} + +func TestLegacyToolInfersShellCommand(t *testing.T) { + var tool ToolInfo + if err := json.Unmarshal([]byte(`{"command":["grep foo file"]}`), &tool); err != nil { + t.Fatal(err) + } + if !tool.ShellCommand { + t.Fatal("legacy one-element command should retain shell rendering") + } + data, err := json.Marshal(ToolInfo{Command: []string{"/tmp/my tool"}}) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(data, &tool); err != nil { + t.Fatal(err) + } + if tool.ShellCommand { + t.Fatal("current direct command should retain argv rendering") + } +} + +func TestLegacyToolInfersDirectSingleArgumentAndVerifiedHash(t *testing.T) { + var tool ToolInfo + if err := json.Unmarshal([]byte( + `{"command":["/tmp/my tool"],"executable":"/tmp/my tool","sha256":"abc"}`, + ), &tool); err != nil { + t.Fatal(err) + } + if tool.ShellCommand || !tool.ProvenanceVerified { + t.Fatalf("legacy direct tool = %+v", tool) + } +} diff --git a/internal/model/stats.go b/internal/model/stats.go index 69f8e68..29fcdba 100644 --- a/internal/model/stats.go +++ b/internal/model/stats.go @@ -13,15 +13,17 @@ func Summarize(runs []Run) Summary { } return result } - physicalValues := values(func(r Run) float64 { return r.PeakPhysicalFootprintBytes }) - var physicalStats *Stats - for _, value := range physicalValues { - if value > 0 { - value := stats(physicalValues) - physicalStats = &value - break + physicalValues := make([]float64, 0, len(runs)) + for _, run := range runs { + if run.PhysicalFootprintValid { + physicalValues = append(physicalValues, run.PeakPhysicalFootprintBytes) } } + var physicalStats *Stats + if len(physicalValues) > 0 { + value := stats(physicalValues) + physicalStats = &value + } return Summary{ WallSeconds: stats(values(func(r Run) float64 { return r.WallSeconds })), CPUTotalSeconds: stats(values(func(r Run) float64 { @@ -66,36 +68,56 @@ func Summarize(runs []Run) Summary { func CalculateStats(values []float64) Stats { return stats(values) } func stats(values []float64) Stats { - if len(values) == 0 { + finite := make([]float64, 0, len(values)) + for _, value := range values { + if !math.IsNaN(value) && !math.IsInf(value, 0) { + finite = append(finite, value) + } + } + if len(finite) == 0 { return Stats{} } - sorted := append([]float64(nil), values...) + sorted := append([]float64(nil), finite...) sort.Float64s(sorted) - total := 0.0 - for _, value := range sorted { - total += value - } - mean := total / float64(len(sorted)) - variance := 0.0 - for _, value := range sorted { - delta := value - mean - variance += delta * delta + mean := 0.0 + m2 := 0.0 + varianceOverflow := false + for index, value := range sorted { + count := float64(index + 1) + previousMean := mean + mean = previousMean*(1-1/count) + value/count + if math.IsInf(mean, 0) { + mean = math.Copysign(math.MaxFloat64, mean) + } + delta := value - previousMean + term := delta * (value - mean) + if math.IsInf(term, 0) || math.IsNaN(term) || math.MaxFloat64-m2 < term { + varianceOverflow = true + m2 = math.MaxFloat64 + } else if term > 0 { + m2 += term + } } - if len(sorted) > 1 { - variance /= float64(len(sorted) - 1) + standardDeviation := 0.0 + if len(sorted) > 1 && varianceOverflow { + standardDeviation = math.MaxFloat64 + } else if len(sorted) > 1 { + standardDeviation = math.Sqrt(m2 / float64(len(sorted)-1)) } - standardDeviation := math.Sqrt(variance) margin := 0.0 validInterval := len(sorted) > 1 if len(sorted) > 1 { - margin = tCritical95(len(sorted)-1) * standardDeviation / - math.Sqrt(float64(len(sorted))) + margin = standardDeviation / math.Sqrt(float64(len(sorted))) * + tCritical95(len(sorted)-1) + if math.IsInf(margin, 0) { + margin = math.MaxFloat64 + } } return Stats{ N: len(sorted), Min: sorted[0], Max: sorted[len(sorted)-1], Mean: mean, StdDev: standardDeviation, Median: percentile(sorted, 0.5), - P95: percentile(sorted, 0.95), CI95Low: mean - margin, - CI95High: mean + margin, CI95Valid: validInterval, + P95: percentile(sorted, 0.95), CI95Low: finiteDifference(mean, margin), + CI95High: finiteSum(mean, margin), CI95Valid: validInterval, } } @@ -106,7 +128,23 @@ func percentile(sorted []float64, percentile float64) float64 { position := percentile * float64(len(sorted)-1) lower, upper := int(math.Floor(position)), int(math.Ceil(position)) weight := position - float64(lower) - return sorted[lower]*(1-weight) + sorted[upper]*weight + return finiteSum(sorted[lower]*(1-weight), sorted[upper]*weight) +} + +func finiteSum(left, right float64) float64 { + result := left + right + if math.IsInf(result, 0) { + return math.Copysign(math.MaxFloat64, result) + } + return result +} + +func finiteDifference(left, right float64) float64 { + result := left - right + if math.IsInf(result, 0) { + return math.Copysign(math.MaxFloat64, result) + } + return result } // Two-sided 95% Student's t critical values for 1..30 degrees of freedom. diff --git a/internal/model/stats_json.go b/internal/model/stats_json.go index bd05ef1..dd844dd 100644 --- a/internal/model/stats_json.go +++ b/internal/model/stats_json.go @@ -1,6 +1,9 @@ package model -import "encoding/json" +import ( + "encoding/json" + "math" +) func (stats Stats) MarshalJSON() ([]byte, error) { type plain Stats @@ -14,3 +17,32 @@ func (stats Stats) MarshalJSON() ([]byte, error) { CI95High *float64 `json:"ci95_high,omitempty"` }{plain(stats), low, high}) } + +func (stats *Stats) UnmarshalJSON(data []byte) error { + type plain Stats + decoded := struct { + plain + CI95Low *float64 `json:"ci95_low"` + CI95High *float64 `json:"ci95_high"` + }{} + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + *stats = Stats(decoded.plain) + if !stats.CI95Valid { + return nil + } + if decoded.CI95Low == nil || decoded.CI95High == nil { + if stats.N < 2 { + stats.CI95Valid = false + return nil + } + margin := stats.StdDev / math.Sqrt(float64(stats.N)) * tCritical95(stats.N-1) + stats.CI95Low = finiteDifference(stats.Mean, margin) + stats.CI95High = finiteSum(stats.Mean, margin) + return nil + } + stats.CI95Low = *decoded.CI95Low + stats.CI95High = *decoded.CI95High + return nil +} diff --git a/internal/model/stats_test.go b/internal/model/stats_test.go index ba32945..543d7fb 100644 --- a/internal/model/stats_test.go +++ b/internal/model/stats_test.go @@ -53,6 +53,67 @@ func TestUnavailablePhysicalFootprintIsOmittedFromJSON(t *testing.T) { } } +func TestPhysicalFootprintUsesExplicitPerRunValidity(t *testing.T) { + summary := Summarize([]Run{ + {PeakPhysicalFootprintBytes: 0, PhysicalFootprintValid: true}, + {PeakPhysicalFootprintBytes: 10, PhysicalFootprintValid: true}, + {PeakPhysicalFootprintBytes: 1000, PhysicalFootprintValid: false}, + }) + if summary.PeakPhysicalFootprintBytes == nil { + t.Fatal("valid zero-valued physical footprint should be retained") + } + if got := summary.PeakPhysicalFootprintBytes; got.N != 2 || got.Mean != 5 { + t.Fatalf("physical footprint stats = %+v, want N=2 mean=5", *got) + } +} + +func TestStatsRemainFiniteForExtremeInputs(t *testing.T) { + result := CalculateStats([]float64{ + math.MaxFloat64, -math.MaxFloat64, math.NaN(), math.Inf(1), + }) + if result.N != 2 { + t.Fatalf("N = %d, want two finite observations", result.N) + } + for name, value := range map[string]float64{ + "mean": result.Mean, "stddev": result.StdDev, "median": result.Median, + "p95": result.P95, "ci low": result.CI95Low, "ci high": result.CI95High, + } { + if math.IsNaN(value) || math.IsInf(value, 0) { + t.Fatalf("%s is not finite: %v", name, value) + } + } + if _, err := json.Marshal(result); err != nil { + t.Fatal(err) + } +} + +func TestStatsJSONRoundTripPreservesConfidenceBounds(t *testing.T) { + original := CalculateStats([]float64{1, 2, 4, 8}) + data, err := json.Marshal(original) + if err != nil { + t.Fatal(err) + } + var decoded Stats + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatal(err) + } + if decoded != original { + t.Fatalf("round trip = %+v, want %+v", decoded, original) + } +} + +func TestStatsUnmarshalAcceptsLegacyConfidenceInterval(t *testing.T) { + var decoded Stats + if err := json.Unmarshal( + []byte(`{"n":2,"mean":3,"stddev":1,"ci95_valid":true}`), &decoded, + ); err != nil { + t.Fatal(err) + } + if !decoded.CI95Valid || decoded.CI95Low == 0 || decoded.CI95High == 0 { + t.Fatalf("legacy confidence interval = %+v", decoded) + } +} + func assertClose(t *testing.T, got, want float64) { t.Helper() if math.Abs(got-want) > 1e-9 { diff --git a/internal/platform/dependencies_darwin.go b/internal/platform/dependencies_darwin.go index 534ea86..a0c89ba 100644 --- a/internal/platform/dependencies_darwin.go +++ b/internal/platform/dependencies_darwin.go @@ -3,31 +3,40 @@ package platform import ( "bufio" "bytes" + "context" "os" "os/exec" "path/filepath" "strings" ) -func LinkedFiles(executable string) []LinkedDependency { +func LinkedFiles(ctx context.Context, executable string) ([]LinkedDependency, error) { type queuedLibrary struct { path string rpaths []string } - executableRPaths := loadRPaths(executable, executable, executable) + executableRPaths := loadRPaths(ctx, executable, executable, executable) queue := []queuedLibrary{{path: executable, rpaths: executableRPaths}} seen := map[string]bool{executable: true} var result []LinkedDependency for len(queue) > 0 { + if err := ctx.Err(); err != nil { + return nil, err + } current := queue[0] queue = queue[1:] - output, err := exec.Command("/usr/bin/otool", "-L", current.path).Output() + output, err := exec.CommandContext(ctx, "/usr/bin/otool", "-L", current.path).Output() if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } continue } - rpaths := append(loadRPaths(current.path, executable, current.path), current.rpaths...) + rpaths := append( + loadRPaths(ctx, current.path, executable, current.path), current.rpaths..., + ) for _, name := range parseOtool(output) { - dependency, ok := resolveDylib(name, executable, current.path, rpaths) + dependency, ok := resolveDylib(ctx, name, executable, current.path, rpaths) if !ok || seen[dependency.Path] { continue } @@ -38,7 +47,7 @@ func LinkedFiles(executable string) []LinkedDependency { } } } - return result + return result, nil } func parseOtool(output []byte) []string { @@ -53,8 +62,8 @@ func parseOtool(output []byte) []string { return result } -func loadRPaths(path, executable, loader string) []string { - output, err := exec.Command("/usr/bin/otool", "-l", path).Output() +func loadRPaths(ctx context.Context, path, executable, loader string) []string { + output, err := exec.CommandContext(ctx, "/usr/bin/otool", "-l", path).Output() if err != nil { return nil } @@ -79,7 +88,9 @@ func loadRPaths(path, executable, loader string) []string { return result } -func resolveDylib(name, executable, loader string, rpaths []string) (LinkedDependency, bool) { +func resolveDylib( + ctx context.Context, name, executable, loader string, rpaths []string, +) (LinkedDependency, bool) { var candidates []string if suffix, found := strings.CutPrefix(name, "@rpath/"); found { for _, rpath := range rpaths { @@ -101,7 +112,7 @@ func resolveDylib(name, executable, loader string, rpaths []string) (LinkedDepen sharedCacheCandidate = resolved } } - if sharedCacheCandidate != "" && inSharedCache(sharedCacheCandidate) { + if sharedCacheCandidate != "" && inSharedCache(ctx, sharedCacheCandidate) { return LinkedDependency{Path: sharedCacheCandidate, SharedCache: true}, true } return LinkedDependency{}, false @@ -117,6 +128,6 @@ func isSharedCachePath(path string) bool { strings.HasPrefix(path, "/System/Library/") } -func inSharedCache(path string) bool { - return exec.Command("/usr/bin/dyld_info", "-dependents", path).Run() == nil +func inSharedCache(ctx context.Context, path string) bool { + return exec.CommandContext(ctx, "/usr/bin/dyld_info", "-dependents", path).Run() == nil } diff --git a/internal/platform/dependencies_darwin_test.go b/internal/platform/dependencies_darwin_test.go index 41f9c59..8596a02 100644 --- a/internal/platform/dependencies_darwin_test.go +++ b/internal/platform/dependencies_darwin_test.go @@ -1,6 +1,7 @@ package platform import ( + "context" "os" "path/filepath" "reflect" @@ -26,12 +27,14 @@ func TestResolveDylibUsesRPathAndIdentifiesSharedCache(t *testing.T) { } library, _ = filepath.EvalSymlinks(library) dependency, ok := resolveDylib( + context.Background(), "@rpath/liblocal.dylib", "/tmp/tool", "/tmp/tool", []string{directory}, ) if !ok || dependency.Path != library || dependency.SharedCache { t.Fatalf("resolved dependency = %+v, valid=%v", dependency, ok) } dependency, ok = resolveDylib( + context.Background(), "/usr/lib/libSystem.B.dylib", "/tmp/tool", "/tmp/tool", nil, ) if !ok || !dependency.SharedCache { diff --git a/internal/platform/dependencies_linux.go b/internal/platform/dependencies_linux.go index 38a20a9..ab87925 100644 --- a/internal/platform/dependencies_linux.go +++ b/internal/platform/dependencies_linux.go @@ -1,19 +1,26 @@ package platform import ( + "context" "os" "os/exec" "path/filepath" "strings" ) -func LinkedFiles(executable string) []LinkedDependency { - output, _ := exec.Command("ldd", executable).CombinedOutput() +func LinkedFiles(ctx context.Context, executable string) ([]LinkedDependency, error) { + output, _ := exec.CommandContext(ctx, "ldd", executable).CombinedOutput() + if err := ctx.Err(); err != nil { + return nil, err + } files := parseLDD(output) if len(files) == 0 { if interpreter := shebangInterpreter(executable); interpreter != "" { files = append(files, interpreter) - output, _ = exec.Command("ldd", interpreter).CombinedOutput() + output, _ = exec.CommandContext(ctx, "ldd", interpreter).CombinedOutput() + if err := ctx.Err(); err != nil { + return nil, err + } files = append(files, parseLDD(output)...) } } @@ -22,7 +29,7 @@ func LinkedFiles(executable string) []LinkedDependency { for index, path := range paths { result[index] = LinkedDependency{Path: path} } - return result + return result, nil } func parseLDD(output []byte) []string { diff --git a/internal/platform/sampler_darwin.go b/internal/platform/sampler_darwin.go index e484419..f1126ed 100644 --- a/internal/platform/sampler_darwin.go +++ b/internal/platform/sampler_darwin.go @@ -13,13 +13,15 @@ static int sample_process(pid_t pid, struct proc_taskinfo *task, } if (proc_pid_rusage(pid, RUSAGE_INFO_V4, (rusage_info_t *)usage) != 0) { usage->ri_phys_footprint = 0; + return 1; } - return 1; + return 2; } */ import "C" import ( + "syscall" "time" "unsafe" ) @@ -32,15 +34,21 @@ func SampleTree(rootPID int) (Metrics, bool) { pids := processGroupPIDs(rootPID) var total Metrics foundRoot := false + physicalFootprintValid := true for _, pid := range pids { if pid <= 0 { continue } var task C.struct_proc_taskinfo var usage C.struct_rusage_info_v4 - if C.sample_process(pid, &task, &usage) == 0 { + sampled := C.sample_process(pid, &task, &usage) + if sampled == 0 { + physicalFootprintValid = false continue } + if sampled < 2 { + physicalFootprintValid = false + } if int(pid) == rootPID { foundRoot = true } @@ -50,6 +58,9 @@ func SampleTree(rootPID int) (Metrics, bool) { total.Processes++ total.Threads += uint64(task.pti_threadnum) } + total.PhysicalFootprintValid = total.Processes > 0 && physicalFootprintValid + total.PhysicalFootprintInvalid = !total.PhysicalFootprintValid && + (foundRoot || syscall.Kill(rootPID, 0) == nil) return total, foundRoot } diff --git a/internal/platform/sampler_linux.go b/internal/platform/sampler_linux.go index a5c1d98..bcc921b 100644 --- a/internal/platform/sampler_linux.go +++ b/internal/platform/sampler_linux.go @@ -15,55 +15,31 @@ func DefaultInterval() time.Duration { return 10 * time.Millisecond } func SampleImmediately() bool { return true } func SampleTree(rootPID int) (Metrics, bool) { - seen := make(map[int]bool) - queue := []int{rootPID} + entries, err := os.ReadDir("/proc") + if err != nil { + return Metrics{}, false + } var total Metrics - for len(queue) > 0 { - pid := queue[0] - queue = queue[1:] - if seen[pid] { + rootFound := false + for _, entry := range entries { + pid, err := strconv.Atoi(entry.Name()) + if err != nil { continue } - seen[pid] = true record, err := readProcess(pid) - if err != nil { - if pid == rootPID { - return Metrics{}, false - } + if err != nil || record.GroupID != rootPID { continue } + if record.PID == rootPID { + rootFound = true + } total.ResidentBytes += record.ResidentBytes total.VirtualBytes += record.VirtualBytes total.Processes++ total.Threads += record.Threads total.FileDescriptors += record.FileDescriptors - queue = append(queue, readChildren(pid)...) - } - return total, true -} - -func readChildren(pid int) []int { - taskDirectory := filepath.Join("/proc", strconv.Itoa(pid), "task") - tasks, err := os.ReadDir(taskDirectory) - if err != nil { - return nil - } - seen := make(map[int]bool) - children := make([]int, 0, 2) - for _, task := range tasks { - data, readErr := os.ReadFile(filepath.Join(taskDirectory, task.Name(), "children")) - if readErr != nil { - continue - } - for _, field := range strings.Fields(string(data)) { - child, parseErr := strconv.Atoi(field) - if parseErr == nil && !seen[child] { - seen[child] = true - children = append(children, child) - } - } } - return children + return total, rootFound } func readProcess(pid int) (Process, error) { @@ -81,6 +57,7 @@ func readProcess(pid int) (Process, error) { return Process{}, errors.New("short process stat") } ppid, _ := strconv.Atoi(fields[1]) + groupID, _ := strconv.Atoi(fields[2]) threads, _ := strconv.ParseUint(fields[17], 10, 64) virtual, _ := strconv.ParseUint(fields[20], 10, 64) rssPages, _ := strconv.ParseInt(fields[21], 10, 64) @@ -88,7 +65,7 @@ func readProcess(pid int) (Process, error) { rssPages = 0 } return Process{ - PID: pid, PPID: ppid, Threads: threads, VirtualBytes: virtual, + PID: pid, PPID: ppid, GroupID: groupID, Threads: threads, VirtualBytes: virtual, ResidentBytes: uint64(rssPages) * uint64(os.Getpagesize()), FileDescriptors: countDirectory(filepath.Join(base, "fd")), }, nil diff --git a/internal/platform/sampler_linux_test.go b/internal/platform/sampler_linux_test.go index 493042e..7ea725b 100644 --- a/internal/platform/sampler_linux_test.go +++ b/internal/platform/sampler_linux_test.go @@ -4,8 +4,13 @@ package platform import ( "os" + "os/exec" "path/filepath" + "strconv" + "strings" + "syscall" "testing" + "time" ) func TestCountDirectoryDoesNotRequireSortedEntries(t *testing.T) { @@ -25,3 +30,43 @@ func TestMissingRootIsNotAZeroSample(t *testing.T) { t.Fatal("missing root process should produce an invalid sample") } } + +func TestSampleTreeIncludesReparentedProcessGroupMembers(t *testing.T) { + childFile := filepath.Join(t.TempDir(), "child-pid") + cmd := exec.Command( + "sh", "-c", + "(sleep 30 & printf '%s' $! > \"$1\"); exec sleep 30", + "sh", childFile, + ) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + _ = cmd.Wait() + }) + + var child Process + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(childFile) + if err == nil { + childPID, parseErr := strconv.Atoi(strings.TrimSpace(string(data))) + if parseErr == nil { + child, err = readProcess(childPID) + if err == nil && child.PPID != cmd.Process.Pid { + break + } + } + } + time.Sleep(10 * time.Millisecond) + } + if child.PID == 0 || child.PPID == cmd.Process.Pid || child.GroupID != cmd.Process.Pid { + t.Fatalf("child process was not reparented in group: %+v", child) + } + metrics, valid := SampleTree(cmd.Process.Pid) + if !valid || metrics.Processes < 2 { + t.Fatalf("group metrics = %+v, valid = %v", metrics, valid) + } +} diff --git a/internal/platform/types.go b/internal/platform/types.go index aa90041..e0d8839 100644 --- a/internal/platform/types.go +++ b/internal/platform/types.go @@ -1,20 +1,22 @@ package platform type Metrics struct { - ResidentBytes uint64 - PhysicalFootprintBytes uint64 - VirtualBytes uint64 - Processes uint64 - Threads uint64 - FileDescriptors uint64 - ResidentByteSamples uint64 - SampleCount uint64 - ResidentByteSeconds float64 - SampleCoverageSeconds float64 + ResidentBytes uint64 + PhysicalFootprintBytes uint64 + PhysicalFootprintValid bool + PhysicalFootprintInvalid bool + VirtualBytes uint64 + Processes uint64 + Threads uint64 + FileDescriptors uint64 + ResidentByteSamples uint64 + SampleCount uint64 + ResidentByteSeconds float64 + SampleCoverageSeconds float64 } type Process struct { - PID, PPID int + PID, PPID, GroupID int ResidentBytes, VirtualBytes uint64 Threads, FileDescriptors uint64 } diff --git a/internal/report/chart_artifacts.go b/internal/report/chart_artifacts.go index 4cc7b15..7f3691d 100644 --- a/internal/report/chart_artifacts.go +++ b/internal/report/chart_artifacts.go @@ -20,6 +20,7 @@ func WriteChartFiles( report model.Report, includeCommands bool, ) ([]ChartArtifact, error) { + report = safeDisplayReport(report) if err := os.MkdirAll(directory, 0o755); err != nil { return nil, err } diff --git a/internal/report/chart_delta.go b/internal/report/chart_delta.go index 6dfcfa1..146d7ef 100644 --- a/internal/report/chart_delta.go +++ b/internal/report/chart_delta.go @@ -25,7 +25,7 @@ func deltaForestChart(report model.Report, group chartGroup) svgChart { maximum := 0.0 for _, metric := range group.metrics { row := metricRows[metric.row] - if !available(row, report.Host.OS) { + if !availableFor(row, report.Host.OS, baseline.Summary) { rows = append(rows, forestRow{ label: metric.name, status: "N/A on " + report.Host.OS, class: "neutral", identity: toolColor(report, baselinePosition), @@ -52,6 +52,13 @@ func deltaForestChart(report model.Report, group chartGroup) svgChart { if index == baselinePosition { continue } + if !availableFor(row, report.Host.OS, candidate.Summary) { + rows = append(rows, forestRow{ + label: metric.name + " Β· " + candidate.Tool.Name, + status: "N/A", class: "neutral", identity: toolColor(report, index), + }) + continue + } value := metric.stats(candidate).Mean delta := compareMetric( baseline, candidate, row, report.Config.IntervalMS/1000, @@ -69,6 +76,13 @@ func deltaForestChart(report model.Report, group chartGroup) svgChart { if index == baselinePosition { continue } + if !availableFor(row, report.Host.OS, candidate.Summary) { + rows = append(rows, forestRow{ + label: metric.name + " Β· " + candidate.Tool.Name, + status: "N/A", class: "neutral", identity: toolColor(report, index), + }) + continue + } delta := compareMetric( baseline, candidate, row, report.Config.IntervalMS/1000, ) diff --git a/internal/report/chart_distribution.go b/internal/report/chart_distribution.go index 4022c85..4d912c7 100644 --- a/internal/report/chart_distribution.go +++ b/internal/report/chart_distribution.go @@ -13,6 +13,11 @@ func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart if !available(metricRows[metric.row], report.Host.OS) { return unavailableChart("RUN DISTRIBUTION Β· "+metric.name, report.Host.OS) } + for _, benchmark := range report.Benchmarks { + if !availableFor(metricRows[metric.row], report.Host.OS, benchmark.Summary) { + return unavailableChart("RUN DISTRIBUTION Β· "+metric.name, "metric unavailable") + } + } minimum, maximum := 0.0, 0.0 for _, benchmark := range report.Benchmarks { stats := metric.stats(benchmark) @@ -57,6 +62,9 @@ func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart left, y-4, left+plotWidth, ) for runIndex, run := range benchmark.Runs { + if !chartRunAvailable(metric, run) { + continue + } jitter := (runIndex%3 - 1) * 5 fmt.Fprintf( &body, ``, x(index), positionY(value), color, @@ -131,7 +140,11 @@ func measurementTrendChart(report model.Report, benchmarkIndex int, group chartG func trendValues(runs []model.Run, metric chartMetric) []float64 { values := make([]float64, len(runs)) for index, run := range runs { - values[index] = metric.run(run) + if chartRunAvailable(metric, run) { + values[index] = metric.run(run) + } else { + values[index] = math.NaN() + } } return values } @@ -139,6 +152,9 @@ func trendValues(runs []model.Run, metric chartMetric) []float64 { func valueRange(values []float64) (float64, float64) { minimum, maximum := math.Inf(1), math.Inf(-1) for _, value := range values { + if math.IsNaN(value) { + continue + } minimum = math.Min(minimum, value) maximum = math.Max(maximum, value) } @@ -152,6 +168,9 @@ func trendLaneRange(runs []model.Run, metrics []chartMetric) (float64, float64) minimum, maximum := math.Inf(1), math.Inf(-1) for _, metric := range metrics { for _, run := range runs { + if !chartRunAvailable(metric, run) { + continue + } value := metric.run(run) minimum = math.Min(minimum, value) maximum = math.Max(maximum, value) @@ -163,8 +182,8 @@ func trendLaneRange(runs []model.Run, metrics []chartMetric) (float64, float64) return minimum, maximum } -func trendLanes(report model.Report, group chartGroup) []trendLane { - metrics := trendMetrics(report, group.metrics) +func trendLanes(report model.Report, benchmarkIndex int, group chartGroup) []trendLane { + metrics := trendMetrics(report, benchmarkIndex, group.metrics) byRow := make(map[int]chartMetric, len(metrics)) for _, metric := range metrics { byRow[metric.row] = metric @@ -224,10 +243,15 @@ func trendLaneDefinitions(byRow map[int]chartMetric, specs ...trendLaneSpec) []t return lanes } -func trendMetrics(report model.Report, metrics []chartMetric) []chartMetric { +func trendMetrics( + report model.Report, benchmarkIndex int, metrics []chartMetric, +) []chartMetric { result := make([]chartMetric, 0, len(metrics)) for _, metric := range metrics { - if available(metricRows[metric.row], report.Host.OS) { + if availableFor( + metricRows[metric.row], report.Host.OS, + report.Benchmarks[benchmarkIndex].Summary, + ) { result = append(result, metric) } } diff --git a/internal/report/command_legend.go b/internal/report/command_legend.go index 2db6f2a..ba02f6f 100644 --- a/internal/report/command_legend.go +++ b/internal/report/command_legend.go @@ -1,11 +1,44 @@ package report import ( + "strconv" "strings" + "unicode" "github.com/shellcell/snailrace/internal/model" ) func fullCommand(benchmark model.Benchmark) string { - return strings.Join(benchmark.Tool.Command, " ") + command := benchmark.Tool.Command + if len(command) == 0 { + return "" + } + if benchmark.Tool.ShellCommand { + return safeCommandText(command[0]) + } + arguments := make([]string, len(command)) + for index, argument := range command { + arguments[index] = shellQuoteArgument(argument) + } + return strings.Join(arguments, " ") +} + +func shellQuoteArgument(argument string) string { + if strings.IndexFunc(argument, unicode.IsControl) >= 0 { + return strconv.QuoteToGraphic(argument) + } + if argument != "" && strings.IndexFunc(argument, func(character rune) bool { + return !(unicode.IsLetter(character) || unicode.IsDigit(character) || + strings.ContainsRune("_@%+=:,./-", character)) + }) < 0 { + return argument + } + return "'" + strings.ReplaceAll(argument, "'", `'"'"'`) + "'" +} + +func safeCommandText(command string) string { + if strings.IndexFunc(command, unicode.IsControl) >= 0 { + return strconv.QuoteToGraphic(command) + } + return command } diff --git a/internal/report/command_legend_test.go b/internal/report/command_legend_test.go new file mode 100644 index 0000000..58160b2 --- /dev/null +++ b/internal/report/command_legend_test.go @@ -0,0 +1,48 @@ +package report + +import ( + "bytes" + "strings" + "testing" + + "github.com/shellcell/snailrace/internal/model" +) + +func TestFullCommandQuotesDirectArgumentsFaithfully(t *testing.T) { + benchmark := model.Benchmark{Tool: model.ToolInfo{Command: []string{ + "tool", "two words", "it's", "", "line\nbreak", + }}} + want := `tool 'two words' 'it'"'"'s' '' "line\nbreak"` + if got := fullCommand(benchmark); got != want { + t.Fatalf("command = %q, want %q", got, want) + } +} + +func TestFullCommandQuotesSingleDirectArgument(t *testing.T) { + benchmark := model.Benchmark{Tool: model.ToolInfo{Command: []string{"/tmp/my tool"}}} + if got, want := fullCommand(benchmark), `'/tmp/my tool'`; got != want { + t.Fatalf("command = %q, want %q", got, want) + } +} + +func TestRenderedLabelsContainNoTerminalControls(t *testing.T) { + run := model.Run{WallSeconds: 1, StopReason: "exited"} + report := model.Report{Verbose: true, Config: model.Config{Baseline: 1}, + Benchmarks: []model.Benchmark{{ + Tool: model.ToolInfo{Name: "safe\n\x1b[31munsafe", Command: []string{"true"}}, + Runs: []model.Run{run}, Summary: model.Summarize([]model.Run{run}), + }}} + for _, format := range []string{"text", "markdown", "html", "svg"} { + var output bytes.Buffer + if err := Write(&output, format, report); err != nil { + t.Fatalf("%s: %v", format, err) + } + if strings.ContainsAny(output.String(), "\n\x1b") && + strings.Contains(output.String(), "safe\n\x1b") { + t.Fatalf("%s retained label control characters", format) + } + if !strings.Contains(output.String(), `safe\n\x1b[31munsafe`) { + t.Fatalf("%s does not contain readable escaped label: %q", format, output.String()) + } + } +} diff --git a/internal/report/comparison.go b/internal/report/comparison.go index f53898d..0b9b631 100644 --- a/internal/report/comparison.go +++ b/internal/report/comparison.go @@ -17,6 +17,8 @@ const ( type deltaResult struct { percent float64 percentAvailable bool + baselineMean float64 + candidateMean float64 difference model.Stats status string class string @@ -36,14 +38,20 @@ func compareMetric( row metricRow, intervalSeconds float64, ) deltaResult { - baseMean := row.stats(baseline.Summary).Mean - candidateMean := row.stats(candidate.Summary).Mean - result := deltaResult{} + valid := func(model.Run) bool { return true } + if row.darwinOnly { + valid = func(run model.Run) bool { return run.PhysicalFootprintValid } + } + baseValues, candidateValues, differences := pairedValues( + baseline.Runs, candidate.Runs, row.run, valid, + ) + baseMean := model.CalculateStats(baseValues).Mean + candidateMean := model.CalculateStats(candidateValues).Mean + result := deltaResult{baselineMean: baseMean, candidateMean: candidateMean} if baseMean != 0 { result.percent = (candidateMean/baseMean - 1) * 100 result.percentAvailable = true } - differences := pairedDifferences(baseline.Runs, candidate.Runs, row.run) result.difference = model.CalculateStats(differences) result.status, result.class = comparisonStatus(result.difference, row.direction) if row.sampled && !samplingReliable(baseline, candidate, intervalSeconds) { @@ -81,22 +89,33 @@ func benchmarkSamplesReliable(benchmark model.Benchmark, intervalSeconds float64 return true } -func pairedDifferences( +func pairedValues( baseline, candidate []model.Run, pick func(model.Run) float64, -) []float64 { + valid func(model.Run) bool, +) ([]float64, []float64, []float64) { baselineByIndex := make(map[int]float64, len(baseline)) for _, run := range baseline { - baselineByIndex[run.Index] = pick(run) + if valid(run) { + baselineByIndex[run.Index] = pick(run) + } } + baseValues := make([]float64, 0, len(candidate)) + candidateValues := make([]float64, 0, len(candidate)) differences := make([]float64, 0, len(candidate)) for _, run := range candidate { + if !valid(run) { + continue + } value, ok := baselineByIndex[run.Index] if ok { - differences = append(differences, pick(run)-value) + candidateValue := pick(run) + baseValues = append(baseValues, value) + candidateValues = append(candidateValues, candidateValue) + differences = append(differences, candidateValue-value) } } - return differences + return baseValues, candidateValues, differences } func comparisonStatus(stats model.Stats, direction metricDirection) (string, string) { diff --git a/internal/report/comparison_html.go b/internal/report/comparison_html.go index 6635780..29843dd 100644 --- a/internal/report/comparison_html.go +++ b/internal/report/comparison_html.go @@ -76,7 +76,7 @@ func writeHTMLComparisonRow( baseline, candidate model.Benchmark, row metricRow, ) { - if !available(row, operatingSystem) { + if !availableFor(row, operatingSystem, baseline.Summary, candidate.Summary) { fmt.Fprintf(writer, "%sN/A", row.name) return } @@ -86,8 +86,8 @@ func writeHTMLComparisonRow( `%s%s%s`+ `%s`+ `%s`, - row.name, row.format(row.stats(baseline.Summary).Mean), delta.class, - row.format(row.stats(candidate.Summary).Mean), delta.class, + row.name, row.format(delta.baselineMean), delta.class, + row.format(delta.candidateMean), delta.class, html.EscapeString(formatDelta(delta, row)), html.EscapeString(formatDeltaInterval(delta, row)), ) diff --git a/internal/report/comparison_markdown.go b/internal/report/comparison_markdown.go index 5da5f28..6c302a6 100644 --- a/internal/report/comparison_markdown.go +++ b/internal/report/comparison_markdown.go @@ -36,7 +36,9 @@ func writeMarkdownComparison(writer io.Writer, report model.Report) { formatBytes(float64(candidate.Tool.DiskFootprintBytes)), staticDelta, ) for _, row := range metricRows { - if !available(row, report.Host.OS) { + if !availableFor( + row, report.Host.OS, baseline.Summary, candidate.Summary, + ) { fmt.Fprintf(writer, "| %s | N/A | N/A | N/A | N/A |\n", row.name) continue } @@ -45,8 +47,7 @@ func writeMarkdownComparison(writer io.Writer, report model.Report) { ) fmt.Fprintf( writer, "| %s | %s | %s | %s | %s |\n", row.name, - row.format(row.stats(baseline.Summary).Mean), - row.format(row.stats(candidate.Summary).Mean), + row.format(delta.baselineMean), row.format(delta.candidateMean), formatDelta(delta, row), formatDeltaInterval(delta, row), ) } diff --git a/internal/report/comparison_test.go b/internal/report/comparison_test.go index cb1f439..6ef1f40 100644 --- a/internal/report/comparison_test.go +++ b/internal/report/comparison_test.go @@ -80,6 +80,27 @@ func TestTooFewValidSamplesMarksMetricsAsLimited(t *testing.T) { } } +func TestPhysicalFootprintComparisonExcludesInvalidPairs(t *testing.T) { + baselineRuns := []model.Run{ + {Index: 1, PeakPhysicalFootprintBytes: 100, PhysicalFootprintValid: true}, + {Index: 2, PeakPhysicalFootprintBytes: 1000}, + } + candidateRuns := []model.Run{ + {Index: 1, PeakPhysicalFootprintBytes: 90, PhysicalFootprintValid: true}, + {Index: 2, PeakPhysicalFootprintBytes: 1}, + } + baseline := model.Benchmark{Runs: baselineRuns, Summary: model.Summarize(baselineRuns)} + candidate := model.Benchmark{Runs: candidateRuns, Summary: model.Summarize(candidateRuns)} + result := compareMetric(baseline, candidate, metricRows[8], 0) + if result.difference.N != 1 || result.difference.Mean != -10 { + t.Fatalf("physical footprint difference = %+v", result.difference) + } + if result.baselineMean != 100 || result.candidateMean != 90 || + math.Abs(result.percent+10) > 1e-9 { + t.Fatalf("paired physical footprint means = %+v", result) + } +} + func benchmarkWithWallTimes(name string, values ...float64) model.Benchmark { runs := make([]model.Run, len(values)) for index, value := range values { diff --git a/internal/report/comparison_text.go b/internal/report/comparison_text.go index 8580bc3..ebf25bf 100644 --- a/internal/report/comparison_text.go +++ b/internal/report/comparison_text.go @@ -32,7 +32,9 @@ func writeTextComparison(writer io.Writer, report model.Report) { formatBytes(float64(candidate.Tool.DiskFootprintBytes)), staticDelta, ) for _, row := range metricRows { - if !available(row, report.Host.OS) { + if !availableFor( + row, report.Host.OS, baseline.Summary, candidate.Summary, + ) { fmt.Fprintf(writer, "%s\tN/A\tN/A\tN/A\tN/A\n", row.name) continue } @@ -41,8 +43,7 @@ func writeTextComparison(writer io.Writer, report model.Report) { ) fmt.Fprintf( writer, "%s\t%s\t%s\t%s\t%s\n", row.name, - row.format(row.stats(baseline.Summary).Mean), - row.format(row.stats(candidate.Summary).Mean), + row.format(delta.baselineMean), row.format(delta.candidateMean), formatDelta(delta, row), formatDeltaInterval(delta, row), ) } diff --git a/internal/report/format.go b/internal/report/format.go index 6a19e4b..663d2c0 100644 --- a/internal/report/format.go +++ b/internal/report/format.go @@ -119,3 +119,20 @@ func available(row metricRow, operatingSystem string) bool { return (operatingSystem != "darwin" || !row.unavailableDarwin) && (!row.darwinOnly || operatingSystem == "darwin") } + +func availableFor( + row metricRow, operatingSystem string, summaries ...model.Summary, +) bool { + if !available(row, operatingSystem) { + return false + } + if !row.darwinOnly { + return true + } + for _, summary := range summaries { + if row.stats(summary).N == 0 { + return false + } + } + return true +} diff --git a/internal/report/html.go b/internal/report/html.go index 1e5042a..e2823fe 100644 --- a/internal/report/html.go +++ b/internal/report/html.go @@ -4,7 +4,6 @@ import ( "fmt" "html" "io" - "strings" "github.com/shellcell/snailrace/internal/model" ) @@ -69,6 +68,16 @@ func writeHTML(writer io.Writer, report model.Report) error { html.EscapeString(report.Host.OS), html.EscapeString(report.Host.Architecture), ) writeCards(writer, report) + fmt.Fprintf( + writer, `

Balanced index dimensions actually included: %s.

`, + html.EscapeString(includedDimensionsText(report)), + ) + for _, caveat := range reliabilityCaveats(report) { + fmt.Fprintf( + writer, `

Reliability: %s.

`, + html.EscapeString(caveat), + ) + } writeHTMLCommandLegend(writer, report) writeHTMLFailures(writer, report) if len(report.Benchmarks) > 1 { @@ -212,7 +221,7 @@ func writeHTMLBenchmark( } fmt.Fprintf( writer, `
%s
`, - html.EscapeString(strings.Join(benchmark.Tool.Command, " ")), + html.EscapeString(fullCommand(benchmark)), ) fmt.Fprintf( writer, @@ -257,7 +266,7 @@ func writeHTMLRow( row metricRow, summary model.Summary, ) { - if !available(row, operatingSystem) { + if !availableFor(row, operatingSystem, summary) { fmt.Fprintf(writer, "%sN/A", row.name) return } diff --git a/internal/report/markdown.go b/internal/report/markdown.go index f08c0b9..2d9e61b 100644 --- a/internal/report/markdown.go +++ b/internal/report/markdown.go @@ -18,6 +18,7 @@ func WriteMarkdownWithCharts( charts []ChartArtifact, chartDirectory string, ) error { + report = safeDisplayReport(report) checked := newErrorWriter(writer) writer = checked fmt.Fprintf( @@ -28,6 +29,13 @@ func WriteMarkdownWithCharts( report.Host.OS, report.Host.Architecture, report.Config.Runs, report.Config.Warmups, report.Config.Mode, ) + fmt.Fprintf( + writer, "Balanced index dimensions actually included: **%s**.\n\n", + escapeMarkdown(includedDimensionsText(report)), + ) + for _, caveat := range reliabilityCaveats(report) { + fmt.Fprintf(writer, "> Reliability: %s.\n\n", escapeMarkdown(caveat)) + } writeMarkdownCommandLegend(writer, report) writeMarkdownFailures(writer, report) if len(report.Benchmarks) > 1 { @@ -79,7 +87,7 @@ func writeMarkdownBenchmark( "Disk footprint: %s executable + %s linked = %s "+ "(%d files; %d dyld-cache dependencies excluded)\n\n", escapeMarkdown(label), - strings.Join(benchmark.Tool.Command, " "), + fullCommand(benchmark), benchmark.Tool.SHA256, formatBytes(float64(benchmark.Tool.SizeBytes)), formatBytes(float64(benchmark.Tool.LinkedSizeBytes)), @@ -105,7 +113,7 @@ func writeMarkdownBenchmark( ) fmt.Fprintln(writer, "|---|---:|---:|---:|---:|---:|") for _, row := range metricRows { - if !available(row, operatingSystem) { + if !availableFor(row, operatingSystem, benchmark.Summary) { fmt.Fprintf(writer, "| %s | N/A | N/A | N/A | N/A | N/A |\n", row.name) continue } diff --git a/internal/report/ranking_json.go b/internal/report/ranking_json.go index 8db795d..08cd2f9 100644 --- a/internal/report/ranking_json.go +++ b/internal/report/ranking_json.go @@ -40,20 +40,25 @@ type jsonRankingRow struct { } type jsonRanking struct { - Available bool `json:"available"` - UnavailableReason string `json:"unavailable_reason,omitempty"` - Method string `json:"method"` - PrimaryMetric string `json:"primary_metric"` - Winners []jsonWinner `json:"winners"` - Rows []jsonRankingRow `json:"rows"` + Available bool `json:"available"` + UnavailableReason string `json:"unavailable_reason,omitempty"` + Method string `json:"method"` + PrimaryMetric string `json:"primary_metric"` + IncludedDimensions []string `json:"included_dimensions"` + Winners []jsonWinner `json:"winners"` + Rows []jsonRankingRow `json:"rows"` } func writeJSON(writer io.Writer, report model.Report) error { payload := struct { model.Report - Ranking jsonRanking `json:"ranking"` - Failures []jsonFailure `json:"failures,omitempty"` - }{Report: report, Ranking: makeJSONRanking(report), Failures: makeJSONFailures(report)} + Ranking jsonRanking `json:"ranking"` + Failures []jsonFailure `json:"failures,omitempty"` + Reliability []string `json:"reliability_caveats,omitempty"` + }{ + Report: report, Ranking: makeJSONRanking(report), + Failures: makeJSONFailures(report), Reliability: reliabilityCaveats(report), + } encoder := json.NewEncoder(writer) encoder.SetIndent("", " ") return encoder.Encode(payload) @@ -78,7 +83,7 @@ func makeJSONRanking(report model.Report) jsonRanking { Available: ranking.available, UnavailableReason: ranking.unavailableReason, Method: "equal-weight geometric mean of normalized category costs; included: " + balancedIndexCategories(ranking), - PrimaryMetric: ranking.primaryLabel, + PrimaryMetric: ranking.primaryLabel, IncludedDimensions: includedDimensions(report), } for _, winner := range ranking.winners { result.Winners = append(result.Winners, jsonWinner{ diff --git a/internal/report/report.go b/internal/report/report.go index 6000c01..8fe5454 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -6,9 +6,13 @@ import ( "strings" "github.com/shellcell/snailrace/internal/model" + "github.com/shellcell/snailrace/internal/style" ) func Write(writer io.Writer, format string, report model.Report) error { + if strings.ToLower(format) != "json" { + report = safeDisplayReport(report) + } switch strings.ToLower(format) { case "text", "txt": return writeText(writer, report) @@ -25,6 +29,16 @@ func Write(writer io.Writer, format string, report model.Report) error { } } +func safeDisplayReport(report model.Report) model.Report { + report.Benchmarks = append([]model.Benchmark(nil), report.Benchmarks...) + for index := range report.Benchmarks { + report.Benchmarks[index].Tool.Name = style.SafeText( + report.Benchmarks[index].Tool.Name, + ) + } + return report +} + func ValidFormat(format string) bool { switch strings.ToLower(format) { case "text", "txt", "json", "markdown", "md", "html", "svg": diff --git a/internal/report/semantics.go b/internal/report/semantics.go new file mode 100644 index 0000000..89d0a6c --- /dev/null +++ b/internal/report/semantics.go @@ -0,0 +1,77 @@ +package report + +import ( + "fmt" + "strings" + + "github.com/shellcell/snailrace/internal/model" +) + +func includedDimensions(report model.Report) []string { + ranking := calculateRanking(report) + var result []string + if ranking.indexPrimary { + result = append(result, "time") + } + if ranking.indexCPU { + result = append(result, "cpu") + } + if ranking.indexRAM { + result = append(result, "ram") + } + if ranking.indexFootprint { + result = append(result, "disk") + } + return result +} + +func includedDimensionsText(report model.Report) string { + dimensions := includedDimensions(report) + if len(dimensions) == 0 { + return "none" + } + return strings.Join(dimensions, ", ") +} + +func reliabilityCaveats(report model.Report) []string { + interval := report.Config.IntervalMS / 1000 + var result []string + for _, benchmark := range report.Benchmarks { + if benchmark.Tool.SHA256 != "" && !benchmark.Tool.ProvenanceVerified { + result = append(result, benchmark.Tool.Name+ + ": post-run executable verification was not completed") + } + if !benchmarkSamplesReliable(benchmark, interval) { + result = append(result, fmt.Sprintf( + "%s: sampled process metrics are limited by sparse observations", + benchmark.Tool.Name, + )) + } + if report.Host.OS != "darwin" { + continue + } + physical := benchmark.Summary.PeakPhysicalFootprintBytes + switch { + case physical == nil: + result = append(result, benchmark.Tool.Name+": physical footprint unavailable") + case physical.N < len(benchmark.Runs): + result = append(result, fmt.Sprintf( + "%s: physical footprint available for %d of %d runs", + benchmark.Tool.Name, physical.N, len(benchmark.Runs), + )) + } + } + return result +} + +func physicalFootprintAvailable(report model.Report) bool { + if report.Host.OS != "darwin" || len(report.Benchmarks) == 0 { + return false + } + for _, benchmark := range report.Benchmarks { + if benchmark.Summary.PeakPhysicalFootprintBytes == nil { + return false + } + } + return true +} diff --git a/internal/report/svg.go b/internal/report/svg.go index dc7ff8d..6017061 100644 --- a/internal/report/svg.go +++ b/internal/report/svg.go @@ -46,9 +46,12 @@ func writeSVG(writer io.Writer, report model.Report) error { } func svgHeader(output *strings.Builder, report model.Report) int { - panelHeight, nextSection := 72, 230 + caveats := reliabilityCaveats(report) + panelHeight := 96 + len(caveats)*22 + nextSection := 254 + len(caveats)*22 if report.Config.Mode == "tui" { - panelHeight, nextSection = 96, 254 + panelHeight += 24 + nextSection += 24 } fmt.Fprint(output, ``) fmt.Fprint(output, `🐌 SNAILRACE CHARTS`) @@ -77,14 +80,25 @@ func svgHeader(output *strings.Builder, report model.Report) int { html.EscapeString(report.Host.LoadBefore), report.Host.ProcessesBefore, html.EscapeString(clip(report.Host.Kernel, 32)), ) + fmt.Fprintf( + output, `balanced dimensions %s`, + html.EscapeString(includedDimensionsText(report)), + ) + for index, caveat := range caveats { + fmt.Fprintf( + output, `reliability %s`, + 227+index*22, html.EscapeString(caveat), + ) + } if report.Config.Mode == "tui" { duration := "until exit" if report.Config.DurationSeconds > 0 { duration = formatDuration(report.Config.DurationSeconds) } fmt.Fprintf( - output, `TUI %s Β· %dx%d`, - duration, report.Config.TerminalWidth, report.Config.TerminalHeight, + output, `TUI %s Β· %dx%d`, + 227+len(caveats)*22, duration, + report.Config.TerminalWidth, report.Config.TerminalHeight, ) } return nextSection diff --git a/internal/report/text.go b/internal/report/text.go index 173f859..20e1588 100644 --- a/internal/report/text.go +++ b/internal/report/text.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "io" - "strings" "text/tabwriter" "golang.org/x/term" @@ -39,6 +38,10 @@ func writeVerboseText(writer io.Writer, report model.Report) error { report.Config.Mode, report.Config.Runs, report.Config.Warmups, formatNumber(report.Config.IntervalMS), ) + fmt.Fprintf(w, "Included dimensions\t%s\n", includedDimensionsText(report)) + for _, caveat := range reliabilityCaveats(report) { + fmt.Fprintf(w, "Reliability\t%s\n", caveat) + } if report.Config.Mode == "tui" { duration := "until exit" if report.Config.DurationSeconds > 0 { @@ -99,7 +102,7 @@ func writeTextBenchmark( label += " [BASELINE]" } fmt.Fprintf(w, "%s\t\n", label) - commandLines := wrapText(strings.Join(benchmark.Tool.Command, " "), 100) + commandLines := wrapText(fullCommand(benchmark), 100) for index, line := range commandLines { field := "" if index == 0 { @@ -125,7 +128,7 @@ func writeTextBenchmark( } fmt.Fprintln(w, "Metric\tMean Β± Οƒ\t95% CI mean\tMedian\tP95\tRange") for _, row := range metricRows { - if !available(row, operatingSystem) { + if !availableFor(row, operatingSystem, benchmark.Summary) { fmt.Fprintf(w, "%s\tN/A\tN/A\tN/A\tN/A\tN/A\n", row.name) continue } diff --git a/internal/report/text_bars.go b/internal/report/text_bars.go index 0339053..761f699 100644 --- a/internal/report/text_bars.go +++ b/internal/report/text_bars.go @@ -18,6 +18,11 @@ func writeCompactText(writer io.Writer, report model.Report) error { terminal := writerIsTerminal(writer) var body strings.Builder body.WriteString("\n") + fmt.Fprintf(&body, "BALANCED DIMENSIONS %s\n", includedDimensionsText(report)) + for _, caveat := range reliabilityCaveats(report) { + fmt.Fprintf(&body, "RELIABILITY %s\n", caveat) + } + body.WriteString("\n") writeTextFailures(&body, report) if len(report.Benchmarks) == 1 { writeCompactSingle(&body, report, terminal) @@ -80,7 +85,7 @@ func compactMetrics(report model.Report, ranking rankingData) []compactMetric { stdDev: func(s model.Summary) float64 { return s.MeanResidentBytes.StdDev }, }, { - name: "PHYS", unit: formatBytes, available: report.Host.OS == "darwin", + name: "PHYS", unit: formatBytes, available: physicalFootprintAvailable(report), value: func(r rankingRow) float64 { return report.Benchmarks[r.benchmark].Summary.PhysicalFootprintStats().Mean }, diff --git a/internal/report/text_test.go b/internal/report/text_test.go index a7cf889..b631409 100644 --- a/internal/report/text_test.go +++ b/internal/report/text_test.go @@ -2,6 +2,7 @@ package report import ( "bytes" + "encoding/json" "strings" "testing" @@ -140,6 +141,43 @@ func TestSamplingLimitedRAMIsShownButExcludedFromIndex(t *testing.T) { } } +func TestAllFormatsExposeActualDimensionsAndReliability(t *testing.T) { + report := samplingLimitedRAMReport() + for _, format := range []string{"text", "markdown", "html", "svg"} { + var output bytes.Buffer + if err := Write(&output, format, report); err != nil { + t.Fatalf("%s: %v", format, err) + } + text := strings.ToLower(output.String()) + if !strings.Contains(text, "time, cpu") { + t.Fatalf("%s omits actual included dimensions", format) + } + if !strings.Contains(text, "reliability") { + t.Fatalf("%s omits reliability caveat", format) + } + if !strings.Contains(text, "first:") || !strings.Contains(text, "second:") { + t.Fatalf("%s omits a per-tool reliability caveat", format) + } + } + var output bytes.Buffer + if err := Write(&output, "json", report); err != nil { + t.Fatal(err) + } + var decoded struct { + Ranking struct { + Included []string `json:"included_dimensions"` + } `json:"ranking"` + Reliability []string `json:"reliability_caveats"` + } + if err := json.Unmarshal(output.Bytes(), &decoded); err != nil { + t.Fatal(err) + } + if strings.Join(decoded.Ranking.Included, ",") != "time,cpu" || + len(decoded.Reliability) == 0 { + t.Fatalf("JSON semantics = %+v", decoded) + } +} + func TestColumnGapsFindsRunsOfAtLeastTwoSpaces(t *testing.T) { gaps := columnGaps("a bb c d") want := [][2]int{{1, 3}, {5, 8}} diff --git a/internal/runner/benchmark.go b/internal/runner/benchmark.go index d75cbda..ece186d 100644 --- a/internal/runner/benchmark.go +++ b/internal/runner/benchmark.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os" "github.com/shellcell/snailrace/internal/model" ) @@ -22,16 +23,45 @@ func Benchmark( if err := validateBenchmarkInputs(ctx, specs, config); err != nil { return nil, err } + specs = append([]Spec(nil), specs...) + for index := range specs { + specs[index].Args = append([]string(nil), specs[index].Args...) + } benchmarks := make([]model.Benchmark, len(specs)) progress := newProgressTracker(options, specs, config) defer progress.finish() inspector := newToolInspector() + pinned := make([]*os.File, len(specs)) + defer func() { + for _, file := range pinned { + if file != nil { + _ = file.Close() + } + } + }() for index, spec := range specs { benchmarks[index].Runs = make([]model.Run, 0, config.Runs) - tool, err := inspector.inspect(spec) + tool, err := inspector.inspect(ctx, spec) if err != nil { return nil, err } + file, err := inspector.pin(ctx, &tool) + if err != nil { + return nil, fmt.Errorf("hash %q: %w", tool.Name, err) + } + pinned[index] = file + pinExecution := !executableIsScript(file) + if pinExecution { + specs[index].executable = file + } + if tool.ShellTarget && pinExecution { + if command, ok := pinShellExecutable( + spec.Shell, pinnedExecutablePath(file), + ); ok { + specs[index].Shell = command + specs[index].shellTarget = true + } + } benchmarks[index].Tool = tool progress.setTool(index, tool) } @@ -109,8 +139,13 @@ func Benchmark( return nil, ctx.Err() } for index := range benchmarks { - if err := inspector.addHash(&benchmarks[index].Tool); err != nil { - return nil, fmt.Errorf("hash %q: %w", benchmarks[index].Tool.Name, err) + if !interrupted { + if err := inspector.verify( + ctx, benchmarks[index].Tool, pinned[index], + ); err != nil { + return nil, fmt.Errorf("verify %q: %w", benchmarks[index].Tool.Name, err) + } + benchmarks[index].Tool.ProvenanceVerified = true } benchmarks[index].Summary = model.Summarize(benchmarks[index].Runs) } @@ -119,3 +154,9 @@ func Benchmark( } return benchmarks, nil } + +func executableIsScript(file *os.File) bool { + var magic [2]byte + count, _ := file.ReadAt(magic[:], 0) + return count == len(magic) && magic == [2]byte{'#', '!'} +} diff --git a/internal/runner/benchmark_linux_test.go b/internal/runner/benchmark_linux_test.go new file mode 100644 index 0000000..78242b9 --- /dev/null +++ b/internal/runner/benchmark_linux_test.go @@ -0,0 +1,21 @@ +package runner + +import ( + "context" + "testing" + "time" +) + +func TestPinnedExecutableDoesNotInflateFileDescriptorCount(t *testing.T) { + benchmarks, err := Benchmark( + context.Background(), + []Spec{{Name: "sleep", Args: []string{"/bin/sleep", "0.05"}}}, + Config{Runs: 1, Interval: time.Millisecond}, Options{}, + ) + if err != nil { + t.Fatal(err) + } + if got := benchmarks[0].Runs[0].PeakFileDescriptors; got > 3 { + t.Fatalf("peak file descriptors = %.0f, want no inherited pin descriptor", got) + } +} diff --git a/internal/runner/benchmark_test.go b/internal/runner/benchmark_test.go index 926a9b6..91e6df5 100644 --- a/internal/runner/benchmark_test.go +++ b/internal/runner/benchmark_test.go @@ -3,6 +3,8 @@ package runner import ( "context" "errors" + "os" + "path/filepath" "testing" "time" ) @@ -33,6 +35,9 @@ func TestBenchmarkInterruptionKeepsCompletedRounds(t *testing.T) { if len(benchmark.Runs) != 1 { t.Fatalf("tool %d recorded %d runs, want 1 completed round", index, len(benchmark.Runs)) } + if benchmark.Tool.SHA256 == "" || benchmark.Tool.ProvenanceVerified { + t.Fatalf("interrupted tool provenance = %+v", benchmark.Tool) + } } } @@ -75,6 +80,55 @@ func TestBenchmarkContinuesAfterNonZeroExit(t *testing.T) { } } +func TestShellBenchmarksInspectDistinctTargetExecutables(t *testing.T) { + benchmarks, err := Benchmark( + context.Background(), + []Spec{{Name: "uname", Shell: "uname"}, {Name: "sleep", Shell: "sleep 0"}}, + Config{Runs: 1, Interval: time.Millisecond}, Options{}, + ) + if err != nil { + t.Fatal(err) + } + if benchmarks[0].Tool.Executable == benchmarks[1].Tool.Executable { + t.Fatalf("shell targets share executable %q", benchmarks[0].Tool.Executable) + } + if !benchmarks[0].Tool.ProvenanceVerified || !benchmarks[1].Tool.ProvenanceVerified { + t.Fatal("completed benchmark should verify executable provenance") + } + if benchmarks[0].Tool.DiskFootprintBytes == benchmarks[1].Tool.DiskFootprintBytes { + t.Fatalf( + "shell targets share disk footprint %d", + benchmarks[0].Tool.DiskFootprintBytes, + ) + } +} + +func TestScriptExecutionPreservesOriginalPath(t *testing.T) { + directory := t.TempDir() + script := filepath.Join(directory, "tool") + if err := os.WriteFile(filepath.Join(directory, "value"), []byte("ok\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + script, []byte("#!/bin/sh\ncat \"$(dirname \"$0\")/value\"\n"), 0o755, + ); err != nil { + t.Fatal(err) + } + benchmarks, err := Benchmark( + context.Background(), + []Spec{{Name: "direct", Args: []string{script}}, {Name: "shell", Shell: script}}, + Config{Runs: 1, Interval: time.Millisecond}, Options{}, + ) + if err != nil { + t.Fatal(err) + } + for _, benchmark := range benchmarks { + if benchmark.Runs[0].ExitCode != 0 { + t.Fatalf("script %s exit code = %d", benchmark.Tool.Name, benchmark.Runs[0].ExitCode) + } + } +} + func TestBenchmarkRejectsInvalidInputs(t *testing.T) { validSpec := Spec{Name: "true", Args: []string{"/bin/true"}} validConfig := Config{Runs: 1, Interval: time.Millisecond} diff --git a/internal/runner/executable_darwin.go b/internal/runner/executable_darwin.go new file mode 100644 index 0000000..4c04b6e --- /dev/null +++ b/internal/runner/executable_darwin.go @@ -0,0 +1,9 @@ +package runner + +import "os" + +func pinnedExecutablePath(*os.File) string { return "/dev/fd/9" } + +func pinnedExtraFiles(file *os.File) []*os.File { + return []*os.File{nil, nil, nil, nil, nil, nil, file} +} diff --git a/internal/runner/executable_linux.go b/internal/runner/executable_linux.go new file mode 100644 index 0000000..9c01528 --- /dev/null +++ b/internal/runner/executable_linux.go @@ -0,0 +1,12 @@ +package runner + +import ( + "fmt" + "os" +) + +func pinnedExecutablePath(file *os.File) string { + return fmt.Sprintf("/proc/%d/fd/%d", os.Getpid(), file.Fd()) +} + +func pinnedExtraFiles(*os.File) []*os.File { return nil } diff --git a/internal/runner/options.go b/internal/runner/options.go index 2ca0306..5a467a5 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -18,6 +18,8 @@ type Config struct { type Options struct { ShowOutput bool Output io.Writer + Input io.Reader + TerminalOut io.Writer TUI bool Duration time.Duration Width uint16 diff --git a/internal/runner/run.go b/internal/runner/run.go index cdde499..6f0a074 100644 --- a/internal/runner/run.go +++ b/internal/runner/run.go @@ -60,6 +60,7 @@ func runOnce( AverageCPUPercent: averageCPUPercent(user, system, elapsed), PeakResidentBytes: float64(peak.ResidentBytes), PeakPhysicalFootprintBytes: float64(peak.PhysicalFootprintBytes), + PhysicalFootprintValid: peak.PhysicalFootprintValid, OSMaxRSSBytes: float64(rusageRSS), MeanResidentBytes: meanResident, PeakVirtualBytes: float64(peak.VirtualBytes), @@ -98,12 +99,18 @@ func monitor( peak.SampleCoverageSeconds += seconds } sample := func() { - if current, valid := platform.SampleTree(pid); valid { - now := time.Now() - addCoverage(now) - updatePeaks(&peak, current) - previousRSS, previousAt = current.ResidentBytes, now + current, valid := platform.SampleTree(pid) + if !valid { + if current.PhysicalFootprintInvalid { + peak.PhysicalFootprintInvalid = true + peak.PhysicalFootprintValid = false + } + return } + now := time.Now() + addCoverage(now) + updatePeaks(&peak, current) + previousRSS, previousAt = current.ResidentBytes, now } if platform.SampleImmediately() { sample() @@ -121,12 +128,19 @@ func monitor( } func updatePeaks(peak *platform.Metrics, current platform.Metrics) { + firstSample := peak.SampleCount == 0 peak.ResidentByteSamples += current.ResidentBytes peak.SampleCount++ peak.ResidentBytes = max(peak.ResidentBytes, current.ResidentBytes) - peak.PhysicalFootprintBytes = max( - peak.PhysicalFootprintBytes, current.PhysicalFootprintBytes, - ) + if current.PhysicalFootprintValid { + peak.PhysicalFootprintBytes = max( + peak.PhysicalFootprintBytes, current.PhysicalFootprintBytes, + ) + } + peak.PhysicalFootprintInvalid = peak.PhysicalFootprintInvalid || + current.PhysicalFootprintInvalid + peak.PhysicalFootprintValid = current.PhysicalFootprintValid && + !peak.PhysicalFootprintInvalid && (firstSample || peak.PhysicalFootprintValid) peak.VirtualBytes = max(peak.VirtualBytes, current.VirtualBytes) peak.Processes = max(peak.Processes, current.Processes) peak.Threads = max(peak.Threads, current.Threads) diff --git a/internal/runner/run_test.go b/internal/runner/run_test.go index d5ff32a..01b98f2 100644 --- a/internal/runner/run_test.go +++ b/internal/runner/run_test.go @@ -52,10 +52,25 @@ func TestMeanResidentUsesObservedTimeWeights(t *testing.T) { } func TestUpdatePeaksTracksPhysicalFootprint(t *testing.T) { - peak := platform.Metrics{PhysicalFootprintBytes: 100} - updatePeaks(&peak, platform.Metrics{PhysicalFootprintBytes: 250}) - updatePeaks(&peak, platform.Metrics{PhysicalFootprintBytes: 200}) + peak := platform.Metrics{PhysicalFootprintBytes: 100, PhysicalFootprintValid: true} + updatePeaks(&peak, platform.Metrics{ + PhysicalFootprintBytes: 250, PhysicalFootprintValid: true, + }) + updatePeaks(&peak, platform.Metrics{ + PhysicalFootprintBytes: 200, PhysicalFootprintValid: true, + }) if peak.PhysicalFootprintBytes != 250 { t.Fatalf("peak physical footprint = %d, want 250", peak.PhysicalFootprintBytes) } } + +func TestUpdatePeaksInvalidatesPartialPhysicalFootprint(t *testing.T) { + var peak platform.Metrics + updatePeaks(&peak, platform.Metrics{ + PhysicalFootprintBytes: 100, PhysicalFootprintValid: true, + }) + updatePeaks(&peak, platform.Metrics{PhysicalFootprintBytes: 200}) + if peak.PhysicalFootprintValid { + t.Fatal("a run with an invalid physical-footprint sample should be unavailable") + } +} diff --git a/internal/runner/spec.go b/internal/runner/spec.go index f11895d..4c6f4ba 100644 --- a/internal/runner/spec.go +++ b/internal/runner/spec.go @@ -2,6 +2,7 @@ package runner import ( "context" + "os" "os/exec" ) @@ -9,11 +10,33 @@ type Spec struct { Name string Args []string Shell string + + executable *os.File + shellTarget bool } func (spec Spec) command(ctx context.Context) *exec.Cmd { + path := "" + if spec.executable != nil { + path = pinnedExecutablePath(spec.executable) + } + var command *exec.Cmd if spec.Shell != "" { - return exec.CommandContext(ctx, "/bin/sh", "-c", spec.Shell) + shell := "/bin/sh" + if spec.executable != nil && !spec.shellTarget { + shell = path + } + command = exec.CommandContext(ctx, shell, "-c", spec.Shell) + command.Args[0] = "/bin/sh" + } else { + if path == "" { + path = spec.Args[0] + } + command = exec.CommandContext(ctx, path, spec.Args[1:]...) + command.Args[0] = spec.Args[0] + } + if spec.executable != nil { + command.ExtraFiles = pinnedExtraFiles(spec.executable) } - return exec.CommandContext(ctx, spec.Args[0], spec.Args[1:]...) + return command } diff --git a/internal/runner/terminal.go b/internal/runner/terminal.go index 5665c6b..568852a 100644 --- a/internal/runner/terminal.go +++ b/internal/runner/terminal.go @@ -6,8 +6,10 @@ import ( "os" "os/signal" "syscall" + "time" "github.com/creack/pty" + "golang.org/x/sys/unix" ) func ResolveTerminalSize(width, height uint16) (uint16, uint16) { @@ -29,28 +31,134 @@ func ResolveTerminalSize(width, height uint16) (uint16, uint16) { return size.Cols, size.Rows } -func terminalSize(width, height uint16) *pty.Winsize { - width, height = ResolveTerminalSize(width, height) - return &pty.Winsize{Cols: width, Rows: height} +func terminalSize(input *os.File, width, height uint16) *pty.Winsize { + if input == nil { + input = os.Stdin + } + size := &pty.Winsize{Cols: 80, Rows: 24} + if current, err := pty.GetsizeFull(input); err == nil { + size = current + } + if width > 0 { + size.Cols = width + } + if height > 0 { + size.Rows = height + } + return size } -func followTerminalResize(ctx context.Context, terminal *os.File) { +func followTerminalResize( + ctx context.Context, input *os.File, terminal *os.File, +) <-chan struct{} { resizes := make(chan os.Signal, 1) + done := make(chan struct{}) signal.Notify(resizes, syscall.SIGWINCH) go func() { + defer close(done) defer signal.Stop(resizes) for { select { case <-resizes: - pty.InheritSize(os.Stdin, terminal) + _ = pty.InheritSize(input, terminal) case <-ctx.Done(): return } } }() + return done } func copyTerminalOutput(writer io.Writer, terminal *os.File, done chan<- struct{}) { defer close(done) - io.Copy(writer, terminal) + _, _ = io.Copy(writer, terminal) +} + +func copyInteractiveTerminalOutput( + ctx context.Context, terminalFD, outputFD int, done chan<- struct{}, +) { + defer close(done) + buffer := make([]byte, 4096) + for { + revents, ok := pollDescriptor(ctx, terminalFD, unix.POLLIN) + if !ok { + return + } + if revents&unix.POLLIN == 0 { + if revents&(unix.POLLHUP|unix.POLLERR|unix.POLLNVAL) != 0 { + return + } + continue + } + count, err := unix.Read(terminalFD, buffer) + if count > 0 && !writeDescriptor(ctx, outputFD, buffer[:count]) { + return + } + if err != nil || count == 0 { + return + } + } +} + +func copyTerminalInput( + ctx context.Context, terminal *os.File, inputFD int, done chan<- struct{}, +) { + defer close(done) + buffer := make([]byte, 4096) + for { + revents, ok := pollDescriptor(ctx, inputFD, unix.POLLIN) + if !ok { + return + } + if revents&unix.POLLIN == 0 { + if revents&(unix.POLLHUP|unix.POLLERR|unix.POLLNVAL) != 0 { + return + } + continue + } + count, err := unix.Read(inputFD, buffer) + if count > 0 && !writeDescriptor(ctx, int(terminal.Fd()), buffer[:count]) { + return + } + if err != nil || count == 0 { + return + } + } +} + +func pollDescriptor(ctx context.Context, fd int, events int16) (int16, bool) { + for { + if ctx.Err() != nil { + return 0, false + } + poll := []unix.PollFd{{Fd: int32(fd), Events: events}} + ready, err := unix.Poll(poll, int((100 * time.Millisecond).Milliseconds())) + if err == unix.EINTR { + continue + } + if err != nil { + return 0, false + } + if ready > 0 { + return poll[0].Revents, true + } + } +} + +func writeDescriptor(ctx context.Context, fd int, data []byte) bool { + for len(data) > 0 { + revents, ok := pollDescriptor(ctx, fd, unix.POLLOUT) + if !ok || revents&(unix.POLLHUP|unix.POLLERR|unix.POLLNVAL) != 0 { + return false + } + count, err := unix.Write(fd, data) + if err == unix.EINTR || err == unix.EAGAIN { + continue + } + if err != nil || count == 0 { + return false + } + data = data[count:] + } + return true } diff --git a/internal/runner/tool.go b/internal/runner/tool.go index 58a185f..c6953f7 100644 --- a/internal/runner/tool.go +++ b/internal/runner/tool.go @@ -1,8 +1,10 @@ package runner import ( + "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "os" @@ -17,25 +19,34 @@ import ( type toolInspector struct { cache map[string]model.ToolInfo hashCache map[string]string + fileCache map[string]os.FileInfo } func newToolInspector() *toolInspector { return &toolInspector{ cache: make(map[string]model.ToolInfo), hashCache: make(map[string]string), + fileCache: make(map[string]os.FileInfo), } } func inspectTool(spec Spec) (model.ToolInfo, error) { - return newToolInspector().inspect(spec) + return newToolInspector().inspect(context.Background(), spec) } -func (inspector *toolInspector) inspect(spec Spec) (model.ToolInfo, error) { +func (inspector *toolInspector) inspect( + ctx context.Context, spec Spec, +) (model.ToolInfo, error) { + if err := ctx.Err(); err != nil { + return model.ToolInfo{}, err + } tool := model.ToolInfo{Name: spec.Name} executable := "/bin/sh" if spec.Shell != "" { tool.Command = []string{spec.Shell} + tool.ShellCommand = true if candidate := shellExecutable(spec.Shell); candidate != "" { executable = candidate + tool.ShellTarget = true } } else { tool.Command = append([]string(nil), spec.Args...) @@ -47,9 +58,14 @@ func (inspector *toolInspector) inspect(spec Spec) (model.ToolInfo, error) { } executable, _ = filepath.Abs(path) } + if resolved, err := filepath.EvalSymlinks(executable); err == nil { + executable = resolved + } tool.Executable = executable if cached, ok := inspector.cache[executable]; ok { cached.Name, cached.Command = tool.Name, tool.Command + cached.ShellCommand = tool.ShellCommand + cached.ShellTarget = tool.ShellTarget return cached, nil } info, err := os.Stat(executable) @@ -59,7 +75,12 @@ func (inspector *toolInspector) inspect(spec Spec) (model.ToolInfo, error) { ) } tool.SizeBytes = info.Size() - for _, dependency := range platform.LinkedFiles(executable) { + inspector.fileCache[executable] = info + dependencies, err := platform.LinkedFiles(ctx, executable) + if err != nil { + return model.ToolInfo{}, err + } + for _, dependency := range dependencies { if dependency.SharedCache { tool.SharedCacheFiles = append(tool.SharedCacheFiles, dependency.Path) continue @@ -78,28 +99,97 @@ func (inspector *toolInspector) inspect(spec Spec) (model.ToolInfo, error) { return tool, nil } -func (inspector *toolInspector) addHash(tool *model.ToolInfo) error { +func (inspector *toolInspector) pin( + ctx context.Context, tool *model.ToolInfo, +) (*os.File, error) { + file, err := os.Open(tool.Executable) + if err != nil { + return nil, err + } + info, err := file.Stat() + if err != nil { + file.Close() + return nil, err + } + if inspected := inspector.fileCache[tool.Executable]; inspected == nil || + !os.SameFile(inspected, info) { + file.Close() + return nil, errors.New("executable changed while being inspected") + } + inspected := inspector.fileCache[tool.Executable] + if inspected.Size() != info.Size() || !inspected.ModTime().Equal(info.ModTime()) { + file.Close() + return nil, errors.New("executable contents changed while being inspected") + } if hash, ok := inspector.hashCache[tool.Executable]; ok { tool.SHA256 = hash - return nil + } else { + hash, hashErr := hashOpenFile(ctx, file) + if hashErr != nil { + file.Close() + return nil, hashErr + } + tool.SHA256 = hash + inspector.hashCache[tool.Executable] = hash } - file, err := os.Open(tool.Executable) + if _, err := file.Seek(0, io.SeekStart); err != nil { + file.Close() + return nil, err + } + return file, nil +} + +func (inspector *toolInspector) verify( + ctx context.Context, tool model.ToolInfo, file *os.File, +) error { + before, ok := inspector.fileCache[tool.Executable] + if !ok { + return errors.New("missing inspected executable identity") + } + after, err := os.Stat(tool.Executable) if err != nil { return err } - defer file.Close() - hash := sha256.New() - if _, err := io.Copy(hash, file); err != nil { + if !os.SameFile(before, after) { + return errors.New("executable was replaced during measurement") + } + if _, err := file.Seek(0, io.SeekStart); err != nil { return err } - tool.SHA256 = hex.EncodeToString(hash.Sum(nil)) - inspector.hashCache[tool.Executable] = tool.SHA256 + hash, err := hashOpenFile(ctx, file) + if err != nil { + return err + } + if hash != tool.SHA256 { + return errors.New("executable changed during measurement") + } return nil } +func hashOpenFile(ctx context.Context, file *os.File) (string, error) { + hash := sha256.New() + buffer := make([]byte, 128*1024) + for { + if err := ctx.Err(); err != nil { + return "", err + } + count, readErr := file.Read(buffer) + if count > 0 { + _, _ = hash.Write(buffer[:count]) + } + if errors.Is(readErr, io.EOF) { + return hex.EncodeToString(hash.Sum(nil)), nil + } + if readErr != nil { + return "", readErr + } + } +} + func shellExecutable(command string) string { fields := strings.Fields(command) - if len(fields) == 0 || strings.ContainsAny(fields[0], "'\"|&;<>()$`") { + if len(fields) == 0 || shellBuiltin(fields[0]) || + strings.ContainsAny(fields[0], "'\"|&;<>()$`") { return "" } path, err := exec.LookPath(fields[0]) @@ -107,8 +197,46 @@ func shellExecutable(command string) string { return "" } absolute, err := filepath.Abs(path) - if err != nil { - return path + if err == nil { + path = absolute + } + if resolved, err := filepath.EvalSymlinks(path); err == nil { + path = resolved + } + return path +} + +func shellBuiltin(name string) bool { + builtins := map[string]bool{ + "!": true, ".": true, ":": true, "[": true, "alias": true, + "bg": true, "break": true, "cd": true, "command": true, + "continue": true, "echo": true, "eval": true, "exec": true, + "exit": true, "export": true, "false": true, "fc": true, + "fg": true, "getopts": true, "hash": true, "jobs": true, + "kill": true, "printf": true, "pwd": true, "read": true, + "readonly": true, "return": true, "set": true, "shift": true, + "test": true, "time": true, "times": true, "trap": true, + "true": true, "type": true, "ulimit": true, "umask": true, + "unalias": true, "unset": true, "wait": true, "{": true, + "builtin": true, "declare": true, "local": true, "source": true, + "typeset": true, + } + return builtins[name] +} + +func pinShellExecutable(command, executable string) (string, bool) { + trimmed := strings.TrimLeft(command, " \t\r\n") + if trimmed == "" { + return command, false + } + end := strings.IndexAny(trimmed, " \t\r\n") + if end < 0 { + end = len(trimmed) + } + if strings.ContainsAny(trimmed[:end], "'\"|&;<>()$`") { + return command, false } - return absolute + prefix := command[:len(command)-len(trimmed)] + quoted := "'" + strings.ReplaceAll(executable, "'", `'"'"'`) + "'" + return prefix + quoted + trimmed[end:], true } diff --git a/internal/runner/tool_test.go b/internal/runner/tool_test.go index e2f076f..dae2e64 100644 --- a/internal/runner/tool_test.go +++ b/internal/runner/tool_test.go @@ -1,19 +1,83 @@ package runner import ( + "context" + "os" "path/filepath" "testing" ) -func TestShellExecutableFindsSimpleCommand(t *testing.T) { - got := shellExecutable("sh -c true") - if filepath.Base(got) != "sh" { - t.Fatalf("executable = %q, want sh", got) +func TestInspectionHonorsCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := newToolInspector().inspect( + ctx, Spec{Name: "true", Args: []string{"/bin/true"}}, + ); err == nil { + t.Fatal("cancelled inspection should fail") } } -func TestShellExecutableRejectsCompoundCommand(t *testing.T) { - if got := shellExecutable("'quoted tool' --flag"); got != "" { - t.Fatalf("executable = %q, want empty", got) +func TestShellInspectionReportsTargetExecutable(t *testing.T) { + tool, err := inspectTool(Spec{Name: "shell", Shell: "sleep 0"}) + if err != nil { + t.Fatal(err) + } + want := shellExecutable("sleep 0") + if tool.Executable != want { + t.Fatalf("shell target executable = %q, want %q", tool.Executable, want) + } +} + +func TestShellInspectionPreservesBuiltins(t *testing.T) { + tool, err := inspectTool(Spec{Name: "shell", Shell: "printf test"}) + if err != nil { + t.Fatal(err) + } + want, err := filepath.EvalSymlinks("/bin/sh") + if err != nil { + t.Fatal(err) + } + if tool.Executable != want || tool.ShellTarget { + t.Fatalf("builtin inspection = %+v, want shell executable", tool) + } +} + +func TestPinShellExecutablePreservesArguments(t *testing.T) { + got, ok := pinShellExecutable(" curl --url 'https://example.com/a b'", "/proc/self/fd/3") + want := " '/proc/self/fd/3' --url 'https://example.com/a b'" + if !ok || got != want { + t.Fatalf("pinned command = %q, %v; want %q, true", got, ok, want) + } +} + +func TestToolVerificationDetectsExecutableReplacement(t *testing.T) { + path := filepath.Join(t.TempDir(), "tool") + data, err := os.ReadFile("/bin/true") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o755); err != nil { + t.Fatal(err) + } + inspector := newToolInspector() + tool, err := inspector.inspect( + context.Background(), Spec{Name: "tool", Args: []string{path}}, + ) + if err != nil { + t.Fatal(err) + } + file, err := inspector.pin(context.Background(), &tool) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if err := os.Rename(path, path+".old"); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o755); err != nil { + t.Fatal(err) + } + if err := inspector.verify(context.Background(), tool, file); err == nil { + t.Fatal("replacement executable should fail provenance verification") } } diff --git a/internal/runner/tui.go b/internal/runner/tui.go index 7cb4cd5..a3f3e2f 100644 --- a/internal/runner/tui.go +++ b/internal/runner/tui.go @@ -22,21 +22,33 @@ func runTUIOnce( interval time.Duration, options Options, ) (model.Run, error) { - if options.Interactive && (!term.IsTerminal(int(os.Stdin.Fd())) || - !term.IsTerminal(int(os.Stdout.Fd()))) { + input := options.Input + if input == nil { + input = os.Stdin + } + output := options.TerminalOut + if output == nil { + output = os.Stdout + } + inputTerminal, inputIsTerminal := input.(interface{ Fd() uintptr }) + outputTerminal, outputIsTerminal := output.(interface{ Fd() uintptr }) + inputFile, inputIsFile := input.(*os.File) + if options.Interactive && (!inputIsTerminal || !outputIsTerminal || !inputIsFile || + !term.IsTerminal(int(inputTerminal.Fd())) || + !term.IsTerminal(int(outputTerminal.Fd()))) { return model.Run{}, errors.New( "interactive TUI mode requires terminal stdin and stdout", ) } - size := terminalSize(options.Width, options.Height) + size := terminalSize(inputFile, options.Width, options.Height) var state *term.State var err error if options.Interactive { - state, err = term.MakeRaw(int(os.Stdin.Fd())) + state, err = term.MakeRaw(int(inputTerminal.Fd())) if err != nil { return model.Run{}, err } - defer term.Restore(int(os.Stdin.Fd()), state) + defer term.Restore(int(inputTerminal.Fd()), state) } cmd := spec.command(ctx) @@ -46,14 +58,23 @@ func runTUIOnce( return model.Run{}, err } defer terminal.Close() - resizeContext, stopResize := context.WithCancel(ctx) - defer stopResize() + inputContext, stopInput := context.WithCancel(ctx) + outputContext, stopOutput := context.WithCancel(ctx) + defer stopInput() + defer stopOutput() outputDone := make(chan struct{}) + var inputDone, resizeDone <-chan struct{} if options.Interactive { - go io.Copy(terminal, os.Stdin) - go copyTerminalOutput(os.Stdout, terminal, outputDone) + inputFinished := make(chan struct{}) + inputDone = inputFinished + go copyTerminalInput( + inputContext, terminal, int(inputTerminal.Fd()), inputFinished, + ) + go copyInteractiveTerminalOutput( + outputContext, int(terminal.Fd()), int(outputTerminal.Fd()), outputDone, + ) if options.FollowResize { - followTerminalResize(resizeContext, terminal) + resizeDone = followTerminalResize(inputContext, inputFile, terminal) } } else { go copyTerminalOutput(io.Discard, terminal, outputDone) @@ -71,6 +92,20 @@ func runTUIOnce( signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) close(stopMonitor) peak := <-monitorDone + stopInput() + if inputDone != nil { + <-inputDone + } + if resizeDone != nil { + <-resizeDone + } + select { + case <-outputDone: + case <-time.After(100 * time.Millisecond): + stopOutput() + _ = terminal.Close() + <-outputDone + } user, system, rusageRSS := platform.ResourceUsage(cmd.ProcessState) peak.Processes = max(peak.Processes, 1) peak.Threads = max(peak.Threads, 1) @@ -84,10 +119,6 @@ func runTUIOnce( !isNonZeroExit(cmd, waitResult.err) { return model.Run{}, waitResult.err } - select { - case <-outputDone: - case <-time.After(100 * time.Millisecond): - } return makeTUIRun( cmd, waitResult.elapsed, user, system, rusageRSS, peak, waitResult.reason, ), nil diff --git a/internal/runner/tui_result.go b/internal/runner/tui_result.go index e964f5e..6c30162 100644 --- a/internal/runner/tui_result.go +++ b/internal/runner/tui_result.go @@ -22,6 +22,7 @@ func makeTUIRun( AverageCPUPercent: averageCPUPercent(user, system, elapsed), PeakResidentBytes: float64(peak.ResidentBytes), PeakPhysicalFootprintBytes: float64(peak.PhysicalFootprintBytes), + PhysicalFootprintValid: peak.PhysicalFootprintValid, OSMaxRSSBytes: float64(rusageRSS), MeanResidentBytes: sampledMeanResident(peak), PeakVirtualBytes: float64(peak.VirtualBytes), diff --git a/internal/runner/tui_test.go b/internal/runner/tui_test.go index d3ad235..9e01742 100644 --- a/internal/runner/tui_test.go +++ b/internal/runner/tui_test.go @@ -2,8 +2,12 @@ package runner import ( "context" + "os" "testing" "time" + + "github.com/creack/pty" + "golang.org/x/sys/unix" ) func TestFixedDurationTUIStopsProcessGroup(t *testing.T) { @@ -74,3 +78,124 @@ func TestTerminalSizeOverridesDimensions(t *testing.T) { t.Fatalf("terminal size = %dx%d, want 132x43", width, height) } } + +func TestTerminalInputPumpStopsWithoutInput(t *testing.T) { + input, inputWriter, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer input.Close() + defer inputWriter.Close() + terminal, terminalReader, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer terminal.Close() + defer terminalReader.Close() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go copyTerminalInput(ctx, terminal, int(input.Fd()), done) + cancel() + select { + case <-done: + case <-time.After(500 * time.Millisecond): + t.Fatal("terminal input pump survived cancellation") + } +} + +func TestInteractiveTUIUsesInjectedTerminalStreams(t *testing.T) { + input, output, err := pty.Open() + if err != nil { + t.Fatal(err) + } + defer input.Close() + defer output.Close() + _, err = runTUIOnce( + context.Background(), Spec{Name: "sleep", Shell: "sleep 30"}, + 5*time.Millisecond, Options{ + TUI: true, Interactive: true, Duration: 30 * time.Millisecond, + Input: input, TerminalOut: output, + }, + ) + if err != nil { + t.Fatal(err) + } +} + +func TestTerminalInputPumpCancelsWhileDestinationIsFull(t *testing.T) { + input, inputWriter, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer input.Close() + defer inputWriter.Close() + terminalReader, terminal, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer terminalReader.Close() + defer terminal.Close() + fillPipe(t, terminal) + if _, err := inputWriter.Write([]byte("input")); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go copyTerminalInput(ctx, terminal, int(input.Fd()), done) + time.Sleep(20 * time.Millisecond) + cancel() + select { + case <-done: + case <-time.After(500 * time.Millisecond): + t.Fatal("terminal input pump blocked on a full destination") + } +} + +func TestTerminalOutputPumpCancelsWhileDestinationIsFull(t *testing.T) { + source, sourceWriter, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer source.Close() + defer sourceWriter.Close() + outputReader, output, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer outputReader.Close() + defer output.Close() + fillPipe(t, output) + if _, err := sourceWriter.Write([]byte("output")); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go copyInteractiveTerminalOutput( + ctx, int(source.Fd()), int(output.Fd()), done, + ) + time.Sleep(20 * time.Millisecond) + cancel() + select { + case <-done: + case <-time.After(500 * time.Millisecond): + t.Fatal("terminal output pump blocked on a full destination") + } +} + +func fillPipe(t *testing.T, pipe *os.File) { + t.Helper() + buffer := []byte{0} + for { + poll := []unix.PollFd{{Fd: int32(pipe.Fd()), Events: unix.POLLOUT}} + ready, err := unix.Poll(poll, 0) + if err != nil { + t.Fatal(err) + } + if ready == 0 { + return + } + if _, err := unix.Write(int(pipe.Fd()), buffer); err != nil { + t.Fatal(err) + } + } +} diff --git a/internal/style/text.go b/internal/style/text.go new file mode 100644 index 0000000..5886958 --- /dev/null +++ b/internal/style/text.go @@ -0,0 +1,20 @@ +package style + +import ( + "strconv" + "strings" + "unicode" +) + +func SafeText(value string) string { + var result strings.Builder + for _, character := range value { + if !unicode.IsControl(character) { + result.WriteRune(character) + continue + } + quoted := strconv.QuoteRuneToGraphic(character) + result.WriteString(quoted[1 : len(quoted)-1]) + } + return result.String() +} From 5a5f1df44516e091e66e5270135b3151671c02b1 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sun, 19 Jul 2026 23:44:12 +0200 Subject: [PATCH 07/11] snailrace v0.0.5: performance improvements Signed-off-by: Polina Simonenko --- internal/analysis/ranking.go | 18 +- internal/analysis/ranking_math.go | 27 ++- internal/app/app.go | 7 +- internal/app/options_test.go | 11 + internal/app/output_bundle.go | 28 ++- internal/app/validation.go | 10 + internal/model/stats.go | 88 +++++--- internal/model/stats_json.go | 8 +- internal/model/stats_test.go | 13 ++ internal/platform/dependencies_darwin.go | 18 +- internal/platform/dependencies_darwin_test.go | 10 + internal/platform/dependencies_linux.go | 100 +++++++-- internal/platform/dependencies_linux_test.go | 55 +++++ internal/platform/sampler_darwin.go | 29 ++- internal/platform/sampler_linux.go | 212 +++++++++++++++--- internal/platform/sampler_linux_bench_test.go | 26 +++ internal/platform/sampler_linux_test.go | 27 +++ internal/report/chart_artifacts.go | 10 +- internal/report/chart_delta.go | 48 ++-- internal/report/chart_distribution.go | 27 ++- internal/report/chart_metrics.go | 21 +- internal/report/chart_ranking.go | 25 ++- internal/report/chart_static.go | 3 + internal/report/chart_test.go | 29 +++ internal/report/chart_tradeoff.go | 16 +- internal/report/chart_trend.go | 36 ++- internal/report/chart_types.go | 51 ++++- internal/report/comparison.go | 18 +- internal/report/comparison_html.go | 12 +- internal/report/comparison_markdown.go | 11 +- internal/report/comparison_test.go | 13 ++ internal/report/comparison_text.go | 11 +- internal/report/format.go | 3 + internal/report/html.go | 21 +- internal/report/markdown.go | 22 +- internal/report/ranking_html.go | 5 +- internal/report/ranking_json.go | 15 +- internal/report/ranking_markdown.go | 3 +- internal/report/ranking_text.go | 3 +- internal/report/render.go | 55 +++++ internal/report/report.go | 17 +- internal/report/report_bench_test.go | 111 +++++++++ internal/report/semantics.go | 6 +- internal/report/svg.go | 20 +- internal/report/text.go | 21 +- internal/report/text_bars.go | 21 +- internal/runner/benchmark_progress.go | 98 +++++++- internal/runner/benchmark_progress_test.go | 62 +++++ internal/runner/benchmark_test.go | 1 + internal/runner/validation.go | 3 + internal/style/palette.go | 6 +- internal/style/palette_test.go | 9 + 52 files changed, 1249 insertions(+), 271 deletions(-) create mode 100644 internal/platform/sampler_linux_bench_test.go create mode 100644 internal/report/render.go create mode 100644 internal/report/report_bench_test.go create mode 100644 internal/style/palette_test.go diff --git a/internal/analysis/ranking.go b/internal/analysis/ranking.go index 2f4648b..f099f8e 100644 --- a/internal/analysis/ranking.go +++ b/internal/analysis/ranking.go @@ -199,15 +199,23 @@ func applyRanks( for _, row := range rows { overall[row.Benchmark] = row.OverallScore } + overallRanks := make([]int, len(eligible)) + if overallAvailable { + overallRanks = ranks(overall, eligible) + } + primaryRanks := ranks(primary, eligible) + cpuRanks := ranks(cpu, eligible) + ramRanks := ranks(ram, eligible) + footprintRanks := ranks(footprint, eligible) for index := range rows { benchmark := rows[index].Benchmark if overallAvailable { - rows[index].OverallRank = rankOf(overall, benchmark, eligible) + rows[index].OverallRank = overallRanks[benchmark] } - rows[index].PrimaryRank = rankOf(primary, benchmark, eligible) - rows[index].CPURank = rankOf(cpu, benchmark, eligible) - rows[index].RAMRank = rankOf(ram, benchmark, eligible) - rows[index].FootprintRank = rankOf(footprint, benchmark, eligible) + rows[index].PrimaryRank = primaryRanks[benchmark] + rows[index].CPURank = cpuRanks[benchmark] + rows[index].RAMRank = ramRanks[benchmark] + rows[index].FootprintRank = footprintRanks[benchmark] } sort.SliceStable(rows, func(i, j int) bool { if !overallAvailable { diff --git a/internal/analysis/ranking_math.go b/internal/analysis/ranking_math.go index d2dbe9c..afdbae5 100644 --- a/internal/analysis/ranking_math.go +++ b/internal/analysis/ranking_math.go @@ -2,6 +2,7 @@ package analysis import ( "math" + "sort" "github.com/shellcell/snailrace/internal/model" ) @@ -107,15 +108,27 @@ func allPositive(values []float64, eligible []bool) bool { return found } -func rankOf(values []float64, target int, eligible []bool) int { - if !eligible[target] || math.IsInf(values[target], 0) || math.IsNaN(values[target]) { - return 0 +func ranks(values []float64, eligible []bool) []int { + type item struct { + index int + value float64 } - rank := 1 + items := make([]item, 0, len(values)) for index, value := range values { - if index != target && eligible[index] && value < values[target] { - rank++ + if eligible[index] && !math.IsInf(value, 0) && !math.IsNaN(value) { + items = append(items, item{index: index, value: value}) } } - return rank + sort.SliceStable(items, func(left, right int) bool { + return items[left].value < items[right].value + }) + result := make([]int, len(values)) + rank := 0 + for position, current := range items { + if position == 0 || current.value != items[position-1].value { + rank = position + 1 + } + result[current.index] = rank + } + return result } diff --git a/internal/app/app.go b/internal/app/app.go index d358c94..a0bd683 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -214,11 +214,14 @@ func writeResult( formats []string, result model.Report, ) error { + renderer := report.NewRenderer(result) var saveErr error if directory != "" { - saveErr = saveReportFormats(stderr, directory, formats, result) + saveErr = saveReportFormatsRenderer( + stderr, directory, formats, result, renderer, + ) } - textErr := report.Write(stdout, "text", result) + textErr := renderer.Write(stdout, "text") return errors.Join(saveErr, textErr) } diff --git a/internal/app/options_test.go b/internal/app/options_test.go index b5fbc87..8d4a159 100644 --- a/internal/app/options_test.go +++ b/internal/app/options_test.go @@ -42,6 +42,17 @@ func TestRejectsLabelCountMismatch(t *testing.T) { } } +func TestRejectsEmptyAndDuplicateLabels(t *testing.T) { + for _, arguments := range [][]string{ + {"-label", " ", "--", "true"}, + {"-label", "same", "-label", "same", "-c", "true", "-c", "false"}, + } { + if _, err := parseOptions(arguments, io.Discard); err == nil { + t.Fatalf("arguments %q should reject ambiguous labels", arguments) + } + } +} + func TestRejectsEmptyShellCommand(t *testing.T) { for _, command := range []string{"", " \t\n"} { if _, err := parseOptions([]string{"-c", command}, io.Discard); err == nil { diff --git a/internal/app/output_bundle.go b/internal/app/output_bundle.go index 84306ed..b04d5e1 100644 --- a/internal/app/output_bundle.go +++ b/internal/app/output_bundle.go @@ -17,6 +17,18 @@ func saveReportFormats( directory string, formats []string, result model.Report, +) error { + return saveReportFormatsRenderer( + stderr, directory, formats, result, report.NewRenderer(result), + ) +} + +func saveReportFormatsRenderer( + stderr io.Writer, + directory string, + formats []string, + result model.Report, + renderer *report.Renderer, ) error { if err := os.MkdirAll(directory, 0o755); err != nil { return err @@ -40,8 +52,8 @@ func saveReportFormats( finalBundle := filepath.Join(directory, stem) if needsCharts { chartDirectory := filepath.Join(stagedBundle, "charts") - charts, err = report.WriteChartFiles( - chartDirectory, result, containsFormat(formats, "svg"), + charts, err = renderer.WriteChartFiles( + chartDirectory, containsFormat(formats, "svg"), ) if err != nil { return err @@ -56,13 +68,13 @@ func saveReportFormats( announcements = append(announcements, filepath.Join(finalBundle, "charts")) case "markdown": path := filepath.Join(stagedBundle, "report.md") - if err := writeMarkdownFile(path, result, charts); err != nil { + if err := writeMarkdownFile(path, renderer, charts); err != nil { return err } announcements = append(announcements, filepath.Join(finalBundle, "report.md")) default: stagedPath := reportPathWithStem(staging, format, stem) - if err := writeReportFile(stagedPath, format, result); err != nil { + if err := writeReportFile(stagedPath, format, renderer); err != nil { return err } finalPath := reportPathWithStem(directory, format, stem) @@ -136,7 +148,7 @@ func reportStemExists(directory, stem string) bool { func writeMarkdownFile( path string, - result model.Report, + renderer *report.Renderer, charts []report.ChartArtifact, ) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { @@ -146,7 +158,7 @@ func writeMarkdownFile( if err != nil { return err } - writeErr := report.WriteMarkdownWithCharts(file, result, charts, "charts") + writeErr := renderer.WriteMarkdownWithCharts(file, charts, "charts") syncErr := error(nil) if writeErr == nil { syncErr = file.Sync() @@ -155,12 +167,12 @@ func writeMarkdownFile( return errors.Join(writeErr, syncErr, closeErr) } -func writeReportFile(path, format string, result model.Report) error { +func writeReportFile(path, format string, renderer *report.Renderer) error { file, err := os.Create(path) if err != nil { return err } - writeErr := report.Write(file, format, result) + writeErr := renderer.Write(file, format) syncErr := error(nil) if writeErr == nil { syncErr = file.Sync() diff --git a/internal/app/validation.go b/internal/app/validation.go index 42f7008..968608f 100644 --- a/internal/app/validation.go +++ b/internal/app/validation.go @@ -83,6 +83,16 @@ func validateLabels(labels, commands []string) error { if len(labels) == 0 { return nil } + seen := make(map[string]bool, len(labels)) + for _, label := range labels { + if strings.TrimSpace(label) == "" { + return errors.New("labels cannot be empty") + } + if seen[label] { + return fmt.Errorf("duplicate label %q", label) + } + seen[label] = true + } if len(commands) > 0 { if len(labels) != len(commands) { return errors.New("label count must match command count") diff --git a/internal/model/stats.go b/internal/model/stats.go index 29fcdba..85eb866 100644 --- a/internal/model/stats.go +++ b/internal/model/stats.go @@ -67,42 +67,80 @@ func Summarize(runs []Run) Summary { func CalculateStats(values []float64) Stats { return stats(values) } +type RunningStats struct { + values []float64 + moments statsMoments +} + +func NewRunningStats(capacity int) RunningStats { + return RunningStats{values: make([]float64, 0, capacity)} +} + +func (running *RunningStats) Add(value float64) { + if math.IsNaN(value) || math.IsInf(value, 0) { + return + } + position := sort.SearchFloat64s(running.values, value) + running.values = append(running.values, 0) + copy(running.values[position+1:], running.values[position:]) + running.values[position] = value + running.moments.add(value) +} + +func (running RunningStats) Snapshot() Stats { + return statsFromSorted(running.values, running.moments) +} + func stats(values []float64) Stats { finite := make([]float64, 0, len(values)) + var moments statsMoments for _, value := range values { if !math.IsNaN(value) && !math.IsInf(value, 0) { finite = append(finite, value) + moments.add(value) } } if len(finite) == 0 { return Stats{} } - sorted := append([]float64(nil), finite...) - sort.Float64s(sorted) - mean := 0.0 - m2 := 0.0 - varianceOverflow := false - for index, value := range sorted { - count := float64(index + 1) - previousMean := mean - mean = previousMean*(1-1/count) + value/count - if math.IsInf(mean, 0) { - mean = math.Copysign(math.MaxFloat64, mean) - } - delta := value - previousMean - term := delta * (value - mean) - if math.IsInf(term, 0) || math.IsNaN(term) || math.MaxFloat64-m2 < term { - varianceOverflow = true - m2 = math.MaxFloat64 - } else if term > 0 { - m2 += term - } + sort.Float64s(finite) + return statsFromSorted(finite, moments) +} + +type statsMoments struct { + count int + mean float64 + m2 float64 + varianceOverflow bool +} + +func (moments *statsMoments) add(value float64) { + moments.count++ + count := float64(moments.count) + previousMean := moments.mean + moments.mean = previousMean*(1-1/count) + value/count + if math.IsInf(moments.mean, 0) { + moments.mean = math.Copysign(math.MaxFloat64, moments.mean) + } + delta := value - previousMean + term := delta * (value - moments.mean) + if math.IsInf(term, 0) || math.IsNaN(term) || math.MaxFloat64-moments.m2 < term { + moments.varianceOverflow = true + moments.m2 = math.MaxFloat64 + } else if term > 0 { + moments.m2 += term + } +} + +func statsFromSorted(sorted []float64, moments statsMoments) Stats { + if len(sorted) == 0 { + return Stats{} } standardDeviation := 0.0 - if len(sorted) > 1 && varianceOverflow { + if len(sorted) > 1 && moments.varianceOverflow { standardDeviation = math.MaxFloat64 } else if len(sorted) > 1 { - standardDeviation = math.Sqrt(m2 / float64(len(sorted)-1)) + standardDeviation = math.Sqrt(moments.m2 / float64(len(sorted)-1)) } margin := 0.0 validInterval := len(sorted) > 1 @@ -114,10 +152,10 @@ func stats(values []float64) Stats { } } return Stats{ - N: len(sorted), Min: sorted[0], Max: sorted[len(sorted)-1], Mean: mean, + N: len(sorted), Min: sorted[0], Max: sorted[len(sorted)-1], Mean: moments.mean, StdDev: standardDeviation, Median: percentile(sorted, 0.5), - P95: percentile(sorted, 0.95), CI95Low: finiteDifference(mean, margin), - CI95High: finiteSum(mean, margin), CI95Valid: validInterval, + P95: percentile(sorted, 0.95), CI95Low: finiteDifference(moments.mean, margin), + CI95High: finiteSum(moments.mean, margin), CI95Valid: validInterval, } } diff --git a/internal/model/stats_json.go b/internal/model/stats_json.go index dd844dd..3722e35 100644 --- a/internal/model/stats_json.go +++ b/internal/model/stats_json.go @@ -29,14 +29,14 @@ func (stats *Stats) UnmarshalJSON(data []byte) error { return err } *stats = Stats(decoded.plain) + if stats.N < 2 { + stats.CI95Valid = false + return nil + } if !stats.CI95Valid { return nil } if decoded.CI95Low == nil || decoded.CI95High == nil { - if stats.N < 2 { - stats.CI95Valid = false - return nil - } margin := stats.StdDev / math.Sqrt(float64(stats.N)) * tCritical95(stats.N-1) stats.CI95Low = finiteDifference(stats.Mean, margin) stats.CI95High = finiteSum(stats.Mean, margin) diff --git a/internal/model/stats_test.go b/internal/model/stats_test.go index 543d7fb..0f09bce 100644 --- a/internal/model/stats_test.go +++ b/internal/model/stats_test.go @@ -114,6 +114,19 @@ func TestStatsUnmarshalAcceptsLegacyConfidenceInterval(t *testing.T) { } } +func TestStatsUnmarshalRejectsSingleObservationInterval(t *testing.T) { + var decoded Stats + if err := json.Unmarshal( + []byte(`{"n":1,"mean":3,"ci95_valid":true,"ci95_low":2,"ci95_high":4}`), + &decoded, + ); err != nil { + t.Fatal(err) + } + if decoded.CI95Valid { + t.Fatal("one observation cannot have a confidence interval") + } +} + func assertClose(t *testing.T, got, want float64) { t.Helper() if math.Abs(got-want) > 1e-9 { diff --git a/internal/platform/dependencies_darwin.go b/internal/platform/dependencies_darwin.go index a0c89ba..e79ff99 100644 --- a/internal/platform/dependencies_darwin.go +++ b/internal/platform/dependencies_darwin.go @@ -54,9 +54,12 @@ func parseOtool(output []byte) []string { lines := strings.Split(string(output), "\n") result := make([]string, 0, len(lines)) for _, line := range lines[1:] { - fields := strings.Fields(line) - if len(fields) > 0 { - result = append(result, fields[0]) + line = strings.TrimSpace(line) + if metadata := strings.LastIndex(line, " (compatibility version "); metadata >= 0 { + line = line[:metadata] + } + if line != "" { + result = append(result, line) } } return result @@ -79,9 +82,12 @@ func loadRPaths(ctx context.Context, path, executable, loader string) []string { if !wantPath || !strings.HasPrefix(line, "path ") { continue } - fields := strings.Fields(line) - if len(fields) >= 2 { - result = append(result, expandDylibPath(fields[1], executable, loader)) + path := strings.TrimSpace(strings.TrimPrefix(line, "path ")) + if metadata := strings.LastIndex(path, " (offset "); metadata >= 0 { + path = path[:metadata] + } + if path != "" { + result = append(result, expandDylibPath(path, executable, loader)) } wantPath = false } diff --git a/internal/platform/dependencies_darwin_test.go b/internal/platform/dependencies_darwin_test.go index 8596a02..9780f1a 100644 --- a/internal/platform/dependencies_darwin_test.go +++ b/internal/platform/dependencies_darwin_test.go @@ -19,6 +19,16 @@ func TestParseOtool(t *testing.T) { } } +func TestParseOtoolPreservesPathsWithSpaces(t *testing.T) { + output := []byte("/tmp/tool:\n" + + "\t/Applications/My Tool/lib helper.dylib " + + "(compatibility version 1.0.0, current version 1.0.0)\n") + want := []string{"/Applications/My Tool/lib helper.dylib"} + if got := parseOtool(output); !reflect.DeepEqual(got, want) { + t.Fatalf("dependencies = %#v, want %#v", got, want) + } +} + func TestResolveDylibUsesRPathAndIdentifiesSharedCache(t *testing.T) { directory := t.TempDir() library := filepath.Join(directory, "liblocal.dylib") diff --git a/internal/platform/dependencies_linux.go b/internal/platform/dependencies_linux.go index ab87925..9d2c449 100644 --- a/internal/platform/dependencies_linux.go +++ b/internal/platform/dependencies_linux.go @@ -2,6 +2,7 @@ package platform import ( "context" + "fmt" "os" "os/exec" "path/filepath" @@ -9,16 +10,16 @@ import ( ) func LinkedFiles(ctx context.Context, executable string) ([]LinkedDependency, error) { - output, _ := exec.CommandContext(ctx, "ldd", executable).CombinedOutput() - if err := ctx.Err(); err != nil { + output, err := runLDD(ctx, executable) + if err != nil { return nil, err } files := parseLDD(output) if len(files) == 0 { - if interpreter := shebangInterpreter(executable); interpreter != "" { + for _, interpreter := range shebangExecutables(executable) { files = append(files, interpreter) - output, _ = exec.CommandContext(ctx, "ldd", interpreter).CombinedOutput() - if err := ctx.Err(); err != nil { + output, err = runLDD(ctx, interpreter) + if err != nil { return nil, err } files = append(files, parseLDD(output)...) @@ -32,15 +33,32 @@ func LinkedFiles(ctx context.Context, executable string) ([]LinkedDependency, er return result, nil } +func runLDD(ctx context.Context, executable string) ([]byte, error) { + output, err := exec.CommandContext(ctx, "ldd", executable).CombinedOutput() + if ctx.Err() != nil { + return nil, ctx.Err() + } + if err == nil { + return output, nil + } + message := strings.ToLower(string(output)) + if strings.Contains(message, "not a dynamic executable") || + strings.Contains(message, "statically linked") { + return output, nil + } + return nil, fmt.Errorf("inspect linked files for %q: %w: %s", + executable, err, strings.TrimSpace(string(output))) +} + func parseLDD(output []byte) []string { var files []string for _, line := range strings.Split(string(output), "\n") { - fields := strings.Fields(line) - candidate := "" - if len(fields) >= 3 && fields[1] == "=>" { - candidate = fields[2] - } else if len(fields) >= 1 && filepath.IsAbs(fields[0]) { - candidate = fields[0] + candidate := strings.TrimSpace(line) + if _, after, found := strings.Cut(candidate, "=>"); found { + candidate = strings.TrimSpace(after) + } + if metadata := strings.LastIndex(candidate, " ("); metadata >= 0 { + candidate = strings.TrimSpace(candidate[:metadata]) } if filepath.IsAbs(candidate) { files = append(files, candidate) @@ -49,17 +67,63 @@ func parseLDD(output []byte) []string { return files } -func shebangInterpreter(path string) string { - data, err := os.ReadFile(path) - if err != nil || !strings.HasPrefix(string(data), "#!") { - return "" +func shebangExecutables(path string) []string { + file, err := os.Open(path) + if err != nil { + return nil } - line, _, _ := strings.Cut(string(data[2:]), "\n") + defer file.Close() + buffer := make([]byte, 4096) + count, err := file.Read(buffer) + if err != nil && count == 0 { + return nil + } + data := string(buffer[:count]) + if !strings.HasPrefix(data, "#!") { + return nil + } + line, _, _ := strings.Cut(data[2:], "\n") fields := strings.Fields(line) if len(fields) == 0 || !filepath.IsAbs(fields[0]) { - return "" + return nil + } + result := []string{fields[0]} + if filepath.Base(fields[0]) != "env" { + return result } - return fields[0] + for index := 1; index < len(fields); index++ { + field := fields[index] + switch field { + case "-u", "--unset", "-C", "--chdir", "-a", "--argv0": + index++ + continue + case "-S", "--split-string", "--": + continue + } + if strings.HasPrefix(field, "-S") && len(field) > 2 { + field = field[2:] + } else if split, found := strings.CutPrefix(field, "--split-string="); found { + field = split + } else if strings.HasPrefix(field, "--unset=") || + strings.HasPrefix(field, "--chdir=") || + strings.HasPrefix(field, "--argv0=") { + continue + } + if strings.HasPrefix(field, "-") || strings.Contains(field, "=") { + continue + } + if split := strings.Fields(field); len(split) > 0 { + field = split[0] + } + if executable, err := exec.LookPath(field); err == nil { + if absolute, absErr := filepath.Abs(executable); absErr == nil { + executable = absolute + } + result = append(result, executable) + } + break + } + return result } func uniqueFiles(paths []string, executable string) []string { diff --git a/internal/platform/dependencies_linux_test.go b/internal/platform/dependencies_linux_test.go index cba74f6..7065fea 100644 --- a/internal/platform/dependencies_linux_test.go +++ b/internal/platform/dependencies_linux_test.go @@ -1,6 +1,9 @@ package platform import ( + "os" + "os/exec" + "path/filepath" "reflect" "testing" ) @@ -19,3 +22,55 @@ libmissing.so => not found t.Fatalf("files = %#v, want %#v", got, want) } } + +func TestParseLDDPreservesPathsWithSpaces(t *testing.T) { + output := []byte("libfoo.so => /tmp/lib dir/libfoo.so (0x123)\n" + + "/tmp/loader dir/ld.so (0x456)\n") + want := []string{"/tmp/lib dir/libfoo.so", "/tmp/loader dir/ld.so"} + if got := parseLDD(output); !reflect.DeepEqual(got, want) { + t.Fatalf("files = %#v, want %#v", got, want) + } +} + +func TestShebangExecutablesResolvesEnvInterpreter(t *testing.T) { + path := filepath.Join(t.TempDir(), "script") + if err := os.WriteFile(path, []byte("#!/usr/bin/env -S sh -eu\n"), 0o700); err != nil { + t.Fatal(err) + } + got := shebangExecutables(path) + sh, err := exec.LookPath("sh") + if err != nil { + t.Fatal(err) + } + sh, _ = filepath.Abs(sh) + want := []string{"/usr/bin/env", sh} + if !reflect.DeepEqual(got, want) { + t.Fatalf("interpreters = %#v, want %#v", got, want) + } +} + +func TestShebangExecutablesSkipsEnvOptionOperands(t *testing.T) { + path := filepath.Join(t.TempDir(), "script") + if err := os.WriteFile( + path, []byte("#!/usr/bin/env -u UNUSED -C /tmp sh -eu\n"), 0o700, + ); err != nil { + t.Fatal(err) + } + got := shebangExecutables(path) + if len(got) != 2 || filepath.Base(got[1]) != "sh" { + t.Fatalf("interpreters = %#v, want env and sh", got) + } +} + +func TestShebangExecutablesHandlesAttachedSplitString(t *testing.T) { + path := filepath.Join(t.TempDir(), "script") + if err := os.WriteFile( + path, []byte("#!/usr/bin/env -a custom --split-string=sh -eu\n"), 0o700, + ); err != nil { + t.Fatal(err) + } + got := shebangExecutables(path) + if len(got) != 2 || filepath.Base(got[1]) != "sh" { + t.Fatalf("interpreters = %#v, want env and sh", got) + } +} diff --git a/internal/platform/sampler_darwin.go b/internal/platform/sampler_darwin.go index f1126ed..9415ead 100644 --- a/internal/platform/sampler_darwin.go +++ b/internal/platform/sampler_darwin.go @@ -43,7 +43,9 @@ func SampleTree(rootPID int) (Metrics, bool) { var usage C.struct_rusage_info_v4 sampled := C.sample_process(pid, &task, &usage) if sampled == 0 { - physicalFootprintValid = false + if syscall.Kill(int(pid), 0) == nil { + physicalFootprintValid = false + } continue } if sampled < 2 { @@ -69,15 +71,20 @@ func processGroupPIDs(groupID int) []C.pid_t { if bytes <= 0 { return nil } - // Leave room for processes created between the sizing and data calls. - count := int(bytes) + 16 - pids := make([]C.pid_t, count) - count = int(C.proc_listpgrppids( - C.pid_t(groupID), unsafe.Pointer(&pids[0]), C.int(len(pids))*C.sizeof_pid_t, - )) - if count <= 0 { - return nil + // Leave room for process churn, and retry if the result fills the buffer. + capacity := int(bytes) + 16 + for attempts := 0; attempts < 3; attempts++ { + pids := make([]C.pid_t, capacity) + count := int(C.proc_listpgrppids( + C.pid_t(groupID), unsafe.Pointer(&pids[0]), C.int(len(pids))*C.sizeof_pid_t, + )) + if count <= 0 { + return nil + } + if count < len(pids) { + return pids[:count] + } + capacity *= 2 } - count = min(count, len(pids)) - return pids[:count] + return nil } diff --git a/internal/platform/sampler_linux.go b/internal/platform/sampler_linux.go index bcc921b..b432f1b 100644 --- a/internal/platform/sampler_linux.go +++ b/internal/platform/sampler_linux.go @@ -1,13 +1,16 @@ package platform import ( + "bytes" "errors" "io" + "math" "os" "path/filepath" "strconv" - "strings" "time" + + "golang.org/x/sys/unix" ) func DefaultInterval() time.Duration { return 10 * time.Millisecond } @@ -15,31 +18,46 @@ func DefaultInterval() time.Duration { return 10 * time.Millisecond } func SampleImmediately() bool { return true } func SampleTree(rootPID int) (Metrics, bool) { - entries, err := os.ReadDir("/proc") + directory, err := unix.Open("/proc", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) if err != nil { return Metrics{}, false } + defer unix.Close(directory) var total Metrics rootFound := false - for _, entry := range entries { - pid, err := strconv.Atoi(entry.Name()) - if err != nil { - continue + statBuffer := make([]byte, 4096) + directoryBuffer := make([]byte, 32*1024) + names := make([]string, 0, 256) + for { + count, readErr := unix.ReadDirent(directory, directoryBuffer) + if readErr != nil { + return Metrics{}, false } - record, err := readProcess(pid) - if err != nil || record.GroupID != rootPID { - continue + if count == 0 { + return total, rootFound } - if record.PID == rootPID { - rootFound = true + _, _, names = unix.ParseDirent(directoryBuffer[:count], -1, names[:0]) + for _, name := range names { + pid, ok := parsePID(name) + if !ok { + continue + } + record, err := readProcessAt( + directory, name, pid, rootPID, statBuffer, + ) + if err != nil || record.GroupID != rootPID { + continue + } + if record.PID == rootPID { + rootFound = true + } + total.ResidentBytes += record.ResidentBytes + total.VirtualBytes += record.VirtualBytes + total.Processes++ + total.Threads += record.Threads + total.FileDescriptors += record.FileDescriptors } - total.ResidentBytes += record.ResidentBytes - total.VirtualBytes += record.VirtualBytes - total.Processes++ - total.Threads += record.Threads - total.FileDescriptors += record.FileDescriptors } - return total, rootFound } func readProcess(pid int) (Process, error) { @@ -48,27 +66,151 @@ func readProcess(pid int) (Process, error) { if err != nil { return Process{}, err } - closeParen := strings.LastIndexByte(string(data), ')') + process, err := parseProcessStat(data, pid) + if err != nil { + return Process{}, err + } + process.FileDescriptors = countDirectory(filepath.Join(base, "fd")) + return process, nil +} + +func readProcessAt( + procFD int, name string, pid, groupID int, buffer []byte, +) (Process, error) { + file, err := unix.Openat(procFD, name+"/stat", unix.O_RDONLY|unix.O_CLOEXEC, 0) + if err != nil { + return Process{}, err + } + count, readErr := unix.Read(file, buffer) + _ = unix.Close(file) + if readErr != nil { + return Process{}, readErr + } + if count == len(buffer) { + return Process{}, errors.New("process stat exceeds buffer") + } + process, err := parseProcessStat(buffer[:count], pid) + if err != nil { + return Process{}, err + } + if process.GroupID == groupID { + process.FileDescriptors = countDirectory("/proc/" + name + "/fd") + } + return process, nil +} + +func parseProcessStat(data []byte, pid int) (Process, error) { + closeParen := bytes.LastIndexByte(data, ')') if closeParen < 0 || closeParen+2 >= len(data) { return Process{}, errors.New("malformed process stat") } - fields := strings.Fields(string(data[closeParen+2:])) - if len(fields) < 22 { - return Process{}, errors.New("short process stat") - } - ppid, _ := strconv.Atoi(fields[1]) - groupID, _ := strconv.Atoi(fields[2]) - threads, _ := strconv.ParseUint(fields[17], 10, 64) - virtual, _ := strconv.ParseUint(fields[20], 10, 64) - rssPages, _ := strconv.ParseInt(fields[21], 10, 64) - if rssPages < 0 { - rssPages = 0 - } - return Process{ - PID: pid, PPID: ppid, GroupID: groupID, Threads: threads, VirtualBytes: virtual, - ResidentBytes: uint64(rssPages) * uint64(os.Getpagesize()), - FileDescriptors: countDirectory(filepath.Join(base, "fd")), - }, nil + fields := data[closeParen+2:] + var process Process + process.PID = pid + field := 0 + for offset := 0; offset < len(fields); { + for offset < len(fields) && (fields[offset] == ' ' || fields[offset] == '\n') { + offset++ + } + start := offset + for offset < len(fields) && fields[offset] != ' ' && fields[offset] != '\n' { + offset++ + } + if start == offset { + break + } + value := fields[start:offset] + var err error + switch field { + case 1: + process.PPID, err = parsePositiveInt(value) + case 2: + process.GroupID, err = parsePositiveInt(value) + case 17: + process.Threads, err = parseUint(value) + case 20: + process.VirtualBytes, err = parseUint(value) + case 21: + var pages int64 + pages, err = parseInt64(value) + if pages > 0 { + pageSize := uint64(os.Getpagesize()) + if uint64(pages) > math.MaxUint64/pageSize { + err = errors.New("resident size overflows") + } else { + process.ResidentBytes = uint64(pages) * pageSize + } + } + } + if err != nil { + return Process{}, err + } + field++ + if field > 21 { + return process, nil + } + } + return Process{}, errors.New("short process stat") +} + +func parsePID(value string) (int, bool) { + if value == "" { + return 0, false + } + result := 0 + for index := range len(value) { + digit := value[index] - '0' + if digit > 9 || result > (math.MaxInt-int(digit))/10 { + return 0, false + } + result = result*10 + int(digit) + } + return result, result > 0 +} + +func parsePositiveInt(value []byte) (int, error) { + parsed, err := parseUint(value) + if err != nil || parsed > uint64(math.MaxInt) { + return 0, errors.New("invalid process integer") + } + return int(parsed), nil +} + +func parseUint(value []byte) (uint64, error) { + if len(value) == 0 { + return 0, errors.New("empty process integer") + } + var result uint64 + for _, character := range value { + digit := character - '0' + if digit > 9 || result > (math.MaxUint64-uint64(digit))/10 { + return 0, errors.New("invalid process integer") + } + result = result*10 + uint64(digit) + } + return result, nil +} + +func parseInt64(value []byte) (int64, error) { + if len(value) == 0 { + return 0, errors.New("empty process integer") + } + negative := value[0] == '-' + if negative { + value = value[1:] + } + parsed, err := parseUint(value) + if err != nil || (!negative && parsed > math.MaxInt64) || + (negative && parsed > uint64(math.MaxInt64)+1) { + return 0, errors.New("invalid process integer") + } + if negative { + if parsed == uint64(math.MaxInt64)+1 { + return math.MinInt64, nil + } + return -int64(parsed), nil + } + return int64(parsed), nil } func countDirectory(path string) uint64 { diff --git a/internal/platform/sampler_linux_bench_test.go b/internal/platform/sampler_linux_bench_test.go new file mode 100644 index 0000000..dedbb74 --- /dev/null +++ b/internal/platform/sampler_linux_bench_test.go @@ -0,0 +1,26 @@ +package platform + +import ( + "os/exec" + "syscall" + "testing" +) + +func BenchmarkSampleTree(b *testing.B) { + command := exec.Command("sleep", "60") + command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := command.Start(); err != nil { + b.Fatal(err) + } + b.Cleanup(func() { + _ = syscall.Kill(-command.Process.Pid, syscall.SIGKILL) + _ = command.Wait() + }) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if _, valid := SampleTree(command.Process.Pid); !valid { + b.Fatal("sampled process disappeared") + } + } +} diff --git a/internal/platform/sampler_linux_test.go b/internal/platform/sampler_linux_test.go index 7ea725b..9b669d4 100644 --- a/internal/platform/sampler_linux_test.go +++ b/internal/platform/sampler_linux_test.go @@ -3,6 +3,7 @@ package platform import ( + "fmt" "os" "os/exec" "path/filepath" @@ -70,3 +71,29 @@ func TestSampleTreeIncludesReparentedProcessGroupMembers(t *testing.T) { t.Fatalf("group metrics = %+v, valid = %v", metrics, valid) } } + +func TestParseProcessStatValidatesRequiredNumbers(t *testing.T) { + fields := make([]string, 22) + for index := range fields { + fields[index] = "0" + } + fields[0], fields[1], fields[2] = "S", "42", "123" + fields[17], fields[20], fields[21] = "7", "4096", "2" + data := []byte(fmt.Sprintf("99 (name with ) parenthesis) %s", strings.Join(fields, " "))) + process, err := parseProcessStat(data, 99) + if err != nil { + t.Fatal(err) + } + if process.PPID != 42 || process.GroupID != 123 || process.Threads != 7 || + process.VirtualBytes != 4096 || process.ResidentBytes == 0 { + t.Fatalf("parsed process = %+v", process) + } + for _, field := range []int{1, 2, 17, 20, 21} { + invalid := append([]string(nil), fields...) + invalid[field] = "invalid" + data = []byte(fmt.Sprintf("99 (name) %s", strings.Join(invalid, " "))) + if _, err := parseProcessStat(data, 99); err == nil { + t.Fatalf("field %d should reject malformed input", field) + } + } +} diff --git a/internal/report/chart_artifacts.go b/internal/report/chart_artifacts.go index 7f3691d..5372dcc 100644 --- a/internal/report/chart_artifacts.go +++ b/internal/report/chart_artifacts.go @@ -20,11 +20,17 @@ func WriteChartFiles( report model.Report, includeCommands bool, ) ([]ChartArtifact, error) { - report = safeDisplayReport(report) + return NewRenderer(report).WriteChartFiles(directory, includeCommands) +} + +func (renderer *Renderer) WriteChartFiles( + directory string, includeCommands bool, +) ([]ChartArtifact, error) { + report := renderer.report if err := os.MkdirAll(directory, 0o755); err != nil { return nil, err } - charts := reportCharts(report) + charts := renderer.reportCharts() if includeCommands { prefix := []svgChart{commandLegendChart(report)} if failures := failureChart(report); failures.height > 0 { diff --git a/internal/report/chart_delta.go b/internal/report/chart_delta.go index 146d7ef..94ca77a 100644 --- a/internal/report/chart_delta.go +++ b/internal/report/chart_delta.go @@ -19,6 +19,11 @@ type forestRow struct { } func deltaForestChart(report model.Report, group chartGroup) svgChart { + return deltaForestChartWith(NewRenderer(report), group) +} + +func deltaForestChartWith(renderer *Renderer, group chartGroup) svgChart { + report := renderer.report baselinePosition := baselineIndex(report) baseline := report.Benchmarks[baselinePosition] rows := make([]forestRow, 0) @@ -35,6 +40,13 @@ func deltaForestChart(report model.Report, group chartGroup) svgChart { } baseStats := metric.stats(baseline) baseMean := baseStats.Mean + if baseStats.N == 0 || !finiteChartValue(baseMean) { + rows = append(rows, forestRow{ + label: metric.name, status: "N/A", class: "neutral", + identity: toolColor(report, baselinePosition), baseline: true, + }) + continue + } baselineRow := forestRow{ label: metric.name + " Β· " + reportToolLabel(report, baselinePosition), status: metric.format(baseMean) + " Β· baseline", @@ -43,8 +55,13 @@ func deltaForestChart(report model.Report, group chartGroup) svgChart { } if baseMean != 0 && baseStats.CI95Valid { baselineRow.low, baselineRow.high = percentInterval(baseStats.CI95Low, baseStats.CI95High, baseMean) - baselineRow.interval = true - maximum = math.Max(maximum, math.Max(math.Abs(baselineRow.low), math.Abs(baselineRow.high))) + baselineRow.interval = finiteChartValue(baselineRow.low) && + finiteChartValue(baselineRow.high) + if baselineRow.interval { + maximum = math.Max( + maximum, math.Max(math.Abs(baselineRow.low), math.Abs(baselineRow.high)), + ) + } } rows = append(rows, baselineRow) if baseMean == 0 { @@ -60,9 +77,7 @@ func deltaForestChart(report model.Report, group chartGroup) svgChart { continue } value := metric.stats(candidate).Mean - delta := compareMetric( - baseline, candidate, row, report.Config.IntervalMS/1000, - ) + delta := renderer.comparison(index, metric.row) rows = append(rows, forestRow{ label: metric.name + " Β· " + candidate.Tool.Name, status: metric.format(value) + " Β· Ξ”% n/a Β· " + delta.status, @@ -83,18 +98,25 @@ func deltaForestChart(report model.Report, group chartGroup) svgChart { }) continue } - delta := compareMetric( - baseline, candidate, row, report.Config.IntervalMS/1000, - ) - low := delta.difference.CI95Low / baseMean * 100 - high := delta.difference.CI95High / baseMean * 100 - maximum = math.Max(maximum, math.Max(math.Abs(low), math.Abs(high))) - maximum = math.Max(maximum, math.Abs(delta.percent)) + delta := renderer.comparison(index, metric.row) + low, high := 0.0, 0.0 + interval := delta.difference.CI95Valid + if interval { + low = delta.difference.CI95Low / baseMean * 100 + high = delta.difference.CI95High / baseMean * 100 + interval = finiteChartValue(low) && finiteChartValue(high) + } + if interval { + maximum = math.Max(maximum, math.Max(math.Abs(low), math.Abs(high))) + } + if delta.percentAvailable { + maximum = math.Max(maximum, math.Abs(delta.percent)) + } rows = append(rows, forestRow{ label: metric.name + " Β· " + candidate.Tool.Name, point: delta.percent, low: low, high: high, status: delta.status, class: delta.class, - identity: toolColor(report, index), interval: delta.difference.CI95Valid, + identity: toolColor(report, index), interval: interval, }) } } diff --git a/internal/report/chart_distribution.go b/internal/report/chart_distribution.go index 4d912c7..46cc21d 100644 --- a/internal/report/chart_distribution.go +++ b/internal/report/chart_distribution.go @@ -21,8 +21,13 @@ func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart minimum, maximum := 0.0, 0.0 for _, benchmark := range report.Benchmarks { stats := metric.stats(benchmark) + if stats.N == 0 || !finiteChartValue(stats.Min) || + !finiteChartValue(stats.Max) || !finiteChartValue(stats.Mean) { + return svgChart{} + } maximum = math.Max(maximum, stats.Max) - if stats.CI95Valid { + if stats.CI95Valid && finiteChartValue(stats.CI95Low) && + finiteChartValue(stats.CI95High) { minimum = math.Min(minimum, stats.CI95Low) maximum = math.Max(maximum, stats.CI95High) } @@ -36,6 +41,11 @@ func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart return left + (value-minimum)/(maximum-minimum)*plotWidth } var body strings.Builder + points := 0 + for _, benchmark := range report.Benchmarks { + points += len(benchmark.Runs) + } + body.Grow(1024 + len(report.Benchmarks)*384 + points*96) body.WriteString(svgChartStyle) fmt.Fprintf( &body, @@ -66,15 +76,18 @@ func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart continue } jitter := (runIndex%3 - 1) * 5 - fmt.Fprintf( - &body, ``, - position(metric.run(run)), y-4+jitter, color, - ) + body.WriteString(``) } stats := metric.stats(benchmark) mean := position(stats.Mean) - if stats.CI95Valid { + if stats.CI95Valid && finiteChartValue(stats.CI95Low) && + finiteChartValue(stats.CI95High) { low, high := position(stats.CI95Low), position(stats.CI95High) fmt.Fprintf( &body, diff --git a/internal/report/chart_metrics.go b/internal/report/chart_metrics.go index e404895..78b4aab 100644 --- a/internal/report/chart_metrics.go +++ b/internal/report/chart_metrics.go @@ -3,13 +3,21 @@ package report import "github.com/shellcell/snailrace/internal/model" func reportCharts(report model.Report) []svgChart { - groups := chartGroups(report) + return NewRenderer(report).reportCharts() +} + +func buildReportCharts(renderer *Renderer) []svgChart { + report := renderer.report + if len(report.Benchmarks) == 0 { + return nil + } + groups := renderer.groups var charts []svgChart if len(report.Benchmarks) > 1 { - charts = append(charts, rankingCharts(report)...) + charts = append(charts, rankingChartsWith(report, renderer.ranking)...) for _, group := range groups { for _, metric := range group.metrics { - chart := deltaForestChart(report, chartGroup{ + chart := deltaForestChartWith(renderer, chartGroup{ name: metric.name, metrics: []chartMetric{metric}, }) if chart.height > 0 { @@ -17,7 +25,7 @@ func reportCharts(report model.Report) []svgChart { } } } - charts = append(charts, tradeoffCharts(report)...) + charts = append(charts, tradeoffCharts(report, renderer.ranking)...) } for _, group := range groups { for _, metric := range group.metrics { @@ -34,7 +42,7 @@ func reportCharts(report model.Report) []svgChart { ); chart.height > 0 { charts = append(charts, chart) } - charts = append(charts, measurementTrendCharts(report)...) + charts = append(charts, measurementTrendCharts(report, groups)...) return charts } @@ -53,7 +61,8 @@ func chartGroups(report model.Report) []chartGroup { } func chartRunAvailable(metric chartMetric, run model.Run) bool { - return metric.row != 8 || run.PhysicalFootprintValid + return (metric.row != 8 || run.PhysicalFootprintValid) && + finiteChartValue(metric.run(run)) } func chartMetricDefinitions() []chartMetric { diff --git a/internal/report/chart_ranking.go b/internal/report/chart_ranking.go index 39ff07f..a19e8dd 100644 --- a/internal/report/chart_ranking.go +++ b/internal/report/chart_ranking.go @@ -18,7 +18,10 @@ type rankingChartMetric struct { } func rankingCharts(report model.Report) []svgChart { - ranking := calculateRanking(report) + return rankingChartsWith(report, calculateRanking(report)) +} + +func rankingChartsWith(report model.Report, ranking rankingData) []svgChart { if len(ranking.rows) == 0 { return nil } @@ -65,12 +68,14 @@ func rankingCharts(report model.Report) []svgChart { func rankByValue(rows []rankingRow, value func(rankingRow) float64) map[int]int { result := make(map[int]int, len(rows)) - for _, row := range rows { - rank := 1 - for _, other := range rows { - if value(other) < value(row) { - rank++ - } + ordered := append([]rankingRow(nil), rows...) + sort.SliceStable(ordered, func(left, right int) bool { + return value(ordered[left]) < value(ordered[right]) + }) + rank := 0 + for position, row := range ordered { + if position == 0 || value(row) != value(ordered[position-1]) { + rank = position + 1 } result[row.benchmark] = rank } @@ -85,7 +90,11 @@ func rankingBarChart( const left, plotWidth, rowHeight = 205, 355, 30 maximum := 0.0 for _, row := range ranking.rows { - maximum = math.Max(maximum, metric.value(row)) + value := metric.value(row) + if !finiteChartValue(value) || value < 0 { + return svgChart{} + } + maximum = math.Max(maximum, value) } if maximum <= 0 { maximum = 1 diff --git a/internal/report/chart_static.go b/internal/report/chart_static.go index 8b65d8f..fba8e7e 100644 --- a/internal/report/chart_static.go +++ b/internal/report/chart_static.go @@ -15,6 +15,9 @@ func staticCostChart( pick func(model.Benchmark) float64, format func(float64) string, ) svgChart { + if len(report.Benchmarks) == 0 { + return svgChart{} + } maximum := 0.0 for _, benchmark := range report.Benchmarks { value := pick(benchmark) diff --git a/internal/report/chart_test.go b/internal/report/chart_test.go index b89208d..8de51b1 100644 --- a/internal/report/chart_test.go +++ b/internal/report/chart_test.go @@ -1,12 +1,41 @@ package report import ( + "math" "strings" "testing" "github.com/shellcell/snailrace/internal/model" ) +func TestEmptyReportBuildsNoCharts(t *testing.T) { + if charts := reportCharts(model.Report{}); len(charts) != 0 { + t.Fatalf("empty report charts = %d, want 0", len(charts)) + } +} + +func TestChartsOmitNonFiniteCoordinates(t *testing.T) { + runs := []model.Run{ + {Index: 1, WallSeconds: math.Inf(1)}, + {Index: 2, WallSeconds: math.NaN()}, + } + report := model.Report{ + Config: model.Config{Baseline: 1, Mode: "command"}, + Host: model.HostInfo{OS: "linux"}, + Benchmarks: []model.Benchmark{{ + Tool: model.ToolInfo{Name: "invalid", DiskFootprintBytes: 1}, + Runs: runs, Summary: model.Summarize(runs), + }}, + } + for _, chart := range reportCharts(report) { + output := chart.html() + if strings.Contains(output, "NaN") || strings.Contains(output, "+Inf") || + strings.Contains(output, "-Inf") { + t.Fatalf("chart %q contains non-finite coordinates", chart.title) + } + } +} + func TestComparisonChartsContainDeltaAndRunViews(t *testing.T) { report := model.Report{ Config: model.Config{Mode: "command", Baseline: 1}, diff --git a/internal/report/chart_tradeoff.go b/internal/report/chart_tradeoff.go index 3ab7a25..43f71c4 100644 --- a/internal/report/chart_tradeoff.go +++ b/internal/report/chart_tradeoff.go @@ -6,7 +6,6 @@ import ( "math" "strings" - "github.com/shellcell/snailrace/internal/analysis" "github.com/shellcell/snailrace/internal/model" ) @@ -16,7 +15,7 @@ type tradeoffMetric struct { value func(model.Benchmark) float64 } -func tradeoffCharts(report model.Report) []svgChart { +func tradeoffCharts(report model.Report, ranking rankingData) []svgChart { if len(report.Benchmarks) < 2 { return nil } @@ -37,13 +36,13 @@ func tradeoffCharts(report model.Report) []svgChart { if mean <= 0 || peak <= 0 { return 0 } - return math.Sqrt(mean * peak) + return math.Sqrt(mean) * math.Sqrt(peak) }} linked := tradeoffMetric{"Linked size", formatBytes, func(b model.Benchmark) float64 { return float64(b.Tool.DiskFootprintBytes) }} pairs := [][2]tradeoffMetric{{primary, cpu}, {primary, linked}} - if analysis.Calculate(report.Config, report.Benchmarks).RAMAvailable { + if ranking.ramAvailable { pairs = append(pairs, [2]tradeoffMetric{primary, ram}, [2]tradeoffMetric{cpu, ram}) } charts := make([]svgChart, 0, len(pairs)) @@ -57,8 +56,13 @@ func tradeoffChart(report model.Report, xMetric, yMetric tradeoffMetric) svgChar const left, top, plotWidth, plotHeight = 62, 62, 390, 240 xMaximum, yMaximum := 0.0, 0.0 for _, benchmark := range report.Benchmarks { - xMaximum = math.Max(xMaximum, xMetric.value(benchmark)) - yMaximum = math.Max(yMaximum, yMetric.value(benchmark)) + xValue, yValue := xMetric.value(benchmark), yMetric.value(benchmark) + if !finiteChartValue(xValue) || !finiteChartValue(yValue) || + xValue < 0 || yValue < 0 { + return svgChart{} + } + xMaximum = math.Max(xMaximum, xValue) + yMaximum = math.Max(yMaximum, yValue) } if xMaximum <= 0 { xMaximum = 1 diff --git a/internal/report/chart_trend.go b/internal/report/chart_trend.go index cb63370..f4d1d09 100644 --- a/internal/report/chart_trend.go +++ b/internal/report/chart_trend.go @@ -16,11 +16,10 @@ type trendLane struct { format func(float64) string } -func measurementTrendCharts(report model.Report) []svgChart { +func measurementTrendCharts(report model.Report, groups []chartGroup) []svgChart { if len(report.Benchmarks) == 0 { return nil } - groups := chartGroups(report) charts := make([]svgChart, 0, len(report.Benchmarks)*len(groups)) for benchmarkIndex, benchmark := range report.Benchmarks { if len(benchmark.Runs) < 2 { @@ -48,6 +47,11 @@ func measurementTrendChart(report model.Report, benchmarkIndex int, group chartG return left + float64(index)/float64(len(runs)-1)*(right-left) } var body strings.Builder + metricCount := 0 + for _, lane := range lanes { + metricCount += len(lane.metrics) + } + body.Grow(1536 + len(runs)*metricCount*96) toolLabel := reportToolLabel(report, benchmarkIndex) toolColorHex := toolColor(report, benchmarkIndex) body.WriteString(svgChartStyle) @@ -92,9 +96,10 @@ func measurementTrendChart(report model.Report, benchmarkIndex int, group chartG minimum, maximum := valueRange(values) color := style.Tool(metric.row).Hex var path strings.Builder + path.Grow(len(values) * 20) started := false for index, value := range values { - if math.IsNaN(value) { + if !finiteChartValue(value) { started = false continue } @@ -102,7 +107,12 @@ func measurementTrendChart(report model.Report, benchmarkIndex int, group chartG if !started { command = "M" } - fmt.Fprintf(&path, "%s %.1f %.1f ", command, x(index), positionY(value)) + path.WriteString(command) + path.WriteByte(' ') + writeChartFloat(&path, x(index)) + path.WriteByte(' ') + writeChartFloat(&path, positionY(value)) + path.WriteByte(' ') started = true } fmt.Fprintf( @@ -111,13 +121,16 @@ func measurementTrendChart(report model.Report, benchmarkIndex int, group chartG path.String(), color, ) for index, value := range values { - if math.IsNaN(value) { + if !finiteChartValue(value) { continue } - fmt.Fprintf( - &body, ``, - x(index), positionY(value), color, - ) + body.WriteString(``) } legendY := top + 4 + metricIndex*26 fmt.Fprintf( @@ -152,7 +165,7 @@ func trendValues(runs []model.Run, metric chartMetric) []float64 { func valueRange(values []float64) (float64, float64) { minimum, maximum := math.Inf(1), math.Inf(-1) for _, value := range values { - if math.IsNaN(value) { + if !finiteChartValue(value) { continue } minimum = math.Min(minimum, value) @@ -248,7 +261,8 @@ func trendMetrics( ) []chartMetric { result := make([]chartMetric, 0, len(metrics)) for _, metric := range metrics { - if availableFor( + stats := metricRows[metric.row].stats(report.Benchmarks[benchmarkIndex].Summary) + if stats.N > 0 && finiteChartValue(stats.Mean) && availableFor( metricRows[metric.row], report.Host.OS, report.Benchmarks[benchmarkIndex].Summary, ) { diff --git a/internal/report/chart_types.go b/internal/report/chart_types.go index 1ffc268..795ca35 100644 --- a/internal/report/chart_types.go +++ b/internal/report/chart_types.go @@ -3,7 +3,9 @@ package report import ( "fmt" "html" + "io" "math" + "strconv" "strings" "unicode" @@ -44,26 +46,38 @@ type chartGroup struct { } func (chart svgChart) html() string { - return fmt.Sprintf( + var output strings.Builder + output.Grow(len(chart.body) + 512) + chart.writeHTML(&output) + return output.String() +} + +func (chart svgChart) writeHTML(writer io.Writer) { + fmt.Fprintf( + writer, `%s%s`, - chartWidth, chart.height, chartWidth, chart.metadata(), chart.body, + `style="max-width:%dpx;font-family:monospace">`, + chartWidth, chart.height, chartWidth, ) + chart.writeMetadata(writer) + fmt.Fprint(writer, chart.body, ``) } -func (chart svgChart) embedded(x, y int) string { - return fmt.Sprintf( +func (chart svgChart) writeEmbedded(writer io.Writer, x, y int) { + fmt.Fprintf( + writer, `%s%s`, + `viewBox="0 0 %d %d">`, x, y, chartWidth, chart.height, chartWidth, chart.height, - chart.metadata(), chart.body, ) + chart.writeMetadata(writer) + fmt.Fprint(writer, chart.body, ``) } -func (chart svgChart) metadata() string { - return fmt.Sprintf( - `%s%s`, +func (chart svgChart) writeMetadata(writer io.Writer) { + fmt.Fprintf( + writer, `%s%s`, html.EscapeString(chart.accessibleTitle()), html.EscapeString(chart.explanation()), ) } @@ -103,7 +117,20 @@ func statusColor(class string) string { } } +func writeChartFloat(output *strings.Builder, value float64) { + buffer := make([]byte, 0, 24) + output.Write(strconv.AppendFloat(buffer, value, 'f', 1, 64)) +} + +func writeChartInt(output *strings.Builder, value int) { + buffer := make([]byte, 0, 12) + output.Write(strconv.AppendInt(buffer, int64(value), 10)) +} + func niceDeltaScale(value float64) float64 { + if math.IsNaN(value) || math.IsInf(value, 0) { + return 1 + } value = math.Max(1, value) power := math.Pow(10, math.Floor(math.Log10(value))) normalized := value / power @@ -119,6 +146,10 @@ func niceDeltaScale(value float64) float64 { } } +func finiteChartValue(value float64) bool { + return !math.IsNaN(value) && !math.IsInf(value, 0) +} + func balanceCharts(charts []svgChart) ([]svgChart, []svgChart) { var left, right []svgChart leftHeight, rightHeight := 0, 0 diff --git a/internal/report/comparison.go b/internal/report/comparison.go index 0b9b631..94c61d9 100644 --- a/internal/report/comparison.go +++ b/internal/report/comparison.go @@ -49,8 +49,11 @@ func compareMetric( candidateMean := model.CalculateStats(candidateValues).Mean result := deltaResult{baselineMean: baseMean, candidateMean: candidateMean} if baseMean != 0 { - result.percent = (candidateMean/baseMean - 1) * 100 - result.percentAvailable = true + percent := (candidateMean/baseMean - 1) * 100 + if finiteChartValue(percent) { + result.percent = percent + result.percentAvailable = true + } } result.difference = model.CalculateStats(differences) result.status, result.class = comparisonStatus(result.difference, row.direction) @@ -76,6 +79,9 @@ func samplingReliable( } func benchmarkSamplesReliable(benchmark model.Benchmark, intervalSeconds float64) bool { + if len(benchmark.Runs) == 0 { + return false + } if intervalSeconds <= 0 { return true } @@ -97,7 +103,10 @@ func pairedValues( baselineByIndex := make(map[int]float64, len(baseline)) for _, run := range baseline { if valid(run) { - baselineByIndex[run.Index] = pick(run) + value := pick(run) + if finiteChartValue(value) { + baselineByIndex[run.Index] = value + } } } baseValues := make([]float64, 0, len(candidate)) @@ -110,6 +119,9 @@ func pairedValues( value, ok := baselineByIndex[run.Index] if ok { candidateValue := pick(run) + if !finiteChartValue(candidateValue) { + continue + } baseValues = append(baseValues, value) candidateValues = append(candidateValues, candidateValue) differences = append(differences, candidateValue-value) diff --git a/internal/report/comparison_html.go b/internal/report/comparison_html.go index 29843dd..179feaf 100644 --- a/internal/report/comparison_html.go +++ b/internal/report/comparison_html.go @@ -8,7 +8,8 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func writeHTMLComparison(writer io.Writer, report model.Report) { +func writeHTMLComparison(writer io.Writer, renderer *Renderer) { + report := renderer.report baselinePosition := baselineIndex(report) baseline := report.Benchmarks[baselinePosition] fmt.Fprintf( @@ -35,10 +36,10 @@ func writeHTMLComparison(writer io.Writer, report model.Report) { `MetricBaseline mean`+ `Candidate meanΞ” mean`) writeHTMLStaticFootprint(writer, baseline, candidate) - for _, row := range metricRows { + for metric, row := range metricRows { writeHTMLComparisonRow( - writer, report.Host.OS, report.Config.IntervalMS/1000, - baseline, candidate, row, + writer, report.Host.OS, baseline, candidate, row, + renderer.comparison(index, metric), ) } fmt.Fprint(writer, "") @@ -72,15 +73,14 @@ func writeHTMLStaticFootprint( func writeHTMLComparisonRow( writer io.Writer, operatingSystem string, - intervalSeconds float64, baseline, candidate model.Benchmark, row metricRow, + delta deltaResult, ) { if !availableFor(row, operatingSystem, baseline.Summary, candidate.Summary) { fmt.Fprintf(writer, "%sN/A", row.name) return } - delta := compareMetric(baseline, candidate, row, intervalSeconds) fmt.Fprintf( writer, `%s%s%s`+ diff --git a/internal/report/comparison_markdown.go b/internal/report/comparison_markdown.go index 6c302a6..03023ac 100644 --- a/internal/report/comparison_markdown.go +++ b/internal/report/comparison_markdown.go @@ -3,11 +3,10 @@ package report import ( "fmt" "io" - - "github.com/shellcell/snailrace/internal/model" ) -func writeMarkdownComparison(writer io.Writer, report model.Report) { +func writeMarkdownComparison(writer io.Writer, renderer *Renderer) { + report := renderer.report baselinePosition := baselineIndex(report) baseline := report.Benchmarks[baselinePosition] fmt.Fprintf( @@ -35,16 +34,14 @@ func writeMarkdownComparison(writer io.Writer, report model.Report) { formatBytes(float64(baseline.Tool.DiskFootprintBytes)), formatBytes(float64(candidate.Tool.DiskFootprintBytes)), staticDelta, ) - for _, row := range metricRows { + for metric, row := range metricRows { if !availableFor( row, report.Host.OS, baseline.Summary, candidate.Summary, ) { fmt.Fprintf(writer, "| %s | N/A | N/A | N/A | N/A |\n", row.name) continue } - delta := compareMetric( - baseline, candidate, row, report.Config.IntervalMS/1000, - ) + delta := renderer.comparison(index, metric) fmt.Fprintf( writer, "| %s | %s | %s | %s | %s |\n", row.name, row.format(delta.baselineMean), row.format(delta.candidateMean), diff --git a/internal/report/comparison_test.go b/internal/report/comparison_test.go index 6ef1f40..5e243ad 100644 --- a/internal/report/comparison_test.go +++ b/internal/report/comparison_test.go @@ -101,6 +101,19 @@ func TestPhysicalFootprintComparisonExcludesInvalidPairs(t *testing.T) { } } +func TestComparisonDropsPairWithNonFiniteValue(t *testing.T) { + baselineRuns := []model.Run{{Index: 1, WallSeconds: 1}} + candidateRuns := []model.Run{{Index: 1, WallSeconds: math.NaN()}} + result := compareMetric( + model.Benchmark{Runs: baselineRuns, Summary: model.Summarize(baselineRuns)}, + model.Benchmark{Runs: candidateRuns, Summary: model.Summarize(candidateRuns)}, + metricRows[0], 0, + ) + if result.difference.N != 0 || result.percentAvailable { + t.Fatalf("non-finite pair produced a comparison: %+v", result) + } +} + func benchmarkWithWallTimes(name string, values ...float64) model.Benchmark { runs := make([]model.Run, len(values)) for index, value := range values { diff --git a/internal/report/comparison_text.go b/internal/report/comparison_text.go index ebf25bf..b272715 100644 --- a/internal/report/comparison_text.go +++ b/internal/report/comparison_text.go @@ -3,11 +3,10 @@ package report import ( "fmt" "io" - - "github.com/shellcell/snailrace/internal/model" ) -func writeTextComparison(writer io.Writer, report model.Report) { +func writeTextComparison(writer io.Writer, renderer *Renderer) { + report := renderer.report baselinePosition := baselineIndex(report) baseline := report.Benchmarks[baselinePosition] for index, candidate := range report.Benchmarks { @@ -31,16 +30,14 @@ func writeTextComparison(writer io.Writer, report model.Report) { formatBytes(float64(baseline.Tool.DiskFootprintBytes)), formatBytes(float64(candidate.Tool.DiskFootprintBytes)), staticDelta, ) - for _, row := range metricRows { + for metric, row := range metricRows { if !availableFor( row, report.Host.OS, baseline.Summary, candidate.Summary, ) { fmt.Fprintf(writer, "%s\tN/A\tN/A\tN/A\tN/A\n", row.name) continue } - delta := compareMetric( - baseline, candidate, row, report.Config.IntervalMS/1000, - ) + delta := renderer.comparison(index, metric) fmt.Fprintf( writer, "%s\t%s\t%s\t%s\t%s\n", row.name, row.format(delta.baselineMean), row.format(delta.candidateMean), diff --git a/internal/report/format.go b/internal/report/format.go index 663d2c0..1bcba1d 100644 --- a/internal/report/format.go +++ b/internal/report/format.go @@ -92,6 +92,9 @@ func formatSignedPercent(value float64) string { } func formatNumber(value float64) string { + if math.IsNaN(value) || math.IsInf(value, 0) { + return "N/A" + } if value == 0 { return "0" } diff --git a/internal/report/html.go b/internal/report/html.go index e2823fe..f008d5e 100644 --- a/internal/report/html.go +++ b/internal/report/html.go @@ -58,6 +58,11 @@ th:first-child,td:first-child{text-align:left}code{color:var(--accent);white-spa
` func writeHTML(writer io.Writer, report model.Report) error { + return writeHTMLRenderer(writer, NewRenderer(report)) +} + +func writeHTMLRenderer(writer io.Writer, renderer *Renderer) error { + report := renderer.report checked := newErrorWriter(writer) writer = checked fmt.Fprint(writer, htmlStart) @@ -70,9 +75,9 @@ func writeHTML(writer io.Writer, report model.Report) error { writeCards(writer, report) fmt.Fprintf( writer, `

Balanced index dimensions actually included: %s.

`, - html.EscapeString(includedDimensionsText(report)), + html.EscapeString(renderer.dimensionText), ) - for _, caveat := range reliabilityCaveats(report) { + for _, caveat := range renderer.caveats { fmt.Fprintf( writer, `

Reliability: %s.

`, html.EscapeString(caveat), @@ -81,14 +86,14 @@ func writeHTML(writer io.Writer, report model.Report) error { writeHTMLCommandLegend(writer, report) writeHTMLFailures(writer, report) if len(report.Benchmarks) > 1 { - writeHTMLRanking(writer, report) + writeHTMLRankingWith(writer, report, renderer.ranking) } - charts := reportCharts(report) + charts := renderer.reportCharts() for _, section := range chartSections(charts) { writeHTMLChartSection(writer, section.title, section.charts) } if len(report.Benchmarks) > 1 { - writeHTMLComparison(writer, report) + writeHTMLComparison(writer, renderer) } fmt.Fprint(writer, "

Statistical detail

") for index, benchmark := range report.Benchmarks { @@ -177,9 +182,11 @@ func writeHTMLChartColumn(writer io.Writer, charts []svgChart) { `
`+ ``+ - `%s
`, - label, explanation, explanation, chart.html(), + ``, + label, explanation, explanation, ) + chart.writeHTML(writer) + fmt.Fprint(writer, ``) } fmt.Fprint(writer, "") } diff --git a/internal/report/markdown.go b/internal/report/markdown.go index 2d9e61b..d0fef62 100644 --- a/internal/report/markdown.go +++ b/internal/report/markdown.go @@ -8,17 +8,21 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func writeMarkdown(writer io.Writer, report model.Report) error { - return WriteMarkdownWithCharts(writer, report, nil, "") -} - func WriteMarkdownWithCharts( writer io.Writer, report model.Report, charts []ChartArtifact, chartDirectory string, ) error { - report = safeDisplayReport(report) + return NewRenderer(report).WriteMarkdownWithCharts(writer, charts, chartDirectory) +} + +func (renderer *Renderer) WriteMarkdownWithCharts( + writer io.Writer, + charts []ChartArtifact, + chartDirectory string, +) error { + report := renderer.report checked := newErrorWriter(writer) writer = checked fmt.Fprintf( @@ -31,19 +35,19 @@ func WriteMarkdownWithCharts( ) fmt.Fprintf( writer, "Balanced index dimensions actually included: **%s**.\n\n", - escapeMarkdown(includedDimensionsText(report)), + escapeMarkdown(renderer.dimensionText), ) - for _, caveat := range reliabilityCaveats(report) { + for _, caveat := range renderer.caveats { fmt.Fprintf(writer, "> Reliability: %s.\n\n", escapeMarkdown(caveat)) } writeMarkdownCommandLegend(writer, report) writeMarkdownFailures(writer, report) if len(report.Benchmarks) > 1 { - writeMarkdownRanking(writer, report) + writeMarkdownRankingWith(writer, report, renderer.ranking) } writeMarkdownCharts(writer, charts, chartDirectory) if len(report.Benchmarks) > 1 { - writeMarkdownComparison(writer, report) + writeMarkdownComparison(writer, renderer) } fmt.Fprintln(writer, "## Statistical Detail") for index, benchmark := range report.Benchmarks { diff --git a/internal/report/ranking_html.go b/internal/report/ranking_html.go index 2461490..dbfd6eb 100644 --- a/internal/report/ranking_html.go +++ b/internal/report/ranking_html.go @@ -10,7 +10,10 @@ import ( ) func writeHTMLRanking(writer io.Writer, report model.Report) { - ranking := calculateRanking(report) + writeHTMLRankingWith(writer, report, calculateRanking(report)) +} + +func writeHTMLRankingWith(writer io.Writer, report model.Report, ranking rankingData) { if len(ranking.rows) == 0 { fmt.Fprintf( writer, `

Ranking unavailable

%s

`, diff --git a/internal/report/ranking_json.go b/internal/report/ranking_json.go index 08cd2f9..b4efb6a 100644 --- a/internal/report/ranking_json.go +++ b/internal/report/ranking_json.go @@ -50,14 +50,19 @@ type jsonRanking struct { } func writeJSON(writer io.Writer, report model.Report) error { + return writeJSONRenderer(writer, NewRenderer(report)) +} + +func writeJSONRenderer(writer io.Writer, renderer *Renderer) error { + report := renderer.raw payload := struct { model.Report Ranking jsonRanking `json:"ranking"` Failures []jsonFailure `json:"failures,omitempty"` Reliability []string `json:"reliability_caveats,omitempty"` }{ - Report: report, Ranking: makeJSONRanking(report), - Failures: makeJSONFailures(report), Reliability: reliabilityCaveats(report), + Report: report, Ranking: makeJSONRanking(report, renderer.ranking), + Failures: makeJSONFailures(report), Reliability: renderer.rawCaveats, } encoder := json.NewEncoder(writer) encoder.SetIndent("", " ") @@ -77,13 +82,13 @@ func makeJSONFailures(report model.Report) []jsonFailure { return result } -func makeJSONRanking(report model.Report) jsonRanking { - ranking := calculateRanking(report) +func makeJSONRanking(report model.Report, ranking rankingData) jsonRanking { result := jsonRanking{ Available: ranking.available, UnavailableReason: ranking.unavailableReason, Method: "equal-weight geometric mean of normalized category costs; included: " + balancedIndexCategories(ranking), - PrimaryMetric: ranking.primaryLabel, IncludedDimensions: includedDimensions(report), + PrimaryMetric: ranking.primaryLabel, + IncludedDimensions: append([]string(nil), includedDimensionsFromRanking(ranking)...), } for _, winner := range ranking.winners { result.Winners = append(result.Winners, jsonWinner{ diff --git a/internal/report/ranking_markdown.go b/internal/report/ranking_markdown.go index 394111f..c44ffdc 100644 --- a/internal/report/ranking_markdown.go +++ b/internal/report/ranking_markdown.go @@ -7,8 +7,7 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func writeMarkdownRanking(writer io.Writer, report model.Report) { - ranking := calculateRanking(report) +func writeMarkdownRankingWith(writer io.Writer, report model.Report, ranking rankingData) { if len(ranking.rows) == 0 { fmt.Fprintf( writer, "## Ranking Unavailable\n\n%s.\n\n", diff --git a/internal/report/ranking_text.go b/internal/report/ranking_text.go index e09a835..1becf17 100644 --- a/internal/report/ranking_text.go +++ b/internal/report/ranking_text.go @@ -8,8 +8,7 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func writeTextRanking(writer io.Writer, report model.Report) { - ranking := calculateRanking(report) +func writeTextRankingWith(writer io.Writer, report model.Report, ranking rankingData) { if len(ranking.rows) == 0 { fmt.Fprintf(writer, "Ranking unavailable\t%s\n\n", ranking.unavailableReason) return diff --git a/internal/report/render.go b/internal/report/render.go new file mode 100644 index 0000000..786c698 --- /dev/null +++ b/internal/report/render.go @@ -0,0 +1,55 @@ +package report + +import "github.com/shellcell/snailrace/internal/model" + +type Renderer struct { + raw model.Report + report model.Report + ranking rankingData + dimensionText string + caveats []string + rawCaveats []string + groups []chartGroup + charts []svgChart + chartsReady bool + comparisons map[comparisonKey]deltaResult +} + +type comparisonKey struct { + candidate int + metric int +} + +func NewRenderer(input model.Report) *Renderer { + display := safeDisplayReport(input) + ranking := calculateRanking(input) + dimensions := includedDimensionsFromRanking(ranking) + return &Renderer{ + raw: input, report: display, ranking: ranking, + dimensionText: dimensionsText(dimensions), + caveats: reliabilityCaveats(display), rawCaveats: reliabilityCaveats(input), + groups: chartGroups(display), comparisons: make(map[comparisonKey]deltaResult), + } +} + +func (renderer *Renderer) comparison(candidate, metric int) deltaResult { + key := comparisonKey{candidate: candidate, metric: metric} + if result, ok := renderer.comparisons[key]; ok { + return result + } + baseline := renderer.report.Benchmarks[baselineIndex(renderer.report)] + result := compareMetric( + baseline, renderer.report.Benchmarks[candidate], metricRows[metric], + renderer.report.Config.IntervalMS/1000, + ) + renderer.comparisons[key] = result + return result +} + +func (renderer *Renderer) reportCharts() []svgChart { + if !renderer.chartsReady { + renderer.charts = buildReportCharts(renderer) + renderer.chartsReady = true + } + return renderer.charts +} diff --git a/internal/report/report.go b/internal/report/report.go index 8fe5454..6bb4470 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -10,20 +10,21 @@ import ( ) func Write(writer io.Writer, format string, report model.Report) error { - if strings.ToLower(format) != "json" { - report = safeDisplayReport(report) - } + return NewRenderer(report).Write(writer, format) +} + +func (renderer *Renderer) Write(writer io.Writer, format string) error { switch strings.ToLower(format) { case "text", "txt": - return writeText(writer, report) + return writeTextRenderer(writer, renderer) case "json": - return writeJSON(writer, report) + return writeJSONRenderer(writer, renderer) case "markdown", "md": - return writeMarkdown(writer, report) + return renderer.WriteMarkdownWithCharts(writer, nil, "") case "html": - return writeHTML(writer, report) + return writeHTMLRenderer(writer, renderer) case "svg": - return writeSVG(writer, report) + return writeSVGRenderer(writer, renderer) default: return fmt.Errorf("unknown report format %q", format) } diff --git a/internal/report/report_bench_test.go b/internal/report/report_bench_test.go new file mode 100644 index 0000000..8c70a7c --- /dev/null +++ b/internal/report/report_bench_test.go @@ -0,0 +1,111 @@ +package report + +import ( + "bytes" + "io" + "testing" + + "github.com/shellcell/snailrace/internal/model" +) + +func TestCachedRendererMatchesIndependentFormats(t *testing.T) { + report := benchmarkReport(3, 4) + renderer := NewRenderer(report) + for _, format := range []string{"html", "svg", "markdown", "json", "text"} { + var cached, independent bytes.Buffer + if err := renderer.Write(&cached, format); err != nil { + t.Fatal(err) + } + if err := Write(&independent, format, report); err != nil { + t.Fatal(err) + } + if cached.String() != independent.String() { + t.Fatalf("cached %s output differs", format) + } + } +} + +func BenchmarkWriteHTML10Tools100Runs(b *testing.B) { + report := benchmarkReport(10, 100) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if err := Write(io.Discard, "html", report); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkReportCharts10Tools100Runs(b *testing.B) { + report := benchmarkReport(10, 100) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if charts := reportCharts(report); len(charts) == 0 { + b.Fatal("no charts") + } + } +} + +func BenchmarkWriteAllFormatsCached10Tools100Runs(b *testing.B) { + report := benchmarkReport(10, 100) + formats := []string{"html", "svg", "markdown", "json", "text"} + b.ReportAllocs() + b.ResetTimer() + for range b.N { + renderer := NewRenderer(report) + for _, format := range formats { + if err := renderer.Write(io.Discard, format); err != nil { + b.Fatal(err) + } + } + } +} + +func BenchmarkWriteAllFormatsIndependent10Tools100Runs(b *testing.B) { + report := benchmarkReport(10, 100) + formats := []string{"html", "svg", "markdown", "json", "text"} + b.ReportAllocs() + b.ResetTimer() + for range b.N { + for _, format := range formats { + if err := Write(io.Discard, format, report); err != nil { + b.Fatal(err) + } + } + } +} + +func benchmarkReport(toolCount, runCount int) model.Report { + benchmarks := make([]model.Benchmark, toolCount) + for tool := range toolCount { + runs := make([]model.Run, runCount) + for run := range runCount { + scale := float64(tool+1) * (1 + float64(run%7)/100) + runs[run] = model.Run{ + Index: run + 1, StopReason: "exited", WallSeconds: scale / 100, + CPUUserSeconds: scale / 200, CPUSystemSeconds: scale / 500, + AverageCPUPercent: 70 + scale, MeanResidentBytes: scale * 1024 * 1024, + PeakResidentBytes: scale * 2 * 1024 * 1024, + OSMaxRSSBytes: scale * 2 * 1024 * 1024, + PeakVirtualBytes: scale * 4 * 1024 * 1024, + PeakProcesses: 2, PeakThreads: 4, PeakFileDescriptors: 8, + SampleCount: 10, SampleCoverageSeconds: scale / 100, + } + } + benchmarks[tool] = model.Benchmark{ + Tool: model.ToolInfo{ + Name: "tool", Command: []string{"tool"}, + DiskFootprintBytes: int64(tool+1) * 1024 * 1024, + }, + Runs: runs, Summary: model.Summarize(runs), + } + } + return model.Report{ + Config: model.Config{ + Runs: runCount, Baseline: 1, Mode: "command", IntervalMS: 10, + IndexDimensions: []string{"time", "cpu", "ram", "disk"}, + }, + Host: model.HostInfo{OS: "linux", LogicalCPUs: 8}, Benchmarks: benchmarks, + } +} diff --git a/internal/report/semantics.go b/internal/report/semantics.go index 89d0a6c..0ba78ed 100644 --- a/internal/report/semantics.go +++ b/internal/report/semantics.go @@ -7,8 +7,7 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func includedDimensions(report model.Report) []string { - ranking := calculateRanking(report) +func includedDimensionsFromRanking(ranking rankingData) []string { var result []string if ranking.indexPrimary { result = append(result, "time") @@ -25,8 +24,7 @@ func includedDimensions(report model.Report) []string { return result } -func includedDimensionsText(report model.Report) string { - dimensions := includedDimensions(report) +func dimensionsText(dimensions []string) string { if len(dimensions) == 0 { return "none" } diff --git a/internal/report/svg.go b/internal/report/svg.go index 6017061..831306d 100644 --- a/internal/report/svg.go +++ b/internal/report/svg.go @@ -22,12 +22,17 @@ const svgReportStyle = `` func writeSVG(writer io.Writer, report model.Report) error { + return writeSVGRenderer(writer, NewRenderer(report)) +} + +func writeSVGRenderer(writer io.Writer, renderer *Renderer) error { + report := renderer.report checked := newErrorWriter(writer) writer = checked var content strings.Builder - y := svgHeader(&content, report) + y := svgHeader(&content, renderer) y = svgCommandLegend(&content, report, y) - charts := reportCharts(report) + charts := renderer.reportCharts() if failures := failureChart(report); failures.height > 0 { charts = append([]svgChart{failures}, charts...) } @@ -45,8 +50,9 @@ func writeSVG(writer io.Writer, report model.Report) error { return checked.Err() } -func svgHeader(output *strings.Builder, report model.Report) int { - caveats := reliabilityCaveats(report) +func svgHeader(output *strings.Builder, renderer *Renderer) int { + report := renderer.report + caveats := renderer.caveats panelHeight := 96 + len(caveats)*22 nextSection := 254 + len(caveats)*22 if report.Config.Mode == "tui" { @@ -82,7 +88,7 @@ func svgHeader(output *strings.Builder, report model.Report) int { ) fmt.Fprintf( output, `balanced dimensions %s`, - html.EscapeString(includedDimensionsText(report)), + html.EscapeString(renderer.dimensionText), ) for index, caveat := range caveats { fmt.Fprintf( @@ -113,11 +119,11 @@ func svgCharts(output *strings.Builder, charts []svgChart, y int) int { left, right := balanceCharts(charts) leftY, rightY := y, y for _, chart := range left { - output.WriteString(chart.embedded(24, leftY)) + chart.writeEmbedded(output, 24, leftY) leftY += chart.height + 16 } for _, chart := range right { - output.WriteString(chart.embedded(756, rightY)) + chart.writeEmbedded(output, 756, rightY) rightY += chart.height + 16 } if rightY > leftY { diff --git a/internal/report/text.go b/internal/report/text.go index 20e1588..1a48aa4 100644 --- a/internal/report/text.go +++ b/internal/report/text.go @@ -12,13 +12,18 @@ import ( ) func writeText(writer io.Writer, report model.Report) error { - if !report.Verbose { - return writeCompactText(writer, report) + return writeTextRenderer(writer, NewRenderer(report)) +} + +func writeTextRenderer(writer io.Writer, renderer *Renderer) error { + if !renderer.report.Verbose { + return writeCompactText(writer, renderer) } - return writeVerboseText(writer, report) + return writeVerboseText(writer, renderer) } -func writeVerboseText(writer io.Writer, report model.Report) error { +func writeVerboseText(writer io.Writer, renderer *Renderer) error { + report := renderer.report var output bytes.Buffer w := tabwriter.NewWriter(&output, 0, 4, 2, ' ', 0) fmt.Fprintf(w, "Snailrace report\t%s\n", report.MeasuredAt.Format( @@ -38,8 +43,8 @@ func writeVerboseText(writer io.Writer, report model.Report) error { report.Config.Mode, report.Config.Runs, report.Config.Warmups, formatNumber(report.Config.IntervalMS), ) - fmt.Fprintf(w, "Included dimensions\t%s\n", includedDimensionsText(report)) - for _, caveat := range reliabilityCaveats(report) { + fmt.Fprintf(w, "Included dimensions\t%s\n", renderer.dimensionText) + for _, caveat := range renderer.caveats { fmt.Fprintf(w, "Reliability\t%s\n", caveat) } if report.Config.Mode == "tui" { @@ -63,11 +68,11 @@ func writeVerboseText(writer io.Writer, report model.Report) error { writeTextCommandLegend(w, report) writeTextFailures(w, report) if len(report.Benchmarks) > 1 { - writeTextRanking(w, report) + writeTextRankingWith(w, report, renderer.ranking) } if len(report.Benchmarks) > 1 { fmt.Fprintln(w, "Detailed baseline deltas\t") - writeTextComparison(w, report) + writeTextComparison(w, renderer) } fmt.Fprintln(w, "Statistical detail\t") for index, benchmark := range report.Benchmarks { diff --git a/internal/report/text_bars.go b/internal/report/text_bars.go index 761f699..bc2049e 100644 --- a/internal/report/text_bars.go +++ b/internal/report/text_bars.go @@ -14,20 +14,21 @@ import ( const compactBarWidth = 20 // writeCompactText renders the short, bar-chart stdout report used by default. -func writeCompactText(writer io.Writer, report model.Report) error { +func writeCompactText(writer io.Writer, renderer *Renderer) error { + report := renderer.report terminal := writerIsTerminal(writer) var body strings.Builder body.WriteString("\n") - fmt.Fprintf(&body, "BALANCED DIMENSIONS %s\n", includedDimensionsText(report)) - for _, caveat := range reliabilityCaveats(report) { + fmt.Fprintf(&body, "BALANCED DIMENSIONS %s\n", renderer.dimensionText) + for _, caveat := range renderer.caveats { fmt.Fprintf(&body, "RELIABILITY %s\n", caveat) } body.WriteString("\n") writeTextFailures(&body, report) if len(report.Benchmarks) == 1 { - writeCompactSingle(&body, report, terminal) + writeCompactSingle(&body, report, renderer.ranking, terminal) } else if len(report.Benchmarks) > 1 { - writeCompactBars(&body, report, terminal) + writeCompactBars(&body, report, renderer.ranking, terminal) } _, err := io.WriteString(writer, body.String()) return err @@ -105,8 +106,9 @@ func cpuCompactUnit(fixedTUI bool, value float64) string { return formatDuration(value) } -func writeCompactBars(body *strings.Builder, report model.Report, terminal bool) { - ranking := calculateRanking(report) +func writeCompactBars( + body *strings.Builder, report model.Report, ranking rankingData, terminal bool, +) { if len(ranking.rows) == 0 { fmt.Fprintf(body, "BALANCED unavailable: %s\n\n", ranking.unavailableReason) return @@ -182,8 +184,9 @@ func writeBarGroup( body.WriteString("\n") } -func writeCompactSingle(body *strings.Builder, report model.Report, terminal bool) { - ranking := calculateRanking(report) +func writeCompactSingle( + body *strings.Builder, report model.Report, ranking rankingData, terminal bool, +) { if len(ranking.rows) == 0 { return } diff --git a/internal/runner/benchmark_progress.go b/internal/runner/benchmark_progress.go index 7f7d560..b17a78c 100644 --- a/internal/runner/benchmark_progress.go +++ b/internal/runner/benchmark_progress.go @@ -14,7 +14,10 @@ type progressTracker struct { tools int fixed time.Duration interval float64 + runs int estimates []ProgressEstimate + summaries []progressSummary + processed []int } func newProgressTracker(options Options, specs []Spec, config Config) *progressTracker { @@ -22,14 +25,18 @@ func newProgressTracker(options Options, specs []Spec, config Config) *progressT callback: options.Progress, started: time.Now(), total: len(specs) * (config.Runs + config.Warmups), tools: len(specs), fixed: options.Duration, + runs: config.Runs, interval: float64(config.Interval) / float64(time.Millisecond), } if tracker.callback == nil { return tracker } tracker.estimates = make([]ProgressEstimate, len(specs)) + tracker.summaries = make([]progressSummary, len(specs)) + tracker.processed = make([]int, len(specs)) for index, spec := range specs { tracker.estimates[index] = ProgressEstimate{ToolName: spec.Name, Total: config.Runs} + tracker.summaries[index] = newProgressSummary(config.Runs) } return tracker } @@ -57,8 +64,16 @@ func (tracker *progressTracker) update( if !warmup { estimate := &tracker.estimates[tool] if len(runs) > 0 && (!estimate.HasEstimate || estimate.Completed != len(runs)) { + if len(runs) < tracker.processed[tool] { + tracker.summaries[tool] = newProgressSummary(tracker.runs) + tracker.processed[tool] = 0 + } + for _, run := range runs[tracker.processed[tool]:] { + tracker.summaries[tool].add(run) + } + tracker.processed[tool] = len(runs) estimate.HasEstimate = true - estimate.Estimate = model.Summarize(runs) + estimate.Estimate = tracker.summaries[tool].snapshot() } estimate.Completed = len(runs) estimate.Runs = runs @@ -84,6 +99,87 @@ func (tracker *progressTracker) update( tracker.callback(event) } +const ( + progressWall = iota + progressCPUTotal + progressCPUUser + progressCPUSystem + progressAverageCPU + progressPeakResident + progressOSMaxRSS + progressMeanResident + progressPeakVirtual + progressPeakProcesses + progressPeakThreads + progressPeakFDs + progressSamples + progressCoverage + progressMetricCount +) + +type progressSummary struct { + metrics [progressMetricCount]model.RunningStats + physical model.RunningStats +} + +func newProgressSummary(capacity int) progressSummary { + var summary progressSummary + for index := range summary.metrics { + summary.metrics[index] = model.NewRunningStats(capacity) + } + summary.physical = model.NewRunningStats(capacity) + return summary +} + +func (summary *progressSummary) add(run model.Run) { + values := [...]float64{ + run.WallSeconds, + run.CPUUserSeconds + run.CPUSystemSeconds, + run.CPUUserSeconds, + run.CPUSystemSeconds, + run.AverageCPUPercent, + run.PeakResidentBytes, + run.OSMaxRSSBytes, + run.MeanResidentBytes, + run.PeakVirtualBytes, + run.PeakProcesses, + run.PeakThreads, + run.PeakFileDescriptors, + float64(run.SampleCount), + run.SampleCoverageSeconds, + } + for index, value := range values { + summary.metrics[index].Add(value) + } + if run.PhysicalFootprintValid { + summary.physical.Add(run.PeakPhysicalFootprintBytes) + } +} + +func (summary progressSummary) snapshot() model.Summary { + result := model.Summary{ + WallSeconds: summary.metrics[progressWall].Snapshot(), + CPUTotalSeconds: summary.metrics[progressCPUTotal].Snapshot(), + CPUUserSeconds: summary.metrics[progressCPUUser].Snapshot(), + CPUSystemSeconds: summary.metrics[progressCPUSystem].Snapshot(), + AverageCPUPercent: summary.metrics[progressAverageCPU].Snapshot(), + PeakResidentBytes: summary.metrics[progressPeakResident].Snapshot(), + OSMaxRSSBytes: summary.metrics[progressOSMaxRSS].Snapshot(), + MeanResidentBytes: summary.metrics[progressMeanResident].Snapshot(), + PeakVirtualBytes: summary.metrics[progressPeakVirtual].Snapshot(), + PeakProcesses: summary.metrics[progressPeakProcesses].Snapshot(), + PeakThreads: summary.metrics[progressPeakThreads].Snapshot(), + PeakFileDescriptors: summary.metrics[progressPeakFDs].Snapshot(), + ValidSampleCount: summary.metrics[progressSamples].Snapshot(), + SampleCoverageSeconds: summary.metrics[progressCoverage].Snapshot(), + } + physical := summary.physical.Snapshot() + if physical.N > 0 { + result.PeakPhysicalFootprintBytes = &physical + } + return result +} + func (tracker *progressTracker) finish() { if tracker.callback != nil { tracker.callback(ProgressEvent{Finished: true}) diff --git a/internal/runner/benchmark_progress_test.go b/internal/runner/benchmark_progress_test.go index 76921fe..13218ca 100644 --- a/internal/runner/benchmark_progress_test.go +++ b/internal/runner/benchmark_progress_test.go @@ -2,8 +2,11 @@ package runner import ( "context" + "reflect" "testing" "time" + + "github.com/shellcell/snailrace/internal/model" ) func TestBenchmarkProgressIncludesRunningEstimate(t *testing.T) { @@ -31,3 +34,62 @@ func TestBenchmarkProgressIncludesRunningEstimate(t *testing.T) { ) } } + +func BenchmarkProgressUpdates1000Runs(b *testing.B) { + runs := make([]model.Run, 1000) + for index := range runs { + runs[index] = model.Run{ + Index: index + 1, WallSeconds: float64(index+1) / 1000, + CPUUserSeconds: 0.01, PeakResidentBytes: float64(index+1) * 1024, + MeanResidentBytes: float64(index+1) * 512, + SampleCount: 10, SampleCoverageSeconds: 0.1, + } + } + b.ReportAllocs() + b.ResetTimer() + for range b.N { + tracker := newProgressTracker( + Options{Progress: func(ProgressEvent) {}}, []Spec{{Name: "tool"}}, + Config{Runs: len(runs), Interval: time.Millisecond}, + ) + for index := range runs { + tracker.update(0, "tool", index+1, len(runs), false, false, runs[:index]) + tracker.update(0, "tool", index+1, len(runs), false, true, runs[:index+1]) + } + } +} + +func TestProgressSummariesMatchFinalStatistics(t *testing.T) { + runs := []model.Run{ + {Index: 1, WallSeconds: 3, CPUUserSeconds: 1, MeanResidentBytes: 30, + PeakPhysicalFootprintBytes: 300, PhysicalFootprintValid: true}, + {Index: 2, WallSeconds: 1, CPUUserSeconds: 2, MeanResidentBytes: 10, + PeakPhysicalFootprintBytes: 100, PhysicalFootprintValid: true}, + {Index: 3, WallSeconds: 2, CPUUserSeconds: 3, MeanResidentBytes: 20, + PeakPhysicalFootprintBytes: 200, PhysicalFootprintValid: true}, + } + var estimates []model.Summary + tracker := newProgressTracker( + Options{Progress: func(event ProgressEvent) { + if event.HasEstimate { + estimates = append(estimates, event.Estimate) + } + }}, + []Spec{{Name: "tool"}}, Config{Runs: len(runs), Interval: time.Millisecond}, + ) + for index := range runs { + tracker.update(0, "tool", index+1, len(runs), false, true, runs[:index+1]) + } + if len(estimates) != len(runs) { + t.Fatalf("estimates = %d, want %d", len(estimates), len(runs)) + } + for index, estimate := range estimates { + want := model.Summarize(runs[:index+1]) + if !reflect.DeepEqual(estimate, want) { + t.Fatalf("estimate %d = %+v, want %+v", index, estimate, want) + } + } + if estimates[0].PeakPhysicalFootprintBytes.Mean != 300 { + t.Fatal("earlier progress event was mutated by a later update") + } +} diff --git a/internal/runner/benchmark_test.go b/internal/runner/benchmark_test.go index 91e6df5..e4b763b 100644 --- a/internal/runner/benchmark_test.go +++ b/internal/runner/benchmark_test.go @@ -139,6 +139,7 @@ func TestBenchmarkRejectsInvalidInputs(t *testing.T) { }{ {"no specs", nil, validConfig}, {"empty spec", []Spec{{Name: "empty"}}, validConfig}, + {"empty name", []Spec{{Args: []string{"true"}}}, validConfig}, {"both command forms", []Spec{{Name: "both", Shell: "true", Args: []string{"true"}}}, validConfig}, {"zero runs", []Spec{validSpec}, Config{Interval: time.Millisecond}}, {"zero interval", []Spec{validSpec}, Config{Runs: 1}}, diff --git a/internal/runner/validation.go b/internal/runner/validation.go index 879e2fb..81537dc 100644 --- a/internal/runner/validation.go +++ b/internal/runner/validation.go @@ -24,6 +24,9 @@ func validateBenchmarkInputs(ctx context.Context, specs []Spec, config Config) e return errors.New("sampling interval must be positive") } for index, spec := range specs { + if strings.TrimSpace(spec.Name) == "" { + return fmt.Errorf("command %d name cannot be blank", index+1) + } if spec.Shell != "" && strings.TrimSpace(spec.Shell) == "" { return fmt.Errorf("command %d shell source cannot be blank", index+1) } diff --git a/internal/style/palette.go b/internal/style/palette.go index fbfca47..8f8d115 100644 --- a/internal/style/palette.go +++ b/internal/style/palette.go @@ -15,5 +15,9 @@ var toolPalette = [...]Color{ } func Tool(index int) Color { - return toolPalette[index%len(toolPalette)] + index %= len(toolPalette) + if index < 0 { + index += len(toolPalette) + } + return toolPalette[index] } diff --git a/internal/style/palette_test.go b/internal/style/palette_test.go new file mode 100644 index 0000000..b3ceaec --- /dev/null +++ b/internal/style/palette_test.go @@ -0,0 +1,9 @@ +package style + +import "testing" + +func TestToolColorHandlesNegativeIndex(t *testing.T) { + if got, want := Tool(-1), Tool(len(toolPalette)-1); got != want { + t.Fatalf("negative color = %+v, want %+v", got, want) + } +} From be4ddfd7f04b1a9ce1444850185a99cab5a0550f Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Mon, 20 Jul 2026 01:00:06 +0200 Subject: [PATCH 08/11] snailrace v0.0.5: refactoring Signed-off-by: Polina Simonenko --- internal/analysis/ranking.go | 25 ++-- internal/analysis/ranking_math.go | 7 +- internal/analysis/ranking_test.go | 30 ++-- internal/app/app.go | 4 +- internal/app/filename.go | 40 +++--- internal/app/filename_test.go | 4 +- internal/app/formats.go | 9 +- internal/app/index.go | 4 +- internal/app/legend.go | 5 +- internal/app/output_bundle.go | 52 +++---- internal/app/output_test.go | 21 ++- internal/app/progress_race.go | 6 +- internal/app/progress_race_test.go | 4 +- internal/model/model.go | 11 ++ internal/model/sampling.go | 18 +++ internal/model/sampling_test.go | 26 ++++ internal/model/stats.go | 54 +++---- internal/model/stats_bench_test.go | 22 +++ internal/model/stats_test.go | 67 +++++++++ internal/model/summary_accumulator.go | 66 +++++++++ internal/platform/sampler_darwin.go | 7 +- internal/platform/sampler_darwin_test.go | 4 +- internal/platform/sampler_linux.go | 133 +++++++++++------- internal/platform/sampler_linux_bench_test.go | 4 +- internal/platform/sampler_linux_test.go | 40 +++++- internal/platform/types.go | 10 +- internal/report/chart_artifacts.go | 12 +- internal/report/chart_commands.go | 2 +- internal/report/chart_delta.go | 56 ++++---- internal/report/chart_distribution.go | 33 +++-- internal/report/chart_metrics.go | 86 ++++------- internal/report/chart_ranking.go | 66 ++++----- internal/report/chart_static.go | 3 +- internal/report/chart_test.go | 56 ++++---- internal/report/chart_tradeoff.go | 8 +- internal/report/chart_trend.go | 78 +++++----- internal/report/chart_types.go | 31 ++-- internal/report/command_legend.go | 34 +---- internal/report/command_legend_svg.go | 2 +- internal/report/command_legend_test.go | 2 +- internal/report/comparison.go | 110 ++++++++------- internal/report/comparison_html.go | 10 +- internal/report/comparison_markdown.go | 8 +- internal/report/comparison_test.go | 16 +-- internal/report/comparison_text.go | 8 +- internal/report/error_writer_test.go | 2 +- internal/report/format.go | 55 +++++--- internal/report/formats.go | 44 ++++++ internal/report/formats_test.go | 29 ++++ internal/report/html.go | 16 +-- internal/report/labels_html.go | 2 +- internal/report/markdown.go | 15 +- internal/report/metric_catalog_test.go | 47 +++++++ internal/report/ranking.go | 25 +--- internal/report/ranking_html.go | 60 ++++---- internal/report/ranking_json.go | 46 +++--- internal/report/ranking_markdown.go | 32 ++--- internal/report/ranking_test.go | 19 +-- internal/report/ranking_text.go | 36 ++--- internal/report/ranking_types.go | 42 ++---- internal/report/ranking_winners.go | 43 +++--- internal/report/render.go | 55 ++++++-- internal/report/report.go | 38 ++--- internal/report/report_bench_test.go | 51 ++++++- internal/report/semantics.go | 10 +- internal/report/svg.go | 16 +-- internal/report/svg_test.go | 7 +- internal/report/text.go | 16 +-- internal/report/text_bars.go | 50 +++---- internal/report/text_test.go | 22 +-- internal/report/ui_test.go | 8 +- internal/runner/benchmark.go | 14 +- internal/runner/benchmark_progress.go | 98 ++----------- internal/runner/benchmark_progress_test.go | 7 + internal/runner/monitor.go | 118 ++++++++++++++++ internal/runner/options.go | 3 +- internal/runner/run.go | 122 ++-------------- internal/runner/run_test.go | 64 ++++++--- internal/runner/spec.go | 14 ++ internal/runner/spec_test.go | 36 +++++ internal/runner/tool.go | 17 ++- internal/runner/tool_test.go | 34 +++++ internal/runner/tui.go | 30 ++-- internal/runner/tui_result.go | 27 ++-- internal/style/command.go | 41 ++++++ internal/style/command_test.go | 14 ++ 86 files changed, 1574 insertions(+), 1145 deletions(-) create mode 100644 internal/model/sampling.go create mode 100644 internal/model/sampling_test.go create mode 100644 internal/model/stats_bench_test.go create mode 100644 internal/model/summary_accumulator.go create mode 100644 internal/report/formats.go create mode 100644 internal/report/formats_test.go create mode 100644 internal/report/metric_catalog_test.go create mode 100644 internal/runner/monitor.go create mode 100644 internal/runner/spec_test.go create mode 100644 internal/style/command.go create mode 100644 internal/style/command_test.go diff --git a/internal/analysis/ranking.go b/internal/analysis/ranking.go index f099f8e..97c2af0 100644 --- a/internal/analysis/ranking.go +++ b/internal/analysis/ranking.go @@ -49,7 +49,7 @@ func Calculate(config model.Config, benchmarks []model.Benchmark) Ranking { result.UnavailableReason = "no benchmarks" return result } - result.FixedTUI = config.Mode == "tui" && config.DurationSeconds > 0 + result.FixedTUI = config.FixedDurationTUI() eligible := make([]bool, count) eligibleCount := 0 primary, cpu := make([]float64, count), make([]float64, count) @@ -142,23 +142,20 @@ func Calculate(config model.Config, benchmarks []model.Benchmark) Ranking { return result } -func AutomaticBaseline(config model.Config, benchmarks []model.Benchmark) (int, bool) { - ranking := Calculate(config, benchmarks) - if !ranking.Available || len(ranking.Rows) == 0 { - return 0, false - } - return ranking.Rows[0].Benchmark + 1, true -} +var indexDimensions = [...]string{"time", "cpu", "ram", "disk"} +var defaultIndexDimensions = [...]string{"time", "cpu", "ram"} -// IndexDimensions are the cost categories that may compose the balanced index. -var IndexDimensions = []string{"time", "cpu", "ram", "disk"} +func IndexDimensions() []string { + return append([]string(nil), indexDimensions[:]...) +} -// DefaultIndexDimensions is the balanced index used when none is configured. -var DefaultIndexDimensions = []string{"time", "cpu", "ram"} +func DefaultIndexDimensions() []string { + return append([]string(nil), defaultIndexDimensions[:]...) +} // ValidIndexDimension reports whether token names a known index dimension. func ValidIndexDimension(token string) bool { - for _, dimension := range IndexDimensions { + for _, dimension := range indexDimensions { if token == dimension { return true } @@ -168,7 +165,7 @@ func ValidIndexDimension(token string) bool { func indexSet(dimensions []string) map[string]bool { if len(dimensions) == 0 { - dimensions = DefaultIndexDimensions + dimensions = defaultIndexDimensions[:] } set := make(map[string]bool, len(dimensions)) for _, dimension := range dimensions { diff --git a/internal/analysis/ranking_math.go b/internal/analysis/ranking_math.go index afdbae5..1b32900 100644 --- a/internal/analysis/ranking_math.go +++ b/internal/analysis/ranking_math.go @@ -75,11 +75,8 @@ func samplesReliable(config model.Config, benchmarks []model.Benchmark, eligible if !eligible[index] { continue } - for _, run := range benchmark.Runs { - if run.WallSeconds < minimum || run.SampleCount < 2 || - run.SampleCoverageSeconds < config.IntervalMS/1000 { - return false - } + if !model.SamplingReliable(benchmark, config.IntervalMS/1000) { + return false } } return true diff --git a/internal/analysis/ranking_test.go b/internal/analysis/ranking_test.go index 07e31a4..62c75fd 100644 --- a/internal/analysis/ranking_test.go +++ b/internal/analysis/ranking_test.go @@ -13,15 +13,17 @@ func TestDefaultIndexExcludesDiskFootprint(t *testing.T) { indexBenchmark("fast-but-fat", 1, 1, 100, 100, 1_000_000), } // Default index (time, cpu, ram) ranks the fast tool first despite its size. - if got, ok := AutomaticBaseline(model.Config{Mode: "command"}, benchmarks); !ok || got != 2 { - t.Fatalf("default baseline = %d, want fast tool (2)", got) + ranking := Calculate(model.Config{Mode: "command"}, benchmarks) + if !ranking.Available || ranking.Rows[0].Benchmark+1 != 2 { + t.Fatalf("default baseline = %d, want fast tool (2)", ranking.Rows[0].Benchmark+1) } // Including disk lets the huge footprint sink the fast tool. withDisk := model.Config{ Mode: "command", IndexDimensions: []string{"time", "cpu", "ram", "disk"}, } - if got, ok := AutomaticBaseline(withDisk, benchmarks); !ok || got != 1 { - t.Fatalf("disk baseline = %d, want compact tool (1)", got) + ranking = Calculate(withDisk, benchmarks) + if !ranking.Available || ranking.Rows[0].Benchmark+1 != 1 { + t.Fatalf("disk baseline = %d, want compact tool (1)", ranking.Rows[0].Benchmark+1) } } @@ -38,9 +40,6 @@ func TestUnavailableRankingUsesExplicitState(t *testing.T) { math.IsInf(ranking.Rows[0].OverallScore, 0) || math.IsNaN(ranking.Rows[0].OverallScore) { t.Fatalf("unavailable row contains invalid ranking values: %+v", ranking.Rows) } - if _, ok := AutomaticBaseline(config, benchmarks); ok { - t.Fatal("unavailable ranking should not select an automatic baseline") - } } func TestFailedBenchmarkIsExcludedFromRanking(t *testing.T) { @@ -55,8 +54,21 @@ func TestFailedBenchmarkIsExcludedFromRanking(t *testing.T) { if !ranking.Available || len(ranking.Rows) != 1 || ranking.Rows[0].Benchmark != 1 { t.Fatalf("failed tool was not excluded: %+v", ranking) } - if baseline, ok := AutomaticBaseline(config, []model.Benchmark{failed, success}); !ok || baseline != 2 { - t.Fatalf("baseline = %d/%v, want successful tool 2", baseline, ok) + if ranking.Rows[0].Benchmark+1 != 2 { + t.Fatalf("baseline = %d, want successful tool 2", ranking.Rows[0].Benchmark+1) + } +} + +func TestDimensionDefaultsCannotBeMutatedByCallers(t *testing.T) { + dimensions := DefaultIndexDimensions() + dimensions[0] = "disk" + if got := DefaultIndexDimensions()[0]; got != "time" { + t.Fatalf("mutated default dimension = %q", got) + } + all := IndexDimensions() + all[0] = "disk" + if got := IndexDimensions()[0]; got != "time" { + t.Fatalf("mutated known dimension = %q", got) } } diff --git a/internal/app/app.go b/internal/app/app.go index a0bd683..1a16b9f 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -217,8 +217,8 @@ func writeResult( renderer := report.NewRenderer(result) var saveErr error if directory != "" { - saveErr = saveReportFormatsRenderer( - stderr, directory, formats, result, renderer, + saveErr = saveReportFormats( + stderr, directory, formats, renderer, ) } textErr := renderer.Write(stdout, "text") diff --git a/internal/app/filename.go b/internal/app/filename.go index da28df4..f5c93a3 100644 --- a/internal/app/filename.go +++ b/internal/app/filename.go @@ -4,43 +4,45 @@ import ( "fmt" "path/filepath" "strings" + "time" "unicode" "github.com/shellcell/snailrace/internal/model" + "github.com/shellcell/snailrace/internal/report" ) -func reportPath(directory, format string, report model.Report) string { - return reportPathWithStem(directory, format, reportStem(report)) -} - func reportPathWithStem(directory, format, stem string) string { extension := strings.ToLower(format) - switch extension { - case "text": - extension = "txt" - case "markdown": - extension = "md" + if info, ok := report.LookupFormat(format); ok { + extension = info.Extension } return filepath.Join(directory, stem+"."+extension) } func reportStem(report model.Report) string { - tool := reportLabel(report.Benchmarks) - timestamp := report.MeasuredAt.Format("20060102-150405") + names := make([]string, len(report.Benchmarks)) + for index, benchmark := range report.Benchmarks { + names[index] = benchmark.Tool.Name + } + return reportStemFor(report.MeasuredAt, names) +} + +func reportStemFor(measuredAt time.Time, toolNames []string) string { + tool := reportLabelNames(toolNames) + timestamp := measuredAt.Format("20060102-150405") return fmt.Sprintf("snail-%s-%s", tool, timestamp) } -func reportLabel(benchmarks []model.Benchmark) string { - if len(benchmarks) == 0 { +func reportLabelNames(names []string) string { + if len(names) == 0 { return "unknown" } - if len(benchmarks) == 1 { - return slug(benchmarks[0].Tool.Name) + if len(names) == 1 { + return slug(names[0]) } - label := slug(benchmarks[0].Tool.Name) + "-vs-" + - slug(benchmarks[1].Tool.Name) - if len(benchmarks) > 2 { - label += fmt.Sprintf("-and-%d", len(benchmarks)-2) + label := slug(names[0]) + "-vs-" + slug(names[1]) + if len(names) > 2 { + label += fmt.Sprintf("-and-%d", len(names)-2) } return label } diff --git a/internal/app/filename_test.go b/internal/app/filename_test.go index 6c636fb..490c7b4 100644 --- a/internal/app/filename_test.go +++ b/internal/app/filename_test.go @@ -15,7 +15,7 @@ func TestReportPathContainsToolAndMeasurementTime(t *testing.T) { Tool: model.ToolInfo{Name: "My Tool!"}, }}, } - got := reportPath("reports", "markdown", report) + got := reportPathWithStem("reports", "markdown", reportStem(report)) want := filepath.Join("reports", "snail-my-tool-20260711-140509.md") if got != want { t.Fatalf("path = %q, want %q", got, want) @@ -31,7 +31,7 @@ func TestComparisonReportPathNamesMeasuredTools(t *testing.T) { {Tool: model.ToolInfo{Name: "awk"}}, }, } - got := filepath.Base(reportPath(".", "json", report)) + got := filepath.Base(reportPathWithStem(".", "json", reportStem(report))) want := "snail-grep-vs-ripgrep-and-1-20260711-140509.json" if got != want { t.Fatalf("name = %q, want %q", got, want) diff --git a/internal/app/formats.go b/internal/app/formats.go index 050df14..1fe4b1b 100644 --- a/internal/app/formats.go +++ b/internal/app/formats.go @@ -1,6 +1,10 @@ package app -import "strings" +import ( + "strings" + + "github.com/shellcell/snailrace/internal/report" +) type formatValues struct { values []string @@ -23,6 +27,9 @@ func (formats *formatValues) Set(value string) error { for _, item := range strings.Split(value, ",") { item = strings.ToLower(strings.TrimSpace(item)) if item != "" { + if info, ok := report.LookupFormat(item); ok { + item = string(info.Name) + } formats.values = append(formats.values, item) } } diff --git a/internal/app/index.go b/internal/app/index.go index e59b874..35dd732 100644 --- a/internal/app/index.go +++ b/internal/app/index.go @@ -13,7 +13,7 @@ type indexValues struct { } func newIndexValues() indexValues { - return indexValues{values: append([]string(nil), analysis.DefaultIndexDimensions...)} + return indexValues{values: analysis.DefaultIndexDimensions()} } func (index *indexValues) String() string { @@ -33,7 +33,7 @@ func (index *indexValues) Set(value string) error { if !analysis.ValidIndexDimension(item) { return fmt.Errorf( "unknown index dimension %q; choose from %s", - item, strings.Join(analysis.IndexDimensions, ", "), + item, strings.Join(analysis.IndexDimensions(), ", "), ) } if !contains(index.values, item) { diff --git a/internal/app/legend.go b/internal/app/legend.go index e0e0f9f..8b65fa6 100644 --- a/internal/app/legend.go +++ b/internal/app/legend.go @@ -10,6 +10,7 @@ import ( "golang.org/x/term" "github.com/shellcell/snailrace/internal/runner" + "github.com/shellcell/snailrace/internal/style" ) // printCommandLegend lists each tool's label and full command before measurement @@ -45,7 +46,7 @@ func printCommandLegend(writer io.Writer, specs []runner.Spec) { func specCommand(spec runner.Spec) string { if spec.Shell != "" { - return spec.Shell + return style.CommandText([]string{spec.Shell}, true) } - return strings.Join(spec.Args, " ") + return style.CommandText(spec.Args, false) } diff --git a/internal/app/output_bundle.go b/internal/app/output_bundle.go index b04d5e1..d8cd4e9 100644 --- a/internal/app/output_bundle.go +++ b/internal/app/output_bundle.go @@ -8,7 +8,6 @@ import ( "path/filepath" "strings" - "github.com/shellcell/snailrace/internal/model" "github.com/shellcell/snailrace/internal/report" ) @@ -16,27 +15,23 @@ func saveReportFormats( stderr io.Writer, directory string, formats []string, - result model.Report, -) error { - return saveReportFormatsRenderer( - stderr, directory, formats, result, report.NewRenderer(result), - ) -} - -func saveReportFormatsRenderer( - stderr io.Writer, - directory string, - formats []string, - result model.Report, renderer *report.Renderer, ) error { if err := os.MkdirAll(directory, 0o755); err != nil { return err } formats = uniqueFormats(formats) - needsCharts := containsFormat(formats, "svg") || - containsFormat(formats, "markdown") - stem, release, err := reserveReportStem(directory, result) + needsCharts := false + includeCommandCharts := false + for _, format := range formats { + info, _ := report.LookupFormat(format) + needsCharts = needsCharts || info.NeedsCharts + includeCommandCharts = includeCommandCharts || info.Name == report.FormatSVG + } + metadata := renderer.Metadata() + stem, release, err := reserveReportStem( + directory, reportStemFor(metadata.MeasuredAt, metadata.ToolNames), + ) if err != nil { return err } @@ -53,7 +48,7 @@ func saveReportFormatsRenderer( if needsCharts { chartDirectory := filepath.Join(stagedBundle, "charts") charts, err = renderer.WriteChartFiles( - chartDirectory, containsFormat(formats, "svg"), + chartDirectory, includeCommandCharts, ) if err != nil { return err @@ -105,9 +100,8 @@ func saveReportFormatsRenderer( func reserveReportStem( directory string, - result model.Report, + base string, ) (string, func(), error) { - base := reportStem(result) for suffix := 1; ; suffix++ { stem := base if suffix > 1 { @@ -185,12 +179,11 @@ func uniqueFormats(formats []string) []string { seen := make(map[string]bool) result := make([]string, 0, len(formats)) for _, format := range formats { - format = strings.ToLower(format) - switch format { - case "txt": - format = "text" - case "md": - format = "markdown" + info, ok := report.LookupFormat(format) + if !ok { + format = strings.ToLower(strings.TrimSpace(format)) + } else { + format = string(info.Name) } if !seen[format] { seen[format] = true @@ -200,15 +193,6 @@ func uniqueFormats(formats []string) []string { return result } -func containsFormat(formats []string, target string) bool { - for _, format := range formats { - if format == target { - return true - } - } - return false -} - func announce(writer io.Writer, path string) error { _, err := fmt.Fprintf(writer, "Report saved to %s\n", path) return err diff --git a/internal/app/output_test.go b/internal/app/output_test.go index b4159b5..6c31457 100644 --- a/internal/app/output_test.go +++ b/internal/app/output_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/shellcell/snailrace/internal/model" + reportpkg "github.com/shellcell/snailrace/internal/report" ) func TestSavingAlsoWritesCompleteReportToStdout(t *testing.T) { @@ -106,7 +107,7 @@ func TestMarkdownAndSVGSavedAsChartBundle(t *testing.T) { {Tool: model.ToolInfo{Name: "second"}, Runs: runs, Summary: model.Summarize(runs)}, }, } - if err := saveReportFormats( + if err := saveReportFormatsForTest( &stderr, directory, []string{"markdown", "svg"}, result, ); err != nil { t.Fatal(err) @@ -197,7 +198,7 @@ func TestSavingSameReportUsesUniqueNames(t *testing.T) { wait.Add(1) go func() { defer wait.Done() - errors <- saveReportFormats(io.Discard, directory, []string{"html"}, result) + errors <- saveReportFormatsForTest(io.Discard, directory, []string{"html"}, result) }() } wait.Wait() @@ -222,10 +223,10 @@ func TestSavingDifferentFormatsDoesNotReuseExistingStem(t *testing.T) { Config: model.Config{Mode: "command", Baseline: 1}, Benchmarks: []model.Benchmark{{Tool: model.ToolInfo{Name: "same"}}}, } - if err := saveReportFormats(io.Discard, directory, []string{"html"}, result); err != nil { + if err := saveReportFormatsForTest(io.Discard, directory, []string{"html"}, result); err != nil { t.Fatal(err) } - if err := saveReportFormats(io.Discard, directory, []string{"markdown"}, result); err != nil { + if err := saveReportFormatsForTest(io.Discard, directory, []string{"markdown"}, result); err != nil { t.Fatal(err) } entries, err := os.ReadDir(directory) @@ -243,7 +244,7 @@ func TestFailedSavePublishesNoPartialReports(t *testing.T) { Config: model.Config{Mode: "command", Baseline: 1}, Benchmarks: []model.Benchmark{{Tool: model.ToolInfo{Name: "tool"}}}, } - if err := saveReportFormats( + if err := saveReportFormatsForTest( io.Discard, directory, []string{"html", "unknown"}, result, ); err == nil { t.Fatal("invalid staged format should fail") @@ -289,7 +290,7 @@ func TestSavedSVGBundleIncludesCommandFailures(t *testing.T) { Runs: []model.Run{failedRun}, Summary: model.Summarize([]model.Run{failedRun}), }}, } - if err := saveReportFormats(io.Discard, directory, []string{"svg"}, result); err != nil { + if err := saveReportFormatsForTest(io.Discard, directory, []string{"svg"}, result); err != nil { t.Fatal(err) } entries, err := os.ReadDir(directory) @@ -317,3 +318,11 @@ func TestFormatAliasesAreCanonicalizedBeforeSaving(t *testing.T) { t.Fatalf("canonical formats = %v", formats) } } + +func saveReportFormatsForTest( + stderr io.Writer, directory string, formats []string, result model.Report, +) error { + return saveReportFormats( + stderr, directory, formats, reportpkg.NewRenderer(result), + ) +} diff --git a/internal/app/progress_race.go b/internal/app/progress_race.go index 4d3c044..2741273 100644 --- a/internal/app/progress_race.go +++ b/internal/app/progress_race.go @@ -78,14 +78,14 @@ func progressScores(event runner.ProgressEvent) []float64 { benchmarks := make([]model.Benchmark, 0, len(event.Estimates)) positions := make([]int, 0, len(event.Estimates)) for index, tool := range event.Estimates { - if !tool.HasEstimate { + if !tool.HasEstimate || tool.Estimate == nil { continue } benchmarks = append(benchmarks, model.Benchmark{ Tool: model.ToolInfo{ Name: tool.ToolName, DiskFootprintBytes: tool.DiskFootprintBytes, }, - Runs: tool.Runs, Summary: tool.Estimate, + Runs: tool.Runs, Summary: *tool.Estimate, }) positions = append(positions, index) } @@ -105,7 +105,7 @@ func progressTrailCharacters(tools []runner.ProgressEstimate) []string { minimum := math.Inf(1) values := make([]float64, len(tools)) for index, tool := range tools { - if !tool.HasEstimate { + if !tool.HasEstimate || tool.Estimate == nil { continue } mean := tool.Estimate.MeanResidentBytes.Mean diff --git a/internal/app/progress_race_test.go b/internal/app/progress_race_test.go index 7cdbd6f..5b3deb9 100644 --- a/internal/app/progress_race_test.go +++ b/internal/app/progress_race_test.go @@ -18,7 +18,7 @@ var progressColorSequence = regexp.MustCompile(`\x1b\[[0-9;]*m`) func TestProgressRaceAdvancesBetterExpectedTool(t *testing.T) { event := runner.ProgressEvent{ - Completed: 2, Total: 4, Elapsed: time.Second, ETA: time.Second, + Completed: 2, Total: 4, ETA: time.Second, Estimates: []runner.ProgressEstimate{ progressEstimate("better", 1), progressEstimate("worse", 2), @@ -202,7 +202,7 @@ func progressEstimate(name string, scale float64) runner.ProgressEstimate { MeanResidentBytes: scale, PeakResidentBytes: scale, }}, DiskFootprintBytes: int64(scale * 100), - Estimate: model.Summary{ + Estimate: &model.Summary{ WallSeconds: stats, CPUTotalSeconds: stats, MeanResidentBytes: stats, PeakResidentBytes: stats, }, diff --git a/internal/model/model.go b/internal/model/model.go index 1b15696..c664ed4 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -22,6 +22,17 @@ type Config struct { OutputMode string `json:"output_mode"` } +const ( + ModeCommand = "command" + ModeTUI = "tui" +) + +func (config Config) IsTUI() bool { return config.Mode == ModeTUI } + +func (config Config) FixedDurationTUI() bool { + return config.IsTUI() && config.DurationSeconds > 0 +} + type HostInfo struct { OS string `json:"os"` Architecture string `json:"architecture"` diff --git a/internal/model/sampling.go b/internal/model/sampling.go new file mode 100644 index 0000000..8e199a3 --- /dev/null +++ b/internal/model/sampling.go @@ -0,0 +1,18 @@ +package model + +func SamplingReliable(benchmark Benchmark, intervalSeconds float64) bool { + if len(benchmark.Runs) == 0 { + return false + } + if intervalSeconds <= 0 { + return true + } + minimumDuration := intervalSeconds * 2 + for _, run := range benchmark.Runs { + if run.WallSeconds < minimumDuration || run.SampleCount < 2 || + run.SampleCoverageSeconds < intervalSeconds { + return false + } + } + return true +} diff --git a/internal/model/sampling_test.go b/internal/model/sampling_test.go new file mode 100644 index 0000000..89f146a --- /dev/null +++ b/internal/model/sampling_test.go @@ -0,0 +1,26 @@ +package model + +import "testing" + +func TestSamplingReliableBoundaries(t *testing.T) { + interval := 0.01 + tests := []struct { + name string + runs []Run + want bool + }{ + {"no runs", nil, false}, + {"reliable", []Run{{WallSeconds: 0.02, SampleCount: 2, SampleCoverageSeconds: 0.01}}, true}, + {"short", []Run{{WallSeconds: 0.019, SampleCount: 2, SampleCoverageSeconds: 0.01}}, false}, + {"few samples", []Run{{WallSeconds: 0.02, SampleCount: 1, SampleCoverageSeconds: 0.01}}, false}, + {"low coverage", []Run{{WallSeconds: 0.02, SampleCount: 2, SampleCoverageSeconds: 0.009}}, false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + benchmark := Benchmark{Runs: test.runs} + if got := SamplingReliable(benchmark, interval); got != test.want { + t.Fatalf("reliable = %v, want %v", got, test.want) + } + }) + } +} diff --git a/internal/model/stats.go b/internal/model/stats.go index 85eb866..8ffb80f 100644 --- a/internal/model/stats.go +++ b/internal/model/stats.go @@ -8,8 +8,8 @@ import ( func Summarize(runs []Run) Summary { values := func(pick func(Run) float64) []float64 { result := make([]float64, len(runs)) - for i, run := range runs { - result[i] = pick(run) + for index, run := range runs { + result[index] = pick(run) } return result } @@ -25,42 +25,28 @@ func Summarize(runs []Run) Summary { physicalStats = &value } return Summary{ - WallSeconds: stats(values(func(r Run) float64 { return r.WallSeconds })), - CPUTotalSeconds: stats(values(func(r Run) float64 { - return r.CPUUserSeconds + r.CPUSystemSeconds - })), - CPUUserSeconds: stats(values(func(r Run) float64 { - return r.CPUUserSeconds - })), - CPUSystemSeconds: stats(values(func(r Run) float64 { - return r.CPUSystemSeconds - })), - AverageCPUPercent: stats(values(func(r Run) float64 { - return r.AverageCPUPercent - })), - PeakResidentBytes: stats(values(func(r Run) float64 { - return r.PeakResidentBytes + WallSeconds: stats(values(func(run Run) float64 { return run.WallSeconds })), + CPUTotalSeconds: stats(values(func(run Run) float64 { + return run.CPUUserSeconds + run.CPUSystemSeconds })), + CPUUserSeconds: stats(values(func(run Run) float64 { return run.CPUUserSeconds })), + CPUSystemSeconds: stats(values(func(run Run) float64 { return run.CPUSystemSeconds })), + AverageCPUPercent: stats(values(func(run Run) float64 { return run.AverageCPUPercent })), + PeakResidentBytes: stats(values(func(run Run) float64 { return run.PeakResidentBytes })), PeakPhysicalFootprintBytes: physicalStats, - OSMaxRSSBytes: stats(values(func(r Run) float64 { - return r.OSMaxRSSBytes - })), - MeanResidentBytes: stats(values(func(r Run) float64 { - return r.MeanResidentBytes - })), - PeakVirtualBytes: stats(values(func(r Run) float64 { - return r.PeakVirtualBytes - })), - PeakProcesses: stats(values(func(r Run) float64 { return r.PeakProcesses })), - PeakThreads: stats(values(func(r Run) float64 { return r.PeakThreads })), - PeakFileDescriptors: stats(values(func(r Run) float64 { - return r.PeakFileDescriptors + OSMaxRSSBytes: stats(values(func(run Run) float64 { return run.OSMaxRSSBytes })), + MeanResidentBytes: stats(values(func(run Run) float64 { return run.MeanResidentBytes })), + PeakVirtualBytes: stats(values(func(run Run) float64 { return run.PeakVirtualBytes })), + PeakProcesses: stats(values(func(run Run) float64 { return run.PeakProcesses })), + PeakThreads: stats(values(func(run Run) float64 { return run.PeakThreads })), + PeakFileDescriptors: stats(values(func(run Run) float64 { + return run.PeakFileDescriptors })), - ValidSampleCount: stats(values(func(r Run) float64 { - return float64(r.SampleCount) + ValidSampleCount: stats(values(func(run Run) float64 { + return float64(run.SampleCount) })), - SampleCoverageSeconds: stats(values(func(r Run) float64 { - return r.SampleCoverageSeconds + SampleCoverageSeconds: stats(values(func(run Run) float64 { + return run.SampleCoverageSeconds })), } } diff --git a/internal/model/stats_bench_test.go b/internal/model/stats_bench_test.go new file mode 100644 index 0000000..61a68b1 --- /dev/null +++ b/internal/model/stats_bench_test.go @@ -0,0 +1,22 @@ +package model + +import "testing" + +func BenchmarkSummarize1000Runs(b *testing.B) { + runs := make([]Run, 1000) + for index := range runs { + value := float64((index * 7919) % len(runs)) + runs[index] = Run{ + WallSeconds: value, CPUUserSeconds: value, CPUSystemSeconds: value, + AverageCPUPercent: value, PeakResidentBytes: value, + OSMaxRSSBytes: value, MeanResidentBytes: value, PeakVirtualBytes: value, + PeakProcesses: value, PeakThreads: value, PeakFileDescriptors: value, + SampleCount: index, SampleCoverageSeconds: value, + } + } + b.ReportAllocs() + b.ResetTimer() + for range b.N { + _ = Summarize(runs) + } +} diff --git a/internal/model/stats_test.go b/internal/model/stats_test.go index 0f09bce..f6958fe 100644 --- a/internal/model/stats_test.go +++ b/internal/model/stats_test.go @@ -3,6 +3,7 @@ package model import ( "encoding/json" "math" + "reflect" "strings" "testing" ) @@ -127,6 +128,72 @@ func TestStatsUnmarshalRejectsSingleObservationInterval(t *testing.T) { } } +func TestSummaryAccumulatorMatchesBatchSummaryAtEveryPrefix(t *testing.T) { + runs := []Run{ + {WallSeconds: 1, CPUUserSeconds: 2, CPUSystemSeconds: 3, + AverageCPUPercent: 4, PeakResidentBytes: 5, OSMaxRSSBytes: 6, + MeanResidentBytes: 7, PeakVirtualBytes: 8, PeakProcesses: 9, + PeakThreads: 10, PeakFileDescriptors: 11, SampleCount: 12, + SampleCoverageSeconds: 13}, + {WallSeconds: 14, CPUUserSeconds: 15, CPUSystemSeconds: 16, + AverageCPUPercent: 17, PeakResidentBytes: 18, + PeakPhysicalFootprintBytes: 19, PhysicalFootprintValid: true, + OSMaxRSSBytes: 20, MeanResidentBytes: 21, PeakVirtualBytes: 22, + PeakProcesses: 23, PeakThreads: 24, PeakFileDescriptors: 25, + SampleCount: 26, SampleCoverageSeconds: 27}, + } + accumulator := NewSummaryAccumulator(len(runs)) + for index, run := range runs { + accumulator.Add(run) + got, want := accumulator.Snapshot(), independentSummary(runs[:index+1]) + if !reflect.DeepEqual(got, want) { + t.Fatalf("prefix %d summary mismatch:\ngot %+v\nwant %+v", index+1, got, want) + } + } +} + +func independentSummary(runs []Run) Summary { + values := func(pick func(Run) float64) []float64 { + result := make([]float64, len(runs)) + for index, run := range runs { + result[index] = pick(run) + } + return result + } + physicalValues := make([]float64, 0, len(runs)) + for _, run := range runs { + if run.PhysicalFootprintValid { + physicalValues = append(physicalValues, run.PeakPhysicalFootprintBytes) + } + } + var physical *Stats + if len(physicalValues) > 0 { + value := CalculateStats(physicalValues) + physical = &value + } + return Summary{ + WallSeconds: CalculateStats(values(func(run Run) float64 { return run.WallSeconds })), + CPUTotalSeconds: CalculateStats(values(func(run Run) float64 { + return run.CPUUserSeconds + run.CPUSystemSeconds + })), + CPUUserSeconds: CalculateStats(values(func(run Run) float64 { return run.CPUUserSeconds })), + CPUSystemSeconds: CalculateStats(values(func(run Run) float64 { return run.CPUSystemSeconds })), + AverageCPUPercent: CalculateStats(values(func(run Run) float64 { return run.AverageCPUPercent })), + PeakResidentBytes: CalculateStats(values(func(run Run) float64 { return run.PeakResidentBytes })), + PeakPhysicalFootprintBytes: physical, + OSMaxRSSBytes: CalculateStats(values(func(run Run) float64 { return run.OSMaxRSSBytes })), + MeanResidentBytes: CalculateStats(values(func(run Run) float64 { return run.MeanResidentBytes })), + PeakVirtualBytes: CalculateStats(values(func(run Run) float64 { return run.PeakVirtualBytes })), + PeakProcesses: CalculateStats(values(func(run Run) float64 { return run.PeakProcesses })), + PeakThreads: CalculateStats(values(func(run Run) float64 { return run.PeakThreads })), + PeakFileDescriptors: CalculateStats(values(func(run Run) float64 { return run.PeakFileDescriptors })), + ValidSampleCount: CalculateStats(values(func(run Run) float64 { return float64(run.SampleCount) })), + SampleCoverageSeconds: CalculateStats(values(func(run Run) float64 { + return run.SampleCoverageSeconds + })), + } +} + func assertClose(t *testing.T, got, want float64) { t.Helper() if math.Abs(got-want) > 1e-9 { diff --git a/internal/model/summary_accumulator.go b/internal/model/summary_accumulator.go new file mode 100644 index 0000000..79335b2 --- /dev/null +++ b/internal/model/summary_accumulator.go @@ -0,0 +1,66 @@ +package model + +type SummaryAccumulator struct { + wall, cpuTotal, cpuUser, cpuSystem RunningStats + averageCPU RunningStats + peakResident, physical, osMaxRSS RunningStats + meanResident, peakVirtual RunningStats + peakProcesses, peakThreads RunningStats + peakFileDescriptors RunningStats + validSampleCount, sampleCoverage RunningStats +} + +func NewSummaryAccumulator(capacity int) SummaryAccumulator { + metric := func() RunningStats { return NewRunningStats(capacity) } + return SummaryAccumulator{ + wall: metric(), cpuTotal: metric(), cpuUser: metric(), cpuSystem: metric(), + averageCPU: metric(), peakResident: metric(), physical: metric(), + osMaxRSS: metric(), meanResident: metric(), peakVirtual: metric(), + peakProcesses: metric(), peakThreads: metric(), peakFileDescriptors: metric(), + validSampleCount: metric(), sampleCoverage: metric(), + } +} + +func (summary *SummaryAccumulator) Add(run Run) { + summary.wall.Add(run.WallSeconds) + summary.cpuTotal.Add(run.CPUUserSeconds + run.CPUSystemSeconds) + summary.cpuUser.Add(run.CPUUserSeconds) + summary.cpuSystem.Add(run.CPUSystemSeconds) + summary.averageCPU.Add(run.AverageCPUPercent) + summary.peakResident.Add(run.PeakResidentBytes) + if run.PhysicalFootprintValid { + summary.physical.Add(run.PeakPhysicalFootprintBytes) + } + summary.osMaxRSS.Add(run.OSMaxRSSBytes) + summary.meanResident.Add(run.MeanResidentBytes) + summary.peakVirtual.Add(run.PeakVirtualBytes) + summary.peakProcesses.Add(run.PeakProcesses) + summary.peakThreads.Add(run.PeakThreads) + summary.peakFileDescriptors.Add(run.PeakFileDescriptors) + summary.validSampleCount.Add(float64(run.SampleCount)) + summary.sampleCoverage.Add(run.SampleCoverageSeconds) +} + +func (summary SummaryAccumulator) Snapshot() Summary { + result := Summary{ + WallSeconds: summary.wall.Snapshot(), + CPUTotalSeconds: summary.cpuTotal.Snapshot(), + CPUUserSeconds: summary.cpuUser.Snapshot(), + CPUSystemSeconds: summary.cpuSystem.Snapshot(), + AverageCPUPercent: summary.averageCPU.Snapshot(), + PeakResidentBytes: summary.peakResident.Snapshot(), + OSMaxRSSBytes: summary.osMaxRSS.Snapshot(), + MeanResidentBytes: summary.meanResident.Snapshot(), + PeakVirtualBytes: summary.peakVirtual.Snapshot(), + PeakProcesses: summary.peakProcesses.Snapshot(), + PeakThreads: summary.peakThreads.Snapshot(), + PeakFileDescriptors: summary.peakFileDescriptors.Snapshot(), + ValidSampleCount: summary.validSampleCount.Snapshot(), + SampleCoverageSeconds: summary.sampleCoverage.Snapshot(), + } + physical := summary.physical.Snapshot() + if physical.N > 0 { + result.PeakPhysicalFootprintBytes = &physical + } + return result +} diff --git a/internal/platform/sampler_darwin.go b/internal/platform/sampler_darwin.go index 9415ead..9ac97f2 100644 --- a/internal/platform/sampler_darwin.go +++ b/internal/platform/sampler_darwin.go @@ -22,15 +22,10 @@ import "C" import ( "syscall" - "time" "unsafe" ) -func DefaultInterval() time.Duration { return 10 * time.Millisecond } - -func SampleImmediately() bool { return true } - -func SampleTree(rootPID int) (Metrics, bool) { +func SampleProcessGroup(rootPID int) (Metrics, bool) { pids := processGroupPIDs(rootPID) var total Metrics foundRoot := false diff --git a/internal/platform/sampler_darwin_test.go b/internal/platform/sampler_darwin_test.go index efcff2a..5636b10 100644 --- a/internal/platform/sampler_darwin_test.go +++ b/internal/platform/sampler_darwin_test.go @@ -6,7 +6,7 @@ import ( "testing" ) -func TestSampleTreeReadsProcessGroup(t *testing.T) { +func TestSampleProcessGroupReadsProcessGroup(t *testing.T) { cmd := exec.Command("/bin/sleep", "1") cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} if err := cmd.Start(); err != nil { @@ -17,7 +17,7 @@ func TestSampleTreeReadsProcessGroup(t *testing.T) { _ = cmd.Wait() }() - metrics, valid := SampleTree(cmd.Process.Pid) + metrics, valid := SampleProcessGroup(cmd.Process.Pid) if !valid { t.Fatal("process group leader was not sampled") } diff --git a/internal/platform/sampler_linux.go b/internal/platform/sampler_linux.go index b432f1b..bc96292 100644 --- a/internal/platform/sampler_linux.go +++ b/internal/platform/sampler_linux.go @@ -7,17 +7,14 @@ import ( "math" "os" "path/filepath" + "runtime" "strconv" - "time" + "unsafe" "golang.org/x/sys/unix" ) -func DefaultInterval() time.Duration { return 10 * time.Millisecond } - -func SampleImmediately() bool { return true } - -func SampleTree(rootPID int) (Metrics, bool) { +func SampleProcessGroup(rootPID int) (Metrics, bool) { directory, err := unix.Open("/proc", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) if err != nil { return Metrics{}, false @@ -27,7 +24,6 @@ func SampleTree(rootPID int) (Metrics, bool) { rootFound := false statBuffer := make([]byte, 4096) directoryBuffer := make([]byte, 32*1024) - names := make([]string, 0, 256) for { count, readErr := unix.ReadDirent(directory, directoryBuffer) if readErr != nil { @@ -36,17 +32,12 @@ func SampleTree(rootPID int) (Metrics, bool) { if count == 0 { return total, rootFound } - _, _, names = unix.ParseDirent(directoryBuffer[:count], -1, names[:0]) - for _, name := range names { - pid, ok := parsePID(name) - if !ok { - continue - } + validDirents := scanProcDirents(directoryBuffer[:count], func(pid int) { record, err := readProcessAt( - directory, name, pid, rootPID, statBuffer, + directory, pid, rootPID, statBuffer, ) if err != nil || record.GroupID != rootPID { - continue + return } if record.PID == rootPID { rootFound = true @@ -56,57 +47,78 @@ func SampleTree(rootPID int) (Metrics, bool) { total.Processes++ total.Threads += record.Threads total.FileDescriptors += record.FileDescriptors + }) + if !validDirents { + return Metrics{}, false } } } -func readProcess(pid int) (Process, error) { +func readProcess(pid int) (process, error) { base := filepath.Join("/proc", strconv.Itoa(pid)) data, err := os.ReadFile(filepath.Join(base, "stat")) if err != nil { - return Process{}, err + return process{}, err } - process, err := parseProcessStat(data, pid) + record, err := parseProcessStat(data, pid) if err != nil { - return Process{}, err + return process{}, err } - process.FileDescriptors = countDirectory(filepath.Join(base, "fd")) - return process, nil + record.FileDescriptors = countDirectory(filepath.Join(base, "fd")) + return record, nil } func readProcessAt( - procFD int, name string, pid, groupID int, buffer []byte, -) (Process, error) { - file, err := unix.Openat(procFD, name+"/stat", unix.O_RDONLY|unix.O_CLOEXEC, 0) + procFD, pid, groupID int, buffer []byte, +) (process, error) { + file, err := openProcessStat(procFD, pid) if err != nil { - return Process{}, err + return process{}, err } count, readErr := unix.Read(file, buffer) _ = unix.Close(file) if readErr != nil { - return Process{}, readErr + return process{}, readErr } if count == len(buffer) { - return Process{}, errors.New("process stat exceeds buffer") + return process{}, errors.New("process stat exceeds buffer") } - process, err := parseProcessStat(buffer[:count], pid) + record, err := parseProcessStat(buffer[:count], pid) if err != nil { - return Process{}, err + return process{}, err } - if process.GroupID == groupID { - process.FileDescriptors = countDirectory("/proc/" + name + "/fd") + if record.GroupID == groupID { + record.FileDescriptors = countDirectory( + "/proc/" + strconv.Itoa(pid) + "/fd", + ) } - return process, nil + return record, nil } -func parseProcessStat(data []byte, pid int) (Process, error) { +func openProcessStat(procFD, pid int) (int, error) { + var storage [32]byte + path := strconv.AppendInt(storage[:0], int64(pid), 10) + path = append(path, '/', 's', 't', 'a', 't', 0) + file, _, errno := unix.Syscall6( + unix.SYS_OPENAT, + uintptr(procFD), uintptr(unsafe.Pointer(&path[0])), + uintptr(unix.O_RDONLY|unix.O_CLOEXEC), 0, 0, 0, + ) + runtime.KeepAlive(path) + if errno != 0 { + return -1, errno + } + return int(file), nil +} + +func parseProcessStat(data []byte, pid int) (process, error) { closeParen := bytes.LastIndexByte(data, ')') if closeParen < 0 || closeParen+2 >= len(data) { - return Process{}, errors.New("malformed process stat") + return process{}, errors.New("malformed process stat") } fields := data[closeParen+2:] - var process Process - process.PID = pid + var result process + result.PID = pid field := 0 for offset := 0; offset < len(fields); { for offset < len(fields) && (fields[offset] == ' ' || fields[offset] == '\n') { @@ -123,13 +135,13 @@ func parseProcessStat(data []byte, pid int) (Process, error) { var err error switch field { case 1: - process.PPID, err = parsePositiveInt(value) + result.PPID, err = parsePositiveInt(value) case 2: - process.GroupID, err = parsePositiveInt(value) + result.GroupID, err = parsePositiveInt(value) case 17: - process.Threads, err = parseUint(value) + result.Threads, err = parseUint(value) case 20: - process.VirtualBytes, err = parseUint(value) + result.VirtualBytes, err = parseUint(value) case 21: var pages int64 pages, err = parseInt64(value) @@ -138,28 +150,53 @@ func parseProcessStat(data []byte, pid int) (Process, error) { if uint64(pages) > math.MaxUint64/pageSize { err = errors.New("resident size overflows") } else { - process.ResidentBytes = uint64(pages) * pageSize + result.ResidentBytes = uint64(pages) * pageSize } } } if err != nil { - return Process{}, err + return process{}, err } field++ if field > 21 { - return process, nil + return result, nil } } - return Process{}, errors.New("short process stat") + return process{}, errors.New("short process stat") } -func parsePID(value string) (int, bool) { - if value == "" { +func scanProcDirents(data []byte, visit func(int)) bool { + // linux_dirent64 stores d_reclen at byte 16 and d_name at byte 19. + const recordHeader = 19 + for offset := 0; offset < len(data); { + if len(data)-offset < recordHeader { + return false + } + recordLength := int(data[offset+16]) | int(data[offset+17])<<8 + if recordLength < recordHeader || recordLength%8 != 0 || + recordLength > len(data)-offset { + return false + } + name := data[offset+recordHeader : offset+recordLength] + nameLength := bytes.IndexByte(name, 0) + if nameLength < 0 { + return false + } + if pid, ok := parsePIDBytes(name[:nameLength]); ok { + visit(pid) + } + offset += recordLength + } + return true +} + +func parsePIDBytes(value []byte) (int, bool) { + if len(value) == 0 { return 0, false } result := 0 - for index := range len(value) { - digit := value[index] - '0' + for _, character := range value { + digit := character - '0' if digit > 9 || result > (math.MaxInt-int(digit))/10 { return 0, false } diff --git a/internal/platform/sampler_linux_bench_test.go b/internal/platform/sampler_linux_bench_test.go index dedbb74..9519172 100644 --- a/internal/platform/sampler_linux_bench_test.go +++ b/internal/platform/sampler_linux_bench_test.go @@ -6,7 +6,7 @@ import ( "testing" ) -func BenchmarkSampleTree(b *testing.B) { +func BenchmarkSampleProcessGroup(b *testing.B) { command := exec.Command("sleep", "60") command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} if err := command.Start(); err != nil { @@ -19,7 +19,7 @@ func BenchmarkSampleTree(b *testing.B) { b.ReportAllocs() b.ResetTimer() for range b.N { - if _, valid := SampleTree(command.Process.Pid); !valid { + if _, valid := SampleProcessGroup(command.Process.Pid); !valid { b.Fatal("sampled process disappeared") } } diff --git a/internal/platform/sampler_linux_test.go b/internal/platform/sampler_linux_test.go index 9b669d4..035daa7 100644 --- a/internal/platform/sampler_linux_test.go +++ b/internal/platform/sampler_linux_test.go @@ -27,12 +27,44 @@ func TestCountDirectoryDoesNotRequireSortedEntries(t *testing.T) { } func TestMissingRootIsNotAZeroSample(t *testing.T) { - if _, valid := SampleTree(1 << 30); valid { + if _, valid := SampleProcessGroup(1 << 30); valid { t.Fatal("missing root process should produce an invalid sample") } } -func TestSampleTreeIncludesReparentedProcessGroupMembers(t *testing.T) { +func TestScanProcDirentsVisitsOnlyNumericNames(t *testing.T) { + var data []byte + for _, name := range []string{".", "self", "123", "98765"} { + length := (19 + len(name) + 1 + 7) &^ 7 + record := make([]byte, length) + record[16], record[17] = byte(length), byte(length>>8) + copy(record[19:], name) + data = append(data, record...) + } + var pids []int + if !scanProcDirents(data, func(pid int) { pids = append(pids, pid) }) { + t.Fatal("valid dirents rejected") + } + if fmt.Sprint(pids) != "[123 98765]" { + t.Fatalf("PIDs = %v", pids) + } + if scanProcDirents(data[:len(data)-1], func(int) {}) { + t.Fatal("truncated dirent should be rejected") + } + missingNUL := make([]byte, 24) + missingNUL[16] = byte(len(missingNUL)) + copy(missingNUL[19:], "12345") + if scanProcDirents(missingNUL, func(int) {}) { + t.Fatal("dirent without a NUL-terminated name should be rejected") + } + misaligned := make([]byte, 23) + misaligned[16] = byte(len(misaligned)) + if scanProcDirents(misaligned, func(int) {}) { + t.Fatal("unaligned dirent should be rejected") + } +} + +func TestSampleProcessGroupIncludesReparentedMembers(t *testing.T) { childFile := filepath.Join(t.TempDir(), "child-pid") cmd := exec.Command( "sh", "-c", @@ -48,7 +80,7 @@ func TestSampleTreeIncludesReparentedProcessGroupMembers(t *testing.T) { _ = cmd.Wait() }) - var child Process + var child process deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { data, err := os.ReadFile(childFile) @@ -66,7 +98,7 @@ func TestSampleTreeIncludesReparentedProcessGroupMembers(t *testing.T) { if child.PID == 0 || child.PPID == cmd.Process.Pid || child.GroupID != cmd.Process.Pid { t.Fatalf("child process was not reparented in group: %+v", child) } - metrics, valid := SampleTree(cmd.Process.Pid) + metrics, valid := SampleProcessGroup(cmd.Process.Pid) if !valid || metrics.Processes < 2 { t.Fatalf("group metrics = %+v, valid = %v", metrics, valid) } diff --git a/internal/platform/types.go b/internal/platform/types.go index e0d8839..9e8f710 100644 --- a/internal/platform/types.go +++ b/internal/platform/types.go @@ -1,5 +1,9 @@ package platform +import "time" + +func DefaultInterval() time.Duration { return 10 * time.Millisecond } + type Metrics struct { ResidentBytes uint64 PhysicalFootprintBytes uint64 @@ -9,13 +13,9 @@ type Metrics struct { Processes uint64 Threads uint64 FileDescriptors uint64 - ResidentByteSamples uint64 - SampleCount uint64 - ResidentByteSeconds float64 - SampleCoverageSeconds float64 } -type Process struct { +type process struct { PID, PPID, GroupID int ResidentBytes, VirtualBytes uint64 Threads, FileDescriptors uint64 diff --git a/internal/report/chart_artifacts.go b/internal/report/chart_artifacts.go index 5372dcc..4e10614 100644 --- a/internal/report/chart_artifacts.go +++ b/internal/report/chart_artifacts.go @@ -4,8 +4,6 @@ import ( "fmt" "os" "path/filepath" - - "github.com/shellcell/snailrace/internal/model" ) type ChartArtifact struct { @@ -15,18 +13,10 @@ type ChartArtifact struct { Filename string } -func WriteChartFiles( - directory string, - report model.Report, - includeCommands bool, -) ([]ChartArtifact, error) { - return NewRenderer(report).WriteChartFiles(directory, includeCommands) -} - func (renderer *Renderer) WriteChartFiles( directory string, includeCommands bool, ) ([]ChartArtifact, error) { - report := renderer.report + report := renderer.displayReport if err := os.MkdirAll(directory, 0o755); err != nil { return nil, err } diff --git a/internal/report/chart_commands.go b/internal/report/chart_commands.go index 94dea41..2a23db7 100644 --- a/internal/report/chart_commands.go +++ b/internal/report/chart_commands.go @@ -20,7 +20,7 @@ func commandLegendChart(report model.Report) svgChart { y := height fmt.Fprintf( &body, `%d. %s`, - y, toolColor(report, index), index+1, + y, toolColor(index), index+1, html.EscapeString(reportToolLabel(report, index)), ) for _, line := range lines { diff --git a/internal/report/chart_delta.go b/internal/report/chart_delta.go index 94ca77a..6c433e4 100644 --- a/internal/report/chart_delta.go +++ b/internal/report/chart_delta.go @@ -5,8 +5,6 @@ import ( "html" "math" "strings" - - "github.com/shellcell/snailrace/internal/model" ) type forestRow struct { @@ -18,45 +16,41 @@ type forestRow struct { baseline bool } -func deltaForestChart(report model.Report, group chartGroup) svgChart { - return deltaForestChartWith(NewRenderer(report), group) -} - -func deltaForestChartWith(renderer *Renderer, group chartGroup) svgChart { - report := renderer.report +func deltaForestChart(renderer *Renderer, group chartGroup) svgChart { + report := renderer.displayReport baselinePosition := baselineIndex(report) baseline := report.Benchmarks[baselinePosition] rows := make([]forestRow, 0) maximum := 0.0 for _, metric := range group.metrics { - row := metricRows[metric.row] + row := metric if !availableFor(row, report.Host.OS, baseline.Summary) { rows = append(rows, forestRow{ - label: metric.name, status: "N/A on " + report.Host.OS, - class: "neutral", identity: toolColor(report, baselinePosition), + label: metric.chartName, status: "N/A on " + report.Host.OS, + class: "neutral", identity: toolColor(baselinePosition), baseline: true, }) continue } - baseStats := metric.stats(baseline) + baseStats := metric.stats(baseline.Summary) baseMean := baseStats.Mean - if baseStats.N == 0 || !finiteChartValue(baseMean) { + if baseStats.N == 0 || !finiteNumber(baseMean) { rows = append(rows, forestRow{ - label: metric.name, status: "N/A", class: "neutral", - identity: toolColor(report, baselinePosition), baseline: true, + label: metric.chartName, status: "N/A", class: "neutral", + identity: toolColor(baselinePosition), baseline: true, }) continue } baselineRow := forestRow{ - label: metric.name + " Β· " + reportToolLabel(report, baselinePosition), + label: metric.chartName + " Β· " + reportToolLabel(report, baselinePosition), status: metric.format(baseMean) + " Β· baseline", - class: "neutral", identity: toolColor(report, baselinePosition), + class: "neutral", identity: toolColor(baselinePosition), baseline: true, } if baseMean != 0 && baseStats.CI95Valid { baselineRow.low, baselineRow.high = percentInterval(baseStats.CI95Low, baseStats.CI95High, baseMean) - baselineRow.interval = finiteChartValue(baselineRow.low) && - finiteChartValue(baselineRow.high) + baselineRow.interval = finiteNumber(baselineRow.low) && + finiteNumber(baselineRow.high) if baselineRow.interval { maximum = math.Max( maximum, math.Max(math.Abs(baselineRow.low), math.Abs(baselineRow.high)), @@ -71,17 +65,17 @@ func deltaForestChartWith(renderer *Renderer, group chartGroup) svgChart { } if !availableFor(row, report.Host.OS, candidate.Summary) { rows = append(rows, forestRow{ - label: metric.name + " Β· " + candidate.Tool.Name, - status: "N/A", class: "neutral", identity: toolColor(report, index), + label: metric.chartName + " Β· " + candidate.Tool.Name, + status: "N/A", class: "neutral", identity: toolColor(index), }) continue } - value := metric.stats(candidate).Mean - delta := renderer.comparison(index, metric.row) + value := metric.stats(candidate.Summary).Mean + delta := renderer.comparison(index, metric.id) rows = append(rows, forestRow{ - label: metric.name + " Β· " + candidate.Tool.Name, + label: metric.chartName + " Β· " + candidate.Tool.Name, status: metric.format(value) + " Β· Ξ”% n/a Β· " + delta.status, - class: delta.class, identity: toolColor(report, index), + class: delta.class, identity: toolColor(index), baseline: true, }) } @@ -93,18 +87,18 @@ func deltaForestChartWith(renderer *Renderer, group chartGroup) svgChart { } if !availableFor(row, report.Host.OS, candidate.Summary) { rows = append(rows, forestRow{ - label: metric.name + " Β· " + candidate.Tool.Name, - status: "N/A", class: "neutral", identity: toolColor(report, index), + label: metric.chartName + " Β· " + candidate.Tool.Name, + status: "N/A", class: "neutral", identity: toolColor(index), }) continue } - delta := renderer.comparison(index, metric.row) + delta := renderer.comparison(index, metric.id) low, high := 0.0, 0.0 interval := delta.difference.CI95Valid if interval { low = delta.difference.CI95Low / baseMean * 100 high = delta.difference.CI95High / baseMean * 100 - interval = finiteChartValue(low) && finiteChartValue(high) + interval = finiteNumber(low) && finiteNumber(high) } if interval { maximum = math.Max(maximum, math.Max(math.Abs(low), math.Abs(high))) @@ -113,10 +107,10 @@ func deltaForestChartWith(renderer *Renderer, group chartGroup) svgChart { maximum = math.Max(maximum, math.Abs(delta.percent)) } rows = append(rows, forestRow{ - label: metric.name + " Β· " + candidate.Tool.Name, + label: metric.chartName + " Β· " + candidate.Tool.Name, point: delta.percent, low: low, high: high, status: delta.status, class: delta.class, - identity: toolColor(report, index), interval: interval, + identity: toolColor(index), interval: interval, }) } } diff --git a/internal/report/chart_distribution.go b/internal/report/chart_distribution.go index 46cc21d..2a2c1af 100644 --- a/internal/report/chart_distribution.go +++ b/internal/report/chart_distribution.go @@ -10,24 +10,24 @@ import ( ) func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart { - if !available(metricRows[metric.row], report.Host.OS) { - return unavailableChart("RUN DISTRIBUTION Β· "+metric.name, report.Host.OS) + if !available(metric, report.Host.OS) { + return unavailableChart("RUN DISTRIBUTION Β· "+metric.chartName, report.Host.OS) } for _, benchmark := range report.Benchmarks { - if !availableFor(metricRows[metric.row], report.Host.OS, benchmark.Summary) { - return unavailableChart("RUN DISTRIBUTION Β· "+metric.name, "metric unavailable") + if !availableFor(metric, report.Host.OS, benchmark.Summary) { + return unavailableChart("RUN DISTRIBUTION Β· "+metric.chartName, "metric unavailable") } } minimum, maximum := 0.0, 0.0 for _, benchmark := range report.Benchmarks { - stats := metric.stats(benchmark) - if stats.N == 0 || !finiteChartValue(stats.Min) || - !finiteChartValue(stats.Max) || !finiteChartValue(stats.Mean) { + stats := metric.stats(benchmark.Summary) + if stats.N == 0 || !finiteNumber(stats.Min) || + !finiteNumber(stats.Max) || !finiteNumber(stats.Mean) { return svgChart{} } maximum = math.Max(maximum, stats.Max) - if stats.CI95Valid && finiteChartValue(stats.CI95Low) && - finiteChartValue(stats.CI95High) { + if stats.CI95Valid && finiteNumber(stats.CI95Low) && + finiteNumber(stats.CI95High) { minimum = math.Min(minimum, stats.CI95Low) maximum = math.Max(maximum, stats.CI95High) } @@ -52,7 +52,7 @@ func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart ``+ `RUN DISTRIBUTION Β· %s`+ `dots = runs Β· diamond = mean Β· whisker = mean 95%% CI`, - html.EscapeString(metric.name), + html.EscapeString(metric.chartName), ) fmt.Fprintf( &body, @@ -60,10 +60,9 @@ func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart `%s`, left, metric.format(minimum), left+plotWidth, metric.format(maximum), ) - baseline := baselineIndex(report) for index, benchmark := range report.Benchmarks { y := 76 + index*rowHeight - color := chartColor(index, baseline) + color := toolColor(index) fmt.Fprintf( &body, `%s`+ @@ -84,10 +83,10 @@ func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart body.WriteString(color) body.WriteString(`" opacity=".65"/>`) } - stats := metric.stats(benchmark) + stats := metric.stats(benchmark.Summary) mean := position(stats.Mean) - if stats.CI95Valid && finiteChartValue(stats.CI95Low) && - finiteChartValue(stats.CI95High) { + if stats.CI95Valid && finiteNumber(stats.CI95Low) && + finiteNumber(stats.CI95High) { low, high := position(stats.CI95Low), position(stats.CI95High) fmt.Fprintf( &body, @@ -108,8 +107,8 @@ func absoluteDistributionChart(report model.Report, metric chartMetric) svgChart ) } return svgChart{ - kind: "distribution", title: metric.name, slug: chartSlug(metric.name), - description: distributionChartDescription(metric.name), + kind: "distribution", title: metric.chartName, slug: chartSlug(metric.chartName), + description: distributionChartDescription(metric.chartName), body: body.String(), height: height, } } diff --git a/internal/report/chart_metrics.go b/internal/report/chart_metrics.go index 78b4aab..fb78043 100644 --- a/internal/report/chart_metrics.go +++ b/internal/report/chart_metrics.go @@ -2,23 +2,19 @@ package report import "github.com/shellcell/snailrace/internal/model" -func reportCharts(report model.Report) []svgChart { - return NewRenderer(report).reportCharts() -} - func buildReportCharts(renderer *Renderer) []svgChart { - report := renderer.report + report := renderer.displayReport if len(report.Benchmarks) == 0 { return nil } - groups := renderer.groups + groups := renderer.chartGroups var charts []svgChart if len(report.Benchmarks) > 1 { - charts = append(charts, rankingChartsWith(report, renderer.ranking)...) + charts = append(charts, rankingCharts(report, renderer.ranking)...) for _, group := range groups { for _, metric := range group.metrics { - chart := deltaForestChartWith(renderer, chartGroup{ - name: metric.name, metrics: []chartMetric{metric}, + chart := deltaForestChart(renderer, chartGroup{ + name: metric.chartName, metrics: []chartMetric{metric}, }) if chart.height > 0 { charts = append(charts, chart) @@ -47,64 +43,34 @@ func buildReportCharts(renderer *Renderer) []svgChart { } func chartGroups(report model.Report) []chartGroup { - metrics := chartMetricDefinitions() groups := []chartGroup{ - {"PERFORMANCE", metrics[0:1]}, - {"CPU COST", metrics[1:5]}, - {"MEMORY COST", metrics[5:10]}, - {"PROCESS STRUCTURE", metrics[10:13]}, + {"PERFORMANCE", chartMetrics(metricWall)}, + {"CPU COST", chartMetrics( + metricCPUTotal, metricCPUUser, metricCPUSystem, metricAverageCPU, + )}, + {"MEMORY COST", chartMetrics( + metricMeanResident, metricPeakResident, metricOSMaxRSS, + metricPhysicalFootprint, metricPeakVirtual, + )}, + {"PROCESS STRUCTURE", chartMetrics( + metricPeakProcesses, metricPeakThreads, metricPeakFDs, + )}, } - if report.Config.Mode == "tui" && report.Config.DurationSeconds > 0 { + if report.Config.FixedDurationTUI() { groups = groups[1:] } return groups } -func chartRunAvailable(metric chartMetric, run model.Run) bool { - return (metric.row != 8 || run.PhysicalFootprintValid) && - finiteChartValue(metric.run(run)) +func chartMetrics(ids ...metricID) []chartMetric { + metrics := make([]chartMetric, len(ids)) + for index, id := range ids { + metrics[index] = metricCatalog[id] + } + return metrics } -func chartMetricDefinitions() []chartMetric { - return []chartMetric{ - {"Wall time", formatDuration, - func(b model.Benchmark) model.Stats { return b.Summary.WallSeconds }, - func(r model.Run) float64 { return r.WallSeconds }, 0}, - {"Total CPU", formatDuration, - func(b model.Benchmark) model.Stats { return b.Summary.CPUTotalSeconds }, - func(r model.Run) float64 { return r.CPUUserSeconds + r.CPUSystemSeconds }, 1}, - {"User CPU", formatDuration, - func(b model.Benchmark) model.Stats { return b.Summary.CPUUserSeconds }, - func(r model.Run) float64 { return r.CPUUserSeconds }, 2}, - {"System CPU", formatDuration, - func(b model.Benchmark) model.Stats { return b.Summary.CPUSystemSeconds }, - func(r model.Run) float64 { return r.CPUSystemSeconds }, 3}, - {"Average CPU", formatPercent, - func(b model.Benchmark) model.Stats { return b.Summary.AverageCPUPercent }, - func(r model.Run) float64 { return r.AverageCPUPercent }, 4}, - {"Mean RSS", formatBytes, - func(b model.Benchmark) model.Stats { return b.Summary.MeanResidentBytes }, - func(r model.Run) float64 { return r.MeanResidentBytes }, 5}, - {"Peak RSS", formatBytes, - func(b model.Benchmark) model.Stats { return b.Summary.PeakResidentBytes }, - func(r model.Run) float64 { return r.PeakResidentBytes }, 6}, - {"OS-reported max RSS", formatBytes, - func(b model.Benchmark) model.Stats { return b.Summary.OSMaxRSSBytes }, - func(r model.Run) float64 { return r.OSMaxRSSBytes }, 7}, - {"Peak physical footprint", formatBytes, - func(b model.Benchmark) model.Stats { return b.Summary.PhysicalFootprintStats() }, - func(r model.Run) float64 { return r.PeakPhysicalFootprintBytes }, 8}, - {"Peak virtual memory", formatBytes, - func(b model.Benchmark) model.Stats { return b.Summary.PeakVirtualBytes }, - func(r model.Run) float64 { return r.PeakVirtualBytes }, 9}, - {"Peak processes", formatCount, - func(b model.Benchmark) model.Stats { return b.Summary.PeakProcesses }, - func(r model.Run) float64 { return r.PeakProcesses }, 10}, - {"Peak threads", formatCount, - func(b model.Benchmark) model.Stats { return b.Summary.PeakThreads }, - func(r model.Run) float64 { return r.PeakThreads }, 11}, - {"Peak FD references", formatCount, - func(b model.Benchmark) model.Stats { return b.Summary.PeakFileDescriptors }, - func(r model.Run) float64 { return r.PeakFileDescriptors }, 12}, - } +func chartRunAvailable(metric chartMetric, run model.Run) bool { + return (metric.id != metricPhysicalFootprint || run.PhysicalFootprintValid) && + finiteNumber(metric.run(run)) } diff --git a/internal/report/chart_ranking.go b/internal/report/chart_ranking.go index a19e8dd..f29477a 100644 --- a/internal/report/chart_ranking.go +++ b/internal/report/chart_ranking.go @@ -17,47 +17,43 @@ type rankingChartMetric struct { rank func(rankingRow) int } -func rankingCharts(report model.Report) []svgChart { - return rankingChartsWith(report, calculateRanking(report)) -} - -func rankingChartsWith(report model.Report, ranking rankingData) []svgChart { - if len(ranking.rows) == 0 { +func rankingCharts(report model.Report, ranking rankingData) []svgChart { + if len(ranking.Rows) == 0 { return nil } var metrics []rankingChartMetric - if ranking.available { + if ranking.Available { metrics = append(metrics, rankingChartMetric{"BALANCED INDEX", formatScore, - func(row rankingRow) float64 { return row.overallScore }, - func(row rankingRow) int { return row.overallRank }}) + func(row rankingRow) float64 { return row.OverallScore }, + func(row rankingRow) int { return row.OverallRank }}) } metrics = append(metrics, rankingChartMetric{ranking.primaryLabel, ranking.primaryUnit, - func(row rankingRow) float64 { return row.primaryValue }, - func(row rankingRow) int { return row.primaryRank }}) - if !(report.Config.Mode == "tui" && report.Config.DurationSeconds > 0) { + func(row rankingRow) float64 { return row.PrimaryValue }, + func(row rankingRow) int { return row.PrimaryRank }}) + if !report.Config.FixedDurationTUI() { metrics = append(metrics, rankingChartMetric{ "CPU COST", formatDuration, - func(row rankingRow) float64 { return row.cpuValue }, - func(row rankingRow) int { return row.cpuRank }, + func(row rankingRow) float64 { return row.CPUValue }, + func(row rankingRow) int { return row.CPURank }, }) } - if ranking.ramPresent { - ramRank := func(row rankingRow) int { return row.ramRank } - if !ranking.ramAvailable { + if ranking.RAMPresent { + ramRank := func(row rankingRow) int { return row.RAMRank } + if !ranking.RAMAvailable { // Sampling-limited RAM has no reliable score; rank descriptively by value. - order := rankByValue(ranking.rows, func(row rankingRow) float64 { return row.ramValue }) - ramRank = func(row rankingRow) int { return order[row.benchmark] } + order := rankByValue(ranking.Rows, func(row rankingRow) float64 { return row.RAMValue }) + ramRank = func(row rankingRow) int { return order[row.Benchmark] } } metrics = append(metrics, rankingChartMetric{ "RAM AGGREGATE", formatBytes, - func(row rankingRow) float64 { return row.ramValue }, + func(row rankingRow) float64 { return row.RAMValue }, ramRank, }) } metrics = append(metrics, rankingChartMetric{ "LINKED SIZE", formatBytes, - func(row rankingRow) float64 { return row.footprintValue }, - func(row rankingRow) int { return row.footprintRank }, + func(row rankingRow) float64 { return row.FootprintValue }, + func(row rankingRow) int { return row.FootprintRank }, }) charts := make([]svgChart, 0, len(metrics)) for _, metric := range metrics { @@ -77,7 +73,7 @@ func rankByValue(rows []rankingRow, value func(rankingRow) float64) map[int]int if position == 0 || value(row) != value(ordered[position-1]) { rank = position + 1 } - result[row.benchmark] = rank + result[row.Benchmark] = rank } return result } @@ -89,9 +85,9 @@ func rankingBarChart( ) svgChart { const left, plotWidth, rowHeight = 205, 355, 30 maximum := 0.0 - for _, row := range ranking.rows { + for _, row := range ranking.Rows { value := metric.value(row) - if !finiteChartValue(value) || value < 0 { + if !finiteNumber(value) || value < 0 { return svgChart{} } maximum = math.Max(maximum, value) @@ -99,11 +95,11 @@ func rankingBarChart( if maximum <= 0 { maximum = 1 } - height := 62 + len(ranking.rows)*rowHeight + height := 62 + len(ranking.Rows)*rowHeight var body strings.Builder body.WriteString(svgChartStyle) subtitle := "descriptive point estimates Β· outline = category leader" - if metric.name == "RAM AGGREGATE" && ranking.ramPresent && !ranking.ramAvailable { + if metric.name == "RAM AGGREGATE" && ranking.RAMPresent && !ranking.RAMAvailable { subtitle = "sampling-limited Β· descriptive only Β· excluded from balanced index" if ranking.samplingInterval != "" { subtitle += " Β· lower -interval (now " + ranking.samplingInterval + ")" @@ -116,7 +112,7 @@ func rankingBarChart( `%s`, html.EscapeString(metric.name), html.EscapeString(subtitle), ) - rows := append([]rankingRow(nil), ranking.rows...) + rows := append([]rankingRow(nil), ranking.Rows...) sort.SliceStable(rows, func(left, right int) bool { return metric.rank(rows[left]) < metric.rank(rows[right]) }) @@ -124,7 +120,7 @@ func rankingBarChart( y := 62 + index*rowHeight value := metric.value(row) width := value / maximum * plotWidth - color := toolColor(report, row.benchmark) + color := toolColor(row.Benchmark) outline := "none" outlineWidth := 0 if metric.rank(row) == 1 { @@ -138,7 +134,7 @@ func rankingBarChart( `fill="%s" stroke="%s" stroke-width="%d"/>`+ `%s`, y, color, metric.rank(row), - html.EscapeString(clip(reportToolLabel(report, row.benchmark), 26)), + html.EscapeString(clip(reportToolLabel(report, row.Benchmark), 26)), left, y-11, width, color, outline, outlineWidth, y, metric.format(value), ) } @@ -167,7 +163,7 @@ func rankingChartDescription(metric string, ranking rankingData) string { "this ranking does not include uncertainty intervals." case "RAM AGGREGATE": caveat := "" - if ranking.ramPresent && !ranking.ramAvailable { + if ranking.RAMPresent && !ranking.RAMAvailable { caveat = " These values are sampling-limited (short runs or too few valid " + "samples), so they are shown for reference but excluded from the balanced index." if ranking.samplingInterval != "" { @@ -190,16 +186,16 @@ func rankingChartDescription(metric string, ranking rankingData) string { func balancedIndexCategories(ranking rankingData) string { var categories []string - if ranking.indexPrimary { + if ranking.IndexPrimary { categories = append(categories, primaryCategoryName(ranking.primaryLabel)) } - if ranking.indexCPU { + if ranking.IndexCPU { categories = append(categories, "CPU cost") } - if ranking.indexRAM { + if ranking.IndexRAM { categories = append(categories, "RAM aggregate") } - if ranking.indexFootprint { + if ranking.IndexFootprint { categories = append(categories, "linked size") } if len(categories) == 0 { diff --git a/internal/report/chart_static.go b/internal/report/chart_static.go index fba8e7e..cfaa8ae 100644 --- a/internal/report/chart_static.go +++ b/internal/report/chart_static.go @@ -43,12 +43,11 @@ func staticCostChart( `%s`, left, left+plotWidth, format(maximum), ) - baseline := baselineIndex(report) for index, benchmark := range report.Benchmarks { y := 72 + index*rowHeight value := pick(benchmark) x := left + value/maximum*plotWidth - color := chartColor(index, baseline) + color := toolColor(index) fmt.Fprintf( &body, `%s`+ diff --git a/internal/report/chart_test.go b/internal/report/chart_test.go index 8de51b1..90bd960 100644 --- a/internal/report/chart_test.go +++ b/internal/report/chart_test.go @@ -9,7 +9,7 @@ import ( ) func TestEmptyReportBuildsNoCharts(t *testing.T) { - if charts := reportCharts(model.Report{}); len(charts) != 0 { + if charts := NewRenderer(model.Report{}).reportCharts(); len(charts) != 0 { t.Fatalf("empty report charts = %d, want 0", len(charts)) } } @@ -27,7 +27,7 @@ func TestChartsOmitNonFiniteCoordinates(t *testing.T) { Runs: runs, Summary: model.Summarize(runs), }}, } - for _, chart := range reportCharts(report) { + for _, chart := range NewRenderer(report).reportCharts() { output := chart.html() if strings.Contains(output, "NaN") || strings.Contains(output, "+Inf") || strings.Contains(output, "-Inf") { @@ -44,7 +44,7 @@ func TestComparisonChartsContainDeltaAndRunViews(t *testing.T) { benchmarkWithWallTimes("candidate", 8, 8, 8), }, } - charts := reportCharts(report) + charts := NewRenderer(report).reportCharts() if len(charts) < 2 { t.Fatalf("charts = %d, want delta and distribution charts", len(charts)) } @@ -73,13 +73,13 @@ func TestBaselineChartsSeparateEveryMetric(t *testing.T) { }, } var titles []string - for _, chart := range reportCharts(report) { + for _, chart := range NewRenderer(report).reportCharts() { if chart.kind == "baseline" { titles = append(titles, chart.title) } } - if len(titles) != len(chartMetricDefinitions()) { - t.Fatalf("baseline charts = %d, want %d: %v", len(titles), len(chartMetricDefinitions()), titles) + if len(titles) != len(metricCatalog) { + t.Fatalf("baseline charts = %d, want %d: %v", len(titles), len(metricCatalog), titles) } for _, expected := range []string{ "Mean RSS", "Peak RSS", "OS-reported max RSS", "Peak virtual memory", @@ -102,8 +102,8 @@ func TestBaselineChartShowsBaselineConfidenceWhisker(t *testing.T) { benchmarkWithWallTimes("candidate", 8, 9, 10), }, } - chart := deltaForestChart(report, chartGroup{ - name: "Wall time", metrics: []chartMetric{chartMetricDefinitions()[0]}, + chart := deltaForestChart(NewRenderer(report), chartGroup{ + name: "Wall time", metrics: chartMetrics(metricWall), }) if !strings.Contains(chart.body, `class="baseline-ci"`) { t.Fatalf("baseline confidence whisker missing: %s", chart.body) @@ -116,16 +116,13 @@ func TestBaselineChartShowsBaselineConfidenceWhisker(t *testing.T) { func TestToolColorsAreDistinctAndStable(t *testing.T) { seen := make(map[string]bool) for index := 0; index < 10; index++ { - color := chartColor(index, 0) + color := toolColor(index) if seen[color] { t.Fatalf("tool %d repeats color %s", index, color) } seen[color] = true - if chartColor(index, 3) != color || chartColor(index, 7) != color { - t.Fatalf("tool %d color changes with baseline", index) - } } - if chartColor(0, 0) != "#88c0d0" { + if toolColor(0) != "#88c0d0" { t.Fatal("first tool should begin with the Frost color") } } @@ -139,9 +136,9 @@ func TestChartLegendsUseToolIdentityColors(t *testing.T) { }, } legend := commandLegendChart(report).body - ranking := rankingCharts(report)[0].body + ranking := rankingCharts(report, calculateRanking(report))[0].body for index := range report.Benchmarks { - color := toolColor(report, index) + color := toolColor(index) style := `style="fill:` + color + `"` if !strings.Contains(legend, style) || !strings.Contains(ranking, style) { t.Fatalf("tool %d color %s missing from chart legend or ranking", index, color) @@ -168,7 +165,7 @@ func TestRankingChartsPutBestAtTopAndWorstAtBottom(t *testing.T) { "RAM AGGREGATE": {"#1 efficient", "#2 middle", "#3 fast"}, "LINKED SIZE": {"#1 efficient", "#2 middle", "#3 fast"}, } - for _, chart := range rankingCharts(report) { + for _, chart := range rankingCharts(report, calculateRanking(report)) { order, ok := expected[chart.title] if !ok { t.Fatalf("missing expected order for %s chart", chart.title) @@ -187,7 +184,7 @@ func TestSingleRunDistributionOmitsUndefinedConfidenceWhisker(t *testing.T) { Config: model.Config{Mode: "command", Baseline: 1}, Benchmarks: []model.Benchmark{benchmarkWithWallTimes("tool", 1)}, } - chart := absoluteDistributionChart(report, chartMetricDefinitions()[0]) + chart := absoluteDistributionChart(report, metricCatalog[metricWall]) if strings.Contains(chart.body, `stroke="#eceff4"`) { t.Fatal("one observation should not render a confidence whisker") } @@ -198,7 +195,7 @@ func TestDistributionSummaryPaintsAboveRawRuns(t *testing.T) { Config: model.Config{Mode: "command", Baseline: 1}, Benchmarks: []model.Benchmark{benchmarkWithWallTimes("tool", 1, 2)}, } - body := absoluteDistributionChart(report, chartMetricDefinitions()[0]).body + body := absoluteDistributionChart(report, metricCatalog[metricWall]).body dot := strings.Index(body, `class="run-dot"`) whisker := strings.Index(body, `class="mean-ci"`) diamond := strings.Index(body, `class="mean-diamond"`) @@ -214,7 +211,7 @@ func TestFixedTUIRankingChartUsesCPUUnits(t *testing.T) { benchmarkWithWallTimes("first", 1), benchmarkWithWallTimes("second", 1), }, } - charts := rankingCharts(report) + charts := rankingCharts(report, calculateRanking(report)) found := false for _, chart := range charts { if chart.title == "CPU" && strings.Contains(chart.body, "%") { @@ -231,7 +228,7 @@ func TestSingleToolReportOmitsRankingCharts(t *testing.T) { Config: model.Config{Mode: "command", Baseline: 1}, Benchmarks: []model.Benchmark{benchmarkWithTrendRuns("tool")}, } - for _, chart := range reportCharts(report) { + for _, chart := range NewRenderer(report).reportCharts() { if chart.kind == "ranking" { t.Fatalf("single-tool report should not include ranking chart %q", chart.title) } @@ -248,7 +245,7 @@ func TestBalancedIndexDescriptionListsIncludedCategories(t *testing.T) { Config: model.Config{Mode: "command", Baseline: 1}, Benchmarks: benchmarks, } - description := rankingCharts(report)[0].description + description := rankingCharts(report, calculateRanking(report))[0].description for _, expected := range []string{ "Included here: wall time, CPU cost, RAM aggregate.", "score = value / best value", @@ -265,7 +262,7 @@ func TestBalancedIndexDescriptionListsIncludedCategories(t *testing.T) { }, Benchmarks: benchmarks, } - description = rankingCharts(withDisk)[0].description + description = rankingCharts(withDisk, calculateRanking(withDisk))[0].description if !strings.Contains(description, "wall time, CPU cost, RAM aggregate, linked size") { t.Fatalf("disk index missing linked size: %s", description) } @@ -278,7 +275,7 @@ func TestSingleToolReportHasMetricGroupTrendCharts(t *testing.T) { } var titles []string var combined strings.Builder - for _, chart := range reportCharts(report) { + for _, chart := range NewRenderer(report).reportCharts() { if chart.kind != "trend" { continue } @@ -309,7 +306,7 @@ func TestTrendChartsAreSectionedByTool(t *testing.T) { }, } var titles []string - for _, section := range chartSections(reportCharts(report)) { + for _, section := range chartSections(NewRenderer(report).reportCharts()) { if strings.HasPrefix(section.title, "Measurement trends") { titles = append(titles, section.title) } @@ -330,7 +327,10 @@ func TestTrendLegendValuesDoNotOverlapLabels(t *testing.T) { Benchmarks: []model.Benchmark{benchmarkWithTrendRuns("tool")}, } chart := measurementTrendChart(report, 0, chartGroup{ - name: "MEMORY COST", metrics: chartMetricDefinitions()[5:9], + name: "MEMORY COST", metrics: chartMetrics( + metricMeanResident, metricPeakResident, metricOSMaxRSS, + metricPhysicalFootprint, + ), }) if strings.Contains(chart.body, `x="704"`) { t.Fatalf("trend legend values should not use overlapping right column: %s", chart.body) @@ -356,7 +356,7 @@ func TestComparisonReportHasPerToolTrendCharts(t *testing.T) { benchmarkWithTrendRuns("second"), }, } - titles := trendChartTitles(reportCharts(report)) + titles := trendChartTitles(NewRenderer(report).reportCharts()) for _, expected := range []string{ "CPU cost trends Β· first [BASELINE]", "Memory cost trends Β· first [BASELINE]", @@ -382,8 +382,8 @@ func TestChartsHaveExplicitDescriptions(t *testing.T) { Benchmarks: []model.Benchmark{benchmarkWithWallTimes("tool", 1, 2)}, } chartSets := [][]svgChart{ - append([]svgChart{commandLegendChart(report)}, reportCharts(report)...), - reportCharts(single), + append([]svgChart{commandLegendChart(report)}, NewRenderer(report).reportCharts()...), + NewRenderer(single).reportCharts(), } for _, charts := range chartSets { for _, chart := range charts { diff --git a/internal/report/chart_tradeoff.go b/internal/report/chart_tradeoff.go index 43f71c4..d045c7c 100644 --- a/internal/report/chart_tradeoff.go +++ b/internal/report/chart_tradeoff.go @@ -22,7 +22,7 @@ func tradeoffCharts(report model.Report, ranking rankingData) []svgChart { primary := tradeoffMetric{"Time", formatDuration, func(b model.Benchmark) float64 { return b.Summary.WallSeconds.Mean }} - if report.Config.Mode == "tui" && report.Config.DurationSeconds > 0 { + if report.Config.FixedDurationTUI() { primary = tradeoffMetric{"Average CPU", formatPercent, func(b model.Benchmark) float64 { return b.Summary.AverageCPUPercent.Mean }} @@ -42,7 +42,7 @@ func tradeoffCharts(report model.Report, ranking rankingData) []svgChart { return float64(b.Tool.DiskFootprintBytes) }} pairs := [][2]tradeoffMetric{{primary, cpu}, {primary, linked}} - if ranking.ramAvailable { + if ranking.RAMAvailable { pairs = append(pairs, [2]tradeoffMetric{primary, ram}, [2]tradeoffMetric{cpu, ram}) } charts := make([]svgChart, 0, len(pairs)) @@ -57,7 +57,7 @@ func tradeoffChart(report model.Report, xMetric, yMetric tradeoffMetric) svgChar xMaximum, yMaximum := 0.0, 0.0 for _, benchmark := range report.Benchmarks { xValue, yValue := xMetric.value(benchmark), yMetric.value(benchmark) - if !finiteChartValue(xValue) || !finiteChartValue(yValue) || + if !finiteNumber(xValue) || !finiteNumber(yValue) || xValue < 0 || yValue < 0 { return svgChart{} } @@ -91,7 +91,7 @@ func tradeoffChart(report model.Report, xMetric, yMetric tradeoffMetric) svgChar xValue, yValue := xMetric.value(benchmark), yMetric.value(benchmark) x := left + xValue/xMaximum*plotWidth y := top + plotHeight - yValue/yMaximum*plotHeight - color := toolColor(report, index) + color := toolColor(index) legendY := 67 + index*26 fmt.Fprintf( &body, diff --git a/internal/report/chart_trend.go b/internal/report/chart_trend.go index f4d1d09..6e7d8f7 100644 --- a/internal/report/chart_trend.go +++ b/internal/report/chart_trend.go @@ -53,7 +53,7 @@ func measurementTrendChart(report model.Report, benchmarkIndex int, group chartG } body.Grow(1536 + len(runs)*metricCount*96) toolLabel := reportToolLabel(report, benchmarkIndex) - toolColorHex := toolColor(report, benchmarkIndex) + toolColorHex := toolColor(benchmarkIndex) body.WriteString(svgChartStyle) fmt.Fprintf( &body, @@ -92,17 +92,17 @@ func measurementTrendChart(report model.Report, benchmarkIndex int, group chartG left, bottom+18, right, bottom+18, len(runs), ) for metricIndex, metric := range lane.metrics { - values := trendValues(runs, metric) - minimum, maximum := valueRange(values) - color := style.Tool(metric.row).Hex + minimum, maximum := trendMetricRange(runs, metric) + color := style.Tool(int(metric.id)).Hex var path strings.Builder - path.Grow(len(values) * 20) + path.Grow(len(runs) * 20) started := false - for index, value := range values { - if !finiteChartValue(value) { + for index, run := range runs { + if !chartRunAvailable(metric, run) { started = false continue } + value := metric.run(run) command := "L" if !started { command = "M" @@ -120,10 +120,11 @@ func measurementTrendChart(report model.Report, benchmarkIndex int, group chartG ``, path.String(), color, ) - for index, value := range values { - if !finiteChartValue(value) { + for index, run := range runs { + if !chartRunAvailable(metric, run) { continue } + value := metric.run(run) body.WriteString(`%s`+ `%s ... %s`, - legendY, color, html.EscapeString(clip(metric.name, 24)), legendY+12, + legendY, color, html.EscapeString(clip(metric.chartName, 24)), legendY+12, metric.format(minimum), metric.format(maximum), ) } @@ -150,24 +151,13 @@ func measurementTrendChart(report model.Report, benchmarkIndex int, group chartG } } -func trendValues(runs []model.Run, metric chartMetric) []float64 { - values := make([]float64, len(runs)) - for index, run := range runs { - if chartRunAvailable(metric, run) { - values[index] = metric.run(run) - } else { - values[index] = math.NaN() - } - } - return values -} - -func valueRange(values []float64) (float64, float64) { +func trendMetricRange(runs []model.Run, metric chartMetric) (float64, float64) { minimum, maximum := math.Inf(1), math.Inf(-1) - for _, value := range values { - if !finiteChartValue(value) { + for _, run := range runs { + if !chartRunAvailable(metric, run) { continue } + value := metric.run(run) minimum = math.Min(minimum, value) maximum = math.Max(maximum, value) } @@ -197,37 +187,41 @@ func trendLaneRange(runs []model.Run, metrics []chartMetric) (float64, float64) func trendLanes(report model.Report, benchmarkIndex int, group chartGroup) []trendLane { metrics := trendMetrics(report, benchmarkIndex, group.metrics) - byRow := make(map[int]chartMetric, len(metrics)) + byRow := make(map[metricID]chartMetric, len(metrics)) for _, metric := range metrics { - byRow[metric.row] = metric + byRow[metric.id] = metric } switch group.name { case "PERFORMANCE": - return trendLaneDefinitions(byRow, trendLaneSpec{"Wall time", []int{0}}) + return trendLaneDefinitions(byRow, trendLaneSpec{"Wall time", []metricID{metricWall}}) case "CPU COST": return trendLaneDefinitions( byRow, - trendLaneSpec{"CPU time", []int{1, 2, 3}}, - trendLaneSpec{"Average CPU", []int{4}}, + trendLaneSpec{"CPU time", []metricID{metricCPUTotal, metricCPUUser, metricCPUSystem}}, + trendLaneSpec{"Average CPU", []metricID{metricAverageCPU}}, ) case "MEMORY COST": return trendLaneDefinitions( byRow, - trendLaneSpec{"Resident memory", []int{7, 6, 5}}, - trendLaneSpec{"Physical footprint", []int{8}}, - trendLaneSpec{"Virtual memory", []int{9}}, + trendLaneSpec{"Resident memory", []metricID{ + metricOSMaxRSS, metricPeakResident, metricMeanResident, + }}, + trendLaneSpec{"Physical footprint", []metricID{metricPhysicalFootprint}}, + trendLaneSpec{"Virtual memory", []metricID{metricPeakVirtual}}, ) case "PROCESS STRUCTURE": return trendLaneDefinitions( byRow, - trendLaneSpec{"Processes and threads", []int{11, 10}}, - trendLaneSpec{"FD references", []int{12}}, + trendLaneSpec{"Processes and threads", []metricID{ + metricPeakThreads, metricPeakProcesses, + }}, + trendLaneSpec{"FD references", []metricID{metricPeakFDs}}, ) default: lanes := make([]trendLane, 0, len(metrics)) for _, metric := range metrics { lanes = append(lanes, trendLane{ - name: metric.name, metrics: []chartMetric{metric}, format: metric.format, + name: metric.chartName, metrics: []chartMetric{metric}, format: metric.format, }) } return lanes @@ -236,10 +230,10 @@ func trendLanes(report model.Report, benchmarkIndex int, group chartGroup) []tre type trendLaneSpec struct { name string - rows []int + rows []metricID } -func trendLaneDefinitions(byRow map[int]chartMetric, specs ...trendLaneSpec) []trendLane { +func trendLaneDefinitions(byRow map[metricID]chartMetric, specs ...trendLaneSpec) []trendLane { lanes := make([]trendLane, 0, len(specs)) for _, spec := range specs { metrics := make([]chartMetric, 0, len(spec.rows)) @@ -261,9 +255,9 @@ func trendMetrics( ) []chartMetric { result := make([]chartMetric, 0, len(metrics)) for _, metric := range metrics { - stats := metricRows[metric.row].stats(report.Benchmarks[benchmarkIndex].Summary) - if stats.N > 0 && finiteChartValue(stats.Mean) && availableFor( - metricRows[metric.row], report.Host.OS, + stats := metric.stats(report.Benchmarks[benchmarkIndex].Summary) + if stats.N > 0 && finiteNumber(stats.Mean) && availableFor( + metric, report.Host.OS, report.Benchmarks[benchmarkIndex].Summary, ) { result = append(result, metric) @@ -320,7 +314,7 @@ func trendLaneNames(lanes []trendLane) string { func trendMetricNames(metrics []chartMetric) string { names := make([]string, 0, len(metrics)) for _, metric := range metrics { - names = append(names, metric.name) + names = append(names, metric.chartName) } return strings.Join(names, ", ") } diff --git a/internal/report/chart_types.go b/internal/report/chart_types.go index 795ca35..4d819fc 100644 --- a/internal/report/chart_types.go +++ b/internal/report/chart_types.go @@ -9,7 +9,6 @@ import ( "strings" "unicode" - "github.com/shellcell/snailrace/internal/model" "github.com/shellcell/snailrace/internal/style" ) @@ -32,13 +31,7 @@ type svgChart struct { height int } -type chartMetric struct { - name string - format func(float64) string - stats func(model.Benchmark) model.Stats - run func(model.Run) float64 - row int -} +type chartMetric = metricDefinition type chartGroup struct { name string @@ -61,7 +54,8 @@ func (chart svgChart) writeHTML(writer io.Writer) { chartWidth, chart.height, chartWidth, ) chart.writeMetadata(writer) - fmt.Fprint(writer, chart.body, ``) + writeChartString(writer, chart.body) + writeChartString(writer, ``) } func (chart svgChart) writeEmbedded(writer io.Writer, x, y int) { @@ -72,7 +66,16 @@ func (chart svgChart) writeEmbedded(writer io.Writer, x, y int) { x, y, chartWidth, chart.height, chartWidth, chart.height, ) chart.writeMetadata(writer) - fmt.Fprint(writer, chart.body, ``) + writeChartString(writer, chart.body) + writeChartString(writer, ``) +} + +func writeChartString(writer io.Writer, value string) { + if stringWriter, ok := writer.(io.StringWriter); ok { + _, _ = stringWriter.WriteString(value) + return + } + _, _ = fmt.Fprint(writer, value) } func (chart svgChart) writeMetadata(writer io.Writer) { @@ -96,11 +99,7 @@ func (chart svgChart) explanation() string { return chart.accessibleTitle() } -func chartColor(index, _ int) string { - return style.Tool(index).Hex -} - -func toolColor(_ model.Report, index int) string { +func toolColor(index int) string { return style.Tool(index).Hex } @@ -146,7 +145,7 @@ func niceDeltaScale(value float64) float64 { } } -func finiteChartValue(value float64) bool { +func finiteNumber(value float64) bool { return !math.IsNaN(value) && !math.IsInf(value, 0) } diff --git a/internal/report/command_legend.go b/internal/report/command_legend.go index ba02f6f..b9d2672 100644 --- a/internal/report/command_legend.go +++ b/internal/report/command_legend.go @@ -1,11 +1,8 @@ package report import ( - "strconv" - "strings" - "unicode" - "github.com/shellcell/snailrace/internal/model" + "github.com/shellcell/snailrace/internal/style" ) func fullCommand(benchmark model.Benchmark) string { @@ -13,32 +10,5 @@ func fullCommand(benchmark model.Benchmark) string { if len(command) == 0 { return "" } - if benchmark.Tool.ShellCommand { - return safeCommandText(command[0]) - } - arguments := make([]string, len(command)) - for index, argument := range command { - arguments[index] = shellQuoteArgument(argument) - } - return strings.Join(arguments, " ") -} - -func shellQuoteArgument(argument string) string { - if strings.IndexFunc(argument, unicode.IsControl) >= 0 { - return strconv.QuoteToGraphic(argument) - } - if argument != "" && strings.IndexFunc(argument, func(character rune) bool { - return !(unicode.IsLetter(character) || unicode.IsDigit(character) || - strings.ContainsRune("_@%+=:,./-", character)) - }) < 0 { - return argument - } - return "'" + strings.ReplaceAll(argument, "'", `'"'"'`) + "'" -} - -func safeCommandText(command string) string { - if strings.IndexFunc(command, unicode.IsControl) >= 0 { - return strconv.QuoteToGraphic(command) - } - return command + return style.CommandText(command, benchmark.Tool.ShellCommand) } diff --git a/internal/report/command_legend_svg.go b/internal/report/command_legend_svg.go index 7f997e4..8a18a1c 100644 --- a/internal/report/command_legend_svg.go +++ b/internal/report/command_legend_svg.go @@ -14,7 +14,7 @@ func svgCommandLegend(output *strings.Builder, report model.Report, y int) int { for index, benchmark := range report.Benchmarks { fmt.Fprintf( output, `%d. %s`, - y, toolColor(report, index), index+1, + y, toolColor(index), index+1, html.EscapeString(clip(reportToolLabel(report, index), 80)), ) y += 18 diff --git a/internal/report/command_legend_test.go b/internal/report/command_legend_test.go index 58160b2..5369fc3 100644 --- a/internal/report/command_legend_test.go +++ b/internal/report/command_legend_test.go @@ -34,7 +34,7 @@ func TestRenderedLabelsContainNoTerminalControls(t *testing.T) { }}} for _, format := range []string{"text", "markdown", "html", "svg"} { var output bytes.Buffer - if err := Write(&output, format, report); err != nil { + if err := NewRenderer(report).Write(&output, format); err != nil { t.Fatalf("%s: %v", format, err) } if strings.ContainsAny(output.String(), "\n\x1b") && diff --git a/internal/report/comparison.go b/internal/report/comparison.go index 94c61d9..b943122 100644 --- a/internal/report/comparison.go +++ b/internal/report/comparison.go @@ -35,22 +35,28 @@ func baselineIndex(report model.Report) int { func compareMetric( baseline model.Benchmark, candidate model.Benchmark, - row metricRow, + row metricDefinition, intervalSeconds float64, ) deltaResult { - valid := func(model.Run) bool { return true } - if row.darwinOnly { - valid = func(run model.Run) bool { return run.PhysicalFootprintValid } - } - baseValues, candidateValues, differences := pairedValues( - baseline.Runs, candidate.Runs, row.run, valid, + return compareMetricWithBaselineIndex( + baseline, indexMetricValues(baseline.Runs, row), candidate, row, intervalSeconds, + ) +} + +func compareMetricWithBaselineIndex( + baseline model.Benchmark, + baselineValues map[int]float64, + candidate model.Benchmark, + row metricDefinition, + intervalSeconds float64, +) deltaResult { + baseMean, candidateMean, differences := pairedDifferences( + baselineValues, candidate.Runs, row, ) - baseMean := model.CalculateStats(baseValues).Mean - candidateMean := model.CalculateStats(candidateValues).Mean result := deltaResult{baselineMean: baseMean, candidateMean: candidateMean} if baseMean != 0 { percent := (candidateMean/baseMean - 1) * 100 - if finiteChartValue(percent) { + if finiteNumber(percent) { result.percent = percent result.percentAvailable = true } @@ -63,6 +69,20 @@ func compareMetric( return result } +func indexMetricValues(runs []model.Run, row metricDefinition) map[int]float64 { + values := make(map[int]float64, len(runs)) + for _, run := range runs { + if row.id == metricPhysicalFootprint && !run.PhysicalFootprintValid { + continue + } + value := row.run(run) + if finiteNumber(value) { + values[run.Index] = value + } + } + return values +} + func samplingReliable( baseline, candidate model.Benchmark, intervalSeconds float64, @@ -71,63 +91,47 @@ func samplingReliable( return true } for _, benchmark := range []model.Benchmark{baseline, candidate} { - if !benchmarkSamplesReliable(benchmark, intervalSeconds) { - return false - } - } - return true -} - -func benchmarkSamplesReliable(benchmark model.Benchmark, intervalSeconds float64) bool { - if len(benchmark.Runs) == 0 { - return false - } - if intervalSeconds <= 0 { - return true - } - minimum := intervalSeconds * 2 - for _, run := range benchmark.Runs { - if run.WallSeconds < minimum || run.SampleCount < 2 || - run.SampleCoverageSeconds < intervalSeconds { + if !model.SamplingReliable(benchmark, intervalSeconds) { return false } } return true } -func pairedValues( - baseline, candidate []model.Run, - pick func(model.Run) float64, - valid func(model.Run) bool, -) ([]float64, []float64, []float64) { - baselineByIndex := make(map[int]float64, len(baseline)) - for _, run := range baseline { - if valid(run) { - value := pick(run) - if finiteChartValue(value) { - baselineByIndex[run.Index] = value - } - } - } - baseValues := make([]float64, 0, len(candidate)) - candidateValues := make([]float64, 0, len(candidate)) +func pairedDifferences( + baseline map[int]float64, + candidate []model.Run, + row metricDefinition, +) (float64, float64, []float64) { differences := make([]float64, 0, len(candidate)) + baseMean, candidateMean := 0.0, 0.0 + count := 0 for _, run := range candidate { - if !valid(run) { + if row.id == metricPhysicalFootprint && !run.PhysicalFootprintValid { continue } - value, ok := baselineByIndex[run.Index] + value, ok := baseline[run.Index] if ok { - candidateValue := pick(run) - if !finiteChartValue(candidateValue) { + candidateValue := row.run(run) + if !finiteNumber(candidateValue) { continue } - baseValues = append(baseValues, value) - candidateValues = append(candidateValues, candidateValue) + count++ + baseMean = runningMean(baseMean, value, count) + candidateMean = runningMean(candidateMean, candidateValue, count) differences = append(differences, candidateValue-value) } } - return baseValues, candidateValues, differences + return baseMean, candidateMean, differences +} + +func runningMean(mean, value float64, count int) float64 { + n := float64(count) + mean = mean*(1-1/n) + value/n + if math.IsInf(mean, 0) { + return math.Copysign(math.MaxFloat64, mean) + } + return mean } func comparisonStatus(stats model.Stats, direction metricDirection) (string, string) { @@ -155,7 +159,7 @@ func comparisonStatus(stats model.Stats, direction metricDirection) (string, str return "inconclusive", "uncertain" } -func formatDelta(delta deltaResult, row metricRow) string { +func formatDelta(delta deltaResult) string { change := "Ξ” n/a" if delta.percentAvailable { change = "Ξ” " + formatSignedPercent(delta.percent) @@ -163,7 +167,7 @@ func formatDelta(delta deltaResult, row metricRow) string { return change + " Β· " + delta.status } -func formatDeltaInterval(delta deltaResult, row metricRow) string { +func formatDeltaInterval(delta deltaResult, row metricDefinition) string { if !delta.difference.CI95Valid { return "insufficient paired runs" } diff --git a/internal/report/comparison_html.go b/internal/report/comparison_html.go index 179feaf..14adbba 100644 --- a/internal/report/comparison_html.go +++ b/internal/report/comparison_html.go @@ -9,7 +9,7 @@ import ( ) func writeHTMLComparison(writer io.Writer, renderer *Renderer) { - report := renderer.report + report := renderer.displayReport baselinePosition := baselineIndex(report) baseline := report.Benchmarks[baselinePosition] fmt.Fprintf( @@ -36,10 +36,10 @@ func writeHTMLComparison(writer io.Writer, renderer *Renderer) { `MetricBaseline mean`+ `Candidate meanΞ” mean`) writeHTMLStaticFootprint(writer, baseline, candidate) - for metric, row := range metricRows { + for _, row := range metricCatalog { writeHTMLComparisonRow( writer, report.Host.OS, baseline, candidate, row, - renderer.comparison(index, metric), + renderer.comparison(index, row.id), ) } fmt.Fprint(writer, "") @@ -74,7 +74,7 @@ func writeHTMLComparisonRow( writer io.Writer, operatingSystem string, baseline, candidate model.Benchmark, - row metricRow, + row metricDefinition, delta deltaResult, ) { if !availableFor(row, operatingSystem, baseline.Summary, candidate.Summary) { @@ -88,7 +88,7 @@ func writeHTMLComparisonRow( `%s`, row.name, row.format(delta.baselineMean), delta.class, row.format(delta.candidateMean), delta.class, - html.EscapeString(formatDelta(delta, row)), + html.EscapeString(formatDelta(delta)), html.EscapeString(formatDeltaInterval(delta, row)), ) } diff --git a/internal/report/comparison_markdown.go b/internal/report/comparison_markdown.go index 03023ac..e22727a 100644 --- a/internal/report/comparison_markdown.go +++ b/internal/report/comparison_markdown.go @@ -6,7 +6,7 @@ import ( ) func writeMarkdownComparison(writer io.Writer, renderer *Renderer) { - report := renderer.report + report := renderer.displayReport baselinePosition := baselineIndex(report) baseline := report.Benchmarks[baselinePosition] fmt.Fprintf( @@ -34,18 +34,18 @@ func writeMarkdownComparison(writer io.Writer, renderer *Renderer) { formatBytes(float64(baseline.Tool.DiskFootprintBytes)), formatBytes(float64(candidate.Tool.DiskFootprintBytes)), staticDelta, ) - for metric, row := range metricRows { + for _, row := range metricCatalog { if !availableFor( row, report.Host.OS, baseline.Summary, candidate.Summary, ) { fmt.Fprintf(writer, "| %s | N/A | N/A | N/A | N/A |\n", row.name) continue } - delta := renderer.comparison(index, metric) + delta := renderer.comparison(index, row.id) fmt.Fprintf( writer, "| %s | %s | %s | %s | %s |\n", row.name, row.format(delta.baselineMean), row.format(delta.candidateMean), - formatDelta(delta, row), formatDeltaInterval(delta, row), + formatDelta(delta), formatDeltaInterval(delta, row), ) } fmt.Fprintln(writer) diff --git a/internal/report/comparison_test.go b/internal/report/comparison_test.go index 5e243ad..e71a84d 100644 --- a/internal/report/comparison_test.go +++ b/internal/report/comparison_test.go @@ -8,7 +8,7 @@ import ( ) func TestPairedComparisonRequiresConfidence(t *testing.T) { - row := metricRows[0] + row := metricCatalog[metricWall] baseline := benchmarkWithWallTimes("baseline", 10, 10, 10) better := benchmarkWithWallTimes("better", 8, 8, 8) delta := compareMetric(baseline, better, row, 0) @@ -27,7 +27,7 @@ func TestSinglePairIsInconclusive(t *testing.T) { delta := compareMetric( benchmarkWithWallTimes("baseline", 10), benchmarkWithWallTimes("candidate", 5), - metricRows[0], 0, + metricCatalog[metricWall], 0, ) if delta.status != "inconclusive" { t.Fatalf("status = %q, want inconclusive", delta.status) @@ -38,9 +38,9 @@ func TestZeroBaselineDeltaIsUnavailable(t *testing.T) { delta := compareMetric( benchmarkWithWallTimes("baseline", 0, 0), benchmarkWithWallTimes("candidate", 1, 1), - metricRows[0], 0, + metricCatalog[metricWall], 0, ) - if got := formatDelta(delta, metricRows[0]); got != "Ξ” n/a Β· worse" { + if got := formatDelta(delta); got != "Ξ” n/a Β· worse" { t.Fatalf("delta = %q, want unavailable percentage", got) } if got, _ := compareStaticCost(0, 100); got != "Ξ” n/a Β· worse" { @@ -57,7 +57,7 @@ func TestShortRunsMarkSampledMetricsAsLimited(t *testing.T) { } baseline.Summary = model.Summarize(baseline.Runs) candidate.Summary = model.Summarize(candidate.Runs) - delta := compareMetric(baseline, candidate, metricRows[5], 0.01) + delta := compareMetric(baseline, candidate, metricCatalog[metricMeanResident], 0.01) if delta.status != "sampling-limited" { t.Fatalf("status = %q, want sampling-limited", delta.status) } @@ -74,7 +74,7 @@ func TestTooFewValidSamplesMarksMetricsAsLimited(t *testing.T) { } baseline.Summary = model.Summarize(baseline.Runs) candidate.Summary = model.Summarize(candidate.Runs) - delta := compareMetric(baseline, candidate, metricRows[5], 0.01) + delta := compareMetric(baseline, candidate, metricCatalog[metricMeanResident], 0.01) if delta.status != "sampling-limited" { t.Fatalf("status = %q, want sampling-limited", delta.status) } @@ -91,7 +91,7 @@ func TestPhysicalFootprintComparisonExcludesInvalidPairs(t *testing.T) { } baseline := model.Benchmark{Runs: baselineRuns, Summary: model.Summarize(baselineRuns)} candidate := model.Benchmark{Runs: candidateRuns, Summary: model.Summarize(candidateRuns)} - result := compareMetric(baseline, candidate, metricRows[8], 0) + result := compareMetric(baseline, candidate, metricCatalog[metricPhysicalFootprint], 0) if result.difference.N != 1 || result.difference.Mean != -10 { t.Fatalf("physical footprint difference = %+v", result.difference) } @@ -107,7 +107,7 @@ func TestComparisonDropsPairWithNonFiniteValue(t *testing.T) { result := compareMetric( model.Benchmark{Runs: baselineRuns, Summary: model.Summarize(baselineRuns)}, model.Benchmark{Runs: candidateRuns, Summary: model.Summarize(candidateRuns)}, - metricRows[0], 0, + metricCatalog[metricWall], 0, ) if result.difference.N != 0 || result.percentAvailable { t.Fatalf("non-finite pair produced a comparison: %+v", result) diff --git a/internal/report/comparison_text.go b/internal/report/comparison_text.go index b272715..108ec46 100644 --- a/internal/report/comparison_text.go +++ b/internal/report/comparison_text.go @@ -6,7 +6,7 @@ import ( ) func writeTextComparison(writer io.Writer, renderer *Renderer) { - report := renderer.report + report := renderer.displayReport baselinePosition := baselineIndex(report) baseline := report.Benchmarks[baselinePosition] for index, candidate := range report.Benchmarks { @@ -30,18 +30,18 @@ func writeTextComparison(writer io.Writer, renderer *Renderer) { formatBytes(float64(baseline.Tool.DiskFootprintBytes)), formatBytes(float64(candidate.Tool.DiskFootprintBytes)), staticDelta, ) - for metric, row := range metricRows { + for _, row := range metricCatalog { if !availableFor( row, report.Host.OS, baseline.Summary, candidate.Summary, ) { fmt.Fprintf(writer, "%s\tN/A\tN/A\tN/A\tN/A\n", row.name) continue } - delta := renderer.comparison(index, metric) + delta := renderer.comparison(index, row.id) fmt.Fprintf( writer, "%s\t%s\t%s\t%s\t%s\n", row.name, row.format(delta.baselineMean), row.format(delta.candidateMean), - formatDelta(delta, row), formatDeltaInterval(delta, row), + formatDelta(delta), formatDeltaInterval(delta, row), ) } fmt.Fprintln(writer) diff --git a/internal/report/error_writer_test.go b/internal/report/error_writer_test.go index e6392fe..d2ddb1c 100644 --- a/internal/report/error_writer_test.go +++ b/internal/report/error_writer_test.go @@ -33,7 +33,7 @@ func TestReportFormatsPropagateWriterErrors(t *testing.T) { } for _, format := range []string{"text", "html", "markdown", "svg", "json"} { t.Run(format, func(t *testing.T) { - err := Write(&limitedWriter{}, format, report) + err := NewRenderer(report).Write(&limitedWriter{}, format) if !errors.Is(err, errWriteLimit) { t.Fatalf("error = %v, want write failure", err) } diff --git a/internal/report/format.go b/internal/report/format.go index 1bcba1d..acfce64 100644 --- a/internal/report/format.go +++ b/internal/report/format.go @@ -8,8 +8,29 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -type metricRow struct { +type metricID uint8 + +const ( + metricWall metricID = iota + metricCPUTotal + metricCPUUser + metricCPUSystem + metricAverageCPU + metricMeanResident + metricPeakResident + metricOSMaxRSS + metricPhysicalFootprint + metricPeakVirtual + metricPeakProcesses + metricPeakThreads + metricPeakFDs + metricCount +) + +type metricDefinition struct { + id metricID name string + chartName string stats func(model.Summary) model.Stats format func(float64) string unavailableDarwin bool @@ -19,32 +40,32 @@ type metricRow struct { sampled bool } -var metricRows = []metricRow{ - {"Wall time", func(s model.Summary) model.Stats { return s.WallSeconds }, formatDuration, false, false, +var metricCatalog = [...]metricDefinition{ + {metricWall, "Wall time", "Wall time", func(s model.Summary) model.Stats { return s.WallSeconds }, formatDuration, false, false, func(r model.Run) float64 { return r.WallSeconds }, lowerIsBetter, false}, - {"CPU total", func(s model.Summary) model.Stats { return s.CPUTotalSeconds }, formatDuration, false, false, + {metricCPUTotal, "CPU total", "Total CPU", func(s model.Summary) model.Stats { return s.CPUTotalSeconds }, formatDuration, false, false, func(r model.Run) float64 { return r.CPUUserSeconds + r.CPUSystemSeconds }, lowerIsBetter, false}, - {"CPU user", func(s model.Summary) model.Stats { return s.CPUUserSeconds }, formatDuration, false, false, + {metricCPUUser, "CPU user", "User CPU", func(s model.Summary) model.Stats { return s.CPUUserSeconds }, formatDuration, false, false, func(r model.Run) float64 { return r.CPUUserSeconds }, lowerIsBetter, false}, - {"CPU system", func(s model.Summary) model.Stats { return s.CPUSystemSeconds }, formatDuration, false, false, + {metricCPUSystem, "CPU system", "System CPU", func(s model.Summary) model.Stats { return s.CPUSystemSeconds }, formatDuration, false, false, func(r model.Run) float64 { return r.CPUSystemSeconds }, lowerIsBetter, false}, - {"Average CPU", func(s model.Summary) model.Stats { return s.AverageCPUPercent }, formatPercent, false, false, + {metricAverageCPU, "Average CPU", "Average CPU", func(s model.Summary) model.Stats { return s.AverageCPUPercent }, formatPercent, false, false, func(r model.Run) float64 { return r.AverageCPUPercent }, neutralDirection, false}, - {"Mean resident", func(s model.Summary) model.Stats { return s.MeanResidentBytes }, formatBytes, false, false, + {metricMeanResident, "Mean resident", "Mean RSS", func(s model.Summary) model.Stats { return s.MeanResidentBytes }, formatBytes, false, false, func(r model.Run) float64 { return r.MeanResidentBytes }, lowerIsBetter, true}, - {"Peak resident", func(s model.Summary) model.Stats { return s.PeakResidentBytes }, formatBytes, false, false, + {metricPeakResident, "Peak resident", "Peak RSS", func(s model.Summary) model.Stats { return s.PeakResidentBytes }, formatBytes, false, false, func(r model.Run) float64 { return r.PeakResidentBytes }, lowerIsBetter, true}, - {"OS-reported max RSS", func(s model.Summary) model.Stats { return s.OSMaxRSSBytes }, formatBytes, false, false, + {metricOSMaxRSS, "OS-reported max RSS", "OS-reported max RSS", func(s model.Summary) model.Stats { return s.OSMaxRSSBytes }, formatBytes, false, false, func(r model.Run) float64 { return r.OSMaxRSSBytes }, lowerIsBetter, false}, - {"Peak physical footprint", func(s model.Summary) model.Stats { return s.PhysicalFootprintStats() }, formatBytes, false, true, + {metricPhysicalFootprint, "Peak physical footprint", "Peak physical footprint", func(s model.Summary) model.Stats { return s.PhysicalFootprintStats() }, formatBytes, false, true, func(r model.Run) float64 { return r.PeakPhysicalFootprintBytes }, lowerIsBetter, true}, - {"Peak virtual", func(s model.Summary) model.Stats { return s.PeakVirtualBytes }, formatBytes, false, false, + {metricPeakVirtual, "Peak virtual", "Peak virtual memory", func(s model.Summary) model.Stats { return s.PeakVirtualBytes }, formatBytes, false, false, func(r model.Run) float64 { return r.PeakVirtualBytes }, lowerIsBetter, true}, - {"Peak processes", func(s model.Summary) model.Stats { return s.PeakProcesses }, formatCount, false, false, + {metricPeakProcesses, "Peak processes", "Peak processes", func(s model.Summary) model.Stats { return s.PeakProcesses }, formatCount, false, false, func(r model.Run) float64 { return r.PeakProcesses }, neutralDirection, true}, - {"Peak threads", func(s model.Summary) model.Stats { return s.PeakThreads }, formatCount, false, false, + {metricPeakThreads, "Peak threads", "Peak threads", func(s model.Summary) model.Stats { return s.PeakThreads }, formatCount, false, false, func(r model.Run) float64 { return r.PeakThreads }, neutralDirection, true}, - {"Peak FD references", func(s model.Summary) model.Stats { return s.PeakFileDescriptors }, formatCount, true, false, + {metricPeakFDs, "Peak FD references", "Peak FD references", func(s model.Summary) model.Stats { return s.PeakFileDescriptors }, formatCount, true, false, func(r model.Run) float64 { return r.PeakFileDescriptors }, lowerIsBetter, true}, } @@ -118,13 +139,13 @@ func formatNumber(value float64) string { return result } -func available(row metricRow, operatingSystem string) bool { +func available(row metricDefinition, operatingSystem string) bool { return (operatingSystem != "darwin" || !row.unavailableDarwin) && (!row.darwinOnly || operatingSystem == "darwin") } func availableFor( - row metricRow, operatingSystem string, summaries ...model.Summary, + row metricDefinition, operatingSystem string, summaries ...model.Summary, ) bool { if !available(row, operatingSystem) { return false diff --git a/internal/report/formats.go b/internal/report/formats.go new file mode 100644 index 0000000..649ab06 --- /dev/null +++ b/internal/report/formats.go @@ -0,0 +1,44 @@ +package report + +import "strings" + +type Format string + +const ( + FormatHTML Format = "html" + FormatSVG Format = "svg" + FormatMarkdown Format = "markdown" + FormatJSON Format = "json" + FormatText Format = "text" +) + +type FormatInfo struct { + Name Format + Extension string + NeedsCharts bool +} + +var formatCatalog = map[Format]FormatInfo{ + FormatHTML: {Name: FormatHTML, Extension: "html"}, + FormatSVG: {Name: FormatSVG, Extension: "svg", NeedsCharts: true}, + FormatMarkdown: {Name: FormatMarkdown, Extension: "md", NeedsCharts: true}, + FormatJSON: {Name: FormatJSON, Extension: "json"}, + FormatText: {Name: FormatText, Extension: "txt"}, +} + +func LookupFormat(value string) (FormatInfo, bool) { + value = strings.ToLower(strings.TrimSpace(value)) + switch value { + case "md": + value = string(FormatMarkdown) + case "txt": + value = string(FormatText) + } + info, ok := formatCatalog[Format(value)] + return info, ok +} + +func ValidFormat(format string) bool { + _, ok := LookupFormat(format) + return ok +} diff --git a/internal/report/formats_test.go b/internal/report/formats_test.go new file mode 100644 index 0000000..cd28260 --- /dev/null +++ b/internal/report/formats_test.go @@ -0,0 +1,29 @@ +package report + +import "testing" + +func TestFormatCatalogCanonicalizesAliasesAndMetadata(t *testing.T) { + tests := []struct { + input string + name Format + extension string + charts bool + }{ + {"HTML", FormatHTML, "html", false}, + {"svg", FormatSVG, "svg", true}, + {"md", FormatMarkdown, "md", true}, + {"markdown", FormatMarkdown, "md", true}, + {"json", FormatJSON, "json", false}, + {"txt", FormatText, "txt", false}, + } + for _, test := range tests { + info, ok := LookupFormat(test.input) + if !ok || info.Name != test.name || info.Extension != test.extension || + info.NeedsCharts != test.charts { + t.Fatalf("format %q = %+v, valid=%v", test.input, info, ok) + } + } + if ValidFormat("unknown") { + t.Fatal("unknown format should be invalid") + } +} diff --git a/internal/report/html.go b/internal/report/html.go index f008d5e..42ac5d6 100644 --- a/internal/report/html.go +++ b/internal/report/html.go @@ -57,12 +57,8 @@ th:first-child,td:first-child{text-align:left}code{color:var(--accent);white-spa .command-item{grid-template-columns:1fr}}
` -func writeHTML(writer io.Writer, report model.Report) error { - return writeHTMLRenderer(writer, NewRenderer(report)) -} - -func writeHTMLRenderer(writer io.Writer, renderer *Renderer) error { - report := renderer.report +func writeHTML(writer io.Writer, renderer *Renderer) error { + report := renderer.displayReport checked := newErrorWriter(writer) writer = checked fmt.Fprint(writer, htmlStart) @@ -86,7 +82,7 @@ func writeHTMLRenderer(writer io.Writer, renderer *Renderer) error { writeHTMLCommandLegend(writer, report) writeHTMLFailures(writer, report) if len(report.Benchmarks) > 1 { - writeHTMLRankingWith(writer, report, renderer.ranking) + writeHTMLRanking(writer, report, renderer.ranking) } charts := renderer.reportCharts() for _, section := range chartSections(charts) { @@ -246,7 +242,7 @@ func writeHTMLBenchmark( formatCount(benchmark.Summary.ValidSampleCount.Mean), formatDuration(benchmark.Summary.SampleCoverageSeconds.Mean), ) - if !benchmarkSamplesReliable(benchmark, report.Config.IntervalMS/1000) { + if !model.SamplingReliable(benchmark, report.Config.IntervalMS/1000) { fmt.Fprint( writer, `

Sampling quality: LIMITED `+ @@ -256,7 +252,7 @@ func writeHTMLBenchmark( fmt.Fprint(writer, `

`+ ``+ ``) - for _, row := range metricRows { + for _, row := range metricCatalog { writeHTMLRow(writer, operatingSystem, row, benchmark.Summary) } fmt.Fprint(writer, "
MetricMean Β± Οƒ95% CI meanMedianP95Range
") @@ -270,7 +266,7 @@ func writeHTMLBenchmark( func writeHTMLRow( writer io.Writer, operatingSystem string, - row metricRow, + row metricDefinition, summary model.Summary, ) { if !availableFor(row, operatingSystem, summary) { diff --git a/internal/report/labels_html.go b/internal/report/labels_html.go index a13a7d3..55d46be 100644 --- a/internal/report/labels_html.go +++ b/internal/report/labels_html.go @@ -10,7 +10,7 @@ import ( func htmlToolLabel(report model.Report, index int, badge bool) string { label := fmt.Sprintf( `%s`, - toolColor(report, index), + toolColor(index), html.EscapeString(report.Benchmarks[index].Tool.Name), ) if badge && index == baselineIndex(report) { diff --git a/internal/report/markdown.go b/internal/report/markdown.go index d0fef62..cc3dae1 100644 --- a/internal/report/markdown.go +++ b/internal/report/markdown.go @@ -8,21 +8,12 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func WriteMarkdownWithCharts( - writer io.Writer, - report model.Report, - charts []ChartArtifact, - chartDirectory string, -) error { - return NewRenderer(report).WriteMarkdownWithCharts(writer, charts, chartDirectory) -} - func (renderer *Renderer) WriteMarkdownWithCharts( writer io.Writer, charts []ChartArtifact, chartDirectory string, ) error { - report := renderer.report + report := renderer.displayReport checked := newErrorWriter(writer) writer = checked fmt.Fprintf( @@ -104,7 +95,7 @@ func writeMarkdownBenchmark( formatCount(benchmark.Summary.ValidSampleCount.Mean), formatDuration(benchmark.Summary.SampleCoverageSeconds.Mean), ) - if !benchmarkSamplesReliable(benchmark, intervalSeconds) { + if !model.SamplingReliable(benchmark, intervalSeconds) { fmt.Fprintln( writer, "> Sampling quality: **LIMITED** (fewer than two valid samples or intervals).", @@ -116,7 +107,7 @@ func writeMarkdownBenchmark( "| Metric | Mean Β± Οƒ | 95% CI mean | Median | P95 | Range |", ) fmt.Fprintln(writer, "|---|---:|---:|---:|---:|---:|") - for _, row := range metricRows { + for _, row := range metricCatalog { if !availableFor(row, operatingSystem, benchmark.Summary) { fmt.Fprintf(writer, "| %s | N/A | N/A | N/A | N/A | N/A |\n", row.name) continue diff --git a/internal/report/metric_catalog_test.go b/internal/report/metric_catalog_test.go new file mode 100644 index 0000000..aa8e613 --- /dev/null +++ b/internal/report/metric_catalog_test.go @@ -0,0 +1,47 @@ +package report + +import ( + "testing" + + "github.com/shellcell/snailrace/internal/model" +) + +func TestMetricCatalogIdentityAndSelectors(t *testing.T) { + if len(metricCatalog) != int(metricCount) { + t.Fatalf("metric catalog length = %d, want %d", len(metricCatalog), metricCount) + } + run := model.Run{ + WallSeconds: 1, CPUUserSeconds: 2, CPUSystemSeconds: 3, + AverageCPUPercent: 4, MeanResidentBytes: 5, PeakResidentBytes: 6, + OSMaxRSSBytes: 7, PeakPhysicalFootprintBytes: 8, + PhysicalFootprintValid: true, PeakVirtualBytes: 9, + PeakProcesses: 10, PeakThreads: 11, PeakFileDescriptors: 12, + } + summary := model.Summarize([]model.Run{run}) + for index, metric := range metricCatalog { + if metric.id != metricID(index) { + t.Fatalf("metric %d has ID %d", index, metric.id) + } + if metric.name == "" || metric.chartName == "" || metric.format == nil { + t.Fatalf("metric %d has incomplete presentation metadata", metric.id) + } + if got, want := metric.stats(summary).Mean, metric.run(run); got != want { + t.Fatalf("metric %d summary = %v, run = %v", metric.id, got, want) + } + } +} + +func TestChartGroupsReferenceEveryMetricOnce(t *testing.T) { + groups := chartGroups(model.Report{Config: model.Config{Mode: "command"}}) + seen := make(map[metricID]int) + for _, group := range groups { + for _, metric := range group.metrics { + seen[metric.id]++ + } + } + for _, metric := range metricCatalog { + if seen[metric.id] != 1 { + t.Fatalf("metric %d appears in %d chart groups", metric.id, seen[metric.id]) + } + } +} diff --git a/internal/report/ranking.go b/internal/report/ranking.go index 78acb63..bb2bc38 100644 --- a/internal/report/ranking.go +++ b/internal/report/ranking.go @@ -8,32 +8,13 @@ import ( func calculateRanking(report model.Report) rankingData { numeric := analysis.Calculate(report.Config, report.Benchmarks) result := rankingData{ - rows: make([]rankingRow, len(numeric.Rows)), - available: numeric.Available, unavailableReason: numeric.UnavailableReason, - ramAvailable: numeric.RAMAvailable, ramPresent: numeric.RAMPresent, - bestOverall: numeric.BestOverall, - primaryRatio: numeric.PrimaryRatio, cpuRatio: numeric.CPURatio, - footprintRatio: numeric.FootprintRatio, - indexPrimary: numeric.IndexPrimary, indexCPU: numeric.IndexCPU, - indexRAM: numeric.IndexRAM, indexFootprint: numeric.IndexFootprint, + Ranking: numeric, } - if result.ramPresent && !result.ramAvailable { + if result.RAMPresent && !result.RAMAvailable { result.samplingInterval = currentIntervalLabel(report.Config) } - for index, row := range numeric.Rows { - result.rows[index] = rankingRow{ - benchmark: row.Benchmark, overallRank: row.OverallRank, - primaryRank: row.PrimaryRank, cpuRank: row.CPURank, - ramRank: row.RAMRank, footprintRank: row.FootprintRank, - overallScore: row.OverallScore, primaryScore: row.PrimaryScore, - cpuScore: row.CPUScore, ramScore: row.RAMScore, - footprintScore: row.FootprintScore, primaryValue: row.PrimaryValue, - cpuValue: row.CPUValue, ramValue: row.RAMValue, - footprintValue: row.FootprintValue, - } - } result.primaryLabel, result.primaryUnit = "TIME", formatDuration - if numeric.FixedTUI { + if result.FixedTUI { result.primaryLabel, result.primaryUnit = "CPU", formatPercent } result.winners = rankingWinners(report, result) diff --git a/internal/report/ranking_html.go b/internal/report/ranking_html.go index dbfd6eb..159d382 100644 --- a/internal/report/ranking_html.go +++ b/internal/report/ranking_html.go @@ -9,15 +9,11 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func writeHTMLRanking(writer io.Writer, report model.Report) { - writeHTMLRankingWith(writer, report, calculateRanking(report)) -} - -func writeHTMLRankingWith(writer io.Writer, report model.Report, ranking rankingData) { - if len(ranking.rows) == 0 { +func writeHTMLRanking(writer io.Writer, report model.Report, ranking rankingData) { + if len(ranking.Rows) == 0 { fmt.Fprintf( writer, `

Ranking unavailable

%s

`, - html.EscapeString(ranking.unavailableReason), + html.EscapeString(ranking.UnavailableReason), ) return } @@ -38,10 +34,10 @@ func writeHTMLRankingWith(writer io.Writer, report model.Report, ranking ranking html.EscapeString(winner.value), ) } - if !ranking.available { + if !ranking.Available { fmt.Fprintf( writer, `

Overall ranking unavailable

%s

`, - html.EscapeString(ranking.unavailableReason), + html.EscapeString(ranking.UnavailableReason), ) return } @@ -60,20 +56,20 @@ func writeHTMLRankingWith(writer io.Writer, report model.Report, ranking ranking rankingHeader(5, "RAM aggregate", "number")+ rankingHeader(6, "Linked size", "number")+ ``) - for _, row := range ranking.rows { + for _, row := range ranking.Rows { ramValue := "N/A" - if ranking.ramAvailable { + if ranking.RAMAvailable { ramValue = rankingHTMLCell( - formatBytes(row.ramValue), row.ramScore, row.ramRank, true, + formatBytes(row.RAMValue), row.RAMScore, row.RAMRank, true, ) - } else if ranking.ramPresent { + } else if ranking.RAMPresent { ramValue = fmt.Sprintf( `%ssampling-limited`, - html.EscapeString(formatBytes(row.ramValue)), + html.EscapeString(formatBytes(row.RAMValue)), ) } class := "" - if row.overallRank == 1 { + if row.OverallRank == 1 { class = ` class="rank-one"` } fmt.Fprintf( @@ -85,29 +81,29 @@ func writeHTMLRankingWith(writer io.Writer, report model.Report, ranking ranking `%s`+ `%s`+ `%s`, - class, row.overallRank, row.overallRank, - html.EscapeString(report.Benchmarks[row.benchmark].Tool.Name), - rankingHTMLTool(report, row.benchmark), - row.overallScore, + class, row.OverallRank, row.OverallRank, + html.EscapeString(report.Benchmarks[row.Benchmark].Tool.Name), + htmlToolLabel(report, row.Benchmark, true), + row.OverallScore, rankingHTMLCell( - formatScore(row.overallScore), row.overallScore/ranking.bestOverall, - row.overallRank, true, + formatScore(row.OverallScore), row.OverallScore/ranking.BestOverall, + row.OverallRank, true, ), - row.primaryValue, + row.PrimaryValue, rankingHTMLCell( - ranking.primaryUnit(row.primaryValue), row.primaryScore, - row.primaryRank, ranking.primaryRatio, + ranking.primaryUnit(row.PrimaryValue), row.PrimaryScore, + row.PrimaryRank, ranking.PrimaryRatio, ), - row.cpuValue, + row.CPUValue, rankingHTMLCell( - cpuRankingValue(report, row), row.cpuScore, row.cpuRank, ranking.cpuRatio, + cpuRankingValue(report, row), row.CPUScore, row.CPURank, ranking.CPURatio, ), - row.ramValue, + row.RAMValue, ramValue, - row.footprintValue, + row.FootprintValue, rankingHTMLCell( - formatBytes(row.footprintValue), row.footprintScore, - row.footprintRank, ranking.footprintRatio, + formatBytes(row.FootprintValue), row.FootprintScore, + row.FootprintRank, ranking.FootprintRatio, ), ) } @@ -130,10 +126,6 @@ const rankingSortScript = `` -func rankingHTMLTool(report model.Report, index int) string { - return htmlToolLabel(report, index, true) -} - func rankingHTMLCell(value string, score float64, rank int, ratioAvailable bool) string { if math.IsInf(score, 1) || math.IsNaN(score) { return "N/A" diff --git a/internal/report/ranking_json.go b/internal/report/ranking_json.go index b4efb6a..9e7c4ef 100644 --- a/internal/report/ranking_json.go +++ b/internal/report/ranking_json.go @@ -49,12 +49,8 @@ type jsonRanking struct { Rows []jsonRankingRow `json:"rows"` } -func writeJSON(writer io.Writer, report model.Report) error { - return writeJSONRenderer(writer, NewRenderer(report)) -} - -func writeJSONRenderer(writer io.Writer, renderer *Renderer) error { - report := renderer.raw +func writeJSON(writer io.Writer, renderer *Renderer) error { + report := renderer.rawReport payload := struct { model.Report Ranking jsonRanking `json:"ranking"` @@ -84,7 +80,7 @@ func makeJSONFailures(report model.Report) []jsonFailure { func makeJSONRanking(report model.Report, ranking rankingData) jsonRanking { result := jsonRanking{ - Available: ranking.available, UnavailableReason: ranking.unavailableReason, + Available: ranking.Available, UnavailableReason: ranking.UnavailableReason, Method: "equal-weight geometric mean of normalized category costs; included: " + balancedIndexCategories(ranking), PrimaryMetric: ranking.primaryLabel, @@ -96,32 +92,32 @@ func makeJSONRanking(report model.Report, ranking rankingData) jsonRanking { Value: winner.value, }) } - for _, row := range ranking.rows { + for _, row := range ranking.Rows { item := jsonRankingRow{ - Tool: report.Benchmarks[row.benchmark].Tool.Name, - PrimaryValue: row.primaryValue, PrimaryRank: row.primaryRank, - CPUValue: row.cpuValue, CPURank: row.cpuRank, - FootprintBytes: row.footprintValue, FootprintRank: row.footprintRank, + Tool: report.Benchmarks[row.Benchmark].Tool.Name, + PrimaryValue: row.PrimaryValue, PrimaryRank: row.PrimaryRank, + CPUValue: row.CPUValue, CPURank: row.CPURank, + FootprintBytes: row.FootprintValue, FootprintRank: row.FootprintRank, } - if ranking.available { - item.Rank = row.overallRank - item.BalancedIndex = floatPointer(row.overallScore) + if ranking.Available { + item.Rank = row.OverallRank + item.BalancedIndex = floatPointer(row.OverallScore) item.BalancedDeltaPercent = floatPointer( - (row.overallScore/ranking.bestOverall - 1) * 100, + (row.OverallScore/ranking.BestOverall - 1) * 100, ) } - if ranking.primaryRatio { - item.PrimaryDeltaPercent = floatPointer((row.primaryScore - 1) * 100) + if ranking.PrimaryRatio { + item.PrimaryDeltaPercent = floatPointer((row.PrimaryScore - 1) * 100) } - if ranking.cpuRatio { - item.CPUDeltaPercent = floatPointer((row.cpuScore - 1) * 100) + if ranking.CPURatio { + item.CPUDeltaPercent = floatPointer((row.CPUScore - 1) * 100) } - if ranking.footprintRatio { - item.FootprintDeltaPercent = floatPointer((row.footprintScore - 1) * 100) + if ranking.FootprintRatio { + item.FootprintDeltaPercent = floatPointer((row.FootprintScore - 1) * 100) } - if ranking.ramAvailable { - item.RAMValueBytes, item.RAMRank = row.ramValue, row.ramRank - item.RAMDeltaPercent = floatPointer((row.ramScore - 1) * 100) + if ranking.RAMAvailable { + item.RAMValueBytes, item.RAMRank = row.RAMValue, row.RAMRank + item.RAMDeltaPercent = floatPointer((row.RAMScore - 1) * 100) } result.Rows = append(result.Rows, item) } diff --git a/internal/report/ranking_markdown.go b/internal/report/ranking_markdown.go index c44ffdc..6d13157 100644 --- a/internal/report/ranking_markdown.go +++ b/internal/report/ranking_markdown.go @@ -8,10 +8,10 @@ import ( ) func writeMarkdownRankingWith(writer io.Writer, report model.Report, ranking rankingData) { - if len(ranking.rows) == 0 { + if len(ranking.Rows) == 0 { fmt.Fprintf( writer, "## Ranking Unavailable\n\n%s.\n\n", - escapeMarkdown(ranking.unavailableReason), + escapeMarkdown(ranking.UnavailableReason), ) return } @@ -28,10 +28,10 @@ func writeMarkdownRankingWith(writer io.Writer, report model.Report, ranking ran writer, "\nComparison baseline: **%s**. Lower balanced index is better.\n", escapeMarkdown(report.Benchmarks[baselineIndex(report)].Tool.Name), ) - if !ranking.available { + if !ranking.Available { fmt.Fprintf( writer, "\n## Overall Ranking Unavailable\n\n%s.\n\n", - escapeMarkdown(ranking.unavailableReason), + escapeMarkdown(ranking.UnavailableReason), ) return } @@ -42,32 +42,32 @@ func writeMarkdownRankingWith(writer io.Writer, report model.Report, ranking ran ranking.primaryLabel, ) fmt.Fprintln(writer, "|---:|---|---:|---:|---:|---:|---:|") - for _, row := range ranking.rows { + for _, row := range ranking.Rows { ramValue := "N/A" - if ranking.ramAvailable { + if ranking.RAMAvailable { ramValue = rankingCell( - formatBytes(row.ramValue), row.ramScore, row.ramRank, true, + formatBytes(row.RAMValue), row.RAMScore, row.RAMRank, true, ) } fmt.Fprintf( writer, "| #%d | %s | %s | %s | %s | %s | %s |\n", - row.overallRank, - escapeMarkdown(reportToolLabel(report, row.benchmark)), + row.OverallRank, + escapeMarkdown(reportToolLabel(report, row.Benchmark)), rankingCell( - formatScore(row.overallScore), row.overallScore/ranking.bestOverall, - row.overallRank, true, + formatScore(row.OverallScore), row.OverallScore/ranking.BestOverall, + row.OverallRank, true, ), rankingCell( - ranking.primaryUnit(row.primaryValue), row.primaryScore, - row.primaryRank, ranking.primaryRatio, + ranking.primaryUnit(row.PrimaryValue), row.PrimaryScore, + row.PrimaryRank, ranking.PrimaryRatio, ), rankingCell( - cpuRankingValue(report, row), row.cpuScore, row.cpuRank, ranking.cpuRatio, + cpuRankingValue(report, row), row.CPUScore, row.CPURank, ranking.CPURatio, ), ramValue, rankingCell( - formatBytes(row.footprintValue), row.footprintScore, - row.footprintRank, ranking.footprintRatio, + formatBytes(row.FootprintValue), row.FootprintScore, + row.FootprintRank, ranking.FootprintRatio, ), ) } diff --git a/internal/report/ranking_test.go b/internal/report/ranking_test.go index dee9845..248ca30 100644 --- a/internal/report/ranking_test.go +++ b/internal/report/ranking_test.go @@ -7,21 +7,22 @@ import ( "github.com/shellcell/snailrace/internal/model" ) -func TestAutomaticBaselineUsesBalancedWinner(t *testing.T) { +func TestBalancedWinnerSelectsBaseline(t *testing.T) { benchmarks := []model.Benchmark{ rankingBenchmark("fast", 1, 3, 300, 300, 300), rankingBenchmark("middle", 2, 2, 200, 200, 200), rankingBenchmark("efficient", 3, 1, 100, 100, 100), } config := model.Config{Mode: "command"} - if got, ok := analysis.AutomaticBaseline(config, benchmarks); !ok || got != 3 { - t.Fatalf("baseline = %d, want efficient tool at index 3", got) + numeric := analysis.Calculate(config, benchmarks) + if !numeric.Available || numeric.Rows[0].Benchmark+1 != 3 { + t.Fatalf("baseline = %d, want efficient tool at index 3", numeric.Rows[0].Benchmark+1) } ranking := calculateRanking(model.Report{Config: config, Benchmarks: benchmarks}) if ranking.winners[1].benchmarks[0] != 0 || ranking.winners[2].benchmarks[0] != 2 { t.Fatalf("unexpected category winners: %+v", ranking.winners) } - if ranking.rows[0].overallScore/ranking.bestOverall != 1 { + if ranking.Rows[0].OverallScore/ranking.BestOverall != 1 { t.Fatal("best overall row should have zero difference from best") } } @@ -35,9 +36,9 @@ func TestCompositeRAMWinnerIsNormalizedToOne(t *testing.T) { ranking := calculateRanking(model.Report{ Config: model.Config{Mode: "command"}, Benchmarks: benchmarks, }) - for _, row := range ranking.rows { - if row.ramRank == 1 && row.ramScore != 1 { - t.Fatalf("RAM winner score = %g, want 1", row.ramScore) + for _, row := range ranking.Rows { + if row.RAMRank == 1 && row.RAMScore != 1 { + t.Fatalf("RAM winner score = %g, want 1", row.RAMScore) } } } @@ -63,10 +64,10 @@ func TestZeroBestCategoryHasNoRatioDelta(t *testing.T) { ranking := calculateRanking(model.Report{ Config: model.Config{Mode: "command"}, Benchmarks: benchmarks, }) - if ranking.cpuRatio { + if ranking.CPURatio { t.Fatal("CPU ratio should be unavailable when the best value is zero") } - if got := rankingCell("0 ns", ranking.rows[0].cpuScore, 1, false); got != "0 ns Β· Ξ” n/a (#1)" { + if got := rankingCell("0 ns", ranking.Rows[0].CPUScore, 1, false); got != "0 ns Β· Ξ” n/a (#1)" { t.Fatalf("zero-best ranking cell = %q", got) } } diff --git a/internal/report/ranking_text.go b/internal/report/ranking_text.go index 1becf17..71d09cd 100644 --- a/internal/report/ranking_text.go +++ b/internal/report/ranking_text.go @@ -9,8 +9,8 @@ import ( ) func writeTextRankingWith(writer io.Writer, report model.Report, ranking rankingData) { - if len(ranking.rows) == 0 { - fmt.Fprintf(writer, "Ranking unavailable\t%s\n\n", ranking.unavailableReason) + if len(ranking.Rows) == 0 { + fmt.Fprintf(writer, "Ranking unavailable\t%s\n\n", ranking.UnavailableReason) return } fmt.Fprintln(writer, "Category leaders (point estimates)\tTool\tValue") @@ -25,8 +25,8 @@ func writeTextRankingWith(writer io.Writer, report model.Report, ranking ranking writer, "Comparison baseline\t%s\n\n", report.Benchmarks[baselineIndex(report)].Tool.Name, ) - if !ranking.available { - fmt.Fprintf(writer, "Overall ranking unavailable\t%s\n\n", ranking.unavailableReason) + if !ranking.Available { + fmt.Fprintf(writer, "Overall ranking unavailable\t%s\n\n", ranking.UnavailableReason) return } fmt.Fprintf( @@ -34,31 +34,31 @@ func writeTextRankingWith(writer io.Writer, report model.Report, ranking ranking "Overall ranking (point estimates)\tTool\tBalanced\t%s\tCPU cost\tRAM aggregate\tLinked size\n", ranking.primaryLabel, ) - for _, row := range ranking.rows { + for _, row := range ranking.Rows { ramValue := "N/A" - if ranking.ramAvailable { + if ranking.RAMAvailable { ramValue = rankingCell( - formatBytes(row.ramValue), row.ramScore, row.ramRank, true, + formatBytes(row.RAMValue), row.RAMScore, row.RAMRank, true, ) } fmt.Fprintf( writer, "#%d\t%s\t%s\t%s\t%s\t%s\t%s\n", - row.overallRank, reportToolLabel(report, row.benchmark), + row.OverallRank, reportToolLabel(report, row.Benchmark), rankingCell( - formatScore(row.overallScore), row.overallScore/ranking.bestOverall, - row.overallRank, true, + formatScore(row.OverallScore), row.OverallScore/ranking.BestOverall, + row.OverallRank, true, ), rankingCell( - ranking.primaryUnit(row.primaryValue), row.primaryScore, - row.primaryRank, ranking.primaryRatio, + ranking.primaryUnit(row.PrimaryValue), row.PrimaryScore, + row.PrimaryRank, ranking.PrimaryRatio, ), rankingCell( - cpuRankingValue(report, row), row.cpuScore, row.cpuRank, ranking.cpuRatio, + cpuRankingValue(report, row), row.CPUScore, row.CPURank, ranking.CPURatio, ), ramValue, rankingCell( - formatBytes(row.footprintValue), row.footprintScore, - row.footprintRank, ranking.footprintRatio, + formatBytes(row.FootprintValue), row.FootprintScore, + row.FootprintRank, ranking.FootprintRatio, ), ) } @@ -78,10 +78,10 @@ func rankingCell(value string, score float64, rank int, ratioAvailable bool) str } func cpuRankingValue(report model.Report, row rankingRow) string { - if report.Config.Mode == "tui" && report.Config.DurationSeconds > 0 { - return formatPercent(row.cpuValue) + if report.Config.FixedDurationTUI() { + return formatPercent(row.CPUValue) } - return formatDuration(row.cpuValue) + return formatDuration(row.CPUValue) } func formatScore(value float64) string { diff --git a/internal/report/ranking_types.go b/internal/report/ranking_types.go index 9999145..a598dcf 100644 --- a/internal/report/ranking_types.go +++ b/internal/report/ranking_types.go @@ -1,22 +1,8 @@ package report -type rankingRow struct { - benchmark int - overallRank int - primaryRank int - cpuRank int - ramRank int - footprintRank int - overallScore float64 - primaryScore float64 - cpuScore float64 - ramScore float64 - footprintScore float64 - primaryValue float64 - cpuValue float64 - ramValue float64 - footprintValue float64 -} +import "github.com/shellcell/snailrace/internal/analysis" + +type rankingRow = analysis.RankingRow type categoryWinner struct { category string @@ -25,21 +11,9 @@ type categoryWinner struct { } type rankingData struct { - rows []rankingRow - winners []categoryWinner - available bool - unavailableReason string - primaryLabel string - primaryUnit func(float64) string - ramAvailable bool - ramPresent bool - samplingInterval string - primaryRatio bool - cpuRatio bool - footprintRatio bool - indexPrimary bool - indexCPU bool - indexRAM bool - indexFootprint bool - bestOverall float64 + analysis.Ranking + winners []categoryWinner + primaryLabel string + primaryUnit func(float64) string + samplingInterval string } diff --git a/internal/report/ranking_winners.go b/internal/report/ranking_winners.go index d542ff0..60f5022 100644 --- a/internal/report/ranking_winners.go +++ b/internal/report/ranking_winners.go @@ -8,7 +8,7 @@ import ( ) func rankingWinners(report model.Report, ranking rankingData) []categoryWinner { - if len(ranking.rows) == 0 { + if len(ranking.Rows) == 0 { return nil } var winners []categoryWinner @@ -17,43 +17,42 @@ func rankingWinners(report model.Report, ranking rankingData) []categoryWinner { winners = append(winners, winner) } } - if ranking.available { - appendWinner(winnerForRank(report, ranking.rows, "OVERALL", func(row rankingRow) int { - return row.overallRank + if ranking.Available { + appendWinner(winnerForRank(ranking.Rows, "OVERALL", func(row rankingRow) int { + return row.OverallRank }, func(row rankingRow) string { - return fmt.Sprintf("%.3fx balanced index", row.overallScore) + return fmt.Sprintf("%.3fx balanced index", row.OverallScore) })) } - appendWinner(winnerForRank(report, ranking.rows, ranking.primaryLabel, func(row rankingRow) int { - return row.primaryRank + appendWinner(winnerForRank(ranking.Rows, ranking.primaryLabel, func(row rankingRow) int { + return row.PrimaryRank }, func(row rankingRow) string { - return ranking.primaryUnit(row.primaryValue) + return ranking.primaryUnit(row.PrimaryValue) })) - if !(report.Config.Mode == "tui" && report.Config.DurationSeconds > 0) { + if !report.Config.FixedDurationTUI() { appendWinner(winnerForRank( - report, ranking.rows, "CPU COST", func(row rankingRow) int { - return row.cpuRank + ranking.Rows, "CPU COST", func(row rankingRow) int { + return row.CPURank }, func(row rankingRow) string { - return formatDuration(row.cpuValue) + return formatDuration(row.CPUValue) })) } - if ranking.ramAvailable { + if ranking.RAMAvailable { appendWinner(winnerForRank( - report, ranking.rows, "RAM COST", - func(row rankingRow) int { return row.ramRank }, - func(row rankingRow) string { return formatBytes(row.ramValue) + " aggregate" }, + ranking.Rows, "RAM COST", + func(row rankingRow) int { return row.RAMRank }, + func(row rankingRow) string { return formatBytes(row.RAMValue) + " aggregate" }, )) } appendWinner(winnerForRank( - report, ranking.rows, "LINKED SIZE", - func(row rankingRow) int { return row.footprintRank }, - func(row rankingRow) string { return formatBytes(row.footprintValue) }, + ranking.Rows, "LINKED SIZE", + func(row rankingRow) int { return row.FootprintRank }, + func(row rankingRow) string { return formatBytes(row.FootprintValue) }, )) return winners } func winnerForRank( - report model.Report, rows []rankingRow, category string, rank func(rankingRow) int, @@ -62,13 +61,13 @@ func winnerForRank( var benchmarks []int for _, row := range rows { if rank(row) == 1 { - benchmarks = append(benchmarks, row.benchmark) + benchmarks = append(benchmarks, row.Benchmark) } } result := categoryWinner{category: category, benchmarks: benchmarks} if len(benchmarks) > 0 { for _, row := range rows { - if row.benchmark == benchmarks[0] { + if row.Benchmark == benchmarks[0] { result.value = value(row) break } diff --git a/internal/report/render.go b/internal/report/render.go index 786c698..d095a00 100644 --- a/internal/report/render.go +++ b/internal/report/render.go @@ -1,46 +1,77 @@ package report -import "github.com/shellcell/snailrace/internal/model" +import ( + "time" + "github.com/shellcell/snailrace/internal/model" +) + +// Renderer caches report analysis and is intended for sequential rendering. +// Callers must not mutate the source report while the renderer is in use. type Renderer struct { - raw model.Report - report model.Report + rawReport model.Report + displayReport model.Report + metadata ReportMetadata ranking rankingData dimensionText string caveats []string rawCaveats []string - groups []chartGroup + chartGroups []chartGroup charts []svgChart chartsReady bool comparisons map[comparisonKey]deltaResult + baselineRuns map[metricID]map[int]float64 +} + +type ReportMetadata struct { + MeasuredAt time.Time + ToolNames []string } type comparisonKey struct { candidate int - metric int + metric metricID } func NewRenderer(input model.Report) *Renderer { display := safeDisplayReport(input) ranking := calculateRanking(input) dimensions := includedDimensionsFromRanking(ranking) + toolNames := make([]string, len(input.Benchmarks)) + for index, benchmark := range input.Benchmarks { + toolNames[index] = benchmark.Tool.Name + } return &Renderer{ - raw: input, report: display, ranking: ranking, + rawReport: input, displayReport: display, ranking: ranking, + metadata: ReportMetadata{MeasuredAt: input.MeasuredAt, ToolNames: toolNames}, dimensionText: dimensionsText(dimensions), caveats: reliabilityCaveats(display), rawCaveats: reliabilityCaveats(input), - groups: chartGroups(display), comparisons: make(map[comparisonKey]deltaResult), + chartGroups: chartGroups(display), comparisons: make(map[comparisonKey]deltaResult), + baselineRuns: make(map[metricID]map[int]float64), } } -func (renderer *Renderer) comparison(candidate, metric int) deltaResult { +func (renderer *Renderer) Metadata() ReportMetadata { + metadata := renderer.metadata + metadata.ToolNames = append([]string(nil), metadata.ToolNames...) + return metadata +} + +func (renderer *Renderer) comparison(candidate int, metric metricID) deltaResult { key := comparisonKey{candidate: candidate, metric: metric} if result, ok := renderer.comparisons[key]; ok { return result } - baseline := renderer.report.Benchmarks[baselineIndex(renderer.report)] - result := compareMetric( - baseline, renderer.report.Benchmarks[candidate], metricRows[metric], - renderer.report.Config.IntervalMS/1000, + baseline := renderer.displayReport.Benchmarks[baselineIndex(renderer.displayReport)] + row := metricCatalog[metric] + baselineRuns, ok := renderer.baselineRuns[metric] + if !ok { + baselineRuns = indexMetricValues(baseline.Runs, row) + renderer.baselineRuns[metric] = baselineRuns + } + result := compareMetricWithBaselineIndex( + baseline, baselineRuns, renderer.displayReport.Benchmarks[candidate], row, + renderer.displayReport.Config.IntervalMS/1000, ) renderer.comparisons[key] = result return result diff --git a/internal/report/report.go b/internal/report/report.go index 6bb4470..3da6de8 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -3,28 +3,27 @@ package report import ( "fmt" "io" - "strings" "github.com/shellcell/snailrace/internal/model" "github.com/shellcell/snailrace/internal/style" ) -func Write(writer io.Writer, format string, report model.Report) error { - return NewRenderer(report).Write(writer, format) -} - func (renderer *Renderer) Write(writer io.Writer, format string) error { - switch strings.ToLower(format) { - case "text", "txt": - return writeTextRenderer(writer, renderer) - case "json": - return writeJSONRenderer(writer, renderer) - case "markdown", "md": + info, ok := LookupFormat(format) + if !ok { + return fmt.Errorf("unknown report format %q", format) + } + switch info.Name { + case FormatText: + return writeText(writer, renderer) + case FormatJSON: + return writeJSON(writer, renderer) + case FormatMarkdown: return renderer.WriteMarkdownWithCharts(writer, nil, "") - case "html": - return writeHTMLRenderer(writer, renderer) - case "svg": - return writeSVGRenderer(writer, renderer) + case FormatHTML: + return writeHTML(writer, renderer) + case FormatSVG: + return writeSVG(writer, renderer) default: return fmt.Errorf("unknown report format %q", format) } @@ -39,12 +38,3 @@ func safeDisplayReport(report model.Report) model.Report { } return report } - -func ValidFormat(format string) bool { - switch strings.ToLower(format) { - case "text", "txt", "json", "markdown", "md", "html", "svg": - return true - default: - return false - } -} diff --git a/internal/report/report_bench_test.go b/internal/report/report_bench_test.go index 8c70a7c..c339648 100644 --- a/internal/report/report_bench_test.go +++ b/internal/report/report_bench_test.go @@ -16,7 +16,7 @@ func TestCachedRendererMatchesIndependentFormats(t *testing.T) { if err := renderer.Write(&cached, format); err != nil { t.Fatal(err) } - if err := Write(&independent, format, report); err != nil { + if err := NewRenderer(report).Write(&independent, format); err != nil { t.Fatal(err) } if cached.String() != independent.String() { @@ -25,12 +25,55 @@ func TestCachedRendererMatchesIndependentFormats(t *testing.T) { } } +func TestRendererOutputIsOrderIndependentAndRepeatable(t *testing.T) { + report := benchmarkReport(3, 4) + formats := []string{"html", "svg", "markdown", "json", "text"} + want := make(map[string]string, len(formats)) + for _, format := range formats { + var output bytes.Buffer + if err := NewRenderer(report).Write(&output, format); err != nil { + t.Fatal(err) + } + want[format] = output.String() + } + for _, order := range [][]string{ + formats, + {"text", "json", "markdown", "svg", "html", "text"}, + } { + renderer := NewRenderer(report) + for _, format := range order { + var output bytes.Buffer + if err := renderer.Write(&output, format); err != nil { + t.Fatal(err) + } + if output.String() != want[format] { + t.Fatalf("%s output depends on rendering order", format) + } + } + } +} + +func TestRendererMetadataIsIndependent(t *testing.T) { + report := benchmarkReport(2, 2) + originalName := report.Benchmarks[0].Tool.Name + renderer := NewRenderer(report) + report.Benchmarks[0].Tool.Name = "changed" + metadata := renderer.Metadata() + if metadata.ToolNames[0] != originalName { + t.Fatalf("renderer metadata changed with source: %+v", metadata) + } + metadata.ToolNames[0] = "also changed" + if renderer.Metadata().ToolNames[0] != originalName { + t.Fatal("renderer returned mutable metadata") + } +} + func BenchmarkWriteHTML10Tools100Runs(b *testing.B) { report := benchmarkReport(10, 100) b.ReportAllocs() b.ResetTimer() for range b.N { - if err := Write(io.Discard, "html", report); err != nil { + if err := NewRenderer(report).Write(io.Discard, "html"); err != nil { b.Fatal(err) } } @@ -41,7 +84,7 @@ func BenchmarkReportCharts10Tools100Runs(b *testing.B) { b.ReportAllocs() b.ResetTimer() for range b.N { - if charts := reportCharts(report); len(charts) == 0 { + if charts := NewRenderer(report).reportCharts(); len(charts) == 0 { b.Fatal("no charts") } } @@ -69,7 +112,7 @@ func BenchmarkWriteAllFormatsIndependent10Tools100Runs(b *testing.B) { b.ResetTimer() for range b.N { for _, format := range formats { - if err := Write(io.Discard, format, report); err != nil { + if err := NewRenderer(report).Write(io.Discard, format); err != nil { b.Fatal(err) } } diff --git a/internal/report/semantics.go b/internal/report/semantics.go index 0ba78ed..0645806 100644 --- a/internal/report/semantics.go +++ b/internal/report/semantics.go @@ -9,16 +9,16 @@ import ( func includedDimensionsFromRanking(ranking rankingData) []string { var result []string - if ranking.indexPrimary { + if ranking.IndexPrimary { result = append(result, "time") } - if ranking.indexCPU { + if ranking.IndexCPU { result = append(result, "cpu") } - if ranking.indexRAM { + if ranking.IndexRAM { result = append(result, "ram") } - if ranking.indexFootprint { + if ranking.IndexFootprint { result = append(result, "disk") } return result @@ -39,7 +39,7 @@ func reliabilityCaveats(report model.Report) []string { result = append(result, benchmark.Tool.Name+ ": post-run executable verification was not completed") } - if !benchmarkSamplesReliable(benchmark, interval) { + if !model.SamplingReliable(benchmark, interval) { result = append(result, fmt.Sprintf( "%s: sampled process metrics are limited by sparse observations", benchmark.Tool.Name, diff --git a/internal/report/svg.go b/internal/report/svg.go index 831306d..6aa15ce 100644 --- a/internal/report/svg.go +++ b/internal/report/svg.go @@ -5,8 +5,6 @@ import ( "html" "io" "strings" - - "github.com/shellcell/snailrace/internal/model" ) const svgReportWidth = 1500 @@ -21,12 +19,8 @@ const svgReportStyle = `` -func writeSVG(writer io.Writer, report model.Report) error { - return writeSVGRenderer(writer, NewRenderer(report)) -} - -func writeSVGRenderer(writer io.Writer, renderer *Renderer) error { - report := renderer.report +func writeSVG(writer io.Writer, renderer *Renderer) error { + report := renderer.displayReport checked := newErrorWriter(writer) writer = checked var content strings.Builder @@ -51,11 +45,11 @@ func writeSVGRenderer(writer io.Writer, renderer *Renderer) error { } func svgHeader(output *strings.Builder, renderer *Renderer) int { - report := renderer.report + report := renderer.displayReport caveats := renderer.caveats panelHeight := 96 + len(caveats)*22 nextSection := 254 + len(caveats)*22 - if report.Config.Mode == "tui" { + if report.Config.IsTUI() { panelHeight += 24 nextSection += 24 } @@ -96,7 +90,7 @@ func svgHeader(output *strings.Builder, renderer *Renderer) int { 227+index*22, html.EscapeString(caveat), ) } - if report.Config.Mode == "tui" { + if report.Config.IsTUI() { duration := "until exit" if report.Config.DurationSeconds > 0 { duration = formatDuration(report.Config.DurationSeconds) diff --git a/internal/report/svg_test.go b/internal/report/svg_test.go index 2c061ec..b07f769 100644 --- a/internal/report/svg_test.go +++ b/internal/report/svg_test.go @@ -24,7 +24,7 @@ func TestSVGIsStandaloneCompleteReport(t *testing.T) { Notes: []string{"No outliers removed."}, } var output bytes.Buffer - if err := writeSVG(&output, report); err != nil { + if err := writeSVG(&output, NewRenderer(report)); err != nil { t.Fatal(err) } decoder := xml.NewDecoder(bytes.NewReader(output.Bytes())) @@ -51,12 +51,13 @@ func TestMarkdownReferencesSeparateSVGCharts(t *testing.T) { benchmarkWithWallTimes("candidate", 9, 9), }, } - charts, err := WriteChartFiles(t.TempDir(), report, false) + renderer := NewRenderer(report) + charts, err := renderer.WriteChartFiles(t.TempDir(), false) if err != nil { t.Fatal(err) } var output bytes.Buffer - if err := WriteMarkdownWithCharts(&output, report, charts, "charts"); err != nil { + if err := renderer.WriteMarkdownWithCharts(&output, charts, "charts"); err != nil { t.Fatal(err) } if strings.Contains(output.String(), " 0 { duration = formatDuration(report.Config.DurationSeconds) @@ -128,11 +124,11 @@ func writeTextBenchmark( formatCount(benchmark.Summary.ValidSampleCount.Mean), formatDuration(benchmark.Summary.SampleCoverageSeconds.Mean), ) - if !benchmarkSamplesReliable(benchmark, intervalSeconds) { + if !model.SamplingReliable(benchmark, intervalSeconds) { fmt.Fprintln(w, "Sampling quality\tLIMITED: fewer than two valid samples or intervals") } fmt.Fprintln(w, "Metric\tMean Β± Οƒ\t95% CI mean\tMedian\tP95\tRange") - for _, row := range metricRows { + for _, row := range metricCatalog { if !availableFor(row, operatingSystem, benchmark.Summary) { fmt.Fprintf(w, "%s\tN/A\tN/A\tN/A\tN/A\tN/A\n", row.name) continue diff --git a/internal/report/text_bars.go b/internal/report/text_bars.go index bc2049e..e3192f2 100644 --- a/internal/report/text_bars.go +++ b/internal/report/text_bars.go @@ -15,7 +15,7 @@ const compactBarWidth = 20 // writeCompactText renders the short, bar-chart stdout report used by default. func writeCompactText(writer io.Writer, renderer *Renderer) error { - report := renderer.report + report := renderer.displayReport terminal := writerIsTerminal(writer) var body strings.Builder body.WriteString("\n") @@ -45,7 +45,7 @@ type compactMetric struct { } func compactMetrics(report model.Report, ranking rankingData) []compactMetric { - fixedTUI := report.Config.Mode == "tui" && report.Config.DurationSeconds > 0 + fixedTUI := report.Config.FixedDurationTUI() primaryStdDev := func(s model.Summary) float64 { return s.WallSeconds.StdDev } cpuStdDev := func(s model.Summary) float64 { return s.CPUTotalSeconds.StdDev } if fixedTUI { @@ -53,7 +53,7 @@ func compactMetrics(report model.Report, ranking rankingData) []compactMetric { cpuStdDev = primaryStdDev } ramNote := "" - if ranking.ramPresent && !ranking.ramAvailable { + if ranking.RAMPresent && !ranking.RAMAvailable { ramNote = " (sampling-limited Β· excluded from balanced index" if ranking.samplingInterval != "" { ramNote += "; lower -interval, now " + ranking.samplingInterval @@ -70,31 +70,31 @@ func compactMetrics(report model.Report, ranking rankingData) []compactMetric { return []compactMetric{ { name: ranking.primaryLabel, unit: ranking.primaryUnit, - available: ranking.primaryRatio, - value: func(r rankingRow) float64 { return r.primaryValue }, + available: ranking.PrimaryRatio, + value: func(r rankingRow) float64 { return r.PrimaryValue }, stdDev: primaryStdDev, }, { name: "CPU", unit: func(v float64) string { return cpuCompactUnit(fixedTUI, v) }, - available: ranking.cpuRatio, - value: func(r rankingRow) float64 { return r.cpuValue }, + available: ranking.CPURatio, + value: func(r rankingRow) float64 { return r.CPUValue }, stdDev: cpuStdDev, }, { - name: "RAM", note: ramNote, unit: formatBytes, available: ranking.ramPresent, - value: func(r rankingRow) float64 { return r.ramValue }, + name: "RAM", note: ramNote, unit: formatBytes, available: ranking.RAMPresent, + value: func(r rankingRow) float64 { return r.RAMValue }, stdDev: func(s model.Summary) float64 { return s.MeanResidentBytes.StdDev }, }, { name: "PHYS", unit: formatBytes, available: physicalFootprintAvailable(report), value: func(r rankingRow) float64 { - return report.Benchmarks[r.benchmark].Summary.PhysicalFootprintStats().Mean + return report.Benchmarks[r.Benchmark].Summary.PhysicalFootprintStats().Mean }, stdDev: func(s model.Summary) float64 { return s.PhysicalFootprintStats().StdDev }, }, { - name: "DISK", note: diskNote, unit: formatBytes, available: ranking.footprintRatio, - value: func(r rankingRow) float64 { return r.footprintValue }, + name: "DISK", note: diskNote, unit: formatBytes, available: ranking.FootprintRatio, + value: func(r rankingRow) float64 { return r.FootprintValue }, }, } } @@ -109,24 +109,24 @@ func cpuCompactUnit(fixedTUI bool, value float64) string { func writeCompactBars( body *strings.Builder, report model.Report, ranking rankingData, terminal bool, ) { - if len(ranking.rows) == 0 { - fmt.Fprintf(body, "BALANCED unavailable: %s\n\n", ranking.unavailableReason) + if len(ranking.Rows) == 0 { + fmt.Fprintf(body, "BALANCED unavailable: %s\n\n", ranking.UnavailableReason) return } - rows := rowsByBenchmark(ranking.rows) + rows := rowsByBenchmark(ranking.Rows) labelWidth := compactLabelWidth(report) - if ranking.available { - winner := report.Benchmarks[ranking.rows[0].benchmark].Tool.Name + if ranking.Available { + winner := report.Benchmarks[ranking.Rows[0].Benchmark].Tool.Name fmt.Fprintf( body, "BALANCED %s winner %s\n", balancedIndexCategories(ranking), winner, ) writeBarGroup(body, report, terminal, labelWidth, rows, func(r rankingRow) float64 { - return r.overallScore - }, func(r rankingRow) string { return formatScore(r.overallScore) }) + return r.OverallScore + }, func(r rankingRow) string { return formatScore(r.OverallScore) }) } else { - fmt.Fprintf(body, "BALANCED unavailable: %s\n\n", ranking.unavailableReason) + fmt.Fprintf(body, "BALANCED unavailable: %s\n\n", ranking.UnavailableReason) } for _, metric := range compactMetrics(report, ranking) { @@ -137,7 +137,7 @@ func writeCompactBars( writeBarGroup(body, report, terminal, labelWidth, rows, metric.value, func(r rankingRow) string { text := metric.unit(metric.value(r)) if metric.stdDev != nil { - sigma := metric.stdDev(report.Benchmarks[r.benchmark].Summary) + sigma := metric.stdDev(report.Benchmarks[r.Benchmark].Summary) text += " Β± " + metric.unit(sigma) } return text @@ -187,17 +187,17 @@ func writeBarGroup( func writeCompactSingle( body *strings.Builder, report model.Report, ranking rankingData, terminal bool, ) { - if len(ranking.rows) == 0 { + if len(ranking.Rows) == 0 { return } - row := ranking.rows[0] + row := ranking.Rows[0] for _, metric := range compactMetrics(report, ranking) { if !metric.available { continue } text := metric.unit(metric.value(row)) if metric.stdDev != nil { - sigma := metric.stdDev(report.Benchmarks[row.benchmark].Summary) + sigma := metric.stdDev(report.Benchmarks[row.Benchmark].Summary) text += " Β± " + metric.unit(sigma) } fmt.Fprintf(body, " %-5s %s%s\n", metric.name, text, redNote(metric.note, terminal)) @@ -227,7 +227,7 @@ func textBar(fraction float64, width int) string { func rowsByBenchmark(rows []rankingRow) map[int]rankingRow { result := make(map[int]rankingRow, len(rows)) for _, row := range rows { - result[row.benchmark] = row + result[row.Benchmark] = row } return result } diff --git a/internal/report/text_test.go b/internal/report/text_test.go index b631409..b447854 100644 --- a/internal/report/text_test.go +++ b/internal/report/text_test.go @@ -19,7 +19,7 @@ func TestFixedTUIComparisonEmphasizesResources(t *testing.T) { }, } var output bytes.Buffer - if err := writeText(&output, input); err != nil { + if err := writeText(&output, NewRenderer(input)); err != nil { t.Fatal(err) } if !strings.Contains(output.String(), "Average CPU") { @@ -40,7 +40,7 @@ func TestCompactReportShowsBarsAndOmitsDetail(t *testing.T) { Notes: []string{"a methodology note"}, } var output bytes.Buffer - if err := writeText(&output, report); err != nil { + if err := writeText(&output, NewRenderer(report)); err != nil { t.Fatal(err) } text := output.String() @@ -67,7 +67,7 @@ func TestVerboseReportKeepsStatisticalDetail(t *testing.T) { Notes: []string{"a methodology note"}, } var output bytes.Buffer - if err := writeText(&output, report); err != nil { + if err := writeText(&output, NewRenderer(report)); err != nil { t.Fatal(err) } if !strings.Contains(output.String(), "Statistical detail") { @@ -81,7 +81,7 @@ func TestCompactSingleToolListsKeyMetrics(t *testing.T) { Benchmarks: []model.Benchmark{rankingBenchmark("solo", 1, 1, 100, 200, 1000)}, } var output bytes.Buffer - if err := writeText(&output, report); err != nil { + if err := writeText(&output, NewRenderer(report)); err != nil { t.Fatal(err) } text := output.String() @@ -105,16 +105,16 @@ func samplingLimitedRAMReport() model.Report { func TestSamplingLimitedRAMIsShownButExcludedFromIndex(t *testing.T) { report := samplingLimitedRAMReport() ranking := calculateRanking(report) - if ranking.ramAvailable || !ranking.ramPresent { + if ranking.RAMAvailable || !ranking.RAMPresent { t.Fatalf("want RAM present but sampling-limited, got present=%v available=%v", - ranking.ramPresent, ranking.ramAvailable) + ranking.RAMPresent, ranking.RAMAvailable) } if strings.Contains(balancedIndexCategories(ranking), "RAM") { t.Fatal("sampling-limited RAM must stay out of the balanced index") } // Compact stdout shows the RAM bars with a remark. var compact bytes.Buffer - if err := writeText(&compact, report); err != nil { + if err := writeText(&compact, NewRenderer(report)); err != nil { t.Fatal(err) } if !strings.Contains(compact.String(), "RAM") || @@ -123,12 +123,12 @@ func TestSamplingLimitedRAMIsShownButExcludedFromIndex(t *testing.T) { } // HTML ranking table and the RAM chart show it with a remark, not N/A. var htmlRanking bytes.Buffer - writeHTMLRanking(&htmlRanking, report) + writeHTMLRanking(&htmlRanking, report, calculateRanking(report)) if !strings.Contains(htmlRanking.String(), "sampling-limited") { t.Fatal("HTML ranking should mark RAM sampling-limited instead of N/A") } foundRAMChart := false - for _, chart := range rankingCharts(report) { + for _, chart := range rankingCharts(report, calculateRanking(report)) { if chart.title == "RAM AGGREGATE" { foundRAMChart = true if !strings.Contains(chart.description, "sampling-limited") { @@ -145,7 +145,7 @@ func TestAllFormatsExposeActualDimensionsAndReliability(t *testing.T) { report := samplingLimitedRAMReport() for _, format := range []string{"text", "markdown", "html", "svg"} { var output bytes.Buffer - if err := Write(&output, format, report); err != nil { + if err := NewRenderer(report).Write(&output, format); err != nil { t.Fatalf("%s: %v", format, err) } text := strings.ToLower(output.String()) @@ -160,7 +160,7 @@ func TestAllFormatsExposeActualDimensionsAndReliability(t *testing.T) { } } var output bytes.Buffer - if err := Write(&output, "json", report); err != nil { + if err := NewRenderer(report).Write(&output, "json"); err != nil { t.Fatal(err) } var decoded struct { diff --git a/internal/report/ui_test.go b/internal/report/ui_test.go index fa21721..3bb487e 100644 --- a/internal/report/ui_test.go +++ b/internal/report/ui_test.go @@ -31,7 +31,7 @@ func TestJSONIncludesComputedRanking(t *testing.T) { }, } var output bytes.Buffer - if err := writeJSON(&output, report); err != nil { + if err := writeJSON(&output, NewRenderer(report)); err != nil { t.Fatal(err) } for _, expected := range []string{"\"ranking\"", "\"winners\"", "balanced_index"} { @@ -50,7 +50,7 @@ func TestUnavailableRankingSerializesWithoutNonFiniteValues(t *testing.T) { Benchmarks: []model.Benchmark{benchmarkWithWallTimes("short", 0.001)}, } var output bytes.Buffer - if err := writeJSON(&output, report); err != nil { + if err := writeJSON(&output, NewRenderer(report)); err != nil { t.Fatal(err) } if !strings.Contains(output.String(), `"available": false`) || @@ -72,7 +72,7 @@ func TestFailedCommandsAreProminentInEveryFormat(t *testing.T) { for _, format := range []string{"text", "html", "markdown", "svg", "json"} { t.Run(format, func(t *testing.T) { var output bytes.Buffer - if err := Write(&output, format, report); err != nil { + if err := NewRenderer(report).Write(&output, format); err != nil { t.Fatal(err) } value := strings.ToLower(output.String()) @@ -92,7 +92,7 @@ func TestHTMLPreservesLongCommandInWrappingBlock(t *testing.T) { }}, } var output bytes.Buffer - if err := writeHTML(&output, report); err != nil { + if err := writeHTML(&output, NewRenderer(report)); err != nil { t.Fatal(err) } if !strings.Contains(output.String(), command) || diff --git a/internal/runner/benchmark.go b/internal/runner/benchmark.go index ece186d..e715975 100644 --- a/internal/runner/benchmark.go +++ b/internal/runner/benchmark.go @@ -27,6 +27,10 @@ func Benchmark( for index := range specs { specs[index].Args = append([]string(nil), specs[index].Args...) } + prepared := make([]preparedSpec, len(specs)) + for index, spec := range specs { + prepared[index] = preparedSpec{Spec: spec} + } benchmarks := make([]model.Benchmark, len(specs)) progress := newProgressTracker(options, specs, config) defer progress.finish() @@ -52,14 +56,14 @@ func Benchmark( pinned[index] = file pinExecution := !executableIsScript(file) if pinExecution { - specs[index].executable = file + prepared[index].executable = file } if tool.ShellTarget && pinExecution { if command, ok := pinShellExecutable( spec.Shell, pinnedExecutablePath(file), ); ok { - specs[index].Shell = command - specs[index].shellTarget = true + prepared[index].Shell = command + prepared[index].shellTarget = true } } benchmarks[index].Tool = tool @@ -74,7 +78,7 @@ func Benchmark( interrupted := false for warmup, order := range warmupOrder { for _, index := range order { - spec := specs[index] + spec := prepared[index] progress.update( index, spec.Name, warmup+1, config.Warmups, true, false, nil, ) @@ -110,7 +114,7 @@ func Benchmark( index, specs[index].Name, runIndex+1, config.Runs, false, false, benchmarks[index].Runs, ) - run, err := runOnce(ctx, specs[index], config.Interval, options) + run, err := runOnce(ctx, prepared[index], config.Interval, options) if err != nil { if ctx.Err() != nil { interrupted = true diff --git a/internal/runner/benchmark_progress.go b/internal/runner/benchmark_progress.go index b17a78c..c0f5b9d 100644 --- a/internal/runner/benchmark_progress.go +++ b/internal/runner/benchmark_progress.go @@ -16,7 +16,7 @@ type progressTracker struct { interval float64 runs int estimates []ProgressEstimate - summaries []progressSummary + summaries []model.SummaryAccumulator processed []int } @@ -32,11 +32,11 @@ func newProgressTracker(options Options, specs []Spec, config Config) *progressT return tracker } tracker.estimates = make([]ProgressEstimate, len(specs)) - tracker.summaries = make([]progressSummary, len(specs)) + tracker.summaries = make([]model.SummaryAccumulator, len(specs)) tracker.processed = make([]int, len(specs)) for index, spec := range specs { tracker.estimates[index] = ProgressEstimate{ToolName: spec.Name, Total: config.Runs} - tracker.summaries[index] = newProgressSummary(config.Runs) + tracker.summaries[index] = model.NewSummaryAccumulator(config.Runs) } return tracker } @@ -65,15 +65,16 @@ func (tracker *progressTracker) update( estimate := &tracker.estimates[tool] if len(runs) > 0 && (!estimate.HasEstimate || estimate.Completed != len(runs)) { if len(runs) < tracker.processed[tool] { - tracker.summaries[tool] = newProgressSummary(tracker.runs) + tracker.summaries[tool] = model.NewSummaryAccumulator(tracker.runs) tracker.processed[tool] = 0 } for _, run := range runs[tracker.processed[tool]:] { - tracker.summaries[tool].add(run) + tracker.summaries[tool].Add(run) } tracker.processed[tool] = len(runs) estimate.HasEstimate = true - estimate.Estimate = tracker.summaries[tool].snapshot() + snapshot := tracker.summaries[tool].Snapshot() + estimate.Estimate = &snapshot } estimate.Completed = len(runs) estimate.Runs = runs @@ -89,97 +90,16 @@ func (tracker *progressTracker) update( Iteration: iteration, Iterations: iterations, Completed: tracker.completed, Total: tracker.total, Warmup: warmup, FixedDuration: tracker.fixed, IntervalMS: tracker.interval, - Elapsed: elapsed, ETA: eta, + ETA: eta, Estimates: append([]ProgressEstimate(nil), tracker.estimates...), } if len(runs) > 0 { event.HasEstimate = true - event.Estimate = tracker.estimates[tool].Estimate + event.Estimate = *tracker.estimates[tool].Estimate } tracker.callback(event) } -const ( - progressWall = iota - progressCPUTotal - progressCPUUser - progressCPUSystem - progressAverageCPU - progressPeakResident - progressOSMaxRSS - progressMeanResident - progressPeakVirtual - progressPeakProcesses - progressPeakThreads - progressPeakFDs - progressSamples - progressCoverage - progressMetricCount -) - -type progressSummary struct { - metrics [progressMetricCount]model.RunningStats - physical model.RunningStats -} - -func newProgressSummary(capacity int) progressSummary { - var summary progressSummary - for index := range summary.metrics { - summary.metrics[index] = model.NewRunningStats(capacity) - } - summary.physical = model.NewRunningStats(capacity) - return summary -} - -func (summary *progressSummary) add(run model.Run) { - values := [...]float64{ - run.WallSeconds, - run.CPUUserSeconds + run.CPUSystemSeconds, - run.CPUUserSeconds, - run.CPUSystemSeconds, - run.AverageCPUPercent, - run.PeakResidentBytes, - run.OSMaxRSSBytes, - run.MeanResidentBytes, - run.PeakVirtualBytes, - run.PeakProcesses, - run.PeakThreads, - run.PeakFileDescriptors, - float64(run.SampleCount), - run.SampleCoverageSeconds, - } - for index, value := range values { - summary.metrics[index].Add(value) - } - if run.PhysicalFootprintValid { - summary.physical.Add(run.PeakPhysicalFootprintBytes) - } -} - -func (summary progressSummary) snapshot() model.Summary { - result := model.Summary{ - WallSeconds: summary.metrics[progressWall].Snapshot(), - CPUTotalSeconds: summary.metrics[progressCPUTotal].Snapshot(), - CPUUserSeconds: summary.metrics[progressCPUUser].Snapshot(), - CPUSystemSeconds: summary.metrics[progressCPUSystem].Snapshot(), - AverageCPUPercent: summary.metrics[progressAverageCPU].Snapshot(), - PeakResidentBytes: summary.metrics[progressPeakResident].Snapshot(), - OSMaxRSSBytes: summary.metrics[progressOSMaxRSS].Snapshot(), - MeanResidentBytes: summary.metrics[progressMeanResident].Snapshot(), - PeakVirtualBytes: summary.metrics[progressPeakVirtual].Snapshot(), - PeakProcesses: summary.metrics[progressPeakProcesses].Snapshot(), - PeakThreads: summary.metrics[progressPeakThreads].Snapshot(), - PeakFileDescriptors: summary.metrics[progressPeakFDs].Snapshot(), - ValidSampleCount: summary.metrics[progressSamples].Snapshot(), - SampleCoverageSeconds: summary.metrics[progressCoverage].Snapshot(), - } - physical := summary.physical.Snapshot() - if physical.N > 0 { - result.PeakPhysicalFootprintBytes = &physical - } - return result -} - func (tracker *progressTracker) finish() { if tracker.callback != nil { tracker.callback(ProgressEvent{Finished: true}) diff --git a/internal/runner/benchmark_progress_test.go b/internal/runner/benchmark_progress_test.go index 13218ca..4cd40c5 100644 --- a/internal/runner/benchmark_progress_test.go +++ b/internal/runner/benchmark_progress_test.go @@ -69,11 +69,15 @@ func TestProgressSummariesMatchFinalStatistics(t *testing.T) { PeakPhysicalFootprintBytes: 200, PhysicalFootprintValid: true}, } var estimates []model.Summary + var raceEstimates []*model.Summary tracker := newProgressTracker( Options{Progress: func(event ProgressEvent) { if event.HasEstimate { estimates = append(estimates, event.Estimate) } + if len(event.Estimates) > 0 && event.Estimates[0].Estimate != nil { + raceEstimates = append(raceEstimates, event.Estimates[0].Estimate) + } }}, []Spec{{Name: "tool"}}, Config{Runs: len(runs), Interval: time.Millisecond}, ) @@ -92,4 +96,7 @@ func TestProgressSummariesMatchFinalStatistics(t *testing.T) { if estimates[0].PeakPhysicalFootprintBytes.Mean != 300 { t.Fatal("earlier progress event was mutated by a later update") } + if raceEstimates[0].WallSeconds.Mean != 3 { + t.Fatal("earlier race estimate was mutated by a later update") + } } diff --git a/internal/runner/monitor.go b/internal/runner/monitor.go new file mode 100644 index 0000000..48cca4e --- /dev/null +++ b/internal/runner/monitor.go @@ -0,0 +1,118 @@ +package runner + +import ( + "time" + + "github.com/shellcell/snailrace/internal/platform" +) + +type processGroupSampler func(int) (platform.Metrics, bool) + +type sampleAggregate struct { + peak platform.Metrics + residentByteSamples uint64 + sampleCount uint64 + residentByteSeconds float64 + sampleCoverageSeconds float64 + previousRSS uint64 + previousAt time.Time +} + +func (aggregate *sampleAggregate) observe(current platform.Metrics, at time.Time) { + aggregate.addCoverage(at) + firstSample := aggregate.sampleCount == 0 + aggregate.residentByteSamples += current.ResidentBytes + aggregate.sampleCount++ + aggregate.peak.ResidentBytes = max(aggregate.peak.ResidentBytes, current.ResidentBytes) + aggregate.peak.VirtualBytes = max(aggregate.peak.VirtualBytes, current.VirtualBytes) + aggregate.peak.Processes = max(aggregate.peak.Processes, current.Processes) + aggregate.peak.Threads = max(aggregate.peak.Threads, current.Threads) + aggregate.peak.FileDescriptors = max( + aggregate.peak.FileDescriptors, current.FileDescriptors, + ) + if current.PhysicalFootprintValid { + aggregate.peak.PhysicalFootprintBytes = max( + aggregate.peak.PhysicalFootprintBytes, current.PhysicalFootprintBytes, + ) + } + aggregate.peak.PhysicalFootprintInvalid = + aggregate.peak.PhysicalFootprintInvalid || current.PhysicalFootprintInvalid + aggregate.peak.PhysicalFootprintValid = current.PhysicalFootprintValid && + !aggregate.peak.PhysicalFootprintInvalid && + (firstSample || aggregate.peak.PhysicalFootprintValid) + aggregate.previousRSS, aggregate.previousAt = current.ResidentBytes, at +} + +func (aggregate *sampleAggregate) invalidatePhysicalFootprint() { + aggregate.peak.PhysicalFootprintInvalid = true + aggregate.peak.PhysicalFootprintValid = false +} + +func (aggregate *sampleAggregate) addCoverage(at time.Time) { + if aggregate.previousAt.IsZero() { + return + } + seconds := at.Sub(aggregate.previousAt).Seconds() + aggregate.residentByteSeconds += float64(aggregate.previousRSS) * seconds + aggregate.sampleCoverageSeconds += seconds +} + +func (aggregate sampleAggregate) meanResident() float64 { + if aggregate.sampleCoverageSeconds > 0 { + return aggregate.residentByteSeconds / aggregate.sampleCoverageSeconds + } + if aggregate.sampleCount == 0 { + return 0 + } + return float64(aggregate.residentByteSamples) / float64(aggregate.sampleCount) +} + +type monitorHandle struct { + stop chan struct{} + done <-chan sampleAggregate +} + +func startMonitor(groupID int, interval time.Duration, sample processGroupSampler) monitorHandle { + stop := make(chan struct{}) + done := make(chan sampleAggregate, 1) + go monitorProcessGroup(groupID, interval, sample, stop, done) + return monitorHandle{stop: stop, done: done} +} + +func (monitor monitorHandle) finish() sampleAggregate { + close(monitor.stop) + return <-monitor.done +} + +func monitorProcessGroup( + groupID int, + interval time.Duration, + sample processGroupSampler, + stop <-chan struct{}, + done chan<- sampleAggregate, +) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + var aggregate sampleAggregate + observe := func() { + current, valid := sample(groupID) + if !valid { + if current.PhysicalFootprintInvalid { + aggregate.invalidatePhysicalFootprint() + } + return + } + aggregate.observe(current, time.Now()) + } + observe() + for { + select { + case <-ticker.C: + observe() + case <-stop: + aggregate.addCoverage(time.Now()) + done <- aggregate + return + } + } +} diff --git a/internal/runner/options.go b/internal/runner/options.go index 5a467a5..b4e90bc 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -44,7 +44,6 @@ type ProgressEvent struct { Estimates []ProgressEstimate FixedDuration time.Duration IntervalMS float64 - Elapsed time.Duration ETA time.Duration } @@ -53,7 +52,7 @@ type ProgressEstimate struct { Completed int Total int HasEstimate bool - Estimate model.Summary + Estimate *model.Summary Runs []model.Run DiskFootprintBytes int64 } diff --git a/internal/runner/run.go b/internal/runner/run.go index 6f0a074..c334c60 100644 --- a/internal/runner/run.go +++ b/internal/runner/run.go @@ -14,7 +14,7 @@ import ( func runOnce( ctx context.Context, - spec Spec, + spec commandSpec, interval time.Duration, options Options, ) (model.Run, error) { @@ -34,18 +34,14 @@ func runOnce( if err := cmd.Start(); err != nil { return model.Run{}, err } - stop := make(chan struct{}) - done := make(chan platform.Metrics, 1) - go monitor(cmd.Process.Pid, interval, stop, done) + groupID := cmd.Process.Pid + monitor := startMonitor(groupID, interval, platform.SampleProcessGroup) waitErr := cmd.Wait() elapsed := time.Since(started) - signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) - close(stop) - peak := <-done + signalProcessGroup(groupID, syscall.SIGKILL) + samples := monitor.finish() user, system, rusageRSS := platform.ResourceUsage(cmd.ProcessState) - peak.Processes = max(peak.Processes, 1) - peak.Threads = max(peak.Threads, 1) if ctx.Err() != nil { return model.Run{}, ctx.Err() @@ -53,24 +49,9 @@ func runOnce( if waitErr != nil && !isNonZeroExit(cmd, waitErr) { return model.Run{}, waitErr } - meanResident := sampledMeanResident(peak) - return model.Run{ - ExitCode: cmd.ProcessState.ExitCode(), WallSeconds: elapsed.Seconds(), - CPUUserSeconds: user, CPUSystemSeconds: system, - AverageCPUPercent: averageCPUPercent(user, system, elapsed), - PeakResidentBytes: float64(peak.ResidentBytes), - PeakPhysicalFootprintBytes: float64(peak.PhysicalFootprintBytes), - PhysicalFootprintValid: peak.PhysicalFootprintValid, - OSMaxRSSBytes: float64(rusageRSS), - MeanResidentBytes: meanResident, - PeakVirtualBytes: float64(peak.VirtualBytes), - PeakProcesses: float64(peak.Processes), - PeakThreads: float64(peak.Threads), - PeakFileDescriptors: float64(peak.FileDescriptors), - StopReason: "exited", - SampleCount: int(peak.SampleCount), - SampleCoverageSeconds: peak.SampleCoverageSeconds, - }, nil + return makeRun( + cmd.ProcessState, elapsed, user, system, rusageRSS, samples, "exited", + ), nil } func isNonZeroExit(cmd *exec.Cmd, err error) bool { @@ -78,90 +59,3 @@ func isNonZeroExit(cmd *exec.Cmd, err error) bool { return errors.As(err, &exitError) && cmd.ProcessState != nil && cmd.ProcessState.ExitCode() > 0 } - -func monitor( - pid int, - interval time.Duration, - stop <-chan struct{}, - done chan<- platform.Metrics, -) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - var peak platform.Metrics - var previousRSS uint64 - var previousAt time.Time - addCoverage := func(now time.Time) { - if previousAt.IsZero() { - return - } - seconds := now.Sub(previousAt).Seconds() - peak.ResidentByteSeconds += float64(previousRSS) * seconds - peak.SampleCoverageSeconds += seconds - } - sample := func() { - current, valid := platform.SampleTree(pid) - if !valid { - if current.PhysicalFootprintInvalid { - peak.PhysicalFootprintInvalid = true - peak.PhysicalFootprintValid = false - } - return - } - now := time.Now() - addCoverage(now) - updatePeaks(&peak, current) - previousRSS, previousAt = current.ResidentBytes, now - } - if platform.SampleImmediately() { - sample() - } - for { - select { - case <-ticker.C: - sample() - case <-stop: - addCoverage(time.Now()) - done <- peak - return - } - } -} - -func updatePeaks(peak *platform.Metrics, current platform.Metrics) { - firstSample := peak.SampleCount == 0 - peak.ResidentByteSamples += current.ResidentBytes - peak.SampleCount++ - peak.ResidentBytes = max(peak.ResidentBytes, current.ResidentBytes) - if current.PhysicalFootprintValid { - peak.PhysicalFootprintBytes = max( - peak.PhysicalFootprintBytes, current.PhysicalFootprintBytes, - ) - } - peak.PhysicalFootprintInvalid = peak.PhysicalFootprintInvalid || - current.PhysicalFootprintInvalid - peak.PhysicalFootprintValid = current.PhysicalFootprintValid && - !peak.PhysicalFootprintInvalid && (firstSample || peak.PhysicalFootprintValid) - peak.VirtualBytes = max(peak.VirtualBytes, current.VirtualBytes) - peak.Processes = max(peak.Processes, current.Processes) - peak.Threads = max(peak.Threads, current.Threads) - peak.FileDescriptors = max( - peak.FileDescriptors, current.FileDescriptors, - ) -} - -func sampledMeanResident(metrics platform.Metrics) float64 { - if metrics.SampleCoverageSeconds > 0 { - return metrics.ResidentByteSeconds / metrics.SampleCoverageSeconds - } - if metrics.SampleCount == 0 { - return 0 - } - return float64(metrics.ResidentByteSamples) / float64(metrics.SampleCount) -} - -func averageCPUPercent(user, system float64, elapsed time.Duration) float64 { - if elapsed <= 0 { - return 0 - } - return (user + system) / elapsed.Seconds() * 100 -} diff --git a/internal/runner/run_test.go b/internal/runner/run_test.go index 01b98f2..fd05c0a 100644 --- a/internal/runner/run_test.go +++ b/internal/runner/run_test.go @@ -42,35 +42,65 @@ func TestRunOnceRecordsNonZeroExit(t *testing.T) { } func TestMeanResidentUsesObservedTimeWeights(t *testing.T) { - metrics := platform.Metrics{ - ResidentByteSeconds: 300, SampleCoverageSeconds: 2, - ResidentByteSamples: 999, SampleCount: 3, - } - if got := sampledMeanResident(metrics); got != 150 { + var samples sampleAggregate + started := time.Unix(0, 0) + samples.observe(platform.Metrics{ResidentBytes: 100}, started) + samples.observe(platform.Metrics{ResidentBytes: 200}, started.Add(time.Second)) + samples.addCoverage(started.Add(2 * time.Second)) + if got := samples.meanResident(); got != 150 { t.Fatalf("weighted mean RSS = %g, want 150", got) } } func TestUpdatePeaksTracksPhysicalFootprint(t *testing.T) { - peak := platform.Metrics{PhysicalFootprintBytes: 100, PhysicalFootprintValid: true} - updatePeaks(&peak, platform.Metrics{ + var samples sampleAggregate + samples.observe(platform.Metrics{ + PhysicalFootprintBytes: 100, PhysicalFootprintValid: true, + }, time.Unix(0, 0)) + samples.observe(platform.Metrics{ PhysicalFootprintBytes: 250, PhysicalFootprintValid: true, - }) - updatePeaks(&peak, platform.Metrics{ + }, time.Unix(1, 0)) + samples.observe(platform.Metrics{ PhysicalFootprintBytes: 200, PhysicalFootprintValid: true, - }) - if peak.PhysicalFootprintBytes != 250 { - t.Fatalf("peak physical footprint = %d, want 250", peak.PhysicalFootprintBytes) + }, time.Unix(2, 0)) + if samples.peak.PhysicalFootprintBytes != 250 { + t.Fatalf("peak physical footprint = %d, want 250", samples.peak.PhysicalFootprintBytes) } } func TestUpdatePeaksInvalidatesPartialPhysicalFootprint(t *testing.T) { - var peak platform.Metrics - updatePeaks(&peak, platform.Metrics{ + var samples sampleAggregate + samples.observe(platform.Metrics{ PhysicalFootprintBytes: 100, PhysicalFootprintValid: true, - }) - updatePeaks(&peak, platform.Metrics{PhysicalFootprintBytes: 200}) - if peak.PhysicalFootprintValid { + }, time.Unix(0, 0)) + samples.observe( + platform.Metrics{PhysicalFootprintBytes: 200}, time.Unix(1, 0), + ) + if samples.peak.PhysicalFootprintValid { t.Fatal("a run with an invalid physical-footprint sample should be unavailable") } } + +func TestMonitorUsesInjectedSampler(t *testing.T) { + monitor := startMonitor(42, time.Hour, func(pid int) (platform.Metrics, bool) { + if pid != 42 { + t.Fatalf("sampled PID = %d, want 42", pid) + } + return platform.Metrics{ResidentBytes: 123, Processes: 1}, true + }) + samples := monitor.finish() + if samples.sampleCount != 1 || samples.peak.ResidentBytes != 123 { + t.Fatalf("samples = %+v", samples) + } +} + +func TestMonitorPreservesInvalidPhysicalFootprintSample(t *testing.T) { + monitor := startMonitor(42, time.Hour, func(int) (platform.Metrics, bool) { + return platform.Metrics{PhysicalFootprintInvalid: true}, false + }) + samples := monitor.finish() + if samples.sampleCount != 0 || !samples.peak.PhysicalFootprintInvalid || + samples.peak.PhysicalFootprintValid { + t.Fatalf("invalid sample = %+v", samples) + } +} diff --git a/internal/runner/spec.go b/internal/runner/spec.go index 4c6f4ba..fc2cc86 100644 --- a/internal/runner/spec.go +++ b/internal/runner/spec.go @@ -10,12 +10,22 @@ type Spec struct { Name string Args []string Shell string +} +type preparedSpec struct { + Spec executable *os.File shellTarget bool } func (spec Spec) command(ctx context.Context) *exec.Cmd { + if spec.Shell != "" { + return exec.CommandContext(ctx, "/bin/sh", "-c", spec.Shell) + } + return exec.CommandContext(ctx, spec.Args[0], spec.Args[1:]...) +} + +func (spec preparedSpec) command(ctx context.Context) *exec.Cmd { path := "" if spec.executable != nil { path = pinnedExecutablePath(spec.executable) @@ -40,3 +50,7 @@ func (spec Spec) command(ctx context.Context) *exec.Cmd { } return command } + +type commandSpec interface { + command(context.Context) *exec.Cmd +} diff --git a/internal/runner/spec_test.go b/internal/runner/spec_test.go new file mode 100644 index 0000000..fb8bd23 --- /dev/null +++ b/internal/runner/spec_test.go @@ -0,0 +1,36 @@ +package runner + +import ( + "context" + "os" + "testing" +) + +func TestPreparedSpecPreservesInvocationIdentity(t *testing.T) { + executable, err := os.Open("/bin/sh") + if err != nil { + t.Fatal(err) + } + defer executable.Close() + + direct := preparedSpec{ + Spec: Spec{Args: []string{"display-name", "argument"}}, executable: executable, + }.command(context.Background()) + if direct.Args[0] != "display-name" || direct.Args[1] != "argument" { + t.Fatalf("direct argv = %v", direct.Args) + } + + shell := preparedSpec{ + Spec: Spec{Shell: "exit 0"}, executable: executable, + }.command(context.Background()) + if shell.Args[0] != "/bin/sh" || shell.Args[1] != "-c" { + t.Fatalf("shell argv = %v", shell.Args) + } + + target := preparedSpec{ + Spec: Spec{Shell: "exec tool"}, executable: executable, shellTarget: true, + }.command(context.Background()) + if target.Path != "/bin/sh" || target.Args[0] != "/bin/sh" { + t.Fatalf("shell target path/argv = %q/%v", target.Path, target.Args) + } +} diff --git a/internal/runner/tool.go b/internal/runner/tool.go index c6953f7..2e14c78 100644 --- a/internal/runner/tool.go +++ b/internal/runner/tool.go @@ -17,15 +17,22 @@ import ( ) type toolInspector struct { - cache map[string]model.ToolInfo - hashCache map[string]string - fileCache map[string]os.FileInfo + cache map[string]model.ToolInfo + hashCache map[string]string + fileCache map[string]os.FileInfo + linkedFiles func(context.Context, string) ([]platform.LinkedDependency, error) } func newToolInspector() *toolInspector { + return newToolInspectorWith(platform.LinkedFiles) +} + +func newToolInspectorWith( + linkedFiles func(context.Context, string) ([]platform.LinkedDependency, error), +) *toolInspector { return &toolInspector{ cache: make(map[string]model.ToolInfo), hashCache: make(map[string]string), - fileCache: make(map[string]os.FileInfo), + fileCache: make(map[string]os.FileInfo), linkedFiles: linkedFiles, } } @@ -76,7 +83,7 @@ func (inspector *toolInspector) inspect( } tool.SizeBytes = info.Size() inspector.fileCache[executable] = info - dependencies, err := platform.LinkedFiles(ctx, executable) + dependencies, err := inspector.linkedFiles(ctx, executable) if err != nil { return model.ToolInfo{}, err } diff --git a/internal/runner/tool_test.go b/internal/runner/tool_test.go index dae2e64..7071ee0 100644 --- a/internal/runner/tool_test.go +++ b/internal/runner/tool_test.go @@ -2,9 +2,12 @@ package runner import ( "context" + "errors" "os" "path/filepath" "testing" + + "github.com/shellcell/snailrace/internal/platform" ) func TestInspectionHonorsCancelledContext(t *testing.T) { @@ -17,6 +20,37 @@ func TestInspectionHonorsCancelledContext(t *testing.T) { } } +func TestToolInspectorUsesInjectedDependencyDiscovery(t *testing.T) { + calls := 0 + inspector := newToolInspectorWith(func( + context.Context, string, + ) ([]platform.LinkedDependency, error) { + calls++ + return nil, nil + }) + for _, name := range []string{"first", "second"} { + if _, err := inspector.inspect( + context.Background(), Spec{Name: name, Args: []string{"/bin/true"}}, + ); err != nil { + t.Fatal(err) + } + } + if calls != 1 { + t.Fatalf("dependency discovery calls = %d, want 1 cached call", calls) + } + expected := errors.New("dependency failure") + inspector = newToolInspectorWith(func( + context.Context, string, + ) ([]platform.LinkedDependency, error) { + return nil, expected + }) + if _, err := inspector.inspect( + context.Background(), Spec{Name: "tool", Args: []string{"/bin/true"}}, + ); !errors.Is(err, expected) { + t.Fatalf("dependency error = %v, want %v", err, expected) + } +} + func TestShellInspectionReportsTargetExecutable(t *testing.T) { tool, err := inspectTool(Spec{Name: "shell", Shell: "sleep 0"}) if err != nil { diff --git a/internal/runner/tui.go b/internal/runner/tui.go index a3f3e2f..a93936a 100644 --- a/internal/runner/tui.go +++ b/internal/runner/tui.go @@ -5,7 +5,6 @@ import ( "errors" "io" "os" - "os/exec" "syscall" "time" @@ -18,7 +17,7 @@ import ( func runTUIOnce( ctx context.Context, - spec Spec, + spec commandSpec, interval time.Duration, options Options, ) (model.Run, error) { @@ -80,18 +79,16 @@ func runTUIOnce( go copyTerminalOutput(io.Discard, terminal, outputDone) } - stopMonitor := make(chan struct{}) - monitorDone := make(chan platform.Metrics, 1) - go monitor(cmd.Process.Pid, interval, stopMonitor, monitorDone) + groupID := cmd.Process.Pid + monitor := startMonitor(groupID, interval, platform.SampleProcessGroup) waitDone := make(chan error, 1) go func() { waitDone <- cmd.Wait() }() waitResult := waitForTUI( - ctx, cmd, options.Duration, started, waitDone, + ctx, groupID, options.Duration, started, waitDone, ) - signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) - close(stopMonitor) - peak := <-monitorDone + signalProcessGroup(groupID, syscall.SIGKILL) + samples := monitor.finish() stopInput() if inputDone != nil { <-inputDone @@ -107,8 +104,6 @@ func runTUIOnce( <-outputDone } user, system, rusageRSS := platform.ResourceUsage(cmd.ProcessState) - peak.Processes = max(peak.Processes, 1) - peak.Threads = max(peak.Threads, 1) if ctx.Err() != nil { return model.Run{}, ctx.Err() } @@ -119,8 +114,9 @@ func runTUIOnce( !isNonZeroExit(cmd, waitResult.err) { return model.Run{}, waitResult.err } - return makeTUIRun( - cmd, waitResult.elapsed, user, system, rusageRSS, peak, waitResult.reason, + return makeRun( + cmd.ProcessState, waitResult.elapsed, user, system, rusageRSS, + samples, waitResult.reason, ), nil } @@ -132,7 +128,7 @@ type tuiWaitResult struct { func waitForTUI( ctx context.Context, - cmd *exec.Cmd, + groupID int, duration time.Duration, started time.Time, waitDone <-chan error, @@ -142,7 +138,7 @@ func waitForTUI( case err := <-waitDone: return tuiWaitResult{err: err, elapsed: time.Since(started), reason: "exited"} case <-ctx.Done(): - signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) + signalProcessGroup(groupID, syscall.SIGKILL) return tuiWaitResult{ err: <-waitDone, elapsed: time.Since(started), reason: "interrupted", } @@ -158,13 +154,13 @@ func waitForTUI( case err := <-waitDone: return tuiWaitResult{err: err, elapsed: time.Since(started), reason: "exited"} case <-ctx.Done(): - signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) + signalProcessGroup(groupID, syscall.SIGKILL) return tuiWaitResult{ err: <-waitDone, elapsed: time.Since(started), reason: "interrupted", } case <-timer.C: elapsed := time.Since(started) - signalProcessGroup(cmd.Process.Pid, syscall.SIGKILL) + signalProcessGroup(groupID, syscall.SIGKILL) return tuiWaitResult{err: <-waitDone, elapsed: elapsed, reason: "duration"} } } diff --git a/internal/runner/tui_result.go b/internal/runner/tui_result.go index 6c30162..10c27d4 100644 --- a/internal/runner/tui_result.go +++ b/internal/runner/tui_result.go @@ -1,36 +1,45 @@ package runner import ( - "os/exec" + "os" "time" "github.com/shellcell/snailrace/internal/model" - "github.com/shellcell/snailrace/internal/platform" ) -func makeTUIRun( - cmd *exec.Cmd, +func makeRun( + state *os.ProcessState, elapsed time.Duration, user, system float64, rusageRSS uint64, - peak platform.Metrics, + samples sampleAggregate, reason string, ) model.Run { + peak := samples.peak + peak.Processes = max(peak.Processes, 1) + peak.Threads = max(peak.Threads, 1) return model.Run{ - ExitCode: cmd.ProcessState.ExitCode(), WallSeconds: elapsed.Seconds(), + ExitCode: state.ExitCode(), WallSeconds: elapsed.Seconds(), CPUUserSeconds: user, CPUSystemSeconds: system, AverageCPUPercent: averageCPUPercent(user, system, elapsed), PeakResidentBytes: float64(peak.ResidentBytes), PeakPhysicalFootprintBytes: float64(peak.PhysicalFootprintBytes), PhysicalFootprintValid: peak.PhysicalFootprintValid, OSMaxRSSBytes: float64(rusageRSS), - MeanResidentBytes: sampledMeanResident(peak), + MeanResidentBytes: samples.meanResident(), PeakVirtualBytes: float64(peak.VirtualBytes), PeakProcesses: float64(peak.Processes), PeakThreads: float64(peak.Threads), PeakFileDescriptors: float64(peak.FileDescriptors), StopReason: reason, - SampleCount: int(peak.SampleCount), - SampleCoverageSeconds: peak.SampleCoverageSeconds, + SampleCount: int(samples.sampleCount), + SampleCoverageSeconds: samples.sampleCoverageSeconds, } } + +func averageCPUPercent(user, system float64, elapsed time.Duration) float64 { + if elapsed <= 0 { + return 0 + } + return (user + system) / elapsed.Seconds() * 100 +} diff --git a/internal/style/command.go b/internal/style/command.go new file mode 100644 index 0000000..e52500e --- /dev/null +++ b/internal/style/command.go @@ -0,0 +1,41 @@ +package style + +import ( + "strconv" + "strings" + "unicode" +) + +func CommandText(command []string, shell bool) string { + if len(command) == 0 { + return "" + } + if shell { + return safeCommandText(command[0]) + } + arguments := make([]string, len(command)) + for index, argument := range command { + arguments[index] = quoteArgument(argument) + } + return strings.Join(arguments, " ") +} + +func quoteArgument(argument string) string { + if strings.IndexFunc(argument, unicode.IsControl) >= 0 { + return strconv.QuoteToGraphic(argument) + } + if argument != "" && strings.IndexFunc(argument, func(character rune) bool { + return !(unicode.IsLetter(character) || unicode.IsDigit(character) || + strings.ContainsRune("_@%+=:,./-", character)) + }) < 0 { + return argument + } + return "'" + strings.ReplaceAll(argument, "'", `'"'"'`) + "'" +} + +func safeCommandText(command string) string { + if strings.IndexFunc(command, unicode.IsControl) >= 0 { + return strconv.QuoteToGraphic(command) + } + return command +} diff --git a/internal/style/command_test.go b/internal/style/command_test.go new file mode 100644 index 0000000..3bb1ddc --- /dev/null +++ b/internal/style/command_test.go @@ -0,0 +1,14 @@ +package style + +import "testing" + +func TestCommandTextQuotesArgumentsWithoutChangingShellCommands(t *testing.T) { + if got, want := CommandText( + []string{"tool", "two words", "", "it's", "line\n"}, false, + ), `tool 'two words' '' 'it'"'"'s' "line\n"`; got != want { + t.Fatalf("command = %q, want %q", got, want) + } + if got := CommandText([]string{"printf '%s' value"}, true); got != "printf '%s' value" { + t.Fatalf("shell command = %q", got) + } +} From b9deccd4ff48e69d5e505435ed4a7c8573418e72 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Mon, 20 Jul 2026 08:02:41 +0200 Subject: [PATCH 09/11] snailrace v0.0.5: fixed macos binary pinning bag Signed-off-by: Polina Simonenko --- go.mod | 2 +- internal/app/output_test.go | 4 ++-- internal/runner/benchmark.go | 2 +- internal/runner/benchmark_test.go | 4 ++-- internal/runner/executable_darwin.go | 11 +++++++---- internal/runner/executable_linux.go | 2 ++ internal/runner/spec.go | 2 +- internal/runner/tool_test.go | 13 +++++++++---- internal/runner/tui_test.go | 2 +- 9 files changed, 26 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index a942b0b..f69df60 100644 --- a/go.mod +++ b/go.mod @@ -9,4 +9,4 @@ require ( golang.org/x/term v0.45.0 ) -require golang.org/x/sys v0.47.0 // indirect +require golang.org/x/sys v0.47.0 diff --git a/internal/app/output_test.go b/internal/app/output_test.go index 6c31457..2ad9714 100644 --- a/internal/app/output_test.go +++ b/internal/app/output_test.go @@ -151,7 +151,7 @@ func TestRunSavesDefaultHTMLInCurrentDirectory(t *testing.T) { t.Chdir(directory) var stdout, stderr bytes.Buffer if err := Run( - []string{"-n", "1", "-warmups", "0", "--", "/bin/true"}, + []string{"-n", "1", "-warmups", "0", "--", "true"}, &stdout, &stderr, ); err != nil { t.Fatal(err) @@ -172,7 +172,7 @@ func TestRunNoSaveLeavesCurrentDirectoryEmpty(t *testing.T) { directory := t.TempDir() t.Chdir(directory) if err := Run( - []string{"-no-save", "-n", "1", "-warmups", "0", "--", "/bin/true"}, + []string{"-no-save", "-n", "1", "-warmups", "0", "--", "true"}, io.Discard, io.Discard, ); err != nil { t.Fatal(err) diff --git a/internal/runner/benchmark.go b/internal/runner/benchmark.go index e715975..c2cf61e 100644 --- a/internal/runner/benchmark.go +++ b/internal/runner/benchmark.go @@ -54,7 +54,7 @@ func Benchmark( return nil, fmt.Errorf("hash %q: %w", tool.Name, err) } pinned[index] = file - pinExecution := !executableIsScript(file) + pinExecution := pinExecutionSupported && !executableIsScript(file) if pinExecution { prepared[index].executable = file } diff --git a/internal/runner/benchmark_test.go b/internal/runner/benchmark_test.go index e4b763b..78b7757 100644 --- a/internal/runner/benchmark_test.go +++ b/internal/runner/benchmark_test.go @@ -60,7 +60,7 @@ func TestBenchmarkContinuesAfterNonZeroExit(t *testing.T) { context.Background(), []Spec{ {Name: "exit", Shell: "exit 7"}, - {Name: "true", Args: []string{"/bin/true"}}, + {Name: "true", Args: []string{"true"}}, }, Config{Runs: 3, Warmups: 1, Interval: time.Millisecond}, Options{}, @@ -130,7 +130,7 @@ func TestScriptExecutionPreservesOriginalPath(t *testing.T) { } func TestBenchmarkRejectsInvalidInputs(t *testing.T) { - validSpec := Spec{Name: "true", Args: []string{"/bin/true"}} + validSpec := Spec{Name: "true", Args: []string{"true"}} validConfig := Config{Runs: 1, Interval: time.Millisecond} tests := []struct { name string diff --git a/internal/runner/executable_darwin.go b/internal/runner/executable_darwin.go index 4c04b6e..c90f578 100644 --- a/internal/runner/executable_darwin.go +++ b/internal/runner/executable_darwin.go @@ -2,8 +2,11 @@ package runner import "os" -func pinnedExecutablePath(*os.File) string { return "/dev/fd/9" } +// macOS cannot execve /dev/fd nodes and has no fexecve, so execution cannot +// be pinned to the inspected descriptor; provenance relies on the before and +// after hash verification instead. +const pinExecutionSupported = false -func pinnedExtraFiles(file *os.File) []*os.File { - return []*os.File{nil, nil, nil, nil, nil, nil, file} -} +func pinnedExecutablePath(*os.File) string { return "" } + +func pinnedExtraFiles(*os.File) []*os.File { return nil } diff --git a/internal/runner/executable_linux.go b/internal/runner/executable_linux.go index 9c01528..48d98be 100644 --- a/internal/runner/executable_linux.go +++ b/internal/runner/executable_linux.go @@ -5,6 +5,8 @@ import ( "os" ) +const pinExecutionSupported = true + func pinnedExecutablePath(file *os.File) string { return fmt.Sprintf("/proc/%d/fd/%d", os.Getpid(), file.Fd()) } diff --git a/internal/runner/spec.go b/internal/runner/spec.go index fc2cc86..3d4f7a3 100644 --- a/internal/runner/spec.go +++ b/internal/runner/spec.go @@ -33,7 +33,7 @@ func (spec preparedSpec) command(ctx context.Context) *exec.Cmd { var command *exec.Cmd if spec.Shell != "" { shell := "/bin/sh" - if spec.executable != nil && !spec.shellTarget { + if path != "" && !spec.shellTarget { shell = path } command = exec.CommandContext(ctx, shell, "-c", spec.Shell) diff --git a/internal/runner/tool_test.go b/internal/runner/tool_test.go index 7071ee0..3b28344 100644 --- a/internal/runner/tool_test.go +++ b/internal/runner/tool_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "os" + "os/exec" "path/filepath" "testing" @@ -14,7 +15,7 @@ func TestInspectionHonorsCancelledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() if _, err := newToolInspector().inspect( - ctx, Spec{Name: "true", Args: []string{"/bin/true"}}, + ctx, Spec{Name: "true", Args: []string{"true"}}, ); err == nil { t.Fatal("cancelled inspection should fail") } @@ -30,7 +31,7 @@ func TestToolInspectorUsesInjectedDependencyDiscovery(t *testing.T) { }) for _, name := range []string{"first", "second"} { if _, err := inspector.inspect( - context.Background(), Spec{Name: name, Args: []string{"/bin/true"}}, + context.Background(), Spec{Name: name, Args: []string{"true"}}, ); err != nil { t.Fatal(err) } @@ -45,7 +46,7 @@ func TestToolInspectorUsesInjectedDependencyDiscovery(t *testing.T) { return nil, expected }) if _, err := inspector.inspect( - context.Background(), Spec{Name: "tool", Args: []string{"/bin/true"}}, + context.Background(), Spec{Name: "tool", Args: []string{"true"}}, ); !errors.Is(err, expected) { t.Fatalf("dependency error = %v, want %v", err, expected) } @@ -86,7 +87,11 @@ func TestPinShellExecutablePreservesArguments(t *testing.T) { func TestToolVerificationDetectsExecutableReplacement(t *testing.T) { path := filepath.Join(t.TempDir(), "tool") - data, err := os.ReadFile("/bin/true") + source, err := exec.LookPath("true") + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(source) if err != nil { t.Fatal(err) } diff --git a/internal/runner/tui_test.go b/internal/runner/tui_test.go index 9e01742..6c0fd08 100644 --- a/internal/runner/tui_test.go +++ b/internal/runner/tui_test.go @@ -31,7 +31,7 @@ func TestFixedDurationTUIStopsProcessGroup(t *testing.T) { func TestFixedDurationTUIRejectsEarlyExit(t *testing.T) { _, err := runTUIOnce( - context.Background(), Spec{Name: "true", Args: []string{"/bin/true"}}, + context.Background(), Spec{Name: "true", Args: []string{"true"}}, time.Millisecond, Options{TUI: true, Duration: 50 * time.Millisecond}, ) if err == nil { From 47829f8450973c4a644fe5e79fc2dc024f5246b6 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Mon, 20 Jul 2026 08:21:30 +0200 Subject: [PATCH 10/11] snailrace v0.0.5: macos ci, small fixes Signed-off-by: Polina Simonenko --- .github/workflows/ci.yml | 21 ++++++++++++ internal/app/output_bundle.go | 53 ++++++++++++++++++++++------- internal/app/output_test.go | 51 +++++++++++++++++++++++++++ internal/app/progress.go | 8 ++--- internal/model/stats.go | 44 ++---------------------- internal/platform/sampler_darwin.go | 24 ++++++++++--- internal/platform/sampler_linux.go | 27 ++++++++++++--- internal/runner/benchmark_test.go | 22 ++++++++++++ internal/runner/executable_linux.go | 3 ++ internal/runner/monitor.go | 26 ++++++++++---- internal/runner/run.go | 2 +- internal/runner/spec.go | 7 ++++ internal/runner/terminal.go | 26 ++++++++------ internal/runner/tool.go | 3 ++ internal/runner/tui.go | 2 +- internal/style/width.go | 33 ++++++++++++++++++ internal/style/width_test.go | 24 +++++++++++++ 17 files changed, 290 insertions(+), 86 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 internal/style/width.go create mode 100644 internal/style/width_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c9762e7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,21 @@ +name: ci + +on: + push: + branches: ['**'] + pull_request: + +jobs: + test: + strategy: + fail-fast: false + matrix: + runner: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + - run: go test -race ./... + - run: go vet ./... diff --git a/internal/app/output_bundle.go b/internal/app/output_bundle.go index d8cd4e9..c7858b1 100644 --- a/internal/app/output_bundle.go +++ b/internal/app/output_bundle.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/shellcell/snailrace/internal/report" ) @@ -98,46 +99,74 @@ func saveReportFormats( return nil } +const maxReportNameAttempts = 10000 + func reserveReportStem( directory string, base string, ) (string, func(), error) { - for suffix := 1; ; suffix++ { + for suffix := 1; suffix <= maxReportNameAttempts; suffix++ { stem := base if suffix > 1 { stem = fmt.Sprintf("%s-%d", base, suffix) } - lockPath := filepath.Join(directory, "."+stem+".lock") + release, reserved, err := tryReserveStem(directory, stem) + if err != nil { + return "", nil, err + } + if reserved { + return stem, release, nil + } + } + return "", nil, fmt.Errorf("no available report name for %q in %s", base, directory) +} + +func tryReserveStem(directory, stem string) (func(), bool, error) { + lockPath := filepath.Join(directory, "."+stem+".lock") + for attempt := range 2 { lock, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) if errors.Is(err, os.ErrExist) { - continue + // Reclaim a lock leaked by a crashed run, then retry this stem once. + if attempt == 0 && staleLockFile(lockPath) && os.Remove(lockPath) == nil { + continue + } + return nil, false, nil } if err != nil { - return "", nil, err + return nil, false, err } if err := lock.Close(); err != nil { os.Remove(lockPath) - return "", nil, err + return nil, false, err } - if reportStemExists(directory, stem) { + exists, err := reportStemExists(directory, stem) + if err != nil || exists { os.Remove(lockPath) - continue + return nil, false, err } - return stem, func() { os.Remove(lockPath) }, nil + return func() { os.Remove(lockPath) }, true, nil } + return nil, false, nil +} + +// Locks are held only while a report is being written, so anything old enough +// to predate the current run by an hour was leaked by a crashed process. +func staleLockFile(path string) bool { + info, err := os.Stat(path) + return err == nil && time.Since(info.ModTime()) > time.Hour } -func reportStemExists(directory, stem string) bool { +func reportStemExists(directory, stem string) (bool, error) { entries, err := os.ReadDir(directory) if err != nil { - return true + return false, err } for _, entry := range entries { if entry.Name() == stem || strings.HasPrefix(entry.Name(), stem+".") { - return true + return true, nil } } - return false + return false, nil } func writeMarkdownFile( diff --git a/internal/app/output_test.go b/internal/app/output_test.go index 2ad9714..defb697 100644 --- a/internal/app/output_test.go +++ b/internal/app/output_test.go @@ -5,9 +5,11 @@ import ( "errors" "io" "os" + "path/filepath" "strings" "sync" "testing" + "time" "github.com/shellcell/snailrace/internal/model" reportpkg "github.com/shellcell/snailrace/internal/report" @@ -217,6 +219,55 @@ func TestSavingSameReportUsesUniqueNames(t *testing.T) { } } +func TestReserveReportStemFailsInUnreadableDirectory(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("directory permissions are not enforced for root") + } + directory := t.TempDir() + if err := os.Chmod(directory, 0o333); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(directory, 0o755) }) + if _, _, err := reserveReportStem(directory, "report"); err == nil { + t.Fatal("unreadable directory should fail instead of retrying forever") + } +} + +func TestReserveReportStemReclaimsStaleLock(t *testing.T) { + directory := t.TempDir() + lockPath := filepath.Join(directory, ".report.lock") + if err := os.WriteFile(lockPath, nil, 0o600); err != nil { + t.Fatal(err) + } + leaked := time.Now().Add(-2 * time.Hour) + if err := os.Chtimes(lockPath, leaked, leaked); err != nil { + t.Fatal(err) + } + stem, release, err := reserveReportStem(directory, "report") + if err != nil { + t.Fatal(err) + } + defer release() + if stem != "report" { + t.Fatalf("stem = %q, want the stale lock reclaimed for %q", stem, "report") + } +} + +func TestReserveReportStemSkipsHeldLock(t *testing.T) { + directory := t.TempDir() + if err := os.WriteFile(filepath.Join(directory, ".report.lock"), nil, 0o600); err != nil { + t.Fatal(err) + } + stem, release, err := reserveReportStem(directory, "report") + if err != nil { + t.Fatal(err) + } + defer release() + if stem != "report-2" { + t.Fatalf("stem = %q, want a fresh lock to skip to %q", stem, "report-2") + } +} + func TestSavingDifferentFormatsDoesNotReuseExistingStem(t *testing.T) { directory := t.TempDir() result := model.Report{ diff --git a/internal/app/progress.go b/internal/app/progress.go index 7d51e18..71fe75c 100644 --- a/internal/app/progress.go +++ b/internal/app/progress.go @@ -13,6 +13,7 @@ import ( "golang.org/x/term" "github.com/shellcell/snailrace/internal/runner" + "github.com/shellcell/snailrace/internal/style" ) func progressRenderer( @@ -147,12 +148,9 @@ func progressDisplayWidth(value string) int { continue } } - r, size := utf8.DecodeRuneInString(value[index:]) + character, size := utf8.DecodeRuneInString(value[index:]) index += size - width++ - if r == '🐌' || r == '🏁' { - width++ - } + width += style.RuneWidth(character) } return width } diff --git a/internal/model/stats.go b/internal/model/stats.go index 8ffb80f..6d34e9a 100644 --- a/internal/model/stats.go +++ b/internal/model/stats.go @@ -6,49 +6,11 @@ import ( ) func Summarize(runs []Run) Summary { - values := func(pick func(Run) float64) []float64 { - result := make([]float64, len(runs)) - for index, run := range runs { - result[index] = pick(run) - } - return result - } - physicalValues := make([]float64, 0, len(runs)) + accumulator := NewSummaryAccumulator(len(runs)) for _, run := range runs { - if run.PhysicalFootprintValid { - physicalValues = append(physicalValues, run.PeakPhysicalFootprintBytes) - } - } - var physicalStats *Stats - if len(physicalValues) > 0 { - value := stats(physicalValues) - physicalStats = &value - } - return Summary{ - WallSeconds: stats(values(func(run Run) float64 { return run.WallSeconds })), - CPUTotalSeconds: stats(values(func(run Run) float64 { - return run.CPUUserSeconds + run.CPUSystemSeconds - })), - CPUUserSeconds: stats(values(func(run Run) float64 { return run.CPUUserSeconds })), - CPUSystemSeconds: stats(values(func(run Run) float64 { return run.CPUSystemSeconds })), - AverageCPUPercent: stats(values(func(run Run) float64 { return run.AverageCPUPercent })), - PeakResidentBytes: stats(values(func(run Run) float64 { return run.PeakResidentBytes })), - PeakPhysicalFootprintBytes: physicalStats, - OSMaxRSSBytes: stats(values(func(run Run) float64 { return run.OSMaxRSSBytes })), - MeanResidentBytes: stats(values(func(run Run) float64 { return run.MeanResidentBytes })), - PeakVirtualBytes: stats(values(func(run Run) float64 { return run.PeakVirtualBytes })), - PeakProcesses: stats(values(func(run Run) float64 { return run.PeakProcesses })), - PeakThreads: stats(values(func(run Run) float64 { return run.PeakThreads })), - PeakFileDescriptors: stats(values(func(run Run) float64 { - return run.PeakFileDescriptors - })), - ValidSampleCount: stats(values(func(run Run) float64 { - return float64(run.SampleCount) - })), - SampleCoverageSeconds: stats(values(func(run Run) float64 { - return run.SampleCoverageSeconds - })), + accumulator.Add(run) } + return accumulator.Snapshot() } func CalculateStats(values []float64) Stats { return stats(values) } diff --git a/internal/platform/sampler_darwin.go b/internal/platform/sampler_darwin.go index 9ac97f2..394fe2e 100644 --- a/internal/platform/sampler_darwin.go +++ b/internal/platform/sampler_darwin.go @@ -25,8 +25,21 @@ import ( "unsafe" ) +// GroupSampler owns the scratch buffers for one monitoring goroutine, so the +// per-tick sampling loop does not generate garbage while a run is measured. +// It is not safe for concurrent use. +type GroupSampler struct { + pids []C.pid_t +} + +func NewGroupSampler() *GroupSampler { return &GroupSampler{} } + func SampleProcessGroup(rootPID int) (Metrics, bool) { - pids := processGroupPIDs(rootPID) + return NewGroupSampler().Sample(rootPID) +} + +func (sampler *GroupSampler) Sample(rootPID int) (Metrics, bool) { + pids := sampler.processGroupPIDs(rootPID) var total Metrics foundRoot := false physicalFootprintValid := true @@ -61,15 +74,18 @@ func SampleProcessGroup(rootPID int) (Metrics, bool) { return total, foundRoot } -func processGroupPIDs(groupID int) []C.pid_t { +func (sampler *GroupSampler) processGroupPIDs(groupID int) []C.pid_t { bytes := C.proc_listpgrppids(C.pid_t(groupID), nil, 0) if bytes <= 0 { return nil } // Leave room for process churn, and retry if the result fills the buffer. capacity := int(bytes) + 16 - for attempts := 0; attempts < 3; attempts++ { - pids := make([]C.pid_t, capacity) + for range 3 { + if cap(sampler.pids) < capacity { + sampler.pids = make([]C.pid_t, capacity) + } + pids := sampler.pids[:capacity] count := int(C.proc_listpgrppids( C.pid_t(groupID), unsafe.Pointer(&pids[0]), C.int(len(pids))*C.sizeof_pid_t, )) diff --git a/internal/platform/sampler_linux.go b/internal/platform/sampler_linux.go index bc96292..0b1e719 100644 --- a/internal/platform/sampler_linux.go +++ b/internal/platform/sampler_linux.go @@ -14,7 +14,26 @@ import ( "golang.org/x/sys/unix" ) +// GroupSampler owns the scratch buffers for one monitoring goroutine, so the +// per-tick sampling loop does not generate garbage while a run is measured. +// It is not safe for concurrent use. +type GroupSampler struct { + statBuffer []byte + directoryBuffer []byte +} + +func NewGroupSampler() *GroupSampler { + return &GroupSampler{ + statBuffer: make([]byte, 4096), + directoryBuffer: make([]byte, 32*1024), + } +} + func SampleProcessGroup(rootPID int) (Metrics, bool) { + return NewGroupSampler().Sample(rootPID) +} + +func (sampler *GroupSampler) Sample(rootPID int) (Metrics, bool) { directory, err := unix.Open("/proc", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) if err != nil { return Metrics{}, false @@ -22,19 +41,17 @@ func SampleProcessGroup(rootPID int) (Metrics, bool) { defer unix.Close(directory) var total Metrics rootFound := false - statBuffer := make([]byte, 4096) - directoryBuffer := make([]byte, 32*1024) for { - count, readErr := unix.ReadDirent(directory, directoryBuffer) + count, readErr := unix.ReadDirent(directory, sampler.directoryBuffer) if readErr != nil { return Metrics{}, false } if count == 0 { return total, rootFound } - validDirents := scanProcDirents(directoryBuffer[:count], func(pid int) { + validDirents := scanProcDirents(sampler.directoryBuffer[:count], func(pid int) { record, err := readProcessAt( - directory, pid, rootPID, statBuffer, + directory, pid, rootPID, sampler.statBuffer, ) if err != nil || record.GroupID != rootPID { return diff --git a/internal/runner/benchmark_test.go b/internal/runner/benchmark_test.go index 78b7757..edc9891 100644 --- a/internal/runner/benchmark_test.go +++ b/internal/runner/benchmark_test.go @@ -80,6 +80,28 @@ func TestBenchmarkContinuesAfterNonZeroExit(t *testing.T) { } } +func TestNativeExecutableRunsExitZero(t *testing.T) { + benchmarks, err := Benchmark( + context.Background(), + []Spec{{Name: "direct", Args: []string{"true"}}, {Name: "shell", Shell: "true"}}, + Config{Runs: 1, Interval: time.Millisecond}, Options{}, + ) + if err != nil { + t.Fatal(err) + } + for _, benchmark := range benchmarks { + if benchmark.Runs[0].ExitCode != 0 { + t.Fatalf( + "native %s exit code = %d, want 0", + benchmark.Tool.Name, benchmark.Runs[0].ExitCode, + ) + } + if !benchmark.Tool.ProvenanceVerified { + t.Fatalf("native %s provenance not verified", benchmark.Tool.Name) + } + } +} + func TestShellBenchmarksInspectDistinctTargetExecutables(t *testing.T) { benchmarks, err := Benchmark( context.Background(), diff --git a/internal/runner/executable_linux.go b/internal/runner/executable_linux.go index 48d98be..8d1c5dc 100644 --- a/internal/runner/executable_linux.go +++ b/internal/runner/executable_linux.go @@ -7,6 +7,9 @@ import ( const pinExecutionSupported = true +// The child resolves the parent's /proc entry, which stays alive as long as +// the parent keeps the descriptor open β€” no fd inheritance is needed, so +// pinnedExtraFiles has nothing to pass down. func pinnedExecutablePath(file *os.File) string { return fmt.Sprintf("/proc/%d/fd/%d", os.Getpid(), file.Fd()) } diff --git a/internal/runner/monitor.go b/internal/runner/monitor.go index 48cca4e..31496c3 100644 --- a/internal/runner/monitor.go +++ b/internal/runner/monitor.go @@ -10,6 +10,8 @@ type processGroupSampler func(int) (platform.Metrics, bool) type sampleAggregate struct { peak platform.Metrics + footprintMissing bool + footprintInvalidated bool residentByteSamples uint64 sampleCount uint64 residentByteSeconds float64 @@ -20,7 +22,6 @@ type sampleAggregate struct { func (aggregate *sampleAggregate) observe(current platform.Metrics, at time.Time) { aggregate.addCoverage(at) - firstSample := aggregate.sampleCount == 0 aggregate.residentByteSamples += current.ResidentBytes aggregate.sampleCount++ aggregate.peak.ResidentBytes = max(aggregate.peak.ResidentBytes, current.ResidentBytes) @@ -30,20 +31,31 @@ func (aggregate *sampleAggregate) observe(current platform.Metrics, at time.Time aggregate.peak.FileDescriptors = max( aggregate.peak.FileDescriptors, current.FileDescriptors, ) + aggregate.observeFootprint(current) + aggregate.previousRSS, aggregate.previousAt = current.ResidentBytes, at +} + +// The aggregate footprint is valid only when every sample carried a valid +// footprint and nothing invalidated it; a sample that merely lacks a footprint +// leaves the aggregate unusable without marking the run invalid. +func (aggregate *sampleAggregate) observeFootprint(current platform.Metrics) { if current.PhysicalFootprintValid { aggregate.peak.PhysicalFootprintBytes = max( aggregate.peak.PhysicalFootprintBytes, current.PhysicalFootprintBytes, ) + } else { + aggregate.footprintMissing = true } - aggregate.peak.PhysicalFootprintInvalid = - aggregate.peak.PhysicalFootprintInvalid || current.PhysicalFootprintInvalid - aggregate.peak.PhysicalFootprintValid = current.PhysicalFootprintValid && - !aggregate.peak.PhysicalFootprintInvalid && - (firstSample || aggregate.peak.PhysicalFootprintValid) - aggregate.previousRSS, aggregate.previousAt = current.ResidentBytes, at + if current.PhysicalFootprintInvalid { + aggregate.footprintInvalidated = true + } + aggregate.peak.PhysicalFootprintValid = !aggregate.footprintMissing && + !aggregate.footprintInvalidated + aggregate.peak.PhysicalFootprintInvalid = aggregate.footprintInvalidated } func (aggregate *sampleAggregate) invalidatePhysicalFootprint() { + aggregate.footprintInvalidated = true aggregate.peak.PhysicalFootprintInvalid = true aggregate.peak.PhysicalFootprintValid = false } diff --git a/internal/runner/run.go b/internal/runner/run.go index c334c60..edd0d44 100644 --- a/internal/runner/run.go +++ b/internal/runner/run.go @@ -35,7 +35,7 @@ func runOnce( return model.Run{}, err } groupID := cmd.Process.Pid - monitor := startMonitor(groupID, interval, platform.SampleProcessGroup) + monitor := startMonitor(groupID, interval, platform.NewGroupSampler().Sample) waitErr := cmd.Wait() elapsed := time.Since(started) diff --git a/internal/runner/spec.go b/internal/runner/spec.go index 3d4f7a3..21ff7b8 100644 --- a/internal/runner/spec.go +++ b/internal/runner/spec.go @@ -12,6 +12,10 @@ type Spec struct { Shell string } +// preparedSpec executes through the inspected descriptor when executable is +// set, so the bytes that were hashed are provably the bytes that run even if +// the path is swapped underneath us. shellTarget means Shell was already +// rewritten to invoke the descriptor as its leading command word. type preparedSpec struct { Spec executable *os.File @@ -37,6 +41,9 @@ func (spec preparedSpec) command(ctx context.Context) *exec.Cmd { shell = path } command = exec.CommandContext(ctx, shell, "-c", spec.Shell) + // argv[0] keeps the conventional name so the measured process and + // anything it reports (ps, error messages) see the usual identity, + // not a descriptor path. command.Args[0] = "/bin/sh" } else { if path == "" { diff --git a/internal/runner/terminal.go b/internal/runner/terminal.go index 568852a..2d4e1d0 100644 --- a/internal/runner/terminal.go +++ b/internal/runner/terminal.go @@ -79,8 +79,9 @@ func copyInteractiveTerminalOutput( ) { defer close(done) buffer := make([]byte, 4096) + scratch := make([]unix.PollFd, 1) for { - revents, ok := pollDescriptor(ctx, terminalFD, unix.POLLIN) + revents, ok := pollDescriptor(ctx, scratch, terminalFD, unix.POLLIN) if !ok { return } @@ -91,7 +92,7 @@ func copyInteractiveTerminalOutput( continue } count, err := unix.Read(terminalFD, buffer) - if count > 0 && !writeDescriptor(ctx, outputFD, buffer[:count]) { + if count > 0 && !writeDescriptor(ctx, scratch, outputFD, buffer[:count]) { return } if err != nil || count == 0 { @@ -105,8 +106,9 @@ func copyTerminalInput( ) { defer close(done) buffer := make([]byte, 4096) + scratch := make([]unix.PollFd, 1) for { - revents, ok := pollDescriptor(ctx, inputFD, unix.POLLIN) + revents, ok := pollDescriptor(ctx, scratch, inputFD, unix.POLLIN) if !ok { return } @@ -117,7 +119,7 @@ func copyTerminalInput( continue } count, err := unix.Read(inputFD, buffer) - if count > 0 && !writeDescriptor(ctx, int(terminal.Fd()), buffer[:count]) { + if count > 0 && !writeDescriptor(ctx, scratch, int(terminal.Fd()), buffer[:count]) { return } if err != nil || count == 0 { @@ -126,13 +128,15 @@ func copyTerminalInput( } } -func pollDescriptor(ctx context.Context, fd int, events int16) (int16, bool) { +func pollDescriptor( + ctx context.Context, scratch []unix.PollFd, fd int, events int16, +) (int16, bool) { for { if ctx.Err() != nil { return 0, false } - poll := []unix.PollFd{{Fd: int32(fd), Events: events}} - ready, err := unix.Poll(poll, int((100 * time.Millisecond).Milliseconds())) + scratch[0] = unix.PollFd{Fd: int32(fd), Events: events} + ready, err := unix.Poll(scratch[:1], int((100 * time.Millisecond).Milliseconds())) if err == unix.EINTR { continue } @@ -140,14 +144,16 @@ func pollDescriptor(ctx context.Context, fd int, events int16) (int16, bool) { return 0, false } if ready > 0 { - return poll[0].Revents, true + return scratch[0].Revents, true } } } -func writeDescriptor(ctx context.Context, fd int, data []byte) bool { +func writeDescriptor( + ctx context.Context, scratch []unix.PollFd, fd int, data []byte, +) bool { for len(data) > 0 { - revents, ok := pollDescriptor(ctx, fd, unix.POLLOUT) + revents, ok := pollDescriptor(ctx, scratch, fd, unix.POLLOUT) if !ok || revents&(unix.POLLHUP|unix.POLLERR|unix.POLLNVAL) != 0 { return false } diff --git a/internal/runner/tool.go b/internal/runner/tool.go index 2e14c78..be3da9b 100644 --- a/internal/runner/tool.go +++ b/internal/runner/tool.go @@ -231,6 +231,9 @@ func shellBuiltin(name string) bool { return builtins[name] } +// pinShellExecutable swaps the leading command word for the pinned descriptor +// path so /bin/sh execs the inspected bytes; it declines when the word carries +// shell syntax that substitution could change the meaning of. func pinShellExecutable(command, executable string) (string, bool) { trimmed := strings.TrimLeft(command, " \t\r\n") if trimmed == "" { diff --git a/internal/runner/tui.go b/internal/runner/tui.go index a93936a..f7914e4 100644 --- a/internal/runner/tui.go +++ b/internal/runner/tui.go @@ -80,7 +80,7 @@ func runTUIOnce( } groupID := cmd.Process.Pid - monitor := startMonitor(groupID, interval, platform.SampleProcessGroup) + monitor := startMonitor(groupID, interval, platform.NewGroupSampler().Sample) waitDone := make(chan error, 1) go func() { waitDone <- cmd.Wait() }() diff --git a/internal/style/width.go b/internal/style/width.go new file mode 100644 index 0000000..f33f239 --- /dev/null +++ b/internal/style/width.go @@ -0,0 +1,33 @@ +package style + +import "unicode" + +// wideRanges lists Unicode blocks that monospace terminals render as two +// cells: East Asian Wide and Fullwidth blocks plus the common emoji planes. +var wideRanges = [][2]rune{ + {0x1100, 0x115F}, {0x2329, 0x232A}, {0x2E80, 0x303E}, {0x3041, 0x33FF}, + {0x3400, 0x4DBF}, {0x4E00, 0x9FFF}, {0xA000, 0xA4CF}, {0xA960, 0xA97F}, + {0xAC00, 0xD7A3}, {0xF900, 0xFAFF}, {0xFE10, 0xFE19}, {0xFE30, 0xFE6F}, + {0xFF00, 0xFF60}, {0xFFE0, 0xFFE6}, {0x1F300, 0x1F64F}, {0x1F680, 0x1F6FF}, + {0x1F900, 0x1FAFF}, {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD}, +} + +func RuneWidth(character rune) int { + if unicode.In(character, unicode.Mn, unicode.Me, unicode.Cf) { + return 0 + } + for _, span := range wideRanges { + if character >= span[0] && character <= span[1] { + return 2 + } + } + return 1 +} + +func DisplayWidth(value string) int { + width := 0 + for _, character := range value { + width += RuneWidth(character) + } + return width +} diff --git a/internal/style/width_test.go b/internal/style/width_test.go new file mode 100644 index 0000000..1755c36 --- /dev/null +++ b/internal/style/width_test.go @@ -0,0 +1,24 @@ +package style + +import "testing" + +func TestDisplayWidth(t *testing.T) { + tests := []struct { + value string + width int + }{ + {"", 0}, + {"curl #1", 7}, + {"🐌", 2}, + {"🏁", 2}, + {"ζ€§θƒ½γƒ†γ‚Ήγƒˆ", 10}, + {"ν•œκΈ€", 4}, + {"café", 4}, + {"a‍b", 2}, + } + for _, test := range tests { + if got := DisplayWidth(test.value); got != test.width { + t.Errorf("DisplayWidth(%q) = %d, want %d", test.value, got, test.width) + } + } +} From 625fabdbfc93c56803ad12a02b2f314c059cc8af Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Mon, 20 Jul 2026 08:46:46 +0200 Subject: [PATCH 11/11] snailrace v0.0.5: fixed linux ci Signed-off-by: Polina Simonenko --- internal/runner/benchmark.go | 27 ++++++++----- internal/runner/benchmark_linux_test.go | 16 ++++++-- internal/runner/benchmark_test.go | 16 +++++++- internal/runner/spec.go | 14 ++++--- internal/runner/spec_test.go | 16 ++++++++ internal/runner/tool.go | 51 ++++++++++++++----------- internal/runner/tool_test.go | 15 ++++++++ 7 files changed, 113 insertions(+), 42 deletions(-) diff --git a/internal/runner/benchmark.go b/internal/runner/benchmark.go index c2cf61e..90093b5 100644 --- a/internal/runner/benchmark.go +++ b/internal/runner/benchmark.go @@ -54,16 +54,23 @@ func Benchmark( return nil, fmt.Errorf("hash %q: %w", tool.Name, err) } pinned[index] = file - pinExecution := pinExecutionSupported && !executableIsScript(file) - if pinExecution { - prepared[index].executable = file - } - if tool.ShellTarget && pinExecution { - if command, ok := pinShellExecutable( - spec.Shell, pinnedExecutablePath(file), - ); ok { - prepared[index].Shell = command - prepared[index].shellTarget = true + if pinnable := !executableIsScript(file); pinnable { + if tool.ShellTarget { + target := tool.Executable + if pinExecutionSupported { + target = pinnedExecutablePath(file) + } + if command, ok := pinShellExecutable(spec.Shell, target); ok { + prepared[index].Shell = command + prepared[index].shellTarget = true + if pinExecutionSupported { + prepared[index].executable = file + } + } + } else if pinExecutionSupported { + prepared[index].executable = file + } else { + prepared[index].executablePath = tool.Executable } } benchmarks[index].Tool = tool diff --git a/internal/runner/benchmark_linux_test.go b/internal/runner/benchmark_linux_test.go index 78242b9..c2bac8e 100644 --- a/internal/runner/benchmark_linux_test.go +++ b/internal/runner/benchmark_linux_test.go @@ -6,16 +6,24 @@ import ( "time" ) -func TestPinnedExecutableDoesNotInflateFileDescriptorCount(t *testing.T) { +func TestPinnedExecutionDoesNotLeakDescriptorsToChild(t *testing.T) { + // The child inspects its own descriptor table with shell builtins: a pin + // descriptor leaked through exec would appear at 3 or above. Sampling + // PeakFileDescriptors instead is unreliable β€” libc startup transiently + // opens files (locale archive, gconv), which a sample may legitimately + // catch. benchmarks, err := Benchmark( context.Background(), - []Spec{{Name: "sleep", Args: []string{"/bin/sleep", "0.05"}}}, + []Spec{{Name: "fds", Args: []string{ + "/bin/sh", "-c", + "[ ! -e /proc/self/fd/3 ] && [ ! -e /proc/self/fd/9 ]", + }}}, Config{Runs: 1, Interval: time.Millisecond}, Options{}, ) if err != nil { t.Fatal(err) } - if got := benchmarks[0].Runs[0].PeakFileDescriptors; got > 3 { - t.Fatalf("peak file descriptors = %.0f, want no inherited pin descriptor", got) + if code := benchmarks[0].Runs[0].ExitCode; code != 0 { + t.Fatalf("child saw inherited descriptors (exit %d)", code) } } diff --git a/internal/runner/benchmark_test.go b/internal/runner/benchmark_test.go index edc9891..987dab5 100644 --- a/internal/runner/benchmark_test.go +++ b/internal/runner/benchmark_test.go @@ -103,9 +103,23 @@ func TestNativeExecutableRunsExitZero(t *testing.T) { } func TestShellBenchmarksInspectDistinctTargetExecutables(t *testing.T) { + // Generated targets with different sizes keep the assertions hermetic: + // real system binaries can collide byte-for-byte (uname and sleep are the + // same size on Ubuntu 24.04). + directory := t.TempDir() + first := filepath.Join(directory, "first-tool") + second := filepath.Join(directory, "second-tool") + if err := os.WriteFile(first, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + second, []byte("#!/bin/sh\n# deliberately larger target\nexit 0\n"), 0o755, + ); err != nil { + t.Fatal(err) + } benchmarks, err := Benchmark( context.Background(), - []Spec{{Name: "uname", Shell: "uname"}, {Name: "sleep", Shell: "sleep 0"}}, + []Spec{{Name: "first", Shell: first}, {Name: "second", Shell: second}}, Config{Runs: 1, Interval: time.Millisecond}, Options{}, ) if err != nil { diff --git a/internal/runner/spec.go b/internal/runner/spec.go index 21ff7b8..6b71236 100644 --- a/internal/runner/spec.go +++ b/internal/runner/spec.go @@ -14,12 +14,16 @@ type Spec struct { // preparedSpec executes through the inspected descriptor when executable is // set, so the bytes that were hashed are provably the bytes that run even if -// the path is swapped underneath us. shellTarget means Shell was already -// rewritten to invoke the descriptor as its leading command word. +// the path is swapped underneath us. executablePath is the weaker fallback +// for platforms without descriptor execution (macOS): runs are pinned to the +// inspected absolute path, preventing PATH re-resolution between runs. +// shellTarget means Shell was already rewritten to invoke the pinned target +// as its leading command word. type preparedSpec struct { Spec - executable *os.File - shellTarget bool + executable *os.File + executablePath string + shellTarget bool } func (spec Spec) command(ctx context.Context) *exec.Cmd { @@ -30,7 +34,7 @@ func (spec Spec) command(ctx context.Context) *exec.Cmd { } func (spec preparedSpec) command(ctx context.Context) *exec.Cmd { - path := "" + path := spec.executablePath if spec.executable != nil { path = pinnedExecutablePath(spec.executable) } diff --git a/internal/runner/spec_test.go b/internal/runner/spec_test.go index fb8bd23..60cbc30 100644 --- a/internal/runner/spec_test.go +++ b/internal/runner/spec_test.go @@ -34,3 +34,19 @@ func TestPreparedSpecPreservesInvocationIdentity(t *testing.T) { t.Fatalf("shell target path/argv = %q/%v", target.Path, target.Args) } } + +func TestPreparedSpecPinsResolvedPathWithoutDescriptor(t *testing.T) { + direct := preparedSpec{ + Spec: Spec{Args: []string{"tool", "arg"}}, executablePath: "/resolved/tool", + }.command(context.Background()) + if direct.Path != "/resolved/tool" || direct.Args[0] != "tool" { + t.Fatalf("direct path/argv = %q/%v", direct.Path, direct.Args) + } + + shell := preparedSpec{ + Spec: Spec{Shell: "exit 0"}, executablePath: "/resolved/sh", + }.command(context.Background()) + if shell.Path != "/resolved/sh" || shell.Args[0] != "/bin/sh" { + t.Fatalf("shell path/argv = %q/%v", shell.Path, shell.Args) + } +} diff --git a/internal/runner/tool.go b/internal/runner/tool.go index be3da9b..e32285c 100644 --- a/internal/runner/tool.go +++ b/internal/runner/tool.go @@ -63,7 +63,10 @@ func (inspector *toolInspector) inspect( "find executable %q: %w", spec.Args[0], err, ) } - executable, _ = filepath.Abs(path) + executable = path + if absolute, err := filepath.Abs(path); err == nil { + executable = absolute + } } if resolved, err := filepath.EvalSymlinks(executable); err == nil { executable = resolved @@ -118,12 +121,11 @@ func (inspector *toolInspector) pin( file.Close() return nil, err } - if inspected := inspector.fileCache[tool.Executable]; inspected == nil || - !os.SameFile(inspected, info) { + inspected := inspector.fileCache[tool.Executable] + if inspected == nil || !os.SameFile(inspected, info) { file.Close() return nil, errors.New("executable changed while being inspected") } - inspected := inspector.fileCache[tool.Executable] if inspected.Size() != info.Size() || !inspected.ModTime().Equal(info.ModTime()) { file.Close() return nil, errors.New("executable contents changed while being inspected") @@ -193,10 +195,16 @@ func hashOpenFile(ctx context.Context, file *os.File) (string, error) { } } +// shellSpecial lists characters that give the leading word of a shell command +// meaning beyond a plain executable name β€” quoting, operators, expansions, +// globs, assignments, comments. Such words are never treated as pinnable +// targets: substituting them could change what the shell would have run. +const shellSpecial = "'\"|&;<>()$`*?[~=#" + func shellExecutable(command string) string { fields := strings.Fields(command) if len(fields) == 0 || shellBuiltin(fields[0]) || - strings.ContainsAny(fields[0], "'\"|&;<>()$`") { + strings.ContainsAny(fields[0], shellSpecial) { return "" } path, err := exec.LookPath(fields[0]) @@ -213,24 +221,23 @@ func shellExecutable(command string) string { return path } -func shellBuiltin(name string) bool { - builtins := map[string]bool{ - "!": true, ".": true, ":": true, "[": true, "alias": true, - "bg": true, "break": true, "cd": true, "command": true, - "continue": true, "echo": true, "eval": true, "exec": true, - "exit": true, "export": true, "false": true, "fc": true, - "fg": true, "getopts": true, "hash": true, "jobs": true, - "kill": true, "printf": true, "pwd": true, "read": true, - "readonly": true, "return": true, "set": true, "shift": true, - "test": true, "time": true, "times": true, "trap": true, - "true": true, "type": true, "ulimit": true, "umask": true, - "unalias": true, "unset": true, "wait": true, "{": true, - "builtin": true, "declare": true, "local": true, "source": true, - "typeset": true, - } - return builtins[name] +var shellBuiltins = map[string]bool{ + "!": true, ".": true, ":": true, "[": true, "alias": true, + "bg": true, "break": true, "cd": true, "command": true, + "continue": true, "echo": true, "eval": true, "exec": true, + "exit": true, "export": true, "false": true, "fc": true, + "fg": true, "getopts": true, "hash": true, "jobs": true, + "kill": true, "printf": true, "pwd": true, "read": true, + "readonly": true, "return": true, "set": true, "shift": true, + "test": true, "time": true, "times": true, "trap": true, + "true": true, "type": true, "ulimit": true, "umask": true, + "unalias": true, "unset": true, "wait": true, "{": true, + "builtin": true, "declare": true, "local": true, "source": true, + "typeset": true, } +func shellBuiltin(name string) bool { return shellBuiltins[name] } + // pinShellExecutable swaps the leading command word for the pinned descriptor // path so /bin/sh execs the inspected bytes; it declines when the word carries // shell syntax that substitution could change the meaning of. @@ -243,7 +250,7 @@ func pinShellExecutable(command, executable string) (string, bool) { if end < 0 { end = len(trimmed) } - if strings.ContainsAny(trimmed[:end], "'\"|&;<>()$`") { + if strings.ContainsAny(trimmed[:end], shellSpecial) { return command, false } prefix := command[:len(command)-len(trimmed)] diff --git a/internal/runner/tool_test.go b/internal/runner/tool_test.go index 3b28344..e33ea3e 100644 --- a/internal/runner/tool_test.go +++ b/internal/runner/tool_test.go @@ -85,6 +85,21 @@ func TestPinShellExecutablePreservesArguments(t *testing.T) { } } +func TestShellTargetRejectsExpansionSyntax(t *testing.T) { + commands := []string{ + "FOO=bar curl url", "curl* url", "cur?l url", "[tool] x", + "~/bin/tool run", "#comment", "$TOOL run", + } + for _, command := range commands { + if got := shellExecutable(command); got != "" { + t.Errorf("shellExecutable(%q) = %q, want rejection", command, got) + } + if _, ok := pinShellExecutable(command, "/proc/self/fd/3"); ok { + t.Errorf("pinShellExecutable(%q) accepted expansion syntax", command) + } + } +} + func TestToolVerificationDetectsExecutableReplacement(t *testing.T) { path := filepath.Join(t.TempDir(), "tool") source, err := exec.LookPath("true")