|
| 1 | +// Package formats is the compat test bundle for §4 (output format) of |
| 2 | +// CONTRACT.md. |
| 3 | +// |
| 4 | +// What is machine-attested here: |
| 5 | +// |
| 6 | +// - The CLI documents `--format` in its `--help` output (root binary |
| 7 | +// for flat CLIs, or each declared subcommand for cobra-based CLIs |
| 8 | +// configured via compat.Runner.Subcommands). |
| 9 | +// - An unknown `--format` value exits non-zero with an error on |
| 10 | +// stderr and an empty stdout. |
| 11 | +// - The unknown-value parse failure performs no network request. |
| 12 | +// - `--format json` exits zero and emits a JSON value on stdout that |
| 13 | +// unmarshals as `[]any`. |
| 14 | +// - `--format csv` exits zero and emits at least one non-empty line |
| 15 | +// on stdout — the header row — even on an empty result set. |
| 16 | +// - The default (no `--format` flag) and `--format markdown` produce |
| 17 | +// byte-identical stdout. This is how the suite pins down "markdown |
| 18 | +// is the default" without having to parse markdown. |
| 19 | +// |
| 20 | +// The data-path subtests (JSONIsArray, CSVHasHeader, DefaultIsMarkdown) |
| 21 | +// invoke the CLI with no extra args beyond `--format`. Integrators |
| 22 | +// whose CLI requires extra args to succeed (e.g. credentials via env) |
| 23 | +// must arrange for those to be present via Runner.Env. Exporters |
| 24 | +// whose data path is not yet runnable from a clean CI environment |
| 25 | +// should wire the bundle in once it is — the parse-time subtests |
| 26 | +// alone are not enough to claim machine attestation for §4. |
| 27 | +// |
| 28 | +// Exporter usage: |
| 29 | +// |
| 30 | +// //go:build compat |
| 31 | +// package mycli_compat_test |
| 32 | +// |
| 33 | +// import ( |
| 34 | +// "os" |
| 35 | +// "testing" |
| 36 | +// "github.com/quantcli/common/compat" |
| 37 | +// "github.com/quantcli/common/compat/formats" |
| 38 | +// ) |
| 39 | +// |
| 40 | +// func TestContractFormats(t *testing.T) { |
| 41 | +// bin := os.Getenv("EXPORT_CLI_BIN") |
| 42 | +// if bin == "" { t.Skip("EXPORT_CLI_BIN not set") } |
| 43 | +// formats.RunContract(t, compat.Runner{Binary: bin}) |
| 44 | +// } |
| 45 | +package formats |
| 46 | + |
| 47 | +import ( |
| 48 | + "context" |
| 49 | + "encoding/json" |
| 50 | + "strings" |
| 51 | + "testing" |
| 52 | + |
| 53 | + "github.com/quantcli/common/compat" |
| 54 | +) |
| 55 | + |
| 56 | +// RunContract runs the full output-format contract test bundle against |
| 57 | +// r. It is the only function exporters are expected to call. |
| 58 | +// |
| 59 | +// Each assertion is a t.Run subtest, so a failure in one does not mask |
| 60 | +// the others. If r.Subcommands is non-empty, RunContract iterates the |
| 61 | +// list and runs the assertions once per subcommand under a |
| 62 | +// "subcommand=NAME" t.Run group, mirroring compat/dates. |
| 63 | +func RunContract(t *testing.T, r compat.Runner) { |
| 64 | + t.Helper() |
| 65 | + if r.Binary == "" { |
| 66 | + t.Fatal("formats: compat.Runner.Binary is empty") |
| 67 | + } |
| 68 | + |
| 69 | + if len(r.Subcommands) == 0 { |
| 70 | + runContractOne(t, r) |
| 71 | + return |
| 72 | + } |
| 73 | + for _, sub := range r.Subcommands { |
| 74 | + sub := sub |
| 75 | + t.Run("subcommand="+sub, func(t *testing.T) { |
| 76 | + runContractOne(t, r.WithSubcommand(sub)) |
| 77 | + }) |
| 78 | + } |
| 79 | +} |
| 80 | + |
| 81 | +// runContractOne runs the §4 assertions against a single invocation |
| 82 | +// surface — either the root binary (when r has no subcommand prefix) |
| 83 | +// or a specific subcommand of it. |
| 84 | +func runContractOne(t *testing.T, r compat.Runner) { |
| 85 | + t.Helper() |
| 86 | + t.Run("HelpDocumentsFormatFlag", func(t *testing.T) { |
| 87 | + helpDocumentsFormatFlag(t, r) |
| 88 | + }) |
| 89 | + t.Run("UnknownFormatFails", func(t *testing.T) { |
| 90 | + unknownFormatFails(t, r) |
| 91 | + }) |
| 92 | + t.Run("FlagValidationIsHermetic", func(t *testing.T) { |
| 93 | + flagValidationIsHermetic(t, r) |
| 94 | + }) |
| 95 | + t.Run("JSONIsArray", func(t *testing.T) { |
| 96 | + jsonIsArray(t, r) |
| 97 | + }) |
| 98 | + t.Run("CSVHasHeader", func(t *testing.T) { |
| 99 | + csvHasHeader(t, r) |
| 100 | + }) |
| 101 | + t.Run("DefaultIsMarkdown", func(t *testing.T) { |
| 102 | + defaultIsMarkdown(t, r) |
| 103 | + }) |
| 104 | +} |
| 105 | + |
| 106 | +// helpDocumentsFormatFlag asserts that the CLI documents `--format` |
| 107 | +// somewhere in its `--help` output. Like the dates equivalent, this |
| 108 | +// is the minimum binding between §4 and the binary: a CLI that quietly |
| 109 | +// drops the `--format` flag will fail this test. |
| 110 | +func helpDocumentsFormatFlag(t *testing.T, r compat.Runner) { |
| 111 | + t.Helper() |
| 112 | + res := r.MustRun(t, "--help") |
| 113 | + if res.ExitCode != 0 { |
| 114 | + t.Fatalf("--help exited %d, want 0; stderr=%q", res.ExitCode, res.StderrString()) |
| 115 | + } |
| 116 | + combined := res.StdoutString() + "\n" + res.StderrString() |
| 117 | + if !strings.Contains(combined, "--format") { |
| 118 | + t.Errorf("--help output does not mention --format; got stdout=%q stderr=%q", |
| 119 | + res.StdoutString(), res.StderrString()) |
| 120 | + } |
| 121 | +} |
| 122 | + |
| 123 | +// unknownFormatFails asserts that a value like `--format frobnicate` |
| 124 | +// causes the CLI to exit non-zero with an error on stderr and an |
| 125 | +// empty stdout. The empty-stdout check is the §4 "stdout is data |
| 126 | +// only" rule: a parse failure must not contaminate the data stream. |
| 127 | +func unknownFormatFails(t *testing.T, r compat.Runner) { |
| 128 | + t.Helper() |
| 129 | + res, err := r.Run(context.Background(), "--format", unknownFormatValue) |
| 130 | + if err != nil { |
| 131 | + t.Fatalf("run failed: %v", err) |
| 132 | + } |
| 133 | + if res.ExitCode == 0 { |
| 134 | + t.Errorf("unknown --format accepted (exit 0); stdout=%q stderr=%q", |
| 135 | + res.StdoutString(), res.StderrString()) |
| 136 | + } |
| 137 | + if len(res.Stdout) != 0 { |
| 138 | + t.Errorf("unknown --format produced stdout output (§4 violation): %q", |
| 139 | + res.StdoutString()) |
| 140 | + } |
| 141 | + if strings.TrimSpace(res.StderrString()) == "" { |
| 142 | + t.Errorf("unknown --format produced no stderr message") |
| 143 | + } |
| 144 | +} |
| 145 | + |
| 146 | +// flagValidationIsHermetic asserts that the unknown-format parse |
| 147 | +// failure does not dial out. Mirrors the dates bundle's parse-failure |
| 148 | +// hermetic test, but exercises the `--format` parse path explicitly |
| 149 | +// in case the CLI looks up format codecs differently than date |
| 150 | +// values. |
| 151 | +// |
| 152 | +// CONTRACT §5: "A CLI run with --help or with a flag-validation |
| 153 | +// failure must not make network requests." |
| 154 | +func flagValidationIsHermetic(t *testing.T, r compat.Runner) { |
| 155 | + t.Helper() |
| 156 | + res, err := r.WithEnv(noNetworkEnv()...).Run(context.Background(), "--format", unknownFormatValue) |
| 157 | + if err != nil { |
| 158 | + t.Fatalf("run failed under no-network env: %v", err) |
| 159 | + } |
| 160 | + if res.ExitCode == 0 { |
| 161 | + t.Errorf("unknown --format accepted under no-network env (exit 0); stderr=%q", |
| 162 | + res.StderrString()) |
| 163 | + } |
| 164 | +} |
| 165 | + |
| 166 | +// jsonIsArray asserts that `--format json` exits zero and emits |
| 167 | +// stdout that unmarshals as a JSON array (`[]any`). The check is |
| 168 | +// row-count agnostic: zero rows is `[]`, N rows is `[{…},…]`, both |
| 169 | +// pass. The §4 empty-result rule (`[]` on no data) is therefore |
| 170 | +// covered implicitly so long as the integrator's data path returns |
| 171 | +// successfully. |
| 172 | +func jsonIsArray(t *testing.T, r compat.Runner) { |
| 173 | + t.Helper() |
| 174 | + res := r.MustRun(t, "--format", "json") |
| 175 | + if res.ExitCode != 0 { |
| 176 | + t.Fatalf("--format json exited %d; stderr=%q", res.ExitCode, res.StderrString()) |
| 177 | + } |
| 178 | + trimmed := strings.TrimSpace(res.StdoutString()) |
| 179 | + if trimmed == "" { |
| 180 | + t.Fatalf("--format json produced empty stdout; want JSON array (`[]` for empty result per §4)") |
| 181 | + } |
| 182 | + var arr []any |
| 183 | + if err := json.Unmarshal([]byte(trimmed), &arr); err != nil { |
| 184 | + t.Errorf("--format json stdout is not a JSON array: %v; stdout=%q", |
| 185 | + err, res.StdoutString()) |
| 186 | + } |
| 187 | +} |
| 188 | + |
| 189 | +// csvHasHeader asserts that `--format csv` exits zero and emits at |
| 190 | +// least one non-empty line on stdout. §4 says an empty result is |
| 191 | +// success with "no rows" for CSV — the header row is still required, |
| 192 | +// so even a zero-row CSV must have one line. |
| 193 | +func csvHasHeader(t *testing.T, r compat.Runner) { |
| 194 | + t.Helper() |
| 195 | + res := r.MustRun(t, "--format", "csv") |
| 196 | + if res.ExitCode != 0 { |
| 197 | + t.Fatalf("--format csv exited %d; stderr=%q", res.ExitCode, res.StderrString()) |
| 198 | + } |
| 199 | + if len(nonEmptyLines(res.StdoutString())) == 0 { |
| 200 | + t.Errorf("--format csv produced no header row on stdout; got %q", |
| 201 | + res.StdoutString()) |
| 202 | + } |
| 203 | +} |
| 204 | + |
| 205 | +// defaultIsMarkdown asserts that the default format (no `--format` |
| 206 | +// flag) produces byte-identical stdout to `--format markdown`. This is |
| 207 | +// the strongest behavioral statement of "markdown is the default" |
| 208 | +// available without parsing markdown. |
| 209 | +func defaultIsMarkdown(t *testing.T, r compat.Runner) { |
| 210 | + t.Helper() |
| 211 | + noFlag := r.MustRun(t) |
| 212 | + explicit := r.MustRun(t, "--format", "markdown") |
| 213 | + if noFlag.ExitCode != 0 { |
| 214 | + t.Fatalf("default (no --format) exited %d; stderr=%q", |
| 215 | + noFlag.ExitCode, noFlag.StderrString()) |
| 216 | + } |
| 217 | + if explicit.ExitCode != 0 { |
| 218 | + t.Fatalf("--format markdown exited %d; stderr=%q", |
| 219 | + explicit.ExitCode, explicit.StderrString()) |
| 220 | + } |
| 221 | + if noFlag.StdoutString() != explicit.StdoutString() { |
| 222 | + t.Errorf("default stdout differs from --format markdown stdout:\n no-flag: %q\n markdown: %q", |
| 223 | + noFlag.StdoutString(), explicit.StdoutString()) |
| 224 | + } |
| 225 | +} |
| 226 | + |
| 227 | +// unknownFormatValue is a sentinel that no contract-compliant CLI |
| 228 | +// should accept as a `--format` value. Kept as a named constant so a |
| 229 | +// future codec adoption (e.g. `yaml`) does not collide silently with |
| 230 | +// the negative-path probe. |
| 231 | +const unknownFormatValue = "obviously-not-a-format" |
| 232 | + |
| 233 | +// nonEmptyLines splits s on '\n' and drops blank/whitespace-only |
| 234 | +// entries — including the trailing empty string from a final newline. |
| 235 | +func nonEmptyLines(s string) []string { |
| 236 | + var out []string |
| 237 | + for _, line := range strings.Split(s, "\n") { |
| 238 | + if strings.TrimSpace(line) != "" { |
| 239 | + out = append(out, line) |
| 240 | + } |
| 241 | + } |
| 242 | + return out |
| 243 | +} |
| 244 | + |
| 245 | +// noNetworkEnv mirrors compat/dates: point every common HTTP proxy |
| 246 | +// env var at an unreachable address so any flag-validation path that |
| 247 | +// accidentally opens a connection fails or stalls. Kept locally |
| 248 | +// rather than exported from compat to keep the two bundles |
| 249 | +// independently auditable. |
| 250 | +func noNetworkEnv() []string { |
| 251 | + const unreachable = "http://127.0.0.1:1" |
| 252 | + return []string{ |
| 253 | + "HTTP_PROXY=" + unreachable, |
| 254 | + "HTTPS_PROXY=" + unreachable, |
| 255 | + "http_proxy=" + unreachable, |
| 256 | + "https_proxy=" + unreachable, |
| 257 | + "NO_PROXY=", |
| 258 | + "no_proxy=", |
| 259 | + "TZ=UTC", |
| 260 | + } |
| 261 | +} |
0 commit comments