From cf7730a072d8e923fcd171f6ff7d9f9ddcb4877b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 03:26:01 +0000 Subject: [PATCH] Version Packages --- ...i-compiled-schema-parse-and-proto-slots.md | 54 -- ...i-hook-error-telemetry-and-proto-cookie.md | 36 -- .../api-route-dispatch-and-request-parsing.md | 45 -- .changeset/api-sync-pipeline.md | 51 -- .../api-workerd-request-signal-allocation.md | 54 -- .changeset/cli-allowed-roots-flag.md | 32 -- .../cli-silent-failures-and-write-safety.md | 64 --- .../examples-ref-fanout-and-invalid-values.md | 67 --- .changeset/flat-schools-show.md | 18 - ...generators-2020-12-structural-semantics.md | 69 --- .../generators-external-schema-registry.md | 56 -- ...rators-multiple-of-and-unicode-patterns.md | 35 -- .changeset/generators-nan-bounds-parity.md | 30 -- .changeset/helpers-id-base-uri-scoping.md | 49 -- .../helpers-ref-naming-and-resolution.md | 48 -- .changeset/json-schema-suite-conformance.md | 41 -- .changeset/lint-filter-execution-and-redos.md | 65 --- .changeset/packaging-manifest-fixes.md | 61 --- .changeset/parsers-ajv-parity.md | 66 --- .../parsers-required-unions-pointers.md | 44 -- .../parsers-strict-keyword-enforcement.md | 40 -- .changeset/readme-benchmarks-remeasured.md | 34 -- .changeset/resolve-refs-id-scoped-pointers.md | 28 - .changeset/resolve-refs-keep-dynamic-refs.md | 42 -- ...efs-local-confinement-and-cache-scoping.md | 67 --- .../resolve-refs-value-position-refs.md | 32 -- .../runtime-validators-honest-guard-type.md | 31 -- .changeset/runtime-validators-id-base-uri.md | 43 -- ...ime-validators-redos-scope-and-hot-path.md | 99 ---- .../runtime-validators-schema-registry.md | 58 --- .changeset/sweet-worms-brake.md | 11 - .changeset/tidy-donkeys-smile.md | 15 - ...validators-typeless-and-boolean-schemas.md | 45 -- ...alidators-unevaluated-and-honest-guards.md | 50 -- ...yaml-block-structure-and-key-properties.md | 53 -- .../yaml-cr-breaks-merge-and-key-paths.md | 49 -- .../yaml-marker-line-and-flow-folding.md | 43 -- .changeset/yaml-syntax-diagnostics.md | 55 -- .../yaml-tab-columns-flow-indent-and-json.md | 75 --- packages/adapters/CHANGELOG.md | 62 +++ packages/adapters/package.json | 2 +- packages/api/CHANGELOG.md | 324 ++++++++++++ packages/api/package.json | 4 +- packages/cli/CHANGELOG.md | 183 +++++++ packages/cli/package.json | 2 +- packages/generate-examples/CHANGELOG.md | 174 +++++++ packages/generate-examples/package.json | 2 +- packages/generate-markdown/CHANGELOG.md | 51 ++ packages/generate-markdown/package.json | 2 +- packages/generate-parsers/CHANGELOG.md | 493 ++++++++++++++++++ packages/generate-parsers/package.json | 2 +- packages/generate-validators/CHANGELOG.md | 398 ++++++++++++++ packages/generate-validators/package.json | 2 +- packages/helpers/CHANGELOG.md | 322 ++++++++++++ packages/helpers/package.json | 2 +- packages/lint/CHANGELOG.md | 199 +++++++ packages/lint/package.json | 2 +- packages/resolve-refs/CHANGELOG.md | 245 +++++++++ packages/resolve-refs/package.json | 2 +- packages/runtime-validators/CHANGELOG.md | 357 +++++++++++++ packages/runtime-validators/package.json | 2 +- packages/yaml/CHANGELOG.md | 312 +++++++++++ packages/yaml/package.json | 2 +- 63 files changed, 3133 insertions(+), 1868 deletions(-) delete mode 100644 .changeset/api-compiled-schema-parse-and-proto-slots.md delete mode 100644 .changeset/api-hook-error-telemetry-and-proto-cookie.md delete mode 100644 .changeset/api-route-dispatch-and-request-parsing.md delete mode 100644 .changeset/api-sync-pipeline.md delete mode 100644 .changeset/api-workerd-request-signal-allocation.md delete mode 100644 .changeset/cli-allowed-roots-flag.md delete mode 100644 .changeset/cli-silent-failures-and-write-safety.md delete mode 100644 .changeset/examples-ref-fanout-and-invalid-values.md delete mode 100644 .changeset/flat-schools-show.md delete mode 100644 .changeset/generators-2020-12-structural-semantics.md delete mode 100644 .changeset/generators-external-schema-registry.md delete mode 100644 .changeset/generators-multiple-of-and-unicode-patterns.md delete mode 100644 .changeset/generators-nan-bounds-parity.md delete mode 100644 .changeset/helpers-id-base-uri-scoping.md delete mode 100644 .changeset/helpers-ref-naming-and-resolution.md delete mode 100644 .changeset/json-schema-suite-conformance.md delete mode 100644 .changeset/lint-filter-execution-and-redos.md delete mode 100644 .changeset/packaging-manifest-fixes.md delete mode 100644 .changeset/parsers-ajv-parity.md delete mode 100644 .changeset/parsers-required-unions-pointers.md delete mode 100644 .changeset/parsers-strict-keyword-enforcement.md delete mode 100644 .changeset/readme-benchmarks-remeasured.md delete mode 100644 .changeset/resolve-refs-id-scoped-pointers.md delete mode 100644 .changeset/resolve-refs-keep-dynamic-refs.md delete mode 100644 .changeset/resolve-refs-local-confinement-and-cache-scoping.md delete mode 100644 .changeset/resolve-refs-value-position-refs.md delete mode 100644 .changeset/runtime-validators-honest-guard-type.md delete mode 100644 .changeset/runtime-validators-id-base-uri.md delete mode 100644 .changeset/runtime-validators-redos-scope-and-hot-path.md delete mode 100644 .changeset/runtime-validators-schema-registry.md delete mode 100644 .changeset/sweet-worms-brake.md delete mode 100644 .changeset/tidy-donkeys-smile.md delete mode 100644 .changeset/validators-typeless-and-boolean-schemas.md delete mode 100644 .changeset/validators-unevaluated-and-honest-guards.md delete mode 100644 .changeset/yaml-block-structure-and-key-properties.md delete mode 100644 .changeset/yaml-cr-breaks-merge-and-key-paths.md delete mode 100644 .changeset/yaml-marker-line-and-flow-folding.md delete mode 100644 .changeset/yaml-syntax-diagnostics.md delete mode 100644 .changeset/yaml-tab-columns-flow-indent-and-json.md diff --git a/.changeset/api-compiled-schema-parse-and-proto-slots.md b/.changeset/api-compiled-schema-parse-and-proto-slots.md deleted file mode 100644 index 62ee3520..00000000 --- a/.changeset/api-compiled-schema-parse-and-proto-slots.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -'@amritk/api': patch ---- - -Close a compiled-engine validation bypass, and stop dropping `__proto__`-named headers and path parameters - -**`compileToModule` baked its schema constants as object literals, which is not -a faithful copy of the JSON they were printed from.** A JavaScript object -literal treats `__proto__` as the prototype setter, so a contract declaring a -property under that name — perfectly ordinary in a schema loaded from a config -file, a database row, or an imported OpenAPI document, where the key really is -an own property — compiled to a constant with that property silently missing. -The compiled engine then validated a schema the runtime engine never had, and -diverged in both directions: it rejected `{"__proto__":"abc"}` under -`additionalProperties: false` that the runtime accepted, and accepted -`{"__proto__":123}` against `{"type":"string","minLength":3}` that the runtime -rejected. The second is a validation bypass in the production engine — the -declared constraint was simply gone. - -Every constant baked from contract data — request schemas for all five slots, -response body and header schemas, and the interpreter's options — now emits as -`JSON.parse('…')`, where each key lands as an own property. The argument is a -correctly-escaped single-quoted string literal (backslashes, single quotes, and -U+2028/U+2029, which are legal unescaped in JSON but were line terminators in -pre-ES2019 JavaScript source), pinned by a round-trip test over hostile input. -There is no startup cost: a JSON string literal evaluates about 13% faster than -the equivalent object literal at module init on a 46 KB schema, and the emitted -module grows by 14 bytes per constant (0.6% on a realistic module). The -precomputed OpenAPI document was never affected — it was already a string -literal. - -The differential corpus gained a route declaring `__proto__` as its path -parameter, header, cookie, *and* body property at once, so the two engines are -now pinned to agree on the correct answer for all of them, and the emitter has -an invariant test that no schema constant may be a bare object literal. - -**Headers and path parameters named `__proto__` are no longer dropped.** The -same write-side bug the cookie parser had: `__proto__` is a valid HTTP field -name (it is a token) and a valid path-template capture name, but a plain -`record[name] = value` runs the prototype setter instead of creating the -property. A contract declaring one saw nothing, and `required: ['__proto__']` -could never be satisfied. Fixed in the route matcher, the params builder, and -the headers builder through a shared `defineOwnProperty`, which the cookie -parser now shares too; the compiled engine unrolls its own params and headers -builders, so it emits the equivalent `Object.defineProperty` for that one name -and pays nothing for every other. - -Also: the schema-derived response serializer now declines any property whose -name shadows an `Object.prototype` member, falling back to `JSON.stringify` — -its `body[""]` reader would otherwise answer with the inherited member -rather than `undefined` when the reply omits the property, so a `__proto__` -property serialized as `{}` and an optional `toString` was emitted on every -reply. This is the same bail the inline guard emitter already made, and the two -now share one list of risky names. diff --git a/.changeset/api-hook-error-telemetry-and-proto-cookie.md b/.changeset/api-hook-error-telemetry-and-proto-cookie.md deleted file mode 100644 index c553cbfd..00000000 --- a/.changeset/api-hook-error-telemetry-and-proto-cookie.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -'@amritk/api': patch ---- - -Report thrown adapter hooks instead of swallowing them, and stop dropping a `__proto__` cookie - -**A throwing `onRequest` gate or `onResponse` decorator is no longer silent.** -Wrapping the hook chains stopped a throwing hook from escaping to the platform, -but the caught error was then dropped on the floor: no log, and the app's own -`onError` — which every routed failure already goes through — was never -consulted. The motivating case is exactly the one that needs telemetry: -`createRequestId({ trustInbound: true })` reflecting a CRLF-bearing inbound -`x-request-id` into `Headers.set` answered a bare `{"error":"internal_error"}` -with no indication that a decorator threw, or which one. Before the wrapping the -throw at least surfaced as a platform-level unhandled error, so the fix traded a -crash for an undiagnosable 500. - -A thrown hook now goes to the app's `onError` (with `route: undefined` — a hook -belongs to no route), whose reply shapes the response exactly like a handler -error's does, raw-`Response` escape hatch included. An app that wired no -`onError` gets a `console.error` instead, because silence is the one outcome -that is never acceptable here; a reporter that throws falls back to the same log -and the bare 500. `Api` gained an optional `onError` so the adapter can reach the -sink the app already configured — the hooks run outside `handle`, so the -pipeline's own boundary never sees them. `compileToModule` emits the identical -helper, and the two-engine differential corpus now pins that both engines report -the same error, through the same sink, with the same log line. - -**A contract declaring a cookie named `__proto__` now actually receives it.** -The read side treated the name as ordinary data, but the write side was a plain -`cookies[name] = value`, which runs `Object.prototype`'s `__proto__` setter -rather than creating a property — so the value silently vanished and -`required: ['__proto__']` failed for every request no matter what the client -sent. Same `defineProperty` fix already applied in `@amritk/generate-validators` -and `@amritk/yaml`. Both engines share this parser, so the compiled engine picks -it up unchanged. diff --git a/.changeset/api-route-dispatch-and-request-parsing.md b/.changeset/api-route-dispatch-and-request-parsing.md deleted file mode 100644 index 03d332bf..00000000 --- a/.changeset/api-route-dispatch-and-request-parsing.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -'@amritk/api': patch ---- - -Bucket route dispatch by shape, and fix two request-parsing defects - -**Route lookup no longer scans every parameterized route.** The runtime engine -kept one list of dynamic routes per method and walked it in registration order, -re-running the segment matcher against each candidate. At 500 routes that was -~7.3 µs per lookup, and a miss cost the same as a hit. Dynamic routes are now -bucketed by segment count and by their first literal segment, so a lookup only -ever touches candidates that could match the shape in front of it: ~0.55 µs at -500 routes, and flat as the table grows. Precedence is unchanged — the buckets -are precomputed with the wildcard-first routes merged into each literal's list -in registration order, so which of two overlapping routes wins is exactly what -it was, greedy tails and static-over-dynamic included. - -**An unroutable path is no longer the most expensive request an API serves.** -Building the 405 `allow` header re-ran the *whole* matcher once per method the -API declares, so a path from a vulnerability scanner cost up to seven times the -scan — ~45 µs of pure dispatch on a 500-route table, versus ~3 µs to serve a -real request. The static half of that answer is now precomputed at startup (the -same table the compiled engine emits as `ALLOW_STATIC`) and the dynamic half -reuses one path split across all methods: ~0.9 µs. The static hit path also -stopped building a `method + ' ' + path` key per request. - -**Duplicate cookie names now resolve first-wins, not last-wins.** Browsers send -the most specific cookie first (RFC 6265 orders by longer path, then earlier -creation), so a `Path=/` cookie planted from a sibling subdomain arrives *after* -the real session cookie — and last-wins let it shadow it. First-wins is what the -`cookie` package behind Express, Hono, and Fastify does, and what the rest of the -stack assumes. Both engines share this parser, so they stay identical. - -**`buildParamPath` rejects `.` and `..` path parameters.** Dots are unreserved, -so `encodeURIComponent` left them alone and `client.getUser({ params: { id: -'..' } })` built `/users/..`, which the URL parser then collapsed *before the -request was sent* — the call silently hit a different endpoint. It now throws. -Greedy `{name+}` tails are checked per segment for the same reason: WHATWG URL -normalizes `%2e%2e` too, so a literal `..` path component cannot be transmitted -at all, which makes one there always an unintended traversal rather than a -directory name. - -The bench harness gains two dispatch cases — `dynamic GET, 500-route table, last -match (runtime)` and `unroutable path, 500-route table (runtime)` — so the PR -delta table catches a regression in either. diff --git a/.changeset/api-sync-pipeline.md b/.changeset/api-sync-pipeline.md deleted file mode 100644 index 17c31ac9..00000000 --- a/.changeset/api-sync-pipeline.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -'@amritk/api': minor ---- - -Keep the runtime pipeline synchronous when the request never suspends - -**`Api.handle` now returns `ApiResponse | Promise`.** It answers -synchronously when nothing along the route's path was asynchronous — no -declared body to read, no `refine`, no context factory, no guards, and a -handler that returned a value rather than a promise. This is the breaking part: -`await api.handle(...)` is unaffected, but code calling `.then()` on the result -directly must handle a plain value. The bench harness in this repo did exactly -that and is updated alongside. - -**Why.** An `async` frame and its promise are not free, and on workerd the -difference is large enough to see. On the static GET, measured inside a real -isolate with `bench:workerd:allocations`, the runtime engine allocated 2115 -bytes per request; it now allocates 1510, a 29% cut. Throughput on that case -went from ~69k to ~93k ops/s — from 0.80x bare Hono to roughly level with it. - -**How.** `runRoute` is no longer one async function. It is a synchronous -dispatcher over three stages that hand off to each other synchronously until -something genuinely asynchronous appears: - -- `runSecuredRoute` — security guards suspend before anything else happens, so - that whole shape stays asynchronous, and rejoins the shared stages after. -- `runSlots` — coercion and validation for params, query, headers, and cookies, - all of which were already synchronous. -- `runBody` — reading a declared body always suspends, so this stage is - unconditionally asynchronous. -- `runTail` — refine, the context factory, guards, and the handler. Synchronous - when none of the first three are configured and the handler returns a value; - otherwise it delegates to `runTailAsync`, which is the original straight-line - code. - -The error tail is factored into one `routeError` helper the synchronous and -asynchronous halves share, so the two cannot drift on payload-too-large -detection or the `onError` contract. `finishReply` stays outside the error -boundary in both, exactly where it was. - -**What did not change.** Ordering is identical: security guards still run -before any parsing, the context factory still runs after validation on -unsecured routes and before it on secured ones, and it still runs exactly once -per request. The differential corpus that holds the runtime and compiled -engines observationally identical passes unchanged. - -**Still open.** The runtime engine's batch-time distribution is still bimodal -under workerd — a p95 around 3.3x its median, which is a major collection -rather than allocation volume. Neither the async work above nor removing -response validation moved it. The compiled engine, which is the production -path, does not show it. diff --git a/.changeset/api-workerd-request-signal-allocation.md b/.changeset/api-workerd-request-signal-allocation.md deleted file mode 100644 index 1348cedf..00000000 --- a/.changeset/api-workerd-request-signal-allocation.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -'@amritk/api': patch ---- - -Stop paying for an AbortSignal on every request, and re-measure the cross-framework tables - -**Both engines materialized a host-backed `AbortSignal` per request.** The -per-request `ApiRequest` was built with `signal: request.signal`, read eagerly. -On workerd that first touch constructs a host object backed by C++ state — -cheap in bytes, expensive to collect — for handlers that overwhelmingly never -look at it. Hono never creates one at all. Reading it through a getter defers -the cost to the handlers that actually want it. `hookApiRequest` still reads it -eagerly: that path runs once per 500 and hands its object straight to an -`onError` reporter. - -**The getter has to be inherited, not owned.** An own accessor pushes the -object out of V8's in-object slots. The compiled engine's request object had no -accessor before this change, and gaining one took it from 852 to 1276 bytes -allocated per request inside workerd. On a shared prototype the instances stay -plain data objects and the deferral is free. Both engines get the same -treatment, as the differential corpus requires. - -**Measured, not inferred.** The README previously reported that workerd stalled -the `@amritk/api` columns far more often than Hono and guessed the cause was -allocating more per request. That guess was wrong: on the static GET the -compiled engine already allocated 852 bytes per request against bare Hono's -1220, and turned a batch of 2048 requests around faster than Hono did. It -allocated less and ran quicker, then periodically got stopped. After the fix it -allocates 816 bytes per request and stalls on 0 of 60 batches, where before it -stalled on 5 and lost 29% of its wall clock to them. The runtime engine still -stalls and still allocates ~2172 bytes per request; that is called out in the -README as open work rather than presented as solved. - -**New: `bun run bench:workerd:allocations`.** Reads the isolate's heap over -workerd's inspector either side of a run of exactly N requests and regresses -the delta against N, so the Miniflare loopback hop lands in the intercept and -cancels; it also times fixed batches inside the isolate and reports how many -ran more than twice the median. workerd accepts `HeapProfiler.startSampling` -but answers with an empty profile, so there is no per-call-frame attribution to -be had from the runtime. `bench/run-workerd.ts` now repeats each cell across -several fresh isolates and reports the median of the per-isolate medians — -a single isolate's median is robust to a paused trial, but isolates differ from -each other by more than that, which was enough to hide effects this size. - -**The cached-`ResponseInit` comment claimed ~40% and was measured on neither -runtime it gets read on.** Measured now: on Node it is worth about 10% on the -static GET (104k vs 93k ops/s against `Response.json`); inside workerd the -cached init, a cached `Headers` instance, and plain `Response.json` are -indistinguishable. The code stays — it costs nothing anywhere and helps on one -runtime — but the comment now says so. - -All three tables were re-measured together on one machine, which is slower than -the one earlier revisions used, so the absolutes moved down across every column -at once. The README says that where the tables are. diff --git a/.changeset/cli-allowed-roots-flag.md b/.changeset/cli-allowed-roots-flag.md deleted file mode 100644 index e64d575f..00000000 --- a/.changeset/cli-allowed-roots-flag.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -'@amritk/mjst': minor ---- - -Add `--allowed-roots` so a split spec can reach a sibling directory again - -`@amritk/resolve-refs` now confines a local `$ref` to the directory holding the -document it appears in, which closed a real path-traversal hole -(`{"$ref": "/etc/passwd"}` used to be read and inlined). The CLI inherited that -default with no way to widen it, so a completely ordinary multi-version layout — -`specs/v1/api.json` referencing `../common/user.json` — started failing, and the -error told the user to "set allowedRoots", a *library* option nothing on the -command line could reach. - -`--allowed-roots ` is that escape hatch, on both `mjst generate` and -`mjst lint`, alongside an `allowedRoots` config-file key. On the generate path it -takes a comma-separated list or the flag repeated (matching `--allowed-hosts`); -on `lint` you repeat the flag (matching its `--allowed-hosts`). Relative entries -resolve against the current working directory, from a config file as readily as -from the flag, which is how `schema` and `outDir` already behave. - -Two things it deliberately does not do. It does not replace the default: the -schema's (or linted document's) own directory stays allowed, so naming a shared -`common/` folder cannot revoke the one directory nobody would think to list. And -it does not widen anything on its own — there is no implicit default drawn from -the config file's location, because a config file usually sits at the repo root -and quietly granting read access to the whole project tree is not a decision -anyone would read into `--config`. A `$ref` that lands outside every named root -is still refused. - -Refusals now name the flag that exists (`pass --allowed-roots …`) instead -of leaving the library's option name as the only lead. diff --git a/.changeset/cli-silent-failures-and-write-safety.md b/.changeset/cli-silent-failures-and-write-safety.md deleted file mode 100644 index 05eda47e..00000000 --- a/.changeset/cli-silent-failures-and-write-safety.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -'@amritk/mjst': minor ---- - -Stop the CLI from succeeding quietly, and make generation safe to write - -**`mjst lint` no longer exits 0 when its file arguments match nothing.** A glob -or path that resolved to zero files fell through to the stdin branch; in CI -there is no TTY, so stdin is an empty pipe and the linter dutifully reported -"No problems found" on an empty document and exited 0. A typo'd path turned a -lint gate into a silent no-op that reported success. Document arguments that -match nothing now exit 2 with `No files matched: …`, and only a run with no -document arguments at all reads stdin. - -**`mjst lint` rejects unknown flags.** yargs was built without `.strict()`, so -`--bogus-flag`, a mistyped `--fail-severity`, or a misspelled `--allowed-hosts` -was dropped and the run silently used the defaults — the opposite of the -generate command's deliberate strictness. A non-numeric `--concurrency` now -reports what is wrong instead of crashing with `Invalid array length`. - -**Generation never clobbers a file it did not write.** A generated name that -collided with a hand-written file (`index.ts` is the common one) overwrote it -without a word, and `--build` then deleted it along with the other intermediate -sources. Each run records what it wrote in a `.mjst-manifest.json` at the root -of the output directory: paths listed there are reclaimed freely, so -regenerating still needs no ceremony, while anything else aborts the run before -a byte is written. The new `--force` flag opts out. This covers every output the -CLI produces — the parser tree, `--validators`, `--examples`, and `--out-file`, -which is the one most likely to be aimed at hand-written source -(`--out-file src/types.ts` used to overwrite that file silently and, under -`--build`, delete it afterwards). For `--out-file` the manifest lands in the -directory holding the file, alongside the `--build` output and any generated -examples. - -**Generation is atomic.** Files are staged under temporary names and renamed -into place only once the whole set has been written, so a mid-run failure (a -`_helpers` path occupied by a regular file, a full disk) leaves the output -directory exactly as it found it instead of a half-generated tree. - -**`--root-type` is validated, and writes are confined to the output -directory.** `--root-type '../../Escaped'` — from the command line or from a -config file — wrote outside `--out-dir` and emitted a type name that could not -compile. The name must now be a TypeScript identifier, and the writer -independently refuses any path that resolves outside the output directory. - -**Config files are validated.** Every key was guarded by a `typeof` check whose -failure branch was "drop it", so `{"strcit": true, "strict": "true"}` generated -non-strict output and exited 0 while the same typo on the command line was -rejected. Unknown keys and wrong types now fail with the offending pointer and -the expected type, and `config.schema.json` closes the object with -`additionalProperties: false`. - -**`-v` / `-h` / `--version` / `--help` are only honored in flag position.** Both -predicates scanned the whole argv, so any flag whose *value* was `-v` (say -`--type-suffix -v`) printed the version, generated nothing, and exited 0. - -Smaller argument-parsing fixes: `--` is accepted as the end-of-flags terminator -instead of being rejected as an unknown flag; `--config` with no value is an -error rather than a silently skipped config file; a missing config file reports -`Config file not found: …` instead of a raw `ENOENT`; an `--out-dir` pointing at -an existing file explains itself instead of surfacing `EEXIST … mkdir`; a stray -positional (`mjst genrate --schema …`) is rejected rather than quietly generating -as if the typo were not there; and `--build --types-only` no longer claims to -have built `.js` files that were never emitted. diff --git a/.changeset/examples-ref-fanout-and-invalid-values.md b/.changeset/examples-ref-fanout-and-invalid-values.md deleted file mode 100644 index a22169c6..00000000 --- a/.changeset/examples-ref-fanout-and-invalid-values.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -'@amritk/generate-examples': patch ---- - -Make generation linear in the `$ref` graph, and stop shipping examples that fail their own schema - -**A `$ref` reachable by several paths is derived once, not once per path.** -`deriveExample` tracked visited refs in a *path-scoped* set with no memo table, -so every fan-out in the definition graph re-expanded the same subtree -exponentially: a 25-definition graph with three refs per definition took ~20 -seconds, and adding two more definitions roughly quadrupled that. Derivation is -now memoized per ref per root document — the pattern -`@amritk/helpers/walk-ref-graph` already uses for `resolveRef` — with the cycle -guard kept exact: a value produced by cutting a cycle is deliberately not -memoized, so a recursive definition still terminates at the same place. The same -graph at 400 definitions now derives in ~7 ms. - -**A validating check no longer carries the whole document.** Every check spliced -the root's entire `$defs` into the schema it validated, and the interpreter -screens each `pattern` in whatever it is handed — so a 959-definition OpenAPI -document paid for all 959 definitions on each of the thousand-odd checks a -generation run makes, and embedded the whole document into every generated file -carrying a validating filter. Only the definitions a schema's `$ref`s actually -reach travel with it now (a reference that cannot be pinned to one definition — -an `$anchor` name, or a `$dynamicRef`/`$recursiveRef`, whose target is picked -from the dynamic scope at validation time — still falls back to the full set). -Generating the OpenAI corpus went -from ~3.6 s to ~0.3 s, and its generated output from 119 MB to 2.7 MB. - -**Generated arbitraries compile under a strict tsconfig.** `fc.constantFrom("a", -"b")` infers `Arbitrary`, which does not fit the `"a" | "b"` the -generated type declares — so *any* schema with an `enum` property produced a -file no consumer on `strict` could build. Scalar members are now spread from a -`const`-asserted tuple, and a filtered arbitrary's predicate is written as a -type guard (`(value): value is Foo => …`), which is what it has always been: the -combinators generate a superset and the runtime validator narrows it. A new -suite type-checks generated files against the real `fast-check` declarations -under `strict`, `exactOptionalPropertyTypes`, and `noUncheckedIndexedAccess`. - -**An example that fails its own schema is now reported instead of shipped -quietly.** Every `fooExample` is validated (formats included) before it is -written; a value that does not satisfy the schema is still emitted, so the -module compiles, but the generator warns and names the type. Several cases that -used to fail silently now produce valid values: `not` gets a perturbation -candidate (`not: { const: 'string' }` no longer returns `"string"`), -`uniqueItems` over a closed value set walks the set instead of suffixing a -string out of its own `enum`, `pattern` sampling honours `minLength` and reads -control escapes (`a\nb` produced `"anb"`), and static examples now cover every -`format` `@amritk/runtime-validators` checks — `duration`, `json-pointer`, -`relative-json-pointer`, `uri-template`, `uri-reference`, `regex`, and the `idn-` -/`iri` variants. A key that `additionalProperties: false` forbids is no longer -invented. The remaining limits are written down in the README. - -**A `__proto__` property survives.** Both the derived value (`out[key] = …` hit -`Object.prototype`'s prototype setter, so the key vanished) and the emitted -source (a *quoted* `"__proto__":` in an object literal is still the setter, in -the example value and in the `fc.record` config — where it also reassigned the -config object's prototype to an `Arbitrary`). The value uses `defineProperty` -and the source uses the computed `["__proto__"]:` form, matching what -`generate-parsers` already does. - -**A schema the validator refuses no longer kills the run — and no longer goes -unmentioned.** A `$ref` pointing outside the document (`#/components/schemas/…` -in a bare fragment) threw out of `buildExampleSchema`. Those checks are opinions -about a candidate value, so an undecidable schema now abstains. It also warns -once, naming the schema and the reason, because a filter that switches itself -off silently is indistinguishable from one that ran and approved of everything. diff --git a/.changeset/flat-schools-show.md b/.changeset/flat-schools-show.md deleted file mode 100644 index 03a11fd1..00000000 --- a/.changeset/flat-schools-show.md +++ /dev/null @@ -1,18 +0,0 @@ ---- ---- - -Bench the runtime-validators interpreter directly on every PR, and exercise the -`engines: node >=20` floor in CI. - -The interpreter was previously timed only through the `api` suite, which runs it -behind a whole request path — an interpreter regression arrived diluted by -request overhead, and one in a keyword the api contracts happen not to use -arrived not at all, even though `@amritk/lint` runs those keywords on every -schema rule. It now has its own `runtime` suite covering both entry points -(`validateGuard` and the error-collecting `validate`) against valid and invalid -input, with parity checked against Ajv. - -`test:dist` — the only thing that runs the shipped artifacts the way a consumer -does — moves into its own job on a Node 20/24 matrix. It previously ran on -whatever Node `ubuntu-latest` happened to carry, so the floor every package -advertises was never exercised. diff --git a/.changeset/generators-2020-12-structural-semantics.md b/.changeset/generators-2020-12-structural-semantics.md deleted file mode 100644 index 84b5963c..00000000 --- a/.changeset/generators-2020-12-structural-semantics.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -'@amritk/generate-validators': minor -'@amritk/generate-parsers': minor ---- - -Make both generators agree with Draft 2020-12 — and with each other — on -structural equality, tuple `items`, `oneOf`, and prototype-member property names. - -The two generators disagreed with Ajv and with one another on the same schemas. -Where they differed, `generate-validators` was usually right: it already shipped -`valuesEqual` / `allUnique`, which `generate-parsers` never adopted. - -**`@amritk/generate-validators`** - -- A property named `__proto__` was silently dropped. The `nullable` rewrite runs - over *every* schema and copied `properties` with a plain assignment, which fires - the `Object.prototype` setter instead of creating a key — so - `{"properties":{"__proto__":{"type":"string","minLength":3}}}` emitted a - validator with no checks at all, and `required: ["__proto__"]` degraded to a - bare presence check. A validation bypass. -- `constructor` / `toString` / `hasOwnProperty` properties were read straight off - the object, so the *prototype's* value answered: a valid document was reported - as `must be string` at `/hasOwnProperty`, and a required `toString` could never - be reported missing (`'toString' in obj` is always true). Reads now go through - an own-property guard, and presence uses `Object.hasOwn` — but only for the - names that can actually be inherited. Every other key keeps the plain `in` it - always had, because `Object.hasOwn` is a call the engine cannot fold the way it - folds `in`, and spending it on `id` or `name` bought nothing while costing - roughly half the throughput on an all-present object. -- `items` alongside `prefixItems` was applied to the prefix positions too. Per - 2020-12 `items` is the tail schema, so `{prefixItems:[{type:'string'}], - items:{type:'number'}}` rejected `["a", 1, 2]` — which Ajv accepts, and which - the `[string?, ...number[]]` type this generator emits already admits. -- `enum` members that are objects or arrays could never match: `.includes` is - SameValueZero, i.e. reference equality. `enum` now compares structurally via - `valuesEqual`, the way `const` always has — which also makes `isX` a sound type - guard again. - -**`@amritk/generate-parsers`** - -- The same `prefixItems` + `items` defect, in both the fast path and the strict - assertion. -- The same `enum`-with-object-members defect. Members are now compared by an - unrolled structural check against the known literal. -- `const` deep equality used `JSON.stringify`, which is key-order sensitive, so - `{b: 2, a: 1}` was rejected against `const: {a: 1, b: 2}` — and it serialized the - whole value on every call to do it. -- `uniqueItems` used a `JSON.stringify` dedupe key (or a bare `Set`, which - compares objects by reference), so `[{a:1,b:2},{b:2,a:1}]` was accepted where - Ajv and `generate-validators` both reject. It now projects through a - key-order-independent canonical form when items may be structural, keeping the - cheap native `Set` when they are provably scalar. A root array of objects - skipped the constraint entirely and now enforces it. -- `oneOf` exclusivity was not enforced — both `oneOf` and `anyOf` compiled to a - plain disjunction, so a value matching two branches was accepted. `oneOf` now - requires exactly one match. -- The array fallback was a bare `[]`, ignoring `prefixItems` and `minItems`. It is - not an instance of its own schema, and against a required closed tuple it is not - even assignable — `TS2322: Type '[]' is not assignable to type '[string, - number]'` made non-strict and `readonly` output fail to compile. -- A schema property named after an `Object.prototype` member produced an - unsatisfiable type: TypeScript reads the inherited `constructor: Function` on the - fallback object literal and rejects it against `constructor?: string`. - -Both packages now type-check their generated output under the repo's real -compiler flags (`exactOptionalPropertyTypes`, `noUncheckedIndexedAccess`), not -`strict` alone — `generate-validators` had no such suite at all — and both pin the -semantics above against Ajv (or, for prototype-member names, against -`@amritk/runtime-validators`, since Ajv has those bugs itself). diff --git a/.changeset/generators-external-schema-registry.md b/.changeset/generators-external-schema-registry.md deleted file mode 100644 index 9c6dc9b2..00000000 --- a/.changeset/generators-external-schema-registry.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -'@amritk/generate-validators': minor -'@amritk/generate-parsers': minor -'@amritk/helpers': minor -'@amritk/runtime-validators': patch ---- - -Take documents you already loaded, so a `$ref` to another document generates - -**On the official JSON Schema Test Suite: `generate-validators` 1238 → 1268 / -1281 (99.0%), `generate-parsers` 1222 → 1237 / 1281 (96.6%).** - -Both generators gain a `schemas` option: documents you have already loaded, keyed -by the absolute URI a `$ref` names them by. It is the build-time counterpart of -`@amritk/runtime-validators`' `ValidateOptions.schemas`, and it keeps the same -promise — nothing is fetched, you cannot pass a URL, only a document. What changes -is that "we do no I/O" no longer also means "we cannot be told". - -A cross-document `$ref` was the single largest gap in both packages, and it is -gone. `refRemote.json` passes in full; so do the `dynamicRef.json` groups that -reach `tree.json` and `extendible-dynamic-ref.json`, and — with the dialect -metaschema registered — `defs.json` and `ref.json`'s "remote ref, containing refs -itself". - -Each registered document becomes a resource of the document being generated: its -`$id`, its `$anchor`s and `$dynamicAnchor`s and its own embedded resources all -resolve, a `$ref` from one registered document into another resolves, and every -definition reached gets a file, a type and a validator/parser by the ordinary -rules. A document with no `$id` resolves its relative `$ref`s against the URI it -was registered under; one whose `$id` disagrees answers to both. Registering more -than the schema uses costs nothing — only the documents actually reached are -emitted — and a `$ref` to a URI nobody registered still stops the build with a -message naming the ref. - -The mechanism is one pass, not a second addressing mode. `@amritk/helpers` gains -`graftExternalSchemas`, which embeds the registered documents into the root before -the `$id` pass, and `pruneExternalSchemas`, which drops the unreferenced ones once -the refs are pointers and reachability is finally knowable. Everything downstream — -the ref-graph walk, the naming, the emitted import graph — keeps working on a -single document and needed no change. `walkRefGraph` carries the option and -memoizes per `(schema, schemas)` by identity. - -**Fixed: a root schema with a union `type` dropped every sibling constraint.** -`{ type: ['object', 'boolean'], properties: {…}, required: [...] }` emitted the -type check and nothing else, so it accepted any object at all. The multi-type root -branch now emits the shared constraint checks the single-type and combinator -branches already did; they carry their own runtime-type guards, so a member of the -union a constraint does not apply to is still untouched. This is the shape the -2020-12 metaschema's own root is written in, which is how it went unnoticed — the -generated dialect validator accepted `{ type: 1 }` as a valid schema. - -`@amritk/runtime-validators` is unchanged in behaviour; its conformance figures are -restated against the corpus that is actually vendored (1281 cases, not 1299 — the -README's count never matched, and upstream's `content.json` is not among the -vendored files). The suite's `remotes/` loader moves to the shared fixtures -bookkeeping so all four conformance suites use one walk. diff --git a/.changeset/generators-multiple-of-and-unicode-patterns.md b/.changeset/generators-multiple-of-and-unicode-patterns.md deleted file mode 100644 index 7b6879fd..00000000 --- a/.changeset/generators-multiple-of-and-unicode-patterns.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -'@amritk/generate-validators': patch -'@amritk/generate-parsers': patch -'@amritk/helpers': patch ---- - -Emit the interpreter's own `multipleOf` check, and compile `pattern` in Unicode -mode — closing the last two places where generated code and -`@amritk/runtime-validators` could disagree about a document. - -`@amritk/helpers/multiple-of-check` claimed to mirror the interpreter and had -drifted from it. The interpreter splits on the divisor (an exact `%` when it is -an integer, a quotient within `2·ε·|q|` when it is not); the emitter still -divided in every case and allowed `1e-8·|q|` — roughly 10⁷× the actual -representation error. Generated validators and parsers therefore **accepted -values the interpreter rejects**: `1000000.005` against `multipleOf: 0.01` (a -half-cent past a whole dollar amount) passed, and so did any value whose quotient -overflows to `Infinity`, because the old fail expression asked `NaN > tolerance` -and got `false`. The emitter now produces the interpreter's two branches -verbatim, so both verdicts flip to invalid and the two implementations agree -again. `0.3` still satisfies `multipleOf: 0.1`, which is what the tolerance is -for. - -A `pattern` now compiles with the `u` flag wherever the pattern admits one, the -same try-`u`-then-fall-back decision the interpreter makes at runtime, taken once -at generation time by the new `regexLiteral` / `regexFlagsFor` in -`@amritk/helpers/escape-regex-pattern`. Without the flag a Unicode property -escape is inert — `\p{Letter}` was read as a literal `p{Letter}` — and `^.$` -rejected a single astral character. Every emit site now goes through -`regexLiteral` rather than interpolating an escaped body into its own `/…/`, so -the flag decision is made in one place instead of at a dozen call sites. - -Measured against the official JSON Schema Test Suite, this closes three cases in -each generator: `@amritk/generate-validators` moves to **1271 / 1281 (99.2%)** -and strict `@amritk/generate-parsers` to **1240 / 1281 (96.8%)**. diff --git a/.changeset/generators-nan-bounds-parity.md b/.changeset/generators-nan-bounds-parity.md deleted file mode 100644 index c4c6b7da..00000000 --- a/.changeset/generators-nan-bounds-parity.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -'@amritk/generate-validators': patch -'@amritk/generate-parsers': patch -'@amritk/api': patch ---- - -Reject `NaN` against a numeric bound, matching `@amritk/runtime-validators`. - -Bounds were emitted as their direct failure condition (`x < minimum`) rather than -the negated pass condition (`!(x >= minimum)`). The two agree on every ordinary -value and are opposite for `NaN`, which compares `false` against every operator: -the direct form read that as "not out of bounds" and let a `NaN` through -`minimum` / `maximum` / `exclusiveMinimum` / `exclusiveMaximum`, where the -interpreter and Ajv both reject it. Generated validators, strict generated -parsers, and the compiled API engine's inlined guards all now write the negated -form — so a `NaN` fails a bounded number everywhere in the toolchain. A bare -`{ type: 'number' }` with no constraint still accepts it, as Ajv does; only a -bound or `multipleOf` rejects it. - -Two internal inconsistencies close with it: `@amritk/generate-parsers` emitted the -un-negated `x >= min` in its inline matchers and the direct `x < min` in its -strict assertions, so the same schema could answer differently depending on which -path ran, and `@amritk/api`'s compiled engine disagreed with its own runtime -engine for a value the two are documented to be observationally identical on. - -`interpreter-parity.test.ts` now covers the numeric keywords — bounds, the -draft-04 boolean `exclusive*` form, and `multipleOf` across integer, fractional, -and quotient-overflowing divisors — over a value set built to separate the two -spellings (`NaN`, `±Infinity`, `1e308`, `1000000.005`). Nothing pinned these -before, which is how the drift got in. diff --git a/.changeset/helpers-id-base-uri-scoping.md b/.changeset/helpers-id-base-uri-scoping.md deleted file mode 100644 index eff493a8..00000000 --- a/.changeset/helpers-id-base-uri-scoping.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -'@amritk/generate-parsers': minor -'@amritk/helpers': minor ---- - -Resolve `$ref` against `$id` as a base URI in the ref graph - -A `$ref` written against an enclosing `$id` — a relative URI (`list`, -`folderInteger.json`), an absolute one, or a URN — used to either stop generation -or, worse, find *a* definition and generate against it, so the emitted parser -enforced a schema its author did not write. On the official JSON Schema Test -Suite, strict-parser generation goes from 1180/1299 to **1222 / 1299 (94.1%)**; -all 29 of the resolve-to-the-wrong-definition cases became right rather than -refused. - -Three new pieces in `@amritk/helpers`, deliberately free of any parser or -validator concepts: - -- **`build-resource-registry`** — one walk producing the document's embedded - resources, anchors and dynamic anchors, each `$id` composed against the base of - its parent. Keyed by JSON Pointer, because that is the currency the rest of the - package already deals in — a registry hit turns straight into a `$ref` string, a - filename, or a type name. Returns `null` for a document with no `$id`, which is - the fast-path switch, and is memoized per document. -- **`resolve-scoped-ref`** — one call covering relative, absolute, absolute-path, - URN, pointer-into-resource and anchor-in-resource forms, plus the plain - `#/$defs/x` that under an enclosing `$id` means *that resource's* `$defs`. -- **`normalize-ref-scopes`** — rewrites every `$ref`/`$dynamicRef` to a - document-root pointer. This is the leverage: everything downstream already - resolves refs by string against the root, so one normalization makes ref - resolution, type naming, the import graph and the strict matcher correct at once, - none of them needing base-URI awareness of their own. - -It is wired into `walkRefGraph`, so `@amritk/generate-validators` and -`@amritk/generate-examples` inherit it. - -`assertIdScopes` keeps its name and signature but changes meaning: it no longer -refuses any document with nested `$id` scoping, only the residue base-URI -resolution cannot place — a fragment ref inside an embedded resource that declares -its own targets and names none of them. That preserves the property worth having: -never silently pick the outer definition. - -`@amritk/generate-parsers` additionally follows the spec on `contains` next to -`unevaluatedItems`: only the items `contains` matched are evaluated, not the whole -array. That is what `@amritk/runtime-validators` does, so the two stop disagreeing -about the same schema. Ajv marks the whole array, so the single fuzz fragment -pairing those keywords leaves the Ajv-oracle corpus (with the reason recorded next -to it) and unit tests plus the conformance suite cover it instead; every other -`contains` and `unevaluated*` fragment keeps fuzzing. diff --git a/.changeset/helpers-ref-naming-and-resolution.md b/.changeset/helpers-ref-naming-and-resolution.md deleted file mode 100644 index 6c8c7a29..00000000 --- a/.changeset/helpers-ref-naming-and-resolution.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -'@amritk/helpers': minor -'@amritk/generate-parsers': minor -'@amritk/generate-validators': minor -'@amritk/generate-examples': minor ---- - -Fix `$ref`-graph naming and reference resolution, and stop degrading silently. - -Generation now fails loudly instead of writing output that cannot work: - -- Two definitions that reduce to one filename (`Pet`/`pet`) or one type name - (`foo-bar`/`foo.bar`/`fooBar` all become `FooBar`) are an error. The filename - case used to drop one definition and give every reference to it the other - one's shape; the type-name case emitted both files and left the importer with - two `import { FooBar }` lines that do not parse. -- An unresolvable `$ref` is an error. It used to warn while the generators still - emitted the type name and the parser/validator call for a file that was never - written. -- A `$dynamicRef` with no `$dynamicAnchor` to bind to is an error. Leaving it in - place made the type generator name the type after the anchor, so the canonical - recursive-tree idiom (`$dynamicAnchor: "node"`) produced a reference to the - DOM's `Node` interface — a clean compile with the wrong type. -- A document that relies on `$id` base-URI scoping is rejected rather than - resolving its inner fragments against the document root and silently selecting - a different definition. -- Every recursive schema walker enforces a nesting cap and reports it by name - instead of dying with a bare stack-overflow. - -And several things that were broken now work: - -- Non-ASCII definition names (CJK, Cyrillic, accented) keep their characters - instead of collapsing onto the single type name `_`, and the generated - `index.ts` barrel re-exports them correctly. -- A root-level `$dynamicAnchor` is generated as the root's own file, so the - 2020-12 recursive-tree idiom produces a real self-referencing type. -- A plain `$anchor` ref (`$ref: "#named"`) resolves, instead of producing the - unloadable import specifier `'./#named.ts'`. -- Derived filenames are normalized: no more `.ts`, `...ts`, `http:--x.ts`, or - characters Windows and ESM specifiers reject. `$ref: "#/__proto__"` no longer - resolves to `Object.prototype`. -- Generated readers guard `Object.prototype` member names (`constructor`, - `toString`, `__proto__`, …) with `Object.hasOwn`, so a schema with a - `constructor` property no longer fails its own shape check for every valid - object, and the parser no longer fabricates a `__proto__` key. -- `x-mjst` `instanceOf` is allow-listed to the classes the generators support, - so an arbitrary identifier is warned about and ignored instead of being - emitted verbatim into the output. diff --git a/.changeset/json-schema-suite-conformance.md b/.changeset/json-schema-suite-conformance.md deleted file mode 100644 index 54a21e78..00000000 --- a/.changeset/json-schema-suite-conformance.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -'@amritk/runtime-validators': patch -'@amritk/generate-parsers': patch -'@amritk/generate-validators': patch -'@amritk/resolve-refs': patch ---- - -Measure every schema-consuming package against the official JSON Schema Test -Suite, the way `@amritk/yaml` is measured against the YAML test suite - -The required Draft 2020-12 tests (46 files, 383 groups, 1299 cases) are vendored -under `fixtures/json-schema-test-suite`, and four packages now run them on every -build. Each carries an expected-failure list naming every case it does not pass -and why, and each suite fails when a case moves in **either** direction — a -regression breaks the build, and so does a case that starts passing while its -entry stays behind. Nothing is published: the corpus and the harnesses live -outside every `files` list. - -| package | measured on | rate | -| --- | --- | --- | -| `@amritk/runtime-validators` | `validate` and `validateGuard` verdicts | 1250 / 1299 (96.2%) | -| `@amritk/generate-parsers` | strict parsers, generated → linked → executed | 1180 / 1299 (90.8%) | -| `@amritk/generate-validators` | generated predicate validators, likewise | 987 / 1299 (76.0%) | -| `@amritk/resolve-refs` | verdict preserved after inlining (`$ref` corpus) | 160 / 170 (94.1%) | - -The generators are measured through the code they emit, not the source text they -emit: each suite schema is generated whole, compiled, and linked in memory, so the -`$ref`'d sibling files and the embedded runtime helpers run too. `resolve-refs` -has no verdicts of its own, so it is held to semantic preservation — the resolved -document must accept exactly what the original did, judged by -`@amritk/runtime-validators` over the cases the interpreter already answers -correctly, which is the population where a resolution bug is visible and nothing -else is. - -Those rates are where the packages *end up*. The suites were written first and -found real defects — a validator that accepted everything for a schema without a -`type`, `required` satisfied by an inherited `toString`, refs that emitted -uncompilable output, `$ref`-shaped data inlined as a reference — each fixed in its -own commit alongside this one. What remains is documented case by case, and each -package's README carries a "Conformance, measured" section with its number and the -reasons behind it. diff --git a/.changeset/lint-filter-execution-and-redos.md b/.changeset/lint-filter-execution-and-redos.md deleted file mode 100644 index 5a18c74e..00000000 --- a/.changeset/lint-filter-execution-and-redos.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -'@amritk/lint': patch ---- - -Stop rulesets from executing code, and close two document-driven hangs - -**A `[?(...)]` filter in a `given` is no longer JavaScript.** Filter bodies were -handed straight to `new Function`, so a `given` string in a YAML or JSON ruleset -— data, to any caller's eye — ran arbitrary code in the linting process: -`$[?(globalThis.x = {home: process.env.HOME})]` leaked the environment, -`import('node:fs')` wrote files. Filters are now parsed into a small AST and -interpreted (`@`, `@.x`, `@['x']`, `@property`, `@parentProperty`, `@parent`, -`@path`, `@root`, `$`, the comparison and logical operators, `!`, numeric -negation, string/number/boolean/null/`undefined`/`void 0` literals, regex -literals, `.length`, and a fixed list of pure methods — `indexOf`, -`lastIndexOf`, `includes`, `startsWith`, `endsWith`, `match`, `test`, -`toLowerCase`, `toUpperCase`, `trim`). Member reads see own properties only, so -`@.constructor` and `@['__proto__']` are plain `undefined`. Verified -node-for-node identical to the old evaluator on the shipped `oas` filters across -every vendored real-world spec. An expression outside the grammar is now a -ruleset error naming the rule, instead of a filter that silently matches -nothing — which also fixes filters quietly disabling themselves wherever -`new Function` is unavailable (CSP, Workers). - -**The `casing` function no longer hangs on a long identifier.** `camel`/`pascal` -compiled to a pattern where digits could be consumed two ways, so a value from -the linted document could force exponential backtracking: a 46-character -`operationId` took over 100 seconds on Node (Bun's regex engine caps -backtracking, which hid it). The patterns are rewritten to be unambiguous, and -verified by brute force to accept exactly the same strings as before. Same for -the second overlap, a separator character the style already uses (`kebab` with -`separator: '-'`), which was exponential from ~40 characters. - -**A deeply nested document is a diagnostic, not a crash.** `'['.repeat(20000)` — -a 40 KB file — took the process down with `RangeError: Maximum call stack size -exceeded`, while every other malformed document came back as findings. JSON -parsing now enforces the same 1000-level nesting limit `@amritk/yaml` does and -reports it as a parser diagnostic, and the JSONPath descent walker is iterative. - -**Rulesets are built once, not per document.** `lintDocument` re-normalized the -ruleset and re-read every `extends` file on every call (~3.6 ms per document -with a 200-rule `extends` file; `fixDocument` paid it up to 11× per document). -The built `Ruleset` is memoized per `(definition object, basePath, restrictTo)`, -and `fixDocument` builds one for the whole loop — 200 lints of a small document -went from 750 ms to ~120 ms — so editing a ruleset file mid-run also stops -changing results half way through. Treat a definition you have passed in as -frozen; pass a fresh object to force a rebuild. - -**`@amritk/lint/rules/openapi` can be bundled.** The four OpenAPI meta-schemas -were loaded through `createRequire` with a computed specifier, invisible to -bundlers (esbuild produced a 524-byte module that threw `Cannot find module -'./oas31.json'`) and unavailable on Workers and Deno. They are now generated -`.ts` modules imported statically, each holding its schema as JSON text that is -still parsed lazily on first use. - -**`fixDocument` reports whether it converged.** The result gains `converged` and -`passes`, and `applied` is de-duplicated by rule code and path: two fixers that -undo each other used to report 11 applied fixes for one problem with no way to -tell a fixpoint from giving up at the pass cap. - -**Also:** `alphabetical` no longer treats `'0x10'`, `'1e2'`, or `' 5'` as -numbers (they were flagged out of order though lexicographically sorted); the -module-level JSONPath, filter, and pattern caches are bounded; `extends` and -custom-function resolution accept an optional `restrictTo` root; and the ruleset -trust boundary is documented in the README and AI.md. diff --git a/.changeset/packaging-manifest-fixes.md b/.changeset/packaging-manifest-fixes.md deleted file mode 100644 index ff6ff26d..00000000 --- a/.changeset/packaging-manifest-fixes.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -'@amritk/mjst': minor -'@amritk/helpers': minor -'@amritk/adapters': minor -'@amritk/generate-parsers': minor -'@amritk/api': patch -'@amritk/generate-examples': patch -'@amritk/generate-markdown': patch -'@amritk/generate-validators': patch -'@amritk/lint': patch -'@amritk/resolve-refs': patch -'@amritk/runtime-validators': patch -'@amritk/yaml': patch ---- - -Fix the published manifests so the packages install, resolve, and dedupe correctly - -**Types resolve on TypeScript's default config.** Every package was -exports-only: nine declared `"module": "./dist/index.js"` (a field neither Node -nor TypeScript reads) and nothing declared `types`. A consumer on -`moduleResolution: "node10"` — still the default when `module` is `commonjs` — -cannot see `exports` at all, so `import { lintDocument } from '@amritk/lint'` -failed with `TS2307: Cannot find module '@amritk/lint' or its corresponding type -declarations`. Each package with a `.` export now also declares `main` and -`types`; `@amritk/helpers` and `@amritk/adapters` have no `.` export (they are -subpath-only), so they declare a `typesVersions` wildcard mapping instead, which -gives their subpaths the same node10 fallback. All of it is ignored under -`node16`/`nodenext`/`bundler`, where `exports` still wins. - -**`workspace:*` resolves to a caret, not an exact pin.** All fourteen -inter-package edges shipped as exact versions, so installing two `@amritk/*` -packages published at different times pulled in two copies of their shared -dependency. That is not merely wasteful: the module-level caches those packages -rely on are per-copy, so the `WeakMap` validator cache in -`@amritk/runtime-validators` silently stopped hitting. Pre-1.0 a caret stays -narrow (`^0.9.1` is `>=0.9.1 <0.10.0`) and breaking changes here already ride a -minor bump. - -**`@amritk/helpers` stops shipping 21 source files it does not need.** Embedded -mode reads four helper sources (`is-object`, `validate-array`, -`validate-record`, `has-ref`) out of the installed package at generation time, -so `src` has to ship — but only those four. `files` now lists them explicitly -instead of globbing all of `src`, cutting the tarball from 78 files / 206 kB to -63 / 112 kB. - -**Two packages no longer declare a dependency they never import.** -`@amritk/mjst` and `@amritk/generate-parsers` both listed -`@amritk/generate-markdown` under `dependencies`, but the only importer is each -package's `scripts/generate-readme.ts`, which is not published. Both moved to -`devDependencies`. `@amritk/adapters` likewise dropped its -`@sinclair/typebox` peer dependency: the TypeBox adapter is purely structural -(it strips symbol keys) and imports nothing. `valibot` stays — it is a genuine -transitive peer of `@valibot/to-json-schema`. - -**`@amritk/mjst` fixes.** `json-schema-typed` moved to `dependencies`, because -the shipped `dist/emit-examples.d.ts` imports types from it. The package gained -an `exports` map, so it is no longer deep-importable in its entirety. And the -build now marks `dist/cli.js` executable: `npm pack` records on-disk modes, and -package managers only `chmod` bin targets when they link them, so flows that -consume the tarball directly (vendoring, Docker `npm pack` + `tar -x`) hit -`EACCES`. diff --git a/.changeset/parsers-ajv-parity.md b/.changeset/parsers-ajv-parity.md deleted file mode 100644 index 09321552..00000000 --- a/.changeset/parsers-ajv-parity.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -'@amritk/generate-parsers': patch -'@amritk/helpers': patch ---- - -Bring the strict parser up to Ajv's assertion vocabulary, and stop refusing the -keywords it can now prove. - -The exact subschema matcher — the thing strict mode enforces `contains`, -`propertyNames`, `not` and `dependentSchemas` through — only understood a -fraction of Draft 2020-12, and every gap in it became either a generation-time -refusal or a keyword nothing checked. It now covers `$ref` (JSON Pointer, -`$anchor`, and the 2020-12 rule that a ref's *siblings* still apply), -`prefixItems` with its `items` tail, `items: false`, `contains` with -`minContains` / `maxContains`, `patternProperties`, a schema-valued -`additionalProperties`, `propertyNames`, `dependentRequired`, -`dependentSchemas`, array-form `type`, structural `const`, and an empty `enum`. - -Built on that: - -- **`unevaluatedProperties` / `unevaluatedItems` are implemented** rather than - rejected at generation time. The emitted check computes the same annotation - coverage the runtime interpreter collects — keys and indices evaluated by - `properties`, `patternProperties`, `additionalProperties`, `prefixItems`, - `items`, a satisfied `contains`, `allOf` members, a `$ref` target, a *matching* - `anyOf` / `oneOf` branch, an `if` / `then` / `else` arm, and a triggered - `dependentSchemas` entry — and applies the unevaluated schema to what is left. -- **A backstop check** now stands behind the per-property assertions, so the - keywords no flat check can express are enforced instead of dropped: a - `$ref` that no imported parser validates (single-file builds, `allOf` members - of a property-less object, array `items`, tuple positions), a `$ref` with - constraining siblings, `items: false`, and constraint keywords with no `type` - to hang them on (`{ minimum: 5 }`, `{ required: ['a'] }`). The fast path and - the shape validator decline for those same shapes, so nothing can skip past - the check. It proves only the keywords that need it — a bare `required` stays - enforceable even when a sibling `allOf` member is too deep to inline — so the - whole 982-schema OpenAPI corpus still generates under `strict`, now covered by - its own pass in the fixture suite. -- **A `type` with more than one non-null member** keeps each family's - constraints: `{ type: ['string','array'], minLength: 3, minItems: 2 }` bounds - the string by length and the array by count. -- **`minLength` / `maxLength` count Unicode code points**, as JSON Schema - specifies — `"💩"` no longer satisfies `minLength: 2`, and `"💩💩"` no longer - violates `maxLength: 2`. The exact count is only scanned inside the narrow band - where the cheap UTF-16 unit count cannot decide, so ASCII input allocates - nothing and `minLength: 1` compiles to a plain length test. -- **A `false` schema rejects every value** instead of casting it through, and a - strict `if` / `then` / `else` root asserts the conditional instead of building - a result from the branch fragments (which invented properties the input never - had). -- **A nullable object root** (`type: ["object","null"]` with `properties`) accepts - `null`, which the object parser's `isObject` guard used to reject. -- **A recursive root `$ref: "#"`** is generated as the root's own type, the way a - root `$dynamicAnchor` already was. It previously emitted an import of a - `ref-.ts` that was never generated — output that did not compile at all. -- The generation-time guard walks *schema* positions only. It used to inspect - every object in the document, so a schema declaring a property named `items`, - `not` or `contains` was checked as though the property name were the keyword. - -Parity is held by a new differential fuzz suite (`parser-vocabulary-conformance`) -over that vocabulary, plus the existing shape, composition and strict fuzzers, -with Ajv 2020 as the oracle. Three departures from Ajv are deliberate and -documented in the README: `format` stays an annotation (Ajv's own default), -`multipleOf` keeps the magnitude-scaled tolerance the whole toolchain shares, and -a type-less schema with `properties` still requires an object because the parser -must return the type it declares. diff --git a/.changeset/parsers-required-unions-pointers.md b/.changeset/parsers-required-unions-pointers.md deleted file mode 100644 index 179e6e4a..00000000 --- a/.changeset/parsers-required-unions-pointers.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -'@amritk/generate-parsers': minor -'@amritk/helpers': minor ---- - -Stop strict parsers accepting what the schema forbids: prototype-inherited -`required`, undiscriminated unions, and `$ref` fragments that were never decoded - -Measured against the official JSON Schema Test Suite, strict-mode generation goes -from 1141/1299 to **1180/1299 (90.8%)**. Four defects, all of them cases where the -generated parser said yes to a document the schema says no to — or refused one it -should have taken. - -- **`required` compiled to `in`, which walks the prototype chain.** `"toString" in {}` - is `true`, so `{ "required": ["__proto__", "toString", "constructor"] }` was - satisfied by an object carrying none of them. `Object.hasOwn` now covers the - names an object can actually inherit, and plain `in` stays everywhere else so - ordinary keys keep the form the engine can fold — the same split - `@amritk/generate-validators` already made. Generated output for ordinary keys is - byte-identical. Applies to `required`, `dependentRequired`, `dependentSchemas`, - and `false`-property absence. -- **A `oneOf`/`anyOf`/`allOf` whose branches carry no `type` compiled to a - pass-through.** `{ "oneOf": [{ "type": "integer" }, { "minimum": 2 }] }` emitted - `parseRoot = (input) => input`: nothing to discriminate on meant nothing was - checked, so a value matching *no* branch — or, for `oneOf`, more than one — was - accepted. Those compositions are now enforced through the existing subschema - matcher, and only where the flat union check declines, so nothing is checked - twice and the common discriminated-union path is unchanged. -- **`$ref` fragments were matched literally, never decoded.** `#/$defs/percent%25field`, - `#/$defs/foo%22bar` and `#/$defs//$defs/` (an empty pointer token) resolved to - nothing, and generation stopped. Tokens are now percent-decoded before `~1`/`~0` - unescaping, per token, and empty tokens are significant. Two consequences worth - knowing: a definition whose name literally contains `%25` must now be written - `%2525`, and `#/$defs//x` now means the `""` member rather than silently meaning - `#/$defs/x`. `#/` still means the document root. -- **A boolean `$defs` entry was not a ref target.** `$defs: { bool: true }` is a - legal definition; the ref graph only named object subschemas, so a `$ref` at it - resolved to nothing. Boolean entries in a definition map now expand to their - object equivalents (`true` → `{}`, `false` → `{ not: {} }`) — confined to - definition maps, because elsewhere `additionalProperties: true` and `{}` generate - different *types*. - -`@amritk/helpers` carries the last two (`resolve-ref`, `walk-ref-graph`) plus the -new `hasOwnCheck`/`missingCheck` emitters in `safe-accessor`. diff --git a/.changeset/parsers-strict-keyword-enforcement.md b/.changeset/parsers-strict-keyword-enforcement.md deleted file mode 100644 index 2a5e904a..00000000 --- a/.changeset/parsers-strict-keyword-enforcement.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -'@amritk/generate-parsers': patch ---- - -Close the strict parser's silent-acceptance gaps across the composition and -constraint keywords. A strict parser promises to throw on anything the schema -rejects; each of these was accepted instead, because the keyword appeared in -neither the fast-path guard nor the slow-path assertions: - -- `const` on a property — scalar, structural (compared by deep equality, key - order included) and `const: null`. -- `minProperties` / `maxProperties` — at the root, on a property, and on the - property-less object and record parsers. -- `required` on a schema with no `properties`. -- `not`, at the root and on a property. Enforced through the exact subschema - matcher; a subschema the matcher cannot prove is now a generation-time error, - joining `contains` / `propertyNames` / `dependentSchemas`. -- `allOf` members that carry constraints rather than an object shape, and object - members of a type-less `allOf` root. -- `if` / `then` / `else`. -- `patternProperties` value schemas, and the value constraints of a - schema-valued `additionalProperties` beyond its bare `type`. -- Array `items` richer than a scalar or enum — a nested array, a union, a - bounded string — which previously contributed no element check at all. -- `minItems` / `maxItems` on a root array whose items are objects or `$ref`s. -- The object shape (`properties`, `required`, …) of a nullable - (`["object","null"]`) property. -- Boolean property schemas: `false` now rejects the key, and `true` no longer - blanks the value to `undefined` — which made a strict parser mutate a value it - had just accepted. - -The fast-path guard and the exported shape validator decline any schema carrying -a keyword they cannot mirror, so a value can no longer be waved through before -the assertions run. `subschemaMatchExpr` gained `allOf` / `anyOf` / `oneOf` / -`not` / `if`-`then`-`else` support (so `contains`, `propertyNames`, -`dependentSchemas` and array items handle combinators too), switched -`uniqueItems` to the structural comparison the rest of the package uses, and -casts its object accessors so the generated code type-checks. - -A new differential fuzz suite holds this surface against Ajv. diff --git a/.changeset/readme-benchmarks-remeasured.md b/.changeset/readme-benchmarks-remeasured.md deleted file mode 100644 index a89e88a9..00000000 --- a/.changeset/readme-benchmarks-remeasured.md +++ /dev/null @@ -1,34 +0,0 @@ ---- ---- - -Re-measure every benchmark quoted in a README, and make the `@amritk/api` cross-framework table reproducible - -All seven benchmark suites were re-run on one machine (Bun 1.3.11 / Node 22, Linux -x64) and the numbers in the root README, `generate-validators`, `generate-parsers`, -`runtime-validators`, `resolve-refs`, `yaml`, `lint`, and `api` READMEs now report -what those suites actually print. Most moved a little; a few moved enough to change -what the surrounding prose could honestly claim, and that prose moved with them — -generated validators are now ~10% ahead of typia on `assert-loose` rather than tied, -the `order` validator gained ~40%, and the runtime-validators cold-start advantage -over Ajv reads ~97–720× rather than ~110–1100×. - -The `api` table was the one that could not be reproduced at all: it compared against -Hono, and no such benchmark existed in the repo — so its numbers could neither be -checked nor refreshed. `packages/api/bench/` now runs that comparison (bare Hono, -Hono + `@hono/zod-validator`, the runtime engine, and the compiled engine, all -`Request` → `Response` on the same three routes, with a status parity check before -timing) on all three runtimes this package targets: **workerd** via Miniflare -(`bun run bench:workerd`), Node (`bun run bench:vs`), and Bun -(`bun run bench:vs:bun`). The stacks live in one shared module so a column cannot -drift between runtimes. - -The workerd run measures inside a real isolate — the loop runs in the Worker itself, -one fresh isolate per cell, because timing from outside would measure Miniflare's -loopback hop and a reused isolate moves the numbers 30–40%. It also changes what the -README can claim: under workerd the compiled engine's *peak* trials match bare Hono -(~153k vs ~154k ops/s on the static GET), but its median is 15–35% lower because -workerd pauses the `@amritk/api` columns far more often than it pauses Hono. That -pattern is consistent with allocating more per request than Hono does, is not yet -diagnosed, and is now stated in the README as open work rather than smoothed over. - -No published code changed — READMEs, benchmarks, and `devDependencies` only. diff --git a/.changeset/resolve-refs-id-scoped-pointers.md b/.changeset/resolve-refs-id-scoped-pointers.md deleted file mode 100644 index 67d5343d..00000000 --- a/.changeset/resolve-refs-id-scoped-pointers.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -'@amritk/resolve-refs': patch ---- - -Resolve a `#/pointer` inside an `$id` scope against the resource that declares it - -A fragment-only ref was hard-coded to resolve against the document root, so -`{ "$id": "…/base.json", "$defs": { "inner": … }, "properties": { "x": { "$ref": "#/$defs/inner" } } }` -nested inside a larger document reported "Cannot resolve internal `$ref`" — the -pointer names a definition of the *embedded resource*, not of the root. It now -looks in the resource named by the base URI in scope first, and falls back to the -document root only when the pointer matches nothing there. The fallback is what -keeps bundled documents working: a bundled OpenAPI file points at -`#/components/schemas/…` from inside an `$id` scope, and when both could match the -resource wins, which is the order the spec asks for. - -That also settles pointer-form `$dynamicRef`s (`#/$defs/items`) inside an `$id` -scope: the spec says a `$dynamicRef` whose fragment is a pointer behaves exactly -like `$ref`, so resolving it against its enclosing resource is right by -construction. - -On the `$ref` corpus of the official JSON Schema Test Suite the package is at -**160 / 170**. The corpus grew from 107 with `@amritk/runtime-validators`' `$id` -work — it is the reference-carrying cases the interpreter answers correctly, which -is the population where a resolution bug is visible at all. What is left is one -documented limit: a `$dynamicRef` binds at evaluation time to the outermost -`$dynamicAnchor` along the *dynamic* scope, so inlining it statically collapses it -to a single target and cannot be right in general. diff --git a/.changeset/resolve-refs-keep-dynamic-refs.md b/.changeset/resolve-refs-keep-dynamic-refs.md deleted file mode 100644 index 9f5d9658..00000000 --- a/.changeset/resolve-refs-keep-dynamic-refs.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -'@amritk/resolve-refs': minor ---- - -Keep a `$dynamicRef` the dynamic scope has to answer, instead of inlining one -wrong target - -A `$dynamicRef` binds at *evaluation* time to the outermost `$dynamicAnchor` of -its name along the chain of resources actually being applied, so the same keyword -can resolve to different schemas depending on where evaluation entered from. -Inlining happens once, which means a resolver that inlines every `$dynamicRef` is -guessing — and a wrong guess changes what the document accepts, in both -directions. - -So it no longer guesses. Where the binding is decidable it inlines as before: - -- a **pointer fragment** (`#`, `#/$defs/items`) is a plain `$ref` per the spec — - there is no anchor to late-bind to; -- an **anchor name declared at most once** in the document has only one schema the - dynamic lookup could ever reach. - -Where it is not decidable — an anchor name declared twice or more — the -`$dynamicRef` stays in the output, along with the scaffolding it needs to resolve -at validation time: the `$dynamicAnchor`s it may bind to and the `$id`s that -delimit the resources those anchors live in. Inlining anything whose copy would -drop a resource out of that chain is held back for the same reason. This is the -move the resolver already makes for a reference cycle — keep the reference rather -than collapse it to one wrong answer — now covering both cases under one rule. - -Consumers see no new API and no new errors: a kept reference is not a failure, it -is a reference the resolver could not answer without changing the document's -meaning, resolvable against the output exactly as it was against the input. -`trackOrigins` records nothing for one, because nothing was copied in its place. - -On the `$ref` corpus of the official JSON Schema Test Suite the package is now at -**170 / 170**. - -Two limits stay, documented in the code: a multi-document `resolveRefsFromFile` -still inlines (preservation only helps when the scaffolding survives into the -output, which a single-document resolve guarantees and a flattened multi-document -one does not), and 2019-09's `$recursiveRef` has the same defect but no corpus to -move it against. diff --git a/.changeset/resolve-refs-local-confinement-and-cache-scoping.md b/.changeset/resolve-refs-local-confinement-and-cache-scoping.md deleted file mode 100644 index faeb6324..00000000 --- a/.changeset/resolve-refs-local-confinement-and-cache-scoping.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -'@amritk/resolve-refs': minor ---- - -Close the local-file, cache-scoping, and fan-out gaps in the resolver's guards - -This package's selling point is a default-deny SSRF guard. These are the places -the guard did not reach. - -**Local `$ref`s are now confined to the root document's directory.** `$ref` -resolution against the filesystem had no containment check and no way to turn it -off: `{"$ref": "../../../etc/passwd"}` (or an absolute path) read whatever the -process could read, and any caller supplying the YAML `parse` callback the docs -recommend got arbitrary text, not just JSON. A local ref must now resolve under -`dirname(rootLocation)`; both the lexical and the symlink-resolved path have to -land inside it, so a symlink planted in the tree cannot be used to escape. - -This is a **behavior change**: a legitimate cross-directory ref -(`../common/schemas.json` — a very normal split-spec layout) now fails until you -widen it with the new `allowedRoots`, which the refusal message names. The -default was chosen to match the stance the remote path already took — deny, then -opt in — since the escaping ref and the traversal attack are the same shape and -only the caller can tell them apart. The root document you name is exempt; it is -what you asked for. `localRefs: false` refuses cross-file reads entirely. - -**The session cache no longer leaks documents across credentials.** It was keyed -by URL alone, so a call carrying one tenant's `Authorization` header handed that -tenant's private document straight to a later call carrying no credentials at -all — and in-flight coalescing additionally made the second caller inherit the -first one's `fetch`, `timeoutMs`, and `maxBytes`. Both keys now include a digest -of the effective headers plus the `fetch`/`parse` identities and the transfer -limits. The cache is also bounded now (10-minute TTL, 256 entries, LRU -eviction) instead of growing for the life of the process, and -`clearRemoteCache(url)` can drop a single document. - -**A resolve is bounded as a whole.** A root document with 500 `$ref`s to distinct -URLs drove 501 fetches with nothing but the per-hop timeout bounding it — an -egress amplifier and a host scanner, from the resolver's network position. New -`maxDocuments` (500) and `totalTimeoutMs` (60s, applied across the whole resolve -and not just per hop) cap it. - -**Deeply nested documents no longer throw.** `'{"a":'.repeat(20000)` raised -`RangeError: Maximum call stack size exceeded` out of the walkers, breaking the -package's stated contract that errors are collected and never thrown. Every -recursive walk is depth-capped (`maxDepth`, default 512); past the limit the -subtree is left unresolved and one `ResolveError` is recorded. - -**The SSRF guard is no longer name-blind.** `metadata.google.internal`, -`metadata.goog`, `metadata`, `instance-data`, and anything under the reserved -`.internal` TLD are refused by name — the IP check missed all of them, because -callers reach the metadata service by name. The new `assertPublicHost` also -resolves each remote hostname and refuses it when *any* address it points at is -non-public, which closes the `127.0.0.1.nip.io` class of bypass; it fails closed, -and `verifyDns: false` (or an `allowedHosts` entry) opts out where names resolve -at an egress proxy. DNS rebinding is narrowed, not closed: pinning the connection -to the verified address is not something Node's `fetch` exposes, and the README -says so rather than overclaiming. - -**Missing IP ranges added:** `fec0::/10` (deprecated site-local), -`198.18.0.0/15` (benchmarking), and `192.0.0.0/24` (IETF protocol assignments). - -**`allowedHosts` is no longer a footgun.** Matching was case-sensitive and -port-exact, so `['example.com']` refused `https://example.com:8443/a.json` and -`['EXAMPLE.com']` refused everything — failing closed, but pushing users toward -`allowPrivateHosts`, which is a real hole. Entries now match case-insensitively; -an entry without a port matches any port, and one with a port must match it -(a URL that omits the port counts as its protocol default). diff --git a/.changeset/resolve-refs-value-position-refs.md b/.changeset/resolve-refs-value-position-refs.md deleted file mode 100644 index c23f9861..00000000 --- a/.changeset/resolve-refs-value-position-refs.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -'@amritk/resolve-refs': patch ---- - -Stop inlining a `$ref`-shaped object that is data, not a reference - -`{ "$defs": { "a_string": { "type": "string" } }, "enum": [ { "$ref": "#/$defs/a_string" } ] }` -references nothing. `enum` holds *instances*, and one of them happens to be an -object with a `$ref` key — but the walk was purely structural, so it inlined that -object and turned "the enum containing `{"$ref": …}`" into "the enum containing -`{"type": "string"}`", changing what the document matches in both directions. The -official suite carries the case under exactly that name: *"naive replacement of -`$ref` with its destination is not correct"*. - -Every structural walk in the package now carries the **role** of the node it is -at — a schema, a map of author-chosen names to schemas, instance data, or -something outside the vocabulary. `enum` / `const` / `default` / `examples` hand -their subtree back untouched; `properties` / `patternProperties` / `$defs` / -`definitions` / `dependentSchemas` / `dependencies` suppress keyword reading one -level down, so a definition legitimately *named* `enum` is still a definition and -a property named `$ref` is still a property — the trap in the naive version of -this fix, which the resource registry had; and an unrecognized keyword yields -`unknown`, which is absorbing, so OpenAPI's `components`/`paths` and `x-` vendor -blocks are walked exactly as before. - -Two consequences beyond the inlining itself: the resource registry no longer -registers an `$id`/`$anchor` that is part of a value or a property name, and -`resolveRefsFromFile` no longer reads a file or opens a network connection for a -`$ref` string sitting inside an `enum`. - -This takes the package to **107 / 107** on the `$ref` corpus of the official JSON -Schema Test Suite, with an empty expected-failure list. diff --git a/.changeset/runtime-validators-honest-guard-type.md b/.changeset/runtime-validators-honest-guard-type.md deleted file mode 100644 index efe9937e..00000000 --- a/.changeset/runtime-validators-honest-guard-type.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -'@amritk/runtime-validators': minor ---- - -`validateGuard` stops narrowing where the inferred type cannot describe every -accepted value - -`FromSchema` infers an object shape from applicator keywords alone, so -`{ properties: { a: { type: 'string' } } }` infers `{ a?: string }`. The -interpreter — correctly — accepts a non-object against that schema, because JSON -Schema's object keywords ignore values that are not objects. The guard was -therefore handing back `input is { a?: string }` for a `42` it had just approved. - -For exactly those schemas — no `type`, `enum`, `const` or `$ref`, but -`properties`, `required`, `additionalProperties`, `patternProperties`, -`prefixItems` or `items` present, recursing through `allOf`/`anyOf`/`oneOf` -branches — `validateGuard` now returns a `Check` instead of a `Guard`: the -same runtime function, no type predicate. Every schema that declares a `type` (or -`enum`/`const`/`$ref`) keeps its predicate, and so does a schema whose type is not -a literal — narrowing is surrendered only when the inference is *demonstrably* -partial, never because the checker could not decide. - -`Check` keeps the erased phantom carrier `Validator` already uses, so -`Infer` still recovers the schema's type rather than collapsing to -`never`, and it is assignable anywhere `(input: unknown) => boolean` is. It reads -as "checks for this, does not claim it". - -The runtime is untouched. This mirrors the same fix in -`@amritk/generate-validators`, whose generated `isX` had the identical hole — the -two now tell one story about the same schemas, and the type-level predicate sits -next to `ImplicitShape` so the keyword lists cannot drift apart. diff --git a/.changeset/runtime-validators-id-base-uri.md b/.changeset/runtime-validators-id-base-uri.md deleted file mode 100644 index f58b656f..00000000 --- a/.changeset/runtime-validators-id-base-uri.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -'@amritk/runtime-validators': minor ---- - -Resolve `$ref` against `$id` as a base URI, and give `$dynamicRef` a real dynamic -scope - -Measured against the official JSON Schema Test Suite, the interpreter goes from -1183/1299 to **1250/1299 (96.2%)**. Sixty-seven cases, one cause: a `$ref` written -against an `$id` — relative (`"list"`), absolute -(`"http://example.com/b/d.json"`), or a URN — had nothing to resolve against and -threw, even when the resource it named sat *inside the same document*. - -The document is now walked once into a registry of its embedded resources: each -`$id` composed against the base of its parent, and each resource's `$anchor`s and -`$dynamicAnchor`s registered under it. A ref resolves against the base in scope at -the referring node — relative, absolute, URN, absolute-path, pointer-into-resource -and anchor-in-resource forms all work — and `$dynamicRef` implements bookending -properly: it goes dynamic only when static resolution already lands on a -`$dynamicAnchor` of that name, then takes the outermost resource in the dynamic -scope declaring it. - -Two behavior changes fall out of that, both spec-correct and both confined to -documents that declare an `$id`: - -- A `#/pointer` inside an `$id` scope resolves within that resource rather than at - the document root. A scoped ref that names nothing in its own resource still - falls back to the document-global lookup, so a bundled schema that worked before - works unchanged — the new path can only *add* an answer. -- `contains` publishes the indices it matched rather than sweeping the whole - array, so an adjacent `unevaluatedItems` sees the right set. This is where the - spec and Ajv disagree; the suite agrees with the spec, and so do we. The pair is - excluded from the Ajv differential corpus and covered by unit tests plus the - suite instead. - -Cost is kept off the common path: the registry is `null` for a document with no -`$id` at all, the `$id` scan is fused into the pattern-screening walk that already -happened, and resolutions are memoized per validator. Eval-free, synchronous, -zero-dependency and no-I/O all hold. - -What remains unimplemented is now one decision rather than two: this package does -no I/O, so a `$ref` naming *another document* (and `$vocabulary`, which means -fetching a metaschema) still throws. Bundle with `@amritk/resolve-refs` first. diff --git a/.changeset/runtime-validators-redos-scope-and-hot-path.md b/.changeset/runtime-validators-redos-scope-and-hot-path.md deleted file mode 100644 index efc667b6..00000000 --- a/.changeset/runtime-validators-redos-scope-and-hot-path.md +++ /dev/null @@ -1,99 +0,0 @@ ---- -'@amritk/runtime-validators': patch ---- - -Widen the ReDoS screen, fix five correctness defects, and cut five allocations off the hot path - -**The ReDoS screen only looked where it expected schemas to be.** It walked a -fixed list of subschema keywords, so an OpenAPI-shaped document — subschemas -parked under `components/schemas` and reached by `$ref` — was declared clean and -its `pattern`s were then compiled and run unscreened. `{ $ref: -'#/components/schemas/A', components: { schemas: { A: { pattern: '^(a+)+$' } } } }` -burned ~1.3 s of CPU on a 31-character input, while the same pattern inlined at -the root was correctly rejected. The walk is now unrestricted: every -string-valued `pattern` key and every `patternProperties` key anywhere in the -document is screened, wherever it sits. Chasing `$ref`s instead would have fixed -that one layout and missed the next unfamiliar one. `const`, `enum`, `default`, -`examples` and `example` are still skipped, because a schema is allowed to carry -arbitrary data there and `{ const: { pattern: '(a+)+' } }` describes an object, -not a regex. This does cost cold build time in proportion to the document -actually being screened — an ordinary component schema is unchanged (~0.016 ms), -but handing `validate` a whole OpenAPI document now costs ~0.25 ms once, where -the old walk visited almost none of it. - -**The screen's documented guarantee was false, and is now both honest and -stronger.** It claimed to "flag a few benign patterns, never the reverse", but it -only recognized *nested* unbounded quantifiers: `^(a|a)+$` is star height 1, so -it passed — and takes over a second on a 29-character input, doubling with each -added character. The screen now also rejects a provably ambiguous alternation -under an unbounded quantifier (two branches that match the same single -character), and the docs say plainly that this is a filter for recognizable -shapes, not a proof of safety — `(a|aa)+` and `a*a*$` still get through. The new -rule is deliberately sound rather than broad: the tempting "overlapping first -characters" test would flag `(ab|ac)+`, which is linear. Zero new flags across a -sweep of 27 ordinary real-world patterns. - -**A deeply nested schema threw an uncatchable `RangeError`.** The pattern screen -and the `$anchor` search both recursed per schema level, and both run before -`maxDepth` applies — so 20,000 nested `{ "not": … }` levels overflowed the native -stack, `isValidationLimitError` returned `false`, and a consumer's limit handler -fell through to a 500. (At 10,000 levels it correctly threw -`ValidationLimitError`.) Both walks are now iterative with an explicit stack, so -the depth cap does its job and an anchor buried 20,000 levels down still -resolves. - -**`required` was silently unenforced for prototype-member names.** The -leftover-required list was built with `k in properties`, which walks -`Object.prototype` — so `'toString' in {}` was `true`, the key looked already -covered and was dropped, and it was absent from the declared-key list too (that -comes from `Object.keys`). Nothing checked it: `{ required: ['constructor'], -properties: {} }` accepted `{}`. Ajv shares this bug by default, so the -differential fuzz could not catch it; there are explicit tests now. - -**`format: 'ipv4'` accepted leading zeros** (`01.2.3.4`), the classic -octal-interpretation allowlist bypass, and the same octets are embedded in the -IPv6 grammar. **`format: 'time'` accepted a bare `12:00:00`** with no offset, -which RFC 3339 `full-time` requires. Both now match Ajv exactly. -**`minProperties`/`maxProperties` counted inherited properties** — a `for…in` -without an own-property guard — so `Object.create({ inherited: 1 })` with one own -key satisfied `minProperties: 2`. - -**Five hot-path costs, measured before and after:** - -- The `enum` failure message was built eagerly and thrown away in guard mode. A - 500-value enum cost 16.4k ops/s on a miss versus 5.1M on a hit — ~99% of the - work was a discarded string. This also hit the *valid* path, because every - non-matching `anyOf`/`oneOf` branch probe runs in guard mode: a 20-branch - discriminated union with `enum` discriminators went 9.5k → 251k ops/s (26×). - The miss itself is now 5.6M ops/s (340–540×). -- `contains` evaluated every element even after it had enough matches. A - 1000-element array matching at index 0 went 8.1k → 5.1M ops/s (630–740×). The early - exit is taken only when `maxContains` is absent and no annotation scope is - active — both need the exact total. -- `dependentRequired` / `dependentSchemas` / `dependencies` rebuilt their - `Object.entries` on every validation. Their entry lists are now memoized on the - per-node metadata alongside the property keys and compiled `patternProperties`, - worth 1.3–1.9× on a one-entry keyword. An `additionalProperties`-only object - schema also stopped allocating a throwaway empty pattern array per call (1.12×). -- `propertyNames` allocated a nine-field interpreter context per key. One scratch - context is now reused across the key loop — safe because the only per-probe - state is the `failed` flag and these probes cannot nest, the key being a string. - A 20-key object gains 1.9–2.5×. -- The own-property count for `minProperties`/`maxProperties` uses - `Object.keys().length`, which measured 92M ops/s against 19M for the old - unguarded `for…in` and 9.5M for a `for…in` with a `hasOwn` guard — so the fix is - also 1.14× faster than the bug. - -**The per-schema validator cache is bounded.** The outer `WeakMap` collects with -the schema, but the inner `Map` keyed on mode/formats/limits lived as long as the -schema did, so a caller deriving `limits` per request pinned a validator forever: -200,000 distinct values retained 82.3 MB. Past 16 configurations it now hands -back an uncached validator (0.5 MB), which costs nothing — there is no compile -step. - -**Two documentation claims corrected.** The README said valid input "and the -entire guard path allocates nothing"; branch probes, annotation trackers and -`uniqueItems` sets all allocate, so it now says what is actually true — nothing -is built for errors that never happen. And the `$ref` cycle-break comment claimed -"stopping here changes no verdict", which holds in a conjunctive position but not -inside a disjunction, where returning valid *is* a verdict. diff --git a/.changeset/runtime-validators-schema-registry.md b/.changeset/runtime-validators-schema-registry.md deleted file mode 100644 index 4ce78317..00000000 --- a/.changeset/runtime-validators-schema-registry.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -'@amritk/runtime-validators': minor ---- - -Take documents the caller already has: `validate(schema, { schemas })` - -The interpreter does no I/O — no `fetch`, no filesystem — which is what lets it -run under a strict CSP and on Workers. Until now that also meant it could not be -*told* about a document it did not receive, so a `$ref` naming another schema -threw and the answer was always "bundle it first". - -`ValidateOptions.schemas` closes that without giving up anything: a plain record -of absolute URI → document, for schemas the caller has already loaded. A -registered document is a full schema resource — walked under its retrieval URI, so -its `$id`, `$anchor`s, `$dynamicAnchor`s and nested embedded resources all -register, a document with no `$id` resolves relative refs against the URI it was -registered under, and one whose `$id` disagrees answers to both. Cross-document -`$dynamicRef` bookending works. A URI that was *not* registered still throws, now -with a message showing how to supply it. - -It is a record rather than an `addSchema` call on purpose: `addSchema` implies -mutable global state, and this package stays a pure function of its inputs. Pass -the registry as an immutable value — the prepared-validator cache keys on its -identity *and* its URI set, so adding or removing a document is a cache miss -rather than a stale hit (swapping the contents under a URI in place is -undetectable, exactly as mutating the schema object is, and is documented as -such). - -With the metaschema registered, `$vocabulary` can finally be read: a custom -dialect that omits the validation vocabulary turns `minimum` and friends into -annotations instead of assertions. Two limits, both documented: it is read from -the root `$schema` rather than per schema resource, and it defaults to enforcing -whenever the metaschema was not registered, which is the stricter answer. - -Nothing changes for callers who pass no registry: the key work is skipped, the -registry build stays gated on the document declaring an `$id`, and the vocabulary -check short-circuits. - -**The package now passes the official JSON Schema Test Suite in full — 1299 / 1299 -required Draft 2020-12 cases.** The harness hands the suite's own `remotes/` -documents to `schemas`, which is the sanctioned equivalent of the HTTP server the -suite would otherwise expect: same documents, same URIs, handed over instead of -fetched, with the interpreter still doing all the base-URI, anchor and -cross-document work the cases exist to test. - -The dialect itself ships alongside, as an opt-in subpath: - -```ts -import { metaschema } from '@amritk/runtime-validators/metaschema' - -validate(userSchema, { schemas: metaschema }) // "is this a valid 2020-12 schema?" -``` - -Eight documents (the dialect plus its seven vocabulary metaschemas), ~7.9 KB of -JSON, reachable only through that subpath — the main entry never imports it, so a -caller who does not ask for it ships none of it. A test holds the copy to Ajv's -vendored specification text by deep equality, which makes Ajv a *check* on the -transcription rather than a runtime dependency of it. diff --git a/.changeset/sweet-worms-brake.md b/.changeset/sweet-worms-brake.md deleted file mode 100644 index 0400a5d1..00000000 --- a/.changeset/sweet-worms-brake.md +++ /dev/null @@ -1,11 +0,0 @@ ---- ---- - -Downgrade the pending `major` bumps to `minor` so every package stays on a -`0.x` line. The breaking changes they describe are unchanged; only the semver -bump they request is. - -Add `bun run versions:check` to CI so this cannot recur: it fails the build on -any package whose next version — projected from the pending changesets, or sitting -in package.json already — leaves `0.x`, and names the changeset files asking for -it. `ALLOW_MAJOR_RELEASE=1` is the escape hatch for the day 1.0 is deliberate. diff --git a/.changeset/tidy-donkeys-smile.md b/.changeset/tidy-donkeys-smile.md deleted file mode 100644 index 389048a8..00000000 --- a/.changeset/tidy-donkeys-smile.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -'@amritk/api': minor ---- - -Security and correctness fixes across the compiled engine, the fetch adapter, and the hook helpers. - -- **`compileToModule` no longer interpolates contract strings into generated source unchecked.** Response status keys, `bodyType`, `method`, and `maxBodyBytes` are validated at emit time and emitted from narrowed values, so a programmatically-built contract (from a config file, a database row, an imported OpenAPI document) can no longer inject code into the module that ships. `defineRoute`/`defineContract` are identity functions with no runtime validation, which is what made this reachable. -- **Guards added or removed after a compile are no longer silently unenforced.** `hashContracts` now fingerprints the *presence* of `guards`, `securityGuards`, and `refine` (their bodies are still excluded, so rewriting one is not staleness), and the emitted module additionally **throws** at init when that shape drifted — a deploy that fails loudly beats one that quietly stops checking credentials. Ordinary schema drift keeps warning and keeps serving. -- **The compiled engine honours the `raw` escape hatch on the error paths.** An `onError` or `errors.*` formatter returning `raw(response)` used to lose its body in the compiled module while the runtime engine sent it. -- **`createETag` enforces `maxBytes` while reading** instead of buffering the whole body first. A large streamed reply was fully buffered just to discover it was over the limit; the cap now bails mid-read and passes the response through without losing already-read chunks. -- **A throwing `onRequest`/`onResponse` hook becomes the pipeline's 500** in both engines instead of escaping to the platform (a Workers 1101, a Bun unhandled rejection). -- **New `writableResponse` export**, used by `createCors`, `createCsrf`, `createRateLimit`, and `createRequestId`. A `Response` from a proxying mount has immutable headers, so mutating them directly threw — and per the previous point, that throw cost the whole reply. -- **`createDocs` escapes `cdn` and `integrity`**, pins the Scalar bundle version (new `SCALAR_VERSION` export and `version` option) instead of floating on `@latest`, and accepts an `integrity` option for subresource integrity. -- **`signCookie`'s imported-key cache is bounded**, so a per-tenant secret-rotation loop no longer retains a `CryptoKey` per distinct secret forever. -- **The package root no longer pulls `node:*` into a Workers or browser bundle.** `node:http`, `node:stream`, and `node:events` reached the root entry through the Node adapters and broke `esbuild --platform=browser` outright (resolution runs before tree-shaking). The adapters now load their built-ins on demand and `waitForDrain` dropped `node:events` entirely; a graph-walking test over `index.ts` pins the invariant. diff --git a/.changeset/validators-typeless-and-boolean-schemas.md b/.changeset/validators-typeless-and-boolean-schemas.md deleted file mode 100644 index 9cde7131..00000000 --- a/.changeset/validators-typeless-and-boolean-schemas.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -'@amritk/generate-validators': minor ---- - -Enforce the keywords a schema declares without a `type`, and stop emitting -validators that call functions nobody wrote - -Measured against the official JSON Schema Test Suite, generation goes from -818/1299 to **987/1299 (76.0%)**. - -- **A schema with no root `type` compiled to `validateRoot = () => true`.** The - generator hung every check off the declared type, so `{ "minLength": 2 }`, - `{ "required": ["a"] }`, `{ "uniqueItems": true }`, `{ "contains": … }`, - `{ "patternProperties": … }`, `{ "propertyNames": … }` and - `{ "dependentRequired": … }` all accepted everything — the largest silent gap - this generator had. Each keyword now emits its check behind its own runtime type - test, so it rejects its own family and ignores every other kind of value, which - is what JSON Schema means by a type-less constraint. The same gate was - suppressing constraint checks next to a root combinator, so - `{ allOf: [{ prefixItems: … }], items: … }` silently dropped its `items` too. -- **Object keywords no longer imply `type: "object"`.** `{ "properties": … }` - ignores a non-object instead of rejecting it, matching the interpreter. Note the - inferred TypeScript type still describes the object case (as - `FromSchema`/`ImplicitShape` in `@amritk/runtime-validators` does), so for that - shape `isX` is a weaker type guard than it was — the verdict, which is the - contract, matches the interpreter exactly. -- **Boolean subschemas do something.** A root of `false` rejects every instance - (it used to accept them all), and a `false` sitting in a `properties`, `allOf`, - `then`/`else`, `patternProperties`, `dependentSchemas`, `prefixItems`, - `contains` or `propertyNames` position now emits a real check. -- **A `not` over a `type` array emitted nothing**, and "no checks" is how the - matcher spells "matches everything" — so the `not` rejected every instance. -- **Unresolvable `$ref`s produced output that does not compile.** For a ref the - walker never queues (a relative path, an absolute path, a URN), the emitter - derived a name from the ref string and called `validateIntJson(…)` without - anything emitting it. Generation now refuses, naming the ref — the same answer - the other unsupported paths already give, and a failure next to its cause rather - than in the consumer's build. -- **String lengths count code points, not UTF-16 units**, in both the validator - and the guard, via the shared `@amritk/helpers/string-length-check` (`.length` - stays the short-circuiting first term). - -`unevaluatedItems`/`unevaluatedProperties` still refuse at generation by design — -flat output cannot carry annotations across the applicator tree — and `$id` -base-URI resolution remains unimplemented, so those refuse rather than guess. diff --git a/.changeset/validators-unevaluated-and-honest-guards.md b/.changeset/validators-unevaluated-and-honest-guards.md deleted file mode 100644 index aadfad62..00000000 --- a/.changeset/validators-unevaluated-and-honest-guards.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -'@amritk/generate-validators': minor ---- - -Generate `unevaluatedItems` / `unevaluatedProperties`, and stop `isX` claiming a -narrowing it cannot make - -**On the official JSON Schema Test Suite: 987/1299 → 1238 / 1299 (95.3%).** - -The position that flat generated code cannot carry annotations across the -applicator tree turned out to be wrong, and it was costing 201 cases — two thirds -of everything this package failed. Both keywords are now emitted as a flat -*expression* computing what the interpreter computes as annotations: for each key -or index, a boolean that is true when some keyword evaluated it. Keywords that -must succeed for the value to be valid at all (`allOf` members, a `$ref` target, a -satisfied `contains`) count unconditionally — sound, because the emitted test is -one conjunct of a validator that also asserts them — while conditional applicators -(`anyOf`/`oneOf` branches, `if`/`then`/`else`, `dependentSchemas`) carry their -condition, hoisted to a `const` before the loop so a per-key sweep reads a boolean -instead of re-running a match. - -`contains` publishes only the indices it matched, per the spec and -`@amritk/runtime-validators`, rather than Ajv's whole-array mark. - -Four shapes still refuse, each with a message naming the shape rather than the -keyword: coverage running through a `$dynamicRef`, an unresolvable or cyclic -`$ref` at the same instance location, a walk deeper than eight applicators, and a -node under `additionalItems`. No case in the suite hits any of them. - -Parity with the interpreter is the contract and is enforced as one: -`interpreter-parity.test.ts` gains six hand-written groups plus a 500-schema × -24-value fuzz pass — 12,000 pairs, no divergence. - -**`isX` no longer lies.** For a schema with no `type`/`enum`/`const`/`$ref` but -object-shaped keywords (recursively, so a union of implicit-object branches counts -too), the emitted type describes the object case while the validator — correctly — -also accepts non-objects. The guard now returns `boolean` for exactly those -schemas instead of `input is X`; the check itself is unchanged and still in -lockstep with `validateX`. Every schema that declares a `type` keeps its type -predicate. - -The complete fix is to widen the emitted type so the narrowing becomes true, which -lives in `@amritk/helpers/generate-type-definition` and has to move together with -`FromSchema`'s `ImplicitShape` in `@amritk/runtime-validators`, since both make the -identical inference for the identical schema. Until they do, a guard that declines -to narrow beats one that narrows wrongly. - -Also inherited from `@amritk/helpers`: `$ref`s written against an enclosing `$id` -now resolve, which closed most of this package's ref failures without a change -here. diff --git a/.changeset/yaml-block-structure-and-key-properties.md b/.changeset/yaml-block-structure-and-key-properties.md deleted file mode 100644 index a73d21e4..00000000 --- a/.changeset/yaml-block-structure-and-key-properties.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -'@amritk/yaml': minor ---- - -Read the block shapes that were folding structure into strings - -Nineteen more YAML test suite cases — 365/402 to **384/402 (95.5%)**. Every one -of them was a document whose structure the parser flattened into text or -orphaned entirely, and none of them produced a diagnostic saying so. - -**Node properties written on a mapping key now apply to the key.** `&a a: b` -anchors the scalar `a`, so `*a` is the string `"a"`; before, the anchor stayed -inside the key's text and every alias to it reported `UNRESOLVED_ALIAS` and -projected to nothing. The same for tags — `!!str 23: v` keyed the mapping by the -literal `"!!str 23"` rather than by `"23"`. Anchor names that hold a `:` work -too (`&a: key: value` anchors `key` as `a:`), which needs the properties scanned -before the key separator is looked for, not after. Properties on a line of their -own above the mapping still describe the mapping. Reaching this costs one -character comparison per mapping entry — the scan itself only runs for a key -that is actually annotated. - -**A `: ` inside a plain scalar is reported** (`BAD_SCALAR_CONTENT`). The spec -ends a plain scalar there, so `a: b: c: d` is an error and not the string -`"b: c: d"`; so is a continuation line that reads as a mapping entry, which is -what a mis-indented `k1: v1` / `⟨space⟩k2: v2` is. The scan is an `indexOf`, so -a scalar with no colon in it — nearly all of them — pays one native pass that -finds nothing. Quoted, block, and flow scalars are unaffected. - -**Block collections may open on an explicit entry's introducer line.** `? a` / -`: - one` is a sequence whose first entry shares the `:` line, and -`? earth: blue` a mapping whose first entry shares the `?` line; both folded -into the value as text. Their remaining entries align under that first one, not -under the introducer. The mirror-image shape after an *implicit* key -(`key: - a`) is invalid YAML and is now reported rather than folded. Relatedly, -a `? ` introducer is settled by the first two characters, so it outranks any -`: ` further along the line — testing the colon first read `? earth: blue` as a -key called `? earth`. - -**Indentation is measured against the parent, not against the node.** The -parser treated "the parent's column" as one less than the column the node -started at, which is only true when a node begins exactly one column in. When it -did not, documents were cut short: a plain scalar stopped at the first -continuation line that stepped back (`a:` / `⟨2 spaces⟩foo` / `⟨1 space⟩bar` -dropped `bar`, and a sequence entry that wrapped was split into two entries), a -zero-indented sequence introduced by a tag or an anchor was orphaned and -reported as stray content (`sequence: !!seq` over a `- entry` list, `seq:` / -`⟨1 space⟩&anchor` over one), and a block scalar counted its indentation -indicator from the wrong column. The document root still measures against -1, so -`--- |2` is unchanged. - -The `parseAllDocuments` / `parseDocument` API, `toJS()` projection, and every -node's `[start, end)` span are unchanged, and the parser's throughput is -unchanged on all three benchmark fixtures. diff --git a/.changeset/yaml-cr-breaks-merge-and-key-paths.md b/.changeset/yaml-cr-breaks-merge-and-key-paths.md deleted file mode 100644 index 4c4ca41c..00000000 --- a/.changeset/yaml-cr-breaks-merge-and-key-paths.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -'@amritk/yaml': minor ---- - -Support lone-`\r` line breaks, stop `<<` dropping inherited-name keys, and make -`nodeAtPath` agree with `toJS()` - -Five silent-data-loss bugs, each in a path the differential corpus and the -yaml-test-suite do not reach. - -**A lone `\r` truncated the document, with no diagnostic at all.** The scanner -skipped to the next line by looking only for `\n`, so a CR-delimited document had -every line after the first jumped over — `a: 1\rb: 2\rc: 3\r` parsed to -`{ a: 1 }` and reported zero errors, and a single stray CR inside an otherwise-LF -file made one key vanish. YAML 1.2 §5.4 makes all three of `CR LF`, `CR`, and -`LF` a line break, and now so do we — in the parser and in `lineCounter`, so -positions stay exact. `CR LF` still counts once. The differential suite re-runs -every case in all three break styles. - -**`<<` dropped any merged key that shares a name with an `Object.prototype` -member.** The "does the target already have this key?" test walked the prototype -chain, so `toString`, `valueOf`, `constructor`, `hasOwnProperty`, -`isPrototypeOf`, `__proto__` and friends were silently discarded from the merge. -Only own keys shadow a merged one now; `__proto__` is still defined as plain data -rather than assigned through the prototype setter, so the pollution guard holds. - -**`nodeAtPath` could not find the keys `toJS()` produces.** It carried its own -simplified key-stringifier that returned `'null'` for a null key, `'*ref'` for an -alias key, and `''` for every collection key — so `null: v`, `*a : v`, and -`[a, b]: v` were unreachable by path, and a `closest: true` lookup quietly -returned the *parent's* source span: a diagnostic pointing at the wrong line. It -now uses the parser's own projection, which is exported as `keyText` for anyone -building paths by hand. - -**An unterminated quoted scalar lost its last character.** The recovery sliced -off a closing quote that was never there, so `a: "abcd` recovered as `"abc"` — -the wrong text for a linter to echo back. The `UNTERMINATED_QUOTE` error was -always correct; the text now is too. - -**`parseDocument` truncated a `---` stream without saying so.** Reading only the -first document is intended, but a caller on `parse()` sees only the data. It now -pushes a `MULTIPLE_DOCUMENTS` warning pointing at the marker and naming -`parseAllDocuments`. A trailing marker with nothing under it stays quiet. - -Also: `lineCounter` builds its index with `indexOf` instead of a per-character -loop — 2.6–3.6× faster, taking it from ~18% of parse+index cost to ~6%. And the -bundle-size benchmark now bundles a consumer of each parser rather than the -barrel, which tree-shook to a 156-byte stub and made the README's size table -fiction; the corrected numbers are in the README. diff --git a/.changeset/yaml-marker-line-and-flow-folding.md b/.changeset/yaml-marker-line-and-flow-folding.md deleted file mode 100644 index f2d3e35a..00000000 --- a/.changeset/yaml-marker-line-and-flow-folding.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -'@amritk/yaml': minor ---- - -Parse the node written on a `---` line, and stop flow scalars losing their type - -Four silent-data-loss bugs, each in a branch that was already cold. - -**A node written on the `---` line was discarded.** The document head skipped -the whole marker line, so `--- |` lost its block-scalar indicator and re-read -the body as a folded plain scalar (line breaks gone), `--- foo` lost the scalar -entirely, and a tag or anchor on the marker line never applied. The node is now -parsed and measured against column 0 rather than the column the marker pushed it -to, so `--- >` may hold content starting at column 0. A block *mapping* on the -marker line is invalid YAML and is now reported. This is also what makes -`--- !!set` / `--- !!omap` reach their `Set` / `Map` projections. - -**A flow scalar that ended its line lost its core-schema type.** `{ a: 1, b: 2 }` -resolved `b` to the number `2`, but the same document wrapped — -`{ a: 1,\n b: 2 }` — resolved it to the *string* `"2"`, because the multi-line -path folded the segments without resolving them. Every entry of a flow -collection written across lines was affected. Such a scalar also now ends at a -`:` or `#` that opens the next line, instead of folding it in — `{foo\n: bar}` -used to key the mapping by `"foo\n"`. - -**Double-quoted folding ran before escapes were resolved,** so it could not tell -an escaped `\t` (content) from a literal trailing tab (padding), and it turned a -`\` line-continuation's break into a space the `\` then absorbed. Escapes are now -resolved per line first, and folding strips only whitespace the document wrote -literally. - -**A block-folded scalar treated only a space as "more indented",** so a break -beside a tab-led line folded to a space and the blank line next to it was lost. - -Also: an unterminated quoted scalar now stops at a `---`/`...` marker instead of -swallowing every document after it; `!!str` over a wrapped plain scalar reads the -folded text rather than un-folding it; and the stream-level directive rules are -enforced — a directive needs a `...` before it and a `---` after it, its version -must parse, a second `%YAML` is an error rather than a warning, and a tag may not -hold a flow indicator. New code: `UNEXPECTED_DIRECTIVE`. - -Conformance against the official YAML test suite is **336/402 (83.6%)**, up from -293/402, with every remaining gap still listed and reasoned. diff --git a/.changeset/yaml-syntax-diagnostics.md b/.changeset/yaml-syntax-diagnostics.md deleted file mode 100644 index 2ffaadb6..00000000 --- a/.changeset/yaml-syntax-diagnostics.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -'@amritk/yaml': minor ---- - -Report the syntax errors that were quietly changing what a document said - -Twenty-nine more YAML test suite cases, every one of them in a branch that was -already cold — a block scalar header, a backslash, a `#`, a node property. - -**A block scalar header was accepted whatever followed it.** `folded: > first -line` dropped `first line` and re-read the body below as the scalar; `|10` took -the `1` as an indentation indicator and threw the `0` away; `>#comment` read a -comment the spec does not allow there; and a repeated indicator (`|--`) silently -kept the last one. The header now ends where its indicators do, and anything -past it is a `BAD_BLOCK_HEADER`. A leading blank line indented deeper than the -block's first content line — which makes the block's own indentation ambiguous — -is a `BAD_INDENT`. - -**A `\` escape the spec does not define passed through as the bare letter,** so -`"a\.b"` became `a.b` and `"it\'s"` became `it's`, each silently dropping a -character the document wrote. Undefined escapes are now `BAD_ESCAPE`; the value -is still produced, so nothing that parsed stops parsing. - -**A `#` with no whitespace before it is not a comment.** `key: "value"# text`, -`[ a, b ]#text`, and `[ a, b,#text` each dropped the rest of the line as though -it were one (`BAD_COMMENT`). The mirror image is fixed too: a comment *does* end -a plain scalar, so `word1 # comment` followed by `word2` no longer folds `word2` -into the value — it is reported as content no node claims. - -**An implicit key has to fit on one line.** A quoted key spanning lines -(`"a\nb": 1`), a flow collection used as a block key across lines (`[23\n]: 42`), -and a compact `[ key\n : value ]` sequence entry are now `BAD_IMPLICIT_KEY`. A -flow *mapping* may still write `{ "foo"\n: bar }` — the spec allows that one. - -**Node properties are checked where they land.** An anchor or tag written on an -alias (`key: &b *a`) was dropped without a word, and two anchors reaching one -scalar kept only the second: both are now `BAD_PROPERTY`. A block sequence -opened on a properties line (`&anchor - entry`) read as the plain scalar -`"- entry"` and is now reported. - -**Also:** a multi-line quoted scalar whose continuation lines do not clear their -parent's indentation is a `BAD_INDENT`; a `-` where a flow entry belongs (`[-]`) -is a `BAD_SCALAR_START`; and a `---`/`...` marker inside a flow collection, or -in the middle of a wrapped flow scalar, ends the document instead of being -absorbed into it. - -New codes: `BAD_BLOCK_HEADER`, `BAD_COMMENT`, `BAD_ESCAPE`, `BAD_IMPLICIT_KEY`, -`BAD_INDENT`, `BAD_PROPERTY`. - -Conformance against the official YAML test suite is **365/402 (90.8%)**, up from -336/402, with every remaining gap still listed and reasoned. Measured against -main with the ABBA bench harness over three full runs, every fixture is within -noise: the one cell that ever flagged — `large (data)`, at -8.9% — came from the -run sharing the machine with a test suite, and read -1.5% and +3.2% in the two -runs that had it to themselves. diff --git a/.changeset/yaml-tab-columns-flow-indent-and-json.md b/.changeset/yaml-tab-columns-flow-indent-and-json.md deleted file mode 100644 index 513f771e..00000000 --- a/.changeset/yaml-tab-columns-flow-indent-and-json.md +++ /dev/null @@ -1,75 +0,0 @@ ---- -'@amritk/yaml': minor -'@amritk/lint': patch ---- - -Read tabs by the column they sit at, hold a flow collection to its parent's -indentation, and pin JSON as the superset it is - -YAML 1.2 test suite conformance goes from **384/402 (95.5%) to 398/402 (99.0%)**. -What is left is four cases where the right answer is not the suite's — one -duplicate-key case that turns on the `uniqueKeys` default, and three tags that -project to a `Uint8Array`/`Set`/`Map` — so this closes the boundary rather than -moving it. Parse throughput is unchanged: an order-balanced A/B over the bench -fixtures lands every case inside run-to-run noise (medians −3% to +1.6% against a -4–6% CV, with min-of-runs favouring the new code). - -**A tab is only an error where indentation belongs.** Indentation in YAML is -spaces (`s-indent ::= s-space × n`), but a tab *past* the indentation is ordinary -separation — and the two are told apart only by the column the tab sits at. -`peekLine` reported any leading tab at all, which cut both ways: it rejected three -valid documents (`\t[…]` and `\t{}` at the document root, and a `foo:` whose value -line reads `⟨space⟩⟨tab⟩bar`) while missing the tabs that really are indentation. -Every caller knows the column its line owes, so it now passes it in, and the same -rule is applied in the three other places a tab can stand for indentation: - -- Inside a block scalar — `foo: |` over a lone `\t` is reported, where the same - line written ` \t` is valid content and still parses to `"\t\n"`. -- In the separation between a block indicator and a **compact collection** opened - on its line. A compact collection takes its indentation from the column it lands - on, so `-\t-`, `?\tkey:` and `:\t- x` are invalid — while `-\tfoo` and `-\t-1` - are ordinary separation and stay valid. -- In a flow collection's continuation lines (below). - -**A flow collection is held to the indentation of the block that holds it.** Flow -scanning is delimiter-driven, so `flow: [a,` over a column-0 `b,` read exactly -like a properly indented collection and parsed clean; it is now reported once per -collection as `BAD_INDENT`. Indentation is counted in spaces, which folds the tab -rule in for free. The closing `]`/`}` is deliberately held one column looser than -the spec asks — to the parent's own column rather than one past it — because -closing a multi-line flow collection at the parent's column is how Prettier and -hand-written manifests both write it, and `yaml` and `js-yaml` both accept it. - -**A tag or anchor inside a flow collection ends at the flow indicator.** In -`{ foo : !!str, }` the tag token swallowed the comma, which the tag-character check -then reported while the missing comma left the mapping looking unterminated and -shifted every entry after it. Outside a flow collection those characters are still -ordinary tag content, so a block-context `!!str,` is still a `BAD_TAG`. - -**Tab-indented JSON parsed to the wrong value.** A wrapped flow line's leading -whitespace is `s-indent(n) s-separate-in-line?`, so tabs sit in it as spaces do — -but the flow scalar scanner skipped only spaces, so the `]` closing a tab-indented -line was never seen as the flow indicator it is and the line folded into the scalar -instead. `JSON.stringify(value, null, '\t')` — what `jq --tab` and every -"indent with tabs" editor setting emit — therefore turned the last entry before a -`]` into a string with a trailing newline: `-1` came back as `"-1\n"`. - -**The 1024-character implicit key limit is enforced in block context.** YAML caps -how far past a key's start its `:` may sit so a processor can recognize a mapping -entry with bounded lookahead; a longer block key is now `BAD_IMPLICIT_KEY`. It is -deliberately *not* enforced in flow context, matching `yaml` (eemeli): a flow -mapping is where JSON lives, `{"…1100 characters…": 1}` is valid JSON, and -rejecting a valid JSON document is the worse of the two errors. Relatedly, an -explicit key in a flow sequence may now put its `:` on the next line -(`[ ? a\n : b ]`) — the one-line rule exists to keep an *implicit* key cheap to -recognize, and a `?` settles that up front. - -**The JSON-superset property is now checked, not assumed.** `@amritk/lint` routes -`.json` documents through the YAML parser and `resolveRefsFromFile` hands it -whatever a `$ref` points at, so "JSON parses as YAML" is load-bearing. -`src/json-superset.test.ts` runs a generated corpus against `JSON.parse` — every -value in six spellings (compact, 2-space, tab-indented, CRLF, and with -leading/trailing blank lines), requiring an identical value *and* zero diagnostics -for each — and `@amritk/lint` gains a matching test holding `parseJson` and -`parseYaml` to identical data, diagnostics, and `line:column` ranges for every path -in a JSON document. diff --git a/packages/adapters/CHANGELOG.md b/packages/adapters/CHANGELOG.md index 9c5dfbf2..25bb457e 100644 --- a/packages/adapters/CHANGELOG.md +++ b/packages/adapters/CHANGELOG.md @@ -1,5 +1,67 @@ # @amritk/adapters +## 0.4.0 + +### Minor Changes + +- 2c9982c: Fix the published manifests so the packages install, resolve, and dedupe correctly + + **Types resolve on TypeScript's default config.** Every package was + exports-only: nine declared `"module": "./dist/index.js"` (a field neither Node + nor TypeScript reads) and nothing declared `types`. A consumer on + `moduleResolution: "node10"` — still the default when `module` is `commonjs` — + cannot see `exports` at all, so `import { lintDocument } from '@amritk/lint'` + failed with `TS2307: Cannot find module '@amritk/lint' or its corresponding type +declarations`. Each package with a `.` export now also declares `main` and + `types`; `@amritk/helpers` and `@amritk/adapters` have no `.` export (they are + subpath-only), so they declare a `typesVersions` wildcard mapping instead, which + gives their subpaths the same node10 fallback. All of it is ignored under + `node16`/`nodenext`/`bundler`, where `exports` still wins. + + **`workspace:*` resolves to a caret, not an exact pin.** All fourteen + inter-package edges shipped as exact versions, so installing two `@amritk/*` + packages published at different times pulled in two copies of their shared + dependency. That is not merely wasteful: the module-level caches those packages + rely on are per-copy, so the `WeakMap` validator cache in + `@amritk/runtime-validators` silently stopped hitting. Pre-1.0 a caret stays + narrow (`^0.9.1` is `>=0.9.1 <0.10.0`) and breaking changes here already ride a + minor bump. + + **`@amritk/helpers` stops shipping 21 source files it does not need.** Embedded + mode reads four helper sources (`is-object`, `validate-array`, + `validate-record`, `has-ref`) out of the installed package at generation time, + so `src` has to ship — but only those four. `files` now lists them explicitly + instead of globbing all of `src`, cutting the tarball from 78 files / 206 kB to + 63 / 112 kB. + + **Two packages no longer declare a dependency they never import.** + `@amritk/mjst` and `@amritk/generate-parsers` both listed + `@amritk/generate-markdown` under `dependencies`, but the only importer is each + package's `scripts/generate-readme.ts`, which is not published. Both moved to + `devDependencies`. `@amritk/adapters` likewise dropped its + `@sinclair/typebox` peer dependency: the TypeBox adapter is purely structural + (it strips symbol keys) and imports nothing. `valibot` stays — it is a genuine + transitive peer of `@valibot/to-json-schema`. + + **`@amritk/mjst` fixes.** `json-schema-typed` moved to `dependencies`, because + the shipped `dist/emit-examples.d.ts` imports types from it. The package gained + an `exports` map, so it is no longer deep-importable in its entirety. And the + build now marks `dist/cli.js` executable: `npm pack` records on-disk modes, and + package managers only `chmod` bin targets when they link them, so flows that + consume the tarball directly (vendoring, Docker `npm pack` + `tar -x`) hit + `EACCES`. + +### Patch Changes + +- Updated dependencies [213ecc4] +- Updated dependencies [9cb45a0] +- Updated dependencies [5afbfd4] +- Updated dependencies [eb80ca6] +- Updated dependencies [2c9982c] +- Updated dependencies [f439570] +- Updated dependencies [fa8620c] + - @amritk/helpers@0.15.0 + ## 0.3.6 ### Patch Changes diff --git a/packages/adapters/package.json b/packages/adapters/package.json index d5975ca0..d15de117 100644 --- a/packages/adapters/package.json +++ b/packages/adapters/package.json @@ -1,6 +1,6 @@ { "name": "@amritk/adapters", - "version": "0.3.6", + "version": "0.4.0", "description": "Convert schemas from external libraries (TypeBox, Zod, ...) into JSON Schema for mjst.", "type": "module", "sideEffects": false, diff --git a/packages/api/CHANGELOG.md b/packages/api/CHANGELOG.md index 0f3a0c5d..52bd4600 100644 --- a/packages/api/CHANGELOG.md +++ b/packages/api/CHANGELOG.md @@ -1,5 +1,329 @@ # @amritk/api +## 0.13.0 + +### Minor Changes + +- 299ed2a: Keep the runtime pipeline synchronous when the request never suspends + + **`Api.handle` now returns `ApiResponse | Promise`.** It answers + synchronously when nothing along the route's path was asynchronous — no + declared body to read, no `refine`, no context factory, no guards, and a + handler that returned a value rather than a promise. This is the breaking part: + `await api.handle(...)` is unaffected, but code calling `.then()` on the result + directly must handle a plain value. The bench harness in this repo did exactly + that and is updated alongside. + + **Why.** An `async` frame and its promise are not free, and on workerd the + difference is large enough to see. On the static GET, measured inside a real + isolate with `bench:workerd:allocations`, the runtime engine allocated 2115 + bytes per request; it now allocates 1510, a 29% cut. Throughput on that case + went from ~69k to ~93k ops/s — from 0.80x bare Hono to roughly level with it. + + **How.** `runRoute` is no longer one async function. It is a synchronous + dispatcher over three stages that hand off to each other synchronously until + something genuinely asynchronous appears: + + - `runSecuredRoute` — security guards suspend before anything else happens, so + that whole shape stays asynchronous, and rejoins the shared stages after. + - `runSlots` — coercion and validation for params, query, headers, and cookies, + all of which were already synchronous. + - `runBody` — reading a declared body always suspends, so this stage is + unconditionally asynchronous. + - `runTail` — refine, the context factory, guards, and the handler. Synchronous + when none of the first three are configured and the handler returns a value; + otherwise it delegates to `runTailAsync`, which is the original straight-line + code. + + The error tail is factored into one `routeError` helper the synchronous and + asynchronous halves share, so the two cannot drift on payload-too-large + detection or the `onError` contract. `finishReply` stays outside the error + boundary in both, exactly where it was. + + **What did not change.** Ordering is identical: security guards still run + before any parsing, the context factory still runs after validation on + unsecured routes and before it on secured ones, and it still runs exactly once + per request. The differential corpus that holds the runtime and compiled + engines observationally identical passes unchanged. + + **Still open.** The runtime engine's batch-time distribution is still bimodal + under workerd — a p95 around 3.3x its median, which is a major collection + rather than allocation volume. Neither the async work above nor removing + response validation moved it. The compiled engine, which is the production + path, does not show it. + +- 0d4bed2: Security and correctness fixes across the compiled engine, the fetch adapter, and the hook helpers. + + - **`compileToModule` no longer interpolates contract strings into generated source unchecked.** Response status keys, `bodyType`, `method`, and `maxBodyBytes` are validated at emit time and emitted from narrowed values, so a programmatically-built contract (from a config file, a database row, an imported OpenAPI document) can no longer inject code into the module that ships. `defineRoute`/`defineContract` are identity functions with no runtime validation, which is what made this reachable. + - **Guards added or removed after a compile are no longer silently unenforced.** `hashContracts` now fingerprints the _presence_ of `guards`, `securityGuards`, and `refine` (their bodies are still excluded, so rewriting one is not staleness), and the emitted module additionally **throws** at init when that shape drifted — a deploy that fails loudly beats one that quietly stops checking credentials. Ordinary schema drift keeps warning and keeps serving. + - **The compiled engine honours the `raw` escape hatch on the error paths.** An `onError` or `errors.*` formatter returning `raw(response)` used to lose its body in the compiled module while the runtime engine sent it. + - **`createETag` enforces `maxBytes` while reading** instead of buffering the whole body first. A large streamed reply was fully buffered just to discover it was over the limit; the cap now bails mid-read and passes the response through without losing already-read chunks. + - **A throwing `onRequest`/`onResponse` hook becomes the pipeline's 500** in both engines instead of escaping to the platform (a Workers 1101, a Bun unhandled rejection). + - **New `writableResponse` export**, used by `createCors`, `createCsrf`, `createRateLimit`, and `createRequestId`. A `Response` from a proxying mount has immutable headers, so mutating them directly threw — and per the previous point, that throw cost the whole reply. + - **`createDocs` escapes `cdn` and `integrity`**, pins the Scalar bundle version (new `SCALAR_VERSION` export and `version` option) instead of floating on `@latest`, and accepts an `integrity` option for subresource integrity. + - **`signCookie`'s imported-key cache is bounded**, so a per-tenant secret-rotation loop no longer retains a `CryptoKey` per distinct secret forever. + - **The package root no longer pulls `node:*` into a Workers or browser bundle.** `node:http`, `node:stream`, and `node:events` reached the root entry through the Node adapters and broke `esbuild --platform=browser` outright (resolution runs before tree-shaking). The adapters now load their built-ins on demand and `waitForDrain` dropped `node:events` entirely; a graph-walking test over `index.ts` pins the invariant. + +### Patch Changes + +- a342117: Close a compiled-engine validation bypass, and stop dropping `__proto__`-named headers and path parameters + + **`compileToModule` baked its schema constants as object literals, which is not + a faithful copy of the JSON they were printed from.** A JavaScript object + literal treats `__proto__` as the prototype setter, so a contract declaring a + property under that name — perfectly ordinary in a schema loaded from a config + file, a database row, or an imported OpenAPI document, where the key really is + an own property — compiled to a constant with that property silently missing. + The compiled engine then validated a schema the runtime engine never had, and + diverged in both directions: it rejected `{"__proto__":"abc"}` under + `additionalProperties: false` that the runtime accepted, and accepted + `{"__proto__":123}` against `{"type":"string","minLength":3}` that the runtime + rejected. The second is a validation bypass in the production engine — the + declared constraint was simply gone. + + Every constant baked from contract data — request schemas for all five slots, + response body and header schemas, and the interpreter's options — now emits as + `JSON.parse('…')`, where each key lands as an own property. The argument is a + correctly-escaped single-quoted string literal (backslashes, single quotes, and + U+2028/U+2029, which are legal unescaped in JSON but were line terminators in + pre-ES2019 JavaScript source), pinned by a round-trip test over hostile input. + There is no startup cost: a JSON string literal evaluates about 13% faster than + the equivalent object literal at module init on a 46 KB schema, and the emitted + module grows by 14 bytes per constant (0.6% on a realistic module). The + precomputed OpenAPI document was never affected — it was already a string + literal. + + The differential corpus gained a route declaring `__proto__` as its path + parameter, header, cookie, _and_ body property at once, so the two engines are + now pinned to agree on the correct answer for all of them, and the emitter has + an invariant test that no schema constant may be a bare object literal. + + **Headers and path parameters named `__proto__` are no longer dropped.** The + same write-side bug the cookie parser had: `__proto__` is a valid HTTP field + name (it is a token) and a valid path-template capture name, but a plain + `record[name] = value` runs the prototype setter instead of creating the + property. A contract declaring one saw nothing, and `required: ['__proto__']` + could never be satisfied. Fixed in the route matcher, the params builder, and + the headers builder through a shared `defineOwnProperty`, which the cookie + parser now shares too; the compiled engine unrolls its own params and headers + builders, so it emits the equivalent `Object.defineProperty` for that one name + and pays nothing for every other. + + Also: the schema-derived response serializer now declines any property whose + name shadows an `Object.prototype` member, falling back to `JSON.stringify` — + its `body[""]` reader would otherwise answer with the inherited member + rather than `undefined` when the reply omits the property, so a `__proto__` + property serialized as `{}` and an optional `toString` was emitted on every + reply. This is the same bail the inline guard emitter already made, and the two + now share one list of risky names. + +- f5a52b7: Report thrown adapter hooks instead of swallowing them, and stop dropping a `__proto__` cookie + + **A throwing `onRequest` gate or `onResponse` decorator is no longer silent.** + Wrapping the hook chains stopped a throwing hook from escaping to the platform, + but the caught error was then dropped on the floor: no log, and the app's own + `onError` — which every routed failure already goes through — was never + consulted. The motivating case is exactly the one that needs telemetry: + `createRequestId({ trustInbound: true })` reflecting a CRLF-bearing inbound + `x-request-id` into `Headers.set` answered a bare `{"error":"internal_error"}` + with no indication that a decorator threw, or which one. Before the wrapping the + throw at least surfaced as a platform-level unhandled error, so the fix traded a + crash for an undiagnosable 500. + + A thrown hook now goes to the app's `onError` (with `route: undefined` — a hook + belongs to no route), whose reply shapes the response exactly like a handler + error's does, raw-`Response` escape hatch included. An app that wired no + `onError` gets a `console.error` instead, because silence is the one outcome + that is never acceptable here; a reporter that throws falls back to the same log + and the bare 500. `Api` gained an optional `onError` so the adapter can reach the + sink the app already configured — the hooks run outside `handle`, so the + pipeline's own boundary never sees them. `compileToModule` emits the identical + helper, and the two-engine differential corpus now pins that both engines report + the same error, through the same sink, with the same log line. + + **A contract declaring a cookie named `__proto__` now actually receives it.** + The read side treated the name as ordinary data, but the write side was a plain + `cookies[name] = value`, which runs `Object.prototype`'s `__proto__` setter + rather than creating a property — so the value silently vanished and + `required: ['__proto__']` failed for every request no matter what the client + sent. Same `defineProperty` fix already applied in `@amritk/generate-validators` + and `@amritk/yaml`. Both engines share this parser, so the compiled engine picks + it up unchanged. + +- 365c6c1: Bucket route dispatch by shape, and fix two request-parsing defects + + **Route lookup no longer scans every parameterized route.** The runtime engine + kept one list of dynamic routes per method and walked it in registration order, + re-running the segment matcher against each candidate. At 500 routes that was + ~7.3 µs per lookup, and a miss cost the same as a hit. Dynamic routes are now + bucketed by segment count and by their first literal segment, so a lookup only + ever touches candidates that could match the shape in front of it: ~0.55 µs at + 500 routes, and flat as the table grows. Precedence is unchanged — the buckets + are precomputed with the wildcard-first routes merged into each literal's list + in registration order, so which of two overlapping routes wins is exactly what + it was, greedy tails and static-over-dynamic included. + + **An unroutable path is no longer the most expensive request an API serves.** + Building the 405 `allow` header re-ran the _whole_ matcher once per method the + API declares, so a path from a vulnerability scanner cost up to seven times the + scan — ~45 µs of pure dispatch on a 500-route table, versus ~3 µs to serve a + real request. The static half of that answer is now precomputed at startup (the + same table the compiled engine emits as `ALLOW_STATIC`) and the dynamic half + reuses one path split across all methods: ~0.9 µs. The static hit path also + stopped building a `method + ' ' + path` key per request. + + **Duplicate cookie names now resolve first-wins, not last-wins.** Browsers send + the most specific cookie first (RFC 6265 orders by longer path, then earlier + creation), so a `Path=/` cookie planted from a sibling subdomain arrives _after_ + the real session cookie — and last-wins let it shadow it. First-wins is what the + `cookie` package behind Express, Hono, and Fastify does, and what the rest of the + stack assumes. Both engines share this parser, so they stay identical. + + **`buildParamPath` rejects `.` and `..` path parameters.** Dots are unreserved, + so `encodeURIComponent` left them alone and `client.getUser({ params: { id: +'..' } })` built `/users/..`, which the URL parser then collapsed _before the + request was sent_ — the call silently hit a different endpoint. It now throws. + Greedy `{name+}` tails are checked per segment for the same reason: WHATWG URL + normalizes `%2e%2e` too, so a literal `..` path component cannot be transmitted + at all, which makes one there always an unintended traversal rather than a + directory name. + + The bench harness gains two dispatch cases — `dynamic GET, 500-route table, last +match (runtime)` and `unroutable path, 500-route table (runtime)` — so the PR + delta table catches a regression in either. + +- 2eed2e5: Stop paying for an AbortSignal on every request, and re-measure the cross-framework tables + + **Both engines materialized a host-backed `AbortSignal` per request.** The + per-request `ApiRequest` was built with `signal: request.signal`, read eagerly. + On workerd that first touch constructs a host object backed by C++ state — + cheap in bytes, expensive to collect — for handlers that overwhelmingly never + look at it. Hono never creates one at all. Reading it through a getter defers + the cost to the handlers that actually want it. `hookApiRequest` still reads it + eagerly: that path runs once per 500 and hands its object straight to an + `onError` reporter. + + **The getter has to be inherited, not owned.** An own accessor pushes the + object out of V8's in-object slots. The compiled engine's request object had no + accessor before this change, and gaining one took it from 852 to 1276 bytes + allocated per request inside workerd. On a shared prototype the instances stay + plain data objects and the deferral is free. Both engines get the same + treatment, as the differential corpus requires. + + **Measured, not inferred.** The README previously reported that workerd stalled + the `@amritk/api` columns far more often than Hono and guessed the cause was + allocating more per request. That guess was wrong: on the static GET the + compiled engine already allocated 852 bytes per request against bare Hono's + 1220, and turned a batch of 2048 requests around faster than Hono did. It + allocated less and ran quicker, then periodically got stopped. After the fix it + allocates 816 bytes per request and stalls on 0 of 60 batches, where before it + stalled on 5 and lost 29% of its wall clock to them. The runtime engine still + stalls and still allocates ~2172 bytes per request; that is called out in the + README as open work rather than presented as solved. + + **New: `bun run bench:workerd:allocations`.** Reads the isolate's heap over + workerd's inspector either side of a run of exactly N requests and regresses + the delta against N, so the Miniflare loopback hop lands in the intercept and + cancels; it also times fixed batches inside the isolate and reports how many + ran more than twice the median. workerd accepts `HeapProfiler.startSampling` + but answers with an empty profile, so there is no per-call-frame attribution to + be had from the runtime. `bench/run-workerd.ts` now repeats each cell across + several fresh isolates and reports the median of the per-isolate medians — + a single isolate's median is robust to a paused trial, but isolates differ from + each other by more than that, which was enough to hide effects this size. + + **The cached-`ResponseInit` comment claimed ~40% and was measured on neither + runtime it gets read on.** Measured now: on Node it is worth about 10% on the + static GET (104k vs 93k ops/s against `Response.json`); inside workerd the + cached init, a cached `Headers` instance, and plain `Response.json` are + indistinguishable. The code stays — it costs nothing anywhere and helps on one + runtime — but the comment now says so. + + All three tables were re-measured together on one machine, which is slower than + the one earlier revisions used, so the absolutes moved down across every column + at once. The README says that where the tables are. + +- ef77708: Reject `NaN` against a numeric bound, matching `@amritk/runtime-validators`. + + Bounds were emitted as their direct failure condition (`x < minimum`) rather than + the negated pass condition (`!(x >= minimum)`). The two agree on every ordinary + value and are opposite for `NaN`, which compares `false` against every operator: + the direct form read that as "not out of bounds" and let a `NaN` through + `minimum` / `maximum` / `exclusiveMinimum` / `exclusiveMaximum`, where the + interpreter and Ajv both reject it. Generated validators, strict generated + parsers, and the compiled API engine's inlined guards all now write the negated + form — so a `NaN` fails a bounded number everywhere in the toolchain. A bare + `{ type: 'number' }` with no constraint still accepts it, as Ajv does; only a + bound or `multipleOf` rejects it. + + Two internal inconsistencies close with it: `@amritk/generate-parsers` emitted the + un-negated `x >= min` in its inline matchers and the direct `x < min` in its + strict assertions, so the same schema could answer differently depending on which + path ran, and `@amritk/api`'s compiled engine disagreed with its own runtime + engine for a value the two are documented to be observationally identical on. + + `interpreter-parity.test.ts` now covers the numeric keywords — bounds, the + draft-04 boolean `exclusive*` form, and `multipleOf` across integer, fractional, + and quotient-overflowing divisors — over a value set built to separate the two + spellings (`NaN`, `±Infinity`, `1e308`, `1000000.005`). Nothing pinned these + before, which is how the drift got in. + +- 2c9982c: Fix the published manifests so the packages install, resolve, and dedupe correctly + + **Types resolve on TypeScript's default config.** Every package was + exports-only: nine declared `"module": "./dist/index.js"` (a field neither Node + nor TypeScript reads) and nothing declared `types`. A consumer on + `moduleResolution: "node10"` — still the default when `module` is `commonjs` — + cannot see `exports` at all, so `import { lintDocument } from '@amritk/lint'` + failed with `TS2307: Cannot find module '@amritk/lint' or its corresponding type +declarations`. Each package with a `.` export now also declares `main` and + `types`; `@amritk/helpers` and `@amritk/adapters` have no `.` export (they are + subpath-only), so they declare a `typesVersions` wildcard mapping instead, which + gives their subpaths the same node10 fallback. All of it is ignored under + `node16`/`nodenext`/`bundler`, where `exports` still wins. + + **`workspace:*` resolves to a caret, not an exact pin.** All fourteen + inter-package edges shipped as exact versions, so installing two `@amritk/*` + packages published at different times pulled in two copies of their shared + dependency. That is not merely wasteful: the module-level caches those packages + rely on are per-copy, so the `WeakMap` validator cache in + `@amritk/runtime-validators` silently stopped hitting. Pre-1.0 a caret stays + narrow (`^0.9.1` is `>=0.9.1 <0.10.0`) and breaking changes here already ride a + minor bump. + + **`@amritk/helpers` stops shipping 21 source files it does not need.** Embedded + mode reads four helper sources (`is-object`, `validate-array`, + `validate-record`, `has-ref`) out of the installed package at generation time, + so `src` has to ship — but only those four. `files` now lists them explicitly + instead of globbing all of `src`, cutting the tarball from 78 files / 206 kB to + 63 / 112 kB. + + **Two packages no longer declare a dependency they never import.** + `@amritk/mjst` and `@amritk/generate-parsers` both listed + `@amritk/generate-markdown` under `dependencies`, but the only importer is each + package's `scripts/generate-readme.ts`, which is not published. Both moved to + `devDependencies`. `@amritk/adapters` likewise dropped its + `@sinclair/typebox` peer dependency: the TypeBox adapter is purely structural + (it strips symbol keys) and imports nothing. `valibot` stays — it is a genuine + transitive peer of `@valibot/to-json-schema`. + + **`@amritk/mjst` fixes.** `json-schema-typed` moved to `dependencies`, because + the shipped `dist/emit-examples.d.ts` imports types from it. The package gained + an `exports` map, so it is no longer deep-importable in its entirety. And the + build now marks `dist/cli.js` executable: `npm pack` records on-disk modes, and + package managers only `chmod` bin targets when they link them, so flows that + consume the tarball directly (vendoring, Docker `npm pack` + `tar -x`) hit + `EACCES`. + +- Updated dependencies [213ecc4] +- Updated dependencies [798fd7a] +- Updated dependencies [2c9982c] +- Updated dependencies [bc09e15] +- Updated dependencies [b152c4e] +- Updated dependencies [15e480e] +- Updated dependencies [140412b] + - @amritk/runtime-validators@0.10.0 + ## 0.12.0 ### Minor Changes diff --git a/packages/api/package.json b/packages/api/package.json index ba0a8db7..ba878e50 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,7 +1,7 @@ { "name": "@amritk/api", - "version": "0.12.0", - "description": "Framework-agnostic, contract-first API layer built on mjst JSON Schema tooling. Typed routes, fast request/response validation, and OpenAPI 3.1 generation with no extra code \u2014 adapters for fetch (Hono, Next.js, Bun, Workers) and Node (Express, Fastify).", + "version": "0.13.0", + "description": "Framework-agnostic, contract-first API layer built on mjst JSON Schema tooling. Typed routes, fast request/response validation, and OpenAPI 3.1 generation with no extra code — adapters for fetch (Hono, Next.js, Bun, Workers) and Node (Express, Fastify).", "main": "./dist/index.js", "module": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index c4a36998..c4710755 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,188 @@ # @amritk/mjst +## 0.14.0 + +### Minor Changes + +- 2c08493: Add `--allowed-roots` so a split spec can reach a sibling directory again + + `@amritk/resolve-refs` now confines a local `$ref` to the directory holding the + document it appears in, which closed a real path-traversal hole + (`{"$ref": "/etc/passwd"}` used to be read and inlined). The CLI inherited that + default with no way to widen it, so a completely ordinary multi-version layout — + `specs/v1/api.json` referencing `../common/user.json` — started failing, and the + error told the user to "set allowedRoots", a _library_ option nothing on the + command line could reach. + + `--allowed-roots ` is that escape hatch, on both `mjst generate` and + `mjst lint`, alongside an `allowedRoots` config-file key. On the generate path it + takes a comma-separated list or the flag repeated (matching `--allowed-hosts`); + on `lint` you repeat the flag (matching its `--allowed-hosts`). Relative entries + resolve against the current working directory, from a config file as readily as + from the flag, which is how `schema` and `outDir` already behave. + + Two things it deliberately does not do. It does not replace the default: the + schema's (or linted document's) own directory stays allowed, so naming a shared + `common/` folder cannot revoke the one directory nobody would think to list. And + it does not widen anything on its own — there is no implicit default drawn from + the config file's location, because a config file usually sits at the repo root + and quietly granting read access to the whole project tree is not a decision + anyone would read into `--config`. A `$ref` that lands outside every named root + is still refused. + + Refusals now name the flag that exists (`pass --allowed-roots …`) instead + of leaving the library's option name as the only lead. + +- a8f96c9: Stop the CLI from succeeding quietly, and make generation safe to write + + **`mjst lint` no longer exits 0 when its file arguments match nothing.** A glob + or path that resolved to zero files fell through to the stdin branch; in CI + there is no TTY, so stdin is an empty pipe and the linter dutifully reported + "No problems found" on an empty document and exited 0. A typo'd path turned a + lint gate into a silent no-op that reported success. Document arguments that + match nothing now exit 2 with `No files matched: …`, and only a run with no + document arguments at all reads stdin. + + **`mjst lint` rejects unknown flags.** yargs was built without `.strict()`, so + `--bogus-flag`, a mistyped `--fail-severity`, or a misspelled `--allowed-hosts` + was dropped and the run silently used the defaults — the opposite of the + generate command's deliberate strictness. A non-numeric `--concurrency` now + reports what is wrong instead of crashing with `Invalid array length`. + + **Generation never clobbers a file it did not write.** A generated name that + collided with a hand-written file (`index.ts` is the common one) overwrote it + without a word, and `--build` then deleted it along with the other intermediate + sources. Each run records what it wrote in a `.mjst-manifest.json` at the root + of the output directory: paths listed there are reclaimed freely, so + regenerating still needs no ceremony, while anything else aborts the run before + a byte is written. The new `--force` flag opts out. This covers every output the + CLI produces — the parser tree, `--validators`, `--examples`, and `--out-file`, + which is the one most likely to be aimed at hand-written source + (`--out-file src/types.ts` used to overwrite that file silently and, under + `--build`, delete it afterwards). For `--out-file` the manifest lands in the + directory holding the file, alongside the `--build` output and any generated + examples. + + **Generation is atomic.** Files are staged under temporary names and renamed + into place only once the whole set has been written, so a mid-run failure (a + `_helpers` path occupied by a regular file, a full disk) leaves the output + directory exactly as it found it instead of a half-generated tree. + + **`--root-type` is validated, and writes are confined to the output + directory.** `--root-type '../../Escaped'` — from the command line or from a + config file — wrote outside `--out-dir` and emitted a type name that could not + compile. The name must now be a TypeScript identifier, and the writer + independently refuses any path that resolves outside the output directory. + + **Config files are validated.** Every key was guarded by a `typeof` check whose + failure branch was "drop it", so `{"strcit": true, "strict": "true"}` generated + non-strict output and exited 0 while the same typo on the command line was + rejected. Unknown keys and wrong types now fail with the offending pointer and + the expected type, and `config.schema.json` closes the object with + `additionalProperties: false`. + + **`-v` / `-h` / `--version` / `--help` are only honored in flag position.** Both + predicates scanned the whole argv, so any flag whose _value_ was `-v` (say + `--type-suffix -v`) printed the version, generated nothing, and exited 0. + + Smaller argument-parsing fixes: `--` is accepted as the end-of-flags terminator + instead of being rejected as an unknown flag; `--config` with no value is an + error rather than a silently skipped config file; a missing config file reports + `Config file not found: …` instead of a raw `ENOENT`; an `--out-dir` pointing at + an existing file explains itself instead of surfacing `EEXIST … mkdir`; a stray + positional (`mjst genrate --schema …`) is rejected rather than quietly generating + as if the typo were not there; and `--build --types-only` no longer claims to + have built `.js` files that were never emitted. + +- 2c9982c: Fix the published manifests so the packages install, resolve, and dedupe correctly + + **Types resolve on TypeScript's default config.** Every package was + exports-only: nine declared `"module": "./dist/index.js"` (a field neither Node + nor TypeScript reads) and nothing declared `types`. A consumer on + `moduleResolution: "node10"` — still the default when `module` is `commonjs` — + cannot see `exports` at all, so `import { lintDocument } from '@amritk/lint'` + failed with `TS2307: Cannot find module '@amritk/lint' or its corresponding type +declarations`. Each package with a `.` export now also declares `main` and + `types`; `@amritk/helpers` and `@amritk/adapters` have no `.` export (they are + subpath-only), so they declare a `typesVersions` wildcard mapping instead, which + gives their subpaths the same node10 fallback. All of it is ignored under + `node16`/`nodenext`/`bundler`, where `exports` still wins. + + **`workspace:*` resolves to a caret, not an exact pin.** All fourteen + inter-package edges shipped as exact versions, so installing two `@amritk/*` + packages published at different times pulled in two copies of their shared + dependency. That is not merely wasteful: the module-level caches those packages + rely on are per-copy, so the `WeakMap` validator cache in + `@amritk/runtime-validators` silently stopped hitting. Pre-1.0 a caret stays + narrow (`^0.9.1` is `>=0.9.1 <0.10.0`) and breaking changes here already ride a + minor bump. + + **`@amritk/helpers` stops shipping 21 source files it does not need.** Embedded + mode reads four helper sources (`is-object`, `validate-array`, + `validate-record`, `has-ref`) out of the installed package at generation time, + so `src` has to ship — but only those four. `files` now lists them explicitly + instead of globbing all of `src`, cutting the tarball from 78 files / 206 kB to + 63 / 112 kB. + + **Two packages no longer declare a dependency they never import.** + `@amritk/mjst` and `@amritk/generate-parsers` both listed + `@amritk/generate-markdown` under `dependencies`, but the only importer is each + package's `scripts/generate-readme.ts`, which is not published. Both moved to + `devDependencies`. `@amritk/adapters` likewise dropped its + `@sinclair/typebox` peer dependency: the TypeBox adapter is purely structural + (it strips symbol keys) and imports nothing. `valibot` stays — it is a genuine + transitive peer of `@valibot/to-json-schema`. + + **`@amritk/mjst` fixes.** `json-schema-typed` moved to `dependencies`, because + the shipped `dist/emit-examples.d.ts` imports types from it. The package gained + an `exports` map, so it is no longer deep-importable in its entirety. And the + build now marks `dist/cli.js` executable: `npm pack` records on-disk modes, and + package managers only `chmod` bin targets when they link them, so flows that + consume the tarball directly (vendoring, Docker `npm pack` + `tar -x`) hit + `EACCES`. + +### Patch Changes + +- Updated dependencies [a342117] +- Updated dependencies [f5a52b7] +- Updated dependencies [365c6c1] +- Updated dependencies [299ed2a] +- Updated dependencies [2eed2e5] +- Updated dependencies [d989bc4] +- Updated dependencies [de0952c] +- Updated dependencies [213ecc4] +- Updated dependencies [9cb45a0] +- Updated dependencies [ef77708] +- Updated dependencies [5afbfd4] +- Updated dependencies [eb80ca6] +- Updated dependencies [798fd7a] +- Updated dependencies [874856f] +- Updated dependencies [2c9982c] +- Updated dependencies [f439570] +- Updated dependencies [fa8620c] +- Updated dependencies [cb0ef39] +- Updated dependencies [08b2833] +- Updated dependencies [d749ee2] +- Updated dependencies [945e8f2] +- Updated dependencies [f9f790a] +- Updated dependencies [0d4bed2] +- Updated dependencies [947d44a] +- Updated dependencies [7757788] +- Updated dependencies [7839a38] +- Updated dependencies [007aa05] +- Updated dependencies [1b720e2] +- Updated dependencies [c1a176f] +- Updated dependencies [00eb0c9] + - @amritk/api@0.13.0 + - @amritk/generate-examples@0.6.0 + - @amritk/generate-validators@0.12.0 + - @amritk/generate-parsers@0.19.0 + - @amritk/helpers@0.15.0 + - @amritk/resolve-refs@0.5.0 + - @amritk/lint@0.4.4 + - @amritk/adapters@0.4.0 + - @amritk/yaml@0.5.0 + ## 0.13.9 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index 96a0e23a..50c98867 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@amritk/mjst", - "version": "0.13.9", + "version": "0.14.0", "description": "Generate TypeScript parsers and type definitions from JSON Schemas.", "type": "module", "engines": { diff --git a/packages/generate-examples/CHANGELOG.md b/packages/generate-examples/CHANGELOG.md index ea8148b5..847daa6a 100644 --- a/packages/generate-examples/CHANGELOG.md +++ b/packages/generate-examples/CHANGELOG.md @@ -1,5 +1,179 @@ # @amritk/generate-examples +## 0.6.0 + +### Minor Changes + +- eb80ca6: Fix `$ref`-graph naming and reference resolution, and stop degrading silently. + + Generation now fails loudly instead of writing output that cannot work: + + - Two definitions that reduce to one filename (`Pet`/`pet`) or one type name + (`foo-bar`/`foo.bar`/`fooBar` all become `FooBar`) are an error. The filename + case used to drop one definition and give every reference to it the other + one's shape; the type-name case emitted both files and left the importer with + two `import { FooBar }` lines that do not parse. + - An unresolvable `$ref` is an error. It used to warn while the generators still + emitted the type name and the parser/validator call for a file that was never + written. + - A `$dynamicRef` with no `$dynamicAnchor` to bind to is an error. Leaving it in + place made the type generator name the type after the anchor, so the canonical + recursive-tree idiom (`$dynamicAnchor: "node"`) produced a reference to the + DOM's `Node` interface — a clean compile with the wrong type. + - A document that relies on `$id` base-URI scoping is rejected rather than + resolving its inner fragments against the document root and silently selecting + a different definition. + - Every recursive schema walker enforces a nesting cap and reports it by name + instead of dying with a bare stack-overflow. + + And several things that were broken now work: + + - Non-ASCII definition names (CJK, Cyrillic, accented) keep their characters + instead of collapsing onto the single type name `_`, and the generated + `index.ts` barrel re-exports them correctly. + - A root-level `$dynamicAnchor` is generated as the root's own file, so the + 2020-12 recursive-tree idiom produces a real self-referencing type. + - A plain `$anchor` ref (`$ref: "#named"`) resolves, instead of producing the + unloadable import specifier `'./#named.ts'`. + - Derived filenames are normalized: no more `.ts`, `...ts`, `http:--x.ts`, or + characters Windows and ESM specifiers reject. `$ref: "#/__proto__"` no longer + resolves to `Object.prototype`. + - Generated readers guard `Object.prototype` member names (`constructor`, + `toString`, `__proto__`, …) with `Object.hasOwn`, so a schema with a + `constructor` property no longer fails its own shape check for every valid + object, and the parser no longer fabricates a `__proto__` key. + - `x-mjst` `instanceOf` is allow-listed to the classes the generators support, + so an arbitrary identifier is warned about and ignored instead of being + emitted verbatim into the output. + +### Patch Changes + +- d989bc4: Make generation linear in the `$ref` graph, and stop shipping examples that fail their own schema + + **A `$ref` reachable by several paths is derived once, not once per path.** + `deriveExample` tracked visited refs in a _path-scoped_ set with no memo table, + so every fan-out in the definition graph re-expanded the same subtree + exponentially: a 25-definition graph with three refs per definition took ~20 + seconds, and adding two more definitions roughly quadrupled that. Derivation is + now memoized per ref per root document — the pattern + `@amritk/helpers/walk-ref-graph` already uses for `resolveRef` — with the cycle + guard kept exact: a value produced by cutting a cycle is deliberately not + memoized, so a recursive definition still terminates at the same place. The same + graph at 400 definitions now derives in ~7 ms. + + **A validating check no longer carries the whole document.** Every check spliced + the root's entire `$defs` into the schema it validated, and the interpreter + screens each `pattern` in whatever it is handed — so a 959-definition OpenAPI + document paid for all 959 definitions on each of the thousand-odd checks a + generation run makes, and embedded the whole document into every generated file + carrying a validating filter. Only the definitions a schema's `$ref`s actually + reach travel with it now (a reference that cannot be pinned to one definition — + an `$anchor` name, or a `$dynamicRef`/`$recursiveRef`, whose target is picked + from the dynamic scope at validation time — still falls back to the full set). + Generating the OpenAI corpus went + from ~3.6 s to ~0.3 s, and its generated output from 119 MB to 2.7 MB. + + **Generated arbitraries compile under a strict tsconfig.** `fc.constantFrom("a", +"b")` infers `Arbitrary`, which does not fit the `"a" | "b"` the + generated type declares — so _any_ schema with an `enum` property produced a + file no consumer on `strict` could build. Scalar members are now spread from a + `const`-asserted tuple, and a filtered arbitrary's predicate is written as a + type guard (`(value): value is Foo => …`), which is what it has always been: the + combinators generate a superset and the runtime validator narrows it. A new + suite type-checks generated files against the real `fast-check` declarations + under `strict`, `exactOptionalPropertyTypes`, and `noUncheckedIndexedAccess`. + + **An example that fails its own schema is now reported instead of shipped + quietly.** Every `fooExample` is validated (formats included) before it is + written; a value that does not satisfy the schema is still emitted, so the + module compiles, but the generator warns and names the type. Several cases that + used to fail silently now produce valid values: `not` gets a perturbation + candidate (`not: { const: 'string' }` no longer returns `"string"`), + `uniqueItems` over a closed value set walks the set instead of suffixing a + string out of its own `enum`, `pattern` sampling honours `minLength` and reads + control escapes (`a\nb` produced `"anb"`), and static examples now cover every + `format` `@amritk/runtime-validators` checks — `duration`, `json-pointer`, + `relative-json-pointer`, `uri-template`, `uri-reference`, `regex`, and the `idn-` + /`iri` variants. A key that `additionalProperties: false` forbids is no longer + invented. The remaining limits are written down in the README. + + **A `__proto__` property survives.** Both the derived value (`out[key] = …` hit + `Object.prototype`'s prototype setter, so the key vanished) and the emitted + source (a _quoted_ `"__proto__":` in an object literal is still the setter, in + the example value and in the `fc.record` config — where it also reassigned the + config object's prototype to an `Arbitrary`). The value uses `defineProperty` + and the source uses the computed `["__proto__"]:` form, matching what + `generate-parsers` already does. + + **A schema the validator refuses no longer kills the run — and no longer goes + unmentioned.** A `$ref` pointing outside the document (`#/components/schemas/…` + in a bare fragment) threw out of `buildExampleSchema`. Those checks are opinions + about a candidate value, so an undecidable schema now abstains. It also warns + once, naming the schema and the reason, because a filter that switches itself + off silently is indistinguishable from one that ran and approved of everything. + +- 2c9982c: Fix the published manifests so the packages install, resolve, and dedupe correctly + + **Types resolve on TypeScript's default config.** Every package was + exports-only: nine declared `"module": "./dist/index.js"` (a field neither Node + nor TypeScript reads) and nothing declared `types`. A consumer on + `moduleResolution: "node10"` — still the default when `module` is `commonjs` — + cannot see `exports` at all, so `import { lintDocument } from '@amritk/lint'` + failed with `TS2307: Cannot find module '@amritk/lint' or its corresponding type +declarations`. Each package with a `.` export now also declares `main` and + `types`; `@amritk/helpers` and `@amritk/adapters` have no `.` export (they are + subpath-only), so they declare a `typesVersions` wildcard mapping instead, which + gives their subpaths the same node10 fallback. All of it is ignored under + `node16`/`nodenext`/`bundler`, where `exports` still wins. + + **`workspace:*` resolves to a caret, not an exact pin.** All fourteen + inter-package edges shipped as exact versions, so installing two `@amritk/*` + packages published at different times pulled in two copies of their shared + dependency. That is not merely wasteful: the module-level caches those packages + rely on are per-copy, so the `WeakMap` validator cache in + `@amritk/runtime-validators` silently stopped hitting. Pre-1.0 a caret stays + narrow (`^0.9.1` is `>=0.9.1 <0.10.0`) and breaking changes here already ride a + minor bump. + + **`@amritk/helpers` stops shipping 21 source files it does not need.** Embedded + mode reads four helper sources (`is-object`, `validate-array`, + `validate-record`, `has-ref`) out of the installed package at generation time, + so `src` has to ship — but only those four. `files` now lists them explicitly + instead of globbing all of `src`, cutting the tarball from 78 files / 206 kB to + 63 / 112 kB. + + **Two packages no longer declare a dependency they never import.** + `@amritk/mjst` and `@amritk/generate-parsers` both listed + `@amritk/generate-markdown` under `dependencies`, but the only importer is each + package's `scripts/generate-readme.ts`, which is not published. Both moved to + `devDependencies`. `@amritk/adapters` likewise dropped its + `@sinclair/typebox` peer dependency: the TypeBox adapter is purely structural + (it strips symbol keys) and imports nothing. `valibot` stays — it is a genuine + transitive peer of `@valibot/to-json-schema`. + + **`@amritk/mjst` fixes.** `json-schema-typed` moved to `dependencies`, because + the shipped `dist/emit-examples.d.ts` imports types from it. The package gained + an `exports` map, so it is no longer deep-importable in its entirety. And the + build now marks `dist/cli.js` executable: `npm pack` records on-disk modes, and + package managers only `chmod` bin targets when they link them, so flows that + consume the tarball directly (vendoring, Docker `npm pack` + `tar -x`) hit + `EACCES`. + +- Updated dependencies [213ecc4] +- Updated dependencies [9cb45a0] +- Updated dependencies [5afbfd4] +- Updated dependencies [eb80ca6] +- Updated dependencies [798fd7a] +- Updated dependencies [2c9982c] +- Updated dependencies [f439570] +- Updated dependencies [fa8620c] +- Updated dependencies [bc09e15] +- Updated dependencies [b152c4e] +- Updated dependencies [15e480e] +- Updated dependencies [140412b] + - @amritk/helpers@0.15.0 + - @amritk/runtime-validators@0.10.0 + ## 0.5.6 ### Patch Changes diff --git a/packages/generate-examples/package.json b/packages/generate-examples/package.json index 9e2d554a..78836f40 100644 --- a/packages/generate-examples/package.json +++ b/packages/generate-examples/package.json @@ -1,6 +1,6 @@ { "name": "@amritk/generate-examples", - "version": "0.5.6", + "version": "0.6.0", "description": "Generate fast-check arbitraries and example values from JSON Schemas.", "main": "./dist/index.js", "module": "./dist/index.js", diff --git a/packages/generate-markdown/CHANGELOG.md b/packages/generate-markdown/CHANGELOG.md index f0998578..6619d8c1 100644 --- a/packages/generate-markdown/CHANGELOG.md +++ b/packages/generate-markdown/CHANGELOG.md @@ -1,5 +1,56 @@ # @amritk/generate-markdown +## 0.4.6 + +### Patch Changes + +- 2c9982c: Fix the published manifests so the packages install, resolve, and dedupe correctly + + **Types resolve on TypeScript's default config.** Every package was + exports-only: nine declared `"module": "./dist/index.js"` (a field neither Node + nor TypeScript reads) and nothing declared `types`. A consumer on + `moduleResolution: "node10"` — still the default when `module` is `commonjs` — + cannot see `exports` at all, so `import { lintDocument } from '@amritk/lint'` + failed with `TS2307: Cannot find module '@amritk/lint' or its corresponding type +declarations`. Each package with a `.` export now also declares `main` and + `types`; `@amritk/helpers` and `@amritk/adapters` have no `.` export (they are + subpath-only), so they declare a `typesVersions` wildcard mapping instead, which + gives their subpaths the same node10 fallback. All of it is ignored under + `node16`/`nodenext`/`bundler`, where `exports` still wins. + + **`workspace:*` resolves to a caret, not an exact pin.** All fourteen + inter-package edges shipped as exact versions, so installing two `@amritk/*` + packages published at different times pulled in two copies of their shared + dependency. That is not merely wasteful: the module-level caches those packages + rely on are per-copy, so the `WeakMap` validator cache in + `@amritk/runtime-validators` silently stopped hitting. Pre-1.0 a caret stays + narrow (`^0.9.1` is `>=0.9.1 <0.10.0`) and breaking changes here already ride a + minor bump. + + **`@amritk/helpers` stops shipping 21 source files it does not need.** Embedded + mode reads four helper sources (`is-object`, `validate-array`, + `validate-record`, `has-ref`) out of the installed package at generation time, + so `src` has to ship — but only those four. `files` now lists them explicitly + instead of globbing all of `src`, cutting the tarball from 78 files / 206 kB to + 63 / 112 kB. + + **Two packages no longer declare a dependency they never import.** + `@amritk/mjst` and `@amritk/generate-parsers` both listed + `@amritk/generate-markdown` under `dependencies`, but the only importer is each + package's `scripts/generate-readme.ts`, which is not published. Both moved to + `devDependencies`. `@amritk/adapters` likewise dropped its + `@sinclair/typebox` peer dependency: the TypeBox adapter is purely structural + (it strips symbol keys) and imports nothing. `valibot` stays — it is a genuine + transitive peer of `@valibot/to-json-schema`. + + **`@amritk/mjst` fixes.** `json-schema-typed` moved to `dependencies`, because + the shipped `dist/emit-examples.d.ts` imports types from it. The package gained + an `exports` map, so it is no longer deep-importable in its entirety. And the + build now marks `dist/cli.js` executable: `npm pack` records on-disk modes, and + package managers only `chmod` bin targets when they link them, so flows that + consume the tarball directly (vendoring, Docker `npm pack` + `tar -x`) hit + `EACCES`. + ## 0.4.5 ### Patch Changes diff --git a/packages/generate-markdown/package.json b/packages/generate-markdown/package.json index 400639e8..8b1301ac 100644 --- a/packages/generate-markdown/package.json +++ b/packages/generate-markdown/package.json @@ -1,6 +1,6 @@ { "name": "@amritk/generate-markdown", - "version": "0.4.5", + "version": "0.4.6", "description": "Generate markdown documentation from JSON Schemas.", "main": "./dist/index.js", "module": "./dist/index.js", diff --git a/packages/generate-parsers/CHANGELOG.md b/packages/generate-parsers/CHANGELOG.md index 35e51e1a..2eac0571 100644 --- a/packages/generate-parsers/CHANGELOG.md +++ b/packages/generate-parsers/CHANGELOG.md @@ -1,5 +1,498 @@ # @amritk/generate-parsers +## 0.19.0 + +### Minor Changes + +- de0952c: Make both generators agree with Draft 2020-12 — and with each other — on + structural equality, tuple `items`, `oneOf`, and prototype-member property names. + + The two generators disagreed with Ajv and with one another on the same schemas. + Where they differed, `generate-validators` was usually right: it already shipped + `valuesEqual` / `allUnique`, which `generate-parsers` never adopted. + + **`@amritk/generate-validators`** + + - A property named `__proto__` was silently dropped. The `nullable` rewrite runs + over _every_ schema and copied `properties` with a plain assignment, which fires + the `Object.prototype` setter instead of creating a key — so + `{"properties":{"__proto__":{"type":"string","minLength":3}}}` emitted a + validator with no checks at all, and `required: ["__proto__"]` degraded to a + bare presence check. A validation bypass. + - `constructor` / `toString` / `hasOwnProperty` properties were read straight off + the object, so the _prototype's_ value answered: a valid document was reported + as `must be string` at `/hasOwnProperty`, and a required `toString` could never + be reported missing (`'toString' in obj` is always true). Reads now go through + an own-property guard, and presence uses `Object.hasOwn` — but only for the + names that can actually be inherited. Every other key keeps the plain `in` it + always had, because `Object.hasOwn` is a call the engine cannot fold the way it + folds `in`, and spending it on `id` or `name` bought nothing while costing + roughly half the throughput on an all-present object. + - `items` alongside `prefixItems` was applied to the prefix positions too. Per + 2020-12 `items` is the tail schema, so `{prefixItems:[{type:'string'}], +items:{type:'number'}}` rejected `["a", 1, 2]` — which Ajv accepts, and which + the `[string?, ...number[]]` type this generator emits already admits. + - `enum` members that are objects or arrays could never match: `.includes` is + SameValueZero, i.e. reference equality. `enum` now compares structurally via + `valuesEqual`, the way `const` always has — which also makes `isX` a sound type + guard again. + + **`@amritk/generate-parsers`** + + - The same `prefixItems` + `items` defect, in both the fast path and the strict + assertion. + - The same `enum`-with-object-members defect. Members are now compared by an + unrolled structural check against the known literal. + - `const` deep equality used `JSON.stringify`, which is key-order sensitive, so + `{b: 2, a: 1}` was rejected against `const: {a: 1, b: 2}` — and it serialized the + whole value on every call to do it. + - `uniqueItems` used a `JSON.stringify` dedupe key (or a bare `Set`, which + compares objects by reference), so `[{a:1,b:2},{b:2,a:1}]` was accepted where + Ajv and `generate-validators` both reject. It now projects through a + key-order-independent canonical form when items may be structural, keeping the + cheap native `Set` when they are provably scalar. A root array of objects + skipped the constraint entirely and now enforces it. + - `oneOf` exclusivity was not enforced — both `oneOf` and `anyOf` compiled to a + plain disjunction, so a value matching two branches was accepted. `oneOf` now + requires exactly one match. + - The array fallback was a bare `[]`, ignoring `prefixItems` and `minItems`. It is + not an instance of its own schema, and against a required closed tuple it is not + even assignable — `TS2322: Type '[]' is not assignable to type '[string, +number]'` made non-strict and `readonly` output fail to compile. + - A schema property named after an `Object.prototype` member produced an + unsatisfiable type: TypeScript reads the inherited `constructor: Function` on the + fallback object literal and rejects it against `constructor?: string`. + + Both packages now type-check their generated output under the repo's real + compiler flags (`exactOptionalPropertyTypes`, `noUncheckedIndexedAccess`), not + `strict` alone — `generate-validators` had no such suite at all — and both pin the + semantics above against Ajv (or, for prototype-member names, against + `@amritk/runtime-validators`, since Ajv has those bugs itself). + +- 213ecc4: Take documents you already loaded, so a `$ref` to another document generates + + **On the official JSON Schema Test Suite: `generate-validators` 1238 → 1268 / + 1281 (99.0%), `generate-parsers` 1222 → 1237 / 1281 (96.6%).** + + Both generators gain a `schemas` option: documents you have already loaded, keyed + by the absolute URI a `$ref` names them by. It is the build-time counterpart of + `@amritk/runtime-validators`' `ValidateOptions.schemas`, and it keeps the same + promise — nothing is fetched, you cannot pass a URL, only a document. What changes + is that "we do no I/O" no longer also means "we cannot be told". + + A cross-document `$ref` was the single largest gap in both packages, and it is + gone. `refRemote.json` passes in full; so do the `dynamicRef.json` groups that + reach `tree.json` and `extendible-dynamic-ref.json`, and — with the dialect + metaschema registered — `defs.json` and `ref.json`'s "remote ref, containing refs + itself". + + Each registered document becomes a resource of the document being generated: its + `$id`, its `$anchor`s and `$dynamicAnchor`s and its own embedded resources all + resolve, a `$ref` from one registered document into another resolves, and every + definition reached gets a file, a type and a validator/parser by the ordinary + rules. A document with no `$id` resolves its relative `$ref`s against the URI it + was registered under; one whose `$id` disagrees answers to both. Registering more + than the schema uses costs nothing — only the documents actually reached are + emitted — and a `$ref` to a URI nobody registered still stops the build with a + message naming the ref. + + The mechanism is one pass, not a second addressing mode. `@amritk/helpers` gains + `graftExternalSchemas`, which embeds the registered documents into the root before + the `$id` pass, and `pruneExternalSchemas`, which drops the unreferenced ones once + the refs are pointers and reachability is finally knowable. Everything downstream — + the ref-graph walk, the naming, the emitted import graph — keeps working on a + single document and needed no change. `walkRefGraph` carries the option and + memoizes per `(schema, schemas)` by identity. + + **Fixed: a root schema with a union `type` dropped every sibling constraint.** + `{ type: ['object', 'boolean'], properties: {…}, required: [...] }` emitted the + type check and nothing else, so it accepted any object at all. The multi-type root + branch now emits the shared constraint checks the single-type and combinator + branches already did; they carry their own runtime-type guards, so a member of the + union a constraint does not apply to is still untouched. This is the shape the + 2020-12 metaschema's own root is written in, which is how it went unnoticed — the + generated dialect validator accepted `{ type: 1 }` as a valid schema. + + `@amritk/runtime-validators` is unchanged in behaviour; its conformance figures are + restated against the corpus that is actually vendored (1281 cases, not 1299 — the + README's count never matched, and upstream's `content.json` is not among the + vendored files). The suite's `remotes/` loader moves to the shared fixtures + bookkeeping so all four conformance suites use one walk. + +- 5afbfd4: Resolve `$ref` against `$id` as a base URI in the ref graph + + A `$ref` written against an enclosing `$id` — a relative URI (`list`, + `folderInteger.json`), an absolute one, or a URN — used to either stop generation + or, worse, find _a_ definition and generate against it, so the emitted parser + enforced a schema its author did not write. On the official JSON Schema Test + Suite, strict-parser generation goes from 1180/1299 to **1222 / 1299 (94.1%)**; + all 29 of the resolve-to-the-wrong-definition cases became right rather than + refused. + + Three new pieces in `@amritk/helpers`, deliberately free of any parser or + validator concepts: + + - **`build-resource-registry`** — one walk producing the document's embedded + resources, anchors and dynamic anchors, each `$id` composed against the base of + its parent. Keyed by JSON Pointer, because that is the currency the rest of the + package already deals in — a registry hit turns straight into a `$ref` string, a + filename, or a type name. Returns `null` for a document with no `$id`, which is + the fast-path switch, and is memoized per document. + - **`resolve-scoped-ref`** — one call covering relative, absolute, absolute-path, + URN, pointer-into-resource and anchor-in-resource forms, plus the plain + `#/$defs/x` that under an enclosing `$id` means _that resource's_ `$defs`. + - **`normalize-ref-scopes`** — rewrites every `$ref`/`$dynamicRef` to a + document-root pointer. This is the leverage: everything downstream already + resolves refs by string against the root, so one normalization makes ref + resolution, type naming, the import graph and the strict matcher correct at once, + none of them needing base-URI awareness of their own. + + It is wired into `walkRefGraph`, so `@amritk/generate-validators` and + `@amritk/generate-examples` inherit it. + + `assertIdScopes` keeps its name and signature but changes meaning: it no longer + refuses any document with nested `$id` scoping, only the residue base-URI + resolution cannot place — a fragment ref inside an embedded resource that declares + its own targets and names none of them. That preserves the property worth having: + never silently pick the outer definition. + + `@amritk/generate-parsers` additionally follows the spec on `contains` next to + `unevaluatedItems`: only the items `contains` matched are evaluated, not the whole + array. That is what `@amritk/runtime-validators` does, so the two stop disagreeing + about the same schema. Ajv marks the whole array, so the single fuzz fragment + pairing those keywords leaves the Ajv-oracle corpus (with the reason recorded next + to it) and unit tests plus the conformance suite cover it instead; every other + `contains` and `unevaluated*` fragment keeps fuzzing. + +- eb80ca6: Fix `$ref`-graph naming and reference resolution, and stop degrading silently. + + Generation now fails loudly instead of writing output that cannot work: + + - Two definitions that reduce to one filename (`Pet`/`pet`) or one type name + (`foo-bar`/`foo.bar`/`fooBar` all become `FooBar`) are an error. The filename + case used to drop one definition and give every reference to it the other + one's shape; the type-name case emitted both files and left the importer with + two `import { FooBar }` lines that do not parse. + - An unresolvable `$ref` is an error. It used to warn while the generators still + emitted the type name and the parser/validator call for a file that was never + written. + - A `$dynamicRef` with no `$dynamicAnchor` to bind to is an error. Leaving it in + place made the type generator name the type after the anchor, so the canonical + recursive-tree idiom (`$dynamicAnchor: "node"`) produced a reference to the + DOM's `Node` interface — a clean compile with the wrong type. + - A document that relies on `$id` base-URI scoping is rejected rather than + resolving its inner fragments against the document root and silently selecting + a different definition. + - Every recursive schema walker enforces a nesting cap and reports it by name + instead of dying with a bare stack-overflow. + + And several things that were broken now work: + + - Non-ASCII definition names (CJK, Cyrillic, accented) keep their characters + instead of collapsing onto the single type name `_`, and the generated + `index.ts` barrel re-exports them correctly. + - A root-level `$dynamicAnchor` is generated as the root's own file, so the + 2020-12 recursive-tree idiom produces a real self-referencing type. + - A plain `$anchor` ref (`$ref: "#named"`) resolves, instead of producing the + unloadable import specifier `'./#named.ts'`. + - Derived filenames are normalized: no more `.ts`, `...ts`, `http:--x.ts`, or + characters Windows and ESM specifiers reject. `$ref: "#/__proto__"` no longer + resolves to `Object.prototype`. + - Generated readers guard `Object.prototype` member names (`constructor`, + `toString`, `__proto__`, …) with `Object.hasOwn`, so a schema with a + `constructor` property no longer fails its own shape check for every valid + object, and the parser no longer fabricates a `__proto__` key. + - `x-mjst` `instanceOf` is allow-listed to the classes the generators support, + so an arbitrary identifier is warned about and ignored instead of being + emitted verbatim into the output. + +- 2c9982c: Fix the published manifests so the packages install, resolve, and dedupe correctly + + **Types resolve on TypeScript's default config.** Every package was + exports-only: nine declared `"module": "./dist/index.js"` (a field neither Node + nor TypeScript reads) and nothing declared `types`. A consumer on + `moduleResolution: "node10"` — still the default when `module` is `commonjs` — + cannot see `exports` at all, so `import { lintDocument } from '@amritk/lint'` + failed with `TS2307: Cannot find module '@amritk/lint' or its corresponding type +declarations`. Each package with a `.` export now also declares `main` and + `types`; `@amritk/helpers` and `@amritk/adapters` have no `.` export (they are + subpath-only), so they declare a `typesVersions` wildcard mapping instead, which + gives their subpaths the same node10 fallback. All of it is ignored under + `node16`/`nodenext`/`bundler`, where `exports` still wins. + + **`workspace:*` resolves to a caret, not an exact pin.** All fourteen + inter-package edges shipped as exact versions, so installing two `@amritk/*` + packages published at different times pulled in two copies of their shared + dependency. That is not merely wasteful: the module-level caches those packages + rely on are per-copy, so the `WeakMap` validator cache in + `@amritk/runtime-validators` silently stopped hitting. Pre-1.0 a caret stays + narrow (`^0.9.1` is `>=0.9.1 <0.10.0`) and breaking changes here already ride a + minor bump. + + **`@amritk/helpers` stops shipping 21 source files it does not need.** Embedded + mode reads four helper sources (`is-object`, `validate-array`, + `validate-record`, `has-ref`) out of the installed package at generation time, + so `src` has to ship — but only those four. `files` now lists them explicitly + instead of globbing all of `src`, cutting the tarball from 78 files / 206 kB to + 63 / 112 kB. + + **Two packages no longer declare a dependency they never import.** + `@amritk/mjst` and `@amritk/generate-parsers` both listed + `@amritk/generate-markdown` under `dependencies`, but the only importer is each + package's `scripts/generate-readme.ts`, which is not published. Both moved to + `devDependencies`. `@amritk/adapters` likewise dropped its + `@sinclair/typebox` peer dependency: the TypeBox adapter is purely structural + (it strips symbol keys) and imports nothing. `valibot` stays — it is a genuine + transitive peer of `@valibot/to-json-schema`. + + **`@amritk/mjst` fixes.** `json-schema-typed` moved to `dependencies`, because + the shipped `dist/emit-examples.d.ts` imports types from it. The package gained + an `exports` map, so it is no longer deep-importable in its entirety. And the + build now marks `dist/cli.js` executable: `npm pack` records on-disk modes, and + package managers only `chmod` bin targets when they link them, so flows that + consume the tarball directly (vendoring, Docker `npm pack` + `tar -x`) hit + `EACCES`. + +- fa8620c: Stop strict parsers accepting what the schema forbids: prototype-inherited + `required`, undiscriminated unions, and `$ref` fragments that were never decoded + + Measured against the official JSON Schema Test Suite, strict-mode generation goes + from 1141/1299 to **1180/1299 (90.8%)**. Four defects, all of them cases where the + generated parser said yes to a document the schema says no to — or refused one it + should have taken. + + - **`required` compiled to `in`, which walks the prototype chain.** `"toString" in {}` + is `true`, so `{ "required": ["__proto__", "toString", "constructor"] }` was + satisfied by an object carrying none of them. `Object.hasOwn` now covers the + names an object can actually inherit, and plain `in` stays everywhere else so + ordinary keys keep the form the engine can fold — the same split + `@amritk/generate-validators` already made. Generated output for ordinary keys is + byte-identical. Applies to `required`, `dependentRequired`, `dependentSchemas`, + and `false`-property absence. + - **A `oneOf`/`anyOf`/`allOf` whose branches carry no `type` compiled to a + pass-through.** `{ "oneOf": [{ "type": "integer" }, { "minimum": 2 }] }` emitted + `parseRoot = (input) => input`: nothing to discriminate on meant nothing was + checked, so a value matching _no_ branch — or, for `oneOf`, more than one — was + accepted. Those compositions are now enforced through the existing subschema + matcher, and only where the flat union check declines, so nothing is checked + twice and the common discriminated-union path is unchanged. + - **`$ref` fragments were matched literally, never decoded.** `#/$defs/percent%25field`, + `#/$defs/foo%22bar` and `#/$defs//$defs/` (an empty pointer token) resolved to + nothing, and generation stopped. Tokens are now percent-decoded before `~1`/`~0` + unescaping, per token, and empty tokens are significant. Two consequences worth + knowing: a definition whose name literally contains `%25` must now be written + `%2525`, and `#/$defs//x` now means the `""` member rather than silently meaning + `#/$defs/x`. `#/` still means the document root. + - **A boolean `$defs` entry was not a ref target.** `$defs: { bool: true }` is a + legal definition; the ref graph only named object subschemas, so a `$ref` at it + resolved to nothing. Boolean entries in a definition map now expand to their + object equivalents (`true` → `{}`, `false` → `{ not: {} }`) — confined to + definition maps, because elsewhere `additionalProperties: true` and `{}` generate + different _types_. + + `@amritk/helpers` carries the last two (`resolve-ref`, `walk-ref-graph`) plus the + new `hasOwnCheck`/`missingCheck` emitters in `safe-accessor`. + +### Patch Changes + +- 9cb45a0: Emit the interpreter's own `multipleOf` check, and compile `pattern` in Unicode + mode — closing the last two places where generated code and + `@amritk/runtime-validators` could disagree about a document. + + `@amritk/helpers/multiple-of-check` claimed to mirror the interpreter and had + drifted from it. The interpreter splits on the divisor (an exact `%` when it is + an integer, a quotient within `2·ε·|q|` when it is not); the emitter still + divided in every case and allowed `1e-8·|q|` — roughly 10⁷× the actual + representation error. Generated validators and parsers therefore **accepted + values the interpreter rejects**: `1000000.005` against `multipleOf: 0.01` (a + half-cent past a whole dollar amount) passed, and so did any value whose quotient + overflows to `Infinity`, because the old fail expression asked `NaN > tolerance` + and got `false`. The emitter now produces the interpreter's two branches + verbatim, so both verdicts flip to invalid and the two implementations agree + again. `0.3` still satisfies `multipleOf: 0.1`, which is what the tolerance is + for. + + A `pattern` now compiles with the `u` flag wherever the pattern admits one, the + same try-`u`-then-fall-back decision the interpreter makes at runtime, taken once + at generation time by the new `regexLiteral` / `regexFlagsFor` in + `@amritk/helpers/escape-regex-pattern`. Without the flag a Unicode property + escape is inert — `\p{Letter}` was read as a literal `p{Letter}` — and `^.$` + rejected a single astral character. Every emit site now goes through + `regexLiteral` rather than interpolating an escaped body into its own `/…/`, so + the flag decision is made in one place instead of at a dozen call sites. + + Measured against the official JSON Schema Test Suite, this closes three cases in + each generator: `@amritk/generate-validators` moves to **1271 / 1281 (99.2%)** + and strict `@amritk/generate-parsers` to **1240 / 1281 (96.8%)**. + +- ef77708: Reject `NaN` against a numeric bound, matching `@amritk/runtime-validators`. + + Bounds were emitted as their direct failure condition (`x < minimum`) rather than + the negated pass condition (`!(x >= minimum)`). The two agree on every ordinary + value and are opposite for `NaN`, which compares `false` against every operator: + the direct form read that as "not out of bounds" and let a `NaN` through + `minimum` / `maximum` / `exclusiveMinimum` / `exclusiveMaximum`, where the + interpreter and Ajv both reject it. Generated validators, strict generated + parsers, and the compiled API engine's inlined guards all now write the negated + form — so a `NaN` fails a bounded number everywhere in the toolchain. A bare + `{ type: 'number' }` with no constraint still accepts it, as Ajv does; only a + bound or `multipleOf` rejects it. + + Two internal inconsistencies close with it: `@amritk/generate-parsers` emitted the + un-negated `x >= min` in its inline matchers and the direct `x < min` in its + strict assertions, so the same schema could answer differently depending on which + path ran, and `@amritk/api`'s compiled engine disagreed with its own runtime + engine for a value the two are documented to be observationally identical on. + + `interpreter-parity.test.ts` now covers the numeric keywords — bounds, the + draft-04 boolean `exclusive*` form, and `multipleOf` across integer, fractional, + and quotient-overflowing divisors — over a value set built to separate the two + spellings (`NaN`, `±Infinity`, `1e308`, `1000000.005`). Nothing pinned these + before, which is how the drift got in. + +- 798fd7a: Measure every schema-consuming package against the official JSON Schema Test + Suite, the way `@amritk/yaml` is measured against the YAML test suite + + The required Draft 2020-12 tests (46 files, 383 groups, 1299 cases) are vendored + under `fixtures/json-schema-test-suite`, and four packages now run them on every + build. Each carries an expected-failure list naming every case it does not pass + and why, and each suite fails when a case moves in **either** direction — a + regression breaks the build, and so does a case that starts passing while its + entry stays behind. Nothing is published: the corpus and the harnesses live + outside every `files` list. + + | package | measured on | rate | + | ----------------------------- | ------------------------------------------------ | ------------------- | + | `@amritk/runtime-validators` | `validate` and `validateGuard` verdicts | 1250 / 1299 (96.2%) | + | `@amritk/generate-parsers` | strict parsers, generated → linked → executed | 1180 / 1299 (90.8%) | + | `@amritk/generate-validators` | generated predicate validators, likewise | 987 / 1299 (76.0%) | + | `@amritk/resolve-refs` | verdict preserved after inlining (`$ref` corpus) | 160 / 170 (94.1%) | + + The generators are measured through the code they emit, not the source text they + emit: each suite schema is generated whole, compiled, and linked in memory, so the + `$ref`'d sibling files and the embedded runtime helpers run too. `resolve-refs` + has no verdicts of its own, so it is held to semantic preservation — the resolved + document must accept exactly what the original did, judged by + `@amritk/runtime-validators` over the cases the interpreter already answers + correctly, which is the population where a resolution bug is visible and nothing + else is. + + Those rates are where the packages _end up_. The suites were written first and + found real defects — a validator that accepted everything for a schema without a + `type`, `required` satisfied by an inherited `toString`, refs that emitted + uncompilable output, `$ref`-shaped data inlined as a reference — each fixed in its + own commit alongside this one. What remains is documented case by case, and each + package's README carries a "Conformance, measured" section with its number and the + reasons behind it. + +- f439570: Bring the strict parser up to Ajv's assertion vocabulary, and stop refusing the + keywords it can now prove. + + The exact subschema matcher — the thing strict mode enforces `contains`, + `propertyNames`, `not` and `dependentSchemas` through — only understood a + fraction of Draft 2020-12, and every gap in it became either a generation-time + refusal or a keyword nothing checked. It now covers `$ref` (JSON Pointer, + `$anchor`, and the 2020-12 rule that a ref's _siblings_ still apply), + `prefixItems` with its `items` tail, `items: false`, `contains` with + `minContains` / `maxContains`, `patternProperties`, a schema-valued + `additionalProperties`, `propertyNames`, `dependentRequired`, + `dependentSchemas`, array-form `type`, structural `const`, and an empty `enum`. + + Built on that: + + - **`unevaluatedProperties` / `unevaluatedItems` are implemented** rather than + rejected at generation time. The emitted check computes the same annotation + coverage the runtime interpreter collects — keys and indices evaluated by + `properties`, `patternProperties`, `additionalProperties`, `prefixItems`, + `items`, a satisfied `contains`, `allOf` members, a `$ref` target, a _matching_ + `anyOf` / `oneOf` branch, an `if` / `then` / `else` arm, and a triggered + `dependentSchemas` entry — and applies the unevaluated schema to what is left. + - **A backstop check** now stands behind the per-property assertions, so the + keywords no flat check can express are enforced instead of dropped: a + `$ref` that no imported parser validates (single-file builds, `allOf` members + of a property-less object, array `items`, tuple positions), a `$ref` with + constraining siblings, `items: false`, and constraint keywords with no `type` + to hang them on (`{ minimum: 5 }`, `{ required: ['a'] }`). The fast path and + the shape validator decline for those same shapes, so nothing can skip past + the check. It proves only the keywords that need it — a bare `required` stays + enforceable even when a sibling `allOf` member is too deep to inline — so the + whole 982-schema OpenAPI corpus still generates under `strict`, now covered by + its own pass in the fixture suite. + - **A `type` with more than one non-null member** keeps each family's + constraints: `{ type: ['string','array'], minLength: 3, minItems: 2 }` bounds + the string by length and the array by count. + - **`minLength` / `maxLength` count Unicode code points**, as JSON Schema + specifies — `"💩"` no longer satisfies `minLength: 2`, and `"💩💩"` no longer + violates `maxLength: 2`. The exact count is only scanned inside the narrow band + where the cheap UTF-16 unit count cannot decide, so ASCII input allocates + nothing and `minLength: 1` compiles to a plain length test. + - **A `false` schema rejects every value** instead of casting it through, and a + strict `if` / `then` / `else` root asserts the conditional instead of building + a result from the branch fragments (which invented properties the input never + had). + - **A nullable object root** (`type: ["object","null"]` with `properties`) accepts + `null`, which the object parser's `isObject` guard used to reject. + - **A recursive root `$ref: "#"`** is generated as the root's own type, the way a + root `$dynamicAnchor` already was. It previously emitted an import of a + `ref-.ts` that was never generated — output that did not compile at all. + - The generation-time guard walks _schema_ positions only. It used to inspect + every object in the document, so a schema declaring a property named `items`, + `not` or `contains` was checked as though the property name were the keyword. + + Parity is held by a new differential fuzz suite (`parser-vocabulary-conformance`) + over that vocabulary, plus the existing shape, composition and strict fuzzers, + with Ajv 2020 as the oracle. Three departures from Ajv are deliberate and + documented in the README: `format` stays an annotation (Ajv's own default), + `multipleOf` keeps the magnitude-scaled tolerance the whole toolchain shares, and + a type-less schema with `properties` still requires an object because the parser + must return the type it declares. + +- cb0ef39: Close the strict parser's silent-acceptance gaps across the composition and + constraint keywords. A strict parser promises to throw on anything the schema + rejects; each of these was accepted instead, because the keyword appeared in + neither the fast-path guard nor the slow-path assertions: + + - `const` on a property — scalar, structural (compared by deep equality, key + order included) and `const: null`. + - `minProperties` / `maxProperties` — at the root, on a property, and on the + property-less object and record parsers. + - `required` on a schema with no `properties`. + - `not`, at the root and on a property. Enforced through the exact subschema + matcher; a subschema the matcher cannot prove is now a generation-time error, + joining `contains` / `propertyNames` / `dependentSchemas`. + - `allOf` members that carry constraints rather than an object shape, and object + members of a type-less `allOf` root. + - `if` / `then` / `else`. + - `patternProperties` value schemas, and the value constraints of a + schema-valued `additionalProperties` beyond its bare `type`. + - Array `items` richer than a scalar or enum — a nested array, a union, a + bounded string — which previously contributed no element check at all. + - `minItems` / `maxItems` on a root array whose items are objects or `$ref`s. + - The object shape (`properties`, `required`, …) of a nullable + (`["object","null"]`) property. + - Boolean property schemas: `false` now rejects the key, and `true` no longer + blanks the value to `undefined` — which made a strict parser mutate a value it + had just accepted. + + The fast-path guard and the exported shape validator decline any schema carrying + a keyword they cannot mirror, so a value can no longer be waved through before + the assertions run. `subschemaMatchExpr` gained `allOf` / `anyOf` / `oneOf` / + `not` / `if`-`then`-`else` support (so `contains`, `propertyNames`, + `dependentSchemas` and array items handle combinators too), switched + `uniqueItems` to the structural comparison the rest of the package uses, and + casts its object accessors so the generated code type-checks. + + A new differential fuzz suite holds this surface against Ajv. + +- Updated dependencies [213ecc4] +- Updated dependencies [9cb45a0] +- Updated dependencies [5afbfd4] +- Updated dependencies [eb80ca6] +- Updated dependencies [2c9982c] +- Updated dependencies [f439570] +- Updated dependencies [fa8620c] + - @amritk/helpers@0.15.0 + ## 0.18.0 ### Minor Changes diff --git a/packages/generate-parsers/package.json b/packages/generate-parsers/package.json index 13fe36cf..4b10fa87 100644 --- a/packages/generate-parsers/package.json +++ b/packages/generate-parsers/package.json @@ -1,6 +1,6 @@ { "name": "@amritk/generate-parsers", - "version": "0.18.0", + "version": "0.19.0", "description": "Generate TypeScript parsers and type definitions from JSON Schemas.", "main": "./dist/index.js", "module": "./dist/index.js", diff --git a/packages/generate-validators/CHANGELOG.md b/packages/generate-validators/CHANGELOG.md index 415f8b96..7b8631b5 100644 --- a/packages/generate-validators/CHANGELOG.md +++ b/packages/generate-validators/CHANGELOG.md @@ -1,5 +1,403 @@ # @amritk/generate-validators +## 0.12.0 + +### Minor Changes + +- de0952c: Make both generators agree with Draft 2020-12 — and with each other — on + structural equality, tuple `items`, `oneOf`, and prototype-member property names. + + The two generators disagreed with Ajv and with one another on the same schemas. + Where they differed, `generate-validators` was usually right: it already shipped + `valuesEqual` / `allUnique`, which `generate-parsers` never adopted. + + **`@amritk/generate-validators`** + + - A property named `__proto__` was silently dropped. The `nullable` rewrite runs + over _every_ schema and copied `properties` with a plain assignment, which fires + the `Object.prototype` setter instead of creating a key — so + `{"properties":{"__proto__":{"type":"string","minLength":3}}}` emitted a + validator with no checks at all, and `required: ["__proto__"]` degraded to a + bare presence check. A validation bypass. + - `constructor` / `toString` / `hasOwnProperty` properties were read straight off + the object, so the _prototype's_ value answered: a valid document was reported + as `must be string` at `/hasOwnProperty`, and a required `toString` could never + be reported missing (`'toString' in obj` is always true). Reads now go through + an own-property guard, and presence uses `Object.hasOwn` — but only for the + names that can actually be inherited. Every other key keeps the plain `in` it + always had, because `Object.hasOwn` is a call the engine cannot fold the way it + folds `in`, and spending it on `id` or `name` bought nothing while costing + roughly half the throughput on an all-present object. + - `items` alongside `prefixItems` was applied to the prefix positions too. Per + 2020-12 `items` is the tail schema, so `{prefixItems:[{type:'string'}], +items:{type:'number'}}` rejected `["a", 1, 2]` — which Ajv accepts, and which + the `[string?, ...number[]]` type this generator emits already admits. + - `enum` members that are objects or arrays could never match: `.includes` is + SameValueZero, i.e. reference equality. `enum` now compares structurally via + `valuesEqual`, the way `const` always has — which also makes `isX` a sound type + guard again. + + **`@amritk/generate-parsers`** + + - The same `prefixItems` + `items` defect, in both the fast path and the strict + assertion. + - The same `enum`-with-object-members defect. Members are now compared by an + unrolled structural check against the known literal. + - `const` deep equality used `JSON.stringify`, which is key-order sensitive, so + `{b: 2, a: 1}` was rejected against `const: {a: 1, b: 2}` — and it serialized the + whole value on every call to do it. + - `uniqueItems` used a `JSON.stringify` dedupe key (or a bare `Set`, which + compares objects by reference), so `[{a:1,b:2},{b:2,a:1}]` was accepted where + Ajv and `generate-validators` both reject. It now projects through a + key-order-independent canonical form when items may be structural, keeping the + cheap native `Set` when they are provably scalar. A root array of objects + skipped the constraint entirely and now enforces it. + - `oneOf` exclusivity was not enforced — both `oneOf` and `anyOf` compiled to a + plain disjunction, so a value matching two branches was accepted. `oneOf` now + requires exactly one match. + - The array fallback was a bare `[]`, ignoring `prefixItems` and `minItems`. It is + not an instance of its own schema, and against a required closed tuple it is not + even assignable — `TS2322: Type '[]' is not assignable to type '[string, +number]'` made non-strict and `readonly` output fail to compile. + - A schema property named after an `Object.prototype` member produced an + unsatisfiable type: TypeScript reads the inherited `constructor: Function` on the + fallback object literal and rejects it against `constructor?: string`. + + Both packages now type-check their generated output under the repo's real + compiler flags (`exactOptionalPropertyTypes`, `noUncheckedIndexedAccess`), not + `strict` alone — `generate-validators` had no such suite at all — and both pin the + semantics above against Ajv (or, for prototype-member names, against + `@amritk/runtime-validators`, since Ajv has those bugs itself). + +- 213ecc4: Take documents you already loaded, so a `$ref` to another document generates + + **On the official JSON Schema Test Suite: `generate-validators` 1238 → 1268 / + 1281 (99.0%), `generate-parsers` 1222 → 1237 / 1281 (96.6%).** + + Both generators gain a `schemas` option: documents you have already loaded, keyed + by the absolute URI a `$ref` names them by. It is the build-time counterpart of + `@amritk/runtime-validators`' `ValidateOptions.schemas`, and it keeps the same + promise — nothing is fetched, you cannot pass a URL, only a document. What changes + is that "we do no I/O" no longer also means "we cannot be told". + + A cross-document `$ref` was the single largest gap in both packages, and it is + gone. `refRemote.json` passes in full; so do the `dynamicRef.json` groups that + reach `tree.json` and `extendible-dynamic-ref.json`, and — with the dialect + metaschema registered — `defs.json` and `ref.json`'s "remote ref, containing refs + itself". + + Each registered document becomes a resource of the document being generated: its + `$id`, its `$anchor`s and `$dynamicAnchor`s and its own embedded resources all + resolve, a `$ref` from one registered document into another resolves, and every + definition reached gets a file, a type and a validator/parser by the ordinary + rules. A document with no `$id` resolves its relative `$ref`s against the URI it + was registered under; one whose `$id` disagrees answers to both. Registering more + than the schema uses costs nothing — only the documents actually reached are + emitted — and a `$ref` to a URI nobody registered still stops the build with a + message naming the ref. + + The mechanism is one pass, not a second addressing mode. `@amritk/helpers` gains + `graftExternalSchemas`, which embeds the registered documents into the root before + the `$id` pass, and `pruneExternalSchemas`, which drops the unreferenced ones once + the refs are pointers and reachability is finally knowable. Everything downstream — + the ref-graph walk, the naming, the emitted import graph — keeps working on a + single document and needed no change. `walkRefGraph` carries the option and + memoizes per `(schema, schemas)` by identity. + + **Fixed: a root schema with a union `type` dropped every sibling constraint.** + `{ type: ['object', 'boolean'], properties: {…}, required: [...] }` emitted the + type check and nothing else, so it accepted any object at all. The multi-type root + branch now emits the shared constraint checks the single-type and combinator + branches already did; they carry their own runtime-type guards, so a member of the + union a constraint does not apply to is still untouched. This is the shape the + 2020-12 metaschema's own root is written in, which is how it went unnoticed — the + generated dialect validator accepted `{ type: 1 }` as a valid schema. + + `@amritk/runtime-validators` is unchanged in behaviour; its conformance figures are + restated against the corpus that is actually vendored (1281 cases, not 1299 — the + README's count never matched, and upstream's `content.json` is not among the + vendored files). The suite's `remotes/` loader moves to the shared fixtures + bookkeeping so all four conformance suites use one walk. + +- eb80ca6: Fix `$ref`-graph naming and reference resolution, and stop degrading silently. + + Generation now fails loudly instead of writing output that cannot work: + + - Two definitions that reduce to one filename (`Pet`/`pet`) or one type name + (`foo-bar`/`foo.bar`/`fooBar` all become `FooBar`) are an error. The filename + case used to drop one definition and give every reference to it the other + one's shape; the type-name case emitted both files and left the importer with + two `import { FooBar }` lines that do not parse. + - An unresolvable `$ref` is an error. It used to warn while the generators still + emitted the type name and the parser/validator call for a file that was never + written. + - A `$dynamicRef` with no `$dynamicAnchor` to bind to is an error. Leaving it in + place made the type generator name the type after the anchor, so the canonical + recursive-tree idiom (`$dynamicAnchor: "node"`) produced a reference to the + DOM's `Node` interface — a clean compile with the wrong type. + - A document that relies on `$id` base-URI scoping is rejected rather than + resolving its inner fragments against the document root and silently selecting + a different definition. + - Every recursive schema walker enforces a nesting cap and reports it by name + instead of dying with a bare stack-overflow. + + And several things that were broken now work: + + - Non-ASCII definition names (CJK, Cyrillic, accented) keep their characters + instead of collapsing onto the single type name `_`, and the generated + `index.ts` barrel re-exports them correctly. + - A root-level `$dynamicAnchor` is generated as the root's own file, so the + 2020-12 recursive-tree idiom produces a real self-referencing type. + - A plain `$anchor` ref (`$ref: "#named"`) resolves, instead of producing the + unloadable import specifier `'./#named.ts'`. + - Derived filenames are normalized: no more `.ts`, `...ts`, `http:--x.ts`, or + characters Windows and ESM specifiers reject. `$ref: "#/__proto__"` no longer + resolves to `Object.prototype`. + - Generated readers guard `Object.prototype` member names (`constructor`, + `toString`, `__proto__`, …) with `Object.hasOwn`, so a schema with a + `constructor` property no longer fails its own shape check for every valid + object, and the parser no longer fabricates a `__proto__` key. + - `x-mjst` `instanceOf` is allow-listed to the classes the generators support, + so an arbitrary identifier is warned about and ignored instead of being + emitted verbatim into the output. + +- 947d44a: Enforce the keywords a schema declares without a `type`, and stop emitting + validators that call functions nobody wrote + + Measured against the official JSON Schema Test Suite, generation goes from + 818/1299 to **987/1299 (76.0%)**. + + - **A schema with no root `type` compiled to `validateRoot = () => true`.** The + generator hung every check off the declared type, so `{ "minLength": 2 }`, + `{ "required": ["a"] }`, `{ "uniqueItems": true }`, `{ "contains": … }`, + `{ "patternProperties": … }`, `{ "propertyNames": … }` and + `{ "dependentRequired": … }` all accepted everything — the largest silent gap + this generator had. Each keyword now emits its check behind its own runtime type + test, so it rejects its own family and ignores every other kind of value, which + is what JSON Schema means by a type-less constraint. The same gate was + suppressing constraint checks next to a root combinator, so + `{ allOf: [{ prefixItems: … }], items: … }` silently dropped its `items` too. + - **Object keywords no longer imply `type: "object"`.** `{ "properties": … }` + ignores a non-object instead of rejecting it, matching the interpreter. Note the + inferred TypeScript type still describes the object case (as + `FromSchema`/`ImplicitShape` in `@amritk/runtime-validators` does), so for that + shape `isX` is a weaker type guard than it was — the verdict, which is the + contract, matches the interpreter exactly. + - **Boolean subschemas do something.** A root of `false` rejects every instance + (it used to accept them all), and a `false` sitting in a `properties`, `allOf`, + `then`/`else`, `patternProperties`, `dependentSchemas`, `prefixItems`, + `contains` or `propertyNames` position now emits a real check. + - **A `not` over a `type` array emitted nothing**, and "no checks" is how the + matcher spells "matches everything" — so the `not` rejected every instance. + - **Unresolvable `$ref`s produced output that does not compile.** For a ref the + walker never queues (a relative path, an absolute path, a URN), the emitter + derived a name from the ref string and called `validateIntJson(…)` without + anything emitting it. Generation now refuses, naming the ref — the same answer + the other unsupported paths already give, and a failure next to its cause rather + than in the consumer's build. + - **String lengths count code points, not UTF-16 units**, in both the validator + and the guard, via the shared `@amritk/helpers/string-length-check` (`.length` + stays the short-circuiting first term). + + `unevaluatedItems`/`unevaluatedProperties` still refuse at generation by design — + flat output cannot carry annotations across the applicator tree — and `$id` + base-URI resolution remains unimplemented, so those refuse rather than guess. + +- 7757788: Generate `unevaluatedItems` / `unevaluatedProperties`, and stop `isX` claiming a + narrowing it cannot make + + **On the official JSON Schema Test Suite: 987/1299 → 1238 / 1299 (95.3%).** + + The position that flat generated code cannot carry annotations across the + applicator tree turned out to be wrong, and it was costing 201 cases — two thirds + of everything this package failed. Both keywords are now emitted as a flat + _expression_ computing what the interpreter computes as annotations: for each key + or index, a boolean that is true when some keyword evaluated it. Keywords that + must succeed for the value to be valid at all (`allOf` members, a `$ref` target, a + satisfied `contains`) count unconditionally — sound, because the emitted test is + one conjunct of a validator that also asserts them — while conditional applicators + (`anyOf`/`oneOf` branches, `if`/`then`/`else`, `dependentSchemas`) carry their + condition, hoisted to a `const` before the loop so a per-key sweep reads a boolean + instead of re-running a match. + + `contains` publishes only the indices it matched, per the spec and + `@amritk/runtime-validators`, rather than Ajv's whole-array mark. + + Four shapes still refuse, each with a message naming the shape rather than the + keyword: coverage running through a `$dynamicRef`, an unresolvable or cyclic + `$ref` at the same instance location, a walk deeper than eight applicators, and a + node under `additionalItems`. No case in the suite hits any of them. + + Parity with the interpreter is the contract and is enforced as one: + `interpreter-parity.test.ts` gains six hand-written groups plus a 500-schema × + 24-value fuzz pass — 12,000 pairs, no divergence. + + **`isX` no longer lies.** For a schema with no `type`/`enum`/`const`/`$ref` but + object-shaped keywords (recursively, so a union of implicit-object branches counts + too), the emitted type describes the object case while the validator — correctly — + also accepts non-objects. The guard now returns `boolean` for exactly those + schemas instead of `input is X`; the check itself is unchanged and still in + lockstep with `validateX`. Every schema that declares a `type` keeps its type + predicate. + + The complete fix is to widen the emitted type so the narrowing becomes true, which + lives in `@amritk/helpers/generate-type-definition` and has to move together with + `FromSchema`'s `ImplicitShape` in `@amritk/runtime-validators`, since both make the + identical inference for the identical schema. Until they do, a guard that declines + to narrow beats one that narrows wrongly. + + Also inherited from `@amritk/helpers`: `$ref`s written against an enclosing `$id` + now resolve, which closed most of this package's ref failures without a change + here. + +### Patch Changes + +- 9cb45a0: Emit the interpreter's own `multipleOf` check, and compile `pattern` in Unicode + mode — closing the last two places where generated code and + `@amritk/runtime-validators` could disagree about a document. + + `@amritk/helpers/multiple-of-check` claimed to mirror the interpreter and had + drifted from it. The interpreter splits on the divisor (an exact `%` when it is + an integer, a quotient within `2·ε·|q|` when it is not); the emitter still + divided in every case and allowed `1e-8·|q|` — roughly 10⁷× the actual + representation error. Generated validators and parsers therefore **accepted + values the interpreter rejects**: `1000000.005` against `multipleOf: 0.01` (a + half-cent past a whole dollar amount) passed, and so did any value whose quotient + overflows to `Infinity`, because the old fail expression asked `NaN > tolerance` + and got `false`. The emitter now produces the interpreter's two branches + verbatim, so both verdicts flip to invalid and the two implementations agree + again. `0.3` still satisfies `multipleOf: 0.1`, which is what the tolerance is + for. + + A `pattern` now compiles with the `u` flag wherever the pattern admits one, the + same try-`u`-then-fall-back decision the interpreter makes at runtime, taken once + at generation time by the new `regexLiteral` / `regexFlagsFor` in + `@amritk/helpers/escape-regex-pattern`. Without the flag a Unicode property + escape is inert — `\p{Letter}` was read as a literal `p{Letter}` — and `^.$` + rejected a single astral character. Every emit site now goes through + `regexLiteral` rather than interpolating an escaped body into its own `/…/`, so + the flag decision is made in one place instead of at a dozen call sites. + + Measured against the official JSON Schema Test Suite, this closes three cases in + each generator: `@amritk/generate-validators` moves to **1271 / 1281 (99.2%)** + and strict `@amritk/generate-parsers` to **1240 / 1281 (96.8%)**. + +- ef77708: Reject `NaN` against a numeric bound, matching `@amritk/runtime-validators`. + + Bounds were emitted as their direct failure condition (`x < minimum`) rather than + the negated pass condition (`!(x >= minimum)`). The two agree on every ordinary + value and are opposite for `NaN`, which compares `false` against every operator: + the direct form read that as "not out of bounds" and let a `NaN` through + `minimum` / `maximum` / `exclusiveMinimum` / `exclusiveMaximum`, where the + interpreter and Ajv both reject it. Generated validators, strict generated + parsers, and the compiled API engine's inlined guards all now write the negated + form — so a `NaN` fails a bounded number everywhere in the toolchain. A bare + `{ type: 'number' }` with no constraint still accepts it, as Ajv does; only a + bound or `multipleOf` rejects it. + + Two internal inconsistencies close with it: `@amritk/generate-parsers` emitted the + un-negated `x >= min` in its inline matchers and the direct `x < min` in its + strict assertions, so the same schema could answer differently depending on which + path ran, and `@amritk/api`'s compiled engine disagreed with its own runtime + engine for a value the two are documented to be observationally identical on. + + `interpreter-parity.test.ts` now covers the numeric keywords — bounds, the + draft-04 boolean `exclusive*` form, and `multipleOf` across integer, fractional, + and quotient-overflowing divisors — over a value set built to separate the two + spellings (`NaN`, `±Infinity`, `1e308`, `1000000.005`). Nothing pinned these + before, which is how the drift got in. + +- 798fd7a: Measure every schema-consuming package against the official JSON Schema Test + Suite, the way `@amritk/yaml` is measured against the YAML test suite + + The required Draft 2020-12 tests (46 files, 383 groups, 1299 cases) are vendored + under `fixtures/json-schema-test-suite`, and four packages now run them on every + build. Each carries an expected-failure list naming every case it does not pass + and why, and each suite fails when a case moves in **either** direction — a + regression breaks the build, and so does a case that starts passing while its + entry stays behind. Nothing is published: the corpus and the harnesses live + outside every `files` list. + + | package | measured on | rate | + | ----------------------------- | ------------------------------------------------ | ------------------- | + | `@amritk/runtime-validators` | `validate` and `validateGuard` verdicts | 1250 / 1299 (96.2%) | + | `@amritk/generate-parsers` | strict parsers, generated → linked → executed | 1180 / 1299 (90.8%) | + | `@amritk/generate-validators` | generated predicate validators, likewise | 987 / 1299 (76.0%) | + | `@amritk/resolve-refs` | verdict preserved after inlining (`$ref` corpus) | 160 / 170 (94.1%) | + + The generators are measured through the code they emit, not the source text they + emit: each suite schema is generated whole, compiled, and linked in memory, so the + `$ref`'d sibling files and the embedded runtime helpers run too. `resolve-refs` + has no verdicts of its own, so it is held to semantic preservation — the resolved + document must accept exactly what the original did, judged by + `@amritk/runtime-validators` over the cases the interpreter already answers + correctly, which is the population where a resolution bug is visible and nothing + else is. + + Those rates are where the packages _end up_. The suites were written first and + found real defects — a validator that accepted everything for a schema without a + `type`, `required` satisfied by an inherited `toString`, refs that emitted + uncompilable output, `$ref`-shaped data inlined as a reference — each fixed in its + own commit alongside this one. What remains is documented case by case, and each + package's README carries a "Conformance, measured" section with its number and the + reasons behind it. + +- 2c9982c: Fix the published manifests so the packages install, resolve, and dedupe correctly + + **Types resolve on TypeScript's default config.** Every package was + exports-only: nine declared `"module": "./dist/index.js"` (a field neither Node + nor TypeScript reads) and nothing declared `types`. A consumer on + `moduleResolution: "node10"` — still the default when `module` is `commonjs` — + cannot see `exports` at all, so `import { lintDocument } from '@amritk/lint'` + failed with `TS2307: Cannot find module '@amritk/lint' or its corresponding type +declarations`. Each package with a `.` export now also declares `main` and + `types`; `@amritk/helpers` and `@amritk/adapters` have no `.` export (they are + subpath-only), so they declare a `typesVersions` wildcard mapping instead, which + gives their subpaths the same node10 fallback. All of it is ignored under + `node16`/`nodenext`/`bundler`, where `exports` still wins. + + **`workspace:*` resolves to a caret, not an exact pin.** All fourteen + inter-package edges shipped as exact versions, so installing two `@amritk/*` + packages published at different times pulled in two copies of their shared + dependency. That is not merely wasteful: the module-level caches those packages + rely on are per-copy, so the `WeakMap` validator cache in + `@amritk/runtime-validators` silently stopped hitting. Pre-1.0 a caret stays + narrow (`^0.9.1` is `>=0.9.1 <0.10.0`) and breaking changes here already ride a + minor bump. + + **`@amritk/helpers` stops shipping 21 source files it does not need.** Embedded + mode reads four helper sources (`is-object`, `validate-array`, + `validate-record`, `has-ref`) out of the installed package at generation time, + so `src` has to ship — but only those four. `files` now lists them explicitly + instead of globbing all of `src`, cutting the tarball from 78 files / 206 kB to + 63 / 112 kB. + + **Two packages no longer declare a dependency they never import.** + `@amritk/mjst` and `@amritk/generate-parsers` both listed + `@amritk/generate-markdown` under `dependencies`, but the only importer is each + package's `scripts/generate-readme.ts`, which is not published. Both moved to + `devDependencies`. `@amritk/adapters` likewise dropped its + `@sinclair/typebox` peer dependency: the TypeBox adapter is purely structural + (it strips symbol keys) and imports nothing. `valibot` stays — it is a genuine + transitive peer of `@valibot/to-json-schema`. + + **`@amritk/mjst` fixes.** `json-schema-typed` moved to `dependencies`, because + the shipped `dist/emit-examples.d.ts` imports types from it. The package gained + an `exports` map, so it is no longer deep-importable in its entirety. And the + build now marks `dist/cli.js` executable: `npm pack` records on-disk modes, and + package managers only `chmod` bin targets when they link them, so flows that + consume the tarball directly (vendoring, Docker `npm pack` + `tar -x`) hit + `EACCES`. + +- Updated dependencies [213ecc4] +- Updated dependencies [9cb45a0] +- Updated dependencies [5afbfd4] +- Updated dependencies [eb80ca6] +- Updated dependencies [2c9982c] +- Updated dependencies [f439570] +- Updated dependencies [fa8620c] + - @amritk/helpers@0.15.0 + ## 0.11.12 ### Patch Changes diff --git a/packages/generate-validators/package.json b/packages/generate-validators/package.json index eca7edd0..f241c1e8 100644 --- a/packages/generate-validators/package.json +++ b/packages/generate-validators/package.json @@ -1,6 +1,6 @@ { "name": "@amritk/generate-validators", - "version": "0.11.12", + "version": "0.12.0", "description": "Generate TypeScript validation functions from JSON Schemas.", "main": "./dist/index.js", "module": "./dist/index.js", diff --git a/packages/helpers/CHANGELOG.md b/packages/helpers/CHANGELOG.md index 73c3b008..8e815990 100644 --- a/packages/helpers/CHANGELOG.md +++ b/packages/helpers/CHANGELOG.md @@ -1,5 +1,327 @@ # @amritk/helpers +## 0.15.0 + +### Minor Changes + +- 213ecc4: Take documents you already loaded, so a `$ref` to another document generates + + **On the official JSON Schema Test Suite: `generate-validators` 1238 → 1268 / + 1281 (99.0%), `generate-parsers` 1222 → 1237 / 1281 (96.6%).** + + Both generators gain a `schemas` option: documents you have already loaded, keyed + by the absolute URI a `$ref` names them by. It is the build-time counterpart of + `@amritk/runtime-validators`' `ValidateOptions.schemas`, and it keeps the same + promise — nothing is fetched, you cannot pass a URL, only a document. What changes + is that "we do no I/O" no longer also means "we cannot be told". + + A cross-document `$ref` was the single largest gap in both packages, and it is + gone. `refRemote.json` passes in full; so do the `dynamicRef.json` groups that + reach `tree.json` and `extendible-dynamic-ref.json`, and — with the dialect + metaschema registered — `defs.json` and `ref.json`'s "remote ref, containing refs + itself". + + Each registered document becomes a resource of the document being generated: its + `$id`, its `$anchor`s and `$dynamicAnchor`s and its own embedded resources all + resolve, a `$ref` from one registered document into another resolves, and every + definition reached gets a file, a type and a validator/parser by the ordinary + rules. A document with no `$id` resolves its relative `$ref`s against the URI it + was registered under; one whose `$id` disagrees answers to both. Registering more + than the schema uses costs nothing — only the documents actually reached are + emitted — and a `$ref` to a URI nobody registered still stops the build with a + message naming the ref. + + The mechanism is one pass, not a second addressing mode. `@amritk/helpers` gains + `graftExternalSchemas`, which embeds the registered documents into the root before + the `$id` pass, and `pruneExternalSchemas`, which drops the unreferenced ones once + the refs are pointers and reachability is finally knowable. Everything downstream — + the ref-graph walk, the naming, the emitted import graph — keeps working on a + single document and needed no change. `walkRefGraph` carries the option and + memoizes per `(schema, schemas)` by identity. + + **Fixed: a root schema with a union `type` dropped every sibling constraint.** + `{ type: ['object', 'boolean'], properties: {…}, required: [...] }` emitted the + type check and nothing else, so it accepted any object at all. The multi-type root + branch now emits the shared constraint checks the single-type and combinator + branches already did; they carry their own runtime-type guards, so a member of the + union a constraint does not apply to is still untouched. This is the shape the + 2020-12 metaschema's own root is written in, which is how it went unnoticed — the + generated dialect validator accepted `{ type: 1 }` as a valid schema. + + `@amritk/runtime-validators` is unchanged in behaviour; its conformance figures are + restated against the corpus that is actually vendored (1281 cases, not 1299 — the + README's count never matched, and upstream's `content.json` is not among the + vendored files). The suite's `remotes/` loader moves to the shared fixtures + bookkeeping so all four conformance suites use one walk. + +- 5afbfd4: Resolve `$ref` against `$id` as a base URI in the ref graph + + A `$ref` written against an enclosing `$id` — a relative URI (`list`, + `folderInteger.json`), an absolute one, or a URN — used to either stop generation + or, worse, find _a_ definition and generate against it, so the emitted parser + enforced a schema its author did not write. On the official JSON Schema Test + Suite, strict-parser generation goes from 1180/1299 to **1222 / 1299 (94.1%)**; + all 29 of the resolve-to-the-wrong-definition cases became right rather than + refused. + + Three new pieces in `@amritk/helpers`, deliberately free of any parser or + validator concepts: + + - **`build-resource-registry`** — one walk producing the document's embedded + resources, anchors and dynamic anchors, each `$id` composed against the base of + its parent. Keyed by JSON Pointer, because that is the currency the rest of the + package already deals in — a registry hit turns straight into a `$ref` string, a + filename, or a type name. Returns `null` for a document with no `$id`, which is + the fast-path switch, and is memoized per document. + - **`resolve-scoped-ref`** — one call covering relative, absolute, absolute-path, + URN, pointer-into-resource and anchor-in-resource forms, plus the plain + `#/$defs/x` that under an enclosing `$id` means _that resource's_ `$defs`. + - **`normalize-ref-scopes`** — rewrites every `$ref`/`$dynamicRef` to a + document-root pointer. This is the leverage: everything downstream already + resolves refs by string against the root, so one normalization makes ref + resolution, type naming, the import graph and the strict matcher correct at once, + none of them needing base-URI awareness of their own. + + It is wired into `walkRefGraph`, so `@amritk/generate-validators` and + `@amritk/generate-examples` inherit it. + + `assertIdScopes` keeps its name and signature but changes meaning: it no longer + refuses any document with nested `$id` scoping, only the residue base-URI + resolution cannot place — a fragment ref inside an embedded resource that declares + its own targets and names none of them. That preserves the property worth having: + never silently pick the outer definition. + + `@amritk/generate-parsers` additionally follows the spec on `contains` next to + `unevaluatedItems`: only the items `contains` matched are evaluated, not the whole + array. That is what `@amritk/runtime-validators` does, so the two stop disagreeing + about the same schema. Ajv marks the whole array, so the single fuzz fragment + pairing those keywords leaves the Ajv-oracle corpus (with the reason recorded next + to it) and unit tests plus the conformance suite cover it instead; every other + `contains` and `unevaluated*` fragment keeps fuzzing. + +- eb80ca6: Fix `$ref`-graph naming and reference resolution, and stop degrading silently. + + Generation now fails loudly instead of writing output that cannot work: + + - Two definitions that reduce to one filename (`Pet`/`pet`) or one type name + (`foo-bar`/`foo.bar`/`fooBar` all become `FooBar`) are an error. The filename + case used to drop one definition and give every reference to it the other + one's shape; the type-name case emitted both files and left the importer with + two `import { FooBar }` lines that do not parse. + - An unresolvable `$ref` is an error. It used to warn while the generators still + emitted the type name and the parser/validator call for a file that was never + written. + - A `$dynamicRef` with no `$dynamicAnchor` to bind to is an error. Leaving it in + place made the type generator name the type after the anchor, so the canonical + recursive-tree idiom (`$dynamicAnchor: "node"`) produced a reference to the + DOM's `Node` interface — a clean compile with the wrong type. + - A document that relies on `$id` base-URI scoping is rejected rather than + resolving its inner fragments against the document root and silently selecting + a different definition. + - Every recursive schema walker enforces a nesting cap and reports it by name + instead of dying with a bare stack-overflow. + + And several things that were broken now work: + + - Non-ASCII definition names (CJK, Cyrillic, accented) keep their characters + instead of collapsing onto the single type name `_`, and the generated + `index.ts` barrel re-exports them correctly. + - A root-level `$dynamicAnchor` is generated as the root's own file, so the + 2020-12 recursive-tree idiom produces a real self-referencing type. + - A plain `$anchor` ref (`$ref: "#named"`) resolves, instead of producing the + unloadable import specifier `'./#named.ts'`. + - Derived filenames are normalized: no more `.ts`, `...ts`, `http:--x.ts`, or + characters Windows and ESM specifiers reject. `$ref: "#/__proto__"` no longer + resolves to `Object.prototype`. + - Generated readers guard `Object.prototype` member names (`constructor`, + `toString`, `__proto__`, …) with `Object.hasOwn`, so a schema with a + `constructor` property no longer fails its own shape check for every valid + object, and the parser no longer fabricates a `__proto__` key. + - `x-mjst` `instanceOf` is allow-listed to the classes the generators support, + so an arbitrary identifier is warned about and ignored instead of being + emitted verbatim into the output. + +- 2c9982c: Fix the published manifests so the packages install, resolve, and dedupe correctly + + **Types resolve on TypeScript's default config.** Every package was + exports-only: nine declared `"module": "./dist/index.js"` (a field neither Node + nor TypeScript reads) and nothing declared `types`. A consumer on + `moduleResolution: "node10"` — still the default when `module` is `commonjs` — + cannot see `exports` at all, so `import { lintDocument } from '@amritk/lint'` + failed with `TS2307: Cannot find module '@amritk/lint' or its corresponding type +declarations`. Each package with a `.` export now also declares `main` and + `types`; `@amritk/helpers` and `@amritk/adapters` have no `.` export (they are + subpath-only), so they declare a `typesVersions` wildcard mapping instead, which + gives their subpaths the same node10 fallback. All of it is ignored under + `node16`/`nodenext`/`bundler`, where `exports` still wins. + + **`workspace:*` resolves to a caret, not an exact pin.** All fourteen + inter-package edges shipped as exact versions, so installing two `@amritk/*` + packages published at different times pulled in two copies of their shared + dependency. That is not merely wasteful: the module-level caches those packages + rely on are per-copy, so the `WeakMap` validator cache in + `@amritk/runtime-validators` silently stopped hitting. Pre-1.0 a caret stays + narrow (`^0.9.1` is `>=0.9.1 <0.10.0`) and breaking changes here already ride a + minor bump. + + **`@amritk/helpers` stops shipping 21 source files it does not need.** Embedded + mode reads four helper sources (`is-object`, `validate-array`, + `validate-record`, `has-ref`) out of the installed package at generation time, + so `src` has to ship — but only those four. `files` now lists them explicitly + instead of globbing all of `src`, cutting the tarball from 78 files / 206 kB to + 63 / 112 kB. + + **Two packages no longer declare a dependency they never import.** + `@amritk/mjst` and `@amritk/generate-parsers` both listed + `@amritk/generate-markdown` under `dependencies`, but the only importer is each + package's `scripts/generate-readme.ts`, which is not published. Both moved to + `devDependencies`. `@amritk/adapters` likewise dropped its + `@sinclair/typebox` peer dependency: the TypeBox adapter is purely structural + (it strips symbol keys) and imports nothing. `valibot` stays — it is a genuine + transitive peer of `@valibot/to-json-schema`. + + **`@amritk/mjst` fixes.** `json-schema-typed` moved to `dependencies`, because + the shipped `dist/emit-examples.d.ts` imports types from it. The package gained + an `exports` map, so it is no longer deep-importable in its entirety. And the + build now marks `dist/cli.js` executable: `npm pack` records on-disk modes, and + package managers only `chmod` bin targets when they link them, so flows that + consume the tarball directly (vendoring, Docker `npm pack` + `tar -x`) hit + `EACCES`. + +- fa8620c: Stop strict parsers accepting what the schema forbids: prototype-inherited + `required`, undiscriminated unions, and `$ref` fragments that were never decoded + + Measured against the official JSON Schema Test Suite, strict-mode generation goes + from 1141/1299 to **1180/1299 (90.8%)**. Four defects, all of them cases where the + generated parser said yes to a document the schema says no to — or refused one it + should have taken. + + - **`required` compiled to `in`, which walks the prototype chain.** `"toString" in {}` + is `true`, so `{ "required": ["__proto__", "toString", "constructor"] }` was + satisfied by an object carrying none of them. `Object.hasOwn` now covers the + names an object can actually inherit, and plain `in` stays everywhere else so + ordinary keys keep the form the engine can fold — the same split + `@amritk/generate-validators` already made. Generated output for ordinary keys is + byte-identical. Applies to `required`, `dependentRequired`, `dependentSchemas`, + and `false`-property absence. + - **A `oneOf`/`anyOf`/`allOf` whose branches carry no `type` compiled to a + pass-through.** `{ "oneOf": [{ "type": "integer" }, { "minimum": 2 }] }` emitted + `parseRoot = (input) => input`: nothing to discriminate on meant nothing was + checked, so a value matching _no_ branch — or, for `oneOf`, more than one — was + accepted. Those compositions are now enforced through the existing subschema + matcher, and only where the flat union check declines, so nothing is checked + twice and the common discriminated-union path is unchanged. + - **`$ref` fragments were matched literally, never decoded.** `#/$defs/percent%25field`, + `#/$defs/foo%22bar` and `#/$defs//$defs/` (an empty pointer token) resolved to + nothing, and generation stopped. Tokens are now percent-decoded before `~1`/`~0` + unescaping, per token, and empty tokens are significant. Two consequences worth + knowing: a definition whose name literally contains `%25` must now be written + `%2525`, and `#/$defs//x` now means the `""` member rather than silently meaning + `#/$defs/x`. `#/` still means the document root. + - **A boolean `$defs` entry was not a ref target.** `$defs: { bool: true }` is a + legal definition; the ref graph only named object subschemas, so a `$ref` at it + resolved to nothing. Boolean entries in a definition map now expand to their + object equivalents (`true` → `{}`, `false` → `{ not: {} }`) — confined to + definition maps, because elsewhere `additionalProperties: true` and `{}` generate + different _types_. + + `@amritk/helpers` carries the last two (`resolve-ref`, `walk-ref-graph`) plus the + new `hasOwnCheck`/`missingCheck` emitters in `safe-accessor`. + +### Patch Changes + +- 9cb45a0: Emit the interpreter's own `multipleOf` check, and compile `pattern` in Unicode + mode — closing the last two places where generated code and + `@amritk/runtime-validators` could disagree about a document. + + `@amritk/helpers/multiple-of-check` claimed to mirror the interpreter and had + drifted from it. The interpreter splits on the divisor (an exact `%` when it is + an integer, a quotient within `2·ε·|q|` when it is not); the emitter still + divided in every case and allowed `1e-8·|q|` — roughly 10⁷× the actual + representation error. Generated validators and parsers therefore **accepted + values the interpreter rejects**: `1000000.005` against `multipleOf: 0.01` (a + half-cent past a whole dollar amount) passed, and so did any value whose quotient + overflows to `Infinity`, because the old fail expression asked `NaN > tolerance` + and got `false`. The emitter now produces the interpreter's two branches + verbatim, so both verdicts flip to invalid and the two implementations agree + again. `0.3` still satisfies `multipleOf: 0.1`, which is what the tolerance is + for. + + A `pattern` now compiles with the `u` flag wherever the pattern admits one, the + same try-`u`-then-fall-back decision the interpreter makes at runtime, taken once + at generation time by the new `regexLiteral` / `regexFlagsFor` in + `@amritk/helpers/escape-regex-pattern`. Without the flag a Unicode property + escape is inert — `\p{Letter}` was read as a literal `p{Letter}` — and `^.$` + rejected a single astral character. Every emit site now goes through + `regexLiteral` rather than interpolating an escaped body into its own `/…/`, so + the flag decision is made in one place instead of at a dozen call sites. + + Measured against the official JSON Schema Test Suite, this closes three cases in + each generator: `@amritk/generate-validators` moves to **1271 / 1281 (99.2%)** + and strict `@amritk/generate-parsers` to **1240 / 1281 (96.8%)**. + +- f439570: Bring the strict parser up to Ajv's assertion vocabulary, and stop refusing the + keywords it can now prove. + + The exact subschema matcher — the thing strict mode enforces `contains`, + `propertyNames`, `not` and `dependentSchemas` through — only understood a + fraction of Draft 2020-12, and every gap in it became either a generation-time + refusal or a keyword nothing checked. It now covers `$ref` (JSON Pointer, + `$anchor`, and the 2020-12 rule that a ref's _siblings_ still apply), + `prefixItems` with its `items` tail, `items: false`, `contains` with + `minContains` / `maxContains`, `patternProperties`, a schema-valued + `additionalProperties`, `propertyNames`, `dependentRequired`, + `dependentSchemas`, array-form `type`, structural `const`, and an empty `enum`. + + Built on that: + + - **`unevaluatedProperties` / `unevaluatedItems` are implemented** rather than + rejected at generation time. The emitted check computes the same annotation + coverage the runtime interpreter collects — keys and indices evaluated by + `properties`, `patternProperties`, `additionalProperties`, `prefixItems`, + `items`, a satisfied `contains`, `allOf` members, a `$ref` target, a _matching_ + `anyOf` / `oneOf` branch, an `if` / `then` / `else` arm, and a triggered + `dependentSchemas` entry — and applies the unevaluated schema to what is left. + - **A backstop check** now stands behind the per-property assertions, so the + keywords no flat check can express are enforced instead of dropped: a + `$ref` that no imported parser validates (single-file builds, `allOf` members + of a property-less object, array `items`, tuple positions), a `$ref` with + constraining siblings, `items: false`, and constraint keywords with no `type` + to hang them on (`{ minimum: 5 }`, `{ required: ['a'] }`). The fast path and + the shape validator decline for those same shapes, so nothing can skip past + the check. It proves only the keywords that need it — a bare `required` stays + enforceable even when a sibling `allOf` member is too deep to inline — so the + whole 982-schema OpenAPI corpus still generates under `strict`, now covered by + its own pass in the fixture suite. + - **A `type` with more than one non-null member** keeps each family's + constraints: `{ type: ['string','array'], minLength: 3, minItems: 2 }` bounds + the string by length and the array by count. + - **`minLength` / `maxLength` count Unicode code points**, as JSON Schema + specifies — `"💩"` no longer satisfies `minLength: 2`, and `"💩💩"` no longer + violates `maxLength: 2`. The exact count is only scanned inside the narrow band + where the cheap UTF-16 unit count cannot decide, so ASCII input allocates + nothing and `minLength: 1` compiles to a plain length test. + - **A `false` schema rejects every value** instead of casting it through, and a + strict `if` / `then` / `else` root asserts the conditional instead of building + a result from the branch fragments (which invented properties the input never + had). + - **A nullable object root** (`type: ["object","null"]` with `properties`) accepts + `null`, which the object parser's `isObject` guard used to reject. + - **A recursive root `$ref: "#"`** is generated as the root's own type, the way a + root `$dynamicAnchor` already was. It previously emitted an import of a + `ref-.ts` that was never generated — output that did not compile at all. + - The generation-time guard walks _schema_ positions only. It used to inspect + every object in the document, so a schema declaring a property named `items`, + `not` or `contains` was checked as though the property name were the keyword. + + Parity is held by a new differential fuzz suite (`parser-vocabulary-conformance`) + over that vocabulary, plus the existing shape, composition and strict fuzzers, + with Ajv 2020 as the oracle. Three departures from Ajv are deliberate and + documented in the README: `format` stays an annotation (Ajv's own default), + `multipleOf` keeps the magnitude-scaled tolerance the whole toolchain shares, and + a type-less schema with `properties` still requires an object because the parser + must return the type it declares. + ## 0.14.0 ### Minor Changes diff --git a/packages/helpers/package.json b/packages/helpers/package.json index 3b068e5d..9bb67d9c 100644 --- a/packages/helpers/package.json +++ b/packages/helpers/package.json @@ -1,6 +1,6 @@ { "name": "@amritk/helpers", - "version": "0.14.0", + "version": "0.15.0", "description": "Shared utilities for the mjst code generation ecosystem.", "type": "module", "sideEffects": false, diff --git a/packages/lint/CHANGELOG.md b/packages/lint/CHANGELOG.md index 2cda2c7c..fe07b7ea 100644 --- a/packages/lint/CHANGELOG.md +++ b/packages/lint/CHANGELOG.md @@ -1,5 +1,204 @@ # @amritk/lint +## 0.4.4 + +### Patch Changes + +- 874856f: Stop rulesets from executing code, and close two document-driven hangs + + **A `[?(...)]` filter in a `given` is no longer JavaScript.** Filter bodies were + handed straight to `new Function`, so a `given` string in a YAML or JSON ruleset + — data, to any caller's eye — ran arbitrary code in the linting process: + `$[?(globalThis.x = {home: process.env.HOME})]` leaked the environment, + `import('node:fs')` wrote files. Filters are now parsed into a small AST and + interpreted (`@`, `@.x`, `@['x']`, `@property`, `@parentProperty`, `@parent`, + `@path`, `@root`, `$`, the comparison and logical operators, `!`, numeric + negation, string/number/boolean/null/`undefined`/`void 0` literals, regex + literals, `.length`, and a fixed list of pure methods — `indexOf`, + `lastIndexOf`, `includes`, `startsWith`, `endsWith`, `match`, `test`, + `toLowerCase`, `toUpperCase`, `trim`). Member reads see own properties only, so + `@.constructor` and `@['__proto__']` are plain `undefined`. Verified + node-for-node identical to the old evaluator on the shipped `oas` filters across + every vendored real-world spec. An expression outside the grammar is now a + ruleset error naming the rule, instead of a filter that silently matches + nothing — which also fixes filters quietly disabling themselves wherever + `new Function` is unavailable (CSP, Workers). + + **The `casing` function no longer hangs on a long identifier.** `camel`/`pascal` + compiled to a pattern where digits could be consumed two ways, so a value from + the linted document could force exponential backtracking: a 46-character + `operationId` took over 100 seconds on Node (Bun's regex engine caps + backtracking, which hid it). The patterns are rewritten to be unambiguous, and + verified by brute force to accept exactly the same strings as before. Same for + the second overlap, a separator character the style already uses (`kebab` with + `separator: '-'`), which was exponential from ~40 characters. + + **A deeply nested document is a diagnostic, not a crash.** `'['.repeat(20000)` — + a 40 KB file — took the process down with `RangeError: Maximum call stack size +exceeded`, while every other malformed document came back as findings. JSON + parsing now enforces the same 1000-level nesting limit `@amritk/yaml` does and + reports it as a parser diagnostic, and the JSONPath descent walker is iterative. + + **Rulesets are built once, not per document.** `lintDocument` re-normalized the + ruleset and re-read every `extends` file on every call (~3.6 ms per document + with a 200-rule `extends` file; `fixDocument` paid it up to 11× per document). + The built `Ruleset` is memoized per `(definition object, basePath, restrictTo)`, + and `fixDocument` builds one for the whole loop — 200 lints of a small document + went from 750 ms to ~120 ms — so editing a ruleset file mid-run also stops + changing results half way through. Treat a definition you have passed in as + frozen; pass a fresh object to force a rebuild. + + **`@amritk/lint/rules/openapi` can be bundled.** The four OpenAPI meta-schemas + were loaded through `createRequire` with a computed specifier, invisible to + bundlers (esbuild produced a 524-byte module that threw `Cannot find module +'./oas31.json'`) and unavailable on Workers and Deno. They are now generated + `.ts` modules imported statically, each holding its schema as JSON text that is + still parsed lazily on first use. + + **`fixDocument` reports whether it converged.** The result gains `converged` and + `passes`, and `applied` is de-duplicated by rule code and path: two fixers that + undo each other used to report 11 applied fixes for one problem with no way to + tell a fixpoint from giving up at the pass cap. + + **Also:** `alphabetical` no longer treats `'0x10'`, `'1e2'`, or `' 5'` as + numbers (they were flagged out of order though lexicographically sorted); the + module-level JSONPath, filter, and pattern caches are bounded; `extends` and + custom-function resolution accept an optional `restrictTo` root; and the ruleset + trust boundary is documented in the README and AI.md. + +- 2c9982c: Fix the published manifests so the packages install, resolve, and dedupe correctly + + **Types resolve on TypeScript's default config.** Every package was + exports-only: nine declared `"module": "./dist/index.js"` (a field neither Node + nor TypeScript reads) and nothing declared `types`. A consumer on + `moduleResolution: "node10"` — still the default when `module` is `commonjs` — + cannot see `exports` at all, so `import { lintDocument } from '@amritk/lint'` + failed with `TS2307: Cannot find module '@amritk/lint' or its corresponding type +declarations`. Each package with a `.` export now also declares `main` and + `types`; `@amritk/helpers` and `@amritk/adapters` have no `.` export (they are + subpath-only), so they declare a `typesVersions` wildcard mapping instead, which + gives their subpaths the same node10 fallback. All of it is ignored under + `node16`/`nodenext`/`bundler`, where `exports` still wins. + + **`workspace:*` resolves to a caret, not an exact pin.** All fourteen + inter-package edges shipped as exact versions, so installing two `@amritk/*` + packages published at different times pulled in two copies of their shared + dependency. That is not merely wasteful: the module-level caches those packages + rely on are per-copy, so the `WeakMap` validator cache in + `@amritk/runtime-validators` silently stopped hitting. Pre-1.0 a caret stays + narrow (`^0.9.1` is `>=0.9.1 <0.10.0`) and breaking changes here already ride a + minor bump. + + **`@amritk/helpers` stops shipping 21 source files it does not need.** Embedded + mode reads four helper sources (`is-object`, `validate-array`, + `validate-record`, `has-ref`) out of the installed package at generation time, + so `src` has to ship — but only those four. `files` now lists them explicitly + instead of globbing all of `src`, cutting the tarball from 78 files / 206 kB to + 63 / 112 kB. + + **Two packages no longer declare a dependency they never import.** + `@amritk/mjst` and `@amritk/generate-parsers` both listed + `@amritk/generate-markdown` under `dependencies`, but the only importer is each + package's `scripts/generate-readme.ts`, which is not published. Both moved to + `devDependencies`. `@amritk/adapters` likewise dropped its + `@sinclair/typebox` peer dependency: the TypeBox adapter is purely structural + (it strips symbol keys) and imports nothing. `valibot` stays — it is a genuine + transitive peer of `@valibot/to-json-schema`. + + **`@amritk/mjst` fixes.** `json-schema-typed` moved to `dependencies`, because + the shipped `dist/emit-examples.d.ts` imports types from it. The package gained + an `exports` map, so it is no longer deep-importable in its entirety. And the + build now marks `dist/cli.js` executable: `npm pack` records on-disk modes, and + package managers only `chmod` bin targets when they link them, so flows that + consume the tarball directly (vendoring, Docker `npm pack` + `tar -x`) hit + `EACCES`. + +- 00eb0c9: Read tabs by the column they sit at, hold a flow collection to its parent's + indentation, and pin JSON as the superset it is + + YAML 1.2 test suite conformance goes from **384/402 (95.5%) to 398/402 (99.0%)**. + What is left is four cases where the right answer is not the suite's — one + duplicate-key case that turns on the `uniqueKeys` default, and three tags that + project to a `Uint8Array`/`Set`/`Map` — so this closes the boundary rather than + moving it. Parse throughput is unchanged: an order-balanced A/B over the bench + fixtures lands every case inside run-to-run noise (medians −3% to +1.6% against a + 4–6% CV, with min-of-runs favouring the new code). + + **A tab is only an error where indentation belongs.** Indentation in YAML is + spaces (`s-indent ::= s-space × n`), but a tab _past_ the indentation is ordinary + separation — and the two are told apart only by the column the tab sits at. + `peekLine` reported any leading tab at all, which cut both ways: it rejected three + valid documents (`\t[…]` and `\t{}` at the document root, and a `foo:` whose value + line reads `⟨space⟩⟨tab⟩bar`) while missing the tabs that really are indentation. + Every caller knows the column its line owes, so it now passes it in, and the same + rule is applied in the three other places a tab can stand for indentation: + + - Inside a block scalar — `foo: |` over a lone `\t` is reported, where the same + line written ` \t` is valid content and still parses to `"\t\n"`. + - In the separation between a block indicator and a **compact collection** opened + on its line. A compact collection takes its indentation from the column it lands + on, so `-\t-`, `?\tkey:` and `:\t- x` are invalid — while `-\tfoo` and `-\t-1` + are ordinary separation and stay valid. + - In a flow collection's continuation lines (below). + + **A flow collection is held to the indentation of the block that holds it.** Flow + scanning is delimiter-driven, so `flow: [a,` over a column-0 `b,` read exactly + like a properly indented collection and parsed clean; it is now reported once per + collection as `BAD_INDENT`. Indentation is counted in spaces, which folds the tab + rule in for free. The closing `]`/`}` is deliberately held one column looser than + the spec asks — to the parent's own column rather than one past it — because + closing a multi-line flow collection at the parent's column is how Prettier and + hand-written manifests both write it, and `yaml` and `js-yaml` both accept it. + + **A tag or anchor inside a flow collection ends at the flow indicator.** In + `{ foo : !!str, }` the tag token swallowed the comma, which the tag-character check + then reported while the missing comma left the mapping looking unterminated and + shifted every entry after it. Outside a flow collection those characters are still + ordinary tag content, so a block-context `!!str,` is still a `BAD_TAG`. + + **Tab-indented JSON parsed to the wrong value.** A wrapped flow line's leading + whitespace is `s-indent(n) s-separate-in-line?`, so tabs sit in it as spaces do — + but the flow scalar scanner skipped only spaces, so the `]` closing a tab-indented + line was never seen as the flow indicator it is and the line folded into the scalar + instead. `JSON.stringify(value, null, '\t')` — what `jq --tab` and every + "indent with tabs" editor setting emit — therefore turned the last entry before a + `]` into a string with a trailing newline: `-1` came back as `"-1\n"`. + + **The 1024-character implicit key limit is enforced in block context.** YAML caps + how far past a key's start its `:` may sit so a processor can recognize a mapping + entry with bounded lookahead; a longer block key is now `BAD_IMPLICIT_KEY`. It is + deliberately _not_ enforced in flow context, matching `yaml` (eemeli): a flow + mapping is where JSON lives, `{"…1100 characters…": 1}` is valid JSON, and + rejecting a valid JSON document is the worse of the two errors. Relatedly, an + explicit key in a flow sequence may now put its `:` on the next line + (`[ ? a\n : b ]`) — the one-line rule exists to keep an _implicit_ key cheap to + recognize, and a `?` settles that up front. + + **The JSON-superset property is now checked, not assumed.** `@amritk/lint` routes + `.json` documents through the YAML parser and `resolveRefsFromFile` hands it + whatever a `$ref` points at, so "JSON parses as YAML" is load-bearing. + `src/json-superset.test.ts` runs a generated corpus against `JSON.parse` — every + value in six spellings (compact, 2-space, tab-indented, CRLF, and with + leading/trailing blank lines), requiring an identical value _and_ zero diagnostics + for each — and `@amritk/lint` gains a matching test holding `parseJson` and + `parseYaml` to identical data, diagnostics, and `line:column` ranges for every path + in a JSON document. + +- Updated dependencies [213ecc4] +- Updated dependencies [798fd7a] +- Updated dependencies [2c9982c] +- Updated dependencies [bc09e15] +- Updated dependencies [b152c4e] +- Updated dependencies [15e480e] +- Updated dependencies [140412b] +- Updated dependencies [7839a38] +- Updated dependencies [007aa05] +- Updated dependencies [1b720e2] +- Updated dependencies [c1a176f] +- Updated dependencies [00eb0c9] + - @amritk/runtime-validators@0.10.0 + - @amritk/yaml@0.5.0 + ## 0.4.3 ### Patch Changes diff --git a/packages/lint/package.json b/packages/lint/package.json index 0176c3a8..0043418c 100644 --- a/packages/lint/package.json +++ b/packages/lint/package.json @@ -1,6 +1,6 @@ { "name": "@amritk/lint", - "version": "0.4.3", + "version": "0.4.4", "description": "A fast, format-agnostic JSON/YAML style-guide linter with JSON Schema and custom rules.", "main": "./dist/index.js", "module": "./dist/index.js", diff --git a/packages/resolve-refs/CHANGELOG.md b/packages/resolve-refs/CHANGELOG.md index edd5b1fc..2adf25b6 100644 --- a/packages/resolve-refs/CHANGELOG.md +++ b/packages/resolve-refs/CHANGELOG.md @@ -1,5 +1,250 @@ # @amritk/resolve-refs +## 0.5.0 + +### Minor Changes + +- d749ee2: Keep a `$dynamicRef` the dynamic scope has to answer, instead of inlining one + wrong target + + A `$dynamicRef` binds at _evaluation_ time to the outermost `$dynamicAnchor` of + its name along the chain of resources actually being applied, so the same keyword + can resolve to different schemas depending on where evaluation entered from. + Inlining happens once, which means a resolver that inlines every `$dynamicRef` is + guessing — and a wrong guess changes what the document accepts, in both + directions. + + So it no longer guesses. Where the binding is decidable it inlines as before: + + - a **pointer fragment** (`#`, `#/$defs/items`) is a plain `$ref` per the spec — + there is no anchor to late-bind to; + - an **anchor name declared at most once** in the document has only one schema the + dynamic lookup could ever reach. + + Where it is not decidable — an anchor name declared twice or more — the + `$dynamicRef` stays in the output, along with the scaffolding it needs to resolve + at validation time: the `$dynamicAnchor`s it may bind to and the `$id`s that + delimit the resources those anchors live in. Inlining anything whose copy would + drop a resource out of that chain is held back for the same reason. This is the + move the resolver already makes for a reference cycle — keep the reference rather + than collapse it to one wrong answer — now covering both cases under one rule. + + Consumers see no new API and no new errors: a kept reference is not a failure, it + is a reference the resolver could not answer without changing the document's + meaning, resolvable against the output exactly as it was against the input. + `trackOrigins` records nothing for one, because nothing was copied in its place. + + On the `$ref` corpus of the official JSON Schema Test Suite the package is now at + **170 / 170**. + + Two limits stay, documented in the code: a multi-document `resolveRefsFromFile` + still inlines (preservation only helps when the scaffolding survives into the + output, which a single-document resolve guarantees and a flattened multi-document + one does not), and 2019-09's `$recursiveRef` has the same defect but no corpus to + move it against. + +- 945e8f2: Close the local-file, cache-scoping, and fan-out gaps in the resolver's guards + + This package's selling point is a default-deny SSRF guard. These are the places + the guard did not reach. + + **Local `$ref`s are now confined to the root document's directory.** `$ref` + resolution against the filesystem had no containment check and no way to turn it + off: `{"$ref": "../../../etc/passwd"}` (or an absolute path) read whatever the + process could read, and any caller supplying the YAML `parse` callback the docs + recommend got arbitrary text, not just JSON. A local ref must now resolve under + `dirname(rootLocation)`; both the lexical and the symlink-resolved path have to + land inside it, so a symlink planted in the tree cannot be used to escape. + + This is a **behavior change**: a legitimate cross-directory ref + (`../common/schemas.json` — a very normal split-spec layout) now fails until you + widen it with the new `allowedRoots`, which the refusal message names. The + default was chosen to match the stance the remote path already took — deny, then + opt in — since the escaping ref and the traversal attack are the same shape and + only the caller can tell them apart. The root document you name is exempt; it is + what you asked for. `localRefs: false` refuses cross-file reads entirely. + + **The session cache no longer leaks documents across credentials.** It was keyed + by URL alone, so a call carrying one tenant's `Authorization` header handed that + tenant's private document straight to a later call carrying no credentials at + all — and in-flight coalescing additionally made the second caller inherit the + first one's `fetch`, `timeoutMs`, and `maxBytes`. Both keys now include a digest + of the effective headers plus the `fetch`/`parse` identities and the transfer + limits. The cache is also bounded now (10-minute TTL, 256 entries, LRU + eviction) instead of growing for the life of the process, and + `clearRemoteCache(url)` can drop a single document. + + **A resolve is bounded as a whole.** A root document with 500 `$ref`s to distinct + URLs drove 501 fetches with nothing but the per-hop timeout bounding it — an + egress amplifier and a host scanner, from the resolver's network position. New + `maxDocuments` (500) and `totalTimeoutMs` (60s, applied across the whole resolve + and not just per hop) cap it. + + **Deeply nested documents no longer throw.** `'{"a":'.repeat(20000)` raised + `RangeError: Maximum call stack size exceeded` out of the walkers, breaking the + package's stated contract that errors are collected and never thrown. Every + recursive walk is depth-capped (`maxDepth`, default 512); past the limit the + subtree is left unresolved and one `ResolveError` is recorded. + + **The SSRF guard is no longer name-blind.** `metadata.google.internal`, + `metadata.goog`, `metadata`, `instance-data`, and anything under the reserved + `.internal` TLD are refused by name — the IP check missed all of them, because + callers reach the metadata service by name. The new `assertPublicHost` also + resolves each remote hostname and refuses it when _any_ address it points at is + non-public, which closes the `127.0.0.1.nip.io` class of bypass; it fails closed, + and `verifyDns: false` (or an `allowedHosts` entry) opts out where names resolve + at an egress proxy. DNS rebinding is narrowed, not closed: pinning the connection + to the verified address is not something Node's `fetch` exposes, and the README + says so rather than overclaiming. + + **Missing IP ranges added:** `fec0::/10` (deprecated site-local), + `198.18.0.0/15` (benchmarking), and `192.0.0.0/24` (IETF protocol assignments). + + **`allowedHosts` is no longer a footgun.** Matching was case-sensitive and + port-exact, so `['example.com']` refused `https://example.com:8443/a.json` and + `['EXAMPLE.com']` refused everything — failing closed, but pushing users toward + `allowPrivateHosts`, which is a real hole. Entries now match case-insensitively; + an entry without a port matches any port, and one with a port must match it + (a URL that omits the port counts as its protocol default). + +### Patch Changes + +- 798fd7a: Measure every schema-consuming package against the official JSON Schema Test + Suite, the way `@amritk/yaml` is measured against the YAML test suite + + The required Draft 2020-12 tests (46 files, 383 groups, 1299 cases) are vendored + under `fixtures/json-schema-test-suite`, and four packages now run them on every + build. Each carries an expected-failure list naming every case it does not pass + and why, and each suite fails when a case moves in **either** direction — a + regression breaks the build, and so does a case that starts passing while its + entry stays behind. Nothing is published: the corpus and the harnesses live + outside every `files` list. + + | package | measured on | rate | + | ----------------------------- | ------------------------------------------------ | ------------------- | + | `@amritk/runtime-validators` | `validate` and `validateGuard` verdicts | 1250 / 1299 (96.2%) | + | `@amritk/generate-parsers` | strict parsers, generated → linked → executed | 1180 / 1299 (90.8%) | + | `@amritk/generate-validators` | generated predicate validators, likewise | 987 / 1299 (76.0%) | + | `@amritk/resolve-refs` | verdict preserved after inlining (`$ref` corpus) | 160 / 170 (94.1%) | + + The generators are measured through the code they emit, not the source text they + emit: each suite schema is generated whole, compiled, and linked in memory, so the + `$ref`'d sibling files and the embedded runtime helpers run too. `resolve-refs` + has no verdicts of its own, so it is held to semantic preservation — the resolved + document must accept exactly what the original did, judged by + `@amritk/runtime-validators` over the cases the interpreter already answers + correctly, which is the population where a resolution bug is visible and nothing + else is. + + Those rates are where the packages _end up_. The suites were written first and + found real defects — a validator that accepted everything for a schema without a + `type`, `required` satisfied by an inherited `toString`, refs that emitted + uncompilable output, `$ref`-shaped data inlined as a reference — each fixed in its + own commit alongside this one. What remains is documented case by case, and each + package's README carries a "Conformance, measured" section with its number and the + reasons behind it. + +- 2c9982c: Fix the published manifests so the packages install, resolve, and dedupe correctly + + **Types resolve on TypeScript's default config.** Every package was + exports-only: nine declared `"module": "./dist/index.js"` (a field neither Node + nor TypeScript reads) and nothing declared `types`. A consumer on + `moduleResolution: "node10"` — still the default when `module` is `commonjs` — + cannot see `exports` at all, so `import { lintDocument } from '@amritk/lint'` + failed with `TS2307: Cannot find module '@amritk/lint' or its corresponding type +declarations`. Each package with a `.` export now also declares `main` and + `types`; `@amritk/helpers` and `@amritk/adapters` have no `.` export (they are + subpath-only), so they declare a `typesVersions` wildcard mapping instead, which + gives their subpaths the same node10 fallback. All of it is ignored under + `node16`/`nodenext`/`bundler`, where `exports` still wins. + + **`workspace:*` resolves to a caret, not an exact pin.** All fourteen + inter-package edges shipped as exact versions, so installing two `@amritk/*` + packages published at different times pulled in two copies of their shared + dependency. That is not merely wasteful: the module-level caches those packages + rely on are per-copy, so the `WeakMap` validator cache in + `@amritk/runtime-validators` silently stopped hitting. Pre-1.0 a caret stays + narrow (`^0.9.1` is `>=0.9.1 <0.10.0`) and breaking changes here already ride a + minor bump. + + **`@amritk/helpers` stops shipping 21 source files it does not need.** Embedded + mode reads four helper sources (`is-object`, `validate-array`, + `validate-record`, `has-ref`) out of the installed package at generation time, + so `src` has to ship — but only those four. `files` now lists them explicitly + instead of globbing all of `src`, cutting the tarball from 78 files / 206 kB to + 63 / 112 kB. + + **Two packages no longer declare a dependency they never import.** + `@amritk/mjst` and `@amritk/generate-parsers` both listed + `@amritk/generate-markdown` under `dependencies`, but the only importer is each + package's `scripts/generate-readme.ts`, which is not published. Both moved to + `devDependencies`. `@amritk/adapters` likewise dropped its + `@sinclair/typebox` peer dependency: the TypeBox adapter is purely structural + (it strips symbol keys) and imports nothing. `valibot` stays — it is a genuine + transitive peer of `@valibot/to-json-schema`. + + **`@amritk/mjst` fixes.** `json-schema-typed` moved to `dependencies`, because + the shipped `dist/emit-examples.d.ts` imports types from it. The package gained + an `exports` map, so it is no longer deep-importable in its entirety. And the + build now marks `dist/cli.js` executable: `npm pack` records on-disk modes, and + package managers only `chmod` bin targets when they link them, so flows that + consume the tarball directly (vendoring, Docker `npm pack` + `tar -x`) hit + `EACCES`. + +- 08b2833: Resolve a `#/pointer` inside an `$id` scope against the resource that declares it + + A fragment-only ref was hard-coded to resolve against the document root, so + `{ "$id": "…/base.json", "$defs": { "inner": … }, "properties": { "x": { "$ref": "#/$defs/inner" } } }` + nested inside a larger document reported "Cannot resolve internal `$ref`" — the + pointer names a definition of the _embedded resource_, not of the root. It now + looks in the resource named by the base URI in scope first, and falls back to the + document root only when the pointer matches nothing there. The fallback is what + keeps bundled documents working: a bundled OpenAPI file points at + `#/components/schemas/…` from inside an `$id` scope, and when both could match the + resource wins, which is the order the spec asks for. + + That also settles pointer-form `$dynamicRef`s (`#/$defs/items`) inside an `$id` + scope: the spec says a `$dynamicRef` whose fragment is a pointer behaves exactly + like `$ref`, so resolving it against its enclosing resource is right by + construction. + + On the `$ref` corpus of the official JSON Schema Test Suite the package is at + **160 / 170**. The corpus grew from 107 with `@amritk/runtime-validators`' `$id` + work — it is the reference-carrying cases the interpreter answers correctly, which + is the population where a resolution bug is visible at all. What is left is one + documented limit: a `$dynamicRef` binds at evaluation time to the outermost + `$dynamicAnchor` along the _dynamic_ scope, so inlining it statically collapses it + to a single target and cannot be right in general. + +- f9f790a: Stop inlining a `$ref`-shaped object that is data, not a reference + + `{ "$defs": { "a_string": { "type": "string" } }, "enum": [ { "$ref": "#/$defs/a_string" } ] }` + references nothing. `enum` holds _instances_, and one of them happens to be an + object with a `$ref` key — but the walk was purely structural, so it inlined that + object and turned "the enum containing `{"$ref": …}`" into "the enum containing + `{"type": "string"}`", changing what the document matches in both directions. The + official suite carries the case under exactly that name: _"naive replacement of + `$ref` with its destination is not correct"_. + + Every structural walk in the package now carries the **role** of the node it is + at — a schema, a map of author-chosen names to schemas, instance data, or + something outside the vocabulary. `enum` / `const` / `default` / `examples` hand + their subtree back untouched; `properties` / `patternProperties` / `$defs` / + `definitions` / `dependentSchemas` / `dependencies` suppress keyword reading one + level down, so a definition legitimately _named_ `enum` is still a definition and + a property named `$ref` is still a property — the trap in the naive version of + this fix, which the resource registry had; and an unrecognized keyword yields + `unknown`, which is absorbing, so OpenAPI's `components`/`paths` and `x-` vendor + blocks are walked exactly as before. + + Two consequences beyond the inlining itself: the resource registry no longer + registers an `$id`/`$anchor` that is part of a value or a property name, and + `resolveRefsFromFile` no longer reads a file or opens a network connection for a + `$ref` string sitting inside an `enum`. + + This takes the package to **107 / 107** on the `$ref` corpus of the official JSON + Schema Test Suite, with an empty expected-failure list. + ## 0.4.5 ### Patch Changes diff --git a/packages/resolve-refs/package.json b/packages/resolve-refs/package.json index 26cc70c2..e972da54 100644 --- a/packages/resolve-refs/package.json +++ b/packages/resolve-refs/package.json @@ -1,6 +1,6 @@ { "name": "@amritk/resolve-refs", - "version": "0.4.5", + "version": "0.5.0", "description": "Resolve and inline JSON Schema / OpenAPI $refs — internal, cross-file, and remote — with session caching and a default-deny SSRF guard.", "main": "./dist/index.js", "module": "./dist/index.js", diff --git a/packages/runtime-validators/CHANGELOG.md b/packages/runtime-validators/CHANGELOG.md index 10e76826..5d3cefb7 100644 --- a/packages/runtime-validators/CHANGELOG.md +++ b/packages/runtime-validators/CHANGELOG.md @@ -1,5 +1,362 @@ # @amritk/runtime-validators +## 0.10.0 + +### Minor Changes + +- bc09e15: `validateGuard` stops narrowing where the inferred type cannot describe every + accepted value + + `FromSchema` infers an object shape from applicator keywords alone, so + `{ properties: { a: { type: 'string' } } }` infers `{ a?: string }`. The + interpreter — correctly — accepts a non-object against that schema, because JSON + Schema's object keywords ignore values that are not objects. The guard was + therefore handing back `input is { a?: string }` for a `42` it had just approved. + + For exactly those schemas — no `type`, `enum`, `const` or `$ref`, but + `properties`, `required`, `additionalProperties`, `patternProperties`, + `prefixItems` or `items` present, recursing through `allOf`/`anyOf`/`oneOf` + branches — `validateGuard` now returns a `Check` instead of a `Guard`: the + same runtime function, no type predicate. Every schema that declares a `type` (or + `enum`/`const`/`$ref`) keeps its predicate, and so does a schema whose type is not + a literal — narrowing is surrendered only when the inference is _demonstrably_ + partial, never because the checker could not decide. + + `Check` keeps the erased phantom carrier `Validator` already uses, so + `Infer` still recovers the schema's type rather than collapsing to + `never`, and it is assignable anywhere `(input: unknown) => boolean` is. It reads + as "checks for this, does not claim it". + + The runtime is untouched. This mirrors the same fix in + `@amritk/generate-validators`, whose generated `isX` had the identical hole — the + two now tell one story about the same schemas, and the type-level predicate sits + next to `ImplicitShape` so the keyword lists cannot drift apart. + +- b152c4e: Resolve `$ref` against `$id` as a base URI, and give `$dynamicRef` a real dynamic + scope + + Measured against the official JSON Schema Test Suite, the interpreter goes from + 1183/1299 to **1250/1299 (96.2%)**. Sixty-seven cases, one cause: a `$ref` written + against an `$id` — relative (`"list"`), absolute + (`"http://example.com/b/d.json"`), or a URN — had nothing to resolve against and + threw, even when the resource it named sat _inside the same document_. + + The document is now walked once into a registry of its embedded resources: each + `$id` composed against the base of its parent, and each resource's `$anchor`s and + `$dynamicAnchor`s registered under it. A ref resolves against the base in scope at + the referring node — relative, absolute, URN, absolute-path, pointer-into-resource + and anchor-in-resource forms all work — and `$dynamicRef` implements bookending + properly: it goes dynamic only when static resolution already lands on a + `$dynamicAnchor` of that name, then takes the outermost resource in the dynamic + scope declaring it. + + Two behavior changes fall out of that, both spec-correct and both confined to + documents that declare an `$id`: + + - A `#/pointer` inside an `$id` scope resolves within that resource rather than at + the document root. A scoped ref that names nothing in its own resource still + falls back to the document-global lookup, so a bundled schema that worked before + works unchanged — the new path can only _add_ an answer. + - `contains` publishes the indices it matched rather than sweeping the whole + array, so an adjacent `unevaluatedItems` sees the right set. This is where the + spec and Ajv disagree; the suite agrees with the spec, and so do we. The pair is + excluded from the Ajv differential corpus and covered by unit tests plus the + suite instead. + + Cost is kept off the common path: the registry is `null` for a document with no + `$id` at all, the `$id` scan is fused into the pattern-screening walk that already + happened, and resolutions are memoized per validator. Eval-free, synchronous, + zero-dependency and no-I/O all hold. + + What remains unimplemented is now one decision rather than two: this package does + no I/O, so a `$ref` naming _another document_ (and `$vocabulary`, which means + fetching a metaschema) still throws. Bundle with `@amritk/resolve-refs` first. + +- 140412b: Take documents the caller already has: `validate(schema, { schemas })` + + The interpreter does no I/O — no `fetch`, no filesystem — which is what lets it + run under a strict CSP and on Workers. Until now that also meant it could not be + _told_ about a document it did not receive, so a `$ref` naming another schema + threw and the answer was always "bundle it first". + + `ValidateOptions.schemas` closes that without giving up anything: a plain record + of absolute URI → document, for schemas the caller has already loaded. A + registered document is a full schema resource — walked under its retrieval URI, so + its `$id`, `$anchor`s, `$dynamicAnchor`s and nested embedded resources all + register, a document with no `$id` resolves relative refs against the URI it was + registered under, and one whose `$id` disagrees answers to both. Cross-document + `$dynamicRef` bookending works. A URI that was _not_ registered still throws, now + with a message showing how to supply it. + + It is a record rather than an `addSchema` call on purpose: `addSchema` implies + mutable global state, and this package stays a pure function of its inputs. Pass + the registry as an immutable value — the prepared-validator cache keys on its + identity _and_ its URI set, so adding or removing a document is a cache miss + rather than a stale hit (swapping the contents under a URI in place is + undetectable, exactly as mutating the schema object is, and is documented as + such). + + With the metaschema registered, `$vocabulary` can finally be read: a custom + dialect that omits the validation vocabulary turns `minimum` and friends into + annotations instead of assertions. Two limits, both documented: it is read from + the root `$schema` rather than per schema resource, and it defaults to enforcing + whenever the metaschema was not registered, which is the stricter answer. + + Nothing changes for callers who pass no registry: the key work is skipped, the + registry build stays gated on the document declaring an `$id`, and the vocabulary + check short-circuits. + + **The package now passes the official JSON Schema Test Suite in full — 1299 / 1299 + required Draft 2020-12 cases.** The harness hands the suite's own `remotes/` + documents to `schemas`, which is the sanctioned equivalent of the HTTP server the + suite would otherwise expect: same documents, same URIs, handed over instead of + fetched, with the interpreter still doing all the base-URI, anchor and + cross-document work the cases exist to test. + + The dialect itself ships alongside, as an opt-in subpath: + + ```ts + import { metaschema } from "@amritk/runtime-validators/metaschema"; + + validate(userSchema, { schemas: metaschema }); // "is this a valid 2020-12 schema?" + ``` + + Eight documents (the dialect plus its seven vocabulary metaschemas), ~7.9 KB of + JSON, reachable only through that subpath — the main entry never imports it, so a + caller who does not ask for it ships none of it. A test holds the copy to Ajv's + vendored specification text by deep equality, which makes Ajv a _check_ on the + transcription rather than a runtime dependency of it. + +### Patch Changes + +- 213ecc4: Take documents you already loaded, so a `$ref` to another document generates + + **On the official JSON Schema Test Suite: `generate-validators` 1238 → 1268 / + 1281 (99.0%), `generate-parsers` 1222 → 1237 / 1281 (96.6%).** + + Both generators gain a `schemas` option: documents you have already loaded, keyed + by the absolute URI a `$ref` names them by. It is the build-time counterpart of + `@amritk/runtime-validators`' `ValidateOptions.schemas`, and it keeps the same + promise — nothing is fetched, you cannot pass a URL, only a document. What changes + is that "we do no I/O" no longer also means "we cannot be told". + + A cross-document `$ref` was the single largest gap in both packages, and it is + gone. `refRemote.json` passes in full; so do the `dynamicRef.json` groups that + reach `tree.json` and `extendible-dynamic-ref.json`, and — with the dialect + metaschema registered — `defs.json` and `ref.json`'s "remote ref, containing refs + itself". + + Each registered document becomes a resource of the document being generated: its + `$id`, its `$anchor`s and `$dynamicAnchor`s and its own embedded resources all + resolve, a `$ref` from one registered document into another resolves, and every + definition reached gets a file, a type and a validator/parser by the ordinary + rules. A document with no `$id` resolves its relative `$ref`s against the URI it + was registered under; one whose `$id` disagrees answers to both. Registering more + than the schema uses costs nothing — only the documents actually reached are + emitted — and a `$ref` to a URI nobody registered still stops the build with a + message naming the ref. + + The mechanism is one pass, not a second addressing mode. `@amritk/helpers` gains + `graftExternalSchemas`, which embeds the registered documents into the root before + the `$id` pass, and `pruneExternalSchemas`, which drops the unreferenced ones once + the refs are pointers and reachability is finally knowable. Everything downstream — + the ref-graph walk, the naming, the emitted import graph — keeps working on a + single document and needed no change. `walkRefGraph` carries the option and + memoizes per `(schema, schemas)` by identity. + + **Fixed: a root schema with a union `type` dropped every sibling constraint.** + `{ type: ['object', 'boolean'], properties: {…}, required: [...] }` emitted the + type check and nothing else, so it accepted any object at all. The multi-type root + branch now emits the shared constraint checks the single-type and combinator + branches already did; they carry their own runtime-type guards, so a member of the + union a constraint does not apply to is still untouched. This is the shape the + 2020-12 metaschema's own root is written in, which is how it went unnoticed — the + generated dialect validator accepted `{ type: 1 }` as a valid schema. + + `@amritk/runtime-validators` is unchanged in behaviour; its conformance figures are + restated against the corpus that is actually vendored (1281 cases, not 1299 — the + README's count never matched, and upstream's `content.json` is not among the + vendored files). The suite's `remotes/` loader moves to the shared fixtures + bookkeeping so all four conformance suites use one walk. + +- 798fd7a: Measure every schema-consuming package against the official JSON Schema Test + Suite, the way `@amritk/yaml` is measured against the YAML test suite + + The required Draft 2020-12 tests (46 files, 383 groups, 1299 cases) are vendored + under `fixtures/json-schema-test-suite`, and four packages now run them on every + build. Each carries an expected-failure list naming every case it does not pass + and why, and each suite fails when a case moves in **either** direction — a + regression breaks the build, and so does a case that starts passing while its + entry stays behind. Nothing is published: the corpus and the harnesses live + outside every `files` list. + + | package | measured on | rate | + | ----------------------------- | ------------------------------------------------ | ------------------- | + | `@amritk/runtime-validators` | `validate` and `validateGuard` verdicts | 1250 / 1299 (96.2%) | + | `@amritk/generate-parsers` | strict parsers, generated → linked → executed | 1180 / 1299 (90.8%) | + | `@amritk/generate-validators` | generated predicate validators, likewise | 987 / 1299 (76.0%) | + | `@amritk/resolve-refs` | verdict preserved after inlining (`$ref` corpus) | 160 / 170 (94.1%) | + + The generators are measured through the code they emit, not the source text they + emit: each suite schema is generated whole, compiled, and linked in memory, so the + `$ref`'d sibling files and the embedded runtime helpers run too. `resolve-refs` + has no verdicts of its own, so it is held to semantic preservation — the resolved + document must accept exactly what the original did, judged by + `@amritk/runtime-validators` over the cases the interpreter already answers + correctly, which is the population where a resolution bug is visible and nothing + else is. + + Those rates are where the packages _end up_. The suites were written first and + found real defects — a validator that accepted everything for a schema without a + `type`, `required` satisfied by an inherited `toString`, refs that emitted + uncompilable output, `$ref`-shaped data inlined as a reference — each fixed in its + own commit alongside this one. What remains is documented case by case, and each + package's README carries a "Conformance, measured" section with its number and the + reasons behind it. + +- 2c9982c: Fix the published manifests so the packages install, resolve, and dedupe correctly + + **Types resolve on TypeScript's default config.** Every package was + exports-only: nine declared `"module": "./dist/index.js"` (a field neither Node + nor TypeScript reads) and nothing declared `types`. A consumer on + `moduleResolution: "node10"` — still the default when `module` is `commonjs` — + cannot see `exports` at all, so `import { lintDocument } from '@amritk/lint'` + failed with `TS2307: Cannot find module '@amritk/lint' or its corresponding type +declarations`. Each package with a `.` export now also declares `main` and + `types`; `@amritk/helpers` and `@amritk/adapters` have no `.` export (they are + subpath-only), so they declare a `typesVersions` wildcard mapping instead, which + gives their subpaths the same node10 fallback. All of it is ignored under + `node16`/`nodenext`/`bundler`, where `exports` still wins. + + **`workspace:*` resolves to a caret, not an exact pin.** All fourteen + inter-package edges shipped as exact versions, so installing two `@amritk/*` + packages published at different times pulled in two copies of their shared + dependency. That is not merely wasteful: the module-level caches those packages + rely on are per-copy, so the `WeakMap` validator cache in + `@amritk/runtime-validators` silently stopped hitting. Pre-1.0 a caret stays + narrow (`^0.9.1` is `>=0.9.1 <0.10.0`) and breaking changes here already ride a + minor bump. + + **`@amritk/helpers` stops shipping 21 source files it does not need.** Embedded + mode reads four helper sources (`is-object`, `validate-array`, + `validate-record`, `has-ref`) out of the installed package at generation time, + so `src` has to ship — but only those four. `files` now lists them explicitly + instead of globbing all of `src`, cutting the tarball from 78 files / 206 kB to + 63 / 112 kB. + + **Two packages no longer declare a dependency they never import.** + `@amritk/mjst` and `@amritk/generate-parsers` both listed + `@amritk/generate-markdown` under `dependencies`, but the only importer is each + package's `scripts/generate-readme.ts`, which is not published. Both moved to + `devDependencies`. `@amritk/adapters` likewise dropped its + `@sinclair/typebox` peer dependency: the TypeBox adapter is purely structural + (it strips symbol keys) and imports nothing. `valibot` stays — it is a genuine + transitive peer of `@valibot/to-json-schema`. + + **`@amritk/mjst` fixes.** `json-schema-typed` moved to `dependencies`, because + the shipped `dist/emit-examples.d.ts` imports types from it. The package gained + an `exports` map, so it is no longer deep-importable in its entirety. And the + build now marks `dist/cli.js` executable: `npm pack` records on-disk modes, and + package managers only `chmod` bin targets when they link them, so flows that + consume the tarball directly (vendoring, Docker `npm pack` + `tar -x`) hit + `EACCES`. + +- 15e480e: Widen the ReDoS screen, fix five correctness defects, and cut five allocations off the hot path + + **The ReDoS screen only looked where it expected schemas to be.** It walked a + fixed list of subschema keywords, so an OpenAPI-shaped document — subschemas + parked under `components/schemas` and reached by `$ref` — was declared clean and + its `pattern`s were then compiled and run unscreened. `{ $ref: +'#/components/schemas/A', components: { schemas: { A: { pattern: '^(a+)+$' } } } }` + burned ~1.3 s of CPU on a 31-character input, while the same pattern inlined at + the root was correctly rejected. The walk is now unrestricted: every + string-valued `pattern` key and every `patternProperties` key anywhere in the + document is screened, wherever it sits. Chasing `$ref`s instead would have fixed + that one layout and missed the next unfamiliar one. `const`, `enum`, `default`, + `examples` and `example` are still skipped, because a schema is allowed to carry + arbitrary data there and `{ const: { pattern: '(a+)+' } }` describes an object, + not a regex. This does cost cold build time in proportion to the document + actually being screened — an ordinary component schema is unchanged (~0.016 ms), + but handing `validate` a whole OpenAPI document now costs ~0.25 ms once, where + the old walk visited almost none of it. + + **The screen's documented guarantee was false, and is now both honest and + stronger.** It claimed to "flag a few benign patterns, never the reverse", but it + only recognized _nested_ unbounded quantifiers: `^(a|a)+$` is star height 1, so + it passed — and takes over a second on a 29-character input, doubling with each + added character. The screen now also rejects a provably ambiguous alternation + under an unbounded quantifier (two branches that match the same single + character), and the docs say plainly that this is a filter for recognizable + shapes, not a proof of safety — `(a|aa)+` and `a*a*$` still get through. The new + rule is deliberately sound rather than broad: the tempting "overlapping first + characters" test would flag `(ab|ac)+`, which is linear. Zero new flags across a + sweep of 27 ordinary real-world patterns. + + **A deeply nested schema threw an uncatchable `RangeError`.** The pattern screen + and the `$anchor` search both recursed per schema level, and both run before + `maxDepth` applies — so 20,000 nested `{ "not": … }` levels overflowed the native + stack, `isValidationLimitError` returned `false`, and a consumer's limit handler + fell through to a 500. (At 10,000 levels it correctly threw + `ValidationLimitError`.) Both walks are now iterative with an explicit stack, so + the depth cap does its job and an anchor buried 20,000 levels down still + resolves. + + **`required` was silently unenforced for prototype-member names.** The + leftover-required list was built with `k in properties`, which walks + `Object.prototype` — so `'toString' in {}` was `true`, the key looked already + covered and was dropped, and it was absent from the declared-key list too (that + comes from `Object.keys`). Nothing checked it: `{ required: ['constructor'], +properties: {} }` accepted `{}`. Ajv shares this bug by default, so the + differential fuzz could not catch it; there are explicit tests now. + + **`format: 'ipv4'` accepted leading zeros** (`01.2.3.4`), the classic + octal-interpretation allowlist bypass, and the same octets are embedded in the + IPv6 grammar. **`format: 'time'` accepted a bare `12:00:00`** with no offset, + which RFC 3339 `full-time` requires. Both now match Ajv exactly. + **`minProperties`/`maxProperties` counted inherited properties** — a `for…in` + without an own-property guard — so `Object.create({ inherited: 1 })` with one own + key satisfied `minProperties: 2`. + + **Five hot-path costs, measured before and after:** + + - The `enum` failure message was built eagerly and thrown away in guard mode. A + 500-value enum cost 16.4k ops/s on a miss versus 5.1M on a hit — ~99% of the + work was a discarded string. This also hit the _valid_ path, because every + non-matching `anyOf`/`oneOf` branch probe runs in guard mode: a 20-branch + discriminated union with `enum` discriminators went 9.5k → 251k ops/s (26×). + The miss itself is now 5.6M ops/s (340–540×). + - `contains` evaluated every element even after it had enough matches. A + 1000-element array matching at index 0 went 8.1k → 5.1M ops/s (630–740×). The early + exit is taken only when `maxContains` is absent and no annotation scope is + active — both need the exact total. + - `dependentRequired` / `dependentSchemas` / `dependencies` rebuilt their + `Object.entries` on every validation. Their entry lists are now memoized on the + per-node metadata alongside the property keys and compiled `patternProperties`, + worth 1.3–1.9× on a one-entry keyword. An `additionalProperties`-only object + schema also stopped allocating a throwaway empty pattern array per call (1.12×). + - `propertyNames` allocated a nine-field interpreter context per key. One scratch + context is now reused across the key loop — safe because the only per-probe + state is the `failed` flag and these probes cannot nest, the key being a string. + A 20-key object gains 1.9–2.5×. + - The own-property count for `minProperties`/`maxProperties` uses + `Object.keys().length`, which measured 92M ops/s against 19M for the old + unguarded `for…in` and 9.5M for a `for…in` with a `hasOwn` guard — so the fix is + also 1.14× faster than the bug. + + **The per-schema validator cache is bounded.** The outer `WeakMap` collects with + the schema, but the inner `Map` keyed on mode/formats/limits lived as long as the + schema did, so a caller deriving `limits` per request pinned a validator forever: + 200,000 distinct values retained 82.3 MB. Past 16 configurations it now hands + back an uncached validator (0.5 MB), which costs nothing — there is no compile + step. + + **Two documentation claims corrected.** The README said valid input "and the + entire guard path allocates nothing"; branch probes, annotation trackers and + `uniqueItems` sets all allocate, so it now says what is actually true — nothing + is built for errors that never happen. And the `$ref` cycle-break comment claimed + "stopping here changes no verdict", which holds in a conjunctive position but not + inside a disjunction, where returning valid _is_ a verdict. + ## 0.9.1 ### Patch Changes diff --git a/packages/runtime-validators/package.json b/packages/runtime-validators/package.json index 559cbeeb..3e24cd84 100644 --- a/packages/runtime-validators/package.json +++ b/packages/runtime-validators/package.json @@ -1,6 +1,6 @@ { "name": "@amritk/runtime-validators", - "version": "0.9.1", + "version": "0.10.0", "description": "Extremely fast, eval-free runtime JSON Schema validation. Interprets a schema you do not know ahead of time — no new Function, no build step, runs anywhere.", "main": "./dist/index.js", "module": "./dist/index.js", diff --git a/packages/yaml/CHANGELOG.md b/packages/yaml/CHANGELOG.md index 9d6c3c48..24b0a270 100644 --- a/packages/yaml/CHANGELOG.md +++ b/packages/yaml/CHANGELOG.md @@ -1,5 +1,317 @@ # @amritk/yaml +## 0.5.0 + +### Minor Changes + +- 7839a38: Read the block shapes that were folding structure into strings + + Nineteen more YAML test suite cases — 365/402 to **384/402 (95.5%)**. Every one + of them was a document whose structure the parser flattened into text or + orphaned entirely, and none of them produced a diagnostic saying so. + + **Node properties written on a mapping key now apply to the key.** `&a a: b` + anchors the scalar `a`, so `*a` is the string `"a"`; before, the anchor stayed + inside the key's text and every alias to it reported `UNRESOLVED_ALIAS` and + projected to nothing. The same for tags — `!!str 23: v` keyed the mapping by the + literal `"!!str 23"` rather than by `"23"`. Anchor names that hold a `:` work + too (`&a: key: value` anchors `key` as `a:`), which needs the properties scanned + before the key separator is looked for, not after. Properties on a line of their + own above the mapping still describe the mapping. Reaching this costs one + character comparison per mapping entry — the scan itself only runs for a key + that is actually annotated. + + **A `: ` inside a plain scalar is reported** (`BAD_SCALAR_CONTENT`). The spec + ends a plain scalar there, so `a: b: c: d` is an error and not the string + `"b: c: d"`; so is a continuation line that reads as a mapping entry, which is + what a mis-indented `k1: v1` / `⟨space⟩k2: v2` is. The scan is an `indexOf`, so + a scalar with no colon in it — nearly all of them — pays one native pass that + finds nothing. Quoted, block, and flow scalars are unaffected. + + **Block collections may open on an explicit entry's introducer line.** `? a` / + `: - one` is a sequence whose first entry shares the `:` line, and + `? earth: blue` a mapping whose first entry shares the `?` line; both folded + into the value as text. Their remaining entries align under that first one, not + under the introducer. The mirror-image shape after an _implicit_ key + (`key: - a`) is invalid YAML and is now reported rather than folded. Relatedly, + a `? ` introducer is settled by the first two characters, so it outranks any + `: ` further along the line — testing the colon first read `? earth: blue` as a + key called `? earth`. + + **Indentation is measured against the parent, not against the node.** The + parser treated "the parent's column" as one less than the column the node + started at, which is only true when a node begins exactly one column in. When it + did not, documents were cut short: a plain scalar stopped at the first + continuation line that stepped back (`a:` / `⟨2 spaces⟩foo` / `⟨1 space⟩bar` + dropped `bar`, and a sequence entry that wrapped was split into two entries), a + zero-indented sequence introduced by a tag or an anchor was orphaned and + reported as stray content (`sequence: !!seq` over a `- entry` list, `seq:` / + `⟨1 space⟩&anchor` over one), and a block scalar counted its indentation + indicator from the wrong column. The document root still measures against -1, so + `--- |2` is unchanged. + + The `parseAllDocuments` / `parseDocument` API, `toJS()` projection, and every + node's `[start, end)` span are unchanged, and the parser's throughput is + unchanged on all three benchmark fixtures. + +- 007aa05: Support lone-`\r` line breaks, stop `<<` dropping inherited-name keys, and make + `nodeAtPath` agree with `toJS()` + + Five silent-data-loss bugs, each in a path the differential corpus and the + yaml-test-suite do not reach. + + **A lone `\r` truncated the document, with no diagnostic at all.** The scanner + skipped to the next line by looking only for `\n`, so a CR-delimited document had + every line after the first jumped over — `a: 1\rb: 2\rc: 3\r` parsed to + `{ a: 1 }` and reported zero errors, and a single stray CR inside an otherwise-LF + file made one key vanish. YAML 1.2 §5.4 makes all three of `CR LF`, `CR`, and + `LF` a line break, and now so do we — in the parser and in `lineCounter`, so + positions stay exact. `CR LF` still counts once. The differential suite re-runs + every case in all three break styles. + + **`<<` dropped any merged key that shares a name with an `Object.prototype` + member.** The "does the target already have this key?" test walked the prototype + chain, so `toString`, `valueOf`, `constructor`, `hasOwnProperty`, + `isPrototypeOf`, `__proto__` and friends were silently discarded from the merge. + Only own keys shadow a merged one now; `__proto__` is still defined as plain data + rather than assigned through the prototype setter, so the pollution guard holds. + + **`nodeAtPath` could not find the keys `toJS()` produces.** It carried its own + simplified key-stringifier that returned `'null'` for a null key, `'*ref'` for an + alias key, and `''` for every collection key — so `null: v`, `*a : v`, and + `[a, b]: v` were unreachable by path, and a `closest: true` lookup quietly + returned the _parent's_ source span: a diagnostic pointing at the wrong line. It + now uses the parser's own projection, which is exported as `keyText` for anyone + building paths by hand. + + **An unterminated quoted scalar lost its last character.** The recovery sliced + off a closing quote that was never there, so `a: "abcd` recovered as `"abc"` — + the wrong text for a linter to echo back. The `UNTERMINATED_QUOTE` error was + always correct; the text now is too. + + **`parseDocument` truncated a `---` stream without saying so.** Reading only the + first document is intended, but a caller on `parse()` sees only the data. It now + pushes a `MULTIPLE_DOCUMENTS` warning pointing at the marker and naming + `parseAllDocuments`. A trailing marker with nothing under it stays quiet. + + Also: `lineCounter` builds its index with `indexOf` instead of a per-character + loop — 2.6–3.6× faster, taking it from ~18% of parse+index cost to ~6%. And the + bundle-size benchmark now bundles a consumer of each parser rather than the + barrel, which tree-shook to a 156-byte stub and made the README's size table + fiction; the corrected numbers are in the README. + +- 1b720e2: Parse the node written on a `---` line, and stop flow scalars losing their type + + Four silent-data-loss bugs, each in a branch that was already cold. + + **A node written on the `---` line was discarded.** The document head skipped + the whole marker line, so `--- |` lost its block-scalar indicator and re-read + the body as a folded plain scalar (line breaks gone), `--- foo` lost the scalar + entirely, and a tag or anchor on the marker line never applied. The node is now + parsed and measured against column 0 rather than the column the marker pushed it + to, so `--- >` may hold content starting at column 0. A block _mapping_ on the + marker line is invalid YAML and is now reported. This is also what makes + `--- !!set` / `--- !!omap` reach their `Set` / `Map` projections. + + **A flow scalar that ended its line lost its core-schema type.** `{ a: 1, b: 2 }` + resolved `b` to the number `2`, but the same document wrapped — + `{ a: 1,\n b: 2 }` — resolved it to the _string_ `"2"`, because the multi-line + path folded the segments without resolving them. Every entry of a flow + collection written across lines was affected. Such a scalar also now ends at a + `:` or `#` that opens the next line, instead of folding it in — `{foo\n: bar}` + used to key the mapping by `"foo\n"`. + + **Double-quoted folding ran before escapes were resolved,** so it could not tell + an escaped `\t` (content) from a literal trailing tab (padding), and it turned a + `\` line-continuation's break into a space the `\` then absorbed. Escapes are now + resolved per line first, and folding strips only whitespace the document wrote + literally. + + **A block-folded scalar treated only a space as "more indented",** so a break + beside a tab-led line folded to a space and the blank line next to it was lost. + + Also: an unterminated quoted scalar now stops at a `---`/`...` marker instead of + swallowing every document after it; `!!str` over a wrapped plain scalar reads the + folded text rather than un-folding it; and the stream-level directive rules are + enforced — a directive needs a `...` before it and a `---` after it, its version + must parse, a second `%YAML` is an error rather than a warning, and a tag may not + hold a flow indicator. New code: `UNEXPECTED_DIRECTIVE`. + + Conformance against the official YAML test suite is **336/402 (83.6%)**, up from + 293/402, with every remaining gap still listed and reasoned. + +- c1a176f: Report the syntax errors that were quietly changing what a document said + + Twenty-nine more YAML test suite cases, every one of them in a branch that was + already cold — a block scalar header, a backslash, a `#`, a node property. + + **A block scalar header was accepted whatever followed it.** `folded: > first +line` dropped `first line` and re-read the body below as the scalar; `|10` took + the `1` as an indentation indicator and threw the `0` away; `>#comment` read a + comment the spec does not allow there; and a repeated indicator (`|--`) silently + kept the last one. The header now ends where its indicators do, and anything + past it is a `BAD_BLOCK_HEADER`. A leading blank line indented deeper than the + block's first content line — which makes the block's own indentation ambiguous — + is a `BAD_INDENT`. + + **A `\` escape the spec does not define passed through as the bare letter,** so + `"a\.b"` became `a.b` and `"it\'s"` became `it's`, each silently dropping a + character the document wrote. Undefined escapes are now `BAD_ESCAPE`; the value + is still produced, so nothing that parsed stops parsing. + + **A `#` with no whitespace before it is not a comment.** `key: "value"# text`, + `[ a, b ]#text`, and `[ a, b,#text` each dropped the rest of the line as though + it were one (`BAD_COMMENT`). The mirror image is fixed too: a comment _does_ end + a plain scalar, so `word1 # comment` followed by `word2` no longer folds `word2` + into the value — it is reported as content no node claims. + + **An implicit key has to fit on one line.** A quoted key spanning lines + (`"a\nb": 1`), a flow collection used as a block key across lines (`[23\n]: 42`), + and a compact `[ key\n : value ]` sequence entry are now `BAD_IMPLICIT_KEY`. A + flow _mapping_ may still write `{ "foo"\n: bar }` — the spec allows that one. + + **Node properties are checked where they land.** An anchor or tag written on an + alias (`key: &b *a`) was dropped without a word, and two anchors reaching one + scalar kept only the second: both are now `BAD_PROPERTY`. A block sequence + opened on a properties line (`&anchor - entry`) read as the plain scalar + `"- entry"` and is now reported. + + **Also:** a multi-line quoted scalar whose continuation lines do not clear their + parent's indentation is a `BAD_INDENT`; a `-` where a flow entry belongs (`[-]`) + is a `BAD_SCALAR_START`; and a `---`/`...` marker inside a flow collection, or + in the middle of a wrapped flow scalar, ends the document instead of being + absorbed into it. + + New codes: `BAD_BLOCK_HEADER`, `BAD_COMMENT`, `BAD_ESCAPE`, `BAD_IMPLICIT_KEY`, + `BAD_INDENT`, `BAD_PROPERTY`. + + Conformance against the official YAML test suite is **365/402 (90.8%)**, up from + 336/402, with every remaining gap still listed and reasoned. Measured against + main with the ABBA bench harness over three full runs, every fixture is within + noise: the one cell that ever flagged — `large (data)`, at -8.9% — came from the + run sharing the machine with a test suite, and read -1.5% and +3.2% in the two + runs that had it to themselves. + +- 00eb0c9: Read tabs by the column they sit at, hold a flow collection to its parent's + indentation, and pin JSON as the superset it is + + YAML 1.2 test suite conformance goes from **384/402 (95.5%) to 398/402 (99.0%)**. + What is left is four cases where the right answer is not the suite's — one + duplicate-key case that turns on the `uniqueKeys` default, and three tags that + project to a `Uint8Array`/`Set`/`Map` — so this closes the boundary rather than + moving it. Parse throughput is unchanged: an order-balanced A/B over the bench + fixtures lands every case inside run-to-run noise (medians −3% to +1.6% against a + 4–6% CV, with min-of-runs favouring the new code). + + **A tab is only an error where indentation belongs.** Indentation in YAML is + spaces (`s-indent ::= s-space × n`), but a tab _past_ the indentation is ordinary + separation — and the two are told apart only by the column the tab sits at. + `peekLine` reported any leading tab at all, which cut both ways: it rejected three + valid documents (`\t[…]` and `\t{}` at the document root, and a `foo:` whose value + line reads `⟨space⟩⟨tab⟩bar`) while missing the tabs that really are indentation. + Every caller knows the column its line owes, so it now passes it in, and the same + rule is applied in the three other places a tab can stand for indentation: + + - Inside a block scalar — `foo: |` over a lone `\t` is reported, where the same + line written ` \t` is valid content and still parses to `"\t\n"`. + - In the separation between a block indicator and a **compact collection** opened + on its line. A compact collection takes its indentation from the column it lands + on, so `-\t-`, `?\tkey:` and `:\t- x` are invalid — while `-\tfoo` and `-\t-1` + are ordinary separation and stay valid. + - In a flow collection's continuation lines (below). + + **A flow collection is held to the indentation of the block that holds it.** Flow + scanning is delimiter-driven, so `flow: [a,` over a column-0 `b,` read exactly + like a properly indented collection and parsed clean; it is now reported once per + collection as `BAD_INDENT`. Indentation is counted in spaces, which folds the tab + rule in for free. The closing `]`/`}` is deliberately held one column looser than + the spec asks — to the parent's own column rather than one past it — because + closing a multi-line flow collection at the parent's column is how Prettier and + hand-written manifests both write it, and `yaml` and `js-yaml` both accept it. + + **A tag or anchor inside a flow collection ends at the flow indicator.** In + `{ foo : !!str, }` the tag token swallowed the comma, which the tag-character check + then reported while the missing comma left the mapping looking unterminated and + shifted every entry after it. Outside a flow collection those characters are still + ordinary tag content, so a block-context `!!str,` is still a `BAD_TAG`. + + **Tab-indented JSON parsed to the wrong value.** A wrapped flow line's leading + whitespace is `s-indent(n) s-separate-in-line?`, so tabs sit in it as spaces do — + but the flow scalar scanner skipped only spaces, so the `]` closing a tab-indented + line was never seen as the flow indicator it is and the line folded into the scalar + instead. `JSON.stringify(value, null, '\t')` — what `jq --tab` and every + "indent with tabs" editor setting emit — therefore turned the last entry before a + `]` into a string with a trailing newline: `-1` came back as `"-1\n"`. + + **The 1024-character implicit key limit is enforced in block context.** YAML caps + how far past a key's start its `:` may sit so a processor can recognize a mapping + entry with bounded lookahead; a longer block key is now `BAD_IMPLICIT_KEY`. It is + deliberately _not_ enforced in flow context, matching `yaml` (eemeli): a flow + mapping is where JSON lives, `{"…1100 characters…": 1}` is valid JSON, and + rejecting a valid JSON document is the worse of the two errors. Relatedly, an + explicit key in a flow sequence may now put its `:` on the next line + (`[ ? a\n : b ]`) — the one-line rule exists to keep an _implicit_ key cheap to + recognize, and a `?` settles that up front. + + **The JSON-superset property is now checked, not assumed.** `@amritk/lint` routes + `.json` documents through the YAML parser and `resolveRefsFromFile` hands it + whatever a `$ref` points at, so "JSON parses as YAML" is load-bearing. + `src/json-superset.test.ts` runs a generated corpus against `JSON.parse` — every + value in six spellings (compact, 2-space, tab-indented, CRLF, and with + leading/trailing blank lines), requiring an identical value _and_ zero diagnostics + for each — and `@amritk/lint` gains a matching test holding `parseJson` and + `parseYaml` to identical data, diagnostics, and `line:column` ranges for every path + in a JSON document. + +### Patch Changes + +- 2c9982c: Fix the published manifests so the packages install, resolve, and dedupe correctly + + **Types resolve on TypeScript's default config.** Every package was + exports-only: nine declared `"module": "./dist/index.js"` (a field neither Node + nor TypeScript reads) and nothing declared `types`. A consumer on + `moduleResolution: "node10"` — still the default when `module` is `commonjs` — + cannot see `exports` at all, so `import { lintDocument } from '@amritk/lint'` + failed with `TS2307: Cannot find module '@amritk/lint' or its corresponding type +declarations`. Each package with a `.` export now also declares `main` and + `types`; `@amritk/helpers` and `@amritk/adapters` have no `.` export (they are + subpath-only), so they declare a `typesVersions` wildcard mapping instead, which + gives their subpaths the same node10 fallback. All of it is ignored under + `node16`/`nodenext`/`bundler`, where `exports` still wins. + + **`workspace:*` resolves to a caret, not an exact pin.** All fourteen + inter-package edges shipped as exact versions, so installing two `@amritk/*` + packages published at different times pulled in two copies of their shared + dependency. That is not merely wasteful: the module-level caches those packages + rely on are per-copy, so the `WeakMap` validator cache in + `@amritk/runtime-validators` silently stopped hitting. Pre-1.0 a caret stays + narrow (`^0.9.1` is `>=0.9.1 <0.10.0`) and breaking changes here already ride a + minor bump. + + **`@amritk/helpers` stops shipping 21 source files it does not need.** Embedded + mode reads four helper sources (`is-object`, `validate-array`, + `validate-record`, `has-ref`) out of the installed package at generation time, + so `src` has to ship — but only those four. `files` now lists them explicitly + instead of globbing all of `src`, cutting the tarball from 78 files / 206 kB to + 63 / 112 kB. + + **Two packages no longer declare a dependency they never import.** + `@amritk/mjst` and `@amritk/generate-parsers` both listed + `@amritk/generate-markdown` under `dependencies`, but the only importer is each + package's `scripts/generate-readme.ts`, which is not published. Both moved to + `devDependencies`. `@amritk/adapters` likewise dropped its + `@sinclair/typebox` peer dependency: the TypeBox adapter is purely structural + (it strips symbol keys) and imports nothing. `valibot` stays — it is a genuine + transitive peer of `@valibot/to-json-schema`. + + **`@amritk/mjst` fixes.** `json-schema-typed` moved to `dependencies`, because + the shipped `dist/emit-examples.d.ts` imports types from it. The package gained + an `exports` map, so it is no longer deep-importable in its entirety. And the + build now marks `dist/cli.js` executable: `npm pack` records on-disk modes, and + package managers only `chmod` bin targets when they link them, so flows that + consume the tarball directly (vendoring, Docker `npm pack` + `tar -x`) hit + `EACCES`. + ## 0.4.0 ### Minor Changes diff --git a/packages/yaml/package.json b/packages/yaml/package.json index e9ea6159..a0442275 100644 --- a/packages/yaml/package.json +++ b/packages/yaml/package.json @@ -1,6 +1,6 @@ { "name": "@amritk/yaml", - "version": "0.4.0", + "version": "0.5.0", "description": "A fast, featherweight, zero-dependency YAML parser for OpenAPI tooling — with exact source positions (line:column) on every node.", "main": "./dist/index.js", "module": "./dist/index.js",