diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 00000000..2f398ec3 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,8 @@ +{ + "permissions": { + "allow": [ + "WebFetch(domain:github.com)", + "WebFetch(domain:raw.githubusercontent.com)" + ] + } +} diff --git a/.gitignore b/.gitignore index f8abb004..d4c3f0dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ build/ +_build/ parser/ .cache/ vcpkg_installed/ __pycache__/ -test/.dev/ +.devbox/ +*.term +machine-config.txt diff --git a/README.md b/README.md index 189c347a..10e468c4 100644 --- a/README.md +++ b/README.md @@ -3,20 +3,46 @@ This repository contains the Zane language compiler and CLI. ## Setup + To setup this project in a new environment or sandbox, run: ```sh curl -fsSL https://get.jetify.com/devbox | bash ``` +Then enter the project development shell: + +```sh +./enter +``` + +The shell provides the project toolchain and adds `dev/bin` to `PATH`. Product +executable sources remain under `bin/`, while development-only command wrappers +live under `dev/bin/`. + ## Building ```sh -devbox shell -just init # initialize submodules, vcpkg dependencies, and meson -just build +./enter +dune build ``` ## Usage -Checkout the `justfile` for all available commands. +Common tools are available directly inside the development shell: + +```sh +compiler +ambiguity profiles +ambiguity search +ambiguity search deep-function-body --timeout 1h --output deep-search.txt +ambiguity check UIDENT LIDENT LPAREN RPAREN LCURLY RCURLY EOF +ambiguity prove 3 +syntax-experiment --max-tokens 12 --timeout 15 --max-witnesses 10 +syntax-experiment --variant semicolon-separated --max-tokens 12 --timeout 15 --max-witnesses 10 +grammar-stat +grammar-conflict +``` + +The `justfile` is reserved for parameterless project actions such as rebuilding, +watching, and running tool tests. diff --git a/ambiguity-searches.toml b/ambiguity-searches.toml new file mode 100644 index 00000000..fb80fa09 --- /dev/null +++ b/ambiguity-searches.toml @@ -0,0 +1,18 @@ +[profiles.quick] +description = "Fast breadth-first feedback while editing the grammar." +tokens = "0..16" +timeout = "2m" +witnesses = 20 + +[profiles.general] +description = "Broad shortest-first search for the regular ambiguity report." +tokens = "0..50" +timeout = "30m" +witnesses = 50 + +[profiles.deep-function-body] +extends = "general" +description = "Rotate deep search waves through statements in a function body." +tokens = "15..50" +prefix-tokens = ["UIDENT", "LIDENT", "LPAREN", "RPAREN", "LCURLY"] +nodes-per-depth = 10 diff --git a/bin/compiler/dune b/bin/compiler/dune new file mode 100644 index 00000000..fda35641 --- /dev/null +++ b/bin/compiler/dune @@ -0,0 +1,6 @@ +(executable + (name main) + (public_name main) + (package zane-compiler) + (libraries cst tree_graph) + (modules main)) diff --git a/bin/compiler/main.ml b/bin/compiler/main.ml new file mode 100644 index 00000000..86ec7c8f --- /dev/null +++ b/bin/compiler/main.ml @@ -0,0 +1,12 @@ +let read_file path = + In_channel.with_open_text path In_channel.input_all + +let () = + let input = read_file "test-parser/main.zn" in + match Cst.parse "test-parser/main.zn" input with + | Ok cst -> + let output = Cst.to_node cst in + Tree_graph.render output + | Error message -> + prerr_string message; + exit 1 diff --git a/bin/zane b/bin/zane deleted file mode 120000 index 7eb2d9e8..00000000 --- a/bin/zane +++ /dev/null @@ -1 +0,0 @@ -../build/zane \ No newline at end of file diff --git a/dev/bin/ambiguity b/dev/bin/ambiguity new file mode 100755 index 00000000..86f9c5d5 --- /dev/null +++ b/dev/bin/ambiguity @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +if [[ "${1:-}" == "__engine" ]]; then + shift + # shellcheck source=../lib/ambiguity-config.sh + . "$ROOT/dev/lib/ambiguity-config.sh" + dune build --root "$ROOT" tools/ambiguity_search.exe + exec "$ROOT/_build/default/tools/ambiguity_search.exe" \ + "$ROOT/lib/cst/parser.mly" \ + "$@" +fi + +# Keep existing scripts working while the profile interface becomes the +# preferred entry point. +case "${1:-}" in + --max-tokens | --min-tokens | --prefix-tokens | --nodes-per-depth | \ + --timeout | --max-witnesses | --check-tokens | --prove) + exec "$0" __engine "$@" + ;; +esac + +exec python3 "$ROOT/tools/ambiguity.py" "$@" diff --git a/dev/bin/compiler b/dev/bin/compiler new file mode 100755 index 00000000..fbb9034d --- /dev/null +++ b/dev/bin/compiler @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" +exec dune exec bin/compiler/main.exe -- "$@" diff --git a/dev/bin/grammar-conflict b/dev/bin/grammar-conflict new file mode 100755 index 00000000..f8248ee9 --- /dev/null +++ b/dev/bin/grammar-conflict @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Find a concrete sentence that reaches a grammar conflict. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=../lib/ambiguity-config.sh +. "$ROOT/dev/lib/ambiguity-config.sh" + +exec "$AMBIGUITY_MENHIR" "$ROOT/lib/cst/parser.mly" \ + --random-sentence-concrete package \ + --random-seed 1 \ + --random-sentence-length 16 \ + "$@" diff --git a/dev/bin/grammar-stat b/dev/bin/grammar-stat new file mode 100755 index 00000000..e96b3b83 --- /dev/null +++ b/dev/bin/grammar-stat @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Report statistics for the grammar. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=../lib/ambiguity-config.sh +. "$ROOT/dev/lib/ambiguity-config.sh" + +exec "$AMBIGUITY_MENHIR" "$ROOT/lib/cst/parser.mly" \ + --no-code-generation \ + --infer \ + "$@" diff --git a/dev/bin/syntax-experiment b/dev/bin/syntax-experiment new file mode 100755 index 00000000..4d82bc97 --- /dev/null +++ b/dev/bin/syntax-experiment @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run one syntax experiment or a selected set of variants. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=../lib/ambiguity-config.sh +. "$ROOT/dev/lib/ambiguity-config.sh" + +dune build --root "$ROOT" tools/ambiguity_search.exe +exec python3 "$ROOT/tools/syntax_experiment.py" \ + "$ROOT/lib/cst/parser.mly" \ + --search-command "$ROOT/_build/default/tools/ambiguity_search.exe" \ + "$@" diff --git a/dev/lib/ambiguity-config.sh b/dev/lib/ambiguity-config.sh new file mode 100644 index 00000000..c5d8787e --- /dev/null +++ b/dev/lib/ambiguity-config.sh @@ -0,0 +1,21 @@ +if [[ -f "$ROOT/machine-config.txt" ]]; then + # machine-config.txt uses shell-compatible KEY=VALUE assignments. + # shellcheck source=/dev/null + . "$ROOT/machine-config.txt" +fi + +for name in \ + AMBIGUITY_MEMORY_MB \ + AMBIGUITY_MAX_FRONTIER_RATIO \ + AMBIGUITY_JOBS \ + AMBIGUITY_MENHIR; do + if [[ -z "${!name:-}" ]]; then + echo "ambiguity: $name must be set in machine-config.txt or the environment" >&2 + exit 2 + fi +done + +export AMBIGUITY_MEMORY_MB +export AMBIGUITY_MAX_FRONTIER_RATIO +export AMBIGUITY_JOBS +export AMBIGUITY_MENHIR diff --git a/devbox.json b/devbox.json index c3ac0c96..57494d06 100644 --- a/devbox.json +++ b/devbox.json @@ -10,14 +10,20 @@ "ninja@latest", "zig@latest", "clang-tools@latest", - "bison@latest", - "re2c@latest" + "opam@latest", + "ocaml@latest", + "gnumake@latest", + "pkg-config@latest", + "m4@latest", + "zstd@latest" ], "shell": { "init_hook": [ "echo 'Welcome to devbox!'", - "export PATH=$(realpath $DEVBOX_PROJECT_ROOT/bin):$PATH", - "export LLVM_CONFIG=$(realpath $DEVBOX_PROJECT_ROOT/.devbox/nix/profile/default/bin/llvm-config)" + "export PATH=\"$DEVBOX_PROJECT_ROOT/dev/bin:$PATH\"", + "eval $(opam env)", + "export LLVM_CONFIG=$(realpath $DEVBOX_PROJECT_ROOT/.devbox/nix/profile/default/bin/llvm-config)", + "export LD_LIBRARY_PATH=/nix/store/s7vmxmhkq439cjb7ag9w198p6dk7kl0w-zstd-1.5.7/lib:$LD_LIBRARY_PATH" ], "scripts": { "test": [ diff --git a/devbox.lock b/devbox.lock index 09fc86ec..1d603860 100644 --- a/devbox.lock +++ b/devbox.lock @@ -1,54 +1,6 @@ { "lockfile_version": "1", "packages": { - "bison@latest": { - "last_modified": "2026-04-23T13:07:47Z", - "resolved": "github:NixOS/nixpkgs/01fbdeef22b76df85ea168fbfe1bfd9e63681b30#bison", - "source": "devbox-search", - "version": "3.8.2", - "systems": { - "aarch64-darwin": { - "outputs": [ - { - "name": "out", - "path": "/nix/store/h3i7bj2a901ms72d6v92m5jdkk5mbv0w-bison-3.8.2", - "default": true - } - ], - "store_path": "/nix/store/h3i7bj2a901ms72d6v92m5jdkk5mbv0w-bison-3.8.2" - }, - "aarch64-linux": { - "outputs": [ - { - "name": "out", - "path": "/nix/store/09i6bzy75cg042106zyf7w0f89ifm4g3-bison-3.8.2", - "default": true - } - ], - "store_path": "/nix/store/09i6bzy75cg042106zyf7w0f89ifm4g3-bison-3.8.2" - }, - "x86_64-darwin": { - "outputs": [ - { - "name": "out", - "path": "/nix/store/lvlg6r94p22mb9k0ws92p4fz4cjm3vx2-bison-3.8.2", - "default": true - } - ], - "store_path": "/nix/store/lvlg6r94p22mb9k0ws92p4fz4cjm3vx2-bison-3.8.2" - }, - "x86_64-linux": { - "outputs": [ - { - "name": "out", - "path": "/nix/store/avysx19spq1m2kzs01nhhqh310zazk40-bison-3.8.2", - "default": true - } - ], - "store_path": "/nix/store/avysx19spq1m2kzs01nhhqh310zazk40-bison-3.8.2" - } - } - }, "clang-tools@latest": { "last_modified": "2026-04-23T13:07:47Z", "resolved": "github:NixOS/nixpkgs/01fbdeef22b76df85ea168fbfe1bfd9e63681b30#clang-tools", @@ -202,96 +154,204 @@ } }, "github:NixOS/nixpkgs/nixpkgs-unstable": { - "last_modified": "2026-04-30T11:26:30Z", - "resolved": "github:NixOS/nixpkgs/7aaa00e7cc9be6c316cb5f6617bd740dd435c59d?lastModified=1777548390&narHash=sha256-WacE23EbHTsBKvr8cu%2B1DFNbP6Rh1brHUH5SDUI0NQI%3D" + "last_modified": "2026-05-27T10:28:13Z", + "resolved": "github:NixOS/nixpkgs/4100e830e085863741bc69b156ec4ccd53ab5be0?lastModified=1779877693&narHash=sha256-NOF9NAREhxr50bbBfVcVOq%2BArCMSoe8dP79Pk2uyARk%3D" + }, + "gnumake@latest": { + "last_modified": "2026-05-23T11:35:32Z", + "resolved": "github:NixOS/nixpkgs/3d8f0f3f72a6cd4d93d0ad13203f2ea1cb7e1456#gnumake", + "source": "devbox-search", + "version": "4.4.1", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/8wwiw8pwyhrkzyq28hqzxfl4z84lks81-gnumake-4.4.1", + "default": true + }, + { + "name": "man", + "path": "/nix/store/wdm9prq35jlgwq6rjyhqxy4jzy833sfr-gnumake-4.4.1-man", + "default": true + }, + { + "name": "info", + "path": "/nix/store/6dbp1ysgxsrrn7xgjgf523faqrm605a5-gnumake-4.4.1-info" + }, + { + "name": "doc", + "path": "/nix/store/sl2ygvgjg4ml8dhyfxj5kh84x2bs1mhc-gnumake-4.4.1-doc" + } + ], + "store_path": "/nix/store/8wwiw8pwyhrkzyq28hqzxfl4z84lks81-gnumake-4.4.1" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/9ngw1ippk25jjj5fjxv36xbp6iq7rxdx-gnumake-4.4.1", + "default": true + }, + { + "name": "man", + "path": "/nix/store/dg3aa71vii59scbfhi02cdxnps64x6ql-gnumake-4.4.1-man", + "default": true + }, + { + "name": "debug", + "path": "/nix/store/mdv67w2xvchbs76zjdzppdkk64z5avki-gnumake-4.4.1-debug" + }, + { + "name": "doc", + "path": "/nix/store/bybbj9avk3g107mwhgvkzkpm6mkbl8gv-gnumake-4.4.1-doc" + }, + { + "name": "info", + "path": "/nix/store/bxdqxx2q13sj15ss8z5l33jd31x55j5l-gnumake-4.4.1-info" + } + ], + "store_path": "/nix/store/9ngw1ippk25jjj5fjxv36xbp6iq7rxdx-gnumake-4.4.1" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/60y2hs45l18g56ykw9789b5vhhi2ls5q-gnumake-4.4.1", + "default": true + }, + { + "name": "man", + "path": "/nix/store/m2ba4bwmfgh994xbg1iqm6zj1mdrq8h7-gnumake-4.4.1-man", + "default": true + }, + { + "name": "info", + "path": "/nix/store/apljbm6plyajbfrnwif0a2d4nx6w0cpf-gnumake-4.4.1-info" + }, + { + "name": "doc", + "path": "/nix/store/s9y438l347nky6rkl1wfk2y3b0d9i636-gnumake-4.4.1-doc" + } + ], + "store_path": "/nix/store/60y2hs45l18g56ykw9789b5vhhi2ls5q-gnumake-4.4.1" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/d3bwqm6bymhy3pdgbvf7vxjqfp31m3j1-gnumake-4.4.1", + "default": true + }, + { + "name": "man", + "path": "/nix/store/bmvqa5ym318mymhxlwwmxq8wxal958p4-gnumake-4.4.1-man", + "default": true + }, + { + "name": "debug", + "path": "/nix/store/q683z843wi7xs96pngh1kinnx88vn2if-gnumake-4.4.1-debug" + }, + { + "name": "doc", + "path": "/nix/store/wxg27bzsljjw4k72m9b2v6a0fx5a68s0-gnumake-4.4.1-doc" + }, + { + "name": "info", + "path": "/nix/store/03d41qgjsrhcr3fzq2z99ry07rahnj0v-gnumake-4.4.1-info" + } + ], + "store_path": "/nix/store/d3bwqm6bymhy3pdgbvf7vxjqfp31m3j1-gnumake-4.4.1" + } + } }, "just@latest": { - "last_modified": "2026-04-23T13:07:47Z", - "resolved": "github:NixOS/nixpkgs/01fbdeef22b76df85ea168fbfe1bfd9e63681b30#just", + "last_modified": "2026-05-12T01:56:55Z", + "resolved": "github:NixOS/nixpkgs/0a0bf409f3593f415d0a554f33acc63dc7dccb43#just", "source": "devbox-search", - "version": "1.50.0", + "version": "1.51.0", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/qxqsxk2614qpcb7l32svpjmmb1sm7ya1-just-1.50.0", + "path": "/nix/store/c9vv24f9bdnhnms10nwijn6q8wjz00ak-just-1.51.0", "default": true }, { "name": "man", - "path": "/nix/store/daf30m3si50cl7xga42g0cqgdwaxj6ys-just-1.50.0-man", + "path": "/nix/store/lxfs1naxd1a3934y9xsd1kid5yyw7jbn-just-1.51.0-man", "default": true }, { "name": "doc", - "path": "/nix/store/9n71gassrpdy0nz4gfcrwvmkb4rwyd3g-just-1.50.0-doc" + "path": "/nix/store/c8jdzd2s7hbldm1q1glrjisavxy1vf54-just-1.51.0-doc" } ], - "store_path": "/nix/store/qxqsxk2614qpcb7l32svpjmmb1sm7ya1-just-1.50.0" + "store_path": "/nix/store/c9vv24f9bdnhnms10nwijn6q8wjz00ak-just-1.51.0" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/ayqw7hgkjs9b0f7v1qh96k316i9jpdw8-just-1.50.0", + "path": "/nix/store/ljwadwqsib4zd6nvg6xi5rnzah9ad2q4-just-1.51.0", "default": true }, { "name": "man", - "path": "/nix/store/59z0bx178i7c3vwmhawgajgpfmv43h5c-just-1.50.0-man", + "path": "/nix/store/5m9rgkzibz19za2nmn83y9l5gsy2z1yh-just-1.51.0-man", "default": true }, { "name": "doc", - "path": "/nix/store/gg298zgvg5vmvzk0lmn9zq7j6kkab3qs-just-1.50.0-doc" + "path": "/nix/store/q80cv2gjn2q640fvsqgdj6ap2iihxwg9-just-1.51.0-doc" } ], - "store_path": "/nix/store/ayqw7hgkjs9b0f7v1qh96k316i9jpdw8-just-1.50.0" + "store_path": "/nix/store/ljwadwqsib4zd6nvg6xi5rnzah9ad2q4-just-1.51.0" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/bmsgb4l7ldxj2zggcalfidzhkrgp03xq-just-1.50.0", + "path": "/nix/store/w0capgph40qykn19a9ghv38fjf06gg97-just-1.51.0", "default": true }, { "name": "man", - "path": "/nix/store/0pvkn1lzh85sav0cj3v36kpigxwypdhb-just-1.50.0-man", + "path": "/nix/store/97jyxj8d9d4qjq25681ng88jnbdak1dx-just-1.51.0-man", "default": true }, { "name": "doc", - "path": "/nix/store/9d2i12jkmsyqw7bkcw3a9d0xki313xrp-just-1.50.0-doc" + "path": "/nix/store/p8pb01d5w82lf8z0ah4kcc6ziwfvxfkr-just-1.51.0-doc" } ], - "store_path": "/nix/store/bmsgb4l7ldxj2zggcalfidzhkrgp03xq-just-1.50.0" + "store_path": "/nix/store/w0capgph40qykn19a9ghv38fjf06gg97-just-1.51.0" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/vr4agmy8jw7f8kqynpizagdaqxy0ayw4-just-1.50.0", + "path": "/nix/store/zn7n9xnw5vx6cqf1az8b8p1pfsmrl9vw-just-1.51.0", "default": true }, { "name": "man", - "path": "/nix/store/6mrjsdff8skgnnn0ngyim2i7wabk0c8x-just-1.50.0-man", + "path": "/nix/store/mgc1wdhgkc9246s9a52j17ymw4745bal-just-1.51.0-man", "default": true }, { "name": "doc", - "path": "/nix/store/nk0p7b0n1fqg0sa2vasgavrhgi7xy9rz-just-1.50.0-doc" + "path": "/nix/store/mkdym7bscwpn6y6dvzrblm5p87q9c2g0-just-1.51.0-doc" } ], - "store_path": "/nix/store/vr4agmy8jw7f8kqynpizagdaqxy0ayw4-just-1.50.0" + "store_path": "/nix/store/zn7n9xnw5vx6cqf1az8b8p1pfsmrl9vw-just-1.51.0" } } }, "llvm_21@latest": { - "last_modified": "2026-04-29T01:19:07Z", - "resolved": "github:NixOS/nixpkgs/ebc08544afa77957cc348ba72dc490ec73b87f68#llvm_21", + "last_modified": "2026-05-20T06:38:13Z", + "resolved": "github:NixOS/nixpkgs/d99b013d5d1931ad77fe3912ed218170dec5d9a4#llvm_21", "source": "devbox-search", "version": "21.1.8", "systems": { @@ -302,10 +362,6 @@ "path": "/nix/store/fib2wxwpvlc1q2qrxbhz2i961l2srcjw-llvm-21.1.8", "default": true }, - { - "name": "python", - "path": "/nix/store/6avba57frkpxqlw7d5mljr5a6ymvlygy-llvm-21.1.8-python" - }, { "name": "dev", "path": "/nix/store/x0msv7ql9xm50fvz3x5p2gb01grnyqln-llvm-21.1.8-dev" @@ -313,6 +369,10 @@ { "name": "lib", "path": "/nix/store/4fk7xkk9faias8kqzh1kagdhb87l5kcw-llvm-21.1.8-lib" + }, + { + "name": "python", + "path": "/nix/store/6avba57frkpxqlw7d5mljr5a6ymvlygy-llvm-21.1.8-python" } ], "store_path": "/nix/store/fib2wxwpvlc1q2qrxbhz2i961l2srcjw-llvm-21.1.8" @@ -324,6 +384,10 @@ "path": "/nix/store/dr58nil7xb6v4di6mmlvki3s1d8iwdym-llvm-21.1.8", "default": true }, + { + "name": "dev", + "path": "/nix/store/0hf3wfrd60fhgh7y1x7857pf7a1hypnm-llvm-21.1.8-dev" + }, { "name": "lib", "path": "/nix/store/lr9kz2ky0ys12cx0xz91rnkmbadaqvj0-llvm-21.1.8-lib" @@ -331,16 +395,26 @@ { "name": "python", "path": "/nix/store/45y119pkm1p76bqn7lhp2i0nx0yk2x44-llvm-21.1.8-python" - }, - { - "name": "dev", - "path": "/nix/store/0hf3wfrd60fhgh7y1x7857pf7a1hypnm-llvm-21.1.8-dev" } ], "store_path": "/nix/store/dr58nil7xb6v4di6mmlvki3s1d8iwdym-llvm-21.1.8" + }, + "x86_64-linux": { + "outputs": [ + { + "path": "/nix/store/jp45dqzv8mjpqsvhj99c93pc4vlmhy16-llvm-21.1.8", + "default": true + } + ] } } }, + "m4@latest": { + "last_modified": "2023-02-24T09:01:09Z", + "resolved": "github:NixOS/nixpkgs/7d0ed7f2e5aea07ab22ccb338d27fbe347ed2f11#m4", + "source": "devbox-search", + "version": "1.4.19" + }, "meson@latest": { "last_modified": "2026-04-23T13:07:47Z", "resolved": "github:NixOS/nixpkgs/01fbdeef22b76df85ea168fbfe1bfd9e63681b30#meson", @@ -453,51 +527,199 @@ } } }, - "re2c@latest": { - "last_modified": "2026-04-23T13:07:47Z", - "resolved": "github:NixOS/nixpkgs/01fbdeef22b76df85ea168fbfe1bfd9e63681b30#re2c", + "ocaml@latest": { + "last_modified": "2026-05-21T08:15:18Z", + "resolved": "github:NixOS/nixpkgs/4a29d733e8a7d5b824c3d8c958a946a9867b3eb2#ocaml", + "source": "devbox-search", + "version": "5.4.1", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/81yy39njdh1d5796qn84inbqq56fh71v-ocaml-5.4.1", + "default": true + } + ], + "store_path": "/nix/store/81yy39njdh1d5796qn84inbqq56fh71v-ocaml-5.4.1" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/zrdg31hnnr0lv3xndwvh74fd160483di-ocaml-5.4.1", + "default": true + } + ], + "store_path": "/nix/store/zrdg31hnnr0lv3xndwvh74fd160483di-ocaml-5.4.1" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/2hrwc25yrvlwz6gmar9cd8zdsy89rfpa-ocaml-5.4.1", + "default": true + } + ], + "store_path": "/nix/store/2hrwc25yrvlwz6gmar9cd8zdsy89rfpa-ocaml-5.4.1" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/ryj2xb3cla5jvx41pxlk8vqm0z93g5cs-ocaml-5.4.1", + "default": true + } + ], + "store_path": "/nix/store/ryj2xb3cla5jvx41pxlk8vqm0z93g5cs-ocaml-5.4.1" + } + } + }, + "opam@latest": { + "last_modified": "2026-05-21T08:15:18Z", + "resolved": "github:NixOS/nixpkgs/4a29d733e8a7d5b824c3d8c958a946a9867b3eb2#opam", "source": "devbox-search", - "version": "4.5.1", + "version": "2.5.1", "systems": { "aarch64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/qbdjjl5qqrhls3ld8x8dlmsz8ddkazib-re2c-4.5.1", + "path": "/nix/store/6cji9in2mij7z9545lyx035vshsi9cag-opam-2.5.1", "default": true + }, + { + "name": "installer", + "path": "/nix/store/c7gjr0dmq98scqkd5i8gpbjhlphb1gk4-opam-2.5.1-installer" } ], - "store_path": "/nix/store/qbdjjl5qqrhls3ld8x8dlmsz8ddkazib-re2c-4.5.1" + "store_path": "/nix/store/6cji9in2mij7z9545lyx035vshsi9cag-opam-2.5.1" }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/b8dp3yrgky5ira1yqwjda0xa5ykf11qh-re2c-4.5.1", + "path": "/nix/store/ci7lqvh196kbhc8lxgd04gz7pz0mm2yq-opam-2.5.1", "default": true + }, + { + "name": "installer", + "path": "/nix/store/1v9bx0wdr2nkfwlwddldi7gscls7qcjm-opam-2.5.1-installer" } ], - "store_path": "/nix/store/b8dp3yrgky5ira1yqwjda0xa5ykf11qh-re2c-4.5.1" + "store_path": "/nix/store/ci7lqvh196kbhc8lxgd04gz7pz0mm2yq-opam-2.5.1" }, "x86_64-darwin": { "outputs": [ { "name": "out", - "path": "/nix/store/3n0qpxbjhw2yziynjjg5pg0zbm2w5ka0-re2c-4.5.1", + "path": "/nix/store/3k4k7yyla8cxj8r8l688gmwd81346wsz-opam-2.5.1", "default": true + }, + { + "name": "installer", + "path": "/nix/store/2m62nyma8h6f37vcr235f9xa560jyii0-opam-2.5.1-installer" } ], - "store_path": "/nix/store/3n0qpxbjhw2yziynjjg5pg0zbm2w5ka0-re2c-4.5.1" + "store_path": "/nix/store/3k4k7yyla8cxj8r8l688gmwd81346wsz-opam-2.5.1" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/zkmywqm90m1wlmjs9x1ibr7hqlvy0rxk-re2c-4.5.1", + "path": "/nix/store/nd0n31sr87n8n6j34ybr30v9h1z8l34l-opam-2.5.1", "default": true + }, + { + "name": "installer", + "path": "/nix/store/xhbyql390bnwyg1if8kssdylp6y3bni9-opam-2.5.1-installer" } ], - "store_path": "/nix/store/zkmywqm90m1wlmjs9x1ibr7hqlvy0rxk-re2c-4.5.1" + "store_path": "/nix/store/nd0n31sr87n8n6j34ybr30v9h1z8l34l-opam-2.5.1" + } + } + }, + "pkg-config@latest": { + "last_modified": "2025-11-23T21:50:36Z", + "resolved": "github:NixOS/nixpkgs/ee09932cedcef15aaf476f9343d1dea2cb77e261#pkg-config", + "source": "devbox-search", + "version": "0.29.2", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/hygaaqwk9ylklp0ybwppqhw75nz8ya41-pkg-config-wrapper-0.29.2", + "default": true + }, + { + "name": "man", + "path": "/nix/store/9px0sji43x3r2w4zxl3j3idwsql7lwxx-pkg-config-wrapper-0.29.2-man", + "default": true + }, + { + "name": "doc", + "path": "/nix/store/hqk44ra6qxw7iixardl6c3hdgb9kq6ns-pkg-config-wrapper-0.29.2-doc" + } + ], + "store_path": "/nix/store/hygaaqwk9ylklp0ybwppqhw75nz8ya41-pkg-config-wrapper-0.29.2" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/lbiigi8qbp7mzf1lpr7p982l1kyf01ql-pkg-config-wrapper-0.29.2", + "default": true + }, + { + "name": "man", + "path": "/nix/store/10060k24qggqyzlwdsfmni9y32zxcg0j-pkg-config-wrapper-0.29.2-man", + "default": true + }, + { + "name": "doc", + "path": "/nix/store/0y4v51ndpyvkj09hwlfqkz0c3h17zfmc-pkg-config-wrapper-0.29.2-doc" + } + ], + "store_path": "/nix/store/lbiigi8qbp7mzf1lpr7p982l1kyf01ql-pkg-config-wrapper-0.29.2" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/vknadizq0q5kffvx6y4379p9gdry9zq3-pkg-config-wrapper-0.29.2", + "default": true + }, + { + "name": "man", + "path": "/nix/store/1nyspra675q22gfhf7hn2nmfpi6rgim5-pkg-config-wrapper-0.29.2-man", + "default": true + }, + { + "name": "doc", + "path": "/nix/store/7lq1axxwrafwljs06n88bzyz9w523rkc-pkg-config-wrapper-0.29.2-doc" + } + ], + "store_path": "/nix/store/vknadizq0q5kffvx6y4379p9gdry9zq3-pkg-config-wrapper-0.29.2" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/8vdiwpbh0g4avsd6x5v4s0di32vcl3dp-pkg-config-wrapper-0.29.2", + "default": true + }, + { + "name": "man", + "path": "/nix/store/j9xfpnrygg3v37svc5pfin9q5bm49r94-pkg-config-wrapper-0.29.2-man", + "default": true + }, + { + "name": "doc", + "path": "/nix/store/x3bypxdxaq20kykybhkf21x4jczsiy8y-pkg-config-wrapper-0.29.2-doc" + } + ], + "store_path": "/nix/store/8vdiwpbh0g4avsd6x5v4s0di32vcl3dp-pkg-config-wrapper-0.29.2" } } }, @@ -612,6 +834,106 @@ "store_path": "/nix/store/5x5y75rc3l43ic0lwal6wzk7wi1ga3m1-zig-0.16.0" } } + }, + "zstd@latest": { + "last_modified": "2026-05-27T10:28:13Z", + "resolved": "github:NixOS/nixpkgs/4100e830e085863741bc69b156ec4ccd53ab5be0#zstd", + "source": "devbox-search", + "version": "1.5.7", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "bin", + "path": "/nix/store/lj773k1dkd7rrnrf0f05v43kwwbzlv4j-zstd-1.5.7-bin", + "default": true + }, + { + "name": "man", + "path": "/nix/store/053fzmw9v3cc6j8j32jyp3lk1017mla5-zstd-1.5.7-man", + "default": true + }, + { + "name": "out", + "path": "/nix/store/ym2h5axmyxqkwmrrrz570gmkbmdlwvbv-zstd-1.5.7" + }, + { + "name": "dev", + "path": "/nix/store/jg2ki5fvwby45jdv5qmwmn6808gfvvks-zstd-1.5.7-dev" + } + ], + "store_path": "/nix/store/lj773k1dkd7rrnrf0f05v43kwwbzlv4j-zstd-1.5.7-bin" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "bin", + "path": "/nix/store/37j8jm7q19xgccgz4lawfg7hr0vf79zl-zstd-1.5.7-bin", + "default": true + }, + { + "name": "man", + "path": "/nix/store/h47ihn6ksn6378j64ya87sci4lxq66mc-zstd-1.5.7-man", + "default": true + }, + { + "name": "out", + "path": "/nix/store/g3vxv9qmkd89xw1ya9ii9gpps78z4q2m-zstd-1.5.7" + }, + { + "name": "dev", + "path": "/nix/store/55aldi2c0mzgqkya8i9ssn76shz0vqp8-zstd-1.5.7-dev" + } + ], + "store_path": "/nix/store/37j8jm7q19xgccgz4lawfg7hr0vf79zl-zstd-1.5.7-bin" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "bin", + "path": "/nix/store/fb8skywpvnjdmwz37azp9r4dvwxwvq76-zstd-1.5.7-bin", + "default": true + }, + { + "name": "man", + "path": "/nix/store/wv4by27faql80s9q8y6xkqcx3ssalvr7-zstd-1.5.7-man", + "default": true + }, + { + "name": "dev", + "path": "/nix/store/g5r73l3qx4pjz4f3g0bx1nq6cihqa629-zstd-1.5.7-dev" + }, + { + "name": "out", + "path": "/nix/store/9db35k8dl5rxk3kk6kzkqwm5732s6ag6-zstd-1.5.7" + } + ], + "store_path": "/nix/store/fb8skywpvnjdmwz37azp9r4dvwxwvq76-zstd-1.5.7-bin" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "bin", + "path": "/nix/store/cwj48a5jfy5srxgq45p4dg37zqdlhzba-zstd-1.5.7-bin", + "default": true + }, + { + "name": "man", + "path": "/nix/store/qb1yjn4gp5g2pqgqy0ywjwv3ms987n5a-zstd-1.5.7-man", + "default": true + }, + { + "name": "dev", + "path": "/nix/store/cbzpsvdi282yhw702d7s42lirqnr0i0g-zstd-1.5.7-dev" + }, + { + "name": "out", + "path": "/nix/store/gd74vdpqzq0qmmikvmrf3r3kghhhnf50-zstd-1.5.7" + } + ], + "store_path": "/nix/store/cwj48a5jfy5srxgq45p4dg37zqdlhzba-zstd-1.5.7-bin" + } + } } } } diff --git a/docs/ambiguity.md b/docs/ambiguity.md new file mode 100644 index 00000000..c81a436e --- /dev/null +++ b/docs/ambiguity.md @@ -0,0 +1,193 @@ +# Ambiguity policy + +Zane's grammar is deliberately not LR(1): the language is parsed with a +GLR+LR hybrid (menhirGLR, previously Elkhound), and constructs may require +unbounded lookahead. Nondeterminism is accepted; ambiguity is not. + +**Policy: the grammar must remain provably unambiguous.** A GLR parser may +fork wherever it needs to, but on every input all forks except one must +eventually die. General context-free ambiguity is undecidable, so no tool +can certify this automatically for arbitrary grammars — instead the proof +is maintained as a finite set of per-conflict obligations, which is +possible because every fork point is an LR conflict state and Menhir +enumerates those exhaustively. + +## Proof obligations + +Every LR conflict state must carry exactly one of: + +1. **A precedence resolution.** The conflict is resolved by a declared + precedence or associativity; the resolution is deliberate and the + intended reading is documented. Resolved conflicts are deterministic and + need no further argument. +2. **A transience argument.** A short written proof that the two branches + of the fork can never both reach acceptance, keyed to the conflict + state's LR items so that grammar changes touching the construct + visibly invalidate the argument. +3. **An open obligation.** Permitted, but tracked: open obligations are the + standing targets of the bounded ambiguity search, and a found witness + turns one into a bug. + +A grammar change that introduces a new conflict state is incomplete until +the state is triaged into one of these categories. + +## Tooling + +- `ambiguity search [PROFILE]` — bounded, parallel GLR search for complete + ambiguous sentences (`tools/ambiguity_search.ml`). A completed bound is a theorem + ("no ambiguous sentence of at most N tokens"), up to the astronomically + unlikely collision of the 124-bit frontier digests used for + deduplication; an interrupted bound is evidence only. + `AMBIGUITY_MEMORY_MB` sets an approximate total resident-memory budget shared + by all workers. The tool derives two per-worker limits from it: a queue cap, + which controls search reach and the live stack set, and an evicting + digest-cache size, which controls deduplication. + `AMBIGUITY_MAX_FRONTIER_RATIO` is the number of digest-cache entries per + queued frontier: raising it trades queue reach for stronger deduplication, + while lowering it does the opposite. The estimate is based on + `queue * (600 + 24 * max_tokens) + cache * 240` bytes per worker. The cache + limit covers both of its generations; it is not multiplied behind the + scenes. Workers also monitor their actual OCaml heap: they compact at 80% of + their share, first release the older (purely optional) dedup generation, + stop admitting new frontiers at 90%, and resume below 85%. The remaining 10% + covers the coordinator, native allocations, and transient compaction/ + copy-on-write overhead. This keeps memory near the configured plateau while + prioritizing queue reach even when real frontiers are larger than the + estimate. Named profiles in `ambiguity-searches.toml` collect search intent + in one reviewable place. `ambiguity profiles` lists them, and command-line + options can temporarily override a profile. The default `general` profile + is breadth-first; `deep-function-body` fixes the function-body prefix and + rotates depth waves so sibling statements continue to receive attention. + The search itself is bounded by the profile's token range and timeout, and a + run that had to drop part of the space reports itself as interrupted. + In an interactive terminal, one transient status line shows the active token + depth, ambiguity families found at that depth, explored and unique frontiers, + elapsed time, and a RAM bar against the configured memory budget. The + coordinator deduplicates ambiguity families across workers before displaying + the count. Redirected output and saved reports contain no progress line. + `--nodes-per-depth N` expands up to `N` queued frontiers at one depth before + descending to the next populated depth. When a deep wave ends, the search + returns to the shallowest unfinished depth, so earlier token choices rotate + instead of one deep subtree monopolizing the run. Smaller values are + narrower and deeper; larger values explore more siblings before descending. + Omitting the option preserves breadth-first scheduling. The scheduler does + not prune queued frontiers, so a run that completes its bound remains + exhaustive. + `--min-tokens N` prevents shorter ambiguities from consuming witness slots. + `--prefix-tokens "TOKENS..."` first advances the GLR parser through a fixed + token prefix and searches from that frontier, which is useful for targeting + contexts such as a function body. Minimum and maximum token counts include + the prefix and `EOF`. + Witnesses are grouped by the conflict states they + traverse, which maps each finding directly onto an obligation above. + Before searching, the tool computes **terminal equivalence classes**: + terminals that behave identically throughout the LR automaton — the same + shifts (up to a state bisimulation), the same reductions as a lookahead, and + the same acceptance — are interchangeable, so swapping one for another is an + automorphism of the recognition relation. The search explores a single + representative per class instead of every interchangeable token, which cuts + the branching factor wherever a construct admits several equivalent terminals + (for the current grammar, the value atoms `FALSE`/`FLOAT`/`STRING`/`TRUE` + collapse to one class, as do same-precedence operator groups such as + `+`/`-` and `*`/`/`). Precedence and associativity are already resolved in + the automaton's concrete actions, so tokens with different precedence never + share a class; and a token that reaches a context the others do not — `INT`, + which is also a const-generic argument — stays in its own class. Because + class members generate isomorphic parse forests, a completed bound is a + theorem up to renaming terminals within their class: an ambiguous sentence + exists with one member iff it exists with every member, so no obligation is + lost. Witnesses render with the representative terminal. +- `ambiguity prove K [PROFILE]` — conservative unambiguity proof mode built + into the ambiguity search. It abstracts GLR stacks to their top-K states and + exhaustively explores pairs of abstract parses of the same input, comparing + reduction chains in lockstep. It does not depend on an external constraint + solver. Three verdicts: exit 0 "PROVEN UNAMBIGUOUS" is a genuine proof with + no sentence-length bound; exit 1 means a concrete ambiguous sentence was + found; exit 3 means not proven — the abstraction reported a candidate the + bounded search could not concretize, so raise the proof level or override the + concretization profile. + Because unambiguity is undecidable in general, the "not proven" verdict can + never be eliminated entirely; the prover is validated against known-ambiguous + grammars, LR(1) grammars, precedence-resolved expression grammars, and + unambiguous non-LR grammars such as palindromes. +- `ambiguity classes` — lists the terminal equivalence classes the search + collapses, so a grammar change that unexpectedly splits or merges a class is + visible. The same classes bound the prover's terminal alphabet. +- `syntax-experiment` — compares candidate grammar changes under + equal search bounds before they are adopted + (`tools/SYNTAX_EXPERIMENT.md`). +- `menhir --explain` — enumerates the conflict states that constitute the + obligation ledger. + +## Local machine configuration + +The ambiguity-tool executables load machine-specific values from the ignored +`machine-config.txt` file. Copy `machine-config.example` before running them. +There are no fallback values: a missing setting is an error. This keeps memory, +worker-count, and executable-path tuning out of normal command invocations and +out of version control. The file supplies `AMBIGUITY_MEMORY_MB`, +`AMBIGUITY_MAX_FRONTIER_RATIO`, `AMBIGUITY_JOBS`, and `AMBIGUITY_MENHIR` as +environment variables; they are deliberately not command-line options. + +Search intent lives in versioned profiles, with concise overrides for one-off +runs. For example, `ambiguity search general --tokens 0..100 --timeout 1h` +searches through 100 tokens for up to one hour, using the local machine budget. +Friendly durations such as `90s`, `30m`, and `1h` are accepted. Add +`--output report.txt` to display and save a report, or `--dry-run` to inspect +the resolved settings without building the engine. + +Every profile setting has an identically named override flag: the TOML key and +the `--flag` share the same kebab-case spelling (`tokens`, `timeout`, +`witnesses`, `prefix-tokens`, `nodes-per-depth`, `breadth-first`, `output`), and +the command line overrides the profile. A single registry in +`tools/ambiguity.py` declares every flag once — value flags, the `breadth-first` +toggle, and the `dry-run` mode alike — and marks which ones profiles may set, so +the flags and the profile keys are one list and cannot drift apart. The only +flag that is not a profile key is `--dry-run`, which is a run mode (show the +resolved settings without running the engine), not saved search intent. + +Scheduling is one slot with two spellings: a profile sets either +`nodes-per-depth = N` (depth waves) or `breadth-first = true` (shortest-first), +never both. Because breadth-first is now a real key, a child profile can reset +an inherited `nodes-per-depth` back to breadth-first (or the reverse) — the +child's choice replaces whichever the parent set. On the command line, +`--breadth-first` overrides a profile's `nodes-per-depth`, and giving both +`--breadth-first` and `--nodes-per-depth` at once is an error. + +The `output` path — whether set as a profile key or passed with `--output` — +may contain placeholders that are filled in when the run starts: `{profile}` is +the resolved profile name, and `{date}`, `{time}`, and `{datetime}` are +timestamps laid out like the existing `reports/` filenames (`2026-07-23`, +`21-38-17`, and `2026-07-23_21-38-17`). Any directories in the expanded path are +created automatically, so `output = "reports/{profile}-{date}.txt"` in a profile +(or `--output reports/{profile}-{date}.txt` on the command line) drops a dated +report into `reports/` without a manual `mkdir`. Write `{{` and `}}` for literal +braces. + +To concentrate a run inside a function body and favor depth over breadth: + +```sh +ambiguity search deep-function-body +``` + +To change just one aspect without creating a profile: + +```sh +ambiguity search deep-function-body --nodes-per-depth 4 --timeout 2h +``` + +Exact witnesses can be checked without quoting their token names: + +```sh +ambiguity check UIDENT LIDENT LPAREN RPAREN LCURLY LIDENT LPAREN RPAREN EOF +``` + +## Why this is sound + +Unambiguity of an arbitrary grammar admits no complete decision procedure, +but a *specific* grammar is proven unambiguous by a finite argument when +its structure supports one. Keeping the obligations discharged is exactly +keeping such a finite argument in existence at all times: determinism +certificates where the grammar is locally LR, human induction arguments +where it is not, and exhaustive bounded search as the continuous attempt at +falsification. diff --git a/docs/bison.md b/docs/bison.md new file mode 100644 index 00000000..3aefd35a --- /dev/null +++ b/docs/bison.md @@ -0,0 +1,57 @@ +for recursion dont need to specify the single node variant when empty a variant. +also: empty doesnt need explicit instantiation. + +``` +parameters + : %empty + | parameters[params] COMMA parameter[p] + { + $params.push_back(std::move($p)); + $$ = std::move($params); + } + ; +``` +only use single node variant if we dont allow empty. + +actually, this is incorrect, as only allowing empty nodes mess with commas, which would only allow: +``` +Void main(, test Int) { + print(2, 2) + Std$print(1 + 1 + 3 + 4) + @Intrinsics$print("hello", "bye") +} +``` +instead of: +``` +Void main(test Int) { + print(2, 2) + Std$print(1 + 1 + 3 + 4) + @Intrinsics$print("hello", "bye") +} +``` + +so we use: +``` +parameters + : %empty + | parameters[params] COMMA parameter[p] + { + $params.push_back(std::move($p)); + $$ = std::move($params); + } + ; +``` +but possible in homologous: +``` +statements + : %empty + { $$ = std::vector>(); } + | statements[list] statement[stmt] + { + $list.push_back(std::make_unique(std::move($stmt))); + $$ = std::move($list); + } + ; +``` + +no ambiguaty must exist that would be resolved through following sub-nodes, because sub-nodes are reduced before matching. otherwise we would use glr, but it is slower and lalr(1) is linear. diff --git a/docs/stages.md b/docs/stages.md new file mode 100644 index 00000000..0da2378e --- /dev/null +++ b/docs/stages.md @@ -0,0 +1,8 @@ +We use 4 stages: +1. parsing: astA (correct terminology: cst, concrete syntax tree) +2. semantics: astB (correct terminology: ast) +3. optimizations: mutate astB +4. codegen: binary + +we use two different ast's. astA only captures the content and doesnt do name resolution or desugaring. +those things are handled in stage 2 semantics. diff --git a/dune-project b/dune-project new file mode 100644 index 00000000..44ca369b --- /dev/null +++ b/dune-project @@ -0,0 +1,18 @@ +(lang dune 3.21) +(name zane-compiler) +(generate_opam_files true) + +(source + (github zane-lang/compiler)) + +(using menhir 3.0) + +(package + (name zane-compiler) + (synopsis "A short synopsis") + (description "A longer description") + (depends ocaml) + (tags + ("add topics" "to describe" your project))) + +; See the complete stanza docs at https://dune.readthedocs.io/en/stable/reference/dune-project/index.html diff --git a/enter b/enter new file mode 100755 index 00000000..2838321c --- /dev/null +++ b/enter @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if ! command -v devbox >/dev/null 2>&1; then + echo "enter: devbox is required; install it from https://www.jetify.com/devbox" >&2 + exit 127 +fi + +cd "$ROOT" +exec devbox shell diff --git a/justfile b/justfile index 8b218f0c..adbb10f7 100644 --- a/justfile +++ b/justfile @@ -1,26 +1,15 @@ -build: - meson compile -C build +default: + just -l -init: - rm -rf build - test -d .git/modules || git submodule update --init --recursive - vcpkg install - CXX=clang++ meson setup build --buildtype=debug --cmake-prefix-path "$(realpath vcpkg_installed/x64-linux)" +rebuild: + dune clean + dune build -release: - rm -rf build - test -d .git/modules || git submodule update --init --recursive - vcpkg install - CXX=clang++ meson setup build --buildtype=release --cmake-prefix-path "$(realpath vcpkg_installed/x64-linux)" +watch: + dune build --watch -check: - clang-check -p build src/*.* +syntax-experiment-test: + python3 -m unittest tools.test_syntax_experiment -v -generate-parser: - scripts/generate_parser.sh - -check-parser: - scripts/check_parser.sh - -link: - ln -sf build/zane bin/zane +ambiguity-test: + python3 -m unittest tools.test_ambiguity -v diff --git a/lib/cst/cst.ml b/lib/cst/cst.ml new file mode 100644 index 00000000..81bd9b51 --- /dev/null +++ b/lib/cst/cst.ml @@ -0,0 +1,13 @@ +module Nodes = Nodes +module Parser = Parser +module Lexer = Lexer +include To_tree_graph + +let parse filename input = + let lexbuf = Sedlexing.Utf8.from_string input in + let tokenizer = Sedlexing.with_tokenizer Lexer.token lexbuf in + try + Ok (MenhirLib.Convert.Simplified.traditional2revised Parser.package tokenizer) + with Parser.Error _tokenizer | Lexer.Lexing_error -> + let (pos_start, pos_end) = Sedlexing.lexing_positions lexbuf in + Error (Parse_error.format_parse_error filename input pos_start pos_end) diff --git a/lib/cst/dune b/lib/cst/dune new file mode 100644 index 00000000..5223e60b --- /dev/null +++ b/lib/cst/dune @@ -0,0 +1,16 @@ +(library + (name cst) + (libraries + menhirLib + sedlex + tree_graph + menhirGLR + ) + (preprocess (pps sedlex.ppx))) + +(menhir + (modules parser) + (infer true) + + (flags --GLR) +) diff --git a/lib/cst/lexer.ml b/lib/cst/lexer.ml new file mode 100644 index 00000000..f1b6ec53 --- /dev/null +++ b/lib/cst/lexer.ml @@ -0,0 +1,83 @@ +open Sedlexing +open Parser + +(* Raised on a character that starts no valid token. Position is read from the + lexbuf by the caller, the same way a [Parser.Error] is located. *) +exception Lexing_error + +let digit = [%sedlex.regexp? '0'..'9'] +let digits = [%sedlex.regexp? Plus digit] +let int_lit = [%sedlex.regexp? digits, Star ('\'', digits)] +let float_lit = [%sedlex.regexp? int_lit, '.', digits] +let str_char = [%sedlex.regexp? Compl ('"' | '\\') | '\\', any] + +(* Any character valid inside an identifier (after the first) *) +let ident_char = [%sedlex.regexp? alphabetic | '0'..'9' | '_'] + +(* Starts with a Unicode lowercase letter, or '_' then one *) +let lower_ident = [%sedlex.regexp? (lowercase | '_', lowercase), Star ident_char] +(* Starts with a Unicode uppercase letter, or '_' then one *) +let upper_ident = [%sedlex.regexp? (uppercase | '_', uppercase), Star ident_char] + +let rec token buf = + match%sedlex buf with + | Plus (' ' | '\t' | '\r' | '\n') -> token buf + | "=>" -> THICK_ARROW + | "==" -> EQEQ + | "<=" -> LESSEQ + | ">=" -> MOREEQ + | '<' -> LESS + | '>' -> MORE + | '=' -> EQUAL + | '(' -> LPAREN + | ')' -> RPAREN + | '{' -> LCURLY + | '}' -> RCURLY + | '[' -> LBRACKET + | ']' -> RBRACKET + | ',' -> COMMA + | '.' -> DOT + | ':' -> COLON + | ';' -> SEMICOLON + | '!' -> EXCL + | "??" -> QSTNQSTN + | '?' -> QSTNMARK + | '~' -> TILDE + | '+' -> PLUS + | '-' -> MINUS + | '*' -> STAR + | '/' -> SLASH + | '$' -> DOLLAR + | '#' -> HASH + | '&' -> AND + | '@' -> AT + | float_lit -> FLOAT (Utf8.lexeme buf) + | int_lit -> INT (Utf8.lexeme buf) + | '"', Star str_char, '"' -> + let s = Utf8.lexeme buf in + STRING (String.sub s 1 (String.length s - 2)) + | "type" -> LTYPE + | "alias" -> ALIAS + | "Type" -> UTYPE + | "Number" -> NUMBER + | "struct" -> STRUCT + | "variant" -> VARIANT + | "tuple" -> TUPLE + | "enum" -> ENUM + | "if" -> IF + | "elif" -> ELIF + | "else" -> ELSE + | "loop" -> LOOP + | "from" -> FROM + | "to" -> TO + | "true" -> TRUE + | "false" -> FALSE + | "this" -> THIS + | "mut" -> MUT + | "abort" -> ABORT + | "return" -> RETURN + | "resolve" -> RESOLVE + | lower_ident -> LIDENT (Utf8.lexeme buf) + | upper_ident -> UIDENT (Utf8.lexeme buf) + | eof -> EOF + | _ -> raise Lexing_error diff --git a/lib/cst/nodes.ml b/lib/cst/nodes.ml new file mode 100644 index 00000000..9ef13cbd --- /dev/null +++ b/lib/cst/nodes.ml @@ -0,0 +1,300 @@ +(* The CST's job is to represent what was parsed, not what's valid. *) + +(* ---------------------------------------------------------------------- *) +(* Leaf types: no back-references into the recursive core, so they live *) +(* outside the [module rec] chain as ordinary modules. *) +(* ---------------------------------------------------------------------- *) + +module Operator = struct + type t = + | Add + | Sub + | Mul + | Div + | Eq + | LessEq + | MoreEq + | Less + | More +end + +module Type_axis = struct + type t = Value | Reference +end + +module Name_type = struct + type t = + | Ident of string + | Qualified of { package : string; ident : string } + (* intrinsic namespace, spelled @package$Ident, e.g. @primitives$I32 *) + | Intrinsic of { package : string; ident : string } +end + +module Concept = struct + type t = Type | Number +end + +module Generic_param = struct + type t = { + name : string; + type_ : Concept.t; + } +end + +(* ---------------------------------------------------------------------- *) +(* Recursive core. Every node is its own module with a [t], so the *) +(* constructor names no longer need disambiguating suffixes: *) +(* FuncCallStat -> Stat.VerbCall, FuncDecl -> Decl.Func, *) +(* NormalType -> Type_expr.Normal, SafeRet -> Ret_type.Safe, ... *) +(* The [: sig .. end = Name] wrapper is required for recursive modules. *) +(* ---------------------------------------------------------------------- *) + +module Name_expr = struct + type t = + | Ident of string + | Qualified of { package : string; ident : string } + (* intrinsic namespace, spelled @package$ident, e.g. @funcs$strToI32 *) + | Intrinsic of { package : string; ident : string } +end + +module rec Expr : sig + type t = + | IntLit of string + | FloatLit of string + | StrLit of string + | BoolLit of bool + | NameExpr of Name_expr.t + | DotAccess of { target : t; field : string } + | Ref of t + | Parenthized of t + | VerbCall of Verb_call.t + | FuncLambda of Func_lambda.t + | MethLambda of Meth_lambda.t +end = Expr + +and Abort_handle : sig + type t = + | Shorthand of Expr.t + | Longhand of { binder: string option; body: Body.t } +end = Abort_handle + +(* needs grouping because then we can unify the abort handling *) +and Verb_call : sig + type t = + | Func of { callee: Expr.t; args: Expr.t list; abort_handle: Abort_handle.t option; } + | Meth of { callee: Expr.t; this: Expr.t; args: Expr.t list; abort_handle: Abort_handle.t option; is_mut: bool; } + | Constructor of { name_type: Name_type.t; args: Expr.t list; abort_handle: Abort_handle.t option; } + | Op of { op: Operator.t; left: Expr.t; right: Expr.t; abort_handle: Abort_handle.t option; } + | Flip of { value: Expr.t; abort_handle: Abort_handle.t option; } +end = Verb_call + +and Body_field : sig + type t = { + name : string; + type_ : Type_expr.t; + } +end = Body_field + +and Mould : sig + type t = + | Struct of Body_field.t list + | Variant of Body_field.t list + | Enum of string list + | Tuple of Type_expr.t list +end = Mould + +and Moulded : sig + type t = { + mould : Mould.t; + axis : Type_axis.t; + } +end = Moulded + +and Generic_arg : sig + type t = + | Type of Type_expr.t + | Number of string + | Inferred of Param.t +end = Generic_arg + +and Verb_type : sig + type t = + | Func of { + params: Param_type.t list; + ret_type: Ret_type.t + } + | Meth of { + this_type: Type_expr.t; + params: Param_type.t list; + ret_type: Ret_type.t; + is_mut: bool + } +end = Verb_type + +and Type_expr : sig + type t = + | Path of { name : Name_type.t; generics : Generic_arg.t list } + | Verb of Verb_type.t + | Parenthesized of t +end = Type_expr + +and Param_type : sig + type t = + | Concrete of Type_expr.t + | Concept of Concept.t +end = Param_type + +and Param : sig + type t = { + name : string; + type_ : Param_type.t; + } +end = Param + +and Cond_block : sig + type t = { + cond : Expr.t; + block : Stat.t list; + } +end = Cond_block + +and Cond_seq : sig + type t = { + if_ : Cond_block.t; + elifs_ : Cond_block.t list; + else_ : Stat.t list option; + } +end = Cond_seq + +and Loop : sig + type t = { + start : Expr.t option; + end_ : Expr.t; + binder : string; + body : Stat.t list; + } +end = Loop + +and Stat : sig + type t = + | VerbCall of Verb_call.t + | Decl of Decl.t + | Abort of Expr.t + | Ret of Expr.t + | Resolve of Expr.t + | CondSeq of Cond_seq.t + | Loop of Loop.t +end = Stat + +and Body : sig + type t = + | Shorthand of Expr.t + | Longhand of Stat.t list +end = Body + +and Ret_type : sig + type t = + | Safe of Type_expr.t + | Abort of { ok : Type_expr.t; abort : Type_expr.t } + | Parenthesized of t +end = Ret_type + +and Func_lambda : sig + type t = { + params : Param.t list; + ret_type : Ret_type.t; + body : Body.t; + } +end = Func_lambda + +and Meth_lambda : sig + type t = { + this_type : Type_expr.t; + params : Param.t list; + ret_type : Ret_type.t; + is_mut : bool; + body : Body.t; + } +end = Meth_lambda + +and Type_or_moulded : sig + type t = + | Raw of Type_expr.t + | Moulded of Moulded.t +end = Type_or_moulded + +and Verb_decl : sig + type t = + | Func of { + name : string; + params : Param.t list; + ret_type : Ret_type.t; + body : Body.t; + } + | Meth of { + name : string; + this_type : Type_expr.t; + params : Param.t list; + ret_type : Ret_type.t; + is_mut : bool; + body : Body.t; + } + | Op of { + op : Operator.t; + params : Param.t list; + ret_type : Ret_type.t; + body : Body.t; + } + | Constructor of { + type_ : Name_type.t; + params : Param.t list; + body : Body.t; + } + | Flip of { + params : Param.t list; + ret_type : Ret_type.t; + body : Body.t; + } +end = Verb_decl + +and Decl : sig + type t = + | Var of { name : string; type_ : Type_expr.t; value : Expr.t } + | VarShorthand of { name : string; constructor : Name_type.t; args : Expr.t list } + | Type of { + name : string; + params : Generic_param.t list; + value : Type_or_moulded.t; + } + | Alias of { + name : string; + params : Generic_param.t list; + value : Type_expr.t; + } + | Verb of Verb_decl.t +end = Decl + +(* ---------------------------------------------------------------------- *) +(* Root + helpers. These are not part of the recursion, so they stay out *) +(* of the [module rec] block and just reference the modules above. *) +(* ---------------------------------------------------------------------- *) + +module Package = struct + type t = { decls : Decl.t list } +end + +let func_type_of_lambda (x : Func_lambda.t) : Type_expr.t = + Type_expr.Verb + (Verb_type.Func { + params = List.map (fun (p : Param.t) -> p.Param.type_) x.Func_lambda.params; + ret_type = x.Func_lambda.ret_type; + }) + +let meth_type_of_lambda (x : Meth_lambda.t) : Type_expr.t = + Type_expr.Verb + (Verb_type.Meth { + this_type = x.Meth_lambda.this_type; + params = List.map (fun (p : Param.t) -> p.Param.type_) x.Meth_lambda.params; + ret_type = x.Meth_lambda.ret_type; + is_mut = x.Meth_lambda.is_mut; + }) diff --git a/lib/cst/parse_error.ml b/lib/cst/parse_error.ml new file mode 100644 index 00000000..a140dfd7 --- /dev/null +++ b/lib/cst/parse_error.ml @@ -0,0 +1,69 @@ +let tab_width = 4 + +let visual_col_of_idx s idx = + let rec aux i vcol = + if i >= idx then vcol + else + let c = s.[i] in + let step = if c = '\t' then tab_width - (vcol mod tab_width) else 1 in + aux (i + 1) (vcol + step) + in + aux 0 0 + +let expand_tabs s = + let buf = Buffer.create (String.length s + 10) in + let rec aux i vcol = + if i >= String.length s then Buffer.contents buf + else + let c = s.[i] in + if c = '\t' then + let spaces = tab_width - (vcol mod tab_width) in + let () = Buffer.add_string buf (String.make spaces ' ') in + aux (i + 1) (vcol + spaces) + else + let () = Buffer.add_char buf c in + aux (i + 1) (vcol + 1) + in + aux 0 0 + +let format_parse_error filename input pos_start pos_end = + let line = pos_start.Lexing.pos_lnum in + let char_start = pos_start.Lexing.pos_cnum - pos_start.Lexing.pos_bol in + let lines = String.split_on_char '\n' input in + let source_line = + match List.nth_opt lines (line - 1) with + | Some l -> + let len = String.length l in + if len > 0 && l.[len - 1] = '\r' then String.sub l 0 (len - 1) else l + | None -> "" + in + let line_len = String.length source_line in + (* The error span may cross newlines. We only display the start line, so the + caret must stop at that line's end rather than following pos_end into + later lines. *) + let same_line = pos_end.Lexing.pos_lnum = line in + let char_end = + if same_line then + min line_len (pos_end.Lexing.pos_cnum - pos_start.Lexing.pos_bol) + else + line_len + in + let char_start = min char_start line_len in + let char_end = max char_start char_end in + + let vcol_start = visual_col_of_idx source_line char_start in + let vcol_end = visual_col_of_idx source_line char_end in + let visual_source_line = expand_tabs source_line in + + let buf = Buffer.create 256 in + Buffer.add_string buf + (Printf.sprintf "File \"%s\", line %d, characters %d-%d:\n" + filename line (vcol_start + 1) (vcol_end + 1)); + Buffer.add_string buf + (Printf.sprintf "%d | %s\n" line visual_source_line); + Buffer.add_string buf + (Printf.sprintf " | %s%s\n" + (String.make vcol_start ' ') + (String.make (max 1 (vcol_end - vcol_start)) '^')); + Buffer.add_string buf "Error: Parse error\n"; + Buffer.contents buf diff --git a/lib/cst/parser.mly b/lib/cst/parser.mly new file mode 100644 index 00000000..f5200b0c --- /dev/null +++ b/lib/cst/parser.mly @@ -0,0 +1,466 @@ +(*****************************) +(* token definitions *) +(*****************************) +%token INT "43" +%token FLOAT "8.647" +%token STRING "\"john\"" +%token LIDENT "length" +%token UIDENT "Int" + +%token LPAREN "(" +%token RPAREN ")" +%token COMMA "," +%token LCURLY "{" +%token RCURLY "}" +%token LBRACKET "[" +%token RBRACKET "]" +%token COLON ":" +%token SEMICOLON ";" +%token DOT "." +%token EQUAL "=" +%token PLUS "+" +%token MINUS "-" +%token STAR "*" +%token SLASH "/" +%token DOLLAR "$" +%token HASH "#" +%token AND "&" +%token AT "@" +%token EXCL "!" +%token QSTNMARK "?" +%token QSTNQSTN "??" +%token TILDE "~" +%token THICK_ARROW "=>" +%token EQEQ "==" +%token LESSEQ "<=" +%token MOREEQ ">=" +%token LESS "<" +%token MORE ">" +%token LTYPE "type" +%token ALIAS "alias" +%token UTYPE "Type" +%token NUMBER "Number" +%token STRUCT "struct" +%token VARIANT "variant" +%token TUPLE "tuple" +%token ENUM "enum" +%token IF "if" +%token ELIF "elif" +%token ELSE "else" +%token TRUE "true" +%token LOOP "loop" +%token FROM "from" +%token TO "to" +%token FALSE "false" +%token THIS "this" +%token MUT "mut" +%token ABORT "abort" +%token RETURN "return" +%token RESOLVE "resolve" +%token EOF "" + +%nonassoc EQEQ LESSEQ MOREEQ LESS MORE /* comparisons */ +%left PLUS MINUS +%left STAR SLASH +%left DOT /* field access */ +%nonassoc TILDE AND /* prefix ~ and & */ +%left LPAREN /* function application */ + +%start package + +(*************************) +(* grammar rules *) +(*************************) +%% + +package: + | decls=list(decl) EOF { { Nodes.Package.decls = decls } } + +%inline func_lambda: + | ret_type=ret_type "(" params=separated_list(COMMA, param) ")" body=body { + { Nodes.Func_lambda.params; ret_type; body } + } + +%inline meth_lambda: + | ret_type=ret_type "(" THIS this_type=type_expr + params=loption(preceded(",", separated_nonempty_list(",", param))) + ")" is_mut=boption(MUT) body=body { + { Nodes.Meth_lambda.this_type; params; ret_type; is_mut; body } + } + +%inline concept: + | "Type" { + Nodes.Concept.Type + } + | "Number" { + Nodes.Concept.Number + } + +%inline generic_arg: + | type_expr=type_expr { + Nodes.Generic_arg.Type type_expr + } + | number=INT { + Nodes.Generic_arg.Number number + } + | param=param { + Nodes.Generic_arg.Inferred param + } + +%inline generics: + | "<" generics=separated_nonempty_list(",", generic_arg) ">" { + generics + } + +%inline type_atom: + | name=name_type generics=loption(generics) { + Nodes.Type_expr.Path { name; generics } + } + | "(" type_=type_expr ")" { + Nodes.Type_expr.Parenthesized type_ + } + +%inline verb_type_suffix: + | "[" params=separated_list(",", param_type) "]" { + fun ret_type -> + Nodes.Type_expr.Verb (Nodes.Verb_type.Func { params; ret_type }) + } + | "[" THIS this_type=type_expr + params=loption(preceded(",", separated_nonempty_list(",", param_type))) + "]" is_mut=boption(MUT) { + fun ret_type -> + Nodes.Type_expr.Verb (Nodes.Verb_type.Meth { + this_type; + params; + ret_type; + is_mut; + }) + } + +(* Verb-type brackets bind more tightly than an unparenthesized abort return: + `Int ? Error[]` is `Int ? (Error[])`. To make the abort return feed the + verb type instead, group it explicitly: `(Int ? Error)[]`. *) +type_expr: + | atom=type_atom suffixes=list(verb_type_suffix) { + List.fold_left + (fun type_ suffix -> suffix (Nodes.Ret_type.Safe type_)) + atom suffixes + } + | "(" ret_type=abort_ret_type ")" + first=verb_type_suffix rest=list(verb_type_suffix) { + let type_ = first (Nodes.Ret_type.Parenthesized ret_type) in + List.fold_left + (fun type_ suffix -> suffix (Nodes.Ret_type.Safe type_)) + type_ rest + } + +%inline generic_param: + | name=UIDENT "Type" { + ({ Nodes.Generic_param.name; type_ = Nodes.Concept.Type } : Nodes.Generic_param.t) + } + | name=LIDENT "Number" { + ({ Nodes.Generic_param.name; type_ = Nodes.Concept.Number } : Nodes.Generic_param.t) + } + +decl: + | name=LIDENT type_=type_expr "=" value=expr { + Nodes.Decl.Var { name; type_; value } + } + | name=LIDENT constructor=name_type "(" args=separated_list(COMMA, expr) ")" { + Nodes.Decl.VarShorthand { name; constructor; args } + } + | name=LIDENT func_lambda=func_lambda { + Nodes.Decl.Var { + name; + type_ = Nodes.func_type_of_lambda func_lambda; + value = Nodes.Expr.FuncLambda func_lambda + } + } + | name=LIDENT meth_lambda=meth_lambda { + Nodes.Decl.Var { + name; + type_ = Nodes.meth_type_of_lambda meth_lambda; + value = Nodes.Expr.MethLambda meth_lambda + } + } + | ret_type=ret_type name=LIDENT + "(" params=separated_list(COMMA, param) ")" body=body { + Nodes.Decl.Verb (Nodes.Verb_decl.Func { + name; + params; + ret_type; + body; + }) + } + | ret_type=ret_type name=LIDENT + "(" THIS this_type=type_expr + params=loption(preceded(",", separated_nonempty_list(",", param))) + ")" is_mut=boption(MUT) body=body { + Nodes.Decl.Verb (Nodes.Verb_decl.Meth { + name; + this_type; + params; + ret_type; + is_mut; + body; + }) + } + | type_=name_type + "(" params=separated_list(COMMA, param) ")" body=body { + Nodes.Decl.Verb (Nodes.Verb_decl.Constructor { + type_; + params; + body; + }) + } + | ret_type=ret_type op=operator "(" params=separated_list(COMMA, param) ")" body=body { + Nodes.Decl.Verb (Nodes.Verb_decl.Op { + op; + params; + ret_type; + body; + }) + } + | ret_type=ret_type "~" "(" params=separated_list(COMMA, param) ")" body=body { + Nodes.Decl.Verb (Nodes.Verb_decl.Flip { + params; + ret_type; + body; + }) + } + | "type" name=UIDENT params=loption(delimited("<", separated_nonempty_list(",", generic_param), ">")) "=" value=type_or_moulded { + Nodes.Decl.Type { + name; + params; + value; + } + } + | "alias" name=UIDENT params=loption(delimited("<", separated_nonempty_list(",", generic_param), ">")) "=" value=type_expr { + Nodes.Decl.Alias { + name; + params; + value; + } + } + +(* value-identifier counterpart to name_type — both segments lowercase + since Name_expr lives in the value namespace (LIDENT), unlike + Name_type's qualified form which ends in a UIDENT type name. *) +%inline name_expr: + | name=LIDENT { Nodes.Name_expr.Ident name } + | pkg=LIDENT "$" name=LIDENT { Nodes.Name_expr.Qualified { package = pkg; ident = name } } + | "@" pkg=LIDENT "$" name=LIDENT { Nodes.Name_expr.Intrinsic { package = pkg; ident = name } } + +%inline comparison_op: + | "==" { Nodes.Operator.Eq } + | "<=" { Nodes.Operator.LessEq } + | ">=" { Nodes.Operator.MoreEq } + | "<" { Nodes.Operator.Less } + | ">" { Nodes.Operator.More } + +%inline additive_op: + | "+" { Nodes.Operator.Add } + | "-" { Nodes.Operator.Sub } + +%inline multiplicative_op: + | "*" { Nodes.Operator.Mul } + | "/" { Nodes.Operator.Div } + +(* used by decl's operator-overload form, e.g. `Int +(other Int) { ... }` — + a single bare operator token, no left/right operands involved there *) +%inline operator: + | op=comparison_op { op } + | op=additive_op { op } + | op=multiplicative_op { op } + +(* atoms: literals, names, parenthesised exprs, lambdas *) +primary: + | i=INT { Nodes.Expr.IntLit i } + | f=FLOAT { Nodes.Expr.FloatLit f } + | s=STRING { Nodes.Expr.StrLit s } + | TRUE { Nodes.Expr.BoolLit true } + | FALSE { Nodes.Expr.BoolLit false } + | name_expr=name_expr { Nodes.Expr.NameExpr name_expr } + | "(" e=expr ")" { Nodes.Expr.Parenthized e } + | func_lambda=func_lambda { Nodes.Expr.FuncLambda func_lambda } + | meth_lambda=meth_lambda { Nodes.Expr.MethLambda meth_lambda } + +(* Calls bind tighter than prefix operators, while field access binds just + below them. Thus `~value().field` is `(~value()).field`. Both remain tighter + than binary operators, so `a!b(c) * d(e)` is `(a!b(c)) * (d(e))`. *) +app: + | primary=primary { primary } + | verb_call=verb_call { Nodes.Expr.VerbCall verb_call } + | target=expr "." field=LIDENT { + Nodes.Expr.DotAccess { target; field } + } + +expr: + | app=app { app } + | left=expr op=comparison_op right=expr abort_handle=ioption(abort_handle) %prec EQEQ { + Nodes.Expr.VerbCall (Nodes.Verb_call.Op { op; left; right; abort_handle }) + } + | left=expr op=additive_op right=expr abort_handle=ioption(abort_handle) %prec PLUS { + Nodes.Expr.VerbCall (Nodes.Verb_call.Op { op; left; right; abort_handle }) + } + | left=expr op=multiplicative_op right=expr abort_handle=ioption(abort_handle) %prec STAR { + Nodes.Expr.VerbCall (Nodes.Verb_call.Op { op; left; right; abort_handle }) + } + | "~" value=expr abort_handle=ioption(abort_handle) %prec TILDE { + Nodes.Expr.VerbCall (Nodes.Verb_call.Flip { value; abort_handle }) + } + | "&" value=expr %prec AND { + Nodes.Expr.Ref value + } + +%inline body_field: + | name=LIDENT type_=type_expr ";" { + { Nodes.Body_field.name; type_ } + } + +%inline mould: + | STRUCT "{" fields=list(body_field) "}" { + Nodes.Mould.Struct fields + } + | VARIANT "{" fields=list(body_field) "}" { + Nodes.Mould.Variant fields + } + | ENUM "[" members=separated_nonempty_list(",", LIDENT) "]" { + Nodes.Mould.Enum members + } + | TUPLE "[" members=separated_nonempty_list(",", type_expr) "]" { + Nodes.Mould.Tuple members + } + +%inline moulded: + | mould=mould { + { Nodes.Moulded.mould; axis = Nodes.Type_axis.Value } + } + | "#" mould=mould { + { Nodes.Moulded.mould; axis = Nodes.Type_axis.Reference } + } + +%inline type_or_moulded: + | type_expr=type_expr { + Nodes.Type_or_moulded.Raw type_expr + } + | moulded=moulded { + Nodes.Type_or_moulded.Moulded moulded + } + +ret_type: + | ret_type=type_expr { + Nodes.Ret_type.Safe ret_type + } + | ret_type=abort_ret_type { + ret_type + } + +abort_ret_type: + | ok=type_expr "?" abort=type_expr { + Nodes.Ret_type.Abort { ok; abort } + } + +body: + | "{" statements=list(stat) "}" { + Nodes.Body.Longhand statements + } + | "=>" value=expr { + Nodes.Body.Shorthand value + } + +%inline abort_handle: + | "?" binder=ioption(LIDENT) body=body { + Nodes.Abort_handle.Longhand { binder; body } + } + | "??" value=expr { + Nodes.Abort_handle.Shorthand value + } + +(* mutable (!) and immutable (:) method calls are structurally identical, so + they share one production; the marker only decides the is_mut payload. *) +%inline meth_marker: + | "!" { true } + | ":" { false } + +%inline meth_part: + | is_mut=meth_marker name=primary { (is_mut, name) } + +(* a single call form: a flexible postfix receiver, then an optional method + part. no method part => function call; a method part => method call, with + the receiver as `this` and the (primary) name as the callee. *) +verb_call: + | receiver=app part=ioption(meth_part) "(" args=separated_list(COMMA, expr) ")" abort_handle=ioption(abort_handle) %prec LPAREN { + match part with + | None -> Nodes.Verb_call.Func { callee = receiver; args; abort_handle } + | Some (is_mut, name) -> + Nodes.Verb_call.Meth { this = receiver; callee = name; args; abort_handle; is_mut } + } + | name_type=name_type "(" args=separated_list(COMMA, expr) ")" abort_handle=ioption(abort_handle) %prec LPAREN { + Nodes.Verb_call.Constructor { name_type; args; abort_handle } + } + +%inline if_: + | IF cond=expr "{" block=list(stat) "}" { + { Nodes.Cond_block.cond; block } + } + +%inline elif_: + | ELIF cond=expr "{" block=list(stat) "}" { + { Nodes.Cond_block.cond; block } + } + +%inline else_: + | ELSE "{" statements=list(stat) "}" { + statements + } + +%inline loop: + | LOOP binder=LIDENT start=ioption(preceded(FROM, expr)) TO end_=expr "{" statements=list(stat) "}" { + ({ Nodes.Loop.start; end_; binder; body = statements } : Nodes.Loop.t) + } + +stat: + | decl=decl { Nodes.Stat.Decl decl } + | verb_call=verb_call { + Nodes.Stat.VerbCall verb_call + } + | ABORT value=expr { + Nodes.Stat.Abort value + } + | RETURN value=expr { + Nodes.Stat.Ret value + } + | RESOLVE value=expr { + Nodes.Stat.Resolve value + } + | if_=if_ elifs_=list(elif_) else_=ioption(else_) { + Nodes.Stat.CondSeq Nodes.Cond_seq.{ if_; elifs_; else_ } + } + | loop=loop { + Nodes.Stat.Loop loop + } + +%inline param_type: + | type_=type_expr { + Nodes.Param_type.Concrete type_ + } + | type_=concept { + Nodes.Param_type.Concept type_ + } + +%inline param: + | name=LIDENT type_=type_expr { + ({ Nodes.Param.name; type_ = Nodes.Param_type.Concrete type_ } : Nodes.Param.t) + } + | name=UIDENT "Type" { + ({ Nodes.Param.name; type_ = Nodes.Param_type.Concept Nodes.Concept.Type } : Nodes.Param.t) + } + | name=LIDENT "Number" { + ({ Nodes.Param.name; type_ = Nodes.Param_type.Concept Nodes.Concept.Number } : Nodes.Param.t) + } + +%inline name_type: + | name=UIDENT { Nodes.Name_type.Ident name } + | pkg=LIDENT "$" name=UIDENT { Nodes.Name_type.Qualified { package = pkg; ident = name } } + | "@" pkg=LIDENT "$" name=UIDENT { Nodes.Name_type.Intrinsic { package = pkg; ident = name } } diff --git a/lib/cst/parser.mly.old b/lib/cst/parser.mly.old new file mode 100644 index 00000000..c3a19792 --- /dev/null +++ b/lib/cst/parser.mly.old @@ -0,0 +1,386 @@ +(*****************************) +(* token definitions *) +(*****************************) +%token INT "43" +%token FLOAT "8.647" +%token STRING "\"john\"" +%token LIDENT "length" +%token UIDENT "Int" + +%token LPAREN "(" +%token RPAREN ")" +%token COMMA "," +%token LCURLY "{" +%token RCURLY "}" +%token LBRACKET "[" +%token RBRACKET "]" +%token COLON ":" +%token SEMICOLON ";" +%token DOT "." +%token EQUAL "=" +%token PLUS "+" +%token MINUS "-" +%token STAR "*" +%token SLASH "/" +%token DOLLAR "$" +%token AND "&" +%token AT "@" +%token EXCL "!" +%token QSTNMARK "?" +%token QSTNQSTN "??" +%token TILDE "~" +%token THICK_ARROW "=>" +%token EQEQ "==" +%token LESSEQ "<=" +%token MOREEQ ">=" +%token LESS "<" +%token MORE ">" +%token LTYPE "type" +%token ALIAS "alias" +%token UTYPE "Type" +%token NUMBER "Number" +%token CLASS "class" +%token STRUCT "struct" +%token VARIANT "variant" +%token TUPLE "tuple" +%token ENUM "enum" +%token IF "if" +%token ELIF "elif" +%token ELSE "else" +%token TRUE "true" +%token LOOP "loop" +%token FROM "from" +%token TO "to" +%token FALSE "false" +%token THIS "this" +%token MUT "mut" +%token ABORT "abort" +%token RETURN "return" +%token RESOLVE "resolve" +%token ERROR "" +%token EOF "" + +%nonassoc EQEQ LESSEQ MOREEQ LESS MORE /* comparisons */ +%left PLUS MINUS +%left STAR SLASH +%left LPAREN /* function application */ +%nonassoc IF +%nonassoc ELSE +%nonassoc TILDE /* prefix ~ */ +%left DOT /* field access */ + +%start package + +(*************************) +(* grammar rules *) +(*************************) +%% + +package: + | decls=list(decl) EOF { { Nodes.Package.decls = decls } } + +%inline func_lambda: + | ret_type=ret_type "(" params=separated_list(COMMA, param) ")" body=body { + { Nodes.Func_lambda.params; ret_type; body } + } + +%inline meth_lambda: + | ret_type=ret_type "(" THIS this_type=type_expr + params=loption(preceded(",", separated_nonempty_list(",", param))) + ")" is_mut=boption(MUT) body=body { + { Nodes.Meth_lambda.this_type; params; ret_type; is_mut; body } + } + +%inline generic_param_type: + | "Type" { + Nodes.Generic_param_type.Type + } + | "Number" { + Nodes.Generic_param_type.Number + } + +%inline generic_param: + | name=UIDENT type_=generic_param_type { + ({ Nodes.Generic_param.name; type_ } : Nodes.Generic_param.t) + } + +decl: + | name=LIDENT type_=type_expr "=" value=expr { + Nodes.Decl.Var { name; type_; value } + } + | name=LIDENT type_=name_type "(" args=separated_list(COMMA, expr) ")" { + Nodes.Decl.VarShorthand { name; type_; args } + } + | name=LIDENT func_lambda=func_lambda { + Nodes.Decl.Var { + name; + type_ = Nodes.func_type_of_lambda func_lambda; + value = Nodes.Expr.FuncLambda func_lambda + } + } + | name=LIDENT meth_lambda=meth_lambda { + Nodes.Decl.Var { + name; + type_ = Nodes.meth_type_of_lambda meth_lambda; + value = Nodes.Expr.MethLambda meth_lambda + } + } + | ret_type=ret_type name=LIDENT + generic_header=ioption(delimited("<", separated_nonempty_list(",", generic_param), ">")) + "(" params=separated_list(COMMA, param) ")" body=body { + Nodes.Decl.Func { + name; + generic_header; + params; + ret_type; + body; + } + } + | ret_type=ret_type name=LIDENT + generic_header=ioption(delimited("<", separated_nonempty_list(",", generic_param), ">")) + "(" THIS this_type=type_expr + params=loption(preceded(",", separated_nonempty_list(",", param))) + ")" is_mut=boption(MUT) body=body { + Nodes.Decl.Meth { + name; + generic_header; + this_type; + params; + ret_type; + is_mut; + body; + } + } + | type_=name_type + "(" params=separated_list(COMMA, param) ")" body=body { + Nodes.Decl.Constructor { + type_; + params; + body; + } + } + | ret_type=ret_type op=operator "(" params=separated_list(COMMA, param) ")" body=body { + Nodes.Decl.Op { + op; + params; + ret_type; + body; + } + } + | ret_type=ret_type "~" "(" params=separated_list(COMMA, param) ")" body=body { + Nodes.Decl.Flip { + params; + ret_type; + body; + } + } + | "type" name=UIDENT params=ioption(delimited("<", separated_nonempty_list(",", generic_param), ">")) "=" value=type_expr { + Nodes.Decl.Type { + name; + params; + value; + } + } + | "alias" name=UIDENT params=ioption(delimited("<", separated_nonempty_list(",", generic_param), ">")) "=" value=type_expr { + Nodes.Decl.Alias { + name; + params; + value; + } + } + +%inline ret_type: + | ret_type=type_expr { + Nodes.Ret_type.Safe ret_type + } + | safe_type=type_expr "?" abort_type=type_expr { + Nodes.Ret_type.Abort (safe_type, abort_type) + } + +body: + | "{" statements=list(stat) "}" { + Nodes.Body.Scope statements + } + | "=>" value=expr { + Nodes.Body.RetShorthand value + } + +func_call: + | callee=expr "(" args=separated_list(COMMA, expr) ")" %prec LPAREN + { Nodes.Func_call.Safe Nodes.Safe_call.{ callee; args } } + | callee=expr "(" args=separated_list(COMMA, expr) ")" "?" binder=ioption(LIDENT) body=body %prec LPAREN + { Nodes.Func_call.Abort + Nodes.Abort_call.{ callee; args; binder; + handle_block = Nodes.Abort_handle.Body body } } + | callee=expr "(" args=separated_list(COMMA, expr) ")" "??" binder=ioption(LIDENT) value=expr %prec LPAREN + { Nodes.Func_call.Abort + Nodes.Abort_call.{ callee; args; binder; + handle_block = Nodes.Abort_handle.Shorthand value } } + +%inline if_: + | IF cond=expr "{" block=list(stat) "}" { + { Nodes.Cond_block.cond; block } + } + +%inline elif_: + | ELIF cond=expr "{" block=list(stat) "}" { + { Nodes.Cond_block.cond; block } + } + +%inline else_: + | ELSE "{" statements=list(stat) "}" { + statements + } + +%inline loop: + | LOOP binder=LIDENT start=ioption(preceded(FROM, expr)) TO end_=expr "{" statements=list(stat) "}" { + ({ Nodes.Loop.start; end_; binder; body = statements } : Nodes.Loop.t) + } + +stat: + | decl=decl { Nodes.Stat.Decl decl } + | func_call=func_call { + Nodes.Stat.FuncCall func_call + } + | ABORT value=expr { + Nodes.Stat.Abort value + } + | RETURN value=expr { + Nodes.Stat.Ret value + } + | RESOLVE value=expr { + Nodes.Stat.Resolve value + } + | if_=if_ elifs_=list(elif_) else_=ioption(else_) { + Nodes.Stat.CondSeq Nodes.Cond_seq.{ if_; elifs_; else_ } + } + | loop=loop { + Nodes.Stat.Loop loop + } + +%inline param_type: + | type_=type_expr { + Nodes.Param_type.Normal type_ + } + | type_=generic_param_type { + Nodes.Param_type.Generic type_ + } + +%inline param: + | name=LIDENT type_=type_expr { + ({ Nodes.Param.name; type_ = Nodes.Param_type.Normal type_ } : Nodes.Param.t) + } + | name=UIDENT type_=generic_param_type { + ({ Nodes.Param.name; type_ = Nodes.Param_type.Generic type_ } : Nodes.Param.t) + } + +%inline name_type: + | name=UIDENT { Nodes.Name_type.Simple name } + | pkg=LIDENT "$" name=UIDENT { Nodes.Name_type.Qualified (pkg, name) } + +%inline call_type: + | ret=ret_type "[" params=separated_list(",", param_type) "]" { + Nodes.Call_type.Func { params; ret_type = ret } + } + | ret=ret_type "[" THIS this_type=type_expr + params=loption(preceded(",", separated_nonempty_list(",", param_type))) + "]" is_mut=boption(MUT) { + Nodes.Call_type.Meth { this_type; params; ret_type = ret; is_mut } + } + +%inline body_field: + | name=LIDENT type_=type_expr ";" { + ({ Nodes.Body_field.name; type_ } : Nodes.Body_field.t) + } + +%inline mould: + | CLASS "{" fields=list(body_field) "}" { + Nodes.Mould.Class fields + } + | STRUCT "{" fields=list(body_field) "}" { + Nodes.Mould.Struct fields + } + | VARIANT "{" fields=list(body_field) "}" { + Nodes.Mould.Variant fields + } + | ENUM "[" cases=separated_list(",", LIDENT) "]" { + Nodes.Mould.Enum cases + } + | TUPLE "[" types=separated_list(",", type_expr) "]" { + Nodes.Mould.Tuple types + } + + +%inline generic_arg: + | type_expr=type_expr { + Nodes.Generic_arg.Type type_expr + } + | number=INT { + Nodes.Generic_arg.Number number + } + + +refable: + | name_type=name_type { + Nodes.Refable.Name name_type + } + | name_type=name_type "<" params=separated_nonempty_list(",", generic_arg) ">" { + Nodes.Refable.Generic (name_type, params) + } + | body_type=body_type { + Nodes.Refable.Body body_type + } + +type_expr: + | call_type=call_type { + Nodes.Type_expr.Call call_type + } + | refable=refable { + Nodes.Type_expr.Normal refable + } + | "&" refable=refable { + Nodes.Type_expr.Ref refable + } + +operator: + | "+" { Nodes.Operator.Add } + | "-" { Nodes.Operator.Sub } + | "*" { Nodes.Operator.Mul } + | "/" { Nodes.Operator.Div } + | "=="{ Nodes.Operator.Eq } + | "<="{ Nodes.Operator.LessEq } + | ">="{ Nodes.Operator.MoreEq } + | "<" { Nodes.Operator.Less } + | ">" { Nodes.Operator.More } + +expr: + | int=INT { Nodes.Expr.IntLit int } + | float=FLOAT { Nodes.Expr.FloatLit float } + | string=STRING { Nodes.Expr.StrLit string } + | TRUE { Nodes.Expr.BoolLit true } + | FALSE { Nodes.Expr.BoolLit false } + | ident=LIDENT { Nodes.Expr.Ident ident } + | pkg=LIDENT "$" ident=LIDENT { Nodes.Expr.QualifiedIdent (pkg, ident) } + | e1=expr "+" e2=expr %prec PLUS { Nodes.Expr.Op { left=e1; right=e2; operator=Nodes.Operator.Add } } + | e1=expr "-" e2=expr %prec MINUS { Nodes.Expr.Op { left=e1; right=e2; operator=Nodes.Operator.Sub } } + | e1=expr "*" e2=expr %prec STAR { Nodes.Expr.Op { left=e1; right=e2; operator=Nodes.Operator.Mul } } + | e1=expr "/" e2=expr %prec SLASH { Nodes.Expr.Op { left=e1; right=e2; operator=Nodes.Operator.Div } } + | e1=expr "==" e2=expr %prec EQEQ { Nodes.Expr.Op { left=e1; right=e2; operator=Nodes.Operator.Eq } } + | e1=expr "<=" e2=expr %prec LESSEQ { Nodes.Expr.Op { left=e1; right=e2; operator=Nodes.Operator.LessEq } } + | e1=expr ">=" e2=expr %prec MOREEQ { Nodes.Expr.Op { left=e1; right=e2; operator=Nodes.Operator.MoreEq } } + | e1=expr "<" e2=expr %prec LESS { Nodes.Expr.Op { left=e1; right=e2; operator=Nodes.Operator.Less } } + | e1=expr ">" e2=expr %prec MORE { Nodes.Expr.Op { left=e1; right=e2; operator=Nodes.Operator.More } } + | "~" value=expr %prec TILDE { Nodes.Expr.Flip value } + | value=expr "." field=LIDENT %prec DOT { Nodes.Expr.DotAccess (value, field) } + | type_=name_type "(" args=separated_list(COMMA, expr) ")" %prec LPAREN + { Nodes.Expr.ConstructorCall { type_; args } } + | "(" e=expr ")" { Nodes.Expr.Parenthized e } + | func_call=func_call { + Nodes.Expr.FuncCall func_call + } + | func_lambda=func_lambda { + Nodes.Expr.FuncLambda func_lambda + } + | meth_lambda=meth_lambda { + Nodes.Expr.MethLambda meth_lambda + } diff --git a/lib/cst/to_tree_graph.ml b/lib/cst/to_tree_graph.ml new file mode 100644 index 00000000..3063ac45 --- /dev/null +++ b/lib/cst/to_tree_graph.ml @@ -0,0 +1,332 @@ +open Tree_graph + +(* =================================================== *) +(* rule: helpers return raw nodes, never self-wrap *) +(* --- *) +(* a helper like name_type_to_node returns the Node *) +(* for its value directly, it never calls group on *) +(* its own output, even if that means the result *) +(* looks "unlabeled" on its own. *) +(* *) +(* let name_type_to_node x = match x with *) +(* | Ident s -> Leaf s *) +(* | Qualified { .. } -> Leaf (...) *) +(* (* no group "name_type" (...) here *) *) +(* *) +(* labeling is the caller's decision, not the callee's *) +(* the same value can be grouped under different names *) +(* depending on where it's used (a "type" field here, *) +(* a "constructor" field there) or left ungrouped *) +(* entirely if it's just one entry among fields [...]. *) +(* if a helper pre-wraps its own result, that decision *) +(* is made once and for all callers, even the ones *) +(* that wanted a different label or no label at all. *) +(* =================================================== *) + +let name_type_to_node (x: Nodes.Name_type.t) = match x with + | Ident s -> Leaf s + | Qualified { package; ident } -> Leaf (package ^ "$" ^ ident) + | Intrinsic { package; ident } -> Leaf ("@" ^ package ^ "$" ^ ident) + +let name_expr_to_node (x: Nodes.Name_expr.t) = match x with + | Ident s -> Leaf s + | Qualified { package; ident } -> Leaf (package ^ "$" ^ ident) + | Intrinsic { package; ident } -> Leaf ("@" ^ package ^ "$" ^ ident) + +let rec concept_to_node (x: Nodes.Concept.t) = match x with + | Type -> Leaf "Type" + | Number -> Leaf "Number" + +and param_type_to_node (x: Nodes.Param_type.t) = match x with + | Concrete x -> type_to_node x + | Concept x -> concept_to_node x + +and verb_type_to_node (x: Nodes.Verb_type.t) = match x with + | Func { params; ret_type } -> + fields [ + ("param", map_seq param_type_to_node params); + ("type", ret_to_node ret_type); + ] + | Meth { this_type; params; ret_type; is_mut } -> + fields [ + ("this_type", type_to_node this_type); + ("param", map_seq param_type_to_node params); + ("type", ret_to_node ret_type); + ("is_mut", Leaf (string_of_bool is_mut)); + ] + +and body_field_to_node (x: Nodes.Body_field.t) = + fields [ + ("name", Leaf x.name); + ("type", type_to_node x.type_); + ] + +and mould_to_node (x: Nodes.Mould.t) = match x with + | Struct x -> group "struct" (map_seq body_field_to_node x) + | Variant x -> group "variant" (map_seq body_field_to_node x) + | Enum x -> group "enum" (map_seq (fun x -> Leaf x) x) + | Tuple x -> group "tuple" (map_seq type_to_node x) + +and generic_arg_to_node (x: Nodes.Generic_arg.t) = match x with + | Type x -> type_to_node x + | Number x -> group "number" (Leaf x) + | Inferred x -> group "param" (param_to_node x) + +and type_to_node (x: Nodes.Type_expr.t) = match x with + | Verb x -> verb_type_to_node x + | Parenthesized x -> group "parenthesized" (type_to_node x) + | Path { name; generics } -> + fields [ + ("qualifier", name_type_to_node name); + ("args", map_seq generic_arg_to_node generics); + ] + +and verb_call_to_node (x: Nodes.Verb_call.t) = match x with + | Func { callee; args; abort_handle } -> + group "func_call" (fields [ + ("callee", expr_to_node callee); + ("args", map_seq expr_to_node args); + ("abort", Option.map abort_handle_to_node abort_handle |> Option.value ~default:(Leaf "none")); + ]) + | Meth { callee; this; args; abort_handle; is_mut } -> + group "meth_call" (fields [ + ("callee", expr_to_node callee); + ("this", expr_to_node this); + ("args", map_seq expr_to_node args); + ("is_mut", Leaf (string_of_bool is_mut)); + ("abort", Option.map abort_handle_to_node abort_handle |> Option.value ~default:(Leaf "none")); + ]) + | Constructor { name_type; args; abort_handle } -> + group "ctor_call" (fields [ + ("type", name_type_to_node name_type); + ("args", map_seq expr_to_node args); + ("abort", Option.map abort_handle_to_node abort_handle |> Option.value ~default:(Leaf "none")); + ]) + | Op { op; left; right; abort_handle } -> + group "op_call" (fields [ + ("op", Leaf (op_to_name op)); + ("left", expr_to_node left); + ("right", expr_to_node right); + ("abort", Option.map abort_handle_to_node abort_handle |> Option.value ~default:(Leaf "none")); + ]) + | Flip { value; abort_handle } -> + group "flip_call" (fields [ + ("value", expr_to_node value); + ("abort", Option.map abort_handle_to_node abort_handle |> Option.value ~default:(Leaf "none")); + ]) + +and expr_to_node (x: Nodes.Expr.t) = match x with + | IntLit x -> Leaf x + | FloatLit x -> Leaf x + | StrLit x -> Leaf x + | BoolLit x -> Leaf (string_of_bool x) + | NameExpr x -> name_expr_to_node x + | DotAccess { target; field } -> + group "dot_access" (fields [ + ("target", expr_to_node target); + ("field", Leaf field); + ]) + | Ref x -> + group "ref" (expr_to_node x) + | Parenthized x -> + group "parenthized" (expr_to_node x) + | FuncLambda x -> + group "func_lambda" (func_lambda_to_node x) + | MethLambda x -> + group "meth_lambda" (meth_lambda_to_node x) + | VerbCall x -> + verb_call_to_node x + +and op_to_name (x: Nodes.Operator.t) = match x with + | Add -> "+" + | Sub -> "-" + | Mul -> "*" + | Div -> "/" + | Eq -> "==" + | LessEq -> "<=" + | MoreEq -> ">=" + | Less -> "<" + | More -> ">" + +and abort_handle_to_node (x: Nodes.Abort_handle.t) = match x with + | Longhand { binder; body } -> + let fs = [("body", body_to_node body)] in + let fs = match binder with + | Some b -> ("binder", Leaf b) :: fs + | None -> fs + in + group "longhand" (fields fs) + | Shorthand x -> group "shorthand" (expr_to_node x) + +and param_to_node (x: Nodes.Param.t) = + fields [ + ("name", Leaf x.name); + ("type", param_type_to_node x.type_); + ] + +and params_to_node (x: Nodes.Param.t list) = + map_seq param_to_node x + +and elif_to_fields (x: Nodes.Cond_block.t) = + fields [ + ("cond", expr_to_node x.cond); + ("block", map_seq stat_to_node x.block); + ] + +and cond_seq_to_node (x: Nodes.Cond_seq.t) = + let cond_seq = [ + ("if", fields [ + ("cond", expr_to_node x.if_.cond); + ("block", map_seq stat_to_node x.if_.block); + ]); + ("elif", map_seq elif_to_fields x.elifs_); + ] in + let cond_seq = match x.else_ with + | Some x -> ("else", fields [ + ("block", map_seq stat_to_node x); + ]) :: cond_seq + | None -> cond_seq + in fields cond_seq + +and loop_to_node (x: Nodes.Loop.t) = + let fs = [ + ("stats", map_seq stat_to_node x.body); + ("end", expr_to_node x.end_); + ("binder", Leaf x.binder); + ] in + let fs = match x.start with + | Some x -> ("start", expr_to_node x) :: fs + | None -> fs + in + fields fs + +and stat_to_node (x: Nodes.Stat.t) = match x with + | VerbCall x -> verb_call_to_node x + | Decl x -> decl_to_node x + | Abort x -> group "abort_stat" (expr_to_node x) + | Ret x -> group "ret_stat" (expr_to_node x) + | Resolve x -> group "resolve_stat" (expr_to_node x) + | CondSeq x -> cond_seq_to_node x + | Loop x -> group "loop" (loop_to_node x) + +and body_to_node (x: Nodes.Body.t) = match x with + | Longhand x -> + group "scope" (fields [ + ("stat", map_seq stat_to_node x); + ]) + | Shorthand x -> + group "ret_shorthand" (expr_to_node x) + +and ret_to_node (x: Nodes.Ret_type.t) = match x with + | Safe ret -> type_to_node ret + | Abort { ok; abort } -> + fields [ + ("safe_type", type_to_node ok); + ("abort_type", type_to_node abort); + ] + | Parenthesized ret -> group "parenthesized" (ret_to_node ret) + +and func_lambda_to_node (x: Nodes.Func_lambda.t) = + fields [ + ("param", params_to_node x.params); + ("ret_type", ret_to_node x.ret_type); + ("body", body_to_node x.body); + ] + +and meth_lambda_to_node (x: Nodes.Meth_lambda.t) = + fields [ + ("this_type", type_to_node x.this_type); + ("param", params_to_node x.params); + ("ret_type", ret_to_node x.ret_type); + ("body", body_to_node x.body); + ("is_mut", Leaf (string_of_bool x.is_mut)); + ] + +and type_axis_to_node (x: Nodes.Type_axis.t) = match x with + | Value -> Leaf "value" + | Reference -> Leaf "reference" + +and moulded_to_node (x: Nodes.Moulded.t) = + fields [ + ("mould", mould_to_node x.mould); + ("type_axis", type_axis_to_node x.axis); + ] + +and type_or_moulded_to_node (x: Nodes.Type_or_moulded.t) = match x with + | Raw x -> type_to_node x + | Moulded x -> moulded_to_node x + +and decl_to_node (x: Nodes.Decl.t) = match x with + | Var { name; type_; value } -> + group "var_decl" (fields [ + ("name", Leaf name); + ("type", type_to_node type_); + ("value", expr_to_node value); + ]) + | VarShorthand { name; constructor; args } -> + group "var_decl_shorthand" (fields [ + ("name", Leaf name); + ("type", name_type_to_node constructor); + ("args", map_seq expr_to_node args); + ]) + | Type x -> + group "type_decl" (fields [ + ("name", Leaf x.name); + ("params", map_seq generic_param_to_node x.params); + ("value", type_or_moulded_to_node x.value); + ]) + | Alias x -> + group "alias_decl" (fields [ + ("name", Leaf x.name); + ("params", map_seq generic_param_to_node x.params); + ("value", type_to_node x.value); + ]) + | Verb x -> verb_decl_to_node x + +and verb_decl_to_node (x: Nodes.Verb_decl.t) = match x with + | Func x -> + group "func_decl" (fields [ + ("name", Leaf x.name); + ("param", params_to_node x.params); + ("ret_type", ret_to_node x.ret_type); + ("body", body_to_node x.body); + ]) + | Meth x -> + group "meth_decl" (fields [ + ("name", Leaf x.name); + ("this_type", type_to_node x.this_type); + ("param", params_to_node x.params); + ("ret_type", ret_to_node x.ret_type); + ("body", body_to_node x.body); + ("is_mut", Leaf (string_of_bool x.is_mut)); + ]) + | Constructor x -> + group "ctor_decl" (fields [ + ("type", name_type_to_node x.type_); + ("param", params_to_node x.params); + ("body", body_to_node x.body); + ]) + | Op x -> + group "op_decl" (fields [ + ("op", Leaf (op_to_name x.op)); + ("param", params_to_node x.params); + ("ret_type", ret_to_node x.ret_type); + ("body", body_to_node x.body); + ]) + | Flip x -> + group "flip_decl" (fields [ + ("param", params_to_node x.params); + ("ret_type", ret_to_node x.ret_type); + ("body", body_to_node x.body); + ]) + +and generic_param_to_node (x: Nodes.Generic_param.t) = + fields [ + ("name", Leaf x.name); + ("type", concept_to_node x.type_); + ] + +let to_node ({ decls }: Nodes.Package.t) = + group "package" (fields [ + ("declarations", map_seq decl_to_node decls); + ]) diff --git a/lib/tree_graph/dune b/lib/tree_graph/dune new file mode 100644 index 00000000..756b7f6e --- /dev/null +++ b/lib/tree_graph/dune @@ -0,0 +1,2 @@ +(library + (name tree_graph)) diff --git a/lib/tree_graph/node.ml b/lib/tree_graph/node.ml new file mode 100644 index 00000000..87ca62aa --- /dev/null +++ b/lib/tree_graph/node.ml @@ -0,0 +1,13 @@ +module StringMap = Map.Make(String) [@@deriving map] + +type node = + | Leaf of string + | Group of { title: string; body: node } + | Fields of node StringMap.t + | Sequence of node list + +(* Smart constructors *) +let group title body = Group { title; body } +let fields pairs = Fields (StringMap.of_list pairs) +let seq xs = Sequence xs +let map_seq f xs = Sequence (List.map f xs) diff --git a/lib/tree_graph/render.ml b/lib/tree_graph/render.ml new file mode 100644 index 00000000..bc6a80a7 --- /dev/null +++ b/lib/tree_graph/render.ml @@ -0,0 +1,58 @@ +type item = + | ILeaf of string * string + | IContainer of string * item list + +let rec collect ~name node = + match node with + | Node.Leaf v -> + [ILeaf (name, v)] + + | Node.Group { title; body } -> + let new_name = if name = "" then title else name ^ " > " ^ title in + collect ~name:new_name body + + | Node.Fields map -> + let nested_items = + Node.StringMap.bindings map + |> List.map (fun (k, v) -> collect ~name:k v) + |> List.flatten + in + if name = "" then nested_items + else [IContainer (name, nested_items)] + + | Node.Sequence list -> + if list = [] then + if name = "" then [] else [IContainer (name, [])] + else + List.mapi (fun i v -> + let element_name = + if name = "" then Printf.sprintf "[%d]" i + else Printf.sprintf "%s[%d]" name i + in + collect ~name:element_name v + ) list |> List.flatten + +let render root_node = + let items = collect ~name:"" root_node in + let rec print_items prefix items = + let len = List.length items in + List.iteri (fun i item -> + let is_last = (i = len - 1) in + let branch = if is_last then "└── " else "├── " in + let nested_prefix = prefix ^ (if is_last then " " else "│ ") in + match item with + | ILeaf (name, value) -> + if name = "" then + Printf.printf "%s%s%s\n" prefix branch value + else + Printf.printf "%s%s%s: %s\n" prefix branch name value + | IContainer (name, nested_items) -> + if name = "" then + print_items prefix nested_items + else begin + Printf.printf "%s%s%s:\n" prefix branch name; + print_items nested_prefix nested_items + end + ) items + in + print_items "" items diff --git a/lib/tree_graph/tree_graph.ml b/lib/tree_graph/tree_graph.ml new file mode 100644 index 00000000..6db1f99d --- /dev/null +++ b/lib/tree_graph/tree_graph.ml @@ -0,0 +1,2 @@ +include Node +let render node = Render.render node diff --git a/machine-config.example b/machine-config.example new file mode 100644 index 00000000..f30b6ef4 --- /dev/null +++ b/machine-config.example @@ -0,0 +1,6 @@ +# Copy this file to machine-config.txt and tune it for the current machine. +# The memory budget is shared by all ambiguity-search workers. +AMBIGUITY_MEMORY_MB=512 +AMBIGUITY_MAX_FRONTIER_RATIO=1.0 +AMBIGUITY_JOBS=4 +AMBIGUITY_MENHIR=menhir diff --git a/reports/2026-07-23_21-38-17.txt b/reports/2026-07-23_21-38-17.txt new file mode 100644 index 00000000..24104a6b --- /dev/null +++ b/reports/2026-07-23_21-38-17.txt @@ -0,0 +1,28 @@ +Search profile: deep-function-body +Rotate deep search waves through statements in a function body. +Tokens 15..50 · timeout 30m · 50 witnesses · 10 nodes per depth +Prefix (5): UIDENT LIDENT LPAREN RPAREN LCURLY + +Memory budget: 6144 MiB total across 4 worker(s); workers compact at 1229 MiB and stop admitting frontiers at 1382 MiB each (10% reserved); per-worker limits are 710564 queued frontiers and 710564 retained dedup frontiers (ratio 1). +Search constraints: 15..50 total tokens; 5-token prefix; 10 nodes per depth. +Prefix tokens: UIDENT LIDENT LPAREN RPAREN LCURLY +Found 4 complete ambiguity families. + +1. Tokens (15): UIDENT LIDENT LPAREN RPAREN LCURLY ABORT TRUE EQEQ AND TILDE FALSE QSTNQSTN FALSE RCURLY EOF + Source: Int length ( ) { abort true == & ~ false ?? false } + Conflict origins: state 214 on QSTNQSTN, state 496 on QSTNQSTN + +2. Tokens (15): UIDENT LIDENT LPAREN RPAREN LCURLY ABORT TRUE LESSEQ TILDE FALSE QSTNQSTN AND STRING RCURLY EOF + Source: Int length ( ) { abort true <= ~ false ?? & "john" } + Conflict origins: state 198 on QSTNQSTN, state 496 on QSTNQSTN + +3. Tokens (17): UIDENT LIDENT LPAREN RPAREN LCURLY ABORT TRUE ABORT AND FALSE LESS TILDE FALSE QSTNQSTN INT RCURLY EOF + Source: Int length ( ) { abort true abort & false < ~ false ?? 43 } + Conflict origins: state 206 on QSTNQSTN, state 496 on QSTNQSTN + +4. Tokens (17): UIDENT LIDENT LPAREN RPAREN LCURLY ABORT UIDENT LBRACKET RBRACKET LPAREN RPAREN THICK_ARROW FALSE DOT LIDENT RCURLY EOF + Source: Int length ( ) { abort Int [ ] ( ) => false . length } + Conflict origins: state 503 on DOT + +Explored 19546256 frontiers (22249396 unique); 415852 conflict seeds. +Search stopped because one or more workers reached a search limit. diff --git a/reports/ambiguities-report.txt b/reports/ambiguities-report.txt new file mode 100644 index 00000000..5e7b1b12 --- /dev/null +++ b/reports/ambiguities-report.txt @@ -0,0 +1,205 @@ +Welcome to devbox! +Found 50 complete ambiguity families. + +1. Tokens (9): LIDENT UIDENT EQUAL TILDE TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = ~ ~ false ?? false + Conflict origins: state 496 on QSTNQSTN + +2. Tokens (10): LIDENT UIDENT EQUAL FALSE EQEQ TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false == ~ false ?? false + Conflict origins: state 214 on QSTNQSTN, state 496 on QSTNQSTN + +3. Tokens (10): LIDENT UIDENT EQUAL FALSE LESS TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false < ~ false ?? false + Conflict origins: state 206 on QSTNQSTN, state 496 on QSTNQSTN + +4. Tokens (10): LIDENT UIDENT EQUAL FALSE LESSEQ TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false <= ~ false ?? false + Conflict origins: state 198 on QSTNQSTN, state 496 on QSTNQSTN + +5. Tokens (10): LIDENT UIDENT EQUAL FALSE MINUS TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false - ~ false ?? false + Conflict origins: state 178 on QSTNQSTN, state 496 on QSTNQSTN + +6. Tokens (10): LIDENT UIDENT EQUAL FALSE MORE TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false > ~ false ?? false + Conflict origins: state 190 on QSTNQSTN, state 496 on QSTNQSTN + +7. Tokens (10): LIDENT UIDENT EQUAL FALSE MOREEQ TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false >= ~ false ?? false + Conflict origins: state 174 on QSTNQSTN, state 496 on QSTNQSTN + +8. Tokens (10): LIDENT UIDENT EQUAL FALSE PLUS TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false + ~ false ?? false + Conflict origins: state 166 on QSTNQSTN, state 496 on QSTNQSTN + +9. Tokens (10): LIDENT UIDENT EQUAL FALSE SLASH TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false / ~ false ?? false + Conflict origins: state 158 on QSTNQSTN, state 496 on QSTNQSTN + +10. Tokens (10): LIDENT UIDENT EQUAL FALSE STAR TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false * ~ false ?? false + Conflict origins: state 148 on QSTNQSTN, state 496 on QSTNQSTN + +11. Tokens (10): LIDENT UIDENT EQUAL TILDE FALSE LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = ~ false ( ) ?? false + Conflict origins: state 135 on LPAREN, state 138 on QSTNQSTN, state 496 on QSTNQSTN + +12. Tokens (10): LIDENT UIDENT EQUAL TILDE TILDE FALSE QSTNMARK LCURLY RCURLY EOF + Source: length Int = ~ ~ false ? { } + Conflict origins: state 496 on QSTNMARK + +13. Tokens (10): LIDENT UIDENT EQUAL TILDE UIDENT LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = ~ Int ( ) ?? false + Conflict origins: state 93 on LPAREN, state 132 on QSTNQSTN, state 496 on QSTNQSTN + +14. Tokens (11): LIDENT UIDENT EQUAL TILDE FALSE QSTNMARK THICK_ARROW FALSE DOT LIDENT EOF + Source: length Int = ~ false ? => false . length + Conflict origins: state 496 on QSTNMARK, state 503 on DOT + +15. Tokens (11): LIDENT UIDENT EQUAL UIDENT LPAREN RPAREN THICK_ARROW FALSE DOT LIDENT EOF + Source: length Int = Int ( ) => false . length + Conflict origins: state 93 on LPAREN, state 503 on DOT + +16. Tokens (12): LIDENT UIDENT EQUAL FALSE EQEQ FALSE QSTNMARK THICK_ARROW FLOAT DOT LIDENT EOF + Source: length Int = false == false ? => 8.647 . length + Conflict origins: state 214 on QSTNMARK, state 503 on DOT + +17. Tokens (12): LIDENT UIDENT EQUAL FALSE LESS FALSE QSTNMARK THICK_ARROW FLOAT DOT LIDENT EOF + Source: length Int = false < false ? => 8.647 . length + Conflict origins: state 206 on QSTNMARK, state 503 on DOT + +18. Tokens (12): LIDENT UIDENT EQUAL FALSE LESSEQ FALSE QSTNMARK THICK_ARROW FLOAT DOT LIDENT EOF + Source: length Int = false <= false ? => 8.647 . length + Conflict origins: state 198 on QSTNMARK, state 503 on DOT + +19. Tokens (12): LIDENT UIDENT EQUAL FALSE LPAREN RPAREN QSTNMARK THICK_ARROW FLOAT DOT LIDENT EOF + Source: length Int = false ( ) ? => 8.647 . length + Conflict origins: state 135 on LPAREN, state 138 on QSTNMARK, state 503 on DOT + +20. Tokens (12): LIDENT UIDENT EQUAL FALSE MINUS FALSE QSTNMARK THICK_ARROW FLOAT DOT LIDENT EOF + Source: length Int = false - false ? => 8.647 . length + Conflict origins: state 178 on QSTNMARK, state 503 on DOT + +21. Tokens (12): LIDENT UIDENT EQUAL FALSE MORE FALSE QSTNMARK THICK_ARROW FLOAT DOT LIDENT EOF + Source: length Int = false > false ? => 8.647 . length + Conflict origins: state 190 on QSTNMARK, state 503 on DOT + +22. Tokens (12): LIDENT UIDENT EQUAL FALSE MOREEQ FALSE QSTNMARK THICK_ARROW FLOAT DOT LIDENT EOF + Source: length Int = false >= false ? => 8.647 . length + Conflict origins: state 174 on QSTNMARK, state 503 on DOT + +23. Tokens (12): LIDENT UIDENT EQUAL FALSE PLUS FALSE QSTNMARK THICK_ARROW FLOAT DOT LIDENT EOF + Source: length Int = false + false ? => 8.647 . length + Conflict origins: state 166 on QSTNMARK, state 503 on DOT + +24. Tokens (12): LIDENT UIDENT EQUAL FALSE SLASH FALSE QSTNMARK THICK_ARROW FLOAT DOT LIDENT EOF + Source: length Int = false / false ? => 8.647 . length + Conflict origins: state 158 on QSTNMARK, state 503 on DOT + +25. Tokens (12): LIDENT UIDENT EQUAL FALSE STAR FALSE QSTNMARK THICK_ARROW FLOAT DOT LIDENT EOF + Source: length Int = false * false ? => 8.647 . length + Conflict origins: state 148 on QSTNMARK, state 503 on DOT + +26. Tokens (12): LIDENT UIDENT EQUAL UIDENT LPAREN RPAREN QSTNMARK THICK_ARROW FLOAT DOT LIDENT EOF + Source: length Int = Int ( ) ? => 8.647 . length + Conflict origins: state 93 on LPAREN, state 132 on QSTNMARK, state 503 on DOT + +27. Tokens (13): LIDENT UIDENT LPAREN AND TILDE TILDE FALSE QSTNQSTN LIDENT DOLLAR LIDENT RPAREN EOF + Source: length Int ( & ~ ~ false ?? length $ length ) + Conflict origins: state 325 on LPAREN, state 496 on QSTNQSTN + +28. Tokens (13): UIDENT PLUS LPAREN RPAREN THICK_ARROW TILDE TILDE FALSE QSTNQSTN STRING DOT LIDENT EOF + Source: Int + ( ) => ~ ~ false ?? "john" . length + Conflict origins: state 496 on QSTNQSTN, state 503 on DOT + +29. Tokens (16): LIDENT UIDENT EQUAL TILDE FALSE QSTNMARK THICK_ARROW FALSE LESS AND AND AT LIDENT DOLLAR LIDENT EOF + Source: length Int = ~ false ? => false < & & @ length $ length + Conflict origins: state 496 on QSTNMARK, state 503 on LESS + +30. Tokens (16): LIDENT UIDENT EQUAL UIDENT LPAREN RPAREN THICK_ARROW FALSE LESS AND AND AT LIDENT DOLLAR LIDENT EOF + Source: length Int = Int ( ) => false < & & @ length $ length + Conflict origins: state 93 on LPAREN, state 503 on LESS + +31. Tokens (17): LIDENT UIDENT EQUAL FALSE EQEQ FALSE QSTNMARK THICK_ARROW LIDENT DOLLAR LIDENT LESSEQ AT LIDENT DOLLAR LIDENT EOF + Source: length Int = false == false ? => length $ length <= @ length $ length + Conflict origins: state 214 on QSTNMARK, state 503 on LESSEQ + +32. Tokens (17): LIDENT UIDENT EQUAL FALSE LESS FALSE QSTNMARK THICK_ARROW LIDENT DOLLAR LIDENT LESSEQ AT LIDENT DOLLAR LIDENT EOF + Source: length Int = false < false ? => length $ length <= @ length $ length + Conflict origins: state 206 on QSTNMARK, state 503 on LESSEQ + +33. Tokens (17): LIDENT UIDENT EQUAL FALSE LESSEQ FALSE QSTNMARK THICK_ARROW LIDENT DOLLAR LIDENT LESSEQ AT LIDENT DOLLAR LIDENT EOF + Source: length Int = false <= false ? => length $ length <= @ length $ length + Conflict origins: state 198 on QSTNMARK, state 503 on LESSEQ + +34. Tokens (17): LIDENT UIDENT EQUAL FALSE LPAREN RPAREN QSTNMARK THICK_ARROW LIDENT DOLLAR LIDENT LESSEQ AT LIDENT DOLLAR LIDENT EOF + Source: length Int = false ( ) ? => length $ length <= @ length $ length + Conflict origins: state 135 on LPAREN, state 138 on QSTNMARK, state 503 on LESSEQ + +35. Tokens (17): LIDENT UIDENT EQUAL FALSE MINUS FALSE QSTNMARK THICK_ARROW LIDENT DOLLAR LIDENT LESSEQ AT LIDENT DOLLAR LIDENT EOF + Source: length Int = false - false ? => length $ length <= @ length $ length + Conflict origins: state 178 on QSTNMARK, state 503 on LESSEQ + +36. Tokens (17): LIDENT UIDENT EQUAL FALSE MORE FALSE QSTNMARK THICK_ARROW LIDENT DOLLAR LIDENT LESSEQ AT LIDENT DOLLAR LIDENT EOF + Source: length Int = false > false ? => length $ length <= @ length $ length + Conflict origins: state 190 on QSTNMARK, state 503 on LESSEQ + +37. Tokens (17): LIDENT UIDENT EQUAL FALSE MOREEQ FALSE QSTNMARK THICK_ARROW LIDENT DOLLAR LIDENT LESSEQ AT LIDENT DOLLAR LIDENT EOF + Source: length Int = false >= false ? => length $ length <= @ length $ length + Conflict origins: state 174 on QSTNMARK, state 503 on LESSEQ + +38. Tokens (17): LIDENT UIDENT EQUAL FALSE PLUS FALSE QSTNMARK THICK_ARROW LIDENT DOLLAR LIDENT LESSEQ AT LIDENT DOLLAR LIDENT EOF + Source: length Int = false + false ? => length $ length <= @ length $ length + Conflict origins: state 166 on QSTNMARK, state 503 on LESSEQ + +39. Tokens (17): LIDENT UIDENT EQUAL FALSE SLASH FALSE QSTNMARK THICK_ARROW LIDENT DOLLAR LIDENT LESSEQ AT LIDENT DOLLAR LIDENT EOF + Source: length Int = false / false ? => length $ length <= @ length $ length + Conflict origins: state 158 on QSTNMARK, state 503 on LESSEQ + +40. Tokens (17): LIDENT UIDENT EQUAL FALSE STAR FALSE QSTNMARK THICK_ARROW LIDENT DOLLAR LIDENT LESSEQ AT LIDENT DOLLAR LIDENT EOF + Source: length Int = false * false ? => length $ length <= @ length $ length + Conflict origins: state 148 on QSTNMARK, state 503 on LESSEQ + +41. Tokens (17): LIDENT UIDENT EQUAL UIDENT LPAREN RPAREN QSTNMARK THICK_ARROW LIDENT DOLLAR LIDENT LESSEQ AT LIDENT DOLLAR LIDENT EOF + Source: length Int = Int ( ) ? => length $ length <= @ length $ length + Conflict origins: state 93 on LPAREN, state 132 on QSTNMARK, state 503 on LESSEQ + +42. Tokens (17): UIDENT LPAREN RPAREN THICK_ARROW TILDE FALSE QSTNMARK THICK_ARROW LIDENT DOLLAR LIDENT LESSEQ AT LIDENT DOLLAR LIDENT EOF + Source: Int ( ) => ~ false ? => length $ length <= @ length $ length + Conflict origins: state 496 on QSTNMARK, state 503 on LESSEQ + +43. Tokens (17): UIDENT LPAREN RPAREN THICK_ARROW UIDENT LPAREN RPAREN THICK_ARROW LIDENT DOLLAR LIDENT LESSEQ AT LIDENT DOLLAR LIDENT EOF + Source: Int ( ) => Int ( ) => length $ length <= @ length $ length + Conflict origins: state 93 on LPAREN, state 503 on LESSEQ + +44. Tokens (19): UIDENT LPAREN RPAREN THICK_ARROW FALSE EQEQ FALSE SLASH FLOAT DOT LIDENT QSTNQSTN AND AND AT LIDENT DOLLAR LIDENT EOF + Source: Int ( ) => false == false / 8.647 . length ?? & & @ length $ length + Conflict origins: state 158 on QSTNQSTN, state 214 on QSTNQSTN, state 503 on EQEQ + +45. Tokens (19): UIDENT LPAREN RPAREN THICK_ARROW FALSE EQEQ FALSE STAR FLOAT DOT LIDENT QSTNQSTN AND AND AT LIDENT DOLLAR LIDENT EOF + Source: Int ( ) => false == false * 8.647 . length ?? & & @ length $ length + Conflict origins: state 148 on QSTNQSTN, state 214 on QSTNQSTN, state 503 on EQEQ + +46. Tokens (19): UIDENT LPAREN RPAREN THICK_ARROW FALSE LESS FALSE SLASH FLOAT DOT LIDENT QSTNQSTN AND AND AT LIDENT DOLLAR LIDENT EOF + Source: Int ( ) => false < false / 8.647 . length ?? & & @ length $ length + Conflict origins: state 158 on QSTNQSTN, state 206 on QSTNQSTN, state 503 on LESS + +47. Tokens (19): UIDENT LPAREN RPAREN THICK_ARROW FALSE LESS FALSE STAR FLOAT DOT LIDENT QSTNQSTN AND AND AT LIDENT DOLLAR LIDENT EOF + Source: Int ( ) => false < false * 8.647 . length ?? & & @ length $ length + Conflict origins: state 148 on QSTNQSTN, state 206 on QSTNQSTN, state 503 on LESS + +48. Tokens (19): UIDENT LPAREN RPAREN THICK_ARROW FALSE LESSEQ FALSE SLASH FLOAT DOT LIDENT QSTNQSTN AND AND AT LIDENT DOLLAR LIDENT EOF + Source: Int ( ) => false <= false / 8.647 . length ?? & & @ length $ length + Conflict origins: state 158 on QSTNQSTN, state 198 on QSTNQSTN, state 503 on LESSEQ + +49. Tokens (19): UIDENT LPAREN RPAREN THICK_ARROW FALSE LESSEQ FALSE STAR FLOAT DOT LIDENT QSTNQSTN AND AND AT LIDENT DOLLAR LIDENT EOF + Source: Int ( ) => false <= false * 8.647 . length ?? & & @ length $ length + Conflict origins: state 148 on QSTNQSTN, state 198 on QSTNQSTN, state 503 on LESSEQ + +50. Tokens (19): UIDENT LPAREN RPAREN THICK_ARROW FALSE MINUS FALSE SLASH FLOAT DOT LIDENT QSTNQSTN AND AND AT LIDENT DOLLAR LIDENT EOF + Source: Int ( ) => false - false / 8.647 . length ?? & & @ length $ length + Conflict origins: state 158 on QSTNQSTN, state 178 on QSTNQSTN, state 503 on MINUS + +Explored 10029149 frontiers (10028786 unique); 65138 conflict seeds. +Search stopped because one or more workers reached a search limit. diff --git a/reports/deep.txt b/reports/deep.txt new file mode 100644 index 00000000..c1fca543 --- /dev/null +++ b/reports/deep.txt @@ -0,0 +1,46 @@ +Search profile: deep-function-body +Rotate deep search waves through statements in a function body. +Tokens 15..50 · timeout 30m · 50 witnesses · 10 nodes per depth +Prefix (5): UIDENT LIDENT LPAREN RPAREN LCURLY + +Memory budget: 6144 MiB total across 4 worker(s); workers compact at 1229 MiB and stop admitting frontiers at 1 +382 MiB each (10% reserved); per-worker limits are 710564 queued frontiers and 710564 retained dedup frontiers +(ratio 1). +Search constraints: 15..50 total tokens; 5-token prefix; 10 nodes per depth. +Prefix tokens: UIDENT LIDENT LPAREN RPAREN LCURLY +Found 7 complete ambiguity families. + +1. Tokens (15): UIDENT LIDENT LPAREN RPAREN LCURLY ABORT TRUE EQEQ AND TILDE FALSE QSTNQSTN FALSE RCURLY EOF + Source: Int length ( ) { abort true == & ~ false ?? false } + Conflict origins: state 214 on QSTNQSTN, state 496 on QSTNQSTN + +2. Tokens (15): UIDENT LIDENT LPAREN RPAREN LCURLY ABORT TRUE LESSEQ TILDE FALSE QSTNQSTN AND STRING RCURLY EOF + Source: Int length ( ) { abort true <= ~ false ?? & "john" } + Conflict origins: state 198 on QSTNQSTN, state 496 on QSTNQSTN + +3. Tokens (15): UIDENT LIDENT LPAREN RPAREN LCURLY RESOLVE TRUE MINUS TILDE FALSE QSTNQSTN AND TRUE RCURLY EOF + Source: Int length ( ) { resolve true - ~ false ?? & true } + Conflict origins: state 178 on QSTNQSTN, state 496 on QSTNQSTN + +4. Tokens (15): UIDENT LIDENT LPAREN RPAREN LCURLY RESOLVE UIDENT LPAREN RPAREN THICK_ARROW FLOAT SLASH STRING +RCURLY EOF + Source: Int length ( ) { resolve Int ( ) => 8.647 / "john" } + Conflict origins: state 93 on LPAREN, state 503 on SLASH + +5. Tokens (16): UIDENT LIDENT LPAREN RPAREN LCURLY RESOLVE TILDE TILDE TILDE AND TRUE QSTNQSTN AND INT RCURLY E +OF + Source: Int length ( ) { resolve ~ ~ ~ & true ?? & 43 } + Conflict origins: state 496 on QSTNQSTN + +6. Tokens (16): UIDENT LIDENT LPAREN RPAREN LCURLY RETURN TRUE LESS TILDE FALSE QSTNQSTN LIDENT MINUS FLOAT RCU +RLY EOF + Source: Int length ( ) { return true < ~ false ?? length - 8.647 } + Conflict origins: state 206 on QSTNQSTN, state 496 on QSTNQSTN + +7. Tokens (17): UIDENT LIDENT LPAREN RPAREN LCURLY ABORT UIDENT LBRACKET RBRACKET LPAREN RPAREN THICK_ARROW FAL +SE DOT LIDENT RCURLY EOF + Source: Int length ( ) { abort Int [ ] ( ) => false . length } + Conflict origins: state 503 on DOT + +Explored 27585724 frontiers (30246349 unique); 455669 conflict seeds. +Search stopped because one or more workers reached a search limit. diff --git a/reports/new-ambiguities-report.txt b/reports/new-ambiguities-report.txt new file mode 100644 index 00000000..bbcdff1b --- /dev/null +++ b/reports/new-ambiguities-report.txt @@ -0,0 +1,205 @@ +Memory budget: 6144 MiB total across 4 worker(s); workers compact at 1229 MiB and stop admitting frontiers at 1382 MiB each (10% reserved); per-worker limits are 710564 queued frontiers and 710564 retained dedup frontiers (ratio 1). +Found 50 complete ambiguity families. + +1. Tokens (9): LIDENT UIDENT EQUAL TILDE TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = ~ ~ false ?? false + Conflict origins: state 496 on QSTNQSTN + +2. Tokens (10): LIDENT UIDENT EQUAL FALSE EQEQ TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false == ~ false ?? false + Conflict origins: state 214 on QSTNQSTN, state 496 on QSTNQSTN + +3. Tokens (10): LIDENT UIDENT EQUAL FALSE LESS TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false < ~ false ?? false + Conflict origins: state 206 on QSTNQSTN, state 496 on QSTNQSTN + +4. Tokens (10): LIDENT UIDENT EQUAL FALSE LESSEQ TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false <= ~ false ?? false + Conflict origins: state 198 on QSTNQSTN, state 496 on QSTNQSTN + +5. Tokens (10): LIDENT UIDENT EQUAL FALSE MINUS TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false - ~ false ?? false + Conflict origins: state 178 on QSTNQSTN, state 496 on QSTNQSTN + +6. Tokens (10): LIDENT UIDENT EQUAL FALSE MORE TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false > ~ false ?? false + Conflict origins: state 190 on QSTNQSTN, state 496 on QSTNQSTN + +7. Tokens (10): LIDENT UIDENT EQUAL FALSE MOREEQ TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false >= ~ false ?? false + Conflict origins: state 174 on QSTNQSTN, state 496 on QSTNQSTN + +8. Tokens (10): LIDENT UIDENT EQUAL FALSE PLUS TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false + ~ false ?? false + Conflict origins: state 166 on QSTNQSTN, state 496 on QSTNQSTN + +9. Tokens (10): LIDENT UIDENT EQUAL FALSE SLASH TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false / ~ false ?? false + Conflict origins: state 158 on QSTNQSTN, state 496 on QSTNQSTN + +10. Tokens (10): LIDENT UIDENT EQUAL FALSE STAR TILDE FALSE QSTNQSTN FALSE EOF + Source: length Int = false * ~ false ?? false + Conflict origins: state 148 on QSTNQSTN, state 496 on QSTNQSTN + +11. Tokens (10): LIDENT UIDENT EQUAL TILDE FALSE LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = ~ false ( ) ?? false + Conflict origins: state 135 on LPAREN, state 138 on QSTNQSTN, state 496 on QSTNQSTN + +12. Tokens (10): LIDENT UIDENT EQUAL TILDE TILDE FALSE QSTNMARK LCURLY RCURLY EOF + Source: length Int = ~ ~ false ? { } + Conflict origins: state 496 on QSTNMARK + +13. Tokens (10): LIDENT UIDENT EQUAL TILDE UIDENT LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = ~ Int ( ) ?? false + Conflict origins: state 93 on LPAREN, state 132 on QSTNQSTN, state 496 on QSTNQSTN + +14. Tokens (10): LIDENT UIDENT LPAREN TILDE TILDE FALSE QSTNQSTN FALSE RPAREN EOF + Source: length Int ( ~ ~ false ?? false ) + Conflict origins: state 325 on LPAREN, state 496 on QSTNQSTN + +15. Tokens (11): LIDENT UIDENT EQUAL FALSE EQEQ FALSE LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = false == false ( ) ?? false + Conflict origins: state 135 on LPAREN, state 138 on QSTNQSTN, state 214 on QSTNQSTN + +16. Tokens (11): LIDENT UIDENT EQUAL FALSE EQEQ FALSE MINUS FALSE QSTNQSTN FALSE EOF + Source: length Int = false == false - false ?? false + Conflict origins: state 178 on QSTNQSTN, state 214 on QSTNQSTN + +17. Tokens (11): LIDENT UIDENT EQUAL FALSE EQEQ FALSE PLUS FALSE QSTNQSTN FALSE EOF + Source: length Int = false == false + false ?? false + Conflict origins: state 166 on QSTNQSTN, state 214 on QSTNQSTN + +18. Tokens (11): LIDENT UIDENT EQUAL FALSE EQEQ FALSE SLASH FALSE QSTNQSTN FALSE EOF + Source: length Int = false == false / false ?? false + Conflict origins: state 158 on QSTNQSTN, state 214 on QSTNQSTN + +19. Tokens (11): LIDENT UIDENT EQUAL FALSE EQEQ FALSE STAR FALSE QSTNQSTN FALSE EOF + Source: length Int = false == false * false ?? false + Conflict origins: state 148 on QSTNQSTN, state 214 on QSTNQSTN + +20. Tokens (11): LIDENT UIDENT EQUAL FALSE EQEQ TILDE FALSE QSTNMARK LCURLY RCURLY EOF + Source: length Int = false == ~ false ? { } + Conflict origins: state 214 on QSTNMARK, state 496 on QSTNMARK + +21. Tokens (11): LIDENT UIDENT EQUAL FALSE EQEQ UIDENT LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = false == Int ( ) ?? false + Conflict origins: state 93 on LPAREN, state 132 on QSTNQSTN, state 214 on QSTNQSTN + +22. Tokens (11): LIDENT UIDENT EQUAL FALSE LESS FALSE LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = false < false ( ) ?? false + Conflict origins: state 135 on LPAREN, state 138 on QSTNQSTN, state 206 on QSTNQSTN + +23. Tokens (11): LIDENT UIDENT EQUAL FALSE LESS FALSE MINUS FALSE QSTNQSTN FALSE EOF + Source: length Int = false < false - false ?? false + Conflict origins: state 178 on QSTNQSTN, state 206 on QSTNQSTN + +24. Tokens (11): LIDENT UIDENT EQUAL FALSE LESS FALSE PLUS FALSE QSTNQSTN FALSE EOF + Source: length Int = false < false + false ?? false + Conflict origins: state 166 on QSTNQSTN, state 206 on QSTNQSTN + +25. Tokens (11): LIDENT UIDENT EQUAL FALSE LESS FALSE SLASH FALSE QSTNQSTN FALSE EOF + Source: length Int = false < false / false ?? false + Conflict origins: state 158 on QSTNQSTN, state 206 on QSTNQSTN + +26. Tokens (11): LIDENT UIDENT EQUAL FALSE LESS FALSE STAR FALSE QSTNQSTN FALSE EOF + Source: length Int = false < false * false ?? false + Conflict origins: state 148 on QSTNQSTN, state 206 on QSTNQSTN + +27. Tokens (11): LIDENT UIDENT EQUAL FALSE LESS TILDE FALSE QSTNMARK LCURLY RCURLY EOF + Source: length Int = false < ~ false ? { } + Conflict origins: state 206 on QSTNMARK, state 496 on QSTNMARK + +28. Tokens (11): LIDENT UIDENT EQUAL FALSE LESS UIDENT LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = false < Int ( ) ?? false + Conflict origins: state 93 on LPAREN, state 132 on QSTNQSTN, state 206 on QSTNQSTN + +29. Tokens (11): LIDENT UIDENT EQUAL FALSE LESSEQ FALSE LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = false <= false ( ) ?? false + Conflict origins: state 135 on LPAREN, state 138 on QSTNQSTN, state 198 on QSTNQSTN + +30. Tokens (11): LIDENT UIDENT EQUAL FALSE LESSEQ FALSE MINUS FALSE QSTNQSTN FALSE EOF + Source: length Int = false <= false - false ?? false + Conflict origins: state 178 on QSTNQSTN, state 198 on QSTNQSTN + +31. Tokens (11): LIDENT UIDENT EQUAL FALSE LESSEQ FALSE PLUS FALSE QSTNQSTN FALSE EOF + Source: length Int = false <= false + false ?? false + Conflict origins: state 166 on QSTNQSTN, state 198 on QSTNQSTN + +32. Tokens (11): LIDENT UIDENT EQUAL FALSE LESSEQ FALSE SLASH FALSE QSTNQSTN FALSE EOF + Source: length Int = false <= false / false ?? false + Conflict origins: state 158 on QSTNQSTN, state 198 on QSTNQSTN + +33. Tokens (11): LIDENT UIDENT EQUAL FALSE LESSEQ FALSE STAR FALSE QSTNQSTN FALSE EOF + Source: length Int = false <= false * false ?? false + Conflict origins: state 148 on QSTNQSTN, state 198 on QSTNQSTN + +34. Tokens (11): LIDENT UIDENT EQUAL FALSE LESSEQ TILDE FALSE QSTNMARK LCURLY RCURLY EOF + Source: length Int = false <= ~ false ? { } + Conflict origins: state 198 on QSTNMARK, state 496 on QSTNMARK + +35. Tokens (11): LIDENT UIDENT EQUAL FALSE LESSEQ UIDENT LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = false <= Int ( ) ?? false + Conflict origins: state 93 on LPAREN, state 132 on QSTNQSTN, state 198 on QSTNQSTN + +36. Tokens (11): LIDENT UIDENT EQUAL FALSE LPAREN RPAREN QSTNQSTN FALSE LPAREN RPAREN EOF + Source: length Int = false ( ) ?? false ( ) + Conflict origins: state 135 on LPAREN, state 138 on QSTNQSTN + +37. Tokens (11): LIDENT UIDENT EQUAL FALSE MINUS FALSE LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = false - false ( ) ?? false + Conflict origins: state 135 on LPAREN, state 138 on QSTNQSTN, state 178 on QSTNQSTN + +38. Tokens (11): LIDENT UIDENT EQUAL FALSE MINUS FALSE SLASH FALSE QSTNQSTN FALSE EOF + Source: length Int = false - false / false ?? false + Conflict origins: state 158 on QSTNQSTN, state 178 on QSTNQSTN + +39. Tokens (11): LIDENT UIDENT EQUAL FALSE MINUS FALSE STAR FALSE QSTNQSTN FALSE EOF + Source: length Int = false - false * false ?? false + Conflict origins: state 148 on QSTNQSTN, state 178 on QSTNQSTN + +40. Tokens (11): LIDENT UIDENT EQUAL FALSE MINUS TILDE FALSE QSTNMARK LCURLY RCURLY EOF + Source: length Int = false - ~ false ? { } + Conflict origins: state 178 on QSTNMARK, state 496 on QSTNMARK + +41. Tokens (11): LIDENT UIDENT EQUAL FALSE MINUS UIDENT LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = false - Int ( ) ?? false + Conflict origins: state 93 on LPAREN, state 132 on QSTNQSTN, state 178 on QSTNQSTN + +42. Tokens (11): LIDENT UIDENT EQUAL FALSE MORE FALSE LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = false > false ( ) ?? false + Conflict origins: state 135 on LPAREN, state 138 on QSTNQSTN, state 190 on QSTNQSTN + +43. Tokens (11): LIDENT UIDENT EQUAL FALSE MORE FALSE MINUS FALSE QSTNQSTN FALSE EOF + Source: length Int = false > false - false ?? false + Conflict origins: state 178 on QSTNQSTN, state 190 on QSTNQSTN + +44. Tokens (11): LIDENT UIDENT EQUAL FALSE MORE FALSE PLUS FALSE QSTNQSTN FALSE EOF + Source: length Int = false > false + false ?? false + Conflict origins: state 166 on QSTNQSTN, state 190 on QSTNQSTN + +45. Tokens (11): LIDENT UIDENT EQUAL FALSE MORE FALSE SLASH FALSE QSTNQSTN FALSE EOF + Source: length Int = false > false / false ?? false + Conflict origins: state 158 on QSTNQSTN, state 190 on QSTNQSTN + +46. Tokens (11): LIDENT UIDENT EQUAL FALSE MORE FALSE STAR FALSE QSTNQSTN FALSE EOF + Source: length Int = false > false * false ?? false + Conflict origins: state 148 on QSTNQSTN, state 190 on QSTNQSTN + +47. Tokens (11): LIDENT UIDENT EQUAL FALSE MORE TILDE FALSE QSTNMARK LCURLY RCURLY EOF + Source: length Int = false > ~ false ? { } + Conflict origins: state 190 on QSTNMARK, state 496 on QSTNMARK + +48. Tokens (11): LIDENT UIDENT EQUAL FALSE MORE UIDENT LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = false > Int ( ) ?? false + Conflict origins: state 93 on LPAREN, state 132 on QSTNQSTN, state 190 on QSTNQSTN + +49. Tokens (11): LIDENT UIDENT EQUAL FALSE MOREEQ FALSE LPAREN RPAREN QSTNQSTN FALSE EOF + Source: length Int = false >= false ( ) ?? false + Conflict origins: state 135 on LPAREN, state 138 on QSTNQSTN, state 174 on QSTNQSTN + +50. Tokens (11): LIDENT UIDENT EQUAL FALSE MOREEQ FALSE MINUS FALSE QSTNQSTN FALSE EOF + Source: length Int = false >= false - false ?? false + Conflict origins: state 174 on QSTNQSTN, state 178 on QSTNQSTN + +Explored 5970121 frontiers (8805696 unique); 572350 conflict seeds. +Search stopped because one or more workers reached a search limit. diff --git a/test-parser/main.zn b/test-parser/main.zn new file mode 100644 index 00000000..6fc37084 --- /dev/null +++ b/test-parser/main.zn @@ -0,0 +1,32 @@ +type Vector2 = struct { + x T; + y T; +} + +Vector2 +(left Vector2, right Vector2) { + return Vector2(left.x + right.x, left.y + right.y) +} + +Void?String main (this Int, a Bool) mut { + x Void[this Int] = "hello" + y String = "world" + content &String = read("wordlist.txt") ?? "" + x Int(3) + + if random() { + results List = crawl(content) ? error { + std$print(error) + abort error + } + } + else { + rifle Weapon("rifle", 20) + value Bool = ~true + 3 + } + + loop i to 3 { + std$print(i) + } +} + +square Int (x Int, y Int) => x * y diff --git a/test-parser/nextStep.zn b/test-parser/nextStep.zn new file mode 100644 index 00000000..7afda325 --- /dev/null +++ b/test-parser/nextStep.zn @@ -0,0 +1,7 @@ +Void main() { + print("hello") + Std$print(1 + 1 + 3 + 4) + @Intrinsics$print("hello") + print(res.value) + print(res:getValue()) +} diff --git a/todo.md b/todo.md new file mode 100644 index 00000000..68dbd128 --- /dev/null +++ b/todo.md @@ -0,0 +1,6 @@ +x ConstructorDecl +x OpDecl +MethodCall + +x rename callables to verb +x refs diff --git a/tools/SYNTAX_EXPERIMENT.md b/tools/SYNTAX_EXPERIMENT.md new file mode 100644 index 00000000..12b5e240 --- /dev/null +++ b/tools/SYNTAX_EXPERIMENT.md @@ -0,0 +1,122 @@ +# Syntax experiments + +`syntax_experiment.py` compares small, explicit changes to Zane's concrete +syntax. It generates a temporary Menhir grammar for every selected variant, +replays the known complete-ambiguity witnesses, runs the bounded ambiguity +search, and produces Markdown and JSON reports. + +This is deliberately not a grammar-rewriting model. Every mutation is named, +reviewable, composable, and assigned an approximate edit cost. + +## Quick start + +```sh +syntax-experiment --list +syntax-experiment +syntax-experiment --variant semicolon-separated +``` + +Reports are written to: + +```text +_build/syntax-experiment/report.md +_build/syntax-experiment/report.json +``` + +Generated grammars normally live only for the duration of a run. To inspect +them directly: + +```sh +python3 tools/syntax_experiment.py --emit-only \ + --emit-dir _build/syntax-experiment/grammars +``` + +## Included ideas + +The predefined candidates cover: + +- semicolons required between adjacent statements; +- semicolons required after every statement; +- only statically named functions, methods, and constructors as call statements; +- significant newline separators; +- `[]`, `{}`, or `group()` expression grouping; +- removing general grouping; +- `[]`, `@()`, or `.()` for all calls; +- ordinary calls for names with `@()` or `[]` reserved for computed callees; +- abort handlers anchored to delimited boundaries (statement calls, + declaration values, `return`/`resolve`/`abort` values, call arguments, and + grouping) instead of every expression production; +- combinations of the most promising statement, grouping, call, and + abort-handler changes. + +The named-statement experiment still permits computed calls inside expressions. +It only prevents a computed call such as `(x)()` from independently reducing to +a statement. Named method calls remain valid statements even when their receiver +is computed. + +The `semicolon-separated` experiment requires a semicolon only where another +statement follows and permits one trailing semicolon. The stricter +`semicolon-terminated` experiment requires a terminator after every declaration +and statement, including the final item in a block. + +## Measurements + +Known witnesses are **respelled in each variant's own syntax** before being +replayed: every transformation that changes surface spelling registers a +matching update in `SPELLINGS`, and each known case builds its intended +program from the resulting spelling profile. A rejection therefore means the +variant genuinely cannot parse the intended program — never that an old +spelling merely became illegal. A case a variant cannot express at all (for +example a computed-call statement under `named-statement-calls`) is labeled +"not expressible" and counted as rejected. + +Alongside the ambiguity witnesses, the case set includes one plain +compatibility probe: an ordinary named call statement, `print("hello")`, +which every variant is expected to parse exactly once (spelled +`print["hello"]` under `bracket-calls`, and so on). + +For each candidate, the report records: + +- whether each known witness has zero, one, or two accepting derivations; +- complete ambiguity profiles found within the configured bounds; +- explored and unique frontiers; +- conflict seeds; +- search limits and elapsed time; +- up to five shortest concrete witnesses; +- an approximate syntax edit cost. + +A candidate that rejects a known program is penalized separately from one that +still parses it ambiguously. A zero-witness result that stopped at a timeout or +frontier limit is marked uncertain rather than ambiguity-free. + +The Pareto marker means that no other tested candidate was at least as good on +every measured axis. It is not an automatic language-design decision: readability, +orthogonality, and whether a changed spelling expresses Zane's intent still need +human judgment. + +## Adding an experiment + +1. Add a transformation function that uses `replace_once` or another checked + structural edit. A changed grammar anchor must fail loudly rather than silently + producing the baseline grammar. +2. Register it in `TRANSFORMS`, and register a matching spelling update in + `SPELLINGS` (the identity update for transforms that do not change how the + known witnesses are spelled). The two tables must cover the same names. +3. Add one or more `Variant` entries, including useful combinations and an edit + cost. +4. Add focused assertions to `test_syntax_experiment.py`. +5. Run `just syntax-experiment-test`, then a short single-variant search before + comparing the full matrix. + +## Limitations + +The ambiguity search is bounded because ambiguity of arbitrary context-free +grammars is undecidable. Results compare evidence found under equal bounds; they +do not prove global unambiguity. + +These variants mutate the parser grammar only. A candidate that introduces a +token such as `NEWLINE` or `group` would also require a corresponding lexer +change before it could become real language syntax. Likewise, compatibility is +currently measured with token witnesses rather than a large source-to-AST corpus. +Adding an AST golden corpus is the natural next step once enough representative +Zane programs exist. diff --git a/tools/ambiguity.py b/tools/ambiguity.py new file mode 100644 index 00000000..1dbde53f --- /dev/null +++ b/tools/ambiguity.py @@ -0,0 +1,692 @@ +#!/usr/bin/env python3 +"""Friendly command-line interface for the ambiguity-search engine.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass, replace +import datetime +from pathlib import Path +import re +import shlex +import string +import subprocess +import sys +import tomllib +from typing import Any, Callable, Sequence, TextIO + + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_PROFILES = ROOT / "ambiguity-searches.toml" +ENGINE_RUNNER = ROOT / "dev" / "bin" / "ambiguity" + +DURATION_PART = re.compile(r"(\d+(?:\.\d+)?)(ms|s|m|h)") +DURATION_FACTORS = {"ms": 0.001, "s": 1.0, "m": 60.0, "h": 3600.0} + + +class ConfigurationError(ValueError): + pass + + +@dataclass(frozen=True) +class SearchProfile: + name: str + description: str + min_tokens: int + max_tokens: int + timeout_seconds: float + witnesses: int + prefix_tokens: tuple[str, ...] = () + nodes_per_depth: int | None = None + output: Path | None = None + + +def parse_token_range(value: str) -> tuple[int, int]: + match = re.fullmatch(r"\s*(\d+)\s*\.\.\s*(\d+)\s*", value) + if match is None: + raise ConfigurationError( + f"invalid token range {value!r}; expected MIN..MAX" + ) + minimum, maximum = map(int, match.groups()) + if minimum > maximum: + raise ConfigurationError("token range minimum must not exceed its maximum") + return minimum, maximum + + +def parse_duration(value: str | int | float) -> float: + if isinstance(value, bool): + raise ConfigurationError("duration must be a number or a value such as 30m") + if isinstance(value, (int, float)): + seconds = float(value) + else: + text = value.strip().lower() + if re.fullmatch(r"\d+(?:\.\d+)?", text): + seconds = float(text) + else: + seconds = 0.0 + position = 0 + for match in DURATION_PART.finditer(text): + if match.start() != position: + break + seconds += float(match.group(1)) * DURATION_FACTORS[match.group(2)] + position = match.end() + if position != len(text) or position == 0: + raise ConfigurationError( + f"invalid duration {value!r}; use values such as 90s, 30m, or 1h" + ) + if seconds < 0: + raise ConfigurationError("duration must be non-negative") + return seconds + + +def format_duration(seconds: float) -> str: + if seconds == 0: + return "0s" + if seconds % 3600 == 0: + return f"{seconds / 3600:g}h" + if seconds % 60 == 0: + return f"{seconds / 60:g}m" + return f"{seconds:g}s" + + +# ----- Search-parameter registry ----- +# +# Every search parameter is declared once here and shared by both surfaces: +# the TOML profile key and the `--key` override flag use the identical +# kebab-case spelling. The registry drives profile validation, argparse flag +# registration, override application, and engine-argument construction, so a +# new parameter is added in exactly one place instead of five. + + +@dataclass(frozen=True) +class Setting: + # One entry per command-line flag. `kind` records how it behaves and + # `profile_key` whether it is also a valid TOML key, so a single registry + # enumerates every flag and marks which ones profiles may set - there is no + # separate cli-only list. + # "value" - takes an argument; loadable from a profile and, unless it is + # purely a wrapper concern, forwarded to the engine. + # "toggle" - a boolean switch (--flag / `flag = true`) that rewrites a + # field rather than carrying its own value. + # "mode" - a boolean switch that changes what the CLI does, never a + # profile setting. + key: str + kind: str + help: str + profile_key: bool = True + metavar: str | None = None + arg_type: Callable[[str], Any] | None = None + required: bool = False + # (raw value, profile name) -> SearchProfile field updates; raises + # ConfigurationError on an invalid value. Accepts both the TOML-typed value + # and the argparse-typed override so a single coercion serves both surfaces. + coerce: Callable[[Any, str], dict[str, Any]] | None = None + to_engine_args: Callable[["SearchProfile"], list[str]] | None = None + + @property + def dest(self) -> str: + return self.key.replace("-", "_") + + +def _coerce_tokens(raw: Any, name: str) -> dict[str, Any]: + if not isinstance(raw, str): + raise ConfigurationError(f"profile {name!r}: tokens must be MIN..MAX") + minimum, maximum = parse_token_range(raw) + return {"min_tokens": minimum, "max_tokens": maximum} + + +def _coerce_timeout(raw: Any, name: str) -> dict[str, Any]: + return {"timeout_seconds": parse_duration(raw)} + + +def _coerce_witnesses(raw: Any, name: str) -> dict[str, Any]: + if isinstance(raw, bool) or not isinstance(raw, int) or raw < 1: + raise ConfigurationError( + f"profile {name!r}: witnesses must be an integer of at least 1" + ) + return {"witnesses": raw} + + +def _coerce_prefix_tokens(raw: Any, name: str) -> dict[str, Any]: + # A profile supplies an array; the CLI override supplies a space-separated + # string. Accept either so both surfaces share one code path. + tokens = raw.split() if isinstance(raw, str) else raw + if not isinstance(tokens, list) or not all( + isinstance(token, str) and token for token in tokens + ): + raise ConfigurationError( + f"profile {name!r}: prefix-tokens must be an array of token names" + ) + return {"prefix_tokens": tuple(tokens)} + + +def _coerce_nodes_per_depth(raw: Any, name: str) -> dict[str, Any]: + if isinstance(raw, bool) or not isinstance(raw, int) or raw < 1: + raise ConfigurationError( + f"profile {name!r}: nodes-per-depth must be an integer of at least 1" + ) + return {"nodes_per_depth": raw} + + +def _coerce_output(raw: Any, name: str) -> dict[str, Any]: + # A profile supplies a string; the --output override supplies a Path. Guard + # like every other coercer so a malformed value (e.g. output = 5) becomes a + # ConfigurationError routed through main() instead of an uncaught TypeError. + if not isinstance(raw, (str, Path)): + raise ConfigurationError(f"profile {name!r}: output must be a string path") + return {"output": Path(raw)} + + +def _coerce_breadth_first(raw: Any, name: str) -> dict[str, Any]: + # Scheduling is a single slot with two spellings: `nodes-per-depth = N` or + # `breadth-first = true`. This toggle rewrites the nodes_per_depth field to + # None; the two keys are mutually exclusive within a table (enforced in + # _profile_tables) and a child overrides whichever the parent chose. + if raw is not True: + raise ConfigurationError( + f"profile {name!r}: breadth-first must be true " + f"(omit it to use nodes-per-depth)" + ) + return {"nodes_per_depth": None} + + +def _tokens_engine_args(profile: SearchProfile) -> list[str]: + return [ + "--max-tokens", + str(profile.max_tokens), + "--min-tokens", + str(profile.min_tokens), + ] + + +def _timeout_engine_args(profile: SearchProfile) -> list[str]: + return ["--timeout", f"{profile.timeout_seconds:g}"] + + +def _witnesses_engine_args(profile: SearchProfile) -> list[str]: + return ["--max-witnesses", str(profile.witnesses)] + + +def _prefix_engine_args(profile: SearchProfile) -> list[str]: + if not profile.prefix_tokens: + return [] + return ["--prefix-tokens", " ".join(profile.prefix_tokens)] + + +def _nodes_engine_args(profile: SearchProfile) -> list[str]: + if profile.nodes_per_depth is None: + return [] + return ["--nodes-per-depth", str(profile.nodes_per_depth)] + + +def _no_engine_args(profile: SearchProfile) -> list[str]: + # The report path is handled by this wrapper, not passed to the engine. + return [] + + +SETTINGS: tuple[Setting, ...] = ( + Setting( + "tokens", + kind="value", + help="override the profile's complete-witness token range", + metavar="MIN..MAX", + required=True, + coerce=_coerce_tokens, + to_engine_args=_tokens_engine_args, + ), + Setting( + "timeout", + kind="value", + help="override the timeout; accepts values such as 90s, 30m, and 1h", + metavar="DURATION", + required=True, + coerce=_coerce_timeout, + to_engine_args=_timeout_engine_args, + ), + Setting( + "witnesses", + kind="value", + help="override the number of ambiguity families to report", + metavar="N", + arg_type=int, + required=True, + coerce=_coerce_witnesses, + to_engine_args=_witnesses_engine_args, + ), + Setting( + "prefix-tokens", + kind="value", + help="override the fixed space-separated token prefix", + metavar="TOKENS", + coerce=_coerce_prefix_tokens, + to_engine_args=_prefix_engine_args, + ), + Setting( + "nodes-per-depth", + kind="value", + help="override how many sibling frontiers a depth wave expands", + metavar="N", + arg_type=int, + coerce=_coerce_nodes_per_depth, + to_engine_args=_nodes_engine_args, + ), + Setting( + "breadth-first", + kind="toggle", + help="use shortest-first scheduling (the breadth-first spelling of the " + "nodes-per-depth slot; a profile may set at most one of the two)", + coerce=_coerce_breadth_first, + ), + Setting( + "output", + kind="value", + help="write the complete report to FILE while also displaying it; " + "{profile}, {date}, {time}, and {datetime} expand in the path " + "(e.g. reports/{profile}-{date}.txt), and missing directories are created", + metavar="FILE", + arg_type=Path, + coerce=_coerce_output, + to_engine_args=_no_engine_args, + ), + Setting( + "dry-run", + kind="mode", + help="show the resolved profile without running the engine", + profile_key=False, + ), +) + +# Scheduling is one concept with two spellings; a profile table may set at most +# one of these keys, and a child profile's choice replaces the parent's. +SCHEDULING_KEYS = ("nodes-per-depth", "breadth-first") + +# The TOML profile keys are exactly the profile-key settings plus the two +# structural keys, so the flags and the profile keys can never drift apart. +PROFILE_KEYS = {"description", "extends"} | { + setting.key for setting in SETTINGS if setting.profile_key +} + + +def _validate_profile(profile: SearchProfile, name: str) -> None: + # max_tokens counts the prefix and the required EOF, so the prefix must + # leave room for at least the EOF token: a prefix of exactly max_tokens can + # never complete a witness. + if len(profile.prefix_tokens) >= profile.max_tokens: + raise ConfigurationError( + f"profile {name!r}: prefix fills the whole token budget, " + f"leaving no room for the EOF token" + ) + + +def expand_output_path(pattern: Path, profile_name: str) -> Path: + """Substitute report-naming placeholders in an --output pattern. + + Placeholders use brace syntax: ``{name}``, with ``{{`` and ``}}`` for + literal braces. The supported names are ``profile`` and three timestamp + forms whose date and time layout matches the existing ``reports/`` + filenames (e.g. ``2026-07-23_21-38-17``).""" + now = datetime.datetime.now() + fields = { + "profile": profile_name, + "date": now.strftime("%Y-%m-%d"), + "time": now.strftime("%H-%M-%S"), + "datetime": now.strftime("%Y-%m-%d_%H-%M-%S"), + } + available = ", ".join("{" + name + "}" for name in fields) + text = str(pattern) + # Expand only bare `{name}` placeholders. Parsing the pattern ourselves — + # rather than str.format_map — keeps indexed or attribute forms such as + # {profile[0]} or {profile.foo} out: format_map would silently expand the + # former to a wrong path and raise an uncaught AttributeError on the latter. + try: + parsed = list(string.Formatter().parse(text)) + except ValueError as error: + raise ConfigurationError( + f"invalid --output pattern {text!r}: {error}; " + "write {{ and }} for literal braces" + ) from error + result: list[str] = [] + for literal, field, spec, conversion in parsed: + result.append(literal) + if field is None: + continue + if field not in fields: + raise ConfigurationError( + f"unknown placeholder {{{field}}} in --output pattern " + f"{text!r}; available placeholders are {available}" + ) + if spec or conversion: + raise ConfigurationError( + f"--output placeholder {{{field}}} takes no format spec or " + f"conversion in pattern {text!r}" + ) + result.append(fields[field]) + return Path("".join(result)) + + +def _profile_tables(path: Path) -> dict[str, dict[str, Any]]: + try: + with path.open("rb") as source: + document = tomllib.load(source) + except OSError as error: + raise ConfigurationError(f"cannot read profiles from {path}: {error}") from error + except tomllib.TOMLDecodeError as error: + raise ConfigurationError(f"invalid TOML in {path}: {error}") from error + tables = document.get("profiles") + if not isinstance(tables, dict) or not tables: + raise ConfigurationError(f"{path} must contain at least one [profiles.NAME]") + result: dict[str, dict[str, Any]] = {} + for name, table in tables.items(): + if not isinstance(table, dict): + raise ConfigurationError(f"profile {name!r} must be a TOML table") + unknown = set(table) - PROFILE_KEYS + if unknown: + keys = ", ".join(sorted(unknown)) + raise ConfigurationError(f"profile {name!r} has unknown settings: {keys}") + if len([key for key in SCHEDULING_KEYS if key in table]) > 1: + raise ConfigurationError( + f"profile {name!r}: set at most one of {' and '.join(SCHEDULING_KEYS)}" + ) + result[name] = table + return result + + +def load_profiles(path: Path = DEFAULT_PROFILES) -> dict[str, SearchProfile]: + tables = _profile_tables(path) + resolved: dict[str, dict[str, Any]] = {} + resolving: list[str] = [] + + def resolve(name: str) -> dict[str, Any]: + if name in resolved: + return resolved[name] + if name not in tables: + raise ConfigurationError(f"unknown inherited profile {name!r}") + if name in resolving: + cycle = " -> ".join([*resolving, name]) + raise ConfigurationError(f"profile inheritance cycle: {cycle}") + resolving.append(name) + table = tables[name] + parent = table.get("extends") + if parent is not None and not isinstance(parent, str): + raise ConfigurationError(f"profile {name!r}: extends must be a string") + merged = dict(resolve(parent)) if parent else {} + child_items = {key: value for key, value in table.items() if key != "extends"} + # Scheduling is one slot: if the child chooses either spelling, drop + # whichever the parent set so the child's choice wins in both directions. + if any(key in child_items for key in SCHEDULING_KEYS): + for key in SCHEDULING_KEYS: + merged.pop(key, None) + merged.update(child_items) + resolving.pop() + resolved[name] = merged + return merged + + profiles: dict[str, SearchProfile] = {} + for name in tables: + settings = resolve(name) + missing = {setting.key for setting in SETTINGS if setting.required} - set( + settings + ) + if missing: + keys = ", ".join(sorted(missing)) + raise ConfigurationError(f"profile {name!r} is missing: {keys}") + description = settings.get("description", "") + if not isinstance(description, str): + raise ConfigurationError(f"profile {name!r}: description must be a string") + fields: dict[str, Any] = {"name": name, "description": description} + for setting in SETTINGS: + if setting.coerce is not None and setting.key in settings: + fields.update(setting.coerce(settings[setting.key], name)) + profile = SearchProfile(**fields) + _validate_profile(profile, name) + profiles[name] = profile + return profiles + + +def apply_overrides( + profile: SearchProfile, arguments: argparse.Namespace +) -> SearchProfile: + fields: dict[str, Any] = {} + for setting in SETTINGS: + if setting.kind != "value": + continue + value = getattr(arguments, setting.dest, None) + if value is not None: + fields.update(setting.coerce(value, profile.name)) + result = replace(profile, **fields) if fields else profile + # The scheduling slot again: --breadth-first rewrites nodes_per_depth to + # None, and the two cannot be given together on one command line. + if getattr(arguments, "breadth_first", False): + if getattr(arguments, "nodes_per_depth", None) is not None: + raise ConfigurationError( + "--breadth-first and --nodes-per-depth cannot be combined" + ) + result = replace(result, **_coerce_breadth_first(True, result.name)) + _validate_profile(result, result.name) + return result + + +def engine_arguments(profile: SearchProfile, prove: int | None = None) -> list[str]: + arguments: list[str] = [] + for setting in SETTINGS: + if setting.to_engine_args is not None: + arguments.extend(setting.to_engine_args(profile)) + if prove is not None: + arguments.extend(["--prove", str(prove)]) + return arguments + + +def profile_summary(profile: SearchProfile, action: str) -> str: + scheduling = ( + "breadth-first" + if profile.nodes_per_depth is None + else f"{profile.nodes_per_depth} node" + f"{'' if profile.nodes_per_depth == 1 else 's'} per depth" + ) + lines = [ + f"{action} profile: {profile.name}", + ( + f"Tokens {profile.min_tokens}..{profile.max_tokens} · " + f"timeout {format_duration(profile.timeout_seconds)} · " + f"{profile.witnesses} witnesses · {scheduling}" + ), + ] + if profile.description: + lines.insert(1, profile.description) + if profile.prefix_tokens: + lines.append( + f"Prefix ({len(profile.prefix_tokens)}): " + + " ".join(profile.prefix_tokens) + ) + return "\n".join(lines) + + +def add_overrides(command: argparse.ArgumentParser) -> None: + # Every flag - value settings, toggles, and modes alike - is registered from + # the one registry, so there is no separate cli-only flag list to maintain. + for setting in SETTINGS: + if setting.kind == "value": + options: dict[str, Any] = { + "metavar": setting.metavar, + "help": setting.help, + } + if setting.arg_type is not None: + options["type"] = setting.arg_type + command.add_argument(f"--{setting.key}", **options) + else: + command.add_argument( + f"--{setting.key}", action="store_true", help=setting.help + ) + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser( + prog="ambiguity", + description="Search for, check, and prove grammar ambiguity.", + ) + result.add_argument( + "--profiles-file", + type=Path, + default=DEFAULT_PROFILES, + help=argparse.SUPPRESS, + ) + commands = result.add_subparsers(dest="command", required=True) + + search = commands.add_parser( + "search", help="run a bounded search using a named profile" + ) + search.add_argument( + "profile", + nargs="?", + default="general", + help="profile from ambiguity-searches.toml (default: general)", + ) + add_overrides(search) + + check = commands.add_parser( + "check", help="check one exact space-separated token sequence" + ) + check.add_argument( + "tokens", + nargs="+", + metavar="TOKEN", + help="terminal names, including EOF when required", + ) + + prove = commands.add_parser( + "prove", help="attempt an unbounded top-K proof, then concretize if needed" + ) + prove.add_argument("level", type=int, metavar="LEVEL") + prove.add_argument( + "profile", + nargs="?", + default="quick", + help="profile for bounded concretization (default: quick)", + ) + add_overrides(prove) + + commands.add_parser("profiles", help="list available search profiles") + commands.add_parser( + "classes", + help="list the terminal equivalence classes the search collapses", + ) + return result + + +def _write_prelude(prelude: str, output: TextIO | None) -> None: + print(prelude) + print() + sys.stdout.flush() + if output is not None: + output.write(prelude + "\n\n") + output.flush() + + +def run_engine( + arguments: Sequence[str], prelude: str, output_path: Path | None +) -> int: + output: TextIO | None = None + try: + if output_path is not None: + output_path.parent.mkdir(parents=True, exist_ok=True) + output = output_path.open("w", encoding="utf-8") + _write_prelude(prelude, output) + process = subprocess.Popen( + [str(ENGINE_RUNNER), "__engine", *arguments], + stdout=subprocess.PIPE, + text=True, + ) + assert process.stdout is not None + for line in process.stdout: + sys.stdout.write(line) + sys.stdout.flush() + if output is not None: + output.write(line) + output.flush() + return process.wait() + except OSError as error: + raise ConfigurationError(f"cannot run ambiguity engine: {error}") from error + finally: + if output is not None: + output.close() + + +def list_profiles(profiles: dict[str, SearchProfile]) -> None: + width = max(map(len, profiles)) + print("Available ambiguity-search profiles:\n") + for name in sorted(profiles): + profile = profiles[name] + print(f" {name:<{width}} {profile.description}") + scheduling = ( + "breadth-first" + if profile.nodes_per_depth is None + else f"{profile.nodes_per_depth} nodes/depth" + ) + print( + f" {'':<{width}} {profile.min_tokens}..{profile.max_tokens} tokens, " + f"{format_duration(profile.timeout_seconds)}, " + f"{profile.witnesses} witnesses, {scheduling}" + ) + if profile.output is not None: + print(f" {'':<{width}} report: {profile.output}") + + +def main(argv: Sequence[str] | None = None) -> int: + cli = parser() + arguments = cli.parse_args(argv) + try: + if arguments.command == "check": + return run_engine( + ["--check-tokens", " ".join(arguments.tokens)], + "Exact ambiguity check", + None, + ) + + if arguments.command == "classes": + return run_engine( + ["--dump-terminal-classes"], + "Terminal equivalence classes", + None, + ) + + profiles = load_profiles(arguments.profiles_file) + if arguments.command == "profiles": + list_profiles(profiles) + return 0 + + if arguments.profile not in profiles: + available = ", ".join(sorted(profiles)) + raise ConfigurationError( + f"unknown profile {arguments.profile!r}; available: {available}" + ) + profile = apply_overrides(profiles[arguments.profile], arguments) + if arguments.command == "prove" and arguments.level < 1: + raise ConfigurationError("proof level must be at least 1") + proof_level = arguments.level if arguments.command == "prove" else None + action = ( + f"Proof level {proof_level}, concretization" + if proof_level is not None + else "Search" + ) + output_path = ( + None + if profile.output is None + else expand_output_path(profile.output, profile.name) + ) + summary = profile_summary(profile, action) + if output_path is not None: + summary += f"\nReport: {output_path}" + engine_args = engine_arguments(profile, proof_level) + if arguments.dry_run: + print(summary) + print("\nEngine arguments:") + print(" " + shlex.join(engine_args)) + return 0 + return run_engine(engine_args, summary, output_path) + except ConfigurationError as error: + cli.error(str(error)) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/ambiguity_search.ml b/tools/ambiguity_search.ml new file mode 100644 index 00000000..63a04934 --- /dev/null +++ b/tools/ambiguity_search.ml @@ -0,0 +1,2029 @@ +(* Bounded complete-ambiguity search for Menhir automata. The search retains + unresolved LR actions as GLR branches, caps derivation counts at two, and + accepts a witness only when two derivations recognize the start symbol. *) + +module StringSet = Set.Make (String) +module IntMap = Map.Make (Int) +module ConflictSet = Set.Make (struct + type t = int * string + let compare = compare +end) + +type reduction = { lhs : string; width : int; prod : int } + +type state = { + transitions : (string, int) Hashtbl.t; + reductions : (string, reduction list) Hashtbl.t; + mutable accepts : StringSet.t; +} + +type automaton = { + states : state array; + terminals : StringSet.t; + aliases : (string, string) Hashtbl.t; + (* Terminals that behave identically everywhere in the automaton share a + class id. Two terminals are in one class when swapping them is an + automorphism of the recognition relation, so the search only needs to try + one representative per class instead of every interchangeable token. *) + terminal_class : (string, int) Hashtbl.t; +} + +let empty_state () = + { + transitions = Hashtbl.create 16; + reductions = Hashtbl.create 16; + accepts = StringSet.empty; + } + +let cap_add a b = min 2 (a + b) + +let words_re = Str.regexp "[ \t]+" + +let words text = + if String.trim text = "" then [] + else Str.split words_re (String.trim text) + +let read_lines path = + let channel = open_in path in + Fun.protect ~finally:(fun () -> close_in channel) (fun () -> + let rec loop result = + match input_line channel with + | line -> loop (line :: result) + | exception End_of_file -> List.rev result + in + loop []) + +let read_file path = + let channel = open_in_bin path in + Fun.protect ~finally:(fun () -> close_in channel) (fun () -> + let length = in_channel_length channel in + really_input_string channel length) + +let split_lines source = + List.map + (fun line -> + let length = String.length line in + if length > 0 && line.[length - 1] = '\r' then + String.sub line 0 (length - 1) + else line) + (String.split_on_char '\n' source) + +let strip_comments source = + let length = String.length source in + let buffer = Buffer.create length in + let rec normal index = + if index >= length then () + else if index + 1 < length && source.[index] = '(' && source.[index + 1] = '*' + then ocaml_comment 1 (index + 2) + else if index + 1 < length && source.[index] = '/' && source.[index + 1] = '*' + then c_comment (index + 2) + else begin + Buffer.add_char buffer source.[index]; + normal (index + 1) + end + and ocaml_comment depth index = + if index >= length then () + else if index + 1 < length && source.[index] = '(' && source.[index + 1] = '*' + then ocaml_comment (depth + 1) (index + 2) + else if index + 1 < length && source.[index] = '*' && source.[index + 1] = ')' + then if depth = 1 then normal (index + 2) else ocaml_comment (depth - 1) (index + 2) + else begin + if source.[index] = '\n' then Buffer.add_char buffer '\n'; + ocaml_comment depth (index + 1) + end + and c_comment index = + if index >= length then () + else if index + 1 < length && source.[index] = '*' && source.[index + 1] = '/' + then normal (index + 2) + else begin + if source.[index] = '\n' then Buffer.add_char buffer '\n'; + c_comment (index + 1) + end + in + normal 0; + Buffer.contents buffer + +let token_decl_re = + Str.regexp "^[ \t]*%token\\([ \t]+<[^>]+>\\)?[ \t]+\\(.*\\)$" + +let is_uppercase = function 'A' .. 'Z' -> true | _ -> false +let is_token_char = function 'A' .. 'Z' | '0' .. '9' | '_' -> true | _ -> false + +let parse_token_specs text add = + let length = String.length text in + let rec skip index = + if index < length && not (is_uppercase text.[index]) then skip (index + 1) + else index + in + let rec token_end index = + if index < length && is_token_char text.[index] then token_end (index + 1) + else index + in + let rec whitespace index = + if index < length && (text.[index] = ' ' || text.[index] = '\t') then + whitespace (index + 1) + else index + in + let rec quoted_end escaped index = + if index >= length then length + else if escaped then quoted_end false (index + 1) + else if text.[index] = '\\' then quoted_end true (index + 1) + else if text.[index] = '"' then index + else quoted_end false (index + 1) + in + let rec loop index = + let start = skip index in + if start < length then begin + let finish = token_end start in + let token = String.sub text start (finish - start) in + let after = whitespace finish in + if after < length && text.[after] = '"' then begin + let quote_end = quoted_end false (after + 1) in + let raw = String.sub text (after + 1) (quote_end - after - 1) in + add token (Some raw); + loop (min length (quote_end + 1)) + end + else begin + add token None; + loop finish + end + end + in + loop 0 + +let parse_tokens grammar = + let terminals = ref StringSet.empty in + let aliases = Hashtbl.create 64 in + let source = strip_comments (read_file grammar) in + List.iter + (fun line -> + if Str.string_match token_decl_re line 0 then + parse_token_specs (Str.matched_group 2 line) (fun token alias -> + terminals := StringSet.add token !terminals; + Option.iter + (fun alias -> + let alias = try Scanf.unescaped alias with _ -> alias in + Hashtbl.replace aliases token alias) + alias)) + (split_lines source); + (!terminals, aliases) + +let quote = Filename.quote + +let run command = + match Sys.command command with + | 0 -> true + | _ -> false + +let prepare_automaton ~menhir ~grammar ~directory = + let expanded = Filename.concat directory "expanded.mly" in + let preprocess_error = Filename.concat directory "preprocess.err" in + let preprocess = + Printf.sprintf "%s --only-preprocess-uu %s > %s 2> %s" + (quote menhir) (quote grammar) (quote expanded) (quote preprocess_error) + in + if not (run preprocess) then begin + List.iter prerr_endline (read_lines preprocess_error); + failwith "Menhir failed to preprocess the grammar" + end; + let base = Filename.concat directory "automaton" in + let build_error = Filename.concat directory "automaton.err" in + let build = + Printf.sprintf "%s --dump --base %s %s > /dev/null 2> %s" + (quote menhir) (quote base) (quote expanded) (quote build_error) + in + ignore (run build); + let automaton = base ^ ".automaton" in + if not (Sys.file_exists automaton) then begin + List.iter prerr_endline (read_lines build_error); + failwith "Menhir did not produce an LR automaton" + end; + automaton + +(* Terminal equivalence classes. + + Two terminals are interchangeable when replacing every occurrence of one by + the other is an automorphism of the recognition relation: they shift to + equivalent states, trigger the same reductions as a lookahead, and are + accepted in the same places. Interchangeable terminals generate isomorphic + parse forests, so an ambiguous sentence exists with one iff it exists with + the other; the search may therefore explore a single representative per + class. Because precedence and associativity are already baked into the + automaton's concrete shift/reduce actions, tokens with different precedence + land in different classes automatically. + + Equivalence is computed in two steps. First a partition refinement over the + states (a bisimulation on recognition behaviour: shift-target blocks, + reduction (lhs, width) sets, nonterminal goto-target blocks, and accepts) + collapses states that differ only in which production they carry - so + [primary -> INT], [primary -> FLOAT] and [primary -> STRING] states become + one block. Then terminals are grouped by their action across every state, + comparing shift targets up to that state partition. This is why FLOAT and + STRING merge even though they shift to distinct states, while INT stays + separate: it is also valid as a const generic argument, a context the other + two never reach. *) +let compute_terminal_classes states terminals = + let count = Array.length states in + let terminal_list = StringSet.elements terminals in + (* "#" is the end marker; it is never shifted but does drive reductions and + acceptance, so it participates in the lookahead-role signatures. *) + let lookaheads = "#" :: terminal_list in + let reduce_signature state token = + match Hashtbl.find_opt state.reductions token with + | None -> [] + | Some reductions -> + List.sort_uniq compare + (List.map (fun reduction -> (reduction.lhs, reduction.width)) reductions) + in + let shift_target state token = + if StringSet.mem token terminals then Hashtbl.find_opt state.transitions token + else None + in + let goto_targets state = + Hashtbl.fold + (fun symbol target result -> + if StringSet.mem symbol terminals then result + else (symbol, target) :: result) + state.transitions [] + |> List.sort compare + in + let block = Array.make (max 1 count) 0 in + let state_signature index = + let state = states.(index) in + let shifts = + List.map + (fun token -> + match shift_target state token with + | Some target -> Some block.(target) + | None -> None) + terminal_list + in + let reduces = List.map (reduce_signature state) lookaheads in + let accepts = + List.map (fun token -> StringSet.mem token state.accepts) lookaheads + in + let gotos = + List.map (fun (symbol, target) -> (symbol, block.(target))) (goto_targets state) + in + (shifts, reduces, accepts, gotos) + in + (* One refinement pass. Keying by the previous block keeps the partition + monotonically finer, so the block count never decreases and the loop + below terminates once it stabilizes. *) + let refine () = + let table = Hashtbl.create 1024 in + let fresh = ref 0 in + let updated = Array.make (max 1 count) 0 in + for index = 0 to count - 1 do + let key = (block.(index), state_signature index) in + match Hashtbl.find_opt table key with + | Some id -> updated.(index) <- id + | None -> + let id = !fresh in + incr fresh; + Hashtbl.add table key id; + updated.(index) <- id + done; + Array.blit updated 0 block 0 count; + !fresh + in + let previous = ref (-1) in + let blocks = ref (refine ()) in + while !blocks <> !previous do + previous := !blocks; + blocks := refine () + done; + let terminal_signature token = + List.init count (fun index -> + let state = states.(index) in + let shift = + match shift_target state token with + | Some target -> Some block.(target) + | None -> None + in + (shift, reduce_signature state token, StringSet.mem token state.accepts)) + in + let signatures = Hashtbl.create 64 in + let terminal_class = Hashtbl.create 64 in + let fresh = ref 0 in + List.iter + (fun token -> + let key = terminal_signature token in + let id = + match Hashtbl.find_opt signatures key with + | Some id -> id + | None -> + let id = !fresh in + incr fresh; + Hashtbl.add signatures key id; + id + in + Hashtbl.add terminal_class token id) + terminal_list; + terminal_class + +(* Keep one terminal per class - the alphabetically first, for reproducible + witnesses - so interchangeable tokens are explored once. A class is either + wholly present in a frontier's possible tokens or wholly absent (its members + act identically in every state), so picking representatives never drops a + reachable token. *) +let class_representatives automaton tokens = + let seen = Hashtbl.create 32 in + StringSet.fold + (fun token result -> + match Hashtbl.find_opt automaton.terminal_class token with + | Some id when Hashtbl.mem seen id -> result + | Some id -> + Hashtbl.add seen id (); + StringSet.add token result + | None -> StringSet.add token result) + tokens StringSet.empty + +let state_re = Str.regexp "^State \\([0-9]+\\):$" +let transition_re = + Str.regexp "^-- On \\([^ ]+\\) shift to state \\([0-9]+\\)$" +let lookahead_re = Str.regexp "^-- On \\(.+\\)$" +let reduction_re = Str.regexp "^-- reduce production \\(.+\\) ->\\(.*\\)$" +let accept_re = Str.regexp "^-- accept \\([^ ]+\\)$" + +let parse_automaton path terminals aliases = + let table = Hashtbl.create 1024 in + let current = ref None in + let lookaheads = ref [] in + let productions = Hashtbl.create 512 in + let intern_production text = + match Hashtbl.find_opt productions text with + | Some id -> id + | None -> + let id = Hashtbl.length productions in + Hashtbl.add productions text id; + id + in + let get_state number = + match Hashtbl.find_opt table number with + | Some state -> state + | None -> + let state = empty_state () in + Hashtbl.add table number state; + state + in + List.iter + (fun line -> + if Str.string_match state_re line 0 then begin + let number = int_of_string (Str.matched_group 1 line) in + current := Some (get_state number); + lookaheads := [] + end + else + match !current with + | None -> () + | Some state -> + if Str.string_match transition_re line 0 then begin + let symbol = Str.matched_group 1 line in + let target = int_of_string (Str.matched_group 2 line) in + Hashtbl.replace state.transitions symbol target; + lookaheads := [] + end + else if Str.string_match lookahead_re line 0 then + lookaheads := words (Str.matched_group 1 line) + else if Str.string_match reduction_re line 0 then begin + let lhs = String.trim (Str.matched_group 1 line) in + let rhs = String.trim (Str.matched_group 2 line) in + let reduction = + { + lhs; + width = List.length (words rhs); + prod = intern_production (lhs ^ " -> " ^ rhs); + } + in + List.iter + (fun token -> + let previous = + Option.value (Hashtbl.find_opt state.reductions token) + ~default:[] + in + Hashtbl.replace state.reductions token + (reduction :: previous)) + !lookaheads + end + else if Str.string_match accept_re line 0 then + List.iter + (fun token -> + state.accepts <- StringSet.add token state.accepts) + !lookaheads) + (read_lines path); + let maximum = Hashtbl.fold (fun number _ value -> max number value) table 0 in + let states = Array.init (maximum + 1) (fun number -> get_state number) in + let terminal_class = compute_terminal_classes states terminals in + { states; terminals; aliases; terminal_class } + +module Stack_pool = struct + type node = { + id : int; + state : int; + parent : node option; + depth : int; + } + + type t = { + by_edge : ((int * int), node) Hashtbl.t; + by_id : (int, node) Hashtbl.t; + mutable next_id : int; + root : node; + } + + let create () = + let root = { id = 0; state = 0; parent = None; depth = 0 } in + let by_id = Hashtbl.create 16_384 in + Hashtbl.add by_id 0 root; + { + by_edge = Hashtbl.create 16_384; + by_id; + next_id = 1; + root; + } + + let find pool id = Hashtbl.find pool.by_id id + + let push pool parent state = + let edge = (parent.id, state) in + match Hashtbl.find_opt pool.by_edge edge with + | Some node -> node + | None -> + let node = + { + id = pool.next_id; + state; + parent = Some parent; + depth = parent.depth + 1; + } + in + pool.next_id <- pool.next_id + 1; + Hashtbl.add pool.by_edge edge node; + Hashtbl.add pool.by_id node.id node; + node + + let rec pop node count = + if count = 0 then Some node + else + match node.parent with + | None -> None + | Some parent -> pop parent (count - 1) + + let size pool = Hashtbl.length pool.by_id + + (* Drop every node not reachable from the root or from [iter_live]'s nodes. + Ids keep increasing across compactions, so a swept node re-created later + gets a fresh id and stale signatures in the dedup cache simply age out. *) + let compact pool iter_live = + let keep : (int, node) Hashtbl.t = Hashtbl.create 16_384 in + let rec mark node = + if not (Hashtbl.mem keep node.id) then begin + Hashtbl.add keep node.id node; + Option.iter mark node.parent + end + in + mark pool.root; + iter_live mark; + Hashtbl.reset pool.by_edge; + Hashtbl.reset pool.by_id; + Hashtbl.iter + (fun _ node -> + Hashtbl.add pool.by_id node.id node; + Option.iter + (fun parent -> + Hashtbl.add pool.by_edge (parent.id, node.state) node) + node.parent) + keep +end + +type frontier = int IntMap.t +type signature = (int * int) list + +let add_count stack_id count frontier = + let old = Option.value (IntMap.find_opt stack_id frontier) ~default:0 in + let updated = cap_add old count in + (IntMap.add stack_id updated frontier, updated - old) + +let signature frontier : signature = IntMap.bindings frontier + +let derivations frontier = + IntMap.fold (fun _ count total -> cap_add total count) frontier 0 + +type engine = { + automaton : automaton; + stacks : Stack_pool.t; + closure_cache : ((int * string), frontier) Hashtbl.t; +} + +let reductions state token = + Option.value (Hashtbl.find_opt state.reductions token) ~default:[] + +let closure_one engine stack_id token = + match Hashtbl.find_opt engine.closure_cache (stack_id, token) with + | Some result -> result + | None -> + let closure = ref (IntMap.singleton stack_id 1) in + let propagated = Hashtbl.create 16 in + let queue = Queue.create () in + Queue.add stack_id queue; + while not (Queue.is_empty queue) do + let current_id = Queue.take queue in + let count = IntMap.find current_id !closure in + let sent = + Option.value (Hashtbl.find_opt propagated current_id) ~default:0 + in + let available = count - sent in + if available > 0 then begin + Hashtbl.replace propagated current_id count; + let current = Stack_pool.find engine.stacks current_id in + let state = engine.automaton.states.(current.state) in + List.iter + (fun reduction -> + match Stack_pool.pop current reduction.width with + | None -> () + | Some base -> + (match + Hashtbl.find_opt + engine.automaton.states.(base.state).transitions + reduction.lhs + with + | None -> () + | Some target -> + let reduced = Stack_pool.push engine.stacks base target in + let updated, delta = + add_count reduced.id available !closure + in + closure := updated; + if delta > 0 then Queue.add reduced.id queue)) + (reductions state token) + end + done; + Hashtbl.add engine.closure_cache (stack_id, token) !closure; + !closure + +let closure engine frontier token = + IntMap.fold + (fun stack_id outer_count result -> + IntMap.fold + (fun reduced_id inner_count result -> + fst + (add_count reduced_id + (min 2 (outer_count * inner_count)) + result)) + (closure_one engine stack_id token) result) + frontier IntMap.empty + +let shift engine frontier token = + IntMap.fold + (fun stack_id count result -> + let stack = Stack_pool.find engine.stacks stack_id in + match + Hashtbl.find_opt engine.automaton.states.(stack.state).transitions token + with + | None -> result + | Some target -> + let shifted = Stack_pool.push engine.stacks stack target in + fst (add_count shifted.id count result)) + (closure engine frontier token) IntMap.empty + +let accepted_count engine frontier = + IntMap.fold + (fun stack_id count total -> + let stack = Stack_pool.find engine.stacks stack_id in + if + StringSet.mem "#" + engine.automaton.states.(stack.state).accepts + then cap_add total count + else total) + (closure engine frontier "#") 0 + +let possible_tokens engine frontier = + IntMap.fold + (fun stack_id _ tokens -> + let stack = Stack_pool.find engine.stacks stack_id in + let state = engine.automaton.states.(stack.state) in + let tokens = + Hashtbl.fold + (fun symbol _ tokens -> + if StringSet.mem symbol engine.automaton.terminals then + StringSet.add symbol tokens + else tokens) + state.transitions tokens + in + Hashtbl.fold + (fun token _ tokens -> + if token = "#" then tokens else StringSet.add token tokens) + state.reductions tokens) + frontier StringSet.empty + +let conflict_states automaton = + Array.mapi + (fun _ state -> + Hashtbl.fold + (fun token reductions conflict -> + conflict + || List.length reductions > 1 + || Hashtbl.mem state.transitions token) + state.reductions false) + automaton.states + +(* Build an optimistic state graph. Terminal shifts cost one token; + nonterminal transitions and reductions cost zero. Reverse shortest paths on + this graph are safe lower bounds, even though stack context is ignored. *) +let reverse_distances automaton targets = + let count = Array.length automaton.states in + let reverse = Array.make count [] in + let add_edge source target cost = + reverse.(target) <- (source, cost) :: reverse.(target) + in + Array.iteri + (fun source state -> + Hashtbl.iter + (fun symbol target -> + let cost = if StringSet.mem symbol automaton.terminals then 1 else 0 in + add_edge source target cost) + state.transitions) + automaton.states; + let gotos = Hashtbl.create 128 in + Array.iter + (fun state -> + Hashtbl.iter + (fun symbol target -> + if not (StringSet.mem symbol automaton.terminals) then + let previous = + Option.value (Hashtbl.find_opt gotos symbol) ~default:[] + in + Hashtbl.replace gotos symbol (target :: previous)) + state.transitions) + automaton.states; + Array.iteri + (fun source state -> + Hashtbl.iter + (fun _ reductions -> + List.iter + (fun reduction -> + List.iter (fun target -> add_edge source target 0) + (Option.value (Hashtbl.find_opt gotos reduction.lhs) ~default:[])) + reductions) + state.reductions) + automaton.states; + let infinity = max_int / 4 in + let distance = Array.make count infinity in + let deque = Queue.create () in + Array.iteri + (fun state is_target -> + if is_target then begin + distance.(state) <- 0; + Queue.add state deque + end) + targets; + (* Repeated relaxation is sufficient here; zero/one weights and 590 states + keep this construction negligible compared with the search. *) + while not (Queue.is_empty deque) do + let target = Queue.take deque in + List.iter + (fun (source, cost) -> + let candidate = distance.(target) + cost in + if candidate < distance.(source) then begin + distance.(source) <- candidate; + Queue.add source deque + end) + reverse.(target) + done; + distance + +let frontier_lower_bound engine distances frontier = + IntMap.fold + (fun stack_id _ best -> + let stack = Stack_pool.find engine.stacks stack_id in + min best distances.(stack.state)) + frontier (max_int / 4) + +(* ----- Conservative unambiguity prover ----- + + Abstract every GLR stack to its top-K states; a suffix shorter than K means + the states below are unknown, which over-approximates every continuation, + including the true stack bottom. Reductions that pop into the unknown part + re-enter through every goto edge on the reduced nonterminal. The abstract + configuration space is finite, so a pair search over runs that consume the + same input terminates. Every concrete ambiguous sentence projects onto an + abstract pair of runs that diverge in their actions and both accept, so if + no such abstract pair exists, the grammar is unambiguous. The converse does + not hold: an abstract diverging pair may be spurious, which is why a found + candidate only downgrades the verdict to "not proven". *) + +let rec drop_states count list = + if count = 0 then list + else match list with [] -> [] | _ :: tail -> drop_states (count - 1) tail + +let truncate_suffix limit list = + let rec take count = function + | [] -> [] + | _ when count = 0 -> [] + | head :: tail -> head :: take (count - 1) tail + in + take limit list + +let goto_edges automaton = + let table = Hashtbl.create 256 in + Array.iteri + (fun source state -> + Hashtbl.iter + (fun symbol target -> + if not (StringSet.mem symbol automaton.terminals) then + Hashtbl.replace table symbol + ((source, target) + :: Option.value (Hashtbl.find_opt table symbol) ~default:[])) + state.transitions) + automaton.states; + table + +(* One micro-step of a single run while consuming a token: apply one + reduction, or terminate the chain by shifting the token (accepting, when + the token is "#"). *) +type side_move = + | Reduce of int * int list (* production id, suffix afterwards *) + | Terminate of int list (* suffix after the shift, or at acceptance *) + +let side_moves automaton gotos limit cache suffix token = + match Hashtbl.find_opt cache (suffix, token) with + | Some moves -> moves + | None -> + let moves = ref [] in + (match suffix with + | [] -> () + | top :: _ -> + let state = automaton.states.(top) in + if token = "#" then begin + if StringSet.mem "#" state.accepts then + moves := Terminate suffix :: !moves + end + else + Option.iter + (fun target -> + moves := + Terminate (truncate_suffix limit (target :: suffix)) :: !moves) + (Hashtbl.find_opt state.transitions token); + List.iter + (fun reduction -> + if reduction.width < List.length suffix then + match drop_states reduction.width suffix with + | [] -> assert false + | base :: _ as remaining -> + Option.iter + (fun target -> + moves := + Reduce + ( reduction.prod, + truncate_suffix limit (target :: remaining) ) + :: !moves) + (Hashtbl.find_opt + automaton.states.(base).transitions reduction.lhs) + else + (* The reduction pops into the unknown part of the stack; the + goto source is the state left on top afterwards. *) + List.iter + (fun (source, target) -> + moves := + Reduce + (reduction.prod, truncate_suffix limit [ target; source ]) + :: !moves) + (Option.value + (Hashtbl.find_opt gotos reduction.lhs) + ~default:[])) + (reductions state token)); + Hashtbl.add cache (suffix, token) !moves; + !moves + +type chain_status = Running of int list | Finished of int list + +(* All (left result, right result, diverged) ways for both runs to consume + [token]. The two reduction chains advance in lockstep: aligned identical + productions carry no divergence, so a reduction cycle both runs share + cancels out instead of poisoning the verdict, while any position where the + chains first differ - two different productions, or one run reducing while + the other shifts - is exactly where two distinct parses of one sentence + must part ways, and marks the pair diverged. Goto and shift-target + differences alone are abstraction artifacts, never a first divergence, so + they are deliberately not compared. *) +let joint_outcomes moves (start_left, start_right) token = + let seen = Hashtbl.create 64 in + let results = Hashtbl.create 16 in + let queue = Queue.create () in + let push node = + if not (Hashtbl.mem seen node) then begin + Hashtbl.add seen node (); + Queue.add node queue + end + in + push (Running start_left, Running start_right, false); + while not (Queue.is_empty queue) do + let (left, right, diverged) = Queue.take queue in + match (left, right) with + | Finished result_left, Finished result_right -> + Hashtbl.replace results (result_left, result_right, diverged) () + | Running suffix_left, Running suffix_right -> + let paired move_left move_right = + match (move_left, move_right) with + | Reduce (p, l), Reduce (q, r) -> + Some (Running l, Running r, diverged || p <> q) + | Reduce (_, l), Terminate r -> Some (Running l, Finished r, true) + | Terminate l, Reduce (_, r) -> Some (Finished l, Running r, true) + | Terminate l, Terminate r -> Some (Finished l, Finished r, diverged) + in + if (not diverged) && suffix_left = suffix_right then + (* The two runs are still the same run: same stack, so an + unknown-base goto resolves identically on both sides. Identical + moves pair diagonally; distinct moves pair only where two real + parses can first part ways - different productions, or reducing + against shifting. Same-production pairs with different gotos are + different possible worlds, never two parses of one sentence. *) + let all = moves suffix_left token in + List.iter + (fun move_left -> + List.iter + (fun move_right -> + let compatible = + match (move_left, move_right) with + | Reduce (p, _), Reduce (q, _) -> p <> q + | Reduce _, Terminate _ | Terminate _, Reduce _ -> true + | Terminate _, Terminate _ -> false + in + if move_left == move_right || compatible then + Option.iter push (paired move_left move_right)) + all) + all + else + let moves_left = moves suffix_left token in + let moves_right = moves suffix_right token in + List.iter + (fun move_left -> + List.iter + (fun move_right -> + Option.iter push (paired move_left move_right)) + moves_right) + moves_left + | Running suffix_left, Finished _ -> + List.iter + (fun move -> + match move with + | Reduce (_, l) -> push (Running l, right, diverged) + | Terminate l -> push (Finished l, right, diverged)) + (moves suffix_left token) + | Finished _, Running suffix_right -> + List.iter + (fun move -> + match move with + | Reduce (_, r) -> push (left, Running r, diverged) + | Terminate r -> push (left, Finished r, diverged)) + (moves suffix_right token) + done; + Hashtbl.fold (fun key () list -> key :: list) results [] + +type prove_result = + | Proven of int + | Abstract_candidate of string list * int + | Pair_overflow of int + +let prove engine limit pair_limit = + let automaton = engine.automaton in + let gotos = goto_edges automaton in + let moves_cache = Hashtbl.create 100_003 in + let moves = side_moves automaton gotos limit moves_cache in + let joint_cache = Hashtbl.create 100_003 in + let joint pair token = + match Hashtbl.find_opt joint_cache (pair, token) with + | Some outcomes -> outcomes + | None -> + let outcomes = joint_outcomes moves pair token in + Hashtbl.add joint_cache (pair, token) outcomes; + outcomes + in + let parents : + ( int list * int list * bool, + (string * (int list * int list * bool)) option ) + Hashtbl.t = + Hashtbl.create 100_003 + in + let queue = Queue.create () in + let overflow = ref false in + let candidate = ref None in + let canonical (left, right, diverged) = + if compare left right <= 0 then (left, right, diverged) + else (right, left, diverged) + in + let push origin node = + let node = canonical node in + if not (Hashtbl.mem parents node) then + if Hashtbl.length parents >= pair_limit then overflow := true + else begin + Hashtbl.add parents node origin; + Queue.add node queue + end + in + let rec trail node = + match Hashtbl.find parents node with + | None -> [] + | Some (token, parent) -> token :: trail parent + in + (* One representative per terminal class: interchangeable lookaheads drive + the same abstract reduction chains, so exploring one covers the class and + shrinks the abstract pair space by the same factor as the search. *) + let terminals = + StringSet.elements (class_representatives automaton automaton.terminals) + in + push None ([ 0 ], [ 0 ], false); + while (not (Queue.is_empty queue)) && !candidate = None && not !overflow do + let (left, right, diverged) as node = Queue.take queue in + List.iter + (fun (_, _, chain_diverged) -> + if (diverged || chain_diverged) && !candidate = None then + candidate := Some (List.rev (trail node))) + (joint (left, right) "#"); + if !candidate = None then + List.iter + (fun token -> + List.iter + (fun (next_left, next_right, chain_diverged) -> + push + (Some (token, node)) + (next_left, next_right, diverged || chain_diverged)) + (joint (left, right) token)) + terminals + done; + let explored = Hashtbl.length parents in + match !candidate with + | Some tokens -> Abstract_candidate (tokens, explored) + | None -> if !overflow then Pair_overflow explored else Proven explored + +type outcome = { + witnesses : ((int * string) list * string list) list; + explored : int; + unique : int; + deepest : int; + stopped : string option; +} + +type search_progress = { + depth : int; + ambiguities : ((int * string) list * int) list; + explored : int; + unique : int; + rss_bytes : float; +} + +let () = Random.self_init () + +let rec temporary_directory () = + let path = + Filename.concat (Filename.get_temp_dir_name ()) + (Printf.sprintf "zane-ambiguity-%d-%08x" (Unix.getpid ()) (Random.bits ())) + in + try + Unix.mkdir path 0o700; + path + with Unix.Unix_error (Unix.EEXIST, _, _) -> temporary_directory () + +let rec remove_tree path = + if Sys.is_directory path then begin + Array.iter (fun name -> remove_tree (Filename.concat path name)) + (Sys.readdir path); + Unix.rmdir path + end + else Sys.remove path + +let render automaton tokens = + tokens + |> List.filter (fun token -> token <> "EOF") + |> List.map (fun token -> + Option.value (Hashtbl.find_opt automaton.aliases token) + ~default:("<" ^ token ^ ">")) + |> String.concat " " + +let conflict_profile engine tokens = + let frontier = ref (IntMap.singleton engine.stacks.root.id 1) in + let conflicts = ref ConflictSet.empty in + let inspect token = + let reduced = closure engine !frontier token in + IntMap.iter + (fun stack_id _ -> + let stack = Stack_pool.find engine.stacks stack_id in + let state = engine.automaton.states.(stack.state) in + let reductions = reductions state token in + if + List.length reductions > 1 + || (reductions <> [] && Hashtbl.mem state.transitions token) + then conflicts := ConflictSet.add (stack.state, token) !conflicts) + reduced + in + List.iter + (fun token -> + inspect token; + frontier := shift engine !frontier token) + tokens; + inspect "#"; + ConflictSet.elements !conflicts + +let compare_witness (_, left) (_, right) = + match compare (List.length left) (List.length right) with + | 0 -> compare left right + | order -> order + +let merge_witnesses limit lists = + let by_profile = Hashtbl.create 128 in + List.iter + (fun (profile, tokens) -> + match Hashtbl.find_opt by_profile profile with + | Some previous when List.length previous <= List.length tokens -> () + | _ -> Hashtbl.replace by_profile profile tokens) + (List.concat lists); + Hashtbl.fold (fun profile tokens result -> (profile, tokens) :: result) + by_profile [] + |> List.sort compare_witness + |> List.to_seq |> Seq.take limit |> List.of_seq + +type directed_item = { + tokens_rev : string list; + depth : int; + frontier : frontier; + branched : bool; +} + +(* Bounded two-generation dedup cache over 124-bit frontier digests. Inserts + and hits go to the young generation; when it fills, the old generation is + dropped and the generations flip, so recently touched entries survive and + stale ones are forgotten. Deduplication is pruning, not correctness - a + forgotten entry costs re-exploration, never a missed witness - so a full + cache degrades the search instead of stopping it or exhausting memory. *) +module Seen_cache = struct + type digest = int * int + + (* The mix constants (and the lane seeds below) need OCaml's 63-bit native + int, so this tool requires a 64-bit platform; Int64 would box every + operation on this hot path. *) + let mix hash value = + let hash = hash + value in + let hash = (hash lxor (hash lsr 30)) * 0x3F58476D1CE4E5B9 in + let hash = (hash lxor (hash lsr 27)) * 0x14D049BB133111EB in + hash lxor (hash lsr 31) + + (* Two independently seeded 62-bit lanes make colliding distinct frontiers + astronomically unlikely even across billions of entries. A collision + could only skip a frontier wrongly, never produce a false witness. *) + let digest branched progress frontier = + let lane seed = + IntMap.fold + (fun stack_id count hash -> mix (mix hash stack_id) count) + frontier + (mix (mix seed (Bool.to_int branched)) progress) + in + (lane 0x2545F4914F6CDD1D, lane 0x27220A95FE4D1D65) + + type t = { + generation_capacity : int; + mutable young : (digest, int) Hashtbl.t; + mutable old : (digest, int) Hashtbl.t; + mutable inserted : int; + } + + (* [capacity] is the total number of entries retained across both + generations. Keeping that meaning literal is important: callers use it + to divide a fixed memory budget between the queue and deduplication. *) + let create capacity = + { + generation_capacity = max 1 (capacity / 2); + young = Hashtbl.create 4_096; + old = Hashtbl.create 4_096; + inserted = 0; + } + + let find cache key = + match Hashtbl.find_opt cache.young key with + | Some _ as found -> found + | None -> Hashtbl.find_opt cache.old key + + let refresh cache key depth = + if + (not (Hashtbl.mem cache.young key)) + && Hashtbl.length cache.young >= cache.generation_capacity + then begin + cache.old <- cache.young; + cache.young <- Hashtbl.create 4_096 + end; + Hashtbl.replace cache.young key depth + + let insert cache key depth = + cache.inserted <- cache.inserted + 1; + refresh cache key depth + + (* Deduplication is an optimization, so the older generation is the safest + memory to reclaim under pressure: forgetting it can cause re-exploration + but cannot hide a witness. *) + let release_old cache = cache.old <- Hashtbl.create 4_096 +end + +let managed_heap_bytes () = + let stats = Gc.quick_stat () in + float_of_int stats.heap_words *. float_of_int (Sys.word_size / 8) + +let resident_memory_bytes () = + try + read_lines "/proc/self/status" + |> List.find_map (fun line -> + if String.starts_with ~prefix:"VmRSS:" line then + try Some (Scanf.sscanf line "VmRSS: %f kB" (fun kib -> kib *. 1024.)) + with _ -> None + else None) + |> Option.value ~default:(managed_heap_bytes ()) + with _ -> managed_heap_bytes () + +let progress_is_visible = Unix.isatty Unix.stderr + +let compact_number value = + let value = float_of_int value in + if value >= 1_000_000_000. then Printf.sprintf "%.1fB" (value /. 1_000_000_000.) + else if value >= 1_000_000. then Printf.sprintf "%.1fM" (value /. 1_000_000.) + else if value >= 1_000. then Printf.sprintf "%.1fk" (value /. 1_000.) + else Printf.sprintf "%.0f" value + +let memory_bar used budget = + let width = 12 in + let ratio = if budget <= 0. then 0. else min 1. (used /. budget) in + let filled = int_of_float (floor ((ratio *. float_of_int width) +. 0.5)) in + String.make filled '#' ^ String.make (width - filled) '-' + +let render_progress ~started ~max_tokens ~memory_budget + (entries : (bool * search_progress) list) = + if progress_is_visible && entries <> [] then begin + let has_active = List.exists fst entries in + let depth = + List.fold_left + (fun current + (is_active, (progress : search_progress)) -> + if has_active && not is_active then current + else max current progress.depth) + 0 entries + in + let profiles = Hashtbl.create 64 in + let explored = ref 0 in + let unique = ref 0 in + let rss_bytes = ref 0. in + List.iter + (fun (_, (progress : search_progress)) -> + explored := !explored + progress.explored; + unique := !unique + progress.unique; + rss_bytes := !rss_bytes +. progress.rss_bytes; + List.iter + (fun (profile, witness_depth) -> + match Hashtbl.find_opt profiles profile with + | Some previous when previous <= witness_depth -> () + | _ -> Hashtbl.replace profiles profile witness_depth) + progress.ambiguities) + entries; + let at_depth = + Hashtbl.fold + (fun _ witness_depth count -> + if witness_depth = depth then count + 1 else count) + profiles 0 + in + let elapsed = max 0. (Unix.gettimeofday () -. started) in + Printf.eprintf + "\r\027[2K● depth %d/%d | ambiguities@depth %d | %s explored | %s unique | RAM [%s] %.1f/%.1f GiB | %02d:%02d" + depth max_tokens at_depth (compact_number !explored) + (compact_number !unique) + (memory_bar !rss_bytes memory_budget) + (!rss_bytes /. 1024. /. 1024. /. 1024.) + (memory_budget /. 1024. /. 1024. /. 1024.) + (int_of_float elapsed / 60) (int_of_float elapsed mod 60); + flush stderr + end + +let clear_progress () = + if progress_is_visible then begin + Printf.eprintf "\r\027[2K"; + flush stderr + end + +let write_progress path (progress : search_progress) = + let temporary = path ^ ".new" in + let channel = open_out_bin temporary in + Marshal.to_channel channel progress []; + close_out channel; + Sys.rename temporary path + +let read_progress path : search_progress option = + try + let channel = open_in_bin path in + Fun.protect ~finally:(fun () -> close_in channel) (fun () -> + Some (Marshal.from_channel channel : search_progress)) + with Sys_error _ | End_of_file | Failure _ -> None + +let unified_search engine initial ~max_tokens ~min_tokens ~nodes_per_depth + ~timeout ~max_frontiers ~max_queue ~max_witnesses ~soft_heap_bytes + ~hard_heap_bytes ~show_progress ~on_progress conflict_distance + accept_distance = + let deadline = Unix.gettimeofday () +. timeout in + let buckets = Array.init (max_tokens + 1) (fun _ -> Queue.create ()) in + let seen = Seen_cache.create max_frontiers in + let queued = ref 0 in + let dropped = ref false in + let admit_new = ref true in + let enqueue item = + incr queued; + Queue.add item buckets.(item.depth) + in + (* max_queue caps the number of queued items - the search reach. A full + queue drops new discoveries (recorded so the result is reported as + incomplete) instead of growing without bound; a dropped frontier can + still be rediscovered once the queue drains, because only enqueued + frontiers enter the dedup cache. max_frontiers is the independent dedup + memory budget (see Seen_cache above): a smaller table simply prunes less. *) + let add item = + if item.depth <= max_tokens then + if derivations item.frontier >= 2 && accepted_count engine item.frontier >= 2 + then enqueue item + else if not !admit_new then dropped := true + else + (* Before [min_tokens], revisiting the same parser frontier at a + greater depth is useful rather than redundant: it can eventually + produce a witness long enough to report. Once the minimum is met, + the usual shortest-path deduplication applies. *) + let progress = min item.depth min_tokens in + let key = Seen_cache.digest item.branched progress item.frontier in + match Seen_cache.find seen key with + | Some depth when depth <= item.depth -> + Seen_cache.refresh seen key depth + | Some _ when !queued < max_queue -> + Seen_cache.refresh seen key item.depth; + enqueue item + | None when !queued < max_queue -> + Seen_cache.insert seen key item.depth; + enqueue item + | _ -> dropped := true + in + List.iter add initial; + let explored = ref 0 in + let deepest = ref 0 in + let conflict_seeds = ref 0 in + let witnesses = Hashtbl.create max_witnesses in + let stopped = ref None in + let current_depth = ref 0 in + let expanded_at_depth = ref 0 in + let last_depth = ref 0 in + let last_progress = ref 0. in + let emit_progress force = + let now = Unix.gettimeofday () in + if show_progress && (force || now -. !last_progress >= 0.2) then begin + last_progress := now; + on_progress + { + depth = !last_depth; + ambiguities = + Hashtbl.fold + (fun profile tokens result -> + (profile, List.length tokens) :: result) + witnesses []; + explored = !explored; + unique = seen.Seen_cache.inserted; + rss_bytes = resident_memory_bytes (); + } + end + in + (* The stack pool and closure cache also grow with the search; sweep them + against the live queue on a geometric schedule so total memory stays + proportional to the queue itself (the live set marked below), whose size + is bounded by max_queue. *) + let stack_budget = max 65_536 (4 * max_queue) in + let compact_floor = ref stack_budget in + let mark_live mark = + Array.iter + (fun bucket -> + Queue.iter + (fun item -> + IntMap.iter + (fun stack_id _ -> + mark (Stack_pool.find engine.stacks stack_id)) + item.frontier) + bucket) + buckets + in + let compact_search_state () = + Stack_pool.compact engine.stacks mark_live; + Hashtbl.reset engine.closure_cache; + compact_floor := max stack_budget (2 * Stack_pool.size engine.stacks) + in + let maybe_compact_stacks () = + if Stack_pool.size engine.stacks > !compact_floor then begin + compact_search_state () + end + in + (* Static entry-size estimates determine the shape of the search, but the + heap guard is what keeps the process inside the requested machine budget. + Near the soft limit, compact once and keep using the reclaimed space. At + the hard limit, pause admission and drain queued work until compaction + brings the heap below the resume watermark. This hysteresis keeps memory + near a plateau instead of repeatedly overshooting the budget. *) + (* soft and hard are 80% and 90% of the worker share respectively. Resume at + 85% so pressure mode holds a narrow 85--90% plateau instead of producing + a large sawtooth while the queue drains and refills. *) + let resume_heap_bytes = 1.0625 *. soft_heap_bytes in + let next_heap_check = ref soft_heap_bytes in + let manage_memory () = + let before = managed_heap_bytes () in + if before >= !next_heap_check then begin + Seen_cache.release_old seen; + compact_search_state (); + Gc.compact (); + let after = managed_heap_bytes () in + if after >= hard_heap_bytes then begin + admit_new := false; + dropped := true; + next_heap_check := 0. + end + else begin + if (not !admit_new) && after <= resume_heap_bytes then + admit_new := true; + next_heap_check := + if !admit_new then + min hard_heap_bytes (max soft_heap_bytes (after *. 1.10)) + else 0. + end + end + in + (* A quota forms a depth wave: expand a bounded number of siblings, descend + through their children, then return to the earliest unfinished siblings + when no deeper bucket remains. Nothing is discarded, so repeated waves + eventually cover the same frontiers as breadth-first search. *) + let first_nonempty start = + let depth = ref start in + while !depth <= max_tokens && Queue.is_empty buckets.(!depth) do + incr depth + done; + if !depth <= max_tokens then Some !depth else None + in + let next_bucket () = + match nodes_per_depth with + | None -> Option.get (first_nonempty 0) + | Some limit -> + let current_available = + !current_depth <= max_tokens + && not (Queue.is_empty buckets.(!current_depth)) + in + if (not current_available) || !expanded_at_depth >= limit then begin + let next = + match first_nonempty (!current_depth + 1) with + | Some depth -> depth + | None -> Option.get (first_nonempty 0) + in + current_depth := next; + expanded_at_depth := 0 + end; + incr expanded_at_depth; + !current_depth + in + while + Hashtbl.length witnesses < max_witnesses + && !queued > 0 + && Unix.gettimeofday () < deadline + do + let bucket = next_bucket () in + if bucket >= 0 && bucket <= max_tokens then begin + maybe_compact_stacks (); + if !admit_new then begin + if !explored mod 4_096 = 0 then manage_memory () + end + else manage_memory (); + let item = Queue.take buckets.(bucket) in + decr queued; + incr explored; + last_depth := item.depth; + deepest := max !deepest item.depth; + if accepted_count engine item.frontier >= 2 then begin + if item.depth >= min_tokens then begin + let tokens = List.rev item.tokens_rev in + let profile = conflict_profile engine tokens in + if not (Hashtbl.mem witnesses profile) then + Hashtbl.add witnesses profile tokens + end + end + else if item.depth < max_tokens then + StringSet.iter + (fun token -> + let next = shift engine item.frontier token in + if not (IntMap.is_empty next) then begin + let branched = item.branched || derivations next >= 2 in + if branched && not item.branched then incr conflict_seeds; + let distance = + if branched then accept_distance else conflict_distance + in + let next_depth = item.depth + 1 in + let lower = frontier_lower_bound engine distance next in + if next_depth + lower <= max_tokens then + add + { + tokens_rev = token :: item.tokens_rev; + depth = next_depth; + frontier = next; + branched; + } + end) + (class_representatives engine.automaton + (possible_tokens engine item.frontier)) + end; + emit_progress false + done; + emit_progress true; + if Unix.gettimeofday () >= deadline then + stopped := Some "the timeout was reached" + else if Hashtbl.length witnesses >= max_witnesses then + stopped := Some "the witness limit was reached" + else if !dropped then + stopped := Some "the memory budget dropped part of the search space"; + ( { + witnesses = + Hashtbl.fold + (fun profile tokens result -> (profile, tokens) :: result) + witnesses []; + explored = !explored; + unique = seen.Seen_cache.inserted; + deepest = !deepest; + stopped = !stopped; + }, + !conflict_seeds ) + +let initial_partitions engine jobs max_tokens initial = + if jobs <= 1 || initial.depth = max_tokens then + ( [| [ initial ] |], + 0, + 0, + 0 ) + else begin + let split_depth = min 3 (max_tokens - initial.depth) in + let current = ref [ initial ] in + let explored = ref 0 in + let unique = ref 1 in + let conflict_seeds = ref 0 in + for _ = 1 to split_depth do + explored := !explored + List.length !current; + let seen = Hashtbl.create 1024 in + let next = ref [] in + List.iter + (fun item -> + if accepted_count engine item.frontier >= 2 then + next := item :: !next + else + StringSet.iter + (fun token -> + let frontier = shift engine item.frontier token in + if not (IntMap.is_empty frontier) then begin + let branched = item.branched || derivations frontier >= 2 in + if branched && not item.branched then incr conflict_seeds; + let item = + { + tokens_rev = token :: item.tokens_rev; + depth = item.depth + 1; + frontier; + branched; + } + in + let key = (branched, signature frontier) in + if not (Hashtbl.mem seen key) then begin + Hashtbl.add seen key (); + next := item :: !next + end + end) + (class_representatives engine.automaton + (possible_tokens engine item.frontier))) + !current; + unique := !unique + Hashtbl.length seen; + current := !next + done; + let buckets = Array.make jobs [] in + List.iter + (fun item -> + let hash = Hashtbl.hash (item.branched, signature item.frontier) land max_int in + let bucket = hash mod jobs in + buckets.(bucket) <- item :: buckets.(bucket)) + !current; + (buckets, !explored, !unique, !conflict_seeds) + end + +let parallel_unified_search engine initial ~jobs ~max_tokens ~min_tokens + ~nodes_per_depth ~timeout ~max_frontiers ~max_queue ~max_witnesses + ~soft_heap_bytes ~hard_heap_bytes conflict_distance accept_distance + temporary = + let started = Unix.gettimeofday () in + let memory_budget = + hard_heap_bytes /. 0.90 *. float_of_int (max 1 jobs) + in + let partitions, prefix_explored, prefix_unique, prefix_seeds = + initial_partitions engine (max 1 jobs) max_tokens initial + in + let prefix_progress = + { + depth = initial.depth; + ambiguities = []; + explored = prefix_explored; + unique = prefix_unique; + rss_bytes = 0.; + } + in + if Array.length partitions = 1 then + let show progress = + render_progress ~started ~max_tokens ~memory_budget + [ (false, prefix_progress); (true, progress) ] + in + let outcome, seeds = + unified_search engine partitions.(0) ~max_tokens ~min_tokens + ~nodes_per_depth ~timeout ~max_frontiers ~max_queue ~max_witnesses + ~soft_heap_bytes ~hard_heap_bytes + ~show_progress:progress_is_visible ~on_progress:show conflict_distance + accept_distance + in + clear_progress (); + ( { + outcome with + explored = prefix_explored + outcome.explored; + unique = prefix_unique + outcome.unique; + }, + prefix_seeds + seeds ) + else begin + (* Do not make every child inherit avoidable garbage or an unnecessarily + sparse major heap: after fork those pages become copy-on-write overhead. *) + Gc.compact (); + (* Anything still buffered would be replayed by every child's exit. *) + flush stdout; + flush stderr; + let children = ref [] in + Array.iteri + (fun index initial -> + let output = Filename.concat temporary (Printf.sprintf "worker-%d" index) in + let progress_path = + Filename.concat temporary (Printf.sprintf "progress-%d" index) + in + match Unix.fork () with + | 0 -> + let report progress = + if progress_is_visible then write_progress progress_path progress + in + let result = + unified_search engine initial ~max_tokens ~min_tokens + ~nodes_per_depth ~timeout ~max_frontiers ~max_queue + ~max_witnesses ~soft_heap_bytes ~hard_heap_bytes + ~show_progress:progress_is_visible ~on_progress:report + conflict_distance accept_distance + in + let channel = open_out_bin output in + Marshal.to_channel channel result []; + close_out channel; + exit 0 + | pid -> children := (pid, output, progress_path) :: !children) + partitions; + let latest = Hashtbl.create (Array.length partitions) in + let worker_result pid output status = + match status with + | Unix.WEXITED 0 -> + let channel = open_in_bin output in + Fun.protect ~finally:(fun () -> close_in channel) (fun () -> + (Marshal.from_channel channel : outcome * int)) + | Unix.WEXITED code -> + failwith + (Printf.sprintf "ambiguity-search worker %d exited with status %d" + pid code) + | Unix.WSIGNALED signal -> + failwith + (Printf.sprintf "ambiguity-search worker %d was killed by signal %d" + pid signal) + | Unix.WSTOPPED signal -> + failwith + (Printf.sprintf "ambiguity-search worker %d stopped on signal %d" + pid signal) + in + let rec collect pending outcomes = + List.iter + (fun (pid, _, progress_path) -> + Option.iter + (fun progress -> Hashtbl.replace latest pid progress) + (read_progress progress_path)) + pending; + let active = Hashtbl.create (List.length pending) in + List.iter (fun (pid, _, _) -> Hashtbl.replace active pid ()) pending; + let entries = + (false, prefix_progress) + :: Hashtbl.fold + (fun pid progress result -> + (Hashtbl.mem active pid, progress) :: result) + latest [] + in + render_progress ~started ~max_tokens ~memory_budget entries; + match pending with + | [] -> List.rev outcomes + | _ -> + let remaining = ref [] in + let completed = ref [] in + List.iter + (fun ((pid, output, progress_path) as child) -> + let waited, status = + try Unix.waitpid [ Unix.WNOHANG ] pid + with Unix.Unix_error (Unix.EINTR, _, _) -> + (0, Unix.WEXITED 0) + in + if waited = 0 then remaining := child :: !remaining + else begin + let ((outcome, _) as result) = + worker_result pid output status + in + let final_progress = + Option.value (read_progress progress_path) + ~default: + { + depth = outcome.deepest; + ambiguities = + List.map + (fun (profile, tokens) -> + (profile, List.length tokens)) + outcome.witnesses; + explored = outcome.explored; + unique = outcome.unique; + rss_bytes = 0.; + } + in + Hashtbl.replace latest pid + { final_progress with rss_bytes = 0. }; + completed := result :: !completed + end) + pending; + if !completed = [] then + ignore (Unix.select [] [] [] 0.1); + collect (List.rev !remaining) (List.rev_append !completed outcomes) + in + let outcomes = collect !children [] in + clear_progress (); + let outcome, seeds = List.fold_left + (fun (combined, seeds) (outcome, worker_seeds) -> + ( { + witnesses = outcome.witnesses @ combined.witnesses; + explored = combined.explored + outcome.explored; + unique = combined.unique + outcome.unique; + deepest = max combined.deepest outcome.deepest; + stopped = + (match (combined.stopped, outcome.stopped) with + | None, None -> None + | _ -> Some "one or more workers reached a search limit"); + }, + seeds + worker_seeds )) + ( { witnesses = []; explored = 0; unique = 0; deepest = 0; stopped = None }, + 0 ) + outcomes + in + ( { + outcome with + witnesses = merge_witnesses max_witnesses [ outcome.witnesses ]; + explored = prefix_explored + outcome.explored; + unique = prefix_unique + outcome.unique; + }, + prefix_seeds + seeds ) + end + +let environment name = + match Sys.getenv_opt name with + | Some value when value <> "" -> value + | _ -> invalid_arg (name ^ " must be set") + +let environment_int name = + match Sys.getenv_opt name with + | None | Some "" -> invalid_arg (name ^ " must be set") + | Some value -> + (match int_of_string_opt value with + | Some parsed -> parsed + | None -> invalid_arg (name ^ " must be an integer")) + +let environment_float name = + match Sys.getenv_opt name with + | None | Some "" -> invalid_arg (name ^ " must be set") + | Some value -> + (try float_of_string value + with Failure _ -> invalid_arg (name ^ " must be a number")) + +let grammar = ref "" +let max_tokens = ref None +let min_tokens = ref 0 +let prefix_tokens = ref [] +let nodes_per_depth = ref None +let timeout = ref None +let max_witnesses = ref None +let check_tokens = ref [] +let prove_level = ref 0 +let dump_classes = ref false + +type memory_limits = { + max_queue : int; + max_frontiers : int; + soft_heap_bytes : float; + hard_heap_bytes : float; +} + +(* Structural estimates choose how to divide the search space between queued + work and deduplication. Actual managed-heap measurements enforce the + budget at runtime, so unexpectedly large frontiers reduce search reach + instead of causing an unbounded memory spike. Ten percent remains outside + the worker high-water marks for the coordinator, native allocations, and + short-lived copy-on-write/compaction overhead. *) +let derive_memory_limits ~memory_mb ~max_frontier_ratio ~jobs ~max_tokens = + let queue_entry_bytes = 600. +. (24. *. float_of_int max_tokens) in + let frontier_entry_bytes = 240. in + let declared_per_worker_bytes = + float_of_int memory_mb *. 1024. *. 1024. /. float_of_int jobs + in + let hard_heap_bytes = 0.90 *. declared_per_worker_bytes in + let soft_heap_bytes = 0.80 *. declared_per_worker_bytes in + let combined_entry_bytes = + queue_entry_bytes +. (max_frontier_ratio *. frontier_entry_bytes) + in + let max_queue_float = floor (hard_heap_bytes /. combined_entry_bytes) in + if max_queue_float < 1. || max_queue_float > float_of_int max_int then + invalid_arg + "AMBIGUITY_MEMORY_MB is too small or too large for AMBIGUITY_JOBS"; + let max_queue = int_of_float max_queue_float in + let max_frontiers_float = + floor (float_of_int max_queue *. max_frontier_ratio) + in + if max_frontiers_float > float_of_int max_int then + invalid_arg + "AMBIGUITY_MEMORY_MB or AMBIGUITY_MAX_FRONTIER_RATIO is too large"; + let max_frontiers = max 1 (int_of_float max_frontiers_float) in + { max_queue; max_frontiers; soft_heap_bytes; hard_heap_bytes } + +let options = + [ + ( "--max-tokens", + Arg.Int (fun value -> max_tokens := Some value), + "N maximum tokens, including EOF (required for search/prove)" ); + ( "--min-tokens", + Arg.Set_int min_tokens, + "N minimum tokens for reported witnesses, including the prefix and EOF \ + (default 0)" ); + ( "--prefix-tokens", + Arg.String (fun value -> prefix_tokens := words value), + "TOKENS consume a space-separated token prefix before searching" ); + ( "--nodes-per-depth", + Arg.Int (fun value -> nodes_per_depth := Some value), + "N queued frontiers to expand at each depth before descending; omitted \ + for breadth-first search" ); + ( "--timeout", + Arg.Float (fun value -> timeout := Some value), + "SECONDS time limit per search phase (required for search/prove)" ); + ( "--max-witnesses", + Arg.Int (fun value -> max_witnesses := Some value), + "N ambiguity families to report (required for search/prove)" ); + ( "--check-tokens", + Arg.String (fun value -> check_tokens := words value), + "TOKENS check one space-separated token sequence" ); + ( "--prove", + Arg.Set_int prove_level, + "K attempt an unambiguity proof with a top-K stack abstraction; \ + exit 0 proven unambiguous, 1 ambiguous, 3 not proven \ + (the derived dedup-frontier limit also bounds the abstract pair count)" ); + ( "--dump-terminal-classes", + Arg.Set dump_classes, + " list the terminal equivalence classes the search collapses, then exit" ); + ] + +let main () = + Arg.parse options (fun value -> grammar := value) + "ambiguity_search [options] GRAMMAR"; + if !grammar = "" then begin + Arg.usage options "ambiguity_search [options] GRAMMAR"; + exit 2 + end; + let menhir = environment "AMBIGUITY_MENHIR" in + let memory_mb = environment_int "AMBIGUITY_MEMORY_MB" in + let max_frontier_ratio = + environment_float "AMBIGUITY_MAX_FRONTIER_RATIO" + in + let jobs = environment_int "AMBIGUITY_JOBS" in + Option.iter + (fun value -> + if value < 0 then invalid_arg "--max-tokens must be at least 0") + !max_tokens; + if !min_tokens < 0 then invalid_arg "--min-tokens must be at least 0"; + Option.iter + (fun value -> + if value < 1 then invalid_arg "--nodes-per-depth must be at least 1") + !nodes_per_depth; + if memory_mb < 1 then invalid_arg "AMBIGUITY_MEMORY_MB must be at least 1"; + if + Float.is_nan max_frontier_ratio + || Float.is_infinite max_frontier_ratio + || max_frontier_ratio <= 0. + then + invalid_arg + "AMBIGUITY_MAX_FRONTIER_RATIO must be finite and greater than 0"; + Option.iter + (fun value -> + if value < 0. then invalid_arg "--timeout must be non-negative") + !timeout; + if jobs < 1 then invalid_arg "AMBIGUITY_JOBS must be at least 1"; + Option.iter + (fun value -> + if value < 1 then invalid_arg "--max-witnesses must be at least 1") + !max_witnesses; + let search_limits = + if !check_tokens <> [] || !dump_classes then None + else + let required name = function + | Some value -> value + | None -> invalid_arg (name ^ " is required for search/prove") + in + Some + ( required "--max-tokens" !max_tokens, + required "--timeout" !timeout, + required "--max-witnesses" !max_witnesses ) + in + let grammar_path = Unix.realpath !grammar in + let temporary = temporary_directory () in + Fun.protect + ~finally:(fun () -> remove_tree temporary) + (fun () -> + let terminals, aliases = parse_tokens grammar_path in + let automaton_path = + prepare_automaton ~menhir ~grammar:grammar_path + ~directory:temporary + in + let automaton = parse_automaton automaton_path terminals aliases in + let stacks = Stack_pool.create () in + let engine = + { automaton; stacks; closure_cache = Hashtbl.create 16_384 } + in + if !dump_classes then begin + let members = Hashtbl.create 64 in + Hashtbl.iter + (fun token id -> + Hashtbl.replace members id + (token :: Option.value (Hashtbl.find_opt members id) ~default:[])) + automaton.terminal_class; + let classes = + Hashtbl.fold (fun _ tokens result -> List.sort compare tokens :: result) + members [] + |> List.sort compare + in + let singletons = List.filter (fun tokens -> List.length tokens = 1) classes in + let merged = List.filter (fun tokens -> List.length tokens > 1) classes in + Printf.printf "%d terminals in %d classes (%d merged, %d singleton).\n" + (StringSet.cardinal automaton.terminals) + (List.length classes) (List.length merged) (List.length singletons); + List.iter + (fun tokens -> Printf.printf " { %s }\n" (String.concat " " tokens)) + merged; + if singletons <> [] then + Printf.printf " singletons: %s\n" + (String.concat " " (List.concat singletons)); + exit 0 + end; + if !check_tokens <> [] then begin + let frontier = + List.fold_left (shift engine) + (IntMap.singleton engine.stacks.root.id 1) + !check_tokens + in + let count = accepted_count engine frontier in + Printf.printf "Accepting derivations: %d\n" count; + exit (if count >= 2 then 1 else 0) + end; + let max_tokens, timeout, max_witnesses = Option.get search_limits in + if !min_tokens > max_tokens then + invalid_arg "--min-tokens must not exceed --max-tokens"; + let prefix_depth = List.length !prefix_tokens in + if prefix_depth > max_tokens then + invalid_arg + "--prefix-tokens must not contain more tokens than --max-tokens"; + let initial_frontier = + List.fold_left (shift engine) + (IntMap.singleton engine.stacks.root.id 1) + !prefix_tokens + in + if IntMap.is_empty initial_frontier then + invalid_arg "--prefix-tokens is not a valid grammar prefix"; + let initial = + { + tokens_rev = List.rev !prefix_tokens; + depth = prefix_depth; + frontier = initial_frontier; + branched = derivations initial_frontier >= 2; + } + in + let memory_limits = + derive_memory_limits ~memory_mb ~max_frontier_ratio ~jobs ~max_tokens + in + Printf.printf + "Memory budget: %d MiB total across %d worker(s); workers compact at %.0f MiB and stop admitting frontiers at %.0f MiB each (10%% reserved); per-worker limits are %d queued frontiers and %d retained dedup frontiers (ratio %g).\n" + memory_mb jobs + (memory_limits.soft_heap_bytes /. 1024. /. 1024.) + (memory_limits.hard_heap_bytes /. 1024. /. 1024.) + memory_limits.max_queue memory_limits.max_frontiers + max_frontier_ratio; + Printf.printf + "Search constraints: %d..%d total tokens; %d-token prefix; %s.\n" + !min_tokens max_tokens prefix_depth + (match !nodes_per_depth with + | None -> "breadth-first scheduling" + | Some 1 -> "1 node per depth" + | Some limit -> Printf.sprintf "%d nodes per depth" limit); + if !prefix_tokens <> [] then + Printf.printf "Prefix tokens: %s\n" (String.concat " " !prefix_tokens); + if !prove_level > 0 then begin + match prove engine !prove_level memory_limits.max_frontiers with + | Proven pairs -> + Printf.printf + "PROVEN UNAMBIGUOUS: no diverging pair of accepting parses \ + exists in the top-%d stack abstraction (%d abstract pairs \ + explored).\n" + !prove_level pairs; + exit 0 + | Pair_overflow pairs -> + Printf.printf + "NOT PROVEN: the abstract pair limit (%d) was reached at \ + abstraction level %d. Raise AMBIGUITY_MEMORY_MB or \ + AMBIGUITY_MAX_FRONTIER_RATIO, or lower --prove.\n" + pairs !prove_level; + exit 3 + | Abstract_candidate (tokens, pairs) -> + Printf.printf + "Abstract ambiguity candidate at level %d after %d pairs \ + (possibly spurious): %s\n" + !prove_level pairs + (String.concat " " tokens); + Printf.printf + "Attempting to concretize with the bounded search...\n\n" + end; + let conflicts = conflict_states automaton in + let conflict_distance = reverse_distances automaton conflicts in + let accept_targets = + Array.map + (fun state -> StringSet.mem "#" state.accepts) + automaton.states + in + let accept_distance = reverse_distances automaton accept_targets in + let outcome, conflict_seeds = + parallel_unified_search engine initial ~jobs ~max_tokens + ~min_tokens:!min_tokens ~nodes_per_depth:!nodes_per_depth ~timeout + ~max_frontiers:memory_limits.max_frontiers + ~max_queue:memory_limits.max_queue ~max_witnesses + ~soft_heap_bytes:memory_limits.soft_heap_bytes + ~hard_heap_bytes:memory_limits.hard_heap_bytes conflict_distance + accept_distance temporary + in + match outcome.witnesses with + | [] -> + (match outcome.stopped with + | None -> + Printf.printf + "No complete ambiguity satisfying the search constraints was found after exploring %d frontiers (%d unique).\n" + outcome.explored outcome.unique + | Some reason -> + Printf.printf + "Search stopped at depth %d because %s; no complete ambiguity was found in %d explored frontiers (%d unique).\n" + outcome.deepest reason outcome.explored outcome.unique); + if !prove_level > 0 then begin + Printf.printf + "NOT PROVEN: the abstract candidate could not be concretized \ + within the search bounds; the grammar is neither proven \ + unambiguous nor shown ambiguous. Raising --prove may remove \ + the spurious candidate.\n"; + exit 3 + end; + Printf.printf "This is a bounded result, not a proof of unambiguity.\n"; + exit 0 + | witnesses -> + Printf.printf "Found %d complete ambiguity %s.\n" + (List.length witnesses) + (if List.length witnesses = 1 then "family" else "families"); + List.iteri + (fun index (profile, witness) -> + Printf.printf "\n%d. Tokens (%d): %s\n Source: %s\n" + (index + 1) (List.length witness) (String.concat " " witness) + (render automaton witness); + Printf.printf " Conflict origins: %s\n" + (profile + |> List.map (fun (state, token) -> + Printf.sprintf "state %d on %s" state token) + |> String.concat ", ")) + witnesses; + Printf.printf "\n"; + Printf.printf "Explored %d frontiers (%d unique); %d conflict seeds.\n" + outcome.explored outcome.unique conflict_seeds; + Option.iter + (fun reason -> + Printf.printf "Search stopped because %s.\n" reason) + outcome.stopped; + exit 1) + +let () = + try main () + with + | Failure message | Invalid_argument message | Sys_error message + | Unix.Unix_error (_, _, message) -> + clear_progress (); + prerr_endline ("error: " ^ message); + exit 2 diff --git a/tools/dune b/tools/dune new file mode 100644 index 00000000..de423c2b --- /dev/null +++ b/tools/dune @@ -0,0 +1,3 @@ +(executable + (name ambiguity_search) + (libraries unix str)) diff --git a/tools/syntax_experiment.py b/tools/syntax_experiment.py new file mode 100644 index 00000000..b927cb75 --- /dev/null +++ b/tools/syntax_experiment.py @@ -0,0 +1,1140 @@ +#!/usr/bin/env python3 +"""Compare small, controlled syntax changes with the ambiguity search. + +Each variant is an explicit composition of named grammar transformations. The +harness emits temporary Menhir grammars, replays known ambiguity witnesses, runs +the bounded complete-ambiguity search, and writes JSON and Markdown reports. + +The result is experimental evidence, not a proof that a grammar is unambiguous. +""" + +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import asdict, dataclass, replace +import json +import math +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys +import tempfile +import time +from typing import Callable, Iterable + + +Transform = Callable[[str], str] + + +@dataclass(frozen=True) +class Variant: + name: str + description: str + transforms: tuple[str, ...] + edit_cost: int + + +@dataclass(frozen=True) +class Spelling: + """How a variant spells the surface syntax the known cases depend on. + + Call and grouping delimiters are (opening, closing) token-name pairs; a + ``None`` group means the variant removed general grouping. Separators and + terminators are token sequences placed between or after statements and + after top-level declarations. + """ + + group: tuple[tuple[str, ...], tuple[str, ...]] | None + named_call: tuple[tuple[str, ...], tuple[str, ...]] + computed_call: tuple[tuple[str, ...], tuple[str, ...]] + computed_call_statement: bool + stat_separator: tuple[str, ...] + stat_terminator: tuple[str, ...] + decl_terminator: tuple[str, ...] + + +BASELINE_SPELLING = Spelling( + group=(("LPAREN",), ("RPAREN",)), + named_call=(("LPAREN",), ("RPAREN",)), + computed_call=(("LPAREN",), ("RPAREN",)), + computed_call_statement=True, + stat_separator=(), + stat_terminator=(), + decl_terminator=(), +) + + +@dataclass(frozen=True) +class KnownCase: + name: str + description: str + spell: Callable[[Spelling], list[str] | None] + + +@dataclass +class CaseResult: + name: str + derivations: int | None + seconds: float + error: str | None = None + tokens: str | None = None + expressible: bool = True + + +@dataclass +class SearchResult: + families: int | None + explored: int | None + unique: int | None + conflict_seeds: int | None + deepest: int | None + stopped: str | None + sources: list[str] + seconds: float + error: str | None = None + + +@dataclass +class VariantResult: + name: str + description: str + transforms: list[str] + edit_cost: int + known_cases: list[CaseResult] + search: SearchResult + rejected_known: int + ambiguous_known: int + pareto: bool = False + + +def spelled_statements(spelling: Spelling, statements: list[list[str]]) -> list[str]: + result: list[str] = [] + for index, statement in enumerate(statements): + if index and not spelling.stat_terminator: + result.extend(spelling.stat_separator) + result.extend(statement) + result.extend(spelling.stat_terminator) + return result + + +def spelled_constructor_decl(spelling: Spelling, body: list[str]) -> list[str]: + """A `Main() { ... }` constructor declaration; its parens are decl syntax, + untouched by the call transforms, so they always stay LPAREN/RPAREN.""" + return [ + "UIDENT", "LPAREN", "RPAREN", "LCURLY", + *body, + "RCURLY", *spelling.decl_terminator, "EOF", + ] + + +def spell_nullable_array_binding(spelling: Spelling) -> list[str] | None: + return [ + "ALIAS", "UIDENT", "EQUAL", "UIDENT", "QSTNMARK", "UIDENT", + "LBRACKET", "RBRACKET", "LBRACKET", "RBRACKET", + *spelling.decl_terminator, "EOF", + ] + + +def spell_adjacent_computed_call(spelling: Spelling) -> list[str] | None: + if spelling.group is None or not spelling.computed_call_statement: + return None + group_open, group_close = spelling.group + named_open, named_close = spelling.named_call + computed_open, computed_close = spelling.computed_call + named = ["LIDENT", *named_open, *named_close] + computed = [*group_open, "LIDENT", *group_close, *computed_open, *computed_close] + return spelled_constructor_decl( + spelling, spelled_statements(spelling, [named, computed]) + ) + + +def spell_abort_handler_attachment(spelling: Spelling) -> list[str] | None: + named_open, named_close = spelling.named_call + statement = [ + "LIDENT", *named_open, + "TILDE", "TILDE", "LIDENT", "QSTNQSTN", "LIDENT", + *named_close, + ] + return spelled_constructor_decl( + spelling, spelled_statements(spelling, [statement]) + ) + + +def spell_named_call_statement(spelling: Spelling) -> list[str] | None: + named_open, named_close = spelling.named_call + statement = ["LIDENT", *named_open, "STRING", *named_close] + return spelled_constructor_decl( + spelling, spelled_statements(spelling, [statement]) + ) + + +KNOWN_CASES = ( + KnownCase( + "nullable-array-binding", + "T ? U[] must bind the array to U, not the nullable result", + spell_nullable_array_binding, + ), + KnownCase( + "adjacent-computed-call", + "f() followed by (x)() must not have both one- and two-statement parses", + spell_adjacent_computed_call, + ), + KnownCase( + "abort-handler-attachment", + "the abort handler must attach to the outer expression", + spell_abort_handler_attachment, + ), + KnownCase( + "named-call-statement", + 'an ordinary named call statement such as print("hello") must keep exactly one parse', + spell_named_call_statement, + ), +) + + +PRIMARY_GROUP = ' | "(" e=expr ")" { Nodes.Expr.Parenthized e }\n' + +VERB_CALL = '''verb_call: + | receiver=app part=ioption(meth_part) "(" args=separated_list(COMMA, expr) ")" abort_handle=ioption(abort_handle) %prec LPAREN { + match part with + | None -> Nodes.Verb_call.Func { callee = receiver; args; abort_handle } + | Some (is_mut, name) -> + Nodes.Verb_call.Meth { this = receiver; callee = name; args; abort_handle; is_mut } + } + | name_type=name_type "(" args=separated_list(COMMA, expr) ")" abort_handle=ioption(abort_handle) %prec LPAREN { + Nodes.Verb_call.Constructor { name_type; args; abort_handle } + } +''' + +STAT_CALL = ''' | verb_call=verb_call { + Nodes.Stat.VerbCall verb_call + } +''' + + +def replace_once(source: str, old: str, new: str, transform: str) -> str: + count = source.count(old) + if count != 1: + raise ValueError( + f"{transform}: expected one grammar anchor, found {count}: {old[:70]!r}" + ) + return source.replace(old, new, 1) + + +def replace_statement_lists(source: str, helper: str, transform: str) -> str: + source = replace_once( + source, + " | decls=list(decl) EOF", + " | decls=decl_sequence EOF", + transform, + ) + if "list(stat)" not in source: + raise ValueError(f"{transform}: no statement-list anchors found") + source = source.replace("list(stat)", "stat_sequence") + return replace_once(source, "\nbody:\n", f"\n{helper}\nbody:\n", transform) + + +def semicolon_separated(source: str) -> str: + helper = '''decl_sequence: + | values=separated_list(";", decl) ioption(";") { values } + +stat_sequence: + | values=separated_list(";", stat) ioption(";") { values } +''' + return replace_statement_lists(source, helper, "semicolon-separated") + + +def semicolon_terminated(source: str) -> str: + helper = '''decl_sequence: + | values=list(terminated(decl, ";")) { values } + +stat_sequence: + | values=list(terminated(stat, ";")) { values } +''' + return replace_statement_lists(source, helper, "semicolon-terminated") + + +def newline_separated(source: str) -> str: + source = replace_once( + source, + '%token SEMICOLON ";"\n', + '%token SEMICOLON ";"\n%token NEWLINE ""\n', + "newline-separated", + ) + helper = '''decl_sequence: + | { [] } + | NEWLINE { [] } + | boption(NEWLINE) first=decl rest=list(preceded(NEWLINE, decl)) boption(NEWLINE) { + first :: rest + } + +stat_sequence: + | { [] } + | NEWLINE { [] } + | boption(NEWLINE) first=stat rest=list(preceded(NEWLINE, stat)) boption(NEWLINE) { + first :: rest + } +''' + return replace_statement_lists(source, helper, "newline-separated") + + +def named_statement_calls(source: str) -> str: + helper = '''statement_verb_call: + | callee=name_expr "(" args=separated_list(COMMA, expr) ")" abort_handle=ioption(abort_handle) { + Nodes.Verb_call.Func { + callee = Nodes.Expr.NameExpr callee; + args; + abort_handle; + } + } + | receiver=app is_mut=meth_marker callee=name_expr + "(" args=separated_list(COMMA, expr) ")" abort_handle=ioption(abort_handle) { + Nodes.Verb_call.Meth { + this = receiver; + callee = Nodes.Expr.NameExpr callee; + args; + abort_handle; + is_mut; + } + } + | name_type=name_type "(" args=separated_list(COMMA, expr) ")" abort_handle=ioption(abort_handle) { + Nodes.Verb_call.Constructor { name_type; args; abort_handle } + } + +''' + source = replace_once(source, "\nstat:\n", f"\n{helper}stat:\n", "named-statement-calls") + return replace_once( + source, + STAT_CALL, + ''' | verb_call=statement_verb_call { + Nodes.Stat.VerbCall verb_call + } +''', + "named-statement-calls", + ) + + +def bracket_grouping(source: str) -> str: + return replace_once( + source, + PRIMARY_GROUP, + ' | "[" e=expr "]" { Nodes.Expr.Parenthized e }\n', + "bracket-grouping", + ) + + +def brace_grouping(source: str) -> str: + return replace_once( + source, + PRIMARY_GROUP, + ' | "{" e=expr "}" { Nodes.Expr.Parenthized e }\n', + "brace-grouping", + ) + + +def keyword_grouping(source: str) -> str: + source = replace_once( + source, + '%token LPAREN "("\n', + '%token GROUP "group"\n%token LPAREN "("\n', + "keyword-grouping", + ) + return replace_once( + source, + PRIMARY_GROUP, + ' | GROUP "(" e=expr ")" { Nodes.Expr.Parenthized e }\n', + "keyword-grouping", + ) + + +def no_grouping(source: str) -> str: + return replace_once(source, PRIMARY_GROUP, "", "no-grouping") + + +def transform_all_calls(source: str, kind: str) -> str: + if kind == "bracket": + replacement = VERB_CALL.replace( + '"(" args=separated_list(COMMA, expr) ")"', + '"[" args=separated_list(COMMA, expr) "]"', + ) + elif kind in {"marked", "dotted"}: + marker = '"@"' if kind == "marked" else '"."' + replacement = VERB_CALL.replace( + '"(" args=separated_list(COMMA, expr) ")"', + f'{marker} "(" args=separated_list(COMMA, expr) ")"', + ) + else: + raise AssertionError(kind) + return replace_once(source, VERB_CALL, replacement, f"{kind}-calls") + + +def bracket_calls(source: str) -> str: + return transform_all_calls(source, "bracket") + + +def marked_calls(source: str) -> str: + return transform_all_calls(source, "marked") + + +def dotted_calls(source: str) -> str: + return transform_all_calls(source, "dotted") + + +def computed_calls(source: str, kind: str) -> str: + if kind == "marked": + opening, closing = '"@" "("', '")"' + elif kind == "bracket": + opening, closing = '"["', '"]"' + else: + raise AssertionError(kind) + replacement = f'''verb_call: + | callee=name_expr "(" args=separated_list(COMMA, expr) ")" abort_handle=ioption(abort_handle) {{ + Nodes.Verb_call.Func {{ + callee = Nodes.Expr.NameExpr callee; + args; + abort_handle; + }} + }} + | receiver=app {opening} args=separated_list(COMMA, expr) {closing} abort_handle=ioption(abort_handle) {{ + Nodes.Verb_call.Func {{ callee = receiver; args; abort_handle }} + }} + | receiver=app part=meth_part "(" args=separated_list(COMMA, expr) ")" abort_handle=ioption(abort_handle) {{ + let (is_mut, name) = part in + Nodes.Verb_call.Meth {{ this = receiver; callee = name; args; abort_handle; is_mut }} + }} + | name_type=name_type "(" args=separated_list(COMMA, expr) ")" abort_handle=ioption(abort_handle) {{ + Nodes.Verb_call.Constructor {{ name_type; args; abort_handle }} + }} +''' + return replace_once(source, VERB_CALL, replacement, f"{kind}-computed-calls") + + +OP_HANDLE_SLOTS = ( + ("comparison_op", "EQEQ"), + ("additive_op", "PLUS"), + ("multiplicative_op", "STAR"), +) + +HANDLED_EXPR = '''handled_expr: + | e=expr { e } + | e=expr abort_handle=abort_handle { + Nodes.Expr.WithAbortHandle { value = e; abort_handle } + } + +''' + + +def anchored_abort_handles(source: str) -> str: + """Abort handlers may only attach at delimited boundaries: statement + calls, declaration values, return/resolve/abort values, call arguments, + and grouping parentheses — never inside an undelimited expression.""" + name = "anchored-abort-handles" + for rule, prec in OP_HANDLE_SLOTS: + source = replace_once( + source, + f" | left=expr op={rule} right=expr abort_handle=ioption(abort_handle) %prec {prec} {{\n" + " Nodes.Expr.VerbCall (Nodes.Verb_call.Op { op; left; right; abort_handle })\n" + " }\n", + f" | left=expr op={rule} right=expr %prec {prec} {{\n" + " Nodes.Expr.VerbCall (Nodes.Verb_call.Op { op; left; right; abort_handle = None })\n" + " }\n", + name, + ) + source = replace_once( + source, + ' | "~" value=expr abort_handle=ioption(abort_handle) %prec TILDE {\n' + " Nodes.Expr.VerbCall (Nodes.Verb_call.Flip { value; abort_handle })\n" + " }\n", + ' | "~" value=expr %prec TILDE {\n' + " Nodes.Expr.VerbCall (Nodes.Verb_call.Flip { value; abort_handle = None })\n" + " }\n", + name, + ) + source = replace_once( + source, + VERB_CALL, + '''verb_call: + | receiver=app part=ioption(meth_part) "(" args=separated_list(COMMA, handled_expr) ")" { + match part with + | None -> Nodes.Verb_call.Func { callee = receiver; args; abort_handle = None } + | Some (is_mut, name) -> + Nodes.Verb_call.Meth { this = receiver; callee = name; args; abort_handle = None; is_mut } + } + | name_type=name_type "(" args=separated_list(COMMA, handled_expr) ")" { + Nodes.Verb_call.Constructor { name_type; args; abort_handle = None } + } +''', + name, + ) + source = replace_once( + source, + STAT_CALL, + ''' | verb_call=verb_call abort_handle=ioption(abort_handle) { + Nodes.Stat.VerbCall (Nodes.with_statement_handle verb_call abort_handle) + } +''', + name, + ) + source = replace_once( + source, + ' | name=LIDENT type_=type_expr "=" value=expr {\n', + ' | name=LIDENT type_=type_expr "=" value=handled_expr {\n', + name, + ) + source = replace_once( + source, + ' | name=LIDENT constructor=name_type "(" args=separated_list(COMMA, expr) ")" {\n', + ' | name=LIDENT constructor=name_type "(" args=separated_list(COMMA, handled_expr) ")" {\n', + name, + ) + for keyword in ("ABORT", "RETURN", "RESOLVE"): + source = replace_once( + source, + f" | {keyword} value=expr {{\n", + f" | {keyword} value=handled_expr {{\n", + name, + ) + source = replace_once( + source, + PRIMARY_GROUP, + ' | "(" e=handled_expr ")" { Nodes.Expr.Parenthized e }\n', + name, + ) + return replace_once(source, "\nexpr:\n", f"\n{HANDLED_EXPR}expr:\n", name) + + +def marked_computed_calls(source: str) -> str: + return computed_calls(source, "marked") + + +def bracket_computed_calls(source: str) -> str: + return computed_calls(source, "bracket") + + +TRANSFORMS: dict[str, Transform] = { + "semicolon-separated": semicolon_separated, + "semicolon-terminated": semicolon_terminated, + "newline-separated": newline_separated, + "named-statement-calls": named_statement_calls, + "bracket-grouping": bracket_grouping, + "brace-grouping": brace_grouping, + "keyword-grouping": keyword_grouping, + "no-grouping": no_grouping, + "bracket-calls": bracket_calls, + "marked-calls": marked_calls, + "dotted-calls": dotted_calls, + "marked-computed-calls": marked_computed_calls, + "bracket-computed-calls": bracket_computed_calls, + "anchored-abort-handles": anchored_abort_handles, +} + + +BRACKET_CALL = (("LBRACKET",), ("RBRACKET",)) +MARKED_CALL = (("AT", "LPAREN"), ("RPAREN",)) +DOTTED_CALL = (("DOT", "LPAREN"), ("RPAREN",)) + +SPELLINGS: dict[str, Callable[[Spelling], Spelling]] = { + "semicolon-separated": lambda s: replace(s, stat_separator=("SEMICOLON",)), + "semicolon-terminated": lambda s: replace( + s, stat_terminator=("SEMICOLON",), decl_terminator=("SEMICOLON",) + ), + "newline-separated": lambda s: replace(s, stat_separator=("NEWLINE",)), + "named-statement-calls": lambda s: replace(s, computed_call_statement=False), + "bracket-grouping": lambda s: replace(s, group=(("LBRACKET",), ("RBRACKET",))), + "brace-grouping": lambda s: replace(s, group=(("LCURLY",), ("RCURLY",))), + "keyword-grouping": lambda s: replace(s, group=(("GROUP", "LPAREN"), ("RPAREN",))), + "no-grouping": lambda s: replace(s, group=None), + "bracket-calls": lambda s: replace( + s, named_call=BRACKET_CALL, computed_call=BRACKET_CALL + ), + "marked-calls": lambda s: replace(s, named_call=MARKED_CALL, computed_call=MARKED_CALL), + "dotted-calls": lambda s: replace(s, named_call=DOTTED_CALL, computed_call=DOTTED_CALL), + "marked-computed-calls": lambda s: replace(s, computed_call=MARKED_CALL), + "bracket-computed-calls": lambda s: replace(s, computed_call=BRACKET_CALL), + # The witnesses spell their abort handlers at positions that stay legal + # (call arguments and declaration values), so the spelling is unchanged. + "anchored-abort-handles": lambda s: s, +} + + +def variant_spelling(variant: Variant) -> Spelling: + spelling = BASELINE_SPELLING + for name in variant.transforms: + spelling = SPELLINGS[name](spelling) + return spelling + + +def case_tokens(case: KnownCase, variant: Variant) -> str | None: + tokens = case.spell(variant_spelling(variant)) + return None if tokens is None else " ".join(tokens) + + +VARIANTS = ( + Variant("baseline", "Current Zane syntax", (), 0), + Variant( + "semicolon-separated", + "Require semicolons between adjacent declarations and statements", + ("semicolon-separated",), + 1, + ), + Variant( + "semicolon-terminated", + "Require every declaration and statement to end in a semicolon", + ("semicolon-terminated",), + 1, + ), + Variant( + "named-statement-calls", + "Only direct named functions, named methods, and constructors may be call statements", + ("named-statement-calls",), + 1, + ), + Variant( + "semicolons-and-named-statements", + "Require separators and restrict call statements to statically named calls", + ("semicolon-separated", "named-statement-calls"), + 2, + ), + Variant( + "terminated-and-named-statements", + "Require terminators and restrict call statements to statically named calls", + ("semicolon-terminated", "named-statement-calls"), + 2, + ), + Variant( + "newline-separated", + "Make newlines grammatical statement separators", + ("newline-separated",), + 2, + ), + Variant( + "newline-and-named-statements", + "Use significant newlines and statically named call statements", + ("newline-separated", "named-statement-calls"), + 3, + ), + Variant("bracket-grouping", "Use [expression] for grouping", ("bracket-grouping",), 1), + Variant("brace-grouping", "Use {expression} for grouping", ("brace-grouping",), 1), + Variant( + "keyword-grouping", + "Use group(expression) for explicit grouping", + ("keyword-grouping",), + 1, + ), + Variant("no-grouping", "Remove general expression grouping", ("no-grouping",), 1), + Variant("bracket-calls", "Use receiver[arguments] for every call", ("bracket-calls",), 1), + Variant("marked-calls", "Use receiver@(arguments) for every call", ("marked-calls",), 1), + Variant("dotted-calls", "Use receiver.(arguments) for every call", ("dotted-calls",), 1), + Variant( + "marked-computed-calls", + "Keep f(args), but require value@(args) for computed callees", + ("marked-computed-calls",), + 2, + ), + Variant( + "bracket-computed-calls", + "Keep f(args), but require value[args] for computed callees", + ("bracket-computed-calls",), + 2, + ), + Variant( + "semicolons-and-marked-computed-calls", + "Combine statement separators with marked computed invocation", + ("semicolon-separated", "marked-computed-calls"), + 3, + ), + Variant( + "named-statements-and-marked-computed-calls", + "Mark computed invocation and forbid it as a standalone statement", + ("named-statement-calls", "marked-computed-calls"), + 3, + ), + Variant( + "bracket-grouping-and-semicolons", + "Combine bracket grouping with statement separators", + ("bracket-grouping", "semicolon-separated"), + 2, + ), + Variant( + "bracket-grouping-and-named-statements", + "Combine bracket grouping with named-only call statements", + ("bracket-grouping", "named-statement-calls"), + 2, + ), + Variant( + "bracket-grouping-semicolons-and-named-statements", + "Combine bracket grouping, separators, and named-only call statements", + ("bracket-grouping", "semicolon-separated", "named-statement-calls"), + 3, + ), + Variant( + "keyword-grouping-and-semicolons", + "Combine keyword grouping with statement separators", + ("keyword-grouping", "semicolon-separated"), + 2, + ), + Variant( + "keyword-grouping-and-named-statements", + "Combine keyword grouping with named-only call statements", + ("keyword-grouping", "named-statement-calls"), + 2, + ), + Variant( + "anchored-abort-handles", + "Attach abort handlers only at delimited boundaries, not inside expressions", + ("anchored-abort-handles",), + 2, + ), + Variant( + "anchored-handles-and-semicolons", + "Combine anchored abort handlers with statement separators", + ("anchored-abort-handles", "semicolon-separated"), + 3, + ), +) + + +DERIVATIONS_RE = re.compile(r"Accepting derivations: (\d+)") +FAMILIES_RE = re.compile(r"Found (\d+) complete ambiguity") +EXPLORED_RE = re.compile(r"Explored (\d+) frontiers \((\d+) unique\); (\d+) conflict seeds") +NO_WITNESS_RE = re.compile( + r"(?:after exploring|found in) (\d+) (?:explored )?frontiers \((\d+) unique\)" +) +DEPTH_RE = re.compile(r"Search stopped at depth (\d+)") +STOPPED_RE = re.compile(r"Search stopped (?:because|at depth \d+ because) ([^.;]+)") +SOURCE_RE = re.compile(r"^\s*Source: (.*)$", re.MULTILINE) + + +def apply_variant(source: str, variant: Variant) -> str: + result = source + for name in variant.transforms: + result = TRANSFORMS[name](result) + header = ( + "(* Generated by tools/syntax_experiment.py.\n" + f" Variant: {variant.name}\n" + f" Transformations: {', '.join(variant.transforms) or 'none'} *)\n" + ) + return header + result + + +def run_process( + command: list[str], env: dict[str, str] | None = None +) -> tuple[int, str, str, float]: + started = time.monotonic() + completed = subprocess.run(command, text=True, capture_output=True, env=env) + return completed.returncode, completed.stdout, completed.stderr, time.monotonic() - started + + +def base_command(args: argparse.Namespace, grammar: Path) -> list[str]: + return [*shlex.split(args.search_command), str(grammar)] + + +def check_case(args: argparse.Namespace, grammar: Path, case: KnownCase, variant: Variant) -> CaseResult: + tokens = case_tokens(case, variant) + if tokens is None: + return CaseResult(case.name, None, 0.0, expressible=False) + command = [*base_command(args, grammar), "--check-tokens", tokens] + code, stdout, stderr, seconds = run_process(command) + match = DERIVATIONS_RE.search(stdout) + if code not in {0, 1} or match is None: + message = (stderr or stdout or f"search exited with status {code}").strip() + return CaseResult(case.name, None, seconds, message, tokens) + return CaseResult(case.name, int(match.group(1)), seconds, tokens=tokens) + + +def search_variant(args: argparse.Namespace, grammar: Path) -> SearchResult: + # Several variants may run concurrently. Give each search an equal share + # so AMBIGUITY_MEMORY_MB remains a total budget for the whole command. + search_memory_mb = args.memory_mb // args.concurrent_searches + environment = os.environ.copy() + environment.update( + { + "AMBIGUITY_MEMORY_MB": str(search_memory_mb), + "AMBIGUITY_MAX_FRONTIER_RATIO": str(args.max_frontier_ratio), + "AMBIGUITY_JOBS": str(args.search_jobs), + "AMBIGUITY_MENHIR": args.menhir, + } + ) + command = [ + *base_command(args, grammar), + "--max-tokens", + str(args.max_tokens), + "--timeout", + str(args.timeout), + "--max-witnesses", + str(args.max_witnesses), + ] + code, stdout, stderr, seconds = run_process(command, env=environment) + if code not in {0, 1}: + message = (stderr or stdout or f"search exited with status {code}").strip() + return SearchResult(None, None, None, None, None, None, [], seconds, message) + + family_match = FAMILIES_RE.search(stdout) + families = int(family_match.group(1)) if family_match else 0 + explored_match = EXPLORED_RE.search(stdout) + no_witness_match = NO_WITNESS_RE.search(stdout) + if explored_match: + explored, unique, seeds = map(int, explored_match.groups()) + elif no_witness_match: + explored, unique = map(int, no_witness_match.groups()) + seeds = None + else: + explored = unique = seeds = None + depth_match = DEPTH_RE.search(stdout) + stopped_match = STOPPED_RE.search(stdout) + return SearchResult( + families, + explored, + unique, + seeds, + int(depth_match.group(1)) if depth_match else None, + stopped_match.group(1) if stopped_match else None, + SOURCE_RE.findall(stdout)[:5], + seconds, + ) + + +def evaluate_variant( + args: argparse.Namespace, source: str, variant: Variant, directory: Path +) -> VariantResult: + grammar = directory / f"{variant.name}.mly" + try: + grammar.write_text(apply_variant(source, variant), encoding="utf-8") + except ValueError as error: + failed = SearchResult(None, None, None, None, None, None, [], 0.0, str(error)) + return VariantResult( + variant.name, + variant.description, + list(variant.transforms), + variant.edit_cost, + [], + failed, + 0, + 0, + ) + + if args.emit_only: + empty = SearchResult(None, None, None, None, None, None, [], 0.0) + return VariantResult( + variant.name, + variant.description, + list(variant.transforms), + variant.edit_cost, + [], + empty, + 0, + 0, + ) + + cases = ( + [] + if args.skip_known + else [check_case(args, grammar, case, variant) for case in KNOWN_CASES] + ) + search = search_variant(args, grammar) + rejected = sum(not case.expressible or case.derivations == 0 for case in cases) + ambiguous = sum( + case.derivations is not None and case.derivations >= 2 for case in cases + ) + return VariantResult( + variant.name, + variant.description, + list(variant.transforms), + variant.edit_cost, + cases, + search, + rejected, + ambiguous, + ) + + +def metric(result: VariantResult) -> tuple[int, int, int, int, int]: + failed = int(result.search.error is not None or any(case.error for case in result.known_cases)) + uncertain_clean = int(result.search.families == 0 and result.search.stopped is not None) + families = result.search.families if result.search.families is not None else 10**9 + return ( + failed, + result.rejected_known, + result.ambiguous_known, + uncertain_clean, + families + result.edit_cost, + ) + + +def mark_pareto(results: list[VariantResult]) -> None: + def objectives(result: VariantResult) -> tuple[int, int, int, int, int, int]: + failed, rejected, ambiguous, uncertain, _ = metric(result) + families = result.search.families if result.search.families is not None else 10**9 + return failed, rejected, ambiguous, uncertain, families, result.edit_cost + + for candidate in results: + values = objectives(candidate) + candidate.pareto = not any( + other is not candidate + and all(left <= right for left, right in zip(objectives(other), values)) + and any(left < right for left, right in zip(objectives(other), values)) + for other in results + ) + + +def render_markdown(args: argparse.Namespace, results: list[VariantResult]) -> str: + lines = [ + "# Zane syntax experiment report", + "", + ( + f"Bounds: {args.max_tokens} tokens, {args.timeout:g}s, " + f"{args.memory_mb:,} MiB total memory, frontier ratio " + f"{args.max_frontier_ratio:g}, {args.max_witnesses} witnesses per variant." + ), + "", + "A Pareto mark means no tested candidate was at least as good on every measured axis. " + "An interrupted zero-witness search is treated as uncertain, not clean.", + "", + "Known witnesses are respelled in each variant's own syntax before checking, so " + "a rejection means the intended program genuinely cannot be parsed, not that an " + "old spelling became illegal. A case a variant cannot express at all counts as " + "rejected and is labeled explicitly.", + "", + "| Pareto | Variant | Edit cost | Rejected known | Ambiguous known | Families | Search stop | Seconds |", + "|---:|---|---:|---:|---:|---:|---|---:|", + ] + for result in sorted(results, key=metric): + search = result.search + families = "error" if search.families is None else str(search.families) + stop = search.error or search.stopped or "bound completed" + stop = stop.replace("|", "\\|").replace("\n", " ")[:100] + lines.append( + f"| {'✓' if result.pareto else ''} | `{result.name}` | {result.edit_cost} | " + f"{result.rejected_known} | {result.ambiguous_known} | {families} | {stop} | " + f"{search.seconds:.2f} |" + ) + + lines.extend(["", "## Candidate details", ""]) + descriptions = {case.name: case.description for case in KNOWN_CASES} + for result in sorted(results, key=metric): + lines.extend( + [ + f"### {result.name}", + "", + result.description, + "", + f"Transforms: `{', '.join(result.transforms) or 'none'}`", + "", + ] + ) + if result.known_cases: + lines.append("Known cases:") + lines.append("") + for case in result.known_cases: + if not case.expressible: + lines.append( + f"- `{case.name}`: not expressible in this variant — " + f"{descriptions[case.name]}" + ) + continue + value = "error" if case.derivations is None else str(case.derivations) + lines.append( + f"- `{case.name}`: {value} accepting derivations — {descriptions[case.name]}" + ) + lines.append("") + if result.search.sources: + lines.append("Shortest reported examples:") + lines.append("") + lines.extend(f"- `{source}`" for source in result.search.sources) + lines.append("") + if result.search.error: + lines.extend(["Error:", "", f"```text\n{result.search.error}\n```", ""]) + return "\n".join(lines) + + +def select_variants(names: list[str] | None) -> list[Variant]: + by_name = {variant.name: variant for variant in VARIANTS} + if not names: + return list(VARIANTS) + unknown = sorted(set(names) - by_name.keys()) + if unknown: + raise ValueError(f"unknown variants: {', '.join(unknown)}") + return [by_name[name] for name in names] + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser(description=__doc__) + result.add_argument("grammar", nargs="?", type=Path, default=Path("lib/cst/parser.mly")) + result.add_argument("--variant", action="append", help="variant to run; repeatable") + result.add_argument("--list", action="store_true", help="list variants and exit") + result.add_argument("--emit-only", action="store_true", help="only materialize grammars") + result.add_argument("--emit-dir", type=Path, help="keep generated grammars in this directory") + result.add_argument( + "--search-command", + default="_build/default/tools/ambiguity_search.exe", + help="command prefix used to invoke the ambiguity search; the default" + " expects a prior `dune build tools/ambiguity_search.exe`", + ) + result.add_argument("--max-tokens", type=int) + result.add_argument("--timeout", type=float) + result.add_argument("--max-witnesses", type=int) + result.add_argument("--skip-known", action="store_true") + result.add_argument( + "--output", + type=Path, + default=Path("_build/syntax-experiment/report"), + help="report path without extension", + ) + return result + + +def load_machine_config(args: argparse.Namespace, cli: argparse.ArgumentParser) -> None: + names = ( + "AMBIGUITY_MEMORY_MB", + "AMBIGUITY_MAX_FRONTIER_RATIO", + "AMBIGUITY_JOBS", + "AMBIGUITY_MENHIR", + ) + missing = [name for name in names if not os.environ.get(name)] + if missing: + cli.error(f"missing machine configuration: {', '.join(missing)}") + args.menhir = os.environ["AMBIGUITY_MENHIR"] + try: + args.memory_mb = int(os.environ["AMBIGUITY_MEMORY_MB"]) + args.max_frontier_ratio = float(os.environ["AMBIGUITY_MAX_FRONTIER_RATIO"]) + args.jobs = int(os.environ["AMBIGUITY_JOBS"]) + except ValueError: + cli.error( + "invalid AMBIGUITY_MEMORY_MB, AMBIGUITY_MAX_FRONTIER_RATIO, " + "or AMBIGUITY_JOBS" + ) + + +def validate_args(args: argparse.Namespace, cli: argparse.ArgumentParser) -> None: + if not args.grammar.is_file(): + cli.error(f"grammar does not exist: {args.grammar}") + search_words = shlex.split(args.search_command) + if not search_words: + cli.error("--search-command must not be empty") + executable = search_words[0] + if not args.emit_only and os.path.dirname(executable) and not Path(executable).is_file(): + cli.error( + f"search executable does not exist: {executable};" + " run `dune build tools/ambiguity_search.exe` first" + ) + missing = [ + name + for name, value in ( + ("--max-tokens", args.max_tokens), + ("--timeout", args.timeout), + ("--max-witnesses", args.max_witnesses), + ) + if value is None + ] + if missing: + cli.error(f"required for experiments: {', '.join(missing)}") + if args.max_tokens < 0: + cli.error("--max-tokens must be at least 0") + if args.timeout < 0: + cli.error("--timeout must be non-negative") + if args.memory_mb < 1 or args.max_witnesses < 1: + cli.error("AMBIGUITY_MEMORY_MB and the witness limit must be at least 1") + if not math.isfinite(args.max_frontier_ratio) or args.max_frontier_ratio <= 0: + cli.error("AMBIGUITY_MAX_FRONTIER_RATIO must be finite and greater than 0") + if args.jobs < 1: + cli.error("AMBIGUITY_JOBS must be at least 1") + + +def main() -> int: + cli = parser() + args = cli.parse_args() + if args.list: + for variant in VARIANTS: + transforms = ", ".join(variant.transforms) or "baseline" + print(f"{variant.name:52} cost={variant.edit_cost} {transforms}") + print(f" {variant.description}") + return 0 + load_machine_config(args, cli) + validate_args(args, cli) + try: + variants = select_variants(args.variant) + except ValueError as error: + cli.error(str(error)) + args.concurrent_searches = min(args.jobs, len(variants)) + args.search_jobs = max(1, args.jobs // args.concurrent_searches) + if args.memory_mb < args.concurrent_searches: + cli.error( + "AMBIGUITY_MEMORY_MB must provide at least 1 MiB per concurrent search" + ) + + source = args.grammar.read_text(encoding="utf-8") + temporary: tempfile.TemporaryDirectory[str] | None = None + if args.emit_dir: + directory = args.emit_dir + directory.mkdir(parents=True, exist_ok=True) + else: + temporary = tempfile.TemporaryDirectory(prefix="zane-syntax-") + directory = Path(temporary.name) + + try: + results: list[VariantResult] = [] + with ThreadPoolExecutor(max_workers=args.jobs) as executor: + futures = { + executor.submit(evaluate_variant, args, source, variant, directory): variant + for variant in variants + } + for future in as_completed(futures): + variant = futures[future] + try: + result = future.result() + except Exception as error: # keep other experiments running + search = SearchResult( + None, None, None, None, None, None, [], 0.0, str(error) + ) + result = VariantResult( + variant.name, + variant.description, + list(variant.transforms), + variant.edit_cost, + [], + search, + 0, + 0, + ) + results.append(result) + status = "generated" if args.emit_only else ( + f"{result.search.families} families" + if result.search.error is None + else "error" + ) + print(f"[{len(results):02d}/{len(variants):02d}] {variant.name}: {status}", file=sys.stderr) + + mark_pareto(results) + results.sort(key=metric) + args.output.parent.mkdir(parents=True, exist_ok=True) + payload = { + "bounds": { + "max_tokens": args.max_tokens, + "timeout": args.timeout, + "memory_mb": args.memory_mb, + "max_frontier_ratio": args.max_frontier_ratio, + "max_witnesses": args.max_witnesses, + }, + "known_cases": [ + {"name": case.name, "description": case.description} + for case in KNOWN_CASES + ], + "results": [asdict(result) for result in results], + } + json_path = args.output.with_suffix(".json") + markdown_path = args.output.with_suffix(".md") + json_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + markdown_path.write_text(render_markdown(args, results) + "\n", encoding="utf-8") + print(f"JSON: {json_path}") + print(f"Markdown: {markdown_path}") + return 0 if all(result.search.error is None for result in results) else 2 + finally: + if temporary is not None: + temporary.cleanup() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/test_ambiguity.py b/tools/test_ambiguity.py new file mode 100644 index 00000000..768fbb8a --- /dev/null +++ b/tools/test_ambiguity.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +import argparse +import os +import shutil +import subprocess +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +from tools import ambiguity + +ROOT = Path(__file__).resolve().parents[1] +ENGINE = ROOT / "_build" / "default" / "tools" / "ambiguity_search.exe" + +# A minimal grammar whose two atoms A and B are interchangeable (both reduce to +# [e] in the same contexts) so they collapse into one terminal class, while the +# unparenthesized [e PLUS e] rule is genuinely ambiguous. It exercises the +# equivalence-class machinery on a grammar small enough for the prover to finish +# instantly. +TINY_GRAMMAR = """\ +%token A "a" +%token B "b" +%token PLUS "+" +%token EOF "" +%start main +%% +main: e EOF { () } +e: + | A { () } + | B { () } + | e PLUS e { () } +""" + + +def engine_environment() -> dict[str, str] | None: + menhir = os.environ.get("AMBIGUITY_MENHIR") or shutil.which("menhir") + if not ENGINE.exists() or menhir is None: + return None + return { + **os.environ, + "AMBIGUITY_MENHIR": menhir, + "AMBIGUITY_MEMORY_MB": "64", + "AMBIGUITY_MAX_FRONTIER_RATIO": "1.0", + "AMBIGUITY_JOBS": "1", + } + + +class ValueParsingTests(unittest.TestCase): + def test_token_range_accepts_whitespace(self) -> None: + self.assertEqual(ambiguity.parse_token_range(" 12 .. 50 "), (12, 50)) + + def test_token_range_rejects_a_descending_range(self) -> None: + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "minimum must not exceed" + ): + ambiguity.parse_token_range("50..12") + + def test_duration_accepts_friendly_and_composed_units(self) -> None: + self.assertEqual(ambiguity.parse_duration("1h30m"), 5400) + self.assertEqual(ambiguity.parse_duration("250ms"), 0.25) + self.assertEqual(ambiguity.parse_duration(90), 90) + + def test_duration_rejects_trailing_text(self) -> None: + with self.assertRaisesRegex(ambiguity.ConfigurationError, "invalid duration"): + ambiguity.parse_duration("30 minutes") + + +class ProfileTests(unittest.TestCase): + def write_profiles(self, contents: str) -> Path: + directory = TemporaryDirectory() + self.addCleanup(directory.cleanup) + path = Path(directory.name) / "searches.toml" + path.write_text(contents, encoding="utf-8") + return path + + def override_namespace(self, **overrides: object) -> argparse.Namespace: + # Mirror argparse defaults: every override dest is None (store_true + # flags are False) unless the test sets it. + namespace = {setting.dest: None for setting in ambiguity.SETTINGS} + namespace["breadth_first"] = False + namespace.update(overrides) + return argparse.Namespace(**namespace) + + def test_inheritance_and_overrides(self) -> None: + path = self.write_profiles( + """ +[profiles.base] +description = "Broad search." +tokens = "0..20" +timeout = "2m" +witnesses = 10 + +[profiles.deep] +extends = "base" +tokens = "12..50" +prefix-tokens = ["UIDENT", "LCURLY"] +nodes-per-depth = 4 +output = "reports/{profile}-{date}.txt" +""" + ) + profile = ambiguity.load_profiles(path)["deep"] + self.assertEqual(profile.description, "Broad search.") + self.assertEqual((profile.min_tokens, profile.max_tokens), (12, 50)) + self.assertEqual(profile.timeout_seconds, 120) + self.assertEqual(profile.prefix_tokens, ("UIDENT", "LCURLY")) + self.assertEqual(profile.nodes_per_depth, 4) + self.assertEqual(profile.output, Path("reports/{profile}-{date}.txt")) + + arguments = self.override_namespace( + tokens="10..30", + timeout="1h", + witnesses=25, + prefix_tokens="UIDENT LIDENT", + breadth_first=True, + ) + overridden = ambiguity.apply_overrides(profile, arguments) + self.assertEqual((overridden.min_tokens, overridden.max_tokens), (10, 30)) + self.assertEqual(overridden.timeout_seconds, 3600) + self.assertEqual(overridden.witnesses, 25) + self.assertEqual(overridden.prefix_tokens, ("UIDENT", "LIDENT")) + self.assertIsNone(overridden.nodes_per_depth) + # The profile's output survives when no --output override is given. + self.assertEqual(overridden.output, Path("reports/{profile}-{date}.txt")) + + def test_output_key_and_override(self) -> None: + path = self.write_profiles( + """ +[profiles.quick] +tokens = "0..10" +timeout = "1m" +witnesses = 5 +output = "reports/{profile}.txt" +""" + ) + profile = ambiguity.load_profiles(path)["quick"] + self.assertEqual(profile.output, Path("reports/{profile}.txt")) + overridden = ambiguity.apply_overrides( + profile, self.override_namespace(output=Path("elsewhere.txt")) + ) + self.assertEqual(overridden.output, Path("elsewhere.txt")) + + def test_output_must_be_a_string_path(self) -> None: + path = self.write_profiles( + """ +[profiles.quick] +tokens = "0..10" +timeout = "1m" +witnesses = 5 +output = 5 +""" + ) + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "output must be a string path" + ): + ambiguity.load_profiles(path) + + def test_prefix_must_leave_room_for_eof(self) -> None: + # A prefix that fills the entire max_tokens budget leaves no slot for the + # required EOF token, so it can never complete a witness. + path = self.write_profiles( + """ +[profiles.quick] +tokens = "0..3" +timeout = "1m" +witnesses = 5 +prefix-tokens = ["UIDENT", "LPAREN", "RPAREN"] +""" + ) + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "no room for the EOF token" + ): + ambiguity.load_profiles(path) + + def test_single_registry_drives_flags_and_keys(self) -> None: + # The whole point of the registry: one list enumerates every flag, and + # each entry marks whether it is also a TOML key. Every setting registers + # a flag; only profile_key settings are valid TOML keys. + search = ambiguity.parser().parse_args(["search"]) + for setting in ambiguity.SETTINGS: + self.assertTrue(hasattr(search, setting.dest)) + self.assertEqual( + setting.profile_key, setting.key in ambiguity.PROFILE_KEYS + ) + # dry-run is a mode, so it is the one flag that is not a TOML key. + self.assertNotIn("dry-run", ambiguity.PROFILE_KEYS) + self.assertIn("breadth-first", ambiguity.PROFILE_KEYS) + + def test_breadth_first_profile_key(self) -> None: + path = self.write_profiles( + """ +[profiles.quick] +tokens = "0..10" +timeout = "1m" +witnesses = 5 +breadth-first = true +""" + ) + self.assertIsNone(ambiguity.load_profiles(path)["quick"].nodes_per_depth) + + def test_breadth_first_must_be_true(self) -> None: + path = self.write_profiles( + """ +[profiles.quick] +tokens = "0..10" +timeout = "1m" +witnesses = 5 +breadth-first = false +""" + ) + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "breadth-first must be true" + ): + ambiguity.load_profiles(path) + + def test_scheduling_keys_are_mutually_exclusive(self) -> None: + path = self.write_profiles( + """ +[profiles.quick] +tokens = "0..10" +timeout = "1m" +witnesses = 5 +nodes-per-depth = 4 +breadth-first = true +""" + ) + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "at most one of nodes-per-depth" + ): + ambiguity.load_profiles(path) + + def test_child_breadth_first_overrides_inherited_depth(self) -> None: + path = self.write_profiles( + """ +[profiles.base] +tokens = "0..10" +timeout = "1m" +witnesses = 5 +nodes-per-depth = 8 + +[profiles.child] +extends = "base" +breadth-first = true +""" + ) + self.assertIsNone(ambiguity.load_profiles(path)["child"].nodes_per_depth) + + def test_child_depth_overrides_inherited_breadth_first(self) -> None: + path = self.write_profiles( + """ +[profiles.base] +tokens = "0..10" +timeout = "1m" +witnesses = 5 +breadth-first = true + +[profiles.child] +extends = "base" +nodes-per-depth = 8 +""" + ) + self.assertEqual(ambiguity.load_profiles(path)["child"].nodes_per_depth, 8) + + def test_dry_run_is_not_a_profile_key(self) -> None: + path = self.write_profiles( + """ +[profiles.quick] +tokens = "0..10" +timeout = "1m" +witnesses = 5 +dry-run = true +""" + ) + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "unknown settings: dry-run" + ): + ambiguity.load_profiles(path) + + def test_snake_case_key_is_rejected(self) -> None: + path = self.write_profiles( + """ +[profiles.quick] +tokens = "0..10" +timeout = "1m" +witnesses = 5 +prefix_tokens = ["UIDENT"] +""" + ) + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "unknown settings: prefix_tokens" + ): + ambiguity.load_profiles(path) + + def test_inheritance_cycle_is_reported(self) -> None: + path = self.write_profiles( + """ +[profiles.one] +extends = "two" + +[profiles.two] +extends = "one" +""" + ) + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "one -> two -> one" + ): + ambiguity.load_profiles(path) + + def test_unknown_setting_is_reported(self) -> None: + path = self.write_profiles( + """ +[profiles.quick] +tokens = "0..10" +timeout = "1m" +witnesses = 5 +surprise = true +""" + ) + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "unknown settings: surprise" + ): + ambiguity.load_profiles(path) + + +class OutputPatternTests(unittest.TestCase): + def test_profile_and_date_placeholders_expand(self) -> None: + path = ambiguity.expand_output_path( + Path("reports/{profile}-{date}.txt"), "general" + ) + self.assertRegex(str(path), r"^reports/general-\d{4}-\d{2}-\d{2}\.txt$") + + def test_timestamp_placeholders_expand(self) -> None: + path = ambiguity.expand_output_path( + Path("{profile}_{datetime}--{time}"), "deep" + ) + self.assertRegex( + str(path), + r"^deep_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}--\d{2}-\d{2}-\d{2}$", + ) + + def test_plain_path_is_unchanged(self) -> None: + self.assertEqual( + ambiguity.expand_output_path(Path("report.txt"), "general"), + Path("report.txt"), + ) + + def test_unknown_placeholder_is_reported(self) -> None: + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "unknown placeholder" + ): + ambiguity.expand_output_path(Path("reports/{oops}.txt"), "general") + + def test_unbalanced_brace_is_reported(self) -> None: + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "invalid --output pattern" + ): + ambiguity.expand_output_path(Path("reports/{profile.txt"), "general") + + def test_literal_braces_are_preserved(self) -> None: + self.assertEqual( + ambiguity.expand_output_path(Path("reports/{{profile}}.txt"), "general"), + Path("reports/{profile}.txt"), + ) + + def test_indexed_placeholder_is_rejected(self) -> None: + # str.format_map would silently expand this to the first character. + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "unknown placeholder" + ): + ambiguity.expand_output_path(Path("{profile[0]}.txt"), "general") + + def test_attribute_placeholder_is_rejected(self) -> None: + # str.format_map would raise an uncaught AttributeError here. + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "unknown placeholder" + ): + ambiguity.expand_output_path(Path("{profile.foo}.txt"), "general") + + def test_format_spec_is_rejected(self) -> None: + with self.assertRaisesRegex( + ambiguity.ConfigurationError, "no format spec or conversion" + ): + ambiguity.expand_output_path(Path("{profile:>10}.txt"), "general") + + +class CommandLineTests(unittest.TestCase): + def test_search_defaults_to_general(self) -> None: + arguments = ambiguity.parser().parse_args(["search"]) + self.assertEqual(arguments.profile, "general") + + def test_prove_defaults_to_quick(self) -> None: + arguments = ambiguity.parser().parse_args(["prove", "3"]) + self.assertEqual(arguments.profile, "quick") + + def test_check_accepts_quoted_or_individual_tokens(self) -> None: + cli = ambiguity.parser() + quoted = cli.parse_args(["check", "UIDENT LPAREN RPAREN EOF"]) + separate = cli.parse_args( + ["check", "UIDENT", "LPAREN", "RPAREN", "EOF"] + ) + self.assertEqual(" ".join(quoted.tokens), " ".join(separate.tokens)) + + def test_engine_arguments_are_stable_and_low_level(self) -> None: + profile = ambiguity.SearchProfile( + name="deep", + description="", + min_tokens=12, + max_tokens=50, + timeout_seconds=1800, + witnesses=50, + prefix_tokens=("UIDENT", "LCURLY"), + nodes_per_depth=10, + ) + self.assertEqual( + ambiguity.engine_arguments(profile), + [ + "--max-tokens", + "50", + "--min-tokens", + "12", + "--timeout", + "1800", + "--max-witnesses", + "50", + "--prefix-tokens", + "UIDENT LCURLY", + "--nodes-per-depth", + "10", + ], + ) + + +class TerminalClassEngineTests(unittest.TestCase): + """End-to-end checks that the search collapses interchangeable terminals + without losing an ambiguity reachable only through a non-representative + member. Skipped when the engine binary or menhir is unavailable, so the + otherwise pure-Python suite still runs without the OCaml toolchain.""" + + def setUp(self) -> None: + self.environment = engine_environment() + if self.environment is None: + self.skipTest( + "requires a built _build/default/tools/ambiguity_search.exe and menhir" + ) + directory = TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.grammar = Path(directory.name) / "tiny.mly" + self.grammar.write_text(TINY_GRAMMAR, encoding="utf-8") + + def engine(self, *arguments: str) -> subprocess.CompletedProcess[str]: + # --timeout only bounds the engine's own search; a process-level timeout + # keeps a hung binary or menhir from blocking the whole suite. + return subprocess.run( + [str(ENGINE), *arguments, str(self.grammar)], + env=self.environment, + text=True, + capture_output=True, + timeout=120, + ) + + def test_interchangeable_atoms_share_one_class(self) -> None: + result = self.engine("--dump-terminal-classes") + self.assertIn("{ A B }", result.stdout) + + def test_ambiguity_holds_through_a_non_representative_member(self) -> None: + # B is not the class representative (A sorts first), so the search never + # shifts it directly; the all-B sentence must still be recognized as + # ambiguous, confirming the representative stands in for the whole class. + result = self.engine("--check-tokens", "B PLUS B PLUS B EOF") + self.assertIn("Accepting derivations: 2", result.stdout) + self.assertEqual(result.returncode, 1) + + def test_prove_finds_the_ambiguity_via_the_representative(self) -> None: + # prove drives the abstract BFS over class representatives, then + # concretizes; it must surface the e-PLUS-e ambiguity even though every + # witness is spelled with the representative atom. + result = self.engine( + "--prove", "2", + "--max-tokens", "8", + "--timeout", "30", + "--max-witnesses", "5", + ) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("complete ambiguity", result.stdout) + # The witness must be spelled with the class representative A, never the + # non-representative B, confirming concretization stays on representatives. + witnesses = [ + line for line in result.stdout.splitlines() if "Tokens (" in line + ] + self.assertTrue(witnesses, result.stdout) + tokens = witnesses[0].split(":", 1)[1].split() + self.assertIn("A", tokens) + self.assertNotIn("B", tokens) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/test_syntax_experiment.py b/tools/test_syntax_experiment.py new file mode 100644 index 00000000..246431ff --- /dev/null +++ b/tools/test_syntax_experiment.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +import unittest +from pathlib import Path + +from tools import syntax_experiment as experiments + + +class SyntaxExperimentTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.source = Path("lib/cst/parser.mly").read_text(encoding="utf-8") + + def test_every_predefined_variant_applies(self) -> None: + names = set() + for variant in experiments.VARIANTS: + self.assertNotIn(variant.name, names) + names.add(variant.name) + grammar = experiments.apply_variant(self.source, variant) + self.assertIn(f"Variant: {variant.name}", grammar) + + def test_semicolon_variants_replace_every_statement_list(self) -> None: + for transform in ( + experiments.semicolon_separated, + experiments.semicolon_terminated, + ): + grammar = transform(self.source) + self.assertNotIn("decls=list(decl)", grammar) + self.assertNotIn("list(stat)", grammar) + self.assertIn("decls=decl_sequence", grammar) + self.assertIn("statements=stat_sequence", grammar) + + def test_named_statement_variant_keeps_computed_calls_in_expressions(self) -> None: + grammar = experiments.named_statement_calls(self.source) + self.assertIn("verb_call:\n | receiver=app", grammar) + self.assertIn("verb_call=statement_verb_call", grammar) + self.assertNotIn(experiments.STAT_CALL, grammar) + + def test_anchored_abort_handles_leave_no_expression_slots(self) -> None: + grammar = experiments.anchored_abort_handles(self.source) + self.assertNotIn("abort_handle=ioption(abort_handle) %prec", grammar) + self.assertIn("handled_expr:", grammar) + self.assertIn( + "| verb_call=verb_call abort_handle=ioption(abort_handle) {", grammar + ) + self.assertIn('value=handled_expr', grammar) + self.assertIn('"(" e=handled_expr ")"', grammar) + self.assertEqual(grammar.count("separated_list(COMMA, handled_expr)"), 3) + + def test_grouping_variants_are_mutually_distinct(self) -> None: + bracket = experiments.bracket_grouping(self.source) + keyword = experiments.keyword_grouping(self.source) + none = experiments.no_grouping(self.source) + self.assertIn('| "[" e=expr "]"', bracket) + self.assertIn('| GROUP "(" e=expr ")"', keyword) + self.assertNotIn("Nodes.Expr.Parenthized", none) + + +class WitnessSpellingTests(unittest.TestCase): + @staticmethod + def variant(name: str) -> experiments.Variant: + return next(v for v in experiments.VARIANTS if v.name == name) + + @staticmethod + def tokens(case_name: str, variant_name: str) -> str | None: + case = next(c for c in experiments.KNOWN_CASES if c.name == case_name) + return experiments.case_tokens( + case, WitnessSpellingTests.variant(variant_name) + ) + + def test_every_transform_has_a_spelling_update(self) -> None: + self.assertEqual( + set(experiments.TRANSFORMS), set(experiments.SPELLINGS) + ) + + def test_baseline_spelling_reproduces_original_witnesses(self) -> None: + expected = { + "nullable-array-binding": "ALIAS UIDENT EQUAL UIDENT QSTNMARK UIDENT " + "LBRACKET RBRACKET LBRACKET RBRACKET EOF", + "adjacent-computed-call": "UIDENT LPAREN RPAREN LCURLY LIDENT LPAREN " + "RPAREN LPAREN LIDENT RPAREN LPAREN RPAREN RCURLY EOF", + "abort-handler-attachment": "UIDENT LPAREN RPAREN LCURLY LIDENT LPAREN " + "TILDE TILDE LIDENT QSTNQSTN LIDENT RPAREN RCURLY EOF", + "named-call-statement": "UIDENT LPAREN RPAREN LCURLY LIDENT LPAREN " + "STRING RPAREN RCURLY EOF", + } + for name, tokens in expected.items(): + self.assertEqual(self.tokens(name, "baseline"), tokens) + + def test_bracket_calls_spell_print_hello_with_brackets(self) -> None: + self.assertEqual( + self.tokens("named-call-statement", "bracket-calls"), + "UIDENT LPAREN RPAREN LCURLY LIDENT LBRACKET STRING RBRACKET RCURLY EOF", + ) + + def test_semicolon_terminated_terminates_statements_and_decls(self) -> None: + self.assertEqual( + self.tokens("named-call-statement", "semicolon-terminated"), + "UIDENT LPAREN RPAREN LCURLY LIDENT LPAREN STRING RPAREN SEMICOLON " + "RCURLY SEMICOLON EOF", + ) + self.assertEqual( + self.tokens("nullable-array-binding", "semicolon-terminated"), + "ALIAS UIDENT EQUAL UIDENT QSTNMARK UIDENT LBRACKET RBRACKET " + "LBRACKET RBRACKET SEMICOLON EOF", + ) + + def test_semicolon_separated_splits_the_two_statement_reading(self) -> None: + self.assertEqual( + self.tokens("adjacent-computed-call", "semicolon-separated"), + "UIDENT LPAREN RPAREN LCURLY LIDENT LPAREN RPAREN SEMICOLON " + "LPAREN LIDENT RPAREN LPAREN RPAREN RCURLY EOF", + ) + + def test_computed_call_statements_can_be_inexpressible(self) -> None: + self.assertIsNone(self.tokens("adjacent-computed-call", "named-statement-calls")) + self.assertIsNone(self.tokens("adjacent-computed-call", "no-grouping")) + + def test_marked_computed_calls_keep_named_calls_plain(self) -> None: + self.assertEqual( + self.tokens("named-call-statement", "marked-computed-calls"), + "UIDENT LPAREN RPAREN LCURLY LIDENT LPAREN STRING RPAREN RCURLY EOF", + ) + self.assertEqual( + self.tokens("adjacent-computed-call", "marked-computed-calls"), + "UIDENT LPAREN RPAREN LCURLY LIDENT LPAREN RPAREN " + "LPAREN LIDENT RPAREN AT LPAREN RPAREN RCURLY EOF", + ) + + def test_transform_spellings_compose(self) -> None: + self.assertEqual( + self.tokens( + "adjacent-computed-call", "bracket-grouping-and-semicolons" + ), + "UIDENT LPAREN RPAREN LCURLY LIDENT LPAREN RPAREN SEMICOLON " + "LBRACKET LIDENT RBRACKET LPAREN RPAREN RCURLY EOF", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/vcpkg-configuration.json b/vcpkg-configuration.json deleted file mode 100644 index 65c85e67..00000000 --- a/vcpkg-configuration.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "default-registry": { - "kind": "git", - "baseline": "4e39a8622e56a113c89228e4e13944dce8654da7", - "repository": "https://github.com/microsoft/vcpkg" - }, - "registries": [ - { - "kind": "artifact", - "location": "https://github.com/microsoft/vcpkg-ce-catalog/archive/refs/heads/main.zip", - "name": "microsoft" - } - ] -} diff --git a/vcpkg.json b/vcpkg.json deleted file mode 100644 index be32b307..00000000 --- a/vcpkg.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "dependencies": [ - "nlohmann-json", - "cereal" - ] -} diff --git a/zane-compiler.opam b/zane-compiler.opam new file mode 100644 index 00000000..1731773f --- /dev/null +++ b/zane-compiler.opam @@ -0,0 +1,28 @@ +# This file is generated by dune, edit dune-project instead +opam-version: "2.0" +synopsis: "A short synopsis" +description: "A longer description" +tags: ["add topics" "to describe" "your" "project"] +homepage: "https://github.com/zane-lang/compiler" +bug-reports: "https://github.com/zane-lang/compiler/issues" +depends: [ + "dune" {>= "3.21"} + "ocaml" + "odoc" {with-doc} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/zane-lang/compiler.git" +x-maintenance-intent: ["(latest)"]