diff --git a/.github/workflows/changed-path-ledger-native.yml b/.github/workflows/changed-path-ledger-native.yml new file mode 100644 index 0000000..c42d7ce --- /dev/null +++ b/.github/workflows/changed-path-ledger-native.yml @@ -0,0 +1,189 @@ +name: Changed-path Ledger Native Gates + +# Required release/merge gate: branch protection and release automation must +# require both matrix jobs from this workflow for the commit being released. + +on: + workflow_call: + inputs: + plan: + description: "cargo-dist plan JSON (accepted for plan-job compatibility)" + required: false + type: string + default: "" + scale_files: + description: "Repository files (100000 or 1000000)" + required: false + type: string + default: "100000" + require_cow: + description: "Require the native COW checkpoint cases" + required: false + type: boolean + default: true + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "17 6 * * 1-5" + - cron: "47 8 * * *" + workflow_dispatch: + inputs: + scale_files: + description: "Repository files (100000 or 1000000)" + required: true + default: "100000" + require_cow: + description: "Require the native COW checkpoint cases" + type: boolean + required: true + default: false + +jobs: + native: + name: ${{ matrix.os }} native ledger + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 720 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Select scheduled scale + id: scale + shell: bash + run: | + requested_files="${{ inputs.scale_files }}" + if [ -n "$requested_files" ]; then + files="$requested_files" + elif [ "${{ github.event.schedule }}" = "47 8 * * *" ]; then + files=1000000 + else + files=100000 + fi + case "$files" in 100000|1000000) ;; *) exit 2 ;; esac + if [ -n "$requested_files" ]; then + require_cow="${{ inputs.require_cow }}" + else + require_cow=true + fi + echo "files=$files" >> "$GITHUB_OUTPUT" + echo "require_cow=$require_cow" >> "$GITHUB_OUTPUT" + if [ "$require_cow" = true ]; then + echo "cow_mode=1" >> "$GITHUB_OUTPUT" + else + echo "cow_mode=auto" >> "$GITHUB_OUTPUT" + fi + - name: Prove native filesystem + id: filesystem + shell: bash + run: | + proof_root="${{ runner.temp }}/trail-cli-scale-native-${{ runner.os }}-${{ github.run_id }}" + mkdir -p "$proof_root" + device="" + case "${{ runner.os }}" in + Linux) + filesystem="$(findmnt --noheadings --output FSTYPE --target "${{ runner.temp }}" | tr -d '[:space:]')" + [ "$filesystem" = ext4 ] || { + echo "native Linux ledger gate requires ext4, found $filesystem" >&2 + exit 1 + } + ;; + macOS) + device="$(df -P "${{ runner.temp }}" | awk 'NR == 2 { print $1 }')" + [ -n "$device" ] || { + echo "could not resolve the device containing ${{ runner.temp }}" >&2 + exit 1 + } + filesystem="$(diskutil info "$device" | awk -F: '/File System Personality/ {gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print tolower($2); exit}')" + [ "$filesystem" = apfs ] || { + echo "native macOS ledger gate requires APFS on $device, found $filesystem" >&2 + exit 1 + } + ;; + *) exit 2 ;; + esac + printf 'runner_os=%s\nfilesystem=%s\ndevice=%s\nroot=%s\n' \ + "${{ runner.os }}" "$filesystem" "$device" "${{ runner.temp }}" \ + >"$proof_root/native-filesystem.txt" + echo "type=$filesystem" >> "$GITHUB_OUTPUT" + - name: Run native qualification suite + shell: bash + run: | + case "${{ runner.os }}" in + Linux) cargo test -p trail --test changed_path_ledger_linux -- --nocapture ;; + macOS) cargo test -p trail --test changed_path_ledger_macos -- --nocapture ;; + *) exit 2 ;; + esac + - name: Nightly daemon restart and crash sampling + if: steps.scale.outputs.files == '1000000' + run: | + cargo test -p trail --test changed_path_ledger_daemon -- --nocapture + cargo test -p trail --test changed_path_ledger_recovery -- --nocapture + - name: Build release binary + run: cargo build -p trail --release + - name: Run native changed-path ledger scale + run: scripts/cli-scale-bench.sh changed-path-ledger + env: + TRAIL_BIN: ${{ github.workspace }}/target/release/trail + TRAIL_SCALE_BASE: ${{ runner.temp }} + TRAIL_SCALE_LABEL: native-${{ runner.os }}-${{ github.run_id }} + TRAIL_SCALE_FILES: ${{ steps.scale.outputs.files }} + TRAIL_CHANGED_PATH_COW: ${{ steps.scale.outputs.cow_mode }} + TRAIL_EXPECTED_NATIVE_FILESYSTEM: ${{ steps.filesystem.outputs.type }} + - name: Enforce native thresholds + shell: bash + run: | + scale="${{ steps.scale.outputs.files }}" + root="${{ runner.temp }}/trail-cli-scale-native-${{ runner.os }}-${{ github.run_id }}/$scale" + case "${{ runner.os }}:$scale" in + Linux:1000000) + status=30 diff=45 record=60 materialized=300 patch=60 cow_seconds=600 cold=900 rss=3221225472 ;; + macOS:1000000) + status=45 diff=60 record=90 materialized=420 patch=90 cow_seconds=900 cold=1200 rss=4294967296 ;; + Linux:100000) + status=15 diff=20 record=30 materialized=120 patch=30 cow_seconds=240 cold=300 rss=3221225472 ;; + macOS:100000) + status=25 diff=35 record=45 materialized=180 patch=45 cow_seconds=360 cold=450 rss=4294967296 ;; + *) exit 2 ;; + esac + cow_args=() + if [ "${{ steps.scale.outputs.require_cow }}" = "true" ]; then + cow_args=(--require-cow) + fi + cold_args=() + if [ "$scale" = 1000000 ]; then + cold_args=(--require-cold-reconcile) + fi + python3 scripts/check-changed-path-ledger-thresholds.py \ + "$root/results.tsv" \ + "$root/structural-metrics.jsonl" \ + --oracle "$root/oracle-results.tsv" \ + "${cow_args[@]}" \ + "${cold_args[@]}" \ + --max-seconds "workspace_status=$status" \ + --max-seconds "workspace_diff=$diff" \ + --max-seconds "workspace_record=$record" \ + --max-seconds "materialized_lane_record=$materialized" \ + --max-seconds "structured_patch=$patch" \ + --max-seconds "cow_checkpoint=$cow_seconds" \ + --max-seconds "cold_reconcile=$cold" \ + --max-rss-bytes "workspace_status=$rss" \ + --max-rss-bytes "workspace_diff=$rss" \ + --max-rss-bytes "workspace_record=$rss" \ + --max-rss-bytes "materialized_lane_record=$rss" \ + --max-rss-bytes "structured_patch=$rss" \ + --max-rss-bytes "cow_checkpoint=$rss" \ + --max-rss-bytes "cold_reconcile=$rss" + - name: Upload native changed-path ledger artifacts + uses: actions/upload-artifact@v4 + if: always() + with: + name: changed-path-ledger-${{ runner.os }}-${{ steps.scale.outputs.files }} + path: ${{ runner.temp }}/trail-cli-scale-native-${{ runner.os }}-${{ github.run_id }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 026e1cf..4c5cda7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,8 +4,31 @@ on: push: pull_request: workflow_dispatch: + schedule: + - cron: "17 9 * * 1" jobs: + acp-v1-reference-interop: + name: ACP v1 official-type interoperability + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Official ACP v1 peer interoperability + run: scripts/test-acp-v1-reference-interop.sh + + acp-v1-schema-drift: + name: ACP v1 upstream schema drift + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Compare pinned ACP v1 artifacts with upstream + run: scripts/check-acp-v1-schema-drift.sh + test: runs-on: ubuntu-latest steps: @@ -24,6 +47,14 @@ jobs: run: cargo test -p trail - name: Scale Benchmark Syntax run: bash -n scripts/cli-scale-bench.sh + - name: Scale Threshold Checker Tests + run: >- + python3 -m unittest + scripts/test_cli_scale_bench_source.py + scripts/test_check_cli_scale_thresholds.py + scripts/test_check_changed_path_ledger_thresholds.py + scripts/test_extract_agent_git_performance.py + scripts/test_extract_path_index_performance.py - name: Scale Benchmark Smoke run: make bench-cli-scale-smoke env: @@ -55,7 +86,13 @@ jobs: daemon_cli_history=10 daemon_cli_code_from=10 agent_apply_patch=30 - merge_queue_run=30 + path_index_rename_patch=30 + path_index_empty_patch=30 + path_index_record=30 + agent_git_apply_dry_run=15 + agent_git_apply=20 + agent_git_apply_missing_mapping=5 + lane_merge_queue_run=30 agent_spawn_sparse=15 agent_read_sparse_hydrate_neighbors=15 agent_sync_sparse_dir=15 @@ -70,6 +107,50 @@ jobs: dbstat_repo_prolly_nodes=32000000 object_kind_repo_TextContent_bytes=16000000 object_count=20000 + agent_git_changed_paths=1 + agent_git_blob_writes=1 + agent_git_plumbing_commands=5 + patch_path_index_lookup_count=5 + patch_path_index_full_root_path_load_count=0 + patch_path_index_full_filesystem_path_scan_count=0 + record_path_index_lookup_count=1 + record_path_index_full_root_path_load_count=0 + record_path_index_full_filesystem_path_scan_count=0 + empty_root_patch_path_index_lookup_count=1 + empty_root_patch_path_index_full_root_path_load_count=0 + empty_root_patch_path_index_full_filesystem_path_scan_count=0 + rename_patch_path_index_lookup_count=2 + rename_patch_path_index_full_root_path_load_count=0 + rename_patch_path_index_full_filesystem_path_scan_count=0 + --metric-equals + agent_git_export_mode=mapped_delta + agent_git_changed_paths=1 + patch_path_index_mode=indexed + record_path_index_mode=indexed + empty_root_patch_path_index_mode=indexed + rename_patch_path_index_mode=indexed + - name: Changed-path Ledger 1k Benchmark + run: scripts/cli-scale-bench.sh changed-path-ledger + env: + TRAIL_SCALE_BASE: ${{ runner.temp }} + TRAIL_SCALE_LABEL: changed-path-ci-${{ github.run_id }} + TRAIL_SCALE_FILES: "1000" + TRAIL_CHANGED_PATH_COW: "0" + - name: Changed-path Ledger 1k Thresholds + run: >- + python3 scripts/check-changed-path-ledger-thresholds.py + "${{ runner.temp }}/trail-cli-scale-changed-path-ci-${{ github.run_id }}/1000/results.tsv" + "${{ runner.temp }}/trail-cli-scale-changed-path-ci-${{ github.run_id }}/1000/structural-metrics.jsonl" + --oracle + "${{ runner.temp }}/trail-cli-scale-changed-path-ci-${{ github.run_id }}/1000/oracle-results.tsv" + --max-seconds workspace_status=15 + --max-seconds workspace_diff=20 + --max-seconds workspace_record=30 + --max-seconds materialized_lane_record=60 + --max-seconds structured_patch=30 + --max-rss-bytes workspace_status=2147483648 + --max-rss-bytes workspace_diff=2147483648 + --max-rss-bytes workspace_record=2147483648 prolly: name: Prolly core @@ -91,25 +172,25 @@ jobs: run: cargo test -p prolly-map - name: Clippy run: >- - cargo clippy -p prolly-map --all-targets --features "tokio sqlite" -- + cargo clippy -p prolly-map --all-targets --features tokio -- -D warnings -A clippy::result-large-err -A clippy::large-enum-variant -A clippy::type-complexity - name: Rustdoc - run: RUSTDOCFLAGS="-D warnings" cargo doc -p prolly-map --no-deps --features "tokio sqlite" + run: RUSTDOCFLAGS="-D warnings" cargo doc -p prolly-map --no-deps --features tokio - name: Inspect binary run: cargo check -p prolly-map --bin prolly-inspect - name: Examples run: cargo check -p prolly-map --examples - name: Bench compile - run: cargo check -p prolly-map --benches --features "tokio sqlite" + run: cargo check -p prolly-map --benches --features tokio - name: Package run: cargo package -p prolly-map --no-verify - name: Tokio feature tests run: cargo test -p prolly-map --features tokio - name: SQLite store contract - run: cargo test -p prolly-map --features sqlite --test sqlite_store + run: cargo test -p prolly-store-sqlite prolly-optional-backends: name: Prolly optional backend (${{ matrix.backend }}) @@ -120,11 +201,9 @@ jobs: matrix: include: - backend: rocksdb - features: rocksdb - test: rocksdb_store + package: prolly-store-rocksdb - backend: slatedb - features: slatedb - test: slatedb_store + package: prolly-store-slatedb steps: - uses: actions/checkout@v4 with: @@ -135,7 +214,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - name: Store contract - run: cargo test -p prolly-map --features "${{ matrix.features }}" --test "${{ matrix.test }}" + run: cargo test -p "${{ matrix.package }}" prolly-pglite-sidecar: name: Prolly PGlite sidecar @@ -155,7 +234,7 @@ jobs: mkdir -p "${{ runner.temp }}/prolly-pglite-node" npm install --prefix "${{ runner.temp }}/prolly-pglite-node" @electric-sql/pglite@0.5.3 - name: PGlite store contract - run: cargo test -p prolly-map --features pglite --test pglite_store + run: cargo test -p prolly-store-pglite env: PROLLY_PGLITE_TEST: "1" PROLLY_PGLITE_NODE_CWD: ${{ runner.temp }}/prolly-pglite-node diff --git a/.github/workflows/layered-workspaces.yml b/.github/workflows/layered-workspaces.yml index 0101d2d..e7c111e 100644 --- a/.github/workflows/layered-workspaces.yml +++ b/.github/workflows/layered-workspaces.yml @@ -4,11 +4,18 @@ on: pull_request: paths: - "trail/**" - - "scripts/verify-linux-overlay-cow-docker.sh" + - "scripts/verify-linux-fuse-cow-docker.sh" - "scripts/verify-layered-lane-scale.sh" - "scripts/verify-rust-cache-isolation-docker.sh" - "scripts/verify-linux-node-layer-docker.sh" - "scripts/install-dokany.ps1" + - "scripts/verify-linux-command-recipe-native.sh" + - "scripts/verify-linux-command-recipe-sandbox.sh" + - "scripts/verify-windows-command-recipe-sandbox.ps1" + - "scripts/verify-macos-nfs-framework-layers.sh" + - "scripts/verify-environment-adapter-plugin.sh" + - "scripts/verify-windows-environment-adapter-plugin.ps1" + - "trail-environment-adapter-sdk/**" - ".github/workflows/layered-workspaces.yml" workflow_dispatch: inputs: @@ -22,6 +29,11 @@ on: required: false default: false type: boolean + run_macos_framework_bench: + description: "Run real Next.js and Vite dependency/build benchmarks through macOS NFS" + required: false + default: false + type: boolean jobs: core: @@ -50,9 +62,25 @@ jobs: - uses: Swatinem/rust-cache@v2 - run: sudo apt-get update && sudo apt-get install -y fuse3 - run: sudo chmod 666 /dev/fuse + - run: cmake --version && cc --version && make --version - run: cargo test -p trail fuse_adapter_runs_shared_mounted_view_suite -- --nocapture env: TRAIL_RUN_FUSE_COW_TESTS: "1" + - run: cargo test -p trail real_cmake_configure_build_and_clean_stay_lane_private -- --nocapture + env: + TRAIL_RUN_FUSE_COW_TESTS: "1" + - run: cargo test -p trail real_python_venvs_embed_lane_paths_and_remain_isolated -- --nocapture + env: + TRAIL_RUN_FUSE_COW_TESTS: "1" + - run: cargo test -p trail failed_mounted_initializer_preserves_previous_generation_and_real_uppers -- --nocapture + env: + TRAIL_RUN_FUSE_COW_TESTS: "1" + - run: cargo test -p trail killing_mounted_python_initialization_never_exposes_a_partial_generation -- --nocapture + env: + TRAIL_RUN_FUSE_COW_TESTS: "1" + - run: cargo test -p trail sync_all_initializes_nested_python_components_at_final_lane_paths -- --nocapture + env: + TRAIL_RUN_FUSE_COW_TESTS: "1" rust-cache-isolation: runs-on: ubuntu-latest @@ -70,6 +98,53 @@ jobs: submodules: recursive - run: scripts/verify-linux-node-layer-docker.sh + command-recipe-sandbox: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: scripts/verify-linux-command-recipe-native.sh + + adapter-plugin-conformance: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: scripts/verify-environment-adapter-plugin.sh + + windows-command-recipe-sandbox: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: ./scripts/install-dokany.ps1 + - shell: pwsh + run: | + cmake --version + cargo test -p trail windows_arguments_follow_command_line_to_argv_w_quoting -- --nocapture + cargo test -p trail windows_recipe_publication_rejects_hard_link_aliases -- --nocapture + $env:TRAIL_RUN_DOKAN_COW_TESTS = "1" + cargo test -p trail real_windows_cmake_build_and_clean_stay_lane_private -- --nocapture + cargo test -p trail real_windows_python_venvs_embed_lane_paths_and_remain_isolated -- --nocapture + cargo test -p trail failed_mounted_initializer_preserves_previous_generation_and_real_uppers -- --nocapture + cargo test -p trail killing_mounted_python_initialization_never_exposes_a_partial_generation -- --nocapture + cargo test -p trail sync_all_initializes_nested_python_components_at_final_lane_paths -- --nocapture + - run: ./scripts/verify-windows-command-recipe-sandbox.ps1 + - run: ./scripts/verify-windows-environment-adapter-plugin.ps1 + nfs-conformance: runs-on: macos-latest steps: @@ -78,6 +153,7 @@ jobs: submodules: recursive - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 + - run: cmake --version && cc --version && make --version - run: cargo test -p trail nfs_adapter_runs_shared_mounted_view_suite -- --nocapture env: TRAIL_RUN_NFS_COW_TESTS: "1" @@ -90,12 +166,39 @@ jobs: - run: cargo test -p trail nfs_git_checkout_reset_and_clean_are_lane_local -- --nocapture env: TRAIL_RUN_NFS_COW_TESTS: "1" - - run: cargo test -p trail nfs_real_node_layer_is_shared_and_writable_installs_are_isolated -- --nocapture + - run: cargo test -p trail nfs_real_node_layer_bulk_replacement_is_isolated -- --nocapture env: TRAIL_RUN_NFS_COW_TESTS: "1" - run: cargo test -p trail nfs_cargo_target_seed_is_shared_and_writable_targets_are_isolated -- --nocapture env: TRAIL_RUN_NFS_COW_TESTS: "1" + - run: cargo test -p trail restricted_command_recipe_publishes_and_activates_multiple_outputs_atomically -- --nocapture + - run: cargo test -p trail real_cmake_configure_build_and_clean_stay_lane_private -- --nocapture + env: + TRAIL_RUN_NFS_COW_TESTS: "1" + - run: cargo test -p trail real_python_venvs_embed_lane_paths_and_remain_isolated -- --nocapture + env: + TRAIL_RUN_NFS_COW_TESTS: "1" + - run: cargo test -p trail failed_mounted_initializer_preserves_previous_generation_and_real_uppers -- --nocapture + env: + TRAIL_RUN_NFS_COW_TESTS: "1" + - run: cargo test -p trail killing_mounted_python_initialization_never_exposes_a_partial_generation -- --nocapture + env: + TRAIL_RUN_NFS_COW_TESTS: "1" + - run: cargo test -p trail sync_all_initializes_nested_python_components_at_final_lane_paths -- --nocapture + env: + TRAIL_RUN_NFS_COW_TESTS: "1" + + nfs-framework-bench: + if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_macos_framework_bench }} + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: scripts/verify-macos-nfs-framework-layers.sh dokan-conformance: if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_dokan_conformance }} diff --git a/.github/workflows/release-automation.yml b/.github/workflows/release-automation.yml index 0df078b..9b350e9 100644 --- a/.github/workflows/release-automation.yml +++ b/.github/workflows/release-automation.yml @@ -108,9 +108,18 @@ jobs: git commit -m "chore: update Cargo.lock for ${release_version}" git push origin "HEAD:${RELEASE_PR_BRANCH}" + exact-sha-native-ledger: + name: Exact-SHA Linux + macOS changed-path gates + needs: release-please + if: ${{ needs.release-please.outputs.token_available == 'true' }} + uses: ./.github/workflows/changed-path-ledger-native.yml + with: + scale_files: "100000" + require_cow: true + tag-release: name: Tag merged release pull request - needs: release-please + needs: [release-please, exact-sha-native-ledger] if: ${{ needs.release-please.outputs.token_available == 'true' }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a620734..61321d6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -87,12 +87,17 @@ jobs: name: artifacts-plan-dist-manifest path: plan-dist-manifest.json + custom-changed-path-ledger-native: + uses: ./.github/workflows/changed-path-ledger-native.yml + secrets: inherit + # Build and packages all the platform-specific things build-local-artifacts: name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) # Let the initial task tell us to not run (currently very blunt) needs: - plan + - custom-changed-path-ledger-native if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} strategy: fail-fast: false diff --git a/.github/workflows/scale.yml b/.github/workflows/scale.yml index a3f5691..d7252fc 100644 --- a/.github/workflows/scale.yml +++ b/.github/workflows/scale.yml @@ -25,6 +25,10 @@ on: description: "Run Git import/update benchmark cases" required: true default: "1" + changed_path_ledger_files: + description: "Changed-path ledger scale (1000, 100000, or 1000000)" + required: true + default: "1000" jobs: scale: @@ -32,6 +36,8 @@ jobs: timeout-minutes: 360 steps: - uses: actions/checkout@v4 + with: + submodules: recursive - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - name: Run CLI scale benchmark @@ -52,16 +58,37 @@ jobs: - name: Check large-agent hot path thresholds if: always() run: | - if [ "${{ github.event.inputs.run_daemon || '1' }}" != "1" ]; then - echo "skipping daemon hot-path thresholds because run_daemon is disabled" - exit 0 - fi root="${{ runner.temp }}/trail-cli-scale-scale-${{ github.run_id }}" mapfile -t result_files < <(find "$root" -name results.tsv -print | sort) if [ "${#result_files[@]}" -eq 0 ]; then echo "no scale benchmark results found under $root" >&2 exit 1 fi + daemon_thresholds=() + if [ "${{ github.event.inputs.run_daemon || '1' }}" = "1" ]; then + daemon_thresholds=( + daemon_wait_for_health=60 + daemon_wait_for_hot_cache=120 + daemon_status=5 + daemon_persisted_snapshot_status=5 + daemon_persisted_snapshot_record_clean=5 + daemon_persisted_snapshot_diff_dirty=10 + daemon_cli_status=5 + daemon_cli_record_dirty=10 + daemon_cli_agent_readiness=5 + daemon_cli_agent_trace_summary=5 + daemon_cli_merge_dry_run=10 + daemon_cli_session_start=10 + daemon_cli_approval_request=10 + daemon_cli_lease_acquire=10 + daemon_cli_timeline=10 + daemon_cli_why=10 + daemon_cli_history=10 + daemon_cli_code_from=10 + ) + else + echo "omitting daemon hot-path thresholds because run_daemon is disabled" + fi git_thresholds=() if [ "${{ github.event.inputs.run_git_import || '1' }}" = "1" ]; then git_thresholds=( @@ -91,38 +118,94 @@ jobs: fi for results in "${result_files[@]}"; do metrics="$(dirname "$results")/metrics.tsv" + scale="$(basename "$(dirname "$results")")" + git_apply_thresholds=() + git_structural_thresholds=() + git_metric_equalities=() + if [ "${{ github.event.inputs.run_git_import || '1' }}" = "1" ]; then + changed_paths=$((scale / 1000)) + if [ "$changed_paths" -lt 1 ]; then changed_paths=1; fi + if [ "$changed_paths" -gt 100 ]; then changed_paths=100; fi + case "$scale" in + 100000) + git_apply_thresholds=( + agent_git_apply_dry_run=2 + agent_git_apply=3 + agent_git_apply_missing_mapping=1 + ) + ;; + 1000000) + git_apply_thresholds=( + agent_git_apply_dry_run=5 + agent_git_apply=8 + agent_git_apply_missing_mapping=1 + ) + ;; + *) + git_apply_thresholds=( + agent_git_apply_dry_run=15 + agent_git_apply=20 + agent_git_apply_missing_mapping=5 + ) + ;; + esac + git_structural_thresholds=( + "agent_git_changed_paths=$changed_paths" + "agent_git_blob_writes=$changed_paths" + agent_git_plumbing_commands=5 + ) + git_metric_equalities=( + agent_git_export_mode=mapped_delta + "agent_git_changed_paths=$changed_paths" + ) + fi + patch_paths=$((scale / 200)) + if [ "$patch_paths" -lt 1 ]; then patch_paths=1; fi + if [ "$patch_paths" -gt 50 ]; then patch_paths=50; fi + path_index_structural_thresholds=( + "patch_path_index_lookup_count=$patch_paths" + patch_path_index_full_root_path_load_count=0 + patch_path_index_full_filesystem_path_scan_count=0 + record_path_index_lookup_count=1 + record_path_index_full_root_path_load_count=0 + record_path_index_full_filesystem_path_scan_count=0 + empty_root_patch_path_index_lookup_count=1 + empty_root_patch_path_index_full_root_path_load_count=0 + empty_root_patch_path_index_full_filesystem_path_scan_count=0 + rename_patch_path_index_lookup_count=2 + rename_patch_path_index_full_root_path_load_count=0 + rename_patch_path_index_full_filesystem_path_scan_count=0 + ) + path_index_metric_equalities=( + patch_path_index_mode=indexed + record_path_index_mode=indexed + empty_root_patch_path_index_mode=indexed + rename_patch_path_index_mode=indexed + ) python3 scripts/check-cli-scale-thresholds.py "$results" \ - daemon_wait_for_health=60 \ - daemon_wait_for_hot_cache=120 \ - daemon_status=5 \ - daemon_persisted_snapshot_status=5 \ - daemon_persisted_snapshot_record_clean=5 \ - daemon_persisted_snapshot_diff_dirty=10 \ - daemon_cli_status=5 \ - daemon_cli_record_dirty=10 \ - daemon_cli_agent_readiness=5 \ - daemon_cli_agent_trace_summary=5 \ - daemon_cli_merge_dry_run=10 \ - daemon_cli_session_start=10 \ - daemon_cli_approval_request=10 \ - daemon_cli_lease_acquire=10 \ - daemon_cli_timeline=10 \ - daemon_cli_why=10 \ - daemon_cli_history=10 \ - daemon_cli_code_from=10 \ + "${daemon_thresholds[@]}" \ agent_apply_patch=10 \ + path_index_rename_patch=10 \ + path_index_empty_patch=10 \ + path_index_record=10 \ agent_readiness=10 \ merge_agent_dry_run=10 \ merge_agent_apply=10 \ - merge_queue_run=10 \ + lane_merge_queue_run=10 \ "${git_thresholds[@]}" \ + "${git_apply_thresholds[@]}" \ "${materialized_thresholds[@]}" \ "${backup_thresholds[@]}" \ --metrics "$metrics" \ sqlite_bytes=4000000000 \ dbstat_repo_prolly_nodes=2500000000 \ object_kind_repo_TextContent_bytes=1200000000 \ - object_count=2000000 + object_count=2000000 \ + "${path_index_structural_thresholds[@]}" \ + "${git_structural_thresholds[@]}" \ + --metric-equals \ + "${path_index_metric_equalities[@]}" \ + "${git_metric_equalities[@]}" done - name: Upload scale results uses: actions/upload-artifact@v4 @@ -130,3 +213,48 @@ jobs: with: name: trail-cli-scale-results path: ${{ runner.temp }}/trail-cli-scale-scale-${{ github.run_id }} + + changed-path-ledger: + name: Changed-path ledger scale gate + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Build release binary + run: cargo build -p trail --release + - name: Run changed-path ledger benchmark + run: scripts/cli-scale-bench.sh changed-path-ledger + env: + TRAIL_BIN: ${{ github.workspace }}/target/release/trail + TRAIL_SCALE_BASE: ${{ runner.temp }} + TRAIL_SCALE_LABEL: changed-path-${{ github.run_id }} + TRAIL_SCALE_FILES: ${{ github.event.inputs.changed_path_ledger_files || '1000' }} + TRAIL_CHANGED_PATH_COW: "0" + - name: Gate changed-path ledger structure and oracle equality + shell: bash + run: | + root="${{ runner.temp }}/trail-cli-scale-changed-path-${{ github.run_id }}" + for scale in ${{ github.event.inputs.changed_path_ledger_files || '1000' }}; do + python3 scripts/check-changed-path-ledger-thresholds.py \ + "$root/$scale/results.tsv" \ + "$root/$scale/structural-metrics.jsonl" \ + --oracle "$root/$scale/oracle-results.tsv" \ + --max-seconds workspace_status=15 \ + --max-seconds workspace_diff=20 \ + --max-seconds workspace_record=30 \ + --max-seconds materialized_lane_record=120 \ + --max-seconds structured_patch=30 \ + --max-rss-bytes workspace_status=2147483648 \ + --max-rss-bytes workspace_diff=2147483648 \ + --max-rss-bytes workspace_record=2147483648 + done + - name: Upload changed-path ledger artifacts + uses: actions/upload-artifact@v4 + if: always() + with: + name: changed-path-ledger-scale-results + path: ${{ runner.temp }}/trail-cli-scale-changed-path-${{ github.run_id }} diff --git a/.gitignore b/.gitignore index 35d2750..f771eb7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /target/ /trail-vscode/ +/.worktrees/ **/*.rs.bk .DS_Store .venv/ diff --git a/.superpowers/sdd/task-6-allocation-approval-review.md b/.superpowers/sdd/task-6-allocation-approval-review.md new file mode 100644 index 0000000..ca20c97 --- /dev/null +++ b/.superpowers/sdd/task-6-allocation-approval-review.md @@ -0,0 +1,47 @@ +### Spec Compliance + +- Issues in schema validation and mkdir/open allocation binding. +- Cannot verify native/crash/durability results from diff. + +### Strengths + +- Journal ordering, retry convergence, retained quiescence, and earlier protections are strong. + +### Issues + +#### Critical + +None. + +#### Important + +1. New allocation/deletion tables are omitted from structural table/FK/row-level validation, and independent FKs do not enforce same scope/epoch/segment join. Malformed v18 can reach mutable open. Fix: complete column/PK/index/FK/row validation, global semantic join/state query, and byte-invariant read-only rejection tests for orphan/cross-wired rows. + +2. `mkdirat` then `openat` is separable; substituted directory can be adopted as Trail-owned. Fix: use creator-bound primitive, preferably nonce-keyed no-replace file quarantine directly under retained descriptor authority, and exact hook/regression between creation/open for old path. + +#### Minor + +None. + +### Assessment + +Task quality: Needs fixes. + +### Allocation approval fixes + +Both Important findings are fixed. + +1. Immutable v18 preflight now validates the allocation and deletion tables' exact columns, primary keys, named/partial unique indexes, and foreign keys. A single read-only semantic query rejects orphan allocations/deletions, non-bound allocations with deletion rows, bound allocations without exactly one deletion, cross-wired scope/epoch/segment/leaf/parent/inode identities, invalid allocation/deletion states, and duplicate active source ownership. New malformed orphan, cross-wired, and state-invalid fixtures prove rejection before mutable open and byte-for-byte invariance of the complete `.trail` tree. + +2. Retirement now journals the exact source inode and a nonce-keyed quarantine leaf directly under the retained scope directory, then uses same-directory atomic no-replace rename. Recovery reopens through the retained descriptor, requires exact journaled source/published inode identity, authenticates the full segment bytes, synchronizes the file and parent directory, and only then publishes allocation/binding state. Existing targets are retained and audited; unsupported no-replace platforms fail closed. The retirement path performs no pathname unlink or directory removal. + +Regression coverage includes the old mkdir/open substitution window, source substitution immediately before rename, target substitution immediately after rename, existing direct-target collisions, and two-segment SIGKILL/reopen recovery at the journal barrier, direct rename, exact verification, file/parent fsync, per-segment setup, transaction commit, and WAL boundaries. + +Verification: + +- `cargo test -p trail --lib`: 568 passed, 1 ignored. +- `cargo test -p trail --test changed_path_ledger_recovery`: 29 passed. +- `cargo test -p trail --release --test schema_v18_hard_cutover`: 9 passed. +- `cargo check -p trail --release --lib`: passed. +- `cargo test -p trail --test e2e`: 204 passed; the pre-existing newer-schema diagnostic wording assertion remains the sole failure. +- The recovery integration target itself is debug-only (`trail::test_support` is gated by `debug_assertions`), so it cannot be compiled as a release integration test; the production release library compiles cleanly and all hook-driven races run in debug. diff --git a/.superpowers/sdd/task-6-orphan-approval-review.md b/.superpowers/sdd/task-6-orphan-approval-review.md new file mode 100644 index 0000000..4966abd --- /dev/null +++ b/.superpowers/sdd/task-6-orphan-approval-review.md @@ -0,0 +1,48 @@ +### Spec Compliance + +- Issue in `recovery.rs:763-777` and `recovery.rs:779-854`. +- Cannot verify tests/native/power-loss behavior from diff. + +### Strengths + +- No production pathname removal; pre-existing namespaces retained/rejected; substitutions preserved. +- Earlier protections remain intact. + +### Issues + +#### Critical + +None. + +#### Important + +1. Quarantine directories are created before deletion rows/scope retirement are durable. Crash/error after mkdir but before durable row/WAL leaves deterministic Trail-owned orphan; retry treats it foreign and permanently rejects retirement. Fix: durably journal allocation ownership before filesystem creation with recoverable allocation state + unique attempt identity; never adopt unjournaled namespace; retain ambiguous abandoned namespaces while safely allocating a new journaled one. Add kill/error coverage after mkdir, before row insertion, between segments, before commit, before WAL barrier. + +#### Minor + +None. + +### Assessment + +Task quality: Needs fixes. + +### Orphan allocation review fix + +- Added a nonce-keyed `changed_path_segment_quarantine_allocations` journal with explicit + `allocating`, `allocated`, `bound`, and `abandoned` states. Every segment allocation is + committed and WAL-barriered before its quarantine directory is created. +- Recovery resumes only an absent journaled `allocating` namespace or an exact-identity + `allocated` namespace. A present namespace without a published identity, a missing or + mismatched allocated namespace, and a create collision are retained and durably marked + `abandoned`; a fresh nonce and directory leaf are then allocated. Pre-existing legacy + namespaces are likewise retained and recorded as abandoned instead of being adopted. +- The final deletion transaction binds each prepared deletion row to its exact allocation + nonce and filesystem identities. Retired-scope retry validates the allocation/deletion + join before reopening deletion authority. +- Extended the real SIGKILL matrix to two segments and covered the allocation journal + barrier, post-mkdir ambiguity, identity publication, the between-segment boundary, + completed allocation setup, pre-commit, post-commit, post-WAL, and the existing + quarantine/quiescence boundaries. The journal-barrier case injects foreign namespaces + and verifies they remain present and audited while fresh attempts complete. +- Verification: exact schema suite passed (8 tests), changed-path recovery integration + suite passed (29 tests), and the two-segment SIGKILL matrix passed all phases. diff --git a/CHANGELOG.md b/CHANGELOG.md index cc0d3be..751303b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,29 @@ All notable changes to Trail are documented in this file. Trail follows ## [Unreleased] +### Changed + +- **Breaking:** Trail CLI human output now uses the unified outcome-first + terminal renderer. The old human layouts and `--no-color` option are removed; + use `--color never` instead. +- **Breaking:** `trail merge-lane` is removed. Use + `trail lane merge --into ` for lane-specific merges; the + `trail merge` command remains for generic branch/ref merges. +- **Breaking:** `POST /v1/branches/{branch}/merge-lane` is removed. Use + `POST /v1/lanes/{lane}/merge` with the target branch in the required `into` + JSON field. +- **Breaking:** the generic merge queue is now lane-only. Use + `trail lane merge-queue`, `/v1/lanes/merges/queue`, and + `trail.lane_merge_queue_*`; the previous CLI, HTTP, MCP, resource, and + `merge_queue` storage contracts are removed without aliases. Generic + branches and refs continue through `trail merge`. +- Added `--format human|plain|json|ndjson`, `--color auto|always|never`, and + `--pager auto|always|never`. `plain` is deterministic text; JSON and NDJSON + are the supported contracts for automation. +- Status, diff, history, lane, agent, maintenance, and diagnostic output now + use responsive tables, ordered checklists, explicit notices, and safe next + actions. Human output is intentionally not stable for parsing. + ## [0.1.0] - 2026-07-10 ### Added diff --git a/Cargo.lock b/Cargo.lock index 14c3a84..68a80cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,7 +15,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -117,9 +119,9 @@ dependencies = [ [[package]] name = "arrayvec" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "askama" @@ -145,7 +147,7 @@ dependencies = [ "memchr", "proc-macro2", "quote", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "serde_derive", "syn", @@ -229,7 +231,7 @@ dependencies = [ "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http 0.61.1", - "aws-smithy-json 0.61.9", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -260,9 +262,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -271,14 +273,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -299,7 +302,7 @@ dependencies = [ "bytes-utils", "fastrand", "http 1.4.2", - "http-body 1.0.1", + "http-body 1.1.0", "percent-encoding", "pin-project-lite", "tracing", @@ -315,8 +318,8 @@ dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http 0.62.5", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -331,17 +334,15 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.102.0" +version = "1.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c82b3ac19f1431854f7ace3a7531674633e286bfdde21976893bfee36fd493b" +checksum = "858007b14d0f1ade2e0124473c2126b24d334dc9486ad12eb7c0ed14757be464" dependencies = [ - "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json 0.62.7", - "aws-smithy-observability", + "aws-smithy-http 0.62.5", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -349,24 +350,22 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.2", + "once_cell", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.104.0" +version = "1.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "321000d2b4c5519ee573f73167f612efd7329322d9b26969ad1979f0427f1913" +checksum = "b83abf3ae8bd10a014933cc2383964a12ca5a3ebbe1948ad26b1b808e7d0d1f2" dependencies = [ - "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json 0.62.7", - "aws-smithy-observability", + "aws-smithy-http 0.62.5", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -374,24 +373,22 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.2", + "once_cell", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sts" -version = "1.107.0" +version = "1.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0d328ba962af23ecfa3c9f23b98d3d35e325fa218d7f13d17a6bf522f8a560" +checksum = "74e8e9ac4a837859c8f1d747054172e1e55933f02ed34728b0b34dea0591ec84" dependencies = [ - "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json 0.62.7", - "aws-smithy-observability", + "aws-smithy-http 0.62.5", + "aws-smithy-json", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -400,7 +397,7 @@ dependencies = [ "aws-types", "fastrand", "http 0.2.12", - "http 1.4.2", + "once_cell", "regex-lite", "tracing", ] @@ -460,9 +457,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.62.6" +version = "0.62.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826141069295752372f8203c17f28e30c464d22899a43a0c9fd9c458d469c88b" +checksum = "445d5d720c99eed0b4aa674ed00d835d9b1427dd73e04adaf2f94c6b2d6f9fca" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -492,7 +489,7 @@ dependencies = [ "futures-core", "futures-util", "http 1.4.2", - "http-body 1.0.1", + "http-body 1.1.0", "http-body-util", "percent-encoding", "pin-project-lite", @@ -517,11 +514,11 @@ dependencies = [ "hyper 0.14.32", "hyper 1.10.1", "hyper-rustls 0.24.2", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.7", "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.41", + "rustls 0.23.42", "rustls-native-certs", "rustls-pki-types", "tokio", @@ -532,21 +529,10 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49fa1213db31ac95288d981476f78d05d9cbb0353d22cdf3472cc05bb02f6551" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-json" -version = "0.62.7" +version = "0.61.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" +checksum = "2db31f727935fc63c6eeae8b37b438847639ec330a9161ece694efba257e0c54" dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-schema", "aws-smithy-types", ] @@ -561,9 +547,9 @@ dependencies = [ [[package]] name = "aws-smithy-query" -version = "0.60.15" +version = "0.60.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" +checksum = "d28a63441360c477465f80c7abac3b9c4d075ca638f982e605b7dc2a2c7156c9" dependencies = [ "aws-smithy-types", "urlencoding", @@ -587,7 +573,7 @@ dependencies = [ "http 0.2.12", "http 1.4.2", "http-body 0.4.6", - "http-body 1.0.1", + "http-body 1.1.0", "http-body-util", "pin-project-lite", "pin-utils", @@ -648,7 +634,7 @@ dependencies = [ "http 0.2.12", "http 1.4.2", "http-body 0.4.6", - "http-body 1.0.1", + "http-body 1.1.0", "http-body-util", "itoa", "num-integer", @@ -663,9 +649,9 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.15" +version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +checksum = "eab77cdd036b11056d2a30a7af7b775789fb024bf216acc13884c6c97752ae56" dependencies = [ "xmlparser", ] @@ -687,9 +673,9 @@ dependencies = [ [[package]] name = "backon" -version = "1.6.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d" dependencies = [ "fastrand", "gloo-timers", @@ -714,9 +700,9 @@ dependencies = [ [[package]] name = "base64ct" -version = "1.8.3" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" [[package]] name = "basic-toml" @@ -783,11 +769,26 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "shlex 1.3.0", "syn", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -821,14 +822,20 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + [[package]] name = "bstr" -version = "1.12.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" dependencies = [ "memchr", - "serde", + "serde_core", ] [[package]] @@ -837,11 +844,17 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "d6aedf8ae72766347502cf3cb4f41cf5e9cc37d28bee90f1fdaaae15f9cf9424" [[package]] name = "byteorder" @@ -851,9 +864,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" dependencies = [ "serde", ] @@ -931,9 +944,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" dependencies = [ "find-msvc-tools", "jobserver", @@ -941,6 +954,12 @@ dependencies = [ "shlex 2.0.1", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -1000,9 +1019,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.5.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "52fa72306bb30daf11bc97773431628e5b4916e97aaa74b7d3f625d4d495da02" dependencies = [ "clap_builder", "clap_derive", @@ -1010,9 +1029,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.5.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "2071365c5c56eae7d77414029dde2f4f4ba151cf68d5a3261c9a40de428ace93" dependencies = [ "anstream", "anstyle", @@ -1022,9 +1041,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.5.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "dec5be1eea072311774b7b84ded287adbd9f293f9d23456817605c6042f4f5e0" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -1034,9 +1053,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "0e78417baa3b3114dc0e95e7357389a249c4da97c3c2b540700079db6171bfd7" [[package]] name = "cmake" @@ -1156,9 +1175,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.4.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" dependencies = [ "crc-catalog", ] @@ -1171,12 +1190,14 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc-fast" -version = "1.10.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +checksum = "2fd92aca2c6001b1bf5ba0ff84ee74ec8501b52bbef0cac80bf25a6c1d87a83d" dependencies = [ + "crc", "digest 0.10.7", - "spin 0.10.0", + "rustversion", + "spin 0.10.1", ] [[package]] @@ -1190,18 +1211,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1209,18 +1230,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] @@ -1237,9 +1258,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crypto-common" @@ -1279,6 +1300,33 @@ dependencies = [ "cmov", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "der" version = "0.7.10" @@ -1292,9 +1340,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] [[package]] name = "digest" @@ -1379,6 +1430,30 @@ dependencies = [ "winnow 0.6.26", ] +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.16.0" @@ -1388,6 +1463,15 @@ dependencies = [ "serde", ] +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -1397,6 +1481,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1453,7 +1557,7 @@ checksum = "c6f4a147ba57fcd64323c54c8f69fd4e045456a99574163010ccf5eea3168aaa" dependencies = [ "log", "once_cell", - "rand 0.9.4", + "rand 0.9.5", "tokio", ] @@ -1469,6 +1573,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fancy-regex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastant" version = "0.1.11" @@ -1485,6 +1600,12 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "figment" version = "0.10.19" @@ -1496,7 +1617,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "toml", + "toml 0.8.23", "uncased", "version_check", ] @@ -1537,6 +1658,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fluent-uri" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1918b65d96df47d3591bed19c5cca17e3fa5d0707318e4b5ef2eae01764df7e5" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1662,7 +1794,7 @@ dependencies = [ "mea", "parking_lot", "pin-project", - "rand 0.9.4", + "rand 0.9.5", "serde", "tracing", "twox-hash", @@ -1678,6 +1810,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "fraction" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fs-err" version = "2.11.0" @@ -1838,7 +1980,7 @@ dependencies = [ "gcloud-metadata", "home", "jsonwebtoken", - "reqwest 0.13.4", + "reqwest 0.13.3", "serde", "serde_json", "thiserror 2.0.18", @@ -1895,7 +2037,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd3152612316be627be52fe9ca72331eb48425059b3a6a700e7adde223e061d5" dependencies = [ - "reqwest 0.13.4", + "reqwest 0.13.3", "thiserror 2.0.18", "tokio", ] @@ -2171,11 +2313,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.12" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2212,9 +2354,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http 1.4.2", @@ -2222,14 +2364,14 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", "http 1.4.2", - "http-body 1.0.1", + "http-body 1.1.0", "pin-project-lite", ] @@ -2247,9 +2389,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" @@ -2296,7 +2438,7 @@ dependencies = [ "futures-core", "h2 0.4.15", "http 1.4.2", - "http-body 1.0.1", + "http-body 1.1.0", "httparse", "itoa", "pin-project-lite", @@ -2322,15 +2464,16 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.4.2", "hyper 1.10.1", "hyper-util", - "rustls 0.23.41", + "rustls 0.23.42", "rustls-native-certs", + "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", "tower-service", @@ -2361,13 +2504,13 @@ dependencies = [ "futures-channel", "futures-util", "http 1.4.2", - "http-body 1.0.1", + "http-body 1.1.0", "hyper 1.10.1", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.4", + "socket2 0.6.5", "tokio", "tower-service", "tracing", @@ -2385,7 +2528,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core", ] [[package]] @@ -2399,23 +2542,21 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" dependencies = [ "displaydoc", - "potential_utf", - "utf8_iter", "yoke", "zerofrom", "zerovec", ] [[package]] -name = "icu_locale_core" -version = "2.2.0" +name = "icu_locid" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" dependencies = [ "displaydoc", "litemap", @@ -2424,61 +2565,99 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" + [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" dependencies = [ + "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", "icu_provider", "smallvec", + "utf16_iter", + "utf8_iter", + "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" [[package]] name = "icu_properties" -version = "2.2.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" dependencies = [ + "displaydoc", "icu_collections", - "icu_locale_core", + "icu_locid_transform", "icu_properties_data", "icu_provider", - "zerotrie", + "tinystr", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" [[package]] name = "icu_provider" -version = "2.2.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" dependencies = [ "displaydoc", - "icu_locale_core", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", "writeable", "yoke", "zerofrom", - "zerotrie", "zerovec", ] +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "idna" version = "1.1.0" @@ -2492,9 +2671,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.2" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" dependencies = [ "icu_normalizer", "icu_properties", @@ -2502,9 +2681,9 @@ dependencies = [ [[package]] name = "ignore" -version = "0.4.26" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b915661dd01db3f05050265b2477bcc6527b3792388e2749b41623cc592be67d" +checksum = "2adf14691c72bcfc1058740436a35bdd3ae9c07d1a941ef00b749e9ea16aefa7" dependencies = [ "crossbeam-deque", "globset", @@ -2518,12 +2697,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown 0.17.1", + "hashbrown 0.16.1", "serde", "serde_core", ] @@ -2547,9 +2726,9 @@ dependencies = [ [[package]] name = "inotify-sys" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d" dependencies = [ "libc", ] @@ -2621,32 +2800,27 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jni" -version = "0.22.4" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" dependencies = [ + "cesu8", "cfg-if", "combine", - "jni-macros", - "jni-sys", + "jni-sys 0.3.1", "log", - "simd_cesu8", - "thiserror 2.0.18", + "thiserror 1.0.69", "walkdir", - "windows-link 0.2.1", + "windows-sys 0.45.0", ] [[package]] -name = "jni-macros" -version = "0.22.4" +name = "jni-sys" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" dependencies = [ - "proc-macro2", - "quote", - "rustc_version", - "simd_cesu8", - "syn", + "jni-sys 0.4.1", ] [[package]] @@ -2689,6 +2863,30 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonschema" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "161c33c3ec738cfea3288c5c53dfcdb32fd4fc2954de86ea06f71b5a1a40bfcd" +dependencies = [ + "ahash", + "base64", + "bytecount", + "email_address", + "fancy-regex", + "fraction", + "idna", + "itoa", + "num-cmp", + "once_cell", + "percent-encoding", + "referencing", + "regex-syntax", + "serde", + "serde_json", + "uuid-simd", +] + [[package]] name = "jsonwebtoken" version = "10.4.0" @@ -2727,13 +2925,24 @@ dependencies = [ "libc", ] +[[package]] +name = "landlock" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "635839550ae8b90d9fd2571460a6645dc0aec070225956ca7a2831ed31d2795d" +dependencies = [ + "enumflags2", + "libc", + "thiserror 2.0.18", +] + [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.9.8", + "spin 0.9.9", ] [[package]] @@ -2822,9 +3031,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" [[package]] name = "lock_api" @@ -2843,9 +3052,9 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9" +checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6" dependencies = [ "hashbrown 0.17.1", ] @@ -2906,9 +3115,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memoffset" @@ -2955,9 +3164,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -2991,9 +3200,9 @@ dependencies = [ [[package]] name = "napi-build" -version = "2.3.2" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9c366d2c8c60b86fa632df75f745509b52f9128f91a6bad4c796e44abb505e1" +checksum = "dcae8ad5609d14afb3a3b91dee88c757016261b151e9dcecabf1b2a31a6cab14" [[package]] name = "napi-derive" @@ -3111,11 +3320,25 @@ dependencies = [ "winapi", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -3132,19 +3355,34 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.6", + "rand 0.8.7", "smallvec", "zeroize", ] [[package]] -name = "num-conv" -version = "0.2.2" +name = "num-cmp" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" [[package]] -name = "num-derive" +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-derive" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" @@ -3165,11 +3403,21 @@ dependencies = [ [[package]] name = "num-iter" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" dependencies = [ - "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", "num-integer", "num-traits", ] @@ -3239,8 +3487,8 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.10.1", - "reqwest 0.13.4", + "rand 0.10.2", + "reqwest 0.13.3", "rustls-pki-types", "serde", "serde_json", @@ -3461,15 +3709,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" -[[package]] -name = "potential_utf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" -dependencies = [ - "zerovec", -] - [[package]] name = "powerfmt" version = "0.2.0" @@ -3513,6 +3752,7 @@ version = "0.1.0" dependencies = [ "hex", "prolly-map", + "prolly-store-sqlite", "serde", "serde_json", "thiserror 1.0.69", @@ -3521,18 +3761,15 @@ dependencies = [ [[package]] name = "prolly-map" -version = "0.1.0" +version = "0.2.0" dependencies = [ "futures-util", - "hex", + "js-sys", "rayon", - "rocksdb", - "rusqlite", "serde", "serde_cbor", "serde_json", "sha2 0.10.9", - "slatedb", "tokio", "xxhash-rust", ] @@ -3598,6 +3835,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "prolly-store-pglite" +version = "0.1.0" +dependencies = [ + "hex", + "prolly-map", + "prolly-store-test", + "serde_json", +] + [[package]] name = "prolly-store-postgres" version = "0.1.0" @@ -3616,6 +3863,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "prolly-store-rocksdb" +version = "0.1.0" +dependencies = [ + "prolly-map", + "prolly-store-test", + "rocksdb", +] + +[[package]] +name = "prolly-store-slatedb" +version = "0.1.0" +dependencies = [ + "futures-util", + "prolly-map", + "prolly-store-test", + "slatedb", + "tokio", +] + [[package]] name = "prolly-store-spanner" version = "0.1.0" @@ -3626,6 +3893,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "prolly-store-sqlite" +version = "0.1.0" +dependencies = [ + "libc", + "prolly-map", + "prolly-store-test", + "rusqlite", + "tempfile", +] + +[[package]] +name = "prolly-store-test" +version = "0.1.0" +dependencies = [ + "prolly-map", +] + [[package]] name = "prolly-wasm" version = "0.1.0" @@ -3636,11 +3921,31 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "proptest" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb0be07becd10686a0bb407298fb425360a5c44a663774406340c59a22de4ce" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.13.0", + "lazy_static", + "num-traits", + "rand 0.9.5", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "prost" -version = "0.14.4" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" dependencies = [ "bytes", "prost-derive", @@ -3648,9 +3953,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.4" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", "itertools 0.14.0", @@ -3661,13 +3966,19 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.4" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" dependencies = [ "prost", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quick-xml" version = "0.40.1" @@ -3680,18 +3991,18 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.11" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ "bytes", "cfg_aliases", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.2", - "rustls 0.23.41", - "socket2 0.6.4", + "rustc-hash 2.1.3", + "rustls 0.23.42", + "socket2 0.6.5", "thiserror 2.0.18", "tokio", "tracing", @@ -3700,18 +4011,18 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.15" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.4", + "rand 0.9.5", "ring", - "rustc-hash 2.1.2", - "rustls 0.23.41", + "rustc-hash 2.1.3", + "rustls 0.23.42", "rustls-pki-types", "slab", "thiserror 2.0.18", @@ -3729,7 +4040,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.4", + "socket2 0.6.5", "tracing", "windows-sys 0.60.2", ] @@ -3757,9 +4068,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -3768,9 +4079,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -3778,9 +4089,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -3831,6 +4142,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "rand_xoshiro" version = "0.7.0" @@ -3879,7 +4199,7 @@ dependencies = [ "pin-project-lite", "ryu", "sha1_smol", - "socket2 0.6.4", + "socket2 0.6.5", "tokio", "tokio-util", "url", @@ -3903,11 +4223,45 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "referencing" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a64b3a635fad9000648b4d8a59c8710c523ab61a23d392a7d91d47683f5adc" +dependencies = [ + "ahash", + "fluent-uri", + "once_cell", + "parking_lot", + "percent-encoding", + "serde_json", +] + [[package]] name = "regex" -version = "1.12.4" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" dependencies = [ "aho-corasick", "memchr", @@ -3917,9 +4271,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" dependencies = [ "aho-corasick", "memchr", @@ -3950,17 +4304,17 @@ dependencies = [ "futures-core", "futures-util", "http 1.4.2", - "http-body 1.0.1", + "http-body 1.1.0", "http-body-util", "hyper 1.10.1", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.7", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.41", + "rustls 0.23.42", "rustls-pki-types", "serde", "serde_json", @@ -3980,9 +4334,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" dependencies = [ "base64", "bytes", @@ -3991,10 +4345,10 @@ dependencies = [ "futures-util", "h2 0.4.15", "http 1.4.2", - "http-body 1.0.1", + "http-body 1.1.0", "http-body-util", "hyper 1.10.1", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.7", "hyper-util", "js-sys", "log", @@ -4002,7 +4356,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.41", + "rustls 0.23.42", "rustls-pki-types", "rustls-platform-verifier", "serde", @@ -4098,9 +4452,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -4138,9 +4492,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "log", @@ -4166,9 +4520,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "web-time", "zeroize", @@ -4176,16 +4530,16 @@ dependencies = [ [[package]] name = "rustls-platform-verifier" -version = "0.7.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" dependencies = [ "core-foundation", "core-foundation-sys", "jni", "log", "once_cell", - "rustls 0.23.41", + "rustls 0.23.42", "rustls-native-certs", "rustls-platform-verifier-android", "rustls-webpki 0.103.13", @@ -4225,9 +4579,21 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] [[package]] name = "ryu" @@ -4291,9 +4657,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.7.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" dependencies = [ "bitflags 2.13.0", "core-foundation", @@ -4332,6 +4698,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_cbor" version = "0.11.2" @@ -4384,6 +4760,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" +dependencies = [ + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4411,9 +4796,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", "cpufeatures 0.2.17", @@ -4482,25 +4867,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" - -[[package]] -name = "simd_cesu8" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" -dependencies = [ - "rustc_version", - "simdutf8", -] - -[[package]] -name = "simdutf8" -version = "0.1.5" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] name = "similar" @@ -4510,9 +4879,9 @@ checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "simple_asn1" -version = "0.6.4" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", @@ -4534,9 +4903,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "slatedb" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01d14107ae93175aff71ab4174a09da323cc95c4cddd7ee253ae1649211d3d9" +checksum = "e6aba890a8fc5d2a3d283a77cfe712c0f27f0501bd75b07e14dece2dfc2cde86" dependencies = [ "async-channel", "async-trait", @@ -4559,7 +4928,7 @@ dependencies = [ "object_store", "ouroboros", "parking_lot", - "rand 0.9.4", + "rand 0.9.5", "serde", "serde_json", "siphasher", @@ -4579,14 +4948,14 @@ dependencies = [ [[package]] name = "slatedb-common" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7696aae130a0a617ec44e7c9e731d6f5408f58d949a91ba5082fcab84a896be" +checksum = "7f3fdad5676b104bec5daa2f367bdb340f4162ef3c1ead16da903297f235fb49" dependencies = [ "chrono", "log", "object_store", - "rand 0.9.4", + "rand 0.9.5", "rand_xoshiro", "serde", "thread_local", @@ -4595,9 +4964,9 @@ dependencies = [ [[package]] name = "slatedb-txn-obj" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a5748b15b53b41beac51663a6b05168ac2af1a7abcada92b7720a483beddd30" +checksum = "5df001443952d76ae7380536b4a050f88b3cc9a7d3dce0f3782ea7ae9a12c29c" dependencies = [ "async-trait", "bytes", @@ -4643,9 +5012,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -4653,15 +5022,15 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" [[package]] name = "spin" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "spki" @@ -4794,7 +5163,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rand 0.8.6", + "rand 0.8.7", "rsa", "serde", "sha1", @@ -4832,7 +5201,7 @@ dependencies = [ "md-5 0.10.6", "memchr", "once_cell", - "rand 0.8.6", + "rand 0.8.7", "serde", "serde_json", "sha2 0.10.9", @@ -4881,9 +5250,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -4942,12 +5311,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", ] +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "textwrap" version = "0.16.2" @@ -4999,38 +5378,39 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] [[package]] name = "time" -version = "0.3.51" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", + "itoa", "num-conv", "powerfmt", - "serde_core", + "serde", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.9" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.30" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ "num-conv", "time-core", @@ -5038,9 +5418,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" dependencies = [ "displaydoc", "zerovec", @@ -5048,9 +5428,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -5078,11 +5458,11 @@ checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", - "mio 1.2.1", + "mio 1.2.2", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.4", + "socket2 0.6.5", "tokio-macros", "windows-sys 0.61.2", ] @@ -5100,9 +5480,9 @@ dependencies = [ [[package]] name = "tokio-retry2" -version = "0.6.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e05e40fd65880de11383bbc693d3c636045ed1b5010ef1265a1d2b41d73334a1" +checksum = "7de2537bbad4f8b2d4237cdab9e7c4948a1f74744e45f54144eeccd05d3ad955" dependencies = [ "pin-project", "tokio", @@ -5124,7 +5504,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.41", + "rustls 0.23.42", "tokio", ] @@ -5161,11 +5541,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", - "serde_spanned", - "toml_datetime", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", "toml_edit", ] +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned 1.0.4", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + [[package]] name = "toml_datetime" version = "0.6.11" @@ -5175,6 +5570,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.22.27" @@ -5183,30 +5587,45 @@ checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap", "serde", - "serde_spanned", - "toml_datetime", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", "toml_write", "winnow 0.7.15", ] +[[package]] +name = "toml_parser" +version = "1.0.10+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420" +dependencies = [ + "winnow 1.0.4", +] + [[package]] name = "toml_write" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "toml_writer" +version = "1.0.7+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17aaa1c6e3dc22b1da4b6bba97d066e354c7945cac2f7852d4e4e7ca7a6b56d" + [[package]] name = "tonic" -version = "0.14.6" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" dependencies = [ "async-trait", "base64", "bytes", "flate2", "http 1.4.2", - "http-body 1.0.1", + "http-body 1.1.0", "http-body-util", "hyper 1.10.1", "hyper-timeout", @@ -5226,9 +5645,9 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.6" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" dependencies = [ "bytes", "prost", @@ -5264,7 +5683,7 @@ dependencies = [ "bytes", "futures-util", "http 1.4.2", - "http-body 1.0.1", + "http-body 1.1.0", "pin-project-lite", "tower", "tower-layer", @@ -5320,20 +5739,31 @@ dependencies = [ name = "trail" version = "0.1.0" dependencies = [ + "anstyle", "async-trait", "bzip2 0.5.2", "clap", + "core-foundation-sys", "dokan", "dokan-sys", + "ed25519-dalek", "flate2", + "fsevent-sys", "fuser", "getrandom 0.2.17", + "globset", "hex", "ignore", + "inotify", + "jsonschema", + "landlock", "libc", "nfsserve", "notify", "prolly-map", + "prolly-store-slatedb", + "prolly-store-sqlite", + "proptest", "rayon", "reqwest 0.12.28", "rusqlite", @@ -5347,16 +5777,35 @@ dependencies = [ "sqlite-vec", "tar", "tempfile", + "terminal_size", "thiserror 1.0.69", + "time", "tokio", - "toml", + "toml 0.8.23", + "trail-environment-adapter-sdk", "unicode-normalization", + "unicode-width", + "url", "walkdir", "widestring", "winapi", "zip", ] +[[package]] +name = "trail-environment-adapter-sdk" +version = "0.1.0" +dependencies = [ + "ed25519-dalek", + "hex", + "serde", + "serde_bytes", + "serde_cbor", + "sha2 0.10.9", + "thiserror 1.0.69", + "toml 0.8.23", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -5369,7 +5818,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" dependencies = [ - "rand 0.9.4", + "rand 0.9.5", ] [[package]] @@ -5384,11 +5833,17 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "rand 0.9.4", + "rand 0.9.5", "serde", "web-time", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "uncased" version = "0.9.10" @@ -5427,9 +5882,15 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-segmentation" -version = "1.13.3" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "uniffi" @@ -5464,7 +5925,7 @@ dependencies = [ "serde", "tempfile", "textwrap", - "toml", + "toml 0.9.12+spec-1.1.0", "uniffi_internal_macros", "uniffi_meta", "uniffi_pipeline", @@ -5509,7 +5970,7 @@ dependencies = [ "quote", "serde", "syn", - "toml", + "toml 0.9.12+spec-1.1.0", "uniffi_meta", ] @@ -5586,6 +6047,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -5600,16 +6067,27 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" dependencies = [ - "getrandom 0.4.3", + "getrandom 0.3.4", "js-sys", "serde_core", "wasm-bindgen", ] +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "uuid", + "vsimd", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -5628,6 +6106,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -5837,7 +6324,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ "windows-collections", - "windows-core 0.61.2", + "windows-core", "windows-future", "windows-link 0.1.3", "windows-numerics", @@ -5849,7 +6336,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" dependencies = [ - "windows-core 0.61.2", + "windows-core", ] [[package]] @@ -5861,21 +6348,8 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link 0.1.3", - "windows-result 0.3.4", - "windows-strings 0.4.2", -] - -[[package]] -name = "windows-core" -version = "0.62.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", + "windows-result", + "windows-strings", ] [[package]] @@ -5884,7 +6358,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ - "windows-core 0.61.2", + "windows-core", "windows-link 0.1.3", "windows-threading", ] @@ -5929,7 +6403,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ - "windows-core 0.61.2", + "windows-core", "windows-link 0.1.3", ] @@ -5942,15 +6416,6 @@ dependencies = [ "windows-link 0.1.3", ] -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link 0.2.1", -] - [[package]] name = "windows-strings" version = "0.4.2" @@ -5961,12 +6426,12 @@ dependencies = [ ] [[package]] -name = "windows-strings" -version = "0.5.1" +name = "windows-sys" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-link 0.2.1", + "windows-targets 0.42.2", ] [[package]] @@ -6014,6 +6479,21 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -6071,6 +6551,12 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -6089,6 +6575,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -6107,6 +6599,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -6137,6 +6635,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -6155,6 +6659,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -6173,6 +6683,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -6191,6 +6707,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -6227,17 +6749,29 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + [[package]] name = "writeable" -version = "0.6.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" [[package]] name = "xattr" @@ -6257,9 +6791,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.15" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72" [[package]] name = "yansi" @@ -6269,10 +6803,11 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.8.3" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ + "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -6280,9 +6815,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.2" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", @@ -6292,18 +6827,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", @@ -6333,40 +6868,29 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.9.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.5.0" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "zerotrie" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - [[package]] name = "zerovec" -version = "0.11.6" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" dependencies = [ "yoke", "zerofrom", @@ -6375,9 +6899,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", @@ -6399,9 +6923,9 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zstd" diff --git a/Cargo.toml b/Cargo.toml index 5fbfcee..f766574 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,22 +1,28 @@ [workspace] members = [ "trail", + "trail-environment-adapter-sdk", "prolly", "prolly/stores/prolly-store-cosmosdb", "prolly/stores/prolly-store-dynamodb", "prolly/stores/prolly-store-mysql", + "prolly/stores/prolly-store-pglite", "prolly/stores/prolly-store-postgres", "prolly/stores/prolly-store-redis", + "prolly/stores/prolly-store-rocksdb", + "prolly/stores/prolly-store-slatedb", + "prolly/stores/prolly-store-sqlite", "prolly/stores/prolly-store-spanner", + "prolly/stores/prolly-store-test", "prolly/bindings/node/native", "prolly/bindings/uniffi", "prolly/bindings/wasm", ] -resolver = "2" +resolver = "3" [workspace.package] -edition = "2021" -rust-version = "1.81" +edition = "2024" +rust-version = "1.89" version = "0.1.0" license = "MIT OR Apache-2.0" authors = ["Crab Build"] @@ -45,6 +51,9 @@ dist = false publish-prereleases = false # Which actions to run on pull requests pr-run-mode = "plan" +# Block every cargo-dist release phase on Linux and macOS native ledger gates +# executed against the exact tag SHA. +plan-jobs = ["./changed-path-ledger-native"] # Checksums to generate for each App checksum = "sha256" # Whether to enable GitHub Attestations @@ -58,13 +67,16 @@ install-updater = false [workspace.dependencies] bzip2 = "0.5" +ed25519-dalek = "2.2" flate2 = "1.0" rayon = "1.10" rocksdb = "0.22" reqwest = { version = "0.12.28", default-features = false, features = ["blocking", "rustls-tls"] } serde = { version = "1.0", features = ["derive"] } +serde_bytes = "0.11" serde_cbor = "0.11" serde_json = "1.0" +globset = "0.4" sha2 = "0.10" rusqlite = { version = "0.31", features = ["bundled"] } sqlite-vec = "=0.1.9" @@ -75,8 +87,13 @@ xxhash-rust = { version = "0.8", features = ["xxh64"] } clap = { version = "4.5", features = ["derive"] } hex = "0.4" ignore = "0.4" +jsonschema = { version = "0.29.1", default-features = false } libc = "0.2" +landlock = "0.4.5" notify = "6.1" +inotify = { version = "0.9.6", default-features = false } +fsevent-sys = "4.1.0" +core-foundation-sys = "0.8.7" getrandom = "0.2" similar = "2.6" tempfile = "3.10" @@ -85,6 +102,11 @@ thiserror = "1.0" tokio = { version = "1.45", features = ["rt-multi-thread", "time"] } toml = "0.8" unicode-normalization = "0.1" +unicode-width = "0.2.2" +url = "2.5" +terminal_size = "0.4.4" +anstyle = "1.0.14" +time = { version = "=0.3.36", features = ["formatting"] } walkdir = "2.5" zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } uniffi = "=0.31.0" diff --git a/Makefile b/Makefile index afaf756..48411d4 100644 --- a/Makefile +++ b/Makefile @@ -326,8 +326,9 @@ info: ## Show build environment info .PHONY: toolchain toolchain: ## Install/verify the correct Rust toolchain - @rustup show active-toolchain 2>/dev/null || rustup toolchain install stable - @rustup component add clippy rustfmt 2>/dev/null || true + @rustup update stable + @rustup default stable + @rustup component add clippy rustfmt --toolchain stable @printf " $(CHECK) toolchain ready\n" # ── CI ───────────────────────────────────────────────────────────── diff --git a/README.md b/README.md index 700846b..1c5da93 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,30 @@ provenance, and lane-coordination layer. Use Git for durable shared source-control history. Use Trail for the messy, high-frequency, local work that happens before a commit is ready. +## Terminal output and automation + +Trail's default terminal output is designed for people: it leads with the +outcome, shows changed paths/checks in responsive layouts, and ends with the +safest next action when one exists. It deliberately is not a parsing API. + +```sh +# Adaptive interactive output (the default). +trail status +trail diff --dirty --patch + +# Deterministic log-friendly text. +trail --format plain lane readiness fix-login + +# Stable automation contracts. +trail --format json status +trail --format ndjson index watch --once +``` + +Use `--color auto|always|never` and `--pager auto|always|never` to control +terminal behavior. The former `--no-color` option is intentionally removed; +use `--color never`. See [the terminal output contract](docs/CLI_TERMINAL_OUTPUT.md) +for the complete behavior matrix and the explicitly raw content modes. + | Need | Git | Trail | | --- | --- | --- | | Shared project history | Excellent: commits, branches, remotes, tags, PR workflows | Complements Git and can import/export mappings | @@ -142,7 +166,7 @@ trail lane spawn fix-login --from main --materialize=true # Work is isolated in the lane until it is reviewed and validated. trail lane record fix-login -m "Fix login validation" trail lane readiness fix-login -trail merge-lane fix-login --into main --dry-run +trail lane merge fix-login --into main --dry-run ``` ## Why This Matters for AI Agents @@ -435,7 +459,8 @@ The Windows binary currently links the Dokany 2.0.6 runtime. Linux FUSE, macFUSE, and the Dokan driver are otherwise relevant only to their corresponding mounted-workspace modes. -To build from source instead, install Rust 1.81 or newer and use the Makefile: +To build from source instead, install Rust 1.89 or newer and use the Makefile. +Trail's parent-owned crates use Rust edition 2024: ```sh # Build the debug binary at target/debug/trail. @@ -460,9 +485,9 @@ For ACP coding-agent setup, keep installation simple and use the guided Trail commands after the binary is installed: ```sh -trail agent doctor --provider claude-code -trail agent doctor --provider codex -trail agent setup +trail agent doctor claude-code +trail agent doctor codex +trail agent acp setup claude-code --editor vscode ``` For a project-local install directory, override `PREFIX`: @@ -531,7 +556,7 @@ trail lane workdir docs-lane trail lane record docs-lane -m "record docs update" trail lane diff docs-lane --patch trail lane readiness docs-lane -trail merge-lane docs-lane --into main --dry-run +trail lane merge docs-lane --into main --dry-run ``` Example CLI output from a tiny workspace looks like this. IDs, object hashes, @@ -541,12 +566,13 @@ workspace IDs, and actor names will differ on your machine. | Prefix | Meaning | Example use | | --- | --- | --- | -| `wk_` | Workspace ID derived when `.trail/` is initialized | Identifies one local Trail workspace | -| `ch_` | Change/operation ID allocated when Trail records an operation | Appears as `Head`, `Initial operation`, timeline entries, and `show` selectors | -| `obj_` | Content-addressed object ID | Identifies stored roots, operations, text objects, blobs, and other durable objects | -| `msg_` | Message ID | Used for durable operation, agent, or review messages | -| `anc_` | Anchor ID | Used for durable labels tied to file and line identity | -| `ch_...:` | Stable file or line identity with an origin change and local sequence | Appears in `why` output as a `Line ID` | +| `workspace_` | Workspace ID derived when `.trail/` is initialized | Identifies one local Trail workspace | +| `change_` | Change/operation ID allocated when Trail records an operation | Appears as `Head`, `Initial operation`, timeline entries, checkpoints, and `show` selectors | +| `object_` | Content-addressed object ID | Identifies stored roots, operations, text objects, blobs, and other durable objects | +| `message_` | Message ID | Used for durable operation, agent, or review messages | +| `anchor_` | Anchor ID | Used for durable labels tied to file and line identity | +| `checkpoint_` | User-facing checkpoint alias | Resolves to the underlying `change_` operation and its resulting `object_` root | +| `line_...:` | User-facing stable line identity | Resolves to an origin `change_` plus its local sequence | ## Example output @@ -557,9 +583,9 @@ workspace IDs, and actor names will differ on your machine. ```text $ trail init --working-tree Initialized Trail workspace -Workspace: wk_24ec99f68d1db8716f4df8a87580e3da +Workspace: workspace_24ec99f68d1db8716f4df8a87580e3da Branch: main -Initial operation: ch_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 +Initial operation: change_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 Imported: 1 files (1 text, 0 opaque, 0 binary) ``` @@ -569,8 +595,8 @@ Imported: 1 files (1 text, 0 opaque, 0 binary) ```text $ trail status Branch: main -Head: ch_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 -Root: obj_46b1a72c6ff5e66a7b3026113243681493e79c2e659b6ef9658a2db57fdac431 +Head: change_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 +Root: object_46b1a72c6ff5e66a7b3026113243681493e79c2e659b6ef9658a2db57fdac431 Worktree: clean ``` @@ -580,8 +606,8 @@ Worktree: clean ```text $ trail status Branch: main -Head: ch_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 -Root: obj_46b1a72c6ff5e66a7b3026113243681493e79c2e659b6ef9658a2db57fdac431 +Head: change_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 +Root: object_46b1a72c6ff5e66a7b3026113243681493e79c2e659b6ef9658a2db57fdac431 Worktree: dirty Modified README.md ``` @@ -591,7 +617,7 @@ Worktree: dirty ```text $ trail record -m "record current edit" -Recorded ch_3d5a38ae49a7cd4b6873f003c97863f30ebc3efa61749b463c222d5d34809bfa +Recorded change_3d5a38ae49a7cd4b6873f003c97863f30ebc3efa61749b463c222d5d34809bfa Modified README.md ``` @@ -600,24 +626,24 @@ Recorded ch_3d5a38ae49a7cd4b6873f003c97863f30ebc3efa61749b463c222d5d34809bfa ```text $ trail timeline --limit 10 -ch_3d5a38ae49a7cd4b6873f003c97863f30ebc3efa61749b463c222d5d34809bfa ManualRecord main record current edit -ch_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 GitImport main Initialize Trail workspace +change_3d5a38ae49a7cd4b6873f003c97863f30ebc3efa61749b463c222d5d34809bfa ManualRecord main record current edit +change_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 GitImport main Initialize Trail workspace ``` 6. Inspect one operation from the timeline. `show` expands the operation kind, actor, message, parent, before/after roots, and path-level summary. ```text -$ trail show ch_3d5a38ae49a7cd4b6873f003c97863f30ebc3efa61749b463c222d5d34809bfa -Operation: ch_3d5a38ae49a7cd4b6873f003c97863f30ebc3efa61749b463c222d5d34809bfa +$ trail show change_3d5a38ae49a7cd4b6873f003c97863f30ebc3efa61749b463c222d5d34809bfa +Operation: change_3d5a38ae49a7cd4b6873f003c97863f30ebc3efa61749b463c222d5d34809bfa Kind: ManualRecord Branch: main Actor: demo Message: record current edit Parents: - ch_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 -Before root: obj_46b1a72c6ff5e66a7b3026113243681493e79c2e659b6ef9658a2db57fdac431 -After root: obj_7e39865c8542fe846b528c28debed69daecc4b53c34ff17f8a4da8bacbb773a4 + change_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 +Before root: object_46b1a72c6ff5e66a7b3026113243681493e79c2e659b6ef9658a2db57fdac431 +After root: object_7e39865c8542fe846b528c28debed69daecc4b53c34ff17f8a4da8bacbb773a4 Modified README.md (+1 -0) ``` @@ -628,9 +654,9 @@ After root: obj_7e39865c8542fe846b528c28debed69daecc4b53c34ff17f8a4da8bacbb773a4 ```text $ trail why README.md:2 README.md:2 First recorded line -Line ID: ch_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6:2 -Introduced by: ch_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 -Last content change: ch_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 +Line ID: line_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6:2 +Introduced by: change_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 +Last content change: change_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 ``` 8. Show file history. `history` lists operations that affected the selected @@ -639,8 +665,8 @@ Last content change: ch_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f ```text $ trail history README.md README.md -ch_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 Added README.md -ch_3d5a38ae49a7cd4b6873f003c97863f30ebc3efa61749b463c222d5d34809bfa Modified README.md +change_5a44178a04acec35b4c27590303d665d462a229aa9bf627bb24e2c0f685fdcd6 Added README.md +change_3d5a38ae49a7cd4b6873f003c97863f30ebc3efa61749b463c222d5d34809bfa Modified README.md ``` 9. After making another edit without recording it, inspect the dirty diff. The @@ -687,7 +713,7 @@ install it with `make install` or replace `trail` with `target/debug/trail`. | `trail merge --into --dry-run` | Preview a branch merge and possible conflicts | | `trail ignore check ` | Check whether ignore policy records or skips a path | | `trail guardrails check --action ` | Preflight a risky action against workspace policy | -| `trail agent setup` | Print editor config for fresh Trail agent tasks | +| `trail agent acp setup claude-code --editor vscode` | Print editor config for fresh Trail agent tasks | | `trail agent continue latest` | Start a fresh follow-up task from the latest task checkpoint | | `trail agent` | Open the agent inbox home view with grouped tasks, review-first hints, and one next action | | `trail agent board` | Show a multi-agent board with low-noise columns and one next action | @@ -777,25 +803,35 @@ install it with `make install` or replace `trail` with `target/debug/trail`. | `trail agent undo latest` | Undo the latest agent turn without copying checkpoint ids | | `trail agent undo-last latest` | Friendly alias for undoing the latest agent turn | | `trail lane spawn --from ` | Create an isolated lane branch | -| `trail agent start --provider codex --workdir-mode nfs-cow` | Run a macOS terminal agent in a loopback NFS copy-on-write workdir | +| `trail agent start codex --workdir-mode nfs-cow` | Run a macOS terminal agent in a loopback NFS copy-on-write workdir | | `trail lane apply-patch --patch ` | Apply a structured patch to a lane branch | | `trail lane review ` | Produce a compact review packet for a lane branch | | `trail lane readiness ` | Report blockers before merging a lane branch | | `trail lane handoff ` | Produce a review and continuation packet for a lane | -| `trail merge-lane --into --dry-run` | Preview merging a lane branch into a target branch | -| `trail merge-queue run` | Run queued lane merges with readiness and conflict checks | +| `trail lane merge --into --dry-run` | Preview merging a lane branch into a target branch | +| `trail lane merge-queue run` | Run queued lane merges with readiness and conflict checks | | `trail daemon` | Start the loopback HTTP daemon for editor and automation integrations | | `trail mcp` | Start the MCP stdio server for agent hosts | -| `trail acp list` | List built-in aliases and current official ACP registry agents | -| `trail acp install --agent claude-code` | Print an ACP relay command and editor snippet | -| `trail acp install --agent codex` | Print the Codex ACP relay command and editor snippet | -| `trail acp doctor --agent claude-code` | Check ACP provider and relay readiness | -| `trail acp sessions` | List captured ACP sessions | +| `trail agent acp status` | List built-in aliases and current official ACP registry agents | +| `trail agent acp setup claude-code --editor generic` | Print a generic Claude Code ACP entry | +| `trail agent acp setup codex --editor zed` | Preview the Codex entry for Zed; add `--yes` to apply it | +| `trail agent acp doctor claude-code` | Report provider readiness and verifiable ACP v1 compatibility evidence | +| `trail agent acp sessions` | List captured ACP sessions | | `trail transcript ` | Read captured prompts, assistant messages, tools, and checkpoints | | `trail doctor` | Run workspace and integration diagnostics | | `trail backup create ` | Create a Trail workspace backup | | `trail fsck` | Verify repository integrity | +Trail is a transparent ACP v1 stdio compatibility layer between an editor and +an agent. It covers all 23 stable ACP v1 methods while adding isolated lanes, +durable transcripts, provenance, and optional Trail MCP injection. Run +`trail agent acp doctor ` for the exact wire version, pinned schema +digests, build-attestation status, capture-journal health, and path-mapping +health. Editor setup and agent installation/authentication are reported +separately from protocol conformance. See +[ACP v1 compatibility](docs/acp-v1-compatibility.md) for the evidence, precise +boundary, exclusions, and opt-in Zed/agent smoke procedures. + ## Core Local Workflows Record the whole worktree: @@ -843,13 +879,12 @@ exact directory where the agent edited files. Configure an ACP editor once: ```sh -trail agent setup +trail agent acp setup claude-code --editor vscode ``` -`agent setup` defaults to Claude Code plus VS Code, and Codex is also available -with `trail agent setup --provider codex`. Use `--editor zed` or -`--editor generic` when you want another snippet. It prints the editor snippet -plus the next verification and review commands. +ACP setup requires a provider. Use `--editor zed` for Trail's exact Zed +adapter, or `--editor vscode`/`generic` for a copyable entry. Setup previews by +default; pass `--yes` to apply an exact supported adapter. Paste the printed snippet into the editor's ACP custom-agent settings. After one prompt, ask Trail what needs attention: @@ -991,7 +1026,7 @@ For terminal-first work, create a fresh materialized lane and launch Claude Code inside it: ```sh -trail agent start --provider claude-code --name docs-edit +trail agent start claude-code --name docs-edit trail agent trail agent ask what should I do next trail agent todo @@ -1254,9 +1289,9 @@ trail lane sync-workdir doc-bot Merge only after review and readiness checks: ```sh -trail merge-lane doc-bot --into main --dry-run -trail merge-queue add doc-bot --into main -trail merge-queue run +trail lane merge doc-bot --into main --dry-run +trail lane merge-queue add doc-bot --into main +trail lane merge-queue run ``` If a merge opens conflicts, inspect and resolve them explicitly: diff --git a/RELEASING.md b/RELEASING.md index cda0626..aa809f2 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -17,7 +17,8 @@ GitHub attestations, and the Homebrew formula. 3. Allow GitHub Actions to create pull requests under repository Settings, Actions, General. 4. Protect `main` and require the normal CI, Release plan, and Release Readiness - checks before merging. + checks before merging. Also require both Linux/ext4 and macOS/APFS jobs from + `Changed-path Ledger Native Gates` for the exact merge SHA. 5. Keep GitHub Actions workflow permissions able to create releases and artifact attestations. @@ -47,10 +48,13 @@ updates one release pull request containing the Cargo version bump, lockfile, changelog, and release manifest. Merge that pull request after its required checks pass. -The merge changes `.release-please-manifest.json`, so the same workflow creates -and pushes the matching annotated `vX.Y.Z` tag. That tag starts the generated -`release.yml` cargo-dist workflow. Cargo-dist builds and attests all platform -artifacts, then publishes the GitHub Release. Publishing the stable release +The merge changes `.release-please-manifest.json`. Before the same workflow can +create the matching annotated `vX.Y.Z` tag, its reusable exact-SHA native gate +must pass on Linux/ext4 and macOS/APFS. That tag starts the generated +`release.yml` cargo-dist workflow, whose configured custom plan job runs the +same two gates against the tag SHA. Every build, host, and publish job depends +on that generated gate. Cargo-dist then builds and attests all platform +artifacts and publishes the GitHub Release. Publishing the stable release completes the generated Release workflow, whose `workflow_run` event starts `publish-homebrew.yml` and updates `Formula/trail.rb` in the shared tap. diff --git a/docs/CLI_TERMINAL_OUTPUT.md b/docs/CLI_TERMINAL_OUTPUT.md new file mode 100644 index 0000000..2ada9b7 --- /dev/null +++ b/docs/CLI_TERMINAL_OUTPUT.md @@ -0,0 +1,96 @@ +# Trail terminal output contract + +Trail's human output is intentionally a presentation surface. Automations must +use `--format json` for a single report or `--format ndjson` for record +streams. + +## Output matrix + +| Mode | TTY | Styling | Paging | Progress | Contract | +| --- | --- | --- | --- | --- | --- | +| `human` | yes | semantic color and Unicode when capable | eligible review documents only | one transient stderr line | interactive presentation | +| `human` | no | no auto color; ASCII-safe layout | never | never | readable redirected log | +| `plain` | any | no ANSI/OSC; ASCII; RFC 3339 timestamps; integer milliseconds | never | never | deterministic text | +| `json` | any | none | never | never | one report value | +| `ndjson` | any | none | never | never | one value per supported record | +| `--quiet` | any | none | never | never | suppresses successful human/plain results only | + +`--color auto` honors `NO_COLOR`, `TERM=dumb`, and stdout TTY detection. +`--color always` and `--color never` override that choice. `--pager auto` is +limited to long interactive review documents; `always` still never pages a +redirected, plain, structured, or quiet command. + +## Renderer inventory and raw exceptions + +Every result emitted by `trail/src/cli/command/render/` is a semantic terminal +document, except for the following intentionally raw payloads: + +| Route | Reason | Structured alternative | +| --- | --- | --- | +| `trail lane read` | exact file content must remain pipeable | `--format json` carries content and metadata | +| agent report Markdown mode | Markdown is copy/paste source rather than terminal prose | default human mode renders a document | +| agent PR title/body modes | requested text is an integration payload | default human mode renders a document | +| ACP/agent configuration snippets | snippets must remain directly pasteable | default human mode wraps them in a document | +| Git patch export and OpenAPI schema output | requested payload is an integration or review artifact | JSON reports where the command provides one | +| native agent hook acknowledgements | provider protocol requires an exact JSON acknowledgement | not applicable | +| JSON and NDJSON | stable automation contracts | not applicable | + +Transient progress is confined to the shared renderer's stderr progress writer. +Persistent diagnostics use the shared error document on stderr. No command +adapter owns ANSI sequences, terminal-width detection, or direct terminal +writes. + +## Breaking migration + +This is a hard human-output cutoff. `--no-color` is removed; use +`--color never`. Shell scripts, editor integrations, and tests that consume +Trail output must select `--format json` for one report or `--format ndjson` +for a supported record stream. Do not parse human or plain output. + +## Dependency decision + +The renderer uses `unicode-width` for display-width measurement, +`terminal_size` for conservative terminal dimensions, `anstyle` for ANSI style +ownership, and `time` for deterministic RFC 3339 timestamps. This keeps the +presentation dependency set platform-neutral and avoids a full TUI runtime or +PTY-only dependency in the CLI binary. + +## Release performance baseline + +Run `scripts/terminal-output-bench.sh` from a release checkout before shipping. +It measures the median of five `trail --help` startups and the renderer's +10,000-row table and roughly 2 MiB patch cases. The release thresholds are: + +| Probe | Threshold | +| --- | --- | +| CLI startup median | 250 ms | +| 10,000-row table render | 750 ms | +| Large patch render | 1.5 s | + +## Verification scope + +The release gate runs formatter, Trail unit/integration tests, source-output +guard, strict OpenSpec validation, and the release benchmark above on the +shipping host. Windows PTY and non-macOS terminal-emulator confirmation remain +release-environment checks: their capability policy defaults conservatively to +unstyled direct output when TTY detection is uncertain, and CI must run the +same smoke cases with `TERM=dumb` and `NO_COLOR` before a platform release. + +### Local verification record (2026-07-12) + +`cargo fmt --all -- --check`, focused renderer/error/source-guard tests, the +Trail unit/integration suite, strict OpenSpec validation, and +`scripts/terminal-output-bench.sh` passed on macOS. The release benchmark +reported an 8.4 ms median warm startup, a 9.4 ms 10,000-row render, and an +8.2 ms large-patch render. + +`cargo test --workspace` is presently blocked outside this change by the +user-owned `prolly/bindings/uniffi` member: its match at `src/lib.rs:1222` +does not handle `prolly::Error::InvalidVersionedMap(_)`. This terminal UX +change does not alter that member, so the failure is recorded rather than +patched here. + +The strict Trail clippy invocation is likewise blocked in the same dependency +tree: `prolly` currently reports existing `large_enum_variant` and +`result_large_err` diagnostics for `TransactionConflict`. Neither failure is +introduced by Trail's terminal renderer. diff --git a/docs/README.md b/docs/README.md index c0b77ef..c9f4783 100644 --- a/docs/README.md +++ b/docs/README.md @@ -63,6 +63,7 @@ These docs are written from the current Rust code, CLI definitions, exported mod - [OpenAPI](integrations/openapi.md) - [MCP](integrations/mcp.md) - [ACP relay design](design/acp-relay.md) +- [Native agent hooks and ACP integration design](design/native-agent-hooks-and-acp.md) - [VS Code ACP chat view design](design/vscode-acp-chat-view.md) - [VS Code extension implementation](https://github.com/crabbuild/trail-vscode) - [Rust library](integrations/rust-library.md) @@ -72,6 +73,7 @@ These docs are written from the current Rust code, CLI definitions, exported mod - [Trail Agent CLI reference](agent/cli-reference.md) - [CLI global options and environment](reference/cli/global-options-and-env.md) +- [Terminal output contract](CLI_TERMINAL_OUTPUT.md) - [CLI command reference](reference/cli/workspace-and-config.md) - [Configuration keys](reference/config.md) - [Patch format](reference/patch-format.md) @@ -80,7 +82,11 @@ These docs are written from the current Rust code, CLI definitions, exported mod - [Data types](reference/data-types.md) - [Design notes](design/architecture.md) - [Layered lane workspaces for large-repository multi-agent development](design/layered-lane-workspaces.md) +- [Universal lane environments](design/universal-lane-environments.md) +- [Environment adapter and specification contract](design/environment-adapter-contract.md) +- [Environment adapter Rust SDK](../trail-environment-adapter-sdk/README.md) - [Distributed Prolly VCS design](design/distributed-prolly-vcs.md) - [ACP relay design](design/acp-relay.md) +- [Native agent hooks and ACP integration design](design/native-agent-hooks-and-acp.md) - [VS Code ACP chat view design](design/vscode-acp-chat-view.md) - [Code fact map](./_meta/code-fact-map.md) diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index db2d5f3..543b3c7 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -10,6 +10,7 @@ Start here: - [First lane workflow](getting-started/first-lane-workflow.md) - [Trail Agent quick start](getting-started/agent-workflow.md) - [Trail Agent user guide](agent/user-guide.md) +- [Native agent hooks, evidence, and recovery](agent/native-hooks.md) - [Lane work model](lanes/work-model.md) - [Record worktree changes](guides/record-worktree-changes.md) - [Inspect history and provenance](guides/inspect-history-and-provenance.md) diff --git a/docs/acp-v1-compatibility.md b/docs/acp-v1-compatibility.md new file mode 100644 index 0000000..12ed580 --- /dev/null +++ b/docs/acp-v1-compatibility.md @@ -0,0 +1,161 @@ +# Trail ACP v1 Compatibility + +Trail is a complete, transparent compatibility layer between a local ACP v1 +client/editor and a local ACP v1 agent over newline-delimited UTF-8 JSON-RPC +stdio. The relay preserves valid protocol behavior while adding Trail lane +isolation, durable capture, provenance, and optional Trail MCP injection. + +## Normative contract + +Trail pins ACP wire version `1` from the official Agent Client Protocol schema: + +- upstream revision: `64cbd71ae520b89aac54164d8c1d364333c8ee5f` +- `schema/v1/schema.json` SHA-256: + `92c1dfcda10dd47e99127500a3763da2b471f9ac61e12b9bf0430c32cf953796` +- `schema/v1/meta.json` SHA-256: + `e0bf36f8123b2544b499174197fdc371ec49a1b4572a35114513d56492741599` +- stable methods: all 23 methods enumerated by that metadata +- transport: stdio + +The offline conformance gate covers every stable method, request and response +direction, success and error response, notification, content/update union, +capability combination, string and numeric ID, concurrent correlation case, +unknown extension, and intentional transformation. Fault tests cover malformed +input, the 16 MiB frame boundary, backpressure, cancellation races, peer +termination, capture failure, and durable spill recovery. Interoperability is +tested both with a peer compiled from the official Rust ACP types and an +independent Python peer. A scheduled drift check compares the vendored v1 +artifacts with the official repository. + +## What the user gets + +- An ACP-capable editor can launch Trail as its agent command; Trail can launch + any conformant local ACP v1 agent. +- Editor-to-agent and agent-to-editor requests remain correlated even when IDs + overlap or responses arrive out of order. +- Prompts, streamed output, tool activity, permissions, filesystem callbacks, + terminal callbacks, cancellation, errors, and extensions remain usable. +- Agent work can run in an isolated Trail lane. Paths inside the workspace are + mapped into the materialized lane; additional roots outside the workspace are + preserved instead of being silently copied or rewritten. +- Capture runs off the forwarding path. Queue pressure spills to a durable + journal, and capture degradation is surfaced without changing wire traffic. +- Trail-owned MCP identity is injected only when requested and is deduplicated + from an equivalent editor-provided server. +- `trail transcript` and the lane/session views provide durable activity and + provenance without making the editor or agent Trail-specific. + +## Read the compatibility evidence + +Run the doctor in a Trail workspace: + +```sh +trail agent acp doctor codex +trail --json agent acp doctor codex +``` + +The report separates these facts: + +- `conformance`: wire version, schema revision and digests, stdio transport, + method count, build identifier, evidence status, and exclusions; +- `provider`, `relay`, and `launch`: whether the selected agent can be launched + on this machine; +- `capture_journal`: whether the durable ingress journal is accessible; +- `path_mapping`: whether workspace isolation and preserved external roots pass + the relay's mapping check; +- `capture_smoke`: intentionally skipped until a real editor/agent prompt has + supplied environment-specific evidence. + +An ordinary local build reports `evidence_status: "unverified"` and never prints +`ACP v1 conformant`. This prevents a modified build from inheriting a release +claim merely because its provider command is available. A release builder may +attest a tested source revision by compiling with both values equal: + +```sh +revision=$(git rev-parse HEAD) +TRAIL_SOURCE_REVISION="$revision" \ +TRAIL_ACP_V1_CONFORMANCE_VERIFIED="$revision" \ + cargo build --release -p trail +``` + +Those variables are a build attestation, not a substitute for running the +release gates. `verified` means the named source revision was built after the +schema, conformance, fault, interoperability, and benchmark gates passed. It +does not mean the selected external provider is installed, authenticated, or +healthy. + +## Exact boundary + +Protocol conformance is independent of setup convenience. The generic, VS Code, +and Zed setup commands only produce or install editor configuration; a missing +adapter does not reduce Trail's wire coverage. Similarly, Trail can diagnose +provider readiness but does not install user credentials, accept third-party +terms, repair a non-conformant peer, or guarantee an external service account. + +Capture fidelity means every parsed ACP frame receives ordered durable evidence +and all stable ACP session activity is projected into Trail's typed views. The +wire remains authoritative: capture timeout, worker panic, journal pressure, or +projection failure cannot rewrite a forwarded frame. A later projection can be +recovered from the journal. + +Lane isolation applies to workspace-owned roots selected for materialization. +Absolute roots outside the workspace remain external and retain their original +paths, as ACP clients expect. Trail does not claim to sandbox those external +roots or the agent process itself. + +The compatibility claim explicitly excludes: + +- ACP v2 semantics; +- the draft remote HTTP transport; +- an editor UI or guarantees about an editor's settings format; +- installation, licensing, authentication, uptime, or correctness of an + external agent. + +## Opt-in real-editor smoke evidence + +Real programs are environment evidence, not part of the deterministic protocol +gate. If a required executable is absent, record the smoke as `skipped`; never +count the skip as passing conformance evidence. + +### Zed with Codex + +```sh +command -v zed >/dev/null || { echo "SKIP: Zed unavailable"; exit 0; } +command -v npx >/dev/null || { echo "SKIP: Codex ACP adapter unavailable"; exit 0; } +trail agent acp doctor codex +trail agent acp setup codex --editor zed --yes +zed . +``` + +In Zed, start the configured Trail/Codex agent, send a prompt that reads and +writes one workspace file, cancel a second prompt, then verify +`trail agent acp sessions` and `trail transcript `. + +### Zed with Claude Code (independent agent) + +```sh +command -v zed >/dev/null || { echo "SKIP: Zed unavailable"; exit 0; } +command -v npx >/dev/null || { echo "SKIP: Claude ACP adapter unavailable"; exit 0; } +trail agent acp doctor claude-code +trail agent acp setup claude-code --editor zed --yes +zed . +``` + +Send a prompt that exercises a permission request and a file edit, then inspect +the session and transcript as above. Authentication failure is provider evidence +and must not be reported as a Trail protocol failure. + +### Zed with Cursor's native ACP agent (second independent implementation) + +```sh +command -v zed >/dev/null || { echo "SKIP: Zed unavailable"; exit 0; } +command -v agent >/dev/null || { echo "SKIP: Cursor agent unavailable"; exit 0; } +trail agent acp doctor cursor +trail agent acp setup cursor --editor zed --yes +zed . +``` + +Send one prompt that uses a terminal and one that reads an additional external +directory. Confirm both editor behavior and Trail's captured session. Record the +program versions, doctor JSON, prompt result, and transcript selector with the +release evidence. diff --git a/docs/agent/cli-reference.md b/docs/agent/cli-reference.md index 28a6ae1..dca2d39 100644 --- a/docs/agent/cli-reference.md +++ b/docs/agent/cli-reference.md @@ -33,23 +33,24 @@ trail --json agent dashboard latest | Command | Purpose and important options | | --- | --- | -| `setup` | Print editor setup. `--provider ` defaults to `claude-code`; `--editor ` defaults to `vscode`. | -| `doctor` | Check provider and workspace readiness. `--provider ` defaults to `claude-code`. | -| `start` | Create a fresh task lane and launch a terminal agent. Supports `--provider`, `--name`, `--from`, `--workdir-mode`, and a command after `--`. | +| `acp setup ` | Preview ACP editor setup. Accepts equivalent `--provider`, plus `--editor`, `--print`, and `--yes`. | +| `hooks setup ` | Preview native hook setup. Accepts equivalent `--provider`, plus `--scope`, `--print`, and `--yes`. | +| `doctor [PROVIDER]` | Check terminal provider and workspace readiness. Accepts equivalent `--provider`. | +| `start [PROVIDER]` | Create a fresh task lane and launch a terminal agent. Accepts equivalent `--provider`, a configured default, `--name`, `--from`, `--workdir-mode`, and a command after `--`. | | `continue [TASK]` | Create a follow-up task from an existing task checkpoint. Supports `--provider`, `--name`, `--workdir-mode`, and a command after `--`. Alias: `follow-up`. | -| `acp` | Hidden stable ACP entry point used by editor integrations. Supports `--provider`, `--name`, `--from`, `--no-mcp`, and a command after `--`. | +| `acp status/doctor/sessions` | Inspect ACP providers, readiness, and captured sessions. Editors use the hidden `acp run` entrypoint. | Built-in terminal profiles are `claude-code`, `codex`, `cursor`, `gemini`, `aider`, and `opencode`. -Valid terminal workdir modes are `full-cow`, `overlay-cow`, and `nfs-cow`; the -default is `full-cow`. +Valid terminal workdir modes are `auto`, `native-cow`, `portable-copy`, +`fuse-cow`, `nfs-cow`, and `dokan-cow`; the default is `auto`. ```sh -trail agent setup --provider codex --editor vscode -trail agent doctor --provider codex -trail agent start --provider codex --name docs-task -trail agent start --provider codex --from main --workdir-mode full-cow +trail agent acp setup codex --editor vscode +trail agent doctor codex +trail agent start codex --name docs-task +trail agent start codex --from main --workdir-mode native-cow trail agent continue latest --provider claude-code ``` @@ -209,6 +210,35 @@ Git branch. ## Recovery +### Native hooks, evidence, and managed capture + +| Command | Purpose and important options | +| --- | --- | +| `hooks add PROVIDER` | Safely install Trail-owned project or user hooks. Use `--scope project|user`, `--lane`, `--dry-run`, and `--force` only after reviewing foreign config. | +| `hooks remove PROVIDER` | Remove only exact Trail-owned entries or files. Supports scope, dry-run, and force. | +| `hooks list` | List all eight adapters and recorded installations; `--installed` filters the catalog. | +| `hooks status PROVIDER` | Compare persisted ownership digests with current provider configuration. | +| `hooks doctor [PROVIDER]` | Report compatibility, drift, receipt failures, spool pressure, capture ownership, transcript fidelity, and stale finalizers. Use `--all` or `--probe`. | +| `hooks events PROVIDER` | List durable redacted receipt rows; `--failed` selects retry, quarantine, and discarded states. | +| `hooks replay --pending` | Drain the secure fallback spool, recover stale processing leases, and replay due receipts with bounded exponential backoff. | +| `hooks retry RECEIPT` | Move a retrying or quarantined receipt back to the replay queue. | +| `hooks discard RECEIPT` | Retain the receipt audit row while explicitly preventing later replay. | +| `capture begin` | Declare a leased managed run with `--owner`, `--session`, optional `--executor`, `--lane`, `--workdir`, `--work-item`, and `--ttl-ms`. | +| `capture renew RUN` | Renew the exact owner/session lease. | +| `capture end RUN` | Idempotently end the exact owner/session run. | +| `capture status` | List active runs; `--all` includes ended and expired runs. | +| `capture reconcile` | Expire abandoned runs and close their open mappings, turns, and sessions as interrupted. | +| `artifacts SESSION` | List immutable native transcript, canonical export, or reconstructed evidence artifacts. | +| `provenance SESSION` | Read factual and explicitly derived causal nodes and edges. | +| `attest create|list|show|verify` | Create and verify content-addressed, chained session attestations. | +| `learnings list|accept|reject` | Review reusable findings; Trail never edits provider context files automatically. | +| `export SESSION` | Write or print the verified portable agent-trace representation; `--attachments` includes bounded attachment bytes. | +| `git-link link|list` | Record or query exact Git commit, Trail change, turn, and session associations. | + +The singular `trail agent hook receive PROVIDER EVENT` command is an internal, +fail-open provider callback. Integration authors may call it, but users should +manage installations through `trail agent hooks`. + | Command | Purpose and important options | | --- | --- | | `undo [TASK]` | Undo the latest completed turn by default. Select with `--last-turn`, `--turn `, `--prompt `, or `--last-operation`. Alias: `undo-last`. | diff --git a/docs/agent/native-hooks.md b/docs/agent/native-hooks.md new file mode 100644 index 0000000..8a1e63f --- /dev/null +++ b/docs/agent/native-hooks.md @@ -0,0 +1,180 @@ +# Native Agent Hooks, Evidence, and Recovery + +Trail can capture Codex, Claude Code, Pi, OpenCode, Cursor, Gemini CLI, +GitHub Copilot CLI, Grok Build, and Kiro without requiring ACP. Native hooks and ACP +are transports over the same versioned lifecycle and evidence model, so users +may use either one or both. + +## Install an integration + +Initialize Trail in the repository, preview the provider mutation, then apply +it: + +```sh +trail agent hooks setup codex --print +trail agent hooks setup codex --yes +trail agent hooks status codex +trail agent hooks doctor codex +``` + +Replace `codex` with `claude-code`, `pi`, `opencode`, `cursor`, `gemini`, +`copilot`, `grok`, or `kiro`. Project scope is the default. User scope writes only to +the provider's declared user location and records the same ownership manifest: + +```sh +trail agent hooks setup claude-code --scope user --yes +``` + +Kiro's automatic target is the current versioned standalone hook contract +(`.kiro/hooks/*.json`, used by the IDE and CLI v3 engine). Kiro CLI package +2.8.0 and newer include the v3 engine, even though `kiro-cli --version` still +reports a 2.x package version; launch it with `kiro-cli --v3` to activate the +standalone hooks. The default v2 engine continues to use hooks embedded in a +selected named custom-agent file and does not read Trail's standalone file. + +Trail structurally merges JSON provider configuration, preserves unrelated +fields and hooks, writes generated plugin/extension files atomically, uses +restrictive permissions, and records exact before/after and ownership digests. +Removal refuses drift it cannot prove it owns: + +```sh +trail agent hooks remove codex --dry-run +trail agent hooks remove codex +``` + +Review a conflict before using `--force`; force does not authorize Trail to +delete unrelated provider configuration. + +## What capture creates + +The provider callback first writes a bounded, centrally redacted receipt to +Trail's durable object journal. Semantic replay then correlates or creates one +native-session mapping and records: + +- a Trail lane session and user-level turns; +- normalized session, turn, message, tool, approval, subagent, compaction, + usage, workspace, and diagnostic events; +- nested root, tool, subagent, and compaction spans; +- at most one workdir checkpoint for a completed changed turn; +- a full canonical export when supplied, otherwise a native transcript + snapshot, otherwise an explicitly low-fidelity reconstructed transcript; +- an immutable exact turn-evidence manifest; +- deterministic factual and derived provenance; +- a chained content-addressed session attestation on finalization. + +Attachments and transcripts are content-addressed and separately redactable. +Their removal or retention state does not rewrite factual receipt, change, or +manifest identities. + +## ACP and hybrid capture + +ACP keeps its streamed messages, structured diffs, permission mirroring, usage, +and tool fidelity. It also emits the common lifecycle envelope and creates the +same evidence manifests and attestations. + +When a native session ID exactly matches an active ACP or upstream session, +Trail creates a hybrid mapping to the existing Trail session. ACP remains the +lifecycle owner while native hooks enrich it with receipts and transcript +evidence. Exact role/body message deduplication and stable event IDs prevent a +second prompt, turn, span, or checkpoint. Ambiguous ACP matches fail closed and +are reported rather than guessed. + +## Managed runs + +A managed run attributes only sessions created while its renewable lease is +active. Matching uses the longest canonical workdir and exact owner or executor +provider; equal-specificity ambiguity fails closed. + +```sh +trail agent capture begin \ + --owner codex --session orchestrator-42 \ + --executor claude-code --lane feature-a \ + --workdir "$PWD" --work-item issue-123 + +trail agent capture renew capture_run_ID \ + --owner codex --session orchestrator-42 + +trail agent capture end capture_run_ID \ + --owner codex --session orchestrator-42 +``` + +Existing direct sessions are never retroactively stamped. If a lease expires, +`trail agent capture reconcile` records the run as expired and closes its open +turn/session as `interrupted`, preserving recoverable evidence without claiming +clean completion. + +`trail agent start` creates and closes this managed capture lease automatically. +The terminal task's pre-created lane session remains the lifecycle owner; native +hooks attach prompt, tool, approval, transcript, and per-turn checkpoint evidence +to that same session. Provider exit performs one final workdir reconciliation, +which is a no-op when the latest hook checkpoint already captured every change. + +## Inspect evidence + +```sh +trail --json agent hooks events codex --last 100 +trail --json agent artifacts SESSION +trail --json agent provenance SESSION +trail --json agent attest list --session SESSION +trail --json agent attest verify ATTESTATION +trail --json agent git-link list SESSION +trail --json agent export SESSION --attachments +``` + +Attestations state exactly which completed turns, changes, and evidence +manifests they cover. A signature proves possession of a configured Trail key; +it does not prove model correctness or human approval. Revoked keys remain +visible in historical verification. + +Portable export is a verified projection, not a second database or hidden Git +branch. Git association occurs only through explicit exact links or a Trail Git +workflow that already knows the exported change boundary. + +## Recovery and degraded operation + +Provider callbacks are fail-open after attempting durable journaling. If Trail +cannot open the database, the callback writes a redacted, atomic 0600 envelope +under `.trail/runtime/agent-hooks-spool` and still returns the provider's +success contract. + +```sh +trail agent hooks replay --pending +trail agent hooks events codex --failed +trail agent hooks retry RECEIPT +trail agent hooks discard RECEIPT +``` + +Replay recovers stale `processing` rows, honors `next_attempt_at`, applies +bounded exponential backoff, quarantines invalid/poison receipts, and uses +event and span idempotency keys when reconstructing partial work. Discard is an +explicit audited operator decision; it does not delete raw receipt identity. + +## Security and privacy + +- HTTP hook ingestion requires a configured daemon bearer or Trail token even + when the rest of the loopback daemon is running in permissive local mode. +- Raw payloads, identifiers, request bodies, config files, transcript files, + artifacts, spool size, row counts, and exports are bounded. +- Transcript locators must resolve inside the workspace or an approved + provider-managed root; leaf symlinks and escapes are rejected. +- Secret-like JSON keys and text are redacted before durable receipt or spool + storage. Diagnostics are bounded and redacted again before display. +- Trail never automatically writes inferred learnings into `CLAUDE.md`, + `GEMINI.md`, or another provider context file. Only explicitly accepted, + scoped learnings are eligible for bounded context injection. + +Use `trail agent hooks doctor --all --probe` for executable/version discovery, +installation drift, recent delivery, receipt failures, spool pressure, managed +run ownership, stale finalizers, and honest transcript/export fidelity. + +## Compatibility policy + +Provider manifests are versioned independently from the lifecycle schema. +Unknown native events are retained as inert `provider..` +evidence. A provider with no verified version range reports compatibility as +`unknown`; Trail does not infer support from executable presence. Local +manifests are built in. Any future remote manifest must be signed and pass the +same constrained schema, path, command, and ownership checks before use. + +For architecture and rationale, see +[Native Agent Hooks and ACP Integration](../design/native-agent-hooks-and-acp.md). diff --git a/docs/agent/overview.md b/docs/agent/overview.md index 74589c2..d79da3f 100644 --- a/docs/agent/overview.md +++ b/docs/agent/overview.md @@ -95,12 +95,15 @@ Terminal tasks support these workdir modes: | Mode | Behavior | Typical use | | --- | --- | --- | -| `full-cow` | Materializes the full task root using filesystem cloning when available. | Portable default. | -| `overlay-cow` | Mounts a FUSE-backed copy-on-write view for the duration of the agent run. | Avoid copying a large tree on Linux/macOS with FUSE available. | +| `auto` | Tries strict native COW, then restarts as portable materialization when native cloning is unavailable. | Default materialized task mode. | +| `native-cow` | Requires every task file to use native filesystem cloning and fails otherwise. | Enforce block-sharing guarantees. | +| `portable-copy` | Clones per file when possible and byte-copies the remainder. | Predictable materialization across filesystems. | +| `fuse-cow` | Mounts a FUSE-backed copy-on-write view for the duration of the agent run. | Avoid copying a large tree on Linux/macOS with FUSE available. | | `nfs-cow` | Mounts a loopback NFSv3 copy-on-write view on macOS. | Large macOS workspaces without macFUSE. | +| `dokan-cow` | Mounts a Dokan-backed copy-on-write view on Windows. | Large Windows workspaces with Dokan 2.x. | -The default for `trail agent start` and `trail agent continue` is `full-cow`. -Overlay and NFS mounts exist only while the terminal agent is running. Trail +The default for `trail agent start` and `trail agent continue` is `auto`. +FUSE, NFS, and Dokan mounts exist only while the terminal agent is running. Trail records their writable changes before unmounting. ## Safety Model diff --git a/docs/agent/troubleshooting.md b/docs/agent/troubleshooting.md index fbb8d14..008293f 100644 --- a/docs/agent/troubleshooting.md +++ b/docs/agent/troubleshooting.md @@ -5,7 +5,7 @@ Start with Trail's state-aware diagnosis commands: ```sh trail agent guide trail agent diagnose latest -trail agent doctor --provider codex +trail agent doctor codex ``` `diagnose` inspects an existing task and suggests safe recovery actions. @@ -17,8 +17,8 @@ Commands using `latest` cannot select a task until an editor prompt or terminal task has created one. Configure an editor or start a terminal task: ```sh -trail agent setup --provider codex -trail agent start --provider codex --name first-task +trail agent acp setup codex +trail agent start codex --name first-task ``` `trail agent action` also publishes first-run setup, doctor, and start actions @@ -41,33 +41,33 @@ Commands accept a task ID, lane, session, or ACP session selector. Run the provider check: ```sh -trail agent doctor --provider claude-code -trail agent doctor --provider codex +trail agent doctor claude-code +trail agent doctor codex ``` Confirm that the provider executable is installed and available in `PATH`. If you need a custom executable or flags, override the profile after `--`: ```sh -trail agent start --provider codex -- /absolute/path/to/codex --flag +trail agent start codex -- /absolute/path/to/codex --flag ``` -## Overlay COW Mount Fails +## FUSE COW Mount Fails -`overlay-cow` requires macFUSE on macOS or FUSE access such as `/dev/fuse` on +`fuse-cow` requires macFUSE on macOS or FUSE access such as `/dev/fuse` on Linux. Trail reports the mount error and does not silently fall back to a full copy. Use the portable mode while diagnosing the host setup: ```sh -trail agent start --provider codex --workdir-mode full-cow +trail agent start codex --workdir-mode native-cow ``` On macOS, `nfs-cow` is another large-workspace option: ```sh -trail agent start --provider codex --workdir-mode nfs-cow +trail agent start codex --workdir-mode nfs-cow ``` See [Spawn and materialize workdirs](../lanes/spawn-and-materialize-workdirs.md) @@ -216,7 +216,7 @@ trail agent summary latest trail agent report latest --markdown trail agent timeline latest trail agent workdir latest -trail agent doctor --provider +trail agent doctor ``` For workspace-level corruption or storage problems, continue with the general diff --git a/docs/agent/user-guide.md b/docs/agent/user-guide.md index 668b10a..2321598 100644 --- a/docs/agent/user-guide.md +++ b/docs/agent/user-guide.md @@ -21,7 +21,7 @@ Trail has terminal profiles for `claude-code`, `codex`, `cursor`, `gemini`, `aider`, and `opencode`. ```sh -trail agent doctor --provider codex +trail agent doctor codex ``` The doctor checks provider and workspace readiness. Fix reported provider, @@ -30,39 +30,53 @@ workspace, or workdir problems before starting a task. For an ACP-capable editor, print a configuration snippet: ```sh -trail agent setup --provider codex --editor vscode +trail agent acp setup codex --editor vscode ``` -`--editor` defaults to `vscode`; `zed` and `generic` are also supported setup -targets. Paste the printed snippet into the editor's custom-agent settings. +Use `zed` for Trail's exact writable adapter, or `vscode`/`generic` for a +copyable entry. Setup previews by default; add `--yes` to apply a supported +adapter. ## 2. Start an Isolated Terminal Task Create a named task and launch the provider inside its workdir: ```sh -trail agent start --provider codex --name update-agent-docs +trail agent start codex --name update-agent-docs ``` Trail creates a fresh task lane from the current base. To start from another Trail ref, task, lane, or checkpoint, use `--from`: ```sh -trail agent start --provider codex --from main --name update-agent-docs +trail agent start codex --from main --name update-agent-docs ``` +Terminal tasks align the child process `PWD`, `OLDPWD`, and Git discovery +ceiling with the materialized lane workdir. This prevents tools from treating +the original parent Git checkout as their project root. On macOS Trail also +launches the provider with workspace write confinement: project files in the +original checkout are read-only to the provider, while the lane workdir and +Trail's internal `.trail` state remain writable. + +When native provider hooks are installed, `agent start` registers a managed +capture run for the new lane before launching the provider. Hook receipts then +enrich the existing terminal task and session rather than creating a second +native-hook task. The terminal wrapper still records the authoritative final +workdir checkpoint when the provider exits. + Choose a workdir mode when the default full materialization is unsuitable: ```sh -trail agent start --provider codex --workdir-mode overlay-cow -trail agent start --provider codex --workdir-mode nfs-cow +trail agent start codex --workdir-mode fuse-cow +trail agent start codex --workdir-mode nfs-cow ``` You can override a provider profile's executable by placing the command after `--`: ```sh -trail agent start --provider codex -- codex --your-provider-flag +trail agent start codex -- codex --your-provider-flag ``` For editor-based ACP work, the configured editor entry point creates a fresh @@ -257,7 +271,7 @@ You can override the provider and workdir mode: ```sh trail agent continue latest \ --provider claude-code \ - --workdir-mode full-cow + --workdir-mode native-cow ``` ## 10. Share the Result diff --git a/docs/concepts/readiness-gates-and-merge-safety.md b/docs/concepts/readiness-gates-and-merge-safety.md index ad4725a..ce0f6f2 100644 --- a/docs/concepts/readiness-gates-and-merge-safety.md +++ b/docs/concepts/readiness-gates-and-merge-safety.md @@ -52,14 +52,14 @@ trail config set lane.required_eval_suites policy-smoke Preview the lane merge first: ```sh -trail merge-lane doc-bot --into main --dry-run +trail lane merge doc-bot --into main --dry-run ``` Use the queue for shared targets: ```sh -trail merge-queue add doc-bot --into main --priority 10 -trail merge-queue run +trail lane merge-queue add doc-bot --into main --priority 10 +trail lane merge-queue run ``` Direct non-dry-run merges into the default branch require `--direct`. diff --git a/docs/design/acp-relay.md b/docs/design/acp-relay.md index 513c89c..91dbe23 100644 --- a/docs/design/acp-relay.md +++ b/docs/design/acp-relay.md @@ -65,15 +65,16 @@ Implemented: - Workdir recording at prompt completion linked to the active prompt turn. - Conservative structured ACP `diff` content capture as Trail `write` patch edits for non-materialized sessions. -- Guided setup and inspection commands: `trail acp install`, `trail acp list`, - `trail acp doctor`, `trail acp sessions`, `trail transcript`, and top-level +- Guided setup and inspection commands: `trail agent acp setup`, + `trail agent acp status`, `trail agent acp doctor`, `trail agent acp sessions`, + `trail transcript`, and top-level `trail turn show`. - Bounded assistant/event capture with truncation events and relay EOF closeout for open turns. Not yet implemented: -- Mutating editor config generation; `acp install` prints snippets only. +- Exact editor adapters beyond Zed; generic and VS Code targets print entries. - Broad structured edit conversion beyond ACP `diff` content with `newText`. - Remote ACP transports while the HTTP transport remains draft. - Long-running assistant message checkpointing before prompt completion. @@ -207,18 +208,18 @@ Useful flags: - `--provider` and `--model` annotate the lane, turns, and session mapping. - `--no-mcp` disables dynamic Trail MCP injection. -Supporting setup commands are exposed through `trail acp`: +Supporting setup commands are exposed through `trail agent acp`: ```sh -trail acp list -trail acp install --agent claude-code --print -trail acp doctor --agent claude-code -trail acp sessions +trail agent acp status +trail agent acp setup claude-code --editor generic --print +trail agent acp doctor claude-code +trail agent acp sessions trail transcript ``` -`acp install` prints editor-specific launch snippets only; it does not mutate -editor configuration. +`agent acp setup` previews editor-specific launch entries and applies exact +supported adapters only with `--yes`. The relay itself must be usable directly from any ACP editor that can launch a local agent command. @@ -520,8 +521,8 @@ state: ```sh trail lane review trail lane diff --patch -trail merge-lane --into main --dry-run -trail merge-queue add --into main +trail lane merge --into main --dry-run +trail lane merge-queue add --into main ``` ## Multi-Agent Coordination @@ -722,7 +723,7 @@ Acceptance criteria: Deliverables: - `trail acp init ` setup helpers. -- `trail acp list` and `trail acp doctor`. +- `trail agent acp status` and `trail agent acp doctor`. - Recommended editor config snippets. - Lane naming policy. - Parallel-session docs. @@ -734,7 +735,7 @@ Acceptance criteria: - A user can configure at least one editor and one upstream ACP agent using docs or generated config. - Two ACP sessions can run concurrently against two Trail lane branches. -- Merge queue and conflict handling work with captured branches. +- Lane merge queue and conflict handling work with captured lane branches. ## Test Plan diff --git a/docs/design/architecture.md b/docs/design/architecture.md index 2233b0b..d36ae50 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -72,7 +72,7 @@ This means CLI behavior usually has two paths: - Local execution through `Trail`. - Daemon-backed execution for selected hot commands, returning the same report shapes. -The daemon path is intentionally partial. It handles `status`, `record`, `diff`, selected `lane` commands, `merge-lane`, and `merge-queue`. Unsupported commands fall back to local execution unless the user explicitly supplied a daemon URL and the command is not daemon-capable. +The daemon path is intentionally partial. It handles `status`, `record`, `diff`, selected `lane` commands including `lane merge`, and `merge-queue`. Unsupported commands fall back to local execution unless the user explicitly supplied a daemon URL and the command is not daemon-capable. ```mermaid sequenceDiagram diff --git a/docs/design/daemon-http-and-mcp.md b/docs/design/daemon-http-and-mcp.md index b0aa1d3..63aeaaa 100644 --- a/docs/design/daemon-http-and-mcp.md +++ b/docs/design/daemon-http-and-mcp.md @@ -264,7 +264,7 @@ Auto-discovery: - Try daemon command. - Fall back to local execution for daemon-unavailable errors when discovery was automatic. -Supported daemon-routed commands include `status`, `record`, `diff`, selected `lane` commands, `merge-lane`, and `merge-queue`. +Supported daemon-routed commands include `status`, `record`, `diff`, selected `lane` commands including `lane merge`, and `merge-queue`. ```mermaid sequenceDiagram diff --git a/docs/design/environment-adapter-contract.md b/docs/design/environment-adapter-contract.md new file mode 100644 index 0000000..68a798a --- /dev/null +++ b/docs/design/environment-adapter-contract.md @@ -0,0 +1,1202 @@ +# Environment Adapter and Specification Contract + +Status: partially implemented; the built-in host contract, Node/Cargo/Go/CMake/Python adapters, +restricted command recipes, and experimental isolated subprocess plugin SDK protocols v1 and v2 are +available. Native macOS, Linux, and Windows enforcement implementations exist; complete +native release evidence, signed catalogs/WASI packaging, complete typed runtime/resource +graphs, and certification remain planned. + +## Current implementation + +Trail now has an internal `WorkspaceEnvironmentAdapter` contract whose adapters emit a +normalized, argv-based plan. The host owns pinned source projection, staging, command +execution, immutable publication, declared-path replacement, component state, and view +generation activation. The first built-ins are: + +- `trail/node@1`: npm, pnpm, Yarn, and Bun frozen dependency trees; +- `trail/cargo-target-seed@1`: conservative `cargo build --locked --offline` target + seeds keyed by the complete source root and Rust toolchain identity; +- `trail/go-vendor@1`: single-module Go vendor trees keyed by the complete source root, + module files, Go executable identity, and platform. Go workspaces fail closed until a + graph-aware multi-module adapter is available; +- `trail/cmake-build@1`: provisions a `writable_private` build tree with no synthetic + shared layer. Configure is deliberately deferred until execution inside the mounted + lane so absolute paths in `CMakeCache.txt` name the stable lane workdir rather than a + disposable staging directory; +- `trail/python-venv@1`: recognizes `pyproject.toml` and the common uv, Poetry, PDM, + Pipenv, and requirements lock/manifest files, provisions a layer-free lane-private + `.venv`, and keys it by every present dependency file plus the resolved Python + executable. Trail automatically creates the virtual environment through an ephemeral + candidate view at the lane's final mountpoint, so scripts and prefix metadata embed + the correct absolute path without exposing partial state; +- `trail/oci-image@1`: reads `trail.oci.toml`, accepts only lowercase SHA-256 + digest-pinned OCI references with an explicit platform, and records provider-owned + image identities without commands, caches, mounts, or manufactured directories; +- `trail/command@1`: repository-declared argv commands with exact/globbed byte inputs, + one to 32 immutable-seeded or writable-private generated outputs, denied + shell/network/scripts/secrets, macOS sandbox enforcement, a Linux + Landlock-plus-seccomp helper, and a + capability-free Windows AppContainer constrained by a one-process Job Object. Kernels + without the required Landlock ABI and unsupported operating systems fail closed. +- external `trail.environment-adapter/v1` and `/v2` plugins: explicitly installed local packages + stored by distribution digest and invoked through bounded length-prefixed CBOR. Plugins + receive only host-selected bytes from a pinned root and return discovery or + denied-by-default command plans through the same native sandbox. V1 proposes one + staging command. V2 adds typed staging and mounted-initialization actions plus + metadata-only external-artifact declarations while the host retains mount authority + and atomic generation activation. + +`trail env adapters` returns the compiled and installed adapter catalog, including versioned identity, +selectors, component kind, discovery markers, implementation provenance, stability, and +description without probing the repository or host tools. Discovery obtains candidate +manifest names from that metadata rather than a central ecosystem filename list. +`trail env sync ` auto-detects an unambiguous adapter, while `--adapter` resolves +one component explicitly. `trail env discover` enumerates nested component proposals +without launching ecosystem tools or repository code; installed plugin planners run only +inside the capability-free sandbox. `trail env sync-all` builds every +proposal before atomically activating all mounts as one generation. `trail env status` +reports logical component identity separately from adapter identity. CLI, HTTP/OpenAPI, +MCP, and Rust APIs share this state. Existing `trail deps` behavior remains a Node +compatibility surface. + +Command recipes and v1/v2 plugins may declare stable logical component dependencies. +`sync-all` validates missing nodes, duplicate/self edges, complete cycles, and mount +collisions before running a command; it then builds in deterministic topological order. +Legacy `depends_on` and protocol-v1 dependencies mean `build_requires`. Protocol v2 and +repository recipes can explicitly select `build_requires`, `runtime_requires`, +`binds_after`, or `invalidates_with`. Trail adds upstream keys for `build_requires` and +`invalidates_with` to the downstream canonical key; runtime/order-only edges select an +exact upstream instance in generation provenance without manufacturing a new artifact +key. Replacing a runtime/order-only provider atomically advances that generation edge +while leaving the consumer ready. Replanning an identity-bearing upstream makes only +identity descendants stale with an exact edge-level explanation. A single +component sync requires every dependency to be ready in the lane and points users to +`sync-all` otherwise; replacing an upstream component alone immediately marks mounted +identity descendants stale without rewriting the dependency keys they were built against. +`env graph` renders that finalized desired graph through CLI, Rust, HTTP/OpenAPI, and +MCP before synchronization, including topological indices, component keys, output +ownership, and exact upstream-key edges without publishing artifacts. CLI, HTTP, and MCP +page by target node with `offset`, `limit`, total node/edge counts, and `next_offset`; +the Rust API also exposes the complete in-process graph. +A repository-wide `sync-all` also treats absence as desired state: components removed +from discovery are unbound with crash-recoverable upper resets in the same activation, +and deleting the final component creates an inspectable empty generation. A scoped +`sync-all --path` never retires components outside that scope. + +Large recipe graphs are batch-planned: Trail parses the complete recipe document twice +(discovery plus planning), not once per component, reuses executable identity checks by +path, and performs target, mount, and source-shadow checks with ordered prefix indexes +instead of quadratic scans. A 1,000-node recipe-chain regression exercises the public +graph path; the lower-level non-recursive resolver covers 10,000 nodes. + +Bindings now contribute runtime path classification, so an adapter-declared dependency +path such as `.venv` receives a private generated upper and stays out of source +checkpoints even though it is not a hard-coded directory name. + +Host execution clears the ambient process environment, supplies an isolated HOME and +temporary directory, and injects only adapter-declared cache/toolchain variables plus +PATH. Built-in keys include adapter implementation provenance and resolved tool +executable digests. Execution preserves dispatcher shim paths such as rustup's `cargo` +shim and verifies the canonical executable digest again immediately before launch. +Cargo performs a locked fetch into managed `CARGO_HOME` followed by +an offline build; MCP and HTTP classify synchronization as open-world execution because +repository-controlled Cargo build scripts and proc macros still execute. Read-only +status reads persisted state and never invokes package managers or compilers. + +The initial Node adapter deliberately rejects workspace-root installs, local file/link +dependencies, pnpm workspace roots, and Yarn Berry/PnP rather than publishing an empty, +escaping, or incorrectly keyed layer. Those forms require explicit workspace-graph and +link contracts before they can graduate. + +The first public Rust protocol crate is `trail-environment-adapter-sdk`. Local executable +packages remain `experimental`: Trail verifies and content-addresses them, records +append-only activation/tombstone history, revalidates their executable before every use, +and executes them without repository, database, mount, process, or network authority. +Detached Ed25519 publisher trust and immediate revocation are implemented. Remote +organization catalogs, WASI components, and independent certification promotion remain +planned. + +Each successful activation now creates an immutable environment generation containing +the pinned source root, sorted component keys, optional exact layer IDs, storage-policy +identities, mount targets, and +predecessor. Failed batch activation leaves the predecessor pointer unchanged and rolls +back every prepared private upper. Retired generations pin their referenced layers so +cache collection cannot invalidate execution provenance. Lane command environments +receive `TRAIL_ENVIRONMENT_GENERATION`. + +Schema v13 adds active and historical external-artifact provenance. Metadata-only +components participate in the same atomic generation transaction but have no layer ID, +mount path, output binding, or cache namespace. OCI declarations retain name, provider, +digest-pinned reference, digest, platform, and explicit `external` cleanup ownership; +Trail layer/cache GC never claims or deletes provider-owned content. + +Every synchronization also owns a durable process-token attempt record. On workspace +open, Trail recovers attempts whose owner died: a component with an attached predecessor +becomes `stale`, a first build with no predecessor becomes `failed`, and a crash after +the activation transaction is recognized as completed. A second synchronization cannot +race an active attempt for the same workspace view. + +Schema v6 adds named component-output bindings and generation-output provenance. One +component command can publish up to 32 directories into a single immutable physical +layer; each output is mounted from a stable layer subpath with its own lane-private +writable upper. Publication and all bindings activate atomically. Replacing a component +also transactionally retires removed output uppers, and the durable reset intent can +distinguish a committed unbind from an interrupted activation so stale private files do +not reappear after recovery. The first output remains projected through legacy singular +report fields while CLI, JSON, HTTP/OpenAPI, MCP, and generation reports expose the full +ordered output list. + +Schema v7 makes output policy and storage identity explicit and permits a generation +output to have no layer. A `writable_private` binding records a content-derived private +identity plus canonical component-key provenance while its bytes remain solely in the +lane's generated upper. Compatible re-sync preserves mutations; a changed key or deleted +private root is rebuilt and replaced through the durable reset transaction. Gate reports +include only actual shared layer IDs, and every public environment surface reports the +nullable layer ID rather than inventing an empty artifact. + +This document specifies how ecosystem knowledge plugs into the +[Universal Lane Environments](universal-lane-environments.md) architecture. It defines +the boundary between an adapter and the Trail host, a declarative repository format, +plugin capabilities, conformance requirements, and reference mappings for common +ecosystems. + +The contract is intentionally narrower than a general extension API. Adapters describe +and inspect; the Trail host authorizes, executes, validates, publishes, attaches, and +persists. + +## How to add an adapter + +Trail should offer an extension ladder instead of forcing every ecosystem integration +into core. Choose the lowest tier that expresses the required behavior: + +| Tier | Use it when | Distribution | Available now | +| --- | --- | --- | --- | +| Command component | One repository needs deterministic generated trees from declared inputs and one command. | Committed `trail.environment.toml` | Yes | +| Versioned profile | Several components or repositories share the same command recipe and policy. | Local included TOML today; signed organization catalogs later | Yes for local includes | +| Isolated plugin | Discovery or semantic parsing is required beyond a declarative recipe. V1 proposes one denied-by-default staging command; v2 adds typed mounted initialization for path-sensitive writable-private outputs, metadata-only pinned external artifacts, and host-managed lane-private OCI services. | Content-addressed signed or unsigned local subprocess package today; remote signed catalogs/WASI later | Yes, experimental local packages | +| Built-in adapter | Trail maintains the ecosystem, migrations, cross-platform behavior, and release fixtures. | Compiled into Trail | Yes, for Trail contributors | + +Promotion is evidence-based. A useful repository recipe can become a profile; a profile +that needs semantic code can become an isolated plugin; a widely used, fully certified +plugin can be proposed as a built-in. Component and adapter identities change explicitly +when semantics change, so promotion never silently reinterprets an existing generation. + +### Add a repository adapter without Rust + +Use `trail/command@1` when the adapter can be represented as pinned byte inputs, an argv +command, and one or more contained generated outputs. The repository may split reusable policy +into local includes: + +```toml +# trail.environment.toml +schema = "trail.environment/v1" +include = ["trail/environments/protobuf.toml"] + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "api.protobuf" +root = "services/api" +extends = ["profile.protobuf"] +``` + +```toml +# trail/environments/protobuf.toml +schema = "trail.environment/v1" + +[profile.protobuf] +version = "1.0.0" +adapter = "trail/command@1" +kind = "generated" +inputs = [ + { path = "{root}/proto/**/*.proto", role = "identity", format = "bytes" }, +] +outputs = [ + { source = "generated", target = "{root}/generated", policy = "immutable_seed_private", portability = "host" }, +] + +[profile.protobuf.build] +command = ["protoc", "--cpp_out=generated", "proto/service.proto"] +cwd = "{root}" +network = "deny" +scripts = "deny" +``` + +Includes are repository-relative committed files. Remote URLs, globs, traversal, +ambiguous duplicate profiles, include cycles, and profile inheritance cycles fail +closed. Includes resolve relative to the including file. Profiles are merged +left-to-right, followed by component overrides: scalar adapter/kind/build values replace +earlier values, inputs accumulate, and a non-empty child output list replaces the +inherited output list. `{root}` is the only template. Canonical expansion, every source +file digest, and every transitive profile version enter the layer key. + +The authoring loop is: + +```text +trail env discover +trail env graph +trail env plan --component api.protobuf +trail env sync --component api.protobuf +trail env status +``` + +Planning is read-only and shows the exact inputs, executable identity, output, key, and +capability grants. Synchronization executes only through the host sandbox and publishes +or privately installs only after output validation. This tier deliberately cannot request a shell, network, +secrets, arbitrary host reads, caches, services, or heterogeneous output policies yet. + +### Add a built-in ecosystem adapter + +A built-in is appropriate only when semantic planning cannot be expressed safely by a +command profile. Examples include parsing workspace graphs, choosing package-manager +strategies, separating content caches from lane-private trees, or validating ABI and +toolchain compatibility. + +The implementation checklist is: + +1. Add `workspace_.rs` beside the existing Node, Cargo, and Go adapters. +2. Define immutable `WorkspaceEnvironmentAdapterMetadata`: canonical identity, + selectors, contract major, implementation version, distribution digest, kind, + storage adapter name, discovery markers, stability, and description. +3. Implement `WorkspaceEnvironmentAdapter`. `detect` must only inspect the pinned Trail + root. `plan` must return a deterministic `WorkspaceEnvironmentPlan`; neither method + may execute tools, write files, publish layers, attach mounts, or update the database. +4. Register the static adapter in `builtin_environment_adapters()`. The same metadata + then drives `env adapters`, discovery, CLI, HTTP/OpenAPI, MCP, and Rust API reports. +5. Put every output-changing fact in `WorkspaceLayerKeyV1`: normalized manifest and + lockfile content, exact executable identity, adapter implementation/distribution, + platform/architecture/ABI, policy, and relevant configuration. Explain the same facts + in `stale_reason`. +6. Describe staged source inputs, host-executed argv commands, cache directories, owned + outputs and mount targets, and the narrowest portability and sandbox policy. For a + provider-owned immutable resource, use kind `external`, include the sorted + `external_artifact_contract` in the canonical key, and declare no action, cache, or + filesystem output. A private service additionally declares a sorted + `runtime_resource_contract` referencing one of those artifacts; Trail retains all + provider, port, health, ownership, and lifecycle authority. + Never teach the adapter about SQLite, NFS, FUSE, overlayfs, or layer publication. +7. Add discovery, deterministic-key, irrelevant-edit, changed-input, two-lane reuse, + private-mutation, failure/no-partial-publication, crash-recovery, security, backend, + and real-scale fixtures. Start at `experimental`; raise certification only when its + platform matrix passes the shared conformance suite. + +Adding a third-party executable to `PATH` does not register it as an adapter. A package +must be installed explicitly: + +```text +trail env plugin install path/to/package +trail env adapters +trail env discover +trail env plan --adapter namespace/name@1 +trail env sync --adapter namespace/name@1 +trail env plugin remove namespace/name@1 +``` + +The package contains `trail-adapter.toml`, one declared executable, and optionally a +detached `trail-adapter.sig`. `env plugin inspect` reports the canonical payload and +distribution digests before mutation. Installation verifies the executable SHA-256, +computes a digest over canonical package metadata plus executable bytes, verifies an +optional Ed25519 signature against the workspace's append-only publisher trust store, +publishes the exact authenticated bytes, and appends an activation record. +Removal appends a tombstone and retains immutable bytes for provenance and repair. A +corrupt active executable fails the catalog closed; reinstalling quarantines the corrupt +copy, while removal remains available for recovery. Trust revocation also fails signed +packages closed immediately. Unsigned packages are visibly `local-experimental`; a +signature authenticates origin but does not grant stable certification. + +The adapter catalog reports each package's planner protocols, supported operating systems, +and architectures. Unsupported plugins remain inspectable but are not auto-discovered or +executed on the current host. `lane readiness` replans every installed component from +the current pinned root through the same built-in, recipe, or plugin path used by sync; +an input, semantic plan, tool, package, executable, platform, capability, command, or +output-contract change marks the attached generation stale. The read-only `env status` +shows the last persisted result and does not execute plugin code by itself. +Newly published artifacts retain the canonical key declaration, not only its digest. +`trail env explain --component ` compares the attached and current keys and +reports added, removed, or modified input/tool edges plus platform, architecture, +adapter, portability, and strategy changes. Values are never rendered. Large monorepo +results are paginated consistently across CLI, HTTP, and MCP; legacy layers explicitly +report incomplete provenance instead of inventing a cause. + +### Plugin implementation and next target + +The [Rust SDK](../../trail-environment-adapter-sdk/README.md) makes an external adapter +return discovery and component plans over the versioned +`trail.environment-adapter/v1` or `/v2` boundary. The host selects the highest protocol +declared by both package and host; packages without protocol metadata retain the exact +v1 canonical payload and behavior. The host supplies bounded pinned-file +bytes, enforces deadlines and request/response/diagnostic limits, resolves and rechecks +tools, executes approved actions itself, validates outputs, and owns all state changes. +Every file exposed to the planner must appear in its identity-input response; Trail +checks exact set equality and keys all of them, preventing inspected-but-unkeyed inputs. +Protocol, implementation, package, executable, semantic input, action phase, tool, output, +platform, and capability identities enter the layer key. + +The conformance fixture verifies discovery, planning, publication, two-lane reuse, +private copy-up, v2 initialization at two distinct final lane paths, declared-input +reads, undeclared source read/write denial, predecessor preservation, input-change +staleness, active-command parent death and abandoned-candidate recovery, timeout, +memory overrun, crash, oversized +output, malformed framing, attempted child execution, executable tampering, removal, +and repair. Native launchers add process/file/memory limits to the denied-by-default +filesystem, network, shell, and child-process sandbox. Ed25519 publisher signatures and +the host trust store authenticate local packages today. Signed catalogs, independent +certification attestations, and WASI transport remain distribution work. + +## Design principles + +1. One contract covers package managers, compilers, build systems, containers, services, + toolchains, and user-defined commands. +2. Adapters return structured plans rather than mutating the lane. +3. All filesystem outputs have an explicit policy and containment root. +4. Commands are argument vectors by default, not shell strings. +5. Fingerprints are deterministic and independently explainable. +6. Secret values are opaque to planning and fingerprinting. +7. Host-owned staging and publication prevent partial shared artifacts. +8. Adapter discovery is side-effect free. +9. Capabilities are denied unless declared and approved. +10. Built-ins, recipes, and isolated plugins produce the same normalized plan. + +## Contract layers + +The contract has three versioned layers: + +| Layer | Responsibility | +| --- | --- | +| Specification schema | Repository-authored desired components, policies, and overrides. | +| Adapter protocol | Discovery, fingerprint inputs, actions, outputs, validation, and binding proposals. | +| Host execution protocol | Capability grants, staged command execution, publication, generation attachment, and reports. | + +Every record carries a schema version. The host rejects unknown required fields and may +ignore only extension fields explicitly marked optional. + +## Adapter identity + +An adapter identity is a tuple: + +```text +(namespace, name, contract_major, implementation_version, distribution_digest) +``` + +Examples: + +```text +trail/node@1 implementation 0.4.0 builtin +trail/cargo@1 implementation 0.4.0 builtin +acme/protobuf@1 implementation 2.3.1 sha256:93d… +``` + +The contract major participates in normalized component identity. The implementation +version participates when it can change output. An adapter may declare a narrower +`fingerprint_compatibility` version only when conformance tests prove output-compatible +behavior. + +## Host/adapter interface + +The following Rust-like interface is conceptual. Concrete Rust, subprocess, and WASI +bindings use serializable request and response types with equivalent semantics. + +```rust +trait EnvironmentAdapter { + fn metadata(&self) -> AdapterMetadata; + + // Read-only and side-effect free. + fn discover(&self, request: DiscoverRequest) -> Result; + + // Returns declarations used by the host's canonical fingerprint engine. + fn fingerprint(&self, request: FingerprintRequest) -> Result; + + // Returns actions and output contracts; does not execute them. + fn plan(&self, request: PlanRequest) -> Result; + + // Optional structured probes around host-executed actions. + fn inspect_staging(&self, request: InspectRequest) -> Result; + + // Returns deterministic validation rules and semantic observations. + fn validate(&self, request: ValidateRequest) -> Result; + + // Returns proposed bindings and runtime resources. + fn bind(&self, request: BindRequest) -> Result; + + // Explains an observed input, incompatibility, or stale transition. + fn explain(&self, request: ExplainRequest) -> Result; +} +``` + +There is intentionally no `publish`, `mount`, `write_database`, `resolve_secret_value`, +or `mark_ready` method. Those actions remain host-owned. + +## Discovery + +Discovery receives: + +- repository and requested discovery root; +- a read-only view restricted to approved paths; +- filenames and small metadata already found by the host, where possible; +- platform and host capability summaries; +- organization and repository policy; +- no secret values and no network access. + +It returns component proposals, input candidates, graph edges, warnings, and ambiguous +choices. Discovery must not run installers, execute repository code, contact registries, +or create files. + +Each proposal contains a stable logical ID. Logical IDs are readable within a +repository, such as `web.dependencies`, `rust.toolchain`, or `dev.postgres`; they are not +global artifact identities. + +When multiple adapters claim the same manifest or output path, the host reports the +conflict unless one adapter declares composition with the other. Priority alone cannot +silently change a safety policy. + +## Fingerprint plan + +The adapter returns inputs, not a precomputed opaque digest. The host reads and hashes +files through its own normalized path and metadata rules. + +```rust +struct FingerprintPlan { + inputs: Vec, + tool_probes: Vec, + parent_edges: Vec, + options: CanonicalValue, + portability: PortabilityClass, + environment_allowlist: Vec, + policy_inputs: PolicyInputs, + output_contract_digest: Digest, +} +``` + +### Input declarations + +An input declaration includes: + +- normalized path or external identity source; +- role: identity, tool identity, policy, runtime, health, secret reference, or ignored; +- content interpretation: bytes, normalized text, parsed semantic value, tree, symlink, + executable probe, or external digest; +- missing behavior: optional, error, or explicit sentinel; +- path case and Unicode normalization policy; +- whether mode and executable bits matter; +- a human explanation used by stale reports. + +Glob expansion is performed by the host and sorted canonically. Globs cannot escape the +discovery root. Semantic parsers must be versioned because normalized parse output is an +identity input. + +### Tool probes + +Tool identity cannot rely on an unqualified version string alone. A probe declares: + +- executable source: bound toolchain component, repository path, or approved host path; +- argument vector and expected output format; +- optional executable content digest; +- version, target, ABI, and configuration fields to extract; +- timeout and locale; +- whether the resolved absolute path is identity-bearing. + +Host tools are conservative `host` portability inputs unless policy maps them to a +trusted toolchain identity. + +## Component plans + +A component plan is a pure description of required actions and outputs: + +```rust +struct ComponentPlan { + component_key_inputs: FingerprintPlan, + actions: Vec, + outputs: Vec, + caches: Vec, + validations: Vec, + bindings: Vec, + runtime_resources: Vec, + capabilities: CapabilityRequest, + recovery: RecoveryPolicy, +} +``` + +Plans are stable for the same canonical request. Volatile discoveries such as a free +host port belong to activation-time runtime allocation, not the component plan. + +## Actions + +Supported host actions begin narrowly: + +- execute a command vector; +- copy or materialize declared inputs into staging; +- create a directory; +- write a generated file whose content is included in the plan; +- unpack a verified archive; +- fetch a pinned external artifact through a host provider; +- invoke another component through a graph edge. + +Each command declares: + +- executable binding or approved path; +- argument vector; +- working directory relative to the sandbox; +- input and output roots; +- normalized non-secret environment additions; +- secret handles required and permitted injection form; +- network mode and endpoint allowlist; +- lifecycle-script policy; +- timeout, resource class, and cancellation behavior; +- expected exit codes; +- redaction fields. + +Shell strings require the `process.shell` capability and a visible policy approval. +Command previews display quoted arguments but never reconstructed secret values. + +Actions execute inside host-owned staging. The host exposes source read-only unless an +output explicitly requires a private writable source projection. An action cannot +write directly into a shared artifact root or active lane binding. + +## Output declarations + +Every produced root declares: + +```rust +struct OutputDeclaration { + name: String, + kind: OutputKind, + source: RelativePath, + policy: StoragePolicy, + mount_target: Option, + portability: PortabilityClass, + mutation_expectation: MutationExpectation, + allowed_file_types: FileTypePolicy, + containment: ContainmentPolicy, + retention: RetentionPolicy, +} +``` + +The host validates: + +- source remains inside staging; +- mount target is normalized and non-reserved; +- no undeclared output overlaps another root; +- symlinks and hard links obey containment rules; +- devices, FIFOs, sockets, setuid bits, and extended attributes obey policy; +- case-folding collisions cannot break the target platform; +- declared portability matches observed native content where detectable; +- immutable output does not contain known transient or secret paths. + +`mutation_expectation` is one of `never`, `consumer_may_copy_up`, `private_mutable`, or +`tool_managed_cache`. It must agree with the selected storage policy. + +## Bindings + +Adapters propose bindings; the host resolves their sources to artifact or private-state +identities and detects collisions. Bindings can target: + +- a workspace-relative path; +- a sandbox-only path; +- an environment variable; +- a file descriptor; +- a socket or allocated port; +- a service name; +- a container volume; +- an executable name in a composed `PATH`. + +Environment composition is deterministic. For variables such as `PATH`, adapters return +typed prepend, append, set, or map-merge operations. Conflicting scalar sets are errors +unless the repository specification explicitly selects precedence. + +## Validation + +Validation combines host-generic and adapter-semantic rules. + +Host-generic validation includes containment, tree manifesting, file-type policy, +portability observations, reserved-path checks, and secret scanning. + +Adapter-semantic validation may include: + +- package manager integrity or lockfile correspondence; +- compiler or linker smoke probes; +- required executable and library presence; +- CMake cache compatibility; +- OCI manifest and platform matching; +- service protocol health; +- framework-specific directory structure. + +A semantic validator reads staged output through a read-only capability. It cannot +repair content. Validation produces rule IDs, results, evidence summaries, and +redaction-safe diagnostics. Non-deterministic network health probes belong to runtime +health, not immutable publication. + +## Declarative repository specification + +The proposed file is `.trail/environment.toml`. The final location remains an open +architecture decision, but the schema below is independent of location. + +```toml +schema = "trail.environment/v1" + +[environment] +name = "developer-and-agent" +discovery_roots = ["."] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "web.dependencies" +adapter = "trail/node@1" +root = "apps/web" +extends = ["profile.nextjs"] + +[[component.input]] +path = "apps/web/package.json" +role = "identity" +format = "json-semantic" + +[[component.input]] +path = "pnpm-lock.yaml" +role = "identity" +format = "yaml-semantic" + +[[component.output]] +name = "modules" +source = "node_modules" +target = "apps/web/node_modules" +policy = "immutable_seed_private" +portability = "abi" + +[[component.cache]] +name = "pnpm-store" +kind = "content" +sharing = "repository" +compatibility = ["os", "arch", "node_abi", "pnpm_major"] + +[component.build] +command = ["pnpm", "install", "--frozen-lockfile"] +cwd = "apps/web" +network = { allow = ["registry.npmjs.org"] } +scripts = "repository-policy" +timeout = "20m" + +[[component.validation]] +kind = "command" +command = ["node", "-e", "require('next/package.json')"] +network = "deny" + +[[component]] +id = "dev.api-token" +adapter = "trail/secret@1" + +[component.secret] +provider = "os-keychain" +name = "team-api-token" +inject = { env = "TEAM_API_TOKEN" } +scope = "command" + +[[component]] +id = "dev.postgres" +adapter = "trail/oci-service@1" + +[component.service] +image = "postgres@sha256:0123456789abcdef" +network = "lane-private" +ports = [{ container = 5432, host = "auto", export_env = "PGPORT" }] +health = { command = ["pg_isready", "-U", "postgres"], timeout = "30s" } +cleanup = "trail-owned" +``` + +The full example remains the target schema. The implemented experimental subset accepts +one of `trail.environment.toml` or `.trail/environment.toml`, strict unknown-field +rejection, repository-local includes, versioned profiles with inheritance and `{root}` +expansion, and `trail/command@1` components. Each command component currently requires: + +- a stable component ID and root; +- optional `depends_on = ["component.id"]` compatibility edges (equivalent to + `build_requires`) or typed `[[component.edge]]` entries with `component` and `type`; + supported types are `build_requires`, `runtime_requires`, `binds_after`, and + `invalidates_with`, and every target must be present in the same discovered graph for + `sync-all`; +- exact file/directory inputs or repository-relative globs, all with `role = "identity"` + and `format = "bytes"`; +- between one and 32 named `kind = "generated"` outputs using + one portability class and either `immutable_seed_private` or `writable_private`. + A component cannot mix the policies yet; +- a non-shell executable name resolved from `PATH` plus an argv vector; +- `network = "deny"` and `scripts = "deny"`; +- no sensitive arguments, environment names, host paths, secret values, caches, + pre-commands, services, or runtime resources. + +`trail env plan` exposes the canonical key, exact executable identity, inputs, output, +and grants before execution. On macOS the command runs under `sandbox-exec`. On Linux a +fresh Trail helper applies a hard-requirement Landlock filesystem allowlist and a +seccomp filter that denies socket, namespace, kernel-module, ptrace, BPF, io_uring, and +related escape syscalls before executing the selected program. On Windows the helper +creates an ephemeral capability-free AppContainer, grants its SID read/execute access to +the staged root and modify access only to output/HOME/tmp, launches the exact executable +suspended, assigns it to a kill-on-close Job Object with an active-process limit of one, +and only then resumes it. A setup failure terminates the suspended process before the +handle is released. The platform backends expose staged inputs for reading, +output/HOME/tmp for writing, the selected executable for launch, and no network or child +processes. Missing kernel facilities and unsupported operating systems fail closed; +discovery and planning remain available. Windows behavior is guarded by a native CI +verifier covering successful publication and reuse plus denied host reads, undeclared +writes, network, child processes, and shells; that native runner, rather than a +cross-compile, is the release evidence for AppContainer and Dokan integration. +When several command components share a root, `--component ` selects one for +`env plan` or `env sync`; `env sync-all` continues to compose every non-conflicting +component atomically in deterministic dependency order. + +### Includes and profiles + +Specifications can include committed files within the repository and extend named +profiles. Includes resolve relative to the including file before canonicalization and +cannot be remote URLs, globs, or traversing paths. The host limits include count, depth, +individual size, and aggregate size, and reports the complete cycle when recursion is +detected. + +Profiles package ecosystem conventions without hiding policy. Examples are +`profile.nextjs`, `profile.vite`, `profile.cargo-workspace`, and +`profile.cmake-ninja`. Expansion is inspectable and contributes the profile version to +the specification digest. + +Organization profiles will be referenceable by signed policy identity, but a generation +records the resolved canonical content so later reproduction does not depend on mutable +profile names. + +### User-defined command components + +Repositories need a general escape hatch that does not require writing a plugin: + +```toml +[[component]] +id = "protobuf.generated" +adapter = "trail/command@1" + +inputs = [ + { path = "proto/**/*.proto", role = "identity", format = "bytes" }, + { component = "tools.protoc", role = "tool_identity" }, +] + +[component.build] +command = ["protoc", "--cpp_out=out", "proto/service.proto"] +cwd = "." +network = "deny" + +[[component.output]] +name = "cpp-sdk" +source = "out" +target = ".trail-generated/protobuf" +policy = "immutable_shared" +portability = "universal" +``` + +The generic command adapter requires complete input/output declarations and defaults to +read-only source, denied network, denied shell, denied scripts, and no secrets. + +## Cache contract + +An adapter declares cache semantics, not only a directory path: + +```rust +struct CacheDeclaration { + name: String, + protocol: CacheProtocol, + compatibility: Vec, + scope: SharingScope, + access: CacheAccess, + authority: CacheAuthority, + eviction: EvictionPolicy, +} +``` + +- `protocol`: content store, concurrent daemon, locked index, or private seeded cache; +- `access`: read-only, single-writer, host-mediated, or tool-concurrent; +- `authority` must be `performance_only` for ordinary caches; +- compatibility includes every dimension that can make an entry unsafe; +- the host can narrow scope but cannot widen it beyond adapter certification. + +Adapters must demonstrate that eviction cannot change correctness. If a tool mixes +authoritative mutable state with cached content, the adapter separates paths or marks +the root writable private. + +The implemented schema-v10 built-in contract derives a `cache_` namespace from +adapter identity, cache name, protocol, access strategy, workspace scope, and a bounded +non-secret compatibility map. Supported protocols are `content_store`, +`compiler_cache`, and `locked_index`; access is either certified `tool_concurrent` or +Trail-mediated `host_exclusive`. Commands reference symbolic cache names, never raw +writable directories. Trail creates the exact host path, records use in component and +generation provenance, holds crash-recoverable leases during execution, and coordinates +GC with an exclusive maintenance barrier. Node, Cargo, sccache, and Go use this path. +Protocol-v2 external plugins can declare the same three protocols for a staging action. +They provide only a bounded non-secret compatibility map and environment-variable to +relative-subpath bindings; the host adds the authenticated distribution digest, +negotiated protocol, OS, and architecture, resolves absolute paths, and projects only +the selected namespace into the native sandbox. External caches are conservatively +`host_exclusive`, so they are safe without trusting a tool's internal locking. Mounted +actions receive no cache access, namespace escape attempts fail in the kernel sandbox, +and planning remains side-effect-free. `tool_concurrent` remains limited to built-ins +until the exact external adapter/tool protocol has independent concurrency +certification. Repository command recipes remain cache-free because repository-authored +commands do not carry an authenticated adapter contract. + +## Runtime and service contract + +Runtime declarations separate immutable identity from allocation: + +```rust +struct RuntimeDeclaration { + kind: RuntimeKind, + immutable_source: Option, + isolation: RuntimeIsolation, + allocation: AllocationRequest, + health: Vec, + reuse: ReusePolicy, + cleanup_owner: CleanupOwner, +} +``` + +Allocation values such as host ports, process IDs, container IDs, and socket paths do +not enter immutable component keys. They enter the environment generation and execution +record. Reuse requires matching immutable identity, compatible configuration, same lane +ownership, and successful health checks. + +An adapter may return external-resource declarations, but it may not create or delete +external infrastructure unless a separate provider capability and cleanup ownership +are explicitly granted. + +## Secret contract + +Planning sees only: + +```rust +struct SecretReference { + provider: String, + logical_name: String, + version_selector: Option, + purpose: String, + injection: SecretInjection, + scope: SecretScope, +} +``` + +The host resolves values. A plugin receives a one-use handle or an already-bound file +descriptor where possible. If a consumer requires an environment value or file, the +host performs injection directly into the child process sandbox. + +Adapters declare which command argument or output fields may echo credentials so the +host can add structured redaction. This declaration supplements, but never replaces, +host-wide exact-value redaction and secret scanning. + +## Plugin protocol and capabilities + +The implemented v1/v2 transport is a natively sandboxed subprocess using one +length-prefixed CBOR request and response. WASI components are a future transport for +the same models. Local plugins are installed by digest and declare: + +- adapter identity and supported planner protocols; +- component kinds and manifest patterns; +- bounded repository-relative read patterns and file/byte limits; +- timeout and response-size limits; +- one executable digest and experimental stability. + +The planner capability set is deliberately fixed rather than open-ended: bounded pinned +bytes in, discovery or plan data out, no direct repository reads or writes, no child +process, no network, no shell, no secrets, no database, and no mount/publication +authority. Host-executed actions separately receive only their declared input/output +contract plus isolated HOME/tmp. V2 adds only host-executed typed action phases. A mounted +action runs in a pinned ephemeral candidate view, reads exact declared inputs, writes +only declared writable-private outputs plus isolated HOME/tmp, and cannot activate +partial state. Commands whose runtime wrapper needs another executable fail closed; +child-runtime permission remains a future separately certified capability. + +Capability examples: + +```text +fs.read.repository:apps/web +fs.read.staging +process.plan +process.inspect_output +network.plan:registry.npmjs.org +secret.reference:team-registry +runtime.plan:oci-container +``` + +`process.plan` lets a plugin propose a command; it does not let the plugin spawn it. +Network access during discovery or a v1/v2 host-executed action is not supported. + +Plugins use request IDs and deterministic serialized values. Unknown enum variants are +handled through contract negotiation rather than silently downgraded. The host applies +deadlines and output-size limits to every call. + +## Ecosystem mappings + +The following mappings show how one policy vocabulary fits very different toolchains. + +### Node, Next.js, and Vite + +| Concern | Inputs | Policy/binding | +| --- | --- | --- | +| Node toolchain | version file, resolved distribution digest, platform/ABI | `immutable_shared`, added to `PATH` | +| Installed dependencies | package manifest, lockfile, package-manager version, Node ABI, install/script policy | `immutable_seed_private` at `node_modules` when consumers may modify it; `immutable_shared` only for proven read-only layouts | +| Package download store | registry identity and package-manager compatibility | `cache_shared_content` | +| Next build state | source/config identity for readiness, but continuously mutable during dev | `writable_private` at `.next` | +| Vite cache/output | framework mode and source/config | `.vite` private or disposable; `dist` immutable only for an explicit build artifact | +| Dev server | generation plus command configuration | `runtime_private`, auto port and lane-private process | + +Next.js and Vite are profiles composed over the Node adapter. They do not duplicate +package installation logic. + +### Rust and Cargo + +| Concern | Inputs | Policy/binding | +| --- | --- | --- | +| Rust toolchain | `rust-toolchain.toml`, distribution digest, target components | `immutable_shared` toolchain | +| Registry crates and Git checkouts | Cargo version, source registry, checksums | `cache_shared_content` with host-mediated indexes | +| Compiled dependency seed | `Cargo.lock`, features, profiles, rustc identity, target, relevant build-script environment | optional `immutable_seed_private` lower for `target` | +| Active target directory | source and commands continuously mutate it | `writable_private` at `target` | +| Compiler cache | compiler identity, target, flags | `cache_shared_compiler` through sccache protocol | +| Credentials | registry token | `secret_runtime`, injected only into Cargo command | + +Cargo build scripts and proc macros make complete hermeticity difficult. Their permitted +environment and network policy are part of the fingerprint/provenance, and portability +is conservative. + +### CMake, Ninja, Make, vcpkg, and Conan + +| Concern | Inputs | Policy/binding | +| --- | --- | --- | +| Toolchain | compiler/linker digests, SDK/sysroot, toolchain file, generator | `immutable_shared` or `external_immutable` | +| Downloaded dependencies | lockfiles/manifests, registry identity, triplet/profile | adapter-specific shared content cache | +| Configure/build tree | `CMakeLists.txt`, presets, generator, absolute source/build paths | `writable_private`; generally host-bound | +| ccache | compiler/flag compatibility | `cache_shared_compiler` | +| Install prefix | explicit install action and full dependency identity | `immutable_shared` if relocatable and validated; otherwise host-bound seed/private | +| Test scratch | test-specific generated files | `disposable` | + +CMake adapters must account for absolute paths in `CMakeCache.txt`. A private build tree +cannot be promoted to a portable immutable artifact merely because compilation +succeeded. + +The implemented `trail/cmake-build@1` adapter covers the safe first slice: discovery, +deterministic host/tool compatibility identity, atomic lane-private build-directory +ownership, generation provenance, mounted-view classification, and crash recovery. +`env sync` provisions `/build` but does not run configure. Run configure and +build inside the lane: + +```sh +trail env sync --adapter trail/cmake-build@1 +trail lane exec -- cmake -S . -B build -G Ninja +trail lane exec -- cmake --build build +``` + +Real Linux/FUSE and macOS/NFS conformance configures and builds the same project in two +lanes, verifies that each cache records its mounted lane path, cleans one build, and +requires the other executable to remain intact. Presets, toolchain files, ccache, +vcpkg/Conan and optional relocation-validated install prefixes remain later slices of +006.7. A native Windows/Dokan two-lane CMake verifier is wired into CI; observed release +evidence remains pending because the local cross-check cannot build `aws-lc-sys` without +the MinGW compiler and cannot substitute for native Dokan execution. + +### Docker and OCI workflows + +| Concern | Inputs | Policy/binding | +| --- | --- | --- | +| Base image | resolved manifest digest and platform | `external_immutable` | +| Image build | Dockerfile, context subset, build args excluding secret values, frontend version | resulting OCI digest is `external_immutable` | +| Build cache | BuildKit protocol and builder identity | tool-managed shared cache with explicit scope | +| Build secrets | provider references | `secret_runtime` mounts; never image layers or keys | +| Running container | image digest, runtime config, lane network | `runtime_private` | +| Named volume | data ownership and lifecycle | `writable_private` if Trail-owned; `external_managed` otherwise | +| Ports and networks | allocation policy | generation-bound runtime resources | + +Tags are discovery inputs; resolved digests are generation identities. Trail must never +delete a user-owned image, container, volume, or remote builder cache. + +### Python + +The experimental `trail/python-venv@1` built-in implements the safe baseline today. It +owns `/.venv` as `writable_private`, publishes no shared layer, and preserves +the directory across a compatible re-sync. Synchronization automatically runs +`python -m venv --without-pip .venv` at the lane's final mountpoint: + +```sh +trail env sync --adapter trail/python-venv@1 +# Optionally let a lockfile-aware tool populate the initialized private path: +trail lane exec -- uv sync --frozen +``` + +Mounted initialization never runs against the active lane upper. Trail constructs an +ephemeral candidate view with the pinned source root, desired immutable bindings, and +prepared private seeds; source and unrelated writes land in disposable uppers. After +the command succeeds, Trail rejects every write outside the newly prepared private +outputs, copies those outputs back to staging, and performs the ordinary atomic +generation activation. A non-zero exit, undeclared write, process kill, or host crash +therefore leaves the predecessor generation and its private upper unchanged. Recovery +also removes abandoned candidate directories. + +Real Linux/FUSE, macOS/NFS, and Windows/Dokan conformance creates two virtual +environments, verifies that `sys.prefix` identifies each mounted lane, mutates one +environment, and requires the other lane to remain unchanged. It also verifies that a +compatible re-sync preserves the private environment and creates no shared layer. +Additional native fixtures cover multi-component `sync-all`, initializer failure, +undeclared source writes, kill-point recovery, and abandoned-candidate cleanup. + +Sharing interpreter distributions and wheel/download content safely remains a separate +typed-cache slice; Trail must not represent a path-bearing virtual environment itself as +a portable immutable artifact without relocation validation. + +| Concern | Inputs | Policy/binding | +| --- | --- | --- | +| Interpreter | distribution digest, version, ABI | immutable toolchain | +| Wheels/downloads | lockfile, index identity, platform tags | shared content cache | +| Virtual environment | lockfile, interpreter identity, install policy | immutable seed/private when tools rewrite entry points; immutable only after relocation validation | +| Bytecode/test caches | source/runtime dependent | private or disposable | +| Index credentials | provider reference | runtime secret | + +### Go + +| Concern | Inputs | Policy/binding | +| --- | --- | --- | +| Go toolchain | distribution digest and target | immutable toolchain | +| Module cache | `go.sum`, proxy identity, tool version | shared content cache with tool-compatible locking | +| Build cache | compiler/target/options | shared compiler cache when tool protocol permits | +| Generated outputs | generator identity plus declared source | immutable artifact or private tree according to consumption | + +These mappings are defaults, not universal truths. A repository can narrow sharing or +choose a private policy when native modules, hooks, absolute paths, or tool behavior make +sharing unsafe. + +## Adapter composition + +Framework adapters extend lower-level adapters through typed composition: + +```text +Next.js profile + requires Node toolchain + requires Node dependency component + adds private .next state + adds optional runtime dev server + adds framework validation +``` + +Composition rules: + +- a child may add inputs, validation, bindings, and runtime resources; +- a child may narrow storage sharing, network, scripts, and portability; +- widening capability or sharing requires explicit user/organization approval; +- output ownership is unique after expansion; +- parent adapter version and normalized profile expansion are identity inputs; +- error explanations retain the responsible adapter and source field. + +## Conformance suite + +Every adapter must pass a host-provided test kit before receiving a certification tier. + +### Determinism + +- identical canonical requests produce identical fingerprint plans and component plans; +- input ordering, directory enumeration, locale, and map ordering do not change keys; +- every observed output-changing input is declared; +- irrelevant source edits do not invalidate the component. + +### Isolation + +- two lanes attach the same artifact while private writes remain invisible to each + other; +- copy-up, whiteout, nested delete, bulk replacement, and rename behave correctly; +- an adapter cannot escape staging through `..`, symlinks, hard links, archives, or + absolute paths; +- commands cannot write repository source unless expressly granted. + +### Publication and recovery + +- failures and cancellation never publish partial output; +- concurrent identical builds publish one result; +- crashes at each action, validation, publication, and activation boundary recover; +- corruption makes an artifact unavailable to new generations without destroying + evidence or old generation records. + +### Security + +- network, scripts, shell, host paths, and secret capabilities are denied by default; +- secret canaries do not appear in keys, manifests, logs, transcripts, diagnostics, + checkpoints, or exports; +- credentials embedded in URLs and tool output are redacted; +- external cleanup respects ownership. + +### Portability + +- platform and ABI changes invalidate when required; +- symlink, executable-mode, case-collision, and absolute-path fixtures narrow or reject + portability correctly; +- every claimed workspace backend passes the shared filesystem suite. + +### Scale + +- real ecosystem fixtures, not only synthetic files, cover large Next.js/Vite + `node_modules`, Cargo registries and `target`, CMake/Ninja builds, and OCI layers; +- warm plan and attach do not recursively scan unchanged artifacts; +- metadata-heavy replacement reports progress and supports cancellation; +- physical byte accounting demonstrates cross-lane sharing. + +## Certification tiers + +| Tier | Meaning | +| --- | --- | +| `experimental` | Contract-valid; incomplete ecosystem and scale coverage; explicit opt-in. | +| `isolated` | Determinism, containment, secret, and multi-lane isolation suites pass. | +| `reproducible` | Hermetic or fully declared inputs, validated portability, and rebuild comparison pass on supported platforms. | +| `builtin` | Maintained with Trail, migration guarantees, all supported backends, and release-gating fixtures. | + +Certification applies to an adapter version and declared platform matrix, not its name +forever. Repository status reports the active tier. + +## Compatibility and evolution + +- Additive optional fields preserve a contract major. +- New required semantics, changed canonicalization, or changed policy meaning requires a + new major. +- The host can run multiple adapter majors so old generations remain inspectable. +- A migration creates a new component identity and generation; it never rewrites an old + manifest. +- Deprecations appear in discovery and plan reports with a mechanical replacement when + possible. +- Unknown policy classes, capabilities, and binding types fail closed. + +## Minimum viable adapter SDK + +The first SDK currently provides: + +- strongly typed request/response models; +- deterministic v1/v2 plan, typed dependency/action, command, output, discovery, and response + authoring helpers with local structural validation and unchanged v1 Rust/wire types; +- length-prefixed binary-safe CBOR framing with pre-allocation limits; +- declarative pinned input, discovery, command, output, portability, and error types; +- a one-request stdio server helper; +- generated-copy, mounted-initializer, direct mounted-tool, and adversarial reference adapters; +- host-side canonical path, identity, capability, resource, and provenance validation; +- host-owned `mounted_initialization` actions for trusted built-ins and authorized v2 + plugins, executed at the stable lane path through disposable candidate uppers before + atomic activation; +- macOS/Linux and Windows-native conformance verifiers. + +Still required before SDK stabilization are standard tool-probe helpers, +signed-package helpers, a standalone fixture host, golden stale-explanation +helpers, secret-canary utilities, progress events, and certification metadata. + +Adapters should not need to know Trail's database schema, mount implementation, NFS +protocol, overlayfs details, or artifact-store layout. + +## Contract acceptance criteria + +The contract is ready for stabilization when: + +- built-in Node and Cargo implementations can express existing behavior without private + host escape hatches; +- a generic command recipe can build and attach a verified immutable artifact; +- CMake can model private build trees and optional immutable install prefixes; +- OCI can separate image identity, BuildKit cache, secret mounts, and lane runtime; +- framework profiles compose without claiming the same output twice; +- deterministic fingerprint and stale-explanation golden tests pass; +- capabilities shown in a plan exactly match those observed during execution; +- plugin failure or timeout cannot corrupt Trail state; +- unknown versions and fields fail predictably; +- the same normalized plan drives CLI, HTTP, MCP, and Rust API reports. diff --git a/docs/design/lane-coordination.md b/docs/design/lane-coordination.md index c15660a..c994c44 100644 --- a/docs/design/lane-coordination.md +++ b/docs/design/lane-coordination.md @@ -297,7 +297,7 @@ This report is the main transfer object for moving lane work between hosts or fr ## Merge Coordination -Direct `merge-lane` checks readiness before merging. Merge queue runs serialize queued entries and record merge results. If a conflict appears, the queue item becomes conflicted and a conflict set persists for resolution. +Direct `lane merge` checks readiness before merging. Merge queue runs serialize queued entries and record merge results. If a conflict appears, the queue item becomes conflicted and a conflict set persists for resolution. This avoids silent overwrites and gives humans and lane-running hosts an explicit conflict-resolution surface. diff --git a/docs/design/layered-lane-workspaces.md b/docs/design/layered-lane-workspaces.md index 4873adb..0ea3643 100644 --- a/docs/design/layered-lane-workspaces.md +++ b/docs/design/layered-lane-workspaces.md @@ -3,6 +3,13 @@ Status: implemented behind platform acceptance gates; native Windows Dokan CI evidence remains required before declaring the rollout complete. +Environment-layer follow-on: the shared adapter host, normalized component state, +Node/Cargo/Go adapters, metadata-driven discovery, atomic environment generations, and +CLI/HTTP/MCP environment surfaces are implemented. Restricted repository command +recipes, local reusable profiles, multi-output atomic publication, and experimental +content-addressed isolated subprocess adapters are implemented; signed catalogs/WASI +packaging and the full universal-environment graph remain tracked by Plan 006. + This document defines the execution and filesystem architecture needed for Trail to become the default local coordination substrate for many coding agents working concurrently in a large repository. It extends Trail's existing lane, @@ -32,9 +39,9 @@ Trail already has the right starting points: - Virtual and sparse lanes. - Full materialization with filesystem clone COW where available. -- FUSE overlay COW on Linux and supported macOS builds. +- FUSE COW on Linux and supported macOS builds. - Loopback NFS COW on macOS. -- Dokan-backed overlay support on Windows. +- Dokan COW support on Windows. - Persistent lane uppers, whiteouts, workdir manifests, sessions, turns, checkpoints, gates, readiness, merge queues, and safe Git apply. @@ -198,8 +205,9 @@ content so filesystem backends can serve ranged reads and use reflink copy-up. ## Current Foundation and Gaps -`LaneWorkdirMode` currently supports `virtual`, `sparse`, `full-cow`, -`overlay-cow`, and `nfs-cow`. Lane spawning records the mode and creates either +`LaneWorkdirMode` currently supports `auto`, `virtual`, `sparse`, `native-cow`, +`portable-copy`, `fuse-cow`, `nfs-cow`, and `dokan-cow`. Lane spawning records +the requested and resolved modes and creates either a materialized workdir or an overlay mountpoint. `trail agent start` holds a mount for the child process, records the lane workdir when the child exits, and then unmounts it. @@ -288,7 +296,7 @@ trail/src/db/lane/view/ fuse.rs Linux/macFUSE adapter nfs.rs macOS loopback NFS adapter dokan.rs Windows adapter - clone.rs full-COW portable fallback + clone.rs native-COW portable fallback cache/ store.rs immutable layer store and pinning builder.rs build leases, staging, atomic publish @@ -631,6 +639,19 @@ sync` or fail with a concrete stale-environment blocker. A successful sync builds or reuses the new layer and swaps bindings at a quiescent view-generation boundary. +Dependency replacement is a Trail administrative operation, not a recursive +filesystem emulation. `trail deps sync` requires an unmounted lane, builds or +reuses the immutable layer first, removes the matching dependency subtree from +the private generated upper, clears whiteouts at or below that mount path, and +then advances the layer binding and view generation. If the process stops +before the binding update, the previous immutable layer remains a valid +fallback. Source-class paths are never accepted by this bulk reset primitive. + +This path matters most for loopback NFS: NFSv3 has no recursive-delete request, +so ordinary `rm -rf node_modules` must still issue one operation per entry. +Agents should use `trail deps sync` for dependency replacement; direct POSIX +deletion remains correct but is intentionally not the optimized control path. + Trail must not infer that a lockfile update is valid merely because an agent mutated `node_modules`. The lockfile and adapter build remain authoritative. @@ -951,7 +972,7 @@ Preferred order: allow an isolated mount. 2. Shared FUSE `ViewCore` adapter when the source root is lazy or native overlay is unavailable. -3. Filesystem reflink full-COW fallback. +3. Filesystem reflink native-COW fallback. 4. Sparse or virtual mode when policy rejects copying. Unprivileged/container environments require explicit `/dev/fuse` or a suitable @@ -964,14 +985,31 @@ Preferred order: 1. Built-in loopback NFS COW for installation-free terminal use. 2. macFUSE when installed and approved. -3. APFS clonefile full-COW fallback. +3. APFS clonefile native-COW fallback. 4. Sparse or virtual mode. The NFS server must bind only to loopback, validate mount ownership, reject unsupported node kinds, and preserve immediate mutation visibility required by -checkpointing. Metadata-heavy dependency workloads need dedicated benchmarks -because disabling NFS attribute caches improves correctness but can reduce -throughput. +checkpointing. macOS mounts use synchronous write requests because its default +`nosync` mode may return after queueing a truncate/write sequence; immediate teardown +could otherwise remount the private upper after the truncate but before the data WRITE. +The userspace server fsyncs each WRITE before reporting NFS `FILE_SYNC`. +Metadata-heavy dependency workloads need dedicated benchmarks because disabling NFS +attribute caches improves correctness but can reduce throughput. + +The real Next.js/Vite benchmark confirms that a global `noac,actimeo=0` policy +does not scale to large Node resolution graphs. Keep strict coherency for +writable uppers and directory mutation boundaries, but allow long-lived +attribute/content caching for immutable source and dependency lowers. Layer +binding generation changes must invalidate those cached lower identities. +Immutable publication now writes a bounded durable verification seal containing the +content-addressed manifest identity, layer summary, and platform filesystem identity of +the published directory. Routine cache-hit reuse and attachment validate that seal +without reopening the artifact tree. Missing, oversized, malformed, or stale seals +fall back to one full scan and are rewritten only after every entry matches. Explicit +`cache verify`, doctor, and readiness remain full-content operations. This removes +hundreds-of-megabytes of rehashing from warm attachment without claiming that bounded +attachment evidence is a full audit. ### Windows @@ -980,13 +1018,14 @@ mapping policy, rename semantics, sharing modes, delete-pending state, and reparse-point safety. A future ProjFS backend can be evaluated without changing `ViewCore`. -### Clone fallback +### Native and portable materialization -`full-cow` remains valuable for tools that are incompatible with mounted -filesystems. It should gain a `require_clone` result: if the filesystem cannot -clone safely, a large-repository policy can reject ordinary materialization. -Reports distinguish reflinked bytes, copied bytes, and files hydrated from -Trail objects. +`native-cow` requires a successful filesystem clone for every file and never +falls back to byte copying. `portable-copy` clones opportunistically and copies +the remainder. Materialized-only `auto` stages a strict attempt first, then +restarts portably only when clone support, filesystem placement, or a complete +validated native source is unavailable. Reports distinguish requested mode, +resolved mode, cloned bytes, and copied bytes. ### Conformance contract @@ -1239,7 +1278,7 @@ view does not satisfy a required current-root gate. ### Phase 0: Specify and benchmark current behavior - Add a backend-independent filesystem conformance model. -- Measure current full-COW, FUSE, NFS, and Dokan behavior. +- Measure current native-COW, FUSE, NFS, and Dokan behavior. - Add large-root mount/start benchmarks and physical-space fixtures. - Document current backend limitations. @@ -1387,17 +1426,15 @@ and GC. Report logical and physical storage after each additional lane. ## Migration and Compatibility -Existing lanes remain valid: +Existing lanes using the current mode vocabulary remain valid: - `virtual` and `sparse` semantics do not change. -- `full-cow`, `overlay-cow`, and `nfs-cow` continue to parse. -- Legacy overlay upper directories can be mounted through a compatibility - adapter, checkpointed, and migrated to split uppers. +- `native-cow`, `portable-copy`, `fuse-cow`, `nfs-cow`, and `dokan-cow` are the supported materialized or mounted COW modes. +- Removed workdir mode names are rejected. Operators must remove and recreate + those lanes with the platform-appropriate current mode. - Existing lane `metadata_json` remains readable while typed workspace-view tables become authoritative for new lanes. -- The default remains `full-cow` until `auto` passes platform conformance and - performance gates; changing the default is a separately documented product - decision. +- New materialized selections default to `auto`; mounted modes remain explicit. Backups from older schemas restore without views. New backups restore source uppers but rebuild caches. JSON reports add fields compatibly where possible; @@ -1491,7 +1528,7 @@ The following decisions require prototypes and measurements: - Whether the persistent directory index belongs in the versioned root object or remains a rebuildable projection keyed by root. -- Whether macOS NFS or APFS full-COW should be the automatic default for +- Whether macOS NFS or APFS native-COW should be the automatic default for metadata-heavy Node workloads. - Whether a workspace-local or user-global immutable cache is the first supported scope. Workspace-local is simpler and safer; global saves more @@ -1515,8 +1552,9 @@ Current implementation areas that this design extends: - Lane spawn and materialization: `trail/src/db/lane/lifecycle.rs` - Workdir lifecycle and manifests: `trail/src/db/lane/workdir.rs` and `trail/src/db/lane/workdir/` -- FUSE and Dokan overlay: `trail/src/db/lane/workdir/overlay.rs` -- macOS loopback NFS overlay: `trail/src/db/lane/workdir/nfs_overlay.rs` +- FUSE COW: `trail/src/db/lane/workdir/fuse.rs` +- Dokan COW: `trail/src/db/lane/workdir/dokan.rs` +- macOS loopback NFS COW: `trail/src/db/lane/workdir/nfs_overlay.rs` - Filesystem clone COW: `trail/src/db/util/fs_cow.rs` - Workdir recording: `trail/src/db/lane/workdir/record.rs` - Agent terminal lifecycle: `trail/src/cli/command/handler/agent.rs` diff --git a/docs/design/native-agent-hooks-and-acp.md b/docs/design/native-agent-hooks-and-acp.md new file mode 100644 index 0000000..bade98e --- /dev/null +++ b/docs/design/native-agent-hooks-and-acp.md @@ -0,0 +1,2383 @@ +# Native Agent Hooks and ACP Integration Design + +Status: Implemented +Audience: Trail maintainers, agent-integration authors, editor/CLI integrators, and reviewers +Research snapshot: 2026-07-11 +Related design: [ACP relay](acp-relay.md) + +## Executive Summary + +Trail should support two first-class ways to observe an AI coding agent: + +1. **ACP relay capture**: Trail sits between an ACP client and ACP agent and observes the structured protocol stream. +2. **Native hook capture**: the agent runs normally in its own terminal or editor and invokes Trail through its lifecycle hook, plugin, or extension system. + +Neither mode depends on the other. A user who does not use ACP can still obtain durable Trail sessions, turns, messages, events, spans, token usage, tool activity, approvals, transcript artifacts, and worktree checkpoints. A user who does use ACP gets the richer protocol-native stream already implemented by Trail. If both integrations are present, they feed the same capture coordinator and must not create duplicate sessions, turns, messages, spans, or checkpoints. + +The core architectural decision is: + +> ACP and native hooks are capture transports. Trail's lane activity model is the product model. + +Provider-specific code should be narrow. It discovers the provider, installs and removes only Trail-owned configuration, parses native payloads, locates native transcripts, and renders provider-specific hook responses. It must not own Trail session orchestration or checkpoint policy. A shared capture coordinator validates and deduplicates receipts, selects a capture owner, applies normalized lifecycle events, imports transcripts, records file changes, and recovers interrupted work. + +This design initially targets: + +- OpenAI Codex +- Anthropic Claude Code +- Pi +- OpenCode +- Cursor +- Google Gemini CLI +- GitHub Copilot CLI +- xAI Grok Build + +The implementation should borrow the strongest ideas from both Entire and Atomic. Entire contributes adapter inversion of control, native transcript preservation, canonical export, safe hook installation, pre-turn transcript offsets, full turn-end snapshots, and broad provider fixtures. Atomic contributes a pure exhaustive lifecycle state machine, managed-run correlation, exact change-set attestations, causal provenance graphs, immutable factual envelopes separated from redactable transcript material, declarative hook manifests, and portable trace export. + +Trail should not copy Entire's hidden Git branch or Atomic's practice of switching the repository's current view when a session starts. Trail remains a local-first operation database, lanes remain explicitly bound work containers, and publication to Git remains a separate boundary. + +## Context + +Trail already has the durable concepts needed for agent observability: + +- lanes and lane workdirs; +- sessions and turns; +- user and assistant messages; +- arbitrary events; +- traces and spans; +- approvals and run state; +- structured patches and workdir synchronization; +- per-turn metadata through `trail.turn_envelope`; +- ACP session mappings and recovery. + +The current ACP relay is the richest integration. It sees prompts, streamed assistant updates, tool calls, structured diffs, permissions, usage, cancellation, and session closure. The existing terminal-provider path is intentionally coarser and generally captures a final checkpoint only when the child process exits. + +That leaves an important product gap. Many users prefer an agent's native terminal or editor UX and do not want an ACP host in the middle. Most major coding agents now expose lifecycle hooks, plugins, or extensions that can observe enough of the session to build a useful Trail record. Trail should use those native extension points without requiring the user to change how they launch or interact with the agent. + +## Goals + +### Product Goals + +- Record agent activity when the user runs an agent directly, without ACP. +- Preserve a consistent Trail review experience across ACP and hook capture. +- Support session, turn, message, event, span, token, tool, approval, transcript, and file-change evidence when the provider exposes it. +- Make installation and removal safe, local, inspectable, and idempotent. +- Allow project-scoped configuration by default and user-scoped configuration on request. +- Recover useful evidence after hook failures, agent crashes, terminal closure, compaction, and resume. +- Make provider limitations explicit through capabilities and diagnostics. +- Keep hooks fast and fail-open for observability-only behavior. +- Preserve provider-native transcripts as evidence rather than pretending Trail can losslessly reconstruct them from normalized events. +- Make adding a new provider primarily an adapter and fixture task. +- Explain not only what happened but, where evidence supports it, the causal path from goal through exploration, modification, and verification. +- Produce exact, reviewable session attestations over the Trail changes actually created by the session. +- Support nested or externally orchestrated agents without attributing unrelated sessions to the outer run. +- Export portable, vendor-neutral trace bundles without making that export the primary storage model. + +### Architecture Goals + +- Reuse the existing lane activity model and capture operations. +- Share orchestration between ACP and native hooks. +- Accept events through an in-process path, a short-lived CLI process, or the Trail daemon without changing semantics. +- Make every receipt idempotent and replayable. +- Correlate the same native session observed through more than one transport. +- Maintain a strict Git boundary: capture in Trail first; publish or export to Git only through explicit Git workflows. +- Keep raw provider payloads versioned and bounded so mappings can be improved later. +- Separate immutable factual evidence from mutable, redactable, or regenerated interpretations. +- Model managed capture runs as renewable leases scoped by workspace, workdir, lane, owner, and executor. + +## Non-Goals + +- Replacing the native transcript viewer of every agent. +- Defining a universal agent-control protocol. ACP already covers the protocol role. +- Making native hooks block tool calls or prompts by default. Trail capture is observability-first. +- Secretly installing global hooks. +- Rewriting arbitrary user configuration into Trail's preferred formatting. +- Treating every provider event as semantically identical. +- Reconstructing private chain-of-thought that the provider does not expose. +- Copying Entire's `entire/checkpoints/v1` hidden branch, commit trailers, or automatic commit coupling. +- Requiring a Git commit to close a Trail turn or session. +- Guaranteeing exact token or cost values when a provider does not emit them. +- Claiming private chain-of-thought as provenance. Trail records exposed plans, summaries, tool evidence, and provider-supplied signed blocks only under explicit policy. +- Automatically rewriting agent context files with inferred learnings. +- Treating a session attestation as proof that the model's output was correct or that a human approved it. + +## Design Principles + +### One Domain Model, Multiple Transports + +ACP, command hooks, HTTP hooks, TypeScript plugins, and native extensions are different delivery mechanisms. Once a payload enters Trail, it should become a normalized lifecycle event and be processed by one coordinator. + +### Preserve Native Evidence + +Normalized events optimize search, review, and cross-provider behavior. They are not a lossless transcript format. Trail should retain the provider's canonical transcript or export, subject to policy and redaction, and attach its digest and provenance to the Trail session. + +### Prefer Canonical Export Over Reconstruction + +If a provider offers a supported export command or SDK API, use it. If it exposes a stable transcript file, copy or incrementally ingest that file. Only reconstruct a transcript from hooks when neither is possible, and mark the artifact as reconstructed. + +### Passive Adapters, Active Framework + +Adapters translate. The framework orchestrates. Provider packages must not directly implement checkpoint policy, create ad hoc database rows, or infer cross-provider ownership independently. + +### Fail Open, Diagnose Loudly + +A recording failure must not normally block the user's agent. Hooks should return the provider's success response even when Trail is unavailable, spool the receipt when safe, and surface the degraded state through `trail agent hooks doctor`, session warnings, and daemon health. + +Policy enforcement is a separate opt-in mode. A future Trail guardrail hook may fail closed, but it must use a different command, configuration marker, and user-visible policy. + +### Monotonic Fidelity + +Additional evidence should enrich an existing Trail record rather than create another record. ACP can contribute structured tool and message updates while a native Stop hook contributes a final transcript and usage summary. The result is one session and one turn. + +### Local-First and Explicit Publication + +Trail writes operational state to its own storage. Git commits, notes, branches, or remote publication are separate actions. This preserves Trail's existing provenance and review model and avoids surprising commits or refs. + +### Facts, Derived Interpretations, and Attestations Are Different + +Provider receipts, messages, tool results, usage counters, structured patches, and Trail checkpoints are factual evidence with explicit source confidence. A rule-based activity classification or generated explanation is a derived interpretation and must link back to its source records. An attestation is a signed or content-addressed statement about a precisely enumerated evidence set. None of these layers may silently masquerade as another. + +### Exact Coverage Beats Ambient History + +Session summaries and attestations use the exact Trail change IDs recorded for the session's turns. They must never scan an entire lane and attribute inherited baseline operations or concurrent human work to the agent. + +### Pure Lifecycle Decisions, Side Effects at the Edge + +The session/turn state machine should be a pure function from current state, normalized event, and bounded context to a new state plus ordered actions. Database writes, transcript imports, checkpointing, and provider responses execute those actions outside the transition function. This makes missing, duplicate, late, resume, and crash events exhaustively testable. + +## User Experience + +### Native Hooks Without ACP + +The primary workflow is: + +```console +$ trail init +$ trail lane spawn feature-auth --workdir-mode native-cow +$ cd "$(trail lane workdir feature-auth)" +$ trail agent hooks setup codex --lane feature-auth --yes +$ trail agent hooks doctor codex +$ codex +``` + +The user interacts with Codex normally. The installed project hooks invoke Trail at session start, prompt submission, tool boundaries, turn stop, compaction, subagent boundaries, and session end. Trail creates or resumes the mapped session and records each turn. + +Equivalent setup should work for every supported provider: + +```console +$ trail agent hooks setup claude-code --yes +$ trail agent hooks setup pi --yes +$ trail agent hooks setup opencode --yes +$ trail agent hooks setup cursor --yes +$ trail agent hooks setup gemini --yes +$ trail agent hooks setup copilot --yes +$ trail agent hooks setup grok --yes +``` + +Project scope is the default. User scope is explicit: + +```console +$ trail agent hooks setup codex --scope user --yes +``` + +User-scoped hooks discover the nearest initialized Trail workspace at runtime. If no workspace exists, they exit successfully without recording. + +### ACP Capture + +The existing ACP workflow remains valid: + +```console +$ trail acp relay codex --lane feature-auth +``` + +The ACP relay uses the same normalized coordinator. It normally becomes the primary structured-event owner for that session because it has the highest-fidelity live stream. + +### Both Installed + +Users should not have to uninstall native hooks before using ACP. The ACP relay exports correlation context to the upstream process: + +```text +TRAIL_CAPTURE_MODE=acp +TRAIL_CAPTURE_RUN_ID= +TRAIL_CAPTURE_WORKSPACE= +TRAIL_CAPTURE_LANE= +``` + +Native hooks still may submit transcript-finalization or audit receipts, but the coordinator recognizes the same run and suppresses duplicate lifecycle mutations. If the provider strips these environment variables, correlation falls back to native session identifiers, working directory identity, provider, and a bounded time window. + +### Inspecting Integration State + +```console +$ trail agent hooks list +$ trail agent hooks status codex --json +$ trail agent hooks doctor --all +$ trail agent hooks events codex --last 20 +$ trail session current feature-auth +$ trail lane handoff feature-auth +``` + +`status` describes installed files, configured events, ownership markers, provider version, detected transcript support, capture mode, and last successful receipt. `doctor` performs safe contract probes and reports fidelity gaps. + +### Removing Hooks + +```console +$ trail agent hooks remove codex +$ trail agent hooks remove codex --scope user +``` + +Removal deletes only entries or files Trail can prove it owns. It preserves unrelated keys, unknown provider fields, comments when the format supports them, and other hook commands. + +## Capture Modes and Fidelity + +Trail should expose the following modes in diagnostics and session metadata: + +| Mode | Launch path | Live messages | Live tools | Per-turn checkpoint | Native transcript | Control/approvals | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| `acp` | Through `trail acp` | Yes | Yes | Yes | Optional enrichment | Yes | +| `native-hooks` | Agent's own CLI/editor | Provider-dependent | Provider-dependent | Yes when turn hooks exist | Preferred | Observe by default | +| `terminal` | Child process launched by Trail | No | No | Usually final only | Provider-dependent | Process-level only | +| `hybrid` | ACP plus native enrichment | Yes | Yes | One deduplicated checkpoint | Preferred | Yes | + +Fidelity is a capability set, not a single support boolean. Every adapter reports: + +```text +session_lifecycle +turn_lifecycle +prompt_text +assistant_text +tool_lifecycle +tool_input +tool_output +permission_lifecycle +subagent_lifecycle +compaction_lifecycle +usage +cost +structured_diff +workdir_checkpoint +native_transcript +canonical_export +resume +context_injection +``` + +The CLI and API return `supported`, `partial`, `unavailable`, or `unknown`, plus a reason and the provider versions verified by Trail. + +## Architecture + +```mermaid +flowchart TB + subgraph Sources[Capture sources] + ACP[ACP relay] + CMD[Command or HTTP hooks] + PLUGIN[Provider plugin or extension] + TERM[Terminal process observer] + end + + ACP --> INGEST[Capture ingress] + CMD --> INGEST + PLUGIN --> INGEST + TERM --> INGEST + + INGEST --> VALIDATE[Validate, bound, redact, persist receipt] + VALIDATE --> ADAPTER[Provider adapter parse] + ADAPTER --> NORMAL[Normalized lifecycle event] + NORMAL --> OWNER[Correlation, ownership, deduplication] + OWNER --> COORD[Shared capture coordinator] + + COORD --> SESSION[Sessions and turns] + COORD --> TIMELINE[Messages and events] + COORD --> TRACE[Traces and spans] + COORD --> CHANGE[Structured patch or workdir checkpoint] + COORD --> ARTIFACT[Native transcript artifacts] + COORD --> APPROVAL[Approvals and run state] + COORD --> PROV[Causal provenance graph] + COORD --> ATTEST[Session attestations] + + SESSION --> TRAIL[(Trail storage)] + TIMELINE --> TRAIL + TRACE --> TRAIL + CHANGE --> TRAIL + ARTIFACT --> TRAIL + APPROVAL --> TRAIL + PROV --> TRAIL + ATTEST --> TRAIL + + TRAIL --> REVIEW[Inspect, handoff, review, search] + TRAIL --> EXPORT[Portable trace export] + TRAIL --> GIT[Explicit Git export/publication] +``` + +### Components + +#### Agent Registry + +The registry maps stable provider names and aliases to adapters. It owns capability reporting, discovery, version probes, and provider construction. + +Provider names are stable API values: + +```text +codex +claude-code +pi +opencode +cursor +gemini +copilot +grok +``` + +Aliases such as `claude`, `gemini-cli`, `copilot-cli`, and `grok-build` resolve at the CLI boundary but are not stored as canonical provider names. + +#### Provider Adapter + +The initial Rust trait should be conceptually equivalent to: + +```rust +trait NativeAgentAdapter { + fn identity(&self) -> AgentIdentity; + fn discover(&self, ctx: &DiscoveryContext) -> Result; + fn capabilities(&self, version: Option<&Version>) -> CapabilityReport; + + fn install_plan(&self, req: &InstallRequest) -> Result; + fn remove_plan(&self, req: &RemoveRequest) -> Result; + fn verify_install(&self, req: &VerifyRequest) -> Result; + + fn parse_receipt(&self, receipt: RawHookReceipt) -> Result>; + fn render_response(&self, outcome: &HookOutcome) -> Result; + + fn locate_transcript(&self, session: &NativeSessionRef) + -> Result>; + fn export_transcript(&self, session: &NativeSessionRef) + -> Result>; + fn resume_command(&self, session: &NativeSessionRef) + -> Result>>; +} +``` + +Optional behavior remains capability-gated. An adapter without canonical export does not implement a fake export. Common validation, redaction, worktree capture, database writes, ownership, and recovery stay outside the adapter. + +#### Declarative Adapter Manifest + +Atomic demonstrates that hook wiring can live in an integration-supplied manifest rather than requiring a Trail binary release for every renamed provider event. Trail should adopt a constrained version of this idea. + +A manifest may declare: + +- canonical provider and adapter bundle version; +- supported provider version range; +- project and user configuration targets; +- native event names and the Trail ingress verb each invokes; +- the provider-native JSON shape to merge; +- generated plugin/extension assets by content digest; +- required trust or feature checks; +- capability overrides and known limitations; +- ownership markers and a complete uninstall inventory. + +The manifest does not contain arbitrary parser code by default. Built-in Rust adapters still validate and normalize security-sensitive payloads. A future external adapter SDK may use a sandboxed WASM parser with explicit byte, time, filesystem, and network limits. + +Manifests are versioned, schema-validated, previewed, and signed when distributed remotely. Local unsigned manifests require an explicit `--allow-unsigned-manifest` confirmation. Unlike Atomic's current generic deep-merge behavior, every Trail-owned non-hook setting must be reversible: the installation inventory records whether Trail created, appended, or replaced each value and removal restores the prior value only when it has not drifted. + +This creates a useful split: + +```text +adapter code = payload semantics, validation, transcript logic +adapter manifest = provider version wiring and installable assets +capture core = state, evidence, checkpoints, provenance, recovery +``` + +#### Host-Local Correlation Buffer + +Rich plugin APIs such as OpenCode may emit token counters, reasoning summaries, message parts, todos, and tool events separately. A provider plugin may keep an ephemeral in-process buffer to aggregate those fragments into a final turn event. That buffer is an optimization and enrichment source, never durable truth. + +The durable receipt journal remains authoritative. If the plugin restarts, Trail reconstructs the turn from already-received receipts, the provider transcript/export, and the workdir. Plugin buffers should therefore: + +- be keyed by native session and turn/message IDs; +- cap accumulated text, tool output, and reasoning blocks; +- flush useful partial data periodically for long turns; +- send monotonic counters or provider step IDs so retries do not double-count; +- tolerate missing start timestamps after a plugin reload; +- clear state only after the durable ingress acknowledges receipt. + +#### Workspace Resolver + +Global hooks must work in the canonical workspace and in lane workdirs without relying on a brittle shell guard such as `test -d .trail`. The internal hook receiver resolves its workspace in this order: + +1. authenticated installation/workspace identifier in the hook command; +2. `TRAIL_CAPTURE_WORKSPACE` set by ACP or a managed launcher; +3. a Trail lane/workdir pointer file created during materialization; +4. upward discovery of `.trail` from the provider-reported canonical cwd; +5. no-op success with a bounded diagnostic if no workspace is found. + +The pointer contains only a workspace locator and lane/workdir identity, is validated against the canonical Trail database, and cannot grant more authority than the installed hook. Trail should invoke the receiver directly and perform discovery in Rust rather than embedding complex portable shell conditionals in provider configuration. + +#### Managed Capture Run Registry + +Outer orchestrators, Trail Agent, ACP hosts, and nested agents need an explicit correlation surface. A managed capture run declares: + +- opaque `capture_run_id`; +- owner agent and owner session; +- optional executor agent; +- Trail workspace, lane, and workdir; +- optional task/work-item ID; +- creation, renewal, and expiry timestamps. + +Runs are renewable leases. Expiry is crash protection, not a maximum session duration. When several runs cover a cwd, Trail selects the longest canonical workdir match and then requires the provider to match the owner or declared executor. Ambiguity is a diagnostic, not permission to guess. + +Hooks are never suppressed merely because a run exists. Newly created native-session mappings are stamped with the governing run and adopt its lane. Pre-existing sessions are not restamped, preventing a concurrent direct session from being attributed to a newly started outer run. Ending the managed run returns the exact sessions, turns, changes, artifacts, and attestations bearing that stamp. + +#### Capture Ingress + +Ingress accepts a raw provider receipt and returns a provider-compatible response. It has three implementations with identical semantics: + +- direct function calls from the ACP relay; +- `trail agent hook receive ` for command hooks; +- `POST /v1/agent-hooks/{provider}/{event}` for providers supporting HTTP hooks or local plugins using the daemon. + +The command name is deliberately singular and low-level. Users manage integrations through `trail agent hooks ...`; provider configuration invokes `trail agent hook receive ...`. + +Example installed command: + +```text +trail agent hook receive codex UserPromptSubmit --installation +``` + +The raw payload is read from stdin. Provider-specific environment fields may supplement it, but arguments must never contain prompt text, tool input, or secrets. + +#### Durable Receipt Journal + +Every accepted hook invocation receives a `receipt_id`. Before semantic processing, Trail stores a bounded receipt record containing: + +- provider and native event name; +- installation ID and workspace identity; +- native session, turn, tool, and subagent identifiers when safely extractable; +- source transport; +- payload digest and optional redacted raw payload reference; +- provider timestamp and Trail receive timestamp; +- processing status and diagnostic; +- deduplication key; +- schema/version hints. + +Persisting before dispatch makes a command hook replayable after a crash. If the main Trail database cannot be opened quickly, the CLI may atomically spool a bounded file under Trail's local runtime directory and exit successfully. The daemon drains the spool later. Spool files use restrictive permissions and the same redaction policy as normal receipts. + +#### Capture Coordinator + +The coordinator consumes normalized events and calls existing Trail primitives in order: + +```text +session.started + -> lane session start or resume mapping + +turn.started + -> lane turn begin + -> user message + -> root agent span + +message.delta / message.completed + -> assistant message buffering and finalization + +tool.started / tool.completed / tool.failed + -> child span start/end + -> optional structured patch + +approval.requested / approval.decided + -> lane approval and run-state update + +turn.completed + -> transcript import + -> structured changes, then workdir fallback + -> assistant message finalization + -> root span end + -> lane turn end + +session.ended + -> close open turn if needed + -> final transcript import + -> lane session end +``` + +This is the same semantic sequence used by the ACP relay. The ACP implementation should be progressively refactored to call this coordinator instead of keeping an ACP-only orchestration path. + +## Normalized Lifecycle Contract + +### Event Envelope + +Every adapter emits one or more versioned envelopes: + +```json +{ + "schema": "trail.agent_lifecycle_event", + "version": 1, + "event_id": "evt_...", + "event_type": "tool.completed", + "occurred_at": 1783791000123, + "received_at": 1783791000189, + "provider": "codex", + "provider_version": "...", + "transport": "native-hooks", + "workspace_id": "...", + "lane_id": "...", + "capture_run_id": "...", + "native": { + "session_id": "...", + "turn_id": "...", + "tool_id": "...", + "subagent_id": null, + "event_name": "PostToolUse", + "sequence": null + }, + "correlation": { + "parent_event_id": null, + "trace_id": "...", + "span_id": "...", + "parent_span_id": "..." + }, + "payload": {}, + "evidence": { + "receipt_id": "...", + "raw_digest": "sha256:...", + "transcript_offset": null, + "confidence": "native-structured" + } +} +``` + +Provider timestamps are evidence, not ordering authority. Trail uses a per-native-session logical receive sequence when the provider supplies no reliable sequence. Late events may enrich closed turns but cannot silently reopen or rewrite their outcome. + +### Event Vocabulary + +Version 1 should recognize: + +- `session.started`, `session.resumed`, `session.updated`, `session.ended`; +- `turn.started`, `turn.completed`, `turn.failed`, `turn.cancelled`; +- `message.user`, `message.assistant.delta`, `message.assistant.completed`; +- `plan.updated`; +- `tool.started`, `tool.completed`, `tool.failed`; +- `approval.requested`, `approval.decided`; +- `subagent.started`, `subagent.completed`, `subagent.failed`; +- `compaction.started`, `compaction.completed`; +- `usage.updated`, `model.updated`; +- `workspace.diff`, `workspace.file_changed`, `workspace.checkpoint`; +- `context.injected`; +- `diagnostic`. + +Unknown provider events are not discarded. They become `provider..` Trail events with bounded, redacted payloads. They do not mutate the session state machine until an adapter version explicitly maps them. + +### Confidence and Provenance + +Each field derived from provider data records one of: + +- `protocol-structured`: observed directly over ACP; +- `native-structured`: supplied by a documented hook or plugin API; +- `native-transcript`: parsed from a provider transcript; +- `canonical-export`: produced by a provider export command/API; +- `worktree-observed`: inferred from Trail's workdir comparison; +- `heuristic`: derived from an unstable or undocumented shape. + +Review reports should expose meaningful degradation, for example: "tool duration is exact; token use was parsed from transcript; changed files were observed at turn end." + +## Session, Turn, and Span State Machine + +### Pure Transition Model + +Trail should encode lifecycle decisions as an exhaustive pure transition table. The persisted session phase is: + +```text +idle no open user turn +active a user turn is open +finalizing a terminal receipt is durable and checkpoint/artifact work is pending +ended provider session ended cleanly +interrupted recovery closed an incomplete session +``` + +The transition function is conceptually: + +```rust +fn transition( + phase: CapturePhase, + event: LifecycleEventKind, + context: TransitionContext, +) -> TransitionResult { + // no I/O +} + +struct TransitionResult { + new_phase: CapturePhase, + actions: Vec, + diagnostics: Vec, +} +``` + +Actions include: + +```text +ensure_session +begin_turn +append_evidence +start_span +end_span +request_turn_finalization +reconcile_workdir +import_transcript +close_turn +close_session +warn_duplicate +recover_interrupted_turn +``` + +The transition table explicitly covers every phase/event pair. Expected disorder is represented, not thrown away: + +| Current | Event | New phase | Important actions | +| --- | --- | --- | --- | +| `idle` | turn start | `active` | begin turn, capture baseline | +| `idle` | turn end | `finalizing` | synthesize/recover start, finalize if changed/evidence exists | +| `active` | turn start | `active` | close prior as interrupted, begin new turn | +| `active` | turn end | `finalizing` | acquire finalization lease, reconcile/import | +| `finalizing` | duplicate turn end | `finalizing` | attach richer evidence or no-op | +| `finalizing` | finalization complete | `idle` | close turn exactly once | +| `active` | session end | `finalizing` | finalize active turn, then close session | +| `idle` | session end | `ended` | final import and attestation | +| `ended` | resume/session start | `idle` | create capture epoch, retain session relation | +| any | unknown provider event | unchanged | append provider event only | + +Database writes execute after the transition is selected and are idempotent by action key. Unit tests enumerate the cross-product of states and event kinds; integration tests verify the resulting durable records. + +### Session Identity + +Trail adds a transport-neutral native session mapping. The natural identity is: + +```text +(workspace_id, provider, native_session_id) +``` + +When a provider does not expose a stable session ID, the adapter derives a scoped ID from the transcript path or provider session-state directory. A random ID is a last resort and is marked non-resumable. + +A native session maps to one Trail lane session. Resume events reopen or continue the same Trail session when the existing session is resumable; they do not create a duplicate just because a new process started. + +### Turn Identity + +Preferred native turn identity order: + +1. provider-supplied turn ID; +2. provider transcript message/turn ID; +3. prompt event ID; +4. deterministic hash of session ID, transcript offset, and prompt receipt; +5. Trail-generated ID persisted in session capture state. + +Repeated identical prompts must remain distinct. Prompt text alone is never a turn key. + +### Root and Child Spans + +Every turn has one root `agent` span. Tool and subagent spans are its children. Nested agent/tool relationships use provider IDs when available. Missing IDs receive deterministic receipt-derived IDs rather than names or input hashes, which can collide during repeated tool calls. + +Unmatched completion events create a synthetic start at the completion timestamp and carry `trail.synthetic_start=true`. Open spans are closed as `interrupted` at turn/session recovery. + +### Compaction + +Compaction is not a new Trail session. Trail records a compaction span and event with before/after transcript offsets, context usage when available, trigger, and summary digest. If the provider changes its native transcript or session identifier during compaction, the mapping table records an alias to the same Trail session. + +### Subagents + +Subagents remain inside the parent lane session by default and receive child spans plus provider-native identity. If the provider exposes a durable child session with its own transcript, Trail records a child native-session mapping and a `parent_native_session_id`. + +A future policy may materialize selected subagents as separate lanes, but capture must not do so implicitly. + +## Turn Capture Algorithm + +### At Session Start + +1. Resolve the initialized Trail workspace from the effective cwd. +2. Verify the hook installation ID belongs to this workspace or is an allowed user-scoped installation. +3. Validate and normalize native identifiers before using them in paths or queries. +4. Correlate or create the native session mapping. +5. Choose the lane according to explicit installation binding, capture environment, current workdir binding, or configured default. +6. Start or resume the Trail lane session. +7. Save provider/model/version, transcript locator, resume metadata, and capture capabilities. +8. Return optional context through the provider's documented response channel. + +### At Turn Start + +1. Deduplicate the prompt receipt. +2. Recover or close an abandoned prior turn. +3. Capture the current lane head and workdir state. +4. Record the native transcript byte offset, event index, or export revision. +5. Begin a Trail turn with a transport-neutral turn envelope. +6. Add the user message, subject to prompt capture policy. +7. Start the root agent span. +8. Persist the active-turn mapping before returning to the provider. + +### During the Turn + +- Buffer assistant deltas by native message ID. +- Start and end tool spans. +- Attach sanitized tool input/output summaries according to policy. +- Apply exact structured patches when the provider supplies them. +- Record permission events without deciding them unless an explicit guardrail integration is enabled. +- Accumulate usage monotonically and retain the raw provider counters. +- Record subagents and compaction. +- Periodically flush long-running state through the daemon; command hooks should keep synchronous work bounded. + +### At Turn End + +1. Make the turn-end receipt durable. +2. Read the transcript delta from the saved offset, if supported. +3. Refresh or export the full canonical transcript. Full snapshots make every turn independently recoverable even when delta parsing changes later. +4. Finalize assistant messages without duplicating ACP-streamed content. +5. Prefer structured diffs already captured during the turn. +6. Reconcile the workdir to catch Bash-generated, formatter-generated, or otherwise unobserved changes. +7. Create at most one Trail checkpoint for the turn. +8. Finalize usage, outcome, capture counts, redaction flags, and fidelity diagnostics. +9. Close remaining tool/subagent spans and the root span. +10. End the Trail turn and persist the new transcript offset. + +The turn-end workdir comparison is mandatory even for providers with edit hooks. Agents can modify files through shell commands, generated tools, external processes, or editor actions that bypass specific tool matchers. + +### At Session End + +Trail closes any active turn as completed, cancelled, failed, or interrupted based on the best available evidence. It performs a final transcript import and workdir reconciliation, closes open spans, and ends the lane session. Cleanup-only events from providers such as Pi should not end a session when the provider is switching or resuming into another session; the adapter supplies the reason. + +## Transcript and Artifact Model + +### Native Artifact Record + +Trail should add an artifact record rather than embedding large transcripts in event payloads: + +```text +artifact_id +workspace_id +lane_id +session_id +turn_id nullable +provider +artifact_kind transcript | export | tool-output | image | other +format provider-specific media type +source copied | exported | reconstructed +source_locator_redacted +content_object_id or blob_id +content_digest +size_bytes +start_offset nullable +end_offset nullable +redaction_profile +created_at +metadata_json +``` + +Artifacts use Trail's content-addressed object/blob storage. Database rows contain metadata and digests, not unbounded JSON or Markdown. + +### Snapshot and Delta Policy + +- Save a complete native transcript snapshot at every completed turn when the size policy permits. +- Also record the provider-native delta boundary so ingestion and review can be incremental. +- Deduplicate content by digest, so repeated full snapshots do not multiply storage when unchanged. +- For very large transcripts, store chunk manifests and only new chunks. +- Never tail a live transcript indefinitely from a short-lived hook process. +- Treat transcript formats as provider-owned and version them in artifact metadata. + +### Transcript Trust + +Transcript paths are untrusted input. Trail must canonicalize them, require them to fall under a provider-specific allowed root or an explicitly approved project path, reject symlink escapes, cap file size, and never construct paths directly from an unchecked session/tool ID. + +## Evidence Integrity and Redaction Layers + +Atomic's split between hashed change metadata and strip-friendly transcript material is a useful model. Trail should express the same distinction using its own object database rather than placing everything into one row or artifact. + +### Immutable Factual Envelope + +At turn closure Trail creates an immutable evidence manifest containing: + +- Trail and native session/turn identifiers; +- provider, adapter, model, and capture transports; +- prompt hash and optional system-instruction hash; +- exact `before_change` and `after_change`; +- exact structured-patch and workdir-checkpoint identifiers; +- source-qualified usage counters and cost; +- terminal outcome and stop reason; +- receipt IDs and payload digests; +- native artifact digests and formats; +- redaction policy active at capture time; +- provenance graph root/digest when present. + +The manifest is content-addressed and linked from the turn envelope. It contains hashes and bounded facts, not full prompts, transcripts, reasoning text, or tool output. + +### Redactable Evidence Attachments + +Full prompts, assistant text, native transcripts, raw hook payloads, tool inputs/outputs, images, and generated explanations remain separate artifacts governed by retention policy. They may be: + +- unavailable from the provider; +- stored locally; +- encrypted; +- exported selectively; +- replaced with a redaction tombstone; +- regenerated into a new derived artifact. + +Redacting an attachment does not rewrite the turn, checkpoint, or evidence manifest. The artifact record retains its original digest, byte count, media type, source, and a redaction status. Review can therefore distinguish "never captured," "captured and retained," and "captured then redacted." + +### Derived Interpretations + +Condensed transcripts, summaries, explanations, learnings, and rule-based classifications are versioned derived artifacts. Each records: + +- generator name and version; +- input evidence IDs/digests; +- model and prompt hash when an LLM was used; +- generation timestamp; +- confidence or deterministic rule identifier; +- superseded artifact, if regenerated. + +Derived artifacts never overwrite native evidence. A new explanation creates a new revision and may be selected as the preferred view. + +## Causal Provenance Graph + +Timeline events and trace spans answer what happened and when. A causal provenance graph adds a reviewable answer to how one action informed another without claiming access to hidden reasoning. + +### Node Vocabulary + +Trail should support the following initial node kinds: + +| Kind | Typical evidence | Meaning | +| --- | --- | --- | +| `goal` | user prompt, task objective | What the human or outer agent asked for | +| `plan` | provider plan/todo update | Explicit proposed work, not hidden thought | +| `exploration` | read, search, browse, diagnostics | Evidence gathering | +| `decision` | explicit agent summary or deterministic consolidation | A stated or inferred strategy choice | +| `commitment` | write/edit/patch tool, structured diff | A proposed filesystem/code mutation | +| `execution` | shell, deployment, package operation | A side effect not primarily verification | +| `verification` | test, lint, build, typecheck, eval | Evidence that checks work | +| `human_gate` | permission/approval request and decision | Human control boundary | +| `subagent` | child agent session/span | Delegated work | +| `checkpoint` | Trail turn change | Durable outcome of the turn | +| `error` | failed tool, agent/session error | Failure evidence | +| `learning` | explicit or derived reusable finding | Knowledge proposed for later context | + +### Edge Vocabulary + +Initial causal relations are: + +```text +led_to +informed_by +implemented_by +verified_by +blocked_by +resumed_after +delegated_to +failed_with +recorded_as +derived_from +supersedes +``` + +Edges are assertions with source and confidence. Temporal adjacency alone creates at most a low-confidence `led_to` edge. Exact provider parent IDs, ACP request relationships, tool-call IDs, and explicit plan/todo IDs create higher-confidence edges. + +### Deterministic Classification + +A versioned rule classifier may map common tool activity: + +- read/search/list/web -> `exploration`; +- edit/write/apply-patch -> `commitment`; +- known test/lint/typecheck/build commands -> `verification`; +- other shell commands -> `execution`; +- error status -> `error`. + +Classification is an overlay. The original event and span remain unchanged, the rule ID is stored, and unknown tools default to `unclassified` rather than being forced into a misleading category. Provider adapters may supply stable semantic tool categories that outrank name-based rules. + +Trail may deterministically consolidate patterns such as explore -> edit -> test or edit -> test-fail -> edit -> test-pass into a `decision` summary. The raw nodes remain present, `consolidated_from` lists them exactly, and the decision carries a confidence. Optional LLM-generated naming is a separate derived artifact and is never required for capture. + +### Privacy Boundary + +Trail must not store private chain-of-thought by default, even if a plugin can observe a field labeled `reasoning`. Safe defaults are: + +- store provider-exposed plan/status summaries when policy allows; +- store reasoning token counts without reasoning text; +- store a provider signature and digest without the signed plaintext when useful; +- accept explicit final explanations as assistant messages; +- require an opt-in policy for raw reasoning blocks; +- label any derived causal explanation as interpretation. + +### Graph Storage + +Nodes link existing Trail objects instead of copying large payloads: + +```text +provenance_node_id +lane_id +session_id +turn_id nullable +node_kind +summary +event_id nullable +span_id nullable +message_id nullable +change_id nullable +artifact_id nullable +source_confidence +classifier_version nullable +created_at +attributes_json +``` + +Edges contain `from_node_id`, `to_node_id`, `relation`, source confidence, and evidence receipt. At turn end Trail writes a content-addressed graph snapshot/manifest for portable review while keeping normalized node/edge indexes for queries. + +## Session Attestations + +An attestation is a statement over an exact set of captured outcomes. It is useful for audit, cost aggregation, handoff, and policy; it is not a correctness certificate. + +### Coverage + +The coordinator appends every successfully created turn checkpoint/change ID to the native-session mapping. Session finalization constructs coverage from that list only. It must not derive coverage by scanning the lane, because a lane may include inherited base history, concurrent human edits, prior sessions, or merged work. + +An attestation segment contains: + +- session and capture-run identity; +- provider/agent/adapter identity; +- human or host principal that authorized capture, when known; +- exact Trail change IDs and turn IDs covered; +- evidence-manifest digests; +- per-model input, output, reasoning, and cache token totals; +- provider-reported and calculated costs, each source-qualified; +- wall-clock and provider/API durations; +- changed-file and line-operation statistics; +- approval/gate outcomes; +- previous attestation ID for resumed sessions; +- attestation schema version, timestamp, and optional signature. + +### Resume Chaining + +When a session resumes after an attestation was created, the next attestation covers only newly recorded turns and references the prior attestation. Walking the chain produces session totals without reattesting old evidence. Creating the same segment twice is idempotent because the coverage set and predecessor form the content key. + +### Principals and Signatures + +Trail should record separate principals rather than encoding the relationship into a display name: + +```text +human_principal who authorized the workspace/session, if known +host_principal ACP client, editor, or outer orchestrator +agent_principal Codex, Claude Code, Pi, etc. +model_principal provider/model/version +capture_principal Trail adapter and version +``` + +An installation approval or managed-run declaration supplies authorization provenance. If Trail has a configured signing identity, it may sign the attestation digest. Provider-supplied reasoning signatures are stored as provider evidence and are never reused as a Trail or human signature. Unsigned attestations remain valid content-addressed summaries labeled `unsigned`. + +### Verification and Revocation + +`trail agent attest verify ` checks: + +- attestation schema and digest/signature; +- predecessor chain; +- existence and digests of covered turn manifests; +- exact change and turn membership; +- aggregate counters recomputed from source evidence; +- artifact retained/redacted/missing state. + +Corrections do not mutate an attestation. Trail issues a superseding attestation or a revocation statement linked to the original. + +## Learnings and Context Flywheel + +Atomic demonstrates the value of feeding repository and workflow learnings into later sessions. Trail should implement this without automatically editing `CLAUDE.md`, `GEMINI.md`, or other provider-specific context files. + +Learnings are explicit or derived records with: + +- `repo`, `workflow`, or `code` scope; +- source session, turn, message, span, and evidence digest; +- confidence and review status; +- stable file/line anchors for code findings; +- supersession/deduplication relation; +- sensitivity and injection policy. + +Only accepted learnings are eligible for automatic context injection. Code findings use Trail's stable line identity so they survive renames and nearby edits. Repository/workflow learnings can appear in bounded SessionStart, prompt-start, ACP context, or compaction summaries. The injection event records exactly which learning IDs were supplied. + +Suggested commands are: + +```text +trail agent explain [--save-derived] +trail agent learnings list [--status proposed|accepted|rejected] +trail agent learnings accept +trail agent learnings reject +``` + +Explanation generation is optional and never runs inside a latency-sensitive hook. + +## Portable Agent Trace Export + +Trail should offer a best-effort vendor-neutral JSONL export for interoperability with external review and analytics tools: + +```console +$ trail agent export --format agent-trace --output traces.jsonl +``` + +Each self-contained record includes: + +- schema version and stable record ID; +- timestamp; +- Trail workspace/lane and change/root revision; +- optional mapped Git commit; +- provider, agent, model, and adapter identity; +- session/turn URI such as `trail://sessions//turns/`; +- file attribution with stable line ranges where known; +- related change, approval, issue, and artifact references; +- provenance graph/attestation extension metadata. + +Export is a projection, not a second source of truth. Importing an external trace creates an artifact plus normalized evidence with its original source; it does not pretend the trace was captured live by Trail. + +## File Change and Checkpoint Strategy + +Trail has three sources of file-change evidence, in descending preference: + +1. **Structured protocol patch**: ACP or a documented native event supplies an exact patch. +2. **Provider tool evidence**: a tool hook supplies the edited path and before/after data. +3. **Workdir reconciliation**: Trail compares the materialized workdir with the turn's `before_change`. + +Structured changes should flow through the same safe patch application path used by ACP. Tool evidence can improve attribution but cannot replace the final workdir comparison. Workdir reconciliation is authoritative for the final filesystem state. + +Only the coordinator may advance a lane head. Duplicate Stop/AfterAgent/agentStop hooks therefore cannot create duplicate operations. The unique logical key is the Trail turn ID plus checkpoint phase. + +If no files changed, the turn closes with `outcome.no_changes=true` and no synthetic operation. + +## Git Association Without Hidden Checkpoint Branches + +Entire's searchable session-to-commit relationship is valuable even though its hidden checkpoint branch is not a fit for Trail. Trail should provide the same navigation through its existing change/root-to-Git mappings. + +### Exact Links Created by Trail + +When `trail git export -m ` or `trail agent land` creates a Git commit, Trail knows: + +- the exported Trail change range; +- the resulting Git commit object; +- the sessions and turns whose `after_change` values contributed to that range; +- the transcript artifacts and spans attached to those turns. + +Trail records exact commit links at that boundary. Queries can then navigate in both directions: + +```text +Git commit -> Trail change range -> contributing turns -> sessions/artifacts +Trail session -> turn checkpoints -> exported Git commits +``` + +### Commits Created Outside Trail + +For ordinary `git commit`, the association is initially unknown. It can become exact or bounded through either: + +1. `trail git import-update`, which maps the current Git state into Trail; or +2. a separate, opt-in lightweight Git `post-commit` integration that records the new Git head and current Trail lane/session mapping. + +Agent-hook installation must not silently install Git hooks. The user enables commit linking explicitly: + +```console +$ trail git hooks add commit-link +``` + +The Git hook records identifiers and mappings only. It does not copy transcripts, create Trail turns, amend the commit, write trailers, push refs, or create hidden branches. If Trail is unavailable, the Git commit still succeeds and the link can be recovered from the reflog/current mappings later. + +### Link Semantics + +A commit may contain work from multiple turns or sessions, and a turn's changes may be split across commits. The relationship is many-to-many and includes a confidence: + +- `exact-export`: Trail created the commit from a known change range; +- `exact-head`: a post-commit receipt matched the lane head and active session; +- `import-bounded`: import mapped the commit/root to a range containing the turn; +- `inferred-overlap`: changed paths and time overlap but no exact mapping exists. + +Only the first three are presented as provenance. Inferred overlap is a search hint and is visibly labeled. + +Suggested storage: + +```sql +CREATE TABLE git_agent_links ( + git_commit TEXT NOT NULL, + lane_id TEXT NOT NULL, + session_id TEXT NOT NULL, + turn_id TEXT, + from_change TEXT, + through_change TEXT, + confidence TEXT NOT NULL, + source TEXT NOT NULL, + created_at INTEGER NOT NULL, + metadata_json TEXT, + PRIMARY KEY(git_commit, session_id, turn_id, source) +); +``` + +This table is a query index over durable Git mappings and Trail activity. It can be rebuilt when all source mappings remain available. Review commands should expose `trail session commits ` and `trail git sessions ` or equivalent JSON/HTTP queries. + +## ACP and Hook Coexistence + +### Capture Ownership + +Trail assigns a lease-like owner for each capture run: + +| Evidence class | Preferred owner | +| --- | --- | +| Prompts and streamed assistant text | ACP, then native structured, then transcript | +| Tool lifecycle and approvals | ACP, then native structured, then transcript | +| Structured patches | ACP/native exact patch, then workdir reconciliation | +| Token and cost usage | Provider authoritative counter, then transcript | +| Native transcript | Native hook/plugin finalizer | +| Turn checkpoint | Shared coordinator only | + +Ownership is per evidence class, not per process. A hybrid run can use ACP for live events and a native hook for the canonical transcript. + +### Correlation + +Strong correlation uses `TRAIL_CAPTURE_RUN_ID`. Fallback correlation requires: + +- same workspace identity; +- same canonical provider; +- same native session ID or transcript identity; +- compatible lane/workdir; +- overlapping process/session time; +- no conflicting explicit run ID. + +Trail must prefer a false negative over merging unrelated sessions. Ambiguous events are retained as receipts and surfaced by doctor/recovery rather than silently attached. + +### Idempotency Keys + +The adapter constructs the strongest available key: + +```text +provider event ID +or (native session, native sequence) +or (native session, native turn, event name, tool/message ID, phase) +or receipt payload digest plus a bounded invocation window +``` + +Payload digest alone is insufficient for repeated identical tool calls. When no provider identifier exists, the receipt journal assigns an invocation ordinal under a serialized native-session stream. + +### Source Precedence + +Field-level precedence is: + +```text +ACP structured + > documented native structured event + > canonical provider export + > native transcript parser + > workdir observation + > heuristic inference +``` + +Lower-precedence evidence may fill missing fields but may not overwrite conflicting higher-precedence values. Conflicts become diagnostics with both evidence references. + +## Storage Changes + +The existing `lane_sessions`, `lane_turns`, `lane_events`, `lane_trace_span_events`, messages, operations, and `lane_acp_sessions` remain the durable activity model. Add transport-neutral integration tables rather than overloading `lane_acp_sessions`. + +The optional `git_agent_links` index described above extends existing Git mappings; it is not required for capture and does not change Git history. + +### `agent_hook_installations` + +```sql +CREATE TABLE agent_hook_installations ( + installation_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + provider TEXT NOT NULL, + scope TEXT NOT NULL, + config_path TEXT NOT NULL, + lane_id TEXT, + manifest_digest TEXT NOT NULL, + manifest_signature_json TEXT, + ownership_inventory_json TEXT NOT NULL, + adapter_version TEXT NOT NULL, + provider_version_range TEXT, + status TEXT NOT NULL, + installed_at INTEGER NOT NULL, + verified_at INTEGER, + metadata_json TEXT +); +``` + +The manifest records only Trail-owned entries and enough original structure to remove them safely. It must not contain secrets. + +### `lane_agent_sessions` + +```sql +CREATE TABLE lane_agent_sessions ( + mapping_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + provider TEXT NOT NULL, + native_session_id TEXT NOT NULL, + parent_native_session_id TEXT, + trail_session_id TEXT NOT NULL, + lane_id TEXT NOT NULL, + capture_run_id TEXT, + primary_transport TEXT NOT NULL, + transcript_identity TEXT, + transcript_offset INTEGER, + resume_json TEXT, + last_attestation_id TEXT, + status TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(workspace_id, provider, native_session_id) +); +``` + +`lane_acp_sessions` can remain during migration. New ACP sessions should also create or link a transport-neutral mapping. Once all ACP reads use `lane_agent_sessions`, the ACP-specific table can become a compatibility projection or be migrated. + +### `agent_hook_receipts` + +```sql +CREATE TABLE agent_hook_receipts ( + receipt_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + installation_id TEXT, + provider TEXT NOT NULL, + native_event TEXT NOT NULL, + native_session_id TEXT, + native_turn_id TEXT, + transport TEXT NOT NULL, + dedupe_key TEXT NOT NULL, + payload_digest TEXT NOT NULL, + raw_artifact_id TEXT, + status TEXT NOT NULL, + diagnostic TEXT, + occurred_at INTEGER, + received_at INTEGER NOT NULL, + processed_at INTEGER, + UNIQUE(workspace_id, provider, dedupe_key) +); +``` + +### `lane_artifacts` + +The artifact metadata described earlier links native evidence to Trail sessions and turns. Foreign keys should follow Trail's current migration and deletion conventions rather than introducing cascading behavior ad hoc. + +### `agent_capture_runs` + +```sql +CREATE TABLE agent_capture_runs ( + capture_run_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + lane_id TEXT, + workdir TEXT NOT NULL, + owner_agent TEXT NOT NULL, + owner_session_id TEXT NOT NULL, + executor_agent TEXT, + work_item_id TEXT, + status TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + metadata_json TEXT +); +``` + +Active-run lookup indexes canonical workdir and expiry. The database enforces opaque ID validation; the coordinator enforces longest-workdir and owner/executor matching. + +### `lane_turn_evidence_manifests` + +```sql +CREATE TABLE lane_turn_evidence_manifests ( + manifest_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL, + session_id TEXT NOT NULL, + turn_id TEXT NOT NULL UNIQUE, + schema_version INTEGER NOT NULL, + object_id TEXT NOT NULL, + digest TEXT NOT NULL, + created_at INTEGER NOT NULL +); +``` + +The content-addressed object contains immutable facts and evidence digests. Mutable artifact retention status stays in `lane_artifacts`. + +### `lane_provenance_nodes` and `lane_provenance_edges` + +```sql +CREATE TABLE lane_provenance_nodes ( + provenance_node_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL, + session_id TEXT NOT NULL, + turn_id TEXT, + node_kind TEXT NOT NULL, + summary TEXT NOT NULL, + event_id TEXT, + span_id TEXT, + message_id TEXT, + change_id TEXT, + artifact_id TEXT, + source_confidence TEXT NOT NULL, + classifier_version TEXT, + created_at INTEGER NOT NULL, + attributes_json TEXT +); + +CREATE TABLE lane_provenance_edges ( + provenance_edge_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL, + session_id TEXT NOT NULL, + from_node_id TEXT NOT NULL, + to_node_id TEXT NOT NULL, + relation TEXT NOT NULL, + source_confidence TEXT NOT NULL, + receipt_id TEXT, + created_at INTEGER NOT NULL, + attributes_json TEXT, + UNIQUE(from_node_id, to_node_id, relation, receipt_id) +); +``` + +Indexes cover session/turn/kind, linked change, and both edge directions. A graph snapshot object is an export/cache; the normalized indexes remain queryable. + +### `lane_session_attestations` + +```sql +CREATE TABLE lane_session_attestations ( + attestation_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL, + session_id TEXT NOT NULL, + capture_run_id TEXT, + previous_attestation_id TEXT, + statement_object_id TEXT NOT NULL, + statement_digest TEXT NOT NULL, + signature_json TEXT, + status TEXT NOT NULL, + created_at INTEGER NOT NULL, + superseded_by TEXT, + metadata_json TEXT +); + +CREATE TABLE lane_session_attestation_turns ( + attestation_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + change_id TEXT, + evidence_manifest_id TEXT NOT NULL, + PRIMARY KEY(attestation_id, turn_id) +); +``` + +Attestation coverage is written from coordinator-owned turn outcomes in the same logical transaction as session finalization, or replayed idempotently afterward. + +### `lane_learnings` + +```sql +CREATE TABLE lane_learnings ( + learning_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL, + session_id TEXT NOT NULL, + turn_id TEXT, + scope TEXT NOT NULL, + body TEXT NOT NULL, + status TEXT NOT NULL, + confidence REAL, + source_artifact_id TEXT, + anchor_json TEXT, + created_at INTEGER NOT NULL, + reviewed_at INTEGER, + reviewer TEXT, + superseded_by TEXT, + metadata_json TEXT +); +``` + +Learning text follows workspace capture/redaction policy. Code anchors use existing Trail identity types rather than raw line numbers as durable identity. + +### Turn Envelope Version 2 + +The current turn envelope is ACP-shaped in its constructor and session fields. Version 2 should retain backward readability and add: + +```json +{ + "kind": "agent_turn", + "protocol": "native-hooks | acp | terminal | hybrid", + "session": { + "trail_session_id": "...", + "capture_run_id": "...", + "native_session_id": "...", + "native_turn_id": "...", + "acp_session_id": "...", + "upstream_session_id": "..." + }, + "capture": { + "transports": ["acp", "native-hooks"], + "fidelity": {}, + "receipt_count": 0, + "artifact_ids": [], + "diagnostic_count": 0, + "evidence_manifest_id": "...", + "provenance_graph_digest": "..." + }, + "principal": { + "human": null, + "host": "vscode", + "agent": "codex", + "model": "...", + "adapter": "trail/codex@..." + } +} +``` + +Existing version 1 ACP envelopes remain valid and readable. The migration should introduce a generic constructor and make `new_acp_prompt` a thin compatibility wrapper. + +## Configuration and Installation Contract + +### Rules + +- Default to project scope. +- Show a dry-run mutation plan before any potentially ambiguous overwrite. +- Parse and write the provider's actual format rather than template-replacing the whole file. +- Preserve unknown keys and unrelated hook entries. +- Identify Trail entries by an installation ID plus a recognizable command prefix or generated-file marker. +- Make repeated installation a no-op when the desired state is already present. +- Use atomic write-and-rename in the same directory. +- Preserve permissions and newline style where practical. +- Refuse to overwrite an unowned plugin/extension file with Trail's chosen filename unless `--force` is explicit. +- On removal, remove only entries matching the installation manifest and ownership marker. +- Validate the resulting file with the provider's schema or parser when available. +- Never enable unrelated experimental provider features without reporting the change. + +### Generated Plugin Files + +Pi and OpenCode require generated TypeScript files. Each file begins with a machine-readable ownership header: + +```text +// Generated by Trail. installation= adapter= +// Manage with: trail agent hooks remove +``` + +The file is self-contained, dependency-light, and invokes `trail` by resolved absolute path when project portability policy permits. Otherwise it uses a small launcher that discovers Trail on `PATH` and reports a diagnostic without breaking the agent. + +### Command Timeouts + +Installed capture commands use a short provider timeout, preferably 5 seconds or less for prompt/tool hooks and at most 30 seconds for turn/session finalization. Trail itself targets: + +- receipt durability under 50 ms with a healthy local daemon; +- parsing and dispatch under 100 ms for ordinary events; +- asynchronous transcript copy/export and workdir reconciliation when the provider response contract allows it. + +If the provider synchronously waits for Stop-hook completion, Trail persists the receipt, schedules heavy work, and returns. Session views may briefly show `finalizing`. + +## Context Injection + +Native hooks can optionally provide Trail context at session or turn start, paralleling ACP's MCP/context capabilities. Examples include lane objective, current head, unresolved approvals, recent handoff, and changed-path claims. + +Context injection is opt-in and bounded. The adapter renders the provider's documented output shape: + +- Claude Code: `additionalContext` or documented SessionStart/UserPromptSubmit stdout behavior; +- Codex: hook response fields supported for the event/version; +- Pi: `before_agent_start` extension result; +- OpenCode: system/message transformation hook; +- Gemini CLI: `hookSpecificOutput.additionalContext` for the appropriate event; +- other providers only when documented. + +Every injection records a `context.injected` event containing a digest, byte count, source records, and redaction result. Trail does not store duplicate plaintext in the event when the underlying context already exists as Trail objects. + +Injection failure never blocks capture. Providers with no safe injection mechanism remain capture-only. + +### Provenance-Aware Compaction Context + +When a provider exposes a pre-compaction context transform, Trail may inject a token-bounded summary built from accepted factual and derived records: + +```text +goals +current plan/todos +accepted decisions with source links +files changed and latest checkpoint +verification outcomes +pending approvals/gates +open errors or deferred items +accepted learnings relevant to the active paths +``` + +The summary targets a configurable budget, defaults to factual records, and excludes raw transcripts and private reasoning. It includes a digest and a `trail://` reference so the full evidence can be inspected outside model context. Trail records both the inputs and exact injected bytes as a `context.injected` derived artifact. Replaying or resuming a compacted session must not reuse stale timestamps, lane heads, or pending approval state without regeneration. + +## Security and Privacy + +### Threat Model + +Hook payloads and transcripts may contain: + +- prompts and proprietary source code; +- command lines and environment fragments; +- secrets in tool input/output; +- attacker-controlled filenames and IDs; +- large or malformed JSON; +- paths outside the workspace; +- terminal escape sequences; +- provider fields whose schema changes unexpectedly. + +### Required Controls + +- Enforce maximum stdin size before JSON decoding. +- Parse with bounded recursion/depth where the library permits. +- Validate IDs against a conservative character/length policy before path use. +- Canonicalize transcript paths and enforce provider allowlists. +- Strip terminal control characters from diagnostics. +- Redact known secret patterns and configured paths before durable raw storage. +- Default tool output capture to bounded summaries; allow full output only by policy. +- Encrypt or exclude sensitive artifacts according to Trail workspace policy. +- Use owner-only permissions for receipt spools and generated secrets-free state. +- Never interpolate payload values into a shell command. +- Execute hook commands without an interactive shell when the provider allows argv configuration. +- Record adapter and provider versions for forensic interpretation. +- Treat context injection as untrusted content and clearly delimit it for the receiving agent. + +### Capture Policy + +Workspace configuration should support: + +```toml +[agent_capture] +enabled = true +prompts = "full" # full | hash | off +assistant_messages = "full" # full | summary | off +tool_inputs = "redacted" # full | redacted | summary | off +tool_outputs = "summary" # full | redacted | summary | off +reasoning_text = "off" # full | redacted | digest-only | off +native_transcripts = "copy" # copy | reference | off +raw_hook_payloads = "redacted" # full | redacted | digest-only +causal_provenance = true +session_attestations = "on-end" # on-end | periodic | manual | off +learning_injection = "accepted" # accepted | off +max_hook_payload_bytes = 1048576 +max_artifact_bytes = 104857600 +fail_mode = "open" +``` + +Policy is evaluated before receipt payload persistence. `digest-only` retains routing fields separately and never writes the full raw JSON. + +## Reliability, Concurrency, and Recovery + +### Ordering + +Providers may launch multiple matching hooks concurrently. Trail must not assume command completion order equals event order. Receipts are serialized per native session for state-machine application while retaining original timestamps. + +Tool calls can remain concurrent. Their spans use independent IDs and close in any order. + +### Finalization Lease + +Providers can deliver duplicate Stop/AfterAgent/session-idle events concurrently. Before transcript import or checkpointing, the coordinator acquires a short renewable lease keyed by `(workspace_id, provider, native_session_id, native_turn_id)`. A competing finalizer may append a higher-fidelity receipt but cannot run workdir reconciliation or close the turn. + +The lease and finalization phase are durable, so a crashed worker can be replaced after expiry. The completed turn ID/evidence manifest is the idempotency result returned to every duplicate receipt. An in-process file lock may reduce contention, but it is never the sole correctness mechanism. + +### Short-Lived Hook Processes + +Command hooks run in separate processes, so in-memory baselines cannot span TurnStart and TurnEnd. Trail stores the pre-turn change/root, workdir manifest/stamps, transcript offset, and active span mapping durably before the start hook returns. Provider plugin buffers can add timing detail, but correctness relies on durable Trail state. + +For provider Stop hooks with very short timeouts, Trail persists the terminal receipt and hands finalization to the local daemon. If no daemon is available, it spools an acknowledged job before returning. A detached child process without a durable receipt is not sufficient because it can be killed with the parent and provides no replay boundary. + +### Missing Events + +- Turn start without session start lazily creates/resumes the session. +- Tool event without turn start lazily creates an inferred active turn only when a prompt/turn identity can be established; otherwise it remains an unattached receipt diagnostic. +- Turn end without turn start can reconstruct from the transcript delta and workdir, marked recovered. +- Session end without turn end closes the active turn as interrupted or provider-indicated status. +- A new turn start closes an abandoned prior turn before opening the next. + +### Crash Recovery + +On workspace open, daemon start, `doctor`, or a new receipt, Trail scans: + +- pending receipt journal entries; +- spooled receipts; +- active native-session mappings with stale heartbeats; +- open turns and spans; +- transcript offsets newer than the last imported artifact; +- workdir changes after the last turn checkpoint. + +Recovery is idempotent. It replays receipts, imports evidence, creates at most one checkpoint per turn, marks synthetic/recovered transitions, and emits a diagnostic event. + +### Rewind and Resume + +Provider resume preserves the Trail session mapping. If the provider rewinds or forks a conversation, Trail records a native session relation: + +- `resume`: same logical session; +- `fork`: child mapping, same lane unless configured otherwise; +- `rewind`: new capture epoch referencing the earlier transcript/artifact boundary. + +Trail history is append-only. A provider rewind does not delete prior Trail turns; it records the branch in conversational history and subsequent worktree checkpoints describe actual state changes. + +## Provider Integration Matrix + +The matrix describes the intended first implementation based on provider contracts verified at the research snapshot. Provider versions change; adapters and `doctor` remain authoritative. + +| Provider | Native mechanism | Project configuration | Session/turn hooks | Tool spans | Transcript/export | ACP | +| --- | --- | --- | --- | --- | --- | --- | +| Codex | Command hooks | `.codex/hooks.json`, trusted project | Strong | Strong | JSONL path may be supplied; unstable format | Yes through supported ACP adapter/host path | +| Claude Code | Command/HTTP hooks | `.claude/settings.json` | Strong | Strong | Native JSONL | Available through ACP adapters, not required | +| Pi | TypeScript extension | `.pi/extensions/trail/index.ts` | Strong | Strong | Native session JSONL | Provider/version-dependent | +| OpenCode | TypeScript plugin | `.opencode/plugins/trail.ts` | Strong via event bus | Strong | `opencode export` preferred | Yes where OpenCode ACP is available | +| Cursor | Command hooks | `.cursor/hooks.json` | Strong on current versions | Strong/partial by surface | Agent transcript JSONL | Yes in Trail's current ACP profiles | +| Gemini CLI | Command hooks | `.gemini/settings.json` | Strong | Strong | Native transcript JSON | ACP support provider/version-dependent | +| Copilot CLI | Command hooks | `.github/hooks/trail.json` | Strong | Strong on current hook contract | Session-state `events.jsonl` | Not required; host/version-dependent | +| Grok Build | Native JSON hooks, Claude/Cursor compatibility, or plugin | `.grok/hooks/trail.json` | Strong on documented events | Strong | `~/.grok/sessions`, Markdown export | Yes: `grok agent stdio` | +| Kiro | Versioned standalone command hooks in IDE/CLI v3 | `.kiro/hooks/trail.json` | Strong turn lifecycle; no standalone SessionEnd | Strong | Stop exposes the final assistant response; no full transcript contract | Host/version-dependent | + +### Adapter Deployment Classes + +Atomic's wider registry demonstrates that support is not limited to JSON hook files. Trail should model deployment independently from event fidelity: + +| Class | Examples | Trail approach | +| --- | --- | --- | +| JSON command config | Codex, Claude Code, Cursor, Gemini, Copilot | Round-trip-safe merge with owned entries | +| Project plugin | OpenCode | Generated or package-managed TypeScript plugin | +| Project extension | Pi | Generated or package-managed TypeScript extension | +| Executable hook directory | Cline-like agents | Owned scripts plus executable/PowerShell variants | +| Versioned standalone hook file | Kiro | Owned `.kiro/hooks/trail.json` using the documented `v1` schema | +| YAML/plugin package | Hermes-like agents | Dedicated parser/package or declarative manifest | +| Self-managed embedded agent | Trail Agent or Sherpa-like hosts | Direct in-process lifecycle API, no config file | +| ACP-native agent | Grok Build and registry agents | ACP relay plus optional native enrichment | + +The verified target list contains the nine providers above. Kiro remains experimental because its standalone hook contract is shared with the IDE and CLI v3 engine. Kiro CLI package 2.8.0 introduced that engine behind `kiro-cli --v3`; the default v2 engine still embeds hooks in named custom-agent configuration. A second compatibility wave may add Cline, Devin, Hermes, and self-managed agents after their current official contracts are verified. Their presence in Atomic is implementation evidence for adapter shapes, not sufficient by itself for Trail to claim compatibility. + +## Provider Designs + +### OpenAI Codex + +#### Installation + +Trail writes owned matchers into `.codex/hooks.json`. Project hooks are subject to Codex project trust and exact hook trust. `doctor` must distinguish: + +- configuration present; +- project trusted; +- hook configuration trusted; +- hook feature disabled; +- installed command resolvable; +- hook invocation recently observed. + +Current Codex documentation describes hooks as enabled by default, with `[features] hooks = false` disabling them and `codex_hooks` retained as a deprecated alias. Trail should not write a feature flag when defaults suffice. If it must remove an explicit disable, it shows that mutation in the install plan. + +#### Hook Coverage + +Target the current lifecycle set where available: + +- `SessionStart` -> session start/resume; +- `UserPromptSubmit` -> turn start and user message; +- `PreToolUse` -> tool span start; +- `PermissionRequest` -> observed approval request; +- `PostToolUse` -> tool span end and possible structured edit; +- `PreCompact` / `PostCompact` -> compaction span; +- `SubagentStart` / `SubagentStop` -> subagent spans; +- `Stop` -> turn completion; +- session closure if/when a documented event is available. + +Codex may invoke multiple matching hook commands concurrently. Trail relies on IDs and receipt ordering, not file order. + +Codex Stop handling must acknowledge quickly. The foreground hook only durably journals the receipt; transcript import, workdir reconciliation, provenance consolidation, and attestation updates run through the daemon/spool finalizer under the per-turn lease. A background process may be an execution mechanism, but the journal acknowledgement is the correctness boundary. + +#### Transcript and Trust Gap + +Codex's `transcript_path` is nullable and the transcript format is not a stable public interface. The adapter therefore treats transcript parsing as versioned, optional enrichment. Turn closure remains correct using hook events plus workdir reconciliation. `doctor` reports when transcript capture is unavailable instead of presenting the session as fully faithful. + +#### ACP Coexistence + +When Codex runs behind Trail's ACP relay, ACP owns prompts, streaming messages, tools, structured diffs, permissions, and usage. Native hooks may supply the transcript and session finalization. Both paths correlate through the capture run environment and native session ID. + +### Anthropic Claude Code + +#### Installation + +Trail merges entries into `.claude/settings.json` at project scope or the appropriate user settings file at user scope. It preserves user hooks, matchers, and unknown fields. Commands use the provider's documented JSON-on-stdin contract. + +#### Hook Coverage + +Claude Code exposes a broad lifecycle. The initial adapter uses: + +- `SessionStart` and `SessionEnd`; +- `UserPromptSubmit`; +- `PreToolUse`, `PostToolUse`, and `PostToolUseFailure`; +- `PermissionRequest` and `PermissionDenied`; +- `Stop`, `StopFailure`; +- `SubagentStart`, `SubagentStop`; +- `PreCompact`, `PostCompact`; +- optionally `FileChanged`, `CwdChanged`, and task/team events as raw or mapped evidence. + +Capture hooks always return a non-blocking success. Trail must not emit exit code 2 or a blocking decision from the observability command. + +#### Transcript + +The hook payload includes a native JSONL transcript path. Trail validates it under Claude's expected project transcript root, imports from the saved offset, and stores a complete content-addressed snapshot at turn end. Resume uses the stable native session ID and transcript path. + +#### Context + +Claude supports context through SessionStart and UserPromptSubmit hook output. Trail can inject a bounded lane handoff or readiness summary and records its digest as evidence. + +### Pi + +#### Installation + +Pi uses a project TypeScript extension at `.pi/extensions/trail/index.ts`. Trail owns that generated file and refuses to overwrite a foreign file. The extension uses Pi's typed extension API and invokes local Trail ingress without bundling Trail logic. + +#### Event Coverage + +The extension observes: + +- `session_start` and `session_shutdown`; +- `before_agent_start`, `agent_start`, and `agent_end`; +- `turn_start` and `turn_end`; +- message lifecycle events for assistant text when stable; +- `tool_call`, tool execution start/end/failure; +- compaction events; +- model-selection and usage events when available. + +Pi distinguishes an agent run for a user prompt from individual model/tool turns. Trail's user-facing `LaneTurn` should correspond to one submitted user prompt (`before_agent_start` through `agent_end`), while inner Pi `turn_start`/`turn_end` pairs become model/tool-cycle spans or events. This avoids exploding one user request into several Trail turns. + +`session_shutdown` reasons such as reload, new, resume, or fork guide whether Trail ends, aliases, or relates the session. + +The extension may maintain ephemeral per-session tool start times and message buffers for precise durations. It sends stable IDs and periodic receipts to Trail; a Pi reload must lose only optional timing precision, never the durable turn or checkpoint. + +#### Transcript and Context + +Pi stores native JSONL sessions and supports session resume. The extension records the resolved session file and native ID. `before_agent_start` can return bounded Trail context or system-prompt modifications through the native extension contract. + +### OpenCode + +#### Installation + +Trail generates `.opencode/plugins/trail.ts`. OpenCode automatically loads project plugins. The file uses the typed plugin API and provider SDK client where necessary. It must not assume Node when OpenCode's plugin runtime is Bun. + +#### Event Coverage + +OpenCode's plugin event bus includes session, message, permission, tool, file, todo, and command events. Map: + +- `session.created` / `session.updated` / `session.idle` / `session.deleted`; +- message updates and parts to prompt/assistant messages; +- `tool.execute.before` / `tool.execute.after` to spans; +- `permission.asked` / `permission.replied` to observed approvals; +- `session.compacted` to compaction; +- `session.diff` and `file.edited` to change evidence; +- `session.error` to failure diagnostics. + +OpenCode child sessions are subagents. Parent-child session relations become nested subagent spans and child native-session mappings. + +OpenCode can emit `chat.message` before the asynchronous `session.created` bus event reaches the plugin. The plugin therefore performs an idempotent `ensure_session` before sending the prompt receipt. Trail's state machine also accepts turn start without session start, so correctness does not depend on plugin event order. + +The plugin may aggregate per-turn model/provider, wall duration, step count, finish reason, token/cache/reasoning counters, cost, todo snapshot, tool durations, structured file diffs, diagnostics, and bounded provider reasoning metadata. It flushes important events as they happen and treats the final `session.idle` payload as a summary enrichment, not the only copy. Counters include provider step IDs or monotonic source sequence so a retried summary cannot double-count usage. + +#### Transcript + +Prefer `opencode export ` or its SDK equivalent over reconstructing an export from event fragments. The canonical JSON export becomes the native artifact. Live message events still provide searchable Trail messages before turn finalization. + +Before compaction, the plugin can request Trail's provenance-aware summary and inject it through OpenCode's compaction/system transform. The injected summary is bounded, excludes private reasoning by default, and points back to the full Trail session. + +### Cursor + +#### Installation + +Trail merges versioned entries into `.cursor/hooks.json`. Cursor hook availability has changed across releases, so discovery records the exact Cursor version and `doctor` validates configured hook names against the installed build. + +#### Event Coverage + +Use current documented events when available: + +- `sessionStart`, `sessionEnd`; +- `beforeSubmitPrompt`; +- `preToolUse`, `postToolUse` and shell/MCP/file-specific before/after hooks where needed; +- `afterAgentResponse`, `afterAgentThought` only when their payload is documented and policy permits capture; +- `stop`; +- `subagentStart`, `subagentStop`; +- `preCompact`; +- workspace/file events as supporting evidence. + +The adapter capability map is version-gated because earlier Cursor builds accepted a smaller event set and had configuration-validation gaps. Missing session start can be recovered lazily from `beforeSubmitPrompt`; it should be reported as degraded rather than fatal. + +#### Transcript + +Cursor transcript locations and schemas are treated as versioned native artifacts. If a reliable transcript cannot be located, Trail still records prompt/stop/tool evidence and performs workdir reconciliation. Resume capability remains `partial` until a documented stable resume locator is available for the relevant surface. + +### Google Gemini CLI + +#### Installation + +Trail merges matchers into `.gemini/settings.json`, preserving unrelated and unknown hook types. Gemini sends JSON on stdin and consumes structured stdout/exit behavior. Capture exits success and does not block agent actions. + +#### Event Coverage + +Gemini offers a broad lifecycle: + +- `SessionStart` / `SessionEnd`; +- `BeforeAgent` / `AfterAgent` for user-level Trail turns; +- `BeforeModel` / `AfterModel` for model-cycle events and usage; +- `BeforeToolSelection`; +- `BeforeTool` / `AfterTool` for spans; +- `PreCompress` for compaction; +- notifications and other events as diagnostic evidence. + +As with Pi, one Trail turn corresponds to one user prompt/agent run, while individual model calls are child spans or events. + +#### Transcript and Context + +Import the native JSON transcript with adapter-version metadata. `BeforeAgent` is the preferred context-injection boundary. Model token counters enrich the turn envelope without assuming all models/providers report cost consistently. + +### GitHub Copilot CLI + +#### Installation + +Project hooks live in an owned `.github/hooks/trail.json`; user hooks live under the Copilot user hook directory. A dedicated project file minimizes configuration collision and makes removal straightforward. Trail still refuses to replace a foreign file with the same name. + +#### Event Coverage + +Use the current hook contract: + +- `sessionStart`, `sessionEnd`; +- `userPromptSubmitted`; +- `preToolUse`, `postToolUse`; +- `agentStop`; +- `subagentStart`, `subagentStop` where available; +- permission, compaction, notification, and error hooks when supported by the installed version. + +Some Copilot versions reuse a tool-call identifier as the child session identifier during subagent lifecycle. The adapter must correlate those events without creating a phantom top-level session. This rule belongs in the Copilot adapter and has a regression fixture. + +#### Transcript + +The native session-state `events.jsonl` is imported from the validated Copilot session directory. Trail records whether the artifact represents the CLI, editor, or cloud-agent surface because hook availability and transcript semantics can differ. + +### xAI Grok Build + +#### Two First-Class Paths + +Grok Build supports ACP through `grok agent stdio`; Trail should provide a built-in ACP profile for that command. Grok also supports hooks/plugins and Claude Code configuration compatibility, allowing direct native capture without ACP. + +#### Native Installation Strategy + +Install a dedicated Trail-owned JSON file at `.grok/hooks/trail.json`. Grok discovers project `.grok/hooks/*.json`, user `~/.grok/hooks/*.json`, enabled-plugin hooks, and compatible Claude Code and Cursor hook files. The dedicated Grok file gives Trail exact ownership and avoids relying on compatibility precedence. Project hooks require `/hooks-trust` or a trusted launch; `doctor` reports both installation and trust state. + +The documented native event set is `SessionStart`, `SessionEnd`, `UserPromptSubmit`, `PreToolUse`, `PostToolUse`, `PostToolUseFailure`, `PermissionDenied`, `Stop`, `StopFailure`, `Notification`, `SubagentStart`, `SubagentStop`, `PreCompact`, and `PostCompact`. Native payloads use camel-case fields including `hookEventName`, `sessionId`, `workspaceRoot`, `toolName`, and `toolInput`. The adapter must accept the explicitly documented Claude/Cursor-compatible shapes as separate fixture variants rather than guessing field aliases at runtime. + +Only `PreToolUse` is blocking. Trail's observability hook always exits zero, emits no passive stdout, and never returns `deny`. Timeouts, crashes, and malformed output are documented as fail-open, which matches Trail's recording policy. + +Grok's changelog also describes host-registered hooks over the agent connection. When that API is stable, the ACP adapter can register enrichment hooks in-memory instead of writing project files for ACP runs. Direct non-ACP use still relies on the project/user native integration. + +#### Contract Probe and Compatibility Drift + +The official contract makes Grok a supported native adapter. An explicit `trail agent hooks doctor grok --probe` remains useful for detecting version drift and compatibility-mode behavior. It should: + +1. inspect the installed Grok version and capabilities; +2. validate hook/plugin directories and trust; +3. generate a harmless probe session when explicitly requested; +4. capture redacted payload shapes; +5. compare them to checked-in fixtures; +6. report whether Grok-native, Claude-compatible, or Cursor-compatible mapping is active. + +Failure to run an optional probe does not make a documented supported version experimental. An unknown newer version reports `unknown` until static inspection or the probe confirms its event and payload shapes. + +#### Transcript and Resume + +Grok stores sessions under `~/.grok/sessions` and can export Markdown. Prefer the supported export command for canonical evidence and retain the session locator for resume. Markdown export is a native artifact; normalized Trail messages still come from ACP or hook events when available. + +## CLI and API Surface + +### User Commands + +```text +trail agent hooks list [--installed] [--json] +trail agent hooks setup [--scope project|user] [--lane ] + [--print] [--yes] [--force] +trail agent hooks remove [--scope project|user] [--dry-run] +trail agent hooks status [--json] +trail agent hooks doctor [|--all] [--probe] [--json] +trail agent hooks events [--last ] [--failed] [--json] +trail agent hooks replay [--receipt |--pending] [--json] + +trail agent capture begin --owner --session + [--executor ] [--lane ] + [--workdir ] [--work-item ] + [--ttl ] [--json] +trail agent capture renew [--ttl ] [--json] +trail agent capture end [--json] +trail agent capture status [--json] + +trail agent provenance show [--json] +trail agent attest list [--session ] [--json] +trail agent attest show [--json] +trail agent attest verify [--json] +trail agent export --format agent-trace [--output ] +trail agent explain [--save-derived] +trail agent learnings list|accept|reject ... +``` + +### Internal Hook Command + +```text +trail agent hook receive + --installation + [--transport command|plugin|http] +``` + +The internal command remains documented for integration authors but is not the normal setup interface. + +### HTTP + +```text +POST /v1/agent-hooks/{provider}/{event} +GET /v1/agent-hooks/installations +GET /v1/agent-hooks/installations/{id} +GET /v1/agent-hooks/receipts +POST /v1/agent-hooks/receipts/{id}/replay +GET /v1/agent-integrations/capabilities +POST /v1/agent-capture-runs +POST /v1/agent-capture-runs/{id}/renew +POST /v1/agent-capture-runs/{id}/end +GET /v1/agent-sessions/{id}/provenance +GET /v1/agent-sessions/{id}/attestations +POST /v1/agent-attestations/{id}/verify +GET /v1/agent-sessions/{id}/export?format=agent-trace +``` + +Local HTTP ingress requires a short-lived installation credential or a Unix-domain-socket trust boundary. It must not expose unauthenticated hook ingestion on a network interface. + +### MCP + +MCP should expose read-oriented integration status and receipt diagnostics. Hook installation is a workspace mutation and should require the same explicit approval as other configuration writes. Agent-facing MCP tools do not need to be called for native capture; the hook infrastructure is out-of-band. + +## Diagnostics + +`doctor` should check: + +- Trail workspace and lane binding; +- provider executable and version; +- native mechanism supported at that version; +- project/user config path and parseability; +- installation ownership and drift; +- provider trust state; +- hook command resolution in a non-interactive environment; +- daemon/socket availability or direct-write fallback; +- recent receipt and processing latency; +- transcript path/export availability; +- native ID and resume support; +- ACP/native duplicate correlation when hybrid; +- governing managed run, executor match, lease expiry, and workdir specificity; +- stale sessions, open turns/spans, pending receipts, and spool backlog; +- stuck/contended finalization leases; +- evidence-manifest and artifact digest consistency; +- attestation coverage, predecessor chain, and signature status; +- adapter manifest signature, provider-version range, and configuration drift; +- capture policy and redaction mode. + +Every warning has a stable code, human message, machine details, and remediation. Examples: + +```text +CODEX_HOOK_TRUST_MISSING +CURSOR_EVENT_UNSUPPORTED_BY_VERSION +TRANSCRIPT_PATH_OUTSIDE_PROVIDER_ROOT +HOOK_COMMAND_NOT_ON_PATH +CAPTURE_RECEIPT_SPOOLING +HYBRID_CORRELATION_AMBIGUOUS +TURN_RECOVERED_WITH_WORKDIR_ONLY +MANAGED_RUN_AMBIGUOUS +MANAGED_RUN_EXECUTOR_MISMATCH +TURN_FINALIZATION_LEASE_STALE +ADAPTER_MANIFEST_UNSIGNED +ADAPTER_MANIFEST_VERSION_MISMATCH +EVIDENCE_ARTIFACT_REDACTED +ATTESTATION_COVERAGE_MISMATCH +``` + +## Testing Strategy + +### Adapter Contract Tests + +Every provider adapter must pass the same suite: + +- fresh install; +- idempotent reinstall; +- dry run; +- force behavior; +- signed, unsigned, expired, and provider-version-incompatible manifests; +- reversible ownership for manifest deep merges and drift-safe uninstall; +- preservation of unrelated and unknown configuration; +- safe removal of only Trail-owned entries; +- refusal to overwrite foreign generated files; +- malformed and oversized payloads; +- invalid identifiers and transcript-path traversal; +- every supported hook fixture maps to expected lifecycle events; +- unknown hooks are retained as provider events; +- provider response never blocks in capture mode; +- context output matches the provider schema; +- capability report matches the fixture version. + +### Lifecycle Tests + +Replay complete fixture sessions through the shared coordinator: + +- normal session with two turns; +- repeated identical prompts and tool calls; +- concurrent tool calls ending out of order; +- tool failure and permission denial; +- subagent nesting; +- compaction and resume; +- missing SessionStart; +- missing Stop followed by a new prompt; +- duplicate delivery; +- hook and ACP delivery of the same run; +- agent crash with transcript/workdir recovery; +- no-change turn; +- large transcript and capture truncation; +- secret redaction. +- exhaustive state/event transition coverage; +- two concurrent terminal receipts competing for the finalization lease; +- managed run with owner/executor filtering and nested workdirs; +- pre-existing session not restamped into a later managed run; +- plugin correlation-buffer restart halfway through a turn; +- exact attestation coverage when the lane contains inherited and concurrent changes; +- resumed-session attestation chaining and idempotent recreation; +- deterministic provenance classification/consolidation with source links; +- transcript redaction preserving the immutable evidence manifest; +- portable trace export roundtrip as an external artifact. + +Tests assert one Trail session, the expected turn count, one checkpoint per changed turn, closed spans, correct source precedence, stable diagnostics, exact attestation coverage, and unchanged factual evidence after derived-artifact regeneration or transcript redaction. + +### Golden Native Fixtures + +Keep redacted payload and transcript fixtures by provider and version: + +```text +trail/tests/fixtures/agent-hooks/// + capabilities.json + session.jsonl + expected-lifecycle.jsonl + transcript.* + config-before.* + config-after.* +``` + +Fixtures must come from documented examples or explicit opt-in capture on test accounts. They contain no real repository content or credentials. + +### Optional Real-Agent E2E + +Real-agent tests are opt-in because they may require login, tokens, network, or spend. A minimal deterministic prompt asks the agent to create one known file and stop. The test verifies hook delivery, transcript acquisition, one turn, one checkpoint, and safe uninstall. + +Grok requires a real contract probe before graduating from experimental. Cursor and Codex should also run version-matrix smoke tests because their hook contracts evolve quickly. + +### Performance Tests + +Measure p50/p95/p99 hook response latency with and without the daemon, concurrent tool hooks, transcript sizes, and workdir sizes. Test that a daemon outage produces bounded spooling and that replay does not duplicate state. + +The v1 release gates use deliberately generous CI budgets so they catch algorithmic +regressions without pretending shared runners are latency laboratories: + +- one hundred concurrent deliveries of the same receipt must converge to one journal + row in less than 15 seconds and grow the workspace database by less than 32 MiB; +- an individual payload is capped at 1 MiB before durable ingress, a transcript or + artifact at 64 MiB, a portable trace at 256 MiB, and the degraded spool at 10,000 + files or 64 MiB; +- local CLI scale smoke tests must remain green at 1,000 files, while native-hook unit + and E2E tests exercise duplicate ingress, replay, database outage, and transcript + rewrite/truncation paths; +- release benchmarking should record p50/p95/p99 rather than turn machine-specific + timings into protocol guarantees. Provider acknowledgement timeouts remain a + compatibility input to each adapter and the durable-spool path is the fallback. + +## Rollout Plan + +### Phase 0: Shared Capture Coordinator + +- Extract ACP session/turn/message/tool/checkpoint orchestration behind transport-neutral inputs. +- Add lifecycle envelope, source precedence, ownership, and idempotency tests. +- Implement the pure exhaustive lifecycle state machine and action executor boundary. +- Add generic turn envelope v2 while preserving v1 reads. +- Keep ACP behavior and performance unchanged. + +Exit criterion: existing ACP e2e tests pass through the shared coordinator and duplicate normalized event replay is a no-op. + +### Phase 1: Receipt Journal and Hook CLI + +- Add installation, native-session, managed-run, receipt, artifact, and finalization-lease storage. +- Implement bounded stdin, redaction, direct dispatch, daemon dispatch, and spool fallback. +- Add `list`, `status`, `doctor`, `events`, and `replay` foundations. +- Implement generic config mutation/ownership helpers. + +Exit criterion: a synthetic provider can record and recover complete sessions through short-lived command invocations. + +### Phase 2: Claude Code and Codex + +- Implement the two broad command-hook adapters. +- Import native transcripts with path validation. +- Implement trust diagnostics, tool/subagent spans, compaction, and context injection. +- Validate hybrid ACP/native deduplication. + +Exit criterion: both agents record multi-turn direct sessions, and Codex hybrid mode creates one record. + +### Phase 3: Gemini CLI, Copilot CLI, and Cursor + +- Add JSON config adapters and versioned capability probes. +- Add transcript importers and provider edge-case fixtures. +- Add editor/CLI surface metadata where relevant. + +Exit criterion: supported versions pass fixture, install-preservation, and opt-in E2E tests. + +### Phase 4: Pi and OpenCode + +- Add generated TypeScript extension/plugin adapters. +- Capture richer message, tool, permission, subagent, and compaction streams. +- Prefer canonical OpenCode export and native Pi session files. + +Exit criterion: generated files are portable, safely owned, and direct sessions produce the same Trail review model as command-hook providers. + +### Phase 5: Grok Build + +- Add built-in `grok agent stdio` ACP profile. +- Check in native camel-case and explicit Claude/Cursor compatibility fixtures. +- Implement the owned `.grok/hooks/trail.json` integration and non-blocking response contract. +- Add export/resume integration and trust diagnostics. + +Exit criterion: documented versions pass native fixtures and direct-session E2E; unknown versions degrade with clear diagnostics rather than silently selecting a compatibility shape. + +### Phase 6: Provenance, Attestation, and Portable Export + +- Add immutable evidence manifests and redactable artifact status. +- Add causal provenance nodes/edges with deterministic classification. +- Add exact session attestation creation, resume chaining, and verification. +- Add accepted learnings with stable code anchors and compaction injection. +- Add vendor-neutral agent-trace export. + +Exit criterion: a multi-turn hybrid session can be audited from goal to exact checkpoints, verified against a chained attestation, redacted without changing factual manifests, and exported as portable JSONL. + +### Phase 7: Product Hardening + +- Search and filtering by provider, model, tool, token usage, and artifact. +- Storage retention, compaction, and artifact export policy. +- Compatibility telemetry that never uploads content without opt-in. +- Signed declarative manifests, documented third-party adapter SDK, and conformance suite. +- Second-wave adapters only after official contract verification. + +## Acceptance Criteria + +The initial feature is complete when: + +1. A user can install hooks for each stable target provider without using ACP. +2. Direct agent sessions create searchable Trail sessions and user-level turns. +3. Supported tool calls become spans with correct nesting and failure state. +4. Every completed changed turn produces at most one Trail checkpoint. +5. Native transcripts or canonical exports are retained when supported and policy allows. +6. ACP and native hooks observing the same run produce one deduplicated record. +7. Hook failures do not block ordinary agent use and are diagnosable. +8. Install and remove preserve foreign configuration and unknown fields. +9. Crash recovery replays receipts without duplicating turns, spans, or operations. +10. All raw payloads, paths, IDs, and artifacts pass the security constraints. +11. Capability reports accurately describe provider/version limitations. +12. Existing ACP and terminal integrations remain backward compatible. +13. Trail-created or explicitly linked Git commits can be queried back to the contributing agent sessions without hidden refs or commit trailers. +14. Managed runs attribute only newly governed sessions from the matching owner/executor and workdir. +15. Every completed turn has an immutable evidence manifest distinct from redactable attachments. +16. Causal provenance nodes link back to source events/spans/changes and clearly label deterministic or generated interpretation. +17. Session attestations cover exactly the turn changes recorded by that session and chain correctly across resume. +18. Transcript redaction and explanation regeneration do not change checkpoint identity or factual manifests. +19. Declarative adapter manifests are previewable, version-gated, signed when remote, and fully reversible for Trail-owned mutations. +20. Sessions can be exported to a portable trace without making the export a second source of truth. + +## Decisions Borrowed From Entire + +The local Entire clone at commit `7cd6662805fbd525f2f418ecf465a247b924af70` informed this design. Trail should borrow ideas, not storage semantics or code blindly. + +| Entire idea | Trail adaptation | +| --- | --- | +| Agent adapter translates native hooks to normalized lifecycle events | `NativeAgentAdapter` emits Trail lifecycle envelopes | +| Generic dispatcher owns lifecycle orchestration | Shared capture coordinator owns sessions, turns, spans, and checkpoints | +| Full native transcript on checkpoints | Content-addressed native artifact at each completed Trail turn | +| Transcript delta offset saved before the turn | Native-session mapping stores transcript offset/export revision | +| Canonical export preferred over event reconstruction | OpenCode export and Grok export are first-class artifacts | +| Idempotent install/uninstall preserving unknown config | Trail-owned mutation manifests and round-trip-safe writers | +| Plugin/extension adapters for Pi and OpenCode | Generated `.pi` extension and `.opencode` plugin | +| Fail-soft hook commands with bounded timeout | Durable receipt first, async finalization, fail-open capture response | +| Central session/tool/subagent identifier validation | Ingress validation before path/query construction | +| Context injection through native responses | Optional lane/handoff injection with evidence digest | +| Provider-specific handling of phantom or reused IDs | Adapter regression rules, notably Copilot subagent IDs | +| Simulated hook tests plus optional real E2E | Golden provider/version fixtures and opt-in agent smoke tests | +| Trust diagnostics for Codex hooks | Provider-specific `doctor` checks for project/hook trust | +| Hidden Git checkpoint branch and commit trailers | **Not adopted**; Trail storage is primary and Git publication is explicit | + +Relevant Entire implementation references include: + +- [Agent integration guide](../../entire/docs/architecture/agent-guide.md) +- [Agent integration checklist](../../entire/docs/architecture/agent-integration-checklist.md) +- [Agent adapter interfaces](../../entire/cmd/entire/cli/agent/agent.go) +- [Generic hook registry](../../entire/cmd/entire/cli/hook_registry.go) +- [Shared lifecycle dispatcher](../../entire/cmd/entire/cli/lifecycle.go) +- Provider packages under `entire/cmd/entire/cli/agent/` + +## Decisions Borrowed From Atomic + +The local Atomic clone at commit `24a5d55a66192e6337f42c2cb18deee4f58ea6fe` was inspected at the implementation level. Its `atomic-agent` library test suite passed with 1,200 tests. Trail should adapt the following ideas to its own operation database and lane model. + +| Atomic idea | Trail adaptation | +| --- | --- | +| Pure `phase + event + context -> actions` transition function | Exhaustive transport-neutral capture state machine with side-effect executor | +| Lightweight local session scratch state | Durable `lane_agent_sessions`, receipt journal, and active-turn mappings | +| Every turn records one native VCS change | Every completed changed turn advances its Trail lane at most once | +| Session envelope embedded in hashed change metadata | Content-addressed factual evidence manifest linked from turn envelope v2 | +| Transcript/reasoning stored in strip-friendly unhashed data | Redactable/encrypted artifacts with retained digest and tombstone state | +| Exact `recorded_change_hashes` used for session attestation | Attestation coverage from coordinator-owned turn change IDs, never lane scans | +| Chained attestations for resumed sessions | New segment covers only new turns and references prior Trail attestation | +| Managed lifecycle lease with owner/executor/view/workdir | Managed capture run with owner/executor/lane/workdir and longest-prefix resolution | +| Only newly created sessions receive a managed-run stamp | Never restamp pre-existing direct sessions into an outer run | +| In-memory provider plugin buffer plus durable CLI state | Ephemeral enrichment buffer backed by durable receipts and replay | +| Turn-end lock suppresses duplicate Stop recording | Durable finalization lease plus idempotent completed result | +| Deterministic tool classification and provenance DAG | Optional causal overlay linked to Trail events, spans, messages, and changes | +| Decision consolidation retains raw nodes | Derived decision nodes list exact `consolidated_from` evidence and confidence | +| Agent identity derived from user authorization | Separate human, host, agent, model, and adapter principals in evidence/attestation | +| Declarative integration hook manifest | Signed/versioned install manifest, built-in parser, reversible ownership inventory | +| Global hooks no-op outside an Atomic/sandbox workspace | Trail receiver resolves workspace/lane pointer and exits success when absent | +| Codex Stop detached foreground worker | Durable terminal receipt plus daemon/spool finalizer; no unacknowledged child reliance | +| Provenance-aware compaction summary | Token-bounded factual/accepted-learning context injected through native or ACP boundary | +| `agent-trace` JSONL projection | `trail agent export --format agent-trace` with Trail revisions and stable line attribution | +| Explain/learnings knowledge flywheel | Reviewed Trail learning records and context injection, no automatic provider-file edits | + +### Atomic Implementation Lessons Not Adopted Directly + +Atomic's source also clarifies several traps: + +- Its common `HookType` currently has only session start/end, turn start/end, and pre/post tool use. Trail's normalized contract must retain messages, plans, approvals, subagents, compaction, usage, model updates, and workspace evidence rather than collapsing them into raw JSON. +- Its fallback watcher is in-process, but command hook invocations are separate processes. Atomic's own turn-end path therefore relies on repository status. Trail must persist the pre-turn baseline and transcript offset so cross-process recovery is exact. +- Its session-start handler may switch Atomic's current view and later restore it. Trail native capture must bind an explicit lane/workdir and never switch a user's Git branch or ambient Trail ref behind their back. +- Several Atomic adapters parse events but intentionally leave installation to separate packages or IDE panels. Trail capability reporting must distinguish `payload_parser`, `automatic_install`, `guided_install`, and `external_package` instead of calling all adapters equally installed. +- Atomic's declarative manifest can deep-merge non-hook settings and intentionally leaves them on uninstall. Trail requires a reversible ownership inventory for every setting it changes. +- Atomic can attach agent metadata directly to its native VCS changes. Trail keeps operational capture in Trail and uses explicit Git mappings/publication, preserving its existing product boundary. +- Atomic's plus-tag display identity is useful UX but not a cryptographic authorization model by itself. Trail records separate principals and signs attestations only with an actual configured key. + +Relevant Atomic implementation references include: + +- [`atomic-agent` overview](../../atomic/atomic-agent/README.md) +- [Normalized hook trait and registry](../../atomic/atomic-agent/src/hooks/mod.rs) +- [Declarative hook manifests](../../atomic/atomic-agent/src/hooks/manifest.rs) +- [Pure lifecycle state machine](../../atomic/atomic-agent/src/turn/phase.rs) +- [Durable session scratch state](../../atomic/atomic-agent/src/turn/session.rs) +- [Turn orchestrator](../../atomic/atomic-agent/src/turn/orchestrator/turn.rs) +- [Managed lifecycle leases](../../atomic/atomic-cli/src/commands/agent/lifecycle.rs) +- [Hook ingress and Codex handoff](../../atomic/atomic-cli/src/commands/agent/hooks.rs) +- [Session evidence envelope](../../atomic/atomic-agent/src/envelope.rs) +- [Causal provenance types](../../atomic/atomic-agent/src/provenance/types.rs) +- [Deterministic provenance classification](../../atomic/atomic-agent/src/provenance/classify.rs) +- [Session attestation construction](../../atomic/atomic-agent/src/turn/orchestrator/attestation.rs) +- [Redactable transcript storage](../../atomic/atomic-agent/src/transcript/storage.rs) +- [Portable trace export](../../atomic/atomic-agent/src/export.rs) +- [OpenCode plugin correlation state](../../atomic/packages/opencode-atomic-hooks/src/session.ts) + +## Rejected Alternatives + +### One Hook Script Per Provider With Direct Database Calls + +Rejected because orchestration, security, and checkpoint behavior would drift across providers and duplicate ACP logic. + +### Transcript-Only Import + +Rejected as the primary path because it loses live tool timing, approvals, structured diffs, and reliable turn boundaries. It remains a recovery and enrichment source. + +### Tool-Hook-Only File Attribution + +Rejected because shell commands, formatters, subprocesses, and provider-specific edit paths can bypass tool matchers. Turn-end workdir reconciliation remains required. + +### Automatically Disable Native Hooks During ACP + +Rejected because native hooks can provide the canonical transcript and session-end evidence ACP lacks. Deduplication and evidence ownership are more robust than mutating configuration for every launch. + +### Store Everything as Generic Events + +Rejected because sessions, turns, messages, spans, approvals, artifacts, and operations have different invariants and query needs. Generic events remain the compatibility escape hatch. + +### Mirror Entire's Hidden Git Branch + +Rejected because it conflicts with Trail's local operation database and explicit Git interoperability. Users may later export a review bundle to Git, but capture must not depend on commits. + +### Switch the Ambient Repository View at Session Start + +Rejected because editor, terminal, human, and concurrent agent processes may share a repository. Native capture binds a Trail lane/workdir; it does not switch Git branches or global Trail refs as a hook side effect. + +### Six Generic Lifecycle Events Only + +Rejected because flattening every provider into session/turn/pre-tool/post-tool loses streamed messages, approvals, subagents, compaction, plans, usage, and structured file evidence. A stable broad vocabulary plus provider-event escape hatch preserves fidelity. + +### Detached Finalizer Without Durable Receipt + +Rejected because a child process can die with the provider, contend with duplicates, or finish without an observable acknowledgement. Trail durably journals first and then finalizes through the daemon, spool replay, or an idempotent foreground path. + +### Automatically Write Derived Learnings Into Agent Files + +Rejected as the default because inferred text can be wrong, sensitive, provider-specific, and noisy in source control. Trail stores proposed learnings, anchors them to evidence, requires review for persistent reuse, and injects accepted learnings through the active transport. + +### Attribute an Entire Lane to a Session + +Rejected because lanes include inherited history and may include concurrent or human changes. Attestations enumerate exact turn change IDs recorded by the coordinator. + +## Open Questions + +### Decisions fixed by the first implementation + +- User-scoped hook commands remain explicitly bound to the Trail workspace that + authorized the installation. The command carries both the workspace locator and + installation ID; it never discovers an arbitrary repository at invocation time. +- Provider-native artifacts use Trail's content-addressed object store plus indexed + `lane_artifacts` metadata. A single artifact is bounded at 64 MiB; larger transcripts + must be imported as ordered offset chunks and may later receive a chunk manifest. +- Context injection stays off unless requested separately. Installing observability + hooks does not change an agent's prompt or system context. +- Unknown provider versions are reported as `unknown` by an explicit version probe + until a checked-in fixture/range confirms compatibility; executable discovery alone + does not silently upgrade a compatibility claim. +- Direct native sessions without an explicit lane/run binding use a deterministic + provider lane and observation-first capture. Workdir checkpointing occurs only when + the selected lane has a materialized, authorized workdir. + +### Version 1 policy choices + +The former open questions are fixed for schema and adapter version 1. Changing one of +these choices requires a new policy or schema version and an upgrade diagnostic: + +1. Canonical exports and native transcripts are copied as bounded, content-addressed + artifacts after central redaction. Privacy-sensitive reference/digest-only capture + remains a future workspace-policy mode; disabling attachments is always allowed. +2. Failed, ambiguous, quarantined, and discarded receipts retain their immutable audit + identity. Automated deletion is disabled until Trail has an explicit retention + configuration and auditable garbage-collection operation. +3. Cursor transcript and resume support stays `unknown` unless a checked-in contract + fixture and version range prove it. Available lifecycle hooks still record receipts. +4. Grok host-hook and canonical-export support follows the same rule. Its ACP profile + is supported independently; unverified native versions never receive a stronger + compatibility claim. +5. Subagents are child spans in the parent Trail session. A separately identified + provider session may have its own mapping, but Trail does not invent child sessions + merely because a subagent hook fired. +6. Late higher-fidelity material may append a new immutable artifact and provenance + edge to a closed turn. It may not rewrite closed messages or factual envelopes; + conflicting semantic enrichment requires an explicit future revision model. +7. A chained attestation is created after every durably finalized session. Periodic and + manual attestations may add coverage but never replace the on-end chain. +8. Attestations use Trail-local Ed25519 keys with explicit key IDs, rotation, and + revocation. Git author identity is descriptive metadata, not signing authority. +9. Provenance uses normalized query rows plus content-addressed evidence and attestation + manifests. Derived activity nodes always link back to immutable factual sources. +10. Trail emits `trail.agent-trace` version 1. Import means bounded parse and complete + verification; it does not merge foreign rows into the local operation database. + Future extensions must use a namespaced schema/version. +11. Learnings are never injected globally. Only explicitly accepted, workspace-scoped, + sensitivity-eligible learnings may be injected through an opted-in transport. +12. Version 1 ships checked-in manifests only. A future remote manifest must be signed, + constrained by the same declarative schema, revocable by publisher/key identity, + and incapable of adding arbitrary shell interpolation or filesystem authority. + +## Documentation and Compatibility Policy + +Each adapter page should publish: + +- verified provider versions and date; +- installed files and exact scope; +- lifecycle and fidelity matrix; +- transcript/export location and retention behavior; +- trust or permission requirements; +- context-injection behavior; +- known gaps and recovery behavior; +- uninstall instructions; +- a link to the provider's primary hook documentation. + +Provider hook contracts evolve faster than Trail's core model. Compatibility claims therefore belong to adapter fixtures and generated capability reports, not hard-coded marketing copy. Unknown newer versions may run in compatible mode with a warning; known incompatible versions fail installation safely and preserve existing configuration. + +## Primary References + +- [Trail ACP relay design](acp-relay.md) +- [Trail data model](data-model.md) +- [Trail integration overview](../integrations/overview.md) +- [Entire agent integration guide](../../entire/docs/architecture/agent-guide.md) +- [Atomic agent integration overview](../../atomic/atomic-agent/README.md) +- [Atomic provenance metadata specification](../../atomic/docs/PROVENANCE-METADATA-SPEC.md) +- [OpenAI Codex hooks](https://learn.chatgpt.com/docs/hooks) +- [OpenAI Codex advanced configuration](https://developers.openai.com/codex/config-advanced) +- [Claude Code hooks reference](https://code.claude.com/docs/en/hooks) +- [Pi extensions](https://pi.dev/docs/latest/extensions) +- [OpenCode plugins](https://opencode.ai/docs/plugins/) +- [Cursor hooks](https://cursor.com/docs/hooks) +- [Gemini CLI hooks reference](https://geminicli.com/docs/hooks/reference/) +- [GitHub Copilot hooks reference](https://docs.github.com/en/copilot/reference/hooks-reference) +- [GitHub Copilot CLI hook configuration](https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/use-hooks) +- [xAI Grok Build overview](https://docs.x.ai/build/overview) +- [xAI Grok Build hooks](https://docs.x.ai/build/features/hooks) +- [xAI Grok Build skills, plugins, and marketplaces](https://docs.x.ai/build/features/skills-plugins-marketplaces) +- [xAI Grok Build headless scripting](https://docs.x.ai/build/cli/headless-scripting) +- [xAI Grok Build changelog](https://x.ai/build/changelog) diff --git a/docs/design/storage-and-indexing.md b/docs/design/storage-and-indexing.md index 5464b76..bd6a822 100644 --- a/docs/design/storage-and-indexing.md +++ b/docs/design/storage-and-indexing.md @@ -64,7 +64,7 @@ The schema contains these major table groups: - Agent activity: `lane_sessions`, `lane_turns`, `lane_events`, `lane_trace_span_events` - Human gates and resumable state: `lane_approvals`, `lane_run_states` - Coordination: `leases` -- Merge state: `merge_queue`, `merge_results`, `conflict_sets`, +- Merge state: `lane_merge_queue`, `merge_results`, `conflict_sets`, `conflict_resolution_suggestions` - Git interop: `git_mappings` - Worktree scan cache: `worktree_file_index` diff --git a/docs/design/text-and-line-identity.md b/docs/design/text-and-line-identity.md index a281fb2..6d8a73b 100644 --- a/docs/design/text-and-line-identity.md +++ b/docs/design/text-and-line-identity.md @@ -101,6 +101,10 @@ flowchart LR - origin `ChangeId` - local sequence +The user-facing form is `line_:`. Trail resolves +that alias to the stored origin `ChangeId` and local sequence; users do not +need to interpret the line as a change identifier. + New lines introduced by an operation get IDs tied to that operation. Existing lines keep their IDs when content or position changes can be matched according to the line preservation logic. ## Preserving Identity Across Edits diff --git a/docs/design/universal-lane-environments.md b/docs/design/universal-lane-environments.md new file mode 100644 index 0000000..6e73963 --- /dev/null +++ b/docs/design/universal-lane-environments.md @@ -0,0 +1,897 @@ +# Universal Lane Environments + +Status: implementation in progress + +This document defines a general environment model for reproducible agent execution in +Trail lanes. It extends the filesystem mechanisms in +[Layered Lane Workspaces](layered-lane-workspaces.md) from dependency directories into a +complete, typed description of everything a command needs: source, toolchains, +dependencies, generated state, caches, secrets, services, network policy, and external +resources. + +The design deliberately distinguishes specification from materialization. An +environment specification says what is required and which policy applies. An +environment generation records the exact artifacts and runtime resources attached to +one lane at one point in time. + +## Executive summary + +Trail should model an execution environment as a directed acyclic graph of typed +components. Every component declares: + +- the inputs that determine its identity; +- the adapter that can plan, build, validate, and bind it; +- whether its contents are immutable, shared, private, disposable, secret, or external; +- the target path, environment variable, service, or runtime resource it contributes; +- the trust and network capabilities required to create or use it; +- how freshness and health are proven. + +Trail, rather than each language integration, owns the safety-critical lifecycle: +canonical fingerprinting, single-flight construction, staging, validation, immutable +publication, lane-quiescence checks, atomic attachment, recovery, provenance, and +garbage collection. Ecosystem adapters supply domain knowledge without receiving +authority to mutate Trail's database or shared artifact references directly. + +The default policy is: + +- share immutable content by digest; +- seed mutable state from immutable content, then give every lane copy-on-write + isolation; +- keep ordinary writable state lane-private; +- make scratch state disposable; +- inject secrets only at execution time; +- represent remote services and OCI images by pinned external identities; +- never infer that an ignored or untracked directory is safe to share merely because + Git does not track it. + +This permits a large monorepo to share terabytes of common dependency and toolchain +content while preserving the behavioral isolation agents expect from separate +worktrees. + +## Goals + +1. Reproduce an agent command from a complete, inspectable environment record. +2. Let adapters for Cargo, npm, CMake, Docker, Python, Go, and future ecosystems compose + under one lifecycle and policy model. +3. Share large immutable content without allowing one lane to corrupt another. +4. Isolate writable build state without eagerly copying it. +5. Make freshness explainable: Trail can identify exactly which input made a component + stale. +6. Keep secret material out of artifact keys, manifests, transcripts, logs, and lane + snapshots. +7. Support local filesystems, Linux overlayfs or FUSE, and the macOS NFS workspace view + without changing environment semantics. +8. Make attachment transactional and recoverable after crashes. +9. Preserve Trail's existing distinction between local operational state and Git + history. +10. Provide consistent CLI, HTTP, MCP, and Rust API reports. + +## Non-goals + +- Replacing ecosystem package managers, build systems, OCI registries, secret managers, + or service orchestrators. +- Claiming arbitrary build scripts are deterministic. Trail records their declared + inputs, policy, outputs, and observations; stronger reproducibility requires a + hermetic adapter or sandbox. +- Treating a cache hit as proof that an environment is correct. +- Storing secrets in content-addressed artifacts. +- Sharing arbitrary writable directories between concurrently active lanes. +- Making Git commits, branches, or merges implicit side effects of environment sync. + +## Terminology + +| Term | Meaning | +| --- | --- | +| Environment specification | Desired graph, policies, commands, and bindings, normally defined by repository configuration plus adapter discovery. | +| Environment graph | A DAG whose nodes are environment components and whose edges express build or runtime dependency. | +| Component | One independently fingerprinted requirement such as a Node dependency tree, Rust toolchain, CMake build tree, OCI image, secret binding, or database service. | +| Artifact | A verified material result with a stable identity and manifest. It may be stored by Trail or managed externally. | +| Binding | The projection of a component into a lane: mount, path, environment variable, socket, port, process, container, or credential handle. | +| Generation | An immutable record of the exact component instances and bindings atomically selected for a lane. | +| Adapter | Trusted built-in, declarative recipe, or capability-constrained plugin that understands an ecosystem or resource type. | +| Fingerprint | Canonical digest of all declared non-secret identity inputs for a component. | +| Seed | Immutable initial content presented through a lane-private copy-on-write upper layer. | +| Runtime resource | A process, container, network, port lease, or remote service needed while commands execute. | + +## Environment graph + +An environment is not a single dependency directory. It is a composition of concerns +with different lifetimes and sharing rules. + +```mermaid +flowchart LR + S[Repository source and config] --> P[Environment planner] + P --> T[Toolchain artifacts] + P --> D[Dependency artifacts] + T --> B[Private build state] + D --> B + P --> X[External images and services] + P --> C[Shared caches] + P --> Q[Secret references] + T --> G[Lane environment generation] + D --> G + B --> G + X --> G + C --> G + Q --> G + G --> E[Agent execution] +``` + +Edges have explicit semantics: + +- `build_requires`: the parent identity contributes to the child's fingerprint; +- `runtime_requires`: the parent must be healthy when the child is used; +- `binds_after`: ordering without identity propagation; +- `invalidates_with`: a declared change invalidates a child even when no artifact is + directly consumed. + +These four edge types are implemented in schema v9, repository command recipes, the +protocol-v2 adapter SDK, canonical keys, generation provenance, graph/plan reports, +CLI, HTTP/OpenAPI, and MCP. Legacy `depends_on` remains a compatible +`build_requires` spelling. Runtime/order-only provider replacement advances the exact +generation edge without rebuilding its consumer. + +Cycles are rejected during planning. A service that refers back to application source +is represented as separate image-build and service-runtime nodes rather than a cyclic +node. + +## Policy classes + +Every output and binding must select a policy. There is no implicit "dependency +directory" policy. + +| Policy | Sharing and mutability | Typical examples | Trail behavior | +| --- | --- | --- | --- | +| `immutable_shared` | Read-only and content-addressed across lanes and repositories allowed by scope | package store, unpacked toolchain, generated SDK, sysroot | Build in staging, validate, publish by digest, mount read-only. | +| `immutable_seed_private` | Immutable shared lower content with a lane-private writable upper | `node_modules`, prebuilt Cargo target seed, SDK that post-install tools modify | Attach lower plus private COW upper; whiteouts and replacements stay in the lane. | +| `cache_shared_content` | Cache entries are immutable; index may be concurrent | npm/pnpm download cache, Cargo registry/git cache, BuildKit content store | Namespace by adapter and compatibility tuple; verify content; tolerate eviction. | +| `cache_shared_compiler` | Concurrent cache with tool-specific correctness protocol | sccache, ccache | Use adapter-approved daemon or locking; never treat cache as authoritative output. | +| `writable_private` | Persistent only for one lane | Cargo `target`, CMake build tree, framework `.next` state, local database files | Store directly in the lane-private generated upper with no fake shared layer; snapshot only when explicitly requested. | +| `disposable` | Private and freely recreated | temp files, test output, sockets, ephemeral logs | Allocate per execution or lane; exclude from checkpoints unless requested; reap by TTL. | +| `secret_runtime` | Never stored as artifact content | tokens, signing keys, registry credentials | Resolve a provider reference at execution time; inject through constrained channel; redact. | +| `external_immutable` | Immutable identity stored elsewhere | OCI image digest, remote toolchain archive, Nix store path | Record provider, digest, and verification evidence; materialize lazily. | +| `external_managed` | Mutable service owned outside Trail | cloud database, SaaS API, shared dev service | Record logical reference and health contract, not state; optionally lease credentials. | +| `runtime_private` | Per-lane process/container/network/port | PostgreSQL container, dev server, emulator | Allocate isolated runtime resources; stop or suspend with the lane. | +| `configuration` | Non-secret execution settings | feature flags, target triple, locale | Canonicalize identity-affecting values; inject remaining values at execution. | + +### Decision rules + +Adapters recommend a policy, but the planner applies these invariants: + +1. If concurrent writers can affect one another's correctness, the content is not + directly shareable. +2. If an output is fully determined by declared inputs and validation can reject partial + results, prefer `immutable_shared`. +3. If consumers conventionally modify an otherwise reusable tree, prefer + `immutable_seed_private`. +4. If deletion must not reveal a lower entry, the workspace backend must preserve a + directory whiteout or replacement marker. +5. If losing content changes only performance, classify it as a cache or disposable. +6. If losing content changes correctness or user state, classify it as private state or + an artifact, never as a cache. +7. If a value grants authority, classify it as a secret even when it is also an input. +8. If Trail cannot own the lifecycle, record it as external and verify its identity or + health at the boundary. +9. Repository ignore rules are evidence about version control only. They do not decide + environment policy. + +Users may override adapter recommendations in repository policy, but unsafe widening +of sharing requires an explicit approval recorded in provenance. + +## Specification sources and precedence + +The effective specification is assembled from narrow, explainable sources: + +1. Trail's built-in safety defaults; +2. organization policy; +3. repository configuration committed with source; +4. adapter discovery from manifests and lockfiles; +5. lane-specific non-secret overrides; +6. execution-specific ephemeral overrides. + +Later sources may narrow capabilities. Widening network access, secret access, host +mounts, shared write access, or executable hooks requires policy approval. The resolved +specification and the origin of every field are inspectable with `trail env explain`. + +Discovery proposes graph nodes; it does not silently execute installers. A plan is +produced before any state-changing build or attachment. + +## Identity and fingerprinting + +For component `c`, Trail computes: + +```text +component_key(c) = H(canonical({ + schema_version, + adapter_id, + adapter_version, + component_kind, + declared_input_digests, + parent_artifact_identities, + toolchain_identities, + os, + architecture, + abi, + target, + portability_class, + canonical_options, + allowlisted_identity_environment, + network_and_script_policy, + output_contract +})) +``` + +Canonicalization uses sorted maps, normalized relative paths, explicit string encodings, +and a versioned hashing algorithm. File modes and symlink targets participate when the +adapter declares them significant. Inputs outside the repository are prohibited unless +represented by an explicit toolchain, external artifact, configuration, or secret node. + +### Input roles + +Adapters classify inputs so invalidation is precise: + +- `identity`: always contributes bytes or a normalized semantic value; +- `tool_identity`: resolved executable identity and version probe; +- `policy_identity`: script, network, target, and sandbox policy; +- `runtime_only`: required at execution but does not rebuild the artifact; +- `health_only`: checked before use without affecting identity; +- `secret_reference`: opaque provider, key name, and optional version identifier; +- `ignored`: explicitly documented as irrelevant. + +Secret values never participate in a key. When credential rotation should invalidate an +output, the provider supplies a non-secret version identifier; Trail stores that opaque +identifier, not the credential. + +### Portability classes + +Each artifact declares one portability class: + +- `universal`: independent of operating system and architecture; +- `os_arch`: bound to operating system and architecture; +- `abi`: additionally bound to libc, SDK, compiler ABI, or platform release; +- `host`: safe only on the creating host; +- `lane`: not publishable outside its originating lane. + +Adapters must conservatively choose a narrower class when native extensions, absolute +paths, case sensitivity, or filesystem semantics are uncertain. + +## Artifacts and manifests + +Trail publishes stored artifacts only after successful validation. A manifest contains: + +- artifact ID and component key; +- adapter and schema versions; +- artifact kind, portability, and sharing scope; +- input and parent artifact identities; +- toolchain identities and normalized build options; +- output roots, file count, logical bytes, and tree digest; +- ownership, modes, symlink targets, hard-link groups, and supported special-file policy; +- creation host compatibility tuple and timestamp; +- build operation and transcript references; +- validation rules and results; +- network and script policy actually used; +- redacted provenance and trust level; +- lifecycle state and corruption observations. + +Manifests never contain secret values, unredacted command environments, or credentials +embedded in source URLs. Publication is an atomic rename or equivalent transaction from +a private staging location. Published immutable content is never repaired in place: a +bad artifact becomes `corrupt`, references are quarantined, and a new artifact is built. + +## Environment generations + +An environment generation is an immutable selection of component instances and +bindings for one lane. It solves two problems that a loose set of mounts cannot: + +- an agent can prove exactly what it executed against; +- a crash cannot expose a half-updated mixture of old and new components. + +A generation records: + +- resolved specification digest and source revision observation; +- component keys and artifact identities; +- private upper-layer identities; +- cache namespaces; +- secret references, never resolved values; +- external resource identities and health observations; +- mount, environment, working-directory, network, and service bindings; +- predecessor generation and attachment operation; +- creation, activation, and retirement timestamps. + +### Atomic sync protocol + +`trail env sync ` performs the following operation: + +1. Acquire a lane environment lease and prove the lane is quiescent, or request an + explicit stop/restart policy for managed processes. +2. Resolve configuration and discovery into a canonical graph. +3. Fingerprint every node and propagate staleness through identity-bearing edges. +4. Reuse healthy artifacts; acquire single-flight leases for missing builds. +5. Build missing content in private staging with declared capabilities only. +6. Validate and publish artifacts, or record a failed build without exposing staging. +7. Prepare private upper layers, caches, runtime allocations, and secret handles. +8. Validate every proposed binding for collision, containment, and backend support. +9. Commit the new generation and lane binding set in one database transaction. +10. Switch the workspace/runtime view to the new generation. +11. Run lightweight post-attach health checks and record readiness evidence. +12. Retire the predecessor only after activation succeeds; keep it available for + rollback until retention policy permits collection. + +Only paths declared as environment bindings may be replaced. Sync never resets the +lane's general source upper layer. Reusing an immutable seed with a new fingerprint +creates a fresh private upper by default; preserving mutations across incompatible seeds +requires an adapter-provided migration and explicit policy. + +If the process crashes before step 9, the old generation remains authoritative. If it +crashes between steps 9 and 10, recovery completes or rolls back the recorded pending +switch. Runtime allocations have idempotent cleanup tokens. + +## Component lifecycle + +```text +unknown -> planned -> building -> verified -> ready -> attached + | | | + v v v + failed corrupt stale +``` + +- `planned`: identity and output contract are known. +- `building`: one lease holder owns private staging; waiters observe progress. +- `verified`: output validation and manifest creation succeeded. +- `ready`: artifact publication or external identity resolution succeeded. +- `attached`: referenced by an active generation. +- `stale`: declared inputs or required runtime health changed. +- `corrupt`: observed content conflicts with its manifest. +- `failed`: the attempt ended without a publishable result. + +States are observations, not destructive commands. A stale immutable artifact can +remain retained for old generations while a replacement is built. + +## Verification tiers + +Verification must match both risk and filesystem cost: + +| Tier | Trigger | Checks | +| --- | --- | --- | +| `attach` | Every generation activation | Manifest exists, artifact state is ready, compatibility tuple matches, root metadata and sentinel checks pass, binding targets are valid. | +| `sample` | Policy interval or suspicious observation | Deterministic sample of content and metadata plus adapter health checks. | +| `full` | Explicit command, publication, corruption suspicion, scheduled audit | Complete tree digest, containment, symlink, mode, and adapter validation. | + +Routine status and attach operations must not recursively hash hundreds of megabytes. +The full digest is computed before publication and thereafter only by explicit or +policy-driven verification. This distinction keeps safety evidence strong without +turning every agent startup into an `O(tree size)` scan. + +## Execution contract + +`trail env exec -- ` captures and enforces a complete execution +envelope: + +- active environment generation; +- source and environment mounts; +- working directory; +- executable resolution and `PATH` ordering; +- normalized non-secret environment values; +- secret injection handles and redaction policy; +- user, group, umask, locale, and time-zone policy; +- network policy and allowed endpoints; +- CPU, memory, process, and timeout limits when configured; +- required runtime services and health gates; +- inherited file descriptors, sockets, and ports; +- sandbox and host-filesystem capabilities; +- transcript and operation provenance. + +The command record references the generation, so later inspection can answer which +compiler, lockfile, dependency tree, service image, and policy were active. Directly +entering a lane shell remains possible, but commands outside `env exec` receive weaker +reproducibility evidence and readiness can report that distinction. + +## Filesystem bindings + +Bindings are typed rather than encoded as opaque mount arguments: + +- `tree_ro`: immutable tree mounted read-only; +- `tree_cow`: immutable lower plus lane-private upper and work metadata; +- `tree_private`: persistent private directory; +- `tree_scratch`: disposable private directory; +- `file_ro` and `file_private`: single-file variants; +- `cache`: adapter-mediated cache namespace; +- `env`: non-secret environment value; +- `secret_env`, `secret_file`, and `secret_fd`: runtime secret injection; +- `socket`, `port`, `service`, and `container`: runtime resources; +- `external`: identity and health reference with no local mount. + +Target paths are normalized relative to the lane workspace unless explicitly declared +as sandbox paths. Bindings cannot overlap unless their ordering and nesting semantics +are declared. Trail internal paths, `.git`, lane control metadata, and secret staging +areas are reserved. + +All workspace backends must preserve the same observable semantics for copy-up, +whiteouts, bulk directory replacement, rename, links, metadata, and deletion. Backend +capability negotiation rejects a generation whose required semantics are unsupported. + +## Shared caches + +Caches improve performance but never determine correctness. Each cache namespace is +derived from: + +- adapter and cache kind; +- compatibility tuple; +- organization/repository sharing scope; +- explicit trust boundary; +- optional tool-native namespace. + +Trail supports three access strategies: + +1. content-addressed entries with an independently locked index; +2. an adapter-approved concurrent cache daemon such as sccache; +3. a lane-private cache seeded from a shared immutable snapshot. + +An adapter may not label a directory `cache_shared_*` unless its producer supports +concurrent writers or Trail mediates writes safely. Cache eviction can make execution +slower but cannot make a ready generation invalid. + +Schema v10 implements this contract for trusted built-ins. Namespace identity hashes +adapter identity, logical cache name, protocol, access policy, workspace scope, and the +complete compatibility map. Node, Cargo, sccache, and Go no longer write untyped global +tool-home paths. Every command holds a process/start-token lease; host-exclusive caches +also acquire deterministic cross-process locks. GC first closes an atomic maintenance +gate, then refuses to rename a namespace with any live lease. Cache rows and exact IDs +are retained in active and historical generation reports, while the bytes remain +performance-only and safely evictable. External plugins remain denied until their cache +protocol receives an explicit certification capability. + +## Secrets + +Environment specifications contain secret references such as provider, logical name, +scope, purpose, and optional version selector. Resolution occurs as late as possible, +normally immediately before command or service start. + +Required guarantees: + +- no secret bytes in component keys, manifests, database values, artifact trees, + command previews, or transcripts; +- redaction operates on exact resolved values and structured provider fields; +- file secrets use private permissions and are unlinked or revoked after use; +- file-descriptor injection is preferred when the consumer supports it; +- provider access is scoped to the lane task and command purpose; +- secret access is an auditable operation without recording the value; +- checkpoint, export, and diagnostics scan for accidental secret persistence; +- adapters receive opaque handles unless their declared capability requires bytes. + +Secret availability affects readiness. Secret rotation alone does not rebuild immutable +artifacts unless a non-secret version identity is deliberately part of the key. + +Schema v15 implements the first fail-closed service-secret provider without storing +bytes. An OCI service or protocol-v2 plugin declares a logical name, `file` or +`environment_file` provider, opaque reference, optional non-secret version, purpose, +requiredness, and a `/run/secrets/...` target. An optional consumer environment variable +receives only that target path for `*_FILE` conventions. `environment_file` resolves an +environment variable to a provider-owned path at reconcile time. Trail never reads that file; it +requires a bounded non-symlink regular file with private permissions and passes only a +read-only mount handle to Docker or Podman. Active and historical tables store references +and availability, and an append-only access audit stores only identity, purpose, provider, +outcome, and time. Missing or revoked required handles stop the owned service and block +readiness. Each managed container carries a digest of its resolved source paths, targets, +and non-secret `*_FILE` bindings—not secret bytes. If a provider rotates a handle to a new +path, reconciliation detects the mismatch and recreates only the Trail-owned container, +so it cannot adopt a stale bind mount. Raw environment-value injection is intentionally +denied for standalone Docker because it would persist the value in inspectable container +configuration; portable non-persistent environment and descriptor injection remain future +provider capabilities. + +## External and runtime resources + +Not everything should become a local tree. + +### External immutable artifacts + +An OCI image should normally be represented by registry, repository, platform, and +content digest. Tags may be accepted as resolution inputs, but the generation records +the resolved digest. Similar rules apply to remote toolchain bundles and package store +objects. + +The first implemented form is a strict pinned declaration: + +```toml +# trail.oci.toml +schema = "trail.oci-images/v1" + +[[image]] +name = "web" +reference = "ghcr.io/example/web@sha256:<64 lowercase hex characters>" +platform = "linux/amd64" +``` + +`trail/oci-image@1` and protocol-v2 plugins normalize this to a metadata-only external +component. Planning is side-effect free and does not contact a registry. Schema v14 +stores the exact identity in active state and immutable generation history; the +component has no fake output directory or layer, and cleanup ownership is `external`. +Tag-to-digest resolution, registry verification/materialization, credentials, and +container lifecycle remain separate provider/runtime work so they cannot contaminate +artifact identity or planning purity. + +### Externally managed services + +A cloud database or shared API contributes: + +- a logical resource reference; +- endpoint discovery policy; +- credential reference; +- compatibility or schema requirement; +- health and readiness probe; +- lease or concurrency rules; +- cleanup ownership, which is explicitly `external`. + +Trail does not snapshot or delete externally managed state. + +### Private runtime services + +Local services and containers are allocated per lane by default. Names, networks, +volumes, sockets, and host ports include lane identity. Runtime state is separate from +immutable image identity. A generation may declare whether a service is restarted, +reused when compatible, or always recreated on activation. + +The first private-service form extends `trail.oci.toml` without giving the adapter +provider authority: + +```toml +[[service]] +name = "database" +image = "postgres-image" +container_port = 5432 +health_timeout_ms = 45000 +restart_policy = "on_failure" +volume_target = "/var/lib/postgresql/data" +``` + +Schema v14 stores the immutable declaration separately from the allocation ID, +provider resource ID, deterministic container/network/volume names, loopback host port, +health state, cleanup token, lifecycle owner, and timestamps. Containers and networks +are generation-scoped; a private data volume is scoped to the logical lane service so a +healthy image rollout does not silently discard database state. Retired containers and +networks are removed after the replacement becomes healthy, while a volume remains as +long as an active generation references it. Docker and Podman are +detected at reconciliation time. A missing pinned image is pulled by digest; Trail then +verifies the observed repository digest before creating anything. Existing names are +adopted only when Trail ownership labels match exactly. A dead lifecycle owner is marked +`orphaned` on reopen and safely inspected on the next reconcile. + +## Monorepos and multiple lanes + +Discovery roots are explicit. A monorepo can have many overlapping environment graphs, +but identical component keys converge to one artifact. For example: + +- several packages can share one pnpm store and toolchain; +- two Next.js applications can share lockfile-derived dependencies but keep separate + `.next` state; +- Rust crates can share registry content and sccache while retaining private target + directories; +- CMake targets can share a toolchain and immutable install prefix while keeping + configuration-specific build trees private. + +Each lane pins a generation independently. Syncing lane A cannot change the active +generation, private upper, services, or secret handles of lane B. A new artifact may be +built once and become available to both, but each lane attaches it through its own +transaction. + +Graph invalidation is component-local. Editing a frontend lockfile need not invalidate +an unrelated Rust toolchain. Editing a shared compiler configuration invalidates every +descendant whose edge carries tool identity. + +## Concurrency and leases + +Trail uses distinct leases for: + +- component build single-flight; +- artifact publication; +- lane generation mutation; +- runtime resource allocation; +- cache maintenance; +- garbage collection. + +Build waiters may stream progress and either reuse the published result or retry after a +failed lease. Leases contain owner operation, heartbeat, expiry, and recovery metadata. +No lease alone proves safety: publication also requires staging ownership and manifest +validation, while activation also requires a lane transaction. + +Garbage collection considers active generations, retained predecessors, checkpoints, +running operations, open backend references, and leases. Logical and physical byte +accounting are reported separately so shared-space savings remain visible. + +## Adapter trust model + +Trail supports three adapter tiers: + +1. **Built-in adapters** run as reviewed Trail code with narrowly scoped host APIs. +2. **Declarative recipes** describe probes, commands, inputs, outputs, and policies; the + Trail host executes them in a sandbox. +3. **Capability-constrained plugins** use a versioned WASI/component interface or + equivalent isolated protocol. They request filesystem, process, network, secret, and + runtime capabilities explicitly. + +Adapters never receive raw database access, shared artifact mutation, arbitrary host +paths, or undeclared secrets. They return plans and observations. The host validates +paths, executes approved actions, publishes artifacts, and commits generations. + +Open-ended shell hooks are disabled by default. Command vectors avoid shell parsing; +shell execution, lifecycle scripts, and network access are separately visible policy +decisions. + +The detailed contract is defined in +[Environment Adapter and Specification Contract](environment-adapter-contract.md). + +## Proposed data model + +The names below are conceptual; implementation may normalize fields differently. + +| Relation | Key fields | Purpose | +| --- | --- | --- | +| `environment_specs` | ID, repository, root, source, canonical digest, schema version | Resolved desired graph and field provenance. | +| `environment_components` | spec ID, component ID, kind, adapter, policy, options | Nodes in the desired graph. | +| `environment_edges` | parent, child, semantics | Build, runtime, ordering, and invalidation relationships. | +| `environment_inputs` | component, role, source, observed digest | Explainable identity and health inputs. | +| `environment_artifacts` | artifact ID, component key, state, portability, manifest | Stored or external immutable results. | +| `environment_builds` | operation, component key, lease, staging, outcome | Construction provenance and recovery. | +| `environment_generations` | lane, generation, spec digest, predecessor, state | Atomic environment snapshots. | +| `environment_bindings` | generation, component, type, source, target, policy | Filesystem, environment, secret, and runtime projections. | +| `environment_private_state` | lane, component, generation lineage, storage ID | Persistent private uppers and directories. | +| `environment_runtime_resources` | generation, kind, provider ID, lease, health, cleanup | Processes, containers, ports, networks, and services. | +| `environment_verifications` | artifact/resource, tier, rule, result, operation | Health and integrity evidence. | + +Foreign keys and state transitions are enforced transactionally. Manifests remain +portable records even if an implementation stores large trees outside the database. + +### Verification tiers + +Immutable filesystem artifacts distinguish bounded attachment evidence from an +explicit integrity audit. Publication and successful full verification write a durable +seal over the content-addressed manifest, layer summary, and platform filesystem +identity. Routine reuse validates that seal in constant filesystem work; a legacy, +oversized, malformed, or stale seal triggers one complete scan and self-heals. Explicit +verification, doctor, and readiness always compare every entry and content hash. The +seal is derived cache state, never authority, and is collected atomically with the +layer. + +## User experience + +The command family is environment-centric while retaining compatibility aliases for +the existing dependency workflow. + +```text +trail env discover [--lane ] +trail env graph [--lane ] [--format table|json|dot] +trail env plan +trail env sync [--offline] [--frozen] [--restart-services] +trail env status [--verify attach|sample|full] +trail env explain [component-or-path] +trail env verify [--full] +trail env exec -- [args...] +trail env gc [--dry-run] +``` + +Commands and gates executed in a layered lane receive `TRAIL_SERVICES_JSON`, keyed by +`component/resource`, with loopback host, port, and address for every healthy active +service. A globally unique resource name also receives +`TRAIL_SERVICE__HOST`, `_PORT`, and `_ADDRESS`. Trail refuses to start the command +when an active required runtime is not healthy, so clients never silently use a stale +endpoint. + +`trail lane deps plan|sync|status` remains as a compatibility view over components with +dependency-related kinds. + +The implemented explanation surface selects a stable component explicitly: + +```text +trail env explain --component [--offset N] [--limit N] +``` + +It diffs canonical input and tool maps plus adapter, platform, architecture, +portability, and strategy fields. Reports expose names and change classes, never key +values, and carry provenance completeness and pagination metadata. + +A plan should be decision-oriented: + +```text +Lane: feature-search +Current generation: envgen_014 +Proposed generation: envgen_015 + +REUSE rust-toolchain immutable_shared art_b71… +REUSE cargo-registry cache_shared_content cache_cargo_linux_x64 +BUILD cargo-target-seed immutable_seed_private stale: Cargo.lock changed +KEEP cargo-target writable_private lane state is compatible +RESOLVE api-token secret_runtime provider: keychain/team-api +START postgres runtime_private image sha256:91c…; port auto + +Network required while building: crates.io (adapter policy) +Source paths replaced: none +Private paths reset: target seed upper only +``` + +`--frozen` rejects discovery or identity resolution that would change the pinned graph. +`--offline` rejects unmaterialized external fetches. Status exits nonzero when readiness +requirements are unmet, while returning a structured reason list. + +## API and report parity + +CLI text is a rendering of shared report types, not an independent implementation. +Minimum reports are: + +- `EnvironmentDiscoveryReport`; +- `EnvironmentGraphReport`; +- `EnvironmentPlanReport`; +- `EnvironmentSyncReport`; +- `EnvironmentStatusReport`; +- `EnvironmentExplanationReport`; +- `EnvironmentVerificationReport`; +- `EnvironmentExecutionReport`. + +HTTP, MCP, CLI JSON, and Rust APIs expose the same stable identifiers, lifecycle states, +policy decisions, stale reasons, operation links, and redaction rules. Long-running +build, verification, and runtime operations use Trail operations with progress events +and cancellation rather than blocking opaque requests. + +## Readiness, claims, and Git handoff + +Environment readiness is one input to lane readiness; it does not replace task claims +or Git conflict checks. + +A lane is environment-ready when: + +- its active generation matches the resolved specification or an allowed pin; +- all required artifacts pass the configured verification tier; +- private bindings are mountable and not owned by another lane; +- required services are healthy; +- required secret references are resolvable for the intended command; +- no generation switch or environment build is in an unsafe state. + +The readiness report identifies environment drift separately from source divergence, +task ownership, guardrail violations, and merge conflicts. Environment sync never +creates a Git commit, moves a branch, or merges code. Git handoff continues through the +explicit Trail lane workflow. + +## Failure handling + +| Failure | Required result | +| --- | --- | +| Adapter discovery fails | Preserve active generation; record structured diagnostic and adapter version. | +| Build command fails | Retain logs under redaction policy; delete or quarantine staging; publish nothing. | +| Builder crashes | Expire lease, inspect staging ownership, and retry or clean idempotently. | +| Validation fails | Mark attempt failed; never attach output. | +| Published artifact changes | Mark corrupt, quarantine from new generations, preserve evidence, rebuild. | +| Backend cannot express whiteout or link semantics | Reject affected binding before activation. | +| Secret provider unavailable | Keep artifact state intact; mark command/service readiness blocked. | +| Service health fails | Apply declared restart policy; otherwise preserve generation and report not ready. | +| Activation crashes | Complete or roll back pending generation from transaction journal. | +| Cache corruption | Evict affected entries; rebuild authoritative outputs. | +| Disk pressure | Reap disposable state, then unreferenced cache and artifacts; never delete active private state. | + +Diagnostics always distinguish corrective actions that are safe to automate from those +requiring user authorization. + +## Platform behavior + +The semantic contract is platform-independent; materialization backends are not. + +- Linux prefers kernel overlayfs where permitted and uses FUSE when policy or substrate + requires it. +- macOS uses Trail's NFS-backed layered workspace view or a future equivalent. +- A materialized-copy backend is a correctness fallback when layering is unavailable, + with explicit space and performance reporting. +- Remote or network filesystems must advertise atomic rename, locking, link, mode, + invalidation, and crash-recovery capabilities rather than being assumed compatible. +- Artifacts whose manifests contain unsupported modes, symlinks, case collisions, or + host-bound paths are rejected or narrowed to `host` portability. + +The conformance suite exercises untracked dependency trees, nested deletion, bulk +directory replacement, rename across layers, symlinks, hard links, executable modes, +concurrent readers, crash recovery, and large metadata-heavy trees on every supported +backend. + +## Performance and observability + +Trail reports: + +- plan, build, validation, publication, attach, and service-start durations; +- logical artifact bytes versus unique physical bytes; +- copy-up bytes and whiteout counts per lane; +- cache hit/miss and invalidation reasons; +- mount/backend round trips where measurable; +- generation switch downtime; +- full-verification throughput; +- garbage-collection candidates and retained-by explanations. + +Initial product targets should include: + +- no full-tree scan on an unchanged routine attach; +- constant-number database transactions for generation activation; +- one shared build for concurrent identical component keys; +- no eager copy of immutable seeds; +- clear progress for metadata-heavy package-manager replacement rather than an + apparently hung command; +- cancellation that leaves the previous generation usable. + +Performance claims must identify dataset shape, backend, host, cold/warm state, and +verification tier. A small synthetic tree is not evidence for real Node, Cargo, CMake, +or container workloads. + +## Compatibility and migration + +The existing workspace-layer implementation becomes the first artifact and binding +backend: + +- workspace layer keys map to component keys; +- workspace layer manifests map to artifact manifests; +- lane workspace mounts map to generation bindings; +- dependency plan/sync/status map to filtered environment operations; +- schema-backed ordering/invalidation edges are finalized in deterministic topological + order, enter downstream component keys, persist in generation provenance, and + propagate exact descendant staleness; +- current Node and Cargo logic become built-in adapters; +- `trail/go-vendor@1` publishes a lock- and toolchain-keyed vendor seed, while Go + workspaces fail closed until graph-aware multi-module planning is implemented; +- `trail/cmake-build@1` provisions a layer-free private build tree and defers configure + to the mounted lane so absolute cache paths remain valid; +- `trail/python-venv@1` provisions a layer-free private `.venv`, keys it by dependency + manifests/locks and interpreter identity, and automatically initializes it through an + ephemeral candidate view at the final mountpoint so embedded prefix paths remain valid + without weakening atomic generation activation; +- existing layer references are imported into an initial environment generation without + copying tree content. + +Migration is additive. Repositories without an environment specification continue to +use adapter discovery and current defaults. Configuration gains a schema version and an +explicit opt-in for policy overrides; existing dependency commands remain supported +until report and behavior parity is proven. + +## Security invariants + +1. A component cannot read undeclared source or host paths during a hermetic build. +2. A component cannot publish outside host-owned staging. +3. An adapter cannot mutate Trail's database or an immutable artifact. +4. A lane cannot write another lane's private state. +5. Shared direct-write state requires a reviewed concurrency protocol. +6. Secrets cannot enter artifact identity or persisted output by default. +7. Network and lifecycle-script use is explicit in plan and provenance. +8. External resources are never deleted unless Trail created them and owns cleanup. +9. Generation activation is atomic and requires lane safety checks. +10. Readiness never claims stronger verification than the evidence tier performed. + +## Acceptance criteria + +The architecture is complete enough to ship when: + +- one graph can compose at least Node, Cargo, CMake, OCI image, secret, and service + components; +- all policy classes have storage, lifecycle, failure, and garbage-collection tests; +- two lanes can concurrently use the same immutable artifacts and caches without sharing + private writes; +- a lockfile change yields an exact stale explanation and only invalidates descendants; +- a crash at every build and activation boundary preserves a valid prior generation; +- secret canary tests find no value in database, logs, transcripts, manifests, + checkpoints, or exported diagnostics; +- Linux overlayfs/FUSE and macOS NFS pass the same filesystem conformance suite; +- full verification detects injected corruption while routine attach avoids a full tree + scan; +- CLI, HTTP, MCP, and Rust API reports are semantically identical; +- a recorded execution can reconstruct or clearly report unavailable external inputs; +- logical-versus-physical accounting demonstrates sharing without hiding private-state + cost. + +## Open decisions + +1. Whether repository specifications live in `trail.toml`, a dedicated + `.trail/environment.toml`, or both through includes. +2. Which sandbox backend provides the minimum portable declarative-command contract. +3. Whether environment generations are embedded in lane checkpoints or referenced as + separately retained immutable records. +4. How organization-level artifact trust and signing are represented across machines. +5. Which cache protocols qualify for shared direct writes in the first release. +6. Whether plugin isolation begins with a subprocess JSON protocol before adopting a + WASI component interface. +7. How long predecessor generations and private seed uppers remain recoverable by + default. + +These decisions can change implementation detail without weakening the policy classes, +identity model, atomic generation lifecycle, or security invariants defined here. diff --git a/docs/design/vscode-acp-chat-view.md b/docs/design/vscode-acp-chat-view.md index dcabdbf..58a3af2 100644 --- a/docs/design/vscode-acp-chat-view.md +++ b/docs/design/vscode-acp-chat-view.md @@ -20,7 +20,7 @@ VS Code Trail extension | | ACP JSON-RPC stdio v -trail agent acp --provider +trail agent acp run | | Trail ACP relay v @@ -177,7 +177,7 @@ turn and whether the turn produced a durable Trail checkpoint. | +-- tool: edit file | +-- diff: docs/integrations/acp.md | -* checkpoint ch_... Ready for review +* checkpoint checkpoint_... Ready for review ``` This is the one distinctive visual move. Everything else should be restrained: @@ -697,7 +697,7 @@ Required reads: Required mutations: -- create/start task through `trail agent acp` +- create/start task through `trail agent acp run` - cancel prompt through ACP - approval decision through ACP and Trail mirror - record/refresh lane workdir when explicitly requested @@ -959,7 +959,7 @@ Acceptance: Deliver: - Provider registry with `claude-code` and custom command. -- ACP stdio client over `trail agent acp`. +- ACP stdio client over `trail agent acp run`. - Chat panel with text messages, plan, tool rows, usage meter, and cancel. - Basic permission prompt. diff --git a/docs/getting-started/agent-workflow.md b/docs/getting-started/agent-workflow.md index a0ed532..7b03694 100644 --- a/docs/getting-started/agent-workflow.md +++ b/docs/getting-started/agent-workflow.md @@ -7,13 +7,13 @@ changes, and applies reviewed work to Git only after a safety preflight. ```sh make install -trail agent doctor --provider codex +trail agent doctor codex ``` -For an ACP editor, print and install its configuration snippet: +For an ACP editor, print its configuration entry: ```sh -trail agent setup --provider codex --editor vscode +trail agent acp setup codex --editor vscode ``` ## 2. Start a Task @@ -21,7 +21,7 @@ trail agent setup --provider codex --editor vscode For terminal-first work: ```sh -trail agent start --provider codex --name first-agent-task +trail agent start codex --name first-agent-task ``` Trail creates a fresh lane and launches the provider inside an isolated diff --git a/docs/getting-started/first-lane-workflow.md b/docs/getting-started/first-lane-workflow.md index c119901..fb208e7 100644 --- a/docs/getting-started/first-lane-workflow.md +++ b/docs/getting-started/first-lane-workflow.md @@ -48,9 +48,9 @@ trail lane gates docs-lane --limit 20 Preview and queue the merge: ```sh -trail merge-lane docs-lane --into main --dry-run -trail merge-queue add docs-lane --into main -trail merge-queue run +trail lane merge docs-lane --into main --dry-run +trail lane merge-queue add docs-lane --into main +trail lane merge-queue run ``` Remove the lane after the work is merged or intentionally abandoned: @@ -135,14 +135,14 @@ audit history. Preview first: ```sh -trail merge-lane docs-lane --into main --dry-run +trail lane merge docs-lane --into main --dry-run ``` For shared branches, use the queue: ```sh -trail merge-queue add docs-lane --into main -trail merge-queue run +trail lane merge-queue add docs-lane --into main +trail lane merge-queue run ``` If conflicts are opened: @@ -159,4 +159,4 @@ trail conflicts show - Patch schema: `trail/src/model/inspect/patch.rs` - Readiness: `trail/src/db/lane/readiness.rs` - Rewind: `trail/src/db/lane/rewind.rs` -- Tests: `lane_patch_can_merge_into_main`, `merge_lane_and_queue_enforce_readiness_blockers`, `merge_queue_pauses_on_conflict` +- Tests: `lane_patch_can_merge_into_main`, `merge_lane_and_queue_enforce_readiness_blockers`, `lane_merge_queue_pauses_on_conflict` diff --git a/docs/getting-started/install-and-build.md b/docs/getting-started/install-and-build.md index 307759b..225c8d3 100644 --- a/docs/getting-started/install-and-build.md +++ b/docs/getting-started/install-and-build.md @@ -5,7 +5,9 @@ Trail is a Rust workspace with two crates: - `trail`: the CLI, library API, HTTP daemon, and MCP server. - `prolly`: the prolly-tree storage library used by Trail. -The workspace declares Rust 1.81 in `Cargo.toml`. +Trail's parent-owned crates use Rust edition 2024 and declare Rust 1.89 as the +minimum supported toolchain in `Cargo.toml`. The independently versioned +`prolly` submodule retains its own edition and compiler policy. ## Install a Release @@ -146,9 +148,9 @@ trail lane readiness docs-lane Merge only after review and readiness checks: ```sh -trail merge-lane docs-lane --into main --dry-run -trail merge-queue add docs-lane --into main -trail merge-queue run +trail lane merge docs-lane --into main --dry-run +trail lane merge-queue add docs-lane --into main +trail lane merge-queue run ``` ## Validate the Local Build diff --git a/docs/guides/branch-checkout-and-merge.md b/docs/guides/branch-checkout-and-merge.md index 68bc48d..9e209f0 100644 --- a/docs/guides/branch-checkout-and-merge.md +++ b/docs/guides/branch-checkout-and-merge.md @@ -43,25 +43,25 @@ When conflicts occur, Trail records structured conflict sets for inspection and ## Lane Merges ```sh -trail merge-lane doc-bot --into main --dry-run +trail lane merge doc-bot --into main --dry-run ``` Non-dry-run lane merges into the default branch use the merge queue by default: ```sh -trail merge-queue add doc-bot --into main -trail merge-queue run +trail lane merge-queue add doc-bot --into main +trail lane merge-queue run ``` -Immediate default-branch merges require `trail merge-lane ... --direct`. +Immediate default-branch merges require `trail lane merge ... --direct`. ## Merge Queue ```sh -trail merge-queue add doc-bot --into main --priority 10 -trail merge-queue list -trail merge-queue run --limit 1 -trail merge-queue remove +trail lane merge-queue add doc-bot --into main --priority 10 +trail lane merge-queue list +trail lane merge-queue run --limit 1 +trail lane merge-queue remove ``` The queue serializes merges and stops on conflicts or failures. diff --git a/docs/guides/hardening-agent-workflows.md b/docs/guides/hardening-agent-workflows.md index df93fba..6367895 100644 --- a/docs/guides/hardening-agent-workflows.md +++ b/docs/guides/hardening-agent-workflows.md @@ -28,9 +28,9 @@ trail config set lane.claim_enforcement reject Use merge queue for shared targets instead of direct merges: ```sh -trail merge-queue add docs-lane --into main -trail merge-queue explain docs-lane -trail merge-queue run +trail lane merge-queue add docs-lane --into main +trail lane merge-queue explain docs-lane +trail lane merge-queue run ``` ## Lane Isolation @@ -96,7 +96,7 @@ mutating the lane. { "op": "replace_line", "path": "README.md", - "line_id": "ch_abc:2", + "line_id": "line_abc:2", "expected_text": "old text", "new_text": "new text" } @@ -154,7 +154,7 @@ example `lane started 14 operations behind main`. Before merging, inspect the queue item: ```sh -trail merge-queue explain docs-lane +trail lane merge-queue explain docs-lane ``` `merge-queue explain` includes readiness blockers, dry-run conflicts, preflight @@ -218,7 +218,7 @@ Useful focused tests for these controls include: - `lane_workdir_record_preview_reports_risks_and_oversized_files` - `lane_workdir_sync_refuses_dirty_and_force_refreshes` - `merge_lane_and_queue_enforce_readiness_blockers` -- `merge_queue_explain_reports_dry_run_conflicts_without_recording_conflict_state` +- `lane_merge_queue_explain_reports_dry_run_conflicts_without_recording_conflict_state` - `conflict_explanations_classify_common_non_line_conflicts` - `local_lane_http_api_can_require_bearer_token` - `local_http_bodyless_mutations_reject_request_bodies` diff --git a/docs/guides/performance-and-scale-benchmarks.md b/docs/guides/performance-and-scale-benchmarks.md index d6cb9bb..385a944 100644 --- a/docs/guides/performance-and-scale-benchmarks.md +++ b/docs/guides/performance-and-scale-benchmarks.md @@ -14,6 +14,41 @@ The smoke target defaults to 1,000 synthetic files and is intended for pull-requ python3 scripts/check-cli-scale-thresholds.py name=max_seconds ... --metrics key=max_value ... ``` +Structural string invariants use `--metric-equals key=value`. The Git handoff +gate combines wall-time ceilings, numeric ceilings, and exact export-mode +checks in one invocation. + +The `Changed-path Ledger Native Gates` workflow is a required merge and +release gate. Branch protection and release automation must require both its +Linux/ext4 and macOS/APFS matrix jobs for the exact commit being merged or +released; scheduled artifacts are additional qualification evidence, not a +substitute for those commit-specific checks. + +The workflow is reusable. `release-automation.yml` calls it before creating a +tag, and cargo-dist declares it as a generated `plan-jobs` dependency so a tag +push cannot build, host, or publish until both native jobs pass for that tag's +exact SHA. The compiled activation manifest names those dependencies, but its +self-hash is only a declaration of the build contract; it is not evidence that +a workflow ran. + +Changed-path structural reports are closed schemas: missing and unknown work +counters fail the gate. Every repository-size-sensitive counter is either +required to be zero or capped by an explicit affine O(k) bound. Selected +worktree-index SQLite work additionally has a typed disposition: + +- workspace status, diff, and record report `not_applicable`, one independent + N/A proof, and zero selected-index work; +- materialized lane record, structured patch, and COW checkpoint report + `complete`, at least one accounting envelope (including k=0), and no N/A + claim; +- absent, mixed, or ambiguous accounting fails even when every numeric SQLite + counter is zero. + +COW checkpoint reports must also declare +`generated_path_accounting=journal_interval`. This proves generated dirty paths +come from the bounded journal interval rather than a recursive upper-directory +inventory. + ## Local and Large Runs ```sh @@ -46,6 +81,23 @@ Optional toggles: - `TRAIL_SCALE_DAEMON=0|1` - `TRAIL_SCALE_GIT_IMPORT=0|1` +Run the path-index structural matrix without the unrelated large artifacts: + +```sh +TRAIL_BIN=target/release/trail \ +TRAIL_SCALE_FILES=1000,100000,1000000 \ +TRAIL_SCALE_MATERIALIZED=0 \ +TRAIL_SCALE_BACKUP=0 \ +TRAIL_SCALE_DAEMON=0 \ +TRAIL_SCALE_GIT_IMPORT=0 \ +scripts/cli-scale-bench.sh +``` + +This still runs an explicitly bounded sparse record case at every scale. It +also covers a content-only patch, a case-only rename combined with delete/add, +and a patch against an empty root. The scheduled workflow applies the same +structural gates to 10k, 100k, and 1M files. + ## Output Each scale writes: @@ -62,6 +114,34 @@ Important storage signals: - `manifest_repo_clean_workdir_bytes` - `manifest_repo_sparse_workdir_bytes` +Patch and record JSON reports include an operation-scoped `path_index` object: + +- `mode`: `indexed` once the operation uses bounded folded-path lookups. +- `lookup_count`: number of folded-key index lookups. This must be no greater + than the unique folded old/new paths touched by the completed operation; + content-only writes to existing exact paths can require zero lookups. +- `full_root_path_load_count`: unbounded traversals that enumerate every path + in a persisted root. A fallback that loads both the previous and target roots + counts twice. +- `full_filesystem_path_scan_count`: unbounded repository-shaped filesystem + traversals used for path validation. A full materialized workdir validation + counts as one, including a clean no-op. Walking an explicitly selected sparse + materialization does not count because its size is bounded by that selection. + +The counters reset at the public patch or record boundary, including failures, +retries, and no-ops. This prevents a prior operation from making a later report +look unbounded. The benchmark extractor independently folds changed paths with +the same NFKC, per-codepoint lowercase, then NFC sequence as the Rust index and +rejects malformed or internally inconsistent reports. + +A full materialized lane patch normally updates a valid clean manifest from its +touched subset. If that manifest is missing or stale, the correctness fallback +loads both complete roots and reports +`full_root_path_load_count=2`. Likewise, a full materialized record without a +usable manifest compares one complete root to the disk manifest and reports +`full_root_path_load_count=1`. These are deliberately visible cold paths; the +no-materialize and explicitly sparse benchmark operations must remain zero. + Important hot-path rows: - `status_clean` @@ -71,6 +151,9 @@ Important hot-path rows: - `git_dirty_diff` - `git_dirty_record` - `git_status_after_dirty_record` +- `agent_git_apply_dry_run` +- `agent_git_apply` +- `agent_git_apply_missing_mapping` - `daemon_wait_for_health` - `daemon_wait_for_hot_cache` - `daemon_persisted_snapshot_status` @@ -78,7 +161,7 @@ Important hot-path rows: - `daemon_persisted_snapshot_diff_dirty` - `lane_apply_patch` - `lane_readiness` -- `merge_queue_run` +- `lane_merge_queue_run` - `daemon_cli_status` - `daemon_cli_session_start` - `daemon_cli_approval_request` @@ -92,8 +175,53 @@ Important hot-path rows: `git_dirty_*` rows measure the non-daemon Git dirty-path fallback for large repositories with a committed Git baseline. This fallback is useful for correctness and smaller repositories, but 1M measurements show it is not a production hot path by itself. `daemon_wait_for_health` and `daemon_wait_for_hot_cache` measure daemon startup and cache warmup, not steady-state command latency. `daemon_persisted_snapshot_*` rows hide daemon endpoint discovery while keeping the live daemon process and persisted watcher snapshot, so they verify separate Trail handles can avoid the full direct fallback without using HTTP RPC. Use `daemon_cli_*` rows for repeated agent-command hot paths. +The `agent_git_apply_*` rows exercise the high-level committed Git handoff, +not only Trail's internal lane merge. The mapped fixture creates a +no-materialize task that changes +`k = max(1, min(100, files / 1000))` paths, records the review gate, and runs +both dry-run and actual apply. Its structural metrics require +`export_mode=mapped_delta`, exactly `k` changed paths, and at most `k` blob +writes. `agent_git_plumbing_commands` is specifically the mapped-export +plumbing subprocess count: `read-tree`, the optional batch `hash-object`, one +batch `update-index`, `write-tree`, and `commit-tree`. It excludes the final +porcelain fast-forward and is gated as a constant ceiling of 5, independent of +`k`; deletion-only exports may be lower because they do not hash new blobs. +Full-snapshot exports do not use this mapped-export counter. The missing-mapping +fixture is initialized from the working tree and must return +`GIT_MAPPING_REQUIRED` without changing Git HEAD, the Git index, or Trail's +mapping table. + ## Current Evidence +The path-index structural matrix was freshly measured on July 13, 2026 with a +release binary and headless optional toggles at +`/tmp/trail-cli-scale-task5-struct-20260713`. All 60 structural and wall/storage +gates passed (20 per scale): + +| Scale | Source files | Init | Content patch | Case rename patch | Empty-root patch | Sparse record | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1k | 1,005 | 0.14s | 0.03s | 0.01s | 0.00s | 0.01s | +| 100k | 100,029 | 9.62s | 0.29s | 0.02s | 0.01s | 0.01s | +| 1M | 1,000,029 | 188.10s | 0.34s | 0.02s | 0.01s | 0.01s | + +At all three scales, all four operation reports used `mode=indexed`, loaded no +full root path set, and performed no unbounded filesystem path scan. The +content patch performed 0 lookups for 5 touched folded keys at 1k and 0 for 50 +at 100k/1M. The case fixture performed 1 lookup for 2 reported folded endpoints, +while empty-root patch and sparse record each performed 1 lookup for 1 touched +folded key. The 1M artifact was 1,584,734,208 SQLite bytes with 1,000,352 +objects; index rebuild took 3.28s. These results are structural evidence that +patch and bounded record work scales with touched paths rather than repository +path count. + +Keep legacy missing-index recovery as a separate correctness proof rather than +mixing it into the performance rows: + +```sh +cargo test -p trail --test e2e \ + cli_path_index_required_human_json_rebuild_and_retry_lifecycle -- --exact +``` + The latest local `/Volumes/Workspace` runs were measured on June 25, 2026 with the release binary: | Scale | Init | Direct clean status | Direct dirty diff | Direct dirty record | Daemon status | Daemon CLI record | Lane patch | Lane readiness | Merge apply | @@ -118,7 +246,7 @@ python3 scripts/check-cli-scale-thresholds.py \ daemon_cli_approval_request=10 daemon_cli_lease_acquire=10 \ daemon_cli_timeline=10 daemon_cli_why=10 daemon_cli_history=10 \ daemon_cli_code_from=10 lane_apply_patch=10 lane_readiness=10 \ - merge_lane_dry_run=10 merge_lane_apply=10 merge_queue_run=10 \ + merge_lane_dry_run=10 merge_lane_apply=10 lane_merge_queue_run=10 \ git_dirty_status=120 git_dirty_diff=120 git_dirty_record=120 \ git_status_after_dirty_record=90 \ --metrics /Volumes/Workspace/trail-cli-scale-codex-1m-git-daemon-20260625/1000000/metrics.tsv \ @@ -151,6 +279,11 @@ Large agent orchestration should use the daemon and no-materialize/sparse workdi - If filesystem access is needed, materialize selected paths with `--paths` and hydrate more paths lazily through `lane read`. - Keep the daemon running for repeated CLI calls so status, record, trace, gate, handoff, and merge queue commands use a hot SQLite connection and watcher-backed dirty path cache. - Separate Trail handles can reuse the daemon's watcher-backed dirty snapshot only while the persisted snapshot is initialized, belongs to the same workspace, and the daemon PID is still alive. If the snapshot is missing, stale, overflowed, or too large, commands fall back to Git dirty paths when a committed Git baseline is available, then to the full persisted-index scan. +- Before high-level `trail agent apply`, reconcile the current Git HEAD with + Trail using `trail git import-update` when `GIT_MAPPING_REQUIRED` is + reported. A mapped apply builds from the trusted HEAD tree and touches only + changed paths. Missing trust is intentionally a fast error; Trail does not + silently scan and hash the full repository to manufacture a mapping. ## Known Limits diff --git a/docs/integrations/acp-agent-usage.md b/docs/integrations/acp-agent-usage.md index 619c460..0e015c8 100644 --- a/docs/integrations/acp-agent-usage.md +++ b/docs/integrations/acp-agent-usage.md @@ -23,19 +23,19 @@ Codex profile and the relay side is provider-neutral. For day-to-day editor use, generate a stable high-level command: ```sh -trail agent setup +trail agent acp setup claude-code --editor vscode ``` The generated editor entry runs: ```sh -trail --workspace /path/to/repo agent acp --provider claude-code +trail --workspace /path/to/repo agent acp run claude-code ``` -Use `--provider codex` for the built-in Codex ACP adapter: +Use `codex` for the built-in Codex ACP adapter: ```sh -trail --workspace /path/to/repo agent acp --provider codex +trail --workspace /path/to/repo agent acp run codex ``` That command creates a fresh lane for the ACP session, launches the real @@ -70,10 +70,10 @@ trail --help Confirm the provider profile: ```sh -trail agent doctor --provider claude-code -trail agent doctor --provider codex -trail agent setup -trail acp list +trail agent doctor claude-code +trail agent doctor codex +trail agent acp setup claude-code --editor vscode +trail agent acp status ``` Provider doctor validates the Trail workspace, the provider profile, the relay @@ -108,14 +108,14 @@ and editing without touching your active branch. Prefer the high-level setup command: ```sh -trail agent setup -trail agent setup --provider claude-code --editor zed -trail agent setup --provider codex --editor zed +trail agent acp setup claude-code --editor vscode +trail agent acp setup claude-code --editor zed +trail agent acp setup codex --editor zed ``` -These snippets use `trail agent acp`, so users do not hard-code or rotate lane -names manually. Use `trail acp install` only when you intentionally want the -lower-level relay command. +These entries use the hidden `trail agent acp run` command, so users do not +hard-code or rotate lane names manually. Use `trail acp relay` only for a custom +ACP host or low-level relay diagnostics. ### Zed @@ -123,7 +123,7 @@ Zed supports ACP External Agents natively. Generate the Trail custom-agent snippet: ```sh -trail agent setup --provider claude-code --editor zed +trail agent acp setup claude-code --editor zed ``` The generated shape is: @@ -316,7 +316,7 @@ After editing, reply with a brief summary and the file path changed. Launch the high-level ACP entrypoint: ```sh -trail --workspace "$PLAYGROUND" agent acp --provider claude-code +trail --workspace "$PLAYGROUND" agent acp run claude-code ``` Then drive the relay over JSON-RPC stdio from your editor or test client. @@ -694,7 +694,7 @@ trail --workspace "$PLAYGROUND" agent undo-last latest --prompt 'Add hook suppor ``` Direct `agent rewind --to ` still works, but -`agent undo-last` avoids copying `ch_...` ids from transcripts. +`agent undo-last` avoids copying `change_...` ids from transcripts. If the task is ready, check apply readiness: @@ -708,7 +708,7 @@ trail --workspace "$PLAYGROUND" agent can-land latest The doctor checks the provider profile and launch command. Run a real ACP prompt through your editor or JSON-RPC driver, then inspect it with -`trail acp sessions`, `trail transcript `, and +`trail agent acp sessions`, `trail transcript `, and `trail lane review `. ### The agent says permission was refused diff --git a/docs/integrations/acp.md b/docs/integrations/acp.md index 1ce5c81..55e7516 100644 --- a/docs/integrations/acp.md +++ b/docs/integrations/acp.md @@ -13,9 +13,9 @@ Trail now treats code-agent integration as three complementary paths: turns/tool events as they stream through the relay. - **MCP server**: context-tool path. Register `trail mcp` in the native agent when the agent supports MCP so it can inspect Trail state directly. -- **Terminal task**: universal CLI path. `trail agent start --provider ` +- **Terminal task**: universal CLI path. `trail agent start ` creates a task lane workdir, runs the provider command there, and records the - final checkpoint when the process exits. Use `--workdir-mode overlay-cow` to + final checkpoint when the process exits. Use `--workdir-mode fuse-cow` to mount a transparent COW view for large repositories instead of creating a full copied workdir. @@ -26,14 +26,14 @@ Trail ships provider profiles for: - `claude-code`, via `@agentclientprotocol/claude-agent-acp` - `codex`, via `@agentclientprotocol/codex-acp` - `cursor`, via the Cursor CLI `agent acp` server -- `gemini`, terminal mode with MCP setup notes +- `gemini`, terminal mode with optional Trail MCP registration - `aider`, terminal mode - `opencode`, terminal mode Trail also reads the [official ACP registry](https://cdn.agentclientprotocol.com/registry/v1/latest/registry.json), so every current registry ID can be launched directly: ```sh -trail acp list +trail agent acp status trail acp relay gemini trail acp relay qwen-code trail acp relay github-copilot-cli @@ -45,7 +45,7 @@ validated registry index locally and uses it if the registry is temporarily unavailable. Built-in aliases remain available without a registry request. An ACP-compatible agent outside the registry can still be used by passing its -upstream command after `--` to `trail agent acp` or `trail acp relay`. +upstream command after `--` to `trail agent acp run` or `trail acp relay`. Any terminal agent can be used by passing its command after `--` to `trail agent start`. @@ -60,17 +60,17 @@ make install Check the agent setup: ```sh -trail agent doctor --provider claude-code -trail agent doctor --provider codex -trail agent doctor --provider cursor -trail agent doctor --provider gemini +trail agent doctor claude-code +trail agent doctor codex +trail agent doctor cursor +trail agent doctor gemini ``` Print editor configuration that creates a fresh Trail task lane for each ACP session: ```sh -trail agent setup +trail agent acp setup claude-code --editor vscode ``` After one prompt: @@ -148,21 +148,21 @@ form: `trail acp relay --provider my-agent -- my-agent acp`. Terminal-first agents use fresh task lanes: ```sh -trail agent start --provider gemini -trail agent start --provider aider -trail agent start --provider opencode -trail agent start --provider custom -- my-agent --flag +trail agent start gemini +trail agent start aider +trail agent start opencode +trail agent start custom -- my-agent --flag ``` -For large repositories, terminal agents can use the overlay COW workdir mode: +For large repositories, terminal agents can use the FUSE COW workdir mode: ```sh -trail agent start --provider codex --workdir-mode overlay-cow -trail agent start --provider custom --workdir-mode overlay-cow -- my-agent --flag -trail agent start --provider codex --workdir-mode nfs-cow +trail agent start codex --workdir-mode fuse-cow +trail agent start custom --workdir-mode fuse-cow -- my-agent --flag +trail agent start codex --workdir-mode nfs-cow ``` -The overlay mount is held only while the terminal process runs and while Trail +The FUSE COW mount is held only while the terminal process runs and while Trail records the checkpoint afterward. On macOS it requires macFUSE; on Linux it requires FUSE access such as `/dev/fuse`. @@ -222,7 +222,7 @@ trail agent apply latest For low-level ACP inspection: ```sh -trail acp sessions +trail agent acp sessions trail transcript trail lane review ``` diff --git a/docs/integrations/mcp.md b/docs/integrations/mcp.md index 9d50996..16d9dba 100644 --- a/docs/integrations/mcp.md +++ b/docs/integrations/mcp.md @@ -61,4 +61,4 @@ known-good state. - Tools: `trail/src/mcp/tools` - Resources: `trail/src/mcp/capabilities/resources.rs` - Prompts: `trail/src/mcp/capabilities/prompts.rs` -- Tests: `mcp_stdio_tools_drive_lane_turn_workflow`, `local_api_and_mcp_drive_merge_queue_and_conflicts` +- Tests: `mcp_stdio_tools_drive_lane_turn_workflow`, `local_api_and_mcp_drive_lane_merge_queue_and_conflicts` diff --git a/docs/integrations/openapi.md b/docs/integrations/openapi.md index 8d43e68..e5c6503 100644 --- a/docs/integrations/openapi.md +++ b/docs/integrations/openapi.md @@ -26,8 +26,8 @@ curl -H "Authorization: Bearer $TOKEN" \ The OpenAPI path builder groups routes as: - Core: health, OpenAPI, doctor, status, record, diff, timeline, why, history, code-from, config, ignore, guardrails. -- Lanes: list/spawn/show/remove, status, review, contribution, gates, readiness, handoff, workdir, diff, read-file, sync-workdir, record, rewind, tests, evals, patches. -- Collaboration: sessions, approvals, leases, claims, anchors, merge queue, conflicts, merge-lane. +- Lanes: list/spawn/show/remove, status, review, contribution, gates, readiness, merge, handoff, workdir, diff, read-file, sync-workdir, record, rewind, tests, evals, patches. +- Collaboration: sessions, approvals, leases, claims, anchors, merge queue, conflicts, and `POST /v1/lanes/{lane}/merge` (used by `trail lane merge`). - Turns and traces: turns, messages, events, spans, runs. ## Code Facts Used diff --git a/docs/lanes/handoff-review-and-merge.md b/docs/lanes/handoff-review-and-merge.md index fff8542..fb821fc 100644 --- a/docs/lanes/handoff-review-and-merge.md +++ b/docs/lanes/handoff-review-and-merge.md @@ -71,9 +71,9 @@ records dirty materialized workdir edits before the rewind when possible. ## Merge ```sh -trail merge-lane doc-bot --into main --dry-run -trail merge-queue add doc-bot --into main -trail merge-queue run +trail lane merge doc-bot --into main --dry-run +trail lane merge-queue add doc-bot --into main +trail lane merge-queue run ``` Use the queue when multiple branches may target the same branch. @@ -84,4 +84,4 @@ Use the queue when multiple branches may target the same branch. - Rewind: `trail/src/db/lane/rewind.rs` - Contribution: `trail/src/db/lane/identity.rs` - Merge queue: `trail/src/db/merge/queue.rs` -- Tests: `merge_queue_runs_lane_branch_into_main`, `merge_lane_and_queue_enforce_readiness_blockers` +- Tests: `lane_merge_queue_runs_lane_branch_into_main`, `merge_lane_and_queue_enforce_readiness_blockers` diff --git a/docs/lanes/overview.md b/docs/lanes/overview.md index 2af3a2c..f9ab5e7 100644 --- a/docs/lanes/overview.md +++ b/docs/lanes/overview.md @@ -50,7 +50,7 @@ cd /path/to/project trail lane record docs-lane -m "record task work" trail lane diff docs-lane --patch trail lane readiness docs-lane -trail merge-lane docs-lane --into main --dry-run +trail lane merge docs-lane --into main --dry-run ``` See [First lane workflow](../getting-started/first-lane-workflow.md) for the @@ -77,7 +77,7 @@ Before merge, inspect contribution, readiness, gates, approvals, and diff. ```sh trail lane contribution doc-bot -trail merge-lane doc-bot --into main --dry-run +trail lane merge doc-bot --into main --dry-run ``` If the branch should be abandoned back to a known-good state, use `lane rewind` diff --git a/docs/lanes/spawn-and-materialize-workdirs.md b/docs/lanes/spawn-and-materialize-workdirs.md index bdb112b..4a9d249 100644 --- a/docs/lanes/spawn-and-materialize-workdirs.md +++ b/docs/lanes/spawn-and-materialize-workdirs.md @@ -1,27 +1,34 @@ # Spawn and Materialize Workdirs Lane branches can stay virtual or be materialized into a filesystem workdir. -New commands and JSON reports expose this as `workdir_mode`: +Commands select `requested_workdir_mode`; JSON reports also expose the resolved +`workdir_mode`, actual `workdir_backend`, and per-file materialization counts: +- `auto`: try strict native COW in staging, then restart as `portable-copy` if + native cloning is unavailable. - `virtual`: no workdir; branch state changes through patches or API calls. - `sparse`: materialize selected paths and hydrate more paths explicitly. -- `full-cow`: materialize the full root, using filesystem clone COW when safe. -- `overlay-cow`: create an empty mountpoint and use a FUSE overlay view at +- `native-cow`: require every file to be an APFS clone or Linux reflink; fail + without copying if any file cannot be cloned. +- `portable-copy`: try a native clone per file and copy bytes when cloning is + unavailable; report `clone`, `mixed`, or `copy` from the actual result. +- `fuse-cow`: create an empty mountpoint and use a FUSE overlay view at runtime; reads come from Trail objects and writes land in a per-lane upper directory. - `nfs-cow`: on macOS, expose the same lower/upper model through a loopback NFSv3 mount without macFUSE. -For `full-cow` and `sparse`, COW means safe file clone during materialization or -hydration. It does not intercept arbitrary writes to unhydrated paths. +For materialized modes, COW means file cloning during materialization. It does +not intercept later writes. Mounted COW modes remain explicit and are never +selected by `auto`. -Overlay COW is different: the visible workdir is mounted while a terminal agent +FUSE COW is different: the visible workdir is mounted while a terminal agent is running. The mountpoint starts empty on disk, the writable upper layer lives -under `.trail/overlay-cow//upper`, and Trail records through the mounted +under `.trail/views//source-upper`, and Trail records through the mounted view before unmounting. On macOS this requires macFUSE; on Linux it requires FUSE access such as `/dev/fuse`. -NFS COW stores writes under `.trail/nfs-cow//upper`, filters macOS +NFS COW stores writes under `.trail/views//source-upper`, filters macOS metadata sidecars, records the upper-layer delta, and unmounts automatically. ## Spawn Without Materialization @@ -37,7 +44,9 @@ The default is controlled by `lane.default_materialize`, and large roots default ```sh trail lane spawn doc-bot --from main --materialize=true -trail lane spawn doc-bot --from main --workdir-mode full-cow +trail lane spawn doc-bot --from main --workdir-mode auto +trail lane spawn doc-bot --from main --workdir-mode native-cow +trail lane spawn doc-bot --from main --workdir-mode portable-copy ``` Use a custom workdir: @@ -48,15 +57,15 @@ trail lane spawn doc-bot --from main --materialize=true --workdir /tmp/doc-bot Custom workdirs must be empty or absent and cannot be symlinks. -## Overlay COW For Terminal Agents +## FUSE COW For Terminal Agents ```sh -trail agent start --provider codex --workdir-mode overlay-cow -trail agent start --provider custom --workdir-mode overlay-cow -- my-agent --flag -trail agent start --provider codex --workdir-mode nfs-cow +trail agent start codex --workdir-mode fuse-cow +trail agent start custom --workdir-mode fuse-cow -- my-agent --flag +trail agent start codex --workdir-mode nfs-cow ``` -`overlay-cow` lets a terminal agent see a normal filesystem tree without first +`fuse-cow` lets a terminal agent see a normal filesystem tree without first copying every file into the lane workdir. Lower files are served from Trail objects. The first write, create, rename, or delete for a path is captured in the lane upper layer and then recorded as the agent checkpoint. @@ -68,11 +77,11 @@ full copy. On macOS with Docker Desktop, verify the Linux path with: ```sh -scripts/verify-linux-overlay-cow-docker.sh +scripts/verify-linux-fuse-cow-docker.sh ``` The script runs a privileged Linux container with `/dev/fuse`, builds Trail, -starts a terminal task with `--workdir-mode overlay-cow`, and asserts that the +starts a terminal task with `--workdir-mode fuse-cow`, and asserts that the agent saw a FUSE filesystem and recorded modified, added, and deleted paths. ## Sparse Materialization diff --git a/docs/lanes/work-model.md b/docs/lanes/work-model.md index 8825492..eae7c86 100644 --- a/docs/lanes/work-model.md +++ b/docs/lanes/work-model.md @@ -113,7 +113,7 @@ lane: docs-lane | +-- message(role=assistant) | +-- event(type=tool_call) | +-- event(type=patch_applied) - | +-- operation(change_id=ch_...) + | +-- operation(change_id=change_...) | +-- turn: turn_002 | @@ -313,7 +313,7 @@ run test/eval gates v review readiness | - +-- ready ---------> merge-lane or merge queue + +-- ready ---------> lane merge or merge queue | +-- not ready -----> continue, approve, resolve conflict, or rewind ``` @@ -349,9 +349,9 @@ trail lane test docs-lane --suite unit -- cargo test trail lane readiness docs-lane # Merge when ready. -trail merge-lane docs-lane --into main --dry-run -trail merge-queue add docs-lane --into main -trail merge-queue run +trail lane merge docs-lane --into main --dry-run +trail lane merge-queue add docs-lane --into main +trail lane merge-queue run ``` ## Readiness Model @@ -453,14 +453,14 @@ trail lane rewind docs-lane \ Merge uses the lane ref as source: ```sh -trail merge-lane docs-lane --into main --dry-run +trail lane merge docs-lane --into main --dry-run ``` Use the merge queue for shared targets: ```sh -trail merge-queue add docs-lane --into main -trail merge-queue run +trail lane merge-queue add docs-lane --into main +trail lane merge-queue run ``` The merge updates Trail's target branch ref. Git history remains separate until @@ -562,7 +562,7 @@ Workdir: DirtyTracked Observed record output: ```text -Recorded lane workdir ch_4ee8... +Recorded lane workdir change_4ee8... Modified README.md ``` @@ -619,15 +619,15 @@ Merge through the queue: ```sh trail session end session-svelte-readme --status completed -trail merge-queue add svelte-readme --into main -trail merge-queue run +trail lane merge-queue add svelte-readme --into main +trail lane merge-queue run ``` Observed merge result: ```text Queued refs/lanes/svelte-readme into refs/branches/main -merged as ch_f348... +merged as change_f348... ``` This demonstrates the intended large-repo pattern: diff --git a/docs/reference/cli/branching-merging-and-conflicts.md b/docs/reference/cli/branching-merging-and-conflicts.md index 2724e2a..c145c13 100644 --- a/docs/reference/cli/branching-merging-and-conflicts.md +++ b/docs/reference/cli/branching-merging-and-conflicts.md @@ -25,30 +25,31 @@ trail merge --into [--strategy ] [--dry-run] Allowed strategies are `conservative`, `line-id-aware`, and `line_id_aware`. -## `merge-lane` +## `lane merge` ```text -trail merge-lane [--into ] [--strategy ] [--dry-run] [--direct] +trail lane merge [--into ] [--strategy ] [--dry-run] [--direct] ``` Default target is `main`. Non-dry-run direct merges into the workspace default -branch are rejected by default so shared targets flow through `merge-queue`. +branch are rejected by default so shared targets flow through +`lane merge-queue`. Use `--dry-run` to preview or `--direct` for an explicit one-off immediate merge. -## `merge-queue` +## `lane merge-queue` ```text -trail merge-queue add --into [--priority ] -trail merge-queue list -trail merge-queue explain -trail merge-queue run [--limit ] -trail merge-queue remove +trail lane merge-queue add --into [--priority ] +trail lane merge-queue list +trail lane merge-queue explain +trail lane merge-queue run [--limit ] +trail lane merge-queue remove ``` Default priority is 0. -`merge-queue explain` resolves a queue id, lane name/ref, or branch name/ref and +`lane merge-queue explain` resolves a queue id, lane id, or lane name and reports readiness blockers, dry-run merge conflicts, preflight errors, warnings, and suggested next steps without mutating refs or recording conflict state. @@ -72,6 +73,6 @@ rejected rather than ignored. ## Code Facts Used -- Args: `trail/src/cli/command/worktree_args.rs`, `trail/src/cli/command/collaboration_args/merge.rs` +- Args: `trail/src/cli/command/worktree_args.rs`, `trail/src/cli/command/lane_args.rs` - Merge logic: `trail/src/db/merge` - Conflict reports: `trail/src/model/reports/merge.rs` diff --git a/docs/reference/cli/global-options-and-env.md b/docs/reference/cli/global-options-and-env.md index 6272acc..e62f8f1 100644 --- a/docs/reference/cli/global-options-and-env.md +++ b/docs/reference/cli/global-options-and-env.md @@ -15,12 +15,13 @@ trail [OPTIONS] | `--workspace ` | Select the workspace root. | | `--db ` | Select the `.trail` database directory directly. | | `--branch ` | Select the default branch for commands that use branch context. | -| `--json` | Render structured JSON output and JSON errors. | -| `--quiet` | Suppress human-oriented success output where renderers support it. | -| `--verbose` | Accepted global flag for verbose mode. | +| `--json` | Compatibility shorthand for `--format json`; renders structured JSON output and JSON errors. | +| `--format ` | `human`, `plain`, `json`, or `ndjson`. Human is adaptive; plain is deterministic ASCII. | +| `--color ` | `auto`, `always`, or `never`. `auto` honors TTY detection, `NO_COLOR`, and `TERM=dumb`. | +| `--pager ` | `auto`, `always`, or `never`. Only eligible long-form human review output can page. | +| `--quiet` | Suppress successful human/plain output; diagnostics remain on stderr. | +| `--verbose` | Reveal full identifiers and secondary technical evidence. | | `--trace` | Accepted global flag for trace mode. | -| `--no-color` | Accepted global flag for color suppression. | -| `--format ` | `human`, `json`, or `ndjson`. | | `--daemon-url ` | Route supported hot commands to a daemon URL. | | `--daemon-token ` | Token for daemon-routed commands. | @@ -31,7 +32,7 @@ trail [OPTIONS] | `TRAIL_WORKSPACE` | Default workspace root. | | `TRAIL_DIR` | Default `.trail` directory. | | `TRAIL_BRANCH` | Default branch. | -| `TRAIL_FORMAT` | `human`, `json`, or `ndjson`; `json` also enables JSON errors. | +| `TRAIL_FORMAT` | `human`, `plain`, `json`, or `ndjson`; structured modes also enable JSON errors. | | `TRAIL_DAEMON_URL` | Default daemon URL. | | `TRAIL_DAEMON_TOKEN` | Default daemon token and daemon startup token source. | @@ -43,7 +44,7 @@ If `--db` is supplied without `--workspace`, the parent directory of the databas ## JSON Errors -Parse errors and runtime errors are rendered as JSON when `--json`, `--format json`, or `TRAIL_FORMAT=json` is used. +Parse errors and runtime errors are rendered as JSON when `--json`, `--format json`, `--format ndjson`, or the corresponding `TRAIL_FORMAT` is used. ```json { @@ -57,7 +58,15 @@ Parse errors and runtime errors are rendered as JSON when `--json`, `--format js ## NDJSON -`--format ndjson` is accepted globally. The current command path that emits newline-delimited JSON is `trail index watch`, which prints one `WorktreeIndexReport` per iteration. +`--format ndjson` is accepted globally. It is a record-stream contract: supported streaming commands (currently `trail index watch` and lane watch commands) emit one report per line. Commands that return a single report use `--format json` instead. + +## Human-output cutoff + +Trail's human output is deliberately not a compatibility API. The former +`--no-color` flag was removed; use `--color never`. Scripts and integrations +must select `--format json` or `--format ndjson`, because human and plain +layouts may evolve for readability. See [the terminal output contract](../../CLI_TERMINAL_OUTPUT.md) +for paging, progress, raw-content exceptions, and examples. ## Code Facts Used @@ -65,4 +74,3 @@ Parse errors and runtime errors are rendered as JSON when `--json`, `--format js - Runtime resolution: `trail/src/cli/command/handler/runtime.rs` - Error rendering: `trail/src/cli/command/handler/errors.rs` - Tests: `cli_json_errors_are_machine_readable`, `cli_env_defaults_select_workspace_db_branch_and_format` - diff --git a/docs/reference/cli/integrations-and-maintenance.md b/docs/reference/cli/integrations-and-maintenance.md index 4f10ef7..dc53db1 100644 --- a/docs/reference/cli/integrations-and-maintenance.md +++ b/docs/reference/cli/integrations-and-maintenance.md @@ -21,41 +21,40 @@ and apply. ### Set up an agent provider ```sh -trail agent doctor --provider claude-code -trail agent setup +trail agent doctor claude-code +trail agent acp setup claude-code --editor vscode ``` -`agent setup` defaults to Claude Code plus VS Code. Use `--provider` for another -agent: +ACP setup requires an explicit provider. Terminal agents do not require setup: ```sh -trail agent setup --provider codex -trail agent setup --provider cursor -trail agent setup --provider gemini --editor generic +trail agent acp setup codex +trail agent acp setup cursor +trail agent start gemini ``` ### Run an agent task from the terminal ```sh -trail agent start --provider claude-code -trail agent start --provider codex -trail agent start --provider cursor -trail agent start --provider gemini -trail agent start --provider aider -trail agent start --provider opencode +trail agent start claude-code +trail agent start codex +trail agent start cursor +trail agent start gemini +trail agent start aider +trail agent start opencode ``` -Use `--workdir-mode overlay-cow` when a large repository should be exposed as a +Use `--workdir-mode fuse-cow` when a large repository should be exposed as a mounted COW filesystem view instead of a full copied workdir: ```sh -trail agent start --provider codex --workdir-mode overlay-cow +trail agent start codex --workdir-mode fuse-cow ``` For an unsupported terminal agent, pass the exact command after `--`: ```sh -trail agent start --provider custom -- my-agent --flag +trail agent start custom -- my-agent --flag ``` ### After the agent runs @@ -78,7 +77,8 @@ validation, apply, or recovery step. | Family | Use it for | | --- | --- | | `agent` | Task-oriented coding-agent workflow | -| `acp` | Low-level ACP relay, install snippets, and captured sessions | +| `agent acp` | ACP editor setup, provider status, diagnostics, and captured sessions | +| `acp` | Low-level ACP relay for custom hosts and diagnostics | | `mcp` | MCP stdio server for agent context tools | | `git` | Export, import, and inspect Git mappings | | `api` | Generate OpenAPI output | @@ -97,7 +97,7 @@ validation, apply, or recovery step. | --- | --- | | Task | One unit of agent work tracked by Trail | | Lane | Isolated Trail branch-like workspace for the task | -| Workdir | Filesystem directory where a terminal agent edits; usually full-cow, optionally overlay-cow | +| Workdir | Filesystem directory where a terminal agent edits; usually native-cow, optionally fuse-cow | | Turn | One prompt or response cycle captured from ACP | | Checkpoint | Recorded code state that can be reviewed, applied, or rewound | | `latest` | The most recent non-archived agent task | @@ -114,9 +114,9 @@ The high-level workflow is: | Provider | ACP | MCP | Terminal default | Notes | | --- | --- | --- | --- | --- | -| `claude-code` | Yes | Yes | `claude` | Default setup provider | +| `claude-code` | Yes | Yes | `claude` | Built-in ACP and terminal provider | | `codex` | Yes | Yes | `codex` | Uses the Codex ACP adapter for ACP mode | -| `cursor` | Yes | Yes | `agent` | Uses `agent acp` for ACP mode | +| `cursor` | Yes | Yes | `agent` | Uses Trail's hidden `agent acp run` entrypoint in ACP mode | | `gemini` | No | Yes | `gemini` | Terminal-first provider | | `aider` | No | No | `aider` | Terminal-first provider | | `opencode` | No | No | `opencode` | Terminal-first provider | @@ -130,14 +130,14 @@ agents direct Trail context tools when they support MCP. | Command | Use it when | | --- | --- | -| `trail agent setup` | Print the default Claude Code + VS Code setup | -| `trail agent setup --provider codex` | Print Codex setup | -| `trail agent setup --provider cursor` | Print Cursor setup | -| `trail agent setup --provider gemini` | Print Gemini setup notes | -| `trail agent doctor --provider ` | Check workspace and provider readiness | +| `trail agent acp setup claude-code --editor vscode` | Print a Claude Code + VS Code ACP entry | +| `trail agent acp setup codex` | Print Codex setup | +| `trail agent acp setup cursor` | Print Cursor setup | +| `trail agent start gemini` | Launch Gemini in an isolated terminal task | +| `trail agent doctor ` | Check workspace and provider readiness | | `trail agent action` | Show runnable setup, review, validation, apply, and recovery actions | -`agent setup` output includes: +`agent acp setup` output includes: - The selected mode: `acp` or `terminal`. - Provider capabilities: ACP, MCP, and terminal. @@ -147,29 +147,33 @@ agents direct Trail context tools when they support MCP. ### Start or continue work ```text -trail agent acp --provider \ +trail agent acp run \ [--name ] [--from ] [--no-mcp] [-- ...] -trail agent start --provider \ - [--name ] [--from ] [--workdir-mode full-cow|overlay-cow|nfs-cow] \ +trail agent start [] \ + [--provider ] \ + [--name ] [--from ] [--workdir-mode auto|native-cow|portable-copy|fuse-cow|nfs-cow|dokan-cow] \ [-- ...] trail agent continue [latest|] \ [--provider ] [--name ] \ - [--workdir-mode full-cow|overlay-cow|nfs-cow] [-- ...] + [--workdir-mode auto|native-cow|portable-copy|fuse-cow|nfs-cow|dokan-cow] [-- ...] ``` -Use `agent acp` as the stable editor entrypoint. It creates a fresh task lane -for each ACP session. +Editors use the hidden `agent acp run` entrypoint generated by ACP setup. It +creates a fresh task lane for each ACP session. Use `agent start` when launching an agent directly from the terminal. It creates a task workdir, runs the agent there, and records a checkpoint when the command -exits. The default `full-cow` mode creates a full materialized workdir using -filesystem clone COW when possible. `overlay-cow` mounts a FUSE view for the +exits. The default `auto` mode first requires native cloning for every file and +restarts portably if native COW is unavailable. Explicit `native-cow` never +copies bytes, while `portable-copy` reports the actual clone/copy mix. +`fuse-cow` mounts a FUSE view for the duration of the run so the agent sees normal files without the initial full copy; it requires macFUSE on macOS or FUSE access on Linux. On macOS, `nfs-cow` provides the same write-time copy-up behavior through the built-in loopback NFS client and requires no kernel extension. +On Windows, `dokan-cow` exposes the same layered semantics through Dokan 2.x. Use `agent continue` after a task has landed or when you want another round of edits from a known checkpoint. `agent follow-up` is an alias. @@ -375,14 +379,14 @@ trail acp relay [AGENT] [--lane ] [--from ] \ [--materialize[=true|false]] [--no-materialize] [--workdir ] \ [--provider ] [--model ] [--no-mcp] [-- ...] -trail acp install --agent \ - [--editor generic|zed] [--dry-run] [--print] +trail agent acp setup [--editor generic|vscode|zed] [--print] [--yes] +trail agent acp setup --provider [--editor generic|vscode|zed] [--print] [--yes] -trail acp doctor --agent \ +trail agent acp doctor \ [--relay-command ...] -trail acp list -trail acp sessions [--lane ] +trail agent acp status +trail agent acp sessions [--lane ] ``` Start a built-in ACP provider with its default upstream adapter: @@ -393,17 +397,17 @@ trail acp relay codex trail acp relay cursor ``` -`AGENT` may also be any ID from the official ACP registry. `trail acp list` +`AGENT` may also be any ID from the official ACP registry. `trail agent acp status` fetches and caches that registry, then shows the current IDs. Registry `npx` and `uvx` entries install through their package runner; matching binary entries download into `.trail/acp/agents/` on first launch. `--materialize` is enabled by default. For an ACP-compatible agent outside the registry, keep the explicit form: `trail acp relay --provider my-agent -- my-agent acp`. -For a terminal-first agent session instead, use `trail agent start --provider codex`. +For a terminal-first agent session instead, use `trail agent start codex`. -Use `acp install` to print setup snippets. It does not mutate editor -configuration. Use `acp sessions` to inspect captured ACP sessions. +Use `agent acp setup` to preview editor configuration. Add `--yes` to apply an +exact supported adapter. Use `agent acp sessions` to inspect captured sessions. ## MCP @@ -414,8 +418,8 @@ trail mcp Starts the MCP stdio server. Register this command in agents that support MCP when you want the agent to ask -Trail for workspace context. `agent setup --provider gemini --editor generic` -prints the same command as part of the Gemini setup notes. +Trail for workspace context. Terminal-only providers can run through +`trail agent start ` without an ACP setup step. ## Git Integration @@ -478,8 +482,8 @@ Runs workspace and integration diagnostics. Use provider-specific diagnostics when the question is about agent setup: ```sh -trail agent doctor --provider claude-code -trail acp doctor --agent claude-code +trail agent doctor claude-code +trail agent acp doctor claude-code ``` ## Backup diff --git a/docs/reference/cli/lanes.md b/docs/reference/cli/lanes.md index b27d71a..953122b 100644 --- a/docs/reference/cli/lanes.md +++ b/docs/reference/cli/lanes.md @@ -31,7 +31,7 @@ workdir. ### Create a lane with a workdir ```sh -trail lane spawn feature-docs --workdir-mode full-cow +trail lane spawn feature-docs --workdir-mode native-cow trail lane workdir feature-docs ``` @@ -100,7 +100,7 @@ The common lifecycle is: ```text trail lane spawn [--from ] \ - [--workdir-mode virtual|sparse|full-cow|overlay-cow|nfs-cow] \ + [--workdir-mode auto|virtual|sparse|native-cow|portable-copy|fuse-cow|nfs-cow|dokan-cow] \ [--materialize[=true|false]] [--no-materialize] \ [--workdir ] [--paths ...] [--include-neighbors] \ [--provider ] [--model ] @@ -115,11 +115,14 @@ trail lane rm [--force] | Mode | What it does | | --- | --- | +| `auto` | Tries strict native COW in staging, then restarts with portable materialization when cloning is unavailable. It never selects a mounted backend. | | `virtual` | Creates no filesystem workdir. This is the high-scale default. | | `sparse` | Materializes only selected paths. | -| `full-cow` | Materializes the full root and tries filesystem clone COW. | -| `overlay-cow` | Creates an empty FUSE mountpoint for a transparent write-time COW view. Reads come from Trail objects; writes land in the lane upper layer. | +| `native-cow` | Requires every file to use a filesystem-native clone/reflink and fails without copying otherwise. | +| `portable-copy` | Opportunistically clones each file, copies bytes when unavailable, and reports `clone`, `mixed`, or `copy`. | +| `fuse-cow` | Creates an empty FUSE mountpoint for a transparent write-time COW view. Reads come from Trail objects; writes land in the lane upper layer. | | `nfs-cow` | On macOS, creates a loopback NFSv3 mount with transparent copy-up and whiteouts, without macFUSE. | +| `dokan-cow` | On Windows, creates a Dokan-backed transparent COW view. | Compatibility flags: @@ -127,14 +130,16 @@ Compatibility flags: - `--no-materialize` keeps the lane virtual. - `--paths ...` implies a sparse materialized workdir. -`overlay-cow` requires FUSE support when mounted by a runtime such as -`trail agent start --workdir-mode overlay-cow`: macFUSE on macOS, or `/dev/fuse` +`fuse-cow` requires FUSE support when mounted by a runtime such as +`trail agent start --workdir-mode fuse-cow`: macFUSE on macOS, or `/dev/fuse` access on Linux. If the mount fails, Trail reports the setup error instead of copying the full workdir. `nfs-cow` is currently macOS-only. The loopback server listens on `127.0.0.1` and the mount is removed automatically after recording the agent checkpoint. +`dokan-cow` is Windows-only and requires Dokan 2.x with its driver service running. + ### Sparse path boundaries `lane spawn --paths ...` creates a sparse workdir scoped to those paths. @@ -305,7 +310,8 @@ Timeline default limit: 30. `lane diff --patch` prints a Git-style unified diff. In an interactive terminal, Trail colorizes patch headers, hunks, additions, and deletions by -default. Pass the global `--no-color` flag, or set `NO_COLOR=1`, for plain text. +default. Use `--color never` or set `NO_COLOR=1` to disable human-mode color; +use `--format plain` for deterministic ASCII logs. `lane rewind` records a `LaneRewind` operation. With `--record-current`, Trail first records dirty materialized workdir edits when possible and preserves the diff --git a/docs/reference/data-types.md b/docs/reference/data-types.md index 2c25e34..d794294 100644 --- a/docs/reference/data-types.md +++ b/docs/reference/data-types.md @@ -151,11 +151,10 @@ This page summarizes public types used across CLI JSON, HTTP API, MCP structured - `AgentApplyReport` - `AgentFinishReport` - `AgentGitApplyPlan` -- `AgentSetupReport` - `AgentRunReport` - `AgentContinueReport` - `AcpProviderProfile` -- `AcpInstallReport` +- `AcpSetupReport` - `AcpDoctorReport` - `AcpDoctorCheck` - `AcpSessionListReport` diff --git a/docs/reference/http-api.md b/docs/reference/http-api.md index 1fdba17..6fa4d8f 100644 --- a/docs/reference/http-api.md +++ b/docs/reference/http-api.md @@ -105,6 +105,7 @@ x-trail-token: | POST | `/v1/lanes/{lane_or_id}/sync-workdir` | Sync workdir. | | POST | `/v1/lanes/{lane_or_id}/record` | Record lane workdir. | | POST | `/v1/lanes/{lane_or_id}/rewind` | Rewind lane branch. | +| POST | `/v1/lanes/{lane}/merge` | Dry-run or explicitly direct-merge this lane into body field `into`. | | POST | `/v1/lanes/{lane_or_id}/tests` | Run test gate. | | POST | `/v1/lanes/{lane_or_id}/evals` | Run eval gate. | | POST | `/v1/lanes/{lane_or_id}/patches` | Apply lane patch. | @@ -117,16 +118,17 @@ resolve the workspace default branch. It reports the target branch/ref, target change, lane base change, `operations_behind`, and whether the lane base is stale. -`POST /v1/lanes` accepts `workdir_mode` values `virtual`, `sparse`, -`full-cow`, `overlay-cow`, and `nfs-cow`. `virtual` creates no workdir, `sparse` requires -`paths`, and `full-cow` creates a full materialized workdir using filesystem -clone COW when available. `overlay-cow` creates an empty workdir mountpoint and -records an overlay backend; a runtime such as `trail agent start ---workdir-mode overlay-cow` mounts the FUSE view and keeps it alive while the -agent runs. The response includes `workdir_mode`, `cow_backend`, `sparse_paths`, -and `overlay_available`. -On macOS, `nfs-cow` reports `cow_backend: "nfs-overlay"` and requires no -macFUSE installation. +`POST /v1/lanes` accepts `workdir_mode` values `auto`, `virtual`, `sparse`, +`native-cow`, `portable-copy`, `fuse-cow`, `nfs-cow`, and `dokan-cow`. +`native-cow` is strict, `portable-copy` permits per-file byte-copy fallback, +and `auto` tries the former before restarting as the latter. `fuse-cow` creates an empty workdir mountpoint and +records the `fuse` backend; a runtime such as `trail agent start +--workdir-mode fuse-cow` mounts the FUSE view and keeps it alive while the +agent runs. The response includes `requested_workdir_mode`, resolved +`workdir_mode`, `workdir_backend`, optional `materialization`, `sparse_paths`, +and `transparent_cow_available`. +On macOS, `nfs-cow` reports `workdir_backend: "nfs"` and requires no +macFUSE installation. On Windows, `dokan-cow` reports `workdir_backend: "dokan"`. `POST /v1/lanes/{lane_or_id}/hydrate` accepts the same body as path-scoped `sync-workdir`, but requires at least one `paths` entry. @@ -144,6 +146,12 @@ entries, oversized changed files, and policy allow/block details. non-committing refresh/rebase preview with operations-behind, conflicts, changed paths, and next steps. +`POST /v1/lanes/{lane_or_id}/merge` requires a JSON `into` target branch and +allows `dry_run=true` preflight. A non-dry-run merge into the workspace default +branch requires `direct=true`; otherwise enqueue with +`POST /v1/lanes/merges/queue` +and run the queue. The former branch-scoped `merge-lane` endpoint was removed. + ## Collaboration Routes | Method | Path | Purpose | @@ -161,15 +169,13 @@ paths, and next steps. | POST | `/v1/lanes/{lane_or_id}/claims` | Claim path. | | GET/POST | `/v1/anchors` | List or create anchors. | | GET/DELETE | `/v1/anchors/{anchor_id}` | Resolve or delete anchor. | -| GET/POST | `/v1/merge-queue` | List or queue merge. | -| POST | `/v1/merge-queue/run` | Run queue. | -| GET | `/v1/merge-queue/explain?selector=` | Explain why a queue item is ready or blocked. | -| GET | `/v1/merge-queue/{selector}/explain` | Explain a queue item by path-safe selector. | -| DELETE | `/v1/merge-queue/{selector}` | Remove queue item. | +| GET/POST | `/v1/lanes/merges/queue` | List or queue a lane merge. | +| POST | `/v1/lanes/merges/queue/run` | Run queued lane merges. | +| GET | `/v1/lanes/merges/queue/{selector}/explain` | Explain a queue item by queue id, lane id, or lane name. | +| DELETE | `/v1/lanes/merges/queue/{selector}` | Remove a lane queue item. | | GET | `/v1/conflicts` | List conflicts. | | GET | `/v1/conflicts/{conflict_set_id}?limit=50` | Show conflict with explanation evidence. | | POST | `/v1/conflicts/{conflict_set_id}/resolve` | Resolve conflict. | -| POST | `/v1/branches/{branch}/merge-lane` | Dry-run or explicitly direct-merge lane branch. | Conflict explanations include the stored `base_root`, `target_root`, and `source_root` snapshots used to reproduce the conflict, plus per-path @@ -181,13 +187,9 @@ signature matches a previously resolved conflict. or `manual`. Manual file values can be plain strings or objects with only `content`, `delete`, and `executable`; unknown keys are rejected. -`POST /v1/branches/{branch}/merge-lane` allows `dry_run=true` preflight. A -non-dry-run merge into the workspace default branch requires `direct=true`; -otherwise enqueue with `POST /v1/merge-queue` and run the queue. - Bodyless mutation routes reject non-empty request bodies. This applies to path-only deletes such as lane removal, lease release, anchor deletion, and -merge-queue removal. +lane merge-queue removal. ## Turn and Trace Routes diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 395a2e4..4691e18 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -289,11 +289,11 @@ before non-dry-run apply. ## Merge and Conflicts -- `trail.merge_queue_add` -- `trail.merge_queue_list` -- `trail.merge_queue_run` -- `trail.merge_queue_explain` -- `trail.merge_queue_remove` +- `trail.lane_merge_queue_add` +- `trail.lane_merge_queue_list` +- `trail.lane_merge_queue_run` +- `trail.lane_merge_queue_explain` +- `trail.lane_merge_queue_remove` - `trail.conflict_list` - `trail.conflict_show` returns conflict details plus deterministic explanation evidence and conservative next steps. - `trail.conflict_resolve` @@ -329,16 +329,17 @@ file values can be plain strings or objects with only `content`, `delete`, and - `trail.sync_workdir` - `trail.read_file` -`trail.lane_spawn` accepts `workdir_mode` values `virtual`, `sparse`, -`full-cow`, `overlay-cow`, and `nfs-cow`. `virtual` creates no workdir, `sparse` requires -`paths`, and `full-cow` creates a full materialized workdir using filesystem -clone COW when available. `overlay-cow` creates an empty workdir mountpoint and -records an overlay backend; a runtime such as `trail agent start ---workdir-mode overlay-cow` mounts the FUSE view and keeps it alive while the -agent runs. Spawn results include `workdir_mode`, `cow_backend`, `sparse_paths`, -and `overlay_available`. -On macOS, `nfs-cow` reports `cow_backend: "nfs-overlay"` and uses the built-in -loopback NFS client. +`trail.lane_spawn` accepts `workdir_mode` values `auto`, `virtual`, `sparse`, +`native-cow`, `portable-copy`, `fuse-cow`, `nfs-cow`, and `dokan-cow`. +`native-cow` is strict, `portable-copy` permits per-file byte-copy fallback, +and `auto` tries the former before restarting as the latter. `fuse-cow` creates an empty workdir mountpoint and +records the `fuse` backend; a runtime such as `trail agent start +--workdir-mode fuse-cow` mounts the FUSE view and keeps it alive while the +agent runs. Spawn results include `requested_workdir_mode`, resolved +`workdir_mode`, `workdir_backend`, optional `materialization`, `sparse_paths`, +and `transparent_cow_available`. +On macOS, `nfs-cow` reports `workdir_backend: "nfs"` and uses the built-in +loopback NFS client. On Windows, `dokan-cow` reports `workdir_backend: "dokan"`. `trail.apply_patch` accepts either native `edits` or compatibility `files`; provide exactly one non-empty array. Native edit objects and compatibility file @@ -355,7 +356,7 @@ workdir path. That directory contains copied recoverable regular files and ## Tool Risk Annotations -The MCP layer annotates tools as read-only, workspace write, destructive write, or open-world write. Read-only tool calls also run under Trail's read-only guard, so they must not persist SQLite database changes or mutate Trail sidecars such as `config.toml`, `HEAD`, ref files, daemon endpoint/token/cache files, or lane workdir metadata manifests while serving the request. The `trail://workspace/status` resource uses the same non-mutating status path. Read-only examples include status, diff, timeline, why, history, agent status/inbox/board/stack/next/guide/dashboard/review-data/review-flow/ask/view/brief/validate/test-plan/report/handoff/story/tools/impact/review-map/risk/confidence/ready/workdir/changes/files/checkpoints/why/turn/compare/diff/review/focus, lane status, review, readiness, handoff, sessions, approvals, runs, leases, anchors, merge queue list, conflict show, event/span queries, and guardrail check. Workspace-write examples include `trail.agent_mark_reviewed`, `trail.agent_mark_file_reviewed`, `trail.agent_archive`, and `trail.agent_unarchive` because they write task metadata markers without deleting provenance. Destructive-write examples include `trail.agent_apply`, `trail.agent_finish`, `trail.agent_undo`, `trail.agent_rewind`, `trail.lane_rewind`, and `trail.merge_queue_run` because they intentionally move lane or shared branch refs and may refresh materialized workdirs. +The MCP layer annotates tools as read-only, workspace write, destructive write, or open-world write. Read-only tool calls also run under Trail's read-only guard, so they must not persist SQLite database changes or mutate Trail sidecars such as `config.toml`, `HEAD`, ref files, daemon endpoint/token/cache files, or lane workdir metadata manifests while serving the request. The `trail://workspace/status` resource uses the same non-mutating status path. Read-only examples include status, diff, timeline, why, history, agent status/inbox/board/stack/next/guide/dashboard/review-data/review-flow/ask/view/brief/validate/test-plan/report/handoff/story/tools/impact/review-map/risk/confidence/ready/workdir/changes/files/checkpoints/why/turn/compare/diff/review/focus, lane status, review, readiness, handoff, sessions, approvals, runs, leases, anchors, lane merge queue list, conflict show, event/span queries, and guardrail check. Workspace-write examples include `trail.agent_mark_reviewed`, `trail.agent_mark_file_reviewed`, `trail.agent_archive`, and `trail.agent_unarchive` because they write task metadata markers without deleting provenance. Destructive-write examples include `trail.agent_apply`, `trail.agent_finish`, `trail.agent_undo`, `trail.agent_rewind`, `trail.lane_rewind`, and `trail.lane_merge_queue_run` because they intentionally move lane or shared branch refs and may refresh materialized workdirs. Open-world write examples are `trail.agent_test`, `trail.agent_eval`, `trail.run_test`, and `trail.run_eval` because they execute commands in lane diff --git a/docs/superpowers/plans/2026-07-12-acp-v1-full-conformance.md b/docs/superpowers/plans/2026-07-12-acp-v1-full-conformance.md new file mode 100644 index 0000000..d8efd1d --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-acp-v1-full-conformance.md @@ -0,0 +1,868 @@ +# ACP v1 Full Conformance Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make Trail a transparent, durable, and exhaustively tested ACP wire-protocol v1 proxy between any conformant stdio client/editor and agent, with complete Trail semantic capture and no known stable-v1 gaps. + +**Architecture:** Keep the relay a transparent byte-preserving proxy. Parse each newline-delimited frame only for classification, negotiation, intentional atomic transformations, and capture. Pin the official ACP v1 schema and method metadata for validation and offline conformance; use `jsonschema` so conformance remains pinned to the raw official artifacts and independent of one peer implementation crate. Move persistence off the forwarding path by appending direction-ordered ACP receipts to Trail's existing durable agent-capture ingress and replaying them asynchronously. + +**Tech Stack:** Rust 2024/MSRV 1.89, `serde`, `serde_json`, `jsonschema` 0.29.1 with default features disabled, `sha2`, stdio JSON-RPC 2.0, SQLite/WAL, Trail's existing agent-receipt and lane-capture models, Cargo unit/integration/E2E tests. + +## Global Constraints + +- The normative source is official ACP commit `64cbd71ae520b89aac54164d8c1d364333c8ee5f`. +- The vendored `schema/v1/schema.json` digest must be `92c1dfcda10dd47e99127500a3763da2b471f9ac61e12b9bf0430c32cf953796`. +- The vendored `schema/v1/meta.json` digest must be `e0bf36f8123b2544b499174197fdc371ec49a1b4572a35114513d56492741599`. +- Preserve a valid input frame byte-for-byte, including property order and insignificant whitespace, unless `transform.rs` performs one of the three approved transformations: `_meta.trail`, Trail MCP injection, or workspace-path mapping. +- Treat editor-to-agent and agent-to-editor request IDs as separate correlation namespaces. +- Enable v1 validation, transformations, and semantic capture only after the upstream `initialize` response selects `protocolVersion: 1`; forward other negotiated versions without claiming conformance. +- Never wait for the Trail database writer lock on a protocol forwarding thread. +- Redact credentials, tokens, environment secrets, and secret-shaped metadata before durable persistence or stderr diagnostics. +- Keep every new production dependency compatible with Rust 1.89 and all five release targets. +- Commit only files named by the active task; preserve unrelated worktree changes. + +--- + +## Task 1: Pin the Official ACP v1 Contract + +**Files:** +- Modify: `Cargo.toml` +- Modify: `trail/Cargo.toml` +- Create: `trail/tests/fixtures/acp/v1/schema.json` +- Create: `trail/tests/fixtures/acp/v1/meta.json` +- Create: `trail/tests/fixtures/acp/v1/source.json` +- Create: `trail/src/acp/schema.rs` +- Modify: `trail/src/acp.rs` + +- [ ] **Step 1: Add the failing artifact-integrity unit test** + +Add this test to `trail/src/acp/schema.rs` before defining its helpers: + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pinned_v1_artifacts_match_manifest_and_compile() { + let contract = AcpV1Contract::load().unwrap(); + assert_eq!(contract.wire_version(), 1); + assert_eq!(contract.method_names().len(), 23); + assert_eq!(contract.schema_sha256(), ACP_V1_SCHEMA_SHA256); + assert_eq!(contract.meta_sha256(), ACP_V1_META_SHA256); + assert!(contract.validator().is_valid(&serde_json::json!({ + "jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": {"protocolVersion": 1, "clientCapabilities": {}} + }))); + } +} +``` + +- [ ] **Step 2: Run the test and verify it fails** + +Run: `cargo test -p trail acp::schema::tests::pinned_v1_artifacts_match_manifest_and_compile -- --exact` + +Expected: FAIL because `acp::schema` and `AcpV1Contract` do not exist. + +- [ ] **Step 3: Add the MSRV-compatible validator dependency** + +Add to `[workspace.dependencies]` in `Cargo.toml`: + +```toml +jsonschema = { version = "0.29.1", default-features = false } +``` + +Add to `[dependencies]` in `trail/Cargo.toml`: + +```toml +jsonschema.workspace = true +``` + +- [ ] **Step 4: Vendor and identify the exact official artifacts** + +Fetch the two files from the pinned upstream revision and verify both digests: + +```bash +mkdir -p trail/tests/fixtures/acp/v1 +curl --fail --location --silent --show-error https://raw.githubusercontent.com/agentclientprotocol/agent-client-protocol/64cbd71ae520b89aac54164d8c1d364333c8ee5f/schema/v1/schema.json --output trail/tests/fixtures/acp/v1/schema.json +curl --fail --location --silent --show-error https://raw.githubusercontent.com/agentclientprotocol/agent-client-protocol/64cbd71ae520b89aac54164d8c1d364333c8ee5f/schema/v1/meta.json --output trail/tests/fixtures/acp/v1/meta.json +shasum -a 256 trail/tests/fixtures/acp/v1/schema.json trail/tests/fixtures/acp/v1/meta.json +``` + +Expected: the output matches the two Global Constraints digests. Create `source.json` with this exact content: + +```json +{ + "repository": "https://github.com/agentclientprotocol/agent-client-protocol", + "commit": "64cbd71ae520b89aac54164d8c1d364333c8ee5f", + "wireVersion": 1, + "schemaSha256": "92c1dfcda10dd47e99127500a3763da2b471f9ac61e12b9bf0430c32cf953796", + "metaSha256": "e0bf36f8123b2544b499174197fdc371ec49a1b4572a35114513d56492741599" +} +``` + +- [ ] **Step 5: Implement the immutable contract loader** + +Define this public-in-crate API in `schema.rs` and register `mod schema;` in `acp.rs`: + +```rust +pub(crate) const ACP_V1_SCHEMA_SHA256: &str = "92c1dfcda10dd47e99127500a3763da2b471f9ac61e12b9bf0430c32cf953796"; +pub(crate) const ACP_V1_META_SHA256: &str = "e0bf36f8123b2544b499174197fdc371ec49a1b4572a35114513d56492741599"; + +pub(crate) struct AcpV1Contract { + schema_sha256: String, + meta_sha256: String, + method_names: std::collections::BTreeSet, + validator: jsonschema::Validator, +} + +impl AcpV1Contract { + pub(crate) fn load() -> crate::Result; + pub(crate) fn wire_version(&self) -> u16 { 1 } + pub(crate) fn schema_sha256(&self) -> &str; + pub(crate) fn meta_sha256(&self) -> &str; + pub(crate) fn method_names(&self) -> &std::collections::BTreeSet; + pub(crate) fn validator(&self) -> &jsonschema::Validator; + pub(crate) fn validate(&self, message: &serde_json::Value) -> crate::Result<()>; +} +``` + +Load fixture bytes with `include_bytes!`, hash the exact bytes, parse `agentMethods`, `clientMethods`, and `protocolMethods`, assert 23 unique values, and compile the schema with `jsonschema::validator_for`. + +- [ ] **Step 6: Run integrity tests and the MSRV check** + +Run: `cargo test -p trail acp::schema::tests::pinned_v1_artifacts_match_manifest_and_compile -- --exact` + +Expected: PASS. + +Run: `cargo +1.89.0 check -p trail --all-targets` + +Expected: PASS, including the new dependency graph. + +If the toolchain is absent, first run `rustup toolchain install 1.89.0 --profile minimal` and repeat the check. + +- [ ] **Step 7: Commit** + +```bash +git add Cargo.toml Cargo.lock trail/Cargo.toml trail/src/acp.rs trail/src/acp/schema.rs trail/tests/fixtures/acp/v1 +git commit -m "test: pin official ACP v1 contract" +``` + +## Task 2: Introduce a Direction-Safe Raw JSON-RPC Model + +**Files:** +- Create: `trail/src/acp/protocol.rs` +- Modify: `trail/src/acp.rs` +- Create: `trail/tests/fixtures/acp/v1/messages.jsonl` + +- [ ] **Step 1: Write failing classification and preservation tests** + +Cover requests, notifications, success responses, error responses, string IDs, integer IDs, null IDs, unknown extension methods, and same-valued IDs in opposite directions. The core assertion is: + +```rust +#[test] +fn frame_preserves_raw_bytes_and_scopes_ids_by_direction() { + let client = Frame::parse(Direction::ClientToAgent, br#" { "id":7, "jsonrpc":"2.0", "method":"session/list", "params":{} }\n"#.to_vec()).unwrap(); + let agent = Frame::parse(Direction::AgentToClient, br#" { "id":7, "jsonrpc":"2.0", "method":"fs/read_text_file", "params":{"sessionId":"s","path":"a"} }\n"#.to_vec()).unwrap(); + assert_eq!(client.forward_bytes(), client.raw_bytes()); + assert_eq!(agent.forward_bytes(), agent.raw_bytes()); + assert_ne!(client.correlation_key(), agent.correlation_key()); +} +``` + +- [ ] **Step 2: Run and verify failure** + +Run: `cargo test -p trail acp::protocol::tests -- --nocapture` + +Expected: FAIL because the protocol model is absent. + +- [ ] **Step 3: Implement the protocol model** + +Use these exact boundary types: + +```rust +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub(crate) enum Direction { ClientToAgent, AgentToClient } + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub(crate) enum RequestId { Null, Number(i64), String(String) } + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub(crate) struct CorrelationKey { pub direction: Direction, pub id: RequestId } + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum EnvelopeKind { Request, Notification, SuccessResponse, ErrorResponse } + +pub(crate) struct Frame { + direction: Direction, + raw: Vec, + parsed: serde_json::Value, + kind: EnvelopeKind, + method: Option, + id: Option, + transformed: Option>, +} + +impl Frame { + pub(crate) fn parse(direction: Direction, raw: Vec) -> std::io::Result; + pub(crate) fn raw_bytes(&self) -> &[u8]; + pub(crate) fn forward_bytes(&self) -> &[u8]; + pub(crate) fn value(&self) -> &serde_json::Value; + pub(crate) fn value_mut_for_transform(&mut self) -> &mut serde_json::Value; + pub(crate) fn commit_transform(&mut self) -> crate::Result<()>; + pub(crate) fn correlation_key(&self) -> Option; +} +``` + +Reject invalid UTF-8, non-object top-level values, wrong/missing `jsonrpc`, fractional IDs, envelopes with both `result` and `error`, and request envelopes without a string `method`. Do not reject unknown methods or unknown fields. + +- [ ] **Step 4: Run focused and regression tests** + +Run: `cargo test -p trail acp::protocol::tests` + +Expected: PASS. + +Run: `cargo test -p trail acp::tests` + +Expected: existing ACP unit tests remain PASS. + +- [ ] **Step 5: Commit** + +```bash +git add trail/src/acp.rs trail/src/acp/protocol.rs trail/tests/fixtures/acp/v1/messages.jsonl +git commit -m "refactor: model ACP JSON-RPC frames" +``` + +## Task 3: Make Transport Byte-Preserving and Concurrent + +**Files:** +- Create: `trail/src/acp/transport.rs` +- Modify: `trail/src/acp.rs` +- Create: `trail/tests/acp_transport.rs` + +- [ ] **Step 1: Add black-box failing transport tests** + +Build a fake client and fake agent using pipes. Assert byte-exact forwarding for unmodified frames, concurrent bidirectional requests with the same ID, out-of-order responses, blank-line behavior, CRLF preservation, stderr isolation, editor EOF, agent EOF, non-zero child exit, and malformed input. Use a 250 ms assertion window to prove forwarding does not wait on capture. + +- [ ] **Step 2: Run and verify the byte-preservation test fails** + +Run: `cargo test -p trail --test acp_transport forwards_unmodified_frames_byte_exactly -- --exact` + +Expected: FAIL because current code parses and reserializes every frame. + +- [ ] **Step 3: Extract the relay transport** + +Define: + +```rust +pub(crate) trait FrameObserver: Send + Sync + 'static { + fn observe(&self, frame: &mut Frame) -> crate::Result<()>; + fn finish(&self, reason: RelayFinishReason); +} + +pub(crate) struct StdioRelay { observer: std::sync::Arc } + +impl StdioRelay { + pub(crate) fn run(self, options: &AcpRelayOptions) -> crate::Result<()>; +} +``` + +Replace `read_json_line`/`write_json_line` with raw `read_until(b'\n', ...)` framing. Parse a copy into `Frame`, let the observer transform it atomically, then write `frame.forward_bytes()` in one direction-local writer loop. Keep upstream stderr copied only to Trail stderr. + +- [ ] **Step 4: Implement bounded shutdown** + +On editor EOF, close child stdin and wait up to two seconds; on agent EOF, stop the editor pump; on timeout, kill and reap the child. Ensure every path calls `FrameObserver::finish` exactly once and joins both pump threads. + +- [ ] **Step 5: Run the transport suite** + +Run: `cargo test -p trail --test acp_transport` + +Expected: PASS with no hangs and no stdout diagnostics. + +- [ ] **Step 6: Commit** + +```bash +git add trail/src/acp.rs trail/src/acp/transport.rs trail/tests/acp_transport.rs +git commit -m "fix: preserve ACP transport semantics" +``` + +## Task 4: Negotiate ACP v1 and Apply Only Atomic Validated Transformations + +**Files:** +- Create: `trail/src/acp/transform.rs` +- Modify: `trail/src/acp.rs` +- Modify: `trail/src/acp/protocol.rs` +- Create: `trail/tests/acp_transform.rs` + +- [ ] **Step 1: Add failing negotiation tests** + +Test forwarded initialize requests offering `1`, offering `[1, 2]` through extension-shaped data, upstream selection of `1`, upstream selection of `2`, initialize error, duplicate initialize, and a session request before negotiation. Assert Trail never rewrites the offered version and activates transformations only after an upstream response selects exactly `1`. + +- [ ] **Step 2: Add failing transformation rollback tests** + +For `_meta.trail`, MCP injection, and path mapping, assert a transformed frame validates against the pinned schema. Force an invalid candidate and assert `forward_bytes()` remains the original raw bytes, with a diagnostic sent to stderr/capture health rather than a partial mutation. + +- [ ] **Step 3: Implement negotiation state** + +```rust +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum NegotiationState { AwaitingInitialize, InitializePending, V1, Other(u16), Failed } + +pub(crate) struct TransformPipeline { + state: NegotiationState, + initialize_id: Option, + contract: std::sync::Arc, + options: TransformOptions, +} + +impl TransformPipeline { + pub(crate) fn apply(&mut self, frame: &mut Frame) -> crate::Result; + pub(crate) fn state(&self) -> NegotiationState; +} +``` + +Correlate the initialize response only in the client-to-agent request namespace. `Other(version)` forwards everything but disables v1 schema validation, intentional transformations, and v1 semantic capture. + +- [ ] **Step 4: Implement atomic candidate validation** + +Clone the parsed `Value`, apply all enabled mutations to the clone, validate the full candidate with `AcpV1Contract`, serialize once with a newline matching the original framing, then commit. Preserve existing `_meta` keys and existing MCP servers; inject one stable Trail server identity only when capabilities allow it. + +- [ ] **Step 5: Run focused tests** + +Run: `cargo test -p trail --test acp_transform` + +Expected: PASS for negotiation, preservation, deduplication, and rollback. + +- [ ] **Step 6: Commit** + +```bash +git add trail/src/acp.rs trail/src/acp/protocol.rs trail/src/acp/transform.rs trail/tests/acp_transform.rs +git commit -m "feat: validate ACP v1 transformations" +``` + +## Task 5: Correct Workspace and Additional-Directory Mapping + +**Files:** +- Modify: `trail/src/acp/transform.rs` +- Modify: `trail/src/model/lane/activity.rs` +- Modify: `trail/src/db/storage/schema/ddl.rs` +- Modify: `trail/src/db/lane/control/acp_sessions.rs` +- Create: `trail/tests/acp_workspace_mapping.rs` + +- [ ] **Step 1: Write failing mapping matrix tests** + +Cover workspace root, nested cwd, normalized `.`/`..`, symlink escape, external cwd, repeated roots, overlapping additional directories, Windows drive-letter paths, and UNC paths. Assert descendants map to the same relative location inside the materialized lane and external roots remain unchanged with `isolated: false` evidence. + +- [ ] **Step 2: Run and verify nested-cwd failure** + +Run: `cargo test -p trail --test acp_workspace_mapping maps_nested_cwd_without_collapsing_to_lane_root -- --exact` + +Expected: FAIL against the current cwd replacement logic. + +- [ ] **Step 3: Implement the mapper boundary** + +```rust +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +pub(crate) struct PathMapping { + pub original: std::path::PathBuf, + pub effective: std::path::PathBuf, + pub isolated: bool, +} + +pub(crate) struct WorkspaceMapper { + workspace_root: std::path::PathBuf, + materialized_root: std::path::PathBuf, +} + +impl WorkspaceMapper { + pub(crate) fn map(&self, path: &std::path::Path) -> crate::Result; + pub(crate) fn map_session_params(&self, params: &mut serde_json::Map) -> crate::Result>; +} +``` + +Canonicalize existing ancestors, reject a relative result that escapes with `..`, retain the exact descendant suffix, deduplicate effective additional roots without reordering the first occurrence, and preserve external paths unchanged. + +- [ ] **Step 4: Persist mappings with ACP sessions** + +Add `path_mappings_json TEXT NOT NULL DEFAULT '[]'` to `lane_acp_sessions`, migrate existing databases idempotently, add `path_mappings: Vec` to `LaneAcpSession`, and round-trip it through create/get/list/update queries. + +- [ ] **Step 5: Run mapping and database tests** + +Run: `cargo test -p trail --test acp_workspace_mapping` + +Run: `cargo test -p trail db::lane::control::acp_sessions` + +Expected: PASS on the host platform; platform-string unit cases pass everywhere. + +- [ ] **Step 6: Commit** + +```bash +git add trail/src/acp/transform.rs trail/src/model/lane/activity.rs trail/src/db/storage/schema/ddl.rs trail/src/db/lane/control/acp_sessions.rs trail/tests/acp_workspace_mapping.rs +git commit -m "fix: preserve ACP workspace path semantics" +``` + +## Task 6: Put Durable Capture Behind a Non-Blocking Ordered Ingress + +**Files:** +- Create: `trail/src/acp/capture.rs` +- Modify: `trail/src/acp.rs` +- Modify: `trail/src/model/agent_capture.rs` +- Modify: `trail/src/db/lane/control/agent_capture.rs` +- Modify: `trail/src/db/storage/schema/agent_capture.rs` +- Create: `trail/tests/acp_capture_journal.rs` + +- [ ] **Step 1: Add failing lock-contention and replay tests** + +Hold Trail's database writer lock, forward 1,000 ACP frames, and assert the final frame reaches the peer within 250 ms. Then release the lock and assert all 1,000 receipts project in strict per-connection sequence. Restart after 500 durable appends but before projection and assert replay produces 1,000 unique ordered events after the remaining frames arrive. + +- [ ] **Step 2: Run and verify current synchronous capture fails** + +Run: `cargo test -p trail --test acp_capture_journal forwarding_never_waits_for_database_writer -- --exact` + +Expected: FAIL because `capture_step` can wait up to 30 seconds. + +- [ ] **Step 3: Extend receipt identity for ACP ordering** + +Add `connection_id TEXT`, `direction TEXT`, and `connection_sequence INTEGER` columns plus a unique index over `(connection_id, direction, connection_sequence)`. Add matching optional fields to `AgentHookReceiptInput` and `AgentHookReceipt`. Retain existing native-hook behavior when they are absent. + +- [ ] **Step 4: Implement the ingress worker** + +```rust +pub(crate) struct CaptureIngress { + tx: std::sync::mpsc::SyncSender, + health: std::sync::Arc, + worker: Option>, +} + +pub(crate) enum CaptureCommand { + Frame(CapturedFrame), + Finish(RelayFinishReason), +} + +pub(crate) struct CapturedFrame { + pub connection_id: String, + pub direction: Direction, + pub sequence: u64, + pub received_at: i64, + pub redacted_message: serde_json::Value, +} + +impl CaptureIngress { + pub(crate) fn append(&self, frame: CapturedFrame) -> crate::Result<()>; + pub(crate) fn shutdown(mut self, timeout: std::time::Duration) -> CaptureShutdownReport; +} +``` + +The forwarding observer performs bounded redaction and enqueue only. The worker persists through `persist_agent_hook_receipt`, assigns deterministic dedupe keys `acp:{connection}:{direction}:{sequence}`, and calls the existing replay pipeline. A full in-memory queue must spill a redacted JSONL record with `sync_data` into `options.db_dir.join("acp-ingress").join(format!("{connection_id}.jsonl"))`; it must never drop a record silently. + +- [ ] **Step 5: Implement recovery and health** + +On startup, replay spill files and pending ACP receipts before accepting new projections. Track `healthy`, `degraded`, `last_error`, `queued`, `spilled`, and `last_projected_sequence`. A permanent append failure marks capture degraded, emits one stderr warning, and preserves the spill file for `trail agent acp doctor`. + +- [ ] **Step 6: Run journal tests** + +Run: `cargo test -p trail --test acp_capture_journal` + +Expected: PASS for contention, crash recovery, idempotency, ordering, bounded shutdown, and queue overflow. + +- [ ] **Step 7: Commit** + +```bash +git add trail/src/acp.rs trail/src/acp/capture.rs trail/src/model/agent_capture.rs trail/src/db/lane/control/agent_capture.rs trail/src/db/storage/schema/agent_capture.rs trail/tests/acp_capture_journal.rs +git commit -m "feat: journal ACP capture off the relay path" +``` + +## Task 7: Capture All Agent-Side Lifecycle and Session Methods + +**Files:** +- Modify: `trail/src/acp/capture.rs` +- Modify: `trail/src/model/agent_capture.rs` +- Modify: `trail/src/db/lane/control/agent_capture.rs` +- Create: `trail/tests/acp_session_semantics.rs` + +- [ ] **Step 1: Add failing lifecycle method tests** + +Cover success and JSON-RPC error outcomes for `initialize`, `authenticate`, `logout`, `session/new`, `session/load`, `session/resume`, `session/close`, `session/list`, and `session/delete`. Assert auth credentials are absent from receipts, pagination tokens remain correlated, and session mappings transition only after successful responses. + +- [ ] **Step 2: Add failing replay reconstruction test** + +Send a `session/load` response followed by its replayed `session/update` notifications containing user, assistant, tool, plan, and usage history. Assert Trail reconstructs ordered transcript turns rather than creating a single synthetic load event. + +- [ ] **Step 3: Implement the pending-request registry** + +```rust +pub(crate) enum PendingOperation { + Initialize { requested_version: serde_json::Value }, + Authenticate { method_id: String }, + Logout, + SessionNew { cwd: PathMapping, additional: Vec }, + SessionLoad { session_id: String, replay: ReplayAccumulator }, + SessionResume { session_id: String }, + SessionClose { session_id: String }, + SessionList { cursor: Option }, + SessionDelete { session_id: String }, + Prompt { session_id: String }, + SetMode { session_id: String, mode_id: String }, + SetConfig { session_id: String, config_id: String, value: serde_json::Value }, + ClientCallback(ClientCallbackOperation), +} +``` + +Store it by `CorrelationKey`, reject duplicate in-flight keys in the same direction as peer violations, and pair responses without removing same-ID operations in the opposite direction. + +- [ ] **Step 4: Project lifecycle semantics** + +Map every request, success, error, and notification to a redacted `AgentLifecycleEvent` with original ACP method, request ID, direction, connection sequence, session ID, outcome, and error code. Reuse `lane_acp_sessions` for durable ACP-to-Trail mapping and the existing agent capture state machine for session/turn transitions. + +- [ ] **Step 5: Run focused tests** + +Run: `cargo test -p trail --test acp_session_semantics` + +Expected: PASS for all nine lifecycle/session methods and replay reconstruction. + +- [ ] **Step 6: Commit** + +```bash +git add trail/src/acp/capture.rs trail/src/model/agent_capture.rs trail/src/db/lane/control/agent_capture.rs trail/tests/acp_session_semantics.rs +git commit -m "feat: capture ACP session lifecycle" +``` + +## Task 8: Capture Prompts, Configuration, Modes, and Both Cancellation Forms + +**Files:** +- Modify: `trail/src/acp/capture.rs` +- Modify: `trail/src/model/agent_capture.rs` +- Create: `trail/tests/acp_turn_semantics.rs` + +- [ ] **Step 1: Add failing method tests** + +Cover `session/prompt`, `session/cancel`, `session/set_mode`, `session/set_config_option`, and `$/cancel_request` from each legal direction. Include cancel-before-response, cancel-after-response, cancel racing prompt completion, and cancellation error code `-32800`. + +- [ ] **Step 2: Add all prompt content fixtures** + +Exercise stable text, image, audio, resource-link, and embedded-resource content blocks with `_meta` and unknown extension keys. Assert the Trail transcript has a lossless redacted structured payload plus the readable text projection. + +- [ ] **Step 3: Implement turn/config state** + +Track one active prompt per ACP session, while allowing prompts in separate sessions concurrently. Persist current mode and every select/boolean config option value after successful responses. Resolve `$/cancel_request.params.requestId` within the sender's request namespace and record the eventual original-request outcome. + +- [ ] **Step 4: Run focused tests** + +Run: `cargo test -p trail --test acp_turn_semantics` + +Expected: PASS for all five methods, all stable content blocks, and cancellation races. + +- [ ] **Step 5: Commit** + +```bash +git add trail/src/acp/capture.rs trail/src/model/agent_capture.rs trail/tests/acp_turn_semantics.rs +git commit -m "feat: capture ACP turns and cancellation" +``` + +## Task 9: Capture Every Session Update Variant + +**Files:** +- Modify: `trail/src/acp/capture.rs` +- Modify: `trail/src/model/agent_capture.rs` +- Create: `trail/tests/fixtures/acp/v1/session_updates.jsonl` +- Create: `trail/tests/acp_update_semantics.rs` + +- [ ] **Step 1: Generate the fixture inventory from the pinned schema** + +Create one schema-valid example for every stable `SessionUpdate` branch: user message chunk, agent message chunk, agent thought chunk, tool call, tool call update, plan, available commands update, current mode update, config option update, and usage update. Include every stable tool-call content/status/kind shape and plan priority/status value. + +- [ ] **Step 2: Add a failing exhaustiveness test** + +```rust +#[test] +fn every_pinned_session_update_variant_has_semantic_projection() { + let cases = fixture_cases("fixtures/acp/v1/session_updates.jsonl"); + assert_eq!(cases.variant_names(), pinned_session_update_variant_names()); + for case in cases { + assert_schema_valid(&case.message); + let events = project(case.message); + assert!(events.iter().any(|event| event.payload["acpVariant"] == case.name)); + } +} +``` + +- [ ] **Step 3: Implement explicit variant projection** + +Use a closed `AcpV1SessionUpdateKind` enum whose `ALL` constant drives the test. Preserve raw redacted structured content on every event, continue the existing human-readable transcript/tool/plan/usage projections, and send unrecognized extension updates to `Extension` without dropping or reordering them. + +- [ ] **Step 4: Run focused tests** + +Run: `cargo test -p trail --test acp_update_semantics` + +Expected: PASS with fixture names exactly matching the stable schema inventory. + +- [ ] **Step 5: Commit** + +```bash +git add trail/src/acp/capture.rs trail/src/model/agent_capture.rs trail/tests/fixtures/acp/v1/session_updates.jsonl trail/tests/acp_update_semantics.rs +git commit -m "feat: capture every ACP session update" +``` + +## Task 10: Capture Permission, Filesystem, and Terminal Callbacks + +**Files:** +- Modify: `trail/src/acp/capture.rs` +- Modify: `trail/src/model/agent_capture.rs` +- Create: `trail/tests/acp_client_callbacks.rs` + +- [ ] **Step 1: Add failing callback method matrix** + +Cover `session/request_permission`, `fs/read_text_file`, `fs/write_text_file`, `terminal/create`, `terminal/output`, `terminal/wait_for_exit`, `terminal/kill`, and `terminal/release`. For each, assert request, success, JSON-RPC error, string/numeric ID, unrelated interleaving, and shutdown-in-flight evidence. + +- [ ] **Step 2: Add callback variant cases** + +Cover all permission option kinds and outcomes; read ranges with optional line/limit fields; write content and errors; terminal environment, cwd, dimensions, output truncation, running/exited state, exit code, signal, kill-before-exit, wait-before-exit, and release-after-exit. + +- [ ] **Step 3: Implement client callback state** + +```rust +pub(crate) enum ClientCallbackOperation { + Permission { session_id: String, tool_call_id: String, options: std::collections::BTreeMap }, + ReadFile { session_id: String, path: String, line: Option, limit: Option }, + WriteFile { session_id: String, path: String, content_sha256: String, byte_len: u64 }, + TerminalCreate { session_id: String, command: Vec, cwd: Option }, + TerminalOutput { session_id: String, terminal_id: String }, + TerminalWait { session_id: String, terminal_id: String }, + TerminalKill { session_id: String, terminal_id: String }, + TerminalRelease { session_id: String, terminal_id: String }, +} +``` + +Persist terminal identity and lifecycle until release. Store written content only through Trail's content-addressed/redacted artifact path; the receipt contains digest and byte length. Record filesystem path mapping identity and terminal output truncation markers. + +- [ ] **Step 4: Run focused tests** + +Run: `cargo test -p trail --test acp_client_callbacks` + +Expected: PASS for all eight methods and all callback variants. + +- [ ] **Step 5: Commit** + +```bash +git add trail/src/acp/capture.rs trail/src/model/agent_capture.rs trail/tests/acp_client_callbacks.rs +git commit -m "feat: capture ACP client callbacks" +``` + +## Task 11: Build the Exhaustive 23-Method Conformance Harness + +**Files:** +- Create: `trail/tests/acp_conformance.rs` +- Create: `trail/tests/support/acp_harness.rs` +- Create: `trail/tests/fixtures/acp/v1/method_cases.json` +- Create: `trail/tests/fixtures/acp/v1/variant_cases.json` +- Modify: `trail/tests/e2e.rs` + +- [ ] **Step 1: Define the exact method inventory fixture** + +The fixture must contain these 23 unique names with side and envelope kind sourced from `meta.json`: `initialize`, `authenticate`, `logout`, `session/new`, `session/load`, `session/resume`, `session/close`, `session/list`, `session/delete`, `session/prompt`, `session/cancel`, `session/set_mode`, `session/set_config_option`, `session/update`, `session/request_permission`, `fs/read_text_file`, `fs/write_text_file`, `terminal/create`, `terminal/output`, `terminal/wait_for_exit`, `terminal/kill`, `terminal/release`, and `$/cancel_request`. + +- [ ] **Step 2: Write a failing inventory equality test** + +Compare fixture method names with the union of all values in pinned `meta.json`; neither set may contain a name absent from the other. + +- [ ] **Step 3: Define and test the stable variant inventory** + +Make `variant_cases.json` name a schema-valid fixture for every branch of these protocol-domain unions: `ContentBlock`, `EmbeddedResourceResource`, `McpServer`, `PermissionOptionKind`, `RequestPermissionOutcome`, `SessionConfigOption`, `SessionUpdate`, `TerminalExitStatus`, `ToolCallContent`, `ToolCallStatus`, `ToolKind`, `PlanEntryPriority`, `PlanEntryStatus`, `Role`, and `StopReason`. Add an inventory test that reads discriminator constants and enum values from the pinned schema and requires exact equality with the fixture manifest. + +Generate and validate every capability combination: all eight `PromptCapabilities` boolean combinations; all four `McpCapabilities` combinations; all eight client filesystem/terminal combinations; omitted, null, and object forms for client session capabilities; and the complete three-state Cartesian product (omitted, null, object) of the five `SessionCapabilities` fields `list`, `delete`, `additionalDirectories`, `resume`, and `close`. For each combination, round-trip initialize through the relay and assert no capability field changes. + +- [ ] **Step 4: Implement the black-box reference harness** + +The harness launches the real `trail acp relay --` binary with the platform-specific command returned by `fixture_agent_command(scenario: &str) -> Vec` and drives both halves over stdio. On Unix that helper writes an executable shell fixture; on Windows it writes a PowerShell fixture. Each method case must assert correct direction, schema-valid success, schema-valid error where a response exists, string/numeric IDs, unknown-field and `_meta` preservation, interleaving, semantic receipt evidence, and deterministic shutdown while in flight. + +- [ ] **Step 5: Run the method and variant matrices** + +Run: `cargo test -p trail --test acp_conformance -- --nocapture` + +Expected: 23 method cases PASS; output names every method once, every stable domain variant once, and every generated capability combination. + +- [ ] **Step 6: Retire overlapping ad hoc E2E helpers** + +Move reusable ACP fake-peer logic from `e2e.rs` into `tests/support/acp_harness.rs`; keep existing user-facing CLI assertions in `e2e.rs`. Do not reduce existing assertions. + +- [ ] **Step 7: Run all ACP integration tests** + +Run: `cargo test -p trail --test acp_conformance --test acp_transport --test acp_transform --test acp_workspace_mapping --test acp_capture_journal --test acp_session_semantics --test acp_turn_semantics --test acp_update_semantics --test acp_client_callbacks` + +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add trail/tests/acp_conformance.rs trail/tests/support/acp_harness.rs trail/tests/fixtures/acp/v1/method_cases.json trail/tests/fixtures/acp/v1/variant_cases.json trail/tests/e2e.rs +git commit -m "test: enforce the ACP v1 method matrix" +``` + +## Task 12: Add Fault, Concurrency, and Interoperability Gates + +**Files:** +- Create: `trail/tests/acp_faults.rs` +- Create: `trail/tests/acp_interop.rs` +- Create: `trail/benches/acp_relay_bench.rs` +- Create: `tools/acp-v1-reference-peer/Cargo.toml` +- Create: `tools/acp-v1-reference-peer/src/main.rs` +- Create: `scripts/check-acp-v1-schema-drift.sh` +- Create: `scripts/test-acp-v1-reference-interop.sh` +- Modify: `trail/Cargo.toml` +- Modify: `.github/workflows/ci.yml` + +- [ ] **Step 1: Add failing fault and concurrency cases** + +Test multiple sessions, 100 concurrent requests per direction, same ID in opposite directions, out-of-order responses, slow reader/writer backpressure, malformed UTF-8, invalid JSON-RPC envelopes, oversized frames at the configured limit and one byte above it, capture worker panic simulation, spill write failure, child crash, editor crash, and cancellation races. + +- [ ] **Step 2: Implement explicit frame and queue limits** + +Add named constants and diagnostics: `ACP_MAX_FRAME_BYTES = 16 * 1024 * 1024`, `ACP_CAPTURE_QUEUE_CAPACITY = 4096`, and `ACP_SHUTDOWN_TIMEOUT = Duration::from_secs(2)`. A frame above the limit terminates that peer direction with an invalid-data error; it must not allocate unbounded memory or emit a JSON-RPC response on behalf of the peer. + +- [ ] **Step 3: Add schema drift enforcement** + +The script downloads the two files from the manifest repository's current stable-v1 revision, prints old/new commit and digests, and exits non-zero on a difference. CI runs the offline digest/conformance tests on every build; the network drift job runs on schedule and manual dispatch so pull requests are not dependent on external availability. + +- [ ] **Step 4: Add the official-type reference peers** + +Create a standalone Cargo package with its own empty `[workspace]` table and exact dependency `agent-client-protocol-schema = "=1.4.0"`. Keep it outside CrabDB's workspace members so the interoperability peer remains independent from Trail's production dependency graph; compile the reference peer with Rust 1.88 or newer in the interoperability job. Its `client` and `agent` modes must serialize and deserialize through official `agent_client_protocol_schema::v1` request, response, notification, content, update, and error types rather than hand-written JSON. + +The script builds the standalone peer into `target/acp-reference`, exports its absolute path as `TRAIL_ACP_REFERENCE_PEER`, and runs `acp_interop`. Exercise official client → Trail → official agent and official client → Trail → fixture agent scenarios. Verify initialize, auth, new/load/resume, prompt streaming, permission, filesystem, terminal, cancellation, close/delete, and clean shutdown. + +- [ ] **Step 5: Add the correctness-preserving relay benchmark** + +Register a harness-free `acp_relay_bench` in `trail/Cargo.toml`. Relay 10,000 mixed request/notification/response frames through in-memory transport with capture enabled, assert byte identity for every untransformed frame and exactly 10,000 unique receipt sequences, then print p50/p95/p99 forwarding latency in microseconds. The benchmark fails on message loss, duplication, reorder, or semantic mismatch; it does not impose a machine-specific absolute latency threshold. + +- [ ] **Step 6: Run fault and interoperability suites** + +Run: `cargo test -p trail --test acp_faults -- --nocapture` + +Run: `scripts/test-acp-v1-reference-interop.sh` + +Expected: PASS without deadlock, dropped capture evidence, or cross-correlation. + +- [ ] **Step 7: Run the benchmark** + +Run: `cargo bench -p trail --bench acp_relay_bench` + +Expected: PASS correctness assertions and print p50/p95/p99. + +- [ ] **Step 8: Run schema drift locally** + +Run: `scripts/check-acp-v1-schema-drift.sh` + +Expected: PASS with both pinned digests unchanged. + +- [ ] **Step 9: Commit** + +```bash +git add trail/tests/acp_faults.rs trail/tests/acp_interop.rs trail/benches/acp_relay_bench.rs tools/acp-v1-reference-peer scripts/check-acp-v1-schema-drift.sh scripts/test-acp-v1-reference-interop.sh trail/Cargo.toml .github/workflows/ci.yml +git commit -m "test: gate ACP v1 faults and interoperability" +``` + +## Task 13: Publish Verifiable Compatibility Evidence + +**Files:** +- Modify: `trail/src/model/lane/activity.rs` +- Modify: `trail/src/cli/command/handler/acp.rs` +- Modify: `trail/src/cli/command/render/acp.rs` +- Modify: `trail/tests/e2e.rs` +- Modify: `README.md` +- Create: `docs/acp-v1-compatibility.md` + +- [ ] **Step 1: Add failing doctor-report assertions** + +Assert JSON and human output include wire version, schema revision/digests, stdio transport, provider readiness, capture journal health, path-mapping health, conformance evidence status, and explicit exclusions for ACP v2 and draft remote transport. A locally modified or unverified build must not print `ACP v1 conformant`. + +- [ ] **Step 2: Extend the report model** + +```rust +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct AcpConformanceEvidence { + pub wire_version: u16, + pub schema_commit: String, + pub schema_sha256: String, + pub meta_sha256: String, + pub transport: String, + pub method_count: u16, + pub evidence_status: String, + pub build_identifier: String, + pub exclusions: Vec, +} +``` + +Add `conformance: AcpConformanceEvidence` and capture/path health checks to `AcpDoctorReport`. Generate `build_identifier` from the package version plus compile-time source revision when available. + +- [ ] **Step 3: Document the precise boundary** + +Document protocol conformance separately from editor setup convenience, agent installation/authentication, capture fidelity, lane isolation, preserved external roots, ACP v2, and draft HTTP transport. Include opt-in smoke commands for Zed and at least two independent ACP agents, with unavailable programs reported as skipped evidence. + +- [ ] **Step 4: Run report and documentation tests** + +Run: `cargo test -p trail --test e2e agent_acp_doctor -- --nocapture` + +Expected: PASS for JSON and human renderers. + +Run: `cargo test -p trail --doc` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add trail/src/model/lane/activity.rs trail/src/cli/command/handler/acp.rs trail/src/cli/command/render/acp.rs trail/tests/e2e.rs README.md docs/acp-v1-compatibility.md +git commit -m "docs: publish ACP v1 compatibility evidence" +``` + +## Task 14: Run the Release Acceptance Gate + +**Files:** +- Modify only files required by failures attributable to this ACP change + +- [ ] **Step 1: Format and inspect the diff** + +Run: `cargo fmt --all -- --check` + +Run: `git diff --check` + +Expected: both PASS. + +- [ ] **Step 2: Run lint and MSRV checks** + +Run: `cargo clippy -p trail --all-targets -- -D warnings` + +Run: `cargo +1.89.0 check -p trail --all-targets` + +Expected: PASS. + +- [ ] **Step 3: Run the complete Trail suite** + +Run: `cargo test -p trail --all-targets` + +Expected: PASS, including all ACP unit, integration, E2E, fault, and interoperability tests. + +- [ ] **Step 4: Run the offline conformance gate explicitly** + +Run: `cargo test -p trail --test acp_conformance --test acp_faults -- --nocapture` + +Expected: all 23 methods, every stable union/capability fixture, concurrency cases, and fault cases PASS. + +Run: `scripts/test-acp-v1-reference-interop.sh` + +Expected: PASS against the peer compiled from official ACP v1 Rust types. + +Run: `cargo bench -p trail --bench acp_relay_bench` + +Expected: byte/capture correctness PASS and p50/p95/p99 printed. + +- [ ] **Step 5: Verify the compatibility claim from a clean build** + +Build the release binary, run `target/release/trail agent acp doctor codex --json`, and confirm the report contains the exact pinned revision/digests and `evidence_status: "verified"` only when the conformance build identifier matches. Codex availability is a separate provider-readiness check. Run configured real-agent smoke tests; absent external agents remain skips and do not fail or satisfy the conformance gate. + +- [ ] **Step 6: Review acceptance evidence against the design** + +Check every item in `docs/superpowers/specs/2026-07-12-acp-v1-full-conformance-design.md` under “Acceptance Gates.” Record the command outputs and platform CI links in the final change/PR description. Do not declare completion if any gate is missing, skipped when required, or flaky. + +- [ ] **Step 7: Commit verification-only corrections if present** + +Stage each file corrected during verification by its explicit path, inspect `git diff --cached`, then run `git commit -m "fix: satisfy ACP v1 release gates"`. If verification required no code or documentation correction, do not create an empty commit. diff --git a/docs/superpowers/plans/2026-07-12-agent-acp-hooks-hard-cutover.md b/docs/superpowers/plans/2026-07-12-agent-acp-hooks-hard-cutover.md new file mode 100644 index 0000000..18f5bc2 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-agent-acp-hooks-hard-cutover.md @@ -0,0 +1,257 @@ +# Agent ACP and hooks hard-cutover implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [x]`) syntax for tracking. + +**Goal:** Replace Trail's overlapping agent setup commands with explicit ACP and native-hook setup groups, positional providers, and a hard cutover. + +**Architecture:** Keep `trail acp relay` as the low-level relay. Move editor-facing setup, diagnostics, sessions, and the hidden task runner below `trail agent acp`. Rename native hook installation to `trail agent hooks setup`. Resolve positional and named providers through one helper, while terminal startup alone may read `agent.default_provider`. + +**Tech stack:** Rust, clap, serde, serde_json, TOML workspace configuration, existing Trail ACP and native-hook modules, cargo test. + +## Global constraints + +- Remove old commands without aliases, migration messages, or compatibility shims +- Prefer positional providers and accept `--provider` as an equivalent named form +- Reject input that supplies both provider forms +- Keep ACP and hook configuration ownership isolated +- Follow red-green-refactor for every behavior change +- Preserve unrelated untracked workspace content + +--- + +### Task 1: New CLI command contract + +**Files:** +- Modify: `trail/src/cli/command.rs` +- Modify: `trail/src/cli/command/acp_args.rs` +- Modify: `trail/src/cli/command/agent_args.rs` +- Test: `trail/src/cli/command.rs` + +**Interfaces:** +- Produces: `AgentAcpCommand`, `AgentAcpSubcommand`, `AgentAcpSetupArgs`, `AgentAcpRunArgs` +- Produces: `resolve_provider_argument(positional, named, fallback)` in the agent handler +- Removes: `AgentSubcommand::Setup`, `AcpSubcommand::Install`, `AgentHooksSubcommand::Add` + +- [x] **Step 1: Write failing parser tests** + +```rust +#[test] +fn parses_new_agent_provider_forms() { + Cli::try_parse_from(["trail", "agent", "start", "codex"]).unwrap(); + Cli::try_parse_from([ + "trail", "agent", "acp", "setup", "codex", "--editor", "zed", + ]) + .unwrap(); + Cli::try_parse_from(["trail", "agent", "hooks", "setup", "codex"]).unwrap(); +} + +#[test] +fn rejects_removed_setup_commands() { + assert!(Cli::try_parse_from(["trail", "agent", "setup"]).is_err()); + assert!(Cli::try_parse_from(["trail", "acp", "install"]).is_err()); + assert!(Cli::try_parse_from(["trail", "agent", "hooks", "add", "codex"]).is_err()); +} +``` + +- [x] **Step 2: Run parser tests and verify RED** + +Run: `cargo test -p trail cli::command::tests::parses_new_agent_provider_forms -- --exact` + +Expected: FAIL because `agent acp setup` and `hooks setup` do not exist. + +- [x] **Step 3: Implement the command enums and argument structs** + +```rust +pub(super) enum AgentAcpSubcommand { + Setup(AgentAcpSetupArgs), + #[command(hide = true)] + Run(AgentAcpRunArgs), + Status(AgentAcpStatusArgs), + Doctor(AgentAcpDoctorArgs), + Sessions(AcpSessionsArgs), +} +``` + +- [x] **Step 4: Run parser tests and verify GREEN** + +Run: `cargo test -p trail cli::command::tests -- --nocapture` + +Expected: PASS. + +### Task 2: Provider defaults and terminal startup + +**Files:** +- Modify: `trail/src/model/domain/config.rs` +- Modify: `trail/src/db/util/config/entries.rs` +- Modify: `trail/src/db/util/config/set.rs` +- Modify: `trail/src/cli/command/handler/agent.rs` +- Test: `trail/src/db/util/config.rs` +- Test: `trail/tests/e2e.rs` + +**Interfaces:** +- Produces: `AgentConfig { default_provider: Option }` +- Produces: config key `agent.default_provider` +- Produces: terminal resolution order positional, named, configured default + +- [x] **Step 1: Write failing config and terminal tests** + +```rust +#[test] +fn agent_default_provider_round_trips() { + let mut config = test_config(); + set_config_value(&mut config, "agent.default_provider", "codex").unwrap(); + assert_eq!(config.agent.default_provider.as_deref(), Some("codex")); +} +``` + +- [x] **Step 2: Run focused tests and verify RED** + +Run: `cargo test -p trail agent_default_provider_round_trips -- --nocapture` + +Expected: FAIL because `TrailConfig` has no agent section. + +- [x] **Step 3: Add the configuration section and terminal resolver** + +```rust +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct AgentConfig { + #[serde(default)] + pub default_provider: Option, +} +``` + +- [x] **Step 4: Run focused tests and verify GREEN** + +Run: `cargo test -p trail agent_default_provider -- --nocapture` + +Expected: PASS. + +### Task 3: ACP setup, status, doctor, sessions, and hidden run + +**Files:** +- Create: `trail/src/acp/setup.rs` +- Modify: `trail/src/acp.rs` +- Modify: `trail/src/cli/command/handler/acp.rs` +- Modify: `trail/src/cli/command/handler/agent.rs` +- Modify: `trail/src/cli/command/render/acp.rs` +- Modify: `trail/src/model/lane/activity.rs` +- Test: `trail/tests/e2e.rs` + +**Interfaces:** +- Produces: `build_acp_setup_plan(workspace, db_dir, provider, editor)` +- Produces: `apply_acp_setup_plan(plan, apply)` +- Produces: generated editor command `trail --workspace agent acp run ` + +- [x] **Step 1: Write failing ACP setup tests** + +```rust +let plan = run_trail_json( + temp.path(), + &["agent", "acp", "setup", "codex", "--editor", "zed", "--print"], +); +assert_eq!(plan["transport"], "acp"); +assert!(plan["command"].as_array().unwrap().iter().any(|v| v == "run")); +``` + +- [x] **Step 2: Run ACP setup test and verify RED** + +Run: `cargo test -p trail agent_acp_setup_uses_hidden_run -- --nocapture` + +Expected: FAIL because the ACP setup group is not handled. + +- [x] **Step 3: Implement plan generation, exact Zed merge, and hidden run** + +```rust +pub struct AcpSetupReport { + pub transport: String, + pub provider: String, + pub editor: String, + pub command: Vec, + pub snippet: String, + pub applied: bool, +} +``` + +- [x] **Step 4: Run ACP tests and verify GREEN** + +Run: `cargo test -p trail acp -- --nocapture` + +Expected: PASS. + +### Task 4: Native hooks setup hard cutover + +**Files:** +- Modify: `trail/src/cli/command/agent_args.rs` +- Modify: `trail/src/cli/command/handler/agent.rs` +- Test: `trail/tests/e2e.rs` + +**Interfaces:** +- Produces: `trail agent hooks setup [--provider ]` +- Preserves: existing hook install planning, atomic apply, ownership, doctor, receipts, and removal + +- [x] **Step 1: Write failing hooks setup tests** + +```rust +let plan = run_trail_json( + temp.path(), + &["agent", "hooks", "setup", "codex", "--print"], +); +assert_eq!(plan["provider"], "codex"); +assert_eq!(plan["dry_run"], true); +``` + +- [x] **Step 2: Run hooks setup test and verify RED** + +Run: `cargo test -p trail agent_hooks_setup -- --nocapture` + +Expected: FAIL because only `hooks add` exists. + +- [x] **Step 3: Route setup through existing install planning** + +```rust +let apply = args.yes; +let report = apply_agent_hook_install_plan(&plan, !apply)?; +``` + +- [x] **Step 4: Run native hook tests and verify GREEN** + +Run: `cargo test -p trail agent_hooks -- --nocapture` + +Expected: PASS. + +### Task 5: Hard-cutover regression and documentation cleanup + +**Files:** +- Modify: `trail/tests/e2e.rs` +- Modify: `docs/agent/*.md` +- Modify: `docs/integrations/*.md` +- Modify: `docs/reference/cli/integrations-and-maintenance.md` +- Modify: `skills/use-trail/references/*.md` + +**Interfaces:** +- Removes all public references to old setup syntax +- Preserves `trail acp relay` as the low-level custom integration surface + +- [x] **Step 1: Add help and removed-command assertions** + +```rust +assert_command_rejected(&["agent", "setup"]); +assert_command_rejected(&["acp", "install"]); +assert_command_rejected(&["agent", "hooks", "add", "codex"]); +``` + +- [x] **Step 2: Run regression tests and verify RED where old expectations remain** + +Run: `cargo test -p trail --test e2e agent -- --nocapture` + +Expected: FAIL at old command expectations. + +- [x] **Step 3: Remove old tests and update current documentation** + +Replace examples with `agent acp setup`, `agent hooks setup`, positional providers, and hidden `agent acp run` where editor configuration requires it. + +- [x] **Step 4: Run full verification** + +Run: `cargo fmt --check && cargo test -p trail && cargo test -p trail` + +Expected: PASS with zero failures. diff --git a/docs/superpowers/plans/2026-07-12-correctness-first-changed-path-ledger.md b/docs/superpowers/plans/2026-07-12-correctness-first-changed-path-ledger.md new file mode 100644 index 0000000..0f2a44a --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-correctness-first-changed-path-ledger.md @@ -0,0 +1,168 @@ +# Changed-Path Ledger Implementation Plan + +Status: superseded by `docs/superpowers/plans/2026-07-13-activated-changed-path-ledger.md` + +The replacement plan implements the approved schema-v18 hard cutover, qualified +Linux and macOS observers, automatic reconciliation, secure daemon startup, and +one atomic activation gate. This earlier dormant/migration-oriented plan must +not be executed. + +Design: `docs/superpowers/specs/2026-07-12-correctness-first-changed-path-ledger-design.md` + +Every task is test-first, independently reviewed, and committed separately. +The ledger remains dormant/feature-disabled through Tasks 2–6. No scope may +enter `trusted`, and no public read path may consume ledger clean proof, until +recovery, policy dependencies, every controlled producer, and a qualified +external adapter are complete. + +## Task 1: Structural metrics and true selected-path locality + +- Add operation-scoped counters for filesystem walks, root ranges, SQLite rows, + input candidates versus final changes, hashes, point lookups, manifest/log + bytes, journal/upper work, Git global work, wall time, and peak RSS. +- Replace `prune_worktree_index_for_selections` global enumeration with indexed + exact/binary-prefix queries. +- Replace repeated selected-map filtering and other `O(k^2)` loops with bounded + unions/batches; batch Prolly reads and index updates. +- Gate existing daemon/Git selected status, dirty diff, and record at 1k with + zero hidden global rows. This task changes locality, not trust semantics. + +## Task 2: Dormant next-schema migration and deep ledger core + +- Add scope, exact-path, prefix, intent path/prefix, reconciliation staging, + and observer segment/lease tables with required indexes/foreign keys. +- Bump the implementation-time schema by one (v16→v17 or v17→v18 if the + path-index recovery migration lands first). Verify exact new-version tables, + columns, constraints, indexes, schema metadata, and `PRAGMA user_version` + even on the startup fast path. +- Implement typed scope/filesystem identity, capabilities, trust states, + source-sequenced evidence, coalescing, intent lifecycle, CAS, and corruption + handling behind a disabled feature gate. +- New/migrated scopes start `legacy_reconcile_required`; `begin_scope` cannot + produce trust. +- Add schema/log-format version, partial migration, stale owner, root/policy/ + filesystem replacement, and concurrent CAS tests. + +## Task 3: Reconciliation, diagnostics, recovery, backup, and GC + +- Implement crash-safe streamed reconciliation staging with full baseline-root + deletion comparison, pinned no-follow reads, full hashing after gaps, and + before/after identity checks. +- Publish only under ref/root/scope/policy/filesystem/provider-fence CAS. +- Permit prefix promotion only for provider-proven complete invalidation; + global gaps always require full scope reconciliation. +- Add stable `CHANGE_LEDGER_RECONCILE_REQUIRED` across human/JSON CLI, daemon + RPC, REST/OpenAPI, and MCP, plus `trail index reconcile` reports. +- Run intent recovery before mutation/GC. Make prepared/published targets GC + roots; specify lane/view retirement, backup fencing/untrusted snapshot, and + restore epoch rotation/untrusted state. +- Keep the ledger disabled after this task. + +## Task 4: Trail policy snapshot and dependency tracking + +- Build Trail's authoritative compiled policy for its walkers/candidate checks. +- Persist the dependency manifest for nested ignore files, Git info/global/ + included config, built-in versions, normalization, modes, and case policy. +- Make adapters declare equivalent versus conservative semantics; unresolved or + unobservable dependencies stale the scope. +- Process raw policy-file events before filtering and avoid full dependency + rediscovery on each fast request. +- Keep all scopes untrusted/disabled. + +## Task 5: Controlled mutation intents and atomic publication + +- Inventory every filesystem producer and assign it to one reviewed class: + observed record/checkpoint, ref-advancing controlled projection, + projection-only alignment, or VFS/COW. +- For ref-advancing projection: prebuild immutable target/operation, + durable-prepare exact/prefix intent, mutate+sync+verify+fence, then atomically + publish operation/ref/ledger. +- For projection-only alignment (checkout materialization, sparse hydration, + ordinary sync): reference the existing target, mutate+sync+verify, then + fence/flush the qualified provider, retain unrelated/later evidence, and + publish only scope baseline/marker and terminal intent under unchanged + ref/epoch/provider-cut CAS—do not fabricate an operation or ref advance. +- Acknowledge only intent-owned/source-sequenced evidence; retain concurrent + same-path watcher events. +- Reconcile ref/manifest mirrors from authoritative SQL and exhaust crash points + around object, intent fsync, filesystem, verification, ref, ledger, and + mirror transitions. +- Install recovery before allowing another mutation. Ledger remains disabled as + read authority until every controlled producer is covered. + +## Task 6: Qualified observer log and Git adapters + +- Implement versioned segments with exclusive epoch/owner lease, monotonic + sequence, checksum/hash chain, durable/folded offsets, fsync/group-commit, + rotation, bounded tail, disk-full/error revocation, and read-only safe-tail + semantics. +- Define/test provider capability matrix and real delivery fence/cursor + contracts. Generic callbacks never qualify clean proof without quiescence. +- Cover delayed delivery, backlog, cursor discontinuity, owner death/restart, + lease replacement, append failure, torn tail, rotation crash, and power-loss + claims. +- Implement Git porcelain-v2 adapter only when ledger baseline exactly equals + mapped HEAD/index tree and sparse/submodule/mode/policy/fsmonitor semantics + qualify; otherwise advisory only. Measure Trace2/index work. +- Ledger remains disabled until Task 7 activation audit. + +## Task 7: Activate main workspace trusted reads + +- Run an activation audit proving Tasks 2–6 cover every workspace filesystem + producer and recovery path; feature gate defaults off until this passes. +- Integrate trusted snapshots into status, read-only status, dirty diff, manual + record, and native-agent checkpoints. +- Implement the observed record/checkpoint state machine for manual record and + native-agent checkpoint: fence c1, read/hash, fence c2, build target, then + atomically publish ref/ledger while acknowledging only c1-covered evidence. +- Snapshot consumers fold/merge durable tails or return reconcile-required; + unread evidence can never be ignored. +- Preserve compatibility: interactive status may perform labeled automatic + reconciliation; unapproved expensive mutations fail with recovery guidance. +- Gate warm `k=0,1,100`, cold start, owner restart, and gap behavior separately. + +## Task 8: Materialized lane v2 marker and deployment contract + +- Create per-lane scopes owned by qualified long-lived observers. +- Publish compact v2 marker and trusted SQL scope identity at one logical cut + only after explicit reconciliation; legacy manifests cannot prove candidates. +- CLI-only/no-qualified-observer lanes become `untrusted_gap` on process exit. +- Make lane status, record, readiness, merge checks, structured-patch workdir + maintenance, and ordinary preview candidate-only only in trusted scopes. +- Keep full ignored/risk traversal as explicit audit/reconciliation. + +## Task 9: COW journal correctness, then compaction + +- VFS callbacks use shared view barrier plus per-view fsynced intent log only; + never workspace lock/primary SQLite. Checkpoint uses workspace lock then + exclusive barrier, folds/verifies, and publishes. +- First prove journal/whiteout atomicity, group commit, active-handle generation + ownership, recovery, and crash behavior. +- Then rotate/compact at checkpoint, roll generations when handles permit, and + replace whole-array whiteouts with an incremental map/log. +- Assert zero upper scan only for qualified trusted journals; gaps explicitly + reconcile. + +## Task 10: Cross-platform scale, fault, and production gates + +- CI 1k; scheduled 100k/1M on actual Linux/macOS/Windows filesystem/provider + tiers. Publish capability matrix with unsupported/network filesystems failing + closed. +- Exercise input candidates `k=0,1,100` separately from final changed output + across workspace status/diff/record, materialized lanes, structured patch, + and COW checkpoint. +- Cover create/content/mode/delete, file/directory/case rename, reverted events, + nested/global policy, delayed delivery/backlog, overflow, disk full, daemon + restart, epoch replacement, concurrent same-path writes, migration, backup/ + restore/GC, and kill points at every fsync/publication boundary. +- Warm trusted gates require zero full walks/ranges/global-index rows, zero + legacy manifest I/O, zero upper recovery, and reads/rows proportional to + authoritative candidates plus unavoidable prefix output. +- Warm gates also require zero external-adapter global work and policy full + discovery, bounded folded tail/log bytes/rows, and calibrated wall-time/RSS + budgets. Persisted caps fail closed when exceeded. +- Cold/gapped O(N) work must be labeled reconciliation. Run a full-scan oracle + after measured fast paths and property-test event permutations for no false + clean. +- Run fmt/check/full tests/docs/OpenAPI/backup plus independent architecture, + security, crash-consistency, and performance review before default enabling. diff --git a/docs/superpowers/plans/2026-07-12-fuse-cow-hard-cutover.md b/docs/superpowers/plans/2026-07-12-fuse-cow-hard-cutover.md new file mode 100644 index 0000000..ea5a9c5 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-fuse-cow-hard-cutover.md @@ -0,0 +1,115 @@ +# COW Workdir Mode Hard-Cutover Implementation Plan + +> **For agentic workers:** Execute inline. The user explicitly waived test-first TDD; +> update tests with each implementation slice and run them as verification gates. + +**Goal:** Rename filesystem materialization to `native-cow`, remove `overlay-cow`, and +expose distinct `fuse-cow` and `dokan-cow` workdir modes across Trail. + +**Architecture:** `LaneWorkdirMode` names the concrete transport. FUSE, NFS, and Dokan +share `ViewCore` semantics but have separate dispatch and user-visible identities. No +old spelling, parser alias, stored-state adoption, or migration remains. + +**Tech Stack:** Rust, Clap, Serde, SQLite lane metadata, FUSE/fuser, Dokan, shell and +PowerShell verification scripts, Markdown documentation. + +## Global constraints + +- Preserve `native-cow` as the portable first-class materialized mode. +- Do not accept removed `full-cow` or `overlay-cow` spellings and symbols anywhere. +- Do not modify unrelated untracked projects or the parent checkout. +- Keep platform dispatch explicit: FUSE on Linux/macOS, Dokan on Windows, NFS on macOS. +- Run formatting, focused tests, the complete Trail suite, and cross-target checks. + +--- + +### Task 1: Workdir-mode model and public contracts + +**Files:** + +- Modify: `trail/src/model/reports/lane.rs` +- Modify: `trail/src/db/lane/lifecycle.rs` +- Modify: `trail/src/cli/command/lane_args.rs` +- Modify: `trail/src/cli/command/agent_args.rs` +- Modify: `trail/src/server/openapi/schemas/lane.rs` +- Modify: `trail/src/mcp/tools/lane.rs` +- Modify: affected parser/report tests in `trail/src/cli/command.rs` and + `trail/tests/e2e.rs` + +Steps: + +- [x] Add `NativeCow`, `FuseCow`, and `DokanCow`; remove the old enum variants. +- [x] Serialize/parse only the new names and return exact backend identifiers. +- [x] Make automatic Windows selection `DokanCow` and Linux FUSE selection `FuseCow`. +- [x] Validate platform availability before lane creation. +- [x] Update CLI/OpenAPI/MCP enums and parser/report assertions. +- [x] Run model, CLI, and lane-spawn focused tests. + +### Task 2: Separate FUSE and Dokan backend dispatch + +**Files:** + +- Rename: `trail/src/db/lane/workdir/overlay.rs` to + `trail/src/db/lane/workdir/fuse.rs` +- Rename: `trail/src/db/lane/workdir/overlay/dokan_overlay.rs` to + `trail/src/db/lane/workdir/dokan.rs` +- Modify: `trail/src/db/lane/workdir.rs` +- Modify: lane lifecycle, view, gate, record, merge, agent, environment, and adapter + call sites under `trail/src/db` and `trail/src/cli` + +Steps: + +- [x] Rename FUSE types/functions to `FuseCow*`/`fuse_cow_*`. +- [x] Rename Dokan types/functions to `DokanCow*`/`dokan_cow_*`. +- [x] Add backend-specific Trail mount/prepare/candidate helpers. +- [x] Dispatch `FuseCow`, `NfsCow`, and `DokanCow` explicitly at every mount site. +- [x] Rename mount FS names, subtypes, diagnostics, and ownership identifiers. +- [x] Run library tests and Windows cross-checks for backend dispatch. + +### Task 3: Metadata hard cutover and lifecycle behavior + +**Files:** + +- Modify: `trail/src/db/lane/lifecycle.rs` +- Modify: `trail/src/db/lane/workdir/lifecycle.rs` +- Modify: `trail/src/db/core/doctor_storage.rs` +- Test: `trail/tests/e2e.rs` + +Steps: + +- [x] Persist only current mode names and exact `cow_backend` values. +- [x] Make old stored metadata fail with the recreate-lane diagnostic. +- [x] Clean only current backend state paths; never adopt `.trail/overlay-cow`. +- [x] Update doctor/backend availability checks. +- [x] Add hard-cutover metadata and lifecycle assertions. + +### Task 4: Scripts, workflows, docs, and skills + +**Files:** + +- Rename FUSE scripts containing `overlay-cow` to `fuse-cow`. +- Modify: `.github/workflows/layered-workspaces.yml` +- Modify: remaining scripts and PowerShell fixtures. +- Modify: current docs under `docs/`, `plans/`, and `skills/use-trail/`. + +Steps: + +- [x] Rename flags, variables, volumes, files, headings, examples, and diagnostics. +- [x] Use `dokan-cow` in Windows fixtures and `fuse-cow` in FUSE fixtures. +- [x] Preserve generic “overlay semantics” only where it describes the algorithm. +- [x] Verify checked-in current sources contain no removed product spelling or symbol + outside explicit hard-cutover rejection assertions. + +### Task 5: Verification and regression repair + +Steps: + +- [x] Run `cargo fmt --all -- --check`. +- [x] Run focused parser, CLI, lane, environment, FUSE, NFS, and Dokan checks. +- [x] Run `cargo test -p trail` and repair every failure caused by or exposed during the + cutover. +- [x] Run `cargo check -p trail --target x86_64-pc-windows-msvc` when the target is + installed; otherwise record the missing external gate. +- [x] Run repository absence scans excluding `.git`, `target`, and the historical design + spec/implementation plan whose purpose is to describe the removed name. +- [x] Inspect the final diff for unrelated changes and run release-level platform checks. diff --git a/docs/superpowers/plans/2026-07-12-lane-merge-http-cutover.md b/docs/superpowers/plans/2026-07-12-lane-merge-http-cutover.md new file mode 100644 index 0000000..c9c29ae --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-lane-merge-http-cutover.md @@ -0,0 +1,184 @@ +# Lane Merge HTTP Cutover Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the legacy branch-scoped lane-merge HTTP API with the lane-scoped hard-cutover endpoint used by `trail lane merge`. + +**Architecture:** The new route identifies the lane in `/v1/lanes/{lane}/merge` and receives the target branch in a required `into` JSON field. The daemon client becomes a direct consumer of that contract. Storage and operation-kind names are intentionally unchanged. + +**Tech Stack:** Rust, serde, existing Trail HTTP router/OpenAPI generator, Rust integration tests, shell benchmark script, Markdown docs. + +## Global Constraints + +- Remove `/v1/branches/{branch}/merge-lane`; do not retain a compatibility route. +- Canonical request fields are `into`, `strategy`, `dry_run`, and `direct`; the lane belongs only in the path. +- Retain `OperationKind::LaneMerge`, `lane_merge`, merge-queue storage, and domain-level `merge_lane*` Rust methods. +- Update client, server, audit, OpenAPI, tests, docs, and benchmark calls in the same change. + +--- + +### Task 1: Define and route the lane-scoped HTTP contract + +**Files:** +- Modify: `trail/src/server/request_types/collaboration.rs:7-19` +- Modify: `trail/src/server/route/lane/collaboration.rs:133-145` +- Modify: `trail/src/server/route/audit.rs:91-124` +- Test: `trail/tests/e2e.rs:10415-10431,19439-19450,19529-19548,19988-19992` + +**Interfaces:** +- Consumes: `POST /v1/lanes/{lane}/merge` and `LaneMergeRequest { into, strategy, dry_run, direct }`. +- Produces: `MergeReport` with the existing readiness, direct-merge, and conflict behavior. + +- [x] **Step 1: Write failing route tests** + +```rust +let response = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/lanes/doc-bot/merge", + serde_json::json!({ "into": "main", "dry_run": true }), + ), +); +assert_eq!(response.status, 200); + +let legacy = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/branches/main/merge-lane", + serde_json::json!({ "lane": "doc-bot", "dry_run": true }), + ), +); +assert_eq!(legacy.status, 400); +``` + +- [x] **Step 2: Verify the new route fails before implementation** + +Run: `cargo test -p trail --test e2e merge_dry_run_reports_without_mutating_refs -- --exact --nocapture` + +Expected: the new lane-scoped request is rejected because no route matches it. + +- [x] **Step 3: Implement the replacement request and route** + +```rust +pub(crate) struct LaneMergeRequest { + pub(crate) into: String, + #[serde(default)] + pub(crate) strategy: Option, + #[serde(default, alias = "dry-run")] + pub(crate) dry_run: bool, + #[serde(default)] + pub(crate) direct: bool, +} + +if parts.len() == 4 + && parts[0] == "v1" + && parts[1] == "lanes" + && parts[3] == "merge" + && request.method == "POST" +{ + let body: LaneMergeRequest = serde_json::from_slice(&request.body)?; + validate_merge_strategy(body.strategy.as_deref())?; + let lane = db.resolve_lane_handle(parts[2])?; + let report = db.merge_lane_user_with_options(&lane, &body.into, body.dry_run, body.direct)?; + return Ok(Some(json_response(200, "OK", &report)?)); +} +``` + +Update audit extraction so this route's path supplies the lane and `body.into` +supplies the target branch ref. Delete the branch-scoped merge-lane cases. + +- [x] **Step 4: Verify the server contract** + +Run: `cargo test -p trail --test e2e merge_dry_run_reports_without_mutating_refs -- --exact --nocapture && cargo test -p trail --test e2e local_api_direct_merge_lane_conflict_records_conflict_set -- --exact --nocapture` + +Expected: the new endpoint succeeds, the legacy route follows normal unknown-route handling (400), and conflict behavior is unchanged. + +### Task 2: Cut over first-party consumers and generated API description + +**Files:** +- Modify: `trail/src/cli/command/handler/daemon_rpc.rs:238-249` +- Modify: `trail/src/server/openapi/paths/collaboration.rs:118-122` +- Modify: `trail/src/server/openapi/schemas/collaboration.rs:5-18` +- Modify: `scripts/cli-scale-bench.sh:494-502` +- Test: `trail/tests/e2e.rs:12679-12689` + +**Interfaces:** +- Consumes: `LaneMergeRequest` and `/v1/lanes/{lane}/merge` from Task 1. +- Produces: daemon CLI merge behavior and OpenAPI operation `laneMerge` with schema `LaneMergeRequest`. + +- [x] **Step 1: Update the daemon route test to expect the new contract** + +```rust +let merge = run_trail_json_daemon( + temp.path(), + &daemon_url, + &["lane", "merge", "rpc-bot", "--into", "main", "--dry-run"], +); +assert_eq!(merge["dry_run"], true); +``` + +- [x] **Step 2: Verify the daemon test fails before the client change** + +Run: `cargo test -p trail --test e2e cli_daemon_url_routes_hot_lane_commands -- --exact --nocapture` + +Expected: the daemon client still posts to the removed route. + +- [x] **Step 3: Cut over daemon, OpenAPI, and benchmark callers** + +```rust +let body = serde_json::json!({ + "into": args.into, + "strategy": args.strategy, + "dry_run": args.dry_run, + "direct": args.direct, +}); +let report: MergeReport = client.post_json( + &format!("/v1/lanes/{}/merge", args.name), + &body, +)?; +``` + +Expose `/v1/lanes/{lane}/merge` as `laneMerge` with required path `lane` and +schema field `into`. Change the benchmark JSON from `lane_id` to `into` and +the URL to `/v1/lanes/daemonbot/merge`. + +- [x] **Step 4: Verify daemon and OpenAPI consumers** + +Run: `cargo test -p trail --test e2e cli_daemon_url_routes_hot_lane_commands -- --exact --nocapture` + +Expected: PASS. + +### Task 3: Update public references and run regression checks + +**Files:** +- Modify: `docs/reference/http-api.md:170-186` +- Modify: `docs/integrations/openapi.md:28-31` +- Modify: `CHANGELOG.md` +- Test: `trail/tests/e2e.rs` + +**Interfaces:** +- Consumes: the endpoint and schema from Tasks 1 and 2. +- Produces: all public documentation referring only to the lane-scoped endpoint. + +- [x] **Step 1: Replace API references and add the breaking-change note** + +```markdown +| POST | `/v1/lanes/{lane_or_id}/merge` | Dry-run or explicitly direct-merge this lane into `into`. | +``` + +Document the request body field `into` and state that the previous +branch-scoped endpoint has been removed. + +- [x] **Step 2: Check no legacy API contract remains** + +Run: `! rg -n '/v1/branches/\{branch\}/merge-lane|/v1/branches/main/merge-lane|MergeLaneRequest|branchMergeLane' trail docs scripts README.md` + +Expected: no matches. + +- [x] **Step 3: Run formatting and focused regressions** + +Run: `cargo fmt --all -- --check && cargo check -p trail && cargo test -p trail --bin trail && cargo test -p trail --test terminal_output_guard && cargo test -p trail --test e2e merge_dry_run_reports_without_mutating_refs -- --exact && cargo test -p trail --test e2e cli_daemon_url_routes_hot_lane_commands -- --exact` + +Expected: every command passes. diff --git a/docs/superpowers/plans/2026-07-12-lane-merge-queue-cutover.md b/docs/superpowers/plans/2026-07-12-lane-merge-queue-cutover.md new file mode 100644 index 0000000..7220781 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-lane-merge-queue-cutover.md @@ -0,0 +1,477 @@ +# Lane Merge Queue Hard Cutover Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace Trail's generic merge queue with a lane-only queue exposed as `trail lane merge-queue` and `/v1/lanes/merges/queue`, with a destructive v16 schema cutover. + +**Architecture:** Queue state stores stable lane ids and resolves the current lane branch only when explaining or executing an entry. The CLI, HTTP, daemon, MCP, reports, renderers, storage methods, and documentation cut over together; generic branch/ref merging remains solely under `trail merge`. + +**Tech Stack:** Rust, clap, serde, rusqlite/SQLite migrations, Trail's JSON HTTP server and OpenAPI generator, MCP JSON-RPC, Rust e2e tests, shell benchmarks, Markdown documentation. + +## Global Constraints + +- Remove `trail merge-queue`, `/v1/merge-queue`, `trail.merge_queue_*`, and `trail://workspace/merge-queue` without aliases. +- Accept only existing lanes in the queue; generic branches and refs continue through `trail merge`. +- Store `lane_id`, not a generic queue `source_ref`, and generate `lmq_` ids. +- Migrate schema v15 to v16 transactionally by discarding legacy queue rows and clearing legacy merge-result queue links. +- Preserve readiness, approval, gate, dirty-workdir, priority, serialization, cancellation, and conflict-pausing behavior. +- Preserve generic `source_ref` and `target_ref` fields in merge results and conflict sets. + +--- + +### Task 1: Cut Storage and Domain Models Over to a Lane Queue + +**Files:** +- Modify: `trail/src/db/mod.rs` +- Modify: `trail/src/db/storage/schema/ddl.rs` +- Modify: `trail/src/db/util/rows.rs` +- Modify: `trail/src/db/merge/queue.rs` +- Modify: `trail/src/db/merge/queue_store.rs` +- Modify: `trail/src/db/lane/identity.rs` +- Modify: `trail/src/db/core/doctor_activity.rs` +- Modify: `trail/src/model/reports/merge.rs` +- Test: `trail/tests/e2e.rs` + +**Interfaces:** +- Consumes: `Trail::lane_details(&str)`/existing lane selector resolution and `Trail::merge_lane_unlocked`. +- Produces: `enqueue_lane_merge`, `list_lane_merge_queue`, `explain_lane_merge_queue`, `run_lane_merge_queue`, `remove_lane_merge_queue`, and `LaneMergeQueue*` reports. + +- [ ] **Step 1: Write failing v16 migration and lane-only queue tests** + +Add focused e2e coverage equivalent to: + +```rust +#[test] +fn schema_v16_discards_generic_merge_queue() { + let (_temp, mut db) = initialized_db(); + db.enqueue_merge("doc-bot", "main", 0).unwrap(); + let conn = db.connection(); + conn.execute_batch("PRAGMA user_version = 15;").unwrap(); + drop(db); + + let reopened = Trail::open(_temp.path()).unwrap(); + assert_eq!(reopened.schema_user_version().unwrap(), 16); + assert!(table_exists(reopened.connection(), "lane_merge_queue")); + assert!(!table_exists(reopened.connection(), "merge_queue")); + assert!(reopened.list_lane_merge_queue().unwrap().is_empty()); +} + +#[test] +fn lane_merge_queue_rejects_branch_sources() { + let (_temp, mut db) = initialized_db(); + let error = db.enqueue_lane_merge("main", "main", 0).unwrap_err(); + assert!(error.to_string().contains("lane")); +} +``` + +Use the repository's actual test fixtures and direct SQLite helpers rather than introducing a second fixture style. + +- [ ] **Step 2: Run the focused tests and confirm red state** + +Run: `cargo test -p trail --test e2e schema_v16_discards_generic_merge_queue -- --exact --nocapture && cargo test -p trail --test e2e lane_merge_queue_rejects_branch_sources -- --exact --nocapture` + +Expected: FAIL because schema version 16, `lane_merge_queue`, and lane-specific methods do not exist. + +- [ ] **Step 3: Implement v16 schema and lane-specific report types** + +Set: + +```rust +const TRAIL_SCHEMA_VERSION: i64 = 16; +``` + +Replace the queue table with: + +```sql +CREATE TABLE IF NOT EXISTS lane_merge_queue ( + queue_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL, + target_ref TEXT NOT NULL, + status TEXT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS lane_merge_queue_active_idx + ON lane_merge_queue(lane_id, target_ref, status); +CREATE INDEX IF NOT EXISTS lane_merge_queue_run_idx + ON lane_merge_queue(status, priority DESC, created_at ASC); +``` + +When upgrading from a version below 16, drop `merge_queue`, create the new table and indexes, rename `merge_results.queue_id` to `lane_queue_id`, and set existing non-null links to `NULL` inside the existing migration savepoint. + +Define the entry shape: + +```rust +pub struct LaneMergeQueueEntry { + pub queue_id: String, + pub lane_id: String, + pub lane: String, + pub target_ref: String, + pub status: String, + pub priority: i64, + pub created_at: i64, + pub updated_at: i64, +} +``` + +Rename all queue report types consistently and update SQLite row mapping to join `lane_merge_queue` with `lanes` for the human-facing lane name. + +- [ ] **Step 4: Implement lane-only queue operations** + +Use signatures: + +```rust +pub fn enqueue_lane_merge( + &mut self, + lane: &str, + target: &str, + priority: i64, +) -> Result; +pub fn list_lane_merge_queue(&self) -> Result>; +pub fn explain_lane_merge_queue(&mut self, selector: &str) -> Result; +pub fn run_lane_merge_queue(&mut self, limit: Option) -> Result; +pub fn remove_lane_merge_queue(&mut self, selector: &str) -> Result; +``` + +Resolve the lane before inserting, seed ids from `lane_id:target_ref:priority:now`, prefix ids with `lmq_`, and resolve `lane_id` again before explain/run. Remove branch fallbacks from queue normalization and execution. + +- [ ] **Step 5: Run storage and safety regressions** + +Run: `cargo test -p trail --test e2e schema_v16_discards_generic_merge_queue -- --exact && cargo test -p trail --test e2e lane_merge_queue_rejects_branch_sources -- --exact && cargo test -p trail --test e2e merge_lane_and_queue_enforce_readiness_blockers -- --exact && cargo test -p trail --test e2e lane_merge_queue_pauses_on_conflict -- --exact` + +Expected: PASS after renaming the existing queue tests to their lane-specific names and assertions. + +- [ ] **Step 6: Commit the storage cutover** + +```bash +git add trail/src/db trail/src/model/reports/merge.rs trail/tests/e2e.rs +git commit -m "refactor(trail)!: make merge queue lane-only" +``` + +### Task 2: Nest the CLI Queue Under `trail lane` + +**Files:** +- Modify: `trail/src/cli/command.rs` +- Modify: `trail/src/cli/command/lane_args.rs` +- Modify: `trail/src/cli/command/collaboration_args.rs` +- Modify: `trail/src/cli/command/collaboration_args/merge.rs` +- Modify: `trail/src/cli/command/handler.rs` +- Modify: `trail/src/cli/command/handler/lane.rs` +- Modify: `trail/src/cli/command/handler/collaboration.rs` +- Modify: `trail/src/cli/command/handler/daemon_rpc.rs` +- Modify: `trail/src/cli/command/render/collaboration/merge.rs` +- Modify: `trail/src/db/merge/lane.rs` +- Test: `trail/src/cli/command.rs` +- Test: `trail/tests/e2e.rs` + +**Interfaces:** +- Consumes: Task 1's lane queue methods and reports. +- Produces: `trail lane merge-queue add|list|explain|run|remove`; the top-level spelling is unparseable. + +- [ ] **Step 1: Write failing parser tests for the hard CLI cutoff** + +```rust +#[test] +fn parses_lane_merge_queue_and_rejects_top_level_form() { + let cli = Cli::try_parse_from([ + "trail", "lane", "merge-queue", "add", "doc-bot", "--into", "main", + ]).expect("lane merge queue should parse"); + assert!(matches!(cli.command, Command::Lane(_))); + assert!(Cli::try_parse_from([ + "trail", "merge-queue", "add", "doc-bot", "--into", "main", + ]).is_err()); +} +``` + +- [ ] **Step 2: Confirm the parser test fails** + +Run: `cargo test -p trail --bin trail parses_lane_merge_queue_and_rejects_top_level_form -- --exact --nocapture` + +Expected: FAIL because `merge-queue` is still top-level. + +- [ ] **Step 3: Move queue args and dispatch into the lane command group** + +Add: + +```rust +pub(super) enum LaneSubcommand { + // existing variants + MergeQueue(LaneMergeQueueCommand), +} +``` + +Rename the argument structs to `LaneMergeQueueCommand`, +`LaneMergeQueueSubcommand`, and `LaneMergeQueue*Args`; change `source` to +`lane`. Delete `Command::MergeQueue` and its top-level dispatch arm. Route local +and daemon-backed lane dispatch through the Task 1 methods and Task 3 HTTP +paths. + +- [ ] **Step 4: Update human output and direct-merge next steps** + +Rename renderer functions to `render_lane_merge_queue_*`, display `Lane` +instead of `Source`, and emit only: + +```text +trail lane merge-queue explain +trail lane merge-queue list +trail lane merge-queue add --into +trail lane merge-queue run +``` + +- [ ] **Step 5: Verify local CLI behavior** + +Run: `cargo test -p trail --bin trail parses_lane_merge_queue_and_rejects_top_level_form -- --exact && cargo test -p trail --test e2e lane_merge_queue_runs_lane_branch_into_main -- --exact && cargo test -p trail --test terminal_output_guard` + +Expected: PASS with no callable top-level queue command. + +- [ ] **Step 6: Commit the CLI cutover** + +```bash +git add trail/src/cli trail/src/db/merge/lane.rs trail/tests +git commit -m "refactor(cli)!: nest merge queue under lanes" +``` + +### Task 3: Replace the HTTP and OpenAPI Contract + +**Files:** +- Modify: `trail/src/server/request_types/collaboration.rs` +- Modify: `trail/src/server/route/lane/collaboration.rs` +- Modify: `trail/src/server/route/audit.rs` +- Modify: `trail/src/server/openapi/paths/collaboration.rs` +- Modify: `trail/src/server/openapi/schemas/collaboration.rs` +- Modify: `trail/src/cli/command/handler/daemon_rpc.rs` +- Test: `trail/tests/e2e.rs` + +**Interfaces:** +- Consumes: Task 1 report/method names and Task 2 daemon arguments. +- Produces: only `/v1/lanes/merges/queue` routes with strict lane-specific JSON. + +- [ ] **Step 1: Write failing HTTP cutoff tests** + +Exercise: + +```rust +let add = api_request( + "POST", + "/v1/lanes/merges/queue", + serde_json::json!({"lane": "doc-bot", "into": "main", "priority": 10}), +); +assert_eq!(handle_http_request(&mut db, &add).status, 201); + +let legacy = api_request( + "POST", + "/v1/merge-queue", + serde_json::json!({"source": "doc-bot", "target": "main"}), +); +assert_eq!(handle_http_request(&mut db, &legacy).status, 400); +``` + +Also assert strict rejection of `source`, `target`, and `target_branch` on the +new route. + +- [ ] **Step 2: Confirm the route test fails** + +Run: `cargo test -p trail --test e2e local_api_drives_lane_merge_queue -- --exact --nocapture` + +Expected: FAIL because the new route is not registered. + +- [ ] **Step 3: Implement strict requests and replacement routes** + +```rust +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct LaneMergeQueueAddRequest { + pub(crate) lane: String, + pub(crate) into: String, + #[serde(default)] + pub(crate) priority: i64, +} +``` + +Route add/list/run/explain/remove under `/v1/lanes/merges/queue`, delete every +`/v1/merge-queue` branch, update audit target extraction, and make the daemon +client call only the replacement routes. + +- [ ] **Step 4: Replace OpenAPI paths and schemas** + +Define `LaneMergeQueueAddRequest`, `LaneMergeQueueEntry`, and lane-specific +report schemas. Expose only: + +```text +/v1/lanes/merges/queue +/v1/lanes/merges/queue/run +/v1/lanes/merges/queue/{selector}/explain +/v1/lanes/merges/queue/{selector} +``` + +- [ ] **Step 5: Verify HTTP, daemon, and OpenAPI behavior** + +Run: `cargo test -p trail --test e2e local_api_drives_lane_merge_queue -- --exact && cargo test -p trail --test e2e cli_daemon_url_routes_hot_lane_commands -- --exact` + +Expected: PASS and old routes return standard unknown-route responses. + +- [ ] **Step 6: Commit the API cutover** + +```bash +git add trail/src/server trail/src/cli/command/handler/daemon_rpc.rs trail/tests/e2e.rs +git commit -m "refactor(api)!: expose lane merge queue routes" +``` + +### Task 4: Cut MCP Over to Lane-Specific Tools and Resource + +**Files:** +- Modify: `trail/src/mcp/audit.rs` +- Modify: `trail/src/mcp/capabilities/resources.rs` +- Modify: `trail/src/mcp/tool_call/merge.rs` +- Modify: `trail/src/mcp/tools/merge.rs` +- Modify: `trail/src/mcp/tools/annotations.rs` +- Modify: `trail/src/mcp/types/merge.rs` +- Modify: `trail/src/mcp/types/constants.rs` +- Test: `trail/tests/e2e.rs` + +**Interfaces:** +- Consumes: Task 1 methods and report types. +- Produces: `trail.lane_merge_queue_*` tools and `trail://workspace/lane-merge-queue`. + +- [ ] **Step 1: Write failing MCP discovery and call assertions** + +```rust +assert!(tools.iter().any(|tool| tool["name"] == "trail.lane_merge_queue_add")); +assert!(!tools.iter().any(|tool| tool["name"] == "trail.merge_queue_add")); +assert!(resources.iter().any(|resource| { + resource["uri"] == "trail://workspace/lane-merge-queue" +})); +``` + +Call add with `{"lane":"doc-bot","target":"main","priority":0}` and assert +the structured content contains the stable `lane_id` and `lane` name. + +- [ ] **Step 2: Confirm MCP tests fail** + +Run: `cargo test -p trail --test e2e local_api_and_mcp_drive_lane_merge_queue_and_conflicts -- --exact --nocapture` + +Expected: FAIL because discovery still exposes generic queue names. + +- [ ] **Step 3: Rename tools, arguments, dispatch, annotations, audit, and resource** + +Use exact names: + +```text +trail.lane_merge_queue_add +trail.lane_merge_queue_list +trail.lane_merge_queue_explain +trail.lane_merge_queue_run +trail.lane_merge_queue_remove +trail://workspace/lane-merge-queue +``` + +The add arguments are `lane`, `target`, and optional `priority`; no `source` +alias remains. Preserve current read/write/destructive annotations. + +- [ ] **Step 4: Verify MCP behavior** + +Run: `cargo test -p trail --test e2e local_api_and_mcp_drive_lane_merge_queue_and_conflicts -- --exact` + +Expected: PASS with old names absent. + +- [ ] **Step 5: Commit the MCP cutover** + +```bash +git add trail/src/mcp trail/tests/e2e.rs +git commit -m "refactor(mcp)!: expose lane merge queue tools" +``` + +### Task 5: Update First-Party References and Run the Full Verification Gate + +**Files:** +- Modify: `README.md` +- Modify: `CHANGELOG.md` +- Modify: `docs/reference/http-api.md` +- Modify: `docs/reference/cli/lanes.md` +- Modify: `docs/guides/branch-checkout-and-merge.md` +- Modify: `docs/guides/hardening-agent-workflows.md` +- Modify: `docs/getting-started/first-lane-workflow.md` +- Modify: `docs/getting-started/install-and-build.md` +- Modify: `docs/concepts/readiness-gates-and-merge-safety.md` +- Modify: `docs/lanes/handoff-review-and-merge.md` +- Modify: `skills/use-trail/SKILL.md` +- Modify: `skills/use-trail/references/lanes.md` +- Modify: `skills/use-trail/references/safety-and-recovery.md` +- Modify: `skills/use-trail/references/integrations.md` +- Modify: `scripts/cli-scale-bench.sh` +- Modify: `scripts/check-cli-scale-thresholds.py` +- Test: repository-wide search and Trail test suites + +**Interfaces:** +- Consumes: all replacement public names from Tasks 1-4. +- Produces: repository guidance, examples, and benchmarks containing no callable legacy queue surface. + +- [ ] **Step 1: Replace public examples and benchmark calls** + +Use: + +```sh +trail lane merge-queue add doc-bot --into main +trail lane merge-queue explain doc-bot +trail lane merge-queue run +``` + +Use `/v1/lanes/merges/queue` for HTTP examples and daemon scale calls. Rename +benchmark metric labels to `lane_merge_queue_*` so reports match the public +domain. + +- [ ] **Step 2: Assert no callable legacy surface remains** + +Run: + +```bash +rg -n 'trail merge-queue|/v1/merge-queue|trail\.merge_queue_|trail://workspace/merge-queue' \ + README.md CHANGELOG.md docs skills scripts trail/src \ + --glob '!docs/superpowers/specs/2026-07-12-lane-merge-queue-cutover-design.md' \ + --glob '!docs/superpowers/plans/2026-07-12-lane-merge-queue-cutover.md' +``` + +Expected: only explicit negative parser, route, and tool assertions that prove +removal. Inspect every match; no help text, production route, callable tool, +resource, documentation example, or benchmark invocation may use a legacy +name. + +- [ ] **Step 3: Format and run focused tests** + +Run: + +```bash +cargo fmt --all -- --check +cargo check -p trail +cargo test -p trail --bin trail +cargo test -p trail --test terminal_output_guard +cargo test -p trail --test e2e schema_v16_discards_generic_merge_queue -- --exact +cargo test -p trail --test e2e lane_merge_queue_rejects_branch_sources -- --exact +cargo test -p trail --test e2e lane_merge_queue_runs_lane_branch_into_main -- --exact +cargo test -p trail --test e2e local_api_drives_lane_merge_queue -- --exact +cargo test -p trail --test e2e local_api_and_mcp_drive_lane_merge_queue_and_conflicts -- --exact +``` + +Expected: every command passes. + +- [ ] **Step 4: Run the complete Trail suite** + +Run: `cargo test -p trail` + +Expected: all unit, integration, e2e, and documentation tests pass. + +- [ ] **Step 5: Commit documentation and benchmark cutover** + +```bash +git add README.md CHANGELOG.md docs skills scripts +git commit -m "docs: document lane merge queue cutover" +``` + +- [ ] **Step 6: Record final evidence** + +Run: `git status --short && git log -6 --oneline` + +Expected: no tracked implementation changes remain uncommitted; unrelated +pre-existing untracked files may still be present. The recent commit list shows +the storage, CLI, API, MCP, documentation, design, and plan commits. diff --git a/docs/superpowers/plans/2026-07-12-nfs-noop-setattr-checkpoint.md b/docs/superpowers/plans/2026-07-12-nfs-noop-setattr-checkpoint.md new file mode 100644 index 0000000..a4d00de --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-nfs-noop-setattr-checkpoint.md @@ -0,0 +1,130 @@ +# NFS No-op `SETATTR` Checkpoint Fix Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Prevent metadata-only NFS `SETATTR` requests from creating false source-file deletions in Trail checkpoints. + +**Architecture:** Enforce the no-op invariant in `ViewCore::setattr`, the shared mutation boundary used by filesystem adapters. A regression test exercises the real lower-root, upper-layer, journal, and checkpoint-candidate flow so the bug cannot recur without failing the test. + +**Tech Stack:** Rust, Trail workspace-view core, Cargo test harness. + +## Global Constraints + +- Calls with neither size nor mode must return visible attributes without copying up or journaling the path. +- Calls with size, mode, or both must retain existing copy-up and journaling behavior. +- Invalid or stale inode errors must remain observable. +- Timestamp, ownership, ACL, and extended-attribute support remain out of scope. +- Existing corrupted lanes are not modified by this implementation. + +--- + +## File Structure + +- Modify and test: `trail/src/db/lane/workdir/view_core.rs` + - `ViewCore::setattr` owns the shared no-op guard. + - The colocated Unix test module owns the lower-file checkpoint regression test. + +### Task 1: Make empty supported-attribute updates side-effect free + +**Files:** +- Modify: `trail/src/db/lane/workdir/view_core.rs:996` +- Test: `trail/src/db/lane/workdir/view_core.rs:2199` + +**Interfaces:** +- Consumes: `ViewCore::setattr(&mut self, ino: u64, size: Option, mode: Option) -> Result` +- Produces: The invariant that `size == None && mode == None` returns current attributes without adding a checkpoint candidate. + +- [x] **Step 1: Write the failing regression test** + +Add this test immediately before `view_core_conformance_truncate_mode_and_symlink_escape`: + +```rust +#[test] +fn view_core_noop_setattr_does_not_create_checkpoint_candidate() { + let (_temp, db, root, upper) = fixture(); + let mut view = lazy_core(&db, &root, upper.clone()); + let readme = view.lookup(VIEW_ROOT_INO, "README.md").unwrap(); + + let attr = view.setattr(readme, None, None).unwrap(); + + assert_eq!(attr.ino, readme); + assert!(!upper.join("README.md").exists()); + let candidates = view.checkpoint_candidates().unwrap(); + assert!(!candidates.paths.contains("README.md")); +} +``` + +- [x] **Step 2: Run the regression test and verify RED** + +Run: + +```sh +cargo test -p trail view_core_noop_setattr_does_not_create_checkpoint_candidate +``` + +Expected: FAIL at `assert!(!candidates.paths.contains("README.md"))`, proving the current no-op `setattr` incorrectly journals the lower file. + +- [x] **Step 3: Add the minimal shared-core guard** + +Change the start of `ViewCore::setattr` to resolve the inode before deciding whether the request mutates supported attributes: + +```rust +pub(crate) fn setattr( + &mut self, + ino: u64, + size: Option, + mode: Option, +) -> std::result::Result { + let path = self.path_for_ino(ino)?; + if size.is_none() && mode.is_none() { + return self.attr(&path); + } + let _barrier = self.begin_mutation()?; +``` + +Keep the existing size, mode, journal, and final-attribute logic unchanged after the guard. + +- [x] **Step 4: Run the regression test and verify GREEN** + +Run: + +```sh +cargo test -p trail view_core_noop_setattr_does_not_create_checkpoint_candidate +``` + +Expected: PASS with one matching test and zero failures. + +- [x] **Step 5: Run focused workspace-view and NFS tests** + +Run: + +```sh +cargo test -p trail view_core +cargo test -p trail nfs_adapter +``` + +Expected: Both commands exit 0 with zero failed tests. The NFS filter may match only macOS-gated tests. + +- [x] **Step 6: Run formatting and broader Trail verification** + +Run: + +```sh +cargo fmt --check +cargo test -p trail --lib +``` + +Expected: Formatting exits 0 and all Trail library tests pass. + +- [x] **Step 7: Review and commit the isolated fix** + +Run: + +```sh +git diff --check +git diff -- trail/src/db/lane/workdir/view_core.rs +git add trail/src/db/lane/workdir/view_core.rs docs/superpowers/plans/2026-07-12-nfs-noop-setattr-checkpoint.md +git commit -m "fix: ignore empty workspace setattr" +``` + +Expected: The commit contains only the implementation plan, regression test, and no-op guard. diff --git a/docs/superpowers/plans/2026-07-12-persistent-path-invariant-index.md b/docs/superpowers/plans/2026-07-12-persistent-path-invariant-index.md new file mode 100644 index 0000000..5cfb113 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-persistent-path-invariant-index.md @@ -0,0 +1,39 @@ +# Persistent Path-Invariant Index Implementation Plan + +## Task 1: Root schema and full builders + +- Add Serde-defaulted `WorktreeRoot.case_fold_map_root`. +- Add helpers to build a sorted folded-key -> canonical-path Prolly tree. +- Populate it in every full root constructor, including workspace-view tests. +- Test ASCII/Unicode collisions and legacy root deserialization. + +## Task 2: Indexed mutation helper + +- Add `validate_and_update_case_fold_index(previous_root, removals, additions)`. +- Query only touched folded keys, apply removals before additions, and write one + mutation batch. +- Return stable `PATH_INDEX_REQUIRED` for a legacy root with no index. +- Test rename, delete-then-add, within-batch collision, existing-root collision, + and SHA/content-addressed node reuse. + +## Task 3: Hot validator integration + +- Replace patch and record final-root full scans with indexed validation. +- Integrate the returned index root into incremental root constructors. +- Ensure validation happens before content/operation/ref writes. +- Add structural counters proving zero full-root path loads and <=k lookups. + +## Task 4: Rebuild and compatibility + +- Extend `trail index rebuild` to backfill the current root's case-fold index + without changing visible files. +- Add `PATH_INDEX_REQUIRED` CLI/JSON diagnostics and recovery guidance. +- Test old-root open/read, mutation failure, rebuild, then successful mutation. + +## Task 5: Scale gates and verification + +- Extend the CLI scale harness with 1k/100k/1M structured patch and record + structural metrics. +- Gate indexed mode, lookup count, and zero full-root path loads. +- Run fmt, focused tests, `cargo test -p trail`, 100k acceptance, and scheduled + 1M automation; independently review before completion. diff --git a/docs/superpowers/plans/2026-07-12-strict-native-cow-materialization.md b/docs/superpowers/plans/2026-07-12-strict-native-cow-materialization.md new file mode 100644 index 0000000..a179524 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-strict-native-cow-materialization.md @@ -0,0 +1,247 @@ +# Strict Native-COW Materialization Implementation Plan + +> **For agentic workers:** Execute inline. The user explicitly waived test-first TDD; +> add focused regression tests with each implementation slice and run them as +> verification gates. + +**Goal:** Make `native-cow` strictly clone every file, add truthful +`portable-copy`, and make materialized-only `auto` restart portably when native COW +is unavailable. + +**Architecture:** A materialization coordinator owns requested/resolved modes, +staging, strict fallback, publication, and reporting. A typed native-clone primitive +separates capability failures from operational errors. Existing mounted workdir modes +remain explicit and independent. + +**Tech Stack:** Rust, Serde, Clap, rusqlite lane metadata, rustix +`fclonefileat`/`ioctl_ficlone`, Rayon, SHA-256 manifests, platform filesystem tests. + +## Global Constraints + +- Explicit `native-cow` must never byte-copy. +- `portable-copy` may clone or copy per file and must report `clone`, `mixed`, or + `copy` from actual outcomes. +- `auto` may restart portably only for clone unsupported, cross-device, or no complete + validated native source. +- Permission, storage, corruption, invalid-path, and I/O errors are hard failures. +- `auto` never selects FUSE, NFS, or Dokan. +- Preserve Trail's immutable-root validation, clean-workdir manifests, dirty-workdir + rescue/refusal, and mounted-view behavior. +- Preserve unrelated untracked workspace content. +- Tests are written after each production slice by explicit user instruction. + +--- + +### Task 1: Public modes and truthful report model + +**Files:** + +- Modify: `trail/src/model/reports/lane.rs` +- Modify: `trail/src/db/lane/lifecycle.rs` +- Modify: `trail/src/cli/command/lane_args.rs` +- Modify: `trail/src/cli/command/agent_args.rs` +- Modify: `trail/src/server/openapi/schemas/lane.rs` +- Modify: `trail/src/mcp/tools/lane.rs` +- Modify: parser/schema tests in the same modules and `trail/tests/e2e.rs` + +**Interfaces:** + +```rust +pub enum LaneWorkdirMode { + Virtual, + Sparse, + NativeCow, + PortableCopy, + FuseCow, + NfsCow, + DokanCow, +} + +pub enum WorkdirBackend { + Clone, + Mixed, + Copy, + Fuse, + Nfs, + Dokan, + Virtual, +} + +pub struct MaterializationReport { + pub cloned_files: u64, + pub cloned_bytes: u64, + pub copied_files: u64, + pub copied_bytes: u64, + pub fallback_reason: Option, +} +``` + +- [ ] Add `portable-copy` parsing/serialization and reject unknown aliases. +- [ ] Add requested mode, resolved mode, actual backend, and optional materialization + details to lane spawn/workdir/sync reports. +- [ ] Keep legacy `cow_backend` readable through Serde defaults/aliases, but stop + deriving strict evidence from the requested mode. +- [ ] Change `auto` resolution to `NativeCow` as a request policy marker handled by + the coordinator, never a mounted backend; add an internal requested-mode value + where persistence needs to preserve `auto`. +- [ ] Update CLI, MCP, HTTP/OpenAPI enums and focused parser/schema tests. +- [ ] Run focused model, CLI, MCP, and OpenAPI tests. +- [ ] Commit the slice. + +### Task 2: Typed native clone primitive + +**Files:** + +- Modify: `trail/src/db/util/fs_cow.rs` +- Modify: `trail/src/db/util/mod.rs` if exports change +- Test: unit tests in `trail/src/db/util/fs_cow.rs` + +**Interfaces:** + +```rust +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum NativeCloneUnavailable { + Unsupported, + CrossDevice, +} + +pub(crate) enum NativeCloneOutcome { + Cloned, + Unavailable(NativeCloneUnavailable), +} + +pub(crate) fn clone_file_native( + source: &Path, + destination: &Path, +) -> Result; +``` + +- [ ] Replace the boolean primitive with typed clone outcomes. +- [ ] Map APFS and Linux unsupported/cross-device errno values conservatively. +- [ ] Remove `EPERM` and `EACCES` from availability classification and include Linux + `ENOTTY` where exposed by rustix. +- [ ] Preserve create-new destination semantics and cleanup after failed ioctls. +- [ ] Keep mounted-view projection explicitly portable by translating unavailable + outcomes into byte copies only in `clone_or_copy_projected_file`. +- [ ] Add classifier, cleanup, clone/copy, permission, and cross-device tests. +- [ ] Run fs-cow unit tests and commit the slice. + +### Task 3: Materialization coordinator and accounting + +**Files:** + +- Create: `trail/src/db/lane/workdir/materialize.rs` +- Modify: `trail/src/db/lane/workdir.rs` +- Modify: `trail/src/db/util/fs_cow.rs` +- Modify: `trail/src/db/lane/lifecycle.rs` +- Test: unit tests in the new module and lifecycle module + +**Interfaces:** + +```rust +pub(crate) enum MaterializationPolicy { + StrictNative, + Portable, + Auto, +} + +pub(crate) struct MaterializationOutcome { + pub resolved_mode: LaneWorkdirMode, + pub backend: WorkdirBackend, + pub report: MaterializationReport, + pub stamps: BTreeMap, +} + +impl Trail { + pub(crate) fn materialize_lane_root_staged( + &self, + root_id: &ObjectId, + destination: &Path, + custom_workdir: bool, + policy: MaterializationPolicy, + ) -> Result; +} +``` + +- [ ] Resolve a complete validated workspace source from the existing root/stamp + checks; return typed native-source-unavailable when incomplete. +- [ ] Build strict attempts only in unique sibling stages. +- [ ] Probe source/destination device identity and native clone support, including an + empty-root probe. +- [ ] Clone every strict file, clear untracked xattrs, apply executable mode, hash the + destination against `FileEntry`, and produce stamps/counters. +- [ ] Abort strict mode on the first unavailable or hard failure without publishing. +- [ ] For `auto`, retire the strict stage and start portable materialization in a new + stage only for the three eligible availability reasons. +- [ ] In portable mode, try clone per file and byte-copy only unavailable files; + aggregate deterministic clone/copy byte and file counts. +- [ ] Verify the clean-workdir manifest before publication. +- [ ] Use bounded Rayon work already present in Trail; stop scheduling new work after + a recorded failure where practical. +- [ ] Add strict success/failure, mixed/copy, fallback, no-source, empty-root, + hardlink-independence, source-mutation, and counter tests. +- [ ] Run coordinator tests and commit the slice. + +### Task 4: Lifecycle, staging publication, and recovery ownership + +**Files:** + +- Modify: `trail/src/db/lane/lifecycle.rs` +- Modify: `trail/src/db/lane/workdir/sync.rs` +- Modify: `trail/src/db/lane/patching.rs` +- Modify: `trail/src/db/lane/control/turn_setup.rs` +- Modify: lane metadata/recovery helpers under `trail/src/db/lane/workdir/` +- Test: lifecycle/sync tests and `trail/tests/e2e.rs` + +- [ ] Route spawn, lazy ensure, full sync, rewind refresh, patch refresh, and the + large-root path through an explicit materialization policy. +- [ ] Persist requested mode separately from resolved mode/backend and refresh actual + reporting after every full rematerialization. +- [ ] Publish initial workdirs with an atomic no-overwrite primitive and make a losing + concurrent creator clean only its own stage. +- [ ] Preserve registered backup/restore behavior for full replacement. +- [ ] Add an operation-owned stage record with preparing/materializing/verified/ + published/failed states using existing Trail database transaction patterns. +- [ ] Reconcile interrupted registered stages/backups without scanning or deleting + unregistered similarly named paths. +- [ ] Ensure legacy best-effort native workdirs have no new verified backend until a + full rematerialization succeeds. +- [ ] Add crash-point, destination-race, cleanup-ownership, backup-restore, + large-root, dirty-workdir, and legacy-metadata tests. +- [ ] Run lifecycle and end-to-end tests and commit the slice. + +### Task 5: User-facing contracts and documentation + +**Files:** + +- Modify: `trail/src/cli/command/render/lane/identity/basic.rs` +- Modify: `trail/src/cli/command/render/lane/work.rs` +- Modify: relevant HTTP/MCP response schemas and tests +- Modify: `docs/lanes/spawn-and-materialize-workdirs.md` +- Modify: `docs/reference/cli/lanes.md` +- Modify: `docs/reference/data-types.md` +- Modify: additional current docs found by repository scan + +- [ ] Render requested mode, resolved mode, actual backend, counts, and bounded + fallback reason without raw persisted OS messages. +- [ ] Document strict native semantics, portable behavior, materialized-only `auto`, + platform support, and explicit mounted modes. +- [ ] Update examples and API schemas from `cow_backend` to `workdir_backend`. +- [ ] Add CLI JSON, HTTP, MCP, and legacy decode regression coverage. +- [ ] Scan current sources for stale claims that `native-cow` may copy silently. +- [ ] Run focused contract tests and commit the slice. + +### Task 6: Complete verification and review + +- [ ] Run `cargo fmt --all -- --check` from `trail/`. +- [ ] Run focused fs-cow, materialization, lifecycle, CLI, MCP, OpenAPI, and e2e tests. +- [ ] Run `cargo clippy -p trail --all-targets -- -D warnings`. +- [ ] Run `cargo test -p trail`. +- [ ] Run native APFS integration tests on the current macOS/APFS host. +- [ ] Record Btrfs and XFS-reflink suites as external platform gates when those + filesystems are not present locally; do not claim they ran locally. +- [ ] Inspect `git diff --check`, `git status --short`, and the complete diff for + unrelated changes. +- [ ] Perform a requirements-to-diff review against the design spec and repair every + critical or important finding. +- [ ] Commit the verified implementation on `main` as explicitly requested. diff --git a/docs/superpowers/plans/2026-07-12-trusted-git-handoff-performance.md b/docs/superpowers/plans/2026-07-12-trusted-git-handoff-performance.md new file mode 100644 index 0000000..926e806 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-trusted-git-handoff-performance.md @@ -0,0 +1,631 @@ +# Trusted Git Handoff Performance Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make high-level Trail agent apply use only mapped changed-path Git export, fail quickly when baseline trust is missing, and prove the behavior at repository scale. + +**Architecture:** Add stable Git handoff errors and an explicit `GitExportPolicy` that separates general full-snapshot export from high-level mapped-delta apply. Gather Git identity once, perform at most one tracked-worktree status query, pass the resulting state into export, and expose export-mode counters in `GitExportReport`. Extend the real CLI scale harness so high-level dry-run and actual apply exercise Git publication rather than only internal lane merge. + +**Tech Stack:** Rust 2021, rusqlite, prolly maps, Git command-line plumbing, shell scale harness, GitHub Actions. + +## Global Constraints + +- High-level agent apply must never select full snapshot export. +- Missing HEAD/base-root trust must fail before loading all root files or writing Git objects. +- Mapped apply work must scale with changed paths `k`, not repository paths `N`. +- Dry-run and actual apply SHALL each ask Git for tracked-worktree cleanliness at most once. +- Dry-run must not insert or repair Git mappings. +- Delta commit construction must preserve Git-tracked paths Trail does not model, including symlinks. +- General `trail git export -m` remains the explicit O(N)-capable full-snapshot operation. +- Tests must run on macOS, Linux, and Windows where the existing Git test gates permit them. + +--- + +### Task 1: Stable Git handoff failures + +**Files:** +- Modify: `trail/src/error.rs` +- Modify: `trail/src/cli/command/handler/errors.rs` + +**Interfaces:** +- Consumes: existing `Error::code()`, `Error::exit_code()`, and `diagnostic_for_error` conventions. +- Produces: `Error::GitMappingRequired`, `Error::GitHeadChanged`, `Error::GitWorktreeDirty`, and `Error::GitDeltaExportRequired` with stable machine codes. + +- [ ] **Step 1: Add failing error-contract tests** + +Add a `#[cfg(test)]` module to `trail/src/error.rs`: + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn git_handoff_errors_have_stable_codes_and_exit_status() { + let errors = [ + (Error::GitMappingRequired("missing mapping".into()), "GIT_MAPPING_REQUIRED"), + (Error::GitHeadChanged("head changed".into()), "GIT_HEAD_CHANGED"), + (Error::GitWorktreeDirty("tracked changes".into()), "GIT_WORKTREE_DIRTY"), + (Error::GitDeltaExportRequired("mapped delta required".into()), "GIT_DELTA_EXPORT_REQUIRED"), + ]; + for (error, code) in errors { + assert_eq!(error.code(), code); + assert_eq!(error.exit_code(), 10); + } + } +} +``` + +Extend the existing diagnostic tests in `trail/src/cli/command/handler/errors.rs`: + +```rust +#[test] +fn missing_git_mapping_recommends_explicit_reconciliation() { + let diagnostic = diagnostic_for_error(&Error::GitMappingRequired("missing".into())); + assert_eq!(diagnostic.code, "GIT_MAPPING_REQUIRED"); + assert_eq!(diagnostic.recovery.unwrap().command, "trail git import-update"); +} +``` + +- [ ] **Step 2: Run the focused tests and verify RED** + +Run: + +```bash +cargo test -p trail git_handoff_errors_have_stable_codes_and_exit_status --lib +``` + +Expected: compilation fails because the four error variants do not exist. + +- [ ] **Step 3: Implement the error variants and diagnostics** + +Add to `Error`: + +```rust +#[error("Git baseline mapping is required: {0}")] +GitMappingRequired(String), +#[error("Git HEAD changed during handoff: {0}")] +GitHeadChanged(String), +#[error("Git tracked worktree is dirty: {0}")] +GitWorktreeDirty(String), +#[error("mapped Git delta export is required: {0}")] +GitDeltaExportRequired(String), +``` + +Map them to the exact codes from Step 1 and exit code `10`. Add dedicated diagnostics; `GitMappingRequired` must recommend `trail git import-update`, `GitHeadChanged` must recommend `git status --short`, and `GitWorktreeDirty` must recommend `git status --short`. + +- [ ] **Step 4: Run focused and diagnostic tests and verify GREEN** + +Run: + +```bash +cargo test -p trail git_handoff_errors_have_stable_codes_and_exit_status --lib +cargo test -p trail missing_git_mapping_recommends_explicit_reconciliation --bin trail +``` + +Expected: both tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add trail/src/error.rs trail/src/cli/command/handler/errors.rs +git commit -m "feat: add stable git handoff errors" +``` + +--- + +### Task 2: Explicit mapped-delta export policy + +**Files:** +- Modify: `trail/src/db/mod.rs` +- Modify: `trail/src/db/merge/git_export.rs` +- Modify: `trail/src/model/reports/worktree.rs` +- Test: `trail/src/db/merge/git_export.rs` + +**Interfaces:** +- Consumes: `git_clean_head_matches_root_mapping`, `diff_root_file_maps`, `git_write_tree_from_head_delta`, and `git_write_tree`. +- Produces: `GitExportPolicy::{RequireMappedDelta, AllowFullSnapshot}`, `Trail::git_export_commit_mapped`, and `GitHandoffMetricsReport` embedded in Git export and agent apply reports. + +- [ ] **Step 1: Write a failing missing-mapping export test** + +Add a module test to `trail/src/db/merge/git_export.rs` that initializes Trail from the working tree inside a clean committed Git repository, records one Trail change, restores the Git worktree, and requests mapped export: + +```rust +#[cfg(test)] +mod tests { +use super::*; + +fn run_git(root: &Path, args: &[&str]) { + let output = Command::new("git").arg("-C").arg(root).args(args).output().unwrap(); + assert!(output.status.success(), "git failed: {}", String::from_utf8_lossy(&output.stderr)); +} + +#[test] +fn mapped_git_export_requires_preexisting_clean_mapping() { + if Command::new("git").arg("--version").output().is_err() { return; } + let temp = tempfile::tempdir().unwrap(); + run_git(temp.path(), &["init"]); + run_git(temp.path(), &["config", "user.email", "trail@example.test"]); + run_git(temp.path(), &["config", "user.name", "Trail Test"]); + fs::write(temp.path().join("README.md"), "one\n").unwrap(); + run_git(temp.path(), &["add", "README.md"]); + run_git(temp.path(), &["commit", "-m", "initial"]); + let init = Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + fs::write(temp.path().join("README.md"), "one\ntwo\n").unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let record = db.record(Some("main"), Some("change".into()), Actor::human(), false).unwrap(); + run_git(temp.path(), &["checkout", "--", "README.md"]); + let range = format!("{}..{}", init.operation.0, record.operation.unwrap().0); + let err = db.git_export_commit_mapped(&range, "mapped", None).unwrap_err(); + assert!(matches!(err, Error::GitMappingRequired(_))); + assert!(db.git_mappings(10).unwrap().is_empty()); +} +} +``` + +- [ ] **Step 2: Run the test and verify RED** + +Run: + +```bash +cargo test -p trail mapped_git_export_requires_preexisting_clean_mapping --lib -- --exact +``` + +Expected: compilation fails because `git_export_commit_mapped` does not exist. + +- [ ] **Step 3: Add policy and report types** + +Add internal types in `trail/src/db/mod.rs`: + +```rust +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum GitExportPolicy { + RequireMappedDelta, + AllowFullSnapshot, +} +``` + +Add a serializable report and embed it in `GitExportReport`: + +```rust +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct GitHandoffMetricsReport { + pub export_mode: String, + pub changed_path_count: u64, + pub blob_write_count: u64, + pub tracked_status_count: u64, + pub full_root_file_count: u64, +} + +pub performance: GitHandoffMetricsReport, +``` + +Refactor `git_export_commit` into a shared implementation: + +```rust +pub fn git_export_commit(&mut self, range: &str, message: &str) -> Result { + let state = self.current_git_state()?.ok_or_else(|| Error::Git("not a Git worktree".into()))?; + self.git_export_commit_with_state(range, message, state, GitExportPolicy::AllowFullSnapshot) +} + +pub(crate) fn git_export_commit_mapped( + &mut self, + range: &str, + message: &str, + state: Option, +) -> Result { + let state = match state { + Some(state) => state, + None => self.current_git_state()? + .ok_or_else(|| Error::Git("not a Git worktree".into()))?, + }; + self.git_export_commit_with_state(range, message, state, GitExportPolicy::RequireMappedDelta) +} +``` + +Inside `git_export_commit_with_state`, query `git_clean_head_matches_root_mapping` directly. Under `RequireMappedDelta`, return `Error::GitMappingRequired` on a miss; do not call `ensure_git_clean_head_root_mapping`, `load_root_files`, or `git_write_tree`. Under `AllowFullSnapshot`, retain the existing full export fallback. + +- [ ] **Step 4: Run mapped and general export tests and verify GREEN** + +Run: + +```bash +cargo test -p trail mapped_git_export_requires_preexisting_clean_mapping --lib -- --exact +cargo test -p trail --test e2e git_export_with_message_creates_commit_object_and_mapping -- --exact +cargo test -p trail --test e2e git_export_uses_clean_head_mapping_for_delta_commit -- --exact +``` + +Expected: all three tests pass; mapped export reports `performance.export_mode == "mapped_delta"`, while an unmapped general export reports `performance.export_mode == "full_snapshot"`. + +- [ ] **Step 5: Commit** + +```bash +git add trail/src/db/mod.rs trail/src/db/merge/git_export.rs trail/src/model/reports/worktree.rs +git commit -m "perf: require mapped delta git export" +``` + +--- + +### Task 3: One Git status query in high-level apply + +**Files:** +- Modify: `trail/src/db/mod.rs` +- Modify: `trail/src/db/core/init.rs` +- Modify: `trail/src/db/storage/git.rs` +- Modify: `trail/src/db/agent.rs` +- Modify: `trail/src/model/lane/activity.rs` +- Test: `trail/tests/e2e.rs` + +**Interfaces:** +- Consumes: `GitState`, `git_export_commit_mapped`, `git_fast_forward`, and existing agent apply reports. +- Produces: `GitIdentity { head, branch }`, `current_git_identity`, `tracked_git_state`, per-handoff structural metrics, and high-level apply that passes one state snapshot into export. + +- [ ] **Step 1: Write a failing structural-metrics regression test** + +Add this helper in `trail/tests/e2e.rs` and use `InitImportMode::GitTracked` for mapped tests or `InitImportMode::WorkingTree` for the missing-mapping test: + +```rust +fn ready_agent_lane_with_mode(mode: InitImportMode) -> (tempfile::TempDir, Trail) { + let temp = tempfile::tempdir().unwrap(); + run_git(temp.path(), &["init"]); + run_git(temp.path(), &["config", "user.email", "trail@example.test"]); + run_git(temp.path(), &["config", "user.name", "Trail Test"]); + fs::write(temp.path().join("README.md"), "base\n").unwrap(); + run_git(temp.path(), &["add", "README.md"]); + run_git(temp.path(), &["commit", "-m", "initial"]); + Trail::init(temp.path(), "main", mode, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("apply-bot", Some("main"), false, None, None).unwrap(); + let patch: PatchDocument = serde_json::from_value(serde_json::json!({ + "message": "one changed path", + "edits": [{"op": "write", "path": "AGENT.md", "content": "agent change\n"}] + })).unwrap(); + apply_lane_patch_at_head(&mut db, "apply-bot", patch).unwrap(); + db.agent_mark_reviewed("apply-bot", None).unwrap(); + (temp, db) +} +``` + +Use it in a new test: + +```rust +#[test] +fn agent_apply_reports_one_tracked_status_query() { + if !git_available() { return; } + let (temp, mut db) = ready_agent_lane_with_mode(InitImportMode::GitTracked); + let dry_run = db.agent_apply("apply-bot", true, None).unwrap(); + assert_eq!(dry_run.performance.tracked_status_count, 1); + assert_eq!(dry_run.performance.full_root_file_count, 0); + assert_eq!(dry_run.performance.export_mode, "mapped_delta"); + drop(temp); +} +``` + +Add a second test using a fresh `ready_agent_lane` fixture and actual apply; assert the same status/full-root counts, `changed_path_count == 1`, and `blob_write_count == 1`. + +Add a third test using a fresh fixture initialized with `InitImportMode::WorkingTree`. `agent_apply("apply-bot", true, None)` must return `Error::GitMappingRequired`, `git_mappings(10)` must remain empty, and Git HEAD must remain unchanged. + +- [ ] **Step 2: Run the focused test and verify RED** + +Run: + +```bash +cargo test -p trail --test e2e agent_apply_reports_one_tracked_status_query -- --exact +``` + +Expected: compilation fails because `AgentApplyReport.performance` does not exist. + +- [ ] **Step 3: Split cheap identity from tracked cleanliness** + +Add: + +```rust +#[derive(Debug, Clone)] +pub(crate) struct GitIdentity { + head: String, + branch: Option, +} +``` + +Add a `Copy + Default` internal counter record and one `Cell` field to `Trail`; initialize it in `Trail::open_at`: + +```rust +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct GitHandoffMetrics { + export_mode: GitExportMode, + changed_path_count: u64, + blob_write_count: u64, + tracked_status_count: u64, + full_root_file_count: u64, +} +``` + +Define `GitExportMode` with variants `Unknown`, `MappedDelta`, and `FullSnapshot`; implement `Default` as `Unknown` and convert the variants to the exact report strings `unknown`, `mapped_delta`, and `full_snapshot`. + +Reset it once at the start of `agent_apply` and general Git export. Increment `tracked_status_count` only where Trail invokes tracked-only status, `blob_write_count` in blob hashing, and `full_root_file_count` only in full snapshot export. Convert it into `GitHandoffMetricsReport` for every `AgentApplyReport` and `GitExportReport` constructor. + +Implement `current_git_identity` with `rev-parse --verify HEAD` and +`symbolic-ref --quiet --short HEAD`; it must not run `git status`. Implement +`tracked_git_state(identity)` with the single tracked-only status command. + +Refactor `agent_apply`: + +1. Resolve `GitIdentity` once. +2. Require the HEAD/base-root mapping with `git_clean_head_matches_root_mapping`. +3. For dry-run, obtain one `GitState` before returning the plan. +4. For actual apply, finish Trail recording/merge first, then obtain one + `GitState`, verify its HEAD equals `GitIdentity.head`, and pass it to + `git_export_commit_mapped`. +5. Remove `current_git_branch` and the full-root fallback in + `ensure_git_head_matches_root`. + +- [ ] **Step 4: Add HEAD-race and dirty-worktree tests** + +Add focused tests that exercise the internal validation helper: + +```rust +assert!(matches!( + validate_git_publication_state("old", &GitState { head: Some("new".into()), dirty: false }), + Err(Error::GitHeadChanged(_)) +)); +assert!(matches!( + validate_git_publication_state("head", &GitState { head: Some("head".into()), dirty: true }), + Err(Error::GitWorktreeDirty(_)) +)); +``` + +- [ ] **Step 5: Run the apply regression set and verify GREEN** + +Run: + +```bash +cargo test -p trail --test e2e agent_apply_reports_one_tracked_status_query -- --exact +cargo test -p trail --test e2e agent_start_custom_command_applies_task_to_git_with_guided_flow -- --exact +cargo test -p trail git_publication_state --lib +``` + +Expected: one Trail-issued status query per dry-run or actual apply; all apply safety behavior remains green. + +- [ ] **Step 6: Commit** + +```bash +git add trail/src/db/mod.rs trail/src/db/core/init.rs trail/src/db/storage/git.rs trail/src/db/agent.rs trail/src/model/lane/activity.rs trail/tests/e2e.rs +git commit -m "perf: reuse git state during agent apply" +``` + +--- + +### Task 4: Characterize Git-only path preservation and dry-run immutability + +**Files:** +- Modify: `trail/tests/e2e.rs` +- Modify: `trail/src/db/merge/git_export.rs` +- Modify: `trail/src/db/agent.rs` + +**Interfaces:** +- Consumes: mapped-delta export and high-level agent apply from Tasks 2-3. +- Produces: end-to-end evidence that symlinks/unmanaged Git paths survive and dry-run does not create mappings or Git objects. + +- [ ] **Step 1: Add the symlink-preservation characterization test** + +On Unix, create and commit `target.md` plus tracked symlink `link.md`, initialize Trail with `GitTracked`, create a ready no-materialize agent lane that adds `README.md`, and apply it. Assert: + +```rust +assert_eq!(git_output(temp.path(), &["show", "HEAD:link.md"]), "target.md"); +assert_eq!(git_output(temp.path(), &["show", "HEAD:target.md"]), "target\n"); +assert_eq!(git_output(temp.path(), &["show", "HEAD:README.md"]), "agent change"); +``` + +- [ ] **Step 2: Run the characterization test** + +Run: + +```bash +cargo test -p trail --test e2e agent_apply_preserves_git_only_symlinks -- --exact +``` + +Expected: PASS through mapped-delta export. A failure proves the mapped temporary-index implementation does not preserve the Git HEAD tree and must be corrected before continuing. + +- [ ] **Step 3: Write the dry-run immutability test** + +Capture before/after values for Git HEAD, `.git/index`, `git_mappings` count, and Git object count around `agent apply --dry-run`. Assert all are unchanged. + +- [ ] **Step 4: Run both tests and verify GREEN** + +Run: + +```bash +cargo test -p trail --test e2e agent_apply_preserves_git_only_symlinks -- --exact +cargo test -p trail --test e2e agent_apply_dry_run_writes_no_git_or_mapping_state -- --exact +``` + +Expected: both pass. + +- [ ] **Step 5: Commit** + +```bash +git add trail/tests/e2e.rs trail/src/db/merge/git_export.rs trail/src/db/agent.rs +git commit -m "test: lock down mapped agent handoff" +``` + +--- + +### Task 5: High-level Git apply scale harness and structural gates + +**Files:** +- Modify: `scripts/cli-scale-bench.sh` +- Modify: `scripts/check-cli-scale-thresholds.py` +- Modify: `.github/workflows/ci.yml` +- Modify: `.github/workflows/scale.yml` +- Modify: `docs/guides/performance-and-scale-benchmarks.md` + +**Interfaces:** +- Consumes: JSON `GitExportReport.performance.export_mode`, `changed_path_count`, and `blob_write_count` from Task 2. +- Produces: result rows `agent_git_apply_dry_run`, `agent_git_apply`, and `agent_git_apply_missing_mapping`; metric keys for export mode and changed/blob counts. + +- [ ] **Step 1: Extend the threshold checker test-first** + +Add Python unit coverage (new file `scripts/test_check_cli_scale_thresholds.py`) proving string equality gates and integer ceilings can be checked together: + +```python +import importlib.util +import pathlib +import tempfile +import unittest + +SCRIPT = pathlib.Path(__file__).with_name("check-cli-scale-thresholds.py") +SPEC = importlib.util.spec_from_file_location("scale_thresholds", SCRIPT) +module = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(module) + +class ThresholdMetricTests(unittest.TestCase): + def test_reads_structural_string_metrics(self): + with tempfile.TemporaryDirectory() as directory: + metrics = pathlib.Path(directory) / "metrics.tsv" + metrics.write_text( + "agent_git_export_mode\tmapped_delta\n" + "agent_git_changed_paths\t1\n" + ) + parsed = module.read_metric_values(metrics) + self.assertEqual(parsed["agent_git_export_mode"], "mapped_delta") + self.assertEqual(parsed["agent_git_changed_paths"], 1.0) +``` + +Run `python3 -m unittest scripts/test_check_cli_scale_thresholds.py` and verify RED because only numeric metrics are supported. + +- [ ] **Step 2: Implement structural metric parsing and gates** + +Add `--metric-equals key=value` support while preserving existing numeric `key=max_value` behavior. Run the unit test and verify GREEN. + +- [ ] **Step 3: Add a committed Git high-level agent scenario** + +In `cli-scale-bench.sh`, create a no-materialize lane from the Git fixture's mapped `main`, apply a structured patch touching `k = max(1, min(100, files / 1000))` files, mark the task reviewed/ready using the same gates as existing agent fixtures, then time: + +```bash +run_timed "$scale" agent_git_apply_dry_run "$BIN" --workspace "$GIT_REPO" --json agent apply latest --dry-run +run_timed "$scale" agent_git_apply "$BIN" --workspace "$GIT_REPO" --json agent apply latest +``` + +Parse the actual apply JSON into: + +```text +agent_git_export_modemapped_delta +agent_git_changed_paths${changed_paths} +agent_git_blob_writes${blob_writes} +``` + +Create a second `InitImportMode::WorkingTree` Git fixture without a mapping and assert `agent apply --dry-run` exits with `GIT_MAPPING_REQUIRED` within the timed row `agent_git_apply_missing_mapping`. + +- [ ] **Step 4: Add CI and scheduled gates** + +CI 1k ceilings: + +```text +agent_git_apply_dry_run=15 +agent_git_apply=20 +agent_git_apply_missing_mapping=5 +``` + +Scheduled ceilings: + +```text +100k: dry-run=2, apply=3, missing=1 +1M: dry-run=5, apply=8, missing=1 +``` + +For every scale, assert `agent_git_export_mode=mapped_delta`, `agent_git_changed_paths=k`, and `agent_git_blob_writes<=k`. + +- [ ] **Step 5: Run the smoke harness** + +Run: + +```bash +TRAIL_SCALE_FILES=1000 TRAIL_SCALE_MATERIALIZED=0 TRAIL_SCALE_BACKUP=0 TRAIL_SCALE_DAEMON=0 TRAIL_SCALE_GIT_IMPORT=1 make bench-cli-scale +``` + +Expected: all new rows exit `0`; structural metric gates pass. + +- [ ] **Step 6: Document the mapped hot path** + +Update the performance guide with the new rows, the explicit reconciliation requirement, and the rule that missing mapping is a fast error rather than a correctness fallback. + +- [ ] **Step 7: Commit** + +```bash +git add scripts/cli-scale-bench.sh scripts/check-cli-scale-thresholds.py scripts/test_check_cli_scale_thresholds.py .github/workflows/ci.yml .github/workflows/scale.yml docs/guides/performance-and-scale-benchmarks.md +git commit -m "perf: gate high-level git apply at scale" +``` + +--- + +### Task 6: Full verification and release evidence + +**Files:** +- Modify only if verification exposes defects in files already listed above. + +**Interfaces:** +- Consumes: all production and test interfaces from Tasks 1-5. +- Produces: passing formatting, lint, focused E2E, workspace tests, smoke scale evidence, and a clean diff. + +- [ ] **Step 1: Format and lint** + +Run: + +```bash +cargo fmt --all --check +cargo clippy -p trail --all-targets -- -D warnings +``` + +Expected: exit `0` with no warnings. + +- [ ] **Step 2: Run focused Git and agent tests** + +Run: + +```bash +cargo test -p trail --test e2e git_export -- --nocapture +cargo test -p trail --test e2e agent_apply -- --nocapture +cargo test -p trail --test e2e agent_start_custom_command_applies_task_to_git_with_guided_flow -- --exact +``` + +Expected: all matching tests pass. + +- [ ] **Step 3: Run the Trail package test suite** + +Run: + +```bash +cargo test -p trail +``` + +Expected: all tests pass. + +- [ ] **Step 4: Run 100k mapped handoff acceptance** + +Run: + +```bash +TRAIL_SCALE_FILES=100000 TRAIL_SCALE_MATERIALIZED=0 TRAIL_SCALE_BACKUP=0 TRAIL_SCALE_DAEMON=1 TRAIL_SCALE_GIT_IMPORT=1 make bench-cli-scale +``` + +Expected: mapped dry-run <= 2 seconds, actual apply <= 3 seconds, missing mapping <= 1 second, and all structural gates pass. + +- [ ] **Step 5: Verify diff integrity** + +Run: + +```bash +git diff --check +git status --short +``` + +Expected: no whitespace errors; status contains only intentional files or is clean after commits. + +- [ ] **Step 6: Commit any verification fixes** + +If verification required code changes, first add a failing regression test for each defect, then commit only the affected files: + +```bash +git add trail/src/error.rs trail/src/cli/command/handler/errors.rs trail/src/db/mod.rs trail/src/db/core/init.rs trail/src/db/storage/git.rs trail/src/db/merge/git_export.rs trail/src/db/agent.rs trail/src/model/reports/worktree.rs trail/src/model/lane/activity.rs trail/tests/e2e.rs scripts/cli-scale-bench.sh scripts/check-cli-scale-thresholds.py scripts/test_check_cli_scale_thresholds.py .github/workflows/ci.yml .github/workflows/scale.yml docs/guides/performance-and-scale-benchmarks.md +git commit -m "fix: close git handoff verification gaps" +``` diff --git a/docs/superpowers/plans/2026-07-13-acp-checkpoint-integrity.md b/docs/superpowers/plans/2026-07-13-acp-checkpoint-integrity.md new file mode 100644 index 0000000..879baf6 --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-acp-checkpoint-integrity.md @@ -0,0 +1,130 @@ +# ACP Checkpoint Integrity Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ensure ACP never reports a materialized edit turn as successful and unchanged when Trail could not create its checkpoint, and prevent ACP startup against legacy live roots that cannot accept mutations. + +**Architecture:** Add a read-only live-root path-index preflight to the Trail core and invoke it from ACP startup and ACP doctor. At prompt completion, convert workdir-record failures into durable turn evidence and a failed outcome while preserving the dirty workdir for recovery. Keep successful prompt-end reconciliation as the authoritative one-operation checkpoint for materialized edits. + +**Tech Stack:** Rust 2024, Trail SQLite/object store, ACP JSON-RPC relay, Cargo unit and integration tests. + +## Global Constraints + +- Do not auto-rebuild legacy indexes in an ACP hot path; return `PATH_INDEX_REQUIRED` with `trail index rebuild` recovery guidance. +- Never discard a workdir checkpoint error. +- Never set `outcome.no_changes=true` when workdir reconciliation observed edits but checkpointing failed. +- Preserve existing unrelated worktree changes. +- Use test-first red-green cycles and verify the real subprocess relay path. + +--- + +### Task 1: Live-root path-index preflight + +**Files:** +- Modify: `trail/src/db/storage/files.rs` +- Modify: `trail/src/acp.rs` +- Modify: `trail/src/cli/command/handler/acp.rs` +- Test: `trail/src/acp.rs` +- Test: `trail/tests/e2e.rs` + +**Interfaces:** +- Produces: `Trail::ensure_live_path_invariant_indexes(&self) -> Result<()>`. +- Consumes: immutable live branch/lane refs and `WorktreeRoot.case_fold_map_root`. + +- [x] **Step 1: Write failing tests** + +Add a coordinator test that converts a non-empty current root to a legacy root and expects `CaptureCoordinator::new` to return `Error::PathIndexRequired`. Add a CLI test expecting ACP doctor to include a failed `path_invariant_index` check and overall failed status. + +- [x] **Step 2: Verify red** + +Run: + +```bash +cargo test -p trail acp_rejects_legacy_live_root_before_start -- --exact +cargo test -p trail --test e2e acp_doctor_reports_legacy_path_index -- --exact +``` + +Expected: the coordinator currently starts and doctor lacks the check. + +- [x] **Step 3: Implement minimal preflight** + +Add a read-only method that loads distinct live `refs/branches/*` and `refs/lanes/*` roots and returns: + +```rust +Error::PathIndexRequired( + "live ref `` has a legacy root with no case-fold index; run `trail index rebuild`" + .to_string(), +) +``` + +Call it from `CaptureCoordinator::new` before the upstream process starts. Add the same result as ACP doctor's `path_invariant_index` check and mark the report failed on error. + +- [x] **Step 4: Verify green** + +Re-run both exact tests and the existing modern-workspace ACP doctor test. + +### Task 2: Durable checkpoint-failure outcome + +**Files:** +- Modify: `trail/src/acp.rs` +- Test: `trail/src/acp.rs` + +**Interfaces:** +- Produces: an `acp_workdir_checkpoint_failed` turn event with stable Trail error code and message. +- Produces: failed turn envelope with `checkpoint=null`, `no_changes=false`, and a checkpoint error summary. + +- [x] **Step 1: Write the failing regression test** + +Start a materialized ACP session on a modern root, create a file in its lane workdir, replace the lane ref root with an equivalent legacy root after startup, and complete the prompt. Assert the turn is failed, does not claim `no_changes`, contains no checkpoint, retains the dirty file, and records `acp_workdir_checkpoint_failed` with `PATH_INDEX_REQUIRED`. + +- [x] **Step 2: Verify red** + +Run: + +```bash +cargo test -p trail acp_checkpoint_failure_is_durable_and_not_no_changes -- --exact +``` + +Expected: current code discards the record error and emits completed/no-changes. + +- [x] **Step 3: Implement minimal failure handling** + +Replace both ignored `record_lane_workdir_for_turn` results with a shared helper. On failure, store the diagnostic event, emit a stderr warning, finalize the turn as failed, preserve `checkpoint=null`, force `no_changes=false`, and retain the original upstream stop reason. Apply the same helper during relay closeout. + +- [x] **Step 4: Verify green** + +Run the exact regression test plus existing checkpoint/no-change envelope tests. + +### Task 3: End-to-end relay verification and Lore recovery + +**Files:** +- Modify: `trail/tests/e2e.rs` +- Modify only through Trail maintenance: `/Users/haipingfu/Github/lore/.trail` + +**Interfaces:** +- Consumes: compiled `trail` binary and fake ACP subprocess. +- Produces: one `LaneRecord` checkpoint tied to a completed prompt with changed paths and clean lane workdir. + +- [x] **Step 1: Strengthen the subprocess success assertion** + +Extend `acp_relay_captures_session_prompt_mcp_and_workdir_edits` to assert a non-null outcome checkpoint, `no_changes=false`, a `workdir_recorded` event, and clean materialized workdir state. + +- [x] **Step 2: Run focused and broad verification** + +Run: + +```bash +cargo fmt --check +cargo test -p trail acp_ +cargo test -p trail --test e2e acp_relay_captures_session_prompt_mcp_and_workdir_edits -- --exact +cargo clippy -p trail --all-targets -- -D warnings +cargo build -p trail --release +``` + +- [x] **Step 3: Back up and migrate Lore** + +Create and verify a Trail backup, run `trail index rebuild`, and re-run the read-only lane-record preview for `agent-claude-code-af2b5ec2c3f5`. + +- [x] **Step 4: Recover and verify the dirty task state** + +Record the ten existing files into a new recoverable lane operation, then verify lane status, timeline, transcript diagnostics, checkpoint visibility, and rewind preview. Do not apply the lane to Git. diff --git a/docs/superpowers/plans/2026-07-13-activated-changed-path-ledger.md b/docs/superpowers/plans/2026-07-13-activated-changed-path-ledger.md new file mode 100644 index 0000000..1938e07 --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-activated-changed-path-ledger.md @@ -0,0 +1,1439 @@ +# Activated Changed-Path Ledger Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make warm `status`, `diff`, and `record` proportional to authoritative changed-path candidates on large Linux and macOS repositories without ever returning false clean. + +**Architecture:** A schema-v18-only `ChangedPathLedger` binds trust and source-sequenced evidence to a workspace/lane/view identity. A securely auto-started per-workspace daemon owns the qualified native observer, appends durable event segments, and reconciles automatically when continuity is unavailable; all readers use the same fenced snapshot API, and all filesystem producers use durable intents. Activation remains compiled off until the schema, recovery, producer inventory, Linux inotify suite, macOS FSEvents suite, and correctness/performance gates pass together. + +**Tech Stack:** Rust 1.81, rusqlite/SQLite WAL, Prolly maps, clap, serde/CBOR, SHA-256, native inotify on Linux, native FSEvents on macOS, existing Trail daemon/REST/MCP surfaces, cargo-nextest-compatible tests. + +## Global Constraints + +- Schema version is exactly `18`; only explicit fresh `trail init` creates it. +- Existing version `0`, version `17`, versions greater than `18`, and partial or malformed version `18` return `SCHEMA_REINITIALIZE_REQUIRED` before any mutable store opens. +- A rejected open leaves `.trail` database files and sidecars byte-for-byte unchanged and instructs the user to back up and run `trail init --force`. +- There are no migrations, backfills, compatibility views, legacy trust states, or repair-on-open paths. +- Only `trusted` may prove clean; `reconciling`, `overflow`, `untrusted_gap`, `stale_baseline`, and `corrupt` fail closed. +- Linux and macOS must pass real native-adapter suites before activation; generic `notify` is advisory only. +- Unsupported platforms retain direct full observation and never persist a trusted ledger scope. +- Observer callbacks never acquire the workspace lock or primary SQLite connection. +- Observer segment and intent durability is explicit; SQLite remains WAL `synchronous=NORMAL`, so dead-owner restart reconciles unless a qualified durable cursor proves continuity. +- The first `status`, `diff`, or `record` securely discovers or starts exactly one daemon for the workspace. +- Stale, overflowed, corrupt, or unavailable ledger state automatically runs a full filesystem reconciliation and continues; `CHANGE_LEDGER_RECONCILE_REQUIRED` is returned only when startup, observation, or reconciliation cannot establish trust. +- Warm trusted paths must have zero full-scope walks, full root ranges, full SQLite index reads, legacy manifest I/O, upper recovery walks, external-adapter global work, and policy dependency rediscovery. +- Every task is test-first, independently reviewable, and committed separately; the final activation commit is the only commit that permits persisted trust to drive public reads. + +--- + +## File Structure + +### New changed-ledger module + +- `trail/src/db/change_ledger/mod.rs`: public crate-level façade and the only entry point consumers call. +- `trail/src/db/change_ledger/types.rs`: identities, trust states, capabilities, cuts, evidence, snapshots, intents, and reconciliation types. +- `trail/src/db/change_ledger/store.rs`: SQLite persistence, binary path/prefix queries, caps, and CAS transitions. +- `trail/src/db/change_ledger/log.rs`: versioned segment codec, checksum/hash chain, rotation, durability offsets, and tail recovery. +- `trail/src/db/change_ledger/snapshot.rs`: live fence acquisition, tail folding, candidate snapshot, and sequence-aware acknowledgement. +- `trail/src/db/change_ledger/reconcile.rs`: streamed no-follow reconciliation and guarded publication. +- `trail/src/db/change_ledger/policy.rs`: authoritative recording policy fingerprint and persisted dependency manifest. +- `trail/src/db/change_ledger/intent.rs`: controlled-writer intent preparation and lifecycle transitions. +- `trail/src/db/change_ledger/recovery.rs`: owner, segment, intent, mirror, backup/restore, and GC recovery decisions. +- `trail/src/db/change_ledger/observer/mod.rs`: observer trait, capability qualification, owner lease, and platform selection. +- `trail/src/db/change_ledger/observer/linux.rs`: native recursive inotify adapter and sentinel fence. +- `trail/src/db/change_ledger/observer/macos.rs`: native FSEvents adapter, persistent event ID, root identity, and synchronous flush fence. + +### Existing files with focused changes + +- `trail/src/db/storage/schema/ddl.rs` and `trail/src/db/storage/schema/changed_path_ledger.rs`: exact one-shot v18 creation and validation. +- `trail/src/db/core/init.rs`: distinguish fresh creation from read-only preflight before Prolly/SQLite mutable opens. +- `trail/src/error.rs`: stable schema-reinitialize and reconcile-required errors. +- `trail/src/db/storage/worktree_index.rs`: remove the persisted daemon JSON cache as an authority; retain its full scan only as reconciliation/oracle code. +- `trail/src/cli/command/handler/daemon_rpc.rs` plus a new `daemon_start.rs`: authenticated per-workspace discovery/startup. +- `trail/src/db/core/status.rs`, `trail/src/db/record/diff.rs`, and `trail/src/db/record/recording/manual.rs`: consume one fenced ledger snapshot flow. +- `trail/src/db/record/checkout.rs`, `trail/src/db/lane/patching.rs`, `trail/src/db/lane/workdir/{materialize,sync,record}.rs`: controlled intents and lane scopes. +- `trail/src/db/lane/workdir/{view_journal,view_core}.rs` and `trail/src/db/lane/workspace_view.rs`: durable view intent journal and compact v2 marker. +- `trail/src/db/core/backup/{create,restore,verify}.rs` and `trail/src/db/storage/lifecycle/gc.rs`: fencing, epoch rotation, and intent GC roots. +- `trail/src/model/reports/{maintenance,worktree,lane}.rs`, CLI rendering, OpenAPI, server routes, and MCP tools: structured diagnostics and metrics. +- `trail/tests/changed_path_ledger_*.rs`, `trail/tests/e2e.rs`, and scale scripts: hard-cutover, state/property, real-adapter, crash, E2E, and locality gates. + +--- + +### Task 1: Replace migration prototype with exact v18 hard cutover + +**Files:** +- Modify: `trail/src/db/mod.rs` +- Modify: `trail/src/db/core/init.rs` +- Modify: `trail/src/db/storage/schema.rs` +- Modify: `trail/src/db/storage/schema/ddl.rs` +- Replace: `trail/src/db/storage/schema/changed_path_ledger.rs` +- Modify: `trail/src/error.rs` +- Test: `trail/tests/e2e.rs` +- Create: `trail/tests/schema_v18_hard_cutover.rs` + +**Interfaces:** +- Consumes: existing `Trail::init_with_options`, `Trail::open_at_without_recovery`, and `TRAIL_SCHEMA_VERSION`. +- Produces: `SchemaOpenMode::{FreshCreate, Existing}`, `preflight_existing_schema(&Path) -> Result<()>`, `create_schema_v18(&Connection) -> Result<()>`, `validate_schema_v18(&Connection) -> Result<()>`, and `Error::SchemaReinitializeRequired { found: String, guidance: String }`. + +- [ ] **Step 1: Write rejection and no-mutation tests before changing open behavior** + +```rust +#[test] +fn existing_v17_is_rejected_without_mutating_any_trail_byte() { + let fixture = SchemaFixture::versioned(17); + let before = fixture.snapshot_tree_bytes(); + let err = Trail::open(fixture.root()).unwrap_err(); + assert_eq!(err.code(), "SCHEMA_REINITIALIZE_REQUIRED"); + assert!(err.to_string().contains("trail init --force")); + assert_eq!(fixture.snapshot_tree_bytes(), before); +} + +#[test] +fn partial_v18_is_rejected_without_repair() { + let fixture = SchemaFixture::partial_v18("changed_path_scopes"); + let before = fixture.snapshot_tree_bytes(); + let err = Trail::open(fixture.root()).unwrap_err(); + assert_eq!(err.code(), "SCHEMA_REINITIALIZE_REQUIRED"); + assert_eq!(fixture.snapshot_tree_bytes(), before); +} +``` + +- [ ] **Step 2: Run the focused hard-cutover tests and capture the expected failure** + +Run: `cargo test -p trail --test schema_v18_hard_cutover -- --nocapture` + +Expected: FAIL because existing schemas are migrated/repaired or mutable sidecars are opened before validation. + +- [ ] **Step 3: Add the stable error contract** + +```rust +#[error("workspace schema {found} cannot be opened; {guidance}")] +SchemaReinitializeRequired { found: String, guidance: String }, +#[error("changed-path ledger reconciliation required for {scope}: {reason}; run `{command}`")] +ChangeLedgerReconcileRequired { + scope: String, + state: String, + reason: String, + command: String, +}, +``` + +Map these to `SCHEMA_REINITIALIZE_REQUIRED`/exit `15` and `CHANGE_LEDGER_RECONCILE_REQUIRED`/exit `16`; add unit assertions for both codes, exit statuses, and exact recovery commands. + +- [ ] **Step 4: Introduce explicit open modes and a read-only preflight** + +```rust +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum SchemaOpenMode { FreshCreate, Existing } + +fn preflight_existing_schema(db_path: &Path) -> Result<()> { + let flags = rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY + | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX; + let conn = rusqlite::Connection::open_with_flags(db_path, flags) + .map_err(schema_reinitialize_error)?; + validate_schema_v18(&conn).map_err(schema_reinitialize_error) +} + +fn schema_reinitialize_error(err: impl std::fmt::Display) -> Error { + Error::SchemaReinitializeRequired { + found: err.to_string(), + guidance: "back up this workspace, then run `trail init --force` to create schema v18".into(), + } +} +``` + +Call `preflight_existing_schema` before `open_prolly_store`, before opening a read-write SQLite connection, and before creating any directory or sidecar. Only `Trail::init_with_options` passes `FreshCreate`; remove its redundant second `init_schema` call. + +- [ ] **Step 5: Replace migration/backfill/repair DDL with a one-savepoint fresh creator** + +```rust +pub(crate) fn create_schema_v18(conn: &Connection) -> Result<()> { + if conn.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))? != 0 { + return Err(Error::Corrupt("fresh schema connection is not empty".into())); + } + conn.execute_batch("SAVEPOINT create_v18;")?; + let result = (|| { + conn.execute_batch(BASE_SCHEMA_V18)?; + conn.execute_batch(CHANGED_PATH_LEDGER_SCHEMA_V18)?; + validate_schema_v18_shape(conn)?; + conn.execute("INSERT INTO schema_meta(key,value) VALUES('schema_version','18')", [])?; + conn.pragma_update(None, "user_version", 18_i64)?; + validate_schema_v18(conn) + })(); + match result { + Ok(()) => conn.execute_batch("RELEASE create_v18;").map_err(Into::into), + Err(err) => { let _ = conn.execute_batch("ROLLBACK TO create_v18; RELEASE create_v18;"); Err(err) } + } +} +``` + +The final ledger DDL must include scopes, entries, prefixes, policy dependencies, intents, intent paths, intent prefixes, reconciliations, observer segments, and observer owners with exact checks/FKs/unique indexes and binary path collation. Do not use `IF NOT EXISTS` in creation or validation. + +- [ ] **Step 6: Add exact-shape validation tests** + +Validate `sqlite_master.sql`, `PRAGMA table_xinfo`, `foreign_key_list`, `index_xinfo`, check/default clauses, schema metadata, `user_version=18`, and the observer log format min/max. Mutate one attribute per test and assert read-only rejection. + +- [ ] **Step 7: Run schema and open-path tests** + +Run: `cargo test -p trail --test schema_v18_hard_cutover && cargo test -p trail db::storage::schema error::tests` + +Expected: PASS; snapshots for v0/v17/v19/partial-v18 are byte-identical before and after attempted open. + +- [ ] **Step 8: Commit only the hard-cutover slice** + +```bash +git add trail/src/db/mod.rs trail/src/db/core/init.rs trail/src/db/storage/schema.rs trail/src/db/storage/schema/ddl.rs trail/src/db/storage/schema/changed_path_ledger.rs trail/src/error.rs trail/tests/e2e.rs trail/tests/schema_v18_hard_cutover.rs +git commit -m "feat: enforce schema v18 hard cutover" +``` + +### Task 2: Add typed ledger state and exact SQLite CAS persistence + +**Files:** +- Create: `trail/src/db/change_ledger/mod.rs` +- Create: `trail/src/db/change_ledger/types.rs` +- Create: `trail/src/db/change_ledger/store.rs` +- Modify: `trail/src/db/mod.rs` +- Create: `trail/tests/changed_path_ledger_state.rs` + +**Interfaces:** +- Consumes: v18 tables from Task 1, existing `ObjectId`, `ChangeId`, and ref generation/root values. +- Produces: `ChangedPathLedger<'a>`, `ScopeId`, `ScopeIdentity`, `BaselineIdentity`, `ProviderCapabilities`, `TrustState`, `EvidenceCut`, `CandidateSnapshot`, `ExpectedScope`, `begin_scope`, `mark_prefix_dirty`, and CAS methods used by every later task. + +- [ ] **Step 1: Write state-machine and stale-CAS tests** + +```rust +#[test] +fn only_trusted_scope_can_return_an_authoritative_snapshot() { + for state in [TrustState::Reconciling, TrustState::Overflow, + TrustState::UntrustedGap, TrustState::StaleBaseline, TrustState::Corrupt] { + let store = fixture_with_state(state); + assert!(matches!(store.snapshot_candidates(expected()), Err(Error::ChangeLedgerReconcileRequired { .. }))); + } +} + +#[test] +fn stale_epoch_or_ref_cannot_advance_baseline() { + let store = trusted_fixture(); + assert_eq!(store.advance_baseline(stale_expected(), target()).unwrap_err().code(), + "CHANGE_LEDGER_RECONCILE_REQUIRED"); +} +``` + +- [ ] **Step 2: Run the state tests to verify missing APIs fail** + +Run: `cargo test -p trail --test changed_path_ledger_state` + +Expected: compile failure for missing `change_ledger` types and methods. + +- [ ] **Step 3: Define serializable identities, capabilities, and states** + +```rust +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum TrustState { Trusted, Reconciling, Overflow, UntrustedGap, StaleBaseline, Corrupt } + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ProviderCapabilities { + pub durable_cursor: bool, + pub linearizable_fence: bool, + pub rename_pairing: bool, + pub overflow_scope: bool, + pub filesystem_supported: bool, + pub clean_proof_allowed: bool, + pub power_loss_durability: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub(crate) struct ScopeId(pub [u8; 32]); +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) enum ScopeKind { Workspace, MaterializedLane, WorkspaceView } +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +pub(crate) struct LedgerPath(pub String); +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ScopeIdentity { pub scope_id: ScopeId, pub kind: ScopeKind, pub owner_id: String } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct BaselineIdentity { pub ref_name: String, pub ref_generation: u64, pub change_id: ChangeId, pub root_id: ObjectId } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PolicyIdentity { pub fingerprint: [u8; 32], pub generation: u64 } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct FilesystemIdentity(pub Vec); +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ProviderIdentity { pub identity: Vec, pub capabilities: ProviderCapabilities } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct EvidenceCut { pub source: EvidenceSource, pub sequence: u64, pub durable_offset: u64 } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct DirtyPrefix { pub path: LedgerPath, pub complete: bool, pub reason: String, pub first_sequence: u64, pub last_sequence: u64 } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct OwnedEvidence { pub source: EvidenceSource, pub through_sequence: u64, pub exact_paths: Vec, pub prefixes: Vec } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CandidateSnapshot { pub expected: ExpectedScope, pub cut: EvidenceCut, pub exact_paths: Vec, pub prefixes: Vec, pub trust: TrustState } + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ExpectedScope { + pub scope_id: ScopeId, + pub epoch: u64, + pub ref_name: String, + pub ref_generation: u64, + pub baseline_root: ObjectId, + pub policy_fingerprint: [u8; 32], + pub policy_generation: u64, + pub filesystem_identity: Vec, + pub provider_identity: Vec, +} +``` + +Also define `EvidenceSource::{Observer, Intent, Reconciliation, GitAdvisory}` and a bitmask `EvidenceFlags` for create/content/mode/delete/rename-from/rename-to. `LedgerPath::parse` rejects absolute, parent-escaping, NUL, and unsupported/non-UTF-8 inputs and preserves exact case. + +- [ ] **Step 4: Implement one façade and CAS store operations** + +```rust +pub(crate) struct ChangedPathLedger<'a> { conn: &'a Connection } + +impl<'a> ChangedPathLedger<'a> { + pub(crate) fn new(conn: &'a Connection) -> Self { Self { conn } } + pub(crate) fn begin_scope(&self, identity: &ScopeIdentity, baseline: &BaselineIdentity, policy: &PolicyIdentity, filesystem: &FilesystemIdentity, provider: &ProviderIdentity) -> Result; + pub(crate) fn mark_prefix_dirty(&self, expected: &ExpectedScope, prefix: &DirtyPrefix) -> Result<()>; + pub(crate) fn mark_untrusted(&self, expected: &ExpectedScope, state: TrustState, reason: &str) -> Result<()>; + pub(crate) fn snapshot_candidates(&self, expected: &ExpectedScope) -> Result; + pub(crate) fn acknowledge(&self, expected: &ExpectedScope, cut: &EvidenceCut, owned: &OwnedEvidence) -> Result<()>; + pub(crate) fn advance_baseline(&self, expected: &ExpectedScope, target: &BaselineIdentity, cut: &EvidenceCut) -> Result<()>; +} +``` + +`begin_scope` inserts `untrusted_gap` with a fresh epoch and cannot accept `trusted` from its caller. Only reconciliation publication may perform the initial promotion. + +Every update includes `WHERE scope_id=? AND epoch=? AND ref_generation=? AND baseline_root=? AND policy_fingerprint=? AND filesystem_identity=? AND provider_identity=?`; zero changed rows is a stale CAS, never success. + +- [ ] **Step 5: Implement binary exact/prefix evidence and caps** + +Use half-open binary ranges `(prefix, prefix_successor)` and indexed `COLLATE BINARY`, merge overlapping complete prefixes while retaining the earliest/latest source sequence and completeness reason, and atomically mark `overflow` when persisted entry/prefix/byte/tail caps are exceeded. + +- [ ] **Step 6: Add property tests for event order, coalescing, and acknowledgement** + +```rust +proptest! { + #[test] + fn acknowledgement_never_clears_later_or_other_source_evidence(events in event_sequences()) { + let (store, c1) = apply_events_and_cut(events); + store.acknowledge(&expected(), &c1, &owned_before(c1)).unwrap(); + prop_assert!(store.all_remaining().iter().all(|e| e.sequence > c1.sequence || e.source != c1.source)); + } +} +``` + +- [ ] **Step 7: Run and commit the typed store** + +Run: `cargo test -p trail --test changed_path_ledger_state` + +Expected: PASS for every trust transition, stale CAS, prefix coalescing, and sequence boundary. + +```bash +git add trail/src/db/mod.rs trail/src/db/change_ledger trail/tests/changed_path_ledger_state.rs +git commit -m "feat: add changed-path ledger state store" +``` + +### Task 3: Implement the durable observer segment protocol + +**Files:** +- Create: `trail/src/db/change_ledger/log.rs` +- Create: `trail/tests/changed_path_ledger_log.rs` +- Modify: `trail/Cargo.toml` + +**Interfaces:** +- Consumes: `ScopeId`, `LedgerPath`, `EvidenceFlags`, and `EvidenceSource` from Task 2. +- Produces: `SegmentWriter::acquire`, `SegmentWriter::append`, `SegmentWriter::flush_durable`, `SegmentWriter::rotate`, `recover_segments`, `DurableCut`, and `RecoveredTail`. + +- [ ] **Step 1: Write codec, torn-tail, hash-chain, and retired-owner tests** + +```rust +#[test] +fn torn_tail_recovers_only_through_last_checked_record() { + let mut bytes = encoded_segment(&[event(1), event(2)]); + bytes.truncate(bytes.len() - 7); + let recovered = recover_bytes(&bytes).unwrap(); + assert_eq!(recovered.records, vec![event(1)]); + assert!(recovered.requires_reconciliation); +} + +#[test] +fn retired_owner_cannot_append_to_reused_epoch() { + let first = fixture.acquire(epoch(3), owner("a")).unwrap(); + fixture.replace_owner(epoch(4), owner("b")).unwrap(); + assert!(first.append(&event(1)).is_err()); +} +``` + +- [ ] **Step 2: Run the log tests and verify they fail before implementation** + +Run: `cargo test -p trail --test changed_path_ledger_log` + +Expected: compile failure for missing segment protocol. + +- [ ] **Step 3: Define a bounded, versioned on-disk record format** + +```rust +const SEGMENT_MAGIC: &[u8; 8] = b"TRAILCPL"; +const LOG_FORMAT_VERSION: u16 = 1; +const MAX_RECORD_BYTES: u32 = 1024 * 1024; + +#[derive(Serialize, Deserialize)] +struct SegmentHeader { + format: u16, + scope_id: ScopeId, + epoch: u64, + owner_token: [u8; 32], + provider_cursor: Vec, + previous_segment_hash: [u8; 32], +} +``` + +Encode `length | sequence | source | payload | previous_record_hash | checksum`; reject unsupported format, non-monotonic sequence, over-limit length, checksum mismatch, broken link, owner mismatch, and segment lineage mismatch. + +- [ ] **Step 4: Implement exclusive owner lease, append, group flush, and rotation** + +The writer verifies the lease token before each batch, syncs record bytes before publishing `durable_end_offset`, syncs the old segment and directory before publishing the next header, and revokes trust in memory immediately on append/flush/disk-full/lease/heartbeat failure. + +- [ ] **Step 5: Implement conservative tail recovery** + +```rust +pub(crate) struct RecoveredTail { + pub records: Vec, + pub durable_end: u64, + pub last_sequence: u64, + pub last_hash: [u8; 32], + pub requires_reconciliation: bool, +} +``` + +Never skip a corrupt middle record. A partial final record is discarded and marks the scope untrusted. Recovery must bound records and bytes before allocation. + +- [ ] **Step 6: Run fault tests and commit** + +Run: `cargo test -p trail --test changed_path_ledger_log` + +Expected: PASS for clean recovery, torn writes, corrupt checksum/linkage, disk-full injection, rotation crash points, stale owner, and cap exhaustion. + +```bash +git add trail/Cargo.toml trail/src/db/change_ledger/log.rs trail/tests/changed_path_ledger_log.rs +git commit -m "feat: add durable changed-path observer log" +``` + +### Task 4: Persist policy dependencies before filtering + +**Files:** +- Create: `trail/src/db/change_ledger/policy.rs` +- Modify: `trail/src/db/storage/worktree_index.rs` +- Create: `trail/tests/changed_path_ledger_policy.rs` + +**Interfaces:** +- Consumes: v18 `changed_path_policy_dependencies`, `ExpectedScope`, existing `RecordingPolicySnapshot`, and Trail/Git ignore walkers. +- Produces: `CompiledPolicy`, `PolicyDependency`, `PolicyManifest`, `compile_policy`, `validate_policy_manifest`, and `raw_event_invalidates_policy`. + +- [ ] **Step 1: Write nested/global/config dependency invalidation tests** + +```rust +#[test] +fn raw_nested_ignore_event_stales_scope_before_ignore_filtering() { + let policy = fixture.compile_policy().unwrap(); + assert!(raw_event_invalidates_policy(&policy, path("src/.gitignore"))); +} + +#[test] +fn unchanged_manifest_reuses_policy_without_tree_discovery() { + let (policy, metrics) = fixture.load_policy_twice().unwrap(); + assert_eq!(policy.fingerprint, fixture.expected_fingerprint()); + assert_eq!(metrics.policy_dependency_full_discovery, 1); +} +``` + +- [ ] **Step 2: Run the focused policy tests** + +Run: `cargo test -p trail --test changed_path_ledger_policy` + +Expected: FAIL because dependencies are not persisted or raw events are filtered first. + +- [ ] **Step 3: Define the authoritative manifest** + +```rust +pub(crate) struct PolicyDependency { + pub identity: String, + pub kind: PolicyDependencyKind, + pub content_identity: [u8; 32], + pub metadata_identity: Vec, + pub observable: bool, + pub generation: u64, + pub last_source_sequence: u64, +} + +pub(crate) struct CompiledPolicy { + pub snapshot: RecordingPolicySnapshot, + pub fingerprint: [u8; 32], + pub dependencies: Vec, + pub adapter_equivalence: AdapterEquivalence, +} +``` + +Cover built-in rule version, Trail configuration, nested `.trailignore`/`.gitignore`, `.git/info/exclude`, `core.excludesFile`, included Git config, global/system config, normalization, mode/symlink handling, and case sensitivity. + +- [ ] **Step 4: Persist and validate manifest identities** + +External or unobservable dependencies set `stale_baseline`; observers route raw dependency events to invalidation before normal path filtering. Reuse is allowed only when every stored identity still validates through bounded direct checks. + +- [ ] **Step 5: Run policy tests and commit** + +Run: `cargo test -p trail --test changed_path_ledger_policy` + +Expected: PASS including nested changes, global config changes, included config, normalization/mode/case changes, and zero repeat discovery. + +```bash +git add trail/src/db/change_ledger/policy.rs trail/src/db/storage/worktree_index.rs trail/tests/changed_path_ledger_policy.rs +git commit -m "feat: persist changed-path policy dependencies" +``` + +### Task 5: Stream full reconciliation and publish by exact CAS + +**Files:** +- Create: `trail/src/db/change_ledger/reconcile.rs` +- Modify: `trail/src/db/storage/worktree_index.rs` +- Modify: `trail/src/db/storage/root_diff.rs` +- Modify: `trail/src/model/reports/maintenance.rs` +- Create: `trail/tests/changed_path_ledger_reconcile.rs` + +**Interfaces:** +- Consumes: `ChangedPathLedger`, `ExpectedScope`, `CompiledPolicy`, provider `ObserverFence`, and existing worktree/root readers. +- Produces: `begin_reconciliation(...) -> ReconciliationAttempt`, `reconcile_full(...)`, `ReconciliationAttempt::observe`, `ReconciliationAttempt::publish`, `ReconcileMode::{Full, ProvenPrefixes}`, and `ChangeLedgerReconcileReport`. + +- [ ] **Step 1: Write correctness and race tests against a full-scan oracle** + +```rust +#[test] +fn reconciliation_finds_add_modify_mode_delete_and_rename() { + let expected = fixture.mutate_all_path_kinds(); + let report = fixture.reconcile_full().unwrap(); + assert_eq!(fixture.ledger_candidates(), fixture.full_scan_oracle()); + assert_eq!(report.observed_candidates, expected); +} + +#[test] +fn ref_or_filesystem_replacement_during_scan_cannot_publish_trust() { + let attempt = fixture.begin_reconciliation().unwrap(); + fixture.replace_scope_root(); + assert!(attempt.publish().is_err()); + assert_ne!(fixture.trust_state(), TrustState::Trusted); +} +``` + +- [ ] **Step 2: Run the reconciliation tests to establish red** + +Run: `cargo test -p trail --test changed_path_ledger_reconcile` + +Expected: compile failure for missing `ReconciliationAttempt` and `reconcile_full`. + +- [ ] **Step 3: Start observation before enumeration and stream staging rows** + +```rust +pub(crate) struct ReconcileExpectation { + pub scope: ExpectedScope, + pub start_fence: ObserverFence, + pub root_handle_identity: Vec, +} + +pub(crate) fn reconcile_full( + ledger: &ChangedPathLedger<'_>, + observer: &dyn QualifiedObserver, + expected: &ReconcileExpectation, + policy: &CompiledPolicy, +) -> Result; +``` + +Open the root no-follow, record identity, enumerate through bounded buffers, hash relevant regular files, verify each before/after identity, stream rows into the attempt staging table, and find deletions by streaming the baseline Prolly range. Never retain the complete repository manifest in memory. + +- [ ] **Step 4: Fence, drain, and publish only on the full identity tuple** + +Obtain the end fence after enumeration, fold all evidence through it into staging, revalidate ref/root/epoch/policy generation/filesystem/provider/root-handle identity, then replace authoritative candidates and transition to `trusted` in one transaction. A mismatch abandons staging and retries; it cannot partially publish. + +- [ ] **Step 5: Limit prefix reconciliation to provider-proven containment** + +`ProvenPrefixes` accepts only persisted complete prefix evidence from a qualified provider. Overflow, owner loss, global policy change, corrupt log, and unknown gaps force `Full`; user-provided prefixes may refresh rows but never promote global trust. + +- [ ] **Step 6: Run reconciliation, memory, and race tests** + +Run: `cargo test -p trail --test changed_path_ledger_reconcile` + +Expected: PASS with complete oracle equality, no false clean under injected races, and bounded peak staging memory for 100,000 fixture paths. + +- [ ] **Step 7: Commit the reconciler** + +```bash +git add trail/src/db/change_ledger/reconcile.rs trail/src/db/storage/worktree_index.rs trail/src/db/storage/root_diff.rs trail/src/model/reports/maintenance.rs trail/tests/changed_path_ledger_reconcile.rs +git commit -m "feat: add streamed changed-path reconciliation" +``` + +### Task 6: Add durable controlled-writer intents, recovery, backup, and GC roots + +**Files:** +- Create: `trail/src/db/change_ledger/intent.rs` +- Create: `trail/src/db/change_ledger/recovery.rs` +- Modify: `trail/src/db/storage/lifecycle/gc.rs` +- Modify: `trail/src/db/core/backup/create.rs` +- Modify: `trail/src/db/core/backup/verify.rs` +- Modify: `trail/src/db/core/backup/restore.rs` +- Create: `trail/tests/changed_path_ledger_recovery.rs` + +**Interfaces:** +- Consumes: `ExpectedScope`, ledger store CAS, observer fence, operation/root IDs, authoritative ref transaction, and backup/GC infrastructure. +- Produces: `IntentId`, `IntentProducer`, `IntentTarget`, `IntentEvidence`, `VerifiedFilesystemCut`, `IntentState`, `prepare_intent`, `mark_filesystem_applied`, `publish_intent`, `recover_scope`, `ChangedPathLedger::recover`, and `ledger_gc_roots`. + +- [ ] **Step 1: Write lifecycle, crash-boundary, restore, and GC tests** + +```rust +#[test] +fn concurrent_same_path_event_survives_intent_acknowledgement() { + let intent = fixture.prepare_checkout_intent("src/lib.rs").unwrap(); + fixture.mark_filesystem_applied(&intent).unwrap(); + fixture.append_external_event_after_intent("src/lib.rs").unwrap(); + fixture.publish_and_ack(&intent).unwrap(); + assert!(fixture.candidates().contains("src/lib.rs")); +} + +#[test] +fn prepared_target_is_a_gc_root_until_terminal_recovery() { + let intent = fixture.prepare_new_target().unwrap(); + fixture.gc().unwrap(); + assert!(fixture.target_exists(&intent)); + fixture.abort_and_recover(intent).unwrap(); + fixture.gc().unwrap(); + assert!(!fixture.target_exists(&intent)); +} +``` + +- [ ] **Step 2: Run the recovery tests and verify missing behavior** + +Run: `cargo test -p trail --test changed_path_ledger_recovery` + +Expected: FAIL because mutations have no durable intent lifecycle or GC roots. + +- [ ] **Step 3: Define producer and lifecycle types** + +```rust +pub(crate) enum IntentProducer { + Checkout, LaneSync, Materialize, StructuredPatchProjection, + RestoreProjection, CowPublication, ObservedCheckpoint, +} +pub(crate) enum IntentState { Prepared, FilesystemApplied, Published, Acknowledged, Aborted } +pub(crate) struct IntentTarget { + pub change_id: ChangeId, + pub root_id: ObjectId, + pub operation_id: Option, +} +pub(crate) struct IntentEvidence { + pub exact_paths: Vec, + pub complete_prefixes: Vec, +} +pub(crate) struct VerifiedFilesystemCut { + pub observer_cut: EvidenceCut, + pub verified_paths: u64, + pub filesystem_identity: Vec, +} +``` + +Persist expected epoch/ref generation/root, target, start cursor, exact paths/prefixes, and source ownership. Sync the intent file before publishing each durable lifecycle transition. + +- [ ] **Step 4: Implement recovery decisions before mutation and GC** + +```rust +pub(crate) enum RecoveryDecision { + FinishPublication, + RetainCandidatesAndAcknowledge, + Abort, + FullReconciliation, +} +``` + +Compare intent state, verified filesystem, authoritative ref/operation, provider cut, and ledger baseline. Ambiguity retains candidates or reconciles; it never clears evidence. Run `recover_scope` before another mutation, snapshot, backup, or GC. + +- [ ] **Step 5: Fence backup and rotate restored epochs** + +Backup either stores a matching fenced SQL+segment cut or marks every backed-up scope untrusted. Restore rotates epoch, clears owner/cursor, discards unmatched tails, rebinds filesystem identity, and writes `untrusted_gap`; it never restores `trusted`. + +- [ ] **Step 6: Add intent targets to GC and retire scopes safely** + +Prepared and published target roots/operations remain GC roots until a terminal state. Lane/view deletion first retires the scope and revokes owner lease transactionally, then removes segments after no reader can reference them. + +- [ ] **Step 7: Execute the crash matrix and commit** + +Run: `cargo test -p trail --test changed_path_ledger_recovery` + +Expected: PASS for process kill before/after intent sync, filesystem sync, operation write, ref CAS, ledger CAS, mirror repair, backup rotation, restore to another filesystem, and GC on both sides of recovery. + +```bash +git add trail/src/db/change_ledger/intent.rs trail/src/db/change_ledger/recovery.rs trail/src/db/storage/lifecycle/gc.rs trail/src/db/core/backup trail/tests/changed_path_ledger_recovery.rs +git commit -m "feat: recover changed-path intents and lifecycle" +``` + +### Task 7: Qualify the native Linux inotify observer + +**Files:** +- Create: `trail/src/db/change_ledger/observer/mod.rs` +- Create: `trail/src/db/change_ledger/observer/linux.rs` +- Modify: `trail/Cargo.toml` +- Modify: root `Cargo.toml` +- Create: `trail/tests/changed_path_ledger_linux.rs` + +**Interfaces:** +- Consumes: `SegmentWriter`, `ObserverRecord`, `ScopeIdentity`, root handle identity, and `ProviderCapabilities`. +- Produces: `QualifiedObserver` trait, `ObserverFence`, `ObserverLease`, `LinuxInotifyObserver`, and `select_observer`. + +- [ ] **Step 1: Write Linux-only real filesystem qualification tests** + +```rust +#[cfg(target_os = "linux")] +#[test] +fn recursive_directory_creation_is_covered_before_children_can_be_clean() { + let observer = real_observer_fixture(); + fixture.create_dir_and_child_while_watch_is_added("a/b/file"); + let cut = observer.fence().unwrap(); + assert!(observer.fold_through(&cut).unwrap().covers("a/b/file")); +} + +#[cfg(target_os = "linux")] +#[test] +fn queue_overflow_revokes_clean_proof() { + let observer = overflow_fixture(); + assert_eq!(observer.fence().unwrap_err().reason(), "inotify_queue_overflow"); +} +``` + +- [ ] **Step 2: Run on Linux and confirm the adapter is absent** + +Run: `cargo test -p trail --test changed_path_ledger_linux -- --nocapture` + +Expected: compile failure for missing `LinuxInotifyObserver`. + +- [ ] **Step 3: Add a direct `inotify` dependency and observer contract** + +```rust +pub(crate) trait QualifiedObserver: Send + Sync { + fn capabilities(&self) -> ProviderCapabilities; + fn root_identity(&self) -> Result>; + fn fence(&self) -> Result; + fn flush_durable(&self, fence: &ObserverFence) -> Result; + fn shutdown(self: Box) -> Result<()>; +} +``` + +The generic `notify` adapter implements an advisory type but always returns `clean_proof_allowed=false`. + +- [ ] **Step 4: Implement recursive coverage, rename cookies, and race handling** + +Add watches root-first; for directory creation/move-in, install the watch then enumerate the new subtree and mark its parent as a complete dirty prefix. Pair `MOVED_FROM`/`MOVED_TO` cookies while retaining both endpoints; expiration becomes a conservative complete parent prefix. `IN_Q_OVERFLOW`, `IN_IGNORED`, unknown watch descriptor, root deletion, and watch-add failure revoke trust globally. + +- [ ] **Step 5: Implement a sentinel delivery fence** + +Create and sync a nonce-named sentinel inside the pinned scope, wait until its exact inotify sequence is durably appended, unlink it, wait for the unlink record, then return the covered cut. The observer qualifies only after a reconciliation began with watching active and ended through this fence. + +- [ ] **Step 6: Run the real Linux suite** + +Run: `cargo test -p trail --test changed_path_ledger_linux -- --nocapture` + +Expected: PASS for create/delete/content/mode, file/directory/case rename, recursive add races, delayed backlog, rename storms, overflow, owner death, filesystem/root replacement, and fence ordering on a native Linux filesystem. + +- [ ] **Step 7: Commit the Linux adapter without enabling authority** + +```bash +git add Cargo.toml trail/Cargo.toml trail/src/db/change_ledger/observer trail/tests/changed_path_ledger_linux.rs +git commit -m "feat: add qualified Linux change observer" +``` + +### Task 8: Qualify the native macOS FSEvents observer + +**Files:** +- Create: `trail/src/db/change_ledger/observer/macos.rs` +- Modify: `trail/src/db/change_ledger/observer/mod.rs` +- Modify: `trail/Cargo.toml` +- Modify: root `Cargo.toml` +- Create: `trail/tests/changed_path_ledger_macos.rs` + +**Interfaces:** +- Consumes: `QualifiedObserver`, `SegmentWriter`, scope/root identity, provider cursors, and reconciliation activation contract. +- Produces: `MacOsFseventsObserver` with persisted event-ID continuity and synchronous flush fence. + +- [ ] **Step 1: Write macOS-only real FSEvents tests** + +```rust +#[cfg(target_os = "macos")] +#[test] +fn must_scan_subdirs_revokes_cursor_resume() { + let observer = fsevents_fixture_with_flag(kFSEventStreamEventFlagMustScanSubDirs); + assert!(observer.resume().is_err()); + assert_eq!(fixture.trust_state(), TrustState::UntrustedGap); +} + +#[cfg(target_os = "macos")] +#[test] +fn synchronous_flush_returns_a_durably_foldable_event_id() { + let observer = real_observer_fixture(); + fixture.write("src/lib.rs", b"new"); + let cut = observer.fence().unwrap(); + assert!(cut.provider_cursor.event_id >= fixture.write_event_id()); +} +``` + +- [ ] **Step 2: Run on macOS and verify the adapter is absent** + +Run: `cargo test -p trail --test changed_path_ledger_macos -- --nocapture` + +Expected: compile failure for missing `MacOsFseventsObserver`. + +- [ ] **Step 3: Add a direct FSEvents binding and file-events stream** + +Request file-level events, persist `FSEventStreamEventId`, device/root identity, stream creation identity, and capability record. Normalize all event paths relative to the pinned root before appending; non-UTF-8 or escaped-root paths revoke qualification. + +- [ ] **Step 4: Enforce continuity and gap flags** + +`MustScanSubDirs`, `UserDropped`, `KernelDropped`, `EventIdsWrapped`, `RootChanged`, `Unmount`, `HistoryDone` inconsistency, root identity change, or a resume ID older than available history marks `untrusted_gap` and starts full reconciliation. Resume without reconciliation is allowed only when event-ID and root continuity are proven. + +- [ ] **Step 5: Implement synchronous flush fencing** + +Call the synchronous stream flush, wait until every callback through the returned/latest event ID has been durably appended, then return that cursor. Persisted cursor alone never qualifies power-loss durability. + +- [ ] **Step 6: Run the real macOS suite** + +Run: `cargo test -p trail --test changed_path_ledger_macos -- --nocapture` + +Expected: PASS for create/delete/content/mode, file/directory/case rename, delayed batches, drop/gap flags, owner death, cursor replacement, root replacement, restart continuity and forced-reconcile cases, and fence ordering on APFS. + +- [ ] **Step 7: Commit the macOS adapter without enabling authority** + +```bash +git add Cargo.toml trail/Cargo.toml trail/src/db/change_ledger/observer trail/tests/changed_path_ledger_macos.rs +git commit -m "feat: add qualified macOS change observer" +``` + +### Task 9: Securely auto-start one per-workspace daemon + +**Files:** +- Create: `trail/src/cli/command/handler/daemon_start.rs` +- Modify: `trail/src/cli/command/handler.rs` +- Modify: `trail/src/cli/command/handler/daemon_rpc.rs` +- Modify: `trail/src/db/storage/worktree_index.rs` +- Modify: `trail/src/db/core/readonly.rs` +- Modify: `trail/src/server.rs` +- Create: `trail/tests/changed_path_ledger_daemon.rs` + +**Interfaces:** +- Consumes: platform observer selection, `recover_scope`, `reconcile_full`, existing daemon RPC client/server, process liveness helpers, and executable identity. +- Produces: `ensure_workspace_daemon_ready(workspace: &Path, token: Option<&str>) -> Result`, authenticated endpoint metadata, and daemon `ledger_fence`/`ledger_reconcile` RPCs. + +- [ ] **Step 1: Write concurrent startup, stale endpoint, and readiness tests** + +```rust +#[test] +fn concurrent_first_status_calls_converge_on_one_owner() { + let results = run_concurrently(16, || ensure_workspace_daemon_ready(fixture.root(), None)); + let owner_nonces = results.into_iter().map(Result::unwrap).map(|r| r.owner_nonce).collect::>(); + assert_eq!(owner_nonces.len(), 1); +} + +#[test] +fn stale_endpoint_is_replaced_only_after_process_identity_mismatch() { + fixture.write_endpoint_for_live_unrelated_process(); + assert!(ensure_workspace_daemon_ready(fixture.root(), None).is_err()); + fixture.mark_endpoint_process_dead(); + assert!(ensure_workspace_daemon_ready(fixture.root(), None).is_ok()); +} +``` + +- [ ] **Step 2: Run the daemon tests before startup support** + +Run: `cargo test -p trail --test changed_path_ledger_daemon` + +Expected: FAIL because auto-discovery currently falls back instead of securely spawning and waiting for ledger readiness. + +- [ ] **Step 3: Define authenticated endpoint metadata** + +```rust +#[derive(Serialize, Deserialize)] +struct WorkspaceDaemonEndpoint { + protocol_version: u16, + pid: u32, + process_start_identity: String, + executable_identity: [u8; 32], + owner_nonce: [u8; 32], + auth_token: [u8; 32], + socket: PathBuf, +} +``` + +Create endpoint/token files with owner-only permissions, publish with write+sync+atomic rename+parent-directory sync, and reject symlinks, unexpected owners/modes, wrong executable identity, PID reuse, nonce mismatch, or failed challenge-response. + +- [ ] **Step 4: Implement exclusive startup convergence** + +Acquire `.trail/index/change-ledger/daemon.lock`, re-check a live authenticated endpoint under the lock, spawn the current executable with an inherited one-shot readiness channel, and wait a bounded interval. Other callers wait for endpoint publication and challenge it. A live process with unverifiable identity is not killed or replaced. + +- [ ] **Step 5: Make readiness include recovery, observation, reconciliation, and fence** + +The daemon publishes ready only after it owns the observer lease, runs recovery, establishes native watch coverage, automatically reconciles any untrusted state, and obtains a valid live fence. Stop reading or writing `worktree-daemon-cache.json` and remove it from read-only sidecar handling; the changed-path ledger is the only persisted candidate authority. + +- [ ] **Step 6: Add authenticated fence and reconcile RPCs** + +```rust +pub(crate) struct DaemonReady { pub owner_nonce: [u8; 32], pub scope: ScopeId, pub cut: EvidenceCut } +pub(crate) struct FenceRequest { pub scope: ScopeId, pub expected_epoch: u64 } +pub(crate) struct ReconcileRequest { pub scope: ScopeId, pub expected: ExpectedScope, pub mode: ReconcileMode } +``` + +Every RPC verifies token, owner nonce, workspace identity, epoch, and executable protocol version; timeouts and broken channels fail closed. + +- [ ] **Step 7: Run daemon security and lifecycle tests** + +Run: `cargo test -p trail --test changed_path_ledger_daemon` + +Expected: PASS for 16-way startup, PID reuse, malicious symlink/permissions, bad nonce/token, stale endpoint replacement, daemon kill/restart reconciliation, and readiness timeout. + +- [ ] **Step 8: Commit daemon lifecycle** + +```bash +git add trail/src/cli/command/handler.rs trail/src/cli/command/handler/daemon_rpc.rs trail/src/cli/command/handler/daemon_start.rs trail/src/db/storage/worktree_index.rs trail/src/db/core/readonly.rs trail/src/server.rs trail/tests/changed_path_ledger_daemon.rs +git commit -m "feat: auto-start authenticated workspace daemon" +``` + +### Task 10: Integrate one fenced snapshot flow into status, diff, and record + +**Files:** +- Create: `trail/src/db/change_ledger/snapshot.rs` +- Modify: `trail/src/db/core/status.rs` +- Modify: `trail/src/db/record/diff.rs` +- Modify: `trail/src/db/record/recording/manual.rs` +- Modify: `trail/src/db/lane/control/agent_capture.rs` +- Modify: `trail/src/cli/command/handler.rs` +- Modify: `trail/src/db/performance.rs` +- Create: `trail/tests/changed_path_ledger_commands.rs` + +**Interfaces:** +- Consumes: ready daemon, `ChangedPathLedger`, `QualifiedObserver`, `recover_scope`, `reconcile_full`, CAS store, and existing root point lookup/hash builders. +- Produces: `ChangedPathLedger::authoritative_snapshot`, `CandidateComparison`, and `ObservedRecordCut` used by workspace and lane record paths. + +- [ ] **Step 1: Write public-flow and sequence-boundary tests** + +```rust +#[test] +fn clean_status_uses_empty_authoritative_candidates_not_a_full_walk() { + let report = fixture.status().unwrap(); + assert!(report.is_clean()); + assert_eq!(report.metrics.full_scope_walks, 0); + assert_eq!(report.metrics.candidate_reads, 0); +} + +#[test] +fn record_retains_event_arriving_between_c1_and_c2() { + fixture.change("a", b"one"); + fixture.inject_after_first_fence(|| fixture.change("a", b"two")); + fixture.record().unwrap(); + assert!(fixture.status().unwrap().changed_paths().contains("a")); +} +``` + +- [ ] **Step 2: Run command tests and observe full-scan behavior** + +Run: `cargo test -p trail --test changed_path_ledger_commands` + +Expected: FAIL because status/diff/record do not consume a live fenced candidate snapshot. + +- [ ] **Step 3: Implement the sole authoritative snapshot entry point** + +```rust +pub(crate) fn authoritative_snapshot( + &self, + observer: &dyn QualifiedObserver, + expected: &ExpectedScope, +) -> Result { + self.recover(expected)?; + let c1 = observer.fence()?; + let durable = observer.flush_durable(&c1)?; + self.fold_tail_through(expected, &durable)?; + self.snapshot_candidates(expected) +} +``` + +If any step cannot prove trust, ask the daemon to run full reconciliation and retry once against the new epoch/cut. Only return `ChangeLedgerReconcileRequired` when automatic recovery cannot establish authority. + +- [ ] **Step 4: Make status and diff candidate-only** + +Load exact paths and complete prefixes, expand only affected baseline ranges/new subtrees, use pinned no-follow reads, batch root point lookups, and compare against the shared `RecordingPolicySnapshot`. Fence through `c2`; retry or retain evidence newer than `c1`. An empty authoritative set is the only fast clean result. + +- [ ] **Step 5: Make manual record an observed checkpoint transaction** + +Acquire workspace lock; capture ref/epoch; obtain/fold `c1`; read/hash bounded candidates; obtain/fold `c2`; build immutable target; and in one SQLite transaction index operation, CAS ref, advance baseline, and acknowledge only evidence covered through `c1`. Use this same state machine for native-agent checkpoints in `lane/control/agent_capture.rs`. Repair mirrors after commit and retain later/different-source same-path evidence. + +- [ ] **Step 6: Record structural metrics** + +```rust +pub(crate) struct ChangedPathMetrics { + pub full_scope_walks: u64, + pub root_full_ranges: u64, + pub sqlite_full_index_rows: u64, + pub authoritative_candidates: u64, + pub candidate_reads: u64, + pub hashes: u64, + pub root_point_lookups: u64, + pub ledger_rows_touched: u64, + pub observer_tail_records_folded: u64, + pub reconciliation_runs: u64, +} +``` + +Keep reconciliation counters separate so an O(N) cold run cannot be reported as a warm fast path. + +- [ ] **Step 7: Run command correctness and locality tests** + +Run: `cargo test -p trail --test changed_path_ledger_commands` + +Expected: PASS for `k=0,1,100`, create/content/mode/delete/file rename/directory rename/case-only rename/revert/ignored paths, c1/c2 races, daemon restart, auto reconciliation, and full-scan oracle equality after each measured call. + +- [ ] **Step 8: Commit command integration while activation remains off** + +```bash +git add trail/src/db/change_ledger/snapshot.rs trail/src/db/core/status.rs trail/src/db/record/diff.rs trail/src/db/record/recording/manual.rs trail/src/db/lane/control/agent_capture.rs trail/src/cli/command/handler.rs trail/src/db/performance.rs trail/tests/changed_path_ledger_commands.rs +git commit -m "feat: consume fenced changed-path snapshots" +``` + +### Task 11: Cover every workspace and materialized-lane filesystem producer + +**Files:** +- Modify: `trail/src/db/record/checkout.rs` +- Modify: `trail/src/db/lane/patching.rs` +- Modify: `trail/src/db/lane/workdir/materialize.rs` +- Modify: `trail/src/db/lane/workdir/sync.rs` +- Modify: `trail/src/db/lane/workdir/record.rs` +- Modify: `trail/src/db/lane/readiness.rs` +- Modify: `trail/src/model/reports/lane.rs` +- Create: `trail/tests/changed_path_ledger_producers.rs` + +**Interfaces:** +- Consumes: intent lifecycle/recovery, fenced snapshots, workspace/lane scope IDs, ref transaction, and immutable target roots. +- Produces: `run_ref_advancing_projection`, `run_projection_alignment`, lane scope reconciliation, and compact `MaterializedLaneMarkerV2`. + +- [ ] **Step 1: Add a checked-in producer inventory test** + +```rust +const CONTROLLED_PRODUCERS: &[IntentProducer] = &[ + IntentProducer::Checkout, + IntentProducer::LaneSync, + IntentProducer::Materialize, + IntentProducer::StructuredPatchProjection, + IntentProducer::RestoreProjection, + IntentProducer::CowPublication, + IntentProducer::ObservedCheckpoint, +]; + +#[test] +fn every_filesystem_producer_has_a_reviewed_protocol() { + assert_eq!(discover_producers_from_source(), CONTROLLED_PRODUCERS); +} +``` + +- [ ] **Step 2: Run producer tests and identify uncovered mutations** + +Run: `cargo test -p trail --test changed_path_ledger_producers` + +Expected: FAIL listing checkout, sync, materialize, structured patch, restoration, COW, or record sites without intent/fence coverage. + +- [ ] **Step 3: Wrap ref-advancing projections** + +```rust +fn run_ref_advancing_projection( + db: &mut Trail, + expected: &ExpectedScope, + target: IntentTarget, + changes: IntentEvidence, + apply: impl FnOnce() -> Result, +) -> Result<()>; +``` + +Prebuild target/operation, durably prepare, apply+sync+verify, fence, then publish operation/ref/ledger in one SQL transaction. Acknowledge only intent-owned/source-covered evidence. + +- [ ] **Step 4: Wrap projection-only alignment** + +Checkout materialization, sparse hydration, and ordinary workdir sync reference the existing target root, durably prepare, apply+sync+verify, fence, and atomically publish only the unchanged-ref scope baseline/marker plus terminal intent. They do not fabricate an operation or advance a ref. + +- [ ] **Step 5: Replace lane manifests with compact v2 markers only after reconciliation** + +```rust +#[derive(Serialize, Deserialize)] +pub(crate) struct MaterializedLaneMarkerV2 { + pub version: u16, + pub scope_id: ScopeId, + pub filesystem_identity: Vec, + pub ref_name: String, + pub ref_generation: u64, + pub root_id: ObjectId, + pub policy_fingerprint: [u8; 32], + pub epoch: u64, + pub provider_cut: EvidenceCut, +} +``` + +Publish marker and trusted scope identity at one logical cut. Missing, unsupported-version, or mismatched markers trigger automatic full lane reconciliation; they never prove candidates. + +- [ ] **Step 6: Route lane record/readiness/merge/preview through candidates** + +Materialized lane status, record, readiness, merge checks, structured-patch maintenance, and normal preview use the lane scope snapshot when trusted. Full ignored/risk traversals remain explicitly labeled audit or reconciliation operations. + +- [ ] **Step 7: Run producer crash and E2E tests** + +Run: `cargo test -p trail --test changed_path_ledger_producers && cargo test -p trail --test e2e lane_` + +Expected: PASS for every producer, same-path external races, kill points, v2 marker mismatch, owner loss/reconcile, and post-operation oracle equality. + +- [ ] **Step 8: Commit the producer inventory and integrations** + +```bash +git add trail/src/db/record/checkout.rs trail/src/db/lane trail/src/model/reports/lane.rs trail/tests/changed_path_ledger_producers.rs +git commit -m "feat: cover workspace and lane producers with intents" +``` + +### Task 12: Make workspace-view/COW journals authoritative only when qualified + +**Files:** +- Modify: `trail/src/db/lane/workdir/view_journal.rs` +- Modify: `trail/src/db/lane/workdir/view_core.rs` +- Modify: `trail/src/db/lane/workspace_view.rs` +- Modify: `trail/src/db/lane/workdir/fuse.rs` +- Modify: `trail/src/db/lane/workdir/nfs_overlay.rs` +- Create: `trail/tests/changed_path_ledger_views.rs` + +**Interfaces:** +- Consumes: workspace-view scope, segment codec primitives, intent recovery, workspace lock, shared/exclusive view barrier, and checkpoint publication. +- Produces: `ViewIntentWriter`, `ViewJournalCut`, `checkpoint_view`, generation ownership, compact incremental whiteout log, and qualified view trust. + +- [ ] **Step 1: Write journal-before-mutation and recovery tests** + +```rust +#[test] +fn semantic_upper_mutation_is_not_visible_before_intent_is_durable() { + let view = fixture.inject_journal_sync_failure(); + assert!(view.write("a", b"new").is_err()); + assert_eq!(view.read("a").unwrap(), b"old"); +} + +#[test] +fn untrusted_view_scans_upper_instead_of_claiming_zero_recovery_walks() { + fixture.corrupt_view_journal(); + let report = fixture.checkpoint().unwrap(); + assert!(report.upper_recovery_walks > 0); +} +``` + +- [ ] **Step 2: Run view tests before changing journal semantics** + +Run: `cargo test -p trail --test changed_path_ledger_views` + +Expected: FAIL because journal/whiteout durability and scope trust are not coupled. + +- [ ] **Step 3: Persist per-view intent before upper/whiteout mutation** + +VFS callbacks take only the shared view barrier, append+sync the intent record, then expose the semantic upper/whiteout change. They never acquire the workspace lock or primary SQLite connection. Group commit may batch only when no mutation becomes externally visible before its intent is durable. + +- [ ] **Step 4: Publish checkpoints under the required lock order** + +Checkpoint takes workspace lock, then exclusive view barrier, folds the journal through a cut, verifies candidates, builds the immutable target, and publishes ref/ledger/marker in one transaction. Replay records after the clean cut into the next generation; retain active-handle ownership of older generations. + +- [ ] **Step 5: Replace whole-array whiteouts with an incremental log/map** + +Each whiteout mutation is atomic and replayable. Rotate at checkpoint, compact only generations with no active handle, and keep the old upper/whiteout set until recovery proves the new generation published. + +- [ ] **Step 6: Run FUSE/NFS/COW crash and concurrency tests** + +Run: `cargo test -p trail --test changed_path_ledger_views && cargo test -p trail db::lane::workdir::fuse db::lane::workdir::nfs_overlay` + +Expected: PASS for journal sync failure, whiteout replay, active handles, checkpoint crash, generation rotation, corrupt/gapped fallback reconciliation, and `upper_recovery_walks=0` only for qualified trusted journals. + +- [ ] **Step 7: Commit view correctness and compaction** + +```bash +git add trail/src/db/lane/workdir/view_journal.rs trail/src/db/lane/workdir/view_core.rs trail/src/db/lane/workspace_view.rs trail/src/db/lane/workdir/fuse.rs trail/src/db/lane/workdir/nfs_overlay.rs trail/tests/changed_path_ledger_views.rs +git commit -m "feat: qualify workspace view change journals" +``` + +### Task 13: Keep Git authoritative only under exact Trail-baseline equivalence + +**Files:** +- Modify: `trail/src/db/storage/git.rs` +- Modify: `trail/src/db/record/recording/git.rs` +- Modify: `trail/src/db/lane/workspace_git.rs` +- Modify: `trail/src/db/change_ledger/types.rs` +- Modify: `trail/src/db/performance.rs` +- Create: `trail/tests/changed_path_ledger_git.rs` + +**Interfaces:** +- Consumes: Trail baseline/ref identity, `CompiledPolicy`, filesystem identity, Git mapping records, and Git command runner. +- Produces: `GitEvidenceQualification`, `qualified_git_candidates`, and Git structural metrics; it never bypasses `ChangedPathLedger::authoritative_snapshot`. + +- [ ] **Step 1: Write equivalence rejection tests** + +```rust +#[test] +fn git_head_match_is_insufficient_when_trail_baseline_is_ahead() { + fixture.record_trail_ahead_of_head(); + fixture.revert_worktree_to_head(); + assert!(!fixture.qualify_git().unwrap().clean_proof_allowed); +} + +#[test] +fn sparse_skip_worktree_or_policy_mismatch_is_advisory_only() { + for mode in [GitMode::Sparse, GitMode::SkipWorktree, GitMode::PolicyMismatch] { + assert!(!fixture.with_mode(mode).qualify_git().unwrap().clean_proof_allowed); + } +} +``` + +- [ ] **Step 2: Run Git qualification tests** + +Run: `cargo test -p trail --test changed_path_ledger_git` + +Expected: FAIL because existing Git fast paths do not bind all semantics to the exact Trail baseline. + +- [ ] **Step 3: Define the complete qualification record** + +```rust +pub(crate) struct GitEvidenceQualification { + pub head_oid: String, + pub index_identity: Vec, + pub mapped_trail_root: ObjectId, + pub filesystem_identity: Vec, + pub policy_fingerprint: [u8; 32], + pub mode_equivalent: bool, + pub symlink_equivalent: bool, + pub sparse_equivalent: bool, + pub submodule_equivalent: bool, + pub clean_proof_allowed: bool, +} +``` + +Require mapped Trail baseline root exactly equal to the ledger baseline and equivalent file-mode, symlink, sparse, submodule, ignore, case, HEAD, and index/split/shared-index identities. + +- [ ] **Step 4: Parse porcelain v2 `-z` conservatively** + +Retain both rename endpoints. `assume-unchanged`, `skip-worktree`, unresolved sparse/submodule state, racy or untrusted fsmonitor, untrusted untracked-cache continuity, or policy mismatch forces advisory evidence. Advisory exact dirty paths may be added but cannot prove completeness or clean. + +- [ ] **Step 5: Record subprocess and hidden-global-work metrics** + +Capture Git subprocess count, index refresh, Trace2 regions/bytes, fsmonitor qualification, and untracked-cache qualification. Any internal global scan sets `external_adapter_global_work>0` and fails warm structural gates. + +- [ ] **Step 6: Run Git oracle tests and commit** + +Run: `cargo test -p trail --test changed_path_ledger_git` + +Expected: PASS for exact equivalence, Trail-ahead reversion, rename endpoints, modes/symlinks, sparse/submodules, split/shared index replacement, racy fsmonitor, policy mismatch, and candidate equality with the direct Git oracle. + +```bash +git add trail/src/db/storage/git.rs trail/src/db/record/recording/git.rs trail/src/db/lane/workspace_git.rs trail/src/db/change_ledger/types.rs trail/src/db/performance.rs trail/tests/changed_path_ledger_git.rs +git commit -m "feat: qualify Git changed-path evidence" +``` + +### Task 14: Expose reconciliation and diagnostics through every public surface + +**Files:** +- Modify: `trail/src/cli/command/maintenance_args.rs` +- Modify: `trail/src/cli/command/handler/maintenance.rs` +- Modify: `trail/src/cli/command/render/maintenance.rs` +- Modify: `trail/src/cli/command/handler/errors.rs` +- Modify: `trail/src/model/reports/maintenance.rs` +- Modify: `trail/src/server/openapi/paths/core.rs` +- Modify: `trail/src/server/openapi/schemas/core.rs` +- Modify: `trail/src/server/openapi.rs` +- Modify: `trail/src/server/request_types.rs` +- Modify: `trail/src/server/route/dispatch.rs` +- Modify: `trail/src/server/route/system.rs` +- Modify: `trail/src/server/route/utils.rs` +- Modify: `trail/src/mcp/tools.rs` +- Modify: `trail/src/mcp/tools/core.rs` +- Modify: `trail/src/mcp/tool_call/core.rs` +- Modify: `trail/src/mcp/types/core.rs` +- Modify: `trail/src/mcp/response.rs` +- Modify: `trail/tests/e2e.rs` +- Create: `trail/tests/changed_path_ledger_api.rs` + +**Interfaces:** +- Consumes: `ChangeLedgerReconcileReport`, stable errors, daemon reconcile RPC, workspace scope, and lane scope lookup. +- Produces: `trail index reconcile [--lane ]`, structured CLI/REST/MCP errors, OpenAPI request/report schemas, and human diagnostics. + +- [ ] **Step 1: Write CLI, JSON, REST, OpenAPI, and MCP contract tests** + +```rust +#[test] +fn reconcile_required_has_identical_structured_recovery_fields() { + let cli = fixture.cli_json_status_failure(); + let rest = fixture.rest_status_failure(); + let mcp = fixture.mcp_status_failure(); + for value in [cli, rest, mcp] { + assert_eq!(value.pointer("/error/code").unwrap(), "CHANGE_LEDGER_RECONCILE_REQUIRED"); + assert_eq!(value.pointer("/error/recovery/command").unwrap(), "trail index reconcile"); + } +} +``` + +- [ ] **Step 2: Run surface contract tests** + +Run: `cargo test -p trail --test changed_path_ledger_api` + +Expected: FAIL because reconcile command/report and structured fields do not exist. + +- [ ] **Step 3: Add command arguments and report type** + +```rust +#[derive(Args)] +pub(super) struct IndexReconcileArgs { + #[arg(long)] + pub(super) lane: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ChangeLedgerReconcileReport { + pub scope_id: String, + pub scope_kind: String, + pub previous_state: String, + pub reason: String, + pub observed_paths: u64, + pub candidates: u64, + pub resulting_epoch: u64, + pub resulting_state: String, +} +``` + +Dispatch `IndexSubcommand::Reconcile`, auto-start the daemon, resolve workspace or lane scope, run full reconciliation, and render the report in human, JSON, and NDJSON formats. + +- [ ] **Step 4: Add stable diagnostic fields everywhere** + +Human guidance names the scope/reason and exact command. JSON, REST, daemon, and MCP include `code`, exit/status code, `scope`, `state`, `reason`, and `recovery.command`. `SCHEMA_REINITIALIZE_REQUIRED` specifically instructs backup plus `trail init --force` and never suggests migration. + +- [ ] **Step 5: Add OpenAPI route and schema checks** + +Expose `POST /v1/index/reconcile` with optional lane, include the report and structured error schemas, regenerate the checked contract through the existing builder, and assert every `$ref` resolves. + +- [ ] **Step 6: Run contract tests and commit** + +Run: `cargo test -p trail --test changed_path_ledger_api && cargo test -p trail server::openapi mcp::` + +Expected: PASS with byte-stable error codes and matching recovery fields across all surfaces. + +```bash +git add trail/src/cli/command/maintenance_args.rs trail/src/cli/command/handler/maintenance.rs trail/src/cli/command/render/maintenance.rs trail/src/cli/command/handler/errors.rs trail/src/model/reports/maintenance.rs trail/src/server trail/src/mcp trail/tests/e2e.rs trail/tests/changed_path_ledger_api.rs +git commit -m "feat: expose changed-path reconciliation diagnostics" +``` + +### Task 15: Pass fault/scale gates, audit producers, and activate on Linux/macOS + +**Files:** +- Modify: `trail/src/db/change_ledger/mod.rs` +- Modify: `trail/src/db/change_ledger/observer/mod.rs` +- Modify: `trail/src/db/performance.rs` +- Modify: `scripts/cli-scale-bench.sh` +- Create: `scripts/check-changed-path-ledger-thresholds.py` +- Create: `scripts/test_check_changed_path_ledger_thresholds.py` +- Modify: `.github/workflows/scale.yml` +- Create: `.github/workflows/changed-path-ledger-native.yml` +- Modify: `trail/tests/e2e.rs` +- Create: `trail/tests/changed_path_ledger_activation.rs` +- Modify: `docs/superpowers/specs/2026-07-12-correctness-first-changed-path-ledger-design.md` + +**Interfaces:** +- Consumes: all Tasks 1–14, producer inventory, native qualification suites, structural metrics, and full-scan oracle. +- Produces: `LEDGER_AUTHORITY_ENABLED` default-on for Linux/macOS only, scheduled native jobs, 1k/100k/1M thresholds, and an auditable activation report. + +- [ ] **Step 1: Write an activation test that initially fails closed** + +```rust +#[test] +fn authority_requires_every_gate_and_supported_platform() { + let complete = ActivationEvidence::from_checked_artifacts().unwrap(); + assert!(complete.schema_hard_cutover); + assert!(complete.producer_inventory_complete); + assert!(complete.linux_native_suite); + assert!(complete.macos_native_suite); + assert!(complete.crash_matrix); + assert!(complete.scale_gates); + assert_eq!(ledger_authority_enabled_for("windows", &complete), false); +} +``` + +- [ ] **Step 2: Run the activation test before flipping the default** + +Run: `cargo test -p trail --test changed_path_ledger_activation` + +Expected: FAIL naming every missing checked artifact or still-disabled platform default. + +- [ ] **Step 3: Complete the process-kill and corruption matrix** + +Exercise kill/failure before and after object write, intent sync, filesystem sync, observer sync, fence, operation index, ref CAS, ledger CAS, mirror repair, backup rotation, restore, segment rotation, and GC. Corrupt checksum, hash chain, durable/folded offsets, owner token, format version, partial SQLite shape, and root identity. Every ambiguous outcome may be a false-positive candidate or reconciliation, never false clean. + +- [ ] **Step 4: Add 1k, 100k, and 1M benchmark modes with separate `k`** + +Run workspace status/diff/record, materialized-lane record, structured-patch maintenance, and COW checkpoint for authoritative input `k=0,1,100`, independently recording final changed output. CI runs 1k; scheduled ext4 and APFS jobs run 100k; nightly jobs run 1M plus cold reconciliation, daemon restart, and crash sampling. + +- [ ] **Step 5: Enforce structural fast-path bounds** + +```python +ZERO = [ + "full_scope_walks", "root_full_ranges", "sqlite_full_index_rows", + "full_manifest_bytes_read", "full_manifest_bytes_written", + "upper_recovery_walks", "external_adapter_global_work", + "policy_dependency_full_discovery", +] +for key in ZERO: + assert metrics[key] == 0, f"{key} must remain zero on a warm trusted run" +assert metrics["candidate_reads"] <= metrics["authoritative_candidates"] + metrics["bounded_prefix_output"] +assert metrics["observer_tail_records_folded"] <= metrics["configured_tail_bound"] +assert metrics["ledger_rows_touched"] <= 8 * (metrics["authoritative_candidates"] + metrics["bounded_prefix_output"] + 1) +``` + +Also enforce persisted caps for candidate/prefix rows and log/segment bytes, plus calibrated wall-time and peak-RSS budgets per host class. Label all reconciliation work separately. + +- [ ] **Step 6: Run the full-scan oracle outside every measured fast path** + +After each warm command completes and metrics are frozen, run direct full observation and assert exact semantic equality. Property-test event permutations, prefix coalescing, rename ambiguity, callback delays, concurrent same-path writes, and acknowledgement boundaries. + +- [ ] **Step 7: Run platform and repository gates** + +Run on Linux: `cargo test -p trail --test changed_path_ledger_linux -- --nocapture && REPO_FILES=100000 scripts/cli-scale-bench.sh changed-path-ledger` + +Run on macOS: `cargo test -p trail --test changed_path_ledger_macos -- --nocapture && REPO_FILES=100000 scripts/cli-scale-bench.sh changed-path-ledger` + +Run everywhere: `cargo fmt --all -- --check && cargo check --workspace --all-targets && cargo test --workspace` + +Expected: all commands PASS; scheduled outputs satisfy structural, time, and RSS thresholds. Other platforms pass direct-observation tests and assert that no scope row can transition to `trusted`. + +- [ ] **Step 8: Flip authority only after checked evidence exists** + +```rust +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub(crate) const LEDGER_AUTHORITY_ENABLED: bool = true; +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub(crate) const LEDGER_AUTHORITY_ENABLED: bool = false; +``` + +Keep runtime qualification mandatory even on supported operating systems: unsupported filesystems/providers automatically reconcile and, if qualification remains impossible, return the stable recovery error rather than persisting trust. + +- [ ] **Step 9: Record the activation audit and commit** + +Append an “Activation evidence” section to the approved design with exact workflow names, artifact paths, producer inventory hash, native adapter results, scale thresholds, and full-workspace test command. Then commit the activation and gates together. + +```bash +git add trail/src/db/change_ledger trail/src/db/performance.rs trail/tests scripts .github/workflows docs/superpowers/specs/2026-07-12-correctness-first-changed-path-ledger-design.md +git commit -m "feat: activate changed-path ledger on Linux and macOS" +``` + +--- + +## Final Verification and Integration + +- [ ] Run `cargo fmt --all -- --check` and expect exit status 0. +- [ ] Run `cargo check --workspace --all-targets` and expect exit status 0. +- [ ] Run `cargo test --workspace` and expect every unit, integration, E2E, property, and contract test to pass. +- [ ] Run `git diff --check` and expect no whitespace errors. +- [ ] Run `rg -n "legacy_reconcile_required|migrate_changed_path|backfill_changed_path|CREATE IF NOT EXISTS.*changed_path" trail/src trail/tests` and expect no matches. +- [ ] Run `rg -n "worktree-daemon-cache.json" trail/src` and expect no matches. +- [ ] On Linux and macOS, run the native observer and 100k commands from Task 15 and archive their machine-readable reports. +- [ ] Open the same 100k fixture on an unsupported platform build and prove it uses direct observation and persists no trusted scope. +- [ ] Inspect `git status --short`, exclude `.superpowers/` scratch state, and ensure every intended source/test/doc file is committed. +- [ ] Merge the completed feature branch into local `main` only after every activation check above passes; do not force, prune, rewrite unrelated work, or include unrelated dirty files. diff --git a/docs/superpowers/plans/2026-07-13-rust-2024-upgrade-and-merge.md b/docs/superpowers/plans/2026-07-13-rust-2024-upgrade-and-merge.md new file mode 100644 index 0000000..2720a82 --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-rust-2024-upgrade-and-merge.md @@ -0,0 +1,173 @@ +# Rust 2024 Upgrade and Merge Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Move Trail's parent-owned Rust crates to edition 2024 with an honest Rust 1.89 minimum, upgrade the installed stable toolchain, verify ACP v1 on that toolchain, and merge the feature branch into local `main`. + +**Architecture:** The root workspace contract owns the edition and MSRV for `trail` and `trail-environment-adapter-sdk`; the independently versioned `prolly` git submodule keeps its own edition contract. Run Cargo's edition migration before changing the root manifest, update user-facing build documentation and the ACP release plan, then verify the feature branch and the merged main checkout separately. + +**Tech Stack:** Rust edition 2024, Rust/Cargo 1.89 minimum, rustup stable, Cargo workspace manifests, Git linked worktrees. + +## Global Constraints + +- Do not modify or remove unrelated untracked files in `/Users/haipingfu/CrabDB`. +- Do not change the independently versioned `prolly` submodule's edition or commit pointer. +- Rust edition 2024 requires Rust 1.85; Trail's resolved dependencies raise the actual minimum to Rust 1.89. +- Merge only after the full Trail and ACP v1 verification commands pass on the upgraded toolchain. +- Remove the owned `.worktrees/acp-v1-conformance` worktree only after the merge and merged-result verification succeed. + +--- + +### Task 1: Upgrade the Toolchain and Workspace Contract + +**Files:** +- Modify: `Cargo.toml` +- Create: `rustfmt.toml` +- Modify: `Makefile` +- Modify: `README.md` +- Modify: `docs/getting-started/install-and-build.md` +- Modify: `docs/superpowers/plans/2026-07-12-acp-v1-full-conformance.md` + +**Interfaces:** +- Consumes: the existing root `[workspace.package]` edition and `rust-version` inherited by Trail-owned crates. +- Produces: edition `2024`, `rust-version = "1.89"`, stable 2021-style formatting, and a `make toolchain` target that updates stable Rust. + +- [ ] **Step 1: Upgrade the installed stable toolchain** + +Run: + +```sh +rustup update stable +rustup component add rustfmt clippy --toolchain stable +rustc +stable --version +cargo +stable --version +``` + +Expected: rustc and Cargo report version 1.89 or newer and both components install successfully. + +- [ ] **Step 2: Run Cargo's edition migration before changing the manifest** + +Run: + +```sh +cargo +stable fix --edition -p trail -p trail-environment-adapter-sdk --all-targets --allow-dirty +``` + +Expected: Cargo applies any source changes required to preserve edition-2021 behavior under edition 2024. + +- [ ] **Step 3: Update the workspace contract and toolchain helper** + +Set the root workspace package values to: + +```toml +edition = "2024" +rust-version = "1.89" +``` + +Change the Makefile toolchain recipe to run `rustup update stable` before installing `clippy` and `rustfmt` for stable. + +Create `rustfmt.toml` with `style_edition = "2021"` so the language upgrade does not introduce a repository-wide cosmetic formatting rewrite. + +- [ ] **Step 4: Update current build and ACP release documentation** + +Replace current Trail build requirements and executable ACP MSRV checks from Rust 1.81/edition 2021 to Rust 1.89/edition 2024. Preserve the official ACP reference peer as an independent interoperability package. + +- [ ] **Step 5: Verify the declared minimum** + +Run: + +```sh +cargo +1.89.0 check -p trail --all-targets +cargo +stable fmt --all -- --check +git diff --check +``` + +Expected: all commands exit zero. + +- [ ] **Step 6: Commit the upgrade** + +```sh +git add Cargo.toml Makefile README.md rustfmt.toml docs/getting-started/install-and-build.md docs/superpowers/plans/2026-07-12-acp-v1-full-conformance.md docs/superpowers/plans/2026-07-13-rust-2024-upgrade-and-merge.md trail trail-environment-adapter-sdk +git diff --cached --check +git commit -m "build: upgrade Trail to Rust 2024" +``` + +### Task 2: Verify Rust 2024 and ACP v1 + +**Files:** +- Modify only files required by failures caused by the edition migration. + +**Interfaces:** +- Consumes: edition-2024 Trail crates and the completed ACP v1 implementation. +- Produces: fresh test, conformance, fault, interoperability, schema-drift, and benchmark evidence. + +- [ ] **Step 1: Run the full Trail suite** + +```sh +cargo +stable test -p trail --all-targets +``` + +Expected: zero failed tests. + +- [ ] **Step 2: Run explicit ACP gates** + +```sh +cargo +stable test -p trail --test acp_conformance --test acp_faults -- --nocapture +scripts/test-acp-v1-reference-interop.sh +scripts/check-acp-v1-schema-drift.sh +cargo +stable bench -p trail --bench acp_relay_bench +``` + +Expected: all 23 methods, stable variants, 266 capability shapes, fault cases, both interoperability peers, schema drift, and the 10,000-frame benchmark pass. + +- [ ] **Step 3: Verify the branch is clean** + +```sh +cargo +stable fmt --all -- --check +git diff --check +git status --short +``` + +Expected: no output from the two Git commands. + +### Task 3: Merge and Verify Main + +**Files:** +- No planned source edits. + +**Interfaces:** +- Consumes: verified `feat/acp-v1-conformance` and local `main`. +- Produces: local `main` containing the ACP v1 and Rust 2024 commits. + +- [ ] **Step 1: Update and merge main without disturbing untracked paths** + +From `/Users/haipingfu/CrabDB`, verify the untracked paths do not collide with incoming tracked paths, update `main` with `git pull --ff-only`, and run: + +```sh +git merge --no-ff feat/acp-v1-conformance +``` + +Expected: merge succeeds without altering the pre-existing untracked paths. + +- [ ] **Step 2: Verify the merged result** + +```sh +cargo +stable check -p trail --all-targets +cargo +stable test -p trail --all-targets +cargo +stable fmt --all -- --check +git diff --check +``` + +Expected: every command exits zero. + +- [ ] **Step 3: Clean up the owned worktree and feature branch** + +From `/Users/haipingfu/CrabDB`: + +```sh +git worktree remove /Users/haipingfu/CrabDB/.worktrees/acp-v1-conformance +git worktree prune +git branch -d feat/acp-v1-conformance +``` + +Expected: `main` remains checked out, the worktree registration is removed, and the merged feature branch is deleted. diff --git a/docs/superpowers/specs/2026-07-12-acp-v1-full-conformance-design.md b/docs/superpowers/specs/2026-07-12-acp-v1-full-conformance-design.md new file mode 100644 index 0000000..25e0b47 --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-acp-v1-full-conformance-design.md @@ -0,0 +1,490 @@ +# ACP v1 Full Conformance Design + +Status: approved for implementation planning on 2026-07-12. + +## Objective + +Make Trail a complete, transparent, and verifiably conformant proxy between any +ACP v1 client/editor and any ACP v1 agent over the stable stdio transport. Trail +must preserve all valid ACP v1 behavior while adding lane isolation, durable +capture, and Trail MCP injection. Compatibility is a release gate, not a best- +effort claim. + +## Normative Baseline + +The implementation targets ACP wire protocol version `1` as defined by the +official Agent Client Protocol repository. The initial pinned conformance +baseline is upstream commit +`64cbd71ae520b89aac54164d8c1d364333c8ee5f`: + +- `schema/v1/schema.json`, SHA-256 + `92c1dfcda10dd47e99127500a3763da2b471f9ac61e12b9bf0430c32cf953796` +- `schema/v1/meta.json`, SHA-256 + `e0bf36f8123b2544b499174197fdc371ec49a1b4572a35114513d56492741599` + +The source of record is +. Trail will +vendor the two v1 artifacts and their source metadata for deterministic, +offline tests. A separate drift check will compare the vendored artifacts with +the latest official stable v1 artifacts. Any upstream stable-v1 change must +fail that check until Trail's fixtures, implementation, and compatibility +evidence are deliberately updated. + +ACP schema package versions and ACP wire versions are different. Trail's public +compatibility statement will name wire protocol version `1` and the pinned +schema artifact revision. + +## Meaning of Full Compatibility + +Trail may claim full ACP v1 compatibility only when all of the following are +true: + +1. Every valid ACP v1 request, response, notification, capability combination, + content variant, session update, error, and extension passes through Trail + with ACP-defined behavior. +2. All 23 stable methods in the official v1 `meta.json` have executable + conformance coverage in the correct direction: + - `initialize` + - `authenticate` + - `logout` + - `session/new` + - `session/load` + - `session/resume` + - `session/close` + - `session/list` + - `session/delete` + - `session/prompt` + - `session/cancel` + - `session/set_mode` + - `session/set_config_option` + - `session/update` + - `session/request_permission` + - `fs/read_text_file` + - `fs/write_text_file` + - `terminal/create` + - `terminal/output` + - `terminal/wait_for_exit` + - `terminal/kill` + - `terminal/release` + - `$/cancel_request` +3. Trail preserves request IDs, direction-specific ID scopes, response + correlation, notification ordering, error objects, `_meta`, and unknown + extension data. +4. Trail's intentional transformations are enumerated, atomic, capability- + aware, and validated against the official v1 schema before forwarding. +5. Multiple sessions and multiple in-flight bidirectional requests on one ACP + connection work without cross-session or cross-direction correlation. +6. Trail-specific capture is causally complete for ACP-visible activity and + cannot silently change wire behavior. +7. The conformance suite passes on every supported Trail platform. + +Passing only a prompt smoke test, forwarding unknown JSON, or passing the +existing ACP-focused tests is insufficient evidence for this claim. + +## Scope + +### Included + +- Stable ACP v1 over newline-delimited UTF-8 JSON-RPC stdio. +- Any conformant editor/client that can launch a local ACP agent command. +- Any conformant local ACP v1 agent, whether built in, registry-provided, or + passed as a custom command. +- Protocol negotiation, authentication, sessions, prompts, streaming updates, + permissions, configuration, modes, filesystem callbacks, terminal callbacks, + cancellation, errors, and extensions. +- Trail lanes, materialized workdirs, prompt checkpoints, transcripts, + approvals, traces, provenance, and MCP injection. +- Correct shutdown, malformed-peer behavior, backpressure, and recovery. + +### Excluded + +- ACP v2 semantics. Non-v1 negotiation must never be misrepresented as v1 + capture or compatibility. +- Streamable HTTP while it remains a draft rather than a stable ACP v1 + transport contract. +- Implementing an editor UI. Editor-specific setup adapters are convenience + surfaces; protocol compatibility must not depend on one editor's settings + format. +- Claiming that a non-conformant editor or agent can be repaired transparently. + Trail must diagnose peer violations without inventing protocol data. + +## Selected Architecture + +Trail will use a schema-driven transparent proxy, not a full protocol +terminator and not an expanded set of ad hoc JSON match arms. + +```text +ACP editor/client + | + | ACP v1 JSON-RPC stdio + v ++---------------- Trail ACP proxy ----------------+ +| frame codec | +| raw envelope + direction-aware correlator | +| negotiated-version and capability state | +| atomic session transformer | +| ordered durable capture journal | +| semantic capture projector | +| conformance diagnostics | ++--------------------------------------------------+ + | + | ACP v1 JSON-RPC stdio + v +ACP agent +``` + +The proxy holds the original JSON value for forwarding and a typed view for +validation and observation. Parsing a typed view must not discard unknown +fields or reserialize untouched messages. Known messages are checked with the +official ACP v1 types or vendored schema; extension messages and extension +fields remain in the raw envelope. + +## Component Boundaries + +The current `trail/src/acp.rs` combines provider discovery, transport, session +transformation, capture, and tests. The implementation will split only the ACP +responsibilities needed for conformance: + +- `trail/src/acp.rs`: public provider and relay entry points plus module wiring. +- `trail/src/acp/transport.rs`: stdio child lifecycle, frame reading/writing, + pumps, shutdown, and stderr forwarding. +- `trail/src/acp/protocol.rs`: raw JSON-RPC envelope classification, + direction-aware request correlation, negotiated v1 state, method catalog, + and schema validation hooks. +- `trail/src/acp/transform.rs`: atomic initialization metadata, MCP injection, + cwd/additional-directory mapping, and validation of transformed requests. +- `trail/src/acp/capture.rs`: ordered journal ingestion and semantic projection + into Trail sessions, turns, messages, events, spans, approvals, and + checkpoints. +- `trail/src/acp/registry.rs`: official registry resolution and package/binary + launch, retained as a separate concern. +- `trail/src/acp/setup.rs`: editor configuration planning and application, + retained as a separate concern. +- `trail/tests/acp_conformance.rs`: black-box protocol conformance harness. +- `trail/tests/fixtures/acp/v1/`: pinned official schema, method metadata, + source manifest, and method/variant fixtures. + +No module may depend on terminal rendering. Transport must not depend on Trail +database report types. Capture may consume protocol observations but may not +write to either protocol stream. + +## Protocol Data Flow + +### Framing + +The frame codec accepts exactly one UTF-8 JSON-RPC object per non-empty line. +Messages are delimited by `\n` and serialized without embedded raw newlines, +matching ACP v1 stdio. Standard JSON string escapes remain valid. Blank lines +may be ignored for resilience but are never emitted. + +Malformed JSON, a non-object top-level value, an invalid JSON-RPC version, or +an invalid request envelope is a peer protocol violation. If an ID can be +recovered safely, Trail emits the appropriate JSON-RPC error to the sender; +otherwise it emits a parse/invalid-request error with a null ID. Trail records +the diagnostic, closes affected in-flight capture state as interrupted, and +shuts down cleanly when continuing would make framing ambiguous. + +### Initialization and version negotiation + +Trail forwards the editor's `initialize` request unchanged. It must not reject +an editor merely because the editor offers a later protocol version: ACP +negotiation allows the downstream agent to select version `1`. + +Trail forwards the agent's initialize response after optionally adding only the +documented `_meta.trail` extension. The modified response is validated before +it replaces the raw response. Trail records the selected version and both +capability sets. + +- If the selected version is `1`, Trail enables v1 validation, transformation, + and semantic capture. +- If the selected version is not `1`, Trail forwards the response so the client + can follow ACP negotiation rules, disables all v1 transformations, reports + that the connection is outside Trail's supported protocol, and does not claim + capture compatibility. +- Session methods received before successful initialization are forwarded only + as raw peer behavior and produce a diagnostic; Trail must not create a lane + from an unnegotiated session. + +Trail never adds or removes advertised editor or agent capabilities. + +### Requests, notifications, and responses + +Request IDs may be strings or numbers. Client-to-agent and agent-to-client +requests have independent ID spaces. Correlation keys therefore include both +direction and the exact JSON ID value. Responses may arrive out of order. + +The proxy forwards requests, notifications, responses, and errors in arrival +order within each input stream. It must not wait for an earlier request's +response before forwarding later traffic. Capture observations carry a +monotonic relay sequence so database projection can reproduce causal ordering +across both directions. + +`$/cancel_request` is correlated with the request in the same sender's request +space. `session/cancel` remains a distinct session notification. Neither may be +translated into the other. + +Unknown methods are forwarded unchanged. Methods beginning with `_` are treated +as implementation extensions. Unknown non-extension methods are also preserved +because stable v1 may gain optional methods without a wire-version change; the +schema-drift gate ensures Trail deliberately learns their semantics before a +new full-compatibility release. + +## Intentional Transformations + +Trail may perform only these protocol mutations: + +1. Add `_meta.trail` to a successful `initialize` response without replacing + other `_meta` keys. +2. Add one Trail stdio MCP server to `mcpServers` for `session/new`, + `session/load`, and `session/resume` when MCP injection is enabled and an + equivalent Trail server is not already present. +3. Replace session workspace paths with their effective lane paths when + materialization is enabled. + +Each transformation follows clone-transform-validate-commit: + +1. Clone the raw JSON value. +2. Apply the complete mutation to the clone. +3. Validate the clone as the corresponding ACP v1 request or response. +4. Replace the forwarded value only after validation succeeds. + +Transformation failure must never leak a partially modified message. A failed +session transformation returns a JSON-RPC error to the editor and does not +forward the request or create an active upstream session. + +### MCP injection + +The injected MCP descriptor uses the ACP v1 stdio server shape with an absolute +Trail executable, `args: ["mcp"]`, and environment entries for the explicit +workspace and Trail directory. Existing servers and their order are preserved; +Trail is appended once. Equivalence checks use the Trail-owned identity rather +than deleting or replacing an editor-provided server with the same display +name. + +### Workspace path mapping + +For a session whose requested `cwd` is the Trail workspace root, the effective +cwd is the lane workdir root. For a requested cwd below the workspace root, +Trail preserves the relative suffix below the lane workdir and verifies that +the mapped directory exists or can be created safely. A path outside the Trail +workspace is preserved unchanged because rewriting it would change ACP +semantics and Trail does not own that tree. + +Every `additionalDirectories` entry is handled independently: + +- A path at or below the Trail workspace root maps to the corresponding lane + workdir path with the same relative suffix. +- An external path is preserved exactly and recorded as a non-isolated external + session root. +- Canonicalization, symlink checks, and platform path rules prevent a mapped + path from escaping its intended root. + +The exact requested/effective root mapping is persisted with the ACP session +and reused by load/resume. Agent-to-editor filesystem and terminal callbacks +already contain paths in the effective session namespace and therefore remain +unchanged. Trail validates that path-bearing callbacks are consistent with the +negotiated effective roots before capture; it does not rewrite editor callback +paths behind either peer's back. + +## Complete Semantic Capture + +Wire compatibility and Trail capture are separate responsibilities. Transport +forwards the protocol message; capture projects a redacted observation. + +The capture projector handles every stable v1 method: + +- Initialization records negotiated versions, implementation information, and + capabilities without recording secrets. +- Authentication/logout record lifecycle and outcome metadata; credentials and + secret-bearing fields are redacted before persistence. +- New/load/resume/close/list/delete maintain durable ACP-to-Trail session state. +- Session load replay reconstructs user and assistant transcript messages, + message IDs, plans, tools, modes, configuration, session information, and + usage without inventing a prompt turn. +- Prompt creates one active Trail turn. All content blocks remain represented; + binary/image/audio data use bounded metadata and digests rather than raw + unbounded attachments. +- Every stable `session/update` variant is projected with its ACP identity. + Agent thought content remains intentionally excluded by privacy policy, but a + redacted fact that an excluded update occurred may be recorded. +- Permission requests and responses preserve option identity and final outcome + in Trail approvals. +- Mode/config requests and updates maintain the current session projection. +- Filesystem requests/responses record the operation, path, result, and linkage + to the active turn. File contents use Trail's existing redaction and bounded- + attachment policies. +- Terminal create/output/wait/kill/release maintain terminal lifecycle, + truncated-output state, exit status, and tool/turn linkage. +- Both cancellation mechanisms record the exact target and eventual outcome. +- Unknown updates and extension methods remain ordered redacted events with raw + digests so they are not silently discarded. + +Capture state is keyed by ACP session and direction-aware request identity. It +supports concurrent prompts in different sessions on one connection. A peer +attempting overlapping prompts for the same session is handled according to the +downstream agent's response and must not merge two Trail turns. + +## Durable Capture Journal and Backpressure + +Database writes must not occur inline with high-volume protocol forwarding. +The relay writes bounded redacted observations to an ordered local journal and +an independent projector batches them into Trail under the workspace writer +lock. + +The journal is opened before Trail accepts the first session request. If a +session requires Trail capture but the journal cannot be opened, Trail returns +a session-creation error instead of pretending the session is captured. Once a +session is active: + +- a journal append is bounded in size and time; +- database writer contention does not block protocol forwarding; +- a temporary projector failure leaves journal entries replayable; +- sequence numbers and idempotency keys prevent loss, duplication, or reorder; +- shutdown drains or durably leaves the journal for recovery; +- permanent journal failure marks capture degraded visibly, closes Trail turn + state as interrupted, and continues forwarding already-active ACP traffic so + Trail does not corrupt the editor-agent conversation. + +This policy distinguishes protocol correctness from local evidence +availability while never silently claiming evidence that was not persisted. + +## Errors, Shutdown, and Recovery + +- Upstream spawn failure is reported before protocol startup. +- Editor EOF closes upstream stdin, gives the agent a bounded graceful-exit + window, then terminates the child if necessary. +- Agent EOF or exit closes open requests and turns with an outcome derived from + the real exit status; a clean idle exit is not mislabeled as a failed prompt. +- SIGINT/SIGTERM propagate to the child, close capture state, and leave a + replayable journal. +- Malformed peer traffic cannot panic the relay or poison later workspace use. +- Capture or transformation diagnostics go to stderr only; stdout remains ACP + JSON-RPC exclusively. +- Secret values are redacted before journal persistence, diagnostic rendering, + command recording, or Trail events. +- No failure path substitutes `--force`, bypasses ignore/guardrail policy, or + mutates Git. + +## Conformance Test Architecture + +### Schema validation + +Every official v1 request, notification, response, and error fixture validates +against the pinned schema. Every transformed message validates again after +mutation. Invalid fixtures prove that validation rejects wrong method shapes, +missing required fields, invalid enums, wrong ID types, and malformed `_meta`. + +### Method matrix + +The black-box relay harness drives all 23 methods through a real Trail relay +process with independent editor and agent fixtures. Each method covers: + +- correct direction; +- success response; +- JSON-RPC error response where applicable; +- string and numeric IDs where applicable; +- preservation of unknown fields and `_meta`; +- ordering with unrelated in-flight traffic; +- semantic capture evidence; +- relay shutdown with the method in flight. + +### Variant matrix + +The suite covers every stable v1 capability and union variant, including: + +- all prompt content blocks and advertised prompt capabilities; +- stdio, HTTP, and SSE MCP descriptors accepted by v1 capabilities; +- every stable session update variant; +- every tool-call content/status/kind shape; +- every permission option and outcome; +- modes, select/boolean config options, commands, plan entries, usage, and + session information; +- all filesystem optional ranges and responses; +- terminal output truncation, running/exited states, signals, kill, wait, and + release; +- session pagination, load replay, resume, close, delete, and additional roots; +- authentication methods, authentication failures, and logout; +- request and session cancellation races. + +### Concurrency and fault matrix + +Tests cover multiple sessions on one connection, bidirectional requests with +identical JSON IDs, out-of-order responses, concurrent permission/filesystem/ +terminal calls, editor EOF, agent EOF, crashes, malformed messages, slow peers, +blocked Trail database writers, journal replay, duplicate observations, and +process interruption. + +### Interoperability + +The release gate includes: + +- the black-box reference client/reference agent harness built from official ACP + v1 types; +- Trail's complete unit and E2E suite; +- existing real-agent smoke coverage for supported built-in adapters when + credentials and executables are available; +- documented opt-in runs for Zed and at least two independent ACP agents; +- macOS, Linux, and Windows CI for platform-neutral protocol behavior. + +Real-provider tests supplement but do not replace the deterministic conformance +suite. + +### Performance + +A release benchmark verifies semantic identity and zero message loss while +measuring forwarding overhead. Uncontended local relay overhead must remain +small relative to agent/model latency, and the benchmark records p50/p95/p99 +rather than hiding tail latency. Performance results cannot waive any +correctness assertion. + +## Public Compatibility Report + +`trail agent acp doctor` will report: + +- supported ACP wire version; +- pinned schema source revision and digests; +- transport support; +- provider launch readiness; +- journal and capture readiness; +- materialization/path-mapping readiness; +- conformance build identifier; +- explicit unsupported items, including v2 and draft remote transport. + +Documentation will distinguish: + +- protocol conformance; +- editor setup convenience; +- agent availability/authentication; +- capture fidelity; +- lane isolation, including preserved external additional roots. + +Trail may print “ACP v1 conformant” only when built from a revision whose full +conformance gate passed. Development or locally modified builds report their +evidence as unverified unless the gate is rerun. + +## Acceptance Gates + +The change is complete only when all gates pass together: + +1. The pinned official schema and metadata digests match the source manifest. +2. The schema drift check reports no unreviewed stable-v1 change. +3. All 23 method-matrix entries pass success, error, preservation, capture, and + shutdown assertions. +4. Every stable capability and union variant has an executable fixture and + passes schema validation and relay round-trip tests. +5. Multiple sessions, direction-scoped IDs, out-of-order responses, and all + cancellation races pass without cross-correlation. +6. Cwd subdirectories and additional directories preserve ACP semantics under + materialization on every supported platform. +7. Capture journal fault/replay tests prove no silent evidence loss, + duplication, or reorder. +8. Full Trail unit, integration, E2E, formatting, lint, and documentation tests + pass from a clean checkout. +9. Reference-client/reference-agent interoperability passes. +10. Available real-editor/real-agent smoke tests pass and unavailable external + dependencies are reported as skipped evidence, never counted as conformance + proof. +11. Documentation and doctor output state the exact verified boundary without + claiming ACP v2 or draft transport support. + +No subset of these gates is sufficient for completion. diff --git a/docs/superpowers/specs/2026-07-12-correctness-first-changed-path-ledger-design.md b/docs/superpowers/specs/2026-07-12-correctness-first-changed-path-ledger-design.md new file mode 100644 index 0000000..0cc13ad --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-correctness-first-changed-path-ledger-design.md @@ -0,0 +1,658 @@ +# Activated Changed-Path Ledger + +Status: approved implementation design + +## Purpose + +Make the common clean and small-change paths for Trail workspaces and +materialized lanes proportional to authoritative changed-path candidates rather +than repository size, without treating incomplete observation as proof that a +worktree is clean. + +This design complements the persistent case-fold path index. That index makes +immutable-root mutation validation local. The changed-path ledger makes mutable +filesystem observation local only when a qualified evidence provider proves +continuity. Linux and macOS ship with qualified observers and activate the +ledger by default. Other platforms retain the direct full-observation command +path and cannot persist ledger trust. + +## Non-negotiable invariants + +Only a `trusted` scope may prove clean or supply a complete candidate set. +`reconciling`, `overflow`, `untrusted_gap`, `stale_baseline`, and `corrupt` +never report clean. + +A callback watermark is not an observation fence. Generic `notify` callbacks +prove only which events have arrived, not that all writes before a snapshot cut +have been delivered. Clean proof requires one of: + +- a continuously owned live provider with complete sequencing and a + linearizable delivery fence; +- a provider with a durable cursor and linearizable delivery fence when trust + is resumed across owner or process restart; +- a controlled writer whose intent and authoritative publication are covered + by Trail's transaction protocol; +- a documented external-writer quiescence protocol held through a full + reconciliation cut; or +- an exact Git/VCS comparison whose baseline and semantics equal the ledger + baseline. + +Observer owner loss or restart rotates the scope epoch and requires +reconciliation unless a qualified durable cursor resumes without a gap. There +is no generic cross-platform shortcut after arbitrary writes occur while no +qualified observer is active. + +Activation is one hard release gate. Schema, recovery, reconciliation, policy +dependency tracking, every controlled producer, both qualified Linux and macOS +adapters, automatic daemon lifecycle, and read-path integration must land and +pass together. There is no partially authoritative rollout and no feature mode +that promotes trust before all activation gates pass. + +Activation and publication are exact-revision gates. The reusable native +workflow must pass on Linux/ext4 and macOS/APFS for the exact SHA before release +automation may create a tag, and the generated cargo-dist graph must run the +same gate for the tag SHA before any build, host, or publish phase. A compiled +manifest hash names this contract but cannot substitute for completed workflow +jobs. + +## Scope and filesystem identity + +Every evidence record binds to: + +- stable scope ID, kind, and owner (`workspace`, `materialized_lane`, or + `workspace_view`); +- pinned canonical scope-root handle/identity and platform-specific filesystem + identity; +- mutable ref name, generation, change ID, and baseline root ID; +- compiled Trail policy fingerprint and persisted dependency manifest; +- ledger epoch and qualified provider cursor/fence state; +- observer capability record and owner token. + +Evidence from a different root, policy, filesystem, epoch, provider, or owner +cannot prove clean. Root replacement, mount change, case-sensitivity change, or +scope directory replacement invalidates the epoch. + +Watcher paths are normalized relative to a pinned no-follow scope root. Case +aliases and case-only renames preserve exact spelling while using the existing +folded-path collision rules. Unsupported/non-UTF-8 names make the affected +scope or provider unqualified; they are never silently dropped. + +## Trust states + +- `trusted`: all controlled producers and a qualified external provider prove + the candidate set complete through the stored cut. +- `reconciling`: a persisted staged reconciliation exists but is not published. +- `overflow`: a qualified provider reported lost evidence or bounds exceeded. +- `untrusted_gap`: no qualified evidence covers an interval. +- `stale_baseline`: ref, root, policy, filesystem, or provider identity changed. +- `corrupt`: ledger, journal, intent, or schema validation failed. + +Only `trusted` can return `Clean` or an authoritative bounded candidate set. +An exact dirty path/prefix may remain useful while a scope is untrusted, but it +cannot prove that no other path changed. + +## Observer capability tiers + +Each adapter persists capabilities rather than relying on its name: + +```text +durable_cursor +linearizable_fence +rename_pairing +overflow_scope +filesystem_supported +clean_proof_allowed +power_loss_durability +``` + +- Generic `notify` has no durable cursor or generic delivery fence and remains + advisory. +- Linux uses a native recursive inotify adapter with explicit watch coverage, + rename cookies, queue-overflow scope, directory-race detection, and a + sentinel delivery fence. It qualifies only while the owner is live and after + a fence-covered reconciliation; restart without a continuous owner requires + reconciliation. +- macOS uses FSEvents file events with persisted event IDs, root identity, + `MustScanSubDirs`/gap handling, and synchronous stream flushing. It resumes a + cursor only when the platform proves continuity; otherwise it reconciles. +- Unknown, NFS, SMB, FUSE, overlay, network, or replaced filesystems default to + reconciliation-required unless an adapter explicitly qualifies them. +- Process-crash durability and host/power-loss durability are separate + capabilities and test claims. + +Actual ext4 and APFS adapters/filesystems are exercised in scheduled jobs; +injected callback tests alone cannot qualify a provider. + +## Persistence and schema completeness + +Schema v18 is a hard cutover, not a migration. Only an explicit fresh +`trail init` may create it. Before opening the mutable stores, every existing +workspace is preflighted through a read-only SQLite connection. Any version +other than 18, or any partial/malformed v18 shape, returns stable +`SCHEMA_REINITIALIZE_REQUIRED` guidance and leaves the database and sidecars +byte-for-byte unchanged. Nothing is repaired on open. + +Fresh creation writes the final v18 schema in one savepoint, records both +`PRAGMA user_version` and schema metadata only after all objects exist, and +validates the exact tables, columns, defaults, checks, foreign keys, uniqueness, +index order/collation, and log-format bounds before release. There are no +legacy columns, upgrade functions, backfills, compatibility views, or +`CREATE IF NOT EXISTS` repair paths for an existing workspace. + +Fresh scopes begin untrusted and can become `trusted` only after qualified +observation and a successfully published reconciliation. + +### Scope and candidate tables + +`changed_path_scopes` stores stable scope/owner identity, filesystem identity, +ref generation/change/root, policy fingerprint/dependency generation, trust +state/reason, epoch, provider capabilities/cursor, durable/folded offsets, +single-writer owner token, heartbeat/error state, and timestamps. It has a +unique `(scope_kind, owner_id)` constraint. + +`changed_path_entries` stores exact normalized paths, event flags, source mask, +first/last sequence, and provider/intent ownership. `changed_path_prefixes` +stores coalesced dirty prefixes with completeness reason and sequence bounds. +Neither stores all clean paths. Exact and prefix queries use indexed binary +ranges, not escaped `LIKE` scans. + +`changed_path_policy_dependencies` stores every policy input needed to reuse a +compiled policy without repository-wide rediscovery: normalized dependency +path or external identity, kind, content/metadata identity, observability, +generation, and last source sequence. Raw dependency events invalidate trust +before normal ignore filtering. + +### Controlled intent tables + +`changed_path_intents` stores producer, expected scope epoch/ref generation and +root, immutable target change/root/operation, start cursor, and lifecycle: +`prepared`, `filesystem_applied`, `published`, `acknowledged`, `aborted`. +`changed_path_intent_paths` and `changed_path_intent_prefixes` store affected +evidence. Prepared and published targets are GC roots until recovery reaches a +terminal state. + +### Reconciliation staging + +`changed_path_reconciliations` stores attempt ID, scope/epoch, expected ref, +filesystem/policy/provider identity, start fence/cursor, mode/reason, +completeness class, staged store location, and state. A crash-safe temporary +table/store streams observed entries and deletions without retaining all files +in memory. Publication is one CAS transaction; abandoned attempts are safe to +discard. + +### Observer segment metadata + +SQLite stores log format version, epoch, owner token, segment ID, first/last +sequence, durable end offset, folded end offset, hash linkage, state, and +rotation lineage. A retired writer cannot append evidence to a reused scope. + +## Versioned observer log protocol + +High-rate callbacks never acquire the workspace lock or primary `Trail` SQLite +connection. A qualified adapter owns an exclusive epoch/owner lease and appends +versioned length-prefixed records to +`.trail/index/change-ledger///`. + +Every segment header contains format, scope, epoch, owner, provider cursor, and +previous-segment hash. Records have monotonic sequences, normalized payload, +checksum/hash linkage, and explicit rename/prefix semantics. The writer +publishes a durable end offset only after the configured fsync/group-commit +boundary. Rotation fsyncs the old tail and directory, publishes the next +header, then updates ownership metadata. + +Append/flush/disk-full/checksum/lease/heartbeat/callback failure revokes clean +proof. Because the same I/O failure may prevent persisting an error marker, a +clean-returning snapshot must obtain a fresh fence nonce/cut from the live epoch +owner after that owner successfully flushes and validates its append path. +Failure to contact, fence, flush, or validate the owner returns untrusted. +Persisted error markers and heartbeat expiry are diagnostic fallbacks, never +the sole revocation mechanism. A snapshot must also fold or conservatively +merge the durable tail; it cannot trust only an older SQLite folded offset. +Read-only clients either obtain this proven live snapshot, fold/merge a +provider-qualified safe tail, or return reconcile-required. + +Observer logs and intent files are explicitly synced before their durability +offset or lifecycle transition is published. SQLite remains WAL +`synchronous=NORMAL`, so a dead owner or host restart always revokes trust and +forces reconciliation unless a platform durable cursor independently proves +continuity. No power-loss clean-proof capability is advertised without a +stronger transaction/fsync path and fault tests. + +## Deep module API + +One `ChangedPathLedger` owns trust, evidence, CAS, and fallback semantics: + +```text +begin_scope(scope, baseline, policy, filesystem, provider) +snapshot(scope, expected_ref, required_capabilities) +prepare_intent(scope, expected, target, paths, prefixes) +mark_filesystem_applied(intent, verified_cut) +publish_intent(intent, ref_and_ledger_transaction) +acknowledge(snapshot, owned_evidence, covered_source_sequences) +mark_prefix_dirty(scope, complete_prefix_proof) +mark_untrusted(scope, state, reason) +begin_reconciliation(scope, expected, mode) +publish_reconciliation(attempt, expected_fence) +advance_baseline(scope, expected, target, covered_evidence) +recover(scope) +``` + +Snapshots include epoch, ref/root, policy/filesystem identity, capabilities, +provider cursor/fence, durable/folded offsets, exact paths/prefixes, source and +last sequence, and trust state. + +## Module boundaries + +The implementation is split so persistence, observation, recovery, and command +integration can be audited independently: + +- `change_ledger/types.rs`: typed identities, capabilities, trust states, cuts, + evidence, intents, and snapshots; +- `change_ledger/store.rs`: exact SQLite persistence, CAS transitions, binary + range queries, and schema-independent state validation; +- `change_ledger/log.rs`: versioned records, checksums, hash chains, rotation, + durable offsets, and safe-tail recovery; +- `change_ledger/observer/linux.rs`: qualified inotify ownership, recursive + coverage, event normalization, overflow, rename, and fence protocol; +- `change_ledger/observer/macos.rs`: qualified FSEvents stream, event-ID + continuity, gap flags, root identity, and flush protocol; +- `change_ledger/reconcile.rs`: streamed full/prefix observation and CAS + publication; +- `change_ledger/intent.rs`: controlled filesystem mutation state machines; +- `change_ledger/recovery.rs`: leases, segments, intents, mirrors, and GC roots; +- `change_ledger/policy.rs`: compiled policy dependency persistence and raw + invalidation; +- `change_ledger/snapshot.rs`: live fencing, tail folding, candidate snapshots, + and sequence-aware acknowledgement. + +Daemon startup and `status`/`diff`/`record` integrations remain thin consumers +of this module. No read path reimplements trust decisions. + +## Automatic per-workspace daemon + +The first `status`, `diff`, or `record` securely discovers or starts one +per-workspace daemon. Startup uses an exclusive process lock, a random +authenticated local endpoint, owner nonce, executable identity, and atomic +endpoint publication. Concurrent clients converge on the same owner and wait +for a bounded readiness result. Stale endpoint files and dead owners are +replaced only after process-identity validation. + +The daemon acquires the observer lease, runs recovery, establishes platform +watch coverage, reconciles when trust is absent, and publishes readiness only +after a valid fence. Failure to start or reconcile returns +`CHANGE_LEDGER_RECONCILE_REQUIRED`; it never exposes an older persisted cache +as authoritative. + +An automatic daemon is an optimization owner, not a correctness dependency. +If it exits, the next command starts a new owner and reconciles any interval +that cannot be proven continuous. + +## Read command flow + +`status` and `diff` follow one flow: + +1. ensure the authenticated daemon is ready; +2. recover unfinished state and validate scope/ref/root/policy/filesystem/ + provider identities; +3. fold the durable observer tail through fence `c1`; +4. automatically run full reconciliation and retry if trust is unavailable; +5. load only exact paths and complete dirty prefixes; +6. compare candidates with pinned no-follow reads and batched root lookups; +7. fence/fold through `c2`, retaining or retrying for events newer than `c1`; +8. return clean only when the complete candidate set is empty at the proven + cut. + +On Linux and macOS, the existing direct full-walk and Git paths are +reconciliation/oracle implementations, not permissive ledger clean-proof +fallbacks. Unsupported platforms continue to use direct full observation and +never publish a trusted ledger scope. + +## Controlled and observed mutation classes + +Every filesystem producer is assigned to exactly one state machine. + +### Observed record/checkpoint + +Manual record, materialized-lane record, and native-agent checkpoint observe +external filesystem state and then publish a new immutable root: + +1. Acquire the workspace lock and capture expected ref/scope epoch. +2. Obtain qualified provider fence `c1` (or hold documented quiescence). +3. Read/hash candidates through pinned no-follow handles. +4. Obtain fence `c2`; retain all evidence newer than `c1`. +5. Build/store immutable target root/operation from the observation. +6. In one SQLite transaction, index the operation, CAS ref, advance the ledger + baseline, and acknowledge only evidence source-sequenced through `c1`. +7. Repair mirrors and retain later/different-source same-path evidence. + +### Ref-advancing controlled projection + +A controlled command that both knows the intended target and advances a ref: + +1. Lock and capture expected ref/scope epoch. +2. Compute/store immutable target root/operation. +3. Durably prepare exact/prefix intent referencing the target. +4. Apply and sync files/directories; verify intended content/mode/type. +5. Fence the qualified provider and retain unrelated/later evidence. +6. In one SQLite transaction, index operation, CAS ref, publish intent, advance + baseline, and acknowledge only intent-owned/covered evidence. +7. Repair mirrors and reach a terminal intent state. + +### Projection-only alignment + +Checkout materialization, sparse hydration, and workdir sync often project an +existing ref/root without creating an operation or advancing a ref. Their +intent references that existing target, applies/syncs/verifies the filesystem, +then fences/flushes the qualified provider, retains unrelated or later +source-sequenced evidence, and atomically publishes only the scope +baseline/marker and terminal intent under unchanged ref/epoch/provider-cut CAS. + +An external same-path event with a later or different source sequence is never +cleared by path equality. A crash leaves a false-positive candidate, not a +missed change. Recovery runs before any later mutation or GC and compares +intent, filesystem verification, ref, operation, and ledger states to finish, +retain candidates, or reconcile. + +## VFS/COW producer exception + +VFS callbacks are a separate producer class because they already hold the +shared view barrier. They acquire only that barrier and append/fsync a per-view +intent record before semantic upper/whiteout mutation. They never acquire the +workspace lock or primary SQLite connection. + +Checkpointing acquires workspace lock, then exclusive view barrier, folds the +per-view journal, verifies candidates, builds immutable targets, and publishes +ref/ledger state. Group-commit acknowledgement must not expose a filesystem +mutation before its intent is durable under the claimed crash model. + +First implement journal/whiteout correctness, active-handle generation +ownership, and crash recovery. Only then compact: replay records after the +clean cut, rotate at checkpoint, roll generations when no active handle uses +the old upper, and retire old upper/whiteouts. `upper_recovery_walks=0` is +permitted only for a qualified trusted journal; untrusted, corrupt, or gapped +views explicitly scan and reconcile. + +Whiteouts use an incremental map/log with atomic mutation semantics rather than +rewriting all class arrays. + +## External observation and snapshot cuts + +Generic `w1`/`w2` callback flushing prevents clearing already-received later +events but is not a delivery fence. A qualified provider must supply a durable +cursor/fence proving all changes through the cut. Without that, external +writers must be quiesced through enumeration/hash/publication or the scope +remains `untrusted_gap`. + +For a qualified provider: + +1. fence and fold through cut `c1`; +2. read/hash authoritative candidates; +3. fence/fold through `c2`; +4. publish/ack only evidence whose source sequence is covered by `c1`; +5. retain later/different-source evidence against the next baseline. + +Delayed delivery, callback backlog, cursor discontinuity, owner death, lease +replacement, and ambiguous rename tests must prove fail-closed behavior. + +## Git adapter + +Git is authoritative only when the ledger baseline root exactly equals the +mapped Git HEAD/index tree under equivalent file-mode, symlink, sparse, +submodule, and ignore semantics. A mapping to an arbitrary Git state is not +enough: after Trail records ahead of HEAD, a reversion toward HEAD can be clean +to Git but dirty to Trail. + +Use porcelain v2 `-z`, retain both rename endpoints, and bind evidence to HEAD, +index/split/shared-index identity, exact mapped Trail root, filesystem identity, +and Trail policy equivalence. `assume-unchanged`, `skip-worktree`, unresolved +sparse/submodule behavior, racy/untrusted fsmonitor, or policy mismatch disables +Git clean proof. Git may remain a conservative advisory source. + +Fsmonitor and untracked-cache improve locality only when their continuity is +qualified. Metrics expose Git subprocess count, index refresh/Trace2 work, and +whether Git internally performed global work. + +## Materialized lanes and deployment contract + +Replace the N-entry JSON manifest only after explicit reconciliation with a +compact v2 marker containing scope ID, filesystem identity, ref/generation/root, +policy fingerprint, epoch, and provider cut. Marker and trusted scope identity +publish at the same logical cut; mismatch is stale. + +Materialized-lane O(k) operation requires the automatically managed qualified +observer or a durable COW journal that proves continuity. Owner loss marks the +scope untrusted and the next command automatically reconciles it. Once trusted, +lane status, record, readiness, merge checks, and ordinary preview consume +ledger candidates. Full recursive ignored/risk scans remain explicit audit or +reconciliation operations. + +## Trail policy and adapter equivalence + +`RecordingPolicySnapshot` is Trail's authoritative semantic policy. It is +shared by Trail walkers and candidate checks; each external adapter declares +whether its semantics are equivalent or only conservative. + +Reconciliation discovers and persists a dependency manifest covering built-in +rule version, configuration, all relevant nested `.trailignore`/`.gitignore`, +`.git/info/exclude`, `core.excludesFile`, included Git config, system/global +inputs, normalization, mode/symlink rules, and case sensitivity. Observers see +raw policy-file events before ignore filtering. Unobserved or unresolved +external dependencies mark the scope stale. Fast paths reuse the persisted +dependency manifest rather than traversing the tree to rediscover it. + +## Path/prefix semantics + +- File deletion: exact candidate; absence against baseline means deletion. +- File rename: both endpoints with source/provider sequence. +- Deleted directory: complete dirty prefix; expand baseline Prolly range. +- Added directory: scan only the new subtree when provider proof is complete. +- Directory rename: expand old baseline prefix and scan new subtree. +- Conservative complete ambiguous event: dirty parent prefix, still trusted. +- Actual event loss/unknown overflow: global untrusted state. +- Coalesce overlapping prefixes without losing completeness provenance. + +Prefix reconciliation may promote trust only when persisted provider proof says +the invalidation is fully contained in those prefixes. User-supplied prefixes +can refresh data but cannot repair global overflow, owner loss, unknown gap, +global policy change, or corruption; those require a full scope reconciliation. + +## Correctness-grade reconciliation + +Add `CHANGE_LEDGER_RECONCILE_REQUIRED` with stable exit code, structured scope, +state/reason, and exact recovery commands: + +```text +trail index reconcile +trail index reconcile --lane +``` + +The error/report propagates through human and JSON CLI, daemon RPC, REST/OpenAPI, +and MCP. Read-only surfaces never ignore an unread log tail. Normal +`status`/`diff`/`record` first attempt automatic full reconciliation; this error +is returned only when daemon startup, observation, or reconciliation cannot +establish trust. + +Gap recovery enumerates the required scope and baseline Prolly range, hashes +all relevant regular-file content through pinned no-follow handles, validates +before/after identity to detect replacement, and finds deletions by comparing +both sides. Cached stamps can optimize only when a gap-free qualified journal +proves they cover the interval. + +The staged attempt publishes only if ref/root, scope epoch, filesystem identity, +policy/dependency generation, provider capabilities, and a real fence/cursor +still match. Races retry or remain untrusted. + +Every normal command automatically performs and labels full reconciliation +when required. Authorization to run the command permits this expense but does +not create an observation fence. Reconciliation promotes trust only with a +qualified provider fence or documented external-writer quiescence; otherwise +the command fails with the stable recovery action. No O(N) run may be +represented as a trusted fast path, and reconciliation metrics are distinct +from warm command metrics. + +## Concurrency and lock order + +Ordinary writers/checkpointing use workspace write lock, then applicable +exclusive scope/view barrier, then a short ledger/authoritative SQLite +transaction. Filesystem reads occur outside long SQL transactions while pinned +to a snapshot identity. + +VFS callbacks use only the shared view barrier and per-view append protocol. +External watcher callbacks use only the observer append path and bounded memory. +Neither acquires the workspace lock or primary connection. Snapshot and +baseline changes use ref/epoch/provider-cut CAS. + +## Backup, restore, deletion, and GC + +Backup either fences/folds and snapshots matching SQL+segment state or marks +backed-up scopes untrusted. Active views require their barriers or restore as +untrusted. Restore always rotates epochs, clears owner/cursor state, discards +unmatched tails, rebinds filesystem identity, and marks scopes +reconcile-required. + +Prepared/published intent target roots and operations are GC roots until +recovery reaches a terminal state. Lane/view deletion transactionally retires +scopes and revokes owners before segments are removed. Tests cover crash during +backup rotation, restore to another filesystem, orphan segments, and GC before +and after intent recovery. + +## Structural observability + +Track per operation: + +- full scope walks/root ranges/SQLite full-index rows; +- authoritative input candidate count versus final changed output; +- candidate reads, hashes, root point lookups, and ledger rows; +- marker bytes, log bytes/segments, folded/tail records, backlog; +- upper recovery walks and journal compaction work; +- policy dependency work and adapter equivalence; +- observer capability/cursor/fence/trust and reconciliation reason; +- Git subprocess/index-refresh/Trace2 global work; +- wall time and peak RSS. + +## Acceptance gates + +Fresh-schema tests prove exact v18 creation. Hard-cutover tests snapshot every +database and sidecar byte, attempt to open v17, version 0 outside explicit +initialization, newer versions, and partial/malformed v18, and prove rejection +without mutation. Existing-schema open contains no DDL or repair path. + +CI uses 1k; scheduled Linux and macOS jobs use 100k on qualified ext4 and APFS +environments; nightly jobs use 1M, cold reconciliation, daemon restart, and +crash matrices. Exercise input candidate counts `k=0,1,100` separately from +final changes for workspace status/diff/record, materialized lane record, +structured patch maintenance, and COW checkpoint. + +Warm trusted assertions: + +```text +full_scope_walks = 0 +root_full_ranges = 0 +sqlite_full_index_rows = 0 +full_manifest_bytes_read = 0 +full_manifest_bytes_written = 0 +upper_recovery_walks = 0 +external_adapter_global_work = 0 +policy_dependency_full_discovery = 0 +observer_tail_records_folded <= configured_tail_bound +candidate_and_prefix_rows <= configured_scope_caps +observer_log_and_segment_bytes <= configured_scope_caps +candidate_reads <= authoritative_candidate_count + bounded prefix expansion/output +root_point_lookups <= small constant * (authoritative candidates + expanded output) +ledger_rows_touched <= small constant * authoritative candidates +``` + +Every candidate, prefix, segment-byte, and unfurled-tail cap is persisted and +enforced; exceeding one fails closed. Scheduled 100k/1M jobs also enforce +calibrated wall-time and peak-RSS regression budgets. Cold start, owner restart, +and cursor-gap scenarios expect labeled reconciliation instead of the warm +bound. A full-scan correctness oracle runs after and outside each measured fast +path. + +State-machine and property tests cover every trust transition, CAS race, +evidence coalescing rule, and acknowledgement boundary. Linux and macOS real +adapter suites cover recursive directory races, rename storms, delayed +delivery, backlog, overflow/gap flags, owner death, cursor replacement, +filesystem replacement, and fence ordering. + +Scenarios include create/content/mode/delete, file/directory/case-only rename, +reverted candidates, ignored subtrees, nested/global policy changes, delayed +delivery, callback backlog, overflow, disk full, log tears, daemon death, +epoch replacement, concurrent same-path writes, backup/restore/GC, and process +kill at every object, intent, filesystem sync, observer sync, ref, ledger, and +mirror boundary. Corruption tests cover checksums, hash chains, offsets, +unsupported formats, and partial SQLite state. Property tests permute event +orders and prove false positives are allowed but false clean is not. + +Activation is compiled on by default only after the controlled-producer +inventory and both real Linux/macOS adapter suites pass. Other platforms use +the direct full-observation command path and never persist trusted ledger +proof. + +## Activation evidence + +Production authority is a compile-time Linux/macOS decision. It does not read +workflow state, downloaded artifacts, environment variables, or mutable files +at runtime. `ActivationEvidence::from_checked_build` authenticates the producer +inventories compiled into the binary and the activation integration suite +executes the recovery, corruption, platform, and metrics gates before release. +The exact compiled audit manifest (schema version, suite and workflow names, +inventory hashes, gate schema, and metrics schema) has SHA-256 +`c68c72d1ba07f777b88b8ed155717d377a75436fa578565a6a62bfdf47e65f54`. + +The approved inventory identities are: + +- `trail/tests/fixtures/changed_path_producers.v1`: + `67027c4bcfba0f3105833978637fe5e81c9cbdb43ba51cbd1a58026a2e067185`; +- `trail/tests/fixtures/changed_path_raw_mutations.v1`: + `38a625a18d607383e865bcea9393792bfbf853ab0597904104cd76c719d6e65c`. + +The pull-request 1k gate is workflow `CLI Scale Benchmark`, job +`changed-path-ledger` in `.github/workflows/scale.yml`. It uploads artifact +`changed-path-ledger-scale-results`, containing `/results.tsv`, +`/structural-metrics.jsonl`, and `/oracle-results.tsv`. The native +scheduled gate is workflow `Changed-path Ledger Native Gates`, job +`native`, in `.github/workflows/changed-path-ledger-native.yml`. That workflow +also runs for pull requests targeting `main` and pushes to `main`. Both its +Linux/ext4 and macOS/APFS matrix jobs are required merge and release checks for +the exact commit being integrated; scheduled artifacts are additional evidence +and cannot replace those commit-specific checks. The matrix runs the real +adapter suites and uploads +`changed-path-ledger--` with the same reports beneath the +native scale root. The weekday schedule checks 100,000 files; the nightly +schedule checks 1,000,000 files plus daemon restart, recovery, and crash +sampling. + +`scripts/check-changed-path-ledger-thresholds.py` checks candidate counts +`k=0,1,100`, separate full-scan oracle equality, persisted scope caps, and zero +global-work structural counters. The 100k native budgets are 15/20/30 seconds +for workspace status/diff/record, 120 seconds for materialized-lane record, and +30 seconds for structured patch, with a 3 GiB RSS ceiling for workspace +commands. The 1M budgets are 30/45/60/300/60 seconds with the same RSS ceiling. +The 1k pull-request budgets are 15/20/30/120/30 seconds with a 2 GiB workspace +RSS ceiling. + +`TRAIL_PERFORMANCE_METRICS_FILE` is inherited by the automatically started +daemon. Each completed outer operation contributes exactly one append write +and one JSON object line. Metrics emission is observational and cannot rewrite +an already-completed command result, so an append error is diagnosed locally; +the benchmark gate makes that condition fatal by rejecting a missing, +malformed, duplicate, or incomplete operation report. + +The checked activation commands are: + +```text +cargo test -p trail --test changed_path_ledger_activation -- --nocapture +cargo test -p trail --test changed_path_ledger_linux -- --nocapture +cargo test -p trail --test changed_path_ledger_macos -- --nocapture +cargo fmt --all -- --check +cargo check --workspace --all-targets +cargo test --workspace +``` + +Unsupported operating systems remain compile-time hard-off. The current native +allowlist is the ext-family on Linux and APFS on macOS; every other filesystem +returns a reconciliation-required error before a native lease can authorize +trust. On allowlisted filesystems, every command still revalidates provider +identity, root identity, capabilities, policy dependencies, cursor continuity, +and the c1/c2 fence; platform support alone is never a clean proof. diff --git a/docs/superpowers/specs/2026-07-12-fuse-cow-hard-cutover-design.md b/docs/superpowers/specs/2026-07-12-fuse-cow-hard-cutover-design.md new file mode 100644 index 0000000..2eff757 --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-fuse-cow-hard-cutover-design.md @@ -0,0 +1,191 @@ +# COW Workdir Mode Hard-Cutover Design + +## Objective + +Replace ambiguous lane workdir mode names everywhere with mechanism-accurate names: + +- `fuse-cow` for the FUSE implementation on Linux and macOS; +- `dokan-cow` for the Dokan implementation on Windows; +- `nfs-cow` for the loopback NFS implementation on macOS; and +- `native-cow` for ordinary materialized directories using clone/reflink COW when + available. + +This is a hard cutover. Trail will not accept, migrate, alias, emit, document, or +suggest the removed `overlay-cow` or `full-cow` modes after this change. + +## Product contract + +The public workdir modes are: + +```text +auto +virtual +sparse +native-cow +fuse-cow +nfs-cow +dokan-cow +``` + +Terminal agent commands accept the materialized modes: + +```text +auto +native-cow +fuse-cow +nfs-cow +dokan-cow +``` + +Platform selection is explicit: + +| Mode | Platform | Transport | +| --- | --- | --- | +| `native-cow` | all supported platforms | ordinary directory; native clone/reflink when available, safe copy fallback otherwise | +| `fuse-cow` | Linux; macOS builds with macFUSE | FUSE mount | +| `nfs-cow` | macOS | loopback NFSv3 mount | +| `dokan-cow` | Windows | Dokan mount | + +Requesting a mode on an unsupported platform fails before lane creation with a +mode-specific diagnostic. `auto` chooses `dokan-cow` on Windows, `nfs-cow` on macOS +when available, `fuse-cow` on Linux when FUSE is available, and otherwise follows the +existing native-COW/large-root policy. + +## Hard-cutover rules + +1. `LaneWorkdirMode::FullCow` becomes `LaneWorkdirMode::NativeCow`; `full-cow` and + `full_cow` are rejected rather than retained as aliases. +2. `LaneWorkdirMode::OverlayCow` becomes `LaneWorkdirMode::FuseCow`. +3. Windows branches that previously overloaded `OverlayCow` become an explicit + `LaneWorkdirMode::DokanCow`. +4. `LaneWorkdirMode::parse` accepts `native-cow`/`native_cow` and the backend-specific + mounted modes. It accepts neither removed mode spelling. +5. CLI value parsers, HTTP/OpenAPI enums, MCP schemas, Rust reports, JSON output, + diagnostics, suggestions, examples, and documentation emit only the new names. +6. Lane metadata written before the cutover is not migrated. Opening or operating on a + lane whose metadata contains `overlay-cow` or `full-cow` returns an unsupported + workdir-mode error instructing the operator to remove and recreate the lane. +7. Persistent transparent-COW state remains backend-neutral under + `.trail/views/`, while mount identifiers use `trail-fuse-cow-*` and Dokan + ownership reports `dokan`. Old `.trail/overlay-cow` state is ignored and never + adopted. +8. Rust symbols use `NativeCow`/`native_cow`, `FuseCow`/`fuse_cow`, and + `DokanCow`/`dokan_cow` for their respective mechanisms. + Generic overlay terminology may remain only where it describes the filesystem + algorithm rather than the removed product mode. +9. Script filenames, environment defaults, Docker volume names, workflow filters, and + benchmark labels use `fuse-cow` when they exercise FUSE. +10. No compatibility warnings or deprecation period are added. + +## Internal architecture + +The workdir mode identifies the user-visible mechanism. The mounted-view semantic core +remains shared. + +```text +LaneWorkdirMode +├── NativeCow -> materialized directory +├── FuseCow -> FUSE transport -> ViewCore +├── NfsCow -> NFS transport -> ViewCore +└── DokanCow -> Dokan transport -> ViewCore +``` + +Mode predicates replace the old two-mode assumptions: + +```rust +pub fn is_transparent_cow(&self) -> bool { + matches!(self, Self::FuseCow | Self::NfsCow | Self::DokanCow) +} + +pub fn cow_backend(&self) -> Option<&'static str> { + match self { + Self::NativeCow => Some("clone"), + Self::FuseCow => Some("fuse"), + Self::NfsCow => Some("nfs"), + Self::DokanCow => Some("dokan"), + Self::Sparse => Some("clone"), + Self::Virtual => None, + } +} +``` + +Mount dispatch must match the mode directly. Dokan must not pass through a method or +module named for FUSE. Shared behavior continues through `ViewCore`, not through an +ambiguous public mode. + +## Metadata and error handling + +New lane metadata stores the exact selected mode in `workdir_mode` and the concrete +mechanism (`clone`, `fuse`, `nfs`, or `dokan`) in `cow_backend`. + +When old metadata is encountered, parsing fails with a stable error equivalent to: + +```text +unsupported lane workdir mode `overlay-cow`; this build uses the hard-cutover modes +`fuse-cow` and `dokan-cow`; remove and recreate the lane with the platform-appropriate +mode +``` + +The removed materialized name similarly reports that `full-cow` was renamed to +`native-cow` and instructs the operator to remove and recreate the lane. + +Trail must not silently infer FUSE versus Dokan from the current host because doing so +could reinterpret copied or restored workspace metadata. + +## Test strategy + +The implementation follows test-driven development. + +### Parser and serialization + +- `native-cow` and `native_cow` parse as `NativeCow`. +- `fuse-cow` and `fuse_cow` parse as `FuseCow`. +- `dokan-cow` and `dokan_cow` parse as `DokanCow`. +- both spellings of `full-cow` and `overlay-cow` fail. +- reports serialize the exact new names and backends. + +### CLI and API contracts + +- lane and agent CLI parsers accept the new values and reject both removed names. +- OpenAPI and MCP schemas contain `native-cow`, `fuse-cow`, and `dokan-cow` and do not + contain either removed name. +- JSON lane spawn reports emit the new mode. + +### Backend dispatch + +- automatic Linux selection returns `FuseCow` when `/dev/fuse` is available. +- automatic Windows selection returns `DokanCow`. +- FUSE mount helpers reject non-FUSE modes. +- Dokan mount helpers reject non-Dokan modes. +- environment initialization, gates, record/checkpoint, update, and agent launch mount + the correct transport. + +### Hard-cutover coverage + +- lane metadata containing `overlay-cow` or `full-cow` fails with an explicit + recreation diagnostic; +- a repository-wide source scan, excluding Git object history and generated build + output, contains no removed public spelling or Rust identifier; and +- renamed FUSE/Dokan scripts and platform workflows run through their native release + gates. + +## Relationship to lane hardening + +This naming cutover is the first bounded change in the broader lane-hardening program. +It does not make `native-cow` secondary. All four COW mechanisms remain first-class and +will be required to satisfy the same environment, process-isolation, cache-provenance, +gate, lifecycle, and reuse contracts. The explicit names make that backend conformance +matrix auditable. + +## Completion criteria + +The cutover is complete only when: + +1. all production code, tests, schemas, scripts, current documentation, and checked-in + skill references use the new names; +2. the old spellings and symbols for both renamed modes are rejected or absent rather + than accepted as aliases; +3. FUSE and Dokan dispatch are distinct and platform-correct; +4. focused parser/CLI/API/backend tests pass; +5. the complete Trail regression passes; and +6. native FUSE, NFS, and Dokan release gates are green on their supported platforms. diff --git a/docs/superpowers/specs/2026-07-12-lane-merge-http-cutover-design.md b/docs/superpowers/specs/2026-07-12-lane-merge-http-cutover-design.md new file mode 100644 index 0000000..993d89e --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-lane-merge-http-cutover-design.md @@ -0,0 +1,56 @@ +# Lane Merge HTTP Cutover Design + +## Goal + +Make the public HTTP API use the same lane-first hierarchy as the CLI: +`trail lane merge` maps to `POST /v1/lanes/{lane}/merge`. + +## Scope + +The canonical request is: + +```http +POST /v1/lanes/{lane}/merge +Content-Type: application/json + +{ + "into": "main", + "strategy": "line-id-aware", + "dry_run": true, + "direct": false +} +``` + +`into` is required at the HTTP boundary. The CLI continues supplying its +existing default of `main` before issuing this request. The lane identifier is +always in the path; request-body aliases such as `lane` and `lane_id` are +removed. + +The old `POST /v1/branches/{branch}/merge-lane` endpoint is removed with no +compatibility route. OpenAPI, audit target extraction, the daemon RPC client, +direct HTTP tests, reference documentation, and scale-benchmark HTTP calls +all use the replacement route. + +## Deliberate Non-Changes + +The persisted domain model remains unchanged. `OperationKind::LaneMerge`, the +`lane_merge` change-id namespace and the then-current queue storage were outside +that cutover. The subsequent lane merge queue hard-cutover design supersedes +the queue-storage non-change described here. + +Internal Rust methods such as `merge_lane_with_options` likewise remain +verb-object domain APIs. They are not CLI aliases and stay idiomatic Rust. + +## Error Handling and Security + +The replacement endpoint keeps the existing merge-strategy validation, +readiness checks, direct-merge guard, conflict behavior, audit lane/target +attribution, status codes, and JSON merge report. Requests to the removed +route follow the server's normal unknown-route behavior. + +## Verification + +Tests must prove the new path accepts a lane path segment and `into` body +field, the old path is not routable, the daemon-backed CLI calls the new route, +and OpenAPI documents only the new operation. Existing CLI parser and merge +flow coverage remains green. diff --git a/docs/superpowers/specs/2026-07-12-lane-merge-queue-cutover-design.md b/docs/superpowers/specs/2026-07-12-lane-merge-queue-cutover-design.md new file mode 100644 index 0000000..5bcc0ca --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-lane-merge-queue-cutover-design.md @@ -0,0 +1,261 @@ +# Lane Merge Queue Hard Cutover Design + +## Goal + +Make Trail's serialized merge queue an explicitly lane-only capability across +the CLI, HTTP API, MCP surface, Rust reports and methods, and SQLite schema. +Generic branch and ref merges remain available through `trail merge` and are +not accepted by the lane merge queue. + +This is an intentional breaking change. The old top-level command, HTTP routes, +MCP tools, resource URI, report types, methods, and database table are removed +without compatibility aliases. + +## Domain Boundaries + +Trail exposes three distinct merge workflows: + +```text +trail merge --into +trail lane merge --into +trail lane merge-queue +``` + +- `trail merge` merges a generic branch or ref. +- `trail lane merge` previews or explicitly direct-merges one lane. +- `trail lane merge-queue` schedules lane merges for serialized execution. + +The lane merge queue never accepts ordinary branch refs. Callers that need to +merge a branch or generic source use `trail merge` directly. + +## CLI Contract + +The canonical commands are: + +```sh +trail lane merge-queue add --into main --priority 10 +trail lane merge-queue list +trail lane merge-queue explain +trail lane merge-queue run --limit 1 +trail lane merge-queue remove +``` + +`add` resolves `` by lane name or stable lane id and rejects selectors +that resolve only to a branch or generic ref. `list` and `run` are +workspace-wide operations even though they live under the lane command group. + +The top-level `trail merge-queue` command is removed. Clap must reject it as an +unknown command. No hidden alias or deprecation period is provided. + +`trail lane merge --direct` remains the explicit queue bypass. A +non-dry-run merge into the configured shared/default branch without `--direct` +continues to fail safely, but its next-step message points only to: + +```sh +trail lane merge-queue add --into +trail lane merge-queue run +``` + +All help output, human rendering, documentation, examples, scripts, and +benchmarks use the new spelling. + +## HTTP API Contract + +The lane merge queue uses the existing plural lane namespace: + +```text +POST /v1/lanes/merges/queue +GET /v1/lanes/merges/queue +POST /v1/lanes/merges/queue/run +GET /v1/lanes/merges/queue/{selector}/explain +DELETE /v1/lanes/merges/queue/{selector} +``` + +This complements the existing single-lane merge operation: + +```text +POST /v1/lanes/{lane}/merge +``` + +The enqueue request is strict: + +```json +{ + "lane": "doc-bot", + "into": "main", + "priority": 10 +} +``` + +`lane` and `into` are required. `priority` defaults to zero. Unknown fields are +rejected. Legacy request keys such as `source`, `target`, and `target_branch` +are not accepted. + +The old `/v1/merge-queue` routes, including the query-form +`/v1/merge-queue/explain?selector=...`, are removed and follow normal +unknown-route behavior. OpenAPI contains only the new routes and lane-specific +schemas. + +## Public Models and Storage + +Public queue types are renamed around the lane domain: + +- `LaneMergeQueueEntry` +- `LaneMergeQueueAddReport` +- `LaneMergeQueueRemoveReport` +- `LaneMergeQueueExplainReport` +- `LaneMergeQueueRunReport` +- `LaneMergeQueueRunItem` + +An entry exposes `queue_id`, `lane_id`, `lane`, `target_ref`, `status`, +`priority`, `created_at`, and `updated_at`. The stable `lane_id` is the stored +identity. `lane` is the resolved human-facing lane name. Generic `source_ref` +is not part of queue requests, entries, or reports. + +The SQLite queue table becomes: + +```sql +CREATE TABLE lane_merge_queue ( + queue_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL, + target_ref TEXT NOT NULL, + status TEXT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); +``` + +Indexes support active-entry lookup by lane and target and queue execution by +status, descending priority, and ascending creation time. New queue ids use the +`lmq_` prefix. + +Database and internal service methods use lane-specific names, including +`enqueue_lane_merge`, `list_lane_merge_queue`, `explain_lane_merge_queue`, +`run_lane_merge_queue`, and `remove_lane_merge_queue`. Normalization of generic +queue source refs is removed. + +`merge_results` and `conflict_sets` retain generic source and target ref fields +because they also represent ordinary `trail merge` operations. The nullable +`merge_results.queue_id` column becomes `lane_queue_id` for new lane-queue +results. + +## Schema Migration + +The schema version advances from v15 to v16. Migration runs within Trail's +existing schema savepoint and is all-or-nothing. + +The v16 migration: + +1. Drops `merge_queue` without copying queued, running, completed, or cancelled + entries. +2. Creates `lane_merge_queue` and its indexes. +3. Renames `merge_results.queue_id` to `lane_queue_id` and clears legacy + non-null values because their generic queue records no longer exist. +4. Records schema version 16 only after every schema operation succeeds. + +Failure rolls the entire migration back, including the version marker. Opening +a successfully migrated workspace never recreates the old table. + +This destructive queue reset is deliberate. Backups remain the user's recovery +mechanism for pre-cutover queue data; Trail does not attempt to infer which +legacy generic entries represented lanes. + +## Queue Behavior and Safety + +Enqueue resolves the supplied lane selector to a current `LaneRecord` and +`LaneBranch`, then stores the stable lane id and normalized target branch ref. +If the same lane and target already have a queued or running entry, enqueue +returns that entry rather than creating a duplicate. + +Execution resolves the stored lane id back to its current lane branch and then +reuses the existing lane merge safety path. Before each item it rechecks: + +- lane existence and branch state; +- dirty materialized workdir state; +- approvals and required gates; +- lane readiness; +- target existence and merge conflicts. + +Entries run by priority descending and creation time ascending. The runner +stops at the first blocker, error, or conflict, preserving the existing +serialized safety property. Completed and cancelled v16 entries remain visible +as audit history. + +`explain` and `remove` accept a new `lmq_` queue id, lane id, or lane name. +Ambiguous or missing selectors return an invalid-input error without mutation. +Branch refs and old `mq_` ids do not resolve through the new API. + +## MCP and Resource Contract + +MCP tools become: + +```text +trail.lane_merge_queue_add +trail.lane_merge_queue_list +trail.lane_merge_queue_explain +trail.lane_merge_queue_run +trail.lane_merge_queue_remove +``` + +The workspace resource becomes: + +```text +trail://workspace/lane-merge-queue +``` + +The old `trail.merge_queue_*` tools and +`trail://workspace/merge-queue` resource are removed. Risk annotations remain +equivalent: list and explain are read-only; add is a write; run and remove are +consequential writes. + +## Error Handling + +- Unknown lane selectors fail before an entry is written. +- Branch and generic ref selectors are rejected as non-lane inputs. +- Invalid targets fail before an entry is written. +- Direct shared-target lane merges continue to explain how to enqueue or use + the explicit `--direct` bypass. +- A lane removed or corrupted after enqueue blocks execution and leaves an + auditable failed/blocked result rather than falling back to a generic ref + merge. +- Queue conflicts use the existing structured conflict-set mechanism and stop + subsequent queue processing. +- Removed CLI, HTTP, and MCP names receive their surfaces' standard + unknown-command, unknown-route, or unknown-tool response. + +## Verification + +Automated coverage must prove: + +1. `trail lane merge-queue` parses every subcommand and + `trail merge-queue` does not parse. +2. Local and daemon-backed CLI commands produce matching lane-specific JSON and + human output. +3. OpenAPI exposes only `/v1/lanes/merges/queue` routes and strict request and + response schemas. +4. Removed HTTP routes return unknown-route responses. +5. MCP discovery exposes only the lane-specific tools and resource; each tool + drives the corresponding database operation. +6. A v15 database with legacy queue rows migrates transactionally to v16, + discards those rows, clears legacy merge-result queue links, and contains no + `merge_queue` table. +7. Failed v16 migration leaves the v15 schema and version marker intact. +8. Branch/ref inputs are rejected while lane name and lane-id inputs enqueue + successfully. +9. Duplicate enqueue, priority ordering, item limits, explanation, + cancellation, readiness blockers, dirty workdirs, approvals, test/eval + gates, and conflict pausing retain their safety behavior. +10. `trail merge` continues to merge generic branches independently of the lane + queue. +11. Documentation, skills, rendered next steps, benchmarks, and repository + source contain no callable use of the removed public names. + +## Deliberate Non-Goals + +- Migrating or preserving legacy queue entries. +- Queueing generic branches or arbitrary refs. +- Renaming generic merge-result or conflict source/target fields. +- Changing lane readiness, approval, gate, merge strategy, or conflict + semantics. +- Adding a compatibility alias, warning period, or legacy HTTP shim. diff --git a/docs/superpowers/specs/2026-07-12-nfs-noop-setattr-checkpoint-design.md b/docs/superpowers/specs/2026-07-12-nfs-noop-setattr-checkpoint-design.md new file mode 100644 index 0000000..f148808 --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-nfs-noop-setattr-checkpoint-design.md @@ -0,0 +1,68 @@ +# NFS No-op `SETATTR` Checkpoint Safety + +## Problem + +Trail's macOS NFS-COW adapter forwards only file size and mode from an NFS +`SETATTR` request. Requests that contain only unsupported metadata, such as +timestamps or ownership, therefore reach `ViewCore::setattr` with both values +absent. + +`ViewCore::setattr` currently journals every call as a source metadata +mutation. Checkpoint recovery treats each journaled source path as a candidate +and scans the source upper layer for its contents. A metadata-only request does +not copy the lower file into that upper layer, so the checkpoint recorder +mistakes the absent upper file for a deletion. + +## Chosen Design + +Make `ViewCore::setattr` treat a call with neither size nor mode as a no-op. It +will return the file's current visible attributes without copying the file into +the upper layer and without appending a mutation-journal record. + +Calls that provide size, mode, or both retain their current behavior: Trail +copies up the file when needed, applies the requested change, and records the +metadata mutation. The NFS adapter continues to ignore unsupported ownership +and timestamp fields; this change only prevents those ignored fields from +creating false source changes. + +The guard belongs in `ViewCore` rather than only in the NFS adapter so every +filesystem adapter shares the invariant that an empty supported-attribute +update cannot dirty a workspace view. + +## Data Flow + +1. The NFS adapter receives `sattr3` and extracts optional size and mode. +2. It calls `ViewCore::setattr` with those two supported values. +3. If both values are absent, `ViewCore` returns the current attributes without + starting a logical mutation. +4. Otherwise, the existing copy-up, mutation, synchronization, and journal + behavior runs unchanged. +5. Checkpoint recovery therefore sees only paths with actual supported + mutations, explicit writes, creations, renames, or whiteouts. + +## Error Handling + +The no-op path still resolves the inode and current visible attributes. Invalid +or stale inode errors therefore remain observable. It does not suppress errors +for real size or mode changes. + +## Tests + +Add a regression test using the real `ViewCore` fixture: + +1. Create a lower-layer source file and look up its inode. +2. Call `setattr(inode, None, None)`. +3. Assert that the visible attributes are returned. +4. Assert that the source upper layer does not contain the file. +5. Assert that checkpoint candidates do not contain the path. + +The test must fail before the implementation change by observing the spurious +checkpoint candidate. After the minimal guard is added, run the focused +workspace-view and NFS-COW tests, followed by the broader Trail test suite. + +## Non-goals + +- Implementing timestamp, ownership, ACL, or extended-attribute mutation. +- Changing checkpoint recovery semantics for real metadata mutations. +- Recovering already-corrupted lanes; recovery remains a separate operator + action. diff --git a/docs/superpowers/specs/2026-07-12-persistent-path-invariant-index-design.md b/docs/superpowers/specs/2026-07-12-persistent-path-invariant-index-design.md new file mode 100644 index 0000000..fe3b57e --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-persistent-path-invariant-index-design.md @@ -0,0 +1,128 @@ +# Persistent Path-Invariant Index Design + +**Status:** Approved as the second production performance slice + +## Objective + +Make case-insensitive path-safety validation proportional to touched paths `k`, +not root size `N`. Structured patches, materialized records, Git imports, lane +merges, and incremental root builders must not load every root path merely to +prove that a small mutation introduces no case-fold collision. + +## Current bottleneck + +`ensure_patch_final_root_paths_safe`, +`ensure_record_final_root_paths_safe_from_summaries`, and several incremental +root builders call `load_root_paths()` and rebuild an in-memory folded-path map. +At 100k/1M paths this adds an `O(N)` read to otherwise incremental operations. + +## Root schema + +Add an optional `case_fold_map_root` to `WorktreeRoot`. Its Prolly map stores: + +```text +NFKC(lowercase(path)), NFC-normalized -> canonical path bytes +``` + +The field is Serde-defaulted for old roots. New non-empty roots carry the index +root. Because an empty Prolly tree has no root CID, `case_fold_map_root=None` +is also a valid empty index exactly when `file_count=0`; no path scan is needed +to establish that state. The index uses the same persistent store/configuration +as the path and file-ID maps, so unchanged nodes are content-addressed and +shared. + +## Mutation contract + +For an indexed root, validation applies removals to an overlay first, then +checks each addition's folded key against the overlay and persistent tree. +Distinct canonical paths at the same folded key return the existing +`InvalidPath` collision error. On success, one Prolly mutation batch produces +the next case-fold tree. Work is `O(k log N)` plus affected-node writes. + +Every root-construction path must keep the path map, file-ID map, and case-fold +map in sync. Full builders construct all three in one pass. Incremental builders +derive removals/additions from the same touched-path set used for the path map. + +## Legacy roots and repair + +An old non-empty root without the index is not silently scanned on a hot +mutation path. Hot callers return stable `PATH_INDEX_REQUIRED` with the +recovery command `trail index rebuild`. A root with no index CID and +`file_count=0` is already a valid empty index and may be mutated directly. The +existing rebuild operation backfills the current root/index state. +Compatibility read/materialization operations may still read old roots; only +mutation safety requires the index. + +Repair covers every mutable live `refs/branches/*` and `refs/lanes/*` head in +one command, not only the checked-out branch. Distinct refs that share a legacy +root share one rebuilt fold tree and one equivalent immutable root. Historical +roots remain unchanged. Each repaired ref advances by CAS through its own +auditable `ManualCheckpoint` maintenance operation with the old head as parent, +the old/new equivalent roots as `before_root`/`after_root`, and no visible file +changes. All roots and lane/Git metadata are preflighted before any ref is +published; authoritative SQLite ref/lane/checkpoint/mapping updates commit as a +unit. Content-addressed nodes or objects created during a failed preflight may +remain unreachable and are reclaimed by normal GC, but no live ref advances. + +Derived baselines follow the equivalent root identity without touching visible +files. A clean checked-out worktree baseline is retargeted, clean lane manifests +and workspace checkpoint markers are retargeted or conservatively invalidated, +and clean Git mapping rows are copied from every distinct clean mapping for the +old root to each repaired root/maintenance change while preserving their Git +head, direction, and branch. Mapping discovery is root-wide rather than tied to +the old ref change, because historical imports and exports can establish valid +trust for the same immutable root under different changes. Duplicate source +tuples are collapsed before insertion. + +Clean-state consumers may reuse a baseline whose root ID predates repair only +after loading both immutable roots and proving equal path-map root, file-ID-map +root, file count, and total text bytes. The exact-ID case remains the fast path; +missing or corrupt roots fail closed. This equivalence applies only to states +already proven clean. Dirty and overflow daemon snapshots are never promoted to +clean, and a live daemon snapshot is not rewritten or deleted by repair. + +The maintenance operation row and its parent rows are indexed inside the same +authoritative SQLite transaction that advances refs and lane heads. Therefore a +process crash immediately after `COMMIT` still leaves operation lookup and +ancestry complete before a later broad index rebuild runs. Ref files remain +derived mirrors of SQLite refs and are reconciled on every rebuild. + +Lane manifests and workspace checkpoint markers have a durable SQLite repair +queue (schema version 17). Each intent stores only ref name, repair kind, old +root, new root, and new change; filesystem paths are resolved again from the +current authoritative lane/view rows. Intents publish in the same transaction +as the ref repair, then drain after commit, at every rebuild, and on open after +an unlocked empty fast check plus a locked recheck. A new or already-retargeted +mirror, a missing scope/mirror, or conservative invalidation clears the intent; +an I/O failure leaves it for retry. Restore and backup verification use a +no-recovery database open so copied absolute paths are never followed. Restore +atomically rewrites lane workdirs and invalidates workspace-view/environment +state that the backup does not contain before draining intents and entering +normal recovery. + +Legacy-root validation is two-pass and bounded by the largest distinct live +legacy root rather than the sum of all roots. The first pass validates one +root's key range, count, normalization, and fold collisions and retains only its +ID/count. After every root passes, the second pass re-ranges and builds one fold +tree at a time. A second rebuild after successful repair publishes no additional +root, operation, ref, or derived intent. + +## Structural evidence + +Expose path-invariant metrics on patch/record scale scenarios: + +- `case_fold_index_mode=indexed` +- `case_fold_index_lookups<=k` +- `full_root_path_loads=0` + +Add 1k CI and 100k/1M scheduled gates. Collision, rename, delete-then-add, +Unicode compatibility, legacy-root recovery, and index corruption tests are +required before rollout. + +## Rollout + +1. Add the backward-compatible root field and full-build index construction. +2. Add indexed lookup/mutation helpers and stable legacy-root error. +3. Replace hot full-path validators and incremental builders. +4. Backfill through `trail index rebuild` and add scale counters/gates. +5. Run full tests plus 100k/1M patch/record acceptance. diff --git a/docs/superpowers/specs/2026-07-12-strict-native-cow-materialization-design.md b/docs/superpowers/specs/2026-07-12-strict-native-cow-materialization-design.md new file mode 100644 index 0000000..b33bb1b --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-strict-native-cow-materialization-design.md @@ -0,0 +1,361 @@ +# Strict Native-COW Materialization Design + +## Objective + +Make Trail's materialized workdir names describe guarantees rather than aspirations. +`native-cow` becomes strict: every regular file is created by a successful +filesystem-native clone operation and Trail fails instead of copying bytes. A new +`portable-copy` mode provides reliable materialization with opportunistic clones and +byte-copy fallback. `auto` tries strict native COW in an isolated stage and, only when +native materialization is unavailable, restarts from scratch as `portable-copy`. + +The design borrows Rift's useful platform primitives—APFS `fclonefileat` and Linux +`FICLONE`—while retaining Trail's immutable-root validation, staged publication, and +workdir lifecycle. Trail does not adopt Rift's whole-tree APFS cloning, Btrfs +subvolume snapshots, two-rename publication, or unowned cleanup behavior. + +## Public contract + +The public lane workdir modes are: + +```text +auto +virtual +sparse +native-cow +portable-copy +fuse-cow +nfs-cow +dokan-cow +``` + +Their contracts are: + +| Requested mode | Resolved mode | Guarantee | +| --- | --- | --- | +| `native-cow` | `native-cow` | Every regular file was created by a successful native clone; otherwise fail. | +| `portable-copy` | `portable-copy` | Materialize every file, cloning when possible and copying bytes otherwise. | +| `auto` | `native-cow` or `portable-copy` | Try strict native COW in staging; on an eligible availability failure, discard the stage and restart portably. | +| `sparse` | `sparse` | Preserve the explicit partial-materialization behavior and report actual per-file clone/copy results. | +| `fuse-cow` | `fuse-cow` | Explicit FUSE-mounted COW view. | +| `nfs-cow` | `nfs-cow` | Explicit loopback-NFS COW view. | +| `dokan-cow` | `dokan-cow` | Explicit Dokan-mounted COW view. | +| `virtual` | `virtual` | No materialized workdir. | + +`auto` is materialized-only. It never selects FUSE, NFS, or Dokan. Those modes remain +explicit because mounting has materially different lifecycle, privilege, and failure +semantics. New materialized lanes default to `auto`. + +`native-cow` initially supports APFS on macOS and reflink-capable Linux filesystems. +Other hosts and filesystems report native COW as unavailable. `portable-copy` remains +subject to ordinary operational failures such as invalid paths, insufficient +permissions, exhausted storage, or I/O errors; "portable" means it has no COW +capability requirement. + +## Strategy architecture + +Materialized workdirs use a small strategy boundary independent of mounted views: + +```text +MaterializationRequest + requested mode + root_id + destination + | + v +MaterializationCoordinator + source resolution + staging + fallback + publication + reporting + | + +---------+----------+ + | | + v v +StrictNativeStrategy PortableStrategy +APFS fclonefileat clone each file +Linux FICLONE copy bytes on CloneUnavailable +all files or failure aggregate clone/mixed/copy +``` + +The native clone primitive returns a typed result: + +```rust +enum NativeCloneOutcome { + Cloned, + Unavailable(NativeCloneUnavailable), +} + +enum NativeCloneUnavailable { + Unsupported, + CrossDevice, +} +``` + +Operational errors remain `Err(Error)` and are never collapsed into +`Unavailable`. In particular, permission failures are errors. The current classifier +must stop treating `EPERM` and `EACCES` as unsupported. Platform-specific tests define +the exact errno mapping; Linux ioctl-not-supported responses include `ENOTTY`. + +The coordinator, rather than the file primitive, owns strictness and fallback. This +keeps the primitive reusable by strict native, portable, sparse, and mounted-view +projection code without letting a low-level helper silently choose product semantics. + +## Native source eligibility + +Strict native materialization requires a complete filesystem projection matching the +requested immutable `root_id`. Initially, candidates are: + +1. the primary workspace when Trail validates every target path against the root; and +2. an existing complete, clean Trail workdir validated against the same root. + +Trail may select candidates deterministically in that order. Object-store blobs, +generated temporary source trees, and partial sparse workdirs are not native sources. +If no complete candidate exists, the attempt returns `NativeSourceUnavailable`. +Explicit `native-cow` fails; `auto` may restart as portable materialization. + +Each source file is opened without following symlinks and validated against its +`FileEntry`. The clone uses the open descriptor. Trail then hashes the staged clone and +checks its content and executable bit against the immutable root before publication. +This destination verification closes the validation/clone race even if a workspace +file changes concurrently. + +Trail's root model contains independent regular-file paths, not inode relationships. +Consequently, each path receives its own clone operation. Accidental hardlinks in the +source are deliberately not preserved, because editing one workdir path must not +modify another. Directory structure is derived from normalized root paths. Existing +Trail behavior for executable modes and removal of untracked cloned xattrs is +preserved. + +## Strict staging and portable restart + +Every materialized creation or full refresh is built in a unique, Trail-owned stage +beside the final destination. Keeping stage and destination under the same parent +filesystem permits atomic publication. Before writing, Trail records an operation ID, +the intended destination, stage path, requested mode, and lifecycle state. + +The strict attempt performs these steps: + +1. validate the requested root and resolve a complete native source; +2. create and register an owned sibling stage; +3. compare source and destination filesystem or volume identity; +4. behavior-probe the native clone primitive inside staging, including for an empty + root; +5. clone every target file independently, with bounded parallelism; +6. verify every staged file and the completed clean-workdir manifest against + `root_id`; and +7. publish the verified directory without overwriting an independently created + destination. + +For non-empty roots, successful per-file clone calls are the strict sharing proof. For +empty roots, Trail combines source/destination filesystem identity with an in-stage +clone probe so explicit strict mode still detects unsupported storage. + +When `auto` receives `CloneUnsupported`, `CrossDevice`, or +`NativeSourceUnavailable`, it closes the strict attempt and removes its entire stage. +It creates a new operation-owned stage and runs `portable-copy` from the beginning. +Files cloned successfully during the failed strict attempt are never reused. Failure +to cleanly retire the strict stage is an operational error rather than permission to +continue with ambiguous ownership. + +Portable materialization tries the same native primitive independently for each file. +`Unavailable` causes that file to be byte-copied; operational errors abort. It may +therefore report all-clone, mixed, or all-copy. Copying preserves existing Trail +content, executable-mode, durability, and manifest behavior. + +## Publication, concurrency, and recovery + +Initial creation uses a no-overwrite publication primitive. Linux uses +`renameat2(RENAME_NOREPLACE)` and macOS uses the equivalent exclusive rename where +available; other platforms use the strongest no-replace primitive they expose. Trail +never implements initial publication as "check destination, then ordinary rename." +A concurrent creator may win, but the loser removes only its registered stage. + +Full refresh retains Trail's clean-workdir, force/rescue, and replacement semantics. +It constructs and verifies a sibling stage before moving the old clean workdir to a +registered backup and publishing the replacement. If publication fails, Trail +restores the registered backup. Dirty workdirs are refused or rescued according to +the existing command contract before materialization begins. + +The operation lifecycle is: + +```text +preparing -> materializing -> verified -> published + \-> failed +``` + +Recovery reconciles operation records rather than scanning and deleting paths by name: + +- `preparing` or `materializing`: remove the registered incomplete stage; +- `verified`: publish if the destination precondition still holds, otherwise retire + the stage and report the collision; +- replacement interrupted after backup: restore the registered backup when the + destination is absent; +- `published` without the final metadata commit: validate the destination marker and + finish the database transition; and +- nonexistent registered paths: mark the operation retired without touching other + filesystem entries. + +Unregistered similarly named directories are never removed. Custom workdir parents +receive the same ownership checks; Trail does not assume that every hidden sibling is +its property. + +## Metadata and reporting + +Reports separate user intent, resolved policy, and actual mechanism: + +```json +{ + "requested_workdir_mode": "auto", + "workdir_mode": "portable-copy", + "workdir_backend": "mixed", + "materialization": { + "cloned_files": 842, + "cloned_bytes": 914358272, + "copied_files": 3, + "copied_bytes": 16384, + "fallback_reason": "clone-unsupported" + } +} +``` + +`requested_workdir_mode` preserves the selection made by the caller. +`workdir_mode` is the resolved mode for the current complete materialization. +`workdir_backend` reports actual results and has these values: + +```text +clone +mixed +copy +fuse +nfs +dokan +virtual +``` + +The mode/backend combinations are constrained: + +- explicit `native-cow` is always `native-cow` plus `clone`; +- `auto` resolves to `native-cow` plus `clone`, or `portable-copy` plus `clone`, + `mixed`, or `copy`; +- explicit `portable-copy` may report `clone`, `mixed`, or `copy`; +- sparse workdirs remain `sparse` and report actual clone/copy counters; and +- mounted and virtual modes do not fabricate per-file materialization counters. + +The bounded fallback reasons are `clone-unsupported`, `cross-device`, and +`native-source-unavailable`. Raw OS errors remain diagnostic context and are not +persisted as schema values. + +`cow_backend` is replaced in new reports by `workdir_backend`. Legacy records remain +readable, but their historical `cow_backend: clone` is not proof that every file was +cloned under the old best-effort implementation. `workdir_backend` remains absent for +such an existing materialization until a full sync rematerializes and verifies it +under the new policy. Existing requested modes are preserved; the default change +applies to new selections. + +## Error behavior + +Only native availability failures are eligible for `auto` restart: + +| Condition | Explicit `native-cow` | `auto` | +| --- | --- | --- | +| clone API/filesystem unsupported | fail `CloneUnsupported` | restart portable | +| different source/destination filesystem | fail `CrossDevice` | restart portable | +| no complete validated source tree | fail `NativeSourceUnavailable` | restart portable | +| permission denied | hard failure | hard failure | +| out of space/quota | hard failure | hard failure | +| corrupt source or staged hash mismatch | hard failure | hard failure | +| invalid/colliding path | hard failure | hard failure | +| ordinary I/O failure | hard failure | hard failure | + +An explicit strict request never copies bytes. Portable fallback is visible in both +the resolved mode and materialization report. + +## Performance constraints + +Clone, verification, and portable-copy work use bounded parallelism rather than one +thread per file. The existing small-tree and streaming paths must call the same +strategy coordinator, so crossing the large-root threshold cannot change semantics. +Counters are reduced deterministically after workers complete. A first failure stops +scheduling new work where practical, while already running workers finish safely +before stage cleanup. + +Strict verification reads every staged byte to prove the immutable root; it does not +write file data and therefore does not break block sharing. Native COW is expected to +reduce allocation and write amplification, not eliminate correctness reads. Future +optimizations require an equally strong immutable-source proof. + +Trail-owned Btrfs snapshot caches are explicitly deferred. They may later provide +another strict native source/strategy, but this change does not create or adopt +foreign subvolumes. + +## Test strategy + +Implementation follows red-green-refactor cycles. + +### Unit and contract tests + +- parse and serialize `portable-copy`, strict `native-cow`, and materialized-only + `auto`; +- reject removed aliases while preserving the already-approved hard cutover; +- classify clone availability separately from permission and operational errors; +- verify exact requested/resolved/backend combinations and counters; +- prove legacy `cow_backend` does not become new strict evidence; and +- exercise empty roots, empty files, source hardlinks, sparse source files, + executable modes, xattr cleanup, case collisions, and invalid paths. + +### Strategy and fallback tests + +- all native clones succeed and strict publication reports `clone`; +- the Nth clone is unsupported: explicit native fails without publication; +- the Nth clone is unsupported: `auto` discards the strict stage, restarts portable, + and reports the portable result; +- a permission, storage, corruption, or I/O failure never falls back; +- absence of a complete native source is eligible only for `auto` restart; +- concurrent source mutation is caught by staged hash verification; and +- the large-root streaming route cannot silently bypass strict policy. + +### Filesystem integration matrix + +- macOS/APFS with `fclonefileat`; +- Linux/Btrfs with `FICLONE`; +- Linux/XFS with reflink enabled; +- Linux ext4 or XFS without reflink for strict failure and automatic portable + restart; and +- cross-device source/destination behavior. + +Successful platform tests modify source and destination independently after cloning +to confirm snapshot isolation. Scheduled privileged CI may use loopback Btrfs/XFS and +extent inspection; ordinary CI exercises real clone calls when available and the +typed fallback contract otherwise. + +### Publication and recovery tests + +Fault injection covers every lifecycle transition, the Nth file operation, +destination races, stage cleanup failure, replacement-backup restoration, and a +crash between filesystem publication and metadata commit. Tests assert that no +unregistered path is removed and no partial destination is published. + +Every materialization entry point is covered: lane spawn, lazy ensure, full sync, +rewind, patch refresh, sparse hydration, and the large-root streaming path. + +## Rollout + +1. Add the strategy types, new mode/report schema, and portable behavior while leaving + the current default unchanged. +2. Route every full and streaming materialization entry point through the coordinator. +3. Make explicit `native-cow` strict and invalidate claims from legacy best-effort + materializations until their next verified full sync. +4. Run the APFS, Btrfs, XFS-reflink, unsupported-filesystem, cross-device, + concurrency, and recovery gates. +5. Change new materialized selections to `auto`. +6. Keep mounted modes explicit and implement Btrfs snapshot caches separately. + +## Completion criteria + +The change is complete only when: + +1. explicit `native-cow` cannot byte-copy through any materialization path; +2. `portable-copy` reports all-clone, mixed, and all-copy results truthfully; +3. `auto` restarts portably only for the three bounded availability conditions; +4. publication and recovery operate only on registered Trail-owned paths; +5. reports distinguish requested mode, resolved mode, and actual backend; +6. legacy best-effort materializations are not presented as verified strict clones; +7. focused unit, integration, fault-injection, formatting, lint, and complete Trail + regression checks pass; and +8. native APFS, Btrfs, and XFS-reflink release gates pass on their supported runners. diff --git a/docs/superpowers/specs/2026-07-12-trusted-git-handoff-performance-design.md b/docs/superpowers/specs/2026-07-12-trusted-git-handoff-performance-design.md new file mode 100644 index 0000000..73a3a83 --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-trusted-git-handoff-performance-design.md @@ -0,0 +1,219 @@ +# Trusted Git Handoff Performance Design + +**Status:** Approved for implementation + +## Objective + +Make high-level Trail agent apply predictable and safe for repositories with +10,000, 100,000, and 1,000,000 files. A mapped apply must scale with the number +of changed paths `k`, not the total Trail or Git path count `N`. Missing trust +must produce an explicit reconciliation requirement rather than an implicit +full-tree scan or export. + +This is the first slice of the production performance program. Later slices +will specify watcher-backed changed-path recording and persistent path-invariant +indexes. Those slices build on the guarantees established here but are not +required to remove the catastrophic Git handoff fallback. + +## Evidence + +The current high-level apply path has four measurable problems: + +1. `agent_apply` obtains Git state through `current_git_branch`, obtains it + again directly, and obtains it a third time inside `git_export_commit`. + Each state lookup runs tracked-only `git status`. +2. A missing Git-to-Trail mapping causes `ensure_git_head_matches_root` to load + the complete Trail root and call `git_write_tree`. +3. `git_write_tree` launches one `git hash-object` process per Trail file. +4. The scale harness covers structured lane patching and internal lane merge, + but not high-level agent apply to Git. + +The local Linux fixture contains 94,837 Git-tracked paths, a 94,506-path Trail +root, and no Git mapping. It therefore selects the catastrophic fallback even +for a one-file agent change. + +## Performance Contract + +For a clean Git worktree whose current HEAD and Trail base root have a trusted +mapping: + +- High-level dry-run and actual apply SHALL NOT load all Trail root paths or + files. +- High-level dry-run and actual apply SHALL NOT run full-tree Git export. +- Trail-side merge and export work SHALL be proportional to changed paths and + the affected prolly-tree nodes. +- Actual apply SHALL ask Git for tracked-worktree cleanliness at most once. +- Dry-run SHALL ask Git for tracked-worktree cleanliness at most once. +- Git commit construction SHALL preserve Git-tracked paths that Trail does not + model, including symlinks, by starting from the mapped Git HEAD tree. +- Dry-run SHALL not insert mappings or otherwise reconcile state. + +When the current Git HEAD and Trail base root do not have a trusted mapping: + +- High-level apply SHALL fail before root materialization or Git object writes. +- The error SHALL distinguish missing baseline trust from a dirty worktree or + a divergent HEAD. +- The error SHALL recommend `trail git import-update` as the explicit + reconciliation workflow. +- Trail SHALL NOT silently scan or hash `N` files to manufacture trust. + +Full snapshot export remains available only through an explicitly selected +general Git export mode. High-level agent apply always requires mapped delta +export. + +## Architecture + +### Git handoff context + +Introduce one internal Git handoff context owned by the Git handoff module. It +contains the workspace identity, branch, expected HEAD, clean/dirty result, +Trail base change/root, and the matching mapping. Callers do not independently +rediscover those facts. + +Planning resolves branch and HEAD without a worktree scan. Dry-run performs its +single clean-state check while creating the plan. Actual apply defers its single +clean-state check until immediately before publication and verifies that HEAD +still equals the expected parent. + +### Export policy + +Git commit export receives an explicit policy: + +- `RequireMappedDelta`: require the expected HEAD/base-root mapping and update a + temporary Git index from only the root diff. +- `AllowFullSnapshot`: permit a general-purpose full Trail snapshot export when + the caller explicitly chose that operation. + +`agent_apply` and `agent_finish` always use `RequireMappedDelta`. The existing +`trail git export -m` command uses `AllowFullSnapshot`; invoking that explicit +general export command is the user's selection of the potentially O(N) mode. + +### Delta construction + +Mapped delta export continues to: + +1. Compute a structural root diff into changed-path maps. +2. Populate a temporary index from the expected Git HEAD. +3. Write or remove only changed entries. +4. Write the new tree and commit with the expected HEAD as parent. + +Git plumbing for multiple changed paths is batched behind the handoff module. +The initial implementation may retain one blob-hash call per changed file, but +it must never issue calls for unchanged files. A later optimization may replace +the adapter with fast-import or an in-process Git implementation without +changing callers. + +### Publication safety + +Immediately before publication, actual apply verifies: + +- Git still points at the expected HEAD. +- The tracked worktree is clean. +- The mapped base root still matches the Trail target base. + +Publication uses Git's fast-forward behavior and records the resulting +Git-to-Trail mapping only after success. A race or mismatch fails without moving +the Git ref or recording a successful mapping. + +## Error Model + +Add stable errors for: + +- `GIT_MAPPING_REQUIRED`: the HEAD/base-root trust record is missing. +- `GIT_HEAD_CHANGED`: Git HEAD changed after planning. +- `GIT_WORKTREE_DIRTY`: tracked worktree changes block publication. +- `GIT_DELTA_EXPORT_REQUIRED`: a high-level caller attempted full snapshot + export. + +Human diagnostics explain the condition and give one safe next command. JSON, +HTTP, and MCP surfaces preserve the stable code and do not parse human text. + +## Benchmark and Regression Design + +Extend the CLI scale harness with a committed Git fixture and a high-level agent +task that changes `k` paths. Measure both dry-run and actual apply at: + +- `N = 10,000`, `100,000`, and `1,000,000` +- `k = 1` and `100` + +Add three scenarios: + +1. Mapped, clean high-level dry-run. +2. Mapped, clean high-level actual apply. +3. Missing mapping, which must fail quickly with `GIT_MAPPING_REQUIRED` and no + Git object or Trail mapping writes. + +The benchmark records wall time, RSS, changed-path count, Git command count, +tracked-status count, full-root load count, blob-hash count, and export mode. +Production gates assert structural invariants before wall-time ceilings: + +- `full_root_load_count = 0` +- `tracked_status_count <= 1` +- `blob_hash_count <= k` +- `changed_path_count = k` +- `export_mode = mapped_delta` + +Reference-machine latency targets are: + +- 100,000 files, `k = 1`: dry-run <= 2 seconds; actual apply <= 3 seconds. +- 1,000,000 files, `k = 1`: dry-run <= 5 seconds; actual apply <= 8 seconds. +- Missing mapping at either scale: fail <= 1 second after Trail opens. + +CI keeps the 1,000-file smoke scenario. Scheduled scale automation enforces the +100,000- and 1,000,000-file structural gates and publishes latency results. + +## Test Strategy + +### Unit tests + +- Mapped-delta policy rejects a missing mapping without loading a root. +- High-level policy cannot select full snapshot export. +- Git handoff context reuses one state snapshot. +- Publication rejects a changed HEAD. +- Dry-run writes no mapping. + +### Integration tests + +- A mapped one-file task creates a commit preserving unrelated Git files and + symlinks. +- Missing mapping returns the stable reconciliation error and writes no Git + objects. +- Dirty tracked files block apply. +- Actual apply records exactly one successful mapping after fast-forward. +- Repeated apply is idempotent. + +### Scale tests + +- High-level agent dry-run and actual apply run at every `N`/`k` matrix point. +- Structural counters prove the hot path did not fall back even when a broad + wall-time threshold would otherwise pass. + +## Rollout + +1. Land failing regression tests and structural instrumentation. +2. Introduce export policy and make high-level apply require mapped delta. +3. Consolidate Git state discovery and final validation. +4. Add high-level scale scenarios and scheduled gates. +5. Update Git interop and performance documentation with the explicit + reconciliation requirement. + +The behavior change is intentionally strict. Repositories initialized without a +valid Git mapping must reconcile once before high-level apply. This converts an +unbounded surprise into a visible, recoverable setup requirement. + +## Subsequent Performance Slices + +The full production objective continues after this slice: + +1. **Changed-path recording ledger:** persist watcher, COW-upper, and structured + patch evidence so normal recording reads only changed paths; fall back only + on explicit overflow reconciliation. +2. **Persistent path-invariant index:** add a case-fold path index to Worktree + roots so adds and renames validate in `O(k log N)` without loading all paths. +3. **Git plumbing batching and end-to-end SLOs:** remove per-changed-file process + overhead where measurements justify it and enforce cross-platform production + release gates. + +Each slice receives its own implementation plan and verification evidence. The +program is complete only after all slices pass their 10,000-, 100,000-, and +1,000,000-file gates. diff --git a/docs/superpowers/specs/2026-07-12-unified-agent-setup-design.md b/docs/superpowers/specs/2026-07-12-unified-agent-setup-design.md new file mode 100644 index 0000000..c62c444 --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-unified-agent-setup-design.md @@ -0,0 +1,362 @@ +--- +meta: + contentType: Conceptual + title: How do you set up Trail agents with ACP and native hooks? +--- + +# How do you set up Trail agents with ACP and native hooks? + +Trail separates Agent Client Protocol (ACP) setup from native hook setup. You choose the transport first, then configure its provider. Both transports record the same Trail tasks, lifecycle events, evidence, and checkpoints. + +## Document plan + +- **Overview**: define the ACP, hooks, and terminal command boundaries +- **Goal**: explain how Trail configures each agent transport through one consistent setup contract +- **Audience**: Trail contributors who implement or review agent setup, provider resolution, adapters, and capture +- **Content plan**: cover commands, provider syntax, planning, configuration ownership, runtime capture, failures, and tests +- **Open questions**: none; the product and safety decisions in this document are approved + +## Why Trail separates ACP and hooks + +Trail currently spreads agent setup across `trail agent setup`, `trail acp install`, `trail agent acp`, and `trail agent hooks add`. These commands use different provider catalogs, editor lists, mutation rules, and terminology. + +The new model exposes two sibling integration categories: + +- `trail agent acp` owns editor and ACP integration +- `trail agent hooks` owns native provider hooks + +Terminal agents need no setup command. `trail agent start` launches an isolated task directly. + +## Product goals + +The change must meet these goals: + +1. Give ACP and native hooks distinct command boundaries +2. Use the same provider syntax across setup, diagnostics, and terminal startup +3. Use one planning and transaction contract for both setup categories +4. Route ACP, native hooks, terminal tasks, and hybrid capture into the same task workflow +5. Preserve unrelated editor and provider configuration +6. Keep every setup idempotent, transactional, and recoverable +7. Remove the old setup model without aliases or migration behavior + +## Non-goals + +This change will not: + +- Add a combined setup orchestrator +- Add a graphical setup application +- Make ACP and native hooks expose identical evidence +- Require setup before a terminal task +- Install configuration without an exact adapter for the target schema +- Change task review, readiness, apply, or finish semantics +- Preserve removed command syntax + +## Public command model + +The public command tree makes the transport boundary visible: + +```text +trail agent +├── acp +│ ├── setup +│ ├── status +│ ├── doctor +│ └── sessions +├── hooks +│ ├── setup +│ ├── status +│ ├── doctor +│ ├── events +│ ├── replay +│ └── remove +└── start +``` + +Configured editors invoke the hidden ACP runner. Provider hook files invoke the hidden singular hook receiver: + +```sh +trail agent acp run codex +trail agent hook receive codex Stop +``` + +`trail acp relay` remains the low-level surface for custom ACP hosts and relay diagnostics. It does not configure editors or replace `trail agent acp setup`. + +## Provider arguments + +Every provider-aware agent command accepts a provider as its preferred positional argument: + +```sh +trail agent start codex +trail agent acp setup codex --editor zed +trail agent hooks setup codex --scope project +trail agent acp doctor codex +trail agent hooks status codex +``` + +The explicit `--provider` form remains part of the new model for automation and callers that prefer named arguments: + +```sh +trail agent start --provider codex +trail agent acp setup --provider codex --editor zed +trail agent hooks setup --provider codex --scope project +``` + +A command rejects input that supplies both forms. After resolution, Trail stores and reports the canonical provider name. + +`trail agent start` uses `agent.default_provider` only when neither form is present. You set that general terminal default explicitly: + +```sh +trail config set agent.default_provider codex +trail agent start +``` + +ACP and hooks setup never change the terminal default. + +Interactive setup may ask you to choose a provider when you omit it. Non-interactive setup requires the positional or named form. + +## Shared setup contract + +Both setup categories use the same operation sequence: + +```text +resolve provider + -> inspect workspace and targets + -> build a read-only plan + -> preview and confirm + -> lock and apply + -> verify + -> commit metadata or roll back +``` + +An interactive invocation previews the plan and requests confirmation. `--print` and JSON output return the plan without writing files or downloading providers. `--yes` applies a fully resolved plan without prompting. + +A non-interactive invocation never waits for input. Without `--yes`, it remains read-only. + +## ACP setup ownership + +`trail agent acp setup` configures editor and ACP integration for one provider: + +```sh +trail agent acp setup codex --editor zed +trail agent acp setup codex --editor vscode --yes +trail --json agent acp setup codex --editor generic +``` + +ACP setup may perform these actions: + +- Resolve a built-in or registry ACP provider +- Report a package runner, download, or cached binary requirement +- Detect supported editors when `--editor` is absent +- Configure exact editor adapters +- Generate a generic copyable entry for unsupported editors +- Inject Trail Model Context Protocol (MCP) tools into the hidden runner +- Record ACP integration ownership and source digests +- Verify the provider, editor entry, workspace path, and launch command + +ACP setup must not install or remove native hooks. It must not change `agent.default_provider`. + +Each editor entry uses a stable Trail-owned identifier such as `trail-codex` and a label such as `Trail · Codex`. The generated entry uses absolute executable and workspace paths: + +```text +/path/to/trail --workspace /path/to/repository agent acp run codex +``` + +The first exact adapter targets Zed external-agent settings. VS Code support requires an adapter for a known ACP client extension and its exact settings schema. Trail prints a generic entry when it cannot identify that schema. + +## Native hook setup ownership + +`trail agent hooks setup` installs or updates Trail-owned native hooks for one provider: + +```sh +trail agent hooks setup codex +trail agent hooks setup claude-code --scope user +trail --json agent hooks setup gemini --scope project +``` + +Hooks setup may perform these actions: + +- Resolve a built-in native hook manifest +- Probe provider compatibility when requested +- Merge Trail-owned entries into shared JSON configuration +- Create an owned plugin, extension, or hook file +- Record ownership inventory and before/after digests +- Verify the installed configuration and provider contract + +Hooks setup must not configure editors, acquire ACP registry providers, or change ACP integration metadata. It must not change `agent.default_provider`. + +Project scope is the default. User scope writes only to the provider’s declared user location. `trail agent hooks remove` removes exact Trail-owned entries or files after ownership verification. + +## Terminal task behavior + +`trail agent start` launches a provider without a setup phase: + +```sh +trail agent start codex +trail agent start custom -- my-agent --flag +``` + +The command creates a fresh task lane, materializes or mounts its workdir, starts a managed capture run, launches the provider, and records the final checkpoint. Installed native hooks may enrich the same task with prompts, tools, approvals, transcripts, and per-turn checkpoints. + +The terminal command resolves its provider from the positional argument, `--provider`, or `agent.default_provider`, in that order. It reports an error when all three are absent. + +## Setup architecture + +The implementation contains one shared setup framework and transport-specific adapters: + +- **Provider resolver**: returns one canonical identity and transport capabilities +- **Detector**: locates the workspace, executable, provider dependencies, editor installations, and target files +- **Planner**: converts detected state and arguments into a serializable plan without mutation +- **ACP adapter**: owns editor detection, scoped merges, generated entries, and verification +- **Hook adapter**: owns provider hook configuration, event contracts, and verification +- **Applier**: locks targets, creates secure backups, writes atomically, and rolls back failures +- **Verifier**: checks every result against the plan before metadata commit + +The planner is the only input to human previews, JSON output, and mutation. The applier rejects a plan when a source digest differs from the inspected value. + +## Setup plans and results + +Every setup plan contains common fields: + +- Transport category +- Workspace root and optional initialization action +- Trail executable path +- Canonical provider and capability summary +- Acquisition actions and network requirements +- Target paths and source digests +- Trail-owned entries or files +- Redacted scoped diffs +- Verification checks +- Confirmation requirement + +ACP plans also contain editor adapter identifiers and fallback entries. Hook plans also contain scope, manifest version, provider version range, and ownership inventory. + +The apply result records every attempted action, its status, rollback status, and verification outcome. Human output summarizes these fields. JSON keeps stable field names for automation. + +## Workspace and provider preparation + +Setup begins with a read-only preflight. In an uninitialized Git repository, the plan may include `trail init --from-git`. Trail shows that action before confirmation. It does not choose a baseline for a non-Git directory. + +Provider resolution reports every Trail-controlled download before mutation. Diagnostics redact environment values, authentication data, unrelated configuration values, and provider output that may contain secrets. + +The ACP and hook resolvers use one provider capability manifest. A provider may support either transport, both transports, or terminal execution only. Each setup command rejects providers that do not support its transport. + +## Configuration ownership and transactions + +Before writing, the applier locks every target and compares its current digest with the plan. It stores user-readable-only backups in Trail’s operating-system application state directory. Editor and provider settings may contain credentials, so backups never enter the repository or `.trail` data. + +Adapters update only their Trail-owned entry or file. Shared formats preserve unrelated keys and hooks. Owned files reject foreign content unless the operator has explicitly authorized replacement through that transport’s setup contract. + +Each setup invocation forms one transaction. Trail writes temporary files, parses each result with its adapter, and replaces targets atomically. A write or verification failure restores every changed target. + +If the plan creates a Trail workspace, rollback removes it only while its digest matches the created state. A concurrent change preserves the workspace and becomes the primary reported conflict. + +Provider acquisition may leave an unreferenced immutable cache artifact after a later failure. It must not leave partial workspace, editor, or hook configuration. + +Repeated setup with the same arguments produces no configuration diff. A concurrent target change aborts the operation and requires a new plan. + +## Runtime capture boundaries + +The hidden `trail agent acp run` command creates a fresh task and starts the ACP relay with MCP injection. The hidden `trail agent hook receive` command journals one bounded, redacted native receipt and returns the provider’s success response. + +ACP, native hooks, terminal wrappers, and hybrid capture use the same versioned lifecycle vocabulary and capture coordinator. Provider adapters translate native events. They do not own task creation, checkpoint policy, deduplication, or finalization. + +When an exact native session identity matches an active ACP session, Trail records hybrid capture. ACP owns lifecycle progression while native hooks add receipts and transcript evidence. Ambiguous matches fail closed. + +## Hard cutover + +The implementation removes the old setup model: + +- Remove `trail agent setup` +- Remove `trail acp install` +- Remove `trail agent hooks add` +- Remove the old leaf form of `trail agent acp` +- Remove old help text, examples, tests, and documentation + +The cutover adds no aliases, deprecation warnings, migration messages, or compatibility shims. Removed syntax receives the command parser’s standard unknown-command or missing-subcommand response. + +`trail acp relay` remains because it serves custom ACP integrations. The explicit `--provider` form also remains because it is part of the new positional-provider contract. + +## Failure behavior + +Each setup category reports failures within its boundary: + +- **Missing workspace**: include Git-based initialization in the confirmed plan +- **Unsupported transport**: report the provider capability that is absent +- **Unsupported editor**: print a generic ACP entry without changing editor settings +- **Malformed target**: report the parse location and preserve the file +- **Missing provider dependency**: show the required runner or acquisition action before mutation +- **Concurrent target change**: abort and require a new preview +- **Write or verification failure**: restore the full setup transaction +- **Rollback failure**: report the unrecovered target as the primary blocker +- **Native delivery failure**: spool the redacted receipt and preserve provider success behavior + +ACP failure never changes hook configuration. Hook failure never changes editor or ACP configuration. + +## Diagnostics and terminology + +Public output names the selected category: + +- `trail agent acp doctor` diagnoses editor, provider, relay, MCP, and capture readiness +- `trail agent hooks doctor` diagnoses compatibility, ownership, drift, delivery, receipts, spool pressure, and transcript fidelity +- `trail agent doctor` remains task-oriented and checks terminal launch plus workspace readiness + +Daily output uses provider, editor agent, terminal agent, native hooks, and Trail task. Protocol terms such as relay, lane, session mapping, and receipt appear only where they explain an ACP or hooks diagnostic. + +## Test strategy + +Implementation follows test-driven development. + +### Command contract tests + +- Accept the positional provider and `--provider` forms +- Reject input that supplies both provider forms +- Resolve `agent.default_provider` only for terminal startup +- Require a provider for non-interactive setup +- Expose only the new command tree in help output +- Reject removed command syntax through the standard parser path + +### Planning and adapter tests + +- Produce no writes or downloads while planning +- Emit the same plan through human and JSON projections +- Redact unrelated configuration values +- Reject providers that lack the selected transport +- Verify ACP editor adapter contracts on each operating system +- Verify native hook adapter contracts for every built-in manifest +- Preserve unrelated configuration and reject malformed targets + +### Transaction tests + +- Store secure backups outside the workspace +- Detect source changes after planning +- Apply every target through atomic replacement +- Restore every changed target after an injected failure +- Report rollback failures without claiming recovery +- Produce no configuration diff after repeated setup + +### Integration and end-to-end tests + +- Exercise confirmation, rejection, `--print`, `--yes`, and JSON behavior +- Keep test homes, workspaces, caches, and editor settings isolated +- Launch a stub editor through hidden `trail agent acp run` +- Deliver stub native events after `trail agent hooks setup` +- Launch terminal tasks with positional, named, and configured providers +- Confirm that every transport creates addressable tasks with the same review and finish actions +- Confirm that hybrid capture deduplicates turns, messages, spans, and checkpoints + +Optional release checks may exercise real editors and providers in disposable profiles. Unit and integration tests do not require network access. + +## Completion criteria + +The change is complete when: + +1. ACP setup configures and verifies supported editor integrations through `trail agent acp setup` +2. Hook setup configures and verifies native provider hooks through `trail agent hooks setup` +3. Both setup categories use the same plan, confirmation, transaction, and verification contract +4. Terminal startup requires no setup and accepts a positional provider +5. The explicit `--provider` form behaves identically to the positional form +6. Generated editors launch the hidden `trail agent acp run` +7. ACP, hooks, terminal, and hybrid capture retain one task and evidence model +8. Removed commands, aliases, documentation, and tests no longer exist +9. Repeated setup is idempotent and injected failures restore every changed target +10. Human and JSON output expose stable, redacted plan and result fields diff --git a/docs/use-cases/conflict-resolution.md b/docs/use-cases/conflict-resolution.md index 9ddbf5b..ca07897 100644 --- a/docs/use-cases/conflict-resolution.md +++ b/docs/use-cases/conflict-resolution.md @@ -64,4 +64,4 @@ trail lane readiness doc-bot - Conflict CLI args: `trail/src/cli/command/collaboration_args/merge.rs` - Manual resolution parsing: `trail/src/cli/command/handler/parsing.rs` -- Tests: `merge_queue_pauses_on_conflict`, `manual_conflict_resolution_works_through_db_cli_http_and_mcp` +- Tests: `lane_merge_queue_pauses_on_conflict`, `manual_conflict_resolution_works_through_db_cli_http_and_mcp` diff --git a/docs/use-cases/daemon-backed-automation.md b/docs/use-cases/daemon-backed-automation.md index 2fb9145..e634673 100644 --- a/docs/use-cases/daemon-backed-automation.md +++ b/docs/use-cases/daemon-backed-automation.md @@ -39,7 +39,7 @@ Daemon routing supports: - `record` - `diff` - selected `lane` commands -- `merge-lane` +- `lane merge` - `merge-queue` If auto-discovery finds a stale daemon endpoint, the CLI falls back to local execution for unavailable daemon errors. diff --git a/docs/use-cases/parallel-lane-work.md b/docs/use-cases/parallel-lane-work.md index ddb9d2a..c6fe24d 100644 --- a/docs/use-cases/parallel-lane-work.md +++ b/docs/use-cases/parallel-lane-work.md @@ -42,9 +42,9 @@ trail lane record docs-bot -m "record docs workdir" ## Merge Safely ```sh -trail merge-queue add docs-bot --into main -trail merge-queue add tests-bot --into main -trail merge-queue run +trail lane merge-queue add docs-bot --into main +trail lane merge-queue add tests-bot --into main +trail lane merge-queue run ``` ## Recover a Bad Attempt diff --git a/docs/use-cases/safe-lane-change-review.md b/docs/use-cases/safe-lane-change-review.md index 121e271..57c31be 100644 --- a/docs/use-cases/safe-lane-change-review.md +++ b/docs/use-cases/safe-lane-change-review.md @@ -32,9 +32,9 @@ trail approvals decide --decision approved ## Merge Flow ```sh -trail merge-lane doc-bot --into main --dry-run -trail merge-queue add doc-bot --into main -trail merge-queue run +trail lane merge doc-bot --into main --dry-run +trail lane merge-queue add doc-bot --into main +trail lane merge-queue run ``` ## Code Facts Used diff --git a/openspec/changes/hard-cutover-terminal-ux/.openspec.yaml b/openspec/changes/hard-cutover-terminal-ux/.openspec.yaml new file mode 100644 index 0000000..8803b47 --- /dev/null +++ b/openspec/changes/hard-cutover-terminal-ux/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-12 diff --git a/openspec/changes/hard-cutover-terminal-ux/design.md b/openspec/changes/hard-cutover-terminal-ux/design.md new file mode 100644 index 0000000..66a3b00 --- /dev/null +++ b/openspec/changes/hard-cutover-terminal-ux/design.md @@ -0,0 +1,220 @@ +## Context + +Trail's CLI exposes a large, coherent domain through report types, but human rendering is implemented as command-local `println!` calls. Most views print internal metadata before the user-visible state, collections are not width-aware, raw Rust enum names leak into the interface, color is largely confined to patches, and failures rarely include recovery guidance. Richer agent and lane reports compound the problem because every field receives similar visual weight. + +The CLI already has useful boundaries: commands return serializable reports, JSON errors have stable codes and exit statuses, `NO_COLOR` is partially honored, and diff rendering detects redirected stdout. The redesign should therefore replace presentation rather than rewrite Trail's operations. It must work on macOS, Linux, and Windows; remain useful over SSH and in CI logs; handle arbitrary user-controlled paths/messages safely; and avoid turning ordinary commands into a full-screen application. + +The intended interaction is a composable terminal UI: fast command output with excellent hierarchy, responsive tables, reviewable diffs, transient progress where useful, and exact next actions. It is not an alternate interactive shell. + +## Goals / Non-Goals + +**Goals:** + +- Give every command an immediately legible outcome and consistent information hierarchy. +- Make the daily flow—orient, inspect, review, validate, recover, and apply—discoverable without requiring Trail's internal model. +- Render attractive wide-terminal output that degrades cleanly to narrow terminals and deterministic logs. +- Centralize presentation policy so command renderers describe semantics rather than spacing, ANSI sequences, or terminal behavior. +- Make warnings, blockers, failures, and recovery actions unambiguous without relying on color. +- Keep JSON/NDJSON, exit codes, and error codes as the stable automation surfaces. +- Complete the cutover across every human renderer in one release, with no legacy-style branch. + +**Non-Goals:** + +- Building a persistent full-screen dashboard, alternate shell, or mouse-driven interface. +- Changing Trail's storage model, operation semantics, readiness rules, or merge safety rules. +- Preserving byte-for-byte human output compatibility. +- Making human or plain output a machine-parsing API; automation must use JSON or NDJSON. +- Adding terminal hyperlinks, image protocols, themes, or user-defined layout templates in the initial cutover. +- Replacing domain reports with view-specific database queries when existing reports already contain the required facts. + +## Decisions + +### 1. Render semantic documents instead of printing from command modules + +Each command adapter will transform its existing report into a `Document` composed from a small set of semantic blocks: + +- `Lead`: outcome text plus `Success`, `Attention`, `Blocked`, `Failure`, `Neutral`, or `Progress` state. +- `Metadata`: low-priority key/value facts. +- `Section`: titled groups with importance and optional empty-state text. +- `Table`: typed headers and rows with per-column alignment, priority, and wrapping policy. +- `ChangeList`: path, change kind, rename source, additions, deletions, and optional line-identity detail. +- `Checklist`: pass/warn/fail/blocked/pending/skip evidence rows. +- `Diagnostic`: stable code, summary, cause, consequence, and recovery actions. +- `Diff`: file summaries and optional unified patch content. +- `NextAction`: exactly one primary safe command and reason, plus optional secondary actions. +- `Notice`: truncation, stale data, skipped detail, or pager fallback. + +A shared `Renderer` will render the document. Command-specific modules will not emit ANSI escapes, compute padding, inspect TTY state, or call stdout/stderr directly. + +This separates domain facts from presentation, permits unit testing without process-global streams, and makes wide, narrow, human, and plain behavior consistent. A loose collection of formatting helpers was rejected because it would retain imperative layout decisions in every command. + +### 2. Resolve one immutable render context at CLI startup + +`RenderContext` will contain output format, verbosity, quiet mode, color policy, glyph policy, terminal width/height, stdout/stderr TTY state, pager policy, clock/timezone formatting, and whether transient progress is allowed. Environment and global flags are resolved once before command dispatch. + +The new global contract is: + +- `--format human`: adaptive terminal output; styling, Unicode glyphs, relative times, progress, and paging are allowed when supported. +- `--format plain`: deterministic line-oriented output with ASCII, no ANSI/OSC sequences, no pager, no transient updates, and absolute machine-independent timestamps. +- `--format json`: one structured JSON value. +- `--format ndjson`: one structured JSON value per record for streaming/list commands. +- `--color auto|always|never`: explicit color policy; `auto` honors TTY state, `NO_COLOR`, and `TERM=dumb`. +- `--pager auto|always|never`: explicit paging policy; `auto` is limited to long human output on an interactive terminal. +- `--verbose`: reveal full IDs, roots, internal refs, secondary evidence, and additional commands. +- `--quiet`: suppress successful human/plain output; errors still render. + +The current `--no-color` switch is removed rather than maintained as an alias. This is an intentional hard cutoff. + +### 3. Use one outcome-first grammar + +Human documents use this order: + +1. Lead outcome or current state. +2. One short context line when needed. +3. Primary evidence or changed items. +4. Warnings, blockers, or diagnostics. +5. One primary next action. +6. Secondary metadata/actions only when useful or verbose. + +Clean and empty outcomes remain compact. A clean `trail status` should fit in two or three lines; a blocked readiness view should spend space on blockers and recovery, not object IDs. Mutation commands start with the completed or previewed action and its scope. + +Metadata-first and label-dump layouts were rejected because users must read several lines before learning whether action is required. + +### 4. Define a restrained visual language + +State is always written as text and optionally reinforced with color: + +| State | Label examples | Color role | +| --- | --- | --- | +| Success | `PASS`, `READY`, `CLEAN`, `RECORDED` | green | +| Attention | `WARN`, `PENDING`, `DIRTY` | yellow | +| Blocked/failure | `BLOCKED`, `FAIL`, `CONFLICT` | red | +| Informational | identifiers, refs, commands | cyan | +| Secondary | roots, timestamps, counts | dim/default | + +Bold is reserved for the lead and section titles. Commands are visually distinct from explanations. Human mode may use `·`, `→`, `…`, and a light horizontal rule; plain mode uses ASCII equivalents. Boxed tables and decorative frames are prohibited because they waste width and dominate logs. + +### 5. Make layouts responsive, not merely truncated + +The renderer will measure display width, including Unicode width, and choose a layout based on the available terminal width: + +- At wide widths, tables expose useful secondary columns and change statistics. +- At standard widths, low-priority columns disappear before content is truncated. +- At narrow widths, each record becomes a compact stacked block with the identity or path first. + +Paths and user messages are the flexible columns. Status, counts, and short identifiers remain intact. Cells never contain raw newlines; multiline content becomes a paragraph beneath the row or uses escaped separators. Truncation always emits an explicit count and a command that reveals the omitted content. + +A fixed 80-column renderer was rejected because it looks sparse on modern terminals and still fails on split panes. + +### 6. Treat tables as a collection tool, not a universal layout + +Borderless tables are used for timelines, branches, queues, tasks, sessions, approvals, gates, checks, mappings, traces, and other multi-row comparisons. Key/value groups are used for one object's metadata. Change lists are used for paths. Paragraphs and diagnostics are used for explanations. + +Column headers appear when they disambiguate three or more rows or when a collection has heterogeneous columns. A one-row result should normally render as a sentence or key/value group. Tables must remain understandable with color disabled. + +### 7. Make diffs progressive and review-oriented + +Diff output has three layers: + +1. A lead naming the comparison. +2. A compact change list with `A/M/D/R/T`, paths, additions/deletions, and proportional change bars. +3. Unified patches only when requested. + +`--stat` produces summary/change statistics without patch bodies. New `--name-only` and `--name-status` options provide focused path output. `--patch` adds patch bodies grouped by file without repeating redundant global headings. Stable line identities remain opt-in and appear beneath the relevant file/hunk. Renames use a readable `old -> new` relationship with an ASCII fallback. + +Patch color distinguishes additions, deletions, hunk headers, and metadata, but prefixes remain authoritative. Long patches may use the pager. Binary, opaque, missing-patch, and truncated cases receive explicit notices rather than disappearing. + +### 8. Translate internal vocabulary at the presentation boundary + +All domain enums receive explicit user-facing labels. Renderers will not use debug formatting for operation kinds, change kinds, worktree states, readiness states, or run states. Examples include `ManualRecord` to `record`, `GitImport` to `import from Git`, and `DirtyTracked` to `unrecorded changes`. + +The default human view prefers task/lane names, checkpoint aliases, branch names, and resolvable selectors. Full change IDs, root object IDs, raw ref names, and database identities are verbose details. An identifier may be shortened only when the shortened form is accepted unambiguously by every command that presents it as a copyable selector; otherwise the renderer uses a full ID or existing alias. + +### 9. Make diagnostics explain recovery + +Human errors and blocking reports use the same diagnostic model. Each diagnostic contains: + +- a stable code; +- a concise user-facing summary; +- the cause when known; +- the consequence or protected action; +- one safest exact recovery command when available; +- optional alternatives ordered by safety. + +Force, bypass, destructive, or approval-sensitive commands are never the primary recovery action and must state their consequence. JSON errors retain their current code/message/exit-code contract. Diagnostics go to stderr; successful result documents go to stdout. + +### 10. Present exactly one primary next action + +Status, guide, dashboard, readiness, dry-run, conflict, and error views may end with `Next:`. It contains one command and one short reason. Additional useful commands appear under `More:` only when they represent genuinely different intents; verbose mode may show a fuller command set. + +The primary action must be safe for the current state, copy/paste ready, and use the selector already visible to the user. Commands requiring confirmation must say so before the user runs them. Views with no meaningful next step omit the section. + +### 11. Keep long-running UX transient and composable + +Long-running scans, checks, backups, cache operations, and agent launches may display a single transient progress line on interactive stderr. The line reports the operation and, when reliable, completed/total work or elapsed time. It is cleared before the final document renders. + +Progress is disabled for plain, JSON, NDJSON, quiet, redirected stderr, and `TERM=dumb`. Trail never emits an unbounded stream of spinner frames into logs. A broken pipe exits cleanly without printing a second error. + +### 12. Page only review content and preserve pipeline behavior + +Automatic paging is limited to long human-mode content such as patches, transcripts, detailed histories, conflict explanations, and large inspection views. Lists intended for composition do not page by default. Paging is disabled when stdout is redirected, in plain/JSON/NDJSON modes, and in quiet mode. + +The pager receives already-rendered content and an explicit color policy. If it cannot be started, Trail writes directly to stdout and, only in verbose mode, emits a non-fatal notice. Trail does not treat a pager's early close as command failure. + +### 13. Sanitize untrusted terminal content + +Paths, operation messages, actor strings, tool output summaries, conflict text, and patch content may contain terminal control sequences. User-controlled content is sanitized before styling: escape characters and non-printing controls are rendered visibly, embedded newlines are normalized according to block type, and renderer-owned ANSI/OSC sequences are added only after sanitization. Tabs remain meaningful in patch bodies but are width-accounted elsewhere. + +This protects terminals and snapshot structure without changing stored values or JSON output. + +### 14. Migrate by command archetype and cut over atomically + +Implementation proceeds through reusable archetypes: + +1. Core infrastructure and diagnostics. +2. Orientation/mutation: init, status, record, checkout, branch. +3. Review: timeline, show, history, why, code-from, diff, transcript. +4. Decisions: guardrails, readiness, doctor, approvals, conflicts, merges, queue. +5. Lane and agent dashboards, actions, evidence, and recovery. +6. Specialist inspection, environment, cache, backup, and integration commands. + +All human renderers must migrate before release. There is no runtime legacy renderer and no mixed-style supported state. JSON/NDJSON rendering bypasses the semantic document renderer and continues to serialize reports directly. + +### 15. Verify semantics and presentation independently + +Tests operate at three levels: + +- Semantic document tests assert lead state, evidence, blocker ordering, and next action without depending on spacing. +- Golden renderer tests cover human and plain modes at representative narrow, standard, and wide widths, with and without color/glyphs. +- PTY/end-to-end tests cover TTY detection, redirection, `NO_COLOR`, paging, progress cleanup, errors, broken pipes, and representative command families. + +Clocks and terminal dimensions are injected. Snapshots do not depend on the developer's locale, timezone, terminal, or current time. A source-level guard prevents new direct `println!`/`eprintln!` calls in command render modules after cutover. + +## Risks / Trade-offs + +- **[Risk] The all-at-once cutover creates a large review surface.** → Migrate by archetype behind an internal compile-time module boundary, require complete command inventory coverage, and merge only when the legacy renderer has no call sites. +- **[Risk] Adaptive output produces more snapshot combinations.** → Keep the primitive set small, test representative widths, and assert semantic documents separately from layout snapshots. +- **[Risk] Hiding internal identifiers may make expert diagnosis slower.** → Keep `--verbose`, `show`, and structured formats complete, and never abbreviate a selector that cannot be pasted back into Trail. +- **[Risk] Terminal capability detection differs across platforms and remote shells.** → Default uncertain cases to plain styling, provide explicit color/pager overrides, and test Windows plus PTY and redirected cases. +- **[Risk] Automatic paging can surprise pipelines or hang automation.** → Restrict it to interactive human review content and disable it whenever either output mode or TTY state is ambiguous. +- **[Risk] Sanitizing patch content can make unusual source bytes look different.** → Render escaped bytes visibly, retain raw content in JSON/object access, and test control-character fixtures. +- **[Risk] Additional presentation dependencies increase maintenance and binary size.** → Prefer focused, platform-neutral libraries; benchmark release size/startup; implement small stable primitives internally when dependency cost is disproportionate. +- **[Risk] Report models may lack precise recovery commands or summaries.** → Add optional semantic fields at the report layer only when the command handler cannot derive them safely; keep those additions shared by CLI, HTTP, and MCP where meaningful. + +## Migration Plan + +1. Inventory every human renderer and capture representative pre-cutover reports as fixtures, not as compatibility snapshots. +2. Add the render context, semantic document model, terminal capability resolver, buffered writer, sanitizer, and test harness. +3. Add the new global format/color/pager contract and remove the old `--no-color` surface. +4. Migrate command families in the archetype order above while JSON/NDJSON continue to bypass the new renderer. +5. Add semantic, golden, and PTY tests; validate Windows and non-TTY behavior; benchmark startup, large tables, and long patches. +6. Delete legacy formatting helpers and direct output calls, then update README examples, reference documentation, shell completions, and changelog with a clear breaking-change notice. +7. Release the new rendering style only after every command is migrated. Rollback is a source/release rollback, not a runtime compatibility flag. + +## Open Questions + +None. The following boundary decisions are resolved for this change: + +- Unique-prefix resolution is not added. Default views use existing resolvable aliases or full IDs; abbreviated decorative IDs are not presented as copyable selectors. +- Pager eligibility is explicit per semantic document. The initial eligible set is patch output, transcripts, detailed history, conflict explanations, and detailed object/text/map inspection. Schema, snippet, path-only, status, queue, and other composition-oriented output never pages automatically. +- Plain timestamps use RFC 3339 UTC and plain durations use integer milliseconds. Human mode may use relative timestamps and compact durations. diff --git a/openspec/changes/hard-cutover-terminal-ux/proposal.md b/openspec/changes/hard-cutover-terminal-ux/proposal.md new file mode 100644 index 0000000..1f21285 --- /dev/null +++ b/openspec/changes/hard-cutover-terminal-ux/proposal.md @@ -0,0 +1,33 @@ +## Why + +Trail exposes rich operational state, but its human output is inconsistent, metadata-first, and difficult to scan across status, review, readiness, merge, and recovery workflows. A hard cutover to one outcome-first terminal language will make the CLI feel trustworthy and immediately understandable while preserving structured machine output as a separate surface. + +## What Changes + +- **BREAKING**: Replace all existing human-readable command output with one new rendering contract; no legacy human style, compatibility switch, or deprecation period is provided. +- Introduce outcome-first layouts that present the current state, the important evidence, attention items, and one safest next action in that order. +- Introduce a shared semantic renderer for statuses, sections, key/value metadata, borderless tables, path changes, diagnostics, command hints, truncation, color, terminal width, and verbosity. +- Replace raw debug enum names and routine internal identifiers with stable user-facing language; keep full technical metadata available in verbose and structured output. +- Redesign status, record, timeline, diff, history, readiness, guardrail, doctor, approval, conflict, merge, lane, and agent-task views around the new grammar. +- Add responsive wide and narrow layouts, terminal-aware color, redirected-output behavior, safe text sanitization, and paging for long interactive content. +- Make failures actionable by pairing stable diagnostic codes with cause, consequence, and an exact safe recovery command. +- Define explicit human, plain, JSON, and NDJSON behavior. JSON/NDJSON report schemas and stable exit/error codes remain machine contracts, but human and plain rendering adopt the new style immediately. +- Add golden and semantic rendering tests across terminal widths, color policies, verbosity levels, empty states, large data sets, and redirected output. + +## Capabilities + +### New Capabilities + +- `terminal-output-ux`: Defines Trail's unified human/plain terminal rendering contract, responsive presentation, diff and table behavior, actionable diagnostics, and command-specific UX expectations. + +### Modified Capabilities + +None. This repository has no existing OpenSpec capability specifications; the new capability establishes the terminal output contract. + +## Impact + +- Affects the CLI argument/runtime context, all modules under `trail/src/cli/command/render`, human error rendering, and selected report models where the renderer lacks data needed for summaries or recovery guidance. +- Replaces direct command-level `println!`/`eprintln!` formatting with shared semantic rendering primitives and buffered writers. +- Changes snapshots, README examples, CLI reference documentation, and downstream consumers that parse human output; those consumers must move to JSON or NDJSON. +- May add small terminal presentation dependencies for display width, wrapping, styling, and paging, subject to Trail's Rust and platform support constraints. +- Does not change Trail's storage format, operational semantics, HTTP/MCP contracts, or JSON/NDJSON report meaning unless a separately documented model addition is required to render missing evidence. diff --git a/openspec/changes/hard-cutover-terminal-ux/specs/terminal-output-ux/spec.md b/openspec/changes/hard-cutover-terminal-ux/specs/terminal-output-ux/spec.md new file mode 100644 index 0000000..1c2d288 --- /dev/null +++ b/openspec/changes/hard-cutover-terminal-ux/specs/terminal-output-ux/spec.md @@ -0,0 +1,270 @@ +## ADDED Requirements + +### Requirement: Hard-cutover human rendering contract +Trail SHALL use the new terminal rendering contract for every human-readable command result and diagnostic, and SHALL NOT provide a legacy rendering mode or mix legacy and new layouts in a supported release. + +#### Scenario: Human command after cutover +- **WHEN** a user runs any Trail command with human output +- **THEN** the command renders exclusively through the new terminal UX primitives + +#### Scenario: Removed compatibility surface +- **WHEN** a user requests the former human style or obsolete `--no-color` option +- **THEN** Trail rejects the obsolete option with an actionable input diagnostic instead of invoking a compatibility renderer + +### Requirement: Explicit output modes +Trail SHALL provide distinct `human`, `plain`, `json`, and `ndjson` formats with behavior appropriate to interactive users, logs, single structured reports, and structured streams respectively. + +#### Scenario: Interactive human output +- **WHEN** `--format human` writes to a capable terminal +- **THEN** Trail may use responsive layout, semantic color, Unicode glyphs, relative time, transient progress, and eligible paging + +#### Scenario: Deterministic plain output +- **WHEN** `--format plain` is selected +- **THEN** Trail emits deterministic ASCII text without ANSI/OSC sequences, relative time, paging, or transient progress + +#### Scenario: Structured output +- **WHEN** JSON or NDJSON output is selected +- **THEN** Trail serializes report data directly without terminal decoration or human document headings + +#### Scenario: Quiet output +- **WHEN** `--quiet` is selected and the command succeeds +- **THEN** Trail suppresses normal human/plain output while still emitting failures on stderr + +### Requirement: Outcome-first information hierarchy +Every non-trivial human result SHALL lead with the user-visible outcome or current state and SHALL order subsequent content as context, primary evidence, attention items, and next action. + +#### Scenario: Dirty workspace +- **WHEN** `trail status` detects unrecorded changes +- **THEN** the first line states that the worktree has unrecorded changes before displaying branch, head, or root metadata + +#### Scenario: Clean workspace +- **WHEN** `trail status` detects no unrecorded changes +- **THEN** Trail emits a compact clean result without empty change or suggestion sections + +#### Scenario: Completed mutation +- **WHEN** a mutating command completes successfully +- **THEN** its lead states the completed action and affected scope before secondary identifiers + +### Requirement: Unified semantic status language +Trail SHALL express state with a shared vocabulary covering success, attention, blocked/failure, informational, and skipped states, and SHALL NOT rely on color as the sole distinction. + +#### Scenario: Readiness checklist without color +- **WHEN** a readiness view is rendered with color disabled +- **THEN** every check remains distinguishable through textual labels such as `PASS`, `WARN`, `FAIL`, `BLOCKED`, `PENDING`, or `SKIP` + +#### Scenario: Semantic color enabled +- **WHEN** human color output is enabled +- **THEN** success, attention, failure, commands, and secondary metadata use the globally defined color roles consistently + +### Requirement: Responsive terminal layouts +Trail SHALL adapt collections and metadata to available display width by removing low-priority columns before truncating flexible content and by switching to stacked records when aligned rows no longer fit. + +#### Scenario: Wide terminal +- **WHEN** a collection is rendered with sufficient width +- **THEN** Trail displays aligned borderless columns including useful secondary data + +#### Scenario: Narrow terminal +- **WHEN** the same collection cannot fit its required columns +- **THEN** Trail renders each item as a compact stacked record without splitting status labels, counts, or identifiers + +#### Scenario: Omitted collection items +- **WHEN** output limits omit one or more records +- **THEN** Trail states the omitted count and provides an exact command or flag that reveals all records + +### Requirement: Selective borderless tables +Trail SHALL use borderless tables for comparable multi-row collections and SHALL use sentences, key/value groups, change lists, or paragraphs when a table would not improve comparison. + +#### Scenario: Timeline collection +- **WHEN** a timeline contains multiple operations +- **THEN** Trail displays consistent operation, kind, branch, time, and message columns subject to responsive priority + +#### Scenario: Single object detail +- **WHEN** a command displays one object's metadata +- **THEN** Trail uses a key/value or narrative layout instead of a one-row boxed table + +### Requirement: Progressive diff presentation +Trail SHALL present diffs as a comparison lead, compact file-level summary, totals, and optional per-file patches, with path-only and name/status projections available. + +#### Scenario: Default diff summary +- **WHEN** a user requests a diff without `--patch` +- **THEN** Trail displays file change markers, paths, additions/deletions, change bars where space permits, and aggregate totals without patch bodies + +#### Scenario: Patch requested +- **WHEN** `--patch` is selected +- **THEN** Trail groups unified patch content beneath each affected file and preserves textual diff prefixes regardless of color + +#### Scenario: Name-only projection +- **WHEN** `--name-only` is selected +- **THEN** Trail emits only affected paths in the selected human/plain format + +#### Scenario: Name-status projection +- **WHEN** `--name-status` is selected +- **THEN** Trail emits stable `A`, `M`, `D`, `R`, or `T` markers with affected paths + +#### Scenario: Non-text change +- **WHEN** an affected file has no textual patch +- **THEN** Trail emits an explicit binary, opaque, unavailable, or truncated notice instead of silently omitting detail + +### Requirement: User-facing terminology and identifiers +Trail SHALL map domain states and operation kinds to explicit user-facing labels and SHALL hide routine internal identifiers from default views unless they are the result or required selector. + +#### Scenario: Domain enum rendering +- **WHEN** a report contains an internal enum such as `ManualRecord` or `DirtyTracked` +- **THEN** Trail renders a documented user-facing phrase and never a Rust debug representation + +#### Scenario: Verbose identifiers +- **WHEN** `--verbose` is selected +- **THEN** Trail includes full change IDs, roots, refs, and other secondary technical metadata relevant to the command + +#### Scenario: Copyable shortened identifier +- **WHEN** Trail renders an abbreviated identifier as a selector +- **THEN** the displayed abbreviation resolves unambiguously in every command that recommends copying it + +### Requirement: Actionable diagnostics +Every human diagnostic for a failed or blocked operation SHALL include a stable code, concise summary, cause or protected consequence when known, and the safest exact recovery action when one can be determined. + +#### Scenario: Dirty-worktree failure +- **WHEN** an operation is rejected to protect unrecorded worktree changes +- **THEN** Trail explains what would be at risk and presents inspection or recording as the primary recovery path + +#### Scenario: Risky bypass exists +- **WHEN** a force, bypass, or destructive alternative exists +- **THEN** Trail places it after the safe action and explicitly states its consequence + +#### Scenario: JSON diagnostic +- **WHEN** the same error is requested as JSON +- **THEN** Trail preserves its stable error code, message, and exit code without terminal-only recovery decoration + +### Requirement: One primary next action +Human orientation, readiness, guide, dry-run, conflict, and recovery views SHALL present at most one primary `Next:` command that is safe, copy/paste ready, and valid for the displayed state. + +#### Scenario: Blocked lane +- **WHEN** lane readiness is blocked by an unrecorded workdir +- **THEN** the primary next command addresses the unrecorded workdir rather than suggesting merge, force, or an unrelated inspection + +#### Scenario: Multiple useful commands +- **WHEN** secondary commands support different user intents +- **THEN** Trail places them after the primary action under a lower-priority `More:` section or reveals them in verbose mode + +#### Scenario: No useful action +- **WHEN** a result requires no follow-up and has no meaningful drill-down +- **THEN** Trail omits the `Next:` section + +### Requirement: Readiness and decision checklists +Readiness, guardrail, doctor, approval, merge-preflight, and validation views SHALL summarize the decision first and render evidence as ordered checks with blockers before warnings. + +#### Scenario: Ready decision +- **WHEN** all required checks pass +- **THEN** the lead states that the item is ready and the checklist identifies the passing evidence + +#### Scenario: Blocked decision +- **WHEN** one or more blocking checks fail +- **THEN** the lead states that the item is blocked, blockers appear before warnings, and the primary next action addresses the highest-priority blocker + +### Requirement: Terminal capability and color policy +Trail SHALL resolve terminal capabilities once per invocation and SHALL support `--color auto|always|never`, with `auto` honoring stream TTY state, `NO_COLOR`, and `TERM=dumb`. + +#### Scenario: Redirected human output +- **WHEN** human stdout is redirected and color policy is `auto` +- **THEN** Trail disables terminal escape styling and automatic paging + +#### Scenario: Explicit color override +- **WHEN** color policy is `always` or `never` +- **THEN** Trail applies the explicit policy consistently to all renderer-owned styling + +#### Scenario: No-color environment +- **WHEN** `NO_COLOR` is present and color policy is `auto` +- **THEN** Trail emits no ANSI color sequences + +### Requirement: Safe paging +Trail SHALL page only eligible long-form human review content and SHALL never automatically page plain, JSON, NDJSON, quiet, or redirected output. + +#### Scenario: Long interactive patch +- **WHEN** a patch exceeds the interactive terminal height and pager policy is `auto` +- **THEN** Trail may send the rendered patch through the configured pager + +#### Scenario: Pipeline output +- **WHEN** stdout is connected to a pipeline +- **THEN** Trail writes directly to stdout without starting a pager + +#### Scenario: Pager closes early +- **WHEN** the pager exits after receiving only part of the output +- **THEN** Trail treats the close as a successful user action and does not print a broken-pipe diagnostic + +### Requirement: Composable progress rendering +Trail SHALL restrict transient progress to long-running human commands with an interactive stderr and SHALL clear progress before rendering the final result. + +#### Scenario: Interactive long operation +- **WHEN** an eligible operation is running with interactive human stderr +- **THEN** Trail may render one bounded transient progress line containing reliable progress or elapsed time + +#### Scenario: Logged operation +- **WHEN** stderr is redirected or output is plain, JSON, NDJSON, or quiet +- **THEN** Trail emits no spinner frames or cursor-control updates + +### Requirement: Terminal-content sanitization +Trail SHALL sanitize user-controlled text before applying renderer-owned styling so paths, messages, tool summaries, conflict text, and patches cannot inject terminal control sequences or corrupt layout. + +#### Scenario: Message contains escape sequence +- **WHEN** an operation message contains ANSI escape bytes +- **THEN** human/plain output displays the bytes visibly or removes their control effect without executing them + +#### Scenario: Table cell contains newline +- **WHEN** a user-controlled table value contains embedded newlines +- **THEN** Trail normalizes or escapes them so the value cannot create forged rows or headings + +#### Scenario: Structured value contains controls +- **WHEN** the same value is emitted through JSON or NDJSON +- **THEN** Trail preserves the stored value using the format's normal escaping rules + +### Requirement: Correct output streams and broken-pipe behavior +Trail SHALL write successful result documents to stdout, diagnostics and transient progress to stderr, and SHALL terminate cleanly when a downstream consumer closes stdout. + +#### Scenario: Successful command +- **WHEN** a command succeeds without warnings requiring stderr +- **THEN** all persistent result content is written to stdout + +#### Scenario: Failed command +- **WHEN** a command returns a Trail error +- **THEN** the persistent diagnostic is written to stderr and Trail exits with the error's stable exit code + +#### Scenario: Consumer closes pipe +- **WHEN** a downstream consumer closes stdout early +- **THEN** Trail exits without a second user-facing I/O failure + +### Requirement: Complete command-family coverage +The hard cutover SHALL cover core workspace, inspection, diff/history, collaboration, maintenance, integration, lane, and agent command families before release. + +#### Scenario: Renderer inventory validation +- **WHEN** the cutover build is validated +- **THEN** every human command route maps to a semantic document renderer or an explicitly documented raw-content renderer such as schema or snippet output + +#### Scenario: New command renderer +- **WHEN** a new command is added after the cutover +- **THEN** automated checks reject direct command-level terminal printing outside approved low-level writer modules + +### Requirement: Rendering verification matrix +Trail SHALL test semantic content separately from layout and SHALL maintain golden coverage for representative terminal capabilities and end-to-end coverage for stream behavior. + +#### Scenario: Width matrix +- **WHEN** renderer golden tests run +- **THEN** representative narrow, standard, and wide widths are verified for both human and plain output + +#### Scenario: Deterministic fixtures +- **WHEN** snapshots include time, duration, terminal size, or identifiers +- **THEN** those inputs are injected or fixed so output does not depend on the developer environment + +#### Scenario: Structured regression check +- **WHEN** the new renderer is introduced +- **THEN** JSON/NDJSON report semantics and stable error/exit codes are regression tested independently of human snapshots + +### Requirement: Rendering performance +Human and plain rendering SHALL be linear in the amount of rendered report data, use buffered writes, and avoid repeating database or worktree queries solely for presentation. + +#### Scenario: Large collection +- **WHEN** Trail renders a report containing thousands of rows +- **THEN** the renderer performs bounded per-row formatting without one write syscall per field + +#### Scenario: Large patch +- **WHEN** Trail renders or pages a large patch +- **THEN** it streams or buffers within an explicit bound and does not duplicate the entire patch multiple times for styling and paging diff --git a/openspec/changes/hard-cutover-terminal-ux/tasks.md b/openspec/changes/hard-cutover-terminal-ux/tasks.md new file mode 100644 index 0000000..3a263cc --- /dev/null +++ b/openspec/changes/hard-cutover-terminal-ux/tasks.md @@ -0,0 +1,115 @@ +## 1. Baseline and Coverage Inventory + +- [x] 1.1 Inventory every persistent and transient stdout/stderr write reachable from Trail CLI commands, map each route to a command family, and record the intentionally raw schema/snippet/content exceptions. +- [x] 1.2 Capture representative typed report fixtures for clean, dirty, empty, success, warning, blocked, conflict, and failure states without treating existing human text as a compatibility baseline. +- [x] 1.3 Define the command-family output matrix covering human, plain, JSON, NDJSON, quiet, verbose, TTY, redirected, narrow, and wide behavior. +- [x] 1.4 Evaluate focused terminal-width, Unicode-width, wrapping, styling, paging, and PTY test dependencies against Rust/platform support, binary size, startup time, and maintenance cost; document the selected minimal set. +- [x] 1.5 Add baseline measurements for CLI startup, a 10,000-row collection render, and a large patch render so the cutover has explicit performance regression thresholds. + +## 2. Semantic Rendering Core + +- [x] 2.1 Create the new terminal UI module structure and a buffered `Renderer` entry point that returns I/O errors rather than printing directly. +- [x] 2.2 Implement the semantic `Document`, `Lead`, `Section`, `Metadata`, `Notice`, and `NextAction` models with importance and visibility metadata. +- [x] 2.3 Implement typed `Table`, `ChangeList`, `Checklist`, `Diagnostic`, and `Diff` blocks without embedding terminal styling in their values. +- [x] 2.4 Implement explicit user-facing label mappings for operation kinds, file changes, worktree states, lane/task states, readiness, approvals, gates, runs, and merge states; prohibit debug enum formatting in render adapters. +- [x] 2.5 Add human-mode style roles for lead, success, attention, blocked/failure, identifiers, commands, headings, and secondary metadata with text labels that remain authoritative without color. +- [x] 2.6 Add human Unicode glyphs and plain ASCII fallbacks for separators, arrows, ellipses, rules, and change bars. +- [x] 2.7 Implement user-content sanitization for cells, paragraphs, paths, messages, diagnostics, conflict text, and patch lines before renderer-owned styling is applied. +- [x] 2.8 Implement display-width measurement, wrapping, indentation, and truncation that account for Unicode width and renderer-owned style sequences. +- [x] 2.9 Implement responsive borderless tables with column priority, alignment, minimum/flexible width, wide/standard layouts, and narrow stacked fallback. +- [x] 2.10 Implement outcome-first document ordering, compact empty/clean states, explicit omitted-item notices, and single-primary-action enforcement. +- [x] 2.11 Implement change-list totals, `A/M/D/R/T` markers, rename presentation, addition/deletion counts, and proportional bars for human and plain modes. +- [x] 2.12 Implement checklist ordering that places blockers before warnings and renders stable textual state labels. +- [x] 2.13 Implement human and deterministic plain render backends, including RFC 3339 UTC timestamps and integer-millisecond durations in plain mode. +- [x] 2.14 Add semantic-document and renderer golden tests at representative narrow, standard, and wide widths with color/glyph policies enabled and disabled. + +## 3. CLI Render Context and Stream Behavior + +- [x] 3.1 Replace global output parsing with explicit `human`, `plain`, `json`, and `ndjson` formats and add parsing/error tests for each format. +- [x] 3.2 Replace `--no-color` with `--color auto|always|never`, honor `NO_COLOR` and `TERM=dumb` in `auto`, and add resolution tests. +- [x] 3.3 Add `--pager auto|always|never` and resolve pager eligibility from output format, command document policy, terminal state, and output size. +- [x] 3.4 Expand `RenderContext` to include verbosity, quiet mode, color/glyph policy, terminal width/height, stdout/stderr TTY state, pager policy, clock, and progress eligibility. +- [x] 3.5 Route persistent successful documents to stdout and diagnostics/progress to stderr through injected buffered writers. +- [x] 3.6 Implement clean broken-pipe handling for direct and paged output so an early downstream close does not produce a second diagnostic. +- [x] 3.7 Implement explicit pager integration for patches, transcripts, detailed history, conflict explanations, and detailed object/text/map views, including direct-output fallback and early-close handling. +- [x] 3.8 Implement one bounded transient progress line for eligible long-running commands and guarantee cleanup before final output. +- [x] 3.9 Implement NDJSON record emission for streaming/list commands and reject or document NDJSON use on commands that cannot produce a record stream. +- [x] 3.10 Add PTY and redirected-stream tests for color auto-detection, `TERM=dumb`, `NO_COLOR`, paging, progress suppression/cleanup, quiet mode, and broken pipes. + +## 4. Actionable Diagnostics + +- [x] 4.1 Add a diagnostic presentation mapping for every stable `trail::Error` code with concise summary, cause/consequence text, and a safe recovery action when determinable. +- [x] 4.2 Convert Clap parse failures to the shared human diagnostic style while preserving JSON error shape and exit status. +- [x] 4.3 Convert daemon, watchdog, sandbox, provider-receipt, and command-execution failures that reach users to the shared stream and diagnostic policy. +- [x] 4.4 Add reusable blocker/warning adapters so readiness, guardrails, doctor, conflicts, and agent diagnosis share diagnostic ordering and recovery presentation. +- [x] 4.5 Ensure force, bypass, destructive, or confirmation-sensitive alternatives are never primary and always describe their consequence. +- [x] 4.6 Add golden and semantic tests for every stable error category, unknown-cause fallback, multiple blockers, risky alternatives, JSON preservation, and stderr routing. + +## 5. Core Workspace and Configuration Cutover + +- [x] 5.1 Migrate init output to a completed-action lead, concise import summary, default branch, and verbose-only workspace/operation details. +- [x] 5.2 Migrate clean, tracked-dirty, and untracked-dirty status output to outcome-first change lists with one state-appropriate next action. +- [x] 5.3 Migrate record and watch output, including no-change, partial path, repeated record, and long-running watch/progress states. +- [x] 5.4 Migrate checkout output for dry-run, alternate output root, dirty-worktree recording, successful materialization, and protected failure states. +- [x] 5.5 Migrate config and ignore list/get/set/add/remove/check outputs using compact outcomes, key/value detail, and responsive collections. +- [x] 5.6 Migrate guardrail checks to decision-first checklists with approval evidence, ignored/denied path detail, blockers, and the safest next action. +- [x] 5.7 Add semantic and width-matrix golden tests for each migrated workspace/configuration state. + +## 6. History, Inspection, and Diff Cutover + +- [x] 6.1 Migrate timeline to a responsive borderless operation table with explicit user-facing kinds, deterministic plain timestamps, and narrow stacked records. +- [x] 6.2 Migrate show output for operation, message, ref, lane, and object variants using narrative/key-value layouts and verbose technical identities. +- [x] 6.3 Migrate history, code-from, and why output to progressive summaries, readable provenance, path/line relationships, and explicit empty states. +- [x] 6.4 Migrate object, root, text, map-range, and map-diff inspection with eligible paging, safe truncation notices, and explicit commands to reveal omitted content. +- [x] 6.5 Refactor diff rendering into semantic file summaries and per-file patch blocks while preserving patch prefixes and line-identity opt-in behavior. +- [x] 6.6 Add and test `--name-only` and `--name-status`; make `--stat` suppress patch bodies and make conflicting projection flags actionable input errors. +- [x] 6.7 Add explicit binary, opaque, missing-patch, and truncated diff notices plus sanitized control-character fixtures. +- [x] 6.8 Migrate ACP profile/install/doctor/session and transcript output, using tables for collections and paging only for long transcripts. +- [x] 6.9 Add semantic, golden width, color, plain, pager, and structured-format regression tests for the migrated review/inspection commands. + +## 7. Collaboration, Merge, and Maintenance Cutover + +- [x] 7.1 Migrate branch create/list/delete/rename and Git import/export/mapping output using concise outcomes and responsive collections. +- [x] 7.2 Migrate anchor and lease create/resolve/list/release/claim outputs with safe coordination status, ownership, expiry, and conflict guidance. +- [x] 7.3 Migrate merge dry-run/result output to decision-first summaries with changed paths, conflicts, and exact safe follow-up. +- [x] 7.4 Migrate merge queue add/list/run/explain/remove output to responsive queue tables, ordered blockers, pause reasons, and one primary action. +- [x] 7.5 Migrate conflict list/detail/explanation/resolve output to progressive path/line evidence, recommendations, safe recovery actions, and eligible paging. +- [x] 7.6 Migrate doctor and FSCK output to ordered checklists with compact healthy states and actionable failed checks. +- [x] 7.7 Migrate backup create/verify/restore, index rebuild, worktree index, and GC output with dry-run distinction, progress, summaries, and verbose integrity metadata. +- [x] 7.8 Add semantic and golden tests for success, empty, warning, blocked, conflict, dry-run, and destructive-confirmation states across collaboration and maintenance commands. + +## 8. Lane Workflow Cutover + +- [x] 8.1 Migrate lane spawn/list/detail/status outputs with user-facing lane state, base freshness, changed paths, workdir state, gates, and queued merges. +- [x] 8.2 Migrate lane contribution, review packet, gate history, readiness, refresh preview, and handoff to decision/checklist layouts with blockers before warnings. +- [x] 8.3 Migrate lane record, preview, rewind, watch, test/eval, workdir, file-read, sync, patch, remove, workspace checkpoint/space/exec/mount outputs. +- [x] 8.4 Migrate session start/current/list/detail/context/end output with responsive session tables and concise lifecycle outcomes. +- [x] 8.5 Migrate turn/message/event/start/detail/end output with progressive evidence and readable checkpoint selectors. +- [x] 8.6 Migrate run pause/resume/list/state output with explicit paused reason, approval dependency, and safe resume guidance. +- [x] 8.7 Migrate trace span start/end/list/summary/detail output with aligned timing/count data and deterministic plain durations. +- [x] 8.8 Migrate approval request/list/detail/decision output to checklist/state language with reviewer evidence and confirmation-sensitive actions. +- [x] 8.9 Migrate dependency environment, adapter environment, workspace cache, and related specialist lane reports discovered by the renderer inventory. +- [x] 8.10 Add semantic and width-matrix golden tests covering clean, dirty, stale, ready, blocked, pending approval, failed gate, paused, conflicted, and empty lane states. + +## 9. Agent Task UX Cutover + +- [x] 9.1 Split the monolithic agent renderer into focused home, review, evidence, validation, lifecycle, and sharing adapters while keeping shared task titles, selectors, risk, and action helpers centralized. +- [x] 9.2 Migrate setup, empty-task guidance, status, guide, inbox, board, stack, list, and next outputs to low-noise orientation layouts. +- [x] 9.3 Migrate dashboard, brief, summary, view, workdir, and review-flow outputs so each emphasizes one current state, one focus item, and one primary action. +- [x] 9.4 Migrate review-data, action palette, review plan, focus, review map, risk, confidence, and ready/can-land output to typed actions and ordered checklists. +- [x] 9.5 Migrate changes, delta, timeline, change-set, files, file, checkpoints, why, turn, diff, review, story, tools, and impact output to progressive review-oriented blocks. +- [x] 9.6 Migrate validation, test plan, diagnosis, compare, and multi-task stack output with consistent evidence, priority, overlap, and blocker presentation. +- [x] 9.7 Migrate new/start/run, mark-reviewed, archive/unarchive, apply/land, finish/ship, continue, undo/recovery, and related lifecycle mutations to completed/preview outcomes and confirmation-aware next actions. +- [x] 9.8 Migrate report, receipt, handoff, and PR output while preserving intentionally raw Markdown/snippet modes as documented exceptions. +- [x] 9.9 Add semantic and width-matrix golden tests covering no tasks, running, needs record, needs review, validation missing/failed/passed, blocked, conflicted, ready, applied, archived, overlap, and recovery states. + +## 10. Hard Cutoff and Documentation + +- [x] 10.1 Update every command handler to pass typed reports through the new render context/document adapters and bypass them only for JSON, NDJSON, and approved raw-content modes. +- [x] 10.2 Delete legacy human formatting helpers, direct ANSI patch styling, obsolete boolean `json/quiet/color` renderer signatures, and the removed `--no-color` argument. +- [x] 10.3 Add an automated source guard that rejects new direct `println!`, `eprintln!`, `print!`, and raw ANSI use in command render/handler modules outside approved writer/progress modules. +- [x] 10.4 Verify the renderer inventory has no unmigrated command route and no mixed legacy/new human output path. +- [x] 10.5 Update CLI help, generated completions, README walkthroughs, CLI reference pages, agent/lane examples, and screenshots/text fixtures to the new output contract. +- [x] 10.6 Add a changelog and migration note declaring the breaking human-output cutoff, removed `--no-color`, new format/color/pager flags, and requirement that automation use JSON or NDJSON. +- [x] 10.7 Run formatting, linting, unit, integration, PTY, platform-relevant, OpenSpec validation, and full workspace tests; record any intentionally deferred platform verification. +- [x] 10.8 Re-run startup/large-table/large-patch benchmarks and confirm rendering remains within the agreed regression thresholds before release. diff --git a/openspec/config.yaml b/openspec/config.yaml new file mode 100644 index 0000000..392946c --- /dev/null +++ b/openspec/config.yaml @@ -0,0 +1,20 @@ +schema: spec-driven + +# Project context (optional) +# This is shown to AI when creating artifacts. +# Add your tech stack, conventions, style guides, domain knowledge, etc. +# Example: +# context: | +# Tech stack: TypeScript, React, Node.js +# We use conventional commits +# Domain: e-commerce platform + +# Per-artifact rules (optional) +# Add custom rules for specific artifacts. +# Example: +# rules: +# proposal: +# - Keep proposals under 500 words +# - Always include a "Non-goals" section +# tasks: +# - Break tasks into chunks of max 2 hours diff --git a/plans/005-layered-lane-workspaces.md b/plans/005-layered-lane-workspaces.md index bf07d2c..89ee11a 100644 --- a/plans/005-layered-lane-workspaces.md +++ b/plans/005-layered-lane-workspaces.md @@ -86,20 +86,51 @@ checkpoint after the build records zero source paths. - Native macOS loopback NFS now runs equivalent real dependency acceptance cases. Two Node lanes share one 1,116-entry lodash/Prettier layer; mounted - package binaries and symlinks execute, while overwrite, `rm -rf - node_modules`, and `npm ci` in lane A leave lane B and the immutable layer - unchanged. A Cargo producer target can be published as an immutable seed for - lane B; the consumer reuses dependency artifacts without private copies, - `cargo clean` cannot alter the producer or seed, and both dependency - checkpoints record zero source paths. These tests also verify that immutable - backing files remain read-only while the NFS COW view exposes writable modes - required by the macOS client. + package binaries and symlinks execute, while lane-A copy-up leaves lane B and + the immutable layer unchanged. Direct `rm -rf node_modules && npm ci` remains + correct but took about 215 seconds because NFSv3 has no recursive-delete RPC. + The supported `trail deps sync` path now bulk-removes private dependency + state and subtree whiteouts off-mount before rebinding the cached layer; the + same real fixture completed replacement in 978 ms (about 220x faster), left + both generated uppers at zero bytes, and recorded zero source paths. A Cargo + producer target can be published as an immutable seed for lane B; the + consumer reuses dependency artifacts without private copies, and `cargo + clean` cannot alter the producer or seed. These tests also verify that + immutable backing files remain read-only while the NFS COW view exposes + writable modes required by the macOS client. - The million-path/twenty-view acceptance test also passes inside Linux: each view starts with one indexed path, all 20 views are created in 91 ms, and unchanged views consume 0 exclusive physical bytes. +- A gated native macOS framework benchmark uses current Next.js 16.2.10, + React 19.2.7, Vite 8.1.4, and `@vitejs/plugin-react` 6.0.3. The Next layer + contains 10,558 entries and 305,041,668 logical bytes: cold sync took 51.2 + seconds, cache-hit attach 27.1 seconds, two mounts 74 ms, full private-tree + materialization 1.9 seconds, and bulk replacement 29.4 seconds. Default + Turbopack did not finish within its 120-second budget. The Vite layer contains + 410 entries and 36,878,763 logical bytes: cold sync took 6.5 seconds, + cache-hit attach 3.2 seconds, two mounts 75 ms, its first CLI probe 53.6 + seconds, production build 7.2 seconds, private-tree materialization 66 ms, + and bulk replacement 3.2 seconds. Both replacement cases preserve lane-B and + immutable-layer hashes; Next returns both dependency uppers to zero bytes, + while Vite retains only its lane-A `dist` output. The reproducible entry point + is `scripts/verify-macos-nfs-framework-layers.sh` (optional argument `next` + or `vite`). +- Cache-hit publication and attachment now validate a bounded durable seal containing + the exact manifest object, layer summary, and platform filesystem identity instead of + hashing every immutable entry. A missing, oversized, malformed, or stale seal falls + back to one full scan and self-heals; explicit `cache verify`, doctor, and readiness + remain full-content checks. The focused regression proves repeated warm reuse performs + zero recursive scans and that root replacement invalidates the seal. The remaining + macOS performance work is lower-aware NFS caching/coherency instead of global + `noac,actimeo=0`. - Native NFS exec/gate/checkpoint/recovery verifies exact source-root, environment-key, layer-ID, and stale-gate readiness behavior with no hosted service. +- macOS NFS mounts request synchronous writes. A parallel regression exposed the + platform's default `nosync` behavior returning before a queued WRITE reached the + userspace server, allowing immediate unmount/remount to observe only the preceding + truncate. Eight concurrent real mount/write/sync/remount flows preserve their private + bytes with the synchronous mount contract. - External process-kill tests stop checkpoint and cache-publish helpers after every durable phase: source sync, ref advance, lane-head update, clean marker, staging sync, publish marker, atomic rename, and ready-state update. diff --git a/plans/006-universal-lane-environments.md b/plans/006-universal-lane-environments.md new file mode 100644 index 0000000..d12dda0 --- /dev/null +++ b/plans/006-universal-lane-environments.md @@ -0,0 +1,638 @@ +# Plan 006: Universal Lane Environments + +Status: IN PROGRESS + +Priority: P0 + +Size: XXL + +Depends on: Plan 005 layered lane workspace semantic core + +Design: + +- [Universal Lane Environments](../docs/design/universal-lane-environments.md) +- [Environment Adapter and Specification Contract](../docs/design/environment-adapter-contract.md) +- [Layered Lane Workspaces](../docs/design/layered-lane-workspaces.md) + +## Outcome + +Trail can discover, plan, materialize, attach, execute, explain, verify, and retire a +complete reproducible lane environment. One typed graph composes toolchains, +dependencies, generated artifacts, private build state, disposable scratch, caches, +secrets, OCI images, and runtime services across multiple ecosystems. + +The implementation shares immutable physical content while every lane receives isolated +writable behavior. A lane generation is atomic, recoverable, and referenced by agent +execution provenance. + +## Implemented foundation + +- Schema v5 separates logical component state from versioned adapter identity and + backfills existing Node dependency rows without copying layer content. +- A host-owned adapter executor materializes declared inputs in private staging, + single-flights immutable publication, preserves predecessor state on failure, and + activates bindings through transactional metadata updates. +- Adapter commands start with a cleared environment, isolated HOME/tmp directories, + declared cache variables, and cache keys containing implementation/tool executable + identities; read-only status never executes adapter tools. +- Built-in `trail/node@1` behavior now runs through that executor. +- Experimental `trail/cargo-target-seed@1` performs locked, offline builds keyed by the + complete source root, Cargo/rustc identity, host target, platform, and output policy. +- `trail/go-vendor@1` proves the host is not Node/Cargo-specific: it stages a pinned + single-module source root, runs `go mod vendor` with isolated module/build caches, and + reuses the immutable vendor tree across lanes. +- Planned executable identities are revalidated immediately before launch while the + selected shim path is preserved for dispatchers such as rustup. +- `trail env status|sync`, `/v1/lanes/{lane}/environment`, and + `trail.env_status|env_sync` expose initial CLI, HTTP/OpenAPI, and MCP parity. +- Side-effect-free `env discover` and atomic `env sync-all` support nested and polyglot + roots. All builds finish before a batch savepoint updates mounts and one generation. +- Built-in adapters publish static registration metadata, so discovery markers are no + longer hard-coded in the scanner. `env adapters`, HTTP/OpenAPI, MCP, and the Rust API + expose the same side-effect-free adapter catalog. +- Experimental `trail/command@1` parses a strict repository specification with bounded + local includes and versioned profile inheritance, canonically expands `{root}` plus + declared exact/glob inputs, emits a normal host plan, exposes grants through `env plan`, + and executes through macOS sandbox-exec, a Linux Landlock-plus-seccomp helper, or a + capability-free Windows AppContainer plus one-process kill-on-close Job Object. The + native policies deny undeclared writes, host reads, other executables, network, shell, + scripts, secrets, and child processes. Failed Windows setup terminates the suspended + process before returning. Missing kernel support and unsupported platforms fail closed. +- Active and retired environment generations persist source root, component keys, + layers, mounts, and predecessor identity; agent execution receives the active + generation ID and cache GC preserves generation-referenced layers. +- Adapter-owned bindings dynamically classify nonstandard paths such as `.venv`, so + their writes remain lane-private and are excluded from source checkpoints. +- Layer replacement writes a durable activation intent and atomically renames the old + private upper. Recovery restores it when SQLite did not commit or finishes cleanup + when the replacement binding committed. +- Synchronization attempts persist process/start-token ownership. Open-time recovery + preserves a usable predecessor as stale, fails a predecessor-less first build, and + recognizes activation that committed before the process exited. +- Schema v6 and the normalized host plan support up to 32 named outputs per component. + One action publishes one composite immutable layer; output subpaths receive independent + bindings and lane-private uppers, activate atomically, persist in generation provenance, + and retire removed uppers through crash-recoverable unbind intents. +- Schema v7 makes output storage policy explicit. `writable_private` outputs have a + durable component/output/key binding identity, a nullable layer ID, canonical key + provenance, and bytes only in the owning lane's generated upper. Activation can seed + or preserve compatible state, never manufactures an empty shared layer, and uses the + same filesystem-intent/SQLite recovery protocol as immutable attachment. CLI, + HTTP/OpenAPI, MCP, plan, sync, generation, gate, and mounted-view reports distinguish + private outputs from immutable-seeded ones. Recipes and v1/v2 plugins can declare either + policy; mixed policies within one component remain rejected until heterogeneous action + publication is implemented. +- Experimental `trail/cmake-build@1` uses that private-output contract to own a build + tree without staging `CMakeCache.txt` under a false absolute path. Synchronization + provisions the directory; configure/build runs inside the mounted lane. A real + Linux/FUSE fixture configures and builds two lanes, verifies mounted cache paths, + cleans one lane, proves the other remains intact, and confirms zero shared layers. + Equivalent macOS/NFS and Windows/Dokan evidence is wired into native workflows. +- `trail-environment-adapter-sdk` defines a bounded framed-CBOR v1 protocol. Explicitly + installed local packages are content-addressed, append-only activated/tombstoned, + digest-checked before every use, and sandboxed without repository/database/network or + child-process authority. Plugins participate in catalog, discovery, plan, sync, and + sync-all through the normalized host plan. Readiness replans built-ins, recipes, and + plugins from the current pinned root, while mount-shadow validation scans the source + once for the complete plan set. Conformance covers reuse, copy-up, stale refresh, + timeout, memory overrun, crash, oversized/malformed output, child execution, + tampering, repair, and removal. +- Immutable layer publication writes a bounded verification seal over the manifest + object, layer summary, and filesystem directory identity. Routine cache reuse and + attachment validate the seal without walking the artifact tree; legacy or invalid + seals perform one full verification and self-heal. Explicit verification and + readiness continue to hash every entry, and GC removes the seal with its layer. +- External packages support detached Ed25519 publisher authentication. A read-only + inspection command reports the canonical payload digest; workspace-local publisher + keys use a content-derived ID and append-only trust/revocation records. Installation + stores the exact authenticated executable bytes, signed distributions include their + attestation in identity, revocation fails active packages closed, and catalog/install + reports distinguish built-ins, unsigned local packages, and publisher-authenticated + experimental packages. +- New immutable manifests retain the canonical host-owned component key, while legacy + manifests remain readable. Stale refresh diffs previous and current input, tool, + platform, architecture, portability, adapter, and strategy edges without exposing + values. Paginated `env explain`, HTTP, and MCP reports return the complete available + change set and explicitly identify legacy/missing provenance. +- Schema v8 persists current component dependency edges and immutable generation edge + keys. Recipes and v1/v2 plugins declare stable component IDs; the host rejects missing, + duplicate, self, and cyclic edges before execution, finalizes a deterministic + topological order, and injects direct upstream component keys into downstream + canonical identity. `sync-all` prepares that order and activates one generation, + readiness propagates desired upstream-key changes through every descendant, and + `env explain` names the exact `dependency:` edge. A single sync requires + ready predecessors, while upstream-only replacement immediately marks mounted + descendants stale and preserves their historical generation edge keys. Tests cover a + 10,000-node non-recursive chain, full cycle diagnostics, missing edges, migration, + rollback, and native macOS shared/private execution. +- Repository-wide `sync-all` now retires components absent from the desired graph in the + same filesystem-intent/SQLite activation. Removed immutable mounts and private uppers + disappear atomically, state rows retire, and removing the last component records an + inspectable empty generation; scoped discovery does not delete out-of-scope state. +- `env graph` exposes the validated desired DAG through one Rust report shared by CLI, + HTTP/OpenAPI, and MCP. Nodes are emitted in deterministic topological order with + roots, adapters, canonical keys, dependencies, and output ownership; edges name the + exact upstream key and their current `ordering_invalidation` semantics. The surface is + read-only and publishes no layer or state. CLI, HTTP, and MCP paginate by target node + with total counts and a stable next offset, while the Rust API retains a full-graph + method for in-process consumers. +- Full recipe graphs load the specification twice (discovery and batch planning), cache + resolved executable identities, and validate target, mount, and source-shadow prefixes + with ordered indexes rather than quadratic scans. A 1,000-component recipe chain + completes through the public graph path while asserting exactly two parses; the host + resolver separately verifies 10,000 nodes without recursion. +- The public plugin SDK provides deterministic validating builders for plans, commands, + outputs, discovery responses, and protocol-matched responses. The helpers reject the + common structural authoring failures locally while preserving the exact v1 wire shape. +- External packages negotiate `trail.environment-adapter/v1` or `/v2`; missing protocol + metadata preserves v1 canonical package bytes and signatures. V2 adds a separate + `AdapterPlanV2` plus typed staging/mounted actions without changing v1 Rust or wire + types. Mounted plugin actions run an authenticated executable copy in the native + sandbox against the pinned ephemeral candidate view, read only exact declared inputs, + write only writable-private outputs plus isolated HOME/tmp, and retain the predecessor + after nonzero exit or undeclared source access. Catalog, CLI JSON/text, HTTP/OpenAPI, + and MCP expose package protocols. Native macOS/NFS conformance exercises two distinct + final paths, compatible private-state preservation, failure rollback, and declared + versus undeclared source access; Linux/FUSE and Windows/Dokan jobs run the same fixture. + The fixture also kills Trail during an active mounted command, requires the + parent-death watchdog to terminate the sandbox helper, reopens the backend, preserves + the predecessor, and removes the abandoned candidate. macOS NFS recovery now uses a + dedicated `nfs-mount.json` backend record and force-detaches a known-dead server before + any potentially blocking mountpoint metadata access. +- Trusted built-ins and authorized protocol-v2 plugins can declare keyed + `mounted_initialization` actions for path-sensitive + writable-private outputs. The host runs them at the final lane mountpoint using a + disposable upper layout, rejects source or unrelated writes, copies only declared + outputs into activation staging, and leaves the predecessor generation untouched on + failure or kill. Python `.venv` creation and nested multi-component `sync-all` exercise + this path on FUSE, NFS, and Dokan CI jobs. +- Schema v9 and the normalized host graph implement typed `build_requires`, + `runtime_requires`, `binds_after`, and `invalidates_with` edges. Legacy recipe/plugin + dependencies remain key-compatible `build_requires` edges. Identity edges enter the + canonical component key and propagate staleness; runtime/order-only edges retain exact + upstream generation keys and advance atomically without rebuilding the consumer. + Repository recipes and protocol-v2 SDK builders author the same semantics, while + graph, plan, generation, CLI/JSON, HTTP/OpenAPI, and MCP reports expose them. Migration + backfills schema-v8 active and historic edges without losing upstream keys. +- Schema v10 replaces built-in adapters' implicit global tool-home paths with + content-derived, workspace-scoped, performance-only cache namespaces. Node package + stores, Cargo registry/Git state, sccache, and Go module/build caches declare protocol, + access strategy, and complete compatibility dimensions. Commands hold crash-recoverable + live leases; `host_exclusive` namespaces serialize users in deterministic order, while + certified `tool_concurrent` caches retain native parallelism. Generation, graph, plan, + CLI/JSON, HTTP/OpenAPI, and MCP provenance expose exact namespace IDs. Cache GC uses an + atomic maintenance barrier, skips live users, supports dry-run, and reclaims inactive + namespace trees without affecting component readiness or immutable artifacts. +- Protocol-v2 external adapters can declare performance-only content-store, + compiler-cache, or locked-index namespaces for their staging action. The host adds the + exact authenticated distribution/protocol/platform identity, injects only declared + relative environment bindings, holds crash-recoverable namespace leases, and grants + the cache root through the macOS/Linux/Windows native sandbox. External access is + forced to `host_exclusive`; mounted actions receive no cache access and + `tool_concurrent` fails closed pending independent certification. Native macOS/NFS and + Linux/FUSE plugin conformance uses distinct component keys across two lanes, proves + one namespace is reused, and denies a parent-directory escape while preserving the + predecessor generation. +- Schema v13 introduces metadata-only external components and active/historical + external-artifact provenance. Built-in `trail/oci-image@1` discovers strict + `trail.oci.toml` declarations, keys digest-pinned OCI references plus platform, and + activates them atomically across lanes without commands, mounts, fake directories, or + GC ownership. Protocol-v2 SDK builders expose the same external-artifact contract and + reject plans that mix provider-owned identities with actions, caches, or filesystem + outputs. Graph, plan, generation, CLI/JSON, HTTP/OpenAPI, and MCP surfaces carry the + exact reference, digest, platform, provider, and cleanup owner. + +Still required for this plan: OCI tag resolution and registry verification/materialization, +runtime resource allocation and health edges, late secret resolution/injection, +heterogeneous component actions, complete CMake presets/toolchains/cache/install-prefix +support, observed native Windows +release-gate evidence, signed remote plugin catalogs, WASI packaging, independent +certification, tool-concurrent external-plugin cache certification, additional +built-ins, and the full cross-platform release gate. + +## Why this follows Plan 005 + +Plan 005 proves the filesystem substrate and initial dependency workflows. Plan 006 +generalizes those mechanisms without replacing them: + +- workspace layer keys become environment component keys; +- layer manifests become typed artifact manifests; +- dependency mount records become generation bindings; +- Node, Cargo, Go, CMake, and Python dependency/build state become built-in environment + adapters with ecosystem-specific sharing policies; +- lane safety, readiness, and operation records remain authoritative. + +Plan 006 must not hide a backend semantic gap behind an adapter. Linux overlayfs/FUSE, +macOS NFS, and fallback materialization must first agree on copy-up, whiteout, bulk +replacement, rename, link, and crash behavior for every binding policy they advertise. + +## Scope + +In scope: + +- environment graph and generation domain model; +- policy classification and sharing enforcement; +- adapter normalization and declarative repository schema; +- artifact build, validation, publication, verification, and collection; +- private state and cache namespaces; +- secret references and runtime injection; +- external immutable artifacts and private runtime services; +- CLI, Rust API, HTTP, MCP, operations, readiness, and provenance; +- built-in Node, Cargo, Go, CMake, Python, command, secret, and OCI/service adapters; +- plugin SDK and conformance kit; +- cross-platform scale, security, and crash testing. + +Out of scope for the first stable release: + +- provisioning arbitrary cloud infrastructure; +- promising bit-for-bit rebuilds for undeclared or non-hermetic tools; +- globally shared arbitrary writable directories; +- transparent interception of every command run outside Trail; +- removing existing `trail lane deps` compatibility commands. + +## Architectural constraints + +1. Trail owns fingerprinting, action execution, staging, validation, publication, + attachment, persistence, and recovery. +2. Adapters return structured declarations and observations; they do not mutate shared + state. +3. Every output selects an explicit policy. +4. Secret values are never persisted or hashed. +5. Routine attach does not recursively verify an unchanged artifact. +6. A generation switch is atomic and requires a quiescent lane or explicit managed + restart policy. +7. Old generations remain inspectable and recoverable through retention policy. +8. CLI, Rust, HTTP, and MCP surfaces render common report types. +9. Environment operations do not commit, branch, merge, or push Git state. +10. Unknown capabilities or policy semantics fail closed. + +## Work packages + +### 006.1 Domain vocabulary and compatibility shell + +Deliver: + +- `EnvironmentSpec`, `EnvironmentComponent`, `EnvironmentEdge`, `EnvironmentArtifact`, + `EnvironmentGeneration`, and `EnvironmentBinding` identifiers and records; +- storage-policy, portability, lifecycle, binding, and input-role enums; +- common structured diagnostic and stale-reason types; +- `trail env` command shell and `trail lane deps` compatibility mapping; +- feature/config schema version and migration boundary. + +Acceptance: + +- old layer and dependency records can be projected into an initial generation without + copying artifact content; +- every existing dependency report has an equivalent environment report field; +- unknown enum values are rejected at mutation boundaries; +- unit tests cover identifier parsing, canonical serialization, and migration rollback. + +### 006.2 Graph resolver and deterministic fingerprints + +Deliver: + +- configuration precedence and field provenance; +- side-effect-free adapter discovery; +- graph validation, cycle detection, and output-ownership collision detection; +- canonical fingerprint engine owned by the host; +- typed build, runtime, ordering, and invalidation edges; +- `env discover`, `env graph`, `env plan`, and `env explain` reports; +- exact staleness propagation and explanation. + +Acceptance: + +- shuffled maps, glob enumeration, locale, and process environment do not change keys; +- irrelevant edits do not invalidate a component; +- every invalidation identifies the input and propagation edge responsible; +- cycles and overlapping binding targets fail before build actions; +- golden tests are stable across Rust versions and supported platforms. + +### 006.3 Typed artifacts, verification, and generations + +Deliver: + +- typed artifact manifest and lifecycle state machine; +- private staging, single-flight build leases, validation, atomic publication, and + quarantine; +- attach, sample, and full verification tiers; +- atomic environment-generation transaction and activation journal; +- predecessor retention and rollback; +- garbage-collection roots and retained-by explanations; +- logical versus unique physical byte accounting. + +Acceptance: + +- concurrent identical builds publish once; +- interruption at every state boundary exposes no partial artifact; +- activation failure preserves or restores the predecessor generation; +- injected corruption is found by full verification and excluded from new generations; +- warm attach performs no full tree walk; +- active and retained generation artifacts survive garbage collection. + +### 006.4 Declarative command adapter and policy engine + +Deliver: + +- `.trail/environment.toml` parser, includes, profiles, and canonical expansion; +- generic command component with argument vectors; +- host-owned sandboxed action executor; +- filesystem, process, shell, network, script, host-path, runtime, and secret capability + policy; +- declared input/output containment and file-type validation; +- plan-time capability presentation and execution-time observation comparison. + +Acceptance: + +- a user-defined code generator produces a reusable verified artifact; +- undeclared filesystem writes, network access, shell execution, and host paths fail; +- cancellation leaves source, active generation, and shared storage intact; +- plan and provenance list the same granted capabilities; +- path traversal, malicious archives, and symlink escapes fail conformance fixtures. + +### 006.5 Node and framework migration + +Deliver: + +- Node toolchain and package-manager built-in adapter; +- npm, pnpm, and supported package-store cache strategies; +- dependency tree immutable/shared versus seed/private policy selection; +- Next.js and Vite profiles with private build state and dev runtime declarations; +- migration from current dependency layer records; +- real application scale fixtures. + +Acceptance: + +- multiple lanes share dependency content while package-manager mutation and deletion stay + isolated; +- `npm ci`, pnpm install, Next build/dev, and Vite build/dev pass on advertised backends; +- lockfile, Node ABI, package-manager version, install flags, and lifecycle-script policy + cause precise invalidation; +- `.next`, Vite cache, dev ports, and processes are lane-private; +- warm attach avoids copying or rehashing the full dependency tree; +- bulk `node_modules` replacement provides bounded-memory progress and cancellation. + +### 006.6 Cargo and Rust migration + +Deliver: + +- Rust toolchain component; +- Cargo registry and Git cache protocol; +- private `target` binding with optional immutable seed; +- sccache integration through the compiler-cache contract; +- build-script/proc-macro environment and network provenance; +- migration from current Cargo dependency logic. + +Acceptance: + +- registry content and compiler cache safely serve concurrent lanes; +- one lane's `cargo clean`, profile change, or target mutation is invisible to another; +- `Cargo.lock`, rustc/toolchain, features, profile, target, relevant `RUSTFLAGS`, and + build-script policy invalidate correctly; +- native and cross-compilation artifacts receive conservative portability; +- real workspace tests cover large registry and target trees. + +### 006.7 CMake and native build adapter + +Status: FOUNDATION IMPLEMENTED. `trail/cmake-build@1` now provides discovery, +deterministic CMake executable/host identity, a layer-free `writable_private` build +binding, atomic generation activation, and real two-lane Linux/FUSE isolation. Configure +is intentionally deferred to the mounted lane to preserve correct absolute paths. +Preset/toolchain/package-manager/compiler-cache/install-prefix semantics and the full +native platform matrix remain in progress. + +Deliver: + +- CMake configure/build graph with compiler, linker, SDK/sysroot, toolchain-file, + generator, preset, vcpkg, and Conan inputs; +- private configure/build tree; +- ccache protocol binding; +- optional immutable install-prefix publication with relocation validation; +- Ninja and Make generator support matrix. + +Acceptance: + +- absolute-path-bearing CMake caches remain host/lane-private; +- compiler, SDK, generator, preset, and toolchain changes invalidate precisely; +- concurrent lanes share ccache only through certified semantics; +- an explicitly installed relocatable artifact can be published and reused read-only; +- configure, incremental compile, clean, and test scratch remain isolated. + +### 006.8 Secrets, external artifacts, and runtime resources + +Deliver: + +- secret provider reference abstraction and injection through environment, file, or file + descriptor; +- structured redaction and secret canary scanner; +- OCI image digest resolver and external immutable artifact record; +- BuildKit cache policy integration; +- lane-private process/container, network, volume, socket, and port allocation; +- service health, restart, reuse, cleanup ownership, and recovery; +- externally managed service references. + +Acceptance: + +- no secret canary appears in database, keys, manifests, logs, transcripts, checkpoints, + exports, or diagnostics; +- tag resolution records an image digest and platform; +- two lanes receive isolated container names, networks, volumes, and port allocations; +- Trail never deletes an externally owned resource; +- service readiness and cleanup recover after daemon or client crashes; +- build secrets never enter OCI layers or cache identities. + +### 006.9 Adapter SDK and plugin isolation + +Deliver: + +- versioned serializable adapter protocol; +- built-in and subprocess/WASI host adapters sharing normalized types; +- capability negotiation and fail-closed version handling; +- SDK helpers for inputs, tools, outputs, validation, diagnostics, and progress; +- fixture host, golden plans, secret canaries, and filesystem conformance kit; +- adapter certification metadata. + +Acceptance: + +- a third-party adapter can discover and build without database or mount knowledge; +- plugin timeout, crash, oversized output, and malformed response cannot corrupt state; +- adapters cannot spawn processes or access network directly through the planning API; +- contract-major coexistence keeps old generations inspectable; +- certification tier and supported platform matrix appear in status. + +### 006.10 Surface integration and readiness + +Deliver: + +- all environment reports in Rust API; +- CLI text/JSON, HTTP/OpenAPI, and MCP parity; +- Trail operations, progress, cancellation, retry, and transcript links; +- lane readiness environment dimension; +- `env exec` execution envelope and generation provenance; +- audit and export behavior under redaction policy. + +Acceptance: + +- report fixtures compare equivalent Rust, CLI JSON, HTTP, and MCP structures; +- every long action returns or links to an operation; +- readiness distinguishes artifact staleness, service health, secret availability, + workspace safety, and Git conflict state; +- command history references the exact environment generation; +- environment sync has no implicit Git mutation. + +### 006.11 Cross-platform scale, fault, and security gate + +Deliver: + +- shared semantic suite for Linux overlayfs, Linux FUSE, macOS NFS, and materialized-copy + fallback; +- real Next.js/Vite, Rust workspace, CMake, and OCI fixtures; +- cold/warm, single/multi-lane, metadata-heavy, and disk-pressure benchmarks; +- crash injection at build, validation, publication, binding, service, and GC boundaries; +- threat model and external security review checklist; +- performance budgets and regression dashboard data. + +Acceptance: + +- every advertised backend passes copy-up, whiteout, bulk replacement, rename, link, + metadata, concurrent-reader, and crash tests; +- four or more concurrent lanes demonstrate shared physical content and isolated private + mutations; +- no benchmark uses only a tiny synthetic dependency tree as ecosystem evidence; +- routine status and attach remain bounded by graph/manifest work rather than artifact + tree size; +- forced cancellation and host reboot recover to a valid generation; +- disk-pressure cleanup never removes active private state or referenced artifacts. + +## Dependency graph + +```text +005 semantic backend + | + v +006.1 domain + compatibility + | + v +006.2 graph + fingerprints + | + v +006.3 artifacts + generations + | + +-----------> 006.4 command + policy + | | + | +-----------+-----------+ + | v v v + | 006.5 Node 006.6 Cargo 006.7 CMake + | \ | / + | +----------+----------+ + | | + +--------------------> 006.8 runtime/secrets + | + 006.9 SDK + | + 006.10 surfaces + | + 006.11 release gate +``` + +Work packages may overlap after their required shared model is stable, but no ecosystem +adapter may bypass the central artifact, policy, or generation lifecycle to ship early. + +## Verification matrix + +| Scenario | Linux overlayfs | Linux FUSE | macOS NFS | Copy fallback | +| --- | --- | --- | --- | --- | +| Immutable read-only tree | Required | Required | Required | Required | +| Immutable seed/private COW | Required | Required | Required | Semantically required | +| Nested whiteout and bulk replacement | Required | Required | Required | Required | +| Node install/build/dev | Required | Required | Required | Required | +| Cargo build/clean/test | Required | Required | Required | Required | +| CMake configure/build/clean | Required | Required | Required | Required | +| Python mounted `.venv` init/failure/kill | Required | Required | Required | N/A to layered mounts | +| OCI runtime | Required when engine available | Required when engine available | Required when engine available | N/A to filesystem | +| Crash recovery | Required | Required | Required | Required | +| Secret canary | Required | Required | Required | Required | + +Every result records operating system, architecture, filesystem, backend version, tool +versions, fixture revision, logical file/byte counts, unique physical bytes, cold/warm +state, and verification tier. + +## Performance budgets + +Budgets should be fixed after baseline measurement, but release gates enforce these +relationships from the start: + +- unchanged warm plan is proportional to graph and declared input size, not artifact + tree size; +- unchanged attach performs only attach-tier checks; +- a shared immutable artifact adds near-zero physical bytes for a second lane before + copy-up; +- copy-up cost tracks bytes actually modified; +- identical concurrent builds execute one producer action; +- metadata-heavy replacement has visible progress, bounded memory, and cancellation; +- status never blocks behind a full integrity audit unless the user requested it. + +## Rollout + +1. Land dormant domain types and report formats behind an experimental feature. +2. Import existing Node/Cargo workspace layers into generated environment records. +3. Add `trail env` as an opt-in command family while retaining dependency commands. +4. Enable generic recipes for trusted repositories with denied-by-default capabilities. +5. Graduate Node and Cargo after scale/backend parity. +6. Add CMake and OCI/runtime support. +7. Open the adapter SDK at `experimental` certification. +8. Make environment readiness part of default lane readiness only after false-positive + and latency budgets pass. +9. Deprecate dependency-only configuration fields after at least one compatibility + release; do not remove aliases until usage telemetry and migration tooling agree. + +## Data migration and rollback + +- Database changes are additive until environment behavior reaches parity. +- Migration records original layer IDs and creates new IDs without rewriting manifests. +- A feature flag chooses the legacy dependency path or environment path per repository. +- Rolling back the binary leaves legacy records usable and ignores new tables safely. +- A generation created by a newer contract remains inspectable even when it cannot be + activated by an older host. +- Artifact storage layout changes use dual-read/single-write before collection of the + old format. + +## Operational metrics + +Track at minimum: + +- plan/sync/attach/verify latency percentiles; +- graph node count and stale-reason distribution; +- build single-flight waiters and duplicate-build rate; +- artifact logical, unique physical, retained, and reclaimable bytes; +- private copy-up bytes and whiteout counts per lane; +- cache hit/miss/corruption/eviction counts; +- activation rollback and crash-recovery counts; +- secret-provider and service-health failure rates; +- backend-specific filesystem operation latency; +- readiness blocks by environment, lane safety, task, and Git reason. + +Metrics and diagnostics must not include repository source, secret values, private URLs, +or command arguments classified as sensitive. + +## Stop conditions + +Stop rollout and keep the feature experimental if any of the following remains true: + +- a lane can observe another lane's private mutation; +- partial build output can become shared or attached; +- a secret can persist outside its approved injection lifetime; +- an adapter can bypass host capability enforcement; +- generation recovery can strand a lane without a valid predecessor; +- backend behavior differs without capability negotiation; +- routine attach requires a recursive scan of large unchanged trees; +- external cleanup ownership is ambiguous; +- environment sync mutates Git state implicitly; +- API surfaces disagree about readiness, generation, or stale reasons. + +## Definition of done + +Plan 006 is complete when the architecture acceptance criteria and adapter contract +acceptance criteria pass, the full verification matrix is recorded for supported +platforms, migration and rollback are exercised on existing repositories, and the +feature is usable for daily multi-agent Node, Rust, native CMake, and OCI-backed +workflows without copying entire workdirs or sharing unsafe writable state. diff --git a/plans/007-native-agent-hooks-and-acp.md b/plans/007-native-agent-hooks-and-acp.md new file mode 100644 index 0000000..a23d354 --- /dev/null +++ b/plans/007-native-agent-hooks-and-acp.md @@ -0,0 +1,444 @@ +# Plan 007: Native Agent Hooks and ACP Integration + +Status: DONE + +Priority: P0 + +Size: XXL + +Depends on: existing lane activity model and ACP relay + +Design: + +- [Native Agent Hooks and ACP Integration](../docs/design/native-agent-hooks-and-acp.md) + +## Outcome + +Trail records agent sessions through native provider hooks, ACP, or both without +duplicating lifecycle records. Codex, Claude Code, Pi, OpenCode, Cursor, Gemini CLI, +GitHub Copilot CLI, and Grok Build share one durable capture coordinator while keeping +provider-specific installation and parsing in narrow adapters. + +The delivered system preserves provider-native evidence, creates exact turn and session +change manifests, correlates events to managed runs and Git history, survives retries and +crashes, and exposes the same state through Rust, CLI, HTTP/OpenAPI, and MCP surfaces. + +## Execution rules + +1. Read the design and this ledger before editing. +2. Keep provider adapters passive: they discover, install, parse, locate artifacts, and + render responses; the shared coordinator owns lifecycle and checkpoint policy. +3. Make every ingress receipt durable and idempotent before semantic dispatch. +4. Treat ACP and native hooks as transports over one domain model. +5. Preserve canonical provider transcripts or exports whenever available. +6. Fail open for recording failures unless the user explicitly chooses strict capture. +7. Never overwrite or remove configuration Trail cannot prove it owns. +8. Update task status only with the acceptance evidence named below. + +Status values are `TODO`, `IN PROGRESS`, `DONE`, `BLOCKED — reason`, and +`REJECTED — rationale`. + +## Milestones + +| ID | Deliverable | Depends on | Status | +| --- | --- | --- | --- | +| NAH-000 | Baseline, drift inventory, and task ledger | - | DONE | +| NAH-100 | Shared lifecycle contract and pure state machine | NAH-000 | DONE | +| NAH-200 | Durable capture schema and repository APIs | NAH-100 | DONE | +| NAH-300 | Receipt ingress, replay, runs, ownership, and recovery | NAH-200 | DONE | +| NAH-400 | Safe installation framework and adapter manifests | NAH-100 | DONE | +| NAH-500 | Eight production provider adapters and fixtures | NAH-300, NAH-400 | DONE | +| NAH-600 | Transcript, artifact, evidence, and checkpoint pipeline | NAH-300 | DONE | +| NAH-700 | Provenance, attestations, learnings, Git links, export | NAH-600 | DONE | +| NAH-800 | CLI, Rust, HTTP/OpenAPI, MCP, and diagnostics parity | NAH-300, NAH-500 | DONE | +| NAH-900 | Security, recovery, performance, compatibility, docs | all prior | DONE | + +## NAH-000: Baseline and delivery map + +- [x] NAH-001 Inspect Trail's ACP relay, lane activity storage, CLI, HTTP, MCP, schema, + and test conventions. +- [x] NAH-002 Clone and inspect Entire for adapter inversion, native transcript capture, + safe hook installation, snapshot/delta behavior, and Git association ideas. +- [x] NAH-003 Clone and inspect Atomic for its lifecycle state machine, managed-run + correlation, evidence manifests, causal graph, attestations, and portable export. +- [x] NAH-004 Write the comprehensive design and link it from the documentation index. +- [x] NAH-005 Create this requirement-to-evidence implementation ledger. + +Acceptance evidence: + +- the design names every shared component, provider, storage table, public surface, + recovery path, rollout phase, and release criterion; +- the reference repositories remain nested, clean clones and are not linked into the + Trail build graph; +- every later task has an explicit dependency and test obligation. + +## NAH-100: Shared lifecycle contract + +### Domain model + +- [x] NAH-101 Add versioned normalized event envelopes, provider/transport identity, + native correlation identifiers, confidence, usage, content references, and bounded + provider payload metadata. +- [x] NAH-102 Add the complete normalized vocabulary for sessions, turns, messages, + plans, tools, approvals, subagents, compaction, usage/model changes, workspace + changes, context injection, and diagnostics. +- [x] NAH-103 Add strict identifier, timestamp, size, and forward-version validation at + mutation boundaries while retaining unknown provider events as inert evidence. +- [x] NAH-104 Define capture states, bounded transition context, ordered side-effect + actions, and terminal outcome semantics. + +### Pure transition function + +- [x] NAH-111 Implement the pure `state + event + context -> state + actions` + transition function with no database or filesystem access. +- [x] NAH-112 Cover missing start events, duplicate starts/ends, implicit turns, resume, + cancellation, failure, late events, compaction, subagents, and unknown events. +- [x] NAH-113 Make terminal receipts enter `finalizing`; only a successful durable + finalization completion can enter `ended` or `interrupted`. +- [x] NAH-114 Add an exhaustive state/event matrix test and property-style idempotency + tests for every terminal transition. + +### Coordinator boundary + +- [x] NAH-121 Define the adapter-neutral capture coordinator interface and action + executor boundary. +- [x] NAH-122 Refactor ACP capture to emit the shared lifecycle contract without + regressing its existing streamed fidelity. +- [x] NAH-123 Add source-precedence and monotonic-enrichment rules for ACP, native + hooks, transcripts, canonical exports, and reconstructed evidence. + +Acceptance evidence: + +- domain serialization golden tests are stable and versioned; +- all state/event pairs are tested and the transition module has no I/O imports; +- existing ACP unit and E2E tests pass unchanged or with explicitly reviewed richer + assertions; +- replaying an identical normalized event produces no duplicate mutation action. + +## NAH-200: Storage and repository APIs + +### Schema + +- [x] NAH-201 Add `agent_hook_installations` with ownership markers, config digests, + provider versions, capability probes, scope, and last-success diagnostics. +- [x] NAH-202 Add `lane_agent_sessions` mapping provider-native sessions to Trail + sessions, lanes, capture owner, managed run, lifecycle state, and finalization lease. +- [x] NAH-203 Add `agent_hook_receipts` with idempotency key, receive sequence, bounded + raw payload reference, processing status, retry state, and error diagnostics. +- [x] NAH-204 Add `lane_artifacts` and `lane_turn_evidence_manifests` with immutable + digest, trust, redaction, coverage, and exact file/change membership. +- [x] NAH-205 Add `agent_capture_runs` with renewable lease, canonical workdir scope, + executor/provider constraints, and stamped output membership. +- [x] NAH-206 Add provenance nodes/edges, session attestations, revocation state, and + learnings with factual/derived separation. +- [x] NAH-207 Add exact Git association records for Trail-created and externally + observed commits without hidden branches or ambient-range attribution. +- [x] NAH-208 Provide transactional schema migration, rollback tests, completeness + checks, indexes, foreign keys, and bounded retention queries. + +### Repository layer + +- [x] NAH-211 Add typed create/read/update APIs for installations, mappings, receipts, + runs, artifacts, manifests, provenance, attestations, learnings, and Git links. +- [x] NAH-212 Enforce compare-and-set lifecycle transitions and renewable finalization + leases under concurrent hook processes. +- [x] NAH-213 Allocate a monotonic receive sequence per native session and reject + cross-workspace identity collisions. +- [x] NAH-214 Provide paginated, deterministic reports shared by CLI, HTTP, and MCP. + +Acceptance evidence: + +- migrations succeed from every supported schema version and rollback cleanly on + injected failure; +- foreign-key and duplicate-key tests prove idempotency and exact ownership; +- concurrent writers cannot produce two active mappings or two finalizers; +- a database reopen preserves every in-flight receipt and lease. + +## NAH-300: Ingress, replay, ownership, and recovery + +### Durable ingress + +- [x] NAH-301 Implement `trail agent hook receive PROVIDER EVENT` with bounded stdin, + timeout, installation identity validation, and provider-compatible output. +- [x] NAH-302 Implement the matching daemon HTTP ingress with identical validation and + response semantics. +- [x] NAH-303 Persist or securely spool the raw receipt before parsing or dispatch; use + restrictive permissions, atomic publication, quotas, and deterministic filenames. +- [x] NAH-304 Add replay workers, retry/backoff, poison-receipt quarantine, operator + inspection, and explicit retry/discard operations. +- [x] NAH-305 Retain unmapped provider events as redacted inert events. + +### Managed runs and ownership + +- [x] NAH-311 Implement managed-run begin/renew/end APIs and longest-canonical-workdir + matching with provider/executor constraints. +- [x] NAH-312 Stamp only newly created native-session mappings; never retroactively + capture an unrelated pre-existing session. +- [x] NAH-313 Implement owner selection among ACP, native hooks, terminal, and hybrid + capture with stable correlation fallbacks and ambiguity diagnostics. +- [x] NAH-314 Implement native-session, turn, message, tool, and checkpoint idempotency + keys plus source-precedence enrichment. + +### Recovery + +- [x] NAH-321 Implement finalization leases, crash takeover, transcript retry, artifact + retry, workdir sync retry, and checkpoint retry. +- [x] NAH-322 Reconstruct incomplete turns from durable receipts plus native artifacts + after daemon or plugin-buffer loss. +- [x] NAH-323 Reconcile expired managed runs and interrupted provider processes without + silently marking incomplete sessions cleanly ended. + +Acceptance evidence: + +- killing Trail between receipt persistence and dispatch loses no event; +- 100 concurrent duplicate receipts create one semantic record; +- nested managed runs select the longest valid workdir match and ambiguity fails closed; +- hooks still return the provider success contract during bounded Trail degradation. + +## NAH-400: Installation framework + +- [x] NAH-401 Define the provider registry, stable canonical names/aliases, capabilities, + discovery, version probing, transcript location, and response rendering traits. +- [x] NAH-402 Define a constrained, versioned adapter manifest for provider event names, + merge fragments, generated assets, commands, response contracts, and supported + provider versions. +- [x] NAH-403 Implement structural configuration merge preserving unknown fields, + comments where the format permits, unrelated hooks/plugins, and user ordering. +- [x] NAH-404 Add ownership markers and before/after digests; removal may delete only + exact Trail-owned entries or files. +- [x] NAH-405 Add repository and user scopes, canonical path validation, symlink/race + defenses, restrictive file modes, atomic writes, rollback, and crash recovery. +- [x] NAH-406 Implement add/remove/list/status/doctor/events flows with dry-run and JSON. +- [x] NAH-407 Add manifest compatibility probes so unsupported provider versions report + `partial`, `unavailable`, or `unknown` rather than guessing. + +Acceptance evidence: + +- golden merge/unmerge fixtures preserve unrelated configuration byte-for-byte where + the format permits; +- install failure rolls back all Trail-owned changes; +- removal refuses modified ownership markers and reports the exact conflict; +- malicious paths, oversized config, symlinks, and concurrent installers fail safely. + +## NAH-500: Provider adapters + +Every provider must complete discovery, version probe, installation, parsing, +transcript/export collection, response rendering, capabilities, diagnostics, golden +fixtures, and coexistence tests. Unsupported native events remain inert evidence. + +- [x] NAH-501 OpenAI Codex: notifications/native hooks where supported, transcript + collection, managed launch fallback, and ACP correlation. +- [x] NAH-502 Anthropic Claude Code: settings merge, complete hook mapping, transcript + offsets/snapshots, context injection, and hook response contract. +- [x] NAH-503 Pi: extension installation, lifecycle/tool/usage events, transcript and + context integration. +- [x] NAH-504 OpenCode: plugin installation, fragment aggregation, todos/parts/tool + mapping, durable acknowledgements, and transcript export. +- [x] NAH-505 Cursor: project/user hook configuration, available lifecycle/tool events, + transcript capability detection, and explicit fidelity gaps. +- [x] NAH-506 Gemini CLI: settings/hooks merge, session/turn/tool/compaction mapping, + transcript discovery, and context support. +- [x] NAH-507 GitHub Copilot CLI: supported hook/plugin wiring, lifecycle/tool mapping, + transcript/export discovery, and version-gated capability reporting. +- [x] NAH-508 Grok Build: native contract probe and adapter where available plus a + managed-wrapper first-class path with honest transcript/resume limitations. +- [x] NAH-509 Cross-provider alias, mixed-installation, upgrade, downgrade, removal, and + hybrid ACP/native test matrix. + +Acceptance evidence for each provider: + +- provider-native golden receipts map to the expected normalized sequence; +- exact install/uninstall fixtures cover repository and user scopes; +- version probes identify the verified range and fidelity level; +- duplicate, late, missing, malformed, and oversized events are covered; +- canonical transcript/export digest tests pass when the provider exposes one. + +## NAH-600: Evidence and checkpoint pipeline + +- [x] NAH-601 Implement native artifact records for transcript, export, tool output, + structured patch, plan, context, and provider metadata with trust/confidence. +- [x] NAH-602 Prefer canonical export, then stable native transcript, then explicitly + marked reconstructed transcript. +- [x] NAH-603 Implement pre-turn offsets, immutable snapshots/deltas, digest validation, + truncation detection, resume chaining, and bounded parser failures. +- [x] NAH-604 Separate immutable factual envelopes from redactable attachments and + derived interpretations; never mutate a factual digest during redaction. +- [x] NAH-605 Capture exact turn-start workdir basis, structured/observed changes, and + final workdir sync through existing lane patch/sync paths. +- [x] NAH-606 Create deterministic turn evidence manifests with exact file/change, + receipt, message, tool, usage, artifact, and span membership. +- [x] NAH-607 Create session-end checkpoints only after durable finalization actions + succeed; support partial/interrupted outcome and retry. + +Acceptance evidence: + +- transcript append, rewrite, truncation, resume, and missing-file fixtures are covered; +- redaction can remove attachment access without invalidating factual envelope hashes; +- exact file manifest tests exclude unrelated concurrent lane changes; +- a failed checkpoint remains retryable and does not transition to `ended`. + +## NAH-700: Provenance and portable history + +- [x] NAH-701 Persist the causal graph node and edge vocabulary from the design with + exact source identifiers and deterministic ordering. +- [x] NAH-702 Implement rule-based activity classification as derived data linked to + source nodes; never claim hidden reasoning or ambient attribution. +- [x] NAH-703 Build content-addressed session attestations over precisely enumerated + evidence coverage, principal identity, previous attestation, and capture policy. +- [x] NAH-704 Implement local signing, verification, key rotation/revocation status, + unsigned mode, and tamper diagnostics. +- [x] NAH-705 Implement explicit, reviewable learnings with source coverage, confidence, + supersession, expiry, and opt-in context injection. +- [x] NAH-706 Implement portable versioned trace export/import with canonical ordering, + digests, bounded attachments, compatibility validation, and no hidden Git branch. +- [x] NAH-707 Link Trail-created commits exactly and observe external commits only when + exact checkpoint/tree/change identity proves the association. + +Acceptance evidence: + +- graph reconstruction is deterministic after shuffled receipt replay; +- tampering with any covered item fails attestation verification; +- revoked keys are reported without destroying historic evidence; +- export-import-export is byte-stable for the same supported version; +- concurrent unrelated commits are never attributed through time ranges alone. + +## NAH-800: Public surfaces and diagnostics + +### CLI and Rust + +- [x] NAH-801 Implement `trail agent hooks add|remove|list|status|doctor|events` and the + internal singular `trail agent hook receive` command. +- [x] NAH-802 Implement capture run, receipt replay, session/artifact/attestation, + provenance, learning, Git-link, and portable export commands from the design. +- [x] NAH-803 Expose stable Rust request/report types backed by the same repository + operations used by the CLI. + +### HTTP/OpenAPI and MCP + +- [x] NAH-811 Add HTTP endpoints for ingress and every inspect/mutate operation with + pagination, authorization, bounded bodies, and consistent errors. +- [x] NAH-812 Update checked-in OpenAPI schemas/examples and contract tests. +- [x] NAH-813 Add MCP tools/resources with the same capability and report model; keep + host-agent capture calls aligned with `begin_turn -> message -> span/event -> patch or + sync -> assistant message -> end_turn`. + +### Diagnostics + +- [x] NAH-821 Report installation drift, provider compatibility, transcript support, + capture owner, last receipt, spool pressure, replay failures, expired leases, partial + finalizations, and fidelity gaps. +- [x] NAH-822 Make diagnostic output actionable and redact secrets in text and JSON. + +Acceptance evidence: + +- CLI/Rust/HTTP/MCP parity tests compare the same canonical reports; +- OpenAPI validation and MCP schema snapshots pass; +- all list surfaces are deterministic and paginated; +- no diagnostic renders raw secrets or unbounded provider payloads. + +## NAH-900: Release hardening and completion audit + +- [x] NAH-901 Run unit, integration, E2E, golden fixture, concurrency, crash/reopen, + security, compatibility, and optional real-provider suites. +- [x] NAH-902 Add ingress latency, replay throughput, transcript size, database growth, + and 100-concurrent-hook performance gates with documented budgets. +- [x] NAH-903 Add operator and user documentation for setup, capture modes, privacy, + retention, troubleshooting, provider limitations, export, and recovery. +- [x] NAH-904 Add schema and adapter compatibility policy, manifest signing/distribution + policy, deprecation windows, and upgrade/downgrade diagnostics. +- [x] NAH-905 Resolve every design open question in an ADR or mark it as an explicit + versioned policy choice with compatibility behavior. +- [x] NAH-906 Perform a line-by-line design acceptance audit and link each requirement + to code plus automated evidence in the completion matrix below. + +## Completion matrix + +This matrix is the final stop gate. No row may be marked complete solely because an API +or stub exists. + +| Requirement | Implementation | Automated evidence | Status | +| --- | --- | --- | --- | +| Native capture works without ACP | receipt replay into Trail session/turn/message/span/checkpoint | durable lifecycle replay unit and native-hook CLI E2E | DONE | +| ACP retains current fidelity | typed lifecycle envelopes alongside streamed ACP events; Grok ACP profile | 12 ACP unit tests plus ACP relay E2E | DONE | +| Hybrid capture does not duplicate | exact ACP/native correlation with ACP lifecycle ownership and monotonic enrichment | hybrid native/ACP idempotency unit test | DONE | +| All nine providers are supported honestly | checked-in declarative adapters; unknown versions report `unknown` | 27 fixture cases plus nine-provider install matrix | DONE | +| Receipts survive retry and crash | durable object journal, bounded spool, retry/backoff/quarantine/discard | reopen/idempotency, stale-recovery, and spool-recovery tests | DONE | +| Exact turn/session file coverage | turn-start basis, observed/structured changes, immutable evidence membership | evidence-manifest and ACP workdir E2E tests | DONE | +| Canonical transcripts are preserved | export > native transcript > reconstructed fallback, with offsets and rewrite/truncation flags | canonical/rewrite/truncate/reconstruct test | DONE | +| Install/remove preserves user config | structural merge, exact markers, owned files, atomic rollback | nine-provider scope/idempotency, drift, symlink, and CLI tests | DONE | +| Finalization is leased and recoverable | renewable compare-and-set owner lease and stale takeover | competing-finalizer, expired-run, and replay tests | DONE | +| Facts, redactions, and derivations are separate | immutable manifest plus separately redactable attachments and derived nodes | digest, artifact-redaction, and classifier tests | DONE | +| Provenance is causal and exact | deterministic session-scoped source/activity nodes and `derived_from` edges | idempotent evidence/provenance test | DONE | +| Attestations cover enumerated evidence | content-addressed incremental chain over exact turn manifests | chain/sign/verify/revoke/tamper test | DONE | +| Learnings are explicit and reviewable | proposed/accepted/rejected records with anchors and opt-in injection | redaction and review-state test | DONE | +| Git association has no hidden branch | exact commit/change/session link records | full-object-ID and idempotency test | DONE | +| Portable trace round-trips | bounded canonical v1 export and verified import parser | export-import-export byte-stability test and CLI E2E | DONE | +| CLI/Rust/HTTP/MCP are consistent | shared report types, paginated routes/tools/resources, strict OpenAPI | receipt-ID and offset parity E2E plus OpenAPI/MCP contract tests | DONE | +| Security and resource bounds hold | auth, input/config/spool/artifact bounds, redaction, symlink and lock defenses | malformed/oversize/symlink/drift/auth/read-only tests | DONE | +| Upgrade and recovery are documented | operator guide, v1 policy choices, compatibility and recovery procedures | documentation and completion audit | DONE | + +### Design acceptance audit + +| # | Design acceptance criterion | Implementation and automated evidence | Result | +| --- | --- | --- | --- | +| 1 | Install every stable provider without ACP | `agent_hooks` registry/install planner and nine-provider project/user asset test | PASS | +| 2 | Direct sessions create searchable sessions/turns | native receipt replay repository test and Codex CLI E2E | PASS | +| 3 | Tools become correctly nested spans | root/tool/subagent/compaction span replay assertions | PASS | +| 4 | At most one checkpoint per changed turn | pure finalization actions plus duplicate-terminal and durable replay tests | PASS | +| 5 | Preserve transcript/export evidence | canonical precedence, offset, rewrite, truncation, and reconstruction test | PASS | +| 6 | Hybrid ACP/native capture deduplicates | exact correlation, ACP lifecycle ownership, message/event idempotency test | PASS | +| 7 | Hook failure is fail-open and diagnosable | database-outage spool/replay E2E and doctor diagnostics | PASS | +| 8 | Preserve foreign provider configuration | structural merge, drift refusal, idempotent uninstall, CLI E2E | PASS | +| 9 | Crash recovery does not duplicate semantics | durable receipt, stale processing, replay, and 100-writer tests | PASS | +| 10 | Enforce payload/path/ID/artifact security | hard size bounds, central redaction, symlink/outside-root/auth tests | PASS | +| 11 | Report capability/version limits honestly | compatibility probe and unknown-version provider reports | PASS | +| 12 | Preserve ACP and terminal compatibility | complete ACP unit/E2E suite and existing package regression suite | PASS | +| 13 | Query exact Git associations without hidden refs | exact full-object-ID Git link repository/CLI/HTTP/MCP surfaces | PASS | +| 14 | Managed runs stamp only governed sessions | longest-workdir, owner/executor, ambiguity, and expiry tests | PASS | +| 15 | Immutable turn manifest is separate from attachments | evidence manifest, artifact redaction, and digest-stability tests | PASS | +| 16 | Causal provenance links to factual sources | deterministic rule classifier and `derived_from` graph test | PASS | +| 17 | Session attestations exactly cover and chain turns | create/sign/verify/revoke/tamper/previous-attestation test | PASS | +| 18 | Redaction does not change factual/checkpoint identity | attachment redaction retains manifest and attestation identity | PASS | +| 19 | Adapters are previewable, gated, and reversible | dry-run plans, ownership manifests, compatibility diagnostics; remote manifests disabled in v1 | PASS | +| 20 | Portable trace is an export, not a second truth | canonical bounded export, verified import parser, byte-stable roundtrip | PASS | + +## Shared validation gate + +Before this plan can be marked `DONE`, run and record successful results for: + +```sh +make fmt-check +cargo check -p trail +cargo test -p trail +make bench-cli-scale-smoke +``` + +Also run every provider fixture suite, schema migration suite, receipt crash/replay suite, +hybrid ACP/native E2E, HTTP/OpenAPI contract suite, MCP parity suite, and security test +listed above. Platform-specific real-provider evidence may be explicitly documented as +unavailable, but the corresponding adapter contract fixtures and honest capability +diagnostics are mandatory on every release platform. + +### Latest implementation evidence + +- `make fmt-check` and scoped `git diff --check`: passed. +- `cargo check -p trail`: passed in the shared repository. +- `cargo test -p trail` executed 279 library tests, 12 binary tests, and 183 E2E tests. + The library and binary suites passed; 181 E2E tests passed in the aggregate run. The + two failures were stale concurrent schema-v15 expectations (`14` instead of `15`), + and both corrected migration tests passed exactly afterward. Combined result: all + 474 tests pass. +- `cargo test -p trail agent_capture --lib`: 29 tests passed, including lifecycle + cross-product/idempotency, 100 concurrent writers under the 15-second/32-MiB budget, + hybrid ownership, recovery, transcript rewrite/truncation, payload bounds, and + transcript symlink/outside-root rejection. +- `cargo test -p trail agent_hooks --lib`: 14 tests passed, including the checked-in + 27-case provider fixture matrix and all-nine-provider project/user installation. +- ACP, evidence, OpenAPI, authenticated HTTP ingress, pagination parity, MCP read-only + annotation, native spool recovery, schema-v12, and portable-trace E2E tests passed. +- `make bench-cli-scale-smoke`: passed at 1,000 files; results were written under + `/tmp/trail-cli-scale-ci-smoke/1000`. +- Real-provider smoke tests were not run because they require provider installations, + authentication, possible network access, and spend. Version compatibility therefore + remains `unknown` unless proved by a checked-in adapter fixture/range, as required by + the v1 policy. diff --git a/plans/README.md b/plans/README.md index 286f9d1..308db45 100644 --- a/plans/README.md +++ b/plans/README.md @@ -13,6 +13,9 @@ Execute in the order below unless dependencies say otherwise. Each executor shou | 003 | Parallelize copy-on-write cloning | P1 | M | 002 | DONE | | 004 | Stream root materialization in chunks | P2 | L | 001, 002 | DONE | | 005 | Layered lane workspaces | P0 | XXL | 004 | BLOCKED — native Windows Dokan acceptance run pending | +| 006 | Universal lane environments | P0 | XXL | 005 semantic core | TODO | +| 006 | Universal lane environments | P0 | XXL | 005 semantic core | TODO | +| 007 | Native agent hooks and ACP integration | P0 | XXL | lane activity model, ACP relay | IN PROGRESS | Status values: TODO, IN PROGRESS, DONE, BLOCKED with a one-line reason, or REJECTED with a one-line rationale. @@ -22,6 +25,9 @@ Status values: TODO, IN PROGRESS, DONE, BLOCKED with a one-line reason, or REJEC - 003 depends on 002 so parallel clone workers can return destination stamps through the same report path. - 004 depends on 001 and 002 because streaming chunks need clean-index stamp reuse and final manifest assembly to avoid reintroducing full scans. - 005 builds on the bounded materialization work but replaces copied agent workdirs with lazy, layered views; its internal task graph and acceptance matrix live in `005-layered-lane-workspaces.md`. +- 006 generalizes the layered workspace substrate into typed, reproducible environment graphs spanning toolchains, dependencies, private state, caches, secrets, external artifacts, and services. Its work packages can begin after the relevant 005 filesystem semantics are stable, even while a platform-specific 005 release gate remains open. +- 006 generalizes the layered workspace substrate into typed, reproducible environment graphs spanning toolchains, dependencies, private state, caches, secrets, external artifacts, and services. Its work packages can begin after the relevant 005 filesystem semantics are stable, even while a platform-specific 005 release gate remains open. +- 007 is additive to the lane activity model and replaces ACP-specific orchestration with a shared capture coordinator incrementally. It does not depend on the unfinished environment release gates. ## Findings considered and deferred diff --git a/prolly b/prolly index 3a992bd..0474896 160000 --- a/prolly +++ b/prolly @@ -1 +1 @@ -Subproject commit 3a992bde3ca2777d9fdb7cf4998a3fb9e0294780 +Subproject commit 04748964758aa6a9f111e50225b2e3646d53f0b5 diff --git a/rustfmt.toml b/rustfmt.toml index 2fc0a4f..ae2a548 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,3 +1,5 @@ -edition = "2021" +edition = "2024" +# Keep style changes bounded while the language edition moves to Rust 2024. +style_edition = "2021" newline_style = "Unix" use_field_init_shorthand = true diff --git a/scripts/check-acp-v1-schema-drift.sh b/scripts/check-acp-v1-schema-drift.sh new file mode 100755 index 0000000..c95bd95 --- /dev/null +++ b/scripts/check-acp-v1-schema-drift.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source_manifest="$repo_root/trail/tests/fixtures/acp/v1/source.json" +fixture_dir="$repo_root/trail/tests/fixtures/acp/v1" +repository="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["repository"])' "$source_manifest")" +old_commit="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["commit"])' "$source_manifest")" +new_commit="${ACP_V1_UPSTREAM_REVISION:-$(git ls-remote "$repository.git" refs/heads/main | awk '{print $1}')}" +if [[ -z "$new_commit" ]]; then + echo "unable to resolve the upstream ACP main revision" >&2 + exit 2 +fi + +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +base="https://raw.githubusercontent.com/agentclientprotocol/agent-client-protocol/$new_commit/schema/v1" +curl --fail --location --silent --show-error "$base/schema.json" --output "$tmp/schema.json" +curl --fail --location --silent --show-error "$base/meta.json" --output "$tmp/meta.json" + +digest() { + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$1" | awk '{print $1}' + else + sha256sum "$1" | awk '{print $1}' + fi +} + +old_schema="$(digest "$fixture_dir/schema.json")" +old_meta="$(digest "$fixture_dir/meta.json")" +new_schema="$(digest "$tmp/schema.json")" +new_meta="$(digest "$tmp/meta.json")" +printf 'ACP v1 pinned commit: %s\n' "$old_commit" +printf 'ACP v1 upstream commit: %s\n' "$new_commit" +printf 'schema.json: %s -> %s\n' "$old_schema" "$new_schema" +printf 'meta.json: %s -> %s\n' "$old_meta" "$new_meta" + +if ! cmp -s "$fixture_dir/schema.json" "$tmp/schema.json" || ! cmp -s "$fixture_dir/meta.json" "$tmp/meta.json"; then + echo "ACP v1 schema drift detected; review and repin the normative contract" >&2 + exit 1 +fi +echo "ACP v1 schema has not drifted" diff --git a/scripts/check-changed-path-ledger-thresholds.py b/scripts/check-changed-path-ledger-thresholds.py new file mode 100755 index 0000000..908644e --- /dev/null +++ b/scripts/check-changed-path-ledger-thresholds.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python3 +"""Gate changed-path-ledger scale artifacts without mixing oracle work into fast paths.""" + +import argparse +import csv +import json +import math +import pathlib +import sys + + +DEFAULT_OPERATIONS = ( + "workspace_status", + "workspace_diff", + "workspace_record", + "materialized_lane_record", + "structured_patch", +) +ORACLE_OPERATIONS = ( + "workspace_status", + "workspace_diff", + "workspace_record", + "materialized_lane_record", + "structured_patch", + "cow_checkpoint", +) +SCOPED_OPERATIONS = { + "workspace_status": "status", + "workspace_diff": "diff", + "workspace_record": "record", + "materialized_lane_record": "materialized_lane_record", + "structured_patch": "structured_patch", + "cow_checkpoint": "cow_checkpoint", +} +ZERO_SCOPE_FIELDS = ( + "full_filesystem_walk_count", + "bounded_filesystem_walk_count", + "full_root_range_count", + "selected_worktree_index_sqlite_full_scan_count", + "policy_dependency_full_discovery", + "reconciliation_run_count", + "manifest_bytes", + "upper_work_count", + "external_adapter_global_work", + "git_global_work_count", + "git_index_refresh_count", + "git_trace2_region_count", + "git_trace2_bytes", + "git_fsmonitor_qualification_count", + "git_untracked_cache_qualification_count", + "git_index_read_count", + "git_index_bytes", + "git_shared_index_read_count", + "git_shared_index_bytes", + "daemon_cumulative_rewrite_count", + "daemon_cumulative_rewrite_bytes", +) +OPERATION_METRIC_FIELDS = ( + "generation", + "operation", + "outcome", + "input_path_count", + "canonical_path_count", + "expanded_path_count", + "final_path_count", + "full_filesystem_walk_count", + "bounded_filesystem_walk_count", + "filesystem_entry_count", + "filesystem_stat_count", + "filesystem_read_count", + "filesystem_read_bytes", + "filesystem_hash_count", + "filesystem_hash_bytes", + "full_root_range_count", + "bounded_root_range_count", + "root_range_row_count", + "root_point_key_count", + "prolly_read_call_count", + "prolly_read_key_count", + "prolly_read_value_count", + "prolly_read_value_bytes", + "prolly_write_call_count", + "prolly_write_key_count", + "prolly_write_value_bytes", + "prolly_tree_batch_call_count", + "prolly_tree_batch_mutation_count", + "selected_worktree_index_sqlite_accounting_complete", + "selected_worktree_index_sqlite_accounting_disposition", + "selected_worktree_index_sqlite_envelope_count", + "selected_worktree_index_sqlite_not_applicable_count", + "selected_worktree_index_sqlite_full_scan_count", + "selected_worktree_index_sqlite_row_read_count", + "selected_worktree_index_sqlite_row_delete_count", + "selected_worktree_index_sqlite_row_upsert_count", + "selected_worktree_index_sqlite_statement_count", + "selected_worktree_index_sqlite_transaction_count", + "selection_comparison_count", + "policy_build_count", + "policy_dependency_full_discovery", + "policy_dependency_bytes", + "policy_dependency_file_count", + "git_subprocess_count", + "git_global_work_count", + "git_index_refresh_count", + "git_trace2_region_count", + "git_trace2_bytes", + "git_fsmonitor_qualification_count", + "git_untracked_cache_qualification_count", + "external_adapter_global_work", + "git_index_read_count", + "git_index_bytes", + "git_shared_index_read_count", + "git_shared_index_bytes", + "git_output_bytes", + "git_output_record_count", + "daemon_snapshot_bytes", + "daemon_snapshot_path_count", + "daemon_cumulative_rewrite_count", + "daemon_cumulative_rewrite_bytes", + "daemon_cumulative_rewrite_count_total", + "daemon_cumulative_rewrite_bytes_total", + "authoritative_candidate_count", + "ledger_row_touch_count", + "observer_tail_record_fold_count", + "reconciliation_run_count", + "manifest_bytes", + "manifest_key_comparison_count", + "journal_bytes", + "upper_work_count", + "wall_time_ns", + "rss_start_bytes", + "rss_end_bytes", + "rss_lifetime_high_water_bytes", +) +REQUIRED_SCOPE_FIELDS = (*OPERATION_METRIC_FIELDS, "configured_tail_bound") + +# Each nonzero work allowance is affine in the exact candidate count. The +# operation multiplier accounts for the different constant number of stages in +# each command while preserving a repository-size-independent O(k) ceiling. +OPERATION_WORK_MULTIPLIER = { + "workspace_status": 1, + "workspace_diff": 2, + "workspace_record": 3, + "materialized_lane_record": 6, + "structured_patch": 6, + "cow_checkpoint": 8, +} +AFFINE_WORK_BOUNDS = { + "input_path_count": (8, 2), + "canonical_path_count": (8, 2), + "filesystem_entry_count": (16, 16), + "filesystem_stat_count": (8, 8), + "filesystem_read_count": (8, 8), + "filesystem_read_bytes": (1 << 20, 1 << 20), + "filesystem_hash_count": (8, 8), + "filesystem_hash_bytes": (1 << 20, 1 << 20), + "bounded_root_range_count": (8, 8), + "root_range_row_count": (16, 16), + "root_point_key_count": (8, 8), + "prolly_read_call_count": (32, 32), + "prolly_read_key_count": (64, 64), + "prolly_read_value_count": (64, 64), + "prolly_read_value_bytes": (1 << 24, 1 << 24), + "prolly_write_call_count": (32, 32), + "prolly_write_key_count": (64, 64), + "prolly_write_value_bytes": (1 << 24, 1 << 24), + "prolly_tree_batch_call_count": (16, 16), + "prolly_tree_batch_mutation_count": (64, 64), + "selection_comparison_count": (64, 64), + "policy_build_count": (8, 8), + "policy_dependency_bytes": (1 << 20, 1 << 20), + "policy_dependency_file_count": (16, 16), + "git_subprocess_count": (16, 4), + "git_output_bytes": (1 << 12, 1 << 12), + "git_output_record_count": (16, 16), + "daemon_snapshot_bytes": (1 << 20, 1 << 20), + "daemon_snapshot_path_count": (8, 8), + "ledger_row_touch_count": (16, 16), + "manifest_key_comparison_count": (64, 64), + "journal_bytes": (1 << 24, 1 << 24), +} + +SIDE_CAR_BASE_FIELDS = { + "benchmark", + "benchmark_operation", + "repo_files", + "authoritative_input_k", + "final_changed_output", + "configured_tail_bound", + "metric_source", + "scope_caps", +} +UPPER_RECOVERY_FIELDS = { + "upper_recovery_walks", +} +GENERATED_PATH_ACCOUNTING_FIELDS = { + "generated_path_accounting", +} +PATH_INDEX_FIELDS = { + "path_index_full_root_path_load_count", + "path_index_full_filesystem_path_scan_count", + "path_index_lookup_count", + "path_index_mode", +} + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("results", type=pathlib.Path) + parser.add_argument("structural_metrics", type=pathlib.Path) + parser.add_argument("--oracle", required=True, type=pathlib.Path) + parser.add_argument("--k", default="0,1,100") + parser.add_argument("--operations", default=",".join(DEFAULT_OPERATIONS)) + parser.add_argument("--require-cow", action="store_true") + parser.add_argument("--require-cold-reconcile", action="store_true") + parser.add_argument("--max-seconds", action="append", default=[]) + parser.add_argument("--max-rss-bytes", action="append", default=[]) + return parser.parse_args(argv) + + +def parse_integer_list(value: str, label: str) -> tuple[int, ...]: + try: + values = tuple(int(item) for item in value.split(",") if item) + except ValueError as error: + raise ValueError(f"invalid {label}: {value}") from error + if not values or any(item < 0 for item in values): + raise ValueError(f"invalid {label}: {value}") + return values + + +def parse_thresholds(specs: list[str], label: str) -> dict[str, float]: + parsed = {} + for spec in specs: + if "=" not in spec: + raise ValueError(f"invalid {label} threshold {spec!r}; expected operation=value") + operation, raw = spec.split("=", 1) + if not operation: + raise ValueError(f"invalid {label} threshold {spec!r}; operation is empty") + try: + value = float(raw) + except ValueError as error: + raise ValueError(f"invalid {label} threshold value in {spec!r}") from error + if value < 0: + raise ValueError(f"invalid {label} threshold value in {spec!r}") + parsed[operation] = value + return parsed + + +def read_results(path: pathlib.Path) -> dict[str, dict[str, str]]: + with path.open(newline="") as handle: + reader = csv.DictReader(handle, delimiter="\t") + fieldnames = reader.fieldnames or [] + rows = list(reader) + required = {"name", "real_seconds", "max_rss_bytes", "exit_code"} + if not required.issubset(fieldnames): + raise ValueError(f"{path}: missing required results columns") + found = {} + for row in rows: + name = row.get("name", "") + if not name: + continue + if name in found: + raise ValueError(f"{path}: duplicate result {name!r}") + found[name] = row + return found + + +def read_jsonl(path: pathlib.Path) -> dict[str, dict[str, object]]: + found = {} + for number, line in enumerate(path.read_text().splitlines(), 1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError(f"{path}:{number}: invalid JSON: {error}") from error + if not isinstance(record, dict): + raise ValueError(f"{path}:{number}: expected a JSON object") + benchmark = record.get("benchmark") + if not isinstance(benchmark, str) or not benchmark: + raise ValueError(f"{path}:{number}: missing benchmark") + if benchmark in found: + raise ValueError(f"{path}:{number}: duplicate benchmark {benchmark!r}") + found[benchmark] = record + return found + + +def read_oracle(path: pathlib.Path) -> dict[str, dict[str, str]]: + with path.open(newline="") as handle: + reader = csv.DictReader(handle, delimiter="\t") + fieldnames = reader.fieldnames or [] + rows = list(reader) + required = { + "benchmark", + "repo_files", + "authoritative_input_k", + "oracle_time_ns", + "measured_paths", + "oracle_paths", + "equal", + } + if not required.issubset(fieldnames): + raise ValueError(f"{path}: missing required oracle columns") + found = {} + for row in rows: + benchmark = row.get("benchmark", "") + if benchmark in found: + raise ValueError(f"{path}: duplicate oracle benchmark {benchmark!r}") + found[benchmark] = row + return found + + +def require_number(record: dict[str, object], key: str, failures: list[str], name: str) -> float: + value = record.get(key) + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + ): + failures.append(f"{name}: missing or nonnumeric structural field {key}") + return 0 + return float(value) + + +def check_scope_caps(record: dict[str, object], failures: list[str], name: str) -> None: + scopes = record.get("scope_caps") + if not isinstance(scopes, list) or not scopes: + failures.append(f"{name}: no persisted changed-path scope caps were reported") + return + comparisons = ( + ("candidate_rows", "candidate_row_cap"), + ("prefix_rows", "prefix_row_cap"), + ("observer_log_bytes", "observer_log_byte_cap"), + ("largest_segment_bytes", "segment_byte_cap"), + ) + expected_fields = {"scope_id"} + for value_key, cap_key in comparisons: + expected_fields.update((value_key, cap_key)) + for scope in scopes: + if not isinstance(scope, dict): + failures.append(f"{name}: malformed scope cap record") + continue + missing = sorted(expected_fields.difference(scope)) + unknown = sorted(set(scope).difference(expected_fields)) + if missing: + failures.append(f"{name}: scope cap omitted {', '.join(missing)}") + if unknown: + failures.append(f"{name}: scope cap has unknown fields {', '.join(unknown)}") + scope_id = scope.get("scope_id", "unknown") + for value_key, cap_key in comparisons: + value = scope.get(value_key) + cap = scope.get(cap_key) + if not isinstance(value, (int, float)) or not isinstance(cap, (int, float)): + failures.append(f"{name}: scope {scope_id} omitted {value_key}/{cap_key}") + elif value > cap: + failures.append(f"{name}: scope {scope_id} {value_key}={value} exceeds {cap_key}={cap}") + + +def check_scoped(record: dict[str, object], operation: str, k: int, failures: list[str], name: str) -> None: + if record.get("metric_source") != "operation_scope": + failures.append(f"{name}: expected operation_scope metrics") + return + missing = [key for key in REQUIRED_SCOPE_FIELDS if key not in record] + if missing: + failures.append(f"{name}: missing structural fields {', '.join(missing)}") + return + allowed_fields = set(OPERATION_METRIC_FIELDS) | SIDE_CAR_BASE_FIELDS + if operation in {"materialized_lane_record", "cow_checkpoint"}: + allowed_fields.update(UPPER_RECOVERY_FIELDS) + if operation == "cow_checkpoint": + allowed_fields.update(GENERATED_PATH_ACCOUNTING_FIELDS) + if operation in {"materialized_lane_record", "structured_patch"}: + allowed_fields.update(PATH_INDEX_FIELDS) + unknown = sorted(set(record).difference(allowed_fields)) + if unknown: + failures.append(f"{name}: unknown structural fields {', '.join(unknown)}") + if record.get("operation") != SCOPED_OPERATIONS[operation]: + failures.append(f"{name}: operation report is {record.get('operation')!r}") + if str(record.get("outcome")).lower() != "success": + failures.append(f"{name}: operation outcome is {record.get('outcome')!r}") + nonnumeric_fields = { + "operation", + "outcome", + "selected_worktree_index_sqlite_accounting_complete", + "selected_worktree_index_sqlite_accounting_disposition", + } + for field in OPERATION_METRIC_FIELDS: + if field not in nonnumeric_fields: + require_number(record, field, failures, name) + sqlite_activity_fields = ( + "selected_worktree_index_sqlite_envelope_count", + "selected_worktree_index_sqlite_not_applicable_count", + "selected_worktree_index_sqlite_full_scan_count", + "selected_worktree_index_sqlite_row_read_count", + "selected_worktree_index_sqlite_row_delete_count", + "selected_worktree_index_sqlite_row_upsert_count", + "selected_worktree_index_sqlite_statement_count", + "selected_worktree_index_sqlite_transaction_count", + ) + sqlite_activity = { + field: require_number(record, field, failures, name) + for field in sqlite_activity_fields + } + sqlite_complete = record.get("selected_worktree_index_sqlite_accounting_complete") + sqlite_disposition = record.get("selected_worktree_index_sqlite_accounting_disposition") + workspace_operation = operation in { + "workspace_status", + "workspace_diff", + "workspace_record", + } + if workspace_operation: + if sqlite_disposition != "not_applicable" or sqlite_complete is not False: + failures.append( + f"{name}: workspace selected worktree-index SQLite disposition must be independently proven not_applicable" + ) + if sqlite_activity["selected_worktree_index_sqlite_not_applicable_count"] != 1: + failures.append( + f"{name}: workspace selected worktree-index SQLite N/A proof count must equal one" + ) + unexpected = { + field: value + for field, value in sqlite_activity.items() + if field != "selected_worktree_index_sqlite_not_applicable_count" and value != 0 + } + if unexpected: + failures.append( + f"{name}: selected worktree-index SQLite work occurred on a not_applicable operation" + ) + else: + if sqlite_disposition != "complete" or sqlite_complete is not True: + failures.append( + f"{name}: selected worktree-index SQLite accounting must be complete" + ) + envelopes = sqlite_activity["selected_worktree_index_sqlite_envelope_count"] + if envelopes <= 0: + failures.append( + f"{name}: selected worktree-index SQLite completeness requires an accounting envelope" + ) + if sqlite_activity["selected_worktree_index_sqlite_not_applicable_count"] != 0: + failures.append( + f"{name}: selected worktree-index SQLite accounting mixes complete and not_applicable claims" + ) + sqlite_bounds = { + "selected_worktree_index_sqlite_envelope_count": k + 1, + "selected_worktree_index_sqlite_row_read_count": 8 * (k + 1), + "selected_worktree_index_sqlite_row_delete_count": 8 * (k + 1), + "selected_worktree_index_sqlite_row_upsert_count": 8 * (k + 1), + "selected_worktree_index_sqlite_statement_count": 32 * (k + 1), + "selected_worktree_index_sqlite_transaction_count": k + 1, + } + for field, bound in sqlite_bounds.items(): + if sqlite_activity[field] > bound: + failures.append( + f"{name}: {field}={sqlite_activity[field]:g} exceeds O(k) bound {bound:g}" + ) + for field in ZERO_SCOPE_FIELDS: + if require_number(record, field, failures, name) != 0: + failures.append(f"{name}: {field} must remain zero on a warm trusted run") + authoritative = require_number(record, "authoritative_candidate_count", failures, name) + expanded = require_number(record, "expanded_path_count", failures, name) + if authoritative != k: + failures.append( + f"{name}: authoritative candidates {authoritative:g} do not equal exact-file k={k}" + ) + if expanded > k: + failures.append( + f"{name}: expanded paths {expanded:g} exceed exact-file k={k}" + ) + operation_multiplier = OPERATION_WORK_MULTIPLIER[operation] + for field, (base, per_candidate) in AFFINE_WORK_BOUNDS.items(): + value = require_number(record, field, failures, name) + bound = base + per_candidate * k * operation_multiplier + if value > bound: + failures.append( + f"{name}: {field}={value:g} exceeds O(k) bound {bound:g}" + ) + folded = require_number(record, "observer_tail_record_fold_count", failures, name) + tail_bound = require_number(record, "configured_tail_bound", failures, name) + if tail_bound <= 0 or tail_bound > 4096: + failures.append(f"{name}: configured_tail_bound={tail_bound:g} exceeds audited cap 4096") + if folded > tail_bound: + failures.append(f"{name}: observer tail folded {folded:g} records, above {tail_bound:g}") + + total_count = require_number( + record, "daemon_cumulative_rewrite_count_total", failures, name + ) + total_bytes = require_number( + record, "daemon_cumulative_rewrite_bytes_total", failures, name + ) + if total_count < 0 or total_bytes < 0: + failures.append(f"{name}: daemon cumulative gauges must be nonnegative") + + +def check_operation_extras( + record: dict[str, object], operation: str, k: int, failures: list[str], name: str +) -> None: + required_zero = [] + if operation in {"materialized_lane_record", "cow_checkpoint"}: + required_zero.append("upper_recovery_walks") + if operation in {"materialized_lane_record", "structured_patch"}: + required_zero.extend( + ( + "path_index_full_root_path_load_count", + "path_index_full_filesystem_path_scan_count", + ) + ) + for field in required_zero: + if require_number(record, field, failures, name) != 0: + failures.append(f"{name}: {field} must remain zero on a warm trusted run") + if operation == "cow_checkpoint" and record.get("generated_path_accounting") != "journal_interval": + failures.append( + f"{name}: generated_path_accounting={record.get('generated_path_accounting')!r}, expected 'journal_interval'" + ) + if operation in {"materialized_lane_record", "structured_patch"}: + lookups = require_number(record, "path_index_lookup_count", failures, name) + lookup_bound = 8 * (k + 1) + if lookups > lookup_bound: + failures.append( + f"{name}: path_index_lookup_count={lookups:g} exceeds O(k) bound {lookup_bound:g}" + ) + mode = record.get("path_index_mode") + if mode != "indexed" and not (k == 0 and mode == "unknown"): + failures.append(f"{name}: path_index_mode={mode!r}, expected 'indexed'") + + +def check_artifacts(args: argparse.Namespace) -> list[str]: + k_values = parse_integer_list(args.k, "candidate counts") + operations = tuple(item for item in args.operations.split(",") if item) + if args.require_cow and "cow_checkpoint" not in operations: + operations += ("cow_checkpoint",) + unknown = sorted(set(operations).difference((*DEFAULT_OPERATIONS, "cow_checkpoint"))) + if unknown: + raise ValueError(f"unknown operations: {', '.join(unknown)}") + time_limits = parse_thresholds(args.max_seconds, "time") + rss_limits = parse_thresholds(args.max_rss_bytes, "RSS") + results = read_results(args.results) + structural = read_jsonl(args.structural_metrics) + oracle = read_oracle(args.oracle) + failures = [] + + if args.require_cold_reconcile: + name = "ledger_cold_reconcile" + result = results.get(name) + if result is None: + failures.append(f"{name}: missing benchmark result") + else: + try: + if int(result["exit_code"]) != 0: + failures.append(f"{name}: exit_code={result['exit_code']}") + seconds = float(result["real_seconds"]) + rss = float(result["max_rss_bytes"]) + if not math.isfinite(seconds) or seconds < 0: + failures.append(f"{name}: invalid real_seconds={result['real_seconds']!r}") + if not math.isfinite(rss) or rss <= 0: + failures.append(f"{name}: invalid max_rss_bytes={result['max_rss_bytes']!r}") + if "cold_reconcile" in time_limits and seconds > time_limits["cold_reconcile"]: + failures.append( + f"{name}: {seconds:.2f}s > {time_limits['cold_reconcile']:.2f}s" + ) + if "cold_reconcile" in rss_limits and rss > rss_limits["cold_reconcile"]: + failures.append( + f"{name}: RSS {rss:.0f} > {rss_limits['cold_reconcile']:.0f}" + ) + except (KeyError, ValueError) as error: + failures.append(f"{name}: malformed benchmark result: {error}") + + for operation in operations: + for k in k_values: + name = f"ledger_{operation}_k{k}" + result = results.get(name) + if result is None: + failures.append(f"{name}: missing benchmark result") + else: + try: + if int(result["exit_code"]) != 0: + failures.append(f"{name}: exit_code={result['exit_code']}") + seconds = float(result["real_seconds"]) + rss = float(result["max_rss_bytes"]) + if not math.isfinite(seconds) or seconds < 0: + failures.append( + f"{name}: invalid real_seconds={result['real_seconds']!r}" + ) + if not math.isfinite(rss) or rss <= 0: + failures.append( + f"{name}: invalid max_rss_bytes={result['max_rss_bytes']!r}" + ) + if operation in time_limits and seconds > time_limits[operation]: + failures.append(f"{name}: {seconds:.2f}s > {time_limits[operation]:.2f}s") + if operation in rss_limits and rss > rss_limits[operation]: + failures.append(f"{name}: RSS {rss:.0f} > {rss_limits[operation]:.0f}") + except (KeyError, ValueError) as error: + failures.append(f"{name}: malformed benchmark result: {error}") + + record = structural.get(name) + if record is None: + failures.append(f"{name}: missing structural metrics report") + continue + if record.get("benchmark_operation") != operation: + failures.append(f"{name}: benchmark_operation={record.get('benchmark_operation')!r}") + if record.get("authoritative_input_k") != k: + failures.append(f"{name}: authoritative_input_k={record.get('authoritative_input_k')!r}") + expected_output = k + if record.get("final_changed_output") != expected_output: + failures.append( + f"{name}: final_changed_output={record.get('final_changed_output')!r}, expected {expected_output}" + ) + check_scoped(record, operation, k, failures, name) + check_operation_extras(record, operation, k, failures, name) + reported_rss = require_number( + record, "rss_lifetime_high_water_bytes", failures, name + ) + if reported_rss <= 0: + failures.append(f"{name}: operation RSS must be positive") + if operation in rss_limits and reported_rss > rss_limits[operation]: + failures.append( + f"{name}: operation RSS {reported_rss:.0f} > {rss_limits[operation]:.0f}" + ) + reported_wall_ns = require_number(record, "wall_time_ns", failures, name) + if reported_wall_ns <= 0: + failures.append(f"{name}: operation wall_time_ns must be positive") + reported_seconds = reported_wall_ns / 1_000_000_000 + if operation in time_limits and reported_seconds > time_limits[operation]: + failures.append( + f"{name}: operation wall time {reported_seconds:.2f}s > {time_limits[operation]:.2f}s" + ) + if record.get("final_path_count") != record.get("final_changed_output"): + failures.append( + f"{name}: final_path_count does not match final_changed_output" + ) + check_scope_caps(record, failures, name) + + if operation in ORACLE_OPERATIONS: + oracle_row = oracle.get(name) + if oracle_row is None: + failures.append(f"{name}: missing separate full-scan oracle result") + else: + try: + if int(oracle_row["equal"]) != 1: + failures.append(f"{name}: oracle equality failed") + measured = int(oracle_row["measured_paths"]) + observed = int(oracle_row["oracle_paths"]) + if int(oracle_row["authoritative_input_k"]) != k: + failures.append(f"{name}: oracle authoritative_input_k mismatch") + if int(oracle_row["repo_files"]) != record.get("repo_files"): + failures.append(f"{name}: oracle repo_files mismatch") + if measured != expected_output or observed != expected_output: + failures.append( + f"{name}: measured/oracle path counts are {measured}/{observed}, expected {expected_output}" + ) + if int(oracle_row["oracle_time_ns"]) < 0: + failures.append(f"{name}: oracle_time_ns is negative") + except (KeyError, ValueError) as error: + failures.append(f"{name}: malformed oracle result: {error}") + return failures + + +def main(argv: list[str] | None = None) -> int: + try: + args = parse_args(argv or sys.argv[1:]) + failures = check_artifacts(args) + except (OSError, ValueError) as error: + print(f"changed-path ledger threshold input error: {error}", file=sys.stderr) + return 2 + if failures: + print("changed-path ledger threshold failures:", file=sys.stderr) + for failure in failures: + print(f" - {failure}", file=sys.stderr) + return 1 + print("changed-path ledger thresholds passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check-cli-scale-thresholds.py b/scripts/check-cli-scale-thresholds.py index abce2e1..73444bf 100644 --- a/scripts/check-cli-scale-thresholds.py +++ b/scripts/check-cli-scale-thresholds.py @@ -8,7 +8,8 @@ def main() -> int: if len(sys.argv) < 3: print( "usage: check-cli-scale-thresholds.py RESULTS.tsv name=max_seconds ... " - "[--metrics METRICS.tsv key=max_value ...]", + "[--metrics METRICS.tsv key=max_value ...] " + "[--metric-equals key=expected_value ...]", file=sys.stderr, ) return 2 @@ -17,14 +18,24 @@ def main() -> int: args = sys.argv[2:] metrics_path = None metric_specs = [] + metric_equality_specs = [] if "--metrics" in args: marker = args.index("--metrics") if marker + 1 >= len(args): print("missing METRICS.tsv after --metrics", file=sys.stderr) return 2 metrics_path = pathlib.Path(args[marker + 1]) - metric_specs = args[marker + 2 :] + metric_args = args[marker + 2 :] args = args[:marker] + if "--metric-equals" in metric_args: + equals_marker = metric_args.index("--metric-equals") + metric_specs = metric_args[:equals_marker] + metric_equality_specs = metric_args[equals_marker + 1 :] + else: + metric_specs = metric_args + elif "--metric-equals" in args: + print("--metric-equals requires --metrics METRICS.tsv", file=sys.stderr) + return 2 thresholds = {} for spec in args: @@ -50,6 +61,23 @@ def main() -> int: print(f"invalid metric threshold value for `{key}`: {value}", file=sys.stderr) return 2 + metric_equalities = {} + for spec in metric_equality_specs: + if "=" not in spec: + print( + f"invalid metric equality `{spec}`; expected key=expected_value", + file=sys.stderr, + ) + return 2 + key, value = spec.split("=", 1) + if not key or not value: + print( + f"invalid metric equality `{spec}`; expected key=expected_value", + file=sys.stderr, + ) + return 2 + metric_equalities[key] = value + with results_path.open(newline="") as handle: rows = { row["name"]: row @@ -71,15 +99,31 @@ def main() -> int: failures.append(f"{name}: {seconds:.2f}s > {max_seconds:.2f}s") if metrics_path is not None: - metrics = read_metrics(metrics_path) + metrics = read_metric_values(metrics_path) missing_hint = format_available_metrics(metrics) for key, max_value in metric_thresholds.items(): value = metrics.get(key) if value is None: failures.append(f"{key}: missing from {metrics_path}{missing_hint}") continue + if not isinstance(value, (int, float)): + failures.append(f"{key}: expected numeric value, found {value!r}") + continue if value > max_value: failures.append(f"{key}: {value:.0f} > {max_value:.0f}") + for key, expected in metric_equalities.items(): + value = metrics.get(key) + if value is None: + failures.append(f"{key}: missing from {metrics_path}{missing_hint}") + continue + equal = str(value) == expected + if isinstance(value, (int, float)): + try: + equal = value == float(expected) + except ValueError: + equal = False + if not equal: + failures.append(f"{key}: {value!r} != {expected!r}") if failures: print("CLI scale threshold failures:", file=sys.stderr) @@ -87,12 +131,12 @@ def main() -> int: print(f" - {failure}", file=sys.stderr) return 1 - checked = len(thresholds) + len(metric_thresholds) + checked = len(thresholds) + len(metric_thresholds) + len(metric_equalities) print(f"checked {checked} CLI scale thresholds") return 0 -def read_metrics(metrics_path: pathlib.Path) -> dict[str, float]: +def read_metric_values(metrics_path: pathlib.Path) -> dict[str, object]: metrics = {} with metrics_path.open(newline="") as handle: for line in handle: @@ -108,12 +152,17 @@ def read_metrics(metrics_path: pathlib.Path) -> dict[str, float]: try: metrics[key] = float(value) except ValueError: - continue + metrics[key] = value add_derived_file_metrics(metrics_path, metrics) return metrics -def add_derived_file_metrics(metrics_path: pathlib.Path, metrics: dict[str, float]) -> None: +def read_metrics(metrics_path: pathlib.Path) -> dict[str, object]: + """Backward-compatible alias for callers importing the old helper name.""" + return read_metric_values(metrics_path) + + +def add_derived_file_metrics(metrics_path: pathlib.Path, metrics: dict[str, object]) -> None: root = metrics_path.parent add_file_size_metric( metrics, @@ -128,7 +177,7 @@ def add_derived_file_metrics(metrics_path: pathlib.Path, metrics: dict[str, floa def add_file_size_metric( - metrics: dict[str, float], + metrics: dict[str, object], key: str, path: pathlib.Path, ) -> None: @@ -136,7 +185,7 @@ def add_file_size_metric( metrics[key] = float(path.stat().st_size) -def format_available_metrics(metrics: dict[str, float]) -> str: +def format_available_metrics(metrics: dict[str, object]) -> str: if not metrics: return "; no valid metrics were read" keys = sorted(metrics) diff --git a/scripts/cli-scale-bench.sh b/scripts/cli-scale-bench.sh index 5e2961d..b13cd18 100755 --- a/scripts/cli-scale-bench.sh +++ b/scripts/cli-scale-bench.sh @@ -2,13 +2,25 @@ set -euo pipefail BIN="${TRAIL_BIN:-}" -SCALES="${TRAIL_SCALE_FILES:-10000}" +MODE="${1:-default}" +if [ "$#" -gt 1 ]; then + printf 'usage: %s [changed-path-ledger]\n' "$0" >&2 + exit 2 +fi +if [ "$MODE" = "changed-path-ledger" ]; then + SCALES="${TRAIL_SCALE_FILES:-${REPO_FILES:-1000,100000,1000000}}" +else + SCALES="${TRAIL_SCALE_FILES:-10000}" +fi BASE_DIR="${TRAIL_SCALE_BASE:-/Volumes/Workspace}" RUN_LABEL="${TRAIL_SCALE_LABEL:-$(date +%Y%m%d-%H%M%S)}" RUN_MATERIALIZED="${TRAIL_SCALE_MATERIALIZED:-1}" RUN_BACKUP="${TRAIL_SCALE_BACKUP:-1}" RUN_DAEMON="${TRAIL_SCALE_DAEMON:-1}" RUN_GIT_IMPORT="${TRAIL_SCALE_GIT_IMPORT:-1}" +LEDGER_K_VALUES="${TRAIL_CHANGED_PATH_K_VALUES:-0,1,100}" +RUN_LEDGER_COW="${TRAIL_CHANGED_PATH_COW:-auto}" +LEDGER_TAIL_BOUND="${TRAIL_CHANGED_PATH_TAIL_BOUND:-4096}" if [ -z "$BIN" ]; then cargo build -p trail --release >/dev/null @@ -41,14 +53,34 @@ run_timed() { local code=$? set -e local real rss - real="$(awk '/^real / {v=$2} END {print v+0}' "$stderr")" - rss="$(awk ' - /maximum resident set size kb/ {v=$NF * 1024} - /maximum resident set size/ && $0 !~ / kb / { - for (i=1;i<=NF;i++) if ($i ~ /^[0-9]+$/) v=$i + if ! real="$(awk ' + $1 == "real" && $2 ~ /^[0-9]+([.][0-9]+)?$/ { value=$2; matches++ } + END { if (matches != 1) exit 1; print value } + ' "$stderr")"; then + printf '%s: missing or ambiguous /usr/bin/time real measurement\n' "$name" >&2 + tail -80 "$stderr" >&2 || true + exit 1 + fi + if ! rss="$(awk ' + /maximum resident set size kb/ { + if ($NF !~ /^[0-9]+$/) exit 2 + value=$NF * 1024 + matches++ + next + } + /maximum resident set size/ { + found="" + for (i=1;i<=NF;i++) if ($i ~ /^[0-9]+$/) { found=$i; break } + if (found == "") exit 2 + value=found + matches++ } - END {print v+0} - ' "$stderr")" + END { if (matches != 1 || value <= 0) exit 1; printf "%.0f\n", value } + ' "$stderr")"; then + printf '%s: missing, ambiguous, or zero /usr/bin/time RSS measurement\n' "$name" >&2 + tail -80 "$stderr" >&2 || true + exit 1 + fi printf '%s\t%s\t%s\t%s\n' "$name" "$real" "$rss" "$code" >> "$results" printf 'scale=%s %-36s %8ss rss=%s exit=%s\n' "$scale" "$name" "$real" "$rss" "$code" if [ "$code" -ne 0 ]; then @@ -58,6 +90,67 @@ run_timed() { fi } +run_timed_expected_error() { + local scale="$1" + local name="$2" + local expected_exit="$3" + local expected_code="$4" + shift 4 + run_timed "$scale" "$name" bash -s -- "$expected_exit" "$expected_code" "$@" <<'SH' +set -euo pipefail +expected_exit="$1" +expected_code="$2" +shift 2 +stdout="$(mktemp)" +stderr="$(mktemp)" +cleanup() { + rm -f "$stdout" "$stderr" +} +trap cleanup EXIT +set +e +"$@" >"$stdout" 2>"$stderr" +code=$? +set -e +cat "$stdout" +cat "$stderr" >&2 +if [ "$code" -ne "$expected_exit" ]; then + printf 'expected exit %s, got %s\n' "$expected_exit" "$code" >&2 + exit 1 +fi +python3 - "$expected_code" "$stdout" "$stderr" <<'PY' +import json, pathlib, sys + +expected = sys.argv[1] + +def codes(value): + if isinstance(value, dict): + for key, child in value.items(): + if key == "code" and isinstance(child, str): + yield child + yield from codes(child) + elif isinstance(value, list): + for child in value: + yield from codes(child) + +found = set() +for filename in sys.argv[2:]: + text = pathlib.Path(filename).read_text(errors="replace") + candidates = [text, *text.splitlines()] + for candidate in candidates: + candidate = candidate.strip() + if not candidate: + continue + try: + found.update(codes(json.loads(candidate))) + except json.JSONDecodeError: + pass +if expected not in found: + print(f"expected JSON error code {expected!r}, found {sorted(found)!r}", file=sys.stderr) + sys.exit(1) +PY +SH +} + repo_source_bytes() { python3 - "$1" <<'PY' import pathlib, sys @@ -259,6 +352,620 @@ daemon_rss_bytes() { fi } +append_changed_path_structural_metrics() { + local scale="$1" + local k="$2" + local operation="$3" + local name="$4" + local source="$5" + local stdout="$6" + local repo="$7" + local destination="$8" + python3 - "$scale" "$k" "$operation" "$name" "$source" "$stdout" "$repo" "$destination" "$LEDGER_TAIL_BOUND" <<'PY' +import json, pathlib, sqlite3, sys + +scale, k, operation, name = map(str, sys.argv[1:5]) +source = pathlib.Path(sys.argv[5]) +stdout = pathlib.Path(sys.argv[6]) +repo = pathlib.Path(sys.argv[7]) +destination = pathlib.Path(sys.argv[8]) +tail_bound = int(sys.argv[9]) + +def load_json(path): + return json.loads(path.read_text()) + +def changed_count(payload, operation): + key = { + "workspace_status": "changed_paths", + "workspace_diff": "files", + "workspace_record": "changed_paths", + "materialized_lane_record": "changed_paths", + "structured_patch": "changed_paths", + "cow_checkpoint": "source_paths", + }[operation] + value = payload.get(key) + if not isinstance(value, list): + raise SystemExit(f"{name}: command report omitted list field {key!r}") + return len(value) + +payload = load_json(stdout) +reports = [json.loads(line) for line in source.read_text().splitlines() if line.strip()] +if len(reports) != 1: + raise SystemExit(f"{name}: expected exactly one operation metrics report, found {len(reports)}") +report = reports[0] +required = { + "generation", "operation", "outcome", "input_path_count", + "canonical_path_count", "expanded_path_count", "final_path_count", + "full_filesystem_walk_count", "bounded_filesystem_walk_count", + "filesystem_entry_count", "filesystem_stat_count", "filesystem_read_count", + "filesystem_read_bytes", "filesystem_hash_count", "filesystem_hash_bytes", + "full_root_range_count", "bounded_root_range_count", "root_range_row_count", + "root_point_key_count", "prolly_read_call_count", "prolly_read_key_count", + "prolly_read_value_count", "prolly_read_value_bytes", "prolly_write_call_count", + "prolly_write_key_count", "prolly_write_value_bytes", + "prolly_tree_batch_call_count", "prolly_tree_batch_mutation_count", + "selected_worktree_index_sqlite_accounting_complete", + "selected_worktree_index_sqlite_accounting_disposition", + "selected_worktree_index_sqlite_envelope_count", + "selected_worktree_index_sqlite_not_applicable_count", + "selected_worktree_index_sqlite_full_scan_count", + "selected_worktree_index_sqlite_row_read_count", + "selected_worktree_index_sqlite_row_delete_count", + "selected_worktree_index_sqlite_row_upsert_count", + "selected_worktree_index_sqlite_statement_count", + "selected_worktree_index_sqlite_transaction_count", "selection_comparison_count", + "policy_build_count", "policy_dependency_full_discovery", "policy_dependency_bytes", + "policy_dependency_file_count", "git_subprocess_count", "git_global_work_count", + "git_index_refresh_count", "git_trace2_region_count", "git_trace2_bytes", + "git_fsmonitor_qualification_count", "git_untracked_cache_qualification_count", + "external_adapter_global_work", "git_index_read_count", "git_index_bytes", + "git_shared_index_read_count", "git_shared_index_bytes", "git_output_bytes", + "git_output_record_count", "daemon_snapshot_bytes", "daemon_snapshot_path_count", + "daemon_cumulative_rewrite_count", "daemon_cumulative_rewrite_bytes", + "daemon_cumulative_rewrite_count_total", "daemon_cumulative_rewrite_bytes_total", + "authoritative_candidate_count", "ledger_row_touch_count", + "observer_tail_record_fold_count", "reconciliation_run_count", "manifest_bytes", + "manifest_key_comparison_count", "journal_bytes", "upper_work_count", + "wall_time_ns", "rss_start_bytes", "rss_end_bytes", + "rss_lifetime_high_water_bytes", +} +missing = sorted(required.difference(report)) +if missing: + raise SystemExit(f"{name}: operation metrics report omitted {', '.join(missing)}") +report["metric_source"] = "operation_scope" +if operation in {"materialized_lane_record", "cow_checkpoint"}: + if "upper_recovery_walks" not in payload: + raise SystemExit(f"{name}: command report omitted upper_recovery_walks") + report["upper_recovery_walks"] = int(payload["upper_recovery_walks"]) +if operation == "cow_checkpoint": + if payload.get("generated_path_accounting") != "journal_interval": + raise SystemExit(f"{name}: command report omitted journal_interval generated-path accounting") + report["generated_path_accounting"] = payload["generated_path_accounting"] +if operation in {"materialized_lane_record", "structured_patch"}: + path_index = payload.get("path_index") + required_path_index = { + "mode", "lookup_count", "full_root_path_load_count", + "full_filesystem_path_scan_count", + } + if not isinstance(path_index, dict) or not required_path_index.issubset(path_index): + raise SystemExit(f"{name}: command report omitted complete path_index metrics") + report.update({ + "path_index_full_root_path_load_count": int(path_index["full_root_path_load_count"]), + "path_index_full_filesystem_path_scan_count": int(path_index["full_filesystem_path_scan_count"]), + "path_index_lookup_count": int(path_index["lookup_count"]), + "path_index_mode": path_index["mode"], + }) + +report.update({ + "benchmark": name, + "benchmark_operation": operation, + "repo_files": int(scale), + "authoritative_input_k": int(k), + "final_changed_output": changed_count(payload, operation), + "configured_tail_bound": tail_bound, +}) + +db = repo / ".trail" / "index" / "trail.sqlite" +report["scope_caps"] = [] +if db.is_file(): + connection = sqlite3.connect(db) + try: + query = """ + SELECT scope.scope_id, + (SELECT COUNT(*) FROM changed_path_entries entry WHERE entry.scope_id=scope.scope_id), + scope.max_candidate_rows, + (SELECT COUNT(*) FROM changed_path_prefixes prefix WHERE prefix.scope_id=scope.scope_id), + scope.max_prefix_rows, + COALESCE((SELECT SUM(segment.durable_end_offset) + FROM changed_path_observer_segments segment + WHERE segment.scope_id=scope.scope_id AND segment.state!='retired'), 0), + scope.max_observer_log_bytes, + COALESCE((SELECT MAX(segment.durable_end_offset) + FROM changed_path_observer_segments segment + WHERE segment.scope_id=scope.scope_id AND segment.state!='retired'), 0), + scope.max_segment_bytes + FROM changed_path_scopes scope + WHERE scope.retired_at IS NULL + ORDER BY scope.scope_id + """ + report["scope_caps"] = [ + { + "scope_id": row[0], "candidate_rows": row[1], "candidate_row_cap": row[2], + "prefix_rows": row[3], "prefix_row_cap": row[4], + "observer_log_bytes": row[5], "observer_log_byte_cap": row[6], + "largest_segment_bytes": row[7], "segment_byte_cap": row[8], + } + for row in connection.execute(query) + ] + finally: + connection.close() + +with destination.open("a") as handle: + handle.write(json.dumps(report, sort_keys=True) + "\n") +PY +} + +run_changed_path_scoped() { + local scale="$1" + local k="$2" + local operation="$3" + local name="ledger_${operation}_k${k}" + local repo="$4" + shift 4 + local metrics="$RUN_ROOT/$scale/changed-path-operation-metrics.jsonl" + local segment="$RUN_ROOT/$scale/out/$name.metrics.jsonl" + local before after expected_operation + before="$(python3 - "$metrics" <<'PY' +import pathlib, sys +path = pathlib.Path(sys.argv[1]) +print(path.stat().st_size if path.exists() else 0) +PY +)" + run_timed "$scale" "$name" "$@" + after="$(python3 - "$metrics" <<'PY' +import pathlib, sys +path = pathlib.Path(sys.argv[1]) +print(path.stat().st_size if path.exists() else 0) +PY +)" + case "$operation" in + workspace_status) expected_operation=status ;; + workspace_diff) expected_operation=diff ;; + workspace_record) expected_operation=record ;; + materialized_lane_record) expected_operation=materialized_lane_record ;; + structured_patch) expected_operation=structured_patch ;; + cow_checkpoint) expected_operation=cow_checkpoint ;; + *) printf 'unsupported scoped operation: %s\n' "$operation" >&2; return 2 ;; + esac + python3 - "$metrics" "$before" "$after" "$expected_operation" "$segment" <<'PY' +import json, pathlib, sys +source, start, end, operation, destination = pathlib.Path(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]), sys.argv[4], pathlib.Path(sys.argv[5]) +if end <= start: + raise SystemExit(f"{operation}: operation metrics sidecar did not grow") +with source.open("rb") as handle: + handle.seek(start) + data = handle.read(end - start).decode() +reports = [json.loads(line) for line in data.splitlines() if line.strip()] +matches = [report for report in reports if report.get("operation") == operation] +if len(matches) != 1: + raise SystemExit(f"{operation}: expected one new matching metrics report, found {len(matches)} among {len(reports)} new reports") +destination.write_text(json.dumps(matches[0], sort_keys=True) + "\n") +PY + append_changed_path_structural_metrics \ + "$scale" "$k" "$operation" "$name" "$segment" \ + "$RUN_ROOT/$scale/out/$name.stdout" "$repo" \ + "$RUN_ROOT/$scale/structural-metrics.jsonl" +} + +json_path_set() { + local operation="$1" + local path="$2" + python3 - "$operation" "$path" <<'PY' +import json, pathlib, sys +operation, filename = sys.argv[1:] +payload = json.loads(pathlib.Path(filename).read_text()) +key = { + "workspace_status": "changed_paths", + "workspace_diff": "files", + "workspace_record": "changed_paths", + "materialized_lane_record": "changed_paths", + "structured_patch": "changed_paths", + "cow_checkpoint": "source_paths", +}[operation] +values = payload.get(key) +if not isinstance(values, list): + raise SystemExit(f"oracle report omitted {key!r}") +paths = [] +for value in values: + if isinstance(value, str): + paths.append(value) + elif isinstance(value, dict) and isinstance(value.get("path"), str): + paths.append(value["path"]) + else: + raise SystemExit(f"oracle report has an invalid path entry: {value!r}") +sys.stdout.write("".join(path + "\n" for path in sorted(set(paths)))) +PY +} + +mutate_changed_path_fixture() { + local root="$1" + local k="$2" + local label="$3" + python3 - "$root" "$k" "$label" <<'PY' +import pathlib, sys +root, k, label = pathlib.Path(sys.argv[1]), int(sys.argv[2]), sys.argv[3] +for i in range(k): + path = root / f"pkg_{i // 100:05d}" / f"module_{i % 100:03d}.rs" + with path.open("a") as handle: + handle.write(f"\n// changed-path-ledger {label} {i}\n") +PY +} + +write_changed_path_oracle_manifest() { + local root="$1" + local destination="$2" + python3 - "$root" "$destination" <<'PY' +import hashlib, json, os, pathlib, stat, sys +root, destination = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]) +manifest = {} +macos_storage_names = { + ".DS_Store", ".Spotlight-V100", ".Trashes", ".fseventsd", + ".metadata_never_index", ".metadata_never_index_unless_rootfs", + ".metadata_direct_scope_only", +} +def is_platform_storage_noise(name): + # Match the native NFS-COW adapter's non-user platform metadata filter so + # AppleDouble/Finder/FSEvents artifacts cannot masquerade as source paths + # in the independent full-scan oracle. + return name.startswith("._") or name in macos_storage_names +for directory, names, files in os.walk(root): + relative_directory = pathlib.Path(directory).relative_to(root) + names[:] = [ + name for name in names + if name not in {".trail", ".git"} and not is_platform_storage_noise(name) + ] + for name in files: + if is_platform_storage_noise(name): + continue + path = pathlib.Path(directory) / name + relative = (relative_directory / name).as_posix() + metadata = path.lstat() + if stat.S_ISLNK(metadata.st_mode): + manifest[relative] = {"kind": "symlink", "target": os.readlink(path)} + elif stat.S_ISREG(metadata.st_mode): + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + manifest[relative] = { + "kind": "file", + "sha256": digest.hexdigest(), + "executable": bool(metadata.st_mode & 0o111), + } +destination.write_text(json.dumps(manifest, sort_keys=True)) +PY +} + +run_changed_path_external_oracle() { + local scale="$1" + local k="$2" + local benchmark="$3" + local operation="$4" + local measured="$5" + local root="$6" + local baseline="$7" + local update_baseline="${8:-0}" + local oracle_dir="$RUN_ROOT/$scale/oracle" + local measured_paths="$oracle_dir/$benchmark.measured.paths" + local oracle_paths="$oracle_dir/$benchmark.oracle.paths" + local current="$oracle_dir/$benchmark.current-manifest.json" + local started finished + started="$(python3 -c 'import time; print(time.monotonic_ns())')" + write_changed_path_oracle_manifest "$root" "$current" + python3 - "$baseline" "$current" "$oracle_paths" <<'PY' +import json, pathlib, sys +before = json.loads(pathlib.Path(sys.argv[1]).read_text()) +after = json.loads(pathlib.Path(sys.argv[2]).read_text()) +changed = sorted(path for path in before.keys() | after.keys() if before.get(path) != after.get(path)) +pathlib.Path(sys.argv[3]).write_text("".join(path + "\n" for path in changed)) +PY + finished="$(python3 -c 'import time; print(time.monotonic_ns())')" + json_path_set "$operation" "$measured" >"$measured_paths" + if ! cmp -s "$measured_paths" "$oracle_paths"; then + diff -u "$oracle_paths" "$measured_paths" >&2 || true + printf '%s: measured output differs from the external full-scan oracle\n' "$benchmark" >&2 + printf '%s\t%s\t%s\t%s\t%s\t%s\t0\n' \ + "$benchmark" "$scale" "$k" "$((finished - started))" \ + "$(wc -l < "$measured_paths" | tr -d ' ')" \ + "$(wc -l < "$oracle_paths" | tr -d ' ')" \ + >> "$RUN_ROOT/$scale/oracle-results.tsv" + return 1 + fi + if [ "$update_baseline" = "1" ]; then + mv "$current" "$baseline" + fi + printf '%s\t%s\t%s\t%s\t%s\t%s\t1\n' \ + "$benchmark" "$scale" "$k" "$((finished - started))" \ + "$(wc -l < "$measured_paths" | tr -d ' ')" \ + "$(wc -l < "$oracle_paths" | tr -d ' ')" \ + >> "$RUN_ROOT/$scale/oracle-results.tsv" +} + +prepare_changed_path_external_oracle() { + local scale="$1" + local benchmark="$2" + local root="$3" + local baseline="$4" + local oracle_dir="$RUN_ROOT/$scale/oracle" + local current="$oracle_dir/$benchmark.current-manifest.json" + local started finished + started="$(python3 -c 'import time; print(time.monotonic_ns())')" + write_changed_path_oracle_manifest "$root" "$current" + python3 - "$baseline" "$current" "$oracle_dir/$benchmark.oracle.paths" <<'PY' +import json, pathlib, sys +before = json.loads(pathlib.Path(sys.argv[1]).read_text()) +after = json.loads(pathlib.Path(sys.argv[2]).read_text()) +changed = sorted(path for path in before.keys() | after.keys() if before.get(path) != after.get(path)) +pathlib.Path(sys.argv[3]).write_text("".join(path + "\n" for path in changed)) +PY + finished="$(python3 -c 'import time; print(time.monotonic_ns())')" + printf '%s\n' "$((finished - started))" >"$oracle_dir/$benchmark.oracle-time-ns" +} + +# NFS/FUSE COW views inherit an already independently scanned main baseline. +# Rewalking and rehashing every base file through the virtual mount would make +# the benchmark oracle itself O(N) and obscure the checkpoint measurement. Read +# the known benchmark mutations back through the mounted view and derive the +# next external manifest from that baseline instead. +prepare_cow_changed_path_external_oracle() { + local scale="$1" + local benchmark="$2" + local root="$3" + local baseline="$4" + local k="$5" + local oracle_dir="$RUN_ROOT/$scale/oracle" + local current="$oracle_dir/$benchmark.current-manifest.json" + local started finished + started="$(python3 -c 'import time; print(time.monotonic_ns())')" + python3 - "$baseline" "$root" "$k" "$current" \ + "$oracle_dir/$benchmark.oracle.paths" <<'PY' +import hashlib, json, pathlib, stat, sys +baseline, root, k, current, changed_paths = ( + pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), int(sys.argv[3]), + pathlib.Path(sys.argv[4]), pathlib.Path(sys.argv[5]), +) +before = json.loads(baseline.read_text()) +after = dict(before) +for index in range(k): + relative = f"cow-{k}-{index:03d}.txt" + path = root / relative + metadata = path.lstat() + if not stat.S_ISREG(metadata.st_mode): + raise SystemExit(f"COW oracle mutation is not a regular file: {relative}") + digest = hashlib.sha256(path.read_bytes()).hexdigest() + after[relative] = { + "kind": "file", + "sha256": digest, + "executable": bool(metadata.st_mode & 0o111), + } +current.write_text(json.dumps(after, sort_keys=True)) +changed = sorted(path for path in before.keys() | after.keys() if before.get(path) != after.get(path)) +changed_paths.write_text("".join(path + "\n" for path in changed)) +PY + finished="$(python3 -c 'import time; print(time.monotonic_ns())')" + printf '%s\n' "$((finished - started))" >"$oracle_dir/$benchmark.oracle-time-ns" +} + +finish_changed_path_external_oracle() { + local scale="$1" + local k="$2" + local benchmark="$3" + local operation="$4" + local measured="$5" + local baseline="$6" + local oracle_dir="$RUN_ROOT/$scale/oracle" + local measured_paths="$oracle_dir/$benchmark.measured.paths" + local oracle_paths="$oracle_dir/$benchmark.oracle.paths" + json_path_set "$operation" "$measured" >"$measured_paths" + local equal=1 + if ! cmp -s "$measured_paths" "$oracle_paths"; then + equal=0 + diff -u "$oracle_paths" "$measured_paths" >&2 || true + fi + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$benchmark" "$scale" "$k" "$(cat "$oracle_dir/$benchmark.oracle-time-ns")" \ + "$(wc -l < "$measured_paths" | tr -d ' ')" \ + "$(wc -l < "$oracle_paths" | tr -d ' ')" "$equal" \ + >> "$RUN_ROOT/$scale/oracle-results.tsv" + if [ "$equal" != "1" ]; then + printf '%s: measured output differs from the external full-scan oracle\n' "$benchmark" >&2 + return 1 + fi + mv "$oracle_dir/$benchmark.current-manifest.json" "$baseline" +} + +run_changed_path_ledger_mode() { + for scale in ${SCALES//,/ }; do + case "$scale" in + 1000|100000|1000000) ;; + *) + printf 'changed-path-ledger mode supports scales 1000, 100000, and 1000000; got %s\n' "$scale" >&2 + return 2 + ;; + esac + local work="$RUN_ROOT/$scale" + local repo="$work/repo" + rm -rf "$work" + mkdir -p "$repo" "$work/out" "$work/oracle" + printf 'name\treal_seconds\tmax_rss_bytes\texit_code\n' >"$work/results.tsv" + printf 'benchmark\trepo_files\tauthoritative_input_k\toracle_time_ns\tmeasured_paths\toracle_paths\tequal\n' \ + >"$work/oracle-results.tsv" + : >"$work/structural-metrics.jsonl" + export TRAIL_PERFORMANCE_METRICS=1 + export TRAIL_PERFORMANCE_METRICS_FILE="$work/changed-path-operation-metrics.jsonl" + : >"$TRAIL_PERFORMANCE_METRICS_FILE" + + run_timed "$scale" ledger_generate_repo python3 - "$repo" "$scale" <<'PY' +import pathlib, sys +root, files = pathlib.Path(sys.argv[1]), int(sys.argv[2]) +for index in range(files): + path = root / f"pkg_{index // 100:05d}" / f"module_{index % 100:03d}.rs" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"pub fn value_{index:08d}() -> usize {{ {index} }}\n") +(root / "README.md").write_text(f"# changed-path ledger scale {files}\n") +(root / ".gitignore").write_text("target/\nnode_modules/\n.DS_Store\n") +PY + git -C "$repo" init --quiet + git -C "$repo" config user.name "Trail Scale Benchmark" + git -C "$repo" config user.email "trail-scale@example.invalid" + run_timed "$scale" ledger_git_add git -C "$repo" add --all + run_timed "$scale" ledger_git_commit git -C "$repo" commit --quiet -m "changed-path ledger baseline" + run_timed "$scale" ledger_init "$BIN" --workspace "$repo" --json init --from-git + run_timed "$scale" ledger_cold_reconcile \ + "$BIN" --workspace "$repo" --json index reconcile + cp "$work/out/ledger_cold_reconcile.stdout" \ + "$work/oracle/ledger_workspace_warm.reconcile.json" + "$BIN" --workspace "$repo" --json status \ + >"$work/oracle/ledger_workspace_warm.status.json" + local workspace_oracle_baseline="$work/oracle/workspace-baseline.json" + write_changed_path_oracle_manifest "$repo" "$workspace_oracle_baseline" + + for k in ${LEDGER_K_VALUES//,/ }; do + case "$k" in + 0|1|100) ;; + *) printf 'invalid changed-path candidate count: %s\n' "$k" >&2; return 2 ;; + esac + mutate_changed_path_fixture "$repo" "$k" "workspace-k$k" + run_changed_path_scoped "$scale" "$k" workspace_status "$repo" \ + "$BIN" --workspace "$repo" --json status + run_changed_path_external_oracle "$scale" "$k" "ledger_workspace_status_k$k" \ + workspace_status "$work/out/ledger_workspace_status_k$k.stdout" "$repo" \ + "$workspace_oracle_baseline" + run_changed_path_scoped "$scale" "$k" workspace_diff "$repo" \ + "$BIN" --workspace "$repo" --json diff --dirty + run_changed_path_external_oracle "$scale" "$k" "ledger_workspace_diff_k$k" \ + workspace_diff "$work/out/ledger_workspace_diff_k$k.stdout" "$repo" \ + "$workspace_oracle_baseline" + run_changed_path_scoped "$scale" "$k" workspace_record "$repo" \ + "$BIN" --workspace "$repo" --json record -m "changed-path ledger workspace k=$k" + run_changed_path_external_oracle "$scale" "$k" "ledger_workspace_record_k$k" \ + workspace_record "$work/out/ledger_workspace_record_k$k.stdout" "$repo" \ + "$workspace_oracle_baseline" 1 + done + + run_timed "$scale" ledger_materialized_spawn \ + "$BIN" --workspace "$repo" --json lane spawn ledger-materialized --from main --materialize + local materialized_workdir + materialized_workdir="$(python3 - "$work/out/ledger_materialized_spawn.stdout" <<'PY' +import json, pathlib, sys +print(json.loads(pathlib.Path(sys.argv[1]).read_text()).get("workdir") or "") +PY +)" + if [ -z "$materialized_workdir" ]; then + printf 'changed-path materialized lane did not return a workdir\n' >&2 + return 1 + fi + "$BIN" --workspace "$repo" --json index reconcile --lane ledger-materialized \ + >"$work/oracle/ledger_materialized_warm.reconcile.json" + local materialized_oracle_baseline="$work/oracle/materialized-baseline.json" + write_changed_path_oracle_manifest "$materialized_workdir" "$materialized_oracle_baseline" + for k in ${LEDGER_K_VALUES//,/ }; do + mutate_changed_path_fixture "$materialized_workdir" "$k" "materialized-k$k" + run_changed_path_scoped "$scale" "$k" materialized_lane_record "$repo" \ + "$BIN" --workspace "$repo" --json lane record ledger-materialized \ + -m "changed-path ledger materialized k=$k" + run_changed_path_external_oracle "$scale" "$k" "ledger_materialized_lane_record_k$k" \ + materialized_lane_record "$work/out/ledger_materialized_lane_record_k$k.stdout" \ + "$materialized_workdir" "$materialized_oracle_baseline" 1 + done + + for k in ${LEDGER_K_VALUES//,/ }; do + python3 - "$materialized_workdir" "$work/structured-patch-k$k.json" "$k" <<'PY' +import json, pathlib, sys +root, output, k = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), int(sys.argv[3]) +edits = [] +for i in range(k): + relative = pathlib.Path(f"pkg_{i // 100:05d}") / f"module_{i % 100:03d}.rs" + edits.append({"op": "write", "path": str(relative), "content": (root / relative).read_text() + f"\n// structured patch {k}:{i}\n"}) +output.write_text(json.dumps({"allow_stale": True, "message": f"changed-path structured patch k={k}", "edits": edits})) +PY + run_changed_path_scoped "$scale" "$k" structured_patch "$repo" \ + "$BIN" --workspace "$repo" --json lane apply-patch ledger-materialized \ + --patch "$work/structured-patch-k$k.json" + run_changed_path_external_oracle "$scale" "$k" "ledger_structured_patch_k$k" \ + structured_patch "$work/out/ledger_structured_patch_k$k.stdout" \ + "$materialized_workdir" "$materialized_oracle_baseline" 1 + done + + local cow_mode="" + if [ "$RUN_LEDGER_COW" = "1" ] || { [ "$RUN_LEDGER_COW" = "auto" ] && [ -e /dev/fuse ]; }; then + if [ "$(uname -s)" = "Darwin" ]; then cow_mode="nfs-cow"; else cow_mode="fuse-cow"; fi + elif [ "$RUN_LEDGER_COW" = "auto" ] && [ "$(uname -s)" = "Darwin" ]; then + cow_mode="nfs-cow" + elif [ "$RUN_LEDGER_COW" != "0" ] && [ "$RUN_LEDGER_COW" != "auto" ]; then + printf 'TRAIL_CHANGED_PATH_COW must be 0, 1, or auto\n' >&2 + return 2 + fi + if [ -n "$cow_mode" ]; then + run_timed "$scale" ledger_cow_spawn \ + "$BIN" --workspace "$repo" --json lane spawn ledger-cow --from main --workdir-mode "$cow_mode" + local cow_workdir + cow_workdir="$(python3 - "$work/out/ledger_cow_spawn.stdout" <<'PY' +import json, pathlib, sys +print(json.loads(pathlib.Path(sys.argv[1]).read_text()).get("workdir") or "") +PY + )" + local cow_oracle_baseline="$work/oracle/cow-baseline.json" + cp "$workspace_oracle_baseline" "$cow_oracle_baseline" + for k in ${LEDGER_K_VALUES//,/ }; do + "$BIN" --workspace "$repo" lane mount ledger-cow \ + >"$work/out/ledger_cow_mount_k$k.stdout" 2>"$work/out/ledger_cow_mount_k$k.stderr" & + local mount_pid=$! + python3 - "$cow_workdir" <<'PY' +import pathlib, sys, time +root = pathlib.Path(sys.argv[1]) +deadline = time.time() + 120 +while time.time() < deadline: + if (root / "README.md").is_file(): + raise SystemExit(0) + time.sleep(0.1) +raise SystemExit("COW mount did not become ready") +PY + python3 - "$cow_workdir" "$k" <<'PY' +import pathlib, sys +root, k = pathlib.Path(sys.argv[1]), int(sys.argv[2]) +for i in range(k): + (root / f"cow-{k}-{i:03d}.txt").write_text(f"COW checkpoint {k}:{i}\n") +PY + prepare_cow_changed_path_external_oracle "$scale" "ledger_cow_checkpoint_k$k" \ + "$cow_workdir" "$cow_oracle_baseline" "$k" + "$BIN" --workspace "$repo" lane unmount ledger-cow >/dev/null + wait "$mount_pid" + run_changed_path_scoped "$scale" "$k" cow_checkpoint "$repo" \ + "$BIN" --workspace "$repo" --json lane checkpoint ledger-cow \ + -m "changed-path ledger COW k=$k" + finish_changed_path_external_oracle "$scale" "$k" "ledger_cow_checkpoint_k$k" \ + cow_checkpoint "$work/out/ledger_cow_checkpoint_k$k.stdout" "$cow_oracle_baseline" + done + else + printf 'scale=%s changed-path COW checkpoint skipped (native mount unavailable)\n' "$scale" + fi + + printf 'scale=%s results=%s structural=%s oracle=%s\n' \ + "$scale" "$work/results.tsv" "$work/structural-metrics.jsonl" "$work/oracle-results.tsv" + done +} + +if [ "$MODE" = "changed-path-ledger" ]; then + run_changed_path_ledger_mode + printf 'run_root=%s\n' "$RUN_ROOT" + exit 0 +fi +if [ "$MODE" != "default" ]; then + printf 'unknown benchmark mode: %s\n' "$MODE" >&2 + exit 2 +fi + for scale in ${SCALES//,/ }; do case "$scale" in ''|*[!0-9]*) @@ -269,7 +976,9 @@ for scale in ${SCALES//,/ }; do WORK="$RUN_ROOT/$scale" REPO="$WORK/repo" + EMPTY_REPO="$WORK/empty-root-repo" GIT_REPO="$WORK/git-repo" + GIT_UNMAPPED_REPO="$WORK/git-unmapped-repo" RESULTS="$WORK/results.tsv" rm -rf "$WORK" mkdir -p "$REPO" "$WORK/out" @@ -367,7 +1076,84 @@ PY git -C "$GIT_REPO" config user.email "trail@example.com" git -C "$GIT_REPO" config user.name "Trail" run_timed "$scale" git_commit_initial git -C "$GIT_REPO" commit -m "scale initial" + run_timed "$scale" git_clone_unmapped git clone --no-local --quiet "$GIT_REPO" "$GIT_UNMAPPED_REPO" + git -C "$GIT_UNMAPPED_REPO" config user.email "trail@example.com" + git -C "$GIT_UNMAPPED_REPO" config user.name "Trail" run_timed "$scale" git_init_from_git "$BIN" --workspace "$GIT_REPO" --json init --from-git + run_timed "$scale" agent_git_spawn "$BIN" --workspace "$GIT_REPO" --json lane spawn agent-gitapplybot --from main --no-materialize + python3 - "$GIT_REPO" "$WORK/agent-git-apply.json" "$scale" "$WORK/out/agent_git_spawn.stdout" <<'PY' +import json, pathlib, sys +root = pathlib.Path(sys.argv[1]) +out = pathlib.Path(sys.argv[2]) +files = int(sys.argv[3]) +base_change = json.loads(pathlib.Path(sys.argv[4]).read_text())["base_change"] +changed_paths = max(1, min(100, files // 1000)) +edits = [] +for i in range(changed_paths): + idx = (i * 7919 + 43) % files + d, f = divmod(idx, 100) + rel = pathlib.Path(f"pkg_{d:05d}") / f"module_{f:03d}.rs" + edits.append({ + "op": "write", + "path": str(rel), + "content": (root / rel).read_text() + f"\n// agent Git apply {i}\n", + }) +json.dump({"base_change": base_change, "message": "scale Git apply", "edits": edits}, out.open("w")) +PY + run_timed "$scale" agent_git_apply_patch "$BIN" --workspace "$GIT_REPO" --json lane apply-patch agent-gitapplybot --patch "$WORK/agent-git-apply.json" + run_timed "$scale" agent_git_mark_reviewed "$BIN" --workspace "$GIT_REPO" --json agent mark-reviewed latest --note "scale Git apply reviewed" + run_timed "$scale" agent_git_ready "$BIN" --workspace "$GIT_REPO" --json agent ready latest + run_timed "$scale" agent_git_apply_dry_run "$BIN" --workspace "$GIT_REPO" --json agent apply latest --dry-run + run_timed "$scale" agent_git_apply "$BIN" --workspace "$GIT_REPO" --json agent apply latest + python3 scripts/extract-agent-git-performance.py \ + "$WORK/out/agent_git_apply.stdout" \ + "$WORK/agent-git-metrics.tsv" + GIT_UNMAPPED_HEAD_BEFORE="$(git -C "$GIT_UNMAPPED_REPO" rev-parse HEAD)" + GIT_UNMAPPED_INDEX_BEFORE="$(git hash-object "$GIT_UNMAPPED_REPO/.git/index")" + run_timed "$scale" git_unmapped_init_working_tree "$BIN" --workspace "$GIT_UNMAPPED_REPO" --json init --working-tree + run_timed "$scale" git_unmapped_mappings "$BIN" --workspace "$GIT_UNMAPPED_REPO" --json git mappings + python3 - "$WORK/out/git_unmapped_mappings.stdout" <<'PY' +import json, pathlib, sys +payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) +if payload != []: + raise SystemExit(f"working-tree init unexpectedly created Git mappings: {payload!r}") +PY + run_timed "$scale" agent_git_unmapped_spawn "$BIN" --workspace "$GIT_UNMAPPED_REPO" --json lane spawn agent-gitunmappedbot --from main --no-materialize + python3 - "$GIT_UNMAPPED_REPO" "$WORK/agent-git-unmapped.json" "$WORK/out/agent_git_unmapped_spawn.stdout" <<'PY' +import json, pathlib, sys +root = pathlib.Path(sys.argv[1]) +out = pathlib.Path(sys.argv[2]) +base_change = json.loads(pathlib.Path(sys.argv[3]).read_text())["base_change"] +rel = pathlib.Path("pkg_00000/module_000.rs") +json.dump({ + "base_change": base_change, + "message": "scale missing Git mapping", + "edits": [{ + "op": "write", + "path": str(rel), + "content": (root / rel).read_text() + "\n// missing Git mapping\n", + }], +}, out.open("w")) +PY + run_timed "$scale" agent_git_unmapped_apply_patch "$BIN" --workspace "$GIT_UNMAPPED_REPO" --json lane apply-patch agent-gitunmappedbot --patch "$WORK/agent-git-unmapped.json" + run_timed "$scale" agent_git_unmapped_mark_reviewed "$BIN" --workspace "$GIT_UNMAPPED_REPO" --json agent mark-reviewed latest --note "scale missing mapping reviewed" + run_timed "$scale" agent_git_unmapped_ready "$BIN" --workspace "$GIT_UNMAPPED_REPO" --json agent ready latest + run_timed_expected_error "$scale" agent_git_apply_missing_mapping 10 GIT_MAPPING_REQUIRED "$BIN" --workspace "$GIT_UNMAPPED_REPO" --json agent apply latest --dry-run + if [ "$(git -C "$GIT_UNMAPPED_REPO" rev-parse HEAD)" != "$GIT_UNMAPPED_HEAD_BEFORE" ]; then + printf 'missing-mapping apply changed Git HEAD\n' >&2 + exit 1 + fi + if [ "$(git hash-object "$GIT_UNMAPPED_REPO/.git/index")" != "$GIT_UNMAPPED_INDEX_BEFORE" ]; then + printf 'missing-mapping apply changed Git index\n' >&2 + exit 1 + fi + run_timed "$scale" git_unmapped_mappings_after_apply "$BIN" --workspace "$GIT_UNMAPPED_REPO" --json git mappings + python3 - "$WORK/out/git_unmapped_mappings_after_apply.stdout" <<'PY' +import json, pathlib, sys +payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) +if payload != []: + raise SystemExit(f"missing-mapping apply unexpectedly wrote mappings: {payload!r}") +PY run_timed "$scale" git_mutate_tracked python3 - "$GIT_REPO" "$scale" <<'PY' import pathlib, sys root = pathlib.Path(sys.argv[1]) @@ -495,11 +1281,11 @@ PY python3 - "$WORK/daemon-merge.json" <<'PY' import json, pathlib, sys pathlib.Path(sys.argv[1]).write_text(json.dumps({ - "lane_id": "daemonbot", + "into": "main", "dry_run": True, })) PY - run_http_timed "$scale" daemon_merge_dry_run "$DAEMON_URL" POST /v1/branches/main/merge-lane "$WORK/daemon-merge.json" + run_http_timed "$scale" daemon_merge_dry_run "$DAEMON_URL" POST /v1/lanes/daemonbot/merge "$WORK/daemon-merge.json" run_timed "$scale" daemon_auto_cli_status "$BIN" --workspace "$REPO" --json status run_timed "$scale" daemon_auto_cli_agent_readiness "$BIN" --workspace "$REPO" --json lane readiness daemonbot run_timed "$scale" daemon_cli_status "$BIN" --workspace "$REPO" --daemon-url "$DAEMON_URL" --json status @@ -613,9 +1399,9 @@ PY run_timed "$scale" daemon_cli_agent_timeline "$BIN" --workspace "$REPO" --daemon-url "$DAEMON_URL" --json lane timeline daemonclibot --limit 20 run_timed "$scale" daemon_cli_timeline "$BIN" --workspace "$REPO" --daemon-url "$DAEMON_URL" --json timeline --lane daemonclibot --limit 20 run_timed "$scale" daemon_cli_agent_handoff "$BIN" --workspace "$REPO" --daemon-url "$DAEMON_URL" --json lane handoff daemonclibot - run_timed "$scale" daemon_cli_merge_dry_run "$BIN" --workspace "$REPO" --daemon-url "$DAEMON_URL" --json merge-lane daemonclibot --into main --dry-run - run_timed "$scale" daemon_cli_merge_queue_list "$BIN" --workspace "$REPO" --daemon-url "$DAEMON_URL" --json merge-queue list - run_timed "$scale" daemon_cli_merge_queue_run_empty "$BIN" --workspace "$REPO" --daemon-url "$DAEMON_URL" --json merge-queue run + run_timed "$scale" daemon_cli_merge_dry_run "$BIN" --workspace "$REPO" --daemon-url "$DAEMON_URL" --json lane merge daemonclibot --into main --dry-run + run_timed "$scale" daemon_cli_lane_merge_queue_list "$BIN" --workspace "$REPO" --daemon-url "$DAEMON_URL" --json lane merge-queue list + run_timed "$scale" daemon_cli_lane_merge_queue_run_empty "$BIN" --workspace "$REPO" --daemon-url "$DAEMON_URL" --json lane merge-queue run DAEMON_RSS="$(daemon_rss_bytes "$DAEMON_PID")" kill "$DAEMON_PID" >/dev/null 2>&1 || true wait "$DAEMON_PID" 2>/dev/null || true @@ -643,9 +1429,73 @@ for i in range(max(1, min(50, files // 200))): json.dump({"base_change": base_change, "message": "scale patchbot", "edits": edits}, out.open("w")) PY run_timed "$scale" agent_apply_patch "$BIN" --workspace "$REPO" --json lane apply-patch patchbot --patch "$WORK/patchbot.json" + python3 scripts/extract-path-index-performance.py \ + "$WORK/out/agent_apply_patch.stdout" patch "$WORK/path-index-patch.tsv" run_timed "$scale" agent_readiness "$BIN" --workspace "$REPO" --json lane readiness patchbot - run_timed "$scale" merge_agent_dry_run "$BIN" --workspace "$REPO" --json merge-lane patchbot --into main --direct --dry-run - run_timed "$scale" merge_agent_apply "$BIN" --workspace "$REPO" --json merge-lane patchbot --into main --direct + run_timed "$scale" merge_agent_dry_run "$BIN" --workspace "$REPO" --json lane merge patchbot --into main --direct --dry-run + run_timed "$scale" merge_agent_apply "$BIN" --workspace "$REPO" --json lane merge patchbot --into main --direct + + run_timed "$scale" path_index_rename_spawn "$BIN" --workspace "$REPO" --json lane spawn pathindexrename --from main --no-materialize + python3 - "$REPO" "$WORK/path-index-rename.json" "$WORK/out/path_index_rename_spawn.stdout" <<'PY' +import json, pathlib, sys +root = pathlib.Path(sys.argv[1]) +out = pathlib.Path(sys.argv[2]) +base_change = json.loads(pathlib.Path(sys.argv[3]).read_text())["base_change"] +module = pathlib.Path("pkg_00000/module_000.rs") +json.dump({ + "base_change": base_change, + "message": "path-index delete/add and case rename", + "edits": [ + {"op": "rename", "from": "README.md", "to": "readme.md"}, + {"op": "delete", "path": str(module)}, + { + "op": "write", + "path": str(module), + "content": (root / module).read_text() + "\n// delete then add\n", + }, + ], +}, out.open("w")) +PY + run_timed "$scale" path_index_rename_patch "$BIN" --workspace "$REPO" --json lane apply-patch pathindexrename --patch "$WORK/path-index-rename.json" + python3 scripts/extract-path-index-performance.py \ + "$WORK/out/path_index_rename_patch.stdout" rename_patch "$WORK/path-index-rename.tsv" + + mkdir -p "$EMPTY_REPO" + run_timed "$scale" path_index_empty_init "$BIN" --workspace "$EMPTY_REPO" --json init + run_timed "$scale" path_index_empty_spawn "$BIN" --workspace "$EMPTY_REPO" --json lane spawn pathindexempty --from main --no-materialize + python3 - "$WORK/path-index-empty.json" "$WORK/out/path_index_empty_spawn.stdout" <<'PY' +import json, pathlib, sys +out = pathlib.Path(sys.argv[1]) +base_change = json.loads(pathlib.Path(sys.argv[2]).read_text())["base_change"] +json.dump({ + "base_change": base_change, + "message": "path-index first file", + "edits": [{"op": "write", "path": "FIRST.md", "content": "first file\n"}], +}, out.open("w")) +PY + run_timed "$scale" path_index_empty_patch "$BIN" --workspace "$EMPTY_REPO" --json lane apply-patch pathindexempty --patch "$WORK/path-index-empty.json" + python3 scripts/extract-path-index-performance.py \ + "$WORK/out/path_index_empty_patch.stdout" empty_root_patch "$WORK/path-index-empty.tsv" + + run_timed "$scale" path_index_record_spawn "$BIN" --workspace "$REPO" --json lane spawn pathindexrecord --from main --paths README.md + PATH_INDEX_RECORD_WORKDIR="$(python3 - "$WORK/out/path_index_record_spawn.stdout" <<'PY' +import json, pathlib, sys +value = json.loads(pathlib.Path(sys.argv[1]).read_text()).get("workdir") +print(value or "") +PY +)" + if [ -z "$PATH_INDEX_RECORD_WORKDIR" ]; then + printf 'bounded path-index record lane did not materialize a workdir\n' >&2 + exit 1 + fi + run_timed "$scale" path_index_record_mutate python3 - "$PATH_INDEX_RECORD_WORKDIR/README.md" <<'PY' +import pathlib, sys +path = pathlib.Path(sys.argv[1]) +path.write_text(path.read_text() + "\npath-index bounded record\n") +PY + run_timed "$scale" path_index_record "$BIN" --workspace "$REPO" --json lane record pathindexrecord -m "path-index bounded record" + python3 scripts/extract-path-index-performance.py \ + "$WORK/out/path_index_record.stdout" record "$WORK/path-index-record.tsv" run_timed "$scale" agent_spawn_queuebot "$BIN" --workspace "$REPO" --json lane spawn queuebot --from main --no-materialize python3 - "$REPO" "$WORK/queuebot.json" "$scale" "$WORK/out/agent_spawn_queuebot.stdout" <<'PY' @@ -667,8 +1517,8 @@ for i in range(max(1, min(25, files // 400))): json.dump({"base_change": base_change, "message": "scale queuebot", "edits": edits}, out.open("w")) PY run_timed "$scale" queuebot_apply_patch "$BIN" --workspace "$REPO" --json lane apply-patch queuebot --patch "$WORK/queuebot.json" - run_timed "$scale" merge_queue_add "$BIN" --workspace "$REPO" --json merge-queue add queuebot --into main - run_timed "$scale" merge_queue_run "$BIN" --workspace "$REPO" --json merge-queue run + run_timed "$scale" lane_merge_queue_add "$BIN" --workspace "$REPO" --json lane merge-queue add queuebot --into main + run_timed "$scale" lane_merge_queue_run "$BIN" --workspace "$REPO" --json lane merge-queue run if [ "$RUN_MATERIALIZED" = "1" ]; then run_timed "$scale" agent_spawn_sparse "$BIN" --workspace "$REPO" --json lane spawn sparsebot --from main --paths README.md @@ -730,6 +1580,10 @@ PY dbstat_bytes "$GIT_REPO" "git" workdir_manifest_bytes "$GIT_REPO" "git" fi + if [ -f "$WORK/agent-git-metrics.tsv" ]; then + cat "$WORK/agent-git-metrics.tsv" + fi + cat "$WORK/path-index-"*.tsv printf 'daemon_rss_bytes\t%s\n' "$DAEMON_RSS" du -sk "$REPO" "$REPO/.trail" 2>/dev/null | awk '{print "du_kb_" $2 "\t" $1}' } > "$WORK/metrics.tsv" diff --git a/scripts/extract-agent-git-performance.py b/scripts/extract-agent-git-performance.py new file mode 100644 index 0000000..05a3856 --- /dev/null +++ b/scripts/extract-agent-git-performance.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +import json +import pathlib +import sys + + +def nonnegative_json_integer(value: object, key: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{key} must be a nonnegative JSON integer") + return value + + +def extract_performance_metrics(payload: object) -> dict[str, object]: + if not isinstance(payload, dict): + raise ValueError("agent Git apply JSON must be an object") + export = payload.get("git_export") + if not isinstance(export, dict): + raise ValueError("agent Git apply JSON must contain a git_export object") + performance = export.get("performance") + if not isinstance(performance, dict): + raise ValueError("agent Git apply JSON must contain git_export.performance") + + export_mode = performance.get("export_mode") + if not isinstance(export_mode, str): + raise ValueError("export_mode must be a string") + changed_paths = nonnegative_json_integer( + performance.get("changed_path_count"), "changed_path_count" + ) + blob_writes = nonnegative_json_integer( + performance.get("blob_write_count"), "blob_write_count" + ) + plumbing_commands = nonnegative_json_integer( + performance.get("git_plumbing_command_count"), "git_plumbing_command_count" + ) + return { + "agent_git_export_mode": export_mode, + "agent_git_changed_paths": changed_paths, + "agent_git_blob_writes": blob_writes, + "agent_git_plumbing_commands": plumbing_commands, + } + + +def extract_file(source: pathlib.Path, output: pathlib.Path) -> None: + metrics = extract_performance_metrics(json.loads(source.read_text())) + output.write_text( + "".join(f"{key}\t{value}\n" for key, value in metrics.items()) + ) + + +def main() -> int: + if len(sys.argv) != 3: + print( + "usage: extract-agent-git-performance.py APPLY.json METRICS.tsv", + file=sys.stderr, + ) + return 2 + try: + extract_file(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2])) + except (OSError, json.JSONDecodeError, ValueError) as error: + print(f"invalid agent Git performance report: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/extract-path-index-performance.py b/scripts/extract-path-index-performance.py new file mode 100644 index 0000000..830271a --- /dev/null +++ b/scripts/extract-path-index-performance.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +import json +import pathlib +import re +import sys +import unicodedata + + +PREFIX = re.compile(r"^[a-z][a-z0-9_]*$") + + +def nonnegative_json_integer(value: object, key: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{key} must be a nonnegative JSON integer") + return value + + +def folded_path_key(path: str) -> str: + nfkc = unicodedata.normalize("NFKC", path) + lowered = "".join(character.lower() for character in nfkc) + return unicodedata.normalize("NFC", lowered) + + +def touched_folded_keys(payload: dict[str, object]) -> set[str]: + changed_paths = payload.get("changed_paths") + if not isinstance(changed_paths, list) or not changed_paths: + raise ValueError("completed operation changed_paths must be a non-empty array") + touched = set() + for index, change in enumerate(changed_paths): + if not isinstance(change, dict): + raise ValueError(f"changed_paths[{index}] must be an object") + path = change.get("path") + if not isinstance(path, str) or not path: + raise ValueError(f"changed_paths[{index}].path must be a string") + touched.add(folded_path_key(path)) + old_path = change.get("old_path") + if old_path is not None: + if not isinstance(old_path, str) or not old_path: + raise ValueError(f"changed_paths[{index}].old_path must be a string or null") + touched.add(folded_path_key(old_path)) + return touched + + +def extract_path_index_metrics(payload: object, prefix: str) -> dict[str, object]: + if not PREFIX.fullmatch(prefix): + raise ValueError("metric prefix must use lowercase letters, digits, and underscores") + if not isinstance(payload, dict): + raise ValueError("patch or record JSON must be an object") + operation = payload.get("operation") + if not isinstance(operation, str) or not operation: + raise ValueError("path-index metrics require one completed operation") + path_index = payload.get("path_index") + if not isinstance(path_index, dict): + raise ValueError("operation JSON must contain a path_index object") + + mode = path_index.get("mode") + if not isinstance(mode, str) or not mode: + raise ValueError("mode must be a string") + lookup_count = nonnegative_json_integer( + path_index.get("lookup_count"), "lookup_count" + ) + full_root_loads = nonnegative_json_integer( + path_index.get("full_root_path_load_count"), "full_root_path_load_count" + ) + full_filesystem_scans = nonnegative_json_integer( + path_index.get("full_filesystem_path_scan_count"), + "full_filesystem_path_scan_count", + ) + touched_count = len(touched_folded_keys(payload)) + if lookup_count > touched_count: + raise ValueError( + f"lookup_count {lookup_count} exceeds {touched_count} unique folded touched keys" + ) + + return { + f"{prefix}_path_index_mode": mode, + f"{prefix}_path_index_lookup_count": lookup_count, + f"{prefix}_path_index_full_root_path_load_count": full_root_loads, + f"{prefix}_path_index_full_filesystem_path_scan_count": full_filesystem_scans, + f"{prefix}_path_index_touched_folded_key_count": touched_count, + } + + +def extract_file(source: pathlib.Path, prefix: str, output: pathlib.Path) -> None: + metrics = extract_path_index_metrics(json.loads(source.read_text()), prefix) + output.write_text("".join(f"{key}\t{value}\n" for key, value in metrics.items())) + + +def main() -> int: + if len(sys.argv) != 4: + print( + "usage: extract-path-index-performance.py REPORT.json PREFIX METRICS.tsv", + file=sys.stderr, + ) + return 2 + try: + extract_file(pathlib.Path(sys.argv[1]), sys.argv[2], pathlib.Path(sys.argv[3])) + except (OSError, json.JSONDecodeError, ValueError) as error: + print(f"invalid path-index performance report: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/perf-overlay-cow-large-repos-docker.sh b/scripts/perf-fuse-cow-large-repos-docker.sh similarity index 93% rename from scripts/perf-overlay-cow-large-repos-docker.sh rename to scripts/perf-fuse-cow-large-repos-docker.sh index 95925f2..e9b6481 100755 --- a/scripts/perf-overlay-cow-large-repos-docker.sh +++ b/scripts/perf-fuse-cow-large-repos-docker.sh @@ -6,8 +6,8 @@ OPENCLAW_PATH=${OPENCLAW_PATH:-/Users/haipingfu/Github/openclaw} IMAGE=${RUST_IMAGE:-rust:bookworm} TARGET_DIR=${TRAIL_DOCKER_TARGET_DIR:-/target} CARGO_HOME_DIR=${TRAIL_DOCKER_CARGO_HOME:-/cargo-home} -CARGO_CACHE_VOLUME=${TRAIL_DOCKER_CARGO_CACHE_VOLUME:-trail-overlay-cow-cargo} -TARGET_CACHE_VOLUME=${TRAIL_DOCKER_TARGET_CACHE_VOLUME:-trail-overlay-cow-target} +CARGO_CACHE_VOLUME=${TRAIL_DOCKER_CARGO_CACHE_VOLUME:-trail-fuse-cow-cargo} +TARGET_CACHE_VOLUME=${TRAIL_DOCKER_TARGET_CACHE_VOLUME:-trail-fuse-cow-target} LINUX_CACHE_VOLUME=${TRAIL_LINUX_CACHE_VOLUME:-trail-linux-source-cache} LINUX_REPO_URL=${LINUX_REPO_URL:-https://github.com/torvalds/linux.git} LINUX_REF=${LINUX_REF:-} @@ -157,7 +157,7 @@ headers = [ "land_ms", "db_mb", "worktrees_kb", - "overlay_kb", + "fuse_view_kb", "commit", ] print("\t".join(headers)) @@ -174,7 +174,7 @@ for row in rows: row["land_ms"], round(row["trail_bytes"] / 1024 / 1024, 1), round(row["worktrees_bytes"] / 1024, 1), - round(row["overlay_bytes"] / 1024, 1), + round(row["fuse_bytes"] / 1024, 1), row["head_after"][:12], ] )) @@ -231,7 +231,7 @@ benchmark_repo() { local clone_ms init_ms agent_ms changes_ms land_dry_run_ms land_ms local tracked_files head_before head_after edit_file new_file marker local agent_out agent_err changes_json land_json status_tracked - local trail_bytes worktrees_bytes overlay_bytes + local trail_bytes worktrees_bytes fuse_bytes echo "== $name: clone ==" if [[ "$clone_kind" == "openclaw" ]]; then @@ -250,8 +250,8 @@ benchmark_repo() { tracked_files=$(git -C "$repo" ls-files | wc -l | awk '{print $1}') head_before=$(git -C "$repo" rev-parse HEAD) edit_file=$(choose_edit_file "$repo") - new_file="trail-perf/${name}-overlay.txt" - marker="trail-overlay-perf-${name}-$(date +%s)-$$" + new_file="trail-perf/${name}-fuse.txt" + marker="trail-fuse-perf-${name}-$(date +%s)-$$" agent_out="$BENCH_ROOT/${name}.agent.out" agent_err="$BENCH_ROOT/${name}.agent.err" changes_json="$BENCH_ROOT/${name}.changes.json" @@ -267,13 +267,13 @@ benchmark_repo() { exit 1 fi - echo "== $name: overlay agent edit ==" + echo "== $name: FUSE COW agent edit ==" export PERF_EDIT_FILE="$edit_file" export PERF_NEW_FILE="$new_file" export PERF_MARKER="$marker" run_timed agent_ms "$TRAIL" --workspace "$repo" agent start \ --provider custom \ - --workdir-mode overlay-cow \ + --workdir-mode fuse-cow \ -- bash -lc ' set -euo pipefail test -f "$PERF_EDIT_FILE" @@ -301,7 +301,7 @@ workdir = Path(data["task"]["workdir"]) if not workdir.is_dir(): raise SystemExit(f"missing lane workdir mountpoint: {workdir}") if any(workdir.iterdir()): - raise SystemExit(f"overlay mountpoint should be empty after unmount: {workdir}") + raise SystemExit(f"FUSE COW mountpoint should be empty after unmount: {workdir}") PY echo "== $name: agent land dry-run ==" @@ -322,7 +322,7 @@ PY echo "== $name: agent land ==" run_timed land_ms "$TRAIL" --workspace "$repo" agent land latest \ - -m "Trail overlay perf test for $name" \ + -m "Trail FUSE COW perf test for $name" \ --json >"$land_json" head_after=$(git -C "$repo" rev-parse HEAD) if [[ "$head_before" == "$head_after" ]]; then @@ -341,7 +341,7 @@ PY trail_bytes=$(du_bytes "$repo/.trail") worktrees_bytes=$(du_bytes "$repo/.trail/worktrees") - overlay_bytes=$(du_bytes "$repo/.trail/overlay-cow") + fuse_bytes=$(du_bytes "$repo/.trail/views") append_result_json \ "repo=$name" \ "tracked_files=$tracked_files" \ @@ -353,7 +353,7 @@ PY "land_ms=$land_ms" \ "trail_bytes=$trail_bytes" \ "worktrees_bytes=$worktrees_bytes" \ - "overlay_bytes=$overlay_bytes" \ + "fuse_bytes=$fuse_bytes" \ "head_before=$head_before" \ "head_after=$head_after" \ "edit_file=$edit_file" \ @@ -365,7 +365,7 @@ configure_git cargo build -p trail TRAIL="$CARGO_TARGET_DIR/debug/trail" -BENCH_ROOT=${BENCH_ROOT:-$(mktemp -d /tmp/trail-overlay-perf.XXXXXX)} +BENCH_ROOT=${BENCH_ROOT:-$(mktemp -d /tmp/trail-fuse-perf.XXXXXX)} mkdir -p "$BENCH_ROOT" RESULTS="$BENCH_ROOT/results.jsonl" echo "bench-root=$BENCH_ROOT" diff --git a/scripts/terminal-output-bench.sh b/scripts/terminal-output-bench.sh new file mode 100644 index 0000000..0d412cf --- /dev/null +++ b/scripts/terminal-output-bench.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Release-quality terminal UX baseline. The Rust test covers renderer hot paths; +# this small probe covers binary startup without adding a benchmark dependency. +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$root" + +bin="${TRAIL_BIN:-target/release/trail}" +if [[ ! -x "$bin" ]]; then + cargo build --release -p trail +fi + +python3 - "$bin" <<'PY' +import statistics +import subprocess +import sys +import time + +binary = sys.argv[1] +samples = [] +for _ in range(5): + start = time.perf_counter() + subprocess.run([binary, "--help"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) + samples.append((time.perf_counter() - start) * 1000) +median = statistics.median(samples) +print(f"terminal startup baseline: median={median:.1f}ms samples={','.join(f'{value:.1f}' for value in samples)}") +if median > 250: + raise SystemExit(f"startup regression: {median:.1f}ms exceeds 250ms") +PY + +cargo test --release -p trail terminal_render_baseline -- --ignored --nocapture diff --git a/scripts/test-acp-v1-reference-interop.sh b/scripts/test-acp-v1-reference-interop.sh new file mode 100755 index 0000000..6c1336b --- /dev/null +++ b/scripts/test-acp-v1-reference-interop.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +peer_manifest="$repo_root/tools/acp-v1-reference-peer/Cargo.toml" +peer_target="$repo_root/target/acp-reference" + +cargo build --manifest-path "$peer_manifest" --target-dir "$peer_target" +peer="$peer_target/debug/acp-v1-reference-peer" +if [[ "${OS:-}" == "Windows_NT" ]]; then + peer="$peer.exe" +fi + +trail_target="${CARGO_TARGET_DIR:-$repo_root/target}" +cargo build -p trail --bin trail +export TRAIL_TEST_BIN="$trail_target/debug/trail" +export TRAIL_ACP_REFERENCE_PEER="$peer" +cargo test -p trail --test acp_interop -- --nocapture diff --git a/scripts/test_check_changed_path_ledger_thresholds.py b/scripts/test_check_changed_path_ledger_thresholds.py new file mode 100644 index 0000000..02e0075 --- /dev/null +++ b/scripts/test_check_changed_path_ledger_thresholds.py @@ -0,0 +1,514 @@ +import json +import pathlib +import subprocess +import sys +import tempfile +import unittest + + +SCRIPT = pathlib.Path(__file__).with_name("check-changed-path-ledger-thresholds.py") +OPERATIONS = ( + "workspace_status", + "workspace_diff", + "workspace_record", + "materialized_lane_record", + "structured_patch", +) +SCOPED = { + "workspace_status": "status", + "workspace_diff": "diff", + "workspace_record": "record", + "materialized_lane_record": "materialized_lane_record", + "structured_patch": "structured_patch", + "cow_checkpoint": "cow_checkpoint", +} + + +def scope_caps(): + return [ + { + "scope_id": "workspace:test", + "candidate_rows": 0, + "candidate_row_cap": 250000, + "prefix_rows": 0, + "prefix_row_cap": 16384, + "observer_log_bytes": 128, + "observer_log_byte_cap": 268435456, + "largest_segment_bytes": 128, + "segment_byte_cap": 16777216, + } + ] + + +def structural_record(operation, k): + name = f"ledger_{operation}_k{k}" + final_output = k + common = { + "benchmark": name, + "benchmark_operation": operation, + "repo_files": 1000, + "authoritative_input_k": k, + "final_changed_output": final_output, + "scope_caps": scope_caps(), + } + if operation in SCOPED: + record = { + **common, + "metric_source": "operation_scope", + "generation": 1, + "operation": SCOPED[operation], + "outcome": "success", + "input_path_count": k, + "canonical_path_count": k, + "authoritative_candidate_count": k, + "final_path_count": k, + "filesystem_entry_count": 0, + "filesystem_stat_count": k, + "filesystem_read_count": k if operation == "workspace_record" else 0, + "filesystem_read_bytes": 32 * k, + "filesystem_hash_count": k if operation == "workspace_record" else 0, + "filesystem_hash_bytes": 32 * k, + "expanded_path_count": 0, + "bounded_root_range_count": 0, + "root_range_row_count": 0, + "root_point_key_count": k, + "prolly_read_call_count": k, + "prolly_read_key_count": k, + "prolly_read_value_count": k, + "prolly_read_value_bytes": 32 * k, + "prolly_write_call_count": k, + "prolly_write_key_count": k, + "prolly_write_value_bytes": 32 * k, + "prolly_tree_batch_call_count": min(k, 1), + "prolly_tree_batch_mutation_count": k, + "ledger_row_touch_count": 2 * k, + "observer_tail_record_fold_count": k, + "configured_tail_bound": 4096, + "wall_time_ns": 1000, + "rss_lifetime_high_water_bytes": 1024, + "full_filesystem_walk_count": 0, + "bounded_filesystem_walk_count": 0, + "full_root_range_count": 0, + "selected_worktree_index_sqlite_full_scan_count": 0, + "selected_worktree_index_sqlite_accounting_complete": operation + not in {"workspace_status", "workspace_diff", "workspace_record"}, + "selected_worktree_index_sqlite_accounting_disposition": ( + "not_applicable" + if operation in {"workspace_status", "workspace_diff", "workspace_record"} + else "complete" + ), + "selected_worktree_index_sqlite_envelope_count": ( + 0 + if operation in {"workspace_status", "workspace_diff", "workspace_record"} + else 1 + ), + "selected_worktree_index_sqlite_not_applicable_count": ( + 1 + if operation in {"workspace_status", "workspace_diff", "workspace_record"} + else 0 + ), + "selected_worktree_index_sqlite_row_read_count": 0, + "selected_worktree_index_sqlite_row_delete_count": 0, + "selected_worktree_index_sqlite_row_upsert_count": 0, + "selected_worktree_index_sqlite_statement_count": 0, + "selected_worktree_index_sqlite_transaction_count": 0, + "selection_comparison_count": k, + "policy_build_count": 1, + "policy_dependency_full_discovery": 0, + "policy_dependency_bytes": 0, + "policy_dependency_file_count": 0, + "git_subprocess_count": 0, + "reconciliation_run_count": 0, + "manifest_bytes": 0, + "manifest_key_comparison_count": k, + "journal_bytes": 32 * k, + "upper_work_count": 0, + "external_adapter_global_work": 0, + "git_global_work_count": 0, + "git_index_refresh_count": 0, + "git_trace2_region_count": 0, + "git_trace2_bytes": 0, + "git_fsmonitor_qualification_count": 0, + "git_untracked_cache_qualification_count": 0, + "git_index_read_count": 0, + "git_index_bytes": 0, + "git_shared_index_read_count": 0, + "git_shared_index_bytes": 0, + "git_output_bytes": 0, + "git_output_record_count": 0, + "daemon_snapshot_bytes": 0, + "daemon_snapshot_path_count": k, + "daemon_cumulative_rewrite_count": 0, + "daemon_cumulative_rewrite_bytes": 0, + "daemon_cumulative_rewrite_count_total": 1, + "daemon_cumulative_rewrite_bytes_total": 128, + "rss_start_bytes": 1024, + "rss_end_bytes": 1024, + } + if operation in {"materialized_lane_record", "cow_checkpoint"}: + record["upper_recovery_walks"] = 0 + if operation == "cow_checkpoint": + record["generated_path_accounting"] = "journal_interval" + if operation in {"materialized_lane_record", "structured_patch"}: + record.update( + { + "path_index_full_root_path_load_count": 0, + "path_index_full_filesystem_path_scan_count": 0, + "path_index_lookup_count": k, + "path_index_mode": "unknown" if k == 0 else "indexed", + } + ) + return record + raise AssertionError(f"missing scoped fixture for {operation}") + + +class ChangedPathLedgerThresholdTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = pathlib.Path(self.directory.name) + self.results = self.root / "results.tsv" + self.structural = self.root / "structural-metrics.jsonl" + self.oracle = self.root / "oracle-results.tsv" + self.records = [ + structural_record(operation, k) + for operation in OPERATIONS + for k in (0, 1, 100) + ] + self.write_fixture() + + def write_fixture(self): + result_lines = ["name\treal_seconds\tmax_rss_bytes\texit_code"] + oracle_lines = [ + "benchmark\trepo_files\tauthoritative_input_k\toracle_time_ns\tmeasured_paths\toracle_paths\tequal" + ] + for record in self.records: + name = record["benchmark"] + result_lines.append(f"{name}\t0.25\t1024\t0") + if record["benchmark_operation"] in { + "workspace_status", + "workspace_diff", + "workspace_record", + "materialized_lane_record", + "structured_patch", + }: + k = record["authoritative_input_k"] + final_output = record["final_changed_output"] + oracle_lines.append( + f"{name}\t1000\t{k}\t5000\t{final_output}\t{final_output}\t1" + ) + self.results.write_text("\n".join(result_lines) + "\n") + self.structural.write_text( + "".join(json.dumps(record) + "\n" for record in self.records) + ) + self.oracle.write_text("\n".join(oracle_lines) + "\n") + + def run_checker(self, *extra): + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(self.results), + str(self.structural), + "--oracle", + str(self.oracle), + *extra, + ], + capture_output=True, + text=True, + check=False, + ) + + def record(self, operation="workspace_status", k=1): + return next( + record + for record in self.records + if record["benchmark_operation"] == operation + and record["authoritative_input_k"] == k + ) + + def assert_gate_failure(self, expected): + self.write_fixture() + result = self.run_checker() + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(expected, result.stderr) + + def test_accepts_complete_warm_artifacts(self): + result = self.run_checker( + "--max-seconds", + "workspace_status=1", + "--max-rss-bytes", + "workspace_status=2048", + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("thresholds passed", result.stdout) + + def test_rejects_missing_sidecar_report(self): + self.records.remove(self.record("workspace_diff", 1)) + self.assert_gate_failure("missing structural metrics report") + + def test_rejects_duplicate_sidecar_report(self): + self.write_fixture() + with self.structural.open("a") as handle: + handle.write(json.dumps(self.record("workspace_status", 0)) + "\n") + result = self.run_checker() + self.assertEqual(result.returncode, 2) + self.assertIn("duplicate benchmark", result.stderr) + + def test_rejects_missing_required_structural_field(self): + del self.record("workspace_status", 1)["filesystem_stat_count"] + self.assert_gate_failure("missing structural fields") + + def test_rejects_any_warm_global_work(self): + self.record("workspace_diff", 100)["full_filesystem_walk_count"] = 1 + self.assert_gate_failure("full_filesystem_walk_count must remain zero") + + def test_rejects_global_work_for_lane_patch_and_cow_operations(self): + cases = ( + ("materialized_lane_record", "external_adapter_global_work"), + ("structured_patch", "bounded_filesystem_walk_count"), + ) + for operation, field in cases: + with self.subTest(operation=operation, field=field): + self.setUp() + self.record(operation, 1)[field] = 1 + self.assert_gate_failure(f"{field} must remain zero") + + self.setUp() + self.records.extend(structural_record("cow_checkpoint", k) for k in (0, 1, 100)) + self.record("cow_checkpoint", 1)["daemon_cumulative_rewrite_count"] = 1 + self.write_fixture() + result = self.run_checker("--require-cow") + self.assertEqual(result.returncode, 1) + self.assertIn("daemon_cumulative_rewrite_count must remain zero", result.stderr) + + def test_rejects_incomplete_sqlite_accounting(self): + self.record("workspace_status", 0)[ + "selected_worktree_index_sqlite_statement_count" + ] = 1 + self.assert_gate_failure("SQLite work occurred on a not_applicable operation") + + def test_rejects_false_complete_sqlite_accounting_claim(self): + self.record("workspace_status", 0)[ + "selected_worktree_index_sqlite_accounting_complete" + ] = True + self.assert_gate_failure("disposition must be independently proven not_applicable") + + def test_rejects_untyped_false_zero_sqlite_accounting(self): + record = self.record("workspace_diff", 0) + record["selected_worktree_index_sqlite_accounting_disposition"] = "ambiguous" + record["selected_worktree_index_sqlite_not_applicable_count"] = 0 + self.assert_gate_failure("independently proven not_applicable") + + def test_rejects_missing_or_mixed_selected_index_envelope(self): + missing = self.record("structured_patch", 0) + missing["selected_worktree_index_sqlite_accounting_complete"] = False + missing["selected_worktree_index_sqlite_accounting_disposition"] = "ambiguous" + missing["selected_worktree_index_sqlite_envelope_count"] = 0 + self.write_fixture() + result = self.run_checker() + self.assertEqual(result.returncode, 1) + self.assertIn("SQLite accounting must be complete", result.stderr) + self.assertIn("requires an accounting envelope", result.stderr) + + self.setUp() + mixed = self.record("materialized_lane_record", 1) + mixed["selected_worktree_index_sqlite_accounting_complete"] = False + mixed["selected_worktree_index_sqlite_accounting_disposition"] = "ambiguous" + mixed["selected_worktree_index_sqlite_not_applicable_count"] = 1 + self.assert_gate_failure("mixes complete and not_applicable claims") + + def test_rejects_unknown_metrics_and_unbounded_tail_configuration(self): + record = self.record("workspace_status", 1) + record["unreviewed_repository_work_count"] = 0 + record["configured_tail_bound"] = 100000 + self.write_fixture() + result = self.run_checker() + self.assertEqual(result.returncode, 1) + self.assertIn("unknown structural fields", result.stderr) + self.assertIn("exceeds audited cap 4096", result.stderr) + + def test_rejects_repository_sized_values_for_every_work_counter_class(self): + cases = ( + "filesystem_entry_count", + "filesystem_hash_count", + "filesystem_hash_bytes", + "root_range_row_count", + "prolly_read_key_count", + "prolly_read_value_bytes", + "prolly_write_key_count", + "selection_comparison_count", + "policy_dependency_file_count", + "policy_dependency_bytes", + "git_subprocess_count", + "git_output_bytes", + "git_output_record_count", + "daemon_snapshot_path_count", + "daemon_snapshot_bytes", + "manifest_key_comparison_count", + "journal_bytes", + ) + for field in cases: + with self.subTest(field=field): + self.setUp() + self.record("workspace_record", 1)[field] = 100_000_000 + self.assert_gate_failure(f"{field}=1e+08 exceeds O(k) bound") + + def test_rejects_repository_sized_sqlite_and_path_index_work(self): + record = self.record("structured_patch", 1) + record["selected_worktree_index_sqlite_statement_count"] = 100_000 + record["selected_worktree_index_sqlite_row_read_count"] = 100_000 + record["path_index_lookup_count"] = 100_000 + self.write_fixture() + result = self.run_checker() + self.assertEqual(result.returncode, 1) + self.assertIn("selected_worktree_index_sqlite_statement_count", result.stderr) + self.assertIn("selected_worktree_index_sqlite_row_read_count", result.stderr) + self.assertIn("path_index_lookup_count", result.stderr) + + def test_rejects_policy_dependency_full_discovery(self): + self.record("workspace_record", 1)["policy_dependency_full_discovery"] = 1 + self.assert_gate_failure("policy_dependency_full_discovery must remain zero") + + def test_rejects_candidate_and_ledger_locality_overruns(self): + record = self.record("workspace_record", 1) + record["filesystem_read_count"] = 99 + record["ledger_row_touch_count"] = 99 + self.write_fixture() + result = self.run_checker() + self.assertEqual(result.returncode, 1) + self.assertIn("filesystem_read_count", result.stderr) + self.assertIn("ledger_row_touch_count", result.stderr) + + def test_rejects_observer_tail_overrun(self): + record = self.record("workspace_status", 100) + record["observer_tail_record_fold_count"] = 4097 + self.assert_gate_failure("observer tail folded") + + def test_rejects_each_persisted_scope_cap(self): + fields = [ + ("candidate_rows", "candidate_row_cap"), + ("prefix_rows", "prefix_row_cap"), + ("observer_log_bytes", "observer_log_byte_cap"), + ("largest_segment_bytes", "segment_byte_cap"), + ] + for value, cap in fields: + with self.subTest(value=value): + self.setUp() + scope = self.record("workspace_status", 0)["scope_caps"][0] + scope[value] = scope[cap] + 1 + self.assert_gate_failure(value) + + def test_rejects_missing_or_mismatched_oracle(self): + lines = self.oracle.read_text().splitlines() + self.oracle.write_text("\n".join(line for line in lines if "workspace_diff_k1" not in line) + "\n") + result = self.run_checker() + self.assertEqual(result.returncode, 1) + self.assertIn("missing separate full-scan oracle", result.stderr) + + self.write_fixture() + self.oracle.write_text(self.oracle.read_text().replace( + "ledger_workspace_status_k1\t1000\t1\t5000\t1\t1\t1", + "ledger_workspace_status_k1\t1000\t1\t5000\t1\t0\t0", + )) + result = self.run_checker() + self.assertEqual(result.returncode, 1) + self.assertIn("oracle equality failed", result.stderr) + + def test_rejects_operation_report_recovery_walk(self): + self.record("materialized_lane_record", 1)["upper_recovery_walks"] = 1 + self.assert_gate_failure("upper_recovery_walks must remain zero") + + def test_cow_requires_typed_journal_interval_accounting(self): + self.records.extend(structural_record("cow_checkpoint", k) for k in (0, 1, 100)) + self.record("cow_checkpoint", 1)["generated_path_accounting"] = "recursive_inventory" + self.write_fixture() + result = self.run_checker("--require-cow") + self.assertEqual(result.returncode, 1) + self.assertIn("expected 'journal_interval'", result.stderr) + + def test_rejects_time_rss_and_output_count(self): + self.record("structured_patch", 100)["final_changed_output"] = 99 + self.write_fixture() + result = self.run_checker( + "--max-seconds", + "workspace_status=0.1", + "--max-rss-bytes", + "workspace_diff=512", + ) + self.assertEqual(result.returncode, 1) + self.assertIn("0.25s > 0.10s", result.stderr) + self.assertIn("RSS 1024", result.stderr) + self.assertIn("final_changed_output=99", result.stderr) + + def test_require_cow_demands_all_candidate_counts(self): + result = self.run_checker("--require-cow") + self.assertEqual(result.returncode, 1) + self.assertIn("ledger_cow_checkpoint_k0: missing", result.stderr) + + def test_require_cold_reconcile_demands_and_gates_timed_result(self): + result = self.run_checker("--require-cold-reconcile") + self.assertEqual(result.returncode, 1) + self.assertIn("ledger_cold_reconcile: missing benchmark result", result.stderr) + + with self.results.open("a") as handle: + handle.write("ledger_cold_reconcile\t2.5\t4096\t0\n") + result = self.run_checker( + "--require-cold-reconcile", + "--max-seconds", + "cold_reconcile=2", + "--max-rss-bytes", + "cold_reconcile=2048", + ) + self.assertEqual(result.returncode, 1) + self.assertIn("ledger_cold_reconcile: 2.50s > 2.00s", result.stderr) + self.assertIn("ledger_cold_reconcile: RSS 4096", result.stderr) + + def test_structured_patch_zero_is_a_real_successful_empty_operation(self): + record = self.record("structured_patch", 0) + self.assertEqual(record["authoritative_input_k"], 0) + self.assertEqual(record["authoritative_candidate_count"], 0) + self.assertEqual(record["final_changed_output"], 0) + result = self.run_checker() + self.assertEqual(result.returncode, 0, result.stderr) + + def test_rejects_exact_file_prefix_expansion_and_bounded_walk(self): + record = self.record("workspace_status", 0) + record["authoritative_candidate_count"] = 1 + record["expanded_path_count"] = 1000 + record["filesystem_stat_count"] = 1000 + record["filesystem_read_count"] = 1000 + record["bounded_filesystem_walk_count"] = 1 + self.write_fixture() + result = self.run_checker() + self.assertEqual(result.returncode, 1) + self.assertIn("bounded_filesystem_walk_count must remain zero", result.stderr) + self.assertIn("do not equal exact-file k=0", result.stderr) + self.assertIn("expanded paths 1000 exceed exact-file k=0", result.stderr) + + def test_rejects_zero_or_nonfinite_measurements(self): + self.write_fixture() + self.results.write_text( + self.results.read_text().replace( + "ledger_materialized_lane_record_k0\t0.25\t1024\t0", + "ledger_materialized_lane_record_k0\t0\t0\t0", + ) + ) + result = self.run_checker() + self.assertEqual(result.returncode, 1) + self.assertIn("invalid max_rss_bytes='0'", result.stderr) + + self.write_fixture() + self.results.write_text( + self.results.read_text().replace( + "ledger_materialized_lane_record_k0\t0.25\t1024\t0", + "ledger_materialized_lane_record_k0\tnan\t1024\t0", + ) + ) + result = self.run_checker() + self.assertEqual(result.returncode, 1) + self.assertIn("invalid real_seconds='nan'", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_check_cli_scale_thresholds.py b/scripts/test_check_cli_scale_thresholds.py new file mode 100644 index 0000000..212548b --- /dev/null +++ b/scripts/test_check_cli_scale_thresholds.py @@ -0,0 +1,95 @@ +import importlib.util +import pathlib +import subprocess +import sys +import tempfile +import unittest + + +SCRIPT = pathlib.Path(__file__).with_name("check-cli-scale-thresholds.py") +SPEC = importlib.util.spec_from_file_location("scale_thresholds", SCRIPT) +module = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(module) + + +class ThresholdMetricTests(unittest.TestCase): + def run_checker(self, metrics_text, *specs): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + root = pathlib.Path(directory.name) + results = root / "results.tsv" + results.write_text( + "name\treal_seconds\tmax_rss_bytes\texit_code\n" + "agent_git_apply\t0.25\t1024\t0\n" + ) + metrics = root / "metrics.tsv" + metrics.write_text(metrics_text) + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(results), + "agent_git_apply=1", + "--metrics", + str(metrics), + *specs, + ], + capture_output=True, + text=True, + check=False, + ) + + def test_reads_structural_string_metrics(self): + with tempfile.TemporaryDirectory() as directory: + metrics = pathlib.Path(directory) / "metrics.tsv" + metrics.write_text( + "agent_git_export_mode\tmapped_delta\n" + "agent_git_changed_paths\t1\n" + ) + parsed = module.read_metric_values(metrics) + self.assertEqual(parsed["agent_git_export_mode"], "mapped_delta") + self.assertEqual(parsed["agent_git_changed_paths"], 1.0) + + def test_checks_numeric_ceiling_and_string_equality_together(self): + result = self.run_checker( + ( + "agent_git_export_mode\tmapped_delta\n" + "agent_git_changed_paths\t1\n" + ), + "agent_git_changed_paths=1", + "--metric-equals", + "agent_git_export_mode=mapped_delta", + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("checked 3 CLI scale thresholds", result.stdout) + + def test_rejects_mismatched_metric_equality(self): + result = self.run_checker( + "agent_git_export_mode\tfull_snapshot\n", + "--metric-equals", + "agent_git_export_mode=mapped_delta", + ) + self.assertEqual(result.returncode, 1) + self.assertIn("agent_git_export_mode", result.stderr) + self.assertIn("!=", result.stderr) + + def test_rejects_missing_metric_equality(self): + result = self.run_checker( + "agent_git_changed_paths\t1\n", + "--metric-equals", + "agent_git_export_mode=mapped_delta", + ) + self.assertEqual(result.returncode, 1) + self.assertIn("agent_git_export_mode: missing", result.stderr) + + def test_rejects_nonnumeric_value_for_numeric_ceiling(self): + result = self.run_checker( + "agent_git_changed_paths\tone\n", + "agent_git_changed_paths=1", + ) + self.assertEqual(result.returncode, 1) + self.assertIn("expected numeric value", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_cli_scale_bench_source.py b/scripts/test_cli_scale_bench_source.py new file mode 100644 index 0000000..a88d72e --- /dev/null +++ b/scripts/test_cli_scale_bench_source.py @@ -0,0 +1,250 @@ +import hashlib +import json +import pathlib +import subprocess +import sys +import tempfile +import unittest + + +SCRIPT = pathlib.Path(__file__).with_name("cli-scale-bench.sh") +ROOT = SCRIPT.parent.parent + + +class CliScaleBenchSourceTests(unittest.TestCase): + def test_changed_path_oracle_excludes_native_macos_storage_noise(self): + source = SCRIPT.read_text() + function = source.split("write_changed_path_oracle_manifest() {", 1)[1].split( + "\nPY\n}", 1 + )[0] + program = function.split("<<'PY'\n", 1)[1].rsplit("\nPY", 1)[0] + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) / "root" + root.mkdir() + (root / "source.txt").write_text("source") + (root / "._source.txt").write_text("appledouble") + (root / ".DS_Store").write_text("finder") + (root / ".fseventsd").mkdir() + (root / ".fseventsd" / "state").write_text("native") + destination = pathlib.Path(directory) / "manifest.json" + result = subprocess.run( + [sys.executable, "-c", program, str(root), str(destination)], + capture_output=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(destination.read_text()), { + "source.txt": { + "kind": "file", + "sha256": "41cf6794ba4200b839c53531555f0f3998df4cbb01a4d5cb0b94e3ca5e23947d", + "executable": False, + } + }) + + def test_cow_oracle_derives_from_scanned_baseline_and_reads_only_mutations(self): + source = SCRIPT.read_text() + function = source.split("prepare_cow_changed_path_external_oracle() {", 1)[1].split( + "\nPY\n", 1 + )[0] + program = function.split("<<'PY'\n", 1)[1] + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) / "mounted-view" + root.mkdir() + mutation = root / "cow-1-000.txt" + mutation.write_text("COW checkpoint 1:0\n") + baseline = pathlib.Path(directory) / "baseline.json" + baseline.write_text(json.dumps({ + "base.txt": { + "kind": "file", + "sha256": "baseline-digest", + "executable": False, + } + })) + current = pathlib.Path(directory) / "current.json" + changed = pathlib.Path(directory) / "changed.paths" + result = subprocess.run( + [ + sys.executable, + "-c", + program, + str(baseline), + str(root), + "1", + str(current), + str(changed), + ], + capture_output=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + manifest = json.loads(current.read_text()) + self.assertEqual(manifest["base.txt"]["sha256"], "baseline-digest") + self.assertEqual( + manifest["cow-1-000.txt"]["sha256"], + hashlib.sha256(mutation.read_bytes()).hexdigest(), + ) + self.assertEqual(changed.read_text(), "cow-1-000.txt\n") + + def test_changed_path_empty_measured_set_is_byte_identical_to_empty_oracle(self): + source = SCRIPT.read_text() + function = source.split("json_path_set() {", 1)[1].split("\n}\n", 1)[0] + program = function.split("<<'PY'\n", 1)[1].rsplit("\nPY", 1)[0] + with tempfile.TemporaryDirectory() as directory: + report = pathlib.Path(directory) / "report.json" + for operation, payload in ( + ("workspace_status", {"changed_paths": []}), + ("workspace_diff", {"files": []}), + ): + with self.subTest(operation=operation): + report.write_text(json.dumps(payload)) + result = subprocess.run( + [sys.executable, "-c", program, operation, str(report)], + capture_output=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, b"") + + def test_changed_path_mode_separates_measured_metrics_and_oracle_work(self): + source = SCRIPT.read_text() + self.assertIn('MODE="${1:-default}"', source) + self.assertIn('TRAIL_PERFORMANCE_METRICS_FILE="$work/changed-path-operation-metrics.jsonl"', source) + self.assertLess( + source.index('export TRAIL_PERFORMANCE_METRICS_FILE='), + source.index('ledger_workspace_warm.status.json'), + ) + self.assertLess( + source.index('export TRAIL_PERFORMANCE_METRICS_FILE='), + source.index('--json init --from-git'), + ) + self.assertIn('handle.seek(start)', source) + self.assertIn('expected one new matching metrics report', source) + self.assertIn('>"$work/structural-metrics.jsonl"', source) + self.assertIn('>"$work/oracle-results.tsv"', source) + self.assertIn('write_changed_path_oracle_manifest', source) + self.assertNotIn('TRAIL_PERFORMANCE_METRICS=0 "$BIN"', source) + self.assertIn('LEDGER_K_VALUES="${TRAIL_CHANGED_PATH_K_VALUES:-0,1,100}"', source) + self.assertIn('git -C "$repo" commit --quiet', source) + self.assertIn('--json init --from-git', source) + self.assertIn('run_timed "$scale" ledger_cold_reconcile', source) + self.assertIn('for i in range(k):', source) + self.assertNotIn('edit_count = 1 if k == 0 else k', source) + self.assertIn('expected exactly one operation metrics report', source) + self.assertIn('bounded_filesystem_walk_count', source) + self.assertIn('missing or ambiguous /usr/bin/time real measurement', source) + self.assertIn('missing, ambiguous, or zero /usr/bin/time RSS measurement', source) + self.assertNotIn('ledger_structured_patch_k0.report.json', source) + self.assertNotIn('ledger_structured_patch_k0 7 PATCH_REJECTED', source) + self.assertIn('selected_worktree_index_sqlite_accounting_complete', source) + self.assertIn('selected_worktree_index_sqlite_accounting_disposition', source) + self.assertIn('selected_worktree_index_sqlite_not_applicable_count', source) + self.assertIn('filesystem_hash_count', source) + self.assertIn('prolly_read_key_count', source) + self.assertIn('daemon_snapshot_path_count', source) + self.assertIn('generated_path_accounting', source) + self.assertIn('journal_interval', source) + self.assertIn('policy_dependency_full_discovery', source) + for operation in [ + "workspace_status", + "workspace_diff", + "workspace_record", + "materialized_lane_record", + "structured_patch", + "cow_checkpoint", + ]: + self.assertIn(operation, source) + self.assertIn('scripts/cli-scale-bench.sh changed-path-ledger', (ROOT / ".github/workflows/scale.yml").read_text()) + ci = (ROOT / ".github/workflows/ci.yml").read_text() + self.assertIn("Changed-path Ledger 1k Benchmark", ci) + self.assertIn("check-changed-path-ledger-thresholds.py", ci) + + def test_native_changed_path_workflow_covers_linux_macos_and_large_scales(self): + source = (ROOT / ".github/workflows/changed-path-ledger-native.yml").read_text() + self.assertIn("ubuntu-latest", source) + self.assertIn("macos-latest", source) + self.assertIn("100000", source) + self.assertIn("1000000", source) + self.assertIn("changed_path_ledger_linux", source) + self.assertIn("changed_path_ledger_macos", source) + self.assertIn("check-changed-path-ledger-thresholds.py", source) + self.assertIn("pull_request:", source) + self.assertIn("branches: [main]", source) + self.assertIn("findmnt --noheadings --output FSTYPE", source) + self.assertIn("diskutil info", source) + self.assertIn('df -P "${{ runner.temp }}"', source) + self.assertIn('diskutil info "$device"', source) + self.assertNotIn('diskutil info "${{ runner.temp }}"', source) + self.assertIn("--require-cow", source) + self.assertIn("--require-cold-reconcile", source) + self.assertIn('--max-seconds "cow_checkpoint=$cow_seconds"', source) + self.assertIn('--max-rss-bytes "cow_checkpoint=$rss"', source) + + def test_exact_sha_native_gates_block_tagging_and_every_dist_release(self): + native = (ROOT / ".github/workflows/changed-path-ledger-native.yml").read_text() + automation = (ROOT / ".github/workflows/release-automation.yml").read_text() + release = (ROOT / ".github/workflows/release.yml").read_text() + cargo = (ROOT / "Cargo.toml").read_text() + activation = ( + ROOT / "trail/src/db/change_ledger/activation.rs" + ).read_text() + + self.assertIn("workflow_call:", native) + self.assertIn('default: "100000"', native) + self.assertIn("default: true", native) + self.assertIn("exact-sha-native-ledger:", automation) + self.assertIn("needs: [release-please, exact-sha-native-ledger]", automation) + self.assertIn('plan-jobs = ["./changed-path-ledger-native"]', cargo) + self.assertIn("custom-changed-path-ledger-native:", release) + custom_gate = release.split("custom-changed-path-ledger-native:", 1)[1] + self.assertIn( + "uses: ./.github/workflows/changed-path-ledger-native.yml", + custom_gate, + ) + build = release.split("build-local-artifacts:", 1)[1] + self.assertIn("- custom-changed-path-ledger-native", build) + self.assertIn("exact_sha_tag_gate=Release Automation/exact-sha-native-ledger", activation) + self.assertIn( + "exact_sha_publish_gate=Release/custom-changed-path-ledger-native", + activation, + ) + + def test_unmapped_clone_uses_transport_instead_of_local_object_copy(self): + source = SCRIPT.read_text() + self.assertIn( + 'git clone --no-local --quiet "$GIT_REPO" "$GIT_UNMAPPED_REPO"', + source, + ) + + def test_git_plumbing_command_ceiling_is_constant_in_ci_and_scale_gates(self): + for relative in [".github/workflows/ci.yml", ".github/workflows/scale.yml"]: + with self.subTest(workflow=relative): + source = (ROOT / relative).read_text() + self.assertIn("agent_git_plumbing_commands=5", source) + + def test_path_index_patch_and_bounded_record_reports_are_extracted(self): + source = SCRIPT.read_text() + self.assertIn('"$WORK/out/agent_apply_patch.stdout" patch', source) + self.assertIn('"$WORK/out/path_index_record.stdout" record', source) + self.assertIn('"$WORK/out/path_index_empty_patch.stdout" empty_root_patch', source) + self.assertIn('"$WORK/out/path_index_rename_patch.stdout" rename_patch', source) + self.assertIn('cat "$WORK/path-index-"*.tsv', source) + + def test_ci_and_scheduled_workflows_gate_path_index_structure(self): + for relative in [".github/workflows/ci.yml", ".github/workflows/scale.yml"]: + with self.subTest(workflow=relative): + source = (ROOT / relative).read_text() + for prefix in ["patch", "record", "empty_root_patch", "rename_patch"]: + self.assertIn( + f"{prefix}_path_index_full_root_path_load_count=0", source + ) + self.assertIn( + f"{prefix}_path_index_full_filesystem_path_scan_count=0", + source, + ) + self.assertIn( + f"{prefix}_path_index_mode=indexed", source + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_extract_agent_git_performance.py b/scripts/test_extract_agent_git_performance.py new file mode 100644 index 0000000..8fc296b --- /dev/null +++ b/scripts/test_extract_agent_git_performance.py @@ -0,0 +1,76 @@ +import importlib.util +import json +import pathlib +import tempfile +import unittest + + +SCRIPT = pathlib.Path(__file__).with_name("extract-agent-git-performance.py") +SPEC = importlib.util.spec_from_file_location("agent_git_performance", SCRIPT) +module = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(module) + + +class AgentGitPerformanceTests(unittest.TestCase): + def report(self, **overrides): + performance = { + "export_mode": "mapped_delta", + "changed_path_count": 1, + "blob_write_count": 1, + "git_plumbing_command_count": 5, + } + performance.update(overrides) + return {"git_export": {"performance": performance}} + + def test_extracts_strict_performance_metrics(self): + self.assertEqual( + module.extract_performance_metrics(self.report()), + { + "agent_git_export_mode": "mapped_delta", + "agent_git_changed_paths": 1, + "agent_git_blob_writes": 1, + "agent_git_plumbing_commands": 5, + }, + ) + + def test_rejects_invalid_export_mode_type(self): + with self.assertRaisesRegex(ValueError, "export_mode must be a string"): + module.extract_performance_metrics(self.report(export_mode=1)) + + def test_rejects_non_integer_and_negative_counters(self): + for key, value in [ + ("changed_path_count", True), + ("changed_path_count", "1"), + ("changed_path_count", 1.5), + ("changed_path_count", -1), + ("blob_write_count", False), + ("blob_write_count", "1"), + ("blob_write_count", 1.5), + ("blob_write_count", -1), + ("git_plumbing_command_count", False), + ("git_plumbing_command_count", "5"), + ("git_plumbing_command_count", 4.5), + ("git_plumbing_command_count", -1), + ]: + with self.subTest(key=key, value=value): + with self.assertRaisesRegex(ValueError, "nonnegative JSON integer"): + module.extract_performance_metrics(self.report(**{key: value})) + + def test_writes_validated_tsv(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + source = root / "apply.json" + output = root / "metrics.tsv" + source.write_text(json.dumps(self.report())) + module.extract_file(source, output) + self.assertEqual( + output.read_text(), + "agent_git_export_mode\tmapped_delta\n" + "agent_git_changed_paths\t1\n" + "agent_git_blob_writes\t1\n" + "agent_git_plumbing_commands\t5\n", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_extract_path_index_performance.py b/scripts/test_extract_path_index_performance.py new file mode 100644 index 0000000..6c038a9 --- /dev/null +++ b/scripts/test_extract_path_index_performance.py @@ -0,0 +1,106 @@ +import importlib.util +import json +import pathlib +import tempfile +import unittest + + +SCRIPT = pathlib.Path(__file__).with_name("extract-path-index-performance.py") +SPEC = importlib.util.spec_from_file_location("path_index_performance", SCRIPT) +module = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(module) + + +class PathIndexPerformanceTests(unittest.TestCase): + def report(self, **path_index_overrides): + path_index = { + "mode": "indexed", + "lookup_count": 2, + "full_root_path_load_count": 0, + "full_filesystem_path_scan_count": 0, + } + path_index.update(path_index_overrides) + return { + "operation": "change-1", + "changed_paths": [ + {"path": "README.md", "old_path": "readme.md"}, + {"path": "src/lib.rs", "old_path": None}, + ], + "path_index": path_index, + } + + def test_extracts_operation_scoped_metrics_and_unique_folded_touches(self): + self.assertEqual( + module.extract_path_index_metrics(self.report(), "patch"), + { + "patch_path_index_mode": "indexed", + "patch_path_index_lookup_count": 2, + "patch_path_index_full_root_path_load_count": 0, + "patch_path_index_full_filesystem_path_scan_count": 0, + "patch_path_index_touched_folded_key_count": 2, + }, + ) + + def test_rejects_missing_or_malformed_metrics(self): + invalid = [ + ({"operation": "change-1", "changed_paths": []}, "path_index"), + (self.report(mode=1), "mode must be a string"), + (self.report(lookup_count=True), "nonnegative JSON integer"), + (self.report(lookup_count=-1), "nonnegative JSON integer"), + ( + self.report(full_root_path_load_count="0"), + "nonnegative JSON integer", + ), + ( + self.report(full_filesystem_path_scan_count=-1), + "nonnegative JSON integer", + ), + ] + for payload, message in invalid: + with self.subTest(message=message): + with self.assertRaisesRegex(ValueError, message): + module.extract_path_index_metrics(payload, "patch") + + def test_rejects_reports_without_one_completed_operation(self): + for operation in [None, ""]: + with self.subTest(operation=operation): + payload = self.report() + payload["operation"] = operation + with self.assertRaisesRegex(ValueError, "completed operation"): + module.extract_path_index_metrics(payload, "record") + + def test_rejects_lookup_count_above_unique_folded_touched_paths(self): + with self.assertRaisesRegex(ValueError, "exceeds 2 unique folded touched keys"): + module.extract_path_index_metrics(self.report(lookup_count=3), "patch") + + def test_matches_rust_per_codepoint_lowercase_for_final_sigma(self): + payload = self.report(lookup_count=1) + payload["changed_paths"] = [{"path": "ΟΣ", "old_path": "οσ"}] + metrics = module.extract_path_index_metrics(payload, "patch") + self.assertEqual(metrics["patch_path_index_touched_folded_key_count"], 1) + + def test_rejects_malformed_changed_path_endpoints(self): + payload = self.report() + payload["changed_paths"] = [{"path": 1}] + with self.assertRaisesRegex(ValueError, "changed_paths.*path must be a string"): + module.extract_path_index_metrics(payload, "patch") + + def test_writes_validated_tsv(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + source = root / "report.json" + output = root / "metrics.tsv" + source.write_text(json.dumps(self.report())) + module.extract_file(source, "record", output) + self.assertEqual( + output.read_text(), + "record_path_index_mode\tindexed\n" + "record_path_index_lookup_count\t2\n" + "record_path_index_full_root_path_load_count\t0\n" + "record_path_index_full_filesystem_path_scan_count\t0\n" + "record_path_index_touched_folded_key_count\t2\n", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify-environment-adapter-plugin.sh b/scripts/verify-environment-adapter-plugin.sh new file mode 100755 index 0000000..1e04e66 --- /dev/null +++ b/scripts/verify-environment-adapter-plugin.sh @@ -0,0 +1,381 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${repo_root}" + +cargo build -p trail +cargo build -p trail-environment-adapter-sdk --example generated-copy-adapter --example mounted-initializer-adapter --example mounted-fixture-tool --example cache-adapter --example cache-fixture-tool --example adversarial-adapter --example fixture-sign-adapter +trail="${CARGO_TARGET_DIR:-${repo_root}/target}/debug/trail" +example_dir="${CARGO_TARGET_DIR:-${repo_root}/target}/debug/examples" + +root="$(mktemp -d)" +packages="$(mktemp -d)" +tool_bin="$(mktemp -d)" +cleanup() { + chmod -R u+w "${root}" "${packages}" "${tool_bin}" 2>/dev/null || true + rm -rf "${root}" "${packages}" "${tool_bin}" +} +trap cleanup EXIT +cp "${example_dir}/mounted-fixture-tool" "${tool_bin}/mounted-fixture-tool" +chmod +x "${tool_bin}/mounted-fixture-tool" +cp "${example_dir}/cache-fixture-tool" "${tool_bin}/cache-fixture-tool" +chmod +x "${tool_bin}/cache-fixture-tool" +export PATH="${tool_bin}:${PATH}" +printf "plugin marker\n" >"${root}/copy.adapter" +printf "success\n" >"${root}/mounted.adapter" +printf "lane-a\n" >"${root}/cache.adapter" +printf "declared input\n" >"${root}/input.txt" + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +write_package() { + local directory="$1" + local identity="$2" + local selector="$3" + local executable_source="$4" + local timeout_ms="$5" + local response_bytes="$6" + mkdir -p "${directory}" + cp "${executable_source}" "${directory}/adapter-plugin" + chmod +x "${directory}/adapter-plugin" + local digest + digest="$(sha256_file "${directory}/adapter-plugin")" + { + printf '%s\n' 'schema = "trail.environment-adapter-package/v1"' + printf '%s\n' '[adapter]' + printf 'canonical_identity = "%s"\n' "${identity}" + printf '%s\n' 'implementation_version = "1.0.0"' + printf 'selectors = ["%s", "%s"]\n' "${identity}" "${selector}" + printf '%s\n' 'kind = "generated"' + printf 'layer_adapter_name = "%s"\n' "${selector}" + printf '%s\n' 'discovery_markers = ["copy.adapter"]' + printf '%s\n' 'stability = "experimental"' + printf 'description = "Fixture for %s"\n' "${identity}" + printf '%s\n' '[executable]' + printf '%s\n' 'path = "adapter-plugin"' + printf 'sha256 = "%s"\n' "${digest}" + printf '%s\n' '[permissions]' + printf '%s\n' 'read_patterns = ["copy.adapter", "input.txt"]' + printf '%s\n' 'max_input_files = 8' + printf '%s\n' 'max_input_bytes = 1048576' + printf 'timeout_ms = %s\n' "${timeout_ms}" + printf 'max_response_bytes = %s\n' "${response_bytes}" + } >"${directory}/trail-adapter.toml" +} + +write_mounted_package() { + local directory="$1" + mkdir -p "${directory}" + cp "${example_dir}/mounted-initializer-adapter" "${directory}/adapter-plugin" + chmod +x "${directory}/adapter-plugin" + local digest + digest="$(sha256_file "${directory}/adapter-plugin")" + { + printf '%s\n' 'schema = "trail.environment-adapter-package/v1"' + printf '%s\n' '[adapter]' + printf '%s\n' 'canonical_identity = "example/mounted@1"' + printf '%s\n' 'implementation_version = "1.0.0"' + printf '%s\n' 'selectors = ["example/mounted@1", "example-mounted"]' + printf '%s\n' 'kind = "generated"' + printf '%s\n' 'layer_adapter_name = "example-mounted"' + printf '%s\n' 'discovery_markers = ["mounted.adapter"]' + printf '%s\n' 'protocols = ["trail.environment-adapter/v2"]' + printf '%s\n' 'stability = "experimental"' + printf '%s\n' 'description = "Mounted initializer protocol-v2 fixture"' + printf '%s\n' '[executable]' + printf '%s\n' 'path = "adapter-plugin"' + printf 'sha256 = "%s"\n' "${digest}" + printf '%s\n' '[permissions]' + printf '%s\n' 'read_patterns = ["mounted.adapter"]' + printf '%s\n' 'max_input_files = 8' + printf '%s\n' 'max_input_bytes = 1048576' + printf '%s\n' 'timeout_ms = 5000' + printf '%s\n' 'max_response_bytes = 1048576' + } >"${directory}/trail-adapter.toml" +} + +write_cache_package() { + local directory="$1" + mkdir -p "${directory}" + cp "${example_dir}/cache-adapter" "${directory}/adapter-plugin" + chmod +x "${directory}/adapter-plugin" + local digest + digest="$(sha256_file "${directory}/adapter-plugin")" + { + printf '%s\n' 'schema = "trail.environment-adapter-package/v1"' + printf '%s\n' '[adapter]' + printf '%s\n' 'canonical_identity = "example/cache@1"' + printf '%s\n' 'implementation_version = "1.0.0"' + printf '%s\n' 'selectors = ["example/cache@1", "example-cache"]' + printf '%s\n' 'kind = "generated"' + printf '%s\n' 'layer_adapter_name = "example-cache"' + printf '%s\n' 'discovery_markers = ["cache.adapter"]' + printf '%s\n' 'protocols = ["trail.environment-adapter/v2"]' + printf '%s\n' 'stability = "experimental"' + printf '%s\n' 'description = "Host-owned cache protocol-v2 fixture"' + printf '%s\n' '[executable]' + printf '%s\n' 'path = "adapter-plugin"' + printf 'sha256 = "%s"\n' "${digest}" + printf '%s\n' '[permissions]' + printf '%s\n' 'read_patterns = ["cache.adapter"]' + printf '%s\n' 'max_input_files = 8' + printf '%s\n' 'max_input_bytes = 1048576' + printf '%s\n' 'timeout_ms = 5000' + printf '%s\n' 'max_response_bytes = 1048576' + } >"${directory}/trail-adapter.toml" +} + +json_field() { + local field="$1" + python3 -c 'import json,sys; print(json.load(sys.stdin)[sys.argv[1]])' "${field}" +} + +first_line() { + python3 -c 'import sys; lines=sys.stdin.read().splitlines(); print(lines[0] if lines else "")' +} + +sync_layer_field() { + local field="$1" + python3 -c 'import json,sys; print(json.load(sys.stdin)["layers"][0][sys.argv[1]])' "${field}" +} + +assert_fails() { + local message="$1" + shift + if "$@" >/dev/null 2>&1; then + printf '%s\n' "${message}" >&2 + exit 1 + fi +} + +write_package "${packages}/copy" "example/copy@1" "example-copy" \ + "${example_dir}/generated-copy-adapter" 5000 1048576 +write_mounted_package "${packages}/mounted" +write_cache_package "${packages}/cache" + +"${trail}" --workspace "${root}" init --working-tree >/dev/null +inspection="$("${trail}" --workspace "${root}" --json env plugin inspect "${packages}/copy")" +payload_digest="$(json_field payload_digest <<<"${inspection}")" +"${example_dir}/fixture-sign-adapter" \ + "example-publisher" \ + "0707070707070707070707070707070707070707070707070707070707070707" \ + "${payload_digest}" \ + "${packages}/copy/trail-adapter.sig" \ + "${packages}/copy/publisher-key.toml" +assert_fails "signed adapter unexpectedly installed before publisher trust" \ + "${trail}" --workspace "${root}" env plugin install "${packages}/copy" +trust_json="$("${trail}" --workspace "${root}" --json env plugin trust add "${packages}/copy/publisher-key.toml")" +publisher_key_id="$(json_field key_id <<<"${trust_json}")" +mode="fuse-cow" +if [[ "$(uname -s)" == "Darwin" ]]; then + mode="nfs-cow" +fi +"${trail}" --workspace "${root}" lane spawn plugin-a --from main --workdir-mode "${mode}" >/dev/null +"${trail}" --workspace "${root}" lane spawn plugin-b --from main --workdir-mode "${mode}" >/dev/null +"${trail}" --workspace "${root}" lane spawn plugin-private --from main --workdir-mode "${mode}" >/dev/null +"${trail}" --workspace "${root}" lane spawn plugin-mounted-a --from main --workdir-mode "${mode}" >/dev/null +"${trail}" --workspace "${root}" lane spawn plugin-mounted-b --from main --workdir-mode "${mode}" >/dev/null +"${trail}" --workspace "${root}" lane spawn plugin-mounted-kill --from main --workdir-mode "${mode}" >/dev/null + +install_json="$("${trail}" --workspace "${root}" --json env plugin install "${packages}/copy")" +distribution="$(json_field distribution_digest <<<"${install_json}")" +grep -q '"trust": "publisher_signed"' <<<"${install_json}" +grep -q '"certification_tier": "publisher-authenticated-experimental"' <<<"${install_json}" +catalog="$("${trail}" --workspace "${root}" --json env adapters)" +grep -q '"canonical_identity": "example/copy@1"' <<<"${catalog}" +grep -q '"source": "plugin"' <<<"${catalog}" +discovery="$("${trail}" --workspace "${root}" --json env discover plugin-a)" +grep -q '"component_id": "plugin.copy"' <<<"${discovery}" +plan="$("${trail}" --workspace "${root}" --json env plan plugin-a --adapter example/copy@1)" +grep -q '"sandbox": "' <<<"${plan}" +grep -q '"network": "deny"' <<<"${plan}" +first="$("${trail}" --workspace "${root}" --json env sync plugin-a --adapter example/copy@1)" +second="$("${trail}" --workspace "${root}" --json env sync plugin-b --adapter example/copy@1)" +first_layer="$(sync_layer_field layer_id <<<"${first}")" +second_layer="$(sync_layer_field layer_id <<<"${second}")" +storage="$(sync_layer_field storage_path <<<"${first}")" +test "${first_layer}" = "${second_layer}" +cmp "${root}/input.txt" "${storage}/copied.txt" +"${trail}" --workspace "${root}" lane exec plugin-private -- \ + sh -c 'printf writable_private >copy.adapter' +"${trail}" --workspace "${root}" lane checkpoint plugin-private -m "select private plugin output" >/dev/null +private_sync="$("${trail}" --workspace "${root}" --json env sync plugin-private --adapter example/copy@1)" +python3 -c ' +import json, sys +report = json.load(sys.stdin) +assert report["layers"] == [], report +output = report["generation"]["components"][0]["outputs"][0] +assert output["policy"] == "writable_private", output +assert output["layer_id"] is None, output +' <<<"${private_sync}" +"${trail}" --workspace "${root}" lane exec plugin-private -- \ + sh -c 'printf private-plugin-mutation >.trail-generated/plugin-copy/copied.txt' +"${trail}" --workspace "${root}" env sync plugin-private --adapter example/copy@1 >/dev/null +"${trail}" --workspace "${root}" lane exec plugin-private -- \ + sh -c 'grep -q private-plugin-mutation .trail-generated/plugin-copy/copied.txt' +"${trail}" --workspace "${root}" lane exec plugin-a -- \ + sh -c 'printf lane-a >.trail-generated/plugin-copy/copied.txt' +"${trail}" --workspace "${root}" lane exec plugin-b -- \ + sh -c 'grep -q "declared input" .trail-generated/plugin-copy/copied.txt' +"${trail}" --workspace "${root}" lane exec plugin-a -- \ + sh -c 'printf changed-input >input.txt' +"${trail}" --workspace "${root}" lane checkpoint plugin-a -m "change plugin input" >/dev/null +readiness="$("${trail}" --workspace "${root}" --json lane readiness plugin-a)" +grep -q 'dependency_environment_stale' <<<"${readiness}" +status="$("${trail}" --workspace "${root}" --json env status plugin-a)" +grep -q '"status": "stale"' <<<"${status}" + +"${trail}" --workspace "${root}" env plugin install "${packages}/cache" >/dev/null +"${trail}" --workspace "${root}" lane exec plugin-b -- sh -c 'printf lane-b >cache.adapter' +"${trail}" --workspace "${root}" lane checkpoint plugin-b -m "give cache fixture a distinct component key" >/dev/null +cache_plan="$("${trail}" --workspace "${root}" --json env plan plugin-a --adapter example/cache@1)" +grep -q '"name": "fixture-store"' <<<"${cache_plan}" +grep -q '"protocol": "content_store"' <<<"${cache_plan}" +grep -q '"access": "host_exclusive"' <<<"${cache_plan}" +cache_a="$("${trail}" --workspace "${root}" --json env sync plugin-a --adapter example/cache@1)" +cache_b="$("${trail}" --workspace "${root}" --json env sync plugin-b --adapter example/cache@1)" +cache_a_namespace="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["generation"]["components"][0]["caches"][0]["namespace_id"])' <<<"${cache_a}")" +cache_b_namespace="$(python3 -c 'import json,sys; print(json.load(sys.stdin)["generation"]["components"][0]["caches"][0]["namespace_id"])' <<<"${cache_b}")" +test "${cache_a_namespace}" = "${cache_b_namespace}" +cache_a_observation="$("${trail}" --workspace "${root}" lane exec plugin-a -- sh -c 'cat .trail-generated/plugin-cache/cache-observation.txt' | first_line | tr -d '\r')" +cache_b_observation="$("${trail}" --workspace "${root}" lane exec plugin-b -- sh -c 'cat .trail-generated/plugin-cache/cache-observation.txt' | first_line | tr -d '\r')" +test "${cache_a_observation}" = "${cache_a_namespace}|1" +test "${cache_b_observation}" = "${cache_b_namespace}|2" +test "$(tr -d '\r\n' <"${root}/.trail/cache/namespaces/${cache_a_namespace}/counter")" = "2" +"${trail}" --workspace "${root}" lane exec plugin-a -- sh -c 'printf escape >cache.adapter' +"${trail}" --workspace "${root}" lane checkpoint plugin-a -m "attempt plugin cache namespace escape" >/dev/null +assert_fails "plugin cache write unexpectedly escaped its namespace" \ + "${trail}" --workspace "${root}" env sync plugin-a --adapter example/cache@1 +test ! -e "${root}/.trail/cache/namespaces/plugin-cache-escape" +"${trail}" --workspace "${root}" lane exec plugin-a -- \ + sh -c 'grep -q "|1$" .trail-generated/plugin-cache/cache-observation.txt' + +"${trail}" --workspace "${root}" env plugin install "${packages}/mounted" >/dev/null +mounted_catalog="$("${trail}" --workspace "${root}" --json env adapters)" +grep -q 'trail.environment-adapter/v2' <<<"${mounted_catalog}" +mounted_plan="$("${trail}" --workspace "${root}" --json env plan plugin-mounted-a --adapter example/mounted@1)" +grep -q '"phase": "mounted_initialization"' <<<"${mounted_plan}" +if grep -q '"phase": "staging"' <<<"${mounted_plan}"; then + printf '%s\n' "mounted-only plugin unexpectedly gained a staging action" >&2 + exit 1 +fi +mounted_a="$("${trail}" --workspace "${root}" --json env sync plugin-mounted-a --adapter example/mounted@1)" +mounted_b="$("${trail}" --workspace "${root}" --json env sync plugin-mounted-b --adapter example/mounted@1)" +python3 -c 'import json,sys; assert json.load(sys.stdin)["layers"] == []' <<<"${mounted_a}" +python3 -c 'import json,sys; assert json.load(sys.stdin)["layers"] == []' <<<"${mounted_b}" +mounted_a_pwd="$("${trail}" --workspace "${root}" lane exec plugin-mounted-a -- pwd | first_line | tr -d '\r')" +mounted_b_pwd="$("${trail}" --workspace "${root}" lane exec plugin-mounted-b -- pwd | first_line | tr -d '\r')" +mounted_a_recorded="$("${trail}" --workspace "${root}" lane exec plugin-mounted-a -- sh -c 'cat .trail-generated/plugin-mounted/initialized.txt' | first_line | tr -d '\r')" +mounted_b_recorded="$("${trail}" --workspace "${root}" lane exec plugin-mounted-b -- sh -c 'cat .trail-generated/plugin-mounted/initialized.txt' | first_line | tr -d '\r')" +mounted_a_recorded="${mounted_a_recorded%|success}" +mounted_b_recorded="${mounted_b_recorded%|success}" +test "${mounted_a_recorded}" = "${mounted_a_pwd}" +test "${mounted_b_recorded}" = "${mounted_b_pwd}" +test "${mounted_a_recorded}" != "${mounted_b_recorded}" +"${trail}" --workspace "${root}" lane exec plugin-mounted-a -- \ + sh -c 'printf lane-a-private >.trail-generated/plugin-mounted/initialized.txt' +"${trail}" --workspace "${root}" env sync plugin-mounted-a --adapter example/mounted@1 >/dev/null +"${trail}" --workspace "${root}" lane exec plugin-mounted-a -- \ + sh -c 'grep -q lane-a-private .trail-generated/plugin-mounted/initialized.txt' +"${trail}" --workspace "${root}" lane exec plugin-mounted-b -- \ + sh -c 'test "$(cat .trail-generated/plugin-mounted/initialized.txt)" != lane-a-private' +"${trail}" --workspace "${root}" lane exec plugin-mounted-a -- sh -c 'printf fail >mounted.adapter' +"${trail}" --workspace "${root}" lane checkpoint plugin-mounted-a -m "fail mounted plugin action" >/dev/null +assert_fails "failed mounted plugin action unexpectedly activated" \ + "${trail}" --workspace "${root}" env sync plugin-mounted-a --adapter example/mounted@1 +"${trail}" --workspace "${root}" lane exec plugin-mounted-a -- \ + sh -c 'grep -q lane-a-private .trail-generated/plugin-mounted/initialized.txt && test ! -e .trail-generated/plugin-mounted/partial.txt' +"${trail}" --workspace "${root}" lane exec plugin-mounted-b -- sh -c 'printf source_write >mounted.adapter' +"${trail}" --workspace "${root}" lane checkpoint plugin-mounted-b -m "attempt mounted source write" >/dev/null +assert_fails "mounted plugin source write unexpectedly escaped its declared output" \ + "${trail}" --workspace "${root}" env sync plugin-mounted-b --adapter example/mounted@1 +"${trail}" --workspace "${root}" lane exec plugin-mounted-b -- \ + sh -c 'test ! -e source-leak.txt && test -f .trail-generated/plugin-mounted/initialized.txt' +"${trail}" --workspace "${root}" lane exec plugin-mounted-b -- sh -c 'printf source_read >mounted.adapter' +"${trail}" --workspace "${root}" lane checkpoint plugin-mounted-b -m "attempt undeclared mounted source read" >/dev/null +assert_fails "mounted plugin undeclared source read unexpectedly succeeded" \ + "${trail}" --workspace "${root}" env sync plugin-mounted-b --adapter example/mounted@1 +"${trail}" --workspace "${root}" lane exec plugin-mounted-b -- \ + sh -c 'test ! -e .trail-generated/plugin-mounted/leaked.txt && test -f .trail-generated/plugin-mounted/initialized.txt' +"${trail}" --workspace "${root}" env sync plugin-mounted-kill --adapter example/mounted@1 >/dev/null +"${trail}" --workspace "${root}" lane exec plugin-mounted-kill -- \ + sh -c 'printf kill-predecessor >.trail-generated/plugin-mounted/initialized.txt; printf hang >mounted.adapter' +"${trail}" --workspace "${root}" lane checkpoint plugin-mounted-kill -m "kill active mounted plugin action" >/dev/null +"${trail}" --workspace "${root}" env sync plugin-mounted-kill --adapter example/mounted@1 \ + >"${packages}/mounted-kill.stdout" 2>"${packages}/mounted-kill.stderr" & +mounted_sync_pid=$! +mounted_ready="" +for _ in $(seq 1 200); do + mounted_ready="$(find "${root}/.trail/cache/staging" -path '*/process/*/home/running' -type f -print -quit 2>/dev/null || true)" + if [[ -n "${mounted_ready}" ]]; then + break + fi + sleep 0.05 +done +if [[ -z "${mounted_ready}" ]]; then + kill -9 "${mounted_sync_pid}" 2>/dev/null || true + wait "${mounted_sync_pid}" 2>/dev/null || true + printf '%s\n' "mounted plugin did not reach its active-command kill point" >&2 + exit 1 +fi +mounted_child_pid="$(tr -d '\r\n' <"${mounted_ready}")" +kill -9 "${mounted_sync_pid}" +wait "${mounted_sync_pid}" 2>/dev/null || true +for _ in $(seq 1 200); do + if ! kill -0 "${mounted_child_pid}" 2>/dev/null; then + break + fi + sleep 0.05 +done +if kill -0 "${mounted_child_pid}" 2>/dev/null; then + printf '%s\n' "mounted plugin action survived Trail process death" >&2 + kill -9 "${mounted_child_pid}" 2>/dev/null || true + exit 1 +fi +"${trail}" --workspace "${root}" env status plugin-mounted-kill >/dev/null +"${trail}" --workspace "${root}" lane exec plugin-mounted-kill -- \ + sh -c 'grep -q kill-predecessor .trail-generated/plugin-mounted/initialized.txt && test ! -e .trail-generated/plugin-mounted/partial.txt' +if find "${root}/.trail/cache/staging" -maxdepth 1 -name 'mounted-environment-*' -print -quit | grep -q .; then + printf '%s\n' "recovery left an abandoned mounted plugin candidate" >&2 + exit 1 +fi + +for behavior in hang crash oversized malformed child memory; do + response_bytes=1048576 + timeout_ms=1000 + if [[ "${behavior}" == "hang" ]]; then + timeout_ms=100 + fi + write_package "${packages}/${behavior}" "example/${behavior}@1" "example-${behavior}" \ + "${example_dir}/adversarial-adapter" "${timeout_ms}" "${response_bytes}" + "${trail}" --workspace "${root}" env plugin install "${packages}/${behavior}" >/dev/null + assert_fails "adversarial ${behavior} adapter unexpectedly succeeded" \ + "${trail}" --workspace "${root}" env plan plugin-a --adapter "example/${behavior}@1" +done + +removed="$("${trail}" --workspace "${root}" --json env plugin remove example/copy@1)" +grep -q "${distribution}" <<<"${removed}" +if "${trail}" --workspace "${root}" --json env adapters | grep -q 'example/copy@1'; then + printf '%s\n' "removed adapter remained active" >&2 + exit 1 +fi + +reinstalled="$("${trail}" --workspace "${root}" --json env plugin install "${packages}/copy")" +package_path="$(json_field package_path <<<"${reinstalled}")" +printf 'tamper\n' >>"${package_path}/adapter-plugin" +assert_fails "tampered adapter executable remained trusted" \ + "${trail}" --workspace "${root}" env adapters +"${trail}" --workspace "${root}" env plugin install "${packages}/copy" >/dev/null +"${trail}" --workspace "${root}" env plugin trust remove "${publisher_key_id}" >/dev/null +assert_fails "revoked publisher key left signed adapter active" \ + "${trail}" --workspace "${root}" env adapters + +printf 'environment-adapter-plugin distribution=%s shared-layer=%s external-cache=%s:shared-host-exclusive-and-sandbox-contained writable-private=verified mounted-v2=isolated-and-atomic active-command-kill=terminated-and-recovered declared-read=allowed undeclared-read-write=denied private-copy-up=isolated stale-refresh=verified publisher-signature=verified revocation=fail-closed timeout=denied memory=denied crash=denied oversized=denied malformed=denied child-process=denied tamper=denied\n' \ + "${distribution}" "${first_layer}" "${cache_a_namespace}" diff --git a/scripts/verify-linux-command-recipe-native.sh b/scripts/verify-linux-command-recipe-native.sh new file mode 100755 index 0000000..2b63778 --- /dev/null +++ b/scripts/verify-linux-command-recipe-native.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${repo_root}" +cargo build -p trail +trail="${CARGO_TARGET_DIR:-${repo_root}/target}/debug/trail" + +write_spec() { + local root="$1" + shift + local policy="${TRAIL_RECIPE_POLICY:-immutable_seed_private}" + printf "declared input\n" >"${root}/input.txt" + { + printf "%s\n" "schema = \"trail.environment/v1\"" + printf "%s\n" "[environment]" + printf "%s\n" "default_network = \"deny\"" + printf "%s\n" "default_scripts = \"deny\"" + printf "%s\n" "[[component]]" + printf "%s\n" "id = \"generated.copy\"" + printf "%s\n" "adapter = \"trail/command@1\"" + printf "%s\n" "root = \".\"" + printf "%s\n" "kind = \"generated\"" + printf "%s\n" "[[component.input]]" + printf "%s\n" "path = \"input.txt\"" + printf "%s\n" "role = \"identity\"" + printf "%s\n" "format = \"bytes\"" + printf "%s\n" "[component.build]" + printf "command = [" + local separator="" + local argument + for argument in "$@"; do + argument="${argument//\\/\\\\}" + argument="${argument//\"/\\\"}" + printf "%s\"%s\"" "${separator}" "${argument}" + separator=", " + done + printf "]\n" + printf "%s\n" "cwd = \".\"" + printf "%s\n" "network = \"deny\"" + printf "%s\n" "scripts = \"deny\"" + printf "%s\n" "[[component.output]]" + printf "%s\n" "source = \"generated\"" + printf "%s\n" "target = \".trail-generated/copy\"" + printf 'policy = "%s"\n' "${policy}" + printf "%s\n" "portability = \"host\"" + } >"${root}/trail.environment.toml" +} + +new_lane_workspace() { + local root="$1" + "${trail}" --workspace "${root}" init --working-tree >/dev/null + "${trail}" --workspace "${root}" lane spawn recipe-a --from main --workdir-mode fuse-cow >/dev/null +} + +success_root="$(mktemp -d)" +write_spec "${success_root}" cp input.txt generated/copied.txt +new_lane_workspace "${success_root}" +"${trail}" --workspace "${success_root}" lane spawn recipe-b --from main --workdir-mode fuse-cow >/dev/null +plan="$("${trail}" --workspace "${success_root}" --json env plan recipe-a --adapter command)" +grep -q "linux-landlock-seccomp" <<<"${plan}" +first="$("${trail}" --workspace "${success_root}" --json env sync recipe-a --adapter command)" +second="$("${trail}" --workspace "${success_root}" --json env sync recipe-b --adapter command)" +first_layer="$(sed -n "s/.*\"layer_id\": \"\([^\"]*\)\".*/\1/p" <<<"${first}" | head -1)" +second_layer="$(sed -n "s/.*\"layer_id\": \"\([^\"]*\)\".*/\1/p" <<<"${second}" | head -1)" +storage_path="$(sed -n "s/.*\"storage_path\": \"\([^\"]*\)\".*/\1/p" <<<"${first}" | head -1)" +test -n "${first_layer}" +test "${first_layer}" = "${second_layer}" +cmp "${success_root}/input.txt" "${storage_path}/copied.txt" + +private_root="$(mktemp -d)" +TRAIL_RECIPE_POLICY=writable_private write_spec "${private_root}" cp input.txt generated/copied.txt +new_lane_workspace "${private_root}" +private_sync="$("${trail}" --workspace "${private_root}" --json env sync recipe-a --adapter command)" +python3 -c ' +import json, sys +report = json.load(sys.stdin) +assert report["layers"] == [], report +output = report["generation"]["components"][0]["outputs"][0] +assert output["policy"] == "writable_private", output +assert output["layer_id"] is None, output +' <<<"${private_sync}" +"${trail}" --workspace "${private_root}" lane exec recipe-a -- \ + sh -c 'printf private-mutation >.trail-generated/copy/copied.txt' +"${trail}" --workspace "${private_root}" env sync recipe-a --adapter command >/dev/null +"${trail}" --workspace "${private_root}" lane exec recipe-a -- \ + sh -c 'grep -q private-mutation .trail-generated/copy/copied.txt' + +multi_root="$(mktemp -d)" +write_spec "${multi_root}" touch generated/a.txt generated-b/b.txt +{ + printf "%s\n" "[[component.output]]" + printf "%s\n" "name = \"beta\"" + printf "%s\n" "source = \"generated-b\"" + printf "%s\n" "target = \".trail-generated/beta\"" + printf "%s\n" "policy = \"immutable_seed_private\"" + printf "%s\n" "portability = \"host\"" +} >>"${multi_root}/trail.environment.toml" +new_lane_workspace "${multi_root}" +"${trail}" --workspace "${multi_root}" lane spawn recipe-b --from main --workdir-mode fuse-cow >/dev/null +multi_plan="$("${trail}" --workspace "${multi_root}" --json env plan recipe-a --adapter command)" +test "$(grep -c '"output_path"' <<<"${multi_plan}")" -ge 3 +multi_first="$("${trail}" --workspace "${multi_root}" --json env sync-all recipe-a)" +multi_second="$("${trail}" --workspace "${multi_root}" --json env sync recipe-b --adapter command)" +multi_first_layer="$(sed -n "s/.*\"layer_id\": \"\([^\"]*\)\".*/\1/p" <<<"${multi_first}" | head -1)" +multi_second_layer="$(sed -n "s/.*\"layer_id\": \"\([^\"]*\)\".*/\1/p" <<<"${multi_second}" | head -1)" +multi_storage="$(sed -n "s/.*\"storage_path\": \"\([^\"]*\)\".*/\1/p" <<<"${multi_first}" | head -1)" +test -f "${multi_storage}/outputs/0000/a.txt" +test -f "${multi_storage}/outputs/0001/b.txt" +test "${multi_first_layer}" = "${multi_second_layer}" +"${trail}" --workspace "${multi_root}" lane exec recipe-a -- \ + sh -c 'test -f .trail-generated/copy/a.txt && test -f .trail-generated/beta/b.txt && printf lane-a >.trail-generated/copy/a.txt' +"${trail}" --workspace "${multi_root}" lane exec recipe-b -- \ + sh -c 'test -f .trail-generated/copy/a.txt && test ! -s .trail-generated/copy/a.txt && test -f .trail-generated/beta/b.txt' + +host_read_root="$(mktemp -d)" +write_spec "${host_read_root}" cp /etc/passwd generated/copied.txt +new_lane_workspace "${host_read_root}" +if "${trail}" --workspace "${host_read_root}" env sync recipe-a --adapter command >/dev/null 2>&1; then + echo "restricted recipe unexpectedly read /etc/passwd" >&2 + exit 1 +fi + +write_root="$(mktemp -d)" +write_spec "${write_root}" cp input.txt escape.txt +new_lane_workspace "${write_root}" +if "${trail}" --workspace "${write_root}" env sync recipe-a --adapter command >/dev/null 2>&1; then + echo "restricted recipe unexpectedly wrote outside its declared output" >&2 + exit 1 +fi + +network_root="$(mktemp -d)" +write_spec "${network_root}" curl --fail --max-time 1 http://127.0.0.1:9 -o generated/network.txt +new_lane_workspace "${network_root}" +if "${trail}" --workspace "${network_root}" env sync recipe-a --adapter command >/dev/null 2>&1; then + echo "restricted recipe unexpectedly used a network socket" >&2 + exit 1 +fi + +shell_root="$(mktemp -d)" +write_spec "${shell_root}" sh -c true +new_lane_workspace "${shell_root}" +if "${trail}" --workspace "${shell_root}" env plan recipe-a --adapter command >/dev/null 2>&1; then + echo "restricted recipe unexpectedly accepted a shell" >&2 + exit 1 +fi + +printf "linux-command-recipe shared-layer=%s multi-output-layer=%s writable-private=verified host-read=denied undeclared-write=denied network=denied shell=denied\n" "${first_layer}" "${multi_first_layer}" diff --git a/scripts/verify-linux-command-recipe-sandbox.sh b/scripts/verify-linux-command-recipe-sandbox.sh new file mode 100755 index 0000000..d4748a6 --- /dev/null +++ b/scripts/verify-linux-command-recipe-sandbox.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +docker run --rm --privileged \ + -v "${repo_root}:/work" \ + -v trail-fuse-cow-cargo:/cargo-home \ + -v trail-fuse-cow-target:/target \ + -w /work \ + -e CARGO_HOME=/cargo-home \ + -e CARGO_TARGET_DIR=/target \ + rust:bookworm \ + bash -lc 'export PATH=/usr/local/cargo/bin:/usr/bin:$PATH; scripts/verify-linux-command-recipe-native.sh' diff --git a/scripts/verify-linux-overlay-cow-docker.sh b/scripts/verify-linux-fuse-cow-docker.sh similarity index 77% rename from scripts/verify-linux-overlay-cow-docker.sh rename to scripts/verify-linux-fuse-cow-docker.sh index 8767b62..4bbc7b7 100755 --- a/scripts/verify-linux-overlay-cow-docker.sh +++ b/scripts/verify-linux-fuse-cow-docker.sh @@ -5,8 +5,8 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) IMAGE=${RUST_IMAGE:-rust:bookworm} TARGET_DIR=${TRAIL_DOCKER_TARGET_DIR:-/target} CARGO_HOME_DIR=${TRAIL_DOCKER_CARGO_HOME:-/cargo-home} -CARGO_CACHE_VOLUME=${TRAIL_DOCKER_CARGO_CACHE_VOLUME:-trail-overlay-cow-cargo} -TARGET_CACHE_VOLUME=${TRAIL_DOCKER_TARGET_CACHE_VOLUME:-trail-overlay-cow-target} +CARGO_CACHE_VOLUME=${TRAIL_DOCKER_CARGO_CACHE_VOLUME:-trail-fuse-cow-cargo} +TARGET_CACHE_VOLUME=${TRAIL_DOCKER_TARGET_CACHE_VOLUME:-trail-fuse-cow-target} docker run --rm --privileged \ -v "$ROOT":/work \ @@ -27,15 +27,15 @@ TRAIL_RUN_FUSE_COW_TESTS=1 cargo test -p trail \ fuse_adapter_runs_shared_mounted_view_suite -- --nocapture cargo build -p trail -tmp=$(mktemp -d /tmp/trail-linux-overlay.XXXXXX) +tmp=$(mktemp -d /tmp/trail-linux-fuse-cow.XXXXXX) printf "hello\n" > "$tmp/README.md" mkdir -p "$tmp/src" printf "pub fn answer() -> u8 { 42 }\n" > "$tmp/src/lib.rs" -"$CARGO_TARGET_DIR/debug/trail" --workspace "$tmp" init --working-tree >/tmp/trail-overlay-init.out +"$CARGO_TARGET_DIR/debug/trail" --workspace "$tmp" init --working-tree >/tmp/trail-fuse-init.out "$CARGO_TARGET_DIR/debug/trail" --workspace "$tmp" agent start \ --provider custom \ - --workdir-mode overlay-cow \ + --workdir-mode fuse-cow \ -- bash -lc '"'"' set -euo pipefail test -f README.md @@ -47,22 +47,22 @@ printf "changed\n" >> README.md mkdir notes printf "new\n" > notes/todo.txt rm src/lib.rs -'"'"' >/tmp/trail-overlay-agent.out +'"'"' >/tmp/trail-fuse-agent.out -"$CARGO_TARGET_DIR/debug/trail" --workspace "$tmp" agent changes latest --json >/tmp/trail-overlay-changes.json -cat /tmp/trail-overlay-agent.out +"$CARGO_TARGET_DIR/debug/trail" --workspace "$tmp" agent changes latest --json >/tmp/trail-fuse-changes.json +cat /tmp/trail-fuse-agent.out python3 - <<'"'"'PY'"'"' import json from pathlib import Path -data = json.loads(Path("/tmp/trail-overlay-changes.json").read_text()) +data = json.loads(Path("/tmp/trail-fuse-changes.json").read_text()) paths = sorted(item["path"] for item in data["total_changed_paths"]) print("changed-paths=" + ",".join(paths)) assert paths == ["README.md", "notes/todo.txt", "src/lib.rs"], paths workdir = Path(data["task"]["workdir"]) assert workdir.is_dir(), workdir -assert not any(workdir.iterdir()), "overlay mountpoint should be empty after unmount" +assert not any(workdir.iterdir()), "FUSE COW mountpoint should be empty after unmount" db_dir = workdir.parents[1] views = sorted((db_dir / "views").iterdir()) @@ -75,11 +75,11 @@ assert (view / "meta" / "source-whiteouts.json").is_file(), view PY "$CARGO_TARGET_DIR/debug/trail" --workspace "$tmp" lane spawn persistent \ - --from main --workdir-mode overlay-cow >/tmp/trail-overlay-spawn.out + --from main --workdir-mode fuse-cow >/tmp/trail-fuse-spawn.out workdir=$("$CARGO_TARGET_DIR/debug/trail" --workspace "$tmp" lane workdir persistent --json | \ python3 -c '"'"'import json,sys; print(json.load(sys.stdin)["workdir"])'"'"') "$CARGO_TARGET_DIR/debug/trail" --workspace "$tmp" lane mount persistent \ - >/tmp/trail-overlay-mount.out 2>&1 & + >/tmp/trail-fuse-mount.out 2>&1 & mount_pid=$! for _ in $(seq 1 100); do test -f "$workdir/README.md" && break @@ -88,16 +88,16 @@ done test -f "$workdir/README.md" printf "persistent\n" > "$workdir/persistent.txt" "$CARGO_TARGET_DIR/debug/trail" --workspace "$tmp" lane unmount persistent \ - >/tmp/trail-overlay-unmount.out + >/tmp/trail-fuse-unmount.out wait "$mount_pid" test ! -e "$workdir/README.md" "$CARGO_TARGET_DIR/debug/trail" --workspace "$tmp" lane checkpoint persistent \ - -m "persistent lifecycle" --json >/tmp/trail-overlay-checkpoint.json + -m "persistent lifecycle" --json >/tmp/trail-fuse-checkpoint.json python3 - <<'"'"'PY'"'"' import json from pathlib import Path -checkpoint = json.loads(Path("/tmp/trail-overlay-checkpoint.json").read_text()) +checkpoint = json.loads(Path("/tmp/trail-fuse-checkpoint.json").read_text()) assert checkpoint["source_paths"] == ["persistent.txt"], checkpoint print("persistent-mount-checkpoint=" + checkpoint["root_id"]) PY diff --git a/scripts/verify-linux-node-layer-docker.sh b/scripts/verify-linux-node-layer-docker.sh index eda76b7..d88e8e7 100755 --- a/scripts/verify-linux-node-layer-docker.sh +++ b/scripts/verify-linux-node-layer-docker.sh @@ -5,9 +5,9 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) IMAGE=${RUST_IMAGE:-rust:bookworm} TARGET_DIR=${TRAIL_DOCKER_TARGET_DIR:-/target} CARGO_HOME_DIR=${TRAIL_DOCKER_CARGO_HOME:-/cargo-home} -CARGO_CACHE_VOLUME=${TRAIL_DOCKER_CARGO_CACHE_VOLUME:-trail-overlay-cow-cargo} -TARGET_CACHE_VOLUME=${TRAIL_DOCKER_TARGET_CACHE_VOLUME:-trail-overlay-cow-target} -NPM_CACHE_VOLUME=${TRAIL_DOCKER_NPM_CACHE_VOLUME:-trail-overlay-cow-npm} +CARGO_CACHE_VOLUME=${TRAIL_DOCKER_CARGO_CACHE_VOLUME:-trail-fuse-cow-cargo} +TARGET_CACHE_VOLUME=${TRAIL_DOCKER_TARGET_CACHE_VOLUME:-trail-fuse-cow-target} +NPM_CACHE_VOLUME=${TRAIL_DOCKER_NPM_CACHE_VOLUME:-trail-fuse-cow-npm} docker run --rm --privileged \ -v "$ROOT":/work \ diff --git a/scripts/verify-macos-nfs-framework-layers.sh b/scripts/verify-macos-nfs-framework-layers.sh new file mode 100755 index 0000000..f68f561 --- /dev/null +++ b/scripts/verify-macos-nfs-framework-layers.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$(uname -s)" != "Darwin" ]]; then + echo "macOS is required for the native nfs-cow framework benchmark" >&2 + exit 2 +fi + +export TRAIL_RUN_NFS_FRAMEWORK_BENCH=1 +if [[ $# -gt 0 ]]; then + export TRAIL_NFS_FRAMEWORK_FILTER="$1" +fi + +cargo test -p trail nfs_large_nextjs_and_vite_layers_build_and_bulk_replace -- --nocapture diff --git a/scripts/verify-rust-cache-isolation-docker.sh b/scripts/verify-rust-cache-isolation-docker.sh index b4b6ee5..03b677e 100755 --- a/scripts/verify-rust-cache-isolation-docker.sh +++ b/scripts/verify-rust-cache-isolation-docker.sh @@ -5,8 +5,8 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) IMAGE=${RUST_IMAGE:-rust:bookworm} TARGET_DIR=${TRAIL_DOCKER_TARGET_DIR:-/target} CARGO_HOME_DIR=${TRAIL_DOCKER_CARGO_HOME:-/cargo-home} -CARGO_CACHE_VOLUME=${TRAIL_DOCKER_CARGO_CACHE_VOLUME:-trail-overlay-cow-cargo} -TARGET_CACHE_VOLUME=${TRAIL_DOCKER_TARGET_CACHE_VOLUME:-trail-overlay-cow-target} +CARGO_CACHE_VOLUME=${TRAIL_DOCKER_CARGO_CACHE_VOLUME:-trail-fuse-cow-cargo} +TARGET_CACHE_VOLUME=${TRAIL_DOCKER_TARGET_CACHE_VOLUME:-trail-fuse-cow-target} docker run --rm --privileged \ -v "$ROOT":/work \ diff --git a/scripts/verify-windows-command-recipe-sandbox.ps1 b/scripts/verify-windows-command-recipe-sandbox.ps1 new file mode 100644 index 0000000..c2d5aba --- /dev/null +++ b/scripts/verify-windows-command-recipe-sandbox.ps1 @@ -0,0 +1,188 @@ +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +if (-not $IsWindows) { + throw "Windows is required for the AppContainer command-recipe verifier" +} + +$repoRoot = Split-Path -Parent $PSScriptRoot +Push-Location $repoRoot +try { + cargo build -p trail + if ($LASTEXITCODE -ne 0) { throw "cargo build failed" } + $trail = Join-Path $repoRoot "target/debug/trail.exe" + + function Convert-ToTomlString([string] $Value) { + return '"' + $Value.Replace('\', '\\').Replace('"', '\"') + '"' + } + + function Write-Recipe([string] $Root, [string[]] $Command, [string] $Policy = "immutable_seed_private") { + New-Item -ItemType Directory -Path $Root -Force | Out-Null + Set-Content -LiteralPath (Join-Path $Root "input.txt") -Value "declared input" -Encoding utf8NoBOM + $commandToml = ($Command | ForEach-Object { Convert-ToTomlString $_ }) -join ", " + $specification = @" +schema = "trail.environment/v1" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "generated.copy" +adapter = "trail/command@1" +root = "." +kind = "generated" + +[[component.input]] +path = "input.txt" +role = "identity" +format = "bytes" + +[component.build] +command = [$commandToml] +cwd = "." +network = "deny" +scripts = "deny" + +[[component.output]] +name = "generated" +source = "generated" +target = ".trail-generated/copy" +policy = "$Policy" +portability = "host" +"@ + Set-Content -LiteralPath (Join-Path $Root "trail.environment.toml") -Value $specification -Encoding utf8NoBOM + } + + function New-RecipeLane([string] $Root, [string] $Name = "recipe-a") { + & $trail --workspace $Root init --working-tree | Out-Null + if ($LASTEXITCODE -ne 0) { throw "trail init failed for $Root" } + & $trail --workspace $Root lane spawn $Name --from main --workdir-mode dokan-cow | Out-Null + if ($LASTEXITCODE -ne 0) { throw "lane spawn failed for $Root" } + } + + function Assert-TrailFails([scriptblock] $Action, [string] $Message) { + & $Action *> $null + if ($LASTEXITCODE -eq 0) { throw $Message } + } + + $successRoot = Join-Path $env:RUNNER_TEMP ("trail-recipe-success-" + [guid]::NewGuid()) + Write-Recipe $successRoot @("tar.exe", "-cf", "generated/out.tar", "input.txt") + New-RecipeLane $successRoot + & $trail --workspace $successRoot lane spawn recipe-b --from main --workdir-mode dokan-cow | Out-Null + if ($LASTEXITCODE -ne 0) { throw "second lane spawn failed" } + $plan = (& $trail --workspace $successRoot --json env plan recipe-a --adapter command) | ConvertFrom-Json + if ($plan.capabilities.sandbox -ne "windows-appcontainer-job") { + throw "plan did not report the Windows AppContainer sandbox" + } + $first = (& $trail --workspace $successRoot --json env sync recipe-a --adapter command) | ConvertFrom-Json + $second = (& $trail --workspace $successRoot --json env sync recipe-b --adapter command) | ConvertFrom-Json + $firstLayer = $first.layers[0] + $secondLayer = $second.layers[0] + if ($firstLayer.layer_id -ne $secondLayer.layer_id) { throw "identical Windows recipes did not reuse one layer" } + if (-not (Test-Path -LiteralPath (Join-Path $firstLayer.storage_path "out.tar") -PathType Leaf)) { + throw "sandboxed Windows recipe did not publish its declared archive" + } + + $privateRoot = Join-Path $env:RUNNER_TEMP ("trail-recipe-private-" + [guid]::NewGuid()) + Write-Recipe $privateRoot @("tar.exe", "-cf", "generated/out.tar", "input.txt") "writable_private" + New-RecipeLane $privateRoot + $private = (& $trail --workspace $privateRoot --json env sync recipe-a --adapter command) | ConvertFrom-Json + if ($private.layers.Count -ne 0) { throw "writable-private Windows recipe manufactured a shared layer" } + $privateOutput = $private.generation.components[0].outputs[0] + if ($privateOutput.policy -ne "writable_private" -or $null -ne $privateOutput.layer_id) { + throw "writable-private Windows recipe reported incorrect storage policy" + } + & $trail --workspace $privateRoot lane exec recipe-a -- cmd.exe /d /c "echo private-mutation>.trail-generated\copy\private.txt" + if ($LASTEXITCODE -ne 0) { throw "could not mutate writable-private Windows output" } + & $trail --workspace $privateRoot env sync recipe-a --adapter command | Out-Null + if ($LASTEXITCODE -ne 0) { throw "could not resynchronize writable-private Windows output" } + & $trail --workspace $privateRoot lane exec recipe-a -- cmd.exe /d /c "findstr private-mutation .trail-generated\copy\private.txt>nul" + if ($LASTEXITCODE -ne 0) { throw "writable-private Windows state was not preserved" } + + $multiRoot = Join-Path $env:RUNNER_TEMP ("trail-recipe-multi-" + [guid]::NewGuid()) + Write-Recipe $multiRoot @("tar.exe", "-cf", "generated/out.tar", "input.txt") + Add-Content -LiteralPath (Join-Path $multiRoot "trail.environment.toml") -Encoding utf8NoBOM -Value @" + +[[component.output]] +name = "beta" +source = "generated-b" +target = ".trail-generated/beta" +policy = "immutable_seed_private" +portability = "host" +"@ + New-RecipeLane $multiRoot + & $trail --workspace $multiRoot lane spawn recipe-b --from main --workdir-mode dokan-cow | Out-Null + if ($LASTEXITCODE -ne 0) { throw "multi-output second lane spawn failed" } + $multiFirst = (& $trail --workspace $multiRoot --json env sync recipe-a --adapter command) | ConvertFrom-Json + $multiSecond = (& $trail --workspace $multiRoot --json env sync recipe-b --adapter command) | ConvertFrom-Json + $multiFirstLayer = $multiFirst.layers[0] + $multiSecondLayer = $multiSecond.layers[0] + if ($multiFirstLayer.layer_id -ne $multiSecondLayer.layer_id) { + throw "identical Windows multi-output recipes did not reuse one layer" + } + if ($multiFirst.generation.components[0].outputs.Count -ne 2) { + throw "Windows recipe did not report both declared outputs" + } + if (-not (Test-Path -LiteralPath (Join-Path $multiFirstLayer.storage_path "outputs/0000/out.tar") -PathType Leaf)) { + throw "Windows recipe did not package its first composite output" + } + if (-not (Test-Path -LiteralPath (Join-Path $multiFirstLayer.storage_path "outputs/0001") -PathType Container)) { + throw "Windows recipe did not package its second composite output" + } + & $trail --workspace $multiRoot lane exec recipe-a -- cmd.exe /d /c "echo lane-a>.trail-generated\beta\lane.txt" + if ($LASTEXITCODE -ne 0) { throw "could not mutate the first lane's private multi-output upper" } + & $trail --workspace $multiRoot lane exec recipe-b -- cmd.exe /d /c "if exist .trail-generated\beta\lane.txt (exit /b 9) else (exit /b 0)" + if ($LASTEXITCODE -ne 0) { throw "a Windows multi-output mutation leaked between lanes" } + + $outside = Join-Path $env:RUNNER_TEMP ("trail-outside-" + [guid]::NewGuid() + ".txt") + Set-Content -LiteralPath $outside -Value "host canary" -Encoding utf8NoBOM + & "$env:SystemRoot/System32/icacls.exe" $outside /inheritance:r /grant:r "${env:USERNAME}:(R,W)" "SYSTEM:(F)" /Q | Out-Null + if ($LASTEXITCODE -ne 0) { throw "could not harden the outside-read canary ACL" } + $readRoot = Join-Path $env:RUNNER_TEMP ("trail-recipe-read-" + [guid]::NewGuid()) + Write-Recipe $readRoot @("tar.exe", "-cf", "generated/out.tar", $outside) + New-RecipeLane $readRoot + Assert-TrailFails { & $trail --workspace $readRoot env sync recipe-a --adapter command } "restricted Windows recipe unexpectedly read an undeclared host file" + + $writeRoot = Join-Path $env:RUNNER_TEMP ("trail-recipe-write-" + [guid]::NewGuid()) + Write-Recipe $writeRoot @("tar.exe", "-cf", "escape.tar", "input.txt") + New-RecipeLane $writeRoot + Assert-TrailFails { & $trail --workspace $writeRoot env sync recipe-a --adapter command } "restricted Windows recipe unexpectedly wrote outside its declared output" + + $listener = [System.Net.Sockets.TcpListener]::new([System.Net.IPAddress]::Loopback, 0) + $listener.Start() + $port = ([System.Net.IPEndPoint] $listener.LocalEndpoint).Port + $listener.Stop() + $server = Start-Process -FilePath "python.exe" -ArgumentList @("-m", "http.server", "$port", "--bind", "127.0.0.1") -PassThru -WindowStyle Hidden + try { + Start-Sleep -Milliseconds 500 + $networkRoot = Join-Path $env:RUNNER_TEMP ("trail-recipe-network-" + [guid]::NewGuid()) + Write-Recipe $networkRoot @("curl.exe", "--fail", "--max-time", "2", "http://127.0.0.1:$port/", "-o", "generated/network.txt") + New-RecipeLane $networkRoot + Assert-TrailFails { & $trail --workspace $networkRoot env sync recipe-a --adapter command } "restricted Windows recipe unexpectedly used a network socket" + } + finally { + Stop-Process -Id $server.Id -Force -ErrorAction SilentlyContinue + } + + $childRoot = Join-Path $env:RUNNER_TEMP ("trail-recipe-child-" + [guid]::NewGuid()) + Write-Recipe $childRoot @("forfiles.exe", "/P", ".", "/M", "input.txt", "/C", "cmd /c echo child>generated\child.txt") + New-RecipeLane $childRoot + & $trail --workspace $childRoot --json env sync recipe-a --adapter command *> $null + if ($LASTEXITCODE -eq 0) { + $childLayer = (& $trail --workspace $childRoot --json env sync recipe-a --adapter command) | ConvertFrom-Json + if (Test-Path -LiteralPath (Join-Path $childLayer.layers[0].storage_path "child.txt") -PathType Leaf) { + throw "restricted Windows recipe unexpectedly launched a child process" + } + } + + $shellRoot = Join-Path $env:RUNNER_TEMP ("trail-recipe-shell-" + [guid]::NewGuid()) + Write-Recipe $shellRoot @("cmd.exe", "/c", "exit", "0") + New-RecipeLane $shellRoot + Assert-TrailFails { & $trail --workspace $shellRoot env plan recipe-a --adapter command } "restricted Windows recipe unexpectedly accepted cmd.exe" + + Write-Output "windows-command-recipe shared-layer=$($firstLayer.layer_id) multi-output-layer=$($multiFirstLayer.layer_id) writable-private=verified private-copy-up=isolated host-read=denied undeclared-write=denied network=denied child-process=denied shell=denied" +} +finally { + Pop-Location +} diff --git a/scripts/verify-windows-environment-adapter-plugin.ps1 b/scripts/verify-windows-environment-adapter-plugin.ps1 new file mode 100644 index 0000000..4362354 --- /dev/null +++ b/scripts/verify-windows-environment-adapter-plugin.ps1 @@ -0,0 +1,348 @@ +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +if (-not $IsWindows) { + throw "Windows is required for the AppContainer adapter-plugin verifier" +} + +$repoRoot = Split-Path -Parent $PSScriptRoot +Push-Location $repoRoot +try { + cargo build -p trail + if ($LASTEXITCODE -ne 0) { throw "cargo build failed" } + cargo build -p trail-environment-adapter-sdk --example generated-copy-adapter --example mounted-initializer-adapter --example mounted-fixture-tool --example cache-adapter --example cache-fixture-tool --example adversarial-adapter --example fixture-sign-adapter + if ($LASTEXITCODE -ne 0) { throw "adapter example build failed" } + $trail = Join-Path $repoRoot "target/debug/trail.exe" + $exampleDir = Join-Path $repoRoot "target/debug/examples" + + function Write-Package( + [string] $Directory, + [string] $Identity, + [string] $Selector, + [string] $ExecutableSource, + [int] $TimeoutMs, + [int] $ResponseBytes + ) { + New-Item -ItemType Directory -Path $Directory -Force | Out-Null + $executable = Join-Path $Directory "adapter-plugin.exe" + Copy-Item -LiteralPath $ExecutableSource -Destination $executable + $digest = (Get-FileHash -LiteralPath $executable -Algorithm SHA256).Hash.ToLowerInvariant() + $manifest = @" +schema = "trail.environment-adapter-package/v1" + +[adapter] +canonical_identity = "$Identity" +implementation_version = "1.0.0" +selectors = ["$Identity", "$Selector"] +kind = "generated" +layer_adapter_name = "$Selector" +discovery_markers = ["copy.adapter"] +stability = "experimental" +description = "Windows fixture for $Identity" + +[executable] +path = "adapter-plugin.exe" +sha256 = "$digest" + +[permissions] +read_patterns = ["copy.adapter", "input.txt"] +max_input_files = 8 +max_input_bytes = 1048576 +timeout_ms = $TimeoutMs +max_response_bytes = $ResponseBytes +"@ + Set-Content -LiteralPath (Join-Path $Directory "trail-adapter.toml") -Value $manifest -Encoding utf8NoBOM + } + + function Assert-TrailFails([scriptblock] $Action, [string] $Message) { + & $Action *> $null + if ($LASTEXITCODE -eq 0) { throw $Message } + } + + function Write-MountedPackage([string] $Directory) { + New-Item -ItemType Directory -Path $Directory -Force | Out-Null + $executable = Join-Path $Directory "adapter-plugin.exe" + Copy-Item -LiteralPath (Join-Path $exampleDir "mounted-initializer-adapter.exe") -Destination $executable + $digest = (Get-FileHash -LiteralPath $executable -Algorithm SHA256).Hash.ToLowerInvariant() + $manifest = @" +schema = "trail.environment-adapter-package/v1" + +[adapter] +canonical_identity = "example/mounted@1" +implementation_version = "1.0.0" +selectors = ["example/mounted@1", "example-mounted"] +kind = "generated" +layer_adapter_name = "example-mounted" +discovery_markers = ["mounted.adapter"] +protocols = ["trail.environment-adapter/v2"] +stability = "experimental" +description = "Windows mounted initializer protocol-v2 fixture" + +[executable] +path = "adapter-plugin.exe" +sha256 = "$digest" + +[permissions] +read_patterns = ["mounted.adapter"] +max_input_files = 8 +max_input_bytes = 1048576 +timeout_ms = 5000 +max_response_bytes = 1048576 +"@ + Set-Content -LiteralPath (Join-Path $Directory "trail-adapter.toml") -Value $manifest -Encoding utf8NoBOM + } + + function Write-CachePackage([string] $Directory) { + New-Item -ItemType Directory -Path $Directory -Force | Out-Null + $executable = Join-Path $Directory "adapter-plugin.exe" + Copy-Item -LiteralPath (Join-Path $exampleDir "cache-adapter.exe") -Destination $executable + $digest = (Get-FileHash -LiteralPath $executable -Algorithm SHA256).Hash.ToLowerInvariant() + $manifest = @" +schema = "trail.environment-adapter-package/v1" + +[adapter] +canonical_identity = "example/cache@1" +implementation_version = "1.0.0" +selectors = ["example/cache@1", "example-cache"] +kind = "generated" +layer_adapter_name = "example-cache" +discovery_markers = ["cache.adapter"] +protocols = ["trail.environment-adapter/v2"] +stability = "experimental" +description = "Windows host-owned cache protocol-v2 fixture" + +[executable] +path = "adapter-plugin.exe" +sha256 = "$digest" + +[permissions] +read_patterns = ["cache.adapter"] +max_input_files = 8 +max_input_bytes = 1048576 +timeout_ms = 5000 +max_response_bytes = 1048576 +"@ + Set-Content -LiteralPath (Join-Path $Directory "trail-adapter.toml") -Value $manifest -Encoding utf8NoBOM + } + + $workspace = Join-Path $env:RUNNER_TEMP ("trail-plugin-" + [guid]::NewGuid()) + $packages = Join-Path $env:RUNNER_TEMP ("trail-plugin-packages-" + [guid]::NewGuid()) + $toolBin = Join-Path $env:RUNNER_TEMP ("trail-plugin-tools-" + [guid]::NewGuid()) + New-Item -ItemType Directory -Path $workspace, $packages, $toolBin -Force | Out-Null + Copy-Item -LiteralPath (Join-Path $exampleDir "mounted-fixture-tool.exe") -Destination (Join-Path $toolBin "mounted-fixture-tool.exe") + Copy-Item -LiteralPath (Join-Path $exampleDir "cache-fixture-tool.exe") -Destination (Join-Path $toolBin "cache-fixture-tool.exe") + $env:PATH = "$toolBin;$($env:PATH)" + Set-Content -LiteralPath (Join-Path $workspace "copy.adapter") -Value "plugin marker" -Encoding utf8NoBOM + Set-Content -LiteralPath (Join-Path $workspace "mounted.adapter") -Value "success" -Encoding utf8NoBOM + Set-Content -LiteralPath (Join-Path $workspace "cache.adapter") -Value "lane-a" -Encoding utf8NoBOM + Set-Content -LiteralPath (Join-Path $workspace "input.txt") -Value "declared input" -Encoding utf8NoBOM + + $copyPackage = Join-Path $packages "copy" + $mountedPackage = Join-Path $packages "mounted" + $cachePackage = Join-Path $packages "cache" + Write-Package $copyPackage "example/copy@1" "example-copy" (Join-Path $exampleDir "generated-copy-adapter.exe") 5000 1048576 + Write-MountedPackage $mountedPackage + Write-CachePackage $cachePackage + & $trail --workspace $workspace init --working-tree | Out-Null + if ($LASTEXITCODE -ne 0) { throw "trail init failed" } + $inspection = (& $trail --workspace $workspace --json env plugin inspect $copyPackage) | ConvertFrom-Json + $signaturePath = Join-Path $copyPackage "trail-adapter.sig" + $keyPath = Join-Path $copyPackage "publisher-key.toml" + & (Join-Path $exampleDir "fixture-sign-adapter.exe") ` + "example-publisher" ` + "0707070707070707070707070707070707070707070707070707070707070707" ` + $inspection.payload_digest ` + $signaturePath ` + $keyPath + if ($LASTEXITCODE -ne 0) { throw "adapter fixture signing failed" } + Assert-TrailFails { & $trail --workspace $workspace env plugin install $copyPackage } "signed Windows adapter installed before publisher trust" + $trusted = (& $trail --workspace $workspace --json env plugin trust add $keyPath) | ConvertFrom-Json + foreach ($lane in @("plugin-a", "plugin-b", "plugin-mounted-a", "plugin-mounted-b", "plugin-mounted-kill")) { + & $trail --workspace $workspace lane spawn $lane --from main --workdir-mode dokan-cow | Out-Null + if ($LASTEXITCODE -ne 0) { throw "lane spawn failed for $lane" } + } + + $installed = (& $trail --workspace $workspace --json env plugin install $copyPackage) | ConvertFrom-Json + if ($installed.trust -ne "publisher_signed" -or $installed.certification_tier -ne "publisher-authenticated-experimental") { + throw "signed Windows adapter did not report publisher authentication" + } + $catalog = (& $trail --workspace $workspace --json env adapters) | ConvertFrom-Json + $catalogEntry = $catalog.adapters | Where-Object { $_.canonical_identity -eq "example/copy@1" } + if ($null -eq $catalogEntry -or $catalogEntry.source -ne "plugin") { + throw "installed Windows plugin was absent from the adapter catalog" + } + $discovery = (& $trail --workspace $workspace --json env discover plugin-a) | ConvertFrom-Json + if ($discovery.components.component_id -notcontains "plugin.copy") { + throw "Windows plugin discovery did not propose plugin.copy" + } + $plan = (& $trail --workspace $workspace --json env plan plugin-a --adapter example/copy@1) | ConvertFrom-Json + if ($plan.capabilities.sandbox -ne "windows-appcontainer-job" -or $plan.capabilities.network -ne "deny") { + throw "Windows plugin plan did not report denied-by-default AppContainer capabilities" + } + $first = (& $trail --workspace $workspace --json env sync plugin-a --adapter example/copy@1) | ConvertFrom-Json + $second = (& $trail --workspace $workspace --json env sync plugin-b --adapter example/copy@1) | ConvertFrom-Json + $firstLayer = $first.layers[0] + $secondLayer = $second.layers[0] + if ($firstLayer.layer_id -ne $secondLayer.layer_id) { throw "Windows plugins did not reuse one layer" } + if (-not (Test-Path -LiteralPath (Join-Path $firstLayer.storage_path "out.tar") -PathType Leaf)) { + throw "Windows plugin did not publish its generated archive" + } + & $trail --workspace $workspace lane exec plugin-a -- cmd.exe /d /c "echo lane-a>.trail-generated\plugin-copy\private.txt" + if ($LASTEXITCODE -ne 0) { throw "could not mutate the first plugin lane" } + & $trail --workspace $workspace lane exec plugin-b -- cmd.exe /d /c "if exist .trail-generated\plugin-copy\private.txt (exit /b 9) else (exit /b 0)" + if ($LASTEXITCODE -ne 0) { throw "Windows plugin private output leaked between lanes" } + & $trail --workspace $workspace lane exec plugin-a -- cmd.exe /d /c "echo changed-input>input.txt" + if ($LASTEXITCODE -ne 0) { throw "could not change the Windows plugin identity input" } + & $trail --workspace $workspace lane checkpoint plugin-a -m "change plugin input" | Out-Null + if ($LASTEXITCODE -ne 0) { throw "could not checkpoint the Windows plugin identity input" } + $readiness = (& $trail --workspace $workspace --json lane readiness plugin-a) | ConvertFrom-Json + if ($readiness.blockers.code -notcontains "dependency_environment_stale") { + throw "Windows plugin input change did not block readiness as stale" + } + $status = (& $trail --workspace $workspace --json env status plugin-a) | ConvertFrom-Json + if ($status.status -notcontains "stale") { throw "Windows plugin state did not persist stale" } + + & $trail --workspace $workspace env plugin install $cachePackage | Out-Null + if ($LASTEXITCODE -ne 0) { throw "could not install Windows cache fixture" } + & $trail --workspace $workspace lane exec plugin-b -- cmd.exe /d /c "echo lane-b>cache.adapter" | Out-Null + if ($LASTEXITCODE -ne 0) { throw "could not change the second Windows cache fixture input" } + & $trail --workspace $workspace lane checkpoint plugin-b -m "give cache fixture a distinct component key" | Out-Null + if ($LASTEXITCODE -ne 0) { throw "could not checkpoint the second Windows cache fixture input" } + $cachePlan = (& $trail --workspace $workspace --json env plan plugin-a --adapter example/cache@1) | ConvertFrom-Json + if ($cachePlan.caches.Count -ne 1 -or $cachePlan.caches[0].access -ne "host_exclusive" -or $cachePlan.caches[0].protocol -ne "content_store") { + throw "Windows cache plugin plan lost its conservative cache contract" + } + $cacheA = (& $trail --workspace $workspace --json env sync plugin-a --adapter example/cache@1) | ConvertFrom-Json + $cacheB = (& $trail --workspace $workspace --json env sync plugin-b --adapter example/cache@1) | ConvertFrom-Json + $cacheANamespace = $cacheA.generation.components[0].caches[0].namespace_id + $cacheBNamespace = $cacheB.generation.components[0].caches[0].namespace_id + if ($cacheANamespace -ne $cacheBNamespace) { throw "Windows cache plugin lanes did not reuse one namespace" } + $cacheAObservation = @(& $trail --workspace $workspace lane exec plugin-a -- cmd.exe /d /c "type .trail-generated\plugin-cache\cache-observation.txt")[0].Trim() + $cacheBObservation = @(& $trail --workspace $workspace lane exec plugin-b -- cmd.exe /d /c "type .trail-generated\plugin-cache\cache-observation.txt")[0].Trim() + if ($cacheAObservation -ne "$cacheANamespace|1" -or $cacheBObservation -ne "$cacheBNamespace|2") { + throw "Windows cache plugin did not serialize and reuse its host namespace" + } + $counter = (Get-Content -LiteralPath (Join-Path $workspace ".trail/cache/namespaces/$cacheANamespace/counter") -Raw).Trim() + if ($counter -ne "2") { throw "Windows cache plugin namespace did not retain both executions" } + & $trail --workspace $workspace lane exec plugin-a -- cmd.exe /d /c "echo escape>cache.adapter" | Out-Null + & $trail --workspace $workspace lane checkpoint plugin-a -m "attempt plugin cache namespace escape" | Out-Null + Assert-TrailFails { & $trail --workspace $workspace env sync plugin-a --adapter example/cache@1 } "Windows plugin cache write escaped its namespace" + if (Test-Path -LiteralPath (Join-Path $workspace ".trail/cache/namespaces/plugin-cache-escape")) { + throw "Windows plugin cache escape created a sibling namespace entry" + } + $cacheAObservation = @(& $trail --workspace $workspace lane exec plugin-a -- cmd.exe /d /c "type .trail-generated\plugin-cache\cache-observation.txt")[0].Trim() + if ($cacheAObservation -ne "$cacheANamespace|1") { throw "failed Windows cache escape replaced its predecessor" } + + & $trail --workspace $workspace env plugin install $mountedPackage | Out-Null + if ($LASTEXITCODE -ne 0) { throw "could not install mounted protocol-v2 fixture" } + $mountedCatalog = (& $trail --workspace $workspace --json env adapters) | ConvertFrom-Json + $mountedEntry = $mountedCatalog.adapters | Where-Object { $_.canonical_identity -eq "example/mounted@1" } + if ($null -eq $mountedEntry -or $mountedEntry.protocols -notcontains "trail.environment-adapter/v2") { + throw "Windows catalog did not expose the mounted adapter protocol" + } + $mountedPlan = (& $trail --workspace $workspace --json env plan plugin-mounted-a --adapter example/mounted@1) | ConvertFrom-Json + if ($mountedPlan.commands.Count -ne 1 -or $mountedPlan.commands[0].phase -ne "mounted_initialization") { + throw "Windows mounted plugin did not report exactly one mounted action" + } + $mountedA = (& $trail --workspace $workspace --json env sync plugin-mounted-a --adapter example/mounted@1) | ConvertFrom-Json + $mountedB = (& $trail --workspace $workspace --json env sync plugin-mounted-b --adapter example/mounted@1) | ConvertFrom-Json + if ($mountedA.layers.Count -ne 0 -or $mountedB.layers.Count -ne 0) { + throw "Windows mounted plugin unexpectedly published a shared layer" + } + $mountedAPwd = @(& $trail --workspace $workspace lane exec plugin-mounted-a -- cmd.exe /d /c cd)[0].Trim() + $mountedBPwd = @(& $trail --workspace $workspace lane exec plugin-mounted-b -- cmd.exe /d /c cd)[0].Trim() + $mountedARecorded = @(& $trail --workspace $workspace lane exec plugin-mounted-a -- cmd.exe /d /c "type .trail-generated\plugin-mounted\initialized.txt")[0].Trim() + $mountedBRecorded = @(& $trail --workspace $workspace lane exec plugin-mounted-b -- cmd.exe /d /c "type .trail-generated\plugin-mounted\initialized.txt")[0].Trim() + $mountedARecorded = $mountedARecorded -replace '\|success$', '' + $mountedBRecorded = $mountedBRecorded -replace '\|success$', '' + if ($mountedARecorded -ne $mountedAPwd -or $mountedBRecorded -ne $mountedBPwd -or $mountedARecorded -eq $mountedBRecorded) { + throw "Windows mounted plugin did not initialize at two distinct final lane paths" + } + & $trail --workspace $workspace lane exec plugin-mounted-a -- cmd.exe /d /c "echo lane-a-private>.trail-generated\plugin-mounted\initialized.txt" | Out-Null + if ($LASTEXITCODE -ne 0) { throw "could not mutate first Windows mounted output" } + & $trail --workspace $workspace env sync plugin-mounted-a --adapter example/mounted@1 | Out-Null + if ($LASTEXITCODE -ne 0) { throw "compatible Windows mounted resync failed" } + $preserved = @(& $trail --workspace $workspace lane exec plugin-mounted-a -- cmd.exe /d /c "type .trail-generated\plugin-mounted\initialized.txt")[0].Trim() + if ($preserved -ne "lane-a-private") { throw "compatible Windows mounted resync replaced private state" } + & $trail --workspace $workspace lane exec plugin-mounted-a -- cmd.exe /d /c "echo fail>mounted.adapter" | Out-Null + & $trail --workspace $workspace lane checkpoint plugin-mounted-a -m "fail mounted plugin action" | Out-Null + Assert-TrailFails { & $trail --workspace $workspace env sync plugin-mounted-a --adapter example/mounted@1 } "failed Windows mounted action unexpectedly activated" + $preserved = @(& $trail --workspace $workspace lane exec plugin-mounted-a -- cmd.exe /d /c "type .trail-generated\plugin-mounted\initialized.txt")[0].Trim() + if ($preserved -ne "lane-a-private") { throw "failed Windows mounted action replaced its predecessor" } + & $trail --workspace $workspace lane exec plugin-mounted-b -- cmd.exe /d /c "echo source_write>mounted.adapter" | Out-Null + & $trail --workspace $workspace lane checkpoint plugin-mounted-b -m "attempt mounted source write" | Out-Null + Assert-TrailFails { & $trail --workspace $workspace env sync plugin-mounted-b --adapter example/mounted@1 } "Windows mounted source write escaped its output contract" + & $trail --workspace $workspace lane exec plugin-mounted-b -- cmd.exe /d /c "if exist source-leak.txt (exit /b 9) else (exit /b 0)" | Out-Null + if ($LASTEXITCODE -ne 0) { throw "Windows mounted source write leaked into the lane" } + & $trail --workspace $workspace lane exec plugin-mounted-b -- cmd.exe /d /c "echo source_read>mounted.adapter" | Out-Null + & $trail --workspace $workspace lane checkpoint plugin-mounted-b -m "attempt undeclared mounted source read" | Out-Null + Assert-TrailFails { & $trail --workspace $workspace env sync plugin-mounted-b --adapter example/mounted@1 } "Windows mounted undeclared source read unexpectedly succeeded" + & $trail --workspace $workspace lane exec plugin-mounted-b -- cmd.exe /d /c "if exist .trail-generated\plugin-mounted\leaked.txt (exit /b 9) else (exit /b 0)" | Out-Null + if ($LASTEXITCODE -ne 0) { throw "Windows mounted undeclared source read leaked content" } + & $trail --workspace $workspace env sync plugin-mounted-kill --adapter example/mounted@1 | Out-Null + & $trail --workspace $workspace lane exec plugin-mounted-kill -- cmd.exe /d /c "echo kill-predecessor>.trail-generated\plugin-mounted\initialized.txt & echo hang>mounted.adapter" | Out-Null + & $trail --workspace $workspace lane checkpoint plugin-mounted-kill -m "kill active mounted plugin action" | Out-Null + $killOut = Join-Path $packages "mounted-kill.stdout" + $killErr = Join-Path $packages "mounted-kill.stderr" + $syncProcess = Start-Process -FilePath $trail -ArgumentList @( + "--workspace", $workspace, "env", "sync", "plugin-mounted-kill", + "--adapter", "example/mounted@1" + ) -RedirectStandardOutput $killOut -RedirectStandardError $killErr -PassThru + $readyFile = $null + for ($attempt = 0; $attempt -lt 200; $attempt++) { + $readyFile = Get-ChildItem -LiteralPath (Join-Path $workspace ".trail/cache/staging") -Filter "running" -File -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($null -ne $readyFile) { break } + Start-Sleep -Milliseconds 50 + } + if ($null -eq $readyFile) { + Stop-Process -Id $syncProcess.Id -Force -ErrorAction SilentlyContinue + throw "Windows mounted plugin did not reach its active-command kill point" + } + $mountedChildPid = [int]((Get-Content -LiteralPath $readyFile.FullName -Raw).Trim()) + Stop-Process -Id $syncProcess.Id -Force + $syncProcess.WaitForExit() + for ($attempt = 0; $attempt -lt 200; $attempt++) { + if ($null -eq (Get-Process -Id $mountedChildPid -ErrorAction SilentlyContinue)) { break } + Start-Sleep -Milliseconds 50 + } + if ($null -ne (Get-Process -Id $mountedChildPid -ErrorAction SilentlyContinue)) { + Stop-Process -Id $mountedChildPid -Force -ErrorAction SilentlyContinue + throw "Windows mounted plugin action survived Trail process death" + } + & $trail --workspace $workspace env status plugin-mounted-kill | Out-Null + $preserved = @(& $trail --workspace $workspace lane exec plugin-mounted-kill -- cmd.exe /d /c "type .trail-generated\plugin-mounted\initialized.txt")[0].Trim() + if ($preserved -ne "kill-predecessor") { throw "Windows active-command kill replaced its predecessor" } + $abandoned = Get-ChildItem -LiteralPath (Join-Path $workspace ".trail/cache/staging") -Directory -Filter "mounted-environment-*" -ErrorAction SilentlyContinue + if ($null -ne $abandoned) { throw "Windows recovery left an abandoned mounted plugin candidate" } + + foreach ($behavior in @("hang", "crash", "oversized", "malformed", "child", "memory")) { + $timeout = if ($behavior -eq "hang") { 100 } else { 1000 } + $package = Join-Path $packages $behavior + Write-Package $package "example/$behavior@1" "example-$behavior" (Join-Path $exampleDir "adversarial-adapter.exe") $timeout 1048576 + & $trail --workspace $workspace env plugin install $package | Out-Null + if ($LASTEXITCODE -ne 0) { throw "could not install $behavior fixture" } + Assert-TrailFails { & $trail --workspace $workspace env plan plugin-a --adapter "example/$behavior@1" } "adversarial Windows $behavior adapter unexpectedly succeeded" + } + + $removed = (& $trail --workspace $workspace --json env plugin remove example/copy@1) | ConvertFrom-Json + if ($removed.removed_distribution_digest -ne $installed.distribution_digest) { + throw "Windows plugin removal lost its active distribution provenance" + } + $reinstalled = (& $trail --workspace $workspace --json env plugin install $copyPackage) | ConvertFrom-Json + Add-Content -LiteralPath (Join-Path $reinstalled.package_path "adapter-plugin.exe") -Value "tamper" -Encoding utf8NoBOM + Assert-TrailFails { & $trail --workspace $workspace env adapters } "tampered Windows adapter executable remained trusted" + & $trail --workspace $workspace env plugin install $copyPackage | Out-Null + if ($LASTEXITCODE -ne 0) { throw "could not repair the tampered Windows plugin" } + & $trail --workspace $workspace env plugin trust remove $trusted.key_id | Out-Null + if ($LASTEXITCODE -ne 0) { throw "could not revoke the Windows publisher key" } + Assert-TrailFails { & $trail --workspace $workspace env adapters } "revoked Windows publisher key left signed adapter active" + & $trail --workspace $workspace env plugin remove example/copy@1 | Out-Null + if ($LASTEXITCODE -ne 0) { throw "could not tombstone the tampered Windows plugin" } + & $trail --workspace $workspace env adapters | Out-Null + if ($LASTEXITCODE -ne 0) { throw "catalog did not recover after plugin tombstone" } + + Write-Output "windows-environment-adapter-plugin distribution=$($installed.distribution_digest) shared-layer=$($firstLayer.layer_id) external-cache=${cacheANamespace}:shared-host-exclusive-and-sandbox-contained mounted-v2=isolated-and-atomic active-command-kill=terminated-and-recovered declared-read=allowed undeclared-read-write=denied private-copy-up=isolated stale-refresh=verified publisher-signature=verified revocation=fail-closed timeout=denied memory=denied crash=denied oversized=denied malformed=denied child-process=denied tamper=denied" +} +finally { + Pop-Location +} diff --git a/scripts/xwin-clang-cl-wrapper.sh b/scripts/xwin-clang-cl-wrapper.sh index 0467a4c..7be29d3 100755 --- a/scripts/xwin-clang-cl-wrapper.sh +++ b/scripts/xwin-clang-cl-wrapper.sh @@ -28,4 +28,4 @@ for arg in "$@"; do fi done -exec clang-cl "$@" +exec "${TRAIL_REAL_CLANG_CL:-clang-cl}" "$@" diff --git a/skills/use-trail/SKILL.md b/skills/use-trail/SKILL.md index 78a1882..2bce68c 100644 --- a/skills/use-trail/SKILL.md +++ b/skills/use-trail/SKILL.md @@ -37,7 +37,7 @@ Do not launch `trail agent start` recursively when already running as the provid 4. Perform only the scoped mutation the user authorized. 5. Re-read state and record evidence such as a lane operation, test/eval gate, review marker, handoff, or receipt. -Treat non-dry-run Git apply/finish, merges into shared refs, merge-queue execution, rewind/undo, conflict resolution, restore, garbage collection, lane removal, and force/bypass flags as consequential. Require clear user intent before using them. Never substitute `--allow-stale`, `--allow-ignored`, `--force`, `--direct`, or `--no-auth` for resolving the underlying safety condition. +Treat non-dry-run Git apply/finish, merges into shared refs, lane merge-queue execution, rewind/undo, conflict resolution, restore, garbage collection, lane removal, and force/bypass flags as consequential. Require clear user intent before using them. Never substitute `--allow-stale`, `--allow-ignored`, `--force`, `--direct`, or `--no-auth` for resolving the underlying safety condition. ## Use Stable Automation Patterns diff --git a/skills/use-trail/references/agent-tasks.md b/skills/use-trail/references/agent-tasks.md index 4a349b0..20310c7 100644 --- a/skills/use-trail/references/agent-tasks.md +++ b/skills/use-trail/references/agent-tasks.md @@ -7,17 +7,17 @@ Use this surface when the user is operating coding-agent tasks. It owns task lan Check prerequisites: ```sh -trail agent doctor --provider codex -trail agent setup --provider codex --editor vscode +trail agent doctor codex +trail agent acp setup codex --editor vscode ``` Start one isolated terminal task: ```sh -trail agent start --provider codex --name +trail agent start codex --name ``` -Profiles include `claude-code`, `codex`, `cursor`, `gemini`, `aider`, and `opencode`. Use `--workdir-mode full-cow` as the portable default. Use `overlay-cow` only when FUSE is available and `nfs-cow` on macOS when its tradeoffs are acceptable. Override the provider command only after `--`. +Profiles include `claude-code`, `codex`, `cursor`, `gemini`, `aider`, and `opencode`. Use `--workdir-mode native-cow` as the portable default. Use `fuse-cow` only when FUSE is available, `nfs-cow` on macOS when its tradeoffs are acceptable, and `dokan-cow` on Windows with Dokan 2.x. Override the provider command only after `--`. If already launched inside the task, edit and test in the provided workdir. Do not create a nested task. diff --git a/skills/use-trail/references/integrations.md b/skills/use-trail/references/integrations.md index b39ab32..0ecdff1 100644 --- a/skills/use-trail/references/integrations.md +++ b/skills/use-trail/references/integrations.md @@ -27,7 +27,7 @@ Honor tool risk annotations: - Read-only: status, reports, diff, readiness, diagnosis, resources. - Workspace write: review markers and archive metadata. -- Destructive write: apply/finish, undo/rewind, merge-queue execution. +- Destructive write: apply/finish, undo/rewind, lane merge-queue execution. - Open-world write: test/eval commands. Require confirmation appropriate to the risk. For non-dry-run apply/finish, call readiness and dry-run first. @@ -52,11 +52,11 @@ Use `trail.run_pause`/`trail.run_resume` across real interruptions. Never fabric Use ACP when an editor should keep its normal agent UX while Trail records turns, events, edits, and checkpoints and injects Trail MCP tools: ```sh -trail agent setup --provider codex --editor vscode +trail agent acp setup codex --editor vscode trail acp relay codex ``` -Use `trail agent start --provider ` for terminal-first agents. ACP relay is the richer streaming-capture path; terminal tasks universally isolate work and record the final checkpoint. +Use `trail agent start ` for terminal-first agents. ACP relay is the richer streaming-capture path; terminal tasks universally isolate work and record the final checkpoint. ## HTTP Daemon diff --git a/skills/use-trail/references/lanes.md b/skills/use-trail/references/lanes.md index b093b34..cf5e8d4 100644 --- a/skills/use-trail/references/lanes.md +++ b/skills/use-trail/references/lanes.md @@ -7,7 +7,7 @@ Use a lane for one bounded unit of active work that needs isolation, provenance, From the original Trail workspace: ```sh -trail lane spawn --from main --workdir-mode full-cow +trail lane spawn --from main --workdir-mode native-cow trail lane status trail lane workdir ``` @@ -16,9 +16,10 @@ Choose intentionally: - `virtual`: no filesystem workdir; use structured patches. - `sparse`: selected paths only; supply `--paths`. -- `full-cow`: portable full materialization. -- `overlay-cow`: runtime-mounted FUSE COW where supported. +- `native-cow`: portable full materialization using native clone/reflink COW when available. +- `fuse-cow`: runtime-mounted FUSE COW where supported. - `nfs-cow`: macOS loopback NFS COW. +- `dokan-cow`: Windows Dokan COW. For a narrow large-repository task: @@ -30,6 +31,76 @@ trail lane claim README.md --ttl-secs 1800 Edit only in the returned lane workdir. From that workdir, pass `--workspace ` to Trail commands unless workspace discovery is known to resolve correctly. +For a layered lane with a single supported environment at the selected root, build or +reuse its immutable environment before starting work: + +```sh +trail env adapters +trail env sync +trail env status +``` + +`trail env adapters` lists canonical identities, accepted selectors, stability, and +manifest names used by side-effect-free discovery. It does not probe package managers, +compilers, or repository files. + +For semantic planning beyond a command profile, install an experimental local adapter +package explicitly: + +```sh +trail env plugin install path/to/package +trail env adapters +trail env plan --adapter namespace/name@1 +trail env plugin remove namespace/name@1 +``` + +Trail content-addresses and revalidates the package, gives its planner only bounded bytes +from the pinned root, and runs it without repository, network, child-process, database, +mount, or publication authority. Local packages are experimental; signed organization +catalogs and WASI distribution are not yet available. + +Auto-detection supports Node, the experimental Cargo target-seed adapter, +single-module Go vendoring, and lane-private CMake build trees. For a polyglot root, +select explicitly with +`--adapter trail/node@1`, `--adapter trail/cargo-target-seed@1`, or +`--adapter trail/go-vendor@1`; use `--adapter trail/cmake-build@1` for CMake and +`--path ` for a nested component. +Environment synchronization requires an unmounted lane because it atomically advances +the environment binding generation. `trail deps sync` remains the Node compatibility +command. + +For CMake, synchronization provisions the lane-private build directory without running +configure in a disposable staging path. Configure and build inside the lane so +`CMakeCache.txt` records the correct mounted path: + +```sh +trail env sync --adapter trail/cmake-build@1 +trail lane exec -- cmake -S . -B build -G Ninja +trail lane exec -- cmake --build build +``` + +Inspect a monorepo before executing installers, then activate every non-conflicting +proposal together: + +```sh +trail env discover +trail env plan +trail env sync-all +trail env generation +``` + +`trail env plan` is read-only and shows the normalized component key, input hashes, +resolved executable identity, argv, mount, portability, and capability grants before +synchronization. Repository-defined `trail/command@1` components may be declared in +`trail.environment.toml`; execution uses macOS sandbox-exec, Linux Landlock plus +seccomp, or a capability-free Windows AppContainer constrained by a one-process Job +Object, and fails closed when the required native enforcement is unavailable. +If discovery reports multiple components at one root, pass `--component ` to +`env plan` or `env sync`, or use `env sync-all` to activate the whole environment. + +`sync-all` builds components before changing mounts; activation advances one durable +generation or leaves the predecessor authoritative. + ## Materialized Workdir Changes Preview before recording: @@ -117,19 +188,31 @@ trail lane diff --patch --show-line-ids trail approvals list --lane ``` +If readiness reports `dependency_environment_stale`, inspect the exact cause before +rebuilding: + +```sh +trail env status +trail env explain --component +trail env plan --component +``` + +Explanation reports name changed inputs, tools, platforms, and policies without +rendering their values. Use `--offset` and `--limit` for large monorepos. + Stop on readiness blockers. Preview refresh and merge: ```sh trail lane refresh-preview --target main -trail merge-lane --into main --dry-run +trail lane merge --into main --dry-run ``` For shared targets, queue rather than directly merging: ```sh -trail merge-queue add --into main -trail merge-queue explain -trail merge-queue run +trail lane merge-queue add --into main +trail lane merge-queue explain +trail lane merge-queue run ``` Queue execution is consequential and re-runs readiness. Do it only when authorized. Remove a lane only after verifying it is merged or intentionally abandoned; `lane rm --force` is not routine cleanup. diff --git a/skills/use-trail/references/safety-and-recovery.md b/skills/use-trail/references/safety-and-recovery.md index 7e323e9..77e7a57 100644 --- a/skills/use-trail/references/safety-and-recovery.md +++ b/skills/use-trail/references/safety-and-recovery.md @@ -7,7 +7,7 @@ Use Trail's safety signals as workflow inputs, not obstacles to bypass. Preview and obtain clear user intent before: - Non-dry-run `trail agent apply` or `finish` because they can record a task workdir, create a Git commit, and fast-forward the current Git branch. -- Direct/shared-ref merge, `merge-queue run`, or conflict resolution. +- Direct/shared-ref merge, `trail lane merge-queue run`, or conflict resolution. - `undo`, `rewind`, checkout into an active workspace, forced workdir sync, lane removal, restore, non-dry-run GC, or destructive branch operations. - Approval decisions, test/eval execution, network/deploy commands, or other open-world actions. @@ -69,7 +69,7 @@ Readiness may block on dirty materialized workdirs, required or failed gates, pe For queued work: ```sh -trail merge-queue explain +trail lane merge-queue explain trail lane refresh-preview --target main ``` diff --git a/tools/acp-v1-reference-peer/Cargo.lock b/tools/acp-v1-reference-peer/Cargo.lock new file mode 100644 index 0000000..ec0d73d --- /dev/null +++ b/tools/acp-v1-reference-peer/Cargo.lock @@ -0,0 +1,771 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "acp-v1-reference-peer" +version = "0.1.0" +dependencies = [ + "agent-client-protocol-schema", + "serde", + "serde_json", +] + +[[package]] +name = "agent-client-protocol-schema" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06679e1542356341f4550ccfb16338b64f37f6af70de2105446ef6fbb078234c" +dependencies = [ + "anyhow", + "derive_more", + "schemars 1.2.1", + "serde", + "serde_json", + "serde_with", + "strum", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cc" +version = "1.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn", + "unicode-xid", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "indexmap 2.14.0", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "time" +version = "0.3.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2f034a4bebf216c9e4b7083603e024cf930873fd67830cfb083c9fa33129d9" diff --git a/tools/acp-v1-reference-peer/Cargo.toml b/tools/acp-v1-reference-peer/Cargo.toml new file mode 100644 index 0000000..2dd4c4b --- /dev/null +++ b/tools/acp-v1-reference-peer/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "acp-v1-reference-peer" +version = "0.1.0" +edition = "2024" +rust-version = "1.88" +publish = false + +[dependencies] +agent-client-protocol-schema = "=1.4.0" +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[workspace] diff --git a/tools/acp-v1-reference-peer/src/main.rs b/tools/acp-v1-reference-peer/src/main.rs new file mode 100644 index 0000000..cf0ba0b --- /dev/null +++ b/tools/acp-v1-reference-peer/src/main.rs @@ -0,0 +1,486 @@ +use std::io::{BufRead, BufReader, Write}; +use std::path::Path; +use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; + +use agent_client_protocol_schema::v1::*; +use serde::Serialize; +use serde::de::DeserializeOwned; +use serde_json::{Value, json}; + +type PeerResult = Result>; + +fn typed(value: Value) -> PeerResult { + Ok(serde_json::from_value(value)?) +} + +fn send(writer: &mut impl Write, message: &T) -> PeerResult { + serde_json::to_writer(&mut *writer, message)?; + writer.write_all(b"\n")?; + writer.flush()?; + Ok(()) +} + +fn read_value(reader: &mut impl BufRead) -> PeerResult { + let mut line = String::new(); + if reader.read_line(&mut line)? == 0 { + return Err("peer closed before the next ACP frame".into()); + } + Ok(serde_json::from_str(&line)?) +} + +fn receive(reader: &mut impl BufRead) -> PeerResult { + typed(read_value(reader)?) +} + +fn typed_send( + writer: &mut impl Write, + value: Value, +) -> PeerResult { + send(writer, &typed::(value)?) +} + +fn response_id(value: &Value) -> Value { + value.get("id").cloned().unwrap_or(Value::Null) +} + +fn run_agent() -> PeerResult { + let stdin = std::io::stdin(); + let mut input = stdin.lock(); + let stdout = std::io::stdout(); + let mut output = stdout.lock(); + let mut cwd = String::new(); + let session_id = "official-session".to_string(); + + while let Ok(message) = read_value(&mut input) { + let method = message.get("method").and_then(Value::as_str).unwrap_or(""); + let id = response_id(&message); + match method { + "initialize" => { + let _: JsonRpcMessage> = typed(message)?; + typed_send::>>( + &mut output, + json!({"jsonrpc":"2.0","id":id,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":true,"sessionCapabilities":{"list":{},"delete":{},"resume":{},"close":{}}}}}), + )?; + } + "authenticate" => { + let _: JsonRpcMessage> = typed(message)?; + typed_send::>>( + &mut output, + json!({"jsonrpc":"2.0","id":id,"error":{"code":-32000,"message":"reference auth rejection","data":{"typed":true}}}), + )?; + } + "session/new" => { + let _: JsonRpcMessage> = typed(message.clone())?; + cwd = message["params"]["cwd"].as_str().unwrap().to_string(); + typed_send::>>( + &mut output, + json!({"jsonrpc":"2.0","id":id,"result":{"sessionId":session_id}}), + )?; + } + "session/load" => { + let _: JsonRpcMessage> = typed(message)?; + typed_send::>>( + &mut output, + json!({"jsonrpc":"2.0","id":id,"result":{}}), + )?; + } + "session/resume" => { + let _: JsonRpcMessage> = typed(message)?; + typed_send::>>( + &mut output, + json!({"jsonrpc":"2.0","id":id,"result":{}}), + )?; + } + "session/close" => { + let _: JsonRpcMessage> = typed(message)?; + typed_send::>>( + &mut output, + json!({"jsonrpc":"2.0","id":id,"result":{}}), + )?; + } + "session/delete" => { + let _: JsonRpcMessage> = typed(message)?; + typed_send::>>( + &mut output, + json!({"jsonrpc":"2.0","id":id,"result":{}}), + )?; + } + "logout" => { + let _: JsonRpcMessage> = typed(message)?; + typed_send::>>( + &mut output, + json!({"jsonrpc":"2.0","id":id,"result":{}}), + )?; + } + "session/prompt" => { + let _: JsonRpcMessage> = typed(message)?; + let cancellation: JsonRpcMessage> = + receive(&mut input)?; + let _ = cancellation; + let rpc_cancellation: JsonRpcMessage> = + receive(&mut input)?; + let _ = rpc_cancellation; + + typed_send::>>( + &mut output, + json!({"jsonrpc":"2.0","method":"session/update","params":{"sessionId":session_id,"update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"official update"}}}}), + )?; + callback::( + &mut input, + &mut output, + json!({"jsonrpc":"2.0","id":"permission","method":"session/request_permission","params":{"sessionId":session_id,"toolCall":{"toolCallId":"official-tool","title":"Official permission"},"options":[{"optionId":"allow","name":"Allow","kind":"allow_once"}]}}), + )?; + callback::( + &mut input, + &mut output, + json!({"jsonrpc":"2.0","id":"read","method":"fs/read_text_file","params":{"sessionId":session_id,"path":format!("{cwd}/README.md")}}), + )?; + callback::( + &mut input, + &mut output, + json!({"jsonrpc":"2.0","id":"write","method":"fs/write_text_file","params":{"sessionId":session_id,"path":format!("{cwd}/official.txt"),"content":"official"}}), + )?; + callback::( + &mut input, + &mut output, + json!({"jsonrpc":"2.0","id":"create","method":"terminal/create","params":{"sessionId":session_id,"command":"printf","args":["official"],"cwd":cwd}}), + )?; + for (request_id, method_name) in [ + ("output", "terminal/output"), + ("wait", "terminal/wait_for_exit"), + ("kill", "terminal/kill"), + ("release", "terminal/release"), + ] { + terminal_callback( + &mut input, + &mut output, + request_id, + method_name, + &session_id, + )?; + } + typed_send::>>( + &mut output, + json!({"jsonrpc":"2.0","id":id,"result":{"stopReason":"cancelled"}}), + )?; + } + _ => return Err(format!("official agent received unexpected method {method}").into()), + } + } + Ok(()) +} + +fn callback(input: &mut impl BufRead, output: &mut impl Write, request: Value) -> PeerResult +where + P: DeserializeOwned + Serialize, + R: DeserializeOwned, +{ + typed_send::>>(output, request)?; + let _: JsonRpcMessage> = receive(input)?; + Ok(()) +} + +fn terminal_callback( + input: &mut impl BufRead, + output: &mut impl Write, + id: &str, + method: &str, + session_id: &str, +) -> PeerResult { + let request = json!({"jsonrpc":"2.0","id":id,"method":method,"params":{"sessionId":session_id,"terminalId":"official-terminal"}}); + match method { + "terminal/output" => { + callback::(input, output, request) + } + "terminal/wait_for_exit" => callback::< + WaitForTerminalExitRequest, + WaitForTerminalExitResponse, + >(input, output, request), + "terminal/kill" => { + callback::(input, output, request) + } + "terminal/release" => { + callback::(input, output, request) + } + _ => unreachable!(), + } +} + +struct RelayClient { + child: Child, + input: ChildStdin, + output: BufReader, +} + +impl RelayClient { + fn spawn(workspace: &Path, trail: &Path, agent: &Path) -> PeerResult { + let mut child = Command::new(trail) + .arg("--workspace") + .arg(workspace) + .args(["acp", "relay", "--"]) + .arg(agent) + .arg("agent") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn()?; + let input = child.stdin.take().ok_or("missing relay stdin")?; + let output = BufReader::new(child.stdout.take().ok_or("missing relay stdout")?); + Ok(Self { + child, + input, + output, + }) + } + + fn exchange(&mut self, request: Value) -> PeerResult>> + where + P: DeserializeOwned + Serialize, + R: DeserializeOwned, + { + typed_send::>>(&mut self.input, request)?; + receive(&mut self.output) + } +} + +fn run_client(workspace: &Path, trail: &Path, agent: &Path) -> PeerResult { + let mut relay = RelayClient::spawn(workspace, trail, agent)?; + let _: JsonRpcMessage> = relay.exchange::( + json!({"jsonrpc":"2.0","id":"init","method":"initialize","params":{"protocolVersion":1,"clientCapabilities":{"fs":{"readTextFile":true,"writeTextFile":true},"terminal":true}}}), + )?; + let auth: JsonRpcMessage> = relay.exchange::( + json!({"jsonrpc":"2.0","id":"auth","method":"authenticate","params":{"methodId":"official"}}), + )?; + assert!(matches!(auth.inner(), Response::Error { error, .. } if error.data.is_some())); + let session: JsonRpcMessage> = relay.exchange::( + json!({"jsonrpc":"2.0","id":"new","method":"session/new","params":{"cwd":workspace,"mcpServers":[]}}), + )?; + assert!(matches!(session.inner(), Response::Result { .. })); + + typed_send::>>( + &mut relay.input, + json!({"jsonrpc":"2.0","id":"prompt","method":"session/prompt","params":{"sessionId":"official-session","prompt":[{"type":"text","text":"official prompt"},{"type":"resource_link","name":"README","uri":"file:///README.md"}]}}), + )?; + typed_send::>>( + &mut relay.input, + json!({"jsonrpc":"2.0","method":"session/cancel","params":{"sessionId":"official-session"}}), + )?; + typed_send::>>( + &mut relay.input, + json!({"jsonrpc":"2.0","method":"$/cancel_request","params":{"requestId":"prompt"}}), + )?; + + loop { + let message = read_value(&mut relay.output)?; + if message.get("method").and_then(Value::as_str) == Some("session/update") { + let update: JsonRpcMessage> = typed(message)?; + assert!(matches!( + update.inner().params, + Some(SessionNotification { + update: SessionUpdate::AgentMessageChunk(_), + .. + }) + )); + continue; + } + if let Some(method) = message + .get("method") + .and_then(Value::as_str) + .map(str::to_string) + { + respond_to_callback(&mut relay.input, &method, message)?; + continue; + } + let response: JsonRpcMessage> = typed(message)?; + assert!( + matches!(response.inner(), Response::Result { result, .. } if result.stop_reason == StopReason::Cancelled) + ); + break; + } + + for (id, method, params, response_kind) in [ + ( + "load", + "session/load", + json!({"sessionId":"official-session","cwd":workspace,"mcpServers":[]}), + "load", + ), + ( + "resume", + "session/resume", + json!({"sessionId":"official-session","cwd":workspace,"mcpServers":[]}), + "resume", + ), + ( + "close", + "session/close", + json!({"sessionId":"official-session"}), + "close", + ), + ( + "delete", + "session/delete", + json!({"sessionId":"official-session"}), + "delete", + ), + ] { + let value = json!({"jsonrpc":"2.0","id":id,"method":method,"params":params}); + match response_kind { + "load" => { + let _: JsonRpcMessage> = + relay.exchange::(value)?; + } + "resume" => { + let _: JsonRpcMessage> = + relay.exchange::(value)?; + } + "close" => { + let _: JsonRpcMessage> = + relay.exchange::(value)?; + } + "delete" => { + let _: JsonRpcMessage> = + relay.exchange::(value)?; + } + _ => unreachable!(), + } + } + let _: JsonRpcMessage> = relay.exchange::( + json!({"jsonrpc":"2.0","id":"logout","method":"logout","params":{}}), + )?; + drop(relay.input); + let status = relay.child.wait()?; + if !status.success() { + return Err(format!("Trail relay exited with {status}").into()); + } + Ok(()) +} + +fn run_basic_client(workspace: &Path, trail: &Path, agent: &Path) -> PeerResult { + let mut relay = RelayClient::spawn(workspace, trail, agent)?; + let _: JsonRpcMessage> = relay.exchange::( + json!({"jsonrpc":"2.0","id":"init","method":"initialize","params":{"protocolVersion":1,"clientCapabilities":{}}}), + )?; + let _: JsonRpcMessage> = relay.exchange::( + json!({"jsonrpc":"2.0","id":"new","method":"session/new","params":{"cwd":workspace,"mcpServers":[]}}), + )?; + typed_send::>>( + &mut relay.input, + json!({"jsonrpc":"2.0","id":"prompt","method":"session/prompt","params":{"sessionId":"fixture-session","prompt":[{"type":"text","text":"official client to fixture agent"}]}}), + )?; + let update: JsonRpcMessage> = receive(&mut relay.output)?; + assert!(matches!( + update.inner().params, + Some(SessionNotification { + update: SessionUpdate::AgentMessageChunk(_), + .. + }) + )); + let response: JsonRpcMessage> = receive(&mut relay.output)?; + assert!(matches!( + response.inner(), + Response::Result { result, .. } if result.stop_reason == StopReason::EndTurn + )); + let _: JsonRpcMessage> = relay + .exchange::(json!({"jsonrpc":"2.0","id":"close","method":"session/close","params":{"sessionId":"fixture-session"}}))?; + drop(relay.input); + let status = relay.child.wait()?; + if !status.success() { + return Err(format!("Trail relay exited with {status}").into()); + } + Ok(()) +} + +fn respond_to_callback(output: &mut impl Write, method: &str, message: Value) -> PeerResult { + let id = response_id(&message); + match method { + "session/request_permission" => { + let _: JsonRpcMessage> = typed(message)?; + typed_send::>>( + output, + json!({"jsonrpc":"2.0","id":id,"result":{"outcome":{"outcome":"selected","optionId":"allow"}}}), + ) + } + "fs/read_text_file" => { + let _: JsonRpcMessage> = typed(message)?; + typed_send::>>( + output, + json!({"jsonrpc":"2.0","id":id,"result":{"content":"official read"}}), + ) + } + "fs/write_text_file" => { + let _: JsonRpcMessage> = typed(message)?; + typed_send::>>( + output, + json!({"jsonrpc":"2.0","id":id,"result":{}}), + ) + } + "terminal/create" => { + let _: JsonRpcMessage> = typed(message)?; + typed_send::>>( + output, + json!({"jsonrpc":"2.0","id":id,"result":{"terminalId":"official-terminal"}}), + ) + } + "terminal/output" => { + typed_callback_response::( + output, + message, + json!({"output":"official","truncated":false,"exitStatus":{"exitCode":0,"signal":null}}), + ) + } + "terminal/wait_for_exit" => typed_callback_response::< + WaitForTerminalExitRequest, + WaitForTerminalExitResponse, + >(output, message, json!({"exitCode":0,"signal":null})), + "terminal/kill" => typed_callback_response::( + output, + message, + json!({}), + ), + "terminal/release" => typed_callback_response::< + ReleaseTerminalRequest, + ReleaseTerminalResponse, + >(output, message, json!({})), + _ => Err(format!("official client received unexpected callback {method}").into()), + } +} + +fn typed_callback_response( + output: &mut impl Write, + message: Value, + result: Value, +) -> PeerResult +where + P: DeserializeOwned, + R: DeserializeOwned + Serialize, +{ + let id = response_id(&message); + let _: JsonRpcMessage> = typed(message)?; + let result = serde_json::to_value(typed::(result)?)?; + typed_send::>>( + output, + json!({"jsonrpc":"2.0","id":id,"result":result}), + ) +} + +fn main() -> PeerResult { + let args = std::env::args_os().collect::>(); + match args.get(1).and_then(|arg| arg.to_str()) { + Some("agent") => run_agent(), + Some("client") if args.len() == 5 => run_client( + Path::new(&args[2]), + Path::new(&args[3]), + Path::new(&args[4]), + ), + Some("client-basic") if args.len() == 5 => run_basic_client( + Path::new(&args[2]), + Path::new(&args[3]), + Path::new(&args[4]), + ), + _ => Err( + "usage: acp-v1-reference-peer agent | (client|client-basic) WORKSPACE TRAIL AGENT" + .into(), + ), + } +} diff --git a/trail-environment-adapter-sdk/Cargo.toml b/trail-environment-adapter-sdk/Cargo.toml new file mode 100644 index 0000000..da48275 --- /dev/null +++ b/trail-environment-adapter-sdk/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "trail-environment-adapter-sdk" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +homepage.workspace = true +description = "Versioned, capability-constrained protocol types for Trail environment adapter plugins" +readme = "README.md" + +[dependencies] +serde.workspace = true +serde_bytes.workspace = true +serde_cbor.workspace = true +thiserror.workspace = true + +[dev-dependencies] +ed25519-dalek.workspace = true +hex.workspace = true +sha2.workspace = true +toml.workspace = true + +[lints] +workspace = true diff --git a/trail-environment-adapter-sdk/README.md b/trail-environment-adapter-sdk/README.md new file mode 100644 index 0000000..5ed9331 --- /dev/null +++ b/trail-environment-adapter-sdk/README.md @@ -0,0 +1,407 @@ +# Trail environment adapter SDK + +Use this crate when a new ecosystem needs semantic discovery or planning that a +repository-declared `trail.environment.toml` command recipe cannot express. An adapter +is an isolated planner: it receives bounded, pinned file bytes and returns data. Trail +resolves tools, runs commands, validates outputs, publishes layers, attaches lane +mounts, updates state, and recovers failures. + +## Choose the smallest extension + +1. Use a command recipe for a fixed argv command with declarative inputs and outputs. +2. Use an isolated plugin when manifests must be parsed or the command/output contract + must be selected semantically. +3. Add a Trail built-in only for a broadly used ecosystem that needs a richer sharing + strategy and can pass Trail's full cross-platform conformance suite. + +Plugins do not receive a repository path, database handle, mount authority, network, +secrets, a shell, or permission to spawn child processes. Protocol v1 proposes one +staging command. Protocol v2 may instead propose typed `staging` and +`mounted_initialization` actions, or a metadata-only set of pinned external artifacts +and host-managed runtime services; +the Trail host decides whether and how to execute actions and never delegates mount +creation, publication, or provider cleanup ownership. + +## Minimal package + +A package directory contains a manifest and its declared executable. Authenticated +packages add the optional detached signature described below: + +```text +my-adapter/ +├── trail-adapter.toml +├── trail-adapter.sig # optional +└── my-adapter +``` + +```toml +schema = "trail.environment-adapter-package/v1" + +[adapter] +canonical_identity = "acme/schema-codegen@1" +implementation_version = "1.0.0" +selectors = ["acme/schema-codegen@1", "schema-codegen"] +kind = "generated" +layer_adapter_name = "schema-codegen" +discovery_markers = ["schema.codegen.toml"] +protocols = ["trail.environment-adapter/v1"] +supported_operating_systems = ["linux", "macos", "windows"] +supported_architectures = ["aarch64", "x86_64"] +stability = "experimental" +description = "Generates code from a pinned schema manifest" + +[executable] +path = "my-adapter" +sha256 = "sha256:" + +[permissions] +read_patterns = ["schema.codegen.toml", "schema/**/*.json"] +max_input_files = 4096 +max_input_bytes = 8388608 +timeout_ms = 5000 +max_response_bytes = 4194304 +``` + +The executable handles one length-prefixed CBOR request and writes one response. Start +from [`examples/generated-copy-adapter.rs`](examples/generated-copy-adapter.rs). Its +core shape is: + +```rust,no_run +use trail_environment_adapter_sdk::{ + serve_once, AdapterRequest, AdapterResponse, AdapterResult, +}; + +fn main() { + serve_once(|request| { + let result = plan_or_discover(&request); + AdapterResponse::for_request(&request, result) + }) + .expect("serve one Trail adapter request"); +} + +# fn plan_or_discover(_: &AdapterRequest) -> AdapterResult { +# AdapterResult::Error { code: "example".into(), message: "implement me".into() } +# } +``` + +For protocol v2, start from +[`examples/mounted-initializer-adapter.rs`](examples/mounted-initializer-adapter.rs). +The companion [`mounted-fixture-tool.rs`](examples/mounted-fixture-tool.rs) is a direct, +child-free executable used by the native FUSE/NFS/Dokan conformance flow. + +Build plans through the SDK helpers to catch missing commands, empty fields, duplicate +inputs/dependencies/outputs, self-dependencies, and invalid output counts locally. Set-like +inputs and dependencies are sorted deterministically without changing the v1 wire format: + +```rust +use trail_environment_adapter_sdk::{AdapterCommand, AdapterOutput, AdapterPlan}; + +let plan = AdapterPlan::builder("api.client", "generated") + .dependency("api.schema") + .identity_inputs(["schema.json", "generator.toml"]) + .semantic_input("strategy", "client-v1") + .command(AdapterCommand::new("schema-codegen", ["--out", "generated"])) + .output(AdapterOutput::immutable_seed_private( + "client", + "generated", + "src/generated", + )) + .stale_reason("schema, generator, or strategy changed") + .build()?; +# Ok::<(), trail_environment_adapter_sdk::AdapterPlanBuildError>(()) +``` + +For `discover`, return either no component or a stable component ID and declared kind. +For a protocol-v1 `plan`, return: + +- every supplied file path in `identity_inputs`—Trail rejects inspected-but-unkeyed + inputs; +- bounded semantic inputs that affect output identity; +- zero or more stable logical component IDs in `dependencies`; Trail validates the full + graph and keys each edge with the finalized upstream component key; +- an executable name plus argv, working directory, and non-sensitive environment; +- one to 32 non-overlapping owned outputs, their lane mount targets, and an explicit + `immutable_seed_private` (default) or `writable_private` policy; +- conservative portability and an actionable stale reason. + +Protocol v2 uses `AdapterPlanV2` and `AdapterAction` without changing the v1 Rust or +wire types. Select it in `trail-adapter.toml`: + +```toml +[adapter] +protocols = ["trail.environment-adapter/v2"] +``` + +A mounted-only plan is authored as: + +```rust +use trail_environment_adapter_sdk::{AdapterCommand, AdapterOutput, AdapterPlanV2}; + +let plan = AdapterPlanV2::builder("api.venv", "dependency") + .identity_input("pyproject.toml") + .build_requires("python.toolchain") + .runtime_requires("dev.database") + .mounted_command(AdapterCommand::new("venv-tool", ["init", ".venv"])) + .output(AdapterOutput::writable_private("venv", ".venv", ".venv")) + .stale_reason("manifest, tool, platform, or action changed") + .build()?; +# Ok::<(), trail_environment_adapter_sdk::AdapterPlanBuildError>(()) +``` + +V2 permits at most one staging action plus eight mounted actions. Any mounted action +requires every output to be `writable_private`. Trail resolves and rechecks every +executable, stages authenticated executable bytes into an isolated process directory, +mounts a pinned ephemeral candidate at the final lane path, allows reads only from +declared identity inputs, and allows writes only below declared outputs plus isolated +HOME/tmp. Network, shell, secrets, and undeclared child execution remain denied. Only +validated output trees are copied into atomic activation staging; failure leaves the +predecessor generation unchanged. A parent-death watchdog terminates the sandbox helper +if Trail is killed during an active action; backend recovery detaches an abandoned mount +before touching its path and removes the candidate attempt. + +V1 `dependencies` and the v2 `.dependency(...)` compatibility helper mean +`build_requires`. V2 additionally provides `.build_requires(...)`, +`.runtime_requires(...)`, `.binds_after(...)`, and `.invalidates_with(...)`. +Build and invalidation edges contribute the exact upstream key to artifact identity; +runtime and binding-order edges are recorded with the exact upstream generation but do +not force an otherwise identical artifact to rebuild. A component ID may occur in only +one dependency declaration. + +Protocol-v2 staging actions may also request performance-only host caches. Start from +[`examples/cache-adapter.rs`](examples/cache-adapter.rs); its companion +[`cache-fixture-tool.rs`](examples/cache-fixture-tool.rs) is exercised by the native +two-lane conformance flow: + +```rust +use trail_environment_adapter_sdk::{ + AdapterCache, AdapterCacheProtocol, AdapterCommand, AdapterOutput, AdapterPlanV2, +}; + +let plan = AdapterPlanV2::builder("api.codegen", "generated") + .cache( + AdapterCache::host_exclusive("package-store", AdapterCacheProtocol::ContentStore) + .compatibility_dimension("generator", "schema-codegen@1") + .environment_variable("SCHEMA_CODEGEN_CACHE", "."), + ) + .staging_command(AdapterCommand::new("schema-codegen", ["build"])) + .output(AdapterOutput::immutable_seed_private( + "client", + "generated", + "src/generated", + )) + .stale_reason("schema, generator, or strategy changed") + .build()?; +# Ok::<(), trail_environment_adapter_sdk::AdapterPlanBuildError>(()) +``` + +The adapter declares semantics and relative environment bindings, never a host path. +Trail adds the authenticated distribution digest, negotiated protocol, OS, and +architecture to namespace compatibility; injects the absolute namespace only while the +staging command runs; holds a crash-recoverable lease and lock; projects only that path +through the native sandbox; records it in plan/generation provenance; and coordinates +garbage collection. Mounted actions receive no cache access. External adapters are +currently restricted to `host_exclusive`; `tool_concurrent` declarations fail closed +until that exact adapter/tool cache protocol has independent concurrency certification. +Cache eviction must affect performance only, never output correctness. + +Protocol v2 can also describe a provider-owned immutable identity without executing an +action or manufacturing a filesystem output: + +```rust +use trail_environment_adapter_sdk::{AdapterExternalArtifact, AdapterPlanV2}; + +let plan = AdapterPlanV2::builder("images", "external") + .identity_input("images.lock") + .external_artifact(AdapterExternalArtifact::pinned_oci_image( + "web", + "ghcr.io/example/web@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "linux/amd64", + )) + .stale_reason("pinned image declaration changed") + .build()?; +# Ok::<(), trail_environment_adapter_sdk::AdapterPlanBuildError>(()) +``` + +External-artifact plans must use kind `external` and cannot mix actions, caches, or +outputs. Trail validates the digest/reference/platform tuple, includes the sorted +contract in the component key, persists it with each generation, and treats cleanup as +provider-owned. Registry access, tag resolution, credentials, and runtime allocation +are deliberately outside the planner. + +An external plan may bind a pinned image to a lane-private service declaration. The +adapter still performs no provider calls and receives no Docker socket, network, port, +volume, or cleanup authority: + +```rust +use trail_environment_adapter_sdk::{ + AdapterExternalArtifact, AdapterPlanV2, AdapterRuntimeResource, AdapterSecretReference, +}; + +let plan = AdapterPlanV2::builder("dev.database", "external") + .identity_input("services.lock") + .external_artifact(AdapterExternalArtifact::pinned_oci_image( + "postgres-image", + "ghcr.io/example/postgres@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "linux/amd64", + )) + .runtime_resource( + AdapterRuntimeResource::oci_container("postgres", "postgres-image", 5432) + .health_timeout_ms(45_000) + .restart_policy("on_failure") + .volume_target("/var/lib/postgresql/data") + .secret( + AdapterSecretReference::file( + "database-password", + "environment_file", + "DATABASE_PASSWORD_FILE", + "/run/secrets/database-password", + "authenticate the database service", + ) + .version("rotation-7") + .environment_variable("POSTGRES_PASSWORD_FILE"), + ), + ) + .stale_reason("service image or runtime contract changed") + .build()?; +# Ok::<(), trail_environment_adapter_sdk::AdapterPlanBuildError>(()) +``` + +Runtime declarations are sorted and keyed with the component. Each name must be unique, +must reference an artifact in the same plan, and currently describes an OCI container +with a TCP port and TCP health check. Trail allocates deterministic lane/generation +container and network names plus a logical lane-service volume; binds a persisted, +Trail-reserved host port only on +`127.0.0.1`; verifies ownership labels before adopting or stopping resources; and keeps +readiness blocked until health succeeds. `trail env sync` performs reconciliation after +activation. Operators can inspect or retry it explicitly: + +```sh +trail env runtime status +trail env runtime reconcile +trail env runtime stop +``` + +Layered-lane commands and gates receive a deterministic `TRAIL_SERVICES_JSON`; services +whose resource name is unique also receive `TRAIL_SERVICE__HOST`, `_PORT`, and +`_ADDRESS`. Command launch fails closed until the active runtime generation is healthy. + +Planning never resolves secret values. Service credentials and late-bound environment, +file, or descriptor injection require the separate opaque secret-reference contract. +The implemented portable provider contract accepts `file` (an absolute provider-owned +file) and `environment_file` (an environment variable containing that path). Trail does +not read the bytes. It validates a non-symlink regular file, a one-MiB bound, and private +permissions, then gives the runtime a read-only bind handle at `/run/secrets/...`. +An optional environment binding receives only that in-container file path, enabling +`*_FILE` conventions without placing the credential itself in container metadata. +Resolution status and an access audit are stored without values. Raw environment-value +and file-descriptor injection remain denied for standalone Docker because its normal +environment path persists values in inspectable container metadata. + +When a provider resolves the same declaration to a different canonical file handle, +Trail compares a non-secret binding digest on the owned container and recreates it before +use. This prevents a restarted or reconciled service from retaining a stale bind mount +after credential rotation. + +Do not hash files yourself or omit files that appear irrelevant after inspection. Trail +keys the exact supplied set, the package and executable digests, protocol and +implementation versions, semantic inputs, tool identity, command, output contract, +platform, architecture, and capability contract. + +Dependencies are `build_requires` edges in protocol v1; they do not grant the +plugin access to another component's staged or mounted files. Use `trail env sync-all` +to construct the whole graph atomically. A single-component sync fails with an +actionable error unless every declared dependency is already ready in that lane. + +Use `writable_private` for output owned by one lane. Protocol v1 must initialize it in +Trail's temporary staging directory. Protocol v2 can request mounted initialization for +path-sensitive state such as virtual environments or configure databases, subject to +the stricter mounted-action contract above. Commands requiring wrapper-spawned runtime +executables must use a future explicitly certified child-runtime capability; Trail does +not silently weaken child-process denial. + +## Publisher authentication + +Unsigned local packages remain supported and are reported as +`local-experimental`. For authenticated distribution, first ask Trail for the canonical +payload digest: + +```sh +trail --workspace /path/to/repo --json env plugin inspect ./my-adapter +``` + +Sign these exact bytes with Ed25519: + +```text +"trail.environment-adapter-signature/v1" || NUL || "sha256:" +``` + +Store the detached signature beside the manifest as `trail-adapter.sig`: + +```toml +schema = "trail.environment-adapter-signature/v1" +publisher = "acme" +key_id = "sha256:" +payload_digest = "sha256:" +signature = "<128 lowercase hex characters>" +``` + +The consuming workspace explicitly trusts the public key: + +```toml +# acme-adapter-key.toml +schema = "trail.environment-adapter-publisher-key/v1" +publisher = "acme" +public_key = "<64 lowercase hex characters>" +``` + +```sh +trail --workspace /path/to/repo env plugin trust add ./acme-adapter-key.toml +trail --workspace /path/to/repo env plugin trust list +``` + +Trail verifies the payload digest, content-derived key ID, trusted publisher ownership, +and Ed25519 signature before installation. The immutable distribution identity includes +the detached attestation, while the signed payload identity remains independently +inspectable. Revoking a key immediately makes active packages authenticated by that key +fail closed: + +```sh +trail --workspace /path/to/repo env plugin trust remove sha256: +``` + +Publisher authentication proves origin and byte integrity; it does not by itself grant a +stable certification tier. Signed external adapters are reported as +`publisher-authenticated-experimental` until a separate conformance certification +system exists. + +## Install and exercise + +```sh +trail --workspace /path/to/repo env plugin install ./my-adapter +trail --workspace /path/to/repo env adapters +trail --workspace /path/to/repo env discover my-lane +trail --workspace /path/to/repo env graph my-lane +trail --workspace /path/to/repo env plan my-lane --adapter acme/schema-codegen@1 +trail --workspace /path/to/repo env sync my-lane --adapter acme/schema-codegen@1 +trail --workspace /path/to/repo lane readiness my-lane +``` + +Installation verifies the manifest and executable digest, stores immutable package +bytes by distribution digest, and activates the identity locally. Unsigned local +packages must remain `experimental`. Removal is an append-only tombstone: + +```sh +trail --workspace /path/to/repo env plugin remove acme/schema-codegen@1 +``` + +Run the repository's native conformance verifier while developing Trail or the example +adapter: + +```sh +scripts/verify-environment-adapter-plugin.sh +``` + +Windows uses `scripts/verify-windows-environment-adapter-plugin.ps1`. Release evidence +must cover every declared OS/architecture, deterministic planning, input-change +staleness, two-lane layer reuse, private copy-up, timeout/crash/memory/output limits, +malformed framing, child/network/filesystem denial, tamper detection, and recovery. diff --git a/trail-environment-adapter-sdk/examples/adversarial-adapter.rs b/trail-environment-adapter-sdk/examples/adversarial-adapter.rs new file mode 100644 index 0000000..7a5609d --- /dev/null +++ b/trail-environment-adapter-sdk/examples/adversarial-adapter.rs @@ -0,0 +1,72 @@ +use std::io::{self, Write}; +use std::process::Command; +use std::time::Duration; + +use trail_environment_adapter_sdk::{ + read_frame, AdapterRequest, AdapterResponse, AdapterResult, MAX_FRAME_BYTES, PROTOCOL_V1, +}; + +fn main() { + let request: AdapterRequest = match read_frame(&mut io::stdin().lock(), MAX_FRAME_BYTES) { + Ok(request) => request, + Err(error) => { + eprintln!("adversarial-adapter could not read request: {error}"); + std::process::exit(2); + } + }; + match request.adapter_identity.as_str() { + "example/hang@1" => { + std::thread::sleep(Duration::from_secs(10)); + let _ = trail_environment_adapter_sdk::write_frame( + &mut io::stdout().lock(), + &AdapterResponse { + protocol: PROTOCOL_V1.to_string(), + request_id: request.request_id, + result: AdapterResult::Discovered { component: None }, + }, + MAX_FRAME_BYTES, + ); + } + "example/crash@1" => std::process::exit(7), + "example/oversized@1" => { + let block = vec![b'x'; 2 * 1024 * 1024]; + let _ = io::stdout().lock().write_all(&block); + } + "example/malformed@1" => { + let _ = io::stdout().lock().write_all(b"not a framed response"); + } + "example/child@1" => { + if Command::new(std::env::current_exe().unwrap()) + .spawn() + .is_ok() + { + let _ = trail_environment_adapter_sdk::write_frame( + &mut io::stdout().lock(), + &AdapterResponse { + protocol: PROTOCOL_V1.to_string(), + request_id: request.request_id, + result: AdapterResult::Discovered { + component: Some(trail_environment_adapter_sdk::DiscoveredComponent { + component_id: "plugin.child-escaped".to_string(), + kind: "generated".to_string(), + }), + }, + }, + MAX_FRAME_BYTES, + ); + } else { + std::process::exit(9); + } + } + "example/memory@1" => { + let mut allocation = Vec::>::new(); + for _ in 0..80 { + allocation.push(vec![0xa5; 8 * 1024 * 1024]); + std::thread::sleep(Duration::from_millis(5)); + } + let _ = allocation.len(); + std::process::exit(10); + } + _ => std::process::exit(3), + } +} diff --git a/trail-environment-adapter-sdk/examples/cache-adapter.rs b/trail-environment-adapter-sdk/examples/cache-adapter.rs new file mode 100644 index 0000000..ccb754e --- /dev/null +++ b/trail-environment-adapter-sdk/examples/cache-adapter.rs @@ -0,0 +1,67 @@ +use trail_environment_adapter_sdk::{ + serve_once, AdapterCache, AdapterCacheProtocol, AdapterCommand, AdapterOperation, + AdapterOutput, AdapterPlanV2, AdapterResponse, AdapterResult, DiscoveredComponent, PROTOCOL_V2, +}; + +fn main() { + if let Err(error) = serve_once(|request| { + if request.protocol != PROTOCOL_V2 || request.adapter_identity != "example/cache@1" { + return AdapterResponse::for_request( + &request, + AdapterResult::Error { + code: "unsupported_request".to_string(), + message: "protocol or adapter identity does not match".to_string(), + }, + ); + } + let result = match &request.operation { + AdapterOperation::Discover { files, .. } => AdapterResult::Discovered { + component: files + .iter() + .any(|file| file.path == "cache.adapter") + .then(|| DiscoveredComponent::new("plugin.cache", "generated")), + }, + AdapterOperation::Plan { + component_id, + files, + .. + } => { + let behavior = files + .iter() + .find(|file| file.path == "cache.adapter") + .map(|file| String::from_utf8_lossy(&file.content).trim().to_string()) + .unwrap_or_else(|| "populate".to_string()); + match AdapterPlanV2::builder(component_id.clone(), "generated") + .identity_input("cache.adapter") + .semantic_input("fixture", "host-exclusive-cache-v1") + .cache( + AdapterCache::host_exclusive( + "fixture-store", + AdapterCacheProtocol::ContentStore, + ) + .compatibility_dimension("fixture_tool", "cache-fixture-tool@1") + .environment_variable("TRAIL_FIXTURE_CACHE", "."), + ) + .staging_command(AdapterCommand::new("cache-fixture-tool", [behavior])) + .output(AdapterOutput::immutable_seed_private( + "generated", + "generated", + ".trail-generated/plugin-cache", + )) + .stale_reason("cache fixture input, executable, platform, or adapter changed") + .build() + { + Ok(plan) => AdapterResult::PlannedV2 { plan }, + Err(error) => AdapterResult::Error { + code: "invalid_plan".to_string(), + message: error.to_string(), + }, + } + } + }; + AdapterResponse::for_request(&request, result) + }) { + eprintln!("cache-adapter: {error}"); + std::process::exit(1); + } +} diff --git a/trail-environment-adapter-sdk/examples/cache-fixture-tool.rs b/trail-environment-adapter-sdk/examples/cache-fixture-tool.rs new file mode 100644 index 0000000..7df318d --- /dev/null +++ b/trail-environment-adapter-sdk/examples/cache-fixture-tool.rs @@ -0,0 +1,53 @@ +use std::env; +use std::fs; +use std::io; +use std::path::PathBuf; + +fn main() { + if let Err(error) = run() { + eprintln!("cache-fixture-tool: {error}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), Box> { + let behavior = env::args().nth(1).unwrap_or_else(|| "populate".to_string()); + let cache = PathBuf::from(env::var_os("TRAIL_FIXTURE_CACHE").ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "TRAIL_FIXTURE_CACHE was not injected by the Trail host", + ) + })?); + fs::create_dir_all(&cache)?; + if behavior == "escape" { + let parent = cache.parent().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "cache namespace has no parent") + })?; + fs::write(parent.join("plugin-cache-escape"), b"escaped\n")?; + return Err(io::Error::other("cache namespace escape unexpectedly succeeded").into()); + } + + let counter_path = cache.join("counter"); + let previous = fs::read_to_string(&counter_path) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + let current = previous + .checked_add(1) + .ok_or_else(|| io::Error::other("cache counter overflow"))?; + fs::write(&counter_path, format!("{current}\n"))?; + + let output = PathBuf::from("generated"); + fs::create_dir_all(&output)?; + let namespace = cache + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "cache namespace is not UTF-8") + })?; + fs::write( + output.join("cache-observation.txt"), + format!("{namespace}|{current}\n"), + )?; + Ok(()) +} diff --git a/trail-environment-adapter-sdk/examples/fixture-sign-adapter.rs b/trail-environment-adapter-sdk/examples/fixture-sign-adapter.rs new file mode 100644 index 0000000..80c4290 --- /dev/null +++ b/trail-environment-adapter-sdk/examples/fixture-sign-adapter.rs @@ -0,0 +1,69 @@ +//! Deterministic conformance-fixture signer. Do not use embedded seed material +//! from tests for real publisher keys. + +use std::env; +use std::fs; +use std::path::PathBuf; + +use ed25519_dalek::{Signer, SigningKey}; +use trail_environment_adapter_sdk::{ + AdapterPackageSignature, AdapterPublisherKey, PACKAGE_SIGNATURE_SCHEMA_V1, + TRUSTED_PUBLISHER_KEY_SCHEMA_V1, +}; + +fn main() { + if let Err(error) = run() { + eprintln!("fixture-sign-adapter: {error}"); + std::process::exit(2); + } +} + +fn run() -> Result<(), String> { + let mut args = env::args().skip(1); + let publisher = args.next().ok_or("missing publisher")?; + let seed = args.next().ok_or("missing 32-byte seed hex")?; + let payload_digest = args.next().ok_or("missing payload digest")?; + let signature_path = PathBuf::from(args.next().ok_or("missing signature path")?); + let key_path = PathBuf::from(args.next().ok_or("missing key path")?); + if args.next().is_some() { + return Err("unexpected extra arguments".to_string()); + } + let seed = hex::decode(seed).map_err(|error| format!("invalid seed hex: {error}"))?; + let seed: [u8; 32] = seed + .try_into() + .map_err(|_| "seed must decode to exactly 32 bytes".to_string())?; + let signing_key = SigningKey::from_bytes(&seed); + let public_key = signing_key.verifying_key().to_bytes(); + let key_id = format!("sha256:{}", sha256_hex(&public_key)); + let message = format!("{PACKAGE_SIGNATURE_SCHEMA_V1}\0{payload_digest}"); + let signature = signing_key.sign(message.as_bytes()); + let signature_document = AdapterPackageSignature { + schema: PACKAGE_SIGNATURE_SCHEMA_V1.to_string(), + publisher: publisher.clone(), + key_id, + payload_digest, + signature: hex::encode(signature.to_bytes()), + }; + let key_document = AdapterPublisherKey { + schema: TRUSTED_PUBLISHER_KEY_SCHEMA_V1.to_string(), + publisher, + public_key: hex::encode(public_key), + }; + fs::write( + signature_path, + toml::to_string(&signature_document).map_err(|error| error.to_string())?, + ) + .map_err(|error| error.to_string())?; + fs::write( + key_path, + toml::to_string(&key_document).map_err(|error| error.to_string())?, + ) + .map_err(|error| error.to_string())?; + Ok(()) +} + +fn sha256_hex(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + + hex::encode(Sha256::digest(bytes)) +} diff --git a/trail-environment-adapter-sdk/examples/generated-copy-adapter.rs b/trail-environment-adapter-sdk/examples/generated-copy-adapter.rs new file mode 100644 index 0000000..a29bfb7 --- /dev/null +++ b/trail-environment-adapter-sdk/examples/generated-copy-adapter.rs @@ -0,0 +1,88 @@ +use trail_environment_adapter_sdk::{ + serve_once, AdapterCommand, AdapterOperation, AdapterOutput, AdapterPlan, AdapterResponse, + AdapterResult, DiscoveredComponent, PROTOCOL_V1, +}; + +fn main() { + if let Err(error) = serve_once(|request| { + if request.protocol != PROTOCOL_V1 || request.adapter_identity != "example/copy@1" { + return AdapterResponse::for_request( + &request, + AdapterResult::Error { + code: "unsupported_request".to_string(), + message: "protocol or adapter identity does not match".to_string(), + }, + ); + } + let result = match &request.operation { + AdapterOperation::Discover { files, .. } => AdapterResult::Discovered { + component: files + .iter() + .any(|file| file.path == "copy.adapter") + .then(|| DiscoveredComponent::new("plugin.copy", "generated")), + }, + AdapterOperation::Plan { + component_id, + files, + .. + } => { + let writable_private = files.iter().any(|file| { + file.path == "copy.adapter" && file.content.starts_with(b"writable_private") + }); + let (program, args) = if request.host.operating_system == "windows" { + ( + "tar.exe".to_string(), + vec![ + "-cf".to_string(), + "generated/out.tar".to_string(), + "input.txt".to_string(), + ], + ) + } else { + ( + "cp".to_string(), + vec!["input.txt".to_string(), "generated/copied.txt".to_string()], + ) + }; + let output = if writable_private { + AdapterOutput::writable_private( + "generated", + "generated", + ".trail-generated/plugin-copy", + ) + } else { + AdapterOutput::immutable_seed_private( + "generated", + "generated", + ".trail-generated/plugin-copy", + ) + }; + match AdapterPlan::builder(component_id.clone(), "generated") + .identity_inputs(files.iter().map(|file| file.path.clone())) + .semantic_input( + "strategy", + if writable_private { + "copy-writable-private-v1" + } else { + "copy-immutable-seed-v1" + }, + ) + .command(AdapterCommand::new(program, args)) + .output(output) + .stale_reason("pinned plugin inputs, executable, platform, or adapter changed") + .build() + { + Ok(plan) => AdapterResult::Planned { plan }, + Err(error) => AdapterResult::Error { + code: "invalid_plan".to_string(), + message: error.to_string(), + }, + } + } + }; + AdapterResponse::for_request(&request, result) + }) { + eprintln!("generated-copy-adapter: {error}"); + std::process::exit(1); + } +} diff --git a/trail-environment-adapter-sdk/examples/mounted-fixture-tool.rs b/trail-environment-adapter-sdk/examples/mounted-fixture-tool.rs new file mode 100644 index 0000000..317a0d3 --- /dev/null +++ b/trail-environment-adapter-sdk/examples/mounted-fixture-tool.rs @@ -0,0 +1,76 @@ +use std::path::Path; + +fn main() { + let mut arguments = std::env::args().skip(1); + let behavior = arguments.next().unwrap_or_default(); + let path = arguments.next().unwrap_or_default(); + let input = arguments.next(); + if path.is_empty() { + eprintln!("mounted-fixture-tool requires an output path"); + std::process::exit(2); + } + let path = Path::new(&path); + let current_directory = std::env::current_dir().unwrap_or_default(); + if let Some(parent) = path.parent() { + if let Err(error) = std::fs::create_dir_all(parent) { + eprintln!("cannot create fixture output parent: {error}"); + std::process::exit(3); + } + } + let result = match behavior.as_str() { + "success" => std::env::current_dir() + .and_then(|directory| { + let input = input.as_deref().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "success requires an input path", + ) + })?; + let input = std::fs::read_to_string(input)?; + Ok(format!( + "{}|{}\n", + directory.to_string_lossy(), + input.trim() + )) + }) + .and_then(|contents| std::fs::write(path, contents).map(|_| String::new())), + "fail" => std::fs::write(path, b"partial").map(|_| String::new()), + "hang" => { + if let Err(error) = std::fs::write(path, b"partial") { + Err(error) + } else { + let home = std::env::var_os("HOME").unwrap_or_default(); + let ready = Path::new(&home).join("running"); + if let Err(error) = std::fs::write(&ready, std::process::id().to_string()) { + Err(error) + } else { + loop { + std::thread::sleep(std::time::Duration::from_secs(1)); + } + } + } + } + "source_write" => std::fs::write(path, b"leak").map(|_| String::new()), + "source_read" => { + let input = input.as_deref().unwrap_or("input.txt"); + std::fs::read_to_string(input) + .and_then(|contents| std::fs::write(path, contents)) + .map(|_| String::new()) + } + _ => { + eprintln!("unknown mounted fixture behavior `{behavior}`"); + std::process::exit(4); + } + }; + if let Err(error) = result { + eprintln!( + "mounted fixture action failed in `{}` for `{}`: {error}", + current_directory.display(), + path.display() + ); + std::process::exit(5); + } + if behavior == "fail" { + std::process::exit(23); + } +} diff --git a/trail-environment-adapter-sdk/examples/mounted-initializer-adapter.rs b/trail-environment-adapter-sdk/examples/mounted-initializer-adapter.rs new file mode 100644 index 0000000..08a50ed --- /dev/null +++ b/trail-environment-adapter-sdk/examples/mounted-initializer-adapter.rs @@ -0,0 +1,81 @@ +use trail_environment_adapter_sdk::{ + serve_once, AdapterCommand, AdapterOperation, AdapterOutput, AdapterPlanV2, AdapterResponse, + AdapterResult, DiscoveredComponent, PROTOCOL_V2, +}; + +fn main() { + if let Err(error) = serve_once(|request| { + if request.protocol != PROTOCOL_V2 || request.adapter_identity != "example/mounted@1" { + return AdapterResponse::for_request( + &request, + AdapterResult::Error { + code: "unsupported_request".to_string(), + message: "protocol or adapter identity does not match".to_string(), + }, + ); + } + let result = match &request.operation { + AdapterOperation::Discover { files, .. } => AdapterResult::Discovered { + component: files + .iter() + .any(|file| file.path == "mounted.adapter") + .then(|| DiscoveredComponent::new("plugin.mounted", "generated")), + }, + AdapterOperation::Plan { + component_id, + files, + .. + } => { + let behavior = files + .iter() + .find(|file| file.path == "mounted.adapter") + .map(|file| String::from_utf8_lossy(&file.content).trim().to_string()) + .unwrap_or_else(|| "success".to_string()); + let (action, path, input) = match behavior.as_str() { + "fail" => ("fail", ".trail-generated/plugin-mounted/partial.txt", None), + "hang" => ("hang", ".trail-generated/plugin-mounted/partial.txt", None), + "source_write" => ("source_write", "source-leak.txt", None), + "source_read" => ( + "source_read", + ".trail-generated/plugin-mounted/leaked.txt", + Some("input.txt"), + ), + _ => ( + "success", + ".trail-generated/plugin-mounted/initialized.txt", + Some("mounted.adapter"), + ), + }; + let mut arguments = vec![action, path]; + arguments.extend(input); + match AdapterPlanV2::builder(component_id.clone(), "generated") + .identity_inputs(files.iter().map(|file| file.path.clone())) + .semantic_input("behavior", behavior) + .mounted_command(AdapterCommand::new( + "mounted-fixture-tool", + arguments, + )) + .output(AdapterOutput::writable_private( + "initialized", + ".trail-generated/plugin-mounted", + ".trail-generated/plugin-mounted", + )) + .stale_reason( + "pinned plugin inputs, mounted action, executable, platform, or adapter changed", + ) + .build() + { + Ok(plan) => AdapterResult::PlannedV2 { plan }, + Err(error) => AdapterResult::Error { + code: "invalid_plan".to_string(), + message: error.to_string(), + }, + } + } + }; + AdapterResponse::for_request(&request, result) + }) { + eprintln!("mounted-initializer-adapter: {error}"); + std::process::exit(1); + } +} diff --git a/trail-environment-adapter-sdk/src/lib.rs b/trail-environment-adapter-sdk/src/lib.rs new file mode 100644 index 0000000..7605ece --- /dev/null +++ b/trail-environment-adapter-sdk/src/lib.rs @@ -0,0 +1,2015 @@ +//! Protocol and authoring helpers for isolated Trail environment adapters. +//! +//! An adapter is a planner, not an executor. Trail sends bounded bytes from a +//! pinned source root; the adapter returns discovery or a normalized action +//! proposal. The Trail host owns tool resolution, sandboxing, execution, +//! validation, publication, bindings, state, and recovery. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{self, Read, Write}; + +use serde::{Deserialize, Serialize}; + +pub const PROTOCOL_V1: &str = "trail.environment-adapter/v1"; +/// Adds host-sandboxed actions that execute against Trail's ephemeral mounted +/// candidate view. V1 remains the default for packages that do not declare a +/// protocol list, so existing adapters keep their exact behavior. +pub const PROTOCOL_V2: &str = "trail.environment-adapter/v2"; +pub const PACKAGE_SCHEMA_V1: &str = "trail.environment-adapter-package/v1"; +pub const PACKAGE_SIGNATURE_SCHEMA_V1: &str = "trail.environment-adapter-signature/v1"; +pub const TRUSTED_PUBLISHER_KEY_SCHEMA_V1: &str = "trail.environment-adapter-publisher-key/v1"; +pub const MAX_FRAME_BYTES: usize = 16 * 1024 * 1024; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterPackageManifest { + pub schema: String, + pub adapter: AdapterMetadata, + pub executable: AdapterExecutable, + #[serde(default)] + pub permissions: AdapterPermissions, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterMetadata { + pub canonical_identity: String, + pub implementation_version: String, + pub selectors: Vec, + pub kind: String, + pub layer_adapter_name: String, + pub discovery_markers: Vec, + /// Protocols understood by the packaged executable. Missing metadata is + /// deliberately interpreted as v1 for package compatibility. + #[serde( + default = "default_adapter_protocols", + skip_serializing_if = "is_default_v1_protocols" + )] + pub protocols: Vec, + #[serde(default = "default_supported_operating_systems")] + pub supported_operating_systems: Vec, + #[serde(default = "default_supported_architectures")] + pub supported_architectures: Vec, + pub stability: String, + pub description: String, +} + +fn default_adapter_protocols() -> Vec { + vec![PROTOCOL_V1.to_string()] +} + +fn is_default_v1_protocols(protocols: &[String]) -> bool { + protocols == [PROTOCOL_V1] +} + +fn default_supported_operating_systems() -> Vec { + ["linux", "macos", "windows"] + .into_iter() + .map(str::to_string) + .collect() +} + +fn default_supported_architectures() -> Vec { + ["aarch64", "x86_64"] + .into_iter() + .map(str::to_string) + .collect() +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterExecutable { + pub path: String, + pub sha256: String, +} + +/// Detached publisher signature stored as `trail-adapter.sig` beside a +/// package. The signature authenticates `payload_digest`, the canonical +/// manifest-plus-executable digest calculated by Trail. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterPackageSignature { + pub schema: String, + pub publisher: String, + pub key_id: String, + pub payload_digest: String, + pub signature: String, +} + +/// Public key document accepted by `trail env plugin trust add`. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterPublisherKey { + pub schema: String, + pub publisher: String, + pub public_key: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +pub struct AdapterPermissions { + pub read_patterns: Vec, + pub max_input_files: u32, + pub max_input_bytes: u64, + pub timeout_ms: u64, + pub max_response_bytes: u64, +} + +impl Default for AdapterPermissions { + fn default() -> Self { + Self { + read_patterns: Vec::new(), + max_input_files: 4_096, + max_input_bytes: 8 * 1024 * 1024, + timeout_ms: 5_000, + max_response_bytes: 4 * 1024 * 1024, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterRequest { + pub protocol: String, + pub request_id: String, + pub adapter_identity: String, + pub distribution_digest: String, + pub host: AdapterHost, + pub source_root: String, + pub operation: AdapterOperation, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterHost { + pub operating_system: String, + pub architecture: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "method", rename_all = "snake_case")] +pub enum AdapterOperation { + Discover { + component_root: String, + files: Vec, + }, + Plan { + component_id: String, + component_root: String, + files: Vec, + }, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct PinnedFile { + pub path: String, + pub content_hash: String, + pub executable: bool, + #[serde(with = "serde_bytes")] + pub content: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterResponse { + pub protocol: String, + pub request_id: String, + pub result: AdapterResult, +} + +impl AdapterResponse { + /// Build a protocol-matched response for one host request. + pub fn for_request(request: &AdapterRequest, result: AdapterResult) -> Self { + Self { + protocol: request.protocol.clone(), + request_id: request.request_id.clone(), + result, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "result", rename_all = "snake_case")] +// Keep the v1 constructor shape source-compatible for adapter authors. Boxing +// `Planned::plan` would not change CBOR but would require every existing +// plugin to wrap its plan solely to optimize this short-lived message enum. +#[allow(clippy::large_enum_variant)] +pub enum AdapterResult { + Discovered { + component: Option, + }, + Planned { + plan: AdapterPlan, + }, + PlannedV2 { + plan: AdapterPlanV2, + }, + Error { + code: String, + message: String, + }, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct DiscoveredComponent { + pub component_id: String, + pub kind: String, +} + +impl DiscoveredComponent { + pub fn new(component_id: impl Into, kind: impl Into) -> Self { + Self { + component_id: component_id.into(), + kind: kind.into(), + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterPlan { + pub component_id: String, + pub kind: String, + /// Stable logical component IDs that must precede this component. This is + /// optional on the wire for backward compatibility with early v1 plans. + #[serde(default)] + pub dependencies: Vec, + pub identity_inputs: Vec, + #[serde(default)] + pub semantic_inputs: BTreeMap, + pub command: AdapterCommand, + pub outputs: Vec, + pub portability: AdapterPortability, + pub stale_reason: String, +} + +/// Protocol-v2 plan with explicitly phased host actions. Keeping this type +/// separate preserves the public construction and exact wire shape of +/// `AdapterPlan` for every existing protocol-v1 adapter. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterPlanV2 { + pub component_id: String, + pub kind: String, + #[serde(default)] + pub dependencies: Vec, + /// Typed dependency edges. `dependencies` remains the protocol-v1-shaped + /// compatibility field and is interpreted as `build_requires`; a + /// component ID may appear in only one of the two collections. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dependency_edges: Vec, + pub identity_inputs: Vec, + #[serde(default)] + pub semantic_inputs: BTreeMap, + /// Performance-only caches used by the single staging action. Trail owns + /// namespace identity, storage, locking, sandbox projection, leases, and + /// garbage collection; adapters receive only the declared environment + /// bindings. Mounted actions never receive cache access. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub caches: Vec, + /// Provider-owned immutable identities. A plan that declares external + /// artifacts is metadata-only: it must not also declare actions, caches, + /// or filesystem outputs. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub external_artifacts: Vec, + /// Per-lane runtime resources derived from declared immutable artifacts. + /// Provider allocation IDs and host ports are assigned by Trail after the + /// environment generation commits. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub runtime_resources: Vec, + pub actions: Vec, + pub outputs: Vec, + pub portability: AdapterPortability, + pub stale_reason: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterExternalArtifact { + pub name: String, + pub artifact_type: String, + pub provider: String, + pub reference: String, + pub digest: String, + pub platform: String, + pub cleanup_owner: String, +} + +impl AdapterExternalArtifact { + pub fn pinned_oci_image( + name: impl Into, + reference: impl Into, + platform: impl Into, + ) -> Self { + let reference = reference.into(); + let digest = reference + .rsplit_once('@') + .map(|(_, digest)| digest.to_string()) + .unwrap_or_default(); + Self { + name: name.into(), + artifact_type: "oci_image".to_string(), + provider: "oci".to_string(), + reference, + digest, + platform: platform.into(), + cleanup_owner: "external".to_string(), + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterRuntimeResource { + pub name: String, + pub runtime_type: String, + pub provider: String, + pub artifact_name: String, + pub container_port: u16, + pub protocol: String, + pub health_type: String, + pub health_timeout_ms: u64, + pub restart_policy: String, + pub cleanup_owner: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub volume_target: Option, + /// Opaque provider references resolved by Trail only while starting the + /// resource. Values never cross the adapter protocol. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub secrets: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterSecretReference { + pub name: String, + pub provider: String, + pub reference: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, + pub purpose: String, + pub injection: String, + pub target: String, + /// Optional environment variable receiving the non-secret target file + /// path (for example `POSTGRES_PASSWORD_FILE`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub environment: Option, + pub required: bool, +} + +impl AdapterSecretReference { + pub fn file( + name: impl Into, + provider: impl Into, + reference: impl Into, + target: impl Into, + purpose: impl Into, + ) -> Self { + Self { + name: name.into(), + provider: provider.into(), + reference: reference.into(), + version: None, + purpose: purpose.into(), + injection: "file".to_string(), + target: target.into(), + environment: None, + required: true, + } + } + + pub fn version(mut self, version: impl Into) -> Self { + self.version = Some(version.into()); + self + } + + pub fn optional(mut self) -> Self { + self.required = false; + self + } + + pub fn environment_variable(mut self, name: impl Into) -> Self { + self.environment = Some(name.into()); + self + } +} + +impl AdapterRuntimeResource { + pub fn oci_container( + name: impl Into, + artifact_name: impl Into, + container_port: u16, + ) -> Self { + Self { + name: name.into(), + runtime_type: "container".to_string(), + provider: "oci".to_string(), + artifact_name: artifact_name.into(), + container_port, + protocol: "tcp".to_string(), + health_type: "tcp".to_string(), + health_timeout_ms: 30_000, + restart_policy: "on_failure".to_string(), + cleanup_owner: "trail".to_string(), + volume_target: None, + secrets: Vec::new(), + } + } + + pub fn health_timeout_ms(mut self, timeout_ms: u64) -> Self { + self.health_timeout_ms = timeout_ms; + self + } + + pub fn restart_policy(mut self, policy: impl Into) -> Self { + self.restart_policy = policy.into(); + self + } + + pub fn volume_target(mut self, target: impl Into) -> Self { + self.volume_target = Some(target.into()); + self + } + + pub fn secret(mut self, secret: AdapterSecretReference) -> Self { + self.secrets.push(secret); + self + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(deny_unknown_fields)] +pub struct AdapterDependency { + pub component_id: String, + pub edge_type: AdapterDependencyType, +} + +impl AdapterDependency { + pub fn new(component_id: impl Into, edge_type: AdapterDependencyType) -> Self { + Self { + component_id: component_id.into(), + edge_type, + } + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "snake_case")] +pub enum AdapterDependencyType { + BuildRequires, + RuntimeRequires, + BindsAfter, + InvalidatesWith, +} + +/// A host-owned performance cache requested by a protocol-v2 adapter. +/// +/// Cache contents must never be required for correctness: Trail may evict a +/// namespace at any time when it has no live users. External adapters are +/// initially authorized only for `HostExclusive` access; `ToolConcurrent` is +/// represented on the wire so independently certified adapters do not need a +/// future protocol shape change. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterCache { + pub name: String, + pub protocol: AdapterCacheProtocol, + pub access: AdapterCacheAccess, + /// Adapter-specific, non-secret compatibility dimensions. Trail adds the + /// authenticated distribution, protocol, platform, and architecture. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub compatibility: BTreeMap, + /// Environment variable to relative namespace subpath. `.` binds the + /// namespace root. Trail resolves and injects absolute paths only after it + /// has validated and acquired the namespace. + pub environment: BTreeMap, +} + +impl AdapterCache { + pub fn host_exclusive(name: impl Into, protocol: AdapterCacheProtocol) -> Self { + Self { + name: name.into(), + protocol, + access: AdapterCacheAccess::HostExclusive, + compatibility: BTreeMap::new(), + environment: BTreeMap::new(), + } + } + + pub fn tool_concurrent(name: impl Into, protocol: AdapterCacheProtocol) -> Self { + Self { + name: name.into(), + protocol, + access: AdapterCacheAccess::ToolConcurrent, + compatibility: BTreeMap::new(), + environment: BTreeMap::new(), + } + } + + pub fn compatibility_dimension( + mut self, + name: impl Into, + value: impl Into, + ) -> Self { + self.compatibility.insert(name.into(), value.into()); + self + } + + pub fn environment_variable( + mut self, + name: impl Into, + relative_subpath: impl Into, + ) -> Self { + self.environment + .insert(name.into(), relative_subpath.into()); + self + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum AdapterCacheProtocol { + ContentStore, + CompilerCache, + LockedIndex, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum AdapterCacheAccess { + HostExclusive, + ToolConcurrent, +} + +impl AdapterPlanV2 { + pub fn builder( + component_id: impl Into, + kind: impl Into, + ) -> AdapterPlanV2Builder { + AdapterPlanV2Builder::new(component_id, kind) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "phase", content = "command", rename_all = "snake_case")] +pub enum AdapterAction { + Staging(AdapterCommand), + MountedInitialization(AdapterCommand), +} + +impl AdapterPlan { + /// Start a deterministic plan builder. This is an authoring convenience; + /// the resulting value has exactly the same v1 wire representation as a + /// directly constructed `AdapterPlan`. + pub fn builder(component_id: impl Into, kind: impl Into) -> AdapterPlanBuilder { + AdapterPlanBuilder::new(component_id, kind) + } +} + +/// Validating builder for the normalized v1 plan returned by an adapter. +/// +/// Trail still performs authoritative host-side validation. The builder gives +/// adapter authors fast local failures for the structural mistakes that most +/// often otherwise appear only during `trail env plan`. +#[derive(Clone, Debug)] +pub struct AdapterPlanBuilder { + component_id: String, + kind: String, + dependencies: Vec, + identity_inputs: Vec, + semantic_inputs: BTreeMap, + command: Option, + outputs: Vec, + portability: AdapterPortability, + stale_reason: Option, +} + +impl AdapterPlanBuilder { + pub fn new(component_id: impl Into, kind: impl Into) -> Self { + Self { + component_id: component_id.into(), + kind: kind.into(), + dependencies: Vec::new(), + identity_inputs: Vec::new(), + semantic_inputs: BTreeMap::new(), + command: None, + outputs: Vec::new(), + portability: AdapterPortability::Host, + stale_reason: None, + } + } + + pub fn dependency(mut self, component_id: impl Into) -> Self { + self.dependencies.push(component_id.into()); + self + } + + pub fn dependencies(mut self, component_ids: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.dependencies + .extend(component_ids.into_iter().map(Into::into)); + self + } + + pub fn identity_input(mut self, path: impl Into) -> Self { + self.identity_inputs.push(path.into()); + self + } + + pub fn identity_inputs(mut self, paths: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.identity_inputs + .extend(paths.into_iter().map(Into::into)); + self + } + + pub fn semantic_input(mut self, name: impl Into, value: impl Into) -> Self { + self.semantic_inputs.insert(name.into(), value.into()); + self + } + + pub fn command(mut self, command: AdapterCommand) -> Self { + self.command = Some(command); + self + } + + pub fn output(mut self, output: AdapterOutput) -> Self { + self.outputs.push(output); + self + } + + pub fn outputs(mut self, outputs: I) -> Self + where + I: IntoIterator, + { + self.outputs.extend(outputs); + self + } + + pub fn portability(mut self, portability: AdapterPortability) -> Self { + self.portability = portability; + self + } + + pub fn stale_reason(mut self, reason: impl Into) -> Self { + self.stale_reason = Some(reason.into()); + self + } + + pub fn build(mut self) -> Result { + require_non_empty(&self.component_id, "component_id")?; + require_non_empty(&self.kind, "kind")?; + for dependency in &self.dependencies { + require_non_empty(dependency, "dependencies")?; + if dependency == &self.component_id { + return Err(AdapterPlanBuildError::SelfDependency { + component_id: self.component_id, + }); + } + } + sort_and_reject_duplicates(&mut self.dependencies, "dependencies")?; + for path in &self.identity_inputs { + require_non_empty(path, "identity_inputs")?; + } + sort_and_reject_duplicates(&mut self.identity_inputs, "identity_inputs")?; + for (name, value) in &self.semantic_inputs { + require_non_empty(name, "semantic_inputs key")?; + if value.contains('\0') { + return Err(AdapterPlanBuildError::NulValue { + field: "semantic_inputs value", + }); + } + } + let command = self + .command + .ok_or(AdapterPlanBuildError::MissingField { field: "command" })?; + validate_adapter_command(&command, "command")?; + if self.outputs.is_empty() || self.outputs.len() > 32 { + return Err(AdapterPlanBuildError::OutputCount { + actual: self.outputs.len(), + }); + } + let mut output_names = Vec::with_capacity(self.outputs.len()); + let mut output_targets = Vec::with_capacity(self.outputs.len()); + for output in &self.outputs { + require_non_empty(&output.name, "outputs.name")?; + require_non_empty(&output.source, "outputs.source")?; + require_non_empty(&output.target, "outputs.target")?; + if !output.create_if_missing { + return Err(AdapterPlanBuildError::OutputCreationRequired { + output: output.name.clone(), + }); + } + output_names.push(output.name.clone()); + output_targets.push(output.target.clone()); + } + sort_and_reject_duplicates(&mut output_names, "outputs.name")?; + sort_and_reject_duplicates(&mut output_targets, "outputs.target")?; + let stale_reason = self + .stale_reason + .ok_or(AdapterPlanBuildError::MissingField { + field: "stale_reason", + })?; + require_non_empty(&stale_reason, "stale_reason")?; + + Ok(AdapterPlan { + component_id: self.component_id, + kind: self.kind, + dependencies: self.dependencies, + identity_inputs: self.identity_inputs, + semantic_inputs: self.semantic_inputs, + command, + outputs: self.outputs, + portability: self.portability, + stale_reason, + }) + } +} + +/// Deterministic validating builder for protocol-v2 action plans. +#[derive(Clone, Debug)] +pub struct AdapterPlanV2Builder { + component_id: String, + kind: String, + dependencies: Vec, + dependency_edges: Vec, + identity_inputs: Vec, + semantic_inputs: BTreeMap, + caches: Vec, + external_artifacts: Vec, + runtime_resources: Vec, + actions: Vec, + outputs: Vec, + portability: AdapterPortability, + stale_reason: Option, +} + +impl AdapterPlanV2Builder { + pub fn new(component_id: impl Into, kind: impl Into) -> Self { + Self { + component_id: component_id.into(), + kind: kind.into(), + dependencies: Vec::new(), + dependency_edges: Vec::new(), + identity_inputs: Vec::new(), + semantic_inputs: BTreeMap::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + actions: Vec::new(), + outputs: Vec::new(), + portability: AdapterPortability::Host, + stale_reason: None, + } + } + + pub fn dependency(mut self, component_id: impl Into) -> Self { + self.dependencies.push(component_id.into()); + self + } + + pub fn dependencies(mut self, component_ids: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.dependencies + .extend(component_ids.into_iter().map(Into::into)); + self + } + + pub fn dependency_edge( + mut self, + component_id: impl Into, + edge_type: AdapterDependencyType, + ) -> Self { + self.dependency_edges + .push(AdapterDependency::new(component_id, edge_type)); + self + } + + pub fn build_requires(self, component_id: impl Into) -> Self { + self.dependency_edge(component_id, AdapterDependencyType::BuildRequires) + } + + pub fn runtime_requires(self, component_id: impl Into) -> Self { + self.dependency_edge(component_id, AdapterDependencyType::RuntimeRequires) + } + + pub fn binds_after(self, component_id: impl Into) -> Self { + self.dependency_edge(component_id, AdapterDependencyType::BindsAfter) + } + + pub fn invalidates_with(self, component_id: impl Into) -> Self { + self.dependency_edge(component_id, AdapterDependencyType::InvalidatesWith) + } + + pub fn identity_input(mut self, path: impl Into) -> Self { + self.identity_inputs.push(path.into()); + self + } + + pub fn identity_inputs(mut self, paths: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.identity_inputs + .extend(paths.into_iter().map(Into::into)); + self + } + + pub fn semantic_input(mut self, name: impl Into, value: impl Into) -> Self { + self.semantic_inputs.insert(name.into(), value.into()); + self + } + + pub fn cache(mut self, cache: AdapterCache) -> Self { + self.caches.push(cache); + self + } + + pub fn caches(mut self, caches: I) -> Self + where + I: IntoIterator, + { + self.caches.extend(caches); + self + } + + pub fn external_artifact(mut self, artifact: AdapterExternalArtifact) -> Self { + self.external_artifacts.push(artifact); + self + } + + pub fn external_artifacts(mut self, artifacts: I) -> Self + where + I: IntoIterator, + { + self.external_artifacts.extend(artifacts); + self + } + + pub fn runtime_resource(mut self, resource: AdapterRuntimeResource) -> Self { + self.runtime_resources.push(resource); + self + } + + pub fn runtime_resources(mut self, resources: I) -> Self + where + I: IntoIterator, + { + self.runtime_resources.extend(resources); + self + } + + pub fn staging_command(mut self, command: AdapterCommand) -> Self { + self.actions.push(AdapterAction::Staging(command)); + self + } + + pub fn mounted_command(mut self, command: AdapterCommand) -> Self { + self.actions + .push(AdapterAction::MountedInitialization(command)); + self + } + + pub fn action(mut self, action: AdapterAction) -> Self { + self.actions.push(action); + self + } + + pub fn output(mut self, output: AdapterOutput) -> Self { + self.outputs.push(output); + self + } + + pub fn outputs(mut self, outputs: I) -> Self + where + I: IntoIterator, + { + self.outputs.extend(outputs); + self + } + + pub fn portability(mut self, portability: AdapterPortability) -> Self { + self.portability = portability; + self + } + + pub fn stale_reason(mut self, reason: impl Into) -> Self { + self.stale_reason = Some(reason.into()); + self + } + + pub fn build(mut self) -> Result { + require_non_empty(&self.component_id, "component_id")?; + require_non_empty(&self.kind, "kind")?; + for dependency in &self.dependencies { + require_non_empty(dependency, "dependencies")?; + if dependency == &self.component_id { + return Err(AdapterPlanBuildError::SelfDependency { + component_id: self.component_id, + }); + } + } + sort_and_reject_duplicates(&mut self.dependencies, "dependencies")?; + self.dependency_edges + .sort_by(|left, right| left.component_id.cmp(&right.component_id)); + let mut all_dependency_ids = self.dependencies.iter().cloned().collect::>(); + for dependency in &self.dependency_edges { + require_non_empty(&dependency.component_id, "dependency_edges.component_id")?; + if dependency.component_id == self.component_id { + return Err(AdapterPlanBuildError::SelfDependency { + component_id: self.component_id, + }); + } + if !all_dependency_ids.insert(dependency.component_id.clone()) { + return Err(AdapterPlanBuildError::DuplicateValue { + field: "dependency_edges.component_id", + value: dependency.component_id.clone(), + }); + } + } + for path in &self.identity_inputs { + require_non_empty(path, "identity_inputs")?; + } + sort_and_reject_duplicates(&mut self.identity_inputs, "identity_inputs")?; + for (name, value) in &self.semantic_inputs { + require_non_empty(name, "semantic_inputs key")?; + if value.contains('\0') { + return Err(AdapterPlanBuildError::NulValue { + field: "semantic_inputs value", + }); + } + } + let metadata_only = !self.external_artifacts.is_empty(); + if metadata_only + && (self.kind != "external" + || !self.actions.is_empty() + || !self.caches.is_empty() + || !self.outputs.is_empty()) + { + return Err(AdapterPlanBuildError::ExternalArtifactPlanConflict); + } + if !metadata_only && self.actions.is_empty() { + return Err(AdapterPlanBuildError::MissingAction); + } + if self.actions.len() > 9 { + return Err(AdapterPlanBuildError::ActionCount { + actual: self.actions.len(), + }); + } + let staging_count = self + .actions + .iter() + .filter(|action| matches!(action, AdapterAction::Staging(_))) + .count(); + if staging_count > 1 { + return Err(AdapterPlanBuildError::StagingCommandCount { + actual: staging_count, + }); + } + validate_adapter_caches(&mut self.caches, staging_count)?; + validate_adapter_external_artifacts(&mut self.external_artifacts)?; + validate_adapter_runtime_resources(&mut self.runtime_resources, &self.external_artifacts)?; + for action in &self.actions { + let (field, command) = match action { + AdapterAction::Staging(command) => ("actions.staging", command), + AdapterAction::MountedInitialization(command) => { + ("actions.mounted_initialization", command) + } + }; + validate_adapter_command(command, field)?; + } + if !metadata_only { + validate_adapter_outputs(&self.outputs)?; + } + let stale_reason = self + .stale_reason + .ok_or(AdapterPlanBuildError::MissingField { + field: "stale_reason", + })?; + require_non_empty(&stale_reason, "stale_reason")?; + Ok(AdapterPlanV2 { + component_id: self.component_id, + kind: self.kind, + dependencies: self.dependencies, + dependency_edges: self.dependency_edges, + identity_inputs: self.identity_inputs, + semantic_inputs: self.semantic_inputs, + caches: self.caches, + external_artifacts: self.external_artifacts, + runtime_resources: self.runtime_resources, + actions: self.actions, + outputs: self.outputs, + portability: self.portability, + stale_reason, + }) + } +} + +#[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] +pub enum AdapterPlanBuildError { + #[error("adapter plan field `{field}` is required")] + MissingField { field: &'static str }, + #[error("adapter plan requires at least one staging or mounted action")] + MissingAction, + #[error("adapter plan field `{field}` contains an empty value")] + EmptyValue { field: &'static str }, + #[error("adapter plan field `{field}` contains a NUL byte")] + NulValue { field: &'static str }, + #[error("adapter plan field `{field}` contains duplicate value `{value}`")] + DuplicateValue { field: &'static str, value: String }, + #[error("adapter component `{component_id}` cannot depend on itself")] + SelfDependency { component_id: String }, + #[error("adapter plans require 1-32 outputs; received {actual}")] + OutputCount { actual: usize }, + #[error("adapter output `{output}` must allow Trail to create a missing directory")] + OutputCreationRequired { output: String }, + #[error("adapter protocol-v2 plans support at most nine actions; received {actual}")] + ActionCount { actual: usize }, + #[error("adapter protocol-v2 plans support at most one staging action; received {actual}")] + StagingCommandCount { actual: usize }, + #[error("adapter protocol-v2 plans support at most sixteen caches; received {actual}")] + CacheCount { actual: usize }, + #[error( + "external-artifact plans require kind `external` and cannot mix actions, caches, or filesystem outputs" + )] + ExternalArtifactPlanConflict, + #[error("adapter protocol-v2 plans support at most 32 external artifacts; received {actual}")] + ExternalArtifactCount { actual: usize }, + #[error("adapter external artifact `{artifact}` is invalid")] + InvalidExternalArtifact { artifact: String }, + #[error("adapter protocol-v2 plans support at most 32 runtime resources; received {actual}")] + RuntimeResourceCount { actual: usize }, + #[error("adapter runtime resource `{resource}` is invalid")] + InvalidRuntimeResource { resource: String }, + #[error("adapter cache `{cache}` requires at least one environment binding")] + CacheBindingRequired { cache: String }, + #[error("adapter cache `{cache}` has invalid relative subpath `{path}`")] + InvalidCacheSubpath { cache: String, path: String }, +} + +fn validate_adapter_external_artifacts( + artifacts: &mut [AdapterExternalArtifact], +) -> Result<(), AdapterPlanBuildError> { + if artifacts.len() > 32 { + return Err(AdapterPlanBuildError::ExternalArtifactCount { + actual: artifacts.len(), + }); + } + artifacts.sort_by(|left, right| left.name.cmp(&right.name)); + for (index, artifact) in artifacts.iter().enumerate() { + let digest = artifact.digest.strip_prefix("sha256:"); + let valid_digest = digest.is_some_and(|digest| { + digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }); + let valid_reference = + artifact + .reference + .rsplit_once('@') + .is_some_and(|(repository, digest)| { + !repository.is_empty() + && !repository.contains('@') + && digest == artifact.digest + && repository + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"._-/:".contains(&byte)) + }); + if artifact.name.is_empty() + || artifact.name.len() > 128 + || !artifact.name.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') + }) + || index > 0 && artifacts[index - 1].name == artifact.name + || artifact.artifact_type != "oci_image" + || artifact.provider != "oci" + || artifact.cleanup_owner != "external" + || !valid_digest + || !valid_reference + || !matches!( + artifact.platform.as_str(), + "linux/amd64" | "linux/arm64" | "windows/amd64" | "windows/arm64" + ) + { + return Err(AdapterPlanBuildError::InvalidExternalArtifact { + artifact: artifact.name.clone(), + }); + } + } + Ok(()) +} + +fn validate_adapter_runtime_resources( + resources: &mut [AdapterRuntimeResource], + artifacts: &[AdapterExternalArtifact], +) -> Result<(), AdapterPlanBuildError> { + if resources.len() > 32 { + return Err(AdapterPlanBuildError::RuntimeResourceCount { + actual: resources.len(), + }); + } + let artifact_names = artifacts + .iter() + .map(|artifact| artifact.name.as_str()) + .collect::>(); + resources.sort_by(|left, right| left.name.cmp(&right.name)); + for resource in resources.iter_mut() { + resource + .secrets + .sort_by(|left, right| left.name.cmp(&right.name)); + } + for (index, resource) in resources.iter().enumerate() { + let valid_volume = resource.volume_target.as_deref().map_or(true, |target| { + target.starts_with('/') + && target.len() <= 4096 + && !target.contains('\\') + && !target.chars().any(char::is_control) + && target + .split('/') + .skip(1) + .all(|segment| !segment.is_empty() && segment != "." && segment != "..") + && !["/proc", "/sys", "/dev", "/run", "/etc"] + .iter() + .any(|reserved| { + target == *reserved || target.starts_with(&format!("{reserved}/")) + }) + }); + if resource.name.is_empty() + || resource.name.len() > 128 + || !resource.name.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') + }) + || index > 0 && resources[index - 1].name == resource.name + || resource.runtime_type != "container" + || resource.provider != "oci" + || !artifact_names.contains(resource.artifact_name.as_str()) + || resource.container_port == 0 + || resource.protocol != "tcp" + || resource.health_type != "tcp" + || !(1_000..=300_000).contains(&resource.health_timeout_ms) + || !matches!( + resource.restart_policy.as_str(), + "never" | "on_failure" | "always" + ) + || resource.cleanup_owner != "trail" + || !valid_volume + || !valid_adapter_secret_references(&resource.secrets) + { + return Err(AdapterPlanBuildError::InvalidRuntimeResource { + resource: resource.name.clone(), + }); + } + } + Ok(()) +} + +fn valid_adapter_secret_references(secrets: &[AdapterSecretReference]) -> bool { + if secrets.len() > 16 { + return false; + } + let mut names = BTreeSet::new(); + let mut targets = BTreeSet::new(); + secrets.iter().all(|secret| { + !secret.name.is_empty() + && secret.name.len() <= 128 + && secret.name.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') + }) + && names.insert(secret.name.clone()) + && matches!(secret.provider.as_str(), "file" | "environment_file") + && !secret.reference.is_empty() + && secret.reference.len() <= 4096 + && !secret.reference.chars().any(char::is_control) + && !secret.purpose.is_empty() + && secret.purpose.len() <= 256 + && !secret.purpose.chars().any(char::is_control) + && secret.injection == "file" + && secret.target.starts_with("/run/secrets/") + && secret.target.len() <= 4096 + && !secret.target.contains('\\') + && !secret.target.chars().any(char::is_control) + && secret + .target + .split('/') + .skip(1) + .all(|segment| !segment.is_empty() && segment != "." && segment != "..") + && targets.insert(secret.target.clone()) + && secret.environment.as_deref().map_or(true, |name| { + !name.is_empty() + && name.len() <= 128 + && !name.as_bytes()[0].is_ascii_digit() + && name.bytes().all(|byte| { + byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_' + }) + }) + && secret.version.as_deref().map_or(true, |version| { + !version.is_empty() + && version.len() <= 256 + && !version.chars().any(char::is_control) + }) + && (secret.provider != "file" || secret.reference.starts_with('/')) + && (secret.provider != "environment_file" + || secret.reference.chars().all(|character| { + character.is_ascii_uppercase() || character.is_ascii_digit() || character == '_' + })) + }) +} + +fn validate_adapter_caches( + caches: &mut [AdapterCache], + staging_count: usize, +) -> Result<(), AdapterPlanBuildError> { + if caches.len() > 16 { + return Err(AdapterPlanBuildError::CacheCount { + actual: caches.len(), + }); + } + if !caches.is_empty() && staging_count != 1 { + return Err(AdapterPlanBuildError::MissingField { + field: "cache staging action", + }); + } + caches.sort_by(|left, right| left.name.cmp(&right.name)); + let mut previous_name: Option<&str> = None; + let mut environment_names = BTreeSet::new(); + for cache in caches { + require_non_empty(&cache.name, "caches.name")?; + if cache.name.len() > 64 + || !cache.name.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '_' | '-') + }) + { + return Err(AdapterPlanBuildError::EmptyValue { + field: "caches.name", + }); + } + if previous_name == Some(cache.name.as_str()) { + return Err(AdapterPlanBuildError::DuplicateValue { + field: "caches.name", + value: cache.name.clone(), + }); + } + previous_name = Some(&cache.name); + if cache.compatibility.len() > 24 { + return Err(AdapterPlanBuildError::CacheCount { + actual: cache.compatibility.len(), + }); + } + for (name, value) in &cache.compatibility { + require_non_empty(name, "caches.compatibility key")?; + require_non_empty(value, "caches.compatibility value")?; + if name.len() > 128 || value.len() > 4096 || name.contains('\0') || value.contains('\0') + { + return Err(AdapterPlanBuildError::NulValue { + field: "caches.compatibility", + }); + } + } + if cache.environment.is_empty() { + return Err(AdapterPlanBuildError::CacheBindingRequired { + cache: cache.name.clone(), + }); + } + if cache.environment.len() > 16 { + return Err(AdapterPlanBuildError::CacheCount { + actual: cache.environment.len(), + }); + } + for (name, path) in &cache.environment { + if name.is_empty() + || name.len() > 128 + || !name + .chars() + .all(|character| character == '_' || character.is_ascii_alphanumeric()) + || !environment_names.insert(name.clone()) + { + return Err(AdapterPlanBuildError::DuplicateValue { + field: "caches.environment", + value: name.clone(), + }); + } + if !valid_cache_subpath(path) { + return Err(AdapterPlanBuildError::InvalidCacheSubpath { + cache: cache.name.clone(), + path: path.clone(), + }); + } + } + } + Ok(()) +} + +fn valid_cache_subpath(path: &str) -> bool { + if path == "." { + return true; + } + !path.is_empty() + && !path.starts_with('/') + && !path.contains('\\') + && path + .split('/') + .all(|segment| !segment.is_empty() && segment != "." && segment != "..") +} + +fn validate_adapter_command( + command: &AdapterCommand, + field: &'static str, +) -> Result<(), AdapterPlanBuildError> { + require_non_empty(&command.program, field)?; + require_non_empty(&command.working_directory, field)?; + if command + .args + .iter() + .chain(command.environment.keys()) + .chain(command.environment.values()) + .any(|value| value.contains('\0')) + { + return Err(AdapterPlanBuildError::NulValue { field }); + } + Ok(()) +} + +fn validate_adapter_outputs(outputs: &[AdapterOutput]) -> Result<(), AdapterPlanBuildError> { + if outputs.is_empty() || outputs.len() > 32 { + return Err(AdapterPlanBuildError::OutputCount { + actual: outputs.len(), + }); + } + let mut output_names = Vec::with_capacity(outputs.len()); + let mut output_targets = Vec::with_capacity(outputs.len()); + for output in outputs { + require_non_empty(&output.name, "outputs.name")?; + require_non_empty(&output.source, "outputs.source")?; + require_non_empty(&output.target, "outputs.target")?; + if !output.create_if_missing { + return Err(AdapterPlanBuildError::OutputCreationRequired { + output: output.name.clone(), + }); + } + output_names.push(output.name.clone()); + output_targets.push(output.target.clone()); + } + sort_and_reject_duplicates(&mut output_names, "outputs.name")?; + sort_and_reject_duplicates(&mut output_targets, "outputs.target") +} + +fn require_non_empty(value: &str, field: &'static str) -> Result<(), AdapterPlanBuildError> { + if value.trim().is_empty() { + Err(AdapterPlanBuildError::EmptyValue { field }) + } else { + Ok(()) + } +} + +fn sort_and_reject_duplicates( + values: &mut [String], + field: &'static str, +) -> Result<(), AdapterPlanBuildError> { + values.sort(); + if let Some(duplicate) = values.windows(2).find(|pair| pair[0] == pair[1]) { + return Err(AdapterPlanBuildError::DuplicateValue { + field, + value: duplicate[0].clone(), + }); + } + Ok(()) +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterCommand { + pub program: String, + pub args: Vec, + pub working_directory: String, + #[serde(default)] + pub environment: BTreeMap, +} + +impl AdapterCommand { + pub fn new(program: impl Into, args: I) -> Self + where + I: IntoIterator, + S: Into, + { + Self { + program: program.into(), + args: args.into_iter().map(Into::into).collect(), + working_directory: ".".to_string(), + environment: BTreeMap::new(), + } + } + + pub fn in_directory(mut self, working_directory: impl Into) -> Self { + self.working_directory = working_directory.into(); + self + } + + pub fn environment_variable( + mut self, + name: impl Into, + value: impl Into, + ) -> Self { + self.environment.insert(name.into(), value.into()); + self + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AdapterOutput { + pub name: String, + pub source: String, + pub target: String, + #[serde(default)] + pub policy: AdapterOutputPolicy, + #[serde(default = "default_create_if_missing")] + pub create_if_missing: bool, +} + +impl AdapterOutput { + pub fn immutable_seed_private( + name: impl Into, + source: impl Into, + target: impl Into, + ) -> Self { + Self::new( + name, + source, + target, + AdapterOutputPolicy::ImmutableSeedPrivate, + ) + } + + pub fn writable_private( + name: impl Into, + source: impl Into, + target: impl Into, + ) -> Self { + Self::new(name, source, target, AdapterOutputPolicy::WritablePrivate) + } + + pub fn with_create_if_missing(mut self, create_if_missing: bool) -> Self { + self.create_if_missing = create_if_missing; + self + } + + fn new( + name: impl Into, + source: impl Into, + target: impl Into, + policy: AdapterOutputPolicy, + ) -> Self { + Self { + name: name.into(), + source: source.into(), + target: target.into(), + policy, + create_if_missing: true, + } + } +} + +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum AdapterOutputPolicy { + #[default] + ImmutableSeedPrivate, + WritablePrivate, +} + +fn default_create_if_missing() -> bool { + true +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum AdapterPortability { + Host, + Platform, +} + +#[derive(Debug, thiserror::Error)] +pub enum ProtocolError { + #[error("adapter protocol I/O failed: {0}")] + Io(#[from] io::Error), + #[error("adapter protocol serialization failed: {0}")] + Cbor(#[from] serde_cbor::Error), + #[error("adapter protocol frame is {actual} bytes; maximum is {maximum}")] + FrameTooLarge { actual: usize, maximum: usize }, + #[error("adapter protocol stream ended before a complete frame was read")] + Truncated, +} + +pub fn read_frame Deserialize<'de>>( + reader: &mut impl Read, + maximum: usize, +) -> Result { + let mut header = [0u8; 4]; + reader.read_exact(&mut header).map_err(|error| { + if error.kind() == io::ErrorKind::UnexpectedEof { + ProtocolError::Truncated + } else { + ProtocolError::Io(error) + } + })?; + let length = u32::from_be_bytes(header) as usize; + if length > maximum { + return Err(ProtocolError::FrameTooLarge { + actual: length, + maximum, + }); + } + let mut body = vec![0u8; length]; + reader.read_exact(&mut body).map_err(|error| { + if error.kind() == io::ErrorKind::UnexpectedEof { + ProtocolError::Truncated + } else { + ProtocolError::Io(error) + } + })?; + Ok(serde_cbor::from_slice(&body)?) +} + +pub fn write_frame( + writer: &mut impl Write, + value: &T, + maximum: usize, +) -> Result<(), ProtocolError> { + let body = serde_cbor::to_vec(value)?; + if body.len() > maximum || body.len() > u32::MAX as usize { + return Err(ProtocolError::FrameTooLarge { + actual: body.len(), + maximum: maximum.min(u32::MAX as usize), + }); + } + writer.write_all(&(body.len() as u32).to_be_bytes())?; + writer.write_all(&body)?; + writer.flush()?; + Ok(()) +} + +pub fn serve_once( + handler: impl FnOnce(AdapterRequest) -> AdapterResponse, +) -> Result<(), ProtocolError> { + let request = read_frame(&mut io::stdin().lock(), MAX_FRAME_BYTES)?; + let response = handler(request); + write_frame(&mut io::stdout().lock(), &response, MAX_FRAME_BYTES) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn framed_protocol_round_trips_binary_pinned_files() { + let request = AdapterRequest { + protocol: PROTOCOL_V1.to_string(), + request_id: "request-1".to_string(), + adapter_identity: "example/test@1".to_string(), + distribution_digest: "sha256:abc".to_string(), + host: AdapterHost { + operating_system: "test".to_string(), + architecture: "test".to_string(), + }, + source_root: "root".to_string(), + operation: AdapterOperation::Discover { + component_root: String::new(), + files: vec![PinnedFile { + path: "manifest.bin".to_string(), + content_hash: "hash".to_string(), + executable: false, + content: vec![0, 1, 0xff], + }], + }, + }; + let mut wire = Vec::new(); + write_frame(&mut wire, &request, MAX_FRAME_BYTES).unwrap(); + let decoded: AdapterRequest = read_frame(&mut wire.as_slice(), MAX_FRAME_BYTES).unwrap(); + assert_eq!(decoded, request); + } + + #[test] + fn framed_protocol_rejects_oversized_frames_before_allocating_body() { + let mut wire = ((MAX_FRAME_BYTES + 1) as u32).to_be_bytes().to_vec(); + wire.extend([0u8; 8]); + assert!(matches!( + read_frame::(&mut wire.as_slice(), MAX_FRAME_BYTES), + Err(ProtocolError::FrameTooLarge { .. }) + )); + } + + #[test] + fn output_policy_is_backward_compatible_and_round_trips_writable_private() { + #[derive(Serialize)] + struct LegacyOutput<'a> { + name: &'a str, + source: &'a str, + target: &'a str, + create_if_missing: bool, + } + let legacy = serde_cbor::to_vec(&LegacyOutput { + name: "generated", + source: "generated", + target: "build", + create_if_missing: true, + }) + .unwrap(); + let legacy: AdapterOutput = serde_cbor::from_slice(&legacy).unwrap(); + assert_eq!(legacy.policy, AdapterOutputPolicy::ImmutableSeedPrivate); + + let private = AdapterOutput { + name: "build-tree".to_string(), + source: "build".to_string(), + target: "build".to_string(), + policy: AdapterOutputPolicy::WritablePrivate, + create_if_missing: true, + }; + let private: AdapterOutput = + serde_cbor::from_slice(&serde_cbor::to_vec(&private).unwrap()).unwrap(); + assert_eq!(private.policy, AdapterOutputPolicy::WritablePrivate); + } + + #[test] + fn dependency_edges_are_backward_compatible_and_round_trip() { + #[derive(Serialize)] + struct LegacyPlan { + component_id: String, + kind: String, + identity_inputs: Vec, + semantic_inputs: BTreeMap, + command: AdapterCommand, + outputs: Vec, + portability: AdapterPortability, + stale_reason: String, + } + let legacy = LegacyPlan { + component_id: "generated".to_string(), + kind: "generated".to_string(), + identity_inputs: vec!["schema.json".to_string()], + semantic_inputs: BTreeMap::new(), + command: AdapterCommand { + program: "generator".to_string(), + args: Vec::new(), + working_directory: ".".to_string(), + environment: BTreeMap::new(), + }, + outputs: vec![AdapterOutput { + name: "generated".to_string(), + source: "generated".to_string(), + target: "generated".to_string(), + policy: AdapterOutputPolicy::ImmutableSeedPrivate, + create_if_missing: true, + }], + portability: AdapterPortability::Host, + stale_reason: "input changed".to_string(), + }; + let decoded: AdapterPlan = + serde_cbor::from_slice(&serde_cbor::to_vec(&legacy).unwrap()).unwrap(); + assert!(decoded.dependencies.is_empty()); + + let mut current = decoded; + current.dependencies = vec!["toolchain".to_string()]; + let current: AdapterPlan = + serde_cbor::from_slice(&serde_cbor::to_vec(¤t).unwrap()).unwrap(); + assert_eq!(current.dependencies, ["toolchain"]); + } + + #[test] + fn v2_builder_supports_mounted_only_initialization() { + let plan = AdapterPlanV2::builder("python", "dependency") + .identity_input("pyproject.toml") + .mounted_command( + AdapterCommand::new("python3", ["-m", "venv", ".venv"]) + .in_directory("services/api"), + ) + .output(AdapterOutput::writable_private("venv", ".venv", ".venv")) + .stale_reason("Python or the dependency manifest changed") + .build() + .unwrap(); + + assert_eq!(plan.actions.len(), 1); + let AdapterAction::MountedInitialization(command) = &plan.actions[0] else { + panic!("v2 builder emitted the wrong action phase"); + }; + assert_eq!(command.working_directory, "services/api"); + let decoded: AdapterPlanV2 = + serde_cbor::from_slice(&serde_cbor::to_vec(&plan).unwrap()).unwrap(); + assert_eq!(decoded, plan); + } + + #[test] + fn v2_builder_supports_metadata_only_pinned_oci_artifacts() { + let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let plan = AdapterPlanV2::builder("images", "external") + .identity_input("images.lock") + .external_artifact(AdapterExternalArtifact::pinned_oci_image( + "web", + format!("ghcr.io/example/web@{digest}"), + "linux/amd64", + )) + .runtime_resource( + AdapterRuntimeResource::oci_container("web-service", "web", 8080) + .health_timeout_ms(45_000) + .volume_target("/var/lib/web") + .secret( + AdapterSecretReference::file( + "api-token", + "environment_file", + "WEB_API_TOKEN_FILE", + "/run/secrets/api-token", + "authenticate the web service", + ) + .version("rotation-7") + .environment_variable("WEB_API_TOKEN_FILE"), + ), + ) + .stale_reason("pinned OCI declaration changed") + .build() + .unwrap(); + + assert!(plan.actions.is_empty()); + assert!(plan.outputs.is_empty()); + assert_eq!(plan.external_artifacts[0].digest, digest); + assert_eq!(plan.runtime_resources[0].container_port, 8080); + assert_eq!(plan.runtime_resources[0].secrets[0].name, "api-token"); + assert_eq!( + plan.runtime_resources[0].secrets[0].reference, + "WEB_API_TOKEN_FILE" + ); + assert_eq!( + plan.runtime_resources[0].secrets[0].environment.as_deref(), + Some("WEB_API_TOKEN_FILE") + ); + let decoded: AdapterPlanV2 = + serde_cbor::from_slice(&serde_cbor::to_vec(&plan).unwrap()).unwrap(); + assert_eq!(decoded, plan); + + let mixed = AdapterPlanV2::builder("images", "external") + .external_artifact(AdapterExternalArtifact::pinned_oci_image( + "web", + format!("ghcr.io/example/web@{digest}"), + "linux/amd64", + )) + .staging_command(AdapterCommand::new("docker", ["pull"])) + .stale_reason("pinned OCI declaration changed") + .build(); + assert_eq!( + mixed, + Err(AdapterPlanBuildError::ExternalArtifactPlanConflict) + ); + } + + #[test] + fn v2_builder_preserves_typed_dependency_semantics() { + let plan = AdapterPlanV2::builder("application", "runtime") + .build_requires("compiler") + .runtime_requires("database") + .binds_after("network") + .invalidates_with("configuration") + .mounted_command(AdapterCommand::new("initializer", ["prepare"])) + .output(AdapterOutput::writable_private("state", "state", "state")) + .stale_reason("typed dependency changed") + .build() + .unwrap(); + + assert_eq!(plan.dependency_edges.len(), 4); + assert_eq!( + plan.dependency_edges[0], + AdapterDependency::new("compiler", AdapterDependencyType::BuildRequires) + ); + let decoded: AdapterPlanV2 = + serde_cbor::from_slice(&serde_cbor::to_vec(&plan).unwrap()).unwrap(); + assert_eq!(decoded, plan); + + let duplicate = AdapterPlanV2::builder("application", "runtime") + .dependency("compiler") + .build_requires("compiler") + .mounted_command(AdapterCommand::new("initializer", ["prepare"])) + .output(AdapterOutput::writable_private("state", "state", "state")) + .stale_reason("typed dependency changed") + .build(); + assert!(matches!( + duplicate, + Err(AdapterPlanBuildError::DuplicateValue { .. }) + )); + } + + #[test] + fn v2_builder_declares_deterministic_host_owned_caches() { + let plan = AdapterPlanV2::builder("generated", "generated") + .cache( + AdapterCache::host_exclusive("package-store", AdapterCacheProtocol::ContentStore) + .compatibility_dimension("tool", "generator@1") + .environment_variable("GENERATOR_CACHE", ".") + .environment_variable("GENERATOR_INDEX", "index"), + ) + .staging_command(AdapterCommand::new("generator", ["build"])) + .output(AdapterOutput::immutable_seed_private( + "generated", + "generated", + "generated", + )) + .stale_reason("generator inputs changed") + .build() + .unwrap(); + + assert_eq!(plan.caches.len(), 1); + assert_eq!(plan.caches[0].access, AdapterCacheAccess::HostExclusive); + assert_eq!(plan.caches[0].environment["GENERATOR_INDEX"], "index"); + let decoded: AdapterPlanV2 = + serde_cbor::from_slice(&serde_cbor::to_vec(&plan).unwrap()).unwrap(); + assert_eq!(decoded, plan); + } + + #[test] + fn v2_builder_rejects_mounted_and_ambiguous_cache_bindings() { + let mounted_only = AdapterPlanV2::builder("generated", "generated") + .cache( + AdapterCache::host_exclusive("cache", AdapterCacheProtocol::ContentStore) + .environment_variable("GENERATOR_CACHE", "."), + ) + .mounted_command(AdapterCommand::new("generator", ["build"])) + .output(AdapterOutput::writable_private( + "generated", + "generated", + "generated", + )) + .stale_reason("generator inputs changed") + .build(); + assert_eq!( + mounted_only, + Err(AdapterPlanBuildError::MissingField { + field: "cache staging action" + }) + ); + + let ambiguous = AdapterPlanV2::builder("generated", "generated") + .caches([ + AdapterCache::host_exclusive("first", AdapterCacheProtocol::ContentStore) + .environment_variable("GENERATOR_CACHE", "first"), + AdapterCache::host_exclusive("second", AdapterCacheProtocol::LockedIndex) + .environment_variable("GENERATOR_CACHE", "second"), + ]) + .staging_command(AdapterCommand::new("generator", ["build"])) + .output(AdapterOutput::immutable_seed_private( + "generated", + "generated", + "generated", + )) + .stale_reason("generator inputs changed") + .build(); + assert!(matches!( + ambiguous, + Err(AdapterPlanBuildError::DuplicateValue { + field: "caches.environment", + .. + }) + )); + } + + #[test] + fn legacy_package_metadata_defaults_to_protocol_v1() { + let manifest: AdapterPackageManifest = toml::from_str( + r#"schema = "trail.environment-adapter-package/v1" +[adapter] +canonical_identity = "example/test@1" +implementation_version = "1" +selectors = ["example/test@1"] +kind = "generated" +layer_adapter_name = "test" +discovery_markers = ["test.adapter"] +stability = "experimental" +description = "test" +[executable] +path = "adapter" +sha256 = "sha256:00" +"#, + ) + .unwrap(); + assert_eq!(manifest.adapter.protocols, [PROTOCOL_V1]); + let encoded = serde_cbor::to_vec(&manifest).unwrap(); + let value: serde_cbor::Value = serde_cbor::from_slice(&encoded).unwrap(); + let serde_cbor::Value::Map(package) = value else { + panic!("package did not encode as a CBOR map"); + }; + let adapter = package + .get(&serde_cbor::Value::Text("adapter".to_string())) + .unwrap(); + let serde_cbor::Value::Map(adapter) = adapter else { + panic!("adapter metadata did not encode as a CBOR map"); + }; + assert!(!adapter.contains_key(&serde_cbor::Value::Text("protocols".to_string()))); + } + + #[test] + fn plan_builder_canonicalizes_sets_and_preserves_wire_shape() { + let plan = AdapterPlan::builder("generated.client", "generated") + .dependencies(["schema", "compiler"]) + .identity_inputs(["schema.json", "generator.toml"]) + .semantic_input("strategy", "client-v1") + .command( + AdapterCommand::new("generator", ["--out", "generated"]) + .in_directory("project") + .environment_variable("GENERATOR_COLOR", "never"), + ) + .output(AdapterOutput::immutable_seed_private( + "client", + "generated", + "src/generated", + )) + .stale_reason("schema, generator, or strategy changed") + .build() + .unwrap(); + + assert_eq!(plan.dependencies, ["compiler", "schema"]); + assert_eq!(plan.identity_inputs, ["generator.toml", "schema.json"]); + assert_eq!(plan.command.working_directory, "project"); + assert_eq!(plan.command.environment["GENERATOR_COLOR"], "never"); + assert_eq!( + plan.outputs[0].policy, + AdapterOutputPolicy::ImmutableSeedPrivate + ); + let decoded: AdapterPlan = + serde_cbor::from_slice(&serde_cbor::to_vec(&plan).unwrap()).unwrap(); + assert_eq!(decoded, plan); + } + + #[test] + fn plan_builder_rejects_duplicate_and_self_dependencies() { + let duplicate = AdapterPlan::builder("generated", "generated") + .dependencies(["schema", "schema"]) + .command(AdapterCommand::new("generator", Vec::::new())) + .output(AdapterOutput::writable_private( + "generated", + "generated", + "generated", + )) + .stale_reason("inputs changed") + .build(); + assert_eq!( + duplicate, + Err(AdapterPlanBuildError::DuplicateValue { + field: "dependencies", + value: "schema".to_string(), + }) + ); + + let self_dependency = AdapterPlan::builder("generated", "generated") + .dependency("generated") + .command(AdapterCommand::new("generator", Vec::::new())) + .output(AdapterOutput::writable_private( + "generated", + "generated", + "generated", + )) + .stale_reason("inputs changed") + .build(); + assert_eq!( + self_dependency, + Err(AdapterPlanBuildError::SelfDependency { + component_id: "generated".to_string(), + }) + ); + } + + #[test] + fn plan_builder_requires_command_output_and_stale_reason() { + assert_eq!( + AdapterPlan::builder("generated", "generated").build(), + Err(AdapterPlanBuildError::MissingField { field: "command" }) + ); + assert_eq!( + AdapterPlan::builder("generated", "generated") + .command(AdapterCommand::new("generator", Vec::::new())) + .stale_reason("inputs changed") + .build(), + Err(AdapterPlanBuildError::OutputCount { actual: 0 }) + ); + assert_eq!( + AdapterPlan::builder("generated", "generated") + .command(AdapterCommand::new("generator", Vec::::new())) + .output(AdapterOutput::immutable_seed_private( + "generated", + "generated", + "generated", + )) + .build(), + Err(AdapterPlanBuildError::MissingField { + field: "stale_reason", + }) + ); + } +} diff --git a/trail/Cargo.toml b/trail/Cargo.toml index 963cc85..5ae6ad3 100644 --- a/trail/Cargo.toml +++ b/trail/Cargo.toml @@ -20,13 +20,17 @@ macfuse = ["dep:fuser"] [dependencies] bzip2.workspace = true clap.workspace = true +ed25519-dalek.workspace = true flate2.workspace = true getrandom.workspace = true hex.workspace = true ignore.workspace = true +jsonschema.workspace = true libc.workspace = true notify.workspace = true -prolly = { package = "prolly-map", path = "../prolly", features = ["slatedb", "sqlite"] } +prolly = { package = "prolly-map", path = "../prolly" } +prolly-store-slatedb = { path = "../prolly/stores/prolly-store-slatedb" } +prolly-store-sqlite = { path = "../prolly/stores/prolly-store-sqlite" } rayon.workspace = true reqwest.workspace = true rusqlite.workspace = true @@ -34,22 +38,34 @@ rustix.workspace = true serde.workspace = true serde_cbor.workspace = true serde_json.workspace = true +globset.workspace = true sha2.workspace = true slatedb.workspace = true similar.workspace = true sqlite-vec.workspace = true tar.workspace = true +tempfile.workspace = true thiserror.workspace = true toml.workspace = true +trail-environment-adapter-sdk = { path = "../trail-environment-adapter-sdk" } unicode-normalization.workspace = true +unicode-width.workspace = true +url.workspace = true +terminal_size.workspace = true +anstyle.workspace = true +time.workspace = true walkdir.workspace = true zip.workspace = true [target.'cfg(target_os = "linux")'.dependencies] fuser = { version = "0.15.1", default-features = false } +inotify.workspace = true +landlock.workspace = true [target.'cfg(target_os = "macos")'.dependencies] async-trait = "0.1" +core-foundation-sys.workspace = true +fsevent-sys.workspace = true fuser = { version = "0.15.1", features = ["macfuse-4-compat"], optional = true } nfsserve = "0.11" tokio = { workspace = true, features = ["io-util", "macros", "net", "sync"] } @@ -58,10 +74,10 @@ tokio = { workspace = true, features = ["io-util", "macros", "net", "sync"] } dokan = "0.3.1" dokan-sys = "0.3.1" widestring = "0.4.3" -winapi = { version = "0.3", features = ["fileapi", "handleapi", "minwinbase", "ntdef", "ntstatus", "processthreadsapi", "winnt"] } +winapi = { version = "0.3", features = ["errhandlingapi", "fileapi", "handleapi", "jobapi2", "minwinbase", "ntdef", "ntstatus", "processthreadsapi", "sddl", "securitybaseapi", "synchapi", "userenv", "winbase", "winerror", "winnt"] } [dev-dependencies] -tempfile.workspace = true +proptest = "=1.8.0" [lints] workspace = true @@ -69,3 +85,7 @@ workspace = true [[bench]] name = "sqlite_vector_memory_bench" harness = false + +[[bench]] +name = "acp_relay_bench" +harness = false diff --git a/trail/benches/acp_relay_bench.rs b/trail/benches/acp_relay_bench.rs new file mode 100644 index 0000000..60bf2dd --- /dev/null +++ b/trail/benches/acp_relay_bench.rs @@ -0,0 +1,98 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::thread; +use std::time::{Duration, Instant}; + +use trail::acp::{benchmark_acp_relay_frames, AcpRelayOptions}; +use trail::{InitImportMode, Trail}; + +const FRAME_COUNT: usize = 10_000; + +fn percentile(sorted: &[u128], numerator: usize, denominator: usize) -> u128 { + let index = ((sorted.len() - 1) * numerator) / denominator; + sorted[index] +} + +fn main() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "ACP relay benchmark\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let options = AcpRelayOptions { + workspace_root: temp.path().to_path_buf(), + db_dir: temp.path().join(".trail"), + lane: None, + from_ref: None, + provider: Some("benchmark".to_string()), + model: None, + materialize: false, + workdir: None, + inject_mcp: false, + upstream_command: vec!["benchmark-agent".to_string()], + upstream_env: BTreeMap::new(), + }; + + let frames = (0..FRAME_COUNT) + .map(|sequence| { + let (agent_to_client, value) = match sequence { + 0 => (false, serde_json::json!({"jsonrpc":"2.0","id":"benchmark-init","method":"initialize","params":{"protocolVersion":1,"clientCapabilities":{}}})), + 1 => (true, serde_json::json!({"jsonrpc":"2.0","id":"benchmark-init","result":{"protocolVersion":1,"agentCapabilities":{}}})), + _ if sequence % 4 == 0 => (false, serde_json::json!({"jsonrpc":"2.0","id":sequence as i64,"method":"ext/request","params":{"sequence":sequence}})), + _ if sequence % 4 == 1 => (true, serde_json::json!({"jsonrpc":"2.0","method":"ext/update","params":{"sequence":sequence}})), + _ if sequence % 4 == 2 => (true, serde_json::json!({"jsonrpc":"2.0","id":sequence as i64,"method":"ext/callback","params":{"sequence":sequence}})), + _ => (false, serde_json::json!({"jsonrpc":"2.0","id":sequence as i64 - 1,"result":{"sequence":sequence}})), + }; + let mut raw = serde_json::to_vec(&value).unwrap(); + raw.push(b'\n'); + (agent_to_client, raw) + }) + .collect::>(); + let expected = frames + .iter() + .map(|(_, raw)| raw.clone()) + .collect::>(); + let samples = benchmark_acp_relay_frames(options, frames).unwrap(); + assert_eq!(samples.len(), FRAME_COUNT); + for (index, sample) in samples.iter().enumerate() { + if !sample.transformed { + assert_eq!(sample.forwarded, expected[index], "frame {index} changed"); + } + } + + let deadline = Instant::now() + Duration::from_secs(30); + let receipts = loop { + let db = Trail::open(temp.path()).unwrap(); + let mut receipts = Vec::new(); + for offset in (0..FRAME_COUNT).step_by(1_000) { + receipts.extend( + db.list_agent_hook_receipts_page(Some("trail-acp"), None, offset, 1_000) + .unwrap(), + ); + } + if receipts.len() == FRAME_COUNT { + break receipts; + } + assert!( + Instant::now() < deadline, + "captured {} of {FRAME_COUNT} benchmark frames", + receipts.len() + ); + thread::sleep(Duration::from_millis(50)); + }; + let sequences = receipts + .iter() + .map(|receipt| receipt.connection_sequence.unwrap()) + .collect::>(); + assert_eq!(sequences.len(), FRAME_COUNT); + + let mut latencies = samples + .iter() + .map(|sample| sample.latency_micros) + .collect::>(); + latencies.sort_unstable(); + println!( + "ACP relay forwarding latency: p50={}us p95={}us p99={}us frames={FRAME_COUNT}", + percentile(&latencies, 50, 100), + percentile(&latencies, 95, 100), + percentile(&latencies, 99, 100), + ); +} diff --git a/trail/src/acp.rs b/trail/src/acp.rs index 998897f..21f4576 100644 --- a/trail/src/acp.rs +++ b/trail/src/acp.rs @@ -1,26 +1,93 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::env; -use std::io::{self, BufRead, BufReader, Read, Write}; -use std::path::PathBuf; -use std::process::{Command, Stdio}; -use std::sync::{mpsc, Arc, Mutex}; -use std::thread; -use std::time::Duration; +use std::ffi::OsString; +use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use url::Url; use crate::model::*; use crate::{Error, PatchDocument, PatchEdit, Result, Trail}; +mod capture; +mod protocol; mod registry; +mod schema; +mod setup; +mod transform; +mod transport; + +use capture::{capture_frame, CaptureIngress}; +use protocol::{Direction, Frame}; +use schema::AcpV1Contract; +use transform::{ + passthrough_session_mappings, PathMapping, TransformOptions, TransformPipeline, WorkspaceMapper, +}; +use transport::{FrameObserver, RelayFinishReason, StdioRelay}; + +pub use setup::{apply_acp_setup_plan, build_acp_setup_plan, AcpSetupReport}; const ACP_CAPTURE_LOCK_WAIT: Duration = Duration::from_secs(30); +const ACP_CALLBACK_CAPTURE_FLUSH_TIMEOUT: Duration = Duration::from_secs(2); const CLAUDE_ACP_ADAPTER: &str = "@agentclientprotocol/claude-agent-acp@latest"; const CODEX_ACP_ADAPTER: &str = "@agentclientprotocol/codex-acp@latest"; const ACP_MAX_PENDING_EVENTS_PER_TURN: usize = 128; const ACP_MAX_ASSISTANT_MESSAGE_BYTES: usize = 256 * 1024; const ACP_MAX_ASSISTANT_TOTAL_BYTES: usize = 1024 * 1024; +/// Returns the immutable contract identity and build attestation exposed by +/// `trail agent acp doctor`. +pub fn acp_v1_conformance_evidence() -> AcpConformanceEvidence { + let source_revision = option_env!("TRAIL_SOURCE_REVISION") + .filter(|revision| !revision.is_empty()) + .unwrap_or("unverified"); + let verified = source_revision != "unverified" + && option_env!("TRAIL_ACP_V1_CONFORMANCE_VERIFIED") == Some(source_revision); + AcpConformanceEvidence { + wire_version: 1, + schema_commit: schema::ACP_V1_SCHEMA_COMMIT.to_string(), + schema_sha256: schema::ACP_V1_SCHEMA_SHA256.to_string(), + meta_sha256: schema::ACP_V1_META_SHA256.to_string(), + transport: "stdio".to_string(), + method_count: 23, + evidence_status: if verified { "verified" } else { "unverified" }.to_string(), + build_identifier: format!("{}+{source_revision}", env!("CARGO_PKG_VERSION")), + exclusions: vec![ + "ACP v2".to_string(), + "draft remote HTTP transport".to_string(), + ], + } +} + +/// Exercises the same workspace mapper used by the relay, including the rule +/// that roots outside the Trail workspace are preserved rather than isolated. +pub fn validate_acp_path_mapping(workspace_root: &std::path::Path) -> Result<()> { + let mapper = WorkspaceMapper::new(workspace_root.to_path_buf(), workspace_root.to_path_buf())?; + let workspace = mapper.map(workspace_root)?; + if !workspace.isolated { + return Err(Error::InvalidPath { + path: workspace_root.display().to_string(), + reason: "ACP workspace root was not recognized as isolated".to_string(), + }); + } + if let Some(external_root) = workspace_root.parent() { + if external_root != workspace_root { + let external = mapper.map(external_root)?; + if external.isolated || external.effective != external.original { + return Err(Error::InvalidPath { + path: external_root.display().to_string(), + reason: "ACP external root was not preserved".to_string(), + }); + } + } + } + Ok(()) +} + #[derive(Clone, Debug)] pub struct AcpRelayOptions { pub workspace_root: PathBuf, @@ -76,6 +143,27 @@ pub fn acp_provider_profile(agent: &str) -> Result { default_terminal_command: Some(vec!["agent".to_string()]), }) } + Some("grok") => { + let available = command_in_path("grok"); + Ok(AcpProviderProfile { + agent: "grok".to_string(), + display_name: "Grok Build".to_string(), + available, + relay_command: built_in_acp_relay_command("grok"), + notes: if available { + vec![ + "uses Grok Build's native ACP server through `grok agent stdio`" + .to_string(), + ] + } else { + vec!["`grok` was not found on PATH".to_string()] + }, + supports_acp: true, + supports_mcp: true, + supports_terminal: true, + default_terminal_command: Some(vec!["grok".to_string()]), + }) + } _ => Err(Error::InvalidInput(format!( "unsupported ACP agent `{agent}`; supported agents: {}; use `trail acp relay -- ...` for another ACP-compatible agent", supported_acp_agents().join(", ") @@ -101,6 +189,11 @@ pub fn acp_provider_upstream_command(agent: &str) -> Result> { CODEX_ACP_ADAPTER.to_string(), ]), "cursor" => Ok(vec!["agent".to_string(), "acp".to_string()]), + "grok" => Ok(vec![ + "grok".to_string(), + "agent".to_string(), + "stdio".to_string(), + ]), _ => Err(Error::InvalidInput(format!( "provider `{}` does not define an ACP upstream command", profile.agent @@ -178,7 +271,7 @@ pub fn agent_provider_profile(provider: &str) -> Result { "runs OpenCode in a Trail materialized task lane", )), _ => Err(Error::InvalidInput(format!( - "unsupported agent provider `{provider}`; supported providers: {}. You can still pass an explicit command after `--` to `trail agent start` or `trail agent acp`.", + "unsupported agent provider `{provider}`; supported providers: {}. You can still pass an explicit command after `--` to `trail agent start` or `trail agent acp run`.", supported_agent_providers().join(", ") ))), } @@ -201,43 +294,8 @@ pub fn terminal_agent_command(provider: &str) -> Result> { }) } -pub fn acp_install_report(agent: &str, editor: &str, dry_run: bool) -> Result { - acp_install_report_with_registry(agent, editor, dry_run, None) -} - -pub fn acp_install_report_with_registry( - agent: &str, - editor: &str, - dry_run: bool, - cache_dir: Option<&std::path::Path>, -) -> Result { - let profile = acp_provider_profile_with_registry(agent, cache_dir)?; - let editor = match editor { - "generic" | "zed" => editor, - other => { - return Err(Error::InvalidInput(format!( - "unsupported ACP editor `{other}`; supported editors: generic, zed" - ))) - } - }; - let snippet = acp_editor_snippet(editor, &profile.agent, &profile.relay_command); - Ok(AcpInstallReport { - agent: profile.agent, - editor: editor.to_string(), - dry_run, - relay_command: profile.relay_command, - snippet, - detected: profile.available, - warnings: if profile.available { - Vec::new() - } else { - profile.notes - }, - }) -} - fn supported_acp_agents() -> Vec<&'static str> { - vec!["claude-code", "codex", "cursor"] + vec!["claude-code", "codex", "cursor", "grok"] } fn supported_agent_providers() -> Vec<&'static str> { @@ -256,6 +314,7 @@ fn canonical_acp_agent(agent: &str) -> Option<&'static str> { "claude-code" | "claude" => Some("claude-code"), "codex" | "codex-cli" | "openai-codex" => Some("codex"), "cursor" | "cursor-agent" => Some("cursor"), + "grok" | "grok-build" | "xai-grok" => Some("grok"), _ => None, } } @@ -332,93 +391,290 @@ pub(crate) fn built_in_acp_relay_command(provider: &str) -> Vec { } pub fn run_stdio_relay(options: AcpRelayOptions) -> Result<()> { - if options.upstream_command.is_empty() { - return Err(Error::InvalidInput( - "ACP relay requires an upstream command after `--`".to_string(), - )); - } + let observer = capture_observer(&options)?; + StdioRelay::new(observer).run(&options) +} - let mut child = Command::new(&options.upstream_command[0]) - .args(&options.upstream_command[1..]) - .envs(&options.upstream_env) - .current_dir(&options.workspace_root) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|err| { - Error::InvalidInput(format!( - "failed to launch upstream ACP agent `{}`: {err}", - options.upstream_command[0] - )) - })?; +fn capture_observer(options: &AcpRelayOptions) -> Result> { + let db = open_capture_db(options)?; + db.ensure_live_path_invariant_indexes()?; + let coordinator = Arc::new(Mutex::new(CaptureCoordinator::new_with_db( + options.clone(), + Some(db), + )?)); + let pipeline = TransformPipeline::new( + Arc::new(AcpV1Contract::load()?), + TransformOptions::from_relay(options), + ); + let connection_id = format!( + "conn_{}", + crate::ids::short_hash( + format!( + "{}:{}:{}", + std::process::id(), + acp_now_millis(), + options.workspace_root.display() + ) + .as_bytes(), + 24, + ) + ); + let ingress = CaptureIngress::new( + options.workspace_root.clone(), + options.db_dir.clone(), + Arc::clone(&coordinator), + connection_id.clone(), + )?; + Ok(Arc::new(CaptureObserver { + coordinator, + pipeline: Mutex::new(pipeline), + ingress, + connection_id, + connection_sequence: AtomicU64::new(0), + finish_reason: Mutex::new(None), + })) +} - let child_stdin = child - .stdin - .take() - .ok_or_else(|| Error::InvalidInput("failed to open upstream ACP stdin pipe".to_string()))?; - let child_stdout = child.stdout.take().ok_or_else(|| { - Error::InvalidInput("failed to open upstream ACP stdout pipe".to_string()) - })?; +#[doc(hidden)] +pub struct AcpRelayBenchmarkSample { + pub forwarded: Vec, + pub latency_micros: u128, + pub transformed: bool, +} - if let Some(stderr) = child.stderr.take() { - thread::spawn(move || { - let _ = copy_upstream_stderr(stderr); +/// Runs raw frames through the same transformation and capture observer used by +/// the stdio relay, without process or pipe overhead. This is intentionally +/// exposed only for the correctness-preserving relay benchmark. +#[doc(hidden)] +pub fn benchmark_acp_relay_frames( + options: AcpRelayOptions, + frames: Vec<(bool, Vec)>, +) -> Result> { + let observer = capture_observer(&options)?; + let mut samples = Vec::with_capacity(frames.len()); + for (agent_to_client, raw) in frames { + let direction = if agent_to_client { + Direction::AgentToClient + } else { + Direction::ClientToAgent + }; + let mut frame = Frame::parse(direction, raw).map_err(Error::Io)?; + let started = Instant::now(); + observer.observe(&mut frame)?; + let transformed = frame.forward_bytes() != frame.raw_bytes(); + samples.push(AcpRelayBenchmarkSample { + forwarded: frame.forward_bytes().to_vec(), + latency_micros: started.elapsed().as_micros(), + transformed, }); } + observer.finish(RelayFinishReason::EditorEof); + if !observer.flush(Duration::from_secs(120)) { + return Err(Error::InvalidInput( + "ACP benchmark capture did not settle before the finalization deadline".to_string(), + )); + } + drop(observer); + Ok(samples) +} - let coordinator = Arc::new(Mutex::new(CaptureCoordinator::new(options)?)); - let (done_tx, done_rx) = mpsc::channel(); +struct CaptureObserver { + coordinator: Arc>, + pipeline: Mutex, + ingress: CaptureIngress, + connection_id: String, + connection_sequence: AtomicU64, + finish_reason: Mutex>, +} - let editor_coordinator = Arc::clone(&coordinator); - let editor_done = done_tx.clone(); - let editor_handle = thread::spawn(move || { - let result = pump_editor_to_agent(io::stdin().lock(), child_stdin, editor_coordinator); - let _ = editor_done.send(PumpDone::Editor(result)); - }); +impl FrameObserver for CaptureObserver { + fn observe(&self, frame: &mut Frame) -> Result<()> { + let outcome = self + .pipeline + .lock() + .map_err(|_| Error::InvalidInput("ACP transform lock poisoned".to_string()))? + .apply(frame)?; + if let Some(diagnostic) = outcome.diagnostic_message() { + eprintln!("trail acp relay negotiation warning: {diagnostic}"); + } + if frame.direction() == Direction::AgentToClient + && frame + .value() + .pointer("/result/protocolVersion") + .and_then(Value::as_u64) + == Some(1) + { + if let Ok(mut coordinator) = self.coordinator.lock() { + coordinator.remember_initialize_selection(frame.value()); + } + } + if !outcome.capture_v1() { + if frame.direction() == Direction::ClientToAgent && frame.method() == Some("initialize") + { + if let Ok(mut coordinator) = self.coordinator.lock() { + coordinator.remember_initialize_request(frame.value()); + } + } + let sequence = self.connection_sequence.fetch_add(1, Ordering::Relaxed); + self.ingress.append(capture_frame( + &self.connection_id, + frame.direction(), + sequence, + frame.value(), + false, + ))?; + return Ok(()); + } - let agent_coordinator = Arc::clone(&coordinator); - let agent_handle = thread::spawn(move || { - let result = pump_agent_to_editor( - BufReader::new(child_stdout), - io::stdout(), - agent_coordinator, - ); - let _ = done_tx.send(PumpDone::Agent(result)); - }); + if frame.direction() == Direction::ClientToAgent && frame.method() == Some("session/prompt") + { + let mut candidate = frame.value().clone(); + let remapped = self + .coordinator + .lock() + .map_err(|_| { + Error::InvalidInput("ACP capture coordinator lock poisoned".to_string()) + })? + .remap_prompt_for_forwarding(&mut candidate); + if let Err(error) = remapped { + eprintln!("trail acp relay prompt mapping warning: {error}"); + } else if candidate != *frame.value() { + self.pipeline + .lock() + .map_err(|_| Error::InvalidInput("ACP transform lock poisoned".to_string()))? + .commit_candidate(frame, candidate)?; + } + } - let first = done_rx.recv().map_err(|err| { - Error::InvalidInput(format!("ACP relay pump failed before startup: {err}")) - })?; - match first { - PumpDone::Editor(result) => { - result.map_err(Error::Io)?; - let status = child.wait().map_err(Error::Io)?; - if let Ok(PumpDone::Agent(result)) = done_rx.recv() { - result.map_err(Error::Io)?; - } - let _ = agent_handle.join(); - let _ = editor_handle.join(); - if status.success() { - Ok(()) - } else { - Err(Error::InvalidInput(format!( - "upstream ACP agent exited with status {status}" - ))) + let inline_session_request = frame.direction() == Direction::ClientToAgent + && matches!( + frame.method(), + Some("session/new" | "session/load" | "session/resume") + ); + let inline_session_response = frame.direction() == Direction::AgentToClient + && self + .coordinator + .lock() + .map(|coordinator| coordinator.is_pending_session_response(frame.value())) + .unwrap_or(false); + let inline_client_callback = frame.direction() == Direction::AgentToClient + && frame.method().is_some_and(is_client_callback_method); + let inline_semantic = + inline_session_request || inline_session_response || inline_client_callback; + if inline_semantic { + let mut candidate = frame.value().clone(); + let captured = capture_step(&self.coordinator, |capture| match frame.direction() { + Direction::ClientToAgent => capture.before_client_message(&mut candidate), + Direction::AgentToClient => capture.before_agent_message(&mut candidate), + }); + if (inline_session_request || inline_client_callback) + && captured + && candidate != *frame.value() + { + if let Err(error) = self + .pipeline + .lock() + .map_err(|_| Error::InvalidInput("ACP transform lock poisoned".to_string()))? + .commit_candidate(frame, candidate) + { + eprintln!("trail acp relay transformation warning: {error}"); + } } } - PumpDone::Agent(result) => { - result.map_err(Error::Io)?; - let status = child.wait().map_err(Error::Io)?; - if status.success() { - Ok(()) - } else { - Err(Error::InvalidInput(format!( - "upstream ACP agent exited with status {status}" - ))) + let sequence = self.connection_sequence.fetch_add(1, Ordering::Relaxed); + self.ingress.append(capture_frame( + &self.connection_id, + frame.direction(), + sequence, + frame.value(), + !inline_semantic, + ))?; + Ok(()) + } + + fn finish(&self, reason: RelayFinishReason) { + if let Ok(mut finish_reason) = self.finish_reason.lock() { + *finish_reason = Some(reason.clone()); + } + self.ingress.finish(reason); + } + + fn flush(&self, timeout: Duration) -> bool { + let has_pending_client_callbacks = self + .coordinator + .lock() + .map(|coordinator| coordinator.has_pending_client_callbacks()) + .unwrap_or(true); + let timeout = if has_pending_client_callbacks { + timeout.max(ACP_CALLBACK_CAPTURE_FLUSH_TIMEOUT) + } else { + timeout + }; + let settled = self.ingress.flush(timeout); + if !settled { + return false; + } + let reason = self + .finish_reason + .lock() + .ok() + .and_then(|mut reason| reason.take()); + if let Some(reason) = reason { + return capture_step(&self.coordinator, |capture| { + capture.capture_callback_shutdown(&reason) + }); + } + true + } +} + +#[allow(clippy::result_large_err)] +fn confined_acp_command( + command: &[String], + workspace_root: &Path, + db_dir: &Path, + materialized: bool, +) -> Result<(OsString, Vec)> { + #[cfg(target_os = "macos")] + { + if materialized { + let sandbox = PathBuf::from("/usr/bin/sandbox-exec"); + if !sandbox.is_file() { + return Err(Error::InvalidInput( + "materialized ACP agents require `/usr/bin/sandbox-exec` on macOS".to_string(), + )); } + let workspace_root = workspace_root.canonicalize()?; + let db_dir = db_dir.canonicalize()?; + let profile = format!( + "(version 1)\n\ + (allow default)\n\ + (deny file-write* (subpath \"{}\"))\n\ + (allow file-write* (subpath \"{}\"))", + sandbox_profile_escape(&workspace_root), + sandbox_profile_escape(&db_dir), + ); + let mut args = vec![ + OsString::from("-p"), + OsString::from(profile), + OsString::from(&command[0]), + ]; + args.extend(command[1..].iter().map(OsString::from)); + return Ok((sandbox.into_os_string(), args)); } } + let _ = (workspace_root, db_dir, materialized); + Ok(( + OsString::from(&command[0]), + command[1..].iter().map(OsString::from).collect(), + )) +} + +#[cfg(target_os = "macos")] +fn sandbox_profile_escape(path: &Path) -> String { + path.to_string_lossy() + .replace('\\', "\\\\") + .replace('"', "\\\"") } pub(crate) fn command_in_path(command: &str) -> bool { @@ -431,107 +687,100 @@ pub(crate) fn command_in_path(command: &str) -> bool { env::split_paths(&path).any(|dir| dir.join(command).is_file()) } -fn acp_editor_snippet(editor: &str, agent: &str, relay_command: &[String]) -> String { - let command = shell_join(relay_command); - match editor { - "zed" => serde_json::to_string_pretty(&serde_json::json!({ - "agent_servers": { - (format!("trail-{agent}")): { - "type": "custom", - "command": relay_command.first().cloned().unwrap_or_default(), - "args": relay_command.iter().skip(1).cloned().collect::>() - } - } - })) - .unwrap_or_else(|_| "{}".to_string()), - _ => format!("ACP command:\n{command}"), - } -} - -fn shell_join(parts: &[String]) -> String { - parts - .iter() - .map(|part| { - if part.chars().all(|ch| { - ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '/' | '.' | '@' | ':') - }) { - part.clone() - } else { - format!("'{}'", part.replace('\'', "'\\''")) - } - }) - .collect::>() - .join(" ") -} - -enum PumpDone { - Editor(io::Result<()>), - Agent(io::Result<()>), -} - -fn pump_editor_to_agent( - mut reader: R, - mut writer: W, - coordinator: Arc>, -) -> io::Result<()> -where - R: BufRead, - W: Write, -{ - loop { - let mut message = match read_json_line(&mut reader) { - Ok(Some(message)) => message, - Ok(None) => break, - Err(err) => { - capture_step(&coordinator, |capture| { - capture.finish_open_turns("failed", "editor sent malformed JSON") - }); - return Err(err); - } - }; - capture_step(&coordinator, |capture| { - capture.before_client_message(&mut message) - }); - write_json_line(&mut writer, &message)?; +#[allow(clippy::too_many_arguments)] +fn record_acp_lifecycle_event( + db: &mut Trail, + lane: &str, + session_id: &str, + turn_id: Option<&str>, + provider: Option<&str>, + acp_session_id: &str, + kind: AgentLifecycleEventKind, + payload: Value, +) -> Result<()> { + let provider = provider.unwrap_or("acp-agent").to_ascii_lowercase(); + let payload = redact_json(payload); + let payload_bytes = serde_json::to_vec(&payload)?; + let event_id = format!( + "acp_event_{}", + crate::ids::short_hash( + format!( + "{}:{}:{}:{}:{}", + db.config().workspace.id.0, + session_id, + turn_id.unwrap_or("session"), + kind.wire_name(), + hex::encode(Sha256::digest(&payload_bytes)) + ) + .as_bytes(), + 24, + ) + ); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| i64::try_from(duration.as_millis()).unwrap_or(i64::MAX)) + .unwrap_or(0); + let event = AgentLifecycleEvent { + schema: AGENT_LIFECYCLE_EVENT_SCHEMA.to_string(), + version: AGENT_LIFECYCLE_EVENT_VERSION, + event_id: event_id.clone(), + event_type: AgentLifecycleEventType::from(kind), + occurred_at: Some(now), + received_at: now, + provider, + provider_version: None, + transport: AgentCaptureTransport::Acp, + workspace_id: db.config().workspace.id.0.clone(), + lane_id: Some(db.lane_branch(lane)?.lane_id), + capture_run_id: None, + native: AgentNativeEventIdentity { + session_id: Some(acp_session_id.to_string()), + turn_id: None, + message_id: payload + .get("message_id") + .and_then(Value::as_str) + .map(str::to_string), + tool_id: payload + .get("tool_id") + .and_then(Value::as_str) + .map(str::to_string), + subagent_id: None, + event_name: format!("acp/{}", kind.wire_name()), + sequence: None, + }, + correlation: AgentEventCorrelation { + trace_id: turn_id.map(acp_trace_id_for_turn), + ..AgentEventCorrelation::default() + }, + payload, + evidence: AgentEventEvidence { + receipt_id: format!("acp:{event_id}"), + raw_digest: Some(format!( + "sha256:{}", + hex::encode(Sha256::digest(payload_bytes)) + )), + transcript_offset: None, + confidence: AgentEvidenceConfidence::ProtocolStructured, + }, + }; + event.validate()?; + let serialized = serde_json::to_value(&event)?; + if let Some(turn_id) = turn_id { + db.add_lane_turn_event(turn_id, kind.wire_name(), Some(serialized), None, None)?; + } else { + db.add_lane_session_event(lane, session_id, kind.wire_name(), Some(serialized))?; } - capture_step(&coordinator, |capture| { - capture.finish_open_turns("cancelled", "editor input closed") - }); - writer.flush() + Ok(()) } -fn pump_agent_to_editor( - mut reader: R, - mut writer: W, - coordinator: Arc>, -) -> io::Result<()> -where - R: BufRead, - W: Write, -{ - loop { - let mut message = match read_json_line(&mut reader) { - Ok(Some(message)) => message, - Ok(None) => break, - Err(err) => { - capture_step(&coordinator, |capture| { - capture.finish_open_turns("failed", "upstream sent malformed JSON") - }); - return Err(err); - } - }; - capture_step(&coordinator, |capture| { - capture.before_agent_message(&mut message) - }); - write_json_line(&mut writer, &message)?; - } - capture_step(&coordinator, |capture| { - capture.finish_open_turns("failed", "upstream output closed") - }); - writer.flush() +fn acp_now_millis() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| i64::try_from(duration.as_millis()).unwrap_or(i64::MAX)) + .unwrap_or(0) } -fn capture_step(coordinator: &Arc>, f: F) +fn capture_step(coordinator: &Arc>, f: F) -> bool where F: FnOnce(&mut CaptureCoordinator) -> Result<()>, { @@ -540,52 +789,17 @@ where let result = Trail::with_write_lock_wait(ACP_CAPTURE_LOCK_WAIT, || f(&mut capture)); if let Err(err) = result { eprintln!("trail acp relay capture warning: {err}"); + return false; } + true } Err(_) => { eprintln!("trail acp relay capture warning: capture coordinator lock poisoned"); + false } } } -fn read_json_line(reader: &mut R) -> io::Result> { - loop { - let mut line = String::new(); - let bytes = reader.read_line(&mut line)?; - if bytes == 0 { - return Ok(None); - } - let line = line.trim_end_matches(['\r', '\n']); - if line.trim().is_empty() { - continue; - } - let value = serde_json::from_str(line) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; - return Ok(Some(value)); - } -} - -fn write_json_line(writer: &mut W, value: &Value) -> io::Result<()> { - serde_json::to_writer(&mut *writer, value) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; - writer.write_all(b"\n")?; - writer.flush() -} - -fn copy_upstream_stderr(reader: R) -> io::Result<()> { - let mut reader = BufReader::new(reader); - let mut buf = [0u8; 8192]; - loop { - let bytes = reader.read(&mut buf)?; - if bytes == 0 { - return Ok(()); - } - let mut stderr = io::stderr().lock(); - stderr.write_all(&buf[..bytes])?; - stderr.flush()?; - } -} - #[derive(Clone, Debug)] struct SessionState { acp_session_id: String, @@ -594,6 +808,8 @@ struct SessionState { trail_session_id: String, original_cwd: String, effective_cwd: String, + materialized_root: Option, + path_mappings: Vec, materialized: bool, } @@ -601,6 +817,7 @@ struct SessionState { struct PendingSession { method: String, session: SessionState, + mapping_existed: bool, } #[derive(Clone, Debug)] @@ -635,6 +852,14 @@ struct ActiveTurn { capture_truncated: bool, } +#[derive(Debug)] +struct TurnTerminalOutcome<'a> { + status: &'a str, + stop_reason: Option, + error_summary: Option, + checkpoint_failed: bool, +} + #[derive(Clone, Debug)] struct BufferedTurnEvent { event_type: String, @@ -741,34 +966,285 @@ impl ActiveTurn { #[derive(Clone, Debug)] struct PendingPermission { approval_id: Option, - options_by_id: HashMap, + options_by_id: BTreeMap, + acp_session_id: String, +} + +#[derive(Clone, Debug)] +enum ClientCallbackOperation { + Permission(PendingPermission), + ReadFile { + acp_session_id: String, + effective_path: String, + forwarded_path: String, + line: Option, + limit: Option, + }, + WriteFile { + acp_session_id: String, + effective_path: String, + forwarded_path: String, + content_sha256: String, + byte_len: u64, + redacted_content: Vec, + }, + TerminalCreate { + acp_session_id: String, + command: Vec, + effective_cwd: Option, + forwarded_cwd: Option, + output_byte_limit: Option, + env_names: Vec, + }, + TerminalOutput { + acp_session_id: String, + terminal_id: String, + }, + TerminalWait { + acp_session_id: String, + terminal_id: String, + }, + TerminalKill { + acp_session_id: String, + terminal_id: String, + }, + TerminalRelease { + acp_session_id: String, + terminal_id: String, + }, +} + +impl ClientCallbackOperation { + fn acp_session_id(&self) -> &str { + match self { + Self::Permission(permission) => &permission.acp_session_id, + Self::ReadFile { acp_session_id, .. } + | Self::WriteFile { acp_session_id, .. } + | Self::TerminalCreate { acp_session_id, .. } + | Self::TerminalOutput { acp_session_id, .. } + | Self::TerminalWait { acp_session_id, .. } + | Self::TerminalKill { acp_session_id, .. } + | Self::TerminalRelease { acp_session_id, .. } => acp_session_id, + } + } + + fn method(&self) -> &'static str { + match self { + Self::Permission(_) => "session/request_permission", + Self::ReadFile { .. } => "fs/read_text_file", + Self::WriteFile { .. } => "fs/write_text_file", + Self::TerminalCreate { .. } => "terminal/create", + Self::TerminalOutput { .. } => "terminal/output", + Self::TerminalWait { .. } => "terminal/wait_for_exit", + Self::TerminalKill { .. } => "terminal/kill", + Self::TerminalRelease { .. } => "terminal/release", + } + } +} + +#[derive(Clone, Debug)] +struct CapturedTerminal { + acp_session_id: String, + state: String, +} + +#[derive(Clone, Debug, Default)] +struct ReplayAccumulator { + updates: Vec<(String, Map)>, +} + +#[derive(Clone, Debug, Default)] +struct ReplayTurn { + user_message_id: Option, + user_text: String, + assistant_message_id: Option, + assistant_text: String, + events: Vec<(String, Value)>, +} + +impl ReplayTurn { + fn has_content(&self) -> bool { + !self.user_text.is_empty() || !self.assistant_text.is_empty() || !self.events.is_empty() + } +} + +#[derive(Clone, Debug)] +enum PendingOperation { + Session(PendingSession), + Prompt(PendingPrompt), + Close { + acp_session_id: String, + }, + ClientCallback(ClientCallbackOperation), + Initialize { + requested_version: Value, + }, + Authenticate { + method_id: String, + }, + Logout, + SessionList { + cursor: Option, + }, + SessionDelete { + acp_session_id: String, + }, + SetMode { + acp_session_id: String, + mode_id: String, + }, + SetConfig { + acp_session_id: String, + config_id: String, + value: Value, + }, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AcpV1SessionUpdateKind { + UserMessageChunk, + AgentMessageChunk, + AgentThoughtChunk, + ToolCall, + ToolCallUpdate, + Plan, + AvailableCommandsUpdate, + CurrentModeUpdate, + ConfigOptionUpdate, + SessionInfoUpdate, + UsageUpdate, + Extension, +} + +impl AcpV1SessionUpdateKind { + #[cfg(test)] + const ALL: [Self; 11] = [ + Self::UserMessageChunk, + Self::AgentMessageChunk, + Self::AgentThoughtChunk, + Self::ToolCall, + Self::ToolCallUpdate, + Self::Plan, + Self::AvailableCommandsUpdate, + Self::CurrentModeUpdate, + Self::ConfigOptionUpdate, + Self::SessionInfoUpdate, + Self::UsageUpdate, + ]; + + fn parse(value: &str) -> Self { + match value { + "user_message_chunk" => Self::UserMessageChunk, + "agent_message_chunk" => Self::AgentMessageChunk, + "agent_thought_chunk" => Self::AgentThoughtChunk, + "tool_call" => Self::ToolCall, + "tool_call_update" => Self::ToolCallUpdate, + "plan" => Self::Plan, + "available_commands_update" => Self::AvailableCommandsUpdate, + "current_mode_update" => Self::CurrentModeUpdate, + "config_option_update" => Self::ConfigOptionUpdate, + "session_info_update" => Self::SessionInfoUpdate, + "usage_update" => Self::UsageUpdate, + _ => Self::Extension, + } + } + + #[cfg(test)] + fn as_str(self) -> &'static str { + match self { + Self::UserMessageChunk => "user_message_chunk", + Self::AgentMessageChunk => "agent_message_chunk", + Self::AgentThoughtChunk => "agent_thought_chunk", + Self::ToolCall => "tool_call", + Self::ToolCallUpdate => "tool_call_update", + Self::Plan => "plan", + Self::AvailableCommandsUpdate => "available_commands_update", + Self::CurrentModeUpdate => "current_mode_update", + Self::ConfigOptionUpdate => "config_option_update", + Self::SessionInfoUpdate => "session_info_update", + Self::UsageUpdate => "usage_update", + Self::Extension => "extension", + } + } + + fn is_stable(self) -> bool { + self != Self::Extension + } } struct CaptureCoordinator { options: AcpRelayOptions, - pending_initialize: HashSet, - pending_sessions: HashMap, - pending_prompts: HashMap, - pending_closes: HashMap, - pending_permissions: HashMap, + db_pool: Arc>>, + pending_operations: HashMap<(Direction, String), PendingOperation>, + replay_by_session: HashMap, + pending_connection_events: Vec, + cancelled_requests: HashSet<(Direction, String)>, sessions_by_acp: HashMap, active_turns: HashMap, + terminals: HashMap, upstream_command_json: Option, } +struct PooledTrail { + db: Option, + pool: Arc>>, +} + +impl std::ops::Deref for PooledTrail { + type Target = Trail; + + fn deref(&self) -> &Self::Target { + self.db.as_ref().expect("pooled Trail handle is present") + } +} + +impl std::ops::DerefMut for PooledTrail { + fn deref_mut(&mut self) -> &mut Self::Target { + self.db.as_mut().expect("pooled Trail handle is present") + } +} + +impl Drop for PooledTrail { + fn drop(&mut self) { + if let Some(db) = self.db.take() { + self.pool + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(db); + } + } +} + impl CaptureCoordinator { + fn has_pending_client_callbacks(&self) -> bool { + self.pending_operations + .values() + .any(|operation| matches!(operation, PendingOperation::ClientCallback(_))) + } + + #[cfg(test)] fn new(options: AcpRelayOptions) -> Result { + let db = if options.db_dir.is_dir() { + Some(open_capture_db(&options)?) + } else { + None + }; + Self::new_with_db(options, db) + } + + fn new_with_db(options: AcpRelayOptions, db: Option) -> Result { let upstream_command_json = serde_json::to_string(&redact_command(&options.upstream_command)).ok(); Ok(Self { options, - pending_initialize: HashSet::new(), - pending_sessions: HashMap::new(), - pending_prompts: HashMap::new(), - pending_closes: HashMap::new(), - pending_permissions: HashMap::new(), + db_pool: Arc::new(Mutex::new(db.into_iter().collect())), + pending_operations: HashMap::new(), + replay_by_session: HashMap::new(), + pending_connection_events: Vec::new(), + cancelled_requests: HashSet::new(), sessions_by_acp: HashMap::new(), active_turns: HashMap::new(), + terminals: HashMap::new(), upstream_command_json, }) } @@ -781,9 +1257,28 @@ impl CaptureCoordinator { match method_name(message) { Some("initialize") => { - if let Some(id) = rpc_id_key(message) { - self.pending_initialize.insert(id); - } + let requested_version = message + .pointer("/params/protocolVersion") + .cloned() + .unwrap_or(Value::Null); + self.register_client_operation( + message, + PendingOperation::Initialize { requested_version }, + )?; + } + Some("authenticate") => { + let method_id = message + .pointer("/params/methodId") + .and_then(Value::as_str) + .unwrap_or("unknown") + .to_string(); + self.register_client_operation( + message, + PendingOperation::Authenticate { method_id }, + )?; + } + Some("logout") => { + self.register_client_operation(message, PendingOperation::Logout)?; } Some("session/new") | Some("session/load") | Some("session/resume") => { self.prepare_session_request(message)?; @@ -794,96 +1289,266 @@ impl CaptureCoordinator { Some("session/cancel") => { self.capture_cancel(message)?; } + Some("$/cancel_request") => { + self.capture_rpc_cancel(message, Direction::ClientToAgent)?; + } + Some("session/set_mode") => { + let params = params_object(message)?; + let acp_session_id = required_string(params, "sessionId", "session/set_mode")?; + let mode_id = required_string(params, "modeId", "session/set_mode")?; + self.register_client_operation( + message, + PendingOperation::SetMode { + acp_session_id, + mode_id, + }, + )?; + } + Some("session/set_config_option") => { + let params = params_object(message)?; + let acp_session_id = + required_string(params, "sessionId", "session/set_config_option")?; + let config_id = required_string(params, "configId", "session/set_config_option")?; + let value = params.get("value").cloned().unwrap_or(Value::Null); + self.register_client_operation( + message, + PendingOperation::SetConfig { + acp_session_id, + config_id, + value, + }, + )?; + } Some("session/close") => { self.prepare_close_request(message)?; } + Some("session/list") => { + let cursor = message + .pointer("/params/cursor") + .and_then(Value::as_str) + .map(str::to_string); + self.register_client_operation(message, PendingOperation::SessionList { cursor })?; + } + Some("session/delete") => { + let acp_session_id = message + .pointer("/params/sessionId") + .and_then(Value::as_str) + .ok_or_else(|| { + Error::InvalidInput("ACP session/delete missing sessionId".to_string()) + })? + .to_string(); + self.register_client_operation( + message, + PendingOperation::SessionDelete { acp_session_id }, + )?; + } _ => {} } Ok(()) } - fn before_agent_message(&mut self, message: &mut Value) -> Result<()> { - if method_name(message).is_none() { - self.capture_agent_response(message)?; + fn remap_prompt_for_forwarding(&self, message: &mut Value) -> Result<()> { + let params = params_object_mut(message)?; + let Some(acp_session_id) = params.get("sessionId").and_then(Value::as_str) else { return Ok(()); - } - - match method_name(message) { - Some("session/update") => self.capture_session_update(message)?, - Some("session/request_permission") => self.capture_permission_request(message)?, - _ => {} + }; + let Some(session) = self.sessions_by_acp.get(acp_session_id) else { + return Ok(()); + }; + if session.materialized { + remap_prompt_resource_uris( + params.get_mut("prompt"), + Path::new(&session.original_cwd), + Path::new(&session.effective_cwd), + ); } Ok(()) } - fn capture_client_response(&mut self, message: &Value) -> Result<()> { + fn remember_initialize_request(&mut self, message: &Value) { let Some(id) = rpc_id_key(message) else { - return Ok(()); + return; }; - let Some(permission) = self.pending_permissions.remove(&id) else { - return Ok(()); + let key = (Direction::ClientToAgent, id); + self.pending_operations + .entry(key) + .or_insert_with(|| PendingOperation::Initialize { + requested_version: message + .pointer("/params/protocolVersion") + .cloned() + .unwrap_or(Value::Null), + }); + } + + fn remember_initialize_selection(&mut self, message: &Value) { + let Some(id) = rpc_id_key(message) else { + return; }; - let Some(approval_id) = permission.approval_id else { - return Ok(()); + let requested_version = match self + .pending_operations + .remove(&(Direction::ClientToAgent, id)) + { + Some(PendingOperation::Initialize { requested_version }) => requested_version, + _ => Value::from(1), }; - let decision = permission_decision(message, &permission.options_by_id); - let mut db = self.open_db()?; - db.decide_lane_approval( - &approval_id, - decision, - Some("acp-editor".to_string()), - Some("mirrored from ACP permission response".to_string()), - )?; - Ok(()) + self.pending_connection_events + .push(redact_json(serde_json::json!({ + "protocol": "acp", + "method": "initialize", + "request_id": message.get("id"), + "outcome": "success", + "result": message.get("result"), + "error": Value::Null, + "error_code": Value::Null, + "context": {"requested_version": requested_version} + }))); } - fn capture_agent_response(&mut self, message: &mut Value) -> Result<()> { + fn register_client_operation( + &mut self, + message: &Value, + operation: PendingOperation, + ) -> Result<()> { let Some(id) = rpc_id_key(message) else { return Ok(()); }; + self.register_pending(Direction::ClientToAgent, id, operation) + } - if self.pending_initialize.remove(&id) { - self.add_initialize_metadata(message); - return Ok(()); - } + fn is_pending_session_response(&self, message: &Value) -> bool { + let Some(id) = rpc_id_key(message) else { + return false; + }; + matches!( + self.pending_operations.get(&(Direction::ClientToAgent, id)), + Some(PendingOperation::Session(_)) + ) + } - if let Some(pending) = self.pending_sessions.remove(&id) { - self.finish_session_request(message, pending)?; - return Ok(()); + fn register_pending( + &mut self, + direction: Direction, + id: String, + operation: PendingOperation, + ) -> Result<()> { + let key = (direction, id.clone()); + if self.pending_operations.contains_key(&key) { + return Err(Error::InvalidInput(format!( + "ACP peer reused in-flight request id `{id}` in {direction:?} direction" + ))); } + self.pending_operations.insert(key, operation); + Ok(()) + } - if let Some(pending) = self.pending_prompts.remove(&id) { - self.finish_prompt_request(message, pending)?; + fn before_agent_message(&mut self, message: &mut Value) -> Result<()> { + if method_name(message).is_none() { + self.capture_agent_response(message)?; return Ok(()); } - if let Some(acp_session_id) = self.pending_closes.remove(&id) { - self.finish_close_request(message, &acp_session_id)?; + match method_name(message) { + Some("session/update") => self.capture_session_update(message)?, + Some("session/request_permission") => self.capture_permission_request(message)?, + Some( + "fs/read_text_file" + | "fs/write_text_file" + | "terminal/create" + | "terminal/output" + | "terminal/wait_for_exit" + | "terminal/kill" + | "terminal/release", + ) => self.capture_client_callback_request(message)?, + Some("$/cancel_request") => { + self.capture_rpc_cancel(message, Direction::AgentToClient)?; + } + _ => {} } - Ok(()) } - fn add_initialize_metadata(&self, message: &mut Value) { - let Some(result) = message.get_mut("result").and_then(Value::as_object_mut) else { - return; + fn capture_client_response(&mut self, message: &Value) -> Result<()> { + let Some(id) = rpc_id_key(message) else { + return Ok(()); }; - let meta = ensure_object_field(result, "_meta"); - meta.insert( - "trail".to_string(), - serde_json::json!({ - "relay": true, - "capture": true, - "workspace": self.options.workspace_root.to_string_lossy(), - "dbDir": self.options.db_dir.to_string_lossy(), - "provider": self.options.provider, - "model": self.options.model - }), - ); + let key = (Direction::AgentToClient, id.clone()); + let was_cancelled = self.cancelled_requests.remove(&key); + let Some(PendingOperation::ClientCallback(operation)) = + self.pending_operations.remove(&key) + else { + return Ok(()); + }; + self.finish_client_callback(message, &id, operation, was_cancelled) + } + + fn capture_agent_response(&mut self, message: &mut Value) -> Result<()> { + let Some(id) = rpc_id_key(message) else { + return Ok(()); + }; + let key = (Direction::ClientToAgent, id); + let was_cancelled = self.cancelled_requests.remove(&key); + let Some(operation) = self.pending_operations.remove(&key) else { + return Ok(()); + }; + match operation { + PendingOperation::Session(pending) => self.finish_session_request(message, pending), + PendingOperation::Prompt(pending) => { + self.finish_prompt_request(message, pending, was_cancelled) + } + PendingOperation::Close { acp_session_id } => { + self.finish_close_request(message, &acp_session_id) + } + PendingOperation::SessionDelete { acp_session_id } => { + self.finish_delete_request(message, &acp_session_id) + } + PendingOperation::Initialize { requested_version } => self + .capture_connection_lifecycle( + "initialize", + message, + Some(serde_json::json!({"requested_version": requested_version})), + ), + PendingOperation::Authenticate { method_id } => self.capture_connection_lifecycle( + "authenticate", + message, + Some(serde_json::json!({"method_id": method_id})), + ), + PendingOperation::Logout => self.capture_connection_lifecycle("logout", message, None), + PendingOperation::SessionList { cursor } => self.capture_connection_lifecycle( + "session/list", + message, + Some(serde_json::json!({ + "request_cursor": cursor, + "cancel_requested": was_cancelled + })), + ), + PendingOperation::SetMode { + acp_session_id, + mode_id, + } => self.finish_configuration_request( + message, + &acp_session_id, + Some(&mode_id), + None, + was_cancelled, + ), + PendingOperation::SetConfig { + acp_session_id, + config_id, + value, + } => self.finish_configuration_request( + message, + &acp_session_id, + None, + Some((&config_id, &value)), + was_cancelled, + ), + PendingOperation::ClientCallback(_) => Ok(()), + } } fn prepare_session_request(&mut self, message: &mut Value) -> Result<()> { let method = method_name(message).unwrap_or_default().to_string(); + let is_load = method == "session/load"; let request_id = rpc_id_key(message); let params = params_object_mut(message)?; let original_cwd = params @@ -895,32 +1560,58 @@ impl CaptureCoordinator { .get("sessionId") .and_then(Value::as_str) .map(str::to_string); - let session = self.ensure_capture_session( + let mapping_existed = if let Some(acp_session_id) = requested_acp_session_id.as_deref() { + self.open_db()? + .try_lane_acp_session(acp_session_id)? + .is_some() + } else { + false + }; + let mut session = self.ensure_capture_session( &method, requested_acp_session_id.as_deref(), &original_cwd, )?; - if session.effective_cwd != original_cwd { - params.insert( - "cwd".to_string(), - Value::String(session.effective_cwd.clone()), - ); + if let Some(materialized_root) = &session.materialized_root { + let mapper = WorkspaceMapper::new( + self.options.workspace_root.clone(), + PathBuf::from(materialized_root), + )?; + let mappings = mapper.map_session_params(params)?; + session.effective_cwd = params + .get("cwd") + .and_then(Value::as_str) + .unwrap_or(&original_cwd) + .to_string(); + session.path_mappings = mappings.into_iter().map(acp_path_mapping).collect(); + } else { + session.path_mappings = passthrough_session_mappings(params)? + .into_iter() + .map(acp_path_mapping) + .collect(); } if self.options.inject_mcp { inject_trail_mcp_server(params, &self.options)?; } if let Some(request_id) = request_id { - self.pending_sessions.insert( + self.register_pending( + Direction::ClientToAgent, request_id, - PendingSession { + PendingOperation::Session(PendingSession { method, session: session.clone(), - }, - ); + mapping_existed, + }), + )?; } if let Some(acp_session_id) = requested_acp_session_id { + if is_load { + self.replay_by_session + .entry(acp_session_id.clone()) + .or_default(); + } self.sessions_by_acp.insert(acp_session_id, session); } Ok(()) @@ -939,16 +1630,18 @@ impl CaptureCoordinator { let db = self.open_db()?; if let Some(mapping) = db.try_lane_acp_session(acp_session_id)? { let lane_name = db.resolve_lane_handle(&mapping.lane_id)?; - let effective_cwd = - self.materialized_cwd_for_existing_lane(&lane_name, original_cwd)?; + let materialized_root = self.materialized_root_for_existing_lane(&lane_name)?; + let effective_cwd = original_cwd.to_string(); let state = SessionState { acp_session_id: acp_session_id.to_string(), upstream_session_id: mapping.upstream_session_id, lane_name, trail_session_id: mapping.trail_session_id, original_cwd: original_cwd.to_string(), - materialized: effective_cwd != original_cwd, effective_cwd, + materialized: materialized_root.is_some(), + materialized_root, + path_mappings: mapping.path_mappings, }; return Ok(state); } @@ -976,15 +1669,23 @@ impl CaptureCoordinator { } let details = db.lane_details(&lane_name)?; - let effective_cwd = if self.options.materialize { - details - .branch - .workdir - .clone() - .unwrap_or_else(|| original_cwd.to_string()) + let materialized_root = if self.options.materialize { + details.branch.workdir.clone() } else { - original_cwd.to_string() + None }; + let initial_mapping = if let Some(root) = &materialized_root { + WorkspaceMapper::new(self.options.workspace_root.clone(), PathBuf::from(root))? + .map(PathBuf::from(original_cwd).as_path())? + } else { + PathMapping { + original: PathBuf::from(original_cwd), + effective: PathBuf::from(original_cwd), + isolated: false, + } + }; + let effective_cwd = initial_mapping.effective.to_string_lossy().to_string(); + let path_mappings = vec![acp_path_mapping(initial_mapping)]; let title = Some(format!("ACP {method}")); let session = db.start_lane_session(&lane_name, title, None)?.session; db.add_lane_session_event( @@ -997,12 +1698,12 @@ impl CaptureCoordinator { "requested_acp_session_id": acp_session_id, "cwd": original_cwd, "effective_cwd": effective_cwd, + "path_mappings": path_mappings, "provider": self.options.provider, "model": self.options.model, "materialized": self.options.materialize }))), )?; - Ok(SessionState { acp_session_id: acp_session_id.map(str::to_string).unwrap_or_else(|| { format!( @@ -1015,33 +1716,48 @@ impl CaptureCoordinator { trail_session_id: session.session_id, original_cwd: original_cwd.to_string(), effective_cwd, - materialized: self.options.materialize && details.branch.workdir.is_some(), + materialized: materialized_root.is_some(), + materialized_root, + path_mappings, }) } - fn materialized_cwd_for_existing_lane( - &self, - lane_name: &str, - original_cwd: &str, - ) -> Result { + fn materialized_root_for_existing_lane(&self, lane_name: &str) -> Result> { if !self.options.materialize { - return Ok(original_cwd.to_string()); + return Ok(None); } let mut db = self.open_db()?; let report = db.ensure_lane_workdir_materialized(lane_name, self.options.workdir.clone())?; - Ok(report.workdir.unwrap_or_else(|| original_cwd.to_string())) + Ok(report.workdir) } fn finish_session_request(&mut self, message: &Value, pending: PendingSession) -> Result<()> { - let status = if message.get("error").is_some() { - "failed" - } else { - match pending.method.as_str() { - "session/load" => "loaded", - "session/resume" => "resumed", - _ => "active", + if message.get("error").is_some() { + if pending.method == "session/load" { + self.replay_by_session + .remove(&pending.session.acp_session_id); + } + let mut db = self.open_db()?; + db.add_lane_session_event( + &pending.session.lane_name, + &pending.session.trail_session_id, + "acp_session_request_failed", + Some(redact_json(serde_json::json!({ + "protocol": "acp", + "method": pending.method, + "error": message.get("error") + }))), + )?; + if !pending.mapping_existed { + let _ = db.end_lane_session(&pending.session.trail_session_id, "failed"); } + return Ok(()); + } + let status = match pending.method.as_str() { + "session/load" => "loaded", + "session/resume" => "resumed", + _ => "active", }; let acp_session_id = response_session_id(message) .or_else(|| { @@ -1061,11 +1777,25 @@ impl CaptureCoordinator { &session.lane_name, &session.trail_session_id, &session.effective_cwd, + &session.path_mappings, self.options.provider.as_deref(), self.options.model.as_deref(), self.upstream_command_json.as_deref(), status, )?; + let initial_mode = message + .pointer("/result/modes/currentModeId") + .and_then(Value::as_str); + if let Some(mode_id) = initial_mode { + db.update_lane_acp_session_configuration(&acp_session_id, Some(mode_id), None)?; + } + let initial_config = message + .pointer("/result/configOptions") + .and_then(Value::as_array) + .map(|options| session_config_values_from_options(options)); + if let Some(config) = initial_config.as_ref() { + db.replace_lane_acp_session_configuration_options(&acp_session_id, config)?; + } db.add_lane_session_event( &session.lane_name, &session.trail_session_id, @@ -1077,36 +1807,77 @@ impl CaptureCoordinator { "upstream_session_id": response_session_id(message), "cwd": session.original_cwd, "effective_cwd": session.effective_cwd, + "path_mappings": session.path_mappings, "status": status }))), )?; - if status == "failed" { - let _ = db.end_lane_session(&session.trail_session_id, "failed"); + if initial_mode.is_some() || initial_config.is_some() { + db.add_lane_session_event( + &session.lane_name, + &session.trail_session_id, + "acp_session_configuration", + Some(redact_json(serde_json::json!({ + "protocol": "acp", + "acp_session_id": acp_session_id, + "modes": message.pointer("/result/modes"), + "config_options": message.pointer("/result/configOptions") + }))), + )?; } + for payload in std::mem::take(&mut self.pending_connection_events) { + db.add_lane_session_event( + &session.lane_name, + &session.trail_session_id, + "acp_connection_lifecycle", + Some(payload), + )?; + } + record_acp_lifecycle_event( + &mut db, + &session.lane_name, + &session.trail_session_id, + None, + self.options.provider.as_deref(), + &acp_session_id, + if status == "resumed" || status == "loaded" { + AgentLifecycleEventKind::SessionResumed + } else { + AgentLifecycleEventKind::SessionStarted + }, + serde_json::json!({"method": pending.method, "status": status}), + )?; self.sessions_by_acp.insert(acp_session_id, session); Ok(()) } - fn prepare_prompt_request(&mut self, message: &Value) -> Result<()> { + fn prepare_prompt_request(&mut self, message: &mut Value) -> Result<()> { let Some(request_id) = rpc_id_key(message) else { return Ok(()); }; - let params = params_object(message)?; + let request_id_value = message.get("id").cloned(); + let params = params_object_mut(message)?; let acp_session_id = params .get("sessionId") .and_then(Value::as_str) - .ok_or_else(|| { - Error::InvalidInput("ACP session/prompt missing sessionId".to_string()) - })?; - let session = self.resolve_session_state(acp_session_id)?; + .ok_or_else(|| Error::InvalidInput("ACP session/prompt missing sessionId".to_string()))? + .to_string(); + self.flush_load_replay(&acp_session_id)?; + let session = self.resolve_session_state(&acp_session_id)?; let prompt_text = prompt_text(params.get("prompt")); + if session.materialized { + remap_prompt_resource_uris( + params.get_mut("prompt"), + Path::new(&session.original_cwd), + Path::new(&session.effective_cwd), + ); + } let mut db = self.open_db()?; let branch = db.lane_branch(&session.lane_name)?; let initial_envelope = TurnEnvelope::new_acp_prompt(TurnEnvelopeAcpPromptInput { provider: self.options.provider.clone(), model: self.options.model.clone(), trail_session_id: session.trail_session_id.clone(), - acp_session_id: acp_session_id.to_string(), + acp_session_id: acp_session_id.clone(), upstream_session_id: session.upstream_session_id.clone(), upstream_command_hash: self.upstream_command_hash(), prompt_hash: prompt_hash(&prompt_text), @@ -1137,13 +1908,48 @@ impl CaptureCoordinator { "acp_prompt_started", Some(redact_json(serde_json::json!({ "protocol": "acp", - "acp_session_id": acp_session_id, - "request_id": message.get("id").cloned(), + "acp_session_id": &acp_session_id, + "request_id": request_id_value, "prompt_summary": summarize_text(&prompt_text), }))), None, None, )?; + db.add_lane_turn_event( + &turn.turn.turn_id, + "acp_prompt_content", + Some(redact_json(serde_json::json!({ + "protocol": "acp", + "acp_session_id": acp_session_id, + "readable_text": prompt_text, + "blocks": bounded_prompt_content(params.get("prompt")) + }))), + None, + Some(&user_message.message_id.0), + )?; + record_acp_lifecycle_event( + &mut db, + &session.lane_name, + &session.trail_session_id, + Some(&turn.turn.turn_id), + self.options.provider.as_deref(), + &acp_session_id, + AgentLifecycleEventKind::TurnStarted, + serde_json::json!({"prompt_summary": summarize_text(&prompt_text)}), + )?; + record_acp_lifecycle_event( + &mut db, + &session.lane_name, + &session.trail_session_id, + Some(&turn.turn.turn_id), + self.options.provider.as_deref(), + &acp_session_id, + AgentLifecycleEventKind::MessageUser, + serde_json::json!({ + "message_id": user_message.message_id, + "prompt_hash": prompt_hash(&prompt_text), + }), + )?; let root_span = db .start_lane_trace_span( &turn.turn.turn_id, @@ -1153,7 +1959,7 @@ impl CaptureCoordinator { None, Some(redact_json(serde_json::json!({ "protocol": "acp", - "acp_session_id": acp_session_id, + "acp_session_id": &acp_session_id, "provider": self.options.provider, "model": self.options.model }))), @@ -1162,15 +1968,19 @@ impl CaptureCoordinator { .map(|report| report.span.span_id); let pending = PendingPrompt { - acp_session_id: acp_session_id.to_string(), + acp_session_id: acp_session_id.clone(), lane_name: session.lane_name.clone(), turn_id: turn.turn.turn_id.clone(), root_span_id: root_span.clone(), materialized: session.materialized, }; - self.pending_prompts.insert(request_id, pending.clone()); + self.register_pending( + Direction::ClientToAgent, + request_id, + PendingOperation::Prompt(pending.clone()), + )?; self.active_turns.insert( - acp_session_id.to_string(), + acp_session_id, ActiveTurn { lane_name: pending.lane_name, trail_session_id: session.trail_session_id, @@ -1196,7 +2006,12 @@ impl CaptureCoordinator { Ok(()) } - fn finish_prompt_request(&mut self, message: &Value, pending: PendingPrompt) -> Result<()> { + fn finish_prompt_request( + &mut self, + message: &Value, + pending: PendingPrompt, + cancel_requested: bool, + ) -> Result<()> { let status = prompt_status(message); let mut active = self.active_turns.remove(&pending.acp_session_id); let mut db = self.open_db()?; @@ -1208,47 +2023,127 @@ impl CaptureCoordinator { self.flush_turn_events(active_turn)?; self.flush_assistant_messages(active_turn, "prompt_completed")?; } - if pending.materialized { - let _ = db.record_lane_workdir_for_turn( + let checkpoint_error = if pending.materialized { + self.record_prompt_workdir_checkpoint( + &mut db, &pending.lane_name, &pending.turn_id, - Some("ACP prompt workdir checkpoint".to_string()), - ); - } + "ACP prompt workdir checkpoint".to_string(), + )? + } else { + None + }; + let checkpoint_failed = checkpoint_error.is_some(); + let final_status = if checkpoint_failed { "failed" } else { status }; + let final_error_summary = match (error_summary(message), checkpoint_error.clone()) { + (Some(upstream), Some(checkpoint)) => Some(format!("{upstream}; {checkpoint}")), + (Some(upstream), None) => Some(upstream), + (None, Some(checkpoint)) => Some(checkpoint), + (None, None) => None, + }; db.add_lane_turn_event( &pending.turn_id, "acp_prompt_finished", Some(redact_json(serde_json::json!({ "protocol": "acp", "acp_session_id": pending.acp_session_id, - "status": status, + "status": final_status, + "upstream_status": status, + "cancel_requested": cancel_requested, "stop_reason": stop_reason(message), - "error": message.get("error").cloned() + "error": message.get("error").cloned(), + "checkpoint_error": checkpoint_error }))), None, None, )?; + let terminal_kind = match final_status { + "failed" => AgentLifecycleEventKind::TurnFailed, + "cancelled" | "interrupted" => AgentLifecycleEventKind::TurnCancelled, + _ => AgentLifecycleEventKind::TurnCompleted, + }; + let session_id = db + .lane_turn(&pending.turn_id)? + .session_id + .ok_or_else(|| Error::Corrupt("ACP turn lost its session identity".to_string()))?; + record_acp_lifecycle_event( + &mut db, + &pending.lane_name, + &session_id, + Some(&pending.turn_id), + self.options.provider.as_deref(), + &pending.acp_session_id, + terminal_kind, + serde_json::json!({ + "status": final_status, + "upstream_status": status, + "stop_reason": stop_reason(message), + "checkpoint_error": checkpoint_error + }), + )?; if let Some(span_id) = pending.root_span_id { let _ = db.end_lane_trace_span( &span_id, - status, + final_status, Some(redact_json(serde_json::json!({ "stop_reason": stop_reason(message) }))), ); } - db.end_lane_turn(&pending.turn_id, status)?; + db.end_lane_turn(&pending.turn_id, final_status)?; + db.create_turn_evidence_manifest(&pending.turn_id)?; + db.classify_session_activity(&session_id, 10_000)?; self.finalize_turn_envelope( &mut db, &pending.turn_id, - status, - stop_reason(message).map(str::to_string), - error_summary(message), + TurnTerminalOutcome { + status: final_status, + stop_reason: stop_reason(message).map(str::to_string), + error_summary: final_error_summary, + checkpoint_failed, + }, active.as_ref(), )?; Ok(()) } + fn record_prompt_workdir_checkpoint( + &self, + db: &mut Trail, + lane: &str, + turn_id: &str, + message: String, + ) -> Result> { + match db.record_lane_workdir_for_turn(lane, turn_id, Some(message)) { + Ok(_) => Ok(None), + Err(error) => { + let error_code = error.code(); + let error_message = error.to_string(); + let summary = + format!("Trail ACP workdir checkpoint failed ({error_code}): {error_message}"); + eprintln!("trail acp relay checkpoint error: {summary}"); + db.add_lane_turn_event( + turn_id, + "acp_workdir_checkpoint_failed", + Some(redact_json(serde_json::json!({ + "protocol": "acp", + "lane": lane, + "error_code": error_code, + "error": error_message, + "recovery": if error_code == "PATH_INDEX_REQUIRED" { + Some("trail index rebuild") + } else { + None + } + }))), + None, + None, + )?; + Ok(Some(summary)) + } + } + } + fn capture_session_update(&mut self, message: &Value) -> Result<()> { let params = params_object(message)?; let Some(acp_session_id) = params.get("sessionId").and_then(Value::as_str) else { @@ -1261,13 +2156,25 @@ impl CaptureCoordinator { .get("sessionUpdate") .and_then(Value::as_str) .unwrap_or("unknown"); + let variant = AcpV1SessionUpdateKind::parse(update_kind); + self.persist_session_update_state(acp_session_id, variant, update)?; + if let Some(replay) = self.replay_by_session.get_mut(acp_session_id) { + replay + .updates + .push((update_kind.to_string(), update.clone())); + return Ok(()); + } let Some(mut active) = self.active_turns.remove(acp_session_id) else { self.capture_session_update_without_turn(acp_session_id, update_kind, update)?; return Ok(()); }; - let result = match update_kind { - "agent_message_chunk" => { + active.push_event( + "acp_session_update", + Some(session_update_projection(variant, update_kind, update)), + ); + let result = match variant { + AcpV1SessionUpdateKind::AgentMessageChunk => { let key = update .get("messageId") .and_then(Value::as_str) @@ -1292,9 +2199,11 @@ impl CaptureCoordinator { } Ok(()) } - "tool_call" => self.capture_tool_call_start(&mut active, update), - "tool_call_update" => self.capture_tool_call_update(&mut active, update), - "plan" => { + AcpV1SessionUpdateKind::ToolCall => self.capture_tool_call_start(&mut active, update), + AcpV1SessionUpdateKind::ToolCallUpdate => { + self.capture_tool_call_update(&mut active, update) + } + AcpV1SessionUpdateKind::Plan => { self.flush_turn_events(&mut active)?; self.flush_assistant_messages(&mut active, "before_plan_update")?; active.push_event( @@ -1303,8 +2212,8 @@ impl CaptureCoordinator { ); Ok(()) } - kind if ignore_session_update(kind) => Ok(()), - "available_commands_update" => { + AcpV1SessionUpdateKind::AgentThoughtChunk => Ok(()), + AcpV1SessionUpdateKind::AvailableCommandsUpdate => { self.flush_turn_events(&mut active)?; self.flush_assistant_messages(&mut active, "before_available_commands_update")?; active.push_event( @@ -1313,7 +2222,7 @@ impl CaptureCoordinator { ); Ok(()) } - "usage_update" => { + AcpV1SessionUpdateKind::UsageUpdate => { active.usage = turn_envelope_usage(update); self.flush_turn_events(&mut active)?; self.flush_assistant_messages(&mut active, "before_usage_update")?; @@ -1323,13 +2232,13 @@ impl CaptureCoordinator { ); Ok(()) } - _ => { + AcpV1SessionUpdateKind::UserMessageChunk + | AcpV1SessionUpdateKind::CurrentModeUpdate + | AcpV1SessionUpdateKind::ConfigOptionUpdate + | AcpV1SessionUpdateKind::SessionInfoUpdate + | AcpV1SessionUpdateKind::Extension => { self.flush_turn_events(&mut active)?; self.flush_assistant_messages(&mut active, "before_session_update")?; - active.push_event( - &format!("acp_{update_kind}"), - Some(redact_json(Value::Object(update.clone()))), - ); Ok(()) } }; @@ -1354,20 +2263,130 @@ impl CaptureCoordinator { let Some(session) = session else { return Ok(()); }; - if ignore_session_update(update_kind) { - return Ok(()); + let variant = AcpV1SessionUpdateKind::parse(update_kind); + let mut db = self.open_db()?; + db.add_lane_session_event( + &session.lane_name, + &session.trail_session_id, + "acp_session_update", + Some(session_update_projection(variant, update_kind, update)), + )?; + if variant == AcpV1SessionUpdateKind::AvailableCommandsUpdate { + db.add_lane_session_event( + &session.lane_name, + &session.trail_session_id, + "acp_available_commands_update", + Some(summarize_available_commands(update)), + )?; } - let payload = if update_kind == "available_commands_update" { - summarize_available_commands(update) - } else { - redact_json(Value::Object(update.clone())) + Ok(()) + } + + fn persist_session_update_state( + &mut self, + acp_session_id: &str, + variant: AcpV1SessionUpdateKind, + update: &Map, + ) -> Result<()> { + match variant { + AcpV1SessionUpdateKind::CurrentModeUpdate => { + let Some(mode_id) = update.get("currentModeId").and_then(Value::as_str) else { + return Ok(()); + }; + self.open_db()?.update_lane_acp_session_configuration( + acp_session_id, + Some(mode_id), + None, + )?; + } + AcpV1SessionUpdateKind::ConfigOptionUpdate => { + let values = session_config_values(update); + self.open_db()? + .replace_lane_acp_session_configuration_options(acp_session_id, &values)?; + } + AcpV1SessionUpdateKind::SessionInfoUpdate if update.contains_key("title") => { + let session = self.resolve_session_state(acp_session_id)?; + let title = update.get("title").and_then(Value::as_str); + self.open_db()? + .update_lane_session_title(&session.trail_session_id, title)?; + } + _ => {} + } + Ok(()) + } + + fn flush_load_replay(&mut self, acp_session_id: &str) -> Result<()> { + let Some(replay) = self.replay_by_session.remove(acp_session_id) else { + return Ok(()); }; + if replay.updates.is_empty() { + return Ok(()); + } + let session = self.resolve_session_state(acp_session_id)?; + let turns = replay_turns(replay); let mut db = self.open_db()?; + for (turn_index, replay_turn) in turns.into_iter().enumerate() { + if !replay_turn.has_content() { + continue; + } + let turn = db.begin_lane_session_turn( + &session.lane_name, + &session.trail_session_id, + Some(redact_json(serde_json::json!({ + "kind": "acp_load_replay", + "protocol": "acp", + "acp_session_id": acp_session_id, + "history_index": turn_index + }))), + )?; + let turn_id = turn.turn.turn_id; + if !replay_turn.user_text.is_empty() { + let message = db.add_lane_turn_message(&turn_id, "user", &replay_turn.user_text)?; + db.add_lane_turn_event( + &turn_id, + "acp_replay_message", + Some(redact_json(serde_json::json!({ + "role": "user", + "acp_message_id": replay_turn.user_message_id + }))), + None, + Some(&message.message_id.0), + )?; + } + if !replay_turn.assistant_text.is_empty() { + let message = + db.add_lane_turn_message(&turn_id, "assistant", &replay_turn.assistant_text)?; + db.add_lane_turn_event( + &turn_id, + "acp_replay_message", + Some(redact_json(serde_json::json!({ + "role": "assistant", + "acp_message_id": replay_turn.assistant_message_id + }))), + None, + Some(&message.message_id.0), + )?; + } + for (event_type, payload) in replay_turn.events { + db.add_lane_turn_event( + &turn_id, + &event_type, + Some(redact_json(payload)), + None, + None, + )?; + } + db.end_lane_turn(&turn_id, "completed")?; + let _ = db.create_turn_evidence_manifest(&turn_id); + } db.add_lane_session_event( &session.lane_name, &session.trail_session_id, - &format!("acp_{update_kind}"), - Some(payload), + "acp_load_replay_completed", + Some(redact_json(serde_json::json!({ + "protocol": "acp", + "acp_session_id": acp_session_id + }))), )?; Ok(()) } @@ -1430,6 +2449,386 @@ impl CaptureCoordinator { Ok(()) } + fn capture_client_callback_request(&mut self, message: &mut Value) -> Result<()> { + let id = rpc_id_key(message).ok_or_else(|| { + Error::InvalidInput("ACP client callback request missing id".to_string()) + })?; + let method = method_name(message) + .ok_or_else(|| Error::InvalidInput("ACP client callback missing method".to_string()))? + .to_string(); + let params = params_object_mut(message)?; + let acp_session_id = required_string(params, "sessionId", &method)?; + let session = self.resolve_session_state(&acp_session_id)?; + let operation = match method.as_str() { + "fs/read_text_file" => { + let effective_path = required_string(params, "path", &method)?; + let forwarded_path = reverse_callback_path(&session, &effective_path); + params.insert("path".to_string(), Value::String(forwarded_path.clone())); + ClientCallbackOperation::ReadFile { + acp_session_id, + effective_path, + forwarded_path, + line: params.get("line").and_then(Value::as_u64), + limit: params.get("limit").and_then(Value::as_u64), + } + } + "fs/write_text_file" => { + let effective_path = required_string(params, "path", &method)?; + let forwarded_path = reverse_callback_path(&session, &effective_path); + let content = params + .get("content") + .and_then(Value::as_str) + .ok_or_else(|| { + Error::InvalidInput("ACP fs/write_text_file missing content".to_string()) + })?; + let redacted_content = crate::db::redact_sensitive_text(content); + let redacted_bytes = redacted_content.into_bytes(); + let content_sha256 = hex::encode(Sha256::digest(&redacted_bytes)); + let byte_len = u64::try_from(content.len()).unwrap_or(u64::MAX); + params.insert("path".to_string(), Value::String(forwarded_path.clone())); + ClientCallbackOperation::WriteFile { + acp_session_id, + effective_path, + forwarded_path, + content_sha256, + byte_len, + redacted_content: redacted_bytes, + } + } + "terminal/create" => { + let command = required_string(params, "command", &method)?; + let mut command_line = vec![command]; + command_line.extend( + params + .get("args") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_string), + ); + let effective_cwd = params + .get("cwd") + .and_then(Value::as_str) + .map(str::to_string); + let forwarded_cwd = effective_cwd + .as_deref() + .map(|cwd| reverse_callback_path(&session, cwd)); + if let Some(cwd) = forwarded_cwd.as_ref() { + params.insert("cwd".to_string(), Value::String(cwd.clone())); + } + ClientCallbackOperation::TerminalCreate { + acp_session_id, + command: redact_command(&command_line), + effective_cwd, + forwarded_cwd, + output_byte_limit: params.get("outputByteLimit").and_then(Value::as_u64), + env_names: params + .get("env") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|variable| variable.get("name").and_then(Value::as_str)) + .map(str::to_string) + .collect(), + } + } + "terminal/output" | "terminal/wait_for_exit" | "terminal/kill" | "terminal/release" => { + let terminal_id = required_string(params, "terminalId", &method)?; + match method.as_str() { + "terminal/output" => ClientCallbackOperation::TerminalOutput { + acp_session_id, + terminal_id, + }, + "terminal/wait_for_exit" => ClientCallbackOperation::TerminalWait { + acp_session_id, + terminal_id, + }, + "terminal/kill" => ClientCallbackOperation::TerminalKill { + acp_session_id, + terminal_id, + }, + _ => ClientCallbackOperation::TerminalRelease { + acp_session_id, + terminal_id, + }, + } + } + _ => return Ok(()), + }; + self.register_pending( + Direction::AgentToClient, + id, + PendingOperation::ClientCallback(operation), + )?; + Ok(()) + } + + fn finish_client_callback( + &mut self, + message: &Value, + request_id: &str, + operation: ClientCallbackOperation, + cancel_requested: bool, + ) -> Result<()> { + let acp_session_id = operation.acp_session_id().to_string(); + self.capture_callback_event( + &acp_session_id, + "acp_client_callback_requested", + client_callback_request_payload(&operation, request_id), + )?; + let success = message.get("error").is_none(); + let mut result_summary = Value::Null; + let mut artifact_id = None; + match &operation { + ClientCallbackOperation::Permission(permission) => { + let has_outcome = message.pointer("/result/outcome").is_some(); + let decision = if has_outcome { + permission_decision(message, &permission.options_by_id) + } else { + "error" + }; + result_summary = serde_json::json!({"decision": decision}); + if has_outcome { + if let Some(approval_id) = permission.approval_id.as_deref() { + self.open_db()?.decide_lane_approval( + approval_id, + decision, + Some("acp-editor".to_string()), + Some("mirrored from ACP permission response".to_string()), + )?; + } + } + self.capture_callback_event( + &acp_session_id, + "acp_permission_finished", + serde_json::json!({ + "protocol": "acp", + "acp_session_id": acp_session_id, + "decision": decision, + "cancel_requested": cancel_requested + }), + )?; + } + ClientCallbackOperation::ReadFile { .. } => { + if let Some(content) = message.pointer("/result/content").and_then(Value::as_str) { + let redacted = crate::db::redact_sensitive_text(content); + result_summary = serde_json::json!({ + "byte_len": content.len(), + "sha256": hex::encode(Sha256::digest(redacted.as_bytes())) + }); + } + } + ClientCallbackOperation::WriteFile { + effective_path, + forwarded_path, + byte_len, + redacted_content, + .. + } => { + let session = self.resolve_session_state(&acp_session_id)?; + let active_turn_id = self + .active_turns + .get(&acp_session_id) + .map(|active| active.turn_id.clone()); + let artifact = self.open_db()?.record_lane_artifact(LaneArtifactInput { + lane: session.lane_name, + session_id: session.trail_session_id, + turn_id: active_turn_id, + provider: self + .options + .provider + .clone() + .unwrap_or_else(|| "acp-agent".to_string()), + artifact_kind: "acp_fs_write".to_string(), + format: "text".to_string(), + source: AgentEvidenceSource::Acp, + source_locator_redacted: Some(forwarded_path.clone()), + content: redacted_content.clone(), + start_offset: None, + end_offset: Some(u64::try_from(redacted_content.len()).unwrap_or(u64::MAX)), + redaction_profile: Some("trail-sensitive-text-v1".to_string()), + trust: "protocol".to_string(), + supersedes_artifact_id: None, + metadata_json: Some( + serde_json::json!({ + "protocol": "acp", + "effective_path": effective_path, + "forwarded_path": forwarded_path, + "requested_byte_len": byte_len + }) + .to_string(), + ), + })?; + artifact_id = Some(artifact.artifact_id); + result_summary = serde_json::json!({"written": success}); + } + ClientCallbackOperation::TerminalCreate { .. } if success => { + if let Some(terminal_id) = message + .pointer("/result/terminalId") + .and_then(Value::as_str) + { + self.terminals.insert( + terminal_id.to_string(), + CapturedTerminal { + acp_session_id: acp_session_id.clone(), + state: "running".to_string(), + }, + ); + result_summary = serde_json::json!({ + "terminal_id": terminal_id, + "state": "running" + }); + } + } + ClientCallbackOperation::TerminalOutput { terminal_id, .. } => { + let output = message + .pointer("/result/output") + .and_then(Value::as_str) + .unwrap_or_default(); + let redacted = crate::db::redact_sensitive_text(output); + let exit_status = message.pointer("/result/exitStatus").cloned(); + if success { + if let Some(terminal) = self.terminals.get_mut(terminal_id) { + terminal.state = + if exit_status.as_ref().is_some_and(|value| !value.is_null()) { + "exited".to_string() + } else { + "running".to_string() + }; + } + } + result_summary = serde_json::json!({ + "terminal_id": terminal_id, + "output_byte_len": output.len(), + "output_sha256": hex::encode(Sha256::digest(redacted.as_bytes())), + "truncated": message.pointer("/result/truncated"), + "exit_status": exit_status + }); + } + ClientCallbackOperation::TerminalWait { terminal_id, .. } => { + if success { + if let Some(terminal) = self.terminals.get_mut(terminal_id) { + terminal.state = "exited".to_string(); + } + } + result_summary = serde_json::json!({ + "terminal_id": terminal_id, + "exit_code": message.pointer("/result/exitCode"), + "signal": message.pointer("/result/signal"), + "state": success.then_some("exited") + }); + } + ClientCallbackOperation::TerminalKill { terminal_id, .. } => { + if success { + if let Some(terminal) = self.terminals.get_mut(terminal_id) { + terminal.state = "killed".to_string(); + } + } + result_summary = serde_json::json!({ + "terminal_id": terminal_id, + "state": success.then_some("killed") + }); + } + ClientCallbackOperation::TerminalRelease { terminal_id, .. } => { + if success { + self.terminals.remove(terminal_id); + } + result_summary = serde_json::json!({ + "terminal_id": terminal_id, + "state": success.then_some("released") + }); + } + _ => {} + } + self.capture_callback_event( + &acp_session_id, + "acp_client_callback_finished", + serde_json::json!({ + "protocol": "acp", + "method": operation.method(), + "request_id": rpc_id_value(request_id), + "success": success, + "cancel_requested": cancel_requested, + "result": result_summary, + "error": message.get("error"), + "artifact_id": artifact_id + }), + ) + } + + fn capture_callback_shutdown(&mut self, reason: &RelayFinishReason) -> Result<()> { + let pending = self + .pending_operations + .iter() + .filter_map(|((direction, request_id), operation)| { + if *direction != Direction::AgentToClient { + return None; + } + let PendingOperation::ClientCallback(callback) = operation else { + return None; + }; + Some((request_id.clone(), callback.clone())) + }) + .collect::>(); + for (request_id, callback) in pending { + self.pending_operations + .remove(&(Direction::AgentToClient, request_id.clone())); + self.finish_client_callback( + &serde_json::json!({ + "jsonrpc": "2.0", + "id": serde_json::from_str::(&request_id).unwrap_or(Value::Null), + "error": { + "code": -32099, + "message": "relay shutdown with callback in flight", + "data": {"finish_reason": format!("{reason:?}")} + } + }), + &request_id, + callback, + false, + )?; + } + let terminals = self.terminals.drain().collect::>(); + for (terminal_id, terminal) in terminals { + self.capture_callback_event( + &terminal.acp_session_id, + "acp_terminal_abandoned", + serde_json::json!({ + "protocol": "acp", + "terminal_id": terminal_id, + "state": terminal.state, + "finish_reason": format!("{reason:?}") + }), + )?; + } + Ok(()) + } + + fn capture_callback_event( + &mut self, + acp_session_id: &str, + event_type: &str, + payload: Value, + ) -> Result<()> { + if let Some(mut active) = self.active_turns.remove(acp_session_id) { + self.flush_turn_events(&mut active)?; + self.flush_assistant_messages(&mut active, "before_client_callback")?; + active.push_event(event_type, Some(redact_json(payload))); + self.flush_turn_events(&mut active)?; + self.active_turns.insert(acp_session_id.to_string(), active); + return Ok(()); + } + let session = self.resolve_session_state(acp_session_id)?; + self.open_db()?.add_lane_session_event( + &session.lane_name, + &session.trail_session_id, + event_type, + Some(redact_json(payload)), + )?; + Ok(()) + } + fn flush_turn_events(&self, active: &mut ActiveTurn) -> Result<()> { if active.pending_events.is_empty() { return Ok(()); @@ -1525,7 +2924,7 @@ impl CaptureCoordinator { }; let options_by_id = permission_options(params.get("options")); let mut approval_id = None; - if let Some(session) = self.resolve_session_state(acp_session_id).ok() { + if let Ok(session) = self.resolve_session_state(acp_session_id) { let active_turn_id = if let Some(mut active) = self.active_turns.remove(acp_session_id) { let turn_id = active.turn_id.clone(); @@ -1551,13 +2950,17 @@ impl CaptureCoordinator { )?; approval_id = Some(report.approval.approval_id); } - self.pending_permissions.insert( + self.register_pending( + Direction::AgentToClient, id, - PendingPermission { - approval_id, - options_by_id, - }, - ); + PendingOperation::ClientCallback(ClientCallbackOperation::Permission( + PendingPermission { + approval_id, + options_by_id, + acp_session_id: acp_session_id.to_string(), + }, + )), + )?; Ok(()) } @@ -1583,6 +2986,69 @@ impl CaptureCoordinator { Ok(()) } + fn capture_rpc_cancel(&mut self, message: &Value, direction: Direction) -> Result<()> { + let target = message.pointer("/params/requestId").ok_or_else(|| { + Error::InvalidInput("ACP $/cancel_request missing requestId".to_string()) + })?; + let target_id = serde_json::to_string(target)?; + let key = (direction, target_id.clone()); + let operation = self.pending_operations.get(&key).cloned(); + let matched = operation.is_some(); + if matched { + self.cancelled_requests.insert(key.clone()); + } + let acp_session_id = match operation { + Some(PendingOperation::Session(pending)) => Some(pending.session.acp_session_id), + Some(PendingOperation::Prompt(pending)) => Some(pending.acp_session_id), + Some(PendingOperation::Close { acp_session_id }) + | Some(PendingOperation::SessionDelete { acp_session_id }) + | Some(PendingOperation::SetMode { acp_session_id, .. }) + | Some(PendingOperation::SetConfig { acp_session_id, .. }) => Some(acp_session_id), + Some(PendingOperation::ClientCallback(operation)) => { + Some(operation.acp_session_id().to_string()) + } + _ => None, + }; + let payload = redact_json(serde_json::json!({ + "protocol": "acp", + "method": "$/cancel_request", + "target_request_id": target, + "request_direction": format!("{direction:?}"), + "matched": matched + })); + if let Some(acp_session_id) = acp_session_id { + if let Some(mut active) = self.active_turns.remove(&acp_session_id) { + active.push_event("acp_request_cancelled", Some(payload.clone())); + self.flush_turn_events(&mut active)?; + self.active_turns.insert(acp_session_id.clone(), active); + return Ok(()); + } + if let Ok(session) = self.resolve_session_state(&acp_session_id) { + self.open_db()?.add_lane_session_event( + &session.lane_name, + &session.trail_session_id, + "acp_request_cancelled", + Some(payload), + )?; + } + } else { + let sessions = self.sessions_by_acp.values().cloned().collect::>(); + if sessions.is_empty() { + self.pending_connection_events.push(payload); + } else { + for session in sessions { + self.open_db()?.add_lane_session_event( + &session.lane_name, + &session.trail_session_id, + "acp_request_cancelled", + Some(payload.clone()), + )?; + } + } + } + Ok(()) + } + fn prepare_close_request(&mut self, message: &Value) -> Result<()> { let Some(id) = rpc_id_key(message) else { return Ok(()); @@ -1591,39 +3057,214 @@ impl CaptureCoordinator { let Some(acp_session_id) = params.get("sessionId").and_then(Value::as_str) else { return Ok(()); }; - self.pending_closes.insert(id, acp_session_id.to_string()); - Ok(()) + self.register_pending( + Direction::ClientToAgent, + id, + PendingOperation::Close { + acp_session_id: acp_session_id.to_string(), + }, + ) } fn finish_close_request(&mut self, message: &Value, acp_session_id: &str) -> Result<()> { let Some(session) = self.sessions_by_acp.get(acp_session_id).cloned() else { return Ok(()); }; - let status = if message.get("error").is_some() { - "failed" - } else { - "closed" - }; + let succeeded = message.get("error").is_none(); + let status = if succeeded { "closed" } else { "active" }; let mut db = self.open_db()?; - db.update_lane_acp_session_status(acp_session_id, status)?; + if succeeded { + db.update_lane_acp_session_status(acp_session_id, status)?; + } db.add_lane_session_event( &session.lane_name, &session.trail_session_id, - "acp_session_closed", + if succeeded { + "acp_session_closed" + } else { + "acp_session_close_failed" + }, Some(redact_json(serde_json::json!({ "protocol": "acp", "acp_session_id": acp_session_id, - "status": status + "status": status, + "error": message.get("error") }))), )?; - if status == "closed" { + if succeeded { + record_acp_lifecycle_event( + &mut db, + &session.lane_name, + &session.trail_session_id, + None, + self.options.provider.as_deref(), + acp_session_id, + AgentLifecycleEventKind::SessionEnded, + serde_json::json!({"status": status}), + )?; let _ = db.end_lane_session(&session.trail_session_id, "completed"); + if db + .lane_session_turns(&session.trail_session_id)? + .iter() + .any(|turn| turn.ended_at.is_some()) + { + let _ = db.create_session_attestation( + &session.trail_session_id, + "acp-on-session-close", + Some(serde_json::json!({"acp_session_id": acp_session_id})), + ); + } self.sessions_by_acp.remove(acp_session_id); } Ok(()) } + fn finish_delete_request(&mut self, message: &Value, acp_session_id: &str) -> Result<()> { + if message.get("error").is_some() { + return self.capture_session_lifecycle_failure( + acp_session_id, + "session/delete", + message.get("error"), + ); + } + let Some(session) = self.resolve_session_state(acp_session_id).ok() else { + return Ok(()); + }; + let mut db = self.open_db()?; + db.update_lane_acp_session_status(acp_session_id, "deleted")?; + db.add_lane_session_event( + &session.lane_name, + &session.trail_session_id, + "acp_session_deleted", + Some(redact_json(serde_json::json!({ + "protocol": "acp", + "method": "session/delete", + "acp_session_id": acp_session_id, + "status": "deleted" + }))), + )?; + self.sessions_by_acp.remove(acp_session_id); + Ok(()) + } + + fn finish_configuration_request( + &mut self, + message: &Value, + acp_session_id: &str, + mode_id: Option<&str>, + config: Option<(&str, &Value)>, + cancel_requested: bool, + ) -> Result<()> { + if message.get("error").is_some() { + return self.capture_session_lifecycle_failure( + acp_session_id, + if mode_id.is_some() { + "session/set_mode" + } else { + "session/set_config_option" + }, + message.get("error"), + ); + } + let session = self.resolve_session_state(acp_session_id)?; + let mut db = self.open_db()?; + let authoritative_config = message + .pointer("/result/configOptions") + .and_then(Value::as_array) + .map(|options| session_config_values_from_options(options)); + let mapping = if let Some(config_options) = authoritative_config.as_ref() { + db.replace_lane_acp_session_configuration_options(acp_session_id, config_options)? + } else { + db.update_lane_acp_session_configuration(acp_session_id, mode_id, config)? + }; + db.add_lane_session_event( + &session.lane_name, + &session.trail_session_id, + if mode_id.is_some() { + "acp_session_mode_changed" + } else { + "acp_session_config_changed" + }, + Some(redact_json(serde_json::json!({ + "protocol": "acp", + "acp_session_id": acp_session_id, + "mode_id": mode_id, + "config_id": config.as_ref().map(|(id, _)| *id), + "value": config.as_ref().map(|(_, value)| *value), + "authoritative_config_options": message.pointer("/result/configOptions"), + "cancel_requested": cancel_requested, + "current_mode_id": mapping.current_mode_id, + "config_options": mapping.config_options + }))), + )?; + Ok(()) + } + + fn capture_session_lifecycle_failure( + &mut self, + acp_session_id: &str, + method: &str, + error: Option<&Value>, + ) -> Result<()> { + let Some(session) = self.resolve_session_state(acp_session_id).ok() else { + return Ok(()); + }; + self.open_db()?.add_lane_session_event( + &session.lane_name, + &session.trail_session_id, + "acp_session_request_failed", + Some(redact_json(serde_json::json!({ + "protocol": "acp", + "method": method, + "acp_session_id": acp_session_id, + "error": error + }))), + )?; + Ok(()) + } + + fn capture_connection_lifecycle( + &mut self, + method: &str, + message: &Value, + context: Option, + ) -> Result<()> { + let outcome = if message.get("error").is_some() { + "error" + } else { + "success" + }; + let payload = redact_json(serde_json::json!({ + "protocol": "acp", + "method": method, + "request_id": message.get("id"), + "outcome": outcome, + "result": message.get("result"), + "error": message.get("error"), + "error_code": message.pointer("/error/code"), + "context": context + })); + let sessions = self.sessions_by_acp.values().cloned().collect::>(); + if sessions.is_empty() { + self.pending_connection_events.push(payload); + return Ok(()); + } + for session in sessions { + self.open_db()?.add_lane_session_event( + &session.lane_name, + &session.trail_session_id, + "acp_connection_lifecycle", + Some(payload.clone()), + )?; + } + Ok(()) + } + fn finish_open_turns(&mut self, status: &str, reason: &str) -> Result<()> { + let replay_sessions = self.replay_by_session.keys().cloned().collect::>(); + for acp_session_id in replay_sessions { + self.flush_load_replay(&acp_session_id)?; + } if self.active_turns.is_empty() { return Ok(()); } @@ -1646,21 +3287,32 @@ impl CaptureCoordinator { ); self.flush_turn_events(&mut active)?; let mut db = self.open_db()?; - if active.materialized { - let _ = db.record_lane_workdir_for_turn( + let checkpoint_error = if active.materialized { + self.record_prompt_workdir_checkpoint( + &mut db, &active.lane_name, &active.turn_id, - Some(format!("ACP prompt workdir checkpoint ({reason})")), - ); - } + format!("ACP prompt workdir checkpoint ({reason})"), + )? + } else { + None + }; + let checkpoint_failed = checkpoint_error.is_some(); + let final_status = if checkpoint_failed { "failed" } else { status }; + let final_error_summary = checkpoint_error + .as_ref() + .map(|checkpoint| format!("{}; {checkpoint}", summarize_text(reason))) + .or_else(|| Some(summarize_text(reason))); db.add_lane_turn_event( &active.turn_id, "acp_prompt_finished", Some(redact_json(serde_json::json!({ "protocol": "acp", "acp_session_id": acp_session_id, - "status": status, - "reason": reason + "status": final_status, + "upstream_status": status, + "reason": reason, + "checkpoint_error": checkpoint_error }))), None, None, @@ -1668,22 +3320,31 @@ impl CaptureCoordinator { if let Some(span_id) = &active.root_span_id { let _ = db.end_lane_trace_span( span_id, - status, + final_status, Some(redact_json(serde_json::json!({ "reason": reason }))), ); } - let _ = db.end_lane_turn(&active.turn_id, status); - let _ = self.finalize_turn_envelope( + db.end_lane_turn(&active.turn_id, final_status)?; + db.create_turn_evidence_manifest(&active.turn_id)?; + self.finalize_turn_envelope( &mut db, &active.turn_id, - status, - None, - Some(summarize_text(reason)), + TurnTerminalOutcome { + status: final_status, + stop_reason: None, + error_summary: final_error_summary, + checkpoint_failed, + }, Some(&active), - ); - let _ = db.update_lane_acp_session_status(&acp_session_id, status); - self.pending_prompts - .retain(|_, pending| pending.acp_session_id != acp_session_id); + )?; + db.update_lane_acp_session_status(&acp_session_id, final_status)?; + self.pending_operations.retain(|_, operation| { + !matches!( + operation, + PendingOperation::Prompt(pending) + if pending.acp_session_id == acp_session_id + ) + }); } Ok(()) } @@ -1705,15 +3366,27 @@ impl CaptureCoordinator { let Some(mapping) = db.try_lane_acp_session(acp_session_id)? else { return Ok(None); }; + if mapping.status == "deleted" { + return Ok(None); + } let lane_name = db.resolve_lane_handle(&mapping.lane_id)?; + let materialized_root = db.lane_details(&lane_name)?.branch.workdir; + let original_cwd = mapping + .path_mappings + .iter() + .find(|path| path.effective == mapping.cwd) + .map(|path| path.original.clone()) + .unwrap_or_else(|| mapping.cwd.clone()); Ok(Some(SessionState { acp_session_id: mapping.acp_session_id, upstream_session_id: mapping.upstream_session_id, lane_name, trail_session_id: mapping.trail_session_id, - original_cwd: mapping.cwd.clone(), + original_cwd, effective_cwd: mapping.cwd, - materialized: false, + materialized: mapping.path_mappings.iter().any(|path| path.isolated), + materialized_root, + path_mappings: mapping.path_mappings, })) } @@ -1741,8 +3414,22 @@ impl CaptureCoordinator { ) } - fn open_db(&self) -> Result { - Trail::open_with_db_dir(&self.options.workspace_root, &self.options.db_dir) + fn open_db(&self) -> Result { + if let Some(db) = self + .db_pool + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .pop() + { + return Ok(PooledTrail { + db: Some(db), + pool: self.db_pool.clone(), + }); + } + Ok(PooledTrail { + db: Some(open_capture_db(&self.options)?), + pool: self.db_pool.clone(), + }) } fn upstream_command_hash(&self) -> Option { @@ -1755,9 +3442,7 @@ impl CaptureCoordinator { &self, db: &mut Trail, turn_id: &str, - status: &str, - stop_reason: Option, - error_summary: Option, + terminal: TurnTerminalOutcome<'_>, active: Option<&ActiveTurn>, ) -> Result<()> { let turn = db.lane_turn(turn_id)?; @@ -1793,21 +3478,230 @@ impl CaptureCoordinator { { envelope.capture.redaction_applied = true; } - envelope.finalize_outcome( - status.to_string(), - stop_reason, - &turn.before_change, - turn.after_change.as_ref(), - error_summary, - ); - db.update_lane_turn_metadata(turn_id, &envelope.to_metadata_value()) + envelope.finalize_outcome( + terminal.status.to_string(), + terminal.stop_reason, + &turn.before_change, + turn.after_change.as_ref(), + terminal.error_summary, + ); + if terminal.checkpoint_failed { + envelope.outcome.checkpoint = None; + envelope.outcome.no_changes = false; + } + db.update_lane_turn_metadata(turn_id, &envelope.to_metadata_value()) + } +} + +fn open_capture_db(options: &AcpRelayOptions) -> Result { + let deadline = Instant::now() + ACP_CAPTURE_LOCK_WAIT; + let mut delay = Duration::from_millis(2); + loop { + match Trail::open_with_db_dir(&options.workspace_root, &options.db_dir) { + Ok(db) => return Ok(db), + Err(Error::SchemaReinitializeRequired { ref found, .. }) + if found == "schema main/WAL/SHM generation changed during mutable handoff" + && Instant::now() < deadline => + { + std::thread::sleep(delay); + delay = (delay * 2).min(Duration::from_millis(50)); + } + Err(error) => return Err(error), + } + } +} + +fn replay_turns(replay: ReplayAccumulator) -> Vec { + let mut turns = Vec::new(); + let mut current = ReplayTurn::default(); + for (sequence, (kind, update)) in replay.updates.into_iter().enumerate() { + let variant = AcpV1SessionUpdateKind::parse(&kind); + let mut projection = session_update_projection(variant, &kind, &update); + if let Some(object) = projection.as_object_mut() { + object.insert( + "replaySequence".to_string(), + Value::from(u64::try_from(sequence).unwrap_or(u64::MAX)), + ); + } + match kind.as_str() { + "user_message_chunk" => { + let message_id = update + .get("messageId") + .and_then(Value::as_str) + .map(str::to_string); + let starts_new_turn = current.has_content() + && (!current.assistant_text.is_empty() + || (message_id.is_some() && current.user_message_id != message_id)); + if starts_new_turn { + turns.push(std::mem::take(&mut current)); + } + if current.user_message_id.is_none() { + current.user_message_id = message_id; + } + current + .user_text + .push_str(&content_text(update.get("content"))); + } + "agent_message_chunk" => { + if current.assistant_message_id.is_none() { + current.assistant_message_id = update + .get("messageId") + .and_then(Value::as_str) + .map(str::to_string); + } + current + .assistant_text + .push_str(&content_text(update.get("content"))); + } + "agent_thought_chunk" => { + current.events.push(( + "acp_agent_thought_chunk_excluded".to_string(), + serde_json::json!({ + "protocol": "acp", + "replay_sequence": sequence, + "excluded": true + }), + )); + } + "plan" => current.events.push(( + "plan_update".to_string(), + replay_event_payload(sequence, update), + )), + "usage_update" => current.events.push(( + "acp_usage_update".to_string(), + replay_event_payload(sequence, update), + )), + "tool_call" | "tool_call_update" => current + .events + .push((kind, replay_event_payload(sequence, update))), + _ => current.events.push(( + format!("acp_{kind}"), + replay_event_payload(sequence, update), + )), + } + current + .events + .push(("acp_session_update".to_string(), projection)); } + if current.has_content() { + turns.push(current); + } + turns +} + +fn replay_event_payload(sequence: usize, mut update: Map) -> Value { + update.insert( + "replaySequence".to_string(), + Value::from(u64::try_from(sequence).unwrap_or(u64::MAX)), + ); + Value::Object(update) } fn method_name(message: &Value) -> Option<&str> { message.get("method").and_then(Value::as_str) } +fn is_client_callback_method(method: &str) -> bool { + matches!( + method, + "session/request_permission" + | "fs/read_text_file" + | "fs/write_text_file" + | "terminal/create" + | "terminal/output" + | "terminal/wait_for_exit" + | "terminal/kill" + | "terminal/release" + ) +} + +fn reverse_callback_path(session: &SessionState, effective_path: &str) -> String { + let path = PathBuf::from(effective_path); + let mapping = session + .path_mappings + .iter() + .filter_map(|mapping| { + path.strip_prefix(PathBuf::from(&mapping.effective)) + .ok() + .map(|suffix| (mapping, suffix.to_path_buf())) + }) + .max_by_key(|(mapping, _)| PathBuf::from(&mapping.effective).components().count()); + let Some((mapping, suffix)) = mapping else { + return effective_path.to_string(); + }; + if suffix.as_os_str().is_empty() { + return mapping.original.clone(); + } + PathBuf::from(&mapping.original) + .join(suffix) + .to_string_lossy() + .to_string() +} + +fn client_callback_request_payload(operation: &ClientCallbackOperation, request_id: &str) -> Value { + let details = match operation { + ClientCallbackOperation::Permission(permission) => serde_json::json!({ + "option_kinds": permission.options_by_id, + "approval_id": permission.approval_id + }), + ClientCallbackOperation::ReadFile { + effective_path, + forwarded_path, + line, + limit, + .. + } => serde_json::json!({ + "effective_path": effective_path, + "forwarded_path": forwarded_path, + "line": line, + "limit": limit + }), + ClientCallbackOperation::WriteFile { + effective_path, + forwarded_path, + content_sha256, + byte_len, + .. + } => serde_json::json!({ + "effective_path": effective_path, + "forwarded_path": forwarded_path, + "content_sha256": content_sha256, + "byte_len": byte_len + }), + ClientCallbackOperation::TerminalCreate { + command, + effective_cwd, + forwarded_cwd, + output_byte_limit, + env_names, + .. + } => serde_json::json!({ + "command": command, + "effective_cwd": effective_cwd, + "forwarded_cwd": forwarded_cwd, + "output_byte_limit": output_byte_limit, + "env_names": env_names + }), + ClientCallbackOperation::TerminalOutput { terminal_id, .. } + | ClientCallbackOperation::TerminalWait { terminal_id, .. } + | ClientCallbackOperation::TerminalKill { terminal_id, .. } + | ClientCallbackOperation::TerminalRelease { terminal_id, .. } => { + serde_json::json!({"terminal_id": terminal_id}) + } + }; + serde_json::json!({ + "protocol": "acp", + "method": operation.method(), + "request_id": rpc_id_value(request_id), + "acp_session_id": operation.acp_session_id(), + "details": details + }) +} + +fn rpc_id_value(serialized_id: &str) -> Value { + serde_json::from_str(serialized_id).unwrap_or_else(|_| Value::String(serialized_id.to_string())) +} + fn rpc_id_key(message: &Value) -> Option { message .get("id") @@ -1828,17 +3722,13 @@ fn params_object_mut(message: &mut Value) -> Result<&mut Map> { .ok_or_else(|| Error::InvalidInput("ACP message missing object params".to_string())) } -fn ensure_object_field<'a>( - object: &'a mut Map, - key: &str, -) -> &'a mut Map { - let value = object - .entry(key.to_string()) - .or_insert_with(|| Value::Object(Map::new())); - if !value.is_object() { - *value = Value::Object(Map::new()); - } - value.as_object_mut().expect("object was just inserted") +fn required_string(params: &Map, field: &str, method: &str) -> Result { + params + .get(field) + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .ok_or_else(|| Error::InvalidInput(format!("ACP {method} missing {field}"))) } fn inject_trail_mcp_server( @@ -1852,18 +3742,44 @@ fn inject_trail_mcp_server( let servers = servers.as_array_mut().ok_or_else(|| { Error::InvalidInput("ACP session mcpServers must be an array".to_string()) })?; - let already_present = servers.iter().any(|server| { - server - .get("name") - .and_then(Value::as_str) - .is_some_and(|name| name == "trail") - }); + let already_present = servers + .iter() + .any(|existing| equivalent_trail_mcp_server(existing, &server)); if !already_present { servers.push(server); } Ok(()) } +fn equivalent_trail_mcp_server(existing: &Value, expected: &Value) -> bool { + if existing.get("command") != expected.get("command") + || existing.get("args") != expected.get("args") + { + return false; + } + let Some(existing_env) = existing.get("env").and_then(Value::as_array) else { + return false; + }; + let Some(expected_env) = expected.get("env").and_then(Value::as_array) else { + return false; + }; + expected_env.iter().all(|expected_variable| { + let name = expected_variable.get("name"); + let value = expected_variable.get("value"); + existing_env.iter().any(|existing_variable| { + existing_variable.get("name") == name && existing_variable.get("value") == value + }) + }) +} + +fn acp_path_mapping(mapping: PathMapping) -> AcpPathMapping { + AcpPathMapping { + original: mapping.original.to_string_lossy().to_string(), + effective: mapping.effective.to_string_lossy().to_string(), + isolated: mapping.isolated, + } +} + fn trail_mcp_server(options: &AcpRelayOptions) -> Value { let command = std::env::current_exe() .ok() @@ -1962,6 +3878,9 @@ fn usage_number_field(update: &Map, keys: &[&str]) -> Option } fn prompt_status(message: &Value) -> &'static str { + if message.pointer("/error/code").and_then(Value::as_i64) == Some(-32800) { + return "cancelled"; + } if message.get("error").is_some() { return "failed"; } @@ -1971,6 +3890,67 @@ fn prompt_status(message: &Value) -> &'static str { } } +fn remap_prompt_resource_uris( + prompt: Option<&mut Value>, + original_cwd: &Path, + effective_cwd: &Path, +) { + let Some(Value::Array(blocks)) = prompt else { + return; + }; + for block in blocks { + let block_type = block + .get("type") + .and_then(Value::as_str) + .map(str::to_string); + let uri = match block_type.as_deref() { + Some("resource") => block + .get_mut("resource") + .and_then(Value::as_object_mut) + .and_then(|resource| resource.get_mut("uri")), + Some("resource_link") => block.get_mut("uri"), + _ => None, + }; + let Some(uri) = uri else { + continue; + }; + let Some(source) = uri.as_str().map(str::to_string) else { + continue; + }; + if let Some(mapped) = remap_workspace_file_uri(&source, original_cwd, effective_cwd) { + *uri = Value::String(mapped); + } + } +} + +fn remap_workspace_file_uri( + uri: &str, + original_cwd: &Path, + effective_cwd: &Path, +) -> Option { + let source_url = Url::parse(uri).ok()?; + if source_url.scheme() != "file" { + return None; + } + let source_path = source_url.to_file_path().ok()?; + let relative = source_path.strip_prefix(original_cwd).ok()?; + if relative.components().any(|component| { + matches!( + component, + Component::ParentDir | Component::RootDir | Component::Prefix(_) + ) + }) { + return None; + } + + let query = source_url.query().map(str::to_string); + let fragment = source_url.fragment().map(str::to_string); + let mut mapped_url = Url::from_file_path(effective_cwd.join(relative)).ok()?; + mapped_url.set_query(query.as_deref()); + mapped_url.set_fragment(fragment.as_deref()); + Some(mapped_url.into()) +} + fn prompt_text(prompt: Option<&Value>) -> String { let text = match prompt { Some(Value::Array(blocks)) => blocks @@ -1989,6 +3969,49 @@ fn prompt_text(prompt: Option<&Value>) -> String { } } +fn bounded_prompt_content(prompt: Option<&Value>) -> Value { + let mut value = redact_json(prompt.cloned().unwrap_or(Value::Array(Vec::new()))); + bound_binary_content(&mut value, None); + value +} + +fn bound_binary_content(value: &mut Value, parent_type: Option<&str>) { + match value { + Value::Array(items) => { + for item in items { + bound_binary_content(item, parent_type); + } + } + Value::Object(object) => { + let content_type = object + .get("type") + .and_then(Value::as_str) + .or(parent_type) + .map(str::to_string); + for key in ["data", "blob"] { + let is_binary = + key == "blob" || matches!(content_type.as_deref(), Some("image" | "audio")); + if is_binary { + if let Some(encoded) = object.get(key).and_then(Value::as_str) { + object.insert( + key.to_string(), + serde_json::json!({ + "encoding": "base64", + "encoded_bytes": encoded.len(), + "sha256": hex::encode(Sha256::digest(encoded.as_bytes())) + }), + ); + } + } + } + for child in object.values_mut() { + bound_binary_content(child, content_type.as_deref()); + } + } + _ => {} + } +} + fn block_text(value: &Value) -> String { if let Some(text) = value.get("text").and_then(Value::as_str) { return text.to_string(); @@ -2117,8 +4140,8 @@ fn relative_path_from_cwd(path: &str, cwd: &std::path::Path) -> Option { } } -fn permission_options(value: Option<&Value>) -> HashMap { - let mut options = HashMap::new(); +fn permission_options(value: Option<&Value>) -> BTreeMap { + let mut options = BTreeMap::new(); let Some(Value::Array(items)) = value else { return options; }; @@ -2136,7 +4159,7 @@ fn permission_options(value: Option<&Value>) -> HashMap { options } -fn permission_decision(message: &Value, options_by_id: &HashMap) -> &'static str { +fn permission_decision(message: &Value, options_by_id: &BTreeMap) -> &'static str { let Some(outcome) = message .get("result") .and_then(|result| result.get("outcome")) @@ -2240,8 +4263,46 @@ fn summarize_available_commands(update: &Map) -> Value { })) } -fn ignore_session_update(update_kind: &str) -> bool { - update_kind == "agent_thought_chunk" +fn session_update_projection( + variant: AcpV1SessionUpdateKind, + update_kind: &str, + update: &Map, +) -> Value { + let mut structured = bounded_prompt_content(Some(&Value::Object(update.clone()))); + let thought_content_excluded = variant == AcpV1SessionUpdateKind::AgentThoughtChunk; + if thought_content_excluded { + if let Some(object) = structured.as_object_mut() { + object.remove("content"); + } + } + serde_json::json!({ + "protocol": "acp", + "acpVariant": update_kind, + "stable": variant.is_stable(), + "thoughtContentExcluded": thought_content_excluded, + "update": structured + }) +} + +fn session_config_values(update: &Map) -> Map { + let Some(options) = update.get("configOptions").and_then(Value::as_array) else { + return Map::new(); + }; + session_config_values_from_options(options) +} + +fn session_config_values_from_options(options: &[Value]) -> Map { + let mut values = Map::new(); + for option in options { + let Some(id) = option.get("id").and_then(Value::as_str) else { + continue; + }; + let Some(value) = option.get("currentValue") else { + continue; + }; + values.insert(id.to_string(), value.clone()); + } + values } fn command_display_name(value: &Value) -> Option { @@ -2351,19 +4412,110 @@ fn redact_text(text: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::InitImportMode; + use crate::{InitImportMode, ObjectId}; use std::fs; + fn make_ref_root_legacy(db: &Trail, ref_name: &str) -> ObjectId { + let head = db.get_ref(ref_name).unwrap(); + let mut root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + assert!(root.file_count > 0); + assert!(root.case_fold_map_root.take().is_some()); + let legacy_root = db + .put_object(WORKTREE_ROOT_KIND, root.version, &root) + .unwrap(); + db.set_ref( + &head.name, + &head.change_id, + &legacy_root, + &head.operation_id, + ) + .unwrap(); + legacy_root + } + + #[test] + fn acp_relay_preflight_rejects_legacy_live_root() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + make_ref_root_legacy(&db, "refs/branches/main"); + drop(db); + + let options = AcpRelayOptions { + workspace_root: temp.path().to_path_buf(), + db_dir: temp.path().join(".trail"), + lane: Some("legacy-preflight".to_string()), + from_ref: None, + provider: Some("test".to_string()), + model: None, + materialize: true, + workdir: None, + inject_mcp: false, + upstream_command: vec!["agent".to_string()], + upstream_env: BTreeMap::new(), + }; + + match capture_observer(&options) { + Err(Error::PathIndexRequired(message)) => { + assert!(message.contains("refs/branches/main")); + assert!(message.contains("trail index rebuild")); + } + Err(other) => panic!("expected PATH_INDEX_REQUIRED, got {other}"), + Ok(observer) => { + drop(observer); + panic!("legacy live root unexpectedly passed ACP relay preflight"); + } + } + } + #[test] - fn json_line_round_trips_single_message() { - let mut input = - BufReader::new(br#"{"jsonrpc":"2.0","id":1,"method":"initialize"}"#.as_slice()); - let value = read_json_line(&mut input).unwrap().unwrap(); - assert_eq!(value["method"], "initialize"); + fn raw_json_frame_preserves_a_single_message() { + let raw = br#" {"jsonrpc":"2.0","id":1,"method":"initialize"} +"# + .to_vec(); + let frame = Frame::parse(Direction::ClientToAgent, raw.clone()).unwrap(); + assert_eq!(frame.value()["method"], "initialize"); + assert_eq!(frame.forward_bytes(), raw); + } - let mut out = Vec::new(); - write_json_line(&mut out, &value).unwrap(); - assert!(std::str::from_utf8(&out).unwrap().ends_with('\n')); + #[test] + fn pending_registry_scopes_same_ids_by_direction_and_rejects_reuse() { + let options = AcpRelayOptions { + workspace_root: PathBuf::from("/tmp/workspace"), + db_dir: PathBuf::from("/tmp/workspace/.trail"), + lane: None, + from_ref: None, + provider: None, + model: None, + materialize: false, + workdir: None, + inject_mcp: false, + upstream_command: vec!["agent".to_string()], + upstream_env: BTreeMap::new(), + }; + let mut coordinator = CaptureCoordinator::new(options).unwrap(); + coordinator + .register_pending( + Direction::ClientToAgent, + "same-id".to_string(), + PendingOperation::Logout, + ) + .unwrap(); + coordinator + .register_pending( + Direction::AgentToClient, + "same-id".to_string(), + PendingOperation::Logout, + ) + .unwrap(); + assert!(coordinator + .register_pending( + Direction::ClientToAgent, + "same-id".to_string(), + PendingOperation::Logout, + ) + .is_err()); } #[test] @@ -2498,10 +4650,43 @@ mod tests { } #[test] - fn agent_thought_chunks_are_not_captured() { - assert!(ignore_session_update("agent_thought_chunk")); - assert!(!ignore_session_update("agent_message_chunk")); - assert!(!ignore_session_update("tool_call")); + fn stable_session_update_inventory_matches_the_pinned_schema() { + let schema: Value = + serde_json::from_str(include_str!("../tests/fixtures/acp/v1/schema.json")).unwrap(); + let pinned = schema["$defs"]["SessionUpdate"]["oneOf"] + .as_array() + .unwrap() + .iter() + .map(|branch| { + branch["properties"]["sessionUpdate"]["const"] + .as_str() + .unwrap() + }) + .collect::>(); + let implemented = AcpV1SessionUpdateKind::ALL + .iter() + .map(|variant| variant.as_str()) + .collect::>(); + assert_eq!(implemented, pinned); + } + + #[test] + fn thought_update_projection_excludes_content_but_preserves_metadata() { + let update = serde_json::json!({ + "sessionUpdate": "agent_thought_chunk", + "messageId": "thought-1", + "content": {"type": "text", "text": "private"}, + "_meta": {"extension": "preserved"} + }); + let projected = session_update_projection( + AcpV1SessionUpdateKind::AgentThoughtChunk, + "agent_thought_chunk", + update.as_object().unwrap(), + ); + assert_eq!(projected["thoughtContentExcluded"], true); + assert!(projected["update"].get("content").is_none()); + assert_eq!(projected["update"]["_meta"]["extension"], "preserved"); + assert!(!projected.to_string().contains("private")); } #[test] @@ -2746,4 +4931,113 @@ mod tests { assert!(envelope.outcome.checkpoint.is_none()); assert!(turn.checkpoint.is_none()); } + + #[test] + fn acp_checkpoint_failure_is_durable_and_not_no_changes() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let cwd = temp.path().to_string_lossy().to_string(); + let lane = "agent-checkpoint-error"; + let mut coordinator = CaptureCoordinator::new(AcpRelayOptions { + workspace_root: temp.path().to_path_buf(), + db_dir: temp.path().join(".trail"), + lane: Some(lane.to_string()), + from_ref: None, + provider: Some("codex".to_string()), + model: Some("gpt-test".to_string()), + materialize: true, + workdir: None, + inject_mcp: false, + upstream_command: vec!["codex".to_string()], + upstream_env: BTreeMap::new(), + }) + .unwrap(); + + let mut session_request = serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "session/new", + "params": { + "sessionId": "client-session", + "cwd": cwd + } + }); + coordinator + .before_client_message(&mut session_request) + .unwrap(); + let mut session_response = serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "result": { + "sessionId": "upstream-session" + } + }); + coordinator + .before_agent_message(&mut session_response) + .unwrap(); + + let mut prompt_request = serde_json::json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "session/prompt", + "params": { + "sessionId": "upstream-session", + "prompt": [ + { "type": "text", "text": "Create RECOVERABLE.md" } + ] + } + }); + coordinator + .before_client_message(&mut prompt_request) + .unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let workdir = PathBuf::from(db.lane_details(lane).unwrap().branch.workdir.unwrap()); + fs::write(workdir.join("RECOVERABLE.md"), "preserve me\n").unwrap(); + let branch = db.lane_branch(lane).unwrap(); + make_ref_root_legacy(&db, &branch.ref_name); + drop(db); + + let mut prompt_response = serde_json::json!({ + "jsonrpc": "2.0", + "id": 2, + "result": { + "stopReason": "end_turn" + } + }); + coordinator + .before_agent_message(&mut prompt_response) + .unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let transcript = db.transcript(lane).unwrap(); + assert_eq!(transcript.turns.len(), 1); + let turn = &transcript.turns[0]; + let envelope = turn.turn_envelope.as_ref().unwrap(); + assert_eq!(envelope.outcome.status.as_deref(), Some("failed")); + assert!(!envelope.outcome.no_changes); + assert!(envelope.outcome.checkpoint.is_none()); + assert!(envelope + .outcome + .error_summary + .as_deref() + .is_some_and(|summary| summary.contains("PATH_INDEX_REQUIRED"))); + assert!(turn.checkpoint.is_none()); + let failure_event = turn + .events + .iter() + .find(|event| event.event_type == "acp_workdir_checkpoint_failed") + .expect("checkpoint failure must be durable"); + assert_eq!( + failure_event.payload.as_ref().unwrap()["error_code"], + "PATH_INDEX_REQUIRED" + ); + let status = db.lane_status(lane).unwrap(); + assert_eq!(status.workdir_state, Some(WorktreeState::DirtyUntracked)); + assert!(status + .workdir_changed_paths + .iter() + .any(|path| path.path == "RECOVERABLE.md")); + } } diff --git a/trail/src/acp/capture.rs b/trail/src/acp/capture.rs new file mode 100644 index 0000000..aac152d --- /dev/null +++ b/trail/src/acp/capture.rs @@ -0,0 +1,1403 @@ +use std::collections::{BTreeMap, HashSet, VecDeque}; +use std::fs::{self, File, OpenOptions}; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{mpsc, Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use super::protocol::Direction; +use super::transport::RelayFinishReason; +use super::CaptureCoordinator; +use crate::model::{AgentCaptureTransport, AgentHookReceiptInput}; +use crate::{Error, Result, Trail}; + +pub(crate) const ACP_CAPTURE_QUEUE_CAPACITY: usize = 4_096; +const CAPTURE_MESSAGE_LIMIT: usize = 512 * 1024; +const CAPTURE_PROJECT_LOCK_WAIT: Duration = Duration::from_millis(250); +const CAPTURE_RETRY_INTERVAL: Duration = Duration::from_millis(25); +const CAPTURE_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2); +const CAPTURE_SHUTDOWN_DRAIN_BUDGET: Duration = Duration::from_millis(500); +static SPILL_CLAIM_COUNTER: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct CapturedFrame { + pub connection_id: String, + pub direction: Direction, + pub sequence: u64, + pub received_at: i64, + pub redacted_message: Value, + pub project: bool, +} + +pub(crate) enum CaptureCommand { + Frame(CapturedFrame), + Finish(RelayFinishReason), + Barrier(mpsc::Sender<()>), + #[cfg(test)] + SimulateWorkerPanic, +} + +#[derive(Default)] +pub(crate) struct CaptureHealth { + healthy: AtomicBool, + degraded: AtomicBool, + last_error: Mutex>, + queued: AtomicUsize, + spilled: AtomicUsize, + last_projected_sequence: AtomicU64, +} + +impl CaptureHealth { + fn new() -> Self { + Self { + healthy: AtomicBool::new(true), + ..Self::default() + } + } + + fn record_error(&self, error: &Error) -> bool { + self.healthy.store(false, Ordering::Release); + let first = !self.degraded.swap(true, Ordering::AcqRel); + if let Ok(mut last_error) = self.last_error.lock() { + *last_error = Some(error.to_string()); + } + first + } +} + +#[allow(dead_code)] +pub(crate) struct CaptureShutdownReport { + pub healthy: bool, + pub degraded: bool, + pub queued: usize, + pub spilled: usize, + pub last_projected_sequence: u64, +} + +pub(crate) struct CaptureIngress { + tx: Option>, + health: Arc, + worker: Option>, + spill: Arc, + spill_mode: Arc, + stopping: Arc, + pending_finish: Arc>>, + done_rx: Mutex>, +} + +impl CaptureIngress { + pub(crate) fn new( + _workspace_root: PathBuf, + db_dir: PathBuf, + coordinator: Arc>, + connection_id: String, + ) -> Result { + let spill = Arc::new(SpillStore::new(db_dir.join("acp-ingress"), connection_id)?); + let health = Arc::new(CaptureHealth::new()); + let spill_mode = Arc::new(AtomicBool::new(false)); + let stopping = Arc::new(AtomicBool::new(false)); + let pending_finish = Arc::new(Mutex::new(None)); + let (tx, rx) = mpsc::sync_channel(ACP_CAPTURE_QUEUE_CAPACITY); + let (done_tx, done_rx) = mpsc::channel(); + let worker_health = Arc::clone(&health); + let worker_spill = Arc::clone(&spill); + let worker_spill_mode = Arc::clone(&spill_mode); + let worker_stopping = Arc::clone(&stopping); + let worker_pending_finish = Arc::clone(&pending_finish); + let worker = thread::Builder::new() + .name("trail-acp-capture".to_string()) + .spawn(move || { + let panic_health = Arc::clone(&worker_health); + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + capture_worker( + rx, + coordinator, + worker_health, + worker_spill, + worker_spill_mode, + worker_stopping, + worker_pending_finish, + ); + })); + if outcome.is_err() { + panic_health.record_error(&Error::InvalidInput( + "ACP capture worker panicked".to_string(), + )); + } + let _ = done_tx.send(()); + }) + .map_err(Error::Io)?; + Ok(Self { + tx: Some(tx), + health, + worker: Some(worker), + spill, + spill_mode, + stopping, + pending_finish, + done_rx: Mutex::new(done_rx), + }) + } + + pub(crate) fn append(&self, frame: CapturedFrame) -> Result<()> { + let Some(tx) = &self.tx else { + return Err(Error::InvalidInput( + "ACP capture ingress is shut down".to_string(), + )); + }; + if self.spill_mode.load(Ordering::Acquire) { + self.spill_or_degrade(&frame); + return Ok(()); + } + self.health.queued.fetch_add(1, Ordering::Relaxed); + match tx.try_send(CaptureCommand::Frame(frame)) { + Ok(()) => Ok(()), + Err(mpsc::TrySendError::Full(CaptureCommand::Frame(frame))) + | Err(mpsc::TrySendError::Disconnected(CaptureCommand::Frame(frame))) => { + self.health.queued.fetch_sub(1, Ordering::Relaxed); + self.spill_mode.store(true, Ordering::Release); + self.spill_or_degrade(&frame); + Ok(()) + } + Err(_) => { + self.health.queued.fetch_sub(1, Ordering::Relaxed); + Ok(()) + } + } + } + + fn spill_or_degrade(&self, frame: &CapturedFrame) { + match self.spill.append(frame) { + Ok(()) => { + self.health.spilled.fetch_add(1, Ordering::Relaxed); + } + Err(error) => { + if self.health.record_error(&error) { + eprintln!("trail acp capture warning: durable spill failed: {error}"); + } + } + } + } + + pub(crate) fn finish(&self, reason: RelayFinishReason) { + if let Err(error) = self.spill.persist_finish(&reason) + && self.health.record_error(&error) + { + eprintln!("trail acp capture warning: durable finish journal failed: {error}"); + } + if let Some(tx) = &self.tx { + if let Err(error) = tx.try_send(CaptureCommand::Finish(reason)) { + let reason = match error { + mpsc::TrySendError::Full(CaptureCommand::Finish(reason)) + | mpsc::TrySendError::Disconnected(CaptureCommand::Finish(reason)) => reason, + _ => return, + }; + if let Ok(mut pending) = self.pending_finish.lock() { + *pending = Some(reason); + } + } + } + } + + pub(crate) fn flush(&self, timeout: Duration) -> bool { + let Some(tx) = &self.tx else { + return false; + }; + let deadline = Instant::now() + timeout; + let (barrier_tx, barrier_rx) = mpsc::channel(); + loop { + match tx.try_send(CaptureCommand::Barrier(barrier_tx.clone())) { + Ok(()) => break, + Err(mpsc::TrySendError::Full(_)) if Instant::now() < deadline => { + thread::sleep(Duration::from_millis(1)); + } + Err(_) => return false, + } + } + let remaining = deadline.saturating_duration_since(Instant::now()); + barrier_rx.recv_timeout(remaining).is_ok() + } + + #[allow(dead_code)] + pub(crate) fn shutdown(mut self, timeout: Duration) -> CaptureShutdownReport { + self.stopping.store(true, Ordering::Release); + self.tx.take(); + if self + .done_rx + .get_mut() + .is_ok_and(|done_rx| done_rx.recv_timeout(timeout).is_ok()) + { + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } + self.report() + } + + #[allow(dead_code)] + fn report(&self) -> CaptureShutdownReport { + CaptureShutdownReport { + healthy: self.health.healthy.load(Ordering::Acquire), + degraded: self.health.degraded.load(Ordering::Acquire), + queued: self.health.queued.load(Ordering::Acquire), + spilled: self.health.spilled.load(Ordering::Acquire), + last_projected_sequence: self.health.last_projected_sequence.load(Ordering::Acquire), + } + } +} + +impl Drop for CaptureIngress { + fn drop(&mut self) { + self.stopping.store(true, Ordering::Release); + self.tx.take(); + if self + .done_rx + .get_mut() + .is_ok_and(|done_rx| done_rx.recv_timeout(CAPTURE_SHUTDOWN_TIMEOUT).is_ok()) + { + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } + } +} + +struct SpillStore { + dir: PathBuf, + connection_id: String, + owner_path: PathBuf, + _owner_lock: File, + state: Mutex, +} + +#[derive(Default)] +struct SpillState { + claimed_paths: Vec, + claimed_finish_paths: Vec, + recovered_finishes: VecDeque, + claimed_owners: Vec<(PathBuf, File)>, +} + +impl SpillStore { + fn new(dir: PathBuf, connection_id: String) -> Result { + fs::create_dir_all(&dir)?; + let owner_path = dir.join(format!("{connection_id}.owner")); + let owner_lock = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&owner_path)?; + lock_spill_owner(&owner_lock)?; + Ok(Self { + dir, + connection_id, + owner_path, + _owner_lock: owner_lock, + state: Mutex::new(SpillState::default()), + }) + } + + fn append(&self, frame: &CapturedFrame) -> Result<()> { + let _guard = self + .state + .lock() + .map_err(|_| Error::InvalidInput("ACP spill lock poisoned".to_string()))?; + self.append_locked(std::slice::from_ref(frame)) + } + + fn append_many(&self, frames: &[CapturedFrame]) -> Result<()> { + let _guard = self + .state + .lock() + .map_err(|_| Error::InvalidInput("ACP spill lock poisoned".to_string()))?; + self.append_locked(frames) + } + + fn append_locked(&self, frames: &[CapturedFrame]) -> Result<()> { + let mut grouped = BTreeMap::>::new(); + for frame in frames { + grouped + .entry(self.path_for(&frame.connection_id)) + .or_default() + .push(frame); + } + for (path, frames) in grouped { + let mut file = OpenOptions::new().create(true).append(true).open(path)?; + for frame in frames { + serde_json::to_writer(&mut file, frame)?; + file.write_all(b"\n")?; + } + file.sync_data()?; + } + Ok(()) + } + + fn persist_finish(&self, reason: &RelayFinishReason) -> Result<()> { + let _guard = self + .state + .lock() + .map_err(|_| Error::InvalidInput("ACP spill lock poisoned".to_string()))?; + let counter = SPILL_CLAIM_COUNTER.fetch_add(1, Ordering::Relaxed); + let temporary = self.dir.join(format!( + "{}.finish.{}.{}.tmp", + self.connection_id, + std::process::id(), + counter + )); + let final_path = self.finish_path_for(&self.connection_id); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary)?; + serde_json::to_writer(&mut file, reason)?; + file.write_all(b"\n")?; + file.sync_all()?; + fs::rename(&temporary, &final_path)?; + File::open(&self.dir)?.sync_all()?; + Ok(()) + } + + fn take_all(&self) -> Result> { + let mut state = self + .state + .lock() + .map_err(|_| Error::InvalidInput("ACP spill lock poisoned".to_string()))?; + if !state.claimed_paths.is_empty() || !state.claimed_finish_paths.is_empty() { + return Ok(Vec::new()); + } + let mut paths = fs::read_dir(&self.dir)? + .filter_map(|entry| entry.ok().map(|entry| entry.path())) + .filter(|path| { + spill_connection_id(path).is_some() + && path.extension().is_some_and(|extension| { + extension == "jsonl" || extension == "json" || extension == "processing" + }) + }) + .collect::>(); + paths.sort(); + let mut claimed = Vec::new(); + let mut claimed_finishes = Vec::new(); + let mut recoverable_connections = HashSet::new(); + recoverable_connections.insert(self.connection_id.clone()); + let mut active_connections = HashSet::new(); + let mut claimed_owners = Vec::new(); + for path in paths { + let Some(connection_id) = spill_connection_id(&path) else { + continue; + }; + if !recoverable_connections.contains(&connection_id) + && !active_connections.contains(&connection_id) + { + let owner_path = self.dir.join(format!("{connection_id}.owner")); + let owner = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&owner_path)?; + if try_lock_spill_owner(&owner)? { + recoverable_connections.insert(connection_id.clone()); + claimed_owners.push((owner_path, owner)); + } else { + active_connections.insert(connection_id.clone()); + } + } + if active_connections.contains(&connection_id) { + continue; + } + let finish = spill_finish_path(&path); + if !path + .extension() + .is_some_and(|extension| extension == "processing") + { + let counter = SPILL_CLAIM_COUNTER.fetch_add(1, Ordering::Relaxed); + let claimed_path = path.with_extension(format!( + "{}.{}.{}.processing", + path.extension() + .and_then(|value| value.to_str()) + .unwrap_or("spill"), + std::process::id(), + counter + )); + fs::rename(&path, &claimed_path)?; + if finish { + claimed_finishes.push(claimed_path); + } else { + claimed.push(claimed_path); + } + } else if finish { + claimed_finishes.push(path); + } else { + claimed.push(path); + } + } + let mut frames = Vec::new(); + for path in &claimed { + let file = OpenOptions::new().read(true).open(path)?; + for line in BufReader::new(file).lines() { + let line = line?; + if !line.trim().is_empty() { + frames.push(serde_json::from_str(&line)?); + } + } + } + let mut finishes = VecDeque::new(); + for path in &claimed_finishes { + let file = OpenOptions::new().read(true).open(path)?; + finishes.push_back(serde_json::from_reader(file)?); + } + state.claimed_paths = claimed; + state.claimed_finish_paths = claimed_finishes; + state.recovered_finishes = finishes; + state.claimed_owners = claimed_owners; + Ok(frames) + } + + fn take_recovered_finishes(&self) -> Vec { + self.state + .lock() + .map(|mut state| state.recovered_finishes.drain(..).collect()) + .unwrap_or_default() + } + + fn complete_claimed_frames(&self) -> Result<()> { + let mut state = self + .state + .lock() + .map_err(|_| Error::InvalidInput("ACP spill lock poisoned".to_string()))?; + for path in state.claimed_paths.drain(..) { + match fs::remove_file(path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(Error::Io(error)), + } + } + if state.claimed_finish_paths.is_empty() { + remove_claimed_owners(&mut state)?; + } + Ok(()) + } + + fn complete_finish(&self) -> Result<()> { + let mut state = self + .state + .lock() + .map_err(|_| Error::InvalidInput("ACP spill lock poisoned".to_string()))?; + for path in state.claimed_finish_paths.drain(..) { + match fs::remove_file(path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(Error::Io(error)), + } + } + match fs::remove_file(self.finish_path_for(&self.connection_id)) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(Error::Io(error)), + } + state.recovered_finishes.clear(); + remove_claimed_owners(&mut state)?; + File::open(&self.dir)?.sync_all()?; + Ok(()) + } + + fn finish_path_for(&self, connection_id: &str) -> PathBuf { + self.dir.join(format!("{connection_id}.finish.json")) + } + + fn release_claimed(&self) { + if let Ok(mut state) = self.state.lock() { + state.claimed_paths.clear(); + state.claimed_finish_paths.clear(); + state.recovered_finishes.clear(); + state.claimed_owners.clear(); + } + } + + fn path_for(&self, connection_id: &str) -> PathBuf { + self.dir.join(format!("{connection_id}.jsonl")) + } +} + +fn remove_claimed_owners(state: &mut SpillState) -> Result<()> { + for (owner_path, _owner) in state.claimed_owners.drain(..) { + match fs::remove_file(owner_path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(Error::Io(error)), + } + } + Ok(()) +} + +impl Drop for SpillStore { + fn drop(&mut self) { + let _ = fs::remove_file(&self.owner_path); + } +} + +fn spill_connection_id(path: &Path) -> Option { + let name = path.file_name()?.to_str()?; + let connection_id = name + .split_once(".jsonl") + .or_else(|| name.split_once(".finish.json"))? + .0; + (!connection_id.is_empty()).then(|| connection_id.to_string()) +} + +fn spill_finish_path(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.contains(".finish.json")) +} + +#[cfg(unix)] +fn lock_spill_owner(file: &File) -> Result<()> { + match rustix::fs::flock(file, rustix::fs::FlockOperation::NonBlockingLockExclusive) { + Ok(()) => Ok(()), + Err(error) if error == rustix::io::Errno::WOULDBLOCK => Err(Error::InvalidInput( + "ACP capture connection identity is already active".to_string(), + )), + Err(error) => Err(Error::Io(error.into())), + } +} + +#[cfg(not(unix))] +fn lock_spill_owner(_file: &File) -> Result<()> { + Ok(()) +} + +#[cfg(unix)] +fn try_lock_spill_owner(file: &File) -> Result { + match rustix::fs::flock(file, rustix::fs::FlockOperation::NonBlockingLockExclusive) { + Ok(()) => Ok(true), + Err(error) if error == rustix::io::Errno::WOULDBLOCK => Ok(false), + Err(error) => Err(Error::Io(error.into())), + } +} + +#[cfg(not(unix))] +fn try_lock_spill_owner(_file: &File) -> Result { + Ok(true) +} + +#[allow(clippy::too_many_arguments)] +fn capture_worker( + rx: mpsc::Receiver, + coordinator: Arc>, + health: Arc, + spill: Arc, + spill_mode: Arc, + stopping: Arc, + pending_finish: Arc>>, +) { + let mut pending = match recovery_frames(&coordinator, &spill) { + Ok(frames) => frames.into_iter().map(|frame| (frame, false)).collect(), + Err(error) => { + if health.record_error(&error) { + eprintln!("trail acp capture warning: recovery failed: {error}"); + } + VecDeque::new() + } + }; + load_recovered_finish(&spill, &pending_finish); + let mut deferred_barriers = Vec::new(); + + loop { + if stopping.load(Ordering::Acquire) { + let mut recovered_spill = true; + match spill.take_all() { + Ok(frames) => { + pending.extend(frames.into_iter().map(|frame| (frame, false))); + load_recovered_finish(&spill, &pending_finish); + } + Err(error) => { + recovered_spill = false; + if health.record_error(&error) { + eprintln!( + "trail acp capture warning: shutdown spill replay failed: {error}" + ); + } + } + } + for command in rx.try_iter() { + match command { + CaptureCommand::Frame(frame) => pending.push_back((frame, true)), + CaptureCommand::Finish(reason) => store_finish(&pending_finish, reason), + CaptureCommand::Barrier(barrier) => { + deferred_barriers.push(barrier); + } + #[cfg(test)] + CaptureCommand::SimulateWorkerPanic => { + panic!("simulated ACP capture worker panic") + } + } + } + sort_pending_frames(&mut pending); + let drain_deadline = Instant::now() + CAPTURE_SHUTDOWN_DRAIN_BUDGET; + while Instant::now() + CAPTURE_PROJECT_LOCK_WAIT < drain_deadline { + let Some((frame, queued)) = pending.pop_front() else { + break; + }; + if queued { + health.queued.fetch_sub(1, Ordering::Relaxed); + } + match process_frame(&coordinator, &frame) { + Ok(()) => { + health.healthy.store(true, Ordering::Release); + health + .last_projected_sequence + .store(frame.sequence, Ordering::Release); + } + Err(error) => { + pending.push_front((frame, false)); + if health.record_error(&error) { + eprintln!("trail acp capture warning: {error}"); + } + break; + } + } + } + let frames = pending + .drain(..) + .map(|(frame, queued)| { + if queued { + health.queued.fetch_sub(1, Ordering::Relaxed); + } + frame + }) + .collect::>(); + let preserved_every_frame = recovered_spill + && match spill.append_many(&frames) { + Err(error) => { + if health.record_error(&error) { + eprintln!("trail acp capture warning: shutdown spill failed: {error}"); + } + false + } + _ => { + if frames.is_empty() { + let _ = spill.complete_claimed_frames(); + } + frames.is_empty() + } + }; + if preserved_every_frame + && settle_pending_finish(&coordinator, &health, &spill, &pending_finish) + { + acknowledge_barriers(&mut deferred_barriers); + } + break; + } + + let mut merged_frames = false; + if spill_mode.load(Ordering::Acquire) { + for command in rx.try_iter() { + match command { + CaptureCommand::Frame(frame) => { + pending.push_back((frame, true)); + merged_frames = true; + } + CaptureCommand::Finish(reason) => store_finish(&pending_finish, reason), + CaptureCommand::Barrier(barrier) => deferred_barriers.push(barrier), + #[cfg(test)] + CaptureCommand::SimulateWorkerPanic => { + panic!("simulated ACP capture worker panic") + } + } + } + match spill.take_all() { + Ok(frames) => { + merged_frames |= !frames.is_empty(); + pending.extend(frames.into_iter().map(|frame| (frame, false))); + load_recovered_finish(&spill, &pending_finish); + } + Err(error) => { + if health.record_error(&error) { + eprintln!("trail acp capture warning: spill replay failed: {error}"); + } + } + } + } + if merged_frames { + sort_pending_frames(&mut pending); + } + + if let Some((frame, queued)) = pending.pop_front() { + if queued { + health.queued.fetch_sub(1, Ordering::Relaxed); + } + match process_frame(&coordinator, &frame) { + Ok(()) => { + health.healthy.store(true, Ordering::Release); + health + .last_projected_sequence + .store(frame.sequence, Ordering::Release); + if pending.is_empty() { + match spill.complete_claimed_frames() { + Err(error) => { + if health.record_error(&error) { + eprintln!( + "trail acp capture warning: spill acknowledgement failed: {error}" + ); + } + } + _ => { + spill_mode.store(false, Ordering::Release); + } + } + } + } + Err(error) => { + let mut retry = vec![frame]; + retry.extend(pending.drain(..).map(|(frame, queued)| { + if queued { + health.queued.fetch_sub(1, Ordering::Relaxed); + } + frame + })); + match spill.append_many(&retry) { + Err(spill_error) => { + if health.record_error(&spill_error) { + eprintln!( + "trail acp capture warning: {error}; failed to preserve spill: {spill_error}" + ); + } + } + _ => { + if health.record_error(&error) { + eprintln!("trail acp capture warning: {error}"); + } + } + } + spill.release_claimed(); + spill_mode.store(true, Ordering::Release); + } + } + continue; + } + + if !spill_mode.load(Ordering::Acquire) + && settle_pending_finish(&coordinator, &health, &spill, &pending_finish) + { + acknowledge_barriers(&mut deferred_barriers); + } + + match rx.recv_timeout(CAPTURE_RETRY_INTERVAL) { + Ok(CaptureCommand::Frame(frame)) => pending.push_back((frame, true)), + Ok(CaptureCommand::Finish(reason)) => store_finish(&pending_finish, reason), + Ok(CaptureCommand::Barrier(barrier)) => { + deferred_barriers.push(barrier); + } + #[cfg(test)] + Ok(CaptureCommand::SimulateWorkerPanic) => { + panic!("simulated ACP capture worker panic") + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + stopping.store(true, Ordering::Release); + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + } +} + +fn sort_pending_frames(pending: &mut VecDeque<(CapturedFrame, bool)>) { + let mut ordered = pending.drain(..).collect::>(); + ordered.sort_by(|(left, _), (right, _)| { + if left.connection_id == right.connection_id { + left.sequence + .cmp(&right.sequence) + .then_with(|| direction_name(left.direction).cmp(direction_name(right.direction))) + } else { + left.received_at + .cmp(&right.received_at) + .then_with(|| left.connection_id.cmp(&right.connection_id)) + } + }); + pending.extend(ordered); +} + +fn settle_pending_finish( + coordinator: &Arc>, + health: &CaptureHealth, + spill: &SpillStore, + pending_finish: &Mutex>, +) -> bool { + let Some(reason) = take_finish(pending_finish) else { + return true; + }; + match process_finish(coordinator, reason.clone()).and_then(|()| spill.complete_finish()) { + Ok(()) => true, + Err(error) => { + store_finish(pending_finish, reason); + health.record_error(&error); + false + } + } +} + +fn load_recovered_finish(spill: &SpillStore, pending_finish: &Mutex>) { + for reason in spill.take_recovered_finishes() { + store_finish(pending_finish, reason); + } +} + +fn acknowledge_barriers(barriers: &mut Vec>) { + for barrier in barriers.drain(..) { + let _ = barrier.send(()); + } +} + +fn process_frame( + coordinator: &Arc>, + frame: &CapturedFrame, +) -> Result<()> { + Trail::with_write_lock_wait(CAPTURE_PROJECT_LOCK_WAIT, || { + let mut db = { + let capture = coordinator.lock().map_err(|_| { + Error::InvalidInput("ACP capture coordinator lock poisoned".to_string()) + })?; + capture.open_db()? + }; + let direction = direction_name(frame.direction); + let report = db.persist_agent_hook_receipt(AgentHookReceiptInput { + installation_id: None, + provider: "trail-acp".to_string(), + native_event: "acp/frame".to_string(), + native_session_id: session_id(&frame.redacted_message), + native_turn_id: None, + transport: AgentCaptureTransport::Acp, + connection_id: Some(frame.connection_id.clone()), + direction: Some(direction.to_string()), + connection_sequence: Some(frame.sequence), + dedupe_key: format!( + "acp:{}:{}:{}", + frame.connection_id, direction, frame.sequence + ), + payload: serde_json::json!({ + "connection_id": frame.connection_id, + "direction": direction, + "sequence": frame.sequence, + "received_at": frame.received_at, + "message": frame.redacted_message.clone(), + "project": frame.project + }), + occurred_at: Some(frame.received_at), + })?; + if frame.project && report.receipt.status != "processed" { + let mut message = frame.redacted_message.clone(); + let mut capture = coordinator.lock().map_err(|_| { + Error::InvalidInput("ACP capture coordinator lock poisoned".to_string()) + })?; + match frame.direction { + Direction::ClientToAgent => capture.before_client_message(&mut message)?, + Direction::AgentToClient => capture.before_agent_message(&mut message)?, + } + } + db.mark_agent_hook_receipt_processed(&report.receipt.receipt_id)?; + Ok(()) + }) +} + +fn recovery_frames( + coordinator: &Arc>, + spill: &SpillStore, +) -> Result> { + let db = { + let capture = coordinator.lock().map_err(|_| { + Error::InvalidInput("ACP capture coordinator lock poisoned".to_string()) + })?; + capture.open_db()? + }; + let payloads = db.pending_acp_capture_payloads()?; + let mut frames = spill.take_all()?; + for payload in payloads { + frames.push(frame_from_receipt_payload(payload)?); + } + let mut seen = HashSet::new(); + frames.retain(|frame| { + seen.insert((frame.connection_id.clone(), frame.direction, frame.sequence)) + }); + frames.sort_by(|left, right| { + if left.connection_id == right.connection_id { + left.sequence + .cmp(&right.sequence) + .then_with(|| direction_name(left.direction).cmp(direction_name(right.direction))) + } else { + left.received_at + .cmp(&right.received_at) + .then_with(|| left.connection_id.cmp(&right.connection_id)) + } + }); + Ok(frames) +} + +fn frame_from_receipt_payload(payload: Value) -> Result { + let direction = match payload.get("direction").and_then(Value::as_str) { + Some("client_to_agent") => Direction::ClientToAgent, + Some("agent_to_client") => Direction::AgentToClient, + other => { + return Err(Error::Corrupt(format!( + "ACP receipt has invalid direction {other:?}" + ))); + } + }; + Ok(CapturedFrame { + connection_id: payload + .get("connection_id") + .and_then(Value::as_str) + .ok_or_else(|| Error::Corrupt("ACP receipt is missing connection_id".to_string()))? + .to_string(), + direction, + sequence: payload + .get("sequence") + .and_then(Value::as_u64) + .ok_or_else(|| Error::Corrupt("ACP receipt is missing sequence".to_string()))?, + received_at: payload + .get("received_at") + .and_then(Value::as_i64) + .ok_or_else(|| Error::Corrupt("ACP receipt is missing received_at".to_string()))?, + redacted_message: payload + .get("message") + .cloned() + .ok_or_else(|| Error::Corrupt("ACP receipt is missing message".to_string()))?, + project: payload + .get("project") + .and_then(Value::as_bool) + .unwrap_or(true), + }) +} + +fn store_finish(target: &Mutex>, reason: RelayFinishReason) { + if let Ok(mut pending) = target.lock() { + *pending = Some(reason); + } +} + +fn take_finish(target: &Mutex>) -> Option { + target.lock().ok().and_then(|mut pending| pending.take()) +} + +fn process_finish( + coordinator: &Arc>, + reason: RelayFinishReason, +) -> Result<()> { + let (status, summary) = match reason { + RelayFinishReason::EditorEof => ("cancelled", "editor input closed"), + RelayFinishReason::EditorError(_) => ("failed", "editor sent malformed JSON"), + RelayFinishReason::AgentEof => ("failed", "upstream output closed"), + RelayFinishReason::AgentError(_) => ("failed", "upstream sent malformed JSON"), + }; + let mut capture = coordinator + .lock() + .map_err(|_| Error::InvalidInput("ACP capture coordinator lock poisoned".to_string()))?; + Trail::with_write_lock_wait(CAPTURE_PROJECT_LOCK_WAIT, || { + capture.finish_open_turns(status, summary) + }) +} + +pub(crate) fn capture_frame( + connection_id: &str, + direction: Direction, + sequence: u64, + message: &Value, + project: bool, +) -> CapturedFrame { + CapturedFrame { + connection_id: connection_id.to_string(), + direction, + sequence, + received_at: now_millis(), + redacted_message: bounded_redacted_message(message), + project, + } +} + +fn bounded_redacted_message(message: &Value) -> Value { + let mut redacted = super::redact_json(message.clone()); + redact_callback_secrets(&mut redacted); + let bytes = serde_json::to_vec(&redacted).unwrap_or_default(); + if bytes.len() <= CAPTURE_MESSAGE_LIMIT { + return redacted; + } + serde_json::json!({ + "jsonrpc": redacted.get("jsonrpc"), + "id": redacted.get("id"), + "method": redacted.get("method"), + "truncated": true, + "bytes": bytes.len(), + "sha256": hex::encode(Sha256::digest(bytes)) + }) +} + +fn redact_callback_secrets(message: &mut Value) { + match message.get("method").and_then(Value::as_str) { + Some("fs/write_text_file") => { + let Some(content) = message + .pointer("/params/content") + .and_then(Value::as_str) + .map(str::to_string) + else { + return; + }; + let redacted = crate::db::redact_sensitive_text(&content); + if let Some(params) = message.get_mut("params").and_then(Value::as_object_mut) { + params.insert( + "content".to_string(), + serde_json::json!({ + "redacted": true, + "byte_len": content.len(), + "sha256": hex::encode(Sha256::digest(redacted.as_bytes())) + }), + ); + } + } + Some("terminal/create") => { + let Some(env) = message + .pointer_mut("/params/env") + .and_then(Value::as_array_mut) + else { + return; + }; + for variable in env { + if let Some(variable) = variable.as_object_mut() { + if variable.contains_key("value") { + variable + .insert("value".to_string(), Value::String("[REDACTED]".to_string())); + } + } + } + } + _ => {} + } +} + +fn direction_name(direction: Direction) -> &'static str { + match direction { + Direction::ClientToAgent => "client_to_agent", + Direction::AgentToClient => "agent_to_client", + } +} + +fn session_id(message: &Value) -> Option { + message + .pointer("/params/sessionId") + .or_else(|| message.pointer("/result/sessionId")) + .and_then(Value::as_str) + .map(str::to_string) +} + +fn now_millis() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| i64::try_from(duration.as_millis()).unwrap_or(i64::MAX)) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::acp::AcpRelayOptions; + use crate::InitImportMode; + + #[test] + fn callback_receipts_replace_file_content_and_terminal_environment_values() { + let write = bounded_redacted_message(&serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "fs/write_text_file", + "params": { + "sessionId": "s", + "path": "/repo/a.txt", + "content": "api_key=super-secret" + } + })); + assert_eq!(write["params"]["content"]["redacted"], true); + assert!(!write.to_string().contains("super-secret")); + + let terminal = bounded_redacted_message(&serde_json::json!({ + "jsonrpc": "2.0", + "id": "terminal", + "method": "terminal/create", + "params": { + "sessionId": "s", + "command": "echo", + "env": [{"name": "API_TOKEN", "value": "opaque-secret"}] + } + })); + assert_eq!(terminal["params"]["env"][0]["value"], "[REDACTED]"); + assert!(!terminal.to_string().contains("opaque-secret")); + } + + #[test] + fn queue_overflow_spills_every_frame_and_shutdown_is_bounded() { + const FRAME_COUNT: u64 = ACP_CAPTURE_QUEUE_CAPACITY as u64 + 2; + + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "overflow fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let options = AcpRelayOptions { + workspace_root: temp.path().to_path_buf(), + db_dir: temp.path().join(".trail"), + lane: None, + from_ref: None, + provider: Some("fixture".to_string()), + model: None, + materialize: false, + workdir: None, + inject_mcp: false, + upstream_command: vec!["fixture".to_string()], + upstream_env: BTreeMap::new(), + }; + let coordinator = Arc::new(Mutex::new(CaptureCoordinator::new(options).unwrap())); + fs::write( + temp.path().join(".trail/lock"), + format!("pid={} created_at=0\n", std::process::id()), + ) + .unwrap(); + let ingress = CaptureIngress::new( + temp.path().to_path_buf(), + temp.path().join(".trail"), + coordinator, + "overflow-connection".to_string(), + ) + .unwrap(); + for sequence in 0..FRAME_COUNT { + ingress + .append(capture_frame( + "overflow-connection", + Direction::AgentToClient, + sequence, + &serde_json::json!({ + "jsonrpc": "2.0", + "method": "session/update", + "params": {"sequence": sequence} + }), + false, + )) + .unwrap(); + } + let shutdown_started = Instant::now(); + let report = ingress.shutdown(CAPTURE_SHUTDOWN_TIMEOUT); + assert!(shutdown_started.elapsed() < CAPTURE_SHUTDOWN_TIMEOUT + Duration::from_millis(250)); + assert!( + report.spilled >= 1, + "the bounded queue never entered spill mode" + ); + fs::remove_file(temp.path().join(".trail/lock")).unwrap(); + + let preserved = fs::read_dir(temp.path().join(".trail/acp-ingress")) + .unwrap() + .filter_map(|entry| entry.ok()) + .map(|entry| fs::read_to_string(entry.path()).unwrap()) + .flat_map(|contents| { + contents + .lines() + .filter(|line| !line.trim().is_empty()) + .map(str::to_string) + .collect::>() + }) + .filter_map(|line| serde_json::from_str::(&line).ok()) + .map(|frame| frame.sequence) + .collect::>(); + assert_eq!(preserved.len(), usize::try_from(FRAME_COUNT).unwrap()); + } + + fn test_ingress(temp: &Path, connection_id: &str) -> CaptureIngress { + fs::write(temp.join("README.md"), "capture fault fixture\n").unwrap(); + Trail::init(temp, "main", InitImportMode::WorkingTree, false).unwrap(); + let options = AcpRelayOptions { + workspace_root: temp.to_path_buf(), + db_dir: temp.join(".trail"), + lane: None, + from_ref: None, + provider: Some("fixture".to_string()), + model: None, + materialize: false, + workdir: None, + inject_mcp: false, + upstream_command: vec!["fixture".to_string()], + upstream_env: BTreeMap::new(), + }; + CaptureIngress::new( + temp.to_path_buf(), + temp.join(".trail"), + Arc::new(Mutex::new(CaptureCoordinator::new(options).unwrap())), + connection_id.to_string(), + ) + .unwrap() + } + + #[test] + fn worker_panic_degrades_capture_and_subsequent_frames_spill_durably() { + let temp = tempfile::tempdir().unwrap(); + let ingress = test_ingress(temp.path(), "panic-connection"); + ingress + .tx + .as_ref() + .unwrap() + .send(CaptureCommand::SimulateWorkerPanic) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(1); + while !ingress.worker.as_ref().unwrap().is_finished() { + assert!(Instant::now() < deadline, "capture worker did not panic"); + thread::sleep(Duration::from_millis(1)); + } + ingress + .append(capture_frame( + "panic-connection", + Direction::AgentToClient, + 1, + &serde_json::json!({"jsonrpc":"2.0","method":"ext/after-panic"}), + false, + )) + .unwrap(); + let report = ingress.shutdown(Duration::from_millis(100)); + assert!(report.degraded); + assert_eq!(report.spilled, 1); + } + + #[test] + fn spill_write_failure_marks_capture_unhealthy_without_blocking_forwarding() { + let temp = tempfile::tempdir().unwrap(); + let ingress = test_ingress(temp.path(), "spill-failure"); + let spill_dir = temp.path().join(".trail/acp-ingress"); + fs::remove_file(&ingress.spill.owner_path).unwrap(); + fs::remove_dir(&spill_dir).unwrap(); + fs::write(&spill_dir, "not a directory").unwrap(); + ingress.spill_mode.store(true, Ordering::Release); + + ingress + .append(capture_frame( + "spill-failure", + Direction::ClientToAgent, + 1, + &serde_json::json!({"jsonrpc":"2.0","method":"ext/spill-failure"}), + false, + )) + .unwrap(); + let report = ingress.shutdown(Duration::from_millis(250)); + assert!(!report.healthy); + assert!(report.degraded); + assert_eq!(report.spilled, 0); + } + + #[test] + fn barrier_waits_for_spill_replay_and_finish_projection() { + let temp = tempfile::tempdir().unwrap(); + let ingress = test_ingress(temp.path(), "barrier-order"); + let writer_lock = crate::db::acquire_workspace_lock(&temp.path().join(".trail")).unwrap(); + + ingress + .append(capture_frame( + "barrier-order", + Direction::AgentToClient, + 1, + &serde_json::json!({ + "jsonrpc": "2.0", + "method": "session/update", + "params": {"sequence": 1} + }), + false, + )) + .unwrap(); + let spill_deadline = Instant::now() + Duration::from_secs(1); + while !ingress.spill_mode.load(Ordering::Acquire) { + assert!( + Instant::now() < spill_deadline, + "capture worker never entered durable spill mode" + ); + thread::sleep(Duration::from_millis(1)); + } + + ingress.finish(RelayFinishReason::EditorEof); + assert!( + !ingress.flush(Duration::from_millis(100)), + "barrier acknowledged before the earlier spill and finish were projected" + ); + assert!( + ingress.spill.finish_path_for("barrier-order").is_file(), + "timed-out finalization lost its durable finish marker" + ); + drop(writer_lock); + assert!( + ingress.flush(Duration::from_secs(2)), + "barrier did not acknowledge after spill replay and finish projection" + ); + assert!( + !ingress.spill.finish_path_for("barrier-order").exists(), + "finish marker remained after terminal projection was acknowledged" + ); + } + + #[test] + fn spill_recovery_never_claims_another_live_connection() { + let temp = tempfile::tempdir().unwrap(); + let dir = temp.path().join("acp-ingress"); + let live = SpillStore::new(dir.clone(), "live-connection".to_string()).unwrap(); + live.append(&capture_frame( + "live-connection", + Direction::AgentToClient, + 7, + &serde_json::json!({"jsonrpc":"2.0","method":"ext/live"}), + false, + )) + .unwrap(); + + let recovery = SpillStore::new(dir, "recovery-connection".to_string()).unwrap(); + assert!( + recovery.take_all().unwrap().is_empty(), + "recovery worker claimed a spill owned by a live relay" + ); + drop(live); + + let recovered = recovery.take_all().unwrap(); + assert_eq!(recovered.len(), 1); + assert_eq!(recovered[0].connection_id, "live-connection"); + assert_eq!(recovered[0].sequence, 7); + recovery.complete_claimed_frames().unwrap(); + } + + #[test] + fn spill_recovery_preserves_the_terminal_reason_until_acknowledged() { + let temp = tempfile::tempdir().unwrap(); + let dir = temp.path().join("acp-ingress"); + let live = SpillStore::new(dir.clone(), "finished-connection".to_string()).unwrap(); + live.persist_finish(&RelayFinishReason::AgentError( + "upstream terminated".to_string(), + )) + .unwrap(); + drop(live); + + let recovery = SpillStore::new(dir.clone(), "recovery-connection".to_string()).unwrap(); + assert!(recovery.take_all().unwrap().is_empty()); + assert_eq!( + recovery.take_recovered_finishes(), + vec![RelayFinishReason::AgentError( + "upstream terminated".to_string() + )] + ); + assert!( + fs::read_dir(&dir) + .unwrap() + .filter_map(|entry| entry.ok()) + .any(|entry| spill_finish_path(&entry.path())), + "finish marker disappeared before terminal projection committed" + ); + recovery.complete_finish().unwrap(); + assert!( + !fs::read_dir(&dir) + .unwrap() + .filter_map(|entry| entry.ok()) + .any(|entry| spill_finish_path(&entry.path())), + "acknowledged finish marker remained in the journal" + ); + } +} diff --git a/trail/src/acp/protocol.rs b/trail/src/acp/protocol.rs new file mode 100644 index 0000000..245510d --- /dev/null +++ b/trail/src/acp/protocol.rs @@ -0,0 +1,339 @@ +use std::io; + +use serde_json::Value; + +#[allow(dead_code)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, serde::Deserialize, serde::Serialize)] +pub(crate) enum Direction { + ClientToAgent, + AgentToClient, +} + +impl Direction { + fn opposite(self) -> Self { + match self { + Self::ClientToAgent => Self::AgentToClient, + Self::AgentToClient => Self::ClientToAgent, + } + } +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub(crate) enum RequestId { + Null, + Number(i64), + String(String), +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub(crate) struct CorrelationKey { + pub direction: Direction, + pub id: RequestId, +} + +#[allow(dead_code)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum EnvelopeKind { + Request, + Notification, + SuccessResponse, + ErrorResponse, +} + +#[allow(dead_code)] +pub(crate) struct Frame { + direction: Direction, + raw: Vec, + parsed: Value, + kind: EnvelopeKind, + method: Option, + id: Option, + transformed: Option>, +} + +#[allow(dead_code)] +impl Frame { + pub(crate) fn parse(direction: Direction, raw: Vec) -> io::Result { + let parsed: Value = serde_json::from_slice(&raw) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + let (kind, method, id) = classify(&parsed)?; + Ok(Self { + direction, + raw, + parsed, + kind, + method, + id, + transformed: None, + }) + } + + pub(crate) fn raw_bytes(&self) -> &[u8] { + &self.raw + } + + pub(crate) fn forward_bytes(&self) -> &[u8] { + self.transformed.as_deref().unwrap_or(&self.raw) + } + + pub(crate) fn kind(&self) -> EnvelopeKind { + self.kind + } + + pub(crate) fn direction(&self) -> Direction { + self.direction + } + + pub(crate) fn method(&self) -> Option<&str> { + self.method.as_deref() + } + + pub(crate) fn value(&self) -> &Value { + &self.parsed + } + + pub(crate) fn value_mut_for_transform(&mut self) -> &mut Value { + &mut self.parsed + } + + pub(crate) fn commit_transform(&mut self) -> io::Result<()> { + let (kind, method, id) = classify(&self.parsed)?; + if kind != self.kind || method != self.method || id != self.id { + return Err(invalid( + "ACP transformations must not change JSON-RPC routing fields", + )); + } + let mut transformed = serde_json::to_vec(&self.parsed) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + if self.raw.ends_with(b"\r\n") { + transformed.extend_from_slice(b"\r\n"); + } else if self.raw.ends_with(b"\n") { + transformed.push(b'\n'); + } + self.transformed = Some(transformed); + Ok(()) + } + + pub(crate) fn replace_value_and_commit(&mut self, candidate: Value) -> io::Result<()> { + let original = std::mem::replace(&mut self.parsed, candidate); + if let Err(error) = self.commit_transform() { + self.parsed = original; + self.transformed = None; + return Err(error); + } + Ok(()) + } + + pub(crate) fn correlation_key(&self) -> Option { + self.id.clone().map(|id| CorrelationKey { + direction: match self.kind { + EnvelopeKind::Request | EnvelopeKind::Notification => self.direction, + EnvelopeKind::SuccessResponse | EnvelopeKind::ErrorResponse => { + self.direction.opposite() + } + }, + id, + }) + } +} + +fn classify(value: &Value) -> io::Result<(EnvelopeKind, Option, Option)> { + let object = value + .as_object() + .ok_or_else(|| invalid("JSON-RPC frame must be an object"))?; + if object.get("jsonrpc").and_then(Value::as_str) != Some("2.0") { + return Err(invalid("JSON-RPC frame must declare version 2.0")); + } + + if let Some(method) = object.get("method") { + let method = method + .as_str() + .ok_or_else(|| invalid("JSON-RPC method must be a string"))?; + if object.contains_key("result") || object.contains_key("error") { + return Err(invalid( + "JSON-RPC request and notification envelopes cannot contain result or error", + )); + } + let id = object.get("id").map(request_id).transpose()?; + let kind = if id.is_some() { + EnvelopeKind::Request + } else { + EnvelopeKind::Notification + }; + return Ok((kind, Some(method.to_string()), id)); + } + + let id = request_id( + object + .get("id") + .ok_or_else(|| invalid("JSON-RPC response must have an id"))?, + )?; + let has_result = object.contains_key("result"); + let has_error = object.contains_key("error"); + match (has_result, has_error) { + (true, false) => Ok((EnvelopeKind::SuccessResponse, None, Some(id))), + (false, true) => { + validate_error_object(&object["error"])?; + Ok((EnvelopeKind::ErrorResponse, None, Some(id))) + } + (true, true) => Err(invalid( + "JSON-RPC response cannot contain both result and error", + )), + (false, false) => Err(invalid( + "JSON-RPC response must contain exactly one of result or error", + )), + } +} + +fn validate_error_object(value: &Value) -> io::Result<()> { + let error = value + .as_object() + .ok_or_else(|| invalid("JSON-RPC error must be an object"))?; + if error.get("code").and_then(Value::as_i64).is_none() { + return Err(invalid("JSON-RPC error code must be an integer")); + } + if error.get("message").and_then(Value::as_str).is_none() { + return Err(invalid("JSON-RPC error message must be a string")); + } + Ok(()) +} + +fn request_id(value: &Value) -> io::Result { + match value { + Value::Null => Ok(RequestId::Null), + Value::Number(number) => number + .as_i64() + .map(RequestId::Number) + .ok_or_else(|| invalid("JSON-RPC numeric ids must be signed 64-bit integers")), + Value::String(value) => Ok(RequestId::String(value.clone())), + _ => Err(invalid("JSON-RPC id must be null, an integer, or a string")), + } +} + +fn invalid(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn frame_preserves_raw_bytes_and_scopes_ids_by_direction() { + let client_raw = br#" { "id":7, "jsonrpc":"2.0", "method":"session/list", "params":{} } +"# + .to_vec(); + let agent_raw = br#" { "id":7, "jsonrpc":"2.0", "method":"fs/read_text_file", "params":{"sessionId":"s","path":"a"} } +"# + .to_vec(); + let client = Frame::parse(Direction::ClientToAgent, client_raw.clone()).unwrap(); + let agent = Frame::parse(Direction::AgentToClient, agent_raw.clone()).unwrap(); + + assert_eq!(client.kind(), EnvelopeKind::Request); + assert_eq!(agent.kind(), EnvelopeKind::Request); + assert_eq!(client.forward_bytes(), client_raw); + assert_eq!(agent.forward_bytes(), agent_raw); + assert_ne!(client.correlation_key(), agent.correlation_key()); + } + + #[test] + fn classifies_every_json_rpc_envelope_and_scopes_response_ids() { + let request = Frame::parse( + Direction::ClientToAgent, + br#"{"jsonrpc":"2.0","id":"same","method":"session/list"} +"# + .to_vec(), + ) + .unwrap(); + let notification = Frame::parse( + Direction::ClientToAgent, + br#"{"jsonrpc":"2.0","method":"session/cancel","params":{"sessionId":"s"}} +"# + .to_vec(), + ) + .unwrap(); + let success = Frame::parse( + Direction::AgentToClient, + br#"{"jsonrpc":"2.0","id":"same","result":{"sessions":[]}} +"# + .to_vec(), + ) + .unwrap(); + let error = Frame::parse( + Direction::AgentToClient, + br#"{"jsonrpc":"2.0","id":9,"error":{"code":-32602,"message":"bad params"}} +"# + .to_vec(), + ) + .unwrap(); + + assert_eq!(request.kind(), EnvelopeKind::Request); + assert_eq!(notification.kind(), EnvelopeKind::Notification); + assert_eq!(success.kind(), EnvelopeKind::SuccessResponse); + assert_eq!(error.kind(), EnvelopeKind::ErrorResponse); + assert_eq!(request.method(), Some("session/list")); + assert_eq!(notification.method(), Some("session/cancel")); + assert_eq!(success.method(), None); + assert_eq!(notification.correlation_key(), None); + assert_eq!(success.correlation_key(), request.correlation_key()); + } + + #[test] + fn rejects_invalid_json_rpc_envelopes() { + let invalid = [ + br#"[]"#.as_slice(), + br#"{"id":1,"method":"session/list"}"#.as_slice(), + br#"{"jsonrpc":"1.0","id":1,"method":"session/list"}"#.as_slice(), + br#"{"jsonrpc":"2.0","id":1.5,"method":"session/list"}"#.as_slice(), + br#"{"jsonrpc":"2.0","id":{},"method":"session/list"}"#.as_slice(), + br#"{"jsonrpc":"2.0","id":1,"method":7}"#.as_slice(), + br#"{"jsonrpc":"2.0","id":1,"result":{},"error":{"code":-1,"message":"x"}}"#.as_slice(), + br#"{"jsonrpc":"2.0","result":{}}"#.as_slice(), + br#"{"jsonrpc":"2.0","id":1}"#.as_slice(), + ]; + for raw in invalid { + assert!( + Frame::parse(Direction::ClientToAgent, raw.to_vec()).is_err(), + "accepted invalid envelope: {}", + String::from_utf8_lossy(raw) + ); + } + } + + #[test] + fn transformation_is_explicit_and_preserves_the_original_line_ending() { + let raw = b" {\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"initialize\",\"params\":{\"protocolVersion\":1}}\r\n".to_vec(); + let mut frame = Frame::parse(Direction::ClientToAgent, raw.clone()).unwrap(); + frame.value_mut_for_transform()["params"]["clientCapabilities"] = serde_json::json!({}); + + assert_eq!(frame.forward_bytes(), raw); + frame.commit_transform().unwrap(); + assert_ne!(frame.forward_bytes(), raw); + assert!(frame.forward_bytes().ends_with(b"\r\n")); + assert_eq!( + frame.value()["params"]["clientCapabilities"], + serde_json::json!({}) + ); + } + + #[test] + fn pinned_message_fixture_covers_all_envelope_kinds() { + let kinds = include_str!("../../tests/fixtures/acp/v1/messages.jsonl") + .lines() + .map(|line| { + Frame::parse(Direction::ClientToAgent, format!("{line}\n").into_bytes()) + .unwrap() + .kind() + }) + .collect::>(); + assert_eq!( + kinds, + vec![ + EnvelopeKind::Request, + EnvelopeKind::Notification, + EnvelopeKind::SuccessResponse, + EnvelopeKind::ErrorResponse, + ] + ); + } +} diff --git a/trail/src/acp/registry.rs b/trail/src/acp/registry.rs index 3413321..547f0e3 100644 --- a/trail/src/acp/registry.rs +++ b/trail/src/acp/registry.rs @@ -96,7 +96,7 @@ pub(super) fn resolve_registry_provider( .find(|agent| agent.id.eq_ignore_ascii_case(&requested)) .ok_or_else(|| { Error::InvalidInput(format!( - "unknown ACP agent `{requested_agent}`; run `trail acp list` to see built-in and registry agents" + "unknown ACP agent `{requested_agent}`; run `trail agent acp status` to see built-in and registry agents" )) })?; let profile = registry_profile(agent); @@ -121,7 +121,7 @@ pub(super) fn registry_provider_profile( .map(registry_profile) .ok_or_else(|| { Error::InvalidInput(format!( - "unknown ACP agent `{requested_agent}`; run `trail acp list` to see built-in and registry agents" + "unknown ACP agent `{requested_agent}`; run `trail agent acp status` to see built-in and registry agents" )) }) } @@ -387,7 +387,7 @@ fn install_registry_binary( } match fs::rename(&staging, &install_dir) { Ok(()) => Ok(command_path), - Err(error) if command_path.is_file() => { + Err(_error) if command_path.is_file() => { let _ = fs::remove_dir_all(&staging); Ok(command_path) } diff --git a/trail/src/acp/schema.rs b/trail/src/acp/schema.rs new file mode 100644 index 0000000..8d6f314 --- /dev/null +++ b/trail/src/acp/schema.rs @@ -0,0 +1,161 @@ +use std::collections::BTreeSet; + +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::{Error, Result}; + +pub(crate) const ACP_V1_SCHEMA_SHA256: &str = + "92c1dfcda10dd47e99127500a3763da2b471f9ac61e12b9bf0430c32cf953796"; +pub(crate) const ACP_V1_META_SHA256: &str = + "e0bf36f8123b2544b499174197fdc371ec49a1b4572a35114513d56492741599"; +pub(crate) const ACP_V1_SCHEMA_COMMIT: &str = "64cbd71ae520b89aac54164d8c1d364333c8ee5f"; + +const ACP_V1_SCHEMA_BYTES: &[u8] = include_bytes!("../../tests/fixtures/acp/v1/schema.json"); +const ACP_V1_META_BYTES: &[u8] = include_bytes!("../../tests/fixtures/acp/v1/meta.json"); +const ACP_V1_SOURCE_BYTES: &[u8] = include_bytes!("../../tests/fixtures/acp/v1/source.json"); + +#[allow(dead_code)] +pub(crate) struct AcpV1Contract { + schema_sha256: String, + meta_sha256: String, + method_names: BTreeSet, + validator: jsonschema::Validator, +} + +#[allow(dead_code)] +impl AcpV1Contract { + pub(crate) fn load() -> Result { + let schema_sha256 = sha256_hex(ACP_V1_SCHEMA_BYTES); + let meta_sha256 = sha256_hex(ACP_V1_META_BYTES); + ensure_digest("schema.json", &schema_sha256, ACP_V1_SCHEMA_SHA256)?; + ensure_digest("meta.json", &meta_sha256, ACP_V1_META_SHA256)?; + + let source: Value = serde_json::from_slice(ACP_V1_SOURCE_BYTES)?; + ensure_source_manifest(&source)?; + + let schema: Value = serde_json::from_slice(ACP_V1_SCHEMA_BYTES)?; + let meta: Value = serde_json::from_slice(ACP_V1_META_BYTES)?; + if meta.get("version").and_then(Value::as_u64) != Some(1) { + return Err(Error::Corrupt( + "vendored ACP metadata does not declare wire version 1".to_string(), + )); + } + + let mut method_names = BTreeSet::new(); + for group in ["agentMethods", "clientMethods", "protocolMethods"] { + let methods = meta.get(group).and_then(Value::as_object).ok_or_else(|| { + Error::Corrupt(format!("vendored ACP metadata is missing `{group}`")) + })?; + for method in methods.values() { + let method = method.as_str().ok_or_else(|| { + Error::Corrupt(format!( + "vendored ACP metadata `{group}` contains a non-string method" + )) + })?; + if !method_names.insert(method.to_string()) { + return Err(Error::Corrupt(format!( + "vendored ACP metadata repeats method `{method}`" + ))); + } + } + } + if method_names.len() != 23 { + return Err(Error::Corrupt(format!( + "vendored ACP metadata contains {} methods instead of 23", + method_names.len() + ))); + } + + let validator = jsonschema::validator_for(&schema).map_err(|err| { + Error::Corrupt(format!("vendored ACP v1 schema does not compile: {err}")) + })?; + Ok(Self { + schema_sha256, + meta_sha256, + method_names, + validator, + }) + } + + pub(crate) fn wire_version(&self) -> u16 { + 1 + } + + pub(crate) fn schema_sha256(&self) -> &str { + &self.schema_sha256 + } + + pub(crate) fn meta_sha256(&self) -> &str { + &self.meta_sha256 + } + + pub(crate) fn method_names(&self) -> &BTreeSet { + &self.method_names + } + + pub(crate) fn validator(&self) -> &jsonschema::Validator { + &self.validator + } + + pub(crate) fn validate(&self, message: &Value) -> Result<()> { + self.validator + .validate(message) + .map_err(|err| Error::InvalidInput(format!("message is not valid ACP v1: {err}"))) + } +} + +fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} + +fn ensure_digest(name: &str, actual: &str, expected: &str) -> Result<()> { + if actual == expected { + Ok(()) + } else { + Err(Error::Corrupt(format!( + "vendored ACP v1 `{name}` digest is `{actual}`, expected `{expected}`" + ))) + } +} + +fn ensure_source_manifest(source: &Value) -> Result<()> { + let expected = [ + ("commit", ACP_V1_SCHEMA_COMMIT), + ("schemaSha256", ACP_V1_SCHEMA_SHA256), + ("metaSha256", ACP_V1_META_SHA256), + ]; + for (field, expected) in expected { + if source.get(field).and_then(Value::as_str) != Some(expected) { + return Err(Error::Corrupt(format!( + "vendored ACP v1 source manifest has the wrong `{field}`" + ))); + } + } + if source.get("wireVersion").and_then(Value::as_u64) != Some(1) { + return Err(Error::Corrupt( + "vendored ACP v1 source manifest has the wrong `wireVersion`".to_string(), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pinned_v1_artifacts_match_manifest_and_compile() { + let contract = AcpV1Contract::load().unwrap(); + assert_eq!(contract.wire_version(), 1); + assert_eq!(contract.method_names().len(), 23); + assert_eq!(contract.schema_sha256(), ACP_V1_SCHEMA_SHA256); + assert_eq!(contract.meta_sha256(), ACP_V1_META_SHA256); + assert!(contract.validator().is_valid(&serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": {"protocolVersion": 1, "clientCapabilities": {}} + }))); + } +} diff --git a/trail/src/acp/setup.rs b/trail/src/acp/setup.rs new file mode 100644 index 0000000..15d9af7 --- /dev/null +++ b/trail/src/acp/setup.rs @@ -0,0 +1,302 @@ +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; + +use super::acp_provider_profile_with_registry; +use crate::{Error, Result}; + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct AcpSetupReport { + pub transport: String, + pub provider: String, + pub editor: String, + pub command: Vec, + pub snippet: String, + pub config_path: Option, + pub action: String, + pub before_digest: Option, + pub after_digest: Option, + pub applied: bool, + pub warnings: Vec, + #[serde(skip)] + before_bytes: Option>, + #[serde(skip)] + desired_bytes: Option>, +} + +pub fn build_acp_setup_plan( + workspace_root: &Path, + db_dir: &Path, + provider: &str, + editor: &str, +) -> Result { + let workspace_root = workspace_root.canonicalize()?; + let executable = env::current_exe()?.canonicalize()?; + let profile = acp_provider_profile_with_registry(provider, Some(db_dir))?; + if !profile.supports_acp { + return Err(Error::InvalidInput(format!( + "provider `{}` does not support ACP", + profile.agent + ))); + } + let mut warnings = profile.notes; + if !matches!(editor, "generic" | "vscode" | "zed") { + warnings.push(format!( + "editor `{editor}` has no exact Trail adapter; returning a generic entry without changing editor settings" + )); + } + let command = vec![ + executable.to_string_lossy().to_string(), + "--workspace".to_string(), + workspace_root.to_string_lossy().to_string(), + "agent".to_string(), + "acp".to_string(), + "run".to_string(), + profile.agent.clone(), + ]; + let snippet = editor_snippet(editor, &profile.agent, &command); + let Some(config_path) = editor_config_path(editor)? else { + return Ok(AcpSetupReport { + transport: "acp".to_string(), + provider: profile.agent, + editor: editor.to_string(), + command, + snippet, + config_path: None, + action: "print".to_string(), + before_digest: None, + after_digest: None, + applied: false, + warnings, + before_bytes: None, + desired_bytes: None, + }); + }; + let before_bytes = match fs::read(&config_path) { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), + }; + let desired_bytes = merge_zed_settings(before_bytes.as_deref(), &profile.agent, &command)?; + let action = match before_bytes.as_deref() { + None => "create", + Some(before) if before == desired_bytes => "noop", + Some(_) => "update", + }; + Ok(AcpSetupReport { + transport: "acp".to_string(), + provider: profile.agent, + editor: editor.to_string(), + command, + snippet, + config_path: Some(config_path), + action: action.to_string(), + before_digest: before_bytes.as_deref().map(content_digest), + after_digest: Some(content_digest(&desired_bytes)), + applied: false, + warnings, + before_bytes, + desired_bytes: Some(desired_bytes), + }) +} + +pub fn apply_acp_setup_plan(mut plan: AcpSetupReport, apply: bool) -> Result { + if !apply || plan.action == "print" { + return Ok(plan); + } + let config_path = plan.config_path.as_ref().ok_or_else(|| { + Error::InvalidInput("ACP setup plan has no writable editor target".to_string()) + })?; + let desired = plan.desired_bytes.as_deref().ok_or_else(|| { + Error::InvalidInput("ACP setup plan has no desired editor configuration".to_string()) + })?; + let current = match fs::read(config_path) { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), + }; + if current.as_deref().map(content_digest) != plan.before_digest { + return Err(Error::Conflict(format!( + "ACP editor config `{}` changed after planning; build a new plan", + config_path.display() + ))); + } + if plan.action != "noop" { + if let Some(before) = plan.before_bytes.as_deref() { + write_backup(&plan.provider, &plan.editor, before)?; + } + atomic_write(config_path, desired)?; + } + plan.applied = true; + Ok(plan) +} + +fn editor_config_path(editor: &str) -> Result> { + if editor != "zed" { + return Ok(None); + } + let home = env::var_os("HOME").map(PathBuf::from).ok_or_else(|| { + Error::InvalidInput("cannot resolve home directory for Zed setup".to_string()) + })?; + #[cfg(target_os = "macos")] + return Ok(Some( + home.join("Library/Application Support/Zed/settings.json"), + )); + #[cfg(target_os = "windows")] + return Ok(Some( + env::var_os("APPDATA") + .map(PathBuf::from) + .unwrap_or(home) + .join("Zed/settings.json"), + )); + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + Ok(Some( + env::var_os("XDG_CONFIG_HOME") + .map(PathBuf::from) + .unwrap_or_else(|| home.join(".config")) + .join("zed/settings.json"), + )) +} + +fn merge_zed_settings( + existing: Option<&[u8]>, + provider: &str, + command: &[String], +) -> Result> { + let mut root = match existing { + None => Map::new(), + Some(bytes) => serde_json::from_slice::(bytes)? + .as_object() + .cloned() + .ok_or_else(|| { + Error::Conflict("Zed settings must contain a JSON object".to_string()) + })?, + }; + let servers = root + .entry("agent_servers".to_string()) + .or_insert_with(|| Value::Object(Map::new())) + .as_object_mut() + .ok_or_else(|| Error::Conflict("Zed agent_servers must be a JSON object".to_string()))?; + servers.insert( + format!("trail-{provider}"), + serde_json::json!({ + "type": "custom", + "command": command.first().cloned().unwrap_or_default(), + "args": command.iter().skip(1).cloned().collect::>() + }), + ); + let mut bytes = serde_json::to_vec_pretty(&Value::Object(root))?; + bytes.push(b'\n'); + Ok(bytes) +} + +fn editor_snippet(editor: &str, provider: &str, command: &[String]) -> String { + if editor == "zed" { + return serde_json::to_string_pretty(&serde_json::json!({ + "agent_servers": { + (format!("trail-{provider}")): { + "type": "custom", + "command": command.first().cloned().unwrap_or_default(), + "args": command.iter().skip(1).cloned().collect::>() + } + } + })) + .unwrap_or_else(|_| "{}".to_string()); + } + if editor == "vscode" { + return serde_json::to_string_pretty(&serde_json::json!({ + (format!("Trail {provider}")): { + "command": command.first().cloned().unwrap_or_default(), + "args": command.iter().skip(1).cloned().collect::>(), + "env": {} + } + })) + .unwrap_or_else(|_| "{}".to_string()); + } + format!("ACP command:\n{}", shell_join(command)) +} + +fn atomic_write(path: &Path, bytes: &[u8]) -> Result<()> { + let parent = path.parent().ok_or_else(|| Error::InvalidPath { + path: path.display().to_string(), + reason: "ACP editor config has no parent directory".to_string(), + })?; + fs::create_dir_all(parent)?; + let temp = parent.join(format!(".trail-acp-{}.tmp", std::process::id())); + fs::write(&temp, bytes)?; + if let Ok(metadata) = fs::metadata(path) { + fs::set_permissions(&temp, metadata.permissions())?; + } else { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&temp, fs::Permissions::from_mode(0o600))?; + } + } + if let Err(error) = fs::rename(&temp, path) { + let _ = fs::remove_file(&temp); + return Err(error.into()); + } + Ok(()) +} + +fn write_backup(provider: &str, editor: &str, bytes: &[u8]) -> Result<()> { + let root = application_state_dir()?.join("trail/backups/agent-acp"); + fs::create_dir_all(&root)?; + let digest = content_digest(bytes); + atomic_write( + &root.join(format!("{editor}-{provider}-{}.json", &digest[..12])), + bytes, + ) +} + +fn application_state_dir() -> Result { + if let Some(path) = env::var_os("XDG_STATE_HOME") { + return Ok(PathBuf::from(path)); + } + #[cfg(target_os = "macos")] + { + let home = env::var_os("HOME").map(PathBuf::from).ok_or_else(|| { + Error::InvalidInput("cannot resolve application state directory".to_string()) + })?; + Ok(home.join("Library/Application Support")) + } + #[cfg(target_os = "windows")] + { + return env::var_os("LOCALAPPDATA") + .map(PathBuf::from) + .ok_or_else(|| Error::InvalidInput("cannot resolve LOCALAPPDATA".to_string())); + } + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + { + let home = env::var_os("HOME").map(PathBuf::from).ok_or_else(|| { + Error::InvalidInput("cannot resolve application state directory".to_string()) + })?; + Ok(home.join(".local/state")) + } +} + +fn content_digest(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn shell_join(parts: &[String]) -> String { + parts + .iter() + .map(|part| { + if part.chars().all(|ch| { + ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '/' | '.' | '@' | ':') + }) { + part.clone() + } else { + format!("'{}'", part.replace('\'', "'\\''")) + } + }) + .collect::>() + .join(" ") +} diff --git a/trail/src/acp/transform.rs b/trail/src/acp/transform.rs new file mode 100644 index 0000000..c107a2d --- /dev/null +++ b/trail/src/acp/transform.rs @@ -0,0 +1,478 @@ +use std::collections::HashSet; +use std::ffi::OsString; +use std::fs; +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; + +use serde_json::{Map, Value}; + +use super::protocol::{CorrelationKey, Direction, EnvelopeKind, Frame, RequestId}; +use super::schema::AcpV1Contract; +use super::AcpRelayOptions; +use crate::{Error, Result}; + +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +pub(crate) struct PathMapping { + pub original: PathBuf, + pub effective: PathBuf, + pub isolated: bool, +} + +pub(crate) struct WorkspaceMapper { + workspace_root: PathBuf, + materialized_root: PathBuf, +} + +impl WorkspaceMapper { + pub(crate) fn new(workspace_root: PathBuf, materialized_root: PathBuf) -> Result { + let workspace_root = resolve_existing_ancestors(&workspace_root)?; + let materialized_root = resolve_existing_ancestors(&materialized_root)?; + Ok(Self { + workspace_root, + materialized_root, + }) + } + + pub(crate) fn map(&self, path: &Path) -> Result { + let original = path.to_path_buf(); + if looks_like_foreign_absolute_path(path) { + return Ok(PathMapping { + effective: original.clone(), + original, + isolated: false, + }); + } + if !path.is_absolute() { + return Err(Error::InvalidPath { + path: path.to_string_lossy().to_string(), + reason: "ACP workspace paths must be absolute".to_string(), + }); + } + let resolved = resolve_existing_ancestors(path)?; + let Ok(relative) = resolved.strip_prefix(&self.workspace_root) else { + return Ok(PathMapping { + effective: original.clone(), + original, + isolated: false, + }); + }; + if relative.components().any(|component| { + matches!( + component, + Component::ParentDir | Component::RootDir | Component::Prefix(_) + ) + }) { + return Err(Error::InvalidPath { + path: path.to_string_lossy().to_string(), + reason: "ACP workspace mapping escapes its workspace root".to_string(), + }); + } + let effective = if relative.as_os_str().is_empty() { + self.materialized_root.clone() + } else { + self.materialized_root.join(relative) + }; + let resolved_effective = resolve_existing_ancestors(&effective)?; + if !resolved_effective.starts_with(&self.materialized_root) { + return Err(Error::InvalidPath { + path: effective.to_string_lossy().to_string(), + reason: "ACP workspace mapping escapes its materialized root".to_string(), + }); + } + fs::create_dir_all(&effective)?; + Ok(PathMapping { + original, + effective, + isolated: true, + }) + } + + pub(crate) fn map_session_params( + &self, + params: &mut Map, + ) -> Result> { + let mut mappings = Vec::new(); + let mut effective_roots = HashSet::new(); + if let Some(cwd) = params.get("cwd").and_then(Value::as_str) { + let mapping = self.map(Path::new(cwd))?; + params.insert( + "cwd".to_string(), + Value::String(mapping.effective.to_string_lossy().to_string()), + ); + effective_roots.insert(mapping.effective.clone()); + mappings.push(mapping); + } + + if let Some(additional) = params.get_mut("additionalDirectories") { + let directories = additional.as_array_mut().ok_or_else(|| { + Error::InvalidInput( + "ACP session additionalDirectories must be an array".to_string(), + ) + })?; + let mut forwarded = Vec::with_capacity(directories.len()); + for directory in directories.iter() { + let path = directory.as_str().ok_or_else(|| { + Error::InvalidInput( + "ACP session additionalDirectories entries must be strings".to_string(), + ) + })?; + let mapping = self.map(Path::new(path))?; + if effective_roots.insert(mapping.effective.clone()) { + forwarded.push(Value::String( + mapping.effective.to_string_lossy().to_string(), + )); + mappings.push(mapping); + } + } + *directories = forwarded; + } + Ok(mappings) + } +} + +pub(crate) fn passthrough_session_mappings( + params: &Map, +) -> Result> { + let mut mappings = Vec::new(); + let mut roots = HashSet::new(); + if let Some(cwd) = params.get("cwd").and_then(Value::as_str) { + let path = PathBuf::from(cwd); + roots.insert(path.clone()); + mappings.push(PathMapping { + original: path.clone(), + effective: path, + isolated: false, + }); + } + if let Some(additional) = params.get("additionalDirectories") { + let directories = additional.as_array().ok_or_else(|| { + Error::InvalidInput("ACP session additionalDirectories must be an array".to_string()) + })?; + for directory in directories { + let path = directory.as_str().ok_or_else(|| { + Error::InvalidInput( + "ACP session additionalDirectories entries must be strings".to_string(), + ) + })?; + let path = PathBuf::from(path); + if roots.insert(path.clone()) { + mappings.push(PathMapping { + original: path.clone(), + effective: path, + isolated: false, + }); + } + } + } + Ok(mappings) +} + +fn resolve_existing_ancestors(path: &Path) -> Result { + let normalized = normalize_absolute(path)?; + let mut ancestor = normalized.as_path(); + let mut suffix = Vec::::new(); + loop { + match fs::canonicalize(ancestor) { + Ok(canonical) => { + return Ok(suffix + .iter() + .rev() + .fold(canonical, |resolved, component| resolved.join(component))); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let name = ancestor.file_name().ok_or_else(|| Error::InvalidPath { + path: path.to_string_lossy().to_string(), + reason: "ACP path has no existing ancestor".to_string(), + })?; + suffix.push(name.to_os_string()); + ancestor = ancestor.parent().ok_or_else(|| Error::InvalidPath { + path: path.to_string_lossy().to_string(), + reason: "ACP path has no existing ancestor".to_string(), + })?; + } + Err(error) => return Err(Error::Io(error)), + } + } +} + +fn normalize_absolute(path: &Path) -> Result { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), + Component::RootDir => normalized.push(component.as_os_str()), + Component::CurDir => {} + Component::Normal(value) => normalized.push(value), + Component::ParentDir => { + if !normalized.pop() { + return Err(Error::InvalidPath { + path: path.to_string_lossy().to_string(), + reason: "ACP path escapes its filesystem root".to_string(), + }); + } + } + } + } + Ok(normalized) +} + +fn looks_like_foreign_absolute_path(path: &Path) -> bool { + if path.is_absolute() { + return false; + } + let value = path.to_string_lossy().as_bytes().to_vec(); + value.starts_with(b"\\\\") + || value.starts_with(b"//") + || (value.len() >= 3 + && value[0].is_ascii_alphabetic() + && value[1] == b':' + && matches!(value[2], b'\\' | b'/')) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum NegotiationState { + AwaitingInitialize, + InitializePending, + V1, + Other(u16), + Failed, +} + +#[derive(Clone, Debug)] +pub(crate) struct TransformOptions { + workspace: String, + db_dir: String, + provider: Option, + model: Option, +} + +impl TransformOptions { + pub(crate) fn from_relay(options: &AcpRelayOptions) -> Self { + Self { + workspace: options.workspace_root.to_string_lossy().to_string(), + db_dir: options.db_dir.to_string_lossy().to_string(), + provider: options.provider.clone(), + model: options.model.clone(), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TransformOutcome { + capture_v1: bool, + diagnostic: Option, +} + +impl TransformOutcome { + fn passthrough(capture_v1: bool) -> Self { + Self { + capture_v1, + diagnostic: None, + } + } + + fn diagnostic(capture_v1: bool, diagnostic: impl Into) -> Self { + Self { + capture_v1, + diagnostic: Some(diagnostic.into()), + } + } + + pub(crate) fn capture_v1(&self) -> bool { + self.capture_v1 + } + + pub(crate) fn diagnostic_message(&self) -> Option<&str> { + self.diagnostic.as_deref() + } +} + +pub(crate) struct TransformPipeline { + state: NegotiationState, + initialize_id: Option, + contract: Arc, + options: TransformOptions, + client_capabilities: Option, + agent_capabilities: Option, +} + +impl TransformPipeline { + pub(crate) fn new(contract: Arc, options: TransformOptions) -> Self { + Self { + state: NegotiationState::AwaitingInitialize, + initialize_id: None, + contract, + options, + client_capabilities: None, + agent_capabilities: None, + } + } + + pub(crate) fn apply(&mut self, frame: &mut Frame) -> Result { + if frame.direction() == Direction::ClientToAgent && frame.method() == Some("initialize") { + return self.observe_initialize_request(frame); + } + + if self.state == NegotiationState::InitializePending + && frame.direction() == Direction::AgentToClient + && self.matches_initialize_response(frame) + { + return self.observe_initialize_response(frame); + } + + match self.state { + NegotiationState::V1 => Ok(TransformOutcome::passthrough(true)), + NegotiationState::AwaitingInitialize => Ok(TransformOutcome::diagnostic( + false, + "ACP message arrived before initialize negotiation", + )), + NegotiationState::InitializePending => Ok(TransformOutcome::passthrough(false)), + NegotiationState::Other(version) => Ok(TransformOutcome::diagnostic( + false, + format!("ACP version {version} is outside Trail's v1 compatibility layer"), + )), + NegotiationState::Failed => Ok(TransformOutcome::diagnostic( + false, + "ACP initialize negotiation failed", + )), + } + } + + pub(crate) fn commit_candidate(&self, frame: &mut Frame, candidate: Value) -> Result<()> { + if self.state != NegotiationState::V1 || candidate == *frame.value() { + return Ok(()); + } + self.contract.validate(&candidate)?; + frame.replace_value_and_commit(candidate).map_err(Error::Io) + } + + #[allow(dead_code)] + pub(crate) fn state(&self) -> NegotiationState { + self.state + } + + fn observe_initialize_request(&mut self, frame: &Frame) -> Result { + if self.state != NegotiationState::AwaitingInitialize { + return Ok(TransformOutcome::diagnostic( + false, + "duplicate ACP initialize request was forwarded without renegotiating", + )); + } + if frame.kind() != EnvelopeKind::Request { + self.state = NegotiationState::Failed; + return Ok(TransformOutcome::diagnostic( + false, + "ACP initialize must be a request with an id", + )); + } + let Some(CorrelationKey { + direction: Direction::ClientToAgent, + id, + }) = frame.correlation_key() + else { + self.state = NegotiationState::Failed; + return Ok(TransformOutcome::diagnostic( + false, + "ACP initialize request could not be correlated", + )); + }; + self.initialize_id = Some(id); + self.client_capabilities = frame.value().pointer("/params/clientCapabilities").cloned(); + self.state = NegotiationState::InitializePending; + Ok(TransformOutcome::passthrough(false)) + } + + fn matches_initialize_response(&self, frame: &Frame) -> bool { + frame.correlation_key().is_some_and(|key| { + key.direction == Direction::ClientToAgent + && self.initialize_id.as_ref() == Some(&key.id) + }) + } + + fn observe_initialize_response(&mut self, frame: &mut Frame) -> Result { + if frame.kind() == EnvelopeKind::ErrorResponse { + self.state = NegotiationState::Failed; + return Ok(TransformOutcome::diagnostic( + false, + "ACP initialize returned an error", + )); + } + let Some(version) = frame + .value() + .pointer("/result/protocolVersion") + .and_then(Value::as_u64) + .and_then(|version| u16::try_from(version).ok()) + else { + self.state = NegotiationState::Failed; + return Ok(TransformOutcome::diagnostic( + false, + "ACP initialize response omitted a valid protocol version", + )); + }; + self.agent_capabilities = frame.value().pointer("/result/agentCapabilities").cloned(); + if version != self.contract.wire_version() { + self.state = NegotiationState::Other(version); + return Ok(TransformOutcome::diagnostic( + false, + format!("upstream selected ACP version {version}; Trail supports v1"), + )); + } + + self.state = NegotiationState::V1; + let mut candidate = frame.value().clone(); + if let Err(error) = self.add_trail_metadata(&mut candidate) { + return Ok(TransformOutcome::diagnostic(true, error.to_string())); + } + if let Err(error) = self.contract.validate(&candidate) { + return Ok(TransformOutcome::diagnostic( + true, + format!("ACP initialize metadata transformation rolled back: {error}"), + )); + } + if let Err(error) = frame.replace_value_and_commit(candidate) { + return Ok(TransformOutcome::diagnostic( + true, + format!("ACP initialize metadata transformation rolled back: {error}"), + )); + } + Ok(TransformOutcome::passthrough(true)) + } + + fn add_trail_metadata(&self, message: &mut Value) -> Result<()> { + let result = message + .get_mut("result") + .and_then(Value::as_object_mut) + .ok_or_else(|| { + Error::InvalidInput("ACP initialize response result must be an object".to_string()) + })?; + let meta = match result.entry("_meta".to_string()) { + serde_json::map::Entry::Vacant(entry) => entry + .insert(Value::Object(Map::new())) + .as_object_mut() + .unwrap(), + serde_json::map::Entry::Occupied(mut entry) if entry.get().is_null() => { + entry.insert(Value::Object(Map::new())); + entry.into_mut().as_object_mut().unwrap() + } + serde_json::map::Entry::Occupied(entry) => { + entry.into_mut().as_object_mut().ok_or_else(|| { + Error::InvalidInput( + "ACP initialize response _meta is not an object".to_string(), + ) + })? + } + }; + meta.insert( + "trail".to_string(), + serde_json::json!({ + "relay": true, + "capture": true, + "workspace": self.options.workspace, + "dbDir": self.options.db_dir, + "provider": self.options.provider, + "model": self.options.model + }), + ); + Ok(()) + } +} diff --git a/trail/src/acp/transport.rs b/trail/src/acp/transport.rs new file mode 100644 index 0000000..d336c85 --- /dev/null +++ b/trail/src/acp/transport.rs @@ -0,0 +1,456 @@ +use std::io::{self, BufRead, BufReader, Read, Write}; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{mpsc, Arc}; +use std::thread; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; + +use super::protocol::{Direction, Frame}; +use super::AcpRelayOptions; +use crate::{Error, Result}; + +pub(crate) const ACP_MAX_FRAME_BYTES: usize = 16 * 1024 * 1024; +pub(crate) const ACP_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2); +const ACP_PUMP_DRAIN_TIMEOUT: Duration = ACP_SHUTDOWN_TIMEOUT; +const ACP_CAPTURE_FLUSH_TIMEOUT: Duration = Duration::from_secs(2); + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) enum RelayFinishReason { + EditorEof, + EditorError(String), + AgentEof, + AgentError(String), +} + +pub(crate) trait FrameObserver: Send + Sync + 'static { + fn observe(&self, frame: &mut Frame) -> Result<()>; + fn finish(&self, reason: RelayFinishReason); + fn flush(&self, _timeout: Duration) -> bool { + true + } +} + +pub(crate) struct StdioRelay { + observer: Arc, +} + +impl StdioRelay { + pub(crate) fn new(observer: Arc) -> Self { + Self { observer } + } + + pub(crate) fn run(self, options: &AcpRelayOptions) -> Result<()> { + if options.upstream_command.is_empty() { + return Err(Error::InvalidInput( + "ACP relay requires an upstream command after `--`".to_string(), + )); + } + + let (upstream_program, upstream_args) = super::confined_acp_command( + &options.upstream_command, + &options.workspace_root, + &options.db_dir, + options.materialize, + )?; + let mut command = Command::new(upstream_program); + command + .args(upstream_args) + .envs(&options.upstream_env) + .current_dir(&options.workspace_root) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.process_group(0); + } + let mut child = command.spawn().map_err(|err| { + Error::InvalidInput(format!( + "failed to launch upstream ACP agent `{}`: {err}", + options.upstream_command[0] + )) + })?; + + let child_stdin = child.stdin.take().ok_or_else(|| { + Error::InvalidInput("failed to open upstream ACP stdin pipe".to_string()) + })?; + let child_stdout = child.stdout.take().ok_or_else(|| { + Error::InvalidInput("failed to open upstream ACP stdout pipe".to_string()) + })?; + if let Some(stderr) = child.stderr.take() { + thread::spawn(move || { + let _ = copy_upstream_stderr(stderr); + }); + } + + let (done_tx, done_rx) = mpsc::channel(); + let stop_editor = Arc::new(AtomicBool::new(false)); + let editor_observer = Arc::clone(&self.observer); + let editor_done = done_tx.clone(); + let editor_stop = Arc::clone(&stop_editor); + let editor_handle = thread::spawn(move || { + let result = pump_editor(child_stdin, editor_observer, editor_stop); + let _ = editor_done.send(PumpDone::Editor(result)); + }); + + let agent_observer = Arc::clone(&self.observer); + let agent_handle = thread::spawn(move || { + let result = pump( + BufReader::new(child_stdout), + io::stdout(), + Direction::AgentToClient, + agent_observer, + ); + let _ = done_tx.send(PumpDone::Agent(result)); + }); + + let first = done_rx.recv().map_err(|err| { + Error::InvalidInput(format!("ACP relay pump failed before startup: {err}")) + })?; + let reason = finish_reason(&first); + if matches!(first, PumpDone::Agent(_)) { + stop_editor.store(true, Ordering::Release); + } + let exit = wait_bounded(&mut child)?; + let second = done_rx.recv_timeout(ACP_PUMP_DRAIN_TIMEOUT).map_err(|_| { + Error::InvalidInput( + "ACP relay pump did not stop before the finalization boundary".to_string(), + ) + })?; + editor_handle.join().map_err(|_| { + Error::InvalidInput("ACP editor pump panicked during shutdown".to_string()) + })?; + agent_handle.join().map_err(|_| { + Error::InvalidInput("ACP agent pump panicked during shutdown".to_string()) + })?; + + let (editor_result, agent_result) = pump_results(first, second)?; + self.observer.finish(reason); + if !self.observer.flush(ACP_CAPTURE_FLUSH_TIMEOUT) { + return Err(Error::InvalidInput( + "ACP capture did not settle before the finalization deadline".to_string(), + )); + } + editor_result.map_err(Error::Io)?; + agent_result.map_err(Error::Io)?; + if exit.timed_out { + Ok(()) + } else { + ensure_success(exit.status) + } + } +} + +enum PumpDone { + Editor(io::Result<()>), + Agent(io::Result<()>), +} + +fn finish_reason(done: &PumpDone) -> RelayFinishReason { + match done { + PumpDone::Editor(Ok(())) => RelayFinishReason::EditorEof, + PumpDone::Editor(Err(err)) => RelayFinishReason::EditorError(err.to_string()), + PumpDone::Agent(Ok(())) => RelayFinishReason::AgentEof, + PumpDone::Agent(Err(err)) => RelayFinishReason::AgentError(err.to_string()), + } +} + +fn pump_results(first: PumpDone, second: PumpDone) -> Result<(io::Result<()>, io::Result<()>)> { + match (first, second) { + (PumpDone::Editor(editor), PumpDone::Agent(agent)) + | (PumpDone::Agent(agent), PumpDone::Editor(editor)) => Ok((editor, agent)), + _ => Err(Error::InvalidInput( + "ACP relay received duplicate pump completion".to_string(), + )), + } +} + +#[cfg(unix)] +fn pump_editor(mut writer: W, observer: Arc, stop: Arc) -> io::Result<()> +where + W: Write, + O: FrameObserver, +{ + let mut pending = Vec::new(); + let mut scan_from = 0; + let mut chunk = [0_u8; 8192]; + loop { + if stop.load(Ordering::Acquire) { + return writer.flush(); + } + let mut descriptor = libc::pollfd { + fd: libc::STDIN_FILENO, + events: libc::POLLIN | libc::POLLHUP | libc::POLLERR, + revents: 0, + }; + let ready = unsafe { libc::poll(&mut descriptor, 1, 25) }; + if ready < 0 { + let error = io::Error::last_os_error(); + if error.kind() == io::ErrorKind::Interrupted { + continue; + } + return Err(error); + } + if ready == 0 || stop.load(Ordering::Acquire) { + continue; + } + let read = + unsafe { libc::read(libc::STDIN_FILENO, chunk.as_mut_ptr().cast(), chunk.len()) }; + if read < 0 { + let error = io::Error::last_os_error(); + if error.kind() == io::ErrorKind::Interrupted { + continue; + } + return Err(error); + } + if read == 0 { + if !pending.is_empty() { + forward_raw_frame( + std::mem::take(&mut pending), + Direction::ClientToAgent, + &observer, + &mut writer, + )?; + } + return writer.flush(); + } + pending.extend_from_slice(&chunk[..usize::try_from(read).unwrap_or(0)]); + while let Some(relative_end) = pending[scan_from..].iter().position(|byte| *byte == b'\n') { + let end = scan_from + relative_end; + let remainder = pending.split_off(end + 1); + let raw = std::mem::replace(&mut pending, remainder); + forward_raw_frame(raw, Direction::ClientToAgent, &observer, &mut writer)?; + scan_from = 0; + } + if pending.len() > ACP_MAX_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("ACP frame exceeds the {ACP_MAX_FRAME_BYTES}-byte transport limit"), + )); + } + scan_from = pending.len(); + } +} + +#[cfg(not(unix))] +fn pump_editor(writer: W, observer: Arc, _stop: Arc) -> io::Result<()> +where + W: Write, + O: FrameObserver, +{ + pump( + io::stdin().lock(), + writer, + Direction::ClientToAgent, + observer, + ) +} + +fn forward_raw_frame( + raw: Vec, + direction: Direction, + observer: &Arc, + writer: &mut W, +) -> io::Result<()> +where + W: Write, + O: FrameObserver, +{ + if raw.len() > ACP_MAX_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("ACP frame exceeds the {ACP_MAX_FRAME_BYTES}-byte transport limit"), + )); + } + if raw.iter().all(|byte| byte.is_ascii_whitespace()) { + return Ok(()); + } + let mut frame = Frame::parse(direction, raw)?; + observer.observe(&mut frame).map_err(io::Error::other)?; + writer.write_all(frame.forward_bytes())?; + writer.flush() +} + +fn pump( + mut reader: R, + mut writer: W, + direction: Direction, + observer: Arc, +) -> io::Result<()> +where + R: BufRead, + W: Write, + O: FrameObserver, +{ + while let Some(mut frame) = read_frame(&mut reader, direction)? { + observer.observe(&mut frame).map_err(io::Error::other)?; + writer.write_all(frame.forward_bytes())?; + writer.flush()?; + } + writer.flush() +} + +fn read_frame(reader: &mut R, direction: Direction) -> io::Result> { + loop { + let mut raw = Vec::new(); + let bytes = reader + .take((ACP_MAX_FRAME_BYTES + 1) as u64) + .read_until(b'\n', &mut raw)?; + if raw.len() > ACP_MAX_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("ACP frame exceeds the {ACP_MAX_FRAME_BYTES}-byte transport limit"), + )); + } + if bytes == 0 { + return Ok(None); + } + if raw.iter().all(|byte| byte.is_ascii_whitespace()) { + continue; + } + return Frame::parse(direction, raw).map(Some); + } +} + +fn copy_upstream_stderr(reader: R) -> io::Result<()> { + let mut reader = BufReader::new(reader); + let mut buf = [0u8; 8192]; + loop { + let bytes = reader.read(&mut buf)?; + if bytes == 0 { + return Ok(()); + } + let mut stderr = io::stderr().lock(); + stderr.write_all(&buf[..bytes])?; + stderr.flush()?; + } +} + +struct ChildExit { + status: ExitStatus, + timed_out: bool, +} + +fn wait_bounded(child: &mut Child) -> Result { + let deadline = Instant::now() + ACP_SHUTDOWN_TIMEOUT; + loop { + if let Some(status) = child.try_wait().map_err(Error::Io)? { + return Ok(ChildExit { + status, + timed_out: false, + }); + } + if Instant::now() >= deadline { + terminate_child_tree(child)?; + return child + .wait() + .map(|status| ChildExit { + status, + timed_out: true, + }) + .map_err(Error::Io); + } + thread::sleep(Duration::from_millis(10)); + } +} + +fn terminate_child_tree(child: &mut Child) -> Result<()> { + #[cfg(unix)] + { + let process_group = i32::try_from(child.id()).map_err(|_| { + Error::InvalidInput("upstream ACP process id does not fit i32".to_string()) + })?; + let result = unsafe { libc::kill(-process_group, libc::SIGKILL) }; + if result == 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ESRCH) { + return Ok(()); + } + Err(Error::Io(error)) + } + #[cfg(not(unix))] + { + child.kill().map_err(Error::Io) + } +} + +fn ensure_success(status: ExitStatus) -> Result<()> { + if status.success() { + Ok(()) + } else { + Err(Error::InvalidInput(format!( + "upstream ACP agent exited with status {status}" + ))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + #[derive(Default)] + struct RecordingObserver { + directions: Mutex>, + } + + impl FrameObserver for RecordingObserver { + fn observe(&self, frame: &mut Frame) -> Result<()> { + self.directions.lock().unwrap().push(frame.direction()); + Ok(()) + } + + fn finish(&self, _reason: RelayFinishReason) {} + } + + #[test] + fn pump_preserves_frames_and_skips_blank_lines() { + let input = b"\n \r\n {\"method\":\"ext/test\",\"jsonrpc\":\"2.0\"} \r\n"; + let mut output = Vec::new(); + let observer = Arc::new(RecordingObserver::default()); + pump( + BufReader::new(input.as_slice()), + &mut output, + Direction::ClientToAgent, + Arc::clone(&observer), + ) + .unwrap(); + assert_eq!(output, &input[b"\n \r\n".len()..]); + assert_eq!( + *observer.directions.lock().unwrap(), + vec![Direction::ClientToAgent] + ); + } + + #[test] + fn frame_limit_accepts_the_boundary_and_rejects_one_byte_above_it() { + let prefix = br#"{"jsonrpc":"2.0","method":"ext/limit","params":{"data":""#; + let suffix = b"\"}}\n"; + let payload_len = ACP_MAX_FRAME_BYTES - prefix.len() - suffix.len(); + let mut boundary = Vec::with_capacity(ACP_MAX_FRAME_BYTES); + boundary.extend_from_slice(prefix); + boundary.resize(boundary.len() + payload_len, b'x'); + boundary.extend_from_slice(suffix); + assert_eq!(boundary.len(), ACP_MAX_FRAME_BYTES); + + let mut reader = io::Cursor::new(boundary.clone()); + let frame = read_frame(&mut reader, Direction::ClientToAgent) + .unwrap() + .unwrap(); + assert_eq!(frame.raw_bytes().len(), ACP_MAX_FRAME_BYTES); + + boundary.insert(boundary.len() - suffix.len(), b'x'); + let mut reader = io::Cursor::new(boundary); + let error = match read_frame(&mut reader, Direction::ClientToAgent) { + Err(error) => error, + Ok(_) => panic!("oversized frame was accepted"), + }; + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("transport limit")); + } +} diff --git a/trail/src/agent_hooks.rs b/trail/src/agent_hooks.rs new file mode 100644 index 0000000..e0465ee --- /dev/null +++ b/trail/src/agent_hooks.rs @@ -0,0 +1,868 @@ +//! Provider registry and declarative native-agent hook contracts. +//! +//! Adapters describe discovery and translation only. Durable lifecycle policy remains +//! in Trail's shared capture coordinator. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; +use std::process::Command; + +use serde::{Deserialize, Serialize}; + +use crate::{Error, Result}; + +mod install; +mod parsing; + +pub use install::{ + apply_agent_hook_install_plan, build_agent_hook_install_plan, inspect_agent_hook_installation, + remove_agent_hook_installation, rollback_agent_hook_install_plan, AgentHookInstallAction, + AgentHookInstallPlan, AgentHookInstallReport, AgentHookInstallRequest, AgentHookInstallScope, + AgentHookInstallationRecord, AgentHookInstallationStatus, +}; +pub use parsing::{parse_agent_hook_payload, AgentHookParseContext}; + +pub const AGENT_ADAPTER_MANIFEST_SCHEMA: &str = "trail.agent_adapter_manifest"; +pub const AGENT_ADAPTER_MANIFEST_VERSION: u16 = 1; + +/// Apply Trail's central secret-redaction policy before writing a fallback spool file. +pub fn redact_agent_hook_payload(value: serde_json::Value) -> serde_json::Value { + crate::db::redact_sensitive_json(value) +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum AgentAdapterDeploymentClass { + JsonCommandConfig, + ProjectPlugin, + ProjectExtension, + NativeOrCompatible, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentProviderSupportLevel { + Supported, + Partial, + Experimental, + Unknown, + Unavailable, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentHookEventBinding { + pub native_event: String, + pub normalized_events: Vec, + pub matcher: Option, + pub response_contract: String, + pub notes: Vec, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentProviderCapabilities { + pub session_lifecycle: AgentProviderSupportLevel, + pub turn_lifecycle: AgentProviderSupportLevel, + pub messages: AgentProviderSupportLevel, + pub tool_spans: AgentProviderSupportLevel, + pub approvals: AgentProviderSupportLevel, + pub subagents: AgentProviderSupportLevel, + pub compaction: AgentProviderSupportLevel, + pub usage: AgentProviderSupportLevel, + pub native_transcript: AgentProviderSupportLevel, + pub canonical_export: AgentProviderSupportLevel, + pub context_injection: AgentProviderSupportLevel, + pub acp: AgentProviderSupportLevel, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentAdapterManifest { + pub schema: String, + pub version: u16, + pub provider: String, + pub display_name: String, + pub aliases: Vec, + pub adapter_version: String, + pub deployment: AgentAdapterDeploymentClass, + pub project_config_path: Option, + pub user_config_hint: Option, + pub provider_version_range: Option, + #[serde(default)] + pub executable_candidates: Vec, + #[serde(default)] + pub transcript_location_hints: Vec, + #[serde(default)] + pub canonical_export_command: Option>, + pub support: AgentProviderSupportLevel, + pub contract_source: Option, + pub events: Vec, + pub capabilities: AgentProviderCapabilities, + pub notes: Vec, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentProviderProbeReport { + pub provider: String, + pub executable: Option, + pub detected_version: Option, + pub support: AgentProviderSupportLevel, + pub compatibility: AgentProviderSupportLevel, + pub transcript_location_hints: Vec, + pub canonical_export_command: Option>, + pub diagnostics: Vec, +} + +impl AgentAdapterManifest { + pub fn validate(&self) -> Result<()> { + if self.schema != AGENT_ADAPTER_MANIFEST_SCHEMA + || self.version != AGENT_ADAPTER_MANIFEST_VERSION + { + return Err(Error::InvalidInput(format!( + "unsupported agent adapter manifest {} version {}", + self.schema, self.version + ))); + } + validate_provider_name(&self.provider)?; + if self.adapter_version.trim().is_empty() || self.display_name.trim().is_empty() { + return Err(Error::InvalidInput( + "agent adapter display name and version cannot be empty".to_string(), + )); + } + let mut native_events = BTreeSet::new(); + for event in &self.events { + if event.native_event.trim().is_empty() || event.normalized_events.is_empty() { + return Err(Error::InvalidInput(format!( + "agent adapter `{}` contains an empty event binding", + self.provider + ))); + } + if !native_events.insert(event.native_event.as_str()) { + return Err(Error::InvalidInput(format!( + "agent adapter `{}` repeats native event `{}`", + self.provider, event.native_event + ))); + } + } + Ok(()) + } +} + +#[derive(Clone, Debug)] +pub struct AgentProviderRegistry { + manifests: BTreeMap, + aliases: BTreeMap, +} + +impl AgentProviderRegistry { + pub fn built_in() -> Result { + Self::new(built_in_agent_adapter_manifests()) + } + + pub fn new(manifests: Vec) -> Result { + let mut by_provider = BTreeMap::new(); + let mut aliases = BTreeMap::new(); + for manifest in manifests { + manifest.validate()?; + let provider = manifest.provider.clone(); + if by_provider + .insert(provider.clone(), manifest.clone()) + .is_some() + { + return Err(Error::InvalidInput(format!( + "duplicate agent provider `{provider}`" + ))); + } + for alias in std::iter::once(provider.as_str()) + .chain(manifest.aliases.iter().map(String::as_str)) + { + validate_provider_name(alias)?; + if let Some(existing) = aliases.insert(alias.to_string(), provider.clone()) { + if existing != provider { + return Err(Error::InvalidInput(format!( + "agent provider alias `{alias}` belongs to both `{existing}` and `{provider}`" + ))); + } + } + } + } + Ok(Self { + manifests: by_provider, + aliases, + }) + } + + pub fn resolve(&self, name: &str) -> Result<&AgentAdapterManifest> { + let normalized = name.trim().to_ascii_lowercase(); + let canonical = self + .aliases + .get(&normalized) + .ok_or_else(|| Error::InvalidInput(format!("unknown agent hook provider `{name}`")))?; + Ok(&self.manifests[canonical]) + } + + pub fn list(&self) -> Vec<&AgentAdapterManifest> { + self.manifests.values().collect() + } + + pub fn probe(&self, name: &str, execute_version: bool) -> Result { + let manifest = self.resolve(name)?; + let executable = manifest + .executable_candidates + .iter() + .find_map(|candidate| executable_in_path(candidate)); + let mut diagnostics = Vec::new(); + let detected_version = if execute_version { + executable.as_ref().and_then(|executable| { + match Command::new(executable).arg("--version").output() { + Ok(output) if output.status.success() => { + let version = String::from_utf8_lossy(&output.stdout).trim().to_string(); + (!version.is_empty()).then_some(version) + } + Ok(output) => { + diagnostics.push(format!( + "`{} --version` exited with {}", + executable.display(), + output.status + )); + None + } + Err(error) => { + diagnostics.push(format!( + "could not execute `{} --version`: {error}", + executable.display() + )); + None + } + } + }) + } else { + None + }; + let compatibility = if executable.is_none() { + AgentProviderSupportLevel::Unavailable + } else if execute_version && detected_version.is_none() { + AgentProviderSupportLevel::Unknown + } else if execute_version { + match ( + manifest.provider_version_range.as_deref(), + detected_version.as_deref(), + ) { + (None, _) => { + diagnostics.push( + "the adapter has no pinned provider version range; treating the detected build as unknown until fixtures confirm it" + .to_string(), + ); + AgentProviderSupportLevel::Unknown + } + (Some(range), Some(version)) => match version_satisfies_range(version, range) { + Some(true) => manifest.support, + Some(false) => { + diagnostics.push(format!( + "detected provider version `{version}` does not satisfy the verified adapter range `{range}`" + )); + AgentProviderSupportLevel::Unavailable + } + None => { + diagnostics.push(format!( + "could not compare detected provider version `{version}` with adapter range `{range}`" + )); + AgentProviderSupportLevel::Unknown + } + }, + (Some(_), None) => AgentProviderSupportLevel::Unknown, + } + } else { + manifest.support + }; + if executable.is_none() { + diagnostics.push(format!( + "none of the provider executables were found on PATH: {}", + manifest.executable_candidates.join(", ") + )); + } + Ok(AgentProviderProbeReport { + provider: manifest.provider.clone(), + executable, + detected_version, + support: manifest.support, + compatibility, + transcript_location_hints: manifest.transcript_location_hints.clone(), + canonical_export_command: manifest.canonical_export_command.clone(), + diagnostics, + }) + } +} + +fn version_satisfies_range(version_output: &str, range: &str) -> Option { + fn parse(value: &str) -> Option<(u64, u64, u64)> { + let start = value.find(|character: char| character.is_ascii_digit())?; + let token = value[start..] + .split(|character: char| !(character.is_ascii_digit() || character == '.')) + .next()?; + let mut parts = token.split('.'); + let major = parts.next()?.parse().ok()?; + let minor = parts.next().unwrap_or("0").parse().ok()?; + let patch = parts.next().unwrap_or("0").parse().ok()?; + Some((major, minor, patch)) + } + + let detected = parse(version_output)?; + if let Some(minimum) = range.strip_prefix(">=") { + return Some(detected >= parse(minimum.trim())?); + } + if let Some(maximum) = range.strip_prefix('<') { + return Some(detected < parse(maximum.trim())?); + } + Some(detected == parse(range.trim())?) +} + +fn executable_in_path(candidate: &str) -> Option { + let candidate_path = PathBuf::from(candidate); + if candidate_path.components().count() > 1 { + return candidate_path.is_file().then_some(candidate_path); + } + std::env::var_os("PATH").and_then(|path| { + std::env::split_paths(&path) + .map(|directory| directory.join(candidate)) + .find(|path| path.is_file()) + }) +} + +fn validate_provider_name(value: &str) -> Result<()> { + if value.is_empty() + || value.len() > 64 + || !value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + { + return Err(Error::InvalidInput(format!( + "invalid canonical agent provider name `{value}`" + ))); + } + Ok(()) +} + +fn binding(native: &str, normalized: &[&str]) -> AgentHookEventBinding { + AgentHookEventBinding { + native_event: native.to_string(), + normalized_events: normalized + .iter() + .map(|value| (*value).to_string()) + .collect(), + matcher: None, + response_contract: "non-blocking-success".to_string(), + notes: Vec::new(), + } +} + +fn capabilities( + session: AgentProviderSupportLevel, + turn: AgentProviderSupportLevel, + tools: AgentProviderSupportLevel, + transcript: AgentProviderSupportLevel, + export: AgentProviderSupportLevel, + acp: AgentProviderSupportLevel, +) -> AgentProviderCapabilities { + AgentProviderCapabilities { + session_lifecycle: session, + turn_lifecycle: turn, + messages: AgentProviderSupportLevel::Partial, + tool_spans: tools, + approvals: AgentProviderSupportLevel::Partial, + subagents: AgentProviderSupportLevel::Partial, + compaction: AgentProviderSupportLevel::Partial, + usage: AgentProviderSupportLevel::Partial, + native_transcript: transcript, + canonical_export: export, + context_injection: AgentProviderSupportLevel::Partial, + acp, + } +} + +fn manifest( + provider: &str, + display_name: &str, + aliases: &[&str], + deployment: AgentAdapterDeploymentClass, + project_config_path: &str, + support: AgentProviderSupportLevel, + events: Vec, + capabilities: AgentProviderCapabilities, +) -> AgentAdapterManifest { + AgentAdapterManifest { + schema: AGENT_ADAPTER_MANIFEST_SCHEMA.to_string(), + version: AGENT_ADAPTER_MANIFEST_VERSION, + provider: provider.to_string(), + display_name: display_name.to_string(), + aliases: aliases.iter().map(|alias| (*alias).to_string()).collect(), + adapter_version: format!("trail/{provider}@2"), + deployment, + project_config_path: Some(project_config_path.to_string()), + user_config_hint: None, + provider_version_range: None, + executable_candidates: match provider { + "codex" => vec!["codex".to_string()], + "claude-code" => vec!["claude".to_string()], + "pi" => vec!["pi".to_string()], + "opencode" => vec!["opencode".to_string()], + "cursor" => vec!["agent".to_string(), "cursor-agent".to_string()], + "gemini" => vec!["gemini".to_string()], + "copilot" => vec!["copilot".to_string()], + "grok" => vec!["grok".to_string()], + "kiro" => vec!["kiro-cli".to_string()], + _ => Vec::new(), + }, + transcript_location_hints: Vec::new(), + canonical_export_command: None, + support, + contract_source: None, + events, + capabilities, + notes: Vec::new(), + } +} + +pub fn built_in_agent_adapter_manifests() -> Vec { + use AgentAdapterDeploymentClass as Deployment; + use AgentProviderSupportLevel as Support; + + let mut codex = manifest( + "codex", + "OpenAI Codex", + &["openai-codex"], + Deployment::JsonCommandConfig, + ".codex/hooks.json", + Support::Supported, + vec![ + binding("SessionStart", &["session.started", "session.resumed"]), + binding("UserPromptSubmit", &["turn.started", "message.user"]), + binding("PreToolUse", &["tool.started"]), + binding("PermissionRequest", &["approval.requested"]), + binding("PostToolUse", &["tool.completed", "tool.failed"]), + binding("PreCompact", &["compaction.started"]), + binding("PostCompact", &["compaction.completed"]), + binding("SubagentStart", &["subagent.started"]), + binding("SubagentStop", &["subagent.completed", "subagent.failed"]), + binding("Stop", &["turn.completed", "turn.failed", "turn.cancelled"]), + ], + capabilities( + Support::Partial, + Support::Supported, + Support::Supported, + Support::Partial, + Support::Unavailable, + Support::Supported, + ), + ); + codex.contract_source = Some("https://learn.chatgpt.com/docs/hooks".to_string()); + codex.notes = vec![ + "Project hooks require the .codex configuration layer and exact hook definition to be trusted." + .to_string(), + "Codex currently exposes SessionStart but no native SessionEnd hook; Stop closes turns." + .to_string(), + "Transcript paths are optional and the transcript format is not a stable hook interface." + .to_string(), + ]; + codex.transcript_location_hints = + vec!["hook payload transcript_path when supplied".to_string()]; + + let mut claude = manifest( + "claude-code", + "Anthropic Claude Code", + &["claude"], + Deployment::JsonCommandConfig, + ".claude/settings.json", + Support::Supported, + vec![ + binding("SessionStart", &["session.started", "session.resumed"]), + binding("SessionEnd", &["session.ended"]), + binding("UserPromptSubmit", &["turn.started", "message.user"]), + binding("PreToolUse", &["tool.started"]), + binding("PostToolUse", &["tool.completed"]), + binding("PostToolUseFailure", &["tool.failed"]), + binding("PermissionRequest", &["approval.requested"]), + binding("PermissionDenied", &["approval.decided"]), + binding("Stop", &["turn.completed"]), + binding("StopFailure", &["turn.failed"]), + binding("SubagentStart", &["subagent.started"]), + binding("SubagentStop", &["subagent.completed", "subagent.failed"]), + binding("PreCompact", &["compaction.started"]), + binding("PostCompact", &["compaction.completed"]), + ], + capabilities( + Support::Supported, + Support::Supported, + Support::Supported, + Support::Supported, + Support::Unavailable, + Support::Partial, + ), + ); + claude.contract_source = Some("https://code.claude.com/docs/en/hooks".to_string()); + claude.transcript_location_hints = + vec!["hook payload transcript_path under Claude's project session directory".to_string()]; + + let mut pi = manifest( + "pi", + "Pi", + &["pi-coding-agent"], + Deployment::ProjectExtension, + ".pi/extensions/trail/index.ts", + Support::Partial, + vec![ + binding("session_start", &["session.started", "session.resumed"]), + binding("session_shutdown", &["session.ended"]), + binding("before_agent_start", &["turn.started", "message.user"]), + binding("agent_start", &["session.updated"]), + binding( + "agent_end", + &["turn.completed", "turn.failed", "turn.cancelled"], + ), + binding("message_update", &["message.assistant.delta"]), + binding("tool_execution_start", &["tool.started"]), + binding("tool_execution_end", &["tool.completed", "tool.failed"]), + binding("session_before_compact", &["compaction.started"]), + binding("session_compact", &["compaction.completed"]), + ], + capabilities( + Support::Supported, + Support::Supported, + Support::Supported, + Support::Supported, + Support::Unavailable, + Support::Unknown, + ), + ); + pi.contract_source = Some("https://pi.dev/docs/latest/extensions".to_string()); + pi.transcript_location_hints = vec![ + "extension context sessionManager.getSessionFile() under ~/.pi/agent/sessions".to_string(), + ]; + + let mut opencode = manifest( + "opencode", + "OpenCode", + &["open-code"], + Deployment::ProjectPlugin, + ".opencode/plugins/trail.ts", + Support::Partial, + vec![ + binding("session.created", &["session.started"]), + binding("session.updated", &["session.updated"]), + binding("session.idle", &["turn.completed"]), + binding("session.deleted", &["session.ended"]), + binding("chat.message", &["turn.started", "message.user"]), + binding("message.updated", &["message.assistant.delta"]), + binding("message.part.updated", &["message.assistant.delta"]), + binding("tool.execute.before", &["tool.started"]), + binding("tool.execute.after", &["tool.completed", "tool.failed"]), + binding("permission.asked", &["approval.requested"]), + binding("permission.replied", &["approval.decided"]), + binding("session.compacted", &["compaction.completed"]), + binding("session.diff", &["workspace.diff"]), + binding("file.edited", &["workspace.file_changed"]), + binding("todo.updated", &["plan.updated"]), + binding("command.executed", &["diagnostic"]), + binding("session.error", &["diagnostic"]), + ], + capabilities( + Support::Supported, + Support::Supported, + Support::Supported, + Support::Partial, + Support::Supported, + Support::Partial, + ), + ); + opencode.contract_source = Some("https://opencode.ai/docs/plugins/".to_string()); + opencode.canonical_export_command = Some(vec![ + "opencode".to_string(), + "export".to_string(), + "{session_id}".to_string(), + ]); + + let mut cursor = manifest( + "cursor", + "Cursor", + &["cursor-agent"], + Deployment::JsonCommandConfig, + ".cursor/hooks.json", + Support::Partial, + vec![ + binding("sessionStart", &["session.started", "session.resumed"]), + binding("sessionEnd", &["session.ended"]), + binding("beforeSubmitPrompt", &["turn.started", "message.user"]), + binding("preToolUse", &["tool.started"]), + binding("postToolUse", &["tool.completed", "tool.failed"]), + binding("postToolUseFailure", &["tool.failed"]), + binding("afterAgentResponse", &["message.assistant.completed"]), + binding("afterAgentThought", &["diagnostic"]), + binding("stop", &["turn.completed", "turn.failed", "turn.cancelled"]), + binding("subagentStart", &["subagent.started"]), + binding("subagentStop", &["subagent.completed", "subagent.failed"]), + binding("preCompact", &["compaction.started"]), + ], + capabilities( + Support::Partial, + Support::Supported, + Support::Partial, + Support::Partial, + Support::Unavailable, + Support::Supported, + ), + ); + cursor.contract_source = Some("https://cursor.com/docs/hooks".to_string()); + cursor.transcript_location_hints = vec![ + "hook payload transcript_path when exposed by the installed Cursor surface".to_string(), + ]; + + let mut gemini = manifest( + "gemini", + "Google Gemini CLI", + &["gemini-cli"], + Deployment::JsonCommandConfig, + ".gemini/settings.json", + Support::Partial, + vec![ + binding("SessionStart", &["session.started", "session.resumed"]), + binding("SessionEnd", &["session.ended"]), + binding("BeforeAgent", &["turn.started", "message.user"]), + binding( + "AfterAgent", + &["turn.completed", "turn.failed", "turn.cancelled"], + ), + binding("BeforeModel", &["model.updated"]), + binding("AfterModel", &["usage.updated"]), + binding("BeforeToolSelection", &["diagnostic"]), + binding("BeforeTool", &["tool.started"]), + binding("AfterTool", &["tool.completed", "tool.failed"]), + binding("PreCompress", &["compaction.started"]), + binding("Notification", &["diagnostic"]), + ], + capabilities( + Support::Supported, + Support::Supported, + Support::Supported, + Support::Supported, + Support::Unavailable, + Support::Unknown, + ), + ); + gemini.contract_source = Some("https://geminicli.com/docs/hooks/reference/".to_string()); + gemini.transcript_location_hints = + vec!["hook payload transcript_path for the native JSON transcript".to_string()]; + + let mut copilot = manifest( + "copilot", + "GitHub Copilot CLI", + &["copilot-cli", "github-copilot"], + Deployment::JsonCommandConfig, + ".github/hooks/trail.json", + Support::Partial, + vec![ + binding("sessionStart", &["session.started", "session.resumed"]), + binding("sessionEnd", &["session.ended"]), + binding("userPromptSubmitted", &["turn.started", "message.user"]), + binding("preToolUse", &["tool.started"]), + binding("postToolUse", &["tool.completed", "tool.failed"]), + binding("postToolUseFailure", &["tool.failed"]), + binding("permissionRequest", &["approval.requested"]), + binding("notification", &["diagnostic"]), + binding("errorOccurred", &["diagnostic"]), + binding("preCompact", &["compaction.started"]), + binding( + "agentStop", + &["turn.completed", "turn.failed", "turn.cancelled"], + ), + binding("subagentStart", &["subagent.started"]), + binding("subagentStop", &["subagent.completed", "subagent.failed"]), + ], + capabilities( + Support::Supported, + Support::Supported, + Support::Supported, + Support::Supported, + Support::Unavailable, + Support::Unknown, + ), + ); + copilot.contract_source = + Some("https://docs.github.com/en/copilot/reference/hooks-reference".to_string()); + copilot.transcript_location_hints = + vec!["validated Copilot session-state events.jsonl".to_string()]; + + let mut grok = manifest( + "grok", + "xAI Grok Build", + &["grok-build"], + Deployment::NativeOrCompatible, + ".grok/hooks/trail.json", + Support::Supported, + vec![ + binding("SessionStart", &["session.started", "session.resumed"]), + binding("SessionEnd", &["session.ended"]), + binding("UserPromptSubmit", &["turn.started", "message.user"]), + binding("PreToolUse", &["tool.started"]), + binding("PostToolUse", &["tool.completed"]), + binding("PostToolUseFailure", &["tool.failed"]), + binding("PermissionDenied", &["approval.decided"]), + binding("Notification", &["diagnostic"]), + binding("Stop", &["turn.completed"]), + binding("StopFailure", &["turn.failed"]), + binding("SubagentStart", &["subagent.started"]), + binding("SubagentStop", &["subagent.completed", "subagent.failed"]), + binding("PreCompact", &["compaction.started"]), + binding("PostCompact", &["compaction.completed"]), + ], + capabilities( + Support::Supported, + Support::Supported, + Support::Supported, + Support::Supported, + Support::Partial, + Support::Supported, + ), + ); + grok.contract_source = Some("https://docs.x.ai/build/features/hooks".to_string()); + grok.transcript_location_hints = vec!["~/.grok/sessions".to_string()]; + grok.canonical_export_command = Some(vec![ + "grok".to_string(), + "export".to_string(), + "{session_id}".to_string(), + ]); + grok.notes = vec![ + "Project hook files require explicit /hooks-trust or a trusted launch.".to_string(), + "Grok also reads Claude Code and Cursor hook configuration, but Trail prefers an owned .grok hook file." + .to_string(), + "ACP through `grok agent stdio` is the stable high-fidelity path when available." + .to_string(), + ]; + + let mut kiro = manifest( + "kiro", + "Kiro", + &["kiro-cli"], + Deployment::NativeOrCompatible, + ".kiro/hooks/trail.json", + Support::Experimental, + vec![ + binding("SessionStart", &["session.started", "session.resumed"]), + binding("UserPromptSubmit", &["turn.started", "message.user"]), + binding("PreToolUse", &["tool.started"]), + binding("PostToolUse", &["tool.completed", "tool.failed"]), + binding( + "Stop", + &[ + "message.assistant.completed", + "turn.completed", + "turn.failed", + "turn.cancelled", + ], + ), + binding("PreTaskExec", &["subagent.started"]), + binding("PostTaskExec", &["subagent.completed", "subagent.failed"]), + ], + capabilities( + Support::Partial, + Support::Supported, + Support::Supported, + Support::Unavailable, + Support::Unavailable, + Support::Unknown, + ), + ); + kiro.adapter_version = "trail/kiro@3".to_string(); + kiro.contract_source = Some("https://kiro.dev/docs/hooks/".to_string()); + kiro.provider_version_range = Some(">=2.8.0".to_string()); + kiro.notes = vec![ + "Trail installs Kiro's versioned standalone hook format under .kiro/hooks for the IDE and the CLI v3 engine. Kiro CLI package 2.8.0 and newer expose that engine with `kiro-cli --v3`; the default v2 engine continues to use embedded custom-agent hooks." + .to_string(), + "Kiro currently exposes SessionStart but no standalone SessionEnd trigger; Stop closes turns." + .to_string(), + ]; + + vec![ + codex, claude, pi, opencode, cursor, gemini, copilot, grok, kiro, + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn built_in_registry_has_nine_unique_providers_and_stable_aliases() { + let registry = AgentProviderRegistry::built_in().unwrap(); + assert_eq!(registry.list().len(), 9); + assert_eq!(registry.resolve("claude").unwrap().provider, "claude-code"); + assert_eq!(registry.resolve("gemini-cli").unwrap().provider, "gemini"); + assert_eq!(registry.resolve("grok-build").unwrap().provider, "grok"); + assert_eq!(registry.resolve("kiro-cli").unwrap().provider, "kiro"); + assert!(registry.resolve("unknown").is_err()); + } + + #[test] + fn codex_manifest_matches_current_documented_command_hook_set() { + let registry = AgentProviderRegistry::built_in().unwrap(); + let codex = registry.resolve("codex").unwrap(); + let events = codex + .events + .iter() + .map(|event| event.native_event.as_str()) + .collect::>(); + assert_eq!( + events, + BTreeSet::from([ + "PermissionRequest", + "PostCompact", + "PostToolUse", + "PreCompact", + "PreToolUse", + "SessionStart", + "Stop", + "SubagentStart", + "SubagentStop", + "UserPromptSubmit", + ]) + ); + assert!(!events.contains("SessionEnd")); + assert_eq!( + codex.contract_source.as_deref(), + Some("https://learn.chatgpt.com/docs/hooks") + ); + } + + #[test] + fn every_binding_targets_normalized_or_provider_namespaced_events() { + for manifest in built_in_agent_adapter_manifests() { + manifest.validate().unwrap(); + for event in manifest.events { + for normalized in event.normalized_events { + let event_type = crate::AgentLifecycleEventType::new(normalized.clone()); + assert!( + event_type.kind() != crate::AgentLifecycleEventKind::Unknown + || normalized.starts_with(&format!("provider.{}.", manifest.provider)), + "{} -> {}", + event.native_event, + normalized + ); + } + } + } + } + + #[test] + fn provider_version_ranges_are_compared_against_version_command_output() { + assert_eq!( + version_satisfies_range("kiro-cli 2.7.0", ">=2.8.0"), + Some(false) + ); + assert_eq!( + version_satisfies_range("kiro-cli 2.8.0", ">=2.8.0"), + Some(true) + ); + assert_eq!( + version_satisfies_range("kiro-cli 2.12.1", ">=2.8.0"), + Some(true) + ); + assert_eq!(version_satisfies_range("unknown", ">=2.8.0"), None); + } +} diff --git a/trail/src/agent_hooks/install.rs b/trail/src/agent_hooks/install.rs new file mode 100644 index 0000000..989597b --- /dev/null +++ b/trail/src/agent_hooks/install.rs @@ -0,0 +1,1433 @@ +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::path::{Component, Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::{json, Map, Value}; +use sha2::{Digest, Sha256}; + +use super::{AgentAdapterDeploymentClass, AgentAdapterManifest, AgentProviderRegistry}; +use crate::{Error, Result}; + +const MAX_CONFIG_BYTES: u64 = 2 * 1024 * 1024; +const OWNERSHIP_PREFIX: &str = "trail "; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentHookInstallScope { + Project, + User, +} + +impl AgentHookInstallScope { + pub fn as_str(self) -> &'static str { + match self { + Self::Project => "project", + Self::User => "user", + } + } +} + +#[derive(Clone, Debug)] +pub struct AgentHookInstallRequest<'a> { + pub registry: &'a AgentProviderRegistry, + pub provider: &'a str, + pub workspace_id: &'a str, + pub workspace_root: &'a Path, + pub home_dir: Option<&'a Path>, + pub scope: AgentHookInstallScope, + pub force: bool, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentHookInstallAction { + Create, + Update, + Noop, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentHookInstallPlan { + pub installation_id: String, + pub provider: String, + pub scope: AgentHookInstallScope, + pub adapter_version: String, + pub provider_version_range: Option, + pub config_path: PathBuf, + pub action: AgentHookInstallAction, + pub before_digest: Option, + pub after_digest: String, + pub manifest_digest: String, + pub ownership_inventory: Vec, + #[serde(skip)] + pub before_bytes: Option>, + pub desired_bytes: Vec, + pub owned_file: bool, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentHookInstallReport { + pub installation_id: String, + pub provider: String, + pub scope: AgentHookInstallScope, + pub config_path: PathBuf, + pub action: AgentHookInstallAction, + pub before_digest: Option, + pub after_digest: String, + pub ownership_inventory: Vec, + pub dry_run: bool, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentHookInstallationRecord { + pub installation_id: String, + pub workspace_id: String, + pub provider: String, + pub scope: AgentHookInstallScope, + pub config_path: PathBuf, + pub lane_id: Option, + pub manifest_digest: String, + pub ownership_inventory: Vec, + pub config_before_digest: Option, + pub config_after_digest: String, + pub adapter_version: String, + pub provider_version_range: Option, + pub detected_provider_version: Option, + pub capability_status: String, + pub status: String, + pub installed_at: i64, + pub verified_at: Option, + pub last_receipt_at: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentHookInstallationStatus { + Missing, + Installed, + Drifted, + Foreign, + Malformed, +} + +pub fn build_agent_hook_install_plan( + request: AgentHookInstallRequest<'_>, +) -> Result { + let manifest = request.registry.resolve(request.provider)?; + let installation_id = + stable_installation_id(request.workspace_id, &manifest.provider, request.scope); + let (base, relative) = target_for( + manifest, + request.scope, + request.workspace_root, + request.home_dir, + )?; + validate_relative_target(&relative)?; + let base = canonicalize_or_create_base(base)?; + let workspace_root = + request + .workspace_root + .canonicalize() + .map_err(|error| Error::InvalidPath { + path: request.workspace_root.display().to_string(), + reason: format!("cannot canonicalize hook workspace root: {error}"), + })?; + let target = base.join(&relative); + validate_target_path(&base, &target)?; + let before = read_bounded_optional(&target)?; + let before_digest = before.as_deref().map(content_digest); + let (desired_bytes, inventory, owned_file) = desired_asset( + manifest, + &installation_id, + &workspace_root, + before.as_deref(), + request.force, + )?; + validate_provider_asset(manifest, &desired_bytes)?; + let after_digest = content_digest(&desired_bytes); + let manifest_digest = manifest_digest(manifest)?; + let action = match before.as_deref() { + None => AgentHookInstallAction::Create, + Some(existing) if existing == desired_bytes => AgentHookInstallAction::Noop, + Some(_) => AgentHookInstallAction::Update, + }; + Ok(AgentHookInstallPlan { + installation_id, + provider: manifest.provider.clone(), + scope: request.scope, + adapter_version: manifest.adapter_version.clone(), + provider_version_range: manifest.provider_version_range.clone(), + config_path: target, + action, + before_digest, + after_digest, + manifest_digest, + ownership_inventory: inventory, + before_bytes: before, + desired_bytes, + owned_file, + }) +} + +pub fn rollback_agent_hook_install_plan(plan: &AgentHookInstallPlan) -> Result<()> { + let _target_lock = acquire_target_lock(&plan.config_path)?; + let current = read_bounded_optional(&plan.config_path)?; + if current.as_deref().map(content_digest).as_deref() != Some(plan.after_digest.as_str()) { + return Err(Error::Conflict(format!( + "cannot roll back `{}` because it changed after Trail installed it", + plan.config_path.display() + ))); + } + if let Some(before) = plan.before_bytes.as_deref() { + atomic_write(&plan.config_path, before, 0o600) + } else { + fs::remove_file(&plan.config_path)?; + sync_parent(&plan.config_path) + } +} + +pub fn apply_agent_hook_install_plan( + plan: &AgentHookInstallPlan, + dry_run: bool, +) -> Result { + if !dry_run && plan.action != AgentHookInstallAction::Noop { + let _target_lock = acquire_target_lock(&plan.config_path)?; + let current = read_bounded_optional(&plan.config_path)?; + if current.as_deref().map(content_digest) != plan.before_digest { + return Err(Error::Conflict(format!( + "agent hook config `{}` changed after planning; build a new install plan", + plan.config_path.display() + ))); + } + atomic_write(&plan.config_path, &plan.desired_bytes, 0o600)?; + } + Ok(AgentHookInstallReport { + installation_id: plan.installation_id.clone(), + provider: plan.provider.clone(), + scope: plan.scope, + config_path: plan.config_path.clone(), + action: plan.action.clone(), + before_digest: plan.before_digest.clone(), + after_digest: plan.after_digest.clone(), + ownership_inventory: plan.ownership_inventory.clone(), + dry_run, + }) +} + +pub fn inspect_agent_hook_installation( + plan: &AgentHookInstallPlan, +) -> Result { + let Some(bytes) = read_bounded_optional(&plan.config_path)? else { + return Ok(AgentHookInstallationStatus::Missing); + }; + if bytes == plan.desired_bytes { + return Ok(AgentHookInstallationStatus::Installed); + } + let manifest = AgentProviderRegistry::built_in()? + .resolve(&plan.provider)? + .clone(); + if validate_provider_asset(&manifest, &bytes).is_err() { + return Ok(AgentHookInstallationStatus::Malformed); + } + if plan.owned_file { + let text = String::from_utf8_lossy(&bytes); + return Ok( + if text.contains(&format!("installation={}", plan.installation_id)) + || owned_json_installation(&bytes).as_deref() == Some(plan.installation_id.as_str()) + { + AgentHookInstallationStatus::Drifted + } else { + AgentHookInstallationStatus::Foreign + }, + ); + } + let root: Value = match serde_json::from_slice(&bytes) { + Ok(value) => value, + Err(_) => return Ok(AgentHookInstallationStatus::Malformed), + }; + Ok( + if json_contains_installation(&root, &plan.installation_id) { + AgentHookInstallationStatus::Drifted + } else { + AgentHookInstallationStatus::Missing + }, + ) +} + +pub fn remove_agent_hook_installation( + plan: &AgentHookInstallPlan, + expected_after_digest: &str, + dry_run: bool, +) -> Result { + let _target_lock = if dry_run { + None + } else { + Some(acquire_target_lock(&plan.config_path)?) + }; + let Some(bytes) = read_bounded_optional(&plan.config_path)? else { + return Ok(AgentHookInstallReport { + installation_id: plan.installation_id.clone(), + provider: plan.provider.clone(), + scope: plan.scope, + config_path: plan.config_path.clone(), + action: AgentHookInstallAction::Noop, + before_digest: None, + after_digest: content_digest(&[]), + ownership_inventory: plan.ownership_inventory.clone(), + dry_run, + }); + }; + let current_digest = content_digest(&bytes); + let removed = if plan.owned_file { + if current_digest != expected_after_digest { + return Err(Error::Conflict(format!( + "refusing to remove modified Trail-owned hook file `{}`: expected {}, found {}", + plan.config_path.display(), + expected_after_digest, + current_digest + ))); + } + None + } else { + Some(remove_owned_json_entries(&bytes, &plan.installation_id)?) + }; + let after_digest = removed + .as_deref() + .map(content_digest) + .unwrap_or_else(|| content_digest(&[])); + if !dry_run { + if let Some(content) = removed.as_deref() { + atomic_write(&plan.config_path, content, 0o600)?; + } else { + fs::remove_file(&plan.config_path)?; + sync_parent(&plan.config_path)?; + } + } + Ok(AgentHookInstallReport { + installation_id: plan.installation_id.clone(), + provider: plan.provider.clone(), + scope: plan.scope, + config_path: plan.config_path.clone(), + action: AgentHookInstallAction::Update, + before_digest: Some(current_digest), + after_digest, + ownership_inventory: plan.ownership_inventory.clone(), + dry_run, + }) +} + +fn stable_installation_id( + workspace_id: &str, + provider: &str, + scope: AgentHookInstallScope, +) -> String { + let digest = Sha256::digest( + format!( + "trail-agent-hook-v1:{workspace_id}:{provider}:{}", + scope.as_str() + ) + .as_bytes(), + ); + format!("hook_{}", hex(&digest[..16])) +} + +fn target_for( + manifest: &AgentAdapterManifest, + scope: AgentHookInstallScope, + workspace_root: &Path, + home: Option<&Path>, +) -> Result<(PathBuf, PathBuf)> { + let project = manifest.project_config_path.as_deref().ok_or_else(|| { + Error::InvalidInput(format!("{} has no install target", manifest.provider)) + })?; + match scope { + AgentHookInstallScope::Project => { + Ok((workspace_root.to_path_buf(), PathBuf::from(project))) + } + AgentHookInstallScope::User => { + let home = home.ok_or_else(|| { + Error::InvalidInput("cannot resolve user home for hook installation".to_string()) + })?; + let relative = match manifest.provider.as_str() { + "codex" => ".codex/hooks.json", + "claude-code" => ".claude/settings.json", + "pi" => ".pi/agent/extensions/trail/index.ts", + "opencode" => ".config/opencode/plugins/trail.ts", + "cursor" => ".cursor/hooks.json", + "gemini" => ".gemini/settings.json", + "copilot" => ".copilot/hooks/trail.json", + "grok" => ".grok/hooks/trail.json", + "kiro" => ".kiro/hooks/trail.json", + other => { + return Err(Error::InvalidInput(format!( + "provider `{other}` does not declare a user hook target" + ))); + } + }; + Ok((home.to_path_buf(), PathBuf::from(relative))) + } + } +} + +fn canonicalize_or_create_base(base: PathBuf) -> Result { + fs::create_dir_all(&base)?; + Ok(base.canonicalize()?) +} + +fn validate_relative_target(path: &Path) -> Result<()> { + if path.is_absolute() + || path.components().any(|component| { + matches!( + component, + Component::ParentDir | Component::RootDir | Component::Prefix(_) + ) + }) + { + return Err(Error::InvalidPath { + path: path.display().to_string(), + reason: "agent hook target must be a relative path without parent traversal" + .to_string(), + }); + } + Ok(()) +} + +fn validate_target_path(base: &Path, target: &Path) -> Result<()> { + if !target.starts_with(base) { + return Err(Error::InvalidPath { + path: target.display().to_string(), + reason: "agent hook target escaped its install scope".to_string(), + }); + } + let relative = target.strip_prefix(base).map_err(|_| Error::InvalidPath { + path: target.display().to_string(), + reason: "agent hook target escaped its install scope".to_string(), + })?; + let mut cursor = base.to_path_buf(); + for component in relative.components() { + cursor.push(component); + match fs::symlink_metadata(&cursor) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(Error::InvalidPath { + path: cursor.display().to_string(), + reason: "agent hook installation refuses symlinked path components".to_string(), + }); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => break, + Err(error) => return Err(error.into()), + } + } + Ok(()) +} + +fn desired_asset( + manifest: &AgentAdapterManifest, + installation_id: &str, + workspace_root: &Path, + existing: Option<&[u8]>, + force: bool, +) -> Result<(Vec, Vec, bool)> { + match manifest.deployment { + AgentAdapterDeploymentClass::ProjectExtension + | AgentAdapterDeploymentClass::ProjectPlugin => { + let content = generated_typescript(manifest, installation_id, workspace_root); + refuse_foreign_owned_file(existing, installation_id, force)?; + Ok((content.into_bytes(), vec!["owned-file".to_string()], true)) + } + AgentAdapterDeploymentClass::NativeOrCompatible + if matches!(manifest.provider.as_str(), "grok" | "kiro") => + { + let value = generated_owned_json(manifest, installation_id, workspace_root); + let content = json_bytes(&value)?; + refuse_foreign_owned_json(existing, installation_id, force)?; + Ok((content, vec!["owned-file".to_string()], true)) + } + AgentAdapterDeploymentClass::JsonCommandConfig if manifest.provider == "copilot" => { + let value = generated_owned_json(manifest, installation_id, workspace_root); + let content = json_bytes(&value)?; + refuse_foreign_owned_json(existing, installation_id, force)?; + Ok((content, vec!["owned-file".to_string()], true)) + } + AgentAdapterDeploymentClass::JsonCommandConfig + | AgentAdapterDeploymentClass::NativeOrCompatible => { + let (content, inventory) = + merge_json_hooks(manifest, installation_id, workspace_root, existing)?; + Ok((content, inventory, false)) + } + } +} + +fn merge_json_hooks( + manifest: &AgentAdapterManifest, + installation_id: &str, + workspace_root: &Path, + existing: Option<&[u8]>, +) -> Result<(Vec, Vec)> { + let mut root = parse_json_object(existing)?; + if manifest.provider == "cursor" { + match root.get("version") { + Some(Value::Number(version)) if version.as_u64() == Some(1) => {} + Some(_) => { + return Err(Error::Conflict( + "Cursor hook config `version` must be the number 1".to_string(), + )); + } + None => { + root.insert("version".to_string(), Value::from(1)); + } + } + } + let hooks = root + .entry("hooks".to_string()) + .or_insert_with(|| Value::Object(Map::new())); + let hooks = hooks.as_object_mut().ok_or_else(|| { + Error::Conflict("provider config `hooks` field is not a JSON object".to_string()) + })?; + remove_owned_from_hook_map(hooks, installation_id)?; + let mut inventory = Vec::new(); + for binding in &manifest.events { + let entries = hooks + .entry(binding.native_event.clone()) + .or_insert_with(|| Value::Array(Vec::new())); + let entries = entries.as_array_mut().ok_or_else(|| { + Error::Conflict(format!( + "provider hook event `{}` is not a JSON array", + binding.native_event + )) + })?; + let command = hook_command( + &manifest.provider, + &binding.native_event, + installation_id, + workspace_root, + ); + let entry = hook_config_entry(manifest.provider.as_str(), binding, command); + entries.push(entry); + inventory.push(format!("hooks.{}:{installation_id}", binding.native_event)); + } + Ok((json_bytes(&Value::Object(root))?, inventory)) +} + +fn generated_owned_json( + manifest: &AgentAdapterManifest, + installation_id: &str, + workspace_root: &Path, +) -> Value { + if manifest.provider == "kiro" { + let hooks = manifest + .events + .iter() + .map(|binding| { + json!({ + "name": format!("trail-{}", kiro_hook_name(&binding.native_event)), + "description": "Record this Kiro lifecycle event in Trail", + "trigger": binding.native_event, + "action": { + "type": "command", + "command": hook_command( + &manifest.provider, + &binding.native_event, + installation_id, + workspace_root, + ) + }, + "timeout": hook_timeout_seconds(&binding.native_event), + "enabled": true + }) + }) + .collect::>(); + return json!({"version": "v1", "hooks": hooks}); + } + + let mut hooks = Map::new(); + for binding in &manifest.events { + let command = hook_command( + &manifest.provider, + &binding.native_event, + installation_id, + workspace_root, + ); + let entry = if manifest.provider == "copilot" { + json!({ + "type": "command", + "bash": command, + "powershell": hook_command_powershell( + &manifest.provider, + &binding.native_event, + installation_id, + workspace_root, + ), + "timeoutSec": hook_timeout_seconds(&binding.native_event) + }) + } else { + hook_config_entry(manifest.provider.as_str(), binding, command) + }; + hooks.insert(binding.native_event.clone(), Value::Array(vec![entry])); + } + let mut root = Map::new(); + if manifest.provider == "copilot" { + root.insert("version".to_string(), Value::from(1)); + } + root.insert("hooks".to_string(), Value::Object(hooks)); + Value::Object(root) +} + +/// Render a provider hook group without serializing absent optional fields as JSON null. +/// Claude Code treats an omitted matcher as match-all; `matcher: null` is not a valid +/// matcher value and causes version-dependent configuration drift. +fn hook_config_entry( + provider: &str, + binding: &crate::agent_hooks::AgentHookEventBinding, + command: String, +) -> Value { + if provider == "cursor" { + let mut entry = Map::new(); + entry.insert("command".to_string(), Value::String(command)); + entry.insert( + "timeout".to_string(), + Value::from(hook_timeout_seconds(&binding.native_event)), + ); + if let Some(matcher) = binding.matcher.as_ref() { + entry.insert("matcher".to_string(), Value::String(matcher.clone())); + } + return Value::Object(entry); + } + let mut entry = Map::new(); + if let Some(matcher) = binding.matcher.as_ref() { + entry.insert("matcher".to_string(), Value::String(matcher.clone())); + } + entry.insert( + "hooks".to_string(), + json!([{ + "type": "command", + "command": command, + "timeout": hook_timeout_value(provider, &binding.native_event) + }]), + ); + Value::Object(entry) +} + +fn generated_typescript( + manifest: &AgentAdapterManifest, + installation_id: &str, + workspace_root: &Path, +) -> String { + let header = format!( + "// Generated by Trail. installation={installation_id} adapter={}\n// Manage with: trail agent hooks remove {}\n", + manifest.adapter_version, manifest.provider + ); + let bindings = manifest + .events + .iter() + .map(|binding| { + format!( + " [{} , {}],", + serde_json::to_string(&binding.native_event).expect("string serialization"), + serde_json::to_string(&hook_command( + &manifest.provider, + &binding.native_event, + installation_id, + workspace_root, + )) + .expect("string serialization") + ) + }) + .collect::>() + .join("\n"); + if manifest.provider == "pi" { + format!( + "{header}import type {{ ExtensionAPI }} from \"@earendil-works/pi-coding-agent\";\nimport {{ spawn }} from \"node:child_process\";\n\nconst hooks = new Map([\n{bindings}\n]);\nfunction send(command: string, payload: unknown): Promise {{\n return new Promise((resolve) => {{\n try {{\n const child = spawn(command, {{ shell: true, stdio: [\"pipe\", \"ignore\", \"ignore\"] }});\n const timer = setTimeout(() => {{ child.kill(); resolve(); }}, 30000);\n child.once(\"error\", () => {{ clearTimeout(timer); resolve(); }});\n child.once(\"close\", () => {{ clearTimeout(timer); resolve(); }});\n child.stdin.end(JSON.stringify(payload));\n }} catch {{ resolve(); }}\n }});\n}}\nexport default function trail(pi: ExtensionAPI): void {{\n for (const [event, command] of hooks) {{\n (pi.on as any)(event, async (payload: Record, ctx: any) => {{\n const sessionFile = ctx?.sessionManager?.getSessionFile?.();\n await send(command, {{ ...payload, session_id: (payload as any).session_id ?? sessionFile, transcript_path: sessionFile, cwd: (payload as any).cwd ?? ctx?.cwd }});\n }});\n }}\n}}\n" + ) + } else { + format!( + "{header}import type {{ Plugin }} from \"@opencode-ai/plugin\";\nimport {{ spawn }} from \"node:child_process\";\n\nconst hooks = new Map([\n{bindings}\n]);\nfunction send(event: string, payload: unknown): Promise {{\n const command = hooks.get(event);\n if (!command) return Promise.resolve();\n return new Promise((resolve) => {{\n try {{\n const child = spawn(command, {{ shell: true, stdio: [\"pipe\", \"ignore\", \"ignore\"] }});\n const timer = setTimeout(() => {{ child.kill(); resolve(); }}, 30000);\n child.once(\"error\", () => {{ clearTimeout(timer); resolve(); }});\n child.once(\"close\", () => {{ clearTimeout(timer); resolve(); }});\n child.stdin.end(JSON.stringify(payload));\n }} catch {{ resolve(); }}\n }});\n}}\nexport const TrailPlugin: Plugin = async (ctx) => ({{\n event: async (input: {{ event: {{ type: string }} }}) => send(input.event.type, {{ ...input.event, cwd: ctx.directory }}),\n \"chat.message\": async (input: unknown, output: unknown) => send(\"chat.message\", {{ input, output, cwd: ctx.directory }}),\n \"tool.execute.before\": async (input: unknown, output: unknown) => send(\"tool.execute.before\", {{ input, output, cwd: ctx.directory }}),\n \"tool.execute.after\": async (input: unknown, output: unknown) => send(\"tool.execute.after\", {{ input, output, cwd: ctx.directory }}),\n}});\n" + ) + } +} + +fn hook_command( + provider: &str, + event: &str, + installation_id: &str, + workspace_root: &Path, +) -> String { + format!( + "trail --workspace {} agent hook receive {} {} --installation {}", + shell_quote(&workspace_root.to_string_lossy()), + shell_quote(provider), + shell_quote(event), + shell_quote(installation_id) + ) +} + +fn shell_quote(value: &str) -> String { + if value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) + { + value.to_string() + } else { + format!("'{}'", value.replace('\'', "'\\''")) + } +} + +fn hook_command_powershell( + provider: &str, + event: &str, + installation_id: &str, + workspace_root: &Path, +) -> String { + fn quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) + } + format!( + "trail --workspace {} agent hook receive {} {} --installation {}", + quote(&workspace_root.to_string_lossy()), + quote(provider), + quote(event), + quote(installation_id), + ) +} + +fn hook_timeout_seconds(event: &str) -> u64 { + if event.to_ascii_lowercase().contains("stop") + || event.to_ascii_lowercase().contains("end") + || event.to_ascii_lowercase().contains("shutdown") + { + 60 + } else { + 30 + } +} + +fn hook_timeout_value(provider: &str, event: &str) -> u64 { + let seconds = hook_timeout_seconds(event); + if provider == "gemini" { + seconds.saturating_mul(1_000) + } else { + seconds + } +} + +fn kiro_hook_name(event: &str) -> String { + event + .chars() + .enumerate() + .flat_map(|(index, character)| { + if character.is_ascii_uppercase() && index > 0 { + vec!['-', character.to_ascii_lowercase()] + } else { + vec![character.to_ascii_lowercase()] + } + }) + .collect() +} + +fn parse_json_object(existing: Option<&[u8]>) -> Result> { + let Some(bytes) = existing else { + return Ok(Map::new()); + }; + let value: Value = serde_json::from_slice(bytes).map_err(|error| { + Error::Conflict(format!("provider hook config is not valid JSON: {error}")) + })?; + value.as_object().cloned().ok_or_else(|| { + Error::Conflict("provider hook config root is not a JSON object".to_string()) + }) +} + +fn remove_owned_json_entries(bytes: &[u8], installation_id: &str) -> Result> { + let mut root = parse_json_object(Some(bytes))?; + let Some(hooks) = root.get_mut("hooks") else { + return Err(Error::Conflict(format!( + "Trail installation `{installation_id}` is absent from provider config" + ))); + }; + let hooks = hooks.as_object_mut().ok_or_else(|| { + Error::Conflict("provider config `hooks` field is not a JSON object".to_string()) + })?; + let removed = remove_owned_from_hook_map(hooks, installation_id)?; + if removed == 0 { + return Err(Error::Conflict(format!( + "Trail installation `{installation_id}` is absent or its ownership marker drifted" + ))); + } + if hooks.is_empty() { + root.remove("hooks"); + } + json_bytes(&Value::Object(root)) +} + +fn remove_owned_from_hook_map( + hooks: &mut Map, + installation_id: &str, +) -> Result { + let marker = format!("--installation {installation_id}"); + let mut removed = 0; + for (event, value) in hooks.iter_mut() { + let entries = value.as_array_mut().ok_or_else(|| { + Error::Conflict(format!("provider hook event `{event}` is not a JSON array")) + })?; + entries.retain_mut(|entry| { + if let Some(inner) = entry.get_mut("hooks") { + let Some(inner) = inner.as_array_mut() else { + return true; + }; + let before = inner.len(); + inner.retain(|hook| !entry_matches_marker(hook, &marker)); + removed += before - inner.len(); + !inner.is_empty() + } else if entry_matches_marker(entry, &marker) { + removed += 1; + false + } else { + true + } + }); + } + hooks.retain(|_, value| value.as_array().is_some_and(|entries| !entries.is_empty())); + Ok(removed) +} + +fn entry_matches_marker(entry: &Value, marker: &str) -> bool { + ["command", "bash", "powershell"].iter().any(|key| { + entry + .get(*key) + .and_then(Value::as_str) + .is_some_and(|command| { + command.starts_with(OWNERSHIP_PREFIX) + && command.contains(" agent hook receive ") + && command.contains(marker) + }) + }) +} + +fn json_contains_installation(value: &Value, installation_id: &str) -> bool { + match value { + Value::String(value) => value.contains(&format!("--installation {installation_id}")), + Value::Array(values) => values + .iter() + .any(|value| json_contains_installation(value, installation_id)), + Value::Object(values) => values + .values() + .any(|value| json_contains_installation(value, installation_id)), + _ => false, + } +} + +fn refuse_foreign_owned_file( + existing: Option<&[u8]>, + installation_id: &str, + force: bool, +) -> Result<()> { + if let Some(bytes) = existing { + let text = String::from_utf8_lossy(bytes); + if !text.contains(&format!( + "Generated by Trail. installation={installation_id}" + )) && !force + { + return Err(Error::Conflict( + "refusing to overwrite a foreign provider plugin/extension file; pass --force only after reviewing it" + .to_string(), + )); + } + } + Ok(()) +} + +fn refuse_foreign_owned_json( + existing: Option<&[u8]>, + installation_id: &str, + force: bool, +) -> Result<()> { + if let Some(bytes) = existing { + let owned = owned_json_installation(bytes).is_some_and(|value| value == installation_id); + if !owned && !force { + return Err(Error::Conflict( + "refusing to overwrite a foreign provider hook file; pass --force only after reviewing it" + .to_string(), + )); + } + } + Ok(()) +} + +fn owned_json_installation(bytes: &[u8]) -> Option { + let value = serde_json::from_slice::(bytes).ok()?; + value + .get("trailOwnership") + .and_then(|value| value.get("installation")) + .and_then(Value::as_str) + .map(str::to_string) + .or_else(|| find_installation_marker(&value)) +} + +fn find_installation_marker(value: &Value) -> Option { + match value { + Value::String(value) => value + .split_whitespace() + .collect::>() + .windows(2) + .find_map(|pair| { + (pair[0] == "--installation" && pair[1].starts_with("hook_")) + .then(|| pair[1].trim_matches(['\'', '"']).to_string()) + }), + Value::Array(values) => values.iter().find_map(find_installation_marker), + Value::Object(values) => values.values().find_map(find_installation_marker), + _ => None, + } +} + +fn validate_provider_asset(manifest: &AgentAdapterManifest, bytes: &[u8]) -> Result<()> { + if matches!( + manifest.deployment, + AgentAdapterDeploymentClass::ProjectExtension | AgentAdapterDeploymentClass::ProjectPlugin + ) { + std::str::from_utf8(bytes).map_err(|error| { + Error::Conflict(format!( + "{} hook extension is not UTF-8: {error}", + manifest.provider + )) + })?; + return Ok(()); + } + + let root: Value = serde_json::from_slice(bytes).map_err(|error| { + Error::Conflict(format!( + "{} hook config is not valid JSON: {error}", + manifest.provider + )) + })?; + let root = root.as_object().ok_or_else(|| { + Error::Conflict(format!( + "{} hook config root must be a JSON object", + manifest.provider + )) + })?; + + match manifest.provider.as_str() { + "kiro" => validate_kiro_hook_config(root), + "cursor" => { + require_version(root, Value::from(1), "Cursor")?; + validate_flat_hook_map(root, "Cursor", &["command", "prompt"]) + } + "copilot" => { + require_version(root, Value::from(1), "Copilot")?; + validate_flat_hook_map( + root, + "Copilot", + &["bash", "powershell", "command", "url", "prompt"], + ) + } + _ => validate_nested_hook_map(root, &manifest.display_name), + } +} + +fn require_version(root: &Map, expected: Value, provider: &str) -> Result<()> { + if root.get("version") != Some(&expected) { + return Err(Error::Conflict(format!( + "{provider} hook config has an unsupported or missing `version`" + ))); + } + Ok(()) +} + +fn validate_kiro_hook_config(root: &Map) -> Result<()> { + require_version(root, Value::String("v1".to_string()), "Kiro")?; + let hooks = root + .get("hooks") + .and_then(Value::as_array) + .ok_or_else(|| Error::Conflict("Kiro hook config `hooks` must be an array".to_string()))?; + for hook in hooks { + let hook = hook + .as_object() + .ok_or_else(|| Error::Conflict("Kiro hook definitions must be objects".to_string()))?; + if hook.get("name").and_then(Value::as_str).is_none() + || hook.get("trigger").and_then(Value::as_str).is_none() + || hook.get("action").and_then(Value::as_object).is_none() + { + return Err(Error::Conflict( + "Kiro hooks require string `name`, string `trigger`, and object `action` fields" + .to_string(), + )); + } + let action = hook["action"].as_object().expect("checked above"); + if action.get("type").and_then(Value::as_str) != Some("command") + || action.get("command").and_then(Value::as_str).is_none() + { + return Err(Error::Conflict( + "Trail Kiro hooks require command actions".to_string(), + )); + } + } + Ok(()) +} + +fn validate_flat_hook_map( + root: &Map, + provider: &str, + command_fields: &[&str], +) -> Result<()> { + let Some(hooks_value) = root.get("hooks") else { + return Ok(()); + }; + let hooks = hooks_value.as_object().ok_or_else(|| { + Error::Conflict(format!("{provider} hook config `hooks` must be an object")) + })?; + for (event, entries) in hooks { + let entries = entries.as_array().ok_or_else(|| { + Error::Conflict(format!("{provider} hook event `{event}` must be an array")) + })?; + for entry in entries { + let entry = entry.as_object().ok_or_else(|| { + Error::Conflict(format!( + "{provider} hook event `{event}` contains a non-object entry" + )) + })?; + if !command_fields + .iter() + .any(|field| entry.get(*field).and_then(Value::as_str).is_some()) + { + return Err(Error::Conflict(format!( + "{provider} hook event `{event}` has no supported command, URL, or prompt field" + ))); + } + } + } + Ok(()) +} + +fn validate_nested_hook_map(root: &Map, provider: &str) -> Result<()> { + let Some(hooks_value) = root.get("hooks") else { + return Ok(()); + }; + let hooks = hooks_value.as_object().ok_or_else(|| { + Error::Conflict(format!("{provider} hook config `hooks` must be an object")) + })?; + for (event, groups) in hooks { + let groups = groups.as_array().ok_or_else(|| { + Error::Conflict(format!("{provider} hook event `{event}` must be an array")) + })?; + for group in groups { + let handlers = group + .get("hooks") + .and_then(Value::as_array) + .ok_or_else(|| { + Error::Conflict(format!( + "{provider} hook event `{event}` groups require a `hooks` array" + )) + })?; + for handler in handlers { + let handler = handler.as_object().ok_or_else(|| { + Error::Conflict(format!( + "{provider} hook event `{event}` contains a non-object handler" + )) + })?; + if handler.get("type").and_then(Value::as_str).is_none() { + return Err(Error::Conflict(format!( + "{provider} hook event `{event}` handler is missing `type`" + ))); + } + } + } + } + Ok(()) +} + +fn read_bounded_optional(path: &Path) -> Result>> { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(Error::InvalidPath { + path: path.display().to_string(), + reason: "agent hook target must be a regular non-symlink file".to_string(), + }); + } + if metadata.len() > MAX_CONFIG_BYTES { + return Err(Error::InvalidInput(format!( + "agent hook config `{}` exceeds {} bytes", + path.display(), + MAX_CONFIG_BYTES + ))); + } + Ok(Some(fs::read(path)?)) +} + +fn atomic_write(path: &Path, bytes: &[u8], mode: u32) -> Result<()> { + let parent = path.parent().ok_or_else(|| Error::InvalidPath { + path: path.display().to_string(), + reason: "agent hook target has no parent directory".to_string(), + })?; + fs::create_dir_all(parent)?; + validate_target_path(&parent.canonicalize()?, path)?; + let name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| Error::InvalidPath { + path: path.display().to_string(), + reason: "agent hook target filename is not valid UTF-8".to_string(), + })?; + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|value| value.as_nanos()) + .unwrap_or_default(); + let temp = parent.join(format!(".{name}.trail-{nonce}.tmp")); + let result = (|| -> Result<()> { + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(mode); + } + let mut file = options.open(&temp)?; + file.write_all(bytes)?; + file.sync_all()?; + fs::rename(&temp, path)?; + sync_parent(path)?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(&temp); + } + result +} + +struct TargetLock { + path: PathBuf, +} + +impl Drop for TargetLock { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +fn acquire_target_lock(target: &Path) -> Result { + let parent = target.parent().ok_or_else(|| Error::InvalidPath { + path: target.display().to_string(), + reason: "agent hook target has no parent directory".to_string(), + })?; + fs::create_dir_all(parent)?; + let name = target + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| Error::InvalidPath { + path: target.display().to_string(), + reason: "agent hook target filename is not valid UTF-8".to_string(), + })?; + let path = parent.join(format!(".{name}.trail-install.lock")); + for attempt in 0..2 { + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + match options.open(&path) { + Ok(mut file) => { + writeln!( + file, + "pid={} created_nanos={}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or_default() + )?; + file.sync_all()?; + return Ok(TargetLock { path }); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists && attempt == 0 => { + let metadata = fs::symlink_metadata(&path)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(Error::InvalidPath { + path: path.display().to_string(), + reason: "agent hook install lock is not a regular file".to_string(), + }); + } + let stale = metadata + .modified() + .ok() + .and_then(|modified| modified.elapsed().ok()) + .is_some_and(|elapsed| elapsed.as_secs() > 600); + if stale { + fs::remove_file(&path)?; + continue; + } + return Err(Error::WorkspaceLocked(format!( + "agent hook config `{}` is being modified by another installer", + target.display() + ))); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + return Err(Error::WorkspaceLocked(format!( + "agent hook config `{}` is being modified by another installer", + target.display() + ))); + } + Err(error) => return Err(error.into()), + } + } + Err(Error::WorkspaceLocked(format!( + "could not acquire agent hook install lock for `{}`", + target.display() + ))) +} + +fn sync_parent(path: &Path) -> Result<()> { + #[cfg(unix)] + { + let parent = path.parent().ok_or_else(|| Error::InvalidPath { + path: path.display().to_string(), + reason: "agent hook target has no parent directory".to_string(), + })?; + OpenOptions::new().read(true).open(parent)?.sync_all()?; + } + Ok(()) +} + +fn manifest_digest(manifest: &AgentAdapterManifest) -> Result { + Ok(content_digest(&serde_json::to_vec(manifest)?)) +} + +fn content_digest(bytes: &[u8]) -> String { + format!("sha256:{}", hex(&Sha256::digest(bytes))) +} + +fn json_bytes(value: &Value) -> Result> { + let mut bytes = serde_json::to_vec_pretty(value)?; + bytes.push(b'\n'); + Ok(bytes) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request<'a>( + registry: &'a AgentProviderRegistry, + provider: &'a str, + root: &'a Path, + ) -> AgentHookInstallRequest<'a> { + AgentHookInstallRequest { + registry, + provider, + workspace_id: "workspace-test", + workspace_root: root, + home_dir: Some(root), + scope: AgentHookInstallScope::Project, + force: false, + } + } + + #[test] + fn json_merge_preserves_unrelated_fields_and_hooks() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir_all(temp.path().join(".codex")).unwrap(); + fs::write( + temp.path().join(".codex/hooks.json"), + br#"{"$schema":"kept","hooks":{"Stop":[{"hooks":[{"type":"command","command":"user stop"}]}]}}"#, + ) + .unwrap(); + let registry = AgentProviderRegistry::built_in().unwrap(); + let plan = build_agent_hook_install_plan(request(®istry, "codex", temp.path())).unwrap(); + apply_agent_hook_install_plan(&plan, false).unwrap(); + let value: Value = serde_json::from_slice(&fs::read(&plan.config_path).unwrap()).unwrap(); + assert_eq!(value["$schema"], "kept"); + assert_eq!(value["hooks"]["Stop"].as_array().unwrap().len(), 2); + assert_eq!( + inspect_agent_hook_installation(&plan).unwrap(), + AgentHookInstallationStatus::Installed + ); + } + + #[test] + fn install_is_idempotent_and_uninstall_removes_only_owned_entries() { + let temp = tempfile::tempdir().unwrap(); + let registry = AgentProviderRegistry::built_in().unwrap(); + let plan = + build_agent_hook_install_plan(request(®istry, "claude", temp.path())).unwrap(); + apply_agent_hook_install_plan(&plan, false).unwrap(); + let second = + build_agent_hook_install_plan(request(®istry, "claude-code", temp.path())).unwrap(); + assert_eq!(second.action, AgentHookInstallAction::Noop); + remove_agent_hook_installation(&second, &plan.after_digest, false).unwrap(); + let value: Value = serde_json::from_slice(&fs::read(&plan.config_path).unwrap()).unwrap(); + assert!(value.get("hooks").is_none()); + } + + #[test] + fn claude_hook_groups_omit_unset_matchers() { + let temp = tempfile::tempdir().unwrap(); + let registry = AgentProviderRegistry::built_in().unwrap(); + let plan = + build_agent_hook_install_plan(request(®istry, "claude-code", temp.path())).unwrap(); + apply_agent_hook_install_plan(&plan, false).unwrap(); + let value: Value = serde_json::from_slice(&fs::read(&plan.config_path).unwrap()).unwrap(); + for groups in value["hooks"].as_object().unwrap().values() { + for group in groups.as_array().unwrap() { + assert!(group.get("matcher").is_none()); + assert!(group["hooks"].is_array()); + } + } + } + + #[test] + fn provider_specific_json_contracts_have_valid_versions_and_timeout_units() { + let registry = AgentProviderRegistry::built_in().unwrap(); + + let cursor_temp = tempfile::tempdir().unwrap(); + let cursor = + build_agent_hook_install_plan(request(®istry, "cursor", cursor_temp.path())) + .unwrap(); + let cursor_json: Value = serde_json::from_slice(&cursor.desired_bytes).unwrap(); + assert_eq!(cursor_json["version"], 1); + assert!(cursor_json["hooks"]["preToolUse"][0]["timeout"] + .as_u64() + .is_some_and(|timeout| timeout >= 30)); + + let gemini_temp = tempfile::tempdir().unwrap(); + let gemini = + build_agent_hook_install_plan(request(®istry, "gemini", gemini_temp.path())) + .unwrap(); + let gemini_json: Value = serde_json::from_slice(&gemini.desired_bytes).unwrap(); + assert_eq!( + gemini_json["hooks"]["BeforeTool"][0]["hooks"][0]["timeout"], + 30_000 + ); + + let copilot_temp = tempfile::tempdir().unwrap(); + let copilot = + build_agent_hook_install_plan(request(®istry, "copilot", copilot_temp.path())) + .unwrap(); + let copilot_json: Value = serde_json::from_slice(&copilot.desired_bytes).unwrap(); + assert_eq!(copilot_json["version"], 1); + assert!(copilot_json.get("trailOwnership").is_none()); + } + + #[test] + fn kiro_uses_the_versioned_standalone_hook_contract() { + let temp = tempfile::tempdir().unwrap(); + let registry = AgentProviderRegistry::built_in().unwrap(); + let plan = build_agent_hook_install_plan(request(®istry, "kiro", temp.path())).unwrap(); + let value: Value = serde_json::from_slice(&plan.desired_bytes).unwrap(); + assert_eq!(value["version"], "v1"); + assert_eq!(plan.provider_version_range.as_deref(), Some(">=2.8.0")); + assert!(value.get("trailOwnership").is_none()); + let hooks = value["hooks"].as_array().unwrap(); + assert_eq!(hooks.len(), registry.resolve("kiro").unwrap().events.len()); + assert!(hooks.iter().all(|hook| { + hook["name"].as_str().is_some() + && hook["trigger"].as_str().is_some() + && hook["action"]["type"] == "command" + && hook["action"]["command"] + .as_str() + .is_some_and(|command| command.contains("--installation hook_")) + })); + } + + #[test] + fn strict_json_validation_reports_malformed_drift() { + let temp = tempfile::tempdir().unwrap(); + let registry = AgentProviderRegistry::built_in().unwrap(); + let plan = + build_agent_hook_install_plan(request(®istry, "cursor", temp.path())).unwrap(); + apply_agent_hook_install_plan(&plan, false).unwrap(); + fs::write(&plan.config_path, br#"{"version":"one","hooks":{}}"#).unwrap(); + assert_eq!( + inspect_agent_hook_installation(&plan).unwrap(), + AgentHookInstallationStatus::Malformed + ); + } + + #[test] + fn malformed_hook_shape_fails_closed() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir_all(temp.path().join(".gemini")).unwrap(); + fs::write( + temp.path().join(".gemini/settings.json"), + br#"{"hooks":{"SessionStart":"not-an-array"}}"#, + ) + .unwrap(); + let registry = AgentProviderRegistry::built_in().unwrap(); + let error = + build_agent_hook_install_plan(request(®istry, "gemini", temp.path())).unwrap_err(); + assert!(error.to_string().contains("not a JSON array")); + } + + #[test] + fn owned_plugin_refuses_foreign_file_and_drifted_removal() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir_all(temp.path().join(".opencode/plugins")).unwrap(); + let path = temp.path().join(".opencode/plugins/trail.ts"); + fs::write(&path, "foreign").unwrap(); + let registry = AgentProviderRegistry::built_in().unwrap(); + let error = + build_agent_hook_install_plan(request(®istry, "opencode", temp.path())).unwrap_err(); + assert!(error.to_string().contains("foreign")); + + fs::remove_file(&path).unwrap(); + let plan = + build_agent_hook_install_plan(request(®istry, "opencode", temp.path())).unwrap(); + apply_agent_hook_install_plan(&plan, false).unwrap(); + fs::write( + &path, + format!("{}// drift", String::from_utf8_lossy(&plan.desired_bytes)), + ) + .unwrap(); + let error = remove_agent_hook_installation(&plan, &plan.after_digest, false).unwrap_err(); + assert!(error.to_string().contains("modified")); + } + + #[cfg(unix)] + #[test] + fn symlinked_config_component_is_rejected() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + symlink(outside.path(), temp.path().join(".codex")).unwrap(); + let registry = AgentProviderRegistry::built_in().unwrap(); + let error = + build_agent_hook_install_plan(request(®istry, "codex", temp.path())).unwrap_err(); + assert!(error.to_string().contains("symlink")); + } + + #[test] + fn all_nine_providers_render_project_and_user_assets() { + let registry = AgentProviderRegistry::built_in().unwrap(); + for manifest in registry.list() { + for scope in [AgentHookInstallScope::Project, AgentHookInstallScope::User] { + let temp = tempfile::tempdir().unwrap(); + let mut req = request(®istry, &manifest.provider, temp.path()); + req.scope = scope; + let plan = build_agent_hook_install_plan(req).unwrap(); + assert!( + !plan.desired_bytes.is_empty(), + "{} {scope:?}", + manifest.provider + ); + apply_agent_hook_install_plan(&plan, false).unwrap(); + let mut second = request(®istry, &manifest.provider, temp.path()); + second.scope = scope; + let second = build_agent_hook_install_plan(second).unwrap(); + assert_eq!( + second.action, + AgentHookInstallAction::Noop, + "{} {scope:?}", + manifest.provider + ); + } + } + } +} diff --git a/trail/src/agent_hooks/parsing.rs b/trail/src/agent_hooks/parsing.rs new file mode 100644 index 0000000..31d4e97 --- /dev/null +++ b/trail/src/agent_hooks/parsing.rs @@ -0,0 +1,544 @@ +use serde::{Deserialize, Serialize}; + +use super::*; +use crate::{ + AgentCaptureTransport, AgentEventCorrelation, AgentEventEvidence, AgentEvidenceConfidence, + AgentLifecycleEvent, AgentLifecycleEventType, AgentNativeEventIdentity, + AGENT_LIFECYCLE_EVENT_SCHEMA, AGENT_LIFECYCLE_EVENT_VERSION, +}; + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentHookParseContext { + pub receipt_id: String, + pub workspace_id: String, + pub lane_id: Option, + pub capture_run_id: Option, + pub provider_version: Option, + pub raw_digest: String, + pub received_at: i64, + pub transport: AgentCaptureTransport, +} + +pub fn parse_agent_hook_payload( + registry: &AgentProviderRegistry, + provider: &str, + native_event: &str, + payload: &serde_json::Value, + context: AgentHookParseContext, +) -> Result> { + let manifest = registry.resolve(provider)?; + let binding = manifest + .events + .iter() + .find(|binding| binding.native_event == native_event) + .or_else(|| { + manifest + .events + .iter() + .find(|binding| binding.native_event.eq_ignore_ascii_case(native_event)) + }); + let normalized = match binding { + Some(binding) => select_normalized_events(&binding.normalized_events, payload), + None => vec![format!( + "provider.{}.{}", + manifest.provider, + normalize_provider_event_name(native_event) + )], + }; + let native = parse_native_identity(native_event, payload); + let occurred_at = payload_i64(payload, &["timestamp", "occurred_at", "occurredAt"]); + let correlation = AgentEventCorrelation { + parent_event_id: payload_string(payload, &["parent_event_id", "parentEventId"]), + trace_id: payload_string(payload, &["trace_id", "traceId"]), + span_id: payload_string(payload, &["span_id", "spanId"]), + parent_span_id: payload_string(payload, &["parent_span_id", "parentSpanId"]), + }; + + normalized + .into_iter() + .enumerate() + .map(|(index, event_type)| { + let event = AgentLifecycleEvent { + schema: AGENT_LIFECYCLE_EVENT_SCHEMA.to_string(), + version: AGENT_LIFECYCLE_EVENT_VERSION, + event_id: format!("{}_{}", context.receipt_id, index), + event_type: AgentLifecycleEventType::new(event_type), + occurred_at, + received_at: context.received_at, + provider: manifest.provider.clone(), + provider_version: context.provider_version.clone(), + transport: context.transport, + workspace_id: context.workspace_id.clone(), + lane_id: context.lane_id.clone(), + capture_run_id: context.capture_run_id.clone(), + native: native.clone(), + correlation: correlation.clone(), + payload: payload.clone(), + evidence: AgentEventEvidence { + receipt_id: context.receipt_id.clone(), + raw_digest: Some(context.raw_digest.clone()), + transcript_offset: payload_u64( + payload, + &["transcript_offset", "transcriptOffset"], + ), + confidence: AgentEvidenceConfidence::NativeStructured, + }, + }; + event.validate()?; + Ok(event) + }) + .collect() +} + +fn parse_native_identity( + native_event: &str, + payload: &serde_json::Value, +) -> AgentNativeEventIdentity { + AgentNativeEventIdentity { + session_id: payload_string( + payload, + &[ + "session_id", + "sessionId", + "sessionID", + "conversation_id", + "conversationId", + "thread_id", + "threadId", + ], + ), + turn_id: payload_string(payload, &["turn_id", "turnId"]), + message_id: payload_string(payload, &["message_id", "messageId", "part_id", "partId"]), + tool_id: payload_string( + payload, + &[ + "tool_use_id", + "toolUseId", + "tool_call_id", + "toolCallId", + "tool_id", + "toolId", + "call_id", + "callId", + ], + ), + subagent_id: payload_string( + payload, + &["agent_id", "agentId", "subagent_id", "subagentId"], + ), + event_name: native_event.to_string(), + sequence: payload_u64( + payload, + &["sequence", "seq", "event_sequence", "eventSequence"], + ), + } +} + +fn select_normalized_events(candidates: &[String], payload: &serde_json::Value) -> Vec { + if candidates.len() <= 1 { + return candidates.to_vec(); + } + + if contains_any( + candidates, + &["turn.completed", "turn.failed", "turn.cancelled"], + ) { + let outcome = payload_outcome(payload); + let selected = match outcome { + ParsedOutcome::Completed => "turn.completed", + ParsedOutcome::Failed => "turn.failed", + ParsedOutcome::Cancelled => "turn.cancelled", + }; + return selected_with_supplemental( + candidates, + &["turn.completed", "turn.failed", "turn.cancelled"], + selected, + ); + } + if contains_any(candidates, &["tool.completed", "tool.failed"]) { + return vec![if payload_failed(payload) { + "tool.failed" + } else { + "tool.completed" + } + .to_string()]; + } + if contains_any(candidates, &["subagent.completed", "subagent.failed"]) { + return vec![if payload_failed(payload) { + "subagent.failed" + } else { + "subagent.completed" + } + .to_string()]; + } + if contains_any(candidates, &["session.started", "session.resumed"]) { + let source = payload_string(payload, &["source", "reason"]) + .unwrap_or_default() + .to_ascii_lowercase(); + return vec![ + if matches!(source.as_str(), "resume" | "resumed" | "continue") { + "session.resumed" + } else { + "session.started" + } + .to_string(), + ]; + } + candidates.to_vec() +} + +fn selected_with_supplemental( + candidates: &[String], + alternatives: &[&str], + selected: &str, +) -> Vec { + let mut selected_emitted = false; + let mut normalized = Vec::new(); + for candidate in candidates { + if alternatives.contains(&candidate.as_str()) { + if !selected_emitted { + normalized.push(selected.to_string()); + selected_emitted = true; + } + } else { + normalized.push(candidate.clone()); + } + } + normalized +} + +fn contains_any(candidates: &[String], expected: &[&str]) -> bool { + expected + .iter() + .all(|expected| candidates.iter().any(|candidate| candidate == expected)) +} + +#[derive(Clone, Copy)] +enum ParsedOutcome { + Completed, + Failed, + Cancelled, +} + +fn payload_outcome(payload: &serde_json::Value) -> ParsedOutcome { + let value = payload_string( + payload, + &["outcome", "status", "reason", "stop_reason", "stopReason"], + ) + .unwrap_or_default() + .to_ascii_lowercase(); + if value.contains("cancel") || value.contains("abort") || value.contains("interrupt") { + ParsedOutcome::Cancelled + } else if payload_failed(payload) { + ParsedOutcome::Failed + } else { + ParsedOutcome::Completed + } +} + +fn payload_failed(payload: &serde_json::Value) -> bool { + if payload + .get("error") + .is_some_and(|value| !value.is_null() && value != false) + { + return true; + } + if ["is_error", "isError", "failed"] + .iter() + .any(|key| payload_value(payload, key).and_then(serde_json::Value::as_bool) == Some(true)) + { + return true; + } + if payload_value(payload, "success").and_then(serde_json::Value::as_bool) == Some(false) { + return true; + } + payload_string( + payload, + &["outcome", "status", "reason", "stop_reason", "stopReason"], + ) + .is_some_and(|value| { + let value = value.to_ascii_lowercase(); + value.contains("fail") || value.contains("error") || value.contains("timeout") + }) +} + +fn payload_string(payload: &serde_json::Value, keys: &[&str]) -> Option { + keys.iter().find_map(|key| { + payload_value(payload, key) + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) + .map(str::to_string) + }) +} + +fn payload_i64(payload: &serde_json::Value, keys: &[&str]) -> Option { + keys.iter().find_map(|key| { + payload_value(payload, key).and_then(|value| { + value + .as_i64() + .or_else(|| value.as_u64().and_then(|value| i64::try_from(value).ok())) + }) + }) +} + +fn payload_u64(payload: &serde_json::Value, keys: &[&str]) -> Option { + keys.iter().find_map(|key| { + payload_value(payload, key).and_then(|value| { + value + .as_u64() + .or_else(|| value.as_i64().and_then(|value| u64::try_from(value).ok())) + }) + }) +} + +fn payload_value<'a>(payload: &'a serde_json::Value, key: &str) -> Option<&'a serde_json::Value> { + payload.get(key).or_else(|| { + [ + "properties", + "input", + "event", + "data", + "session", + "output", + "result", + "tool_response", + "toolResponse", + "tool_result", + "toolResult", + ] + .iter() + .find_map(|container| payload.get(*container).and_then(|value| value.get(key))) + }) +} + +fn normalize_provider_event_name(value: &str) -> String { + let normalized = value + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() { + character.to_ascii_lowercase() + } else { + '_' + } + }) + .collect::(); + let normalized = normalized.trim_matches('_'); + if normalized.is_empty() { + "unknown".to_string() + } else { + normalized.chars().take(80).collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::AgentLifecycleEventKind; + + fn context() -> AgentHookParseContext { + AgentHookParseContext { + receipt_id: "receipt_test".to_string(), + workspace_id: "workspace_test".to_string(), + lane_id: Some("lane_test".to_string()), + capture_run_id: None, + provider_version: Some("test".to_string()), + raw_digest: "sha256:test".to_string(), + received_at: 10, + transport: AgentCaptureTransport::NativeHooks, + } + } + + #[test] + fn codex_prompt_yields_turn_and_message_with_native_ids() { + let registry = AgentProviderRegistry::built_in().unwrap(); + let events = parse_agent_hook_payload( + ®istry, + "codex", + "UserPromptSubmit", + &serde_json::json!({ + "session_id": "session-1", + "turn_id": "turn-1", + "prompt": "fix it", + "timestamp": 4 + }), + context(), + ) + .unwrap(); + assert_eq!(events.len(), 2); + assert_eq!( + events[0].event_type.kind(), + AgentLifecycleEventKind::TurnStarted + ); + assert_eq!( + events[1].event_type.kind(), + AgentLifecycleEventKind::MessageUser + ); + assert_eq!(events[0].native.session_id.as_deref(), Some("session-1")); + assert_eq!(events[0].native.turn_id.as_deref(), Some("turn-1")); + } + + #[test] + fn terminal_and_tool_bindings_select_one_observed_outcome() { + let registry = AgentProviderRegistry::built_in().unwrap(); + let turn = parse_agent_hook_payload( + ®istry, + "gemini", + "AfterAgent", + &serde_json::json!({"sessionId": "s", "status": "cancelled"}), + context(), + ) + .unwrap(); + assert_eq!(turn.len(), 1); + assert_eq!( + turn[0].event_type.kind(), + AgentLifecycleEventKind::TurnCancelled + ); + + let tool = parse_agent_hook_payload( + ®istry, + "codex", + "PostToolUse", + &serde_json::json!({"session_id": "s", "error": "failed"}), + context(), + ) + .unwrap(); + assert_eq!(tool.len(), 1); + assert_eq!( + tool[0].event_type.kind(), + AgentLifecycleEventKind::ToolFailed + ); + } + + #[test] + fn kiro_stop_preserves_the_assistant_message_alongside_the_turn_outcome() { + let registry = AgentProviderRegistry::built_in().unwrap(); + let events = parse_agent_hook_payload( + ®istry, + "kiro", + "Stop", + &serde_json::json!({ + "session_id": "kiro-session", + "assistant_response": "Implemented and tested." + }), + context(), + ) + .unwrap(); + assert_eq!(events.len(), 2); + assert_eq!( + events[0].event_type.kind(), + AgentLifecycleEventKind::MessageAssistantCompleted + ); + assert_eq!( + events[1].event_type.kind(), + AgentLifecycleEventKind::TurnCompleted + ); + } + + #[test] + fn unknown_native_events_are_retained_but_inert() { + let registry = AgentProviderRegistry::built_in().unwrap(); + let events = parse_agent_hook_payload( + ®istry, + "grok", + "Future Event/V2", + &serde_json::json!({"sessionId": "s"}), + context(), + ) + .unwrap(); + assert_eq!(events.len(), 1); + assert_eq!( + events[0].event_type.kind(), + AgentLifecycleEventKind::Unknown + ); + assert_eq!( + events[0].event_type.as_str(), + "provider.grok.future_event_v2" + ); + } + + #[test] + fn copilot_pascal_case_contract_resolves_case_insensitively() { + let registry = AgentProviderRegistry::built_in().unwrap(); + let events = parse_agent_hook_payload( + ®istry, + "copilot-cli", + "SessionStart", + &serde_json::json!({"session_id": "s", "source": "resume"}), + context(), + ) + .unwrap(); + assert_eq!( + events[0].event_type.kind(), + AgentLifecycleEventKind::SessionResumed + ); + } + + #[test] + fn checked_in_provider_contract_fixtures_normalize_all_nine_adapters() { + let registry = AgentProviderRegistry::built_in().unwrap(); + let fixtures = [ + ( + "codex", + include_str!("../../tests/fixtures/agent-hooks/codex/contracts.json"), + ), + ( + "claude-code", + include_str!("../../tests/fixtures/agent-hooks/claude-code/contracts.json"), + ), + ( + "pi", + include_str!("../../tests/fixtures/agent-hooks/pi/contracts.json"), + ), + ( + "opencode", + include_str!("../../tests/fixtures/agent-hooks/opencode/contracts.json"), + ), + ( + "cursor", + include_str!("../../tests/fixtures/agent-hooks/cursor/contracts.json"), + ), + ( + "gemini", + include_str!("../../tests/fixtures/agent-hooks/gemini/contracts.json"), + ), + ( + "copilot", + include_str!("../../tests/fixtures/agent-hooks/copilot/contracts.json"), + ), + ( + "grok", + include_str!("../../tests/fixtures/agent-hooks/grok/contracts.json"), + ), + ( + "kiro", + include_str!("../../tests/fixtures/agent-hooks/kiro/contracts.json"), + ), + ]; + for (provider, bytes) in fixtures { + let cases: serde_json::Value = serde_json::from_str(bytes).unwrap(); + let cases = cases.as_array().unwrap(); + assert_eq!(cases.len(), 3, "{provider} fixture coverage"); + for case in cases { + let event = case["event"].as_str().unwrap(); + let normalized = parse_agent_hook_payload( + ®istry, + provider, + event, + &case["payload"], + context(), + ) + .unwrap() + .into_iter() + .map(|event| event.event_type.as_str().to_string()) + .collect::>(); + let expected = case["expected"] + .as_array() + .unwrap() + .iter() + .map(|value| value.as_str().unwrap().to_string()) + .collect::>(); + assert_eq!(normalized, expected, "{provider} {event}"); + } + } + } +} diff --git a/trail/src/cli/command.rs b/trail/src/cli/command.rs index 80bdc7b..80fb321 100644 --- a/trail/src/cli/command.rs +++ b/trail/src/cli/command.rs @@ -47,8 +47,10 @@ struct Cli { verbose: bool, #[arg(long, global = true)] trace: bool, - #[arg(long, global = true)] - no_color: bool, + #[arg(long, global = true, value_enum, default_value_t = ColorArg::Auto)] + color: ColorArg, + #[arg(long, global = true, value_enum, default_value_t = PagerArg::Auto)] + pager: PagerArg, #[arg(long, global = true)] format: Option, #[arg(long, global = true, value_name = "URL")] @@ -59,13 +61,48 @@ struct Cli { command: Command, } -#[derive(Clone, Debug, clap::ValueEnum)] -enum OutputFormat { +#[derive(Clone, Debug, clap::ValueEnum, PartialEq, Eq)] +pub(crate) enum OutputFormat { Human, + Plain, Json, Ndjson, } +#[derive(Clone, Debug, clap::ValueEnum)] +enum ColorArg { + Auto, + Always, + Never, +} + +impl ColorArg { + fn as_policy(&self) -> render::ColorPolicy { + match self { + Self::Auto => render::ColorPolicy::Auto, + Self::Always => render::ColorPolicy::Always, + Self::Never => render::ColorPolicy::Never, + } + } +} + +#[derive(Clone, Debug, clap::ValueEnum)] +enum PagerArg { + Auto, + Always, + Never, +} + +impl PagerArg { + fn as_policy(&self) -> render::PagerPolicy { + match self { + Self::Auto => render::PagerPolicy::Auto, + Self::Always => render::PagerPolicy::Always, + Self::Never => render::PagerPolicy::Never, + } + } +} + #[derive(Subcommand)] enum Command { /// Initialize a new Trail workspace and default branch state. @@ -125,6 +162,8 @@ enum Command { Lane(LaneCommand), /// Build, attach, and inspect reproducible dependency environments. Deps(DepsCommand), + /// Inspect and synchronize adapter-owned workspace environments. + Env(EnvironmentCommand), /// Inspect and verify immutable local workspace cache layers. Cache(CacheCommand), /// Run Agent Client Protocol relay integrations for coding agents. @@ -139,12 +178,6 @@ enum Command { Session(SessionCommand), /// Handle sensitive action approvals and reviewer decisions. Approvals(ApprovalsCommand), - /// Merge a lane branch into a standard branch with readiness checks. - /// Applies the same conflict rules as queue/manual merges. - MergeLane(MergeLaneArgs), - /// Schedule and run controlled serialized merges using a queue. - /// Merge items pause on first conflict and keep audit history. - MergeQueue(MergeQueueCommand), /// Inspect and resolve conflict sets opened by merge operations. Conflicts(ConflictsCommand), /// Create and resolve stable anchors that survive nearby line churn. @@ -171,3 +204,352 @@ enum Command { /// Prune unused objects and stale index references. Gc(GcArgs), } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_environment_adapter_catalog() { + let cli = Cli::try_parse_from(["trail", "env", "adapters"]) + .expect("environment adapter catalog command should parse"); + let Command::Env(EnvironmentCommand { + command: EnvironmentSubcommand::Adapters, + }) = cli.command + else { + panic!("expected environment adapters command"); + }; + } + + #[test] + fn parses_environment_status() { + let cli = Cli::try_parse_from(["trail", "env", "status", "lane-a"]) + .expect("environment status command should parse"); + + let Command::Env(EnvironmentCommand { + command: EnvironmentSubcommand::Status(args), + }) = cli.command + else { + panic!("expected environment status command"); + }; + assert_eq!(args.lane, "lane-a"); + } + + #[test] + fn parses_environment_discovery_with_component_root() { + let cli = Cli::try_parse_from(["trail", "env", "discover", "lane-a", "--path", "apps/web"]) + .expect("environment discovery command should parse"); + let Command::Env(EnvironmentCommand { + command: EnvironmentSubcommand::Discover(args), + }) = cli.command + else { + panic!("expected environment discovery command"); + }; + assert_eq!(args.lane, "lane-a"); + assert_eq!(args.path.as_deref(), Some("apps/web")); + } + + #[test] + fn parses_environment_graph_with_component_root() { + let cli = Cli::try_parse_from([ + "trail", "env", "graph", "lane-a", "--path", "apps", "--offset", "10", "--limit", "20", + ]) + .expect("environment graph command should parse"); + let Command::Env(EnvironmentCommand { + command: EnvironmentSubcommand::Graph(args), + }) = cli.command + else { + panic!("expected environment graph command"); + }; + assert_eq!(args.lane, "lane-a"); + assert_eq!(args.path.as_deref(), Some("apps")); + assert_eq!(args.offset, 10); + assert_eq!(args.limit, 20); + } + + #[test] + fn parses_environment_generation() { + let cli = Cli::try_parse_from(["trail", "env", "generation", "lane-a"]) + .expect("environment generation command should parse"); + let Command::Env(EnvironmentCommand { + command: EnvironmentSubcommand::Generation(args), + }) = cli.command + else { + panic!("expected environment generation command"); + }; + assert_eq!(args.lane, "lane-a"); + } + + #[test] + fn parses_environment_plan_with_adapter_and_root() { + let cli = Cli::try_parse_from([ + "trail", + "env", + "plan", + "lane-a", + "--adapter", + "command", + "--path", + "tools/schema", + "--component", + "protobuf.generated", + ]) + .expect("environment plan command should parse"); + let Command::Env(EnvironmentCommand { + command: EnvironmentSubcommand::Plan(args), + }) = cli.command + else { + panic!("expected environment plan command"); + }; + assert_eq!(args.lane, "lane-a"); + assert_eq!(args.adapter, "command"); + assert_eq!(args.path.as_deref(), Some("tools/schema")); + assert_eq!(args.component.as_deref(), Some("protobuf.generated")); + } + + #[test] + fn environment_sync_defaults_to_auto_detection_and_accepts_a_component_root() { + let cli = Cli::try_parse_from(["trail", "env", "sync", "lane-a", "--path", "apps/web"]) + .expect("environment sync command should parse"); + + let Command::Env(EnvironmentCommand { + command: EnvironmentSubcommand::Sync(args), + }) = cli.command + else { + panic!("expected environment sync command"); + }; + assert_eq!(args.lane, "lane-a"); + assert_eq!(args.adapter, "auto"); + assert_eq!(args.path.as_deref(), Some("apps/web")); + assert_eq!(args.component, None); + } + + #[test] + fn environment_sync_accepts_an_explicit_adapter() { + let cli = Cli::try_parse_from([ + "trail", + "env", + "sync", + "lane-a", + "--adapter", + "example/python@1", + ]) + .expect("environment sync command should parse"); + + let Command::Env(EnvironmentCommand { + command: EnvironmentSubcommand::Sync(args), + }) = cli.command + else { + panic!("expected environment sync command"); + }; + assert_eq!(args.adapter, "example/python@1"); + assert_eq!(args.path, None); + assert_eq!(args.component, None); + } + + #[test] + fn environment_sync_all_accepts_a_discovery_root() { + let cli = Cli::try_parse_from(["trail", "env", "sync-all", "lane-a", "--path", "apps"]) + .expect("environment sync-all command should parse"); + let Command::Env(EnvironmentCommand { + command: EnvironmentSubcommand::SyncAll(args), + }) = cli.command + else { + panic!("expected environment sync-all command"); + }; + assert_eq!(args.lane, "lane-a"); + assert_eq!(args.path.as_deref(), Some("apps")); + } + + #[test] + fn parses_environment_runtime_lifecycle_commands() { + for (action, expected) in [ + ("status", "status"), + ("reconcile", "reconcile"), + ("stop", "stop"), + ] { + let cli = Cli::try_parse_from(["trail", "env", "runtime", action, "lane-a"]) + .expect("environment runtime command should parse"); + let Command::Env(EnvironmentCommand { + command: EnvironmentSubcommand::Runtime(runtime), + }) = cli.command + else { + panic!("expected environment runtime command"); + }; + let (actual, lane) = match runtime.command { + EnvironmentRuntimeSubcommand::Status(args) => ("status", args.lane), + EnvironmentRuntimeSubcommand::Reconcile(args) => ("reconcile", args.lane), + EnvironmentRuntimeSubcommand::Stop(args) => ("stop", args.lane), + }; + assert_eq!(actual, expected); + assert_eq!(lane, "lane-a"); + } + } + + #[test] + fn parses_terminal_rendering_options_and_diff_projections() { + let cli = Cli::try_parse_from([ + "trail", + "--format", + "plain", + "--color", + "never", + "--pager", + "always", + "diff", + "--dirty", + "--name-status", + ]) + .expect("terminal rendering options should parse"); + assert!(matches!(cli.format, Some(OutputFormat::Plain))); + assert!(matches!(cli.color, ColorArg::Never)); + assert!(matches!(cli.pager, PagerArg::Always)); + let Command::Diff(args) = cli.command else { + panic!("expected diff command"); + }; + assert!(args.name_status); + assert!(!args.name_only); + } + + #[test] + fn top_level_merge_lane_is_removed_from_the_cli() { + assert!(Cli::try_parse_from(["trail", "merge-lane", "fix-login"]).is_err()); + } + + #[test] + fn agent_cli_parses_new_positional_provider_commands() { + for command in [ + vec!["trail", "agent", "start", "codex"], + vec!["trail", "agent", "acp", "setup", "codex", "--editor", "zed"], + vec!["trail", "agent", "acp", "run", "codex"], + vec!["trail", "agent", "acp", "doctor", "codex"], + vec!["trail", "agent", "hooks", "setup", "codex"], + vec!["trail", "agent", "hooks", "status", "codex"], + ] { + Cli::try_parse_from(command.clone()).unwrap_or_else(|error| { + panic!("new agent command {command:?} should parse: {error}") + }); + } + } + + #[test] + fn agent_cli_accepts_named_provider_and_rejects_both_forms() { + Cli::try_parse_from([ + "trail", + "agent", + "acp", + "setup", + "--provider", + "codex", + "--editor", + "zed", + ]) + .expect("named ACP provider should parse"); + Cli::try_parse_from(["trail", "agent", "hooks", "setup", "--provider", "codex"]) + .expect("named hook provider should parse"); + assert!(Cli::try_parse_from([ + "trail", + "agent", + "start", + "codex", + "--provider", + "claude-code", + ]) + .is_err()); + } + + #[test] + fn agent_cli_rejects_removed_setup_commands() { + assert!(Cli::try_parse_from(["trail", "agent", "setup"]).is_err()); + assert!(Cli::try_parse_from(["trail", "acp", "install"]).is_err()); + assert!(Cli::try_parse_from(["trail", "agent", "hooks", "add", "codex"]).is_err()); + } + + #[test] + fn workdir_mode_cli_uses_the_hard_cutover_names() { + for mode in [ + "auto", + "native-cow", + "portable-copy", + "fuse-cow", + "dokan-cow", + ] { + Cli::try_parse_from([ + "trail", + "lane", + "spawn", + "mode-check", + "--workdir-mode", + mode, + ]) + .unwrap_or_else(|error| panic!("current workdir mode `{mode}` should parse: {error}")); + } + + for removed in ["full-cow", "full_cow", "overlay-cow", "overlay_cow"] { + assert!( + Cli::try_parse_from([ + "trail", + "lane", + "spawn", + "mode-check", + "--workdir-mode", + removed, + ]) + .is_err(), + "removed workdir mode `{removed}` must be rejected" + ); + } + } + + #[test] + fn parses_lane_merge_with_lane_specific_options() { + let cli = Cli::try_parse_from([ + "trail", + "lane", + "merge", + "fix-login", + "--into", + "main", + "--dry-run", + ]) + .expect("lane merge command should parse"); + let Command::Lane(LaneCommand { + command: LaneSubcommand::Merge(args), + }) = cli.command + else { + panic!("expected lane merge command"); + }; + assert_eq!(args.name, "fix-login"); + assert_eq!(args.into, "main"); + assert!(args.dry_run); + } + + #[test] + fn parses_lane_merge_queue_and_rejects_top_level_form() { + let cli = Cli::try_parse_from([ + "trail", + "lane", + "merge-queue", + "add", + "doc-bot", + "--into", + "main", + ]) + .expect("lane merge queue should parse"); + let Command::Lane(LaneCommand { + command: LaneSubcommand::MergeQueue(queue), + }) = cli.command + else { + panic!("expected lane merge queue command"); + }; + let LaneMergeQueueSubcommand::Add(args) = queue.command else { + panic!("expected lane merge queue add command"); + }; + assert_eq!(args.lane, "doc-bot"); + assert_eq!(args.into, "main"); + assert!( + Cli::try_parse_from(["trail", "merge-queue", "add", "doc-bot", "--into", "main",]) + .is_err() + ); + } +} diff --git a/trail/src/cli/command/acp_args.rs b/trail/src/cli/command/acp_args.rs index 1140a94..393a4a6 100644 --- a/trail/src/cli/command/acp_args.rs +++ b/trail/src/cli/command/acp_args.rs @@ -4,14 +4,6 @@ use clap::{Args, Subcommand}; #[derive(Subcommand)] pub(super) enum AcpSubcommand { - /// Print guided setup for an ACP provider without mutating editor config. - Install(AcpInstallArgs), - /// Run ACP integration diagnostics. - Doctor(AcpDoctorArgs), - /// List supported ACP provider profiles. - List, - /// List captured ACP sessions. - Sessions(AcpSessionsArgs), /// Run a local ACP stdio relay in front of a real ACP coding agent. Relay(AcpRelayArgs), } @@ -22,47 +14,11 @@ pub(super) struct AcpCommand { pub(super) command: AcpSubcommand, } -#[derive(Args)] -pub(super) struct AcpInstallArgs { - #[arg( - long, - default_value = "claude-code", - value_name = "AGENT", - help = "Built-in alias or ACP registry agent ID (run `trail acp list`)" - )] - pub(super) agent: String, - #[arg(long, default_value = "generic")] - pub(super) editor: String, - #[arg(long)] - pub(super) dry_run: bool, - #[arg(long = "print")] - pub(super) print_only: bool, -} - -#[derive(Args)] -pub(super) struct AcpDoctorArgs { - #[arg( - long, - default_value = "claude-code", - value_name = "AGENT", - help = "Built-in alias or ACP registry agent ID (run `trail acp list`)" - )] - pub(super) agent: String, - #[arg(long = "relay-command", num_args = 1.., allow_hyphen_values = true)] - pub(super) relay_command: Vec, -} - -#[derive(Args)] -pub(super) struct AcpSessionsArgs { - #[arg(long)] - pub(super) lane: Option, -} - #[derive(Args)] pub(super) struct AcpRelayArgs { #[arg( value_name = "AGENT", - help = "Built-in ACP alias or registry agent ID (run `trail acp list`)" + help = "Built-in ACP alias or registry agent ID (run `trail agent acp status`)" )] pub(super) agent: Option, #[arg(long)] diff --git a/trail/src/cli/command/agent_args.rs b/trail/src/cli/command/agent_args.rs index a998efa..ede0c93 100644 --- a/trail/src/cli/command/agent_args.rs +++ b/trail/src/cli/command/agent_args.rs @@ -1,12 +1,28 @@ -use clap::{Args, Subcommand}; +use clap::{Args, Subcommand, ValueEnum}; #[derive(Subcommand)] pub(super) enum AgentSubcommand { - /// Print editor setup for the high-level agent task workflow. - Setup(AgentSetupArgs), - /// Run a stable ACP entrypoint that creates a fresh Trail lane per task. + /// Internal native-agent hook ingress. Use `agent hooks` to manage integrations. #[command(hide = true)] - Acp(AgentAcpArgs), + Hook(AgentHookCommand), + /// Install, inspect, diagnose, and remove native agent lifecycle hooks. + Hooks(AgentHooksCommand), + /// Declare and manage an explicit ACP/native/terminal capture ownership run. + Capture(AgentCaptureCommand), + /// List immutable native transcripts, exports, and other captured artifacts. + Artifacts(AgentArtifactsArgs), + /// Inspect the deterministic causal provenance graph for a session. + Provenance(AgentProvenanceArgs), + /// Create and verify exact session evidence attestations. + Attest(AgentAttestCommand), + /// Review captured, evidence-anchored learnings. + Learnings(AgentLearningsCommand), + /// Export a portable, canonical agent trace. + Export(AgentTraceExportArgs), + /// Link an exact Git commit to enumerated Trail session/change evidence. + GitLink(AgentGitLinkCommand), + /// Configure, diagnose, and run Agent Client Protocol integrations. + Acp(AgentAcpCommand), /// Create a fresh task lane and launch a terminal agent. Start(AgentStartArgs), /// Continue from an existing agent task checkpoint in a fresh task lane. @@ -194,6 +210,350 @@ pub(super) enum AgentSubcommand { Doctor(AgentDoctorArgs), } +#[derive(Args)] +pub(super) struct AgentHookCommand { + #[command(subcommand)] + pub(super) command: AgentHookSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum AgentHookSubcommand { + /// Durably journal one provider hook payload from stdin. + Receive(AgentHookReceiveArgs), +} + +#[derive(Args)] +pub(super) struct AgentHookReceiveArgs { + pub(super) provider: String, + pub(super) native_event: String, + #[arg(long)] + pub(super) installation: Option, + #[arg(long, hide = true)] + pub(super) dedupe_key: Option, +} + +#[derive(Args)] +pub(super) struct AgentHooksCommand { + #[command(subcommand)] + pub(super) command: AgentHooksSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum AgentHooksSubcommand { + /// Install or update Trail-owned hooks for a provider. + Setup(AgentHooksSetupArgs), + /// Remove only the entries or file owned by a Trail installation. + Remove(AgentHooksRemoveArgs), + /// List supported providers and their recorded installations. + List(AgentHooksListArgs), + /// Inspect one provider's configured installation and drift state. + Status(AgentHooksProviderArgs), + /// Diagnose provider support, configuration, and recent delivery. + Doctor(AgentHooksDoctorArgs), + /// List durable native receipts for a provider. + Events(AgentHooksEventsArgs), + /// Replay one receipt or the pending durable receipt queue. + Replay(AgentHooksReplayArgs), + /// Move a retry or quarantined receipt back to the replay queue. + Retry(AgentHookReceiptArgs), + /// Explicitly discard a received, retrying, or quarantined receipt. + Discard(AgentHookReceiptArgs), +} + +#[derive(Clone, Copy, Debug, ValueEnum)] +pub(super) enum AgentHooksScopeArg { + Project, + User, +} + +#[derive(Args)] +pub(super) struct AgentHooksSetupArgs { + #[arg(value_name = "PROVIDER", conflicts_with = "provider_flag")] + pub(super) provider: Option, + #[arg( + long = "provider", + value_name = "PROVIDER", + conflicts_with = "provider" + )] + pub(super) provider_flag: Option, + #[arg(long, value_enum, default_value = "project")] + pub(super) scope: AgentHooksScopeArg, + #[arg(long)] + pub(super) lane: Option, + #[arg(long = "print")] + pub(super) print_only: bool, + #[arg(long)] + pub(super) yes: bool, + #[arg(long)] + pub(super) force: bool, +} + +#[derive(Args)] +pub(super) struct AgentHooksRemoveArgs { + #[arg(value_name = "PROVIDER", conflicts_with = "provider_flag")] + pub(super) provider: Option, + #[arg( + long = "provider", + value_name = "PROVIDER", + conflicts_with = "provider" + )] + pub(super) provider_flag: Option, + #[arg(long, value_enum, default_value = "project")] + pub(super) scope: AgentHooksScopeArg, + #[arg(long)] + pub(super) dry_run: bool, +} + +#[derive(Args)] +pub(super) struct AgentHooksListArgs { + #[arg(long)] + pub(super) installed: bool, +} + +#[derive(Args)] +pub(super) struct AgentHooksProviderArgs { + #[arg(value_name = "PROVIDER", conflicts_with = "provider_flag")] + pub(super) provider: Option, + #[arg( + long = "provider", + value_name = "PROVIDER", + conflicts_with = "provider" + )] + pub(super) provider_flag: Option, +} + +#[derive(Args)] +pub(super) struct AgentHooksDoctorArgs { + #[arg(value_name = "PROVIDER", conflicts_with_all = ["provider_flag", "all"])] + pub(super) provider: Option, + #[arg(long = "provider", value_name = "PROVIDER", conflicts_with_all = ["provider", "all"])] + pub(super) provider_flag: Option, + #[arg(long, conflicts_with_all = ["provider", "provider_flag"])] + pub(super) all: bool, + #[arg(long)] + pub(super) probe: bool, +} + +#[derive(Args)] +pub(super) struct AgentHooksEventsArgs { + #[arg(value_name = "PROVIDER", conflicts_with = "provider_flag")] + pub(super) provider: Option, + #[arg( + long = "provider", + value_name = "PROVIDER", + conflicts_with = "provider" + )] + pub(super) provider_flag: Option, + #[arg(long, default_value_t = 20)] + pub(super) last: usize, + #[arg(long, default_value_t = 0)] + pub(super) offset: usize, + #[arg(long)] + pub(super) failed: bool, +} + +#[derive(Args)] +pub(super) struct AgentHooksReplayArgs { + #[arg(long, conflicts_with = "pending")] + pub(super) receipt: Option, + #[arg(long)] + pub(super) pending: bool, + #[arg(long, default_value_t = 100)] + pub(super) limit: usize, +} + +#[derive(Args)] +pub(super) struct AgentHookReceiptArgs { + pub(super) receipt: String, +} + +#[derive(Args)] +pub(super) struct AgentCaptureCommand { + #[command(subcommand)] + pub(super) command: AgentCaptureSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum AgentCaptureSubcommand { + Begin(AgentCaptureBeginArgs), + Renew(AgentCaptureRenewArgs), + End(AgentCaptureEndArgs), + Status(AgentCaptureStatusArgs), + Reconcile, +} + +#[derive(Args)] +pub(super) struct AgentCaptureBeginArgs { + #[arg(long)] + pub(super) owner: String, + #[arg(long)] + pub(super) session: String, + #[arg(long)] + pub(super) executor: Option, + #[arg(long)] + pub(super) lane: Option, + #[arg(long)] + pub(super) workdir: Option, + #[arg(long)] + pub(super) work_item: Option, + #[arg(long, default_value_t = 300_000)] + pub(super) ttl_ms: u64, +} + +#[derive(Args)] +pub(super) struct AgentCaptureRenewArgs { + pub(super) run_id: String, + #[arg(long)] + pub(super) owner: String, + #[arg(long)] + pub(super) session: String, + #[arg(long, default_value_t = 300_000)] + pub(super) ttl_ms: u64, +} + +#[derive(Args)] +pub(super) struct AgentCaptureEndArgs { + pub(super) run_id: String, + #[arg(long)] + pub(super) owner: String, + #[arg(long)] + pub(super) session: String, +} + +#[derive(Args)] +pub(super) struct AgentCaptureStatusArgs { + #[arg(long)] + pub(super) all: bool, + #[arg(long, default_value_t = 100)] + pub(super) limit: usize, + #[arg(long, default_value_t = 0)] + pub(super) offset: usize, +} + +#[derive(Args)] +pub(super) struct AgentArtifactsArgs { + pub(super) session: String, + #[arg(long)] + pub(super) turn: Option, + #[arg(long, default_value_t = 100)] + pub(super) limit: usize, + #[arg(long, default_value_t = 0)] + pub(super) offset: usize, +} + +#[derive(Args)] +pub(super) struct AgentProvenanceArgs { + pub(super) session: String, + #[arg(long, default_value_t = 1_000)] + pub(super) limit: usize, + #[arg(long, default_value_t = 0)] + pub(super) offset: usize, +} + +#[derive(Args)] +pub(super) struct AgentAttestCommand { + #[command(subcommand)] + pub(super) command: AgentAttestSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum AgentAttestSubcommand { + Create(AgentAttestCreateArgs), + List(AgentSessionIdArgs), + Show(AgentAttestationIdArgs), + Verify(AgentAttestationIdArgs), +} + +#[derive(Args)] +pub(super) struct AgentAttestCreateArgs { + pub(super) session: String, + #[arg(long, default_value = "manual")] + pub(super) policy: String, +} + +#[derive(Args)] +pub(super) struct AgentSessionIdArgs { + pub(super) session: String, + #[arg(long, default_value_t = 100)] + pub(super) limit: usize, + #[arg(long, default_value_t = 0)] + pub(super) offset: usize, +} + +#[derive(Args)] +pub(super) struct AgentAttestationIdArgs { + pub(super) attestation_id: String, +} + +#[derive(Args)] +pub(super) struct AgentLearningsCommand { + #[command(subcommand)] + pub(super) command: AgentLearningsSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum AgentLearningsSubcommand { + List(AgentLearningsListArgs), + Accept(AgentLearningReviewArgs), + Reject(AgentLearningReviewArgs), +} + +#[derive(Args)] +pub(super) struct AgentLearningsListArgs { + #[arg(long)] + pub(super) session: Option, + #[arg(long)] + pub(super) status: Option, + #[arg(long, default_value_t = 100)] + pub(super) limit: usize, + #[arg(long, default_value_t = 0)] + pub(super) offset: usize, +} + +#[derive(Args)] +pub(super) struct AgentLearningReviewArgs { + pub(super) learning_id: String, + #[arg(long)] + pub(super) reviewer: String, +} + +#[derive(Args)] +pub(super) struct AgentTraceExportArgs { + pub(super) session: String, + #[arg(long)] + pub(super) output: Option, + #[arg(long)] + pub(super) attachments: bool, +} + +#[derive(Args)] +pub(super) struct AgentGitLinkCommand { + #[command(subcommand)] + pub(super) command: AgentGitLinkSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum AgentGitLinkSubcommand { + Link(AgentGitLinkArgs), + List(AgentSessionIdArgs), +} + +#[derive(Args)] +pub(super) struct AgentGitLinkArgs { + pub(super) session: String, + pub(super) git_commit: String, + #[arg(long)] + pub(super) turn: Option, + #[arg(long)] + pub(super) from_change: Option, + #[arg(long)] + pub(super) through_change: Option, + #[arg(long, default_value = "exact-change")] + pub(super) confidence: String, + #[arg(long, default_value = "explicit-cli")] + pub(super) source: String, +} + #[derive(Args)] #[command(after_help = "Daily path: trail agent guide @@ -209,27 +569,54 @@ pub(super) struct AgentCommand { } #[derive(Args)] -pub(super) struct AgentSetupArgs { +pub(super) struct AgentAcpCommand { + #[command(subcommand)] + pub(super) command: AgentAcpSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum AgentAcpSubcommand { + /// Configure an editor to launch an ACP-backed Trail task. + Setup(AgentAcpSetupArgs), + /// Run the editor-facing ACP task entrypoint. + #[command(hide = true)] + Run(AgentAcpRunArgs), + /// List available ACP providers and configured status. + Status(AgentAcpStatusArgs), + /// Run ACP integration diagnostics. + Doctor(AgentAcpDoctorArgs), + /// List captured ACP sessions. + Sessions(AgentAcpSessionsArgs), +} + +#[derive(Args)] +pub(super) struct AgentAcpSetupArgs { + #[arg(value_name = "PROVIDER", conflicts_with = "provider_flag")] + pub(super) provider: Option, #[arg( - long, - default_value = "claude-code", + long = "provider", value_name = "PROVIDER", - help = "Provider profile: claude-code, codex, cursor, gemini, aider, opencode" + conflicts_with = "provider" )] - pub(super) provider: String, - #[arg(long, default_value = "vscode")] + pub(super) provider_flag: Option, + #[arg(long, default_value = "generic")] pub(super) editor: String, + #[arg(long = "print")] + pub(super) print_only: bool, + #[arg(long)] + pub(super) yes: bool, } #[derive(Args)] -pub(super) struct AgentAcpArgs { +pub(super) struct AgentAcpRunArgs { + #[arg(value_name = "PROVIDER", conflicts_with = "provider_flag")] + pub(super) provider: Option, #[arg( - long, - default_value = "claude-code", + long = "provider", value_name = "PROVIDER", - help = "Built-in or ACP registry provider (see `trail acp list`)" + conflicts_with = "provider" )] - pub(super) provider: String, + pub(super) provider_flag: Option, #[arg(long)] pub(super) name: Option, #[arg(long)] @@ -240,15 +627,48 @@ pub(super) struct AgentAcpArgs { pub(super) command: Vec, } +#[derive(Args)] +pub(super) struct AgentAcpStatusArgs { + #[arg(value_name = "PROVIDER", conflicts_with = "provider_flag")] + pub(super) provider: Option, + #[arg( + long = "provider", + value_name = "PROVIDER", + conflicts_with = "provider" + )] + pub(super) provider_flag: Option, +} + +#[derive(Args)] +pub(super) struct AgentAcpDoctorArgs { + #[arg(value_name = "PROVIDER", conflicts_with = "provider_flag")] + pub(super) provider: Option, + #[arg( + long = "provider", + value_name = "PROVIDER", + conflicts_with = "provider" + )] + pub(super) provider_flag: Option, + #[arg(long = "relay-command", num_args = 1.., allow_hyphen_values = true)] + pub(super) relay_command: Vec, +} + +#[derive(Args)] +pub(super) struct AgentAcpSessionsArgs { + #[arg(long)] + pub(super) lane: Option, +} + #[derive(Args)] pub(super) struct AgentStartArgs { + #[arg(value_name = "PROVIDER", conflicts_with = "provider_flag")] + pub(super) provider: Option, #[arg( - long, - default_value = "claude-code", + long = "provider", value_name = "PROVIDER", - help = "Terminal provider profile: claude-code, codex, cursor, gemini, aider, opencode" + conflicts_with = "provider" )] - pub(super) provider: String, + pub(super) provider_flag: Option, #[arg(long)] pub(super) name: Option, #[arg( @@ -258,8 +678,8 @@ pub(super) struct AgentStartArgs { pub(super) from: Option, #[arg( long = "workdir-mode", - value_parser = ["auto", "full-cow", "overlay-cow", "nfs-cow"], - default_value = "full-cow", + value_parser = ["auto", "native-cow", "portable-copy", "fuse-cow", "nfs-cow", "dokan-cow"], + default_value = "auto", help = "Filesystem view for the terminal agent task" )] pub(super) workdir_mode: String, @@ -284,8 +704,8 @@ pub(super) struct AgentContinueArgs { pub(super) name: Option, #[arg( long = "workdir-mode", - value_parser = ["auto", "full-cow", "overlay-cow", "nfs-cow"], - default_value = "full-cow", + value_parser = ["auto", "native-cow", "portable-copy", "fuse-cow", "nfs-cow", "dokan-cow"], + default_value = "auto", help = "Filesystem view for the terminal follow-up task" )] pub(super) workdir_mode: String, @@ -751,6 +1171,12 @@ pub(super) struct AgentUndoArgs { #[derive(Args)] pub(super) struct AgentDoctorArgs { - #[arg(long, default_value = "claude-code")] - pub(super) provider: String, + #[arg(value_name = "PROVIDER", conflicts_with = "provider_flag")] + pub(super) provider: Option, + #[arg( + long = "provider", + value_name = "PROVIDER", + conflicts_with = "provider" + )] + pub(super) provider_flag: Option, } diff --git a/trail/src/cli/command/collaboration_args/merge.rs b/trail/src/cli/command/collaboration_args/merge.rs index ade6bb5..c354e36 100644 --- a/trail/src/cli/command/collaboration_args/merge.rs +++ b/trail/src/cli/command/collaboration_args/merge.rs @@ -2,42 +2,29 @@ use std::path::PathBuf; use clap::{Args, Subcommand}; -#[derive(Args)] -pub(in crate::cli::command) struct MergeLaneArgs { - pub(in crate::cli::command) name: String, - #[arg(long, default_value = "main")] - pub(in crate::cli::command) into: String, - #[arg(long)] - pub(in crate::cli::command) strategy: Option, - #[arg(long)] - pub(in crate::cli::command) dry_run: bool, - #[arg(long)] - pub(in crate::cli::command) direct: bool, -} - #[derive(Subcommand)] -pub(in crate::cli::command) enum MergeQueueSubcommand { - /// Add a source ref to the merge queue. - Add(MergeQueueAddArgs), +pub(in crate::cli::command) enum LaneMergeQueueSubcommand { + /// Add a lane to the serialized merge queue. + Add(LaneMergeQueueAddArgs), /// List queued merge candidates and states. List, /// Explain why a queued merge is ready or blocked. - Explain(MergeQueueExplainArgs), + Explain(LaneMergeQueueExplainArgs), /// Run queued merges up to optional item limit. - Run(MergeQueueRunArgs), + Run(LaneMergeQueueRunArgs), /// Remove a queued item before execution. - Remove(MergeQueueRemoveArgs), + Remove(LaneMergeQueueRemoveArgs), } #[derive(Args)] -pub(in crate::cli::command) struct MergeQueueCommand { +pub(in crate::cli::command) struct LaneMergeQueueCommand { #[command(subcommand)] - pub(in crate::cli::command) command: MergeQueueSubcommand, + pub(in crate::cli::command) command: LaneMergeQueueSubcommand, } #[derive(Args)] -pub(in crate::cli::command) struct MergeQueueAddArgs { - pub(in crate::cli::command) source: String, +pub(in crate::cli::command) struct LaneMergeQueueAddArgs { + pub(in crate::cli::command) lane: String, #[arg(long)] pub(in crate::cli::command) into: String, #[arg(long, default_value_t = 0)] @@ -45,18 +32,18 @@ pub(in crate::cli::command) struct MergeQueueAddArgs { } #[derive(Args)] -pub(in crate::cli::command) struct MergeQueueRunArgs { +pub(in crate::cli::command) struct LaneMergeQueueRunArgs { #[arg(long)] pub(in crate::cli::command) limit: Option, } #[derive(Args)] -pub(in crate::cli::command) struct MergeQueueExplainArgs { +pub(in crate::cli::command) struct LaneMergeQueueExplainArgs { pub(in crate::cli::command) selector: String, } #[derive(Args)] -pub(in crate::cli::command) struct MergeQueueRemoveArgs { +pub(in crate::cli::command) struct LaneMergeQueueRemoveArgs { pub(in crate::cli::command) selector: String, } diff --git a/trail/src/cli/command/environment_args.rs b/trail/src/cli/command/environment_args.rs index a48dcac..9ede937 100644 --- a/trail/src/cli/command/environment_args.rs +++ b/trail/src/cli/command/environment_args.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use clap::{Args, Subcommand}; #[derive(Args)] @@ -10,7 +12,7 @@ pub(super) struct DepsCommand { pub(super) enum DepsSubcommand { /// Show expected, attached, ready, stale, or failed dependency environments. Status(DepsStatusArgs), - /// Build or reuse a frozen Node dependency layer and attach it to a lane. + /// Build or reuse a frozen Node layer and bulk-replace an unmounted lane's dependency state. Sync(DepsSyncArgs), } @@ -26,6 +28,167 @@ pub(super) struct DepsSyncArgs { pub(super) path: Option, } +#[derive(Args)] +pub(super) struct EnvironmentCommand { + #[command(subcommand)] + pub(super) command: EnvironmentSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum EnvironmentSubcommand { + /// List registered environment adapters and their discovery metadata. + Adapters, + /// Install or remove capability-constrained external adapter packages. + Plugin(EnvironmentPluginCommand), + /// Detect environment components without running tools or repository code. + Discover(EnvironmentDiscoverArgs), + /// Show the validated desired component DAG in deterministic build order. + Graph(EnvironmentGraphArgs), + /// Show normalized component and adapter state for a layered lane. + Status(EnvironmentStatusArgs), + /// Show the exact active component/storage generation and its predecessor. + Generation(EnvironmentStatusArgs), + /// Explain every canonical input, tool, platform, or policy edge that made a component stale. + Explain(EnvironmentExplainArgs), + /// Preview the normalized component key, actions, outputs, and capabilities without executing. + Plan(EnvironmentSyncArgs), + /// Prepare shared and/or writable-private outputs and attach one component atomically. + Sync(EnvironmentSyncArgs), + /// Prepare all discovered components, then activate one atomic generation. + SyncAll(EnvironmentDiscoverArgs), + /// Inspect, reconcile, or stop lane-private runtime services. + Runtime(EnvironmentRuntimeCommand), +} + +#[derive(Args)] +pub(super) struct EnvironmentRuntimeCommand { + #[command(subcommand)] + pub(super) command: EnvironmentRuntimeSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum EnvironmentRuntimeSubcommand { + /// Show the active generation's persisted runtime allocations and health. + Status(EnvironmentStatusArgs), + /// Idempotently create or adopt declared resources and wait for health. + Reconcile(EnvironmentStatusArgs), + /// Stop active containers while retaining their lane-private volumes and networks. + Stop(EnvironmentStatusArgs), +} + +#[derive(Args)] +pub(super) struct EnvironmentPluginCommand { + #[command(subcommand)] + pub(super) command: EnvironmentPluginSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum EnvironmentPluginSubcommand { + /// Validate a package and print the exact payload/distribution digests without installing it. + Inspect(EnvironmentPluginInstallArgs), + /// Verify, content-address, and activate a local adapter package directory. + Install(EnvironmentPluginInstallArgs), + /// Deactivate an installed adapter identity without deleting provenance bytes. + Remove(EnvironmentPluginRemoveArgs), + /// Manage publisher public keys trusted to authenticate adapter packages. + Trust(EnvironmentPluginTrustCommand), +} + +#[derive(Args)] +pub(super) struct EnvironmentPluginInstallArgs { + /// Directory containing trail-adapter.toml and its declared executable. + pub(super) package: PathBuf, +} + +#[derive(Args)] +pub(super) struct EnvironmentPluginRemoveArgs { + /// Canonical adapter identity, for example acme/python@1. + pub(super) identity: String, +} + +#[derive(Args)] +pub(super) struct EnvironmentPluginTrustCommand { + #[command(subcommand)] + pub(super) command: EnvironmentPluginTrustSubcommand, +} + +#[derive(Subcommand)] +pub(super) enum EnvironmentPluginTrustSubcommand { + /// Add or rotate a publisher public key from a TOML document. + Add(EnvironmentPluginTrustAddArgs), + /// List currently trusted publisher keys. + List, + /// Revoke a publisher key; signed packages using it fail closed immediately. + Remove(EnvironmentPluginTrustRemoveArgs), +} + +#[derive(Args)] +pub(super) struct EnvironmentPluginTrustAddArgs { + /// TOML document containing schema, publisher, and 32-byte Ed25519 public key. + pub(super) key: PathBuf, +} + +#[derive(Args)] +pub(super) struct EnvironmentPluginTrustRemoveArgs { + /// Content-derived publisher key ID in sha256: form. + pub(super) key_id: String, +} + +#[derive(Args)] +pub(super) struct EnvironmentStatusArgs { + pub(super) lane: String, +} + +#[derive(Args)] +pub(super) struct EnvironmentExplainArgs { + pub(super) lane: String, + /// Stable logical component ID from `trail env discover` or `trail env status`. + #[arg(long, value_name = "ID")] + pub(super) component: String, + /// Zero-based change offset for large monorepo explanations. + #[arg(long, default_value_t = 0)] + pub(super) offset: u64, + /// Maximum changes returned in one page (1-1000). + #[arg(long, default_value_t = 256)] + pub(super) limit: u64, +} + +#[derive(Args)] +pub(super) struct EnvironmentDiscoverArgs { + pub(super) lane: String, + /// Restrict discovery to one source-relative component root. + #[arg(long, value_name = "ROOT")] + pub(super) path: Option, +} + +#[derive(Args)] +pub(super) struct EnvironmentGraphArgs { + pub(super) lane: String, + /// Restrict discovery to one source-relative component root. + #[arg(long, value_name = "ROOT")] + pub(super) path: Option, + /// Zero-based topological node offset. + #[arg(long, default_value_t = 0)] + pub(super) offset: u64, + /// Maximum target nodes returned in one page (1-1000). + #[arg(long, default_value_t = 256)] + pub(super) limit: u64, +} + +#[derive(Args)] +pub(super) struct EnvironmentSyncArgs { + pub(super) lane: String, + /// Stable logical component ID from `trail env discover`. + #[arg(long, value_name = "ID")] + pub(super) component: Option, + /// Versioned environment adapter identity. + #[arg(long, default_value = "auto")] + pub(super) adapter: String, + /// Adapter component root relative to the lane source root. + #[arg(long, value_name = "ROOT")] + pub(super) path: Option, +} + #[derive(Args)] pub(super) struct CacheCommand { #[command(subcommand)] diff --git a/trail/src/cli/command/handler.rs b/trail/src/cli/command/handler.rs index 2050226..2f8ec2c 100644 --- a/trail/src/cli/command/handler.rs +++ b/trail/src/cli/command/handler.rs @@ -6,13 +6,14 @@ use super::{render::*, *}; use trail::{ acp::AcpRelayOptions, Actor, Error, InitImportMode, LaneGateOptions, OperationKind, - PatchDocument, RecordOptions, Result, Trail, + PatchDocument, RecordOptions, Result, StructuredErrorEnvelope, Trail, }; mod acp; mod agent; mod collaboration; mod daemon_rpc; +mod daemon_start; mod errors; mod inspect; mod lane; @@ -26,6 +27,24 @@ use errors::*; use parsing::*; use runtime::*; +fn resolve_agent_provider_argument( + positional: Option, + named: Option, + fallback: Option<&str>, +) -> Result { + match (positional, named) { + (Some(_), Some(_)) => Err(Error::InvalidInput( + "provider may be supplied either positionally or with --provider, not both".to_string(), + )), + (Some(provider), None) | (None, Some(provider)) => Ok(provider), + (None, None) => fallback.map(ToString::to_string).ok_or_else(|| { + Error::InvalidInput( + "choose a provider positionally or with --provider ".to_string(), + ) + }), + } +} + pub(crate) fn run_cli() { let json_errors = args_request_json_errors(std::env::args_os().skip(1)) || env_requests_json_errors(); @@ -34,7 +53,10 @@ pub(crate) fn run_cli() { Err(err) => handle_cli_parse_error(err, json_errors), }; let json_errors = cli.json - || matches!(cli.format.as_ref(), Some(OutputFormat::Json)) + || matches!( + cli.format.as_ref(), + Some(OutputFormat::Json | OutputFormat::Ndjson) + ) || env_requests_json_errors(); if let Err(err) = run(cli) { render_error(&err, json_errors); @@ -45,6 +67,18 @@ pub(crate) fn run_cli() { fn run(cli: Cli) -> Result<()> { let format = resolve_output_format(cli.format)?; let json = cli.json || matches!(format, OutputFormat::Json); + let render_mode = if matches!(format, OutputFormat::Plain) { + RenderMode::Plain + } else { + RenderMode::Human + }; + let render = RenderOptions::from_environment( + render_mode, + cli.color.as_policy(), + cli.pager.as_policy(), + cli.verbose, + cli.quiet, + ); let workspace = cli .workspace .clone() @@ -73,10 +107,16 @@ fn run(cli: Cli) -> Result<()> { branch, json, quiet: cli.quiet, - color: !cli.no_color && std::env::var_os("NO_COLOR").is_none(), format, + render, }; let command = cli.command; + if matches!(ctx.format, OutputFormat::Ndjson) && !supports_ndjson(&command) { + return Err(Error::InvalidInput( + "--format ndjson is available only for streaming watch commands; use --format json for a single report" + .to_string(), + )); + } if let Some(daemon_url) = daemon_url { if daemon_rpc::try_handle_daemon_command( &ctx, @@ -110,7 +150,7 @@ fn run(cli: Cli) -> Result<()> { args.text_policy.as_ref().map(TextPolicyArg::as_str), args.prolly_backend.as_ref().map(ProllyBackendArg::as_str), )?; - render_init(&report, ctx.json, ctx.quiet) + render_init(&report, ctx.json, &ctx.render) } Command::Config(config) => workspace::handle_config_command(&ctx, config), Command::Ignore(ignore) => workspace::handle_ignore_command(&ctx, ignore), @@ -133,6 +173,7 @@ fn run(cli: Cli) -> Result<()> { Command::CodeFrom(args) => inspect::handle_code_from_command(&ctx, args), Command::Lane(lane_command) => lane::handle_lane_command(&ctx, lane_command), Command::Deps(deps) => handle_deps_command(&ctx, deps), + Command::Env(environment) => handle_environment_command(&ctx, environment), Command::Cache(cache) => handle_cache_command(&ctx, cache), Command::Acp(acp_command) => acp::handle_acp_command(&ctx, acp_command), Command::Agent(agent_command) => agent::handle_agent_command(&ctx, agent_command), @@ -144,8 +185,6 @@ fn run(cli: Cli) -> Result<()> { Command::Approvals(approvals_command) => { collaboration::handle_approvals_command(&ctx, approvals_command) } - Command::MergeLane(args) => collaboration::handle_merge_lane_command(&ctx, args), - Command::MergeQueue(queue) => collaboration::handle_merge_queue_command(&ctx, queue), Command::Conflicts(conflicts) => collaboration::handle_conflicts_command(&ctx, conflicts), Command::Anchor(anchor) => collaboration::handle_anchor_command(&ctx, anchor), Command::Lease(lease) => collaboration::handle_lease_command(&ctx, lease), @@ -161,44 +200,170 @@ fn run(cli: Cli) -> Result<()> { } } +fn supports_ndjson(command: &Command) -> bool { + match command { + Command::Watch(_) => true, + Command::Index(IndexCommand { + command: IndexSubcommand::Watch(_), + }) => true, + Command::Index(IndexCommand { + command: IndexSubcommand::Reconcile(_), + }) => true, + Command::Lane(LaneCommand { + command: LaneSubcommand::Watch(args), + }) => !args.once, + _ => false, + } +} + +fn render_specialist( + ctx: &RuntimeContext, + title: &str, + report: &T, +) -> Result<()> { + render_semantic_report(title, report, ctx.json, &ctx.render) +} + +fn handle_environment_command(ctx: &RuntimeContext, environment: EnvironmentCommand) -> Result<()> { + let db = open_db(ctx)?; + match environment.command { + EnvironmentSubcommand::Adapters => render_specialist( + ctx, + "Environment adapters", + &db.workspace_environment_adapters()?, + ), + EnvironmentSubcommand::Plugin(plugin) => match plugin.command { + EnvironmentPluginSubcommand::Inspect(args) => render_specialist( + ctx, + "Environment adapter package", + &db.inspect_environment_adapter_plugin_package(&args.package)?, + ), + EnvironmentPluginSubcommand::Install(args) => render_specialist( + ctx, + "Installed environment adapter", + &db.install_environment_adapter_plugin(&args.package)?, + ), + EnvironmentPluginSubcommand::Remove(args) => render_specialist( + ctx, + "Updated environment adapter", + &db.remove_environment_adapter_plugin(&args.identity)?, + ), + EnvironmentPluginSubcommand::Trust(args) => match args.command { + EnvironmentPluginTrustSubcommand::Add(args) => render_specialist( + ctx, + "Trusted adapter publisher", + &db.trust_environment_adapter_publisher_key(&args.key)?, + ), + EnvironmentPluginTrustSubcommand::List => render_specialist( + ctx, + "Trusted adapter publishers", + &db.environment_adapter_publisher_trust()?, + ), + EnvironmentPluginTrustSubcommand::Remove(args) => render_specialist( + ctx, + "Removed adapter publisher trust", + &db.remove_environment_adapter_publisher_key(&args.key_id)?, + ), + }, + }, + EnvironmentSubcommand::Discover(args) => render_specialist( + ctx, + "Environment discovery", + &db.discover_workspace_environment(&args.lane, args.path.as_deref())?, + ), + EnvironmentSubcommand::Graph(args) => render_specialist( + ctx, + "Environment graph", + &db.workspace_environment_graph_page( + &args.lane, + args.path.as_deref(), + args.offset, + args.limit, + )?, + ), + EnvironmentSubcommand::Status(args) => render_specialist( + ctx, + "Environment status", + &db.environment_component_status(&args.lane)?, + ), + EnvironmentSubcommand::Generation(args) => render_specialist( + ctx, + "Environment generation", + &db.active_environment_generation(&args.lane)?, + ), + EnvironmentSubcommand::Explain(args) => render_specialist( + ctx, + "Environment staleness", + &db.explain_workspace_environment_staleness_page( + &args.lane, + &args.component, + args.offset, + args.limit, + )?, + ), + EnvironmentSubcommand::Plan(args) => render_specialist( + ctx, + "Environment plan", + &db.plan_workspace_environment_component( + &args.lane, + &args.adapter, + args.path.as_deref(), + args.component.as_deref(), + )?, + ), + EnvironmentSubcommand::Sync(args) => render_specialist( + ctx, + "Synchronized environment", + &db.sync_workspace_environment_component_with_runtime( + &args.lane, + &args.adapter, + args.path.as_deref(), + args.component.as_deref(), + )?, + ), + EnvironmentSubcommand::SyncAll(args) => render_specialist( + ctx, + "Synchronized environments", + &db.sync_all_workspace_environments_with_runtime(&args.lane, args.path.as_deref())?, + ), + EnvironmentSubcommand::Runtime(runtime) => { + let generation = match runtime.command { + EnvironmentRuntimeSubcommand::Status(args) => db + .active_environment_generation(&args.lane)? + .ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{}` has no active environment generation", + args.lane + )) + })?, + EnvironmentRuntimeSubcommand::Reconcile(args) => { + db.reconcile_workspace_environment_runtime(&args.lane)? + } + EnvironmentRuntimeSubcommand::Stop(args) => { + db.stop_workspace_environment_runtime(&args.lane)? + } + }; + render_specialist(ctx, "Environment runtime", &generation) + } + } +} + fn handle_deps_command(ctx: &RuntimeContext, deps: DepsCommand) -> Result<()> { + let db = open_db(ctx)?; match deps.command { DepsSubcommand::Status(args) => { - let db = open_db(ctx)?; db.refresh_workspace_environment_staleness(&args.lane)?; - let report = db.workspace_environment_status(&args.lane)?; - if ctx.json { - render_json(&report) - } else { - if !ctx.quiet { - for environment in report { - println!( - "{} {} expected={} attached={}", - environment.adapter, - environment.status, - environment.expected_key, - environment.attached_key.as_deref().unwrap_or("-") - ); - } - } - Ok(()) - } - } - DepsSubcommand::Sync(args) => { - let db = open_db(ctx)?; - let report = db.sync_node_dependencies(&args.lane, args.path.as_deref())?; - if ctx.json { - render_json(&report) - } else { - if !ctx.quiet { - println!( - "Attached layer {} ({}, {} bytes)", - report.layer_id, report.adapter, report.logical_bytes - ); - } - Ok(()) - } + render_specialist( + ctx, + "Dependency status", + &db.workspace_environment_status(&args.lane)?, + ) } + DepsSubcommand::Sync(args) => render_specialist( + ctx, + "Synchronized dependencies", + &db.sync_node_dependencies(&args.lane, args.path.as_deref())?, + ), } } @@ -206,72 +371,21 @@ fn handle_cache_command(ctx: &RuntimeContext, cache: CacheCommand) -> Result<()> let db = open_db(ctx)?; match cache.command { CacheSubcommand::List => { - let report = db.list_workspace_layers()?; - if ctx.json { - render_json(&report) - } else { - if !ctx.quiet { - for layer in report { - println!( - "{} {} {} logical={} physical={}", - layer.layer_id, - layer.adapter, - layer.state, - layer.logical_bytes, - layer - .physical_bytes - .map(|value| value.to_string()) - .unwrap_or_else(|| "unknown".to_string()) - ); - } - } - Ok(()) - } - } - CacheSubcommand::Verify(args) | CacheSubcommand::Inspect(args) => { - let report = db.verify_workspace_layer(&args.layer)?; - if ctx.json { - render_json(&report) - } else { - if !ctx.quiet { - println!( - "{} {} {} entries={} bytes={}", - report.layer_id, - report.adapter, - report.state, - report.entry_count, - report.logical_bytes - ); - } - Ok(()) - } + render_specialist(ctx, "Workspace cache", &db.list_workspace_layers()?) } - CacheSubcommand::Gc(args) => { - let report = db.workspace_cache_gc(args.dry_run, args.retention_secs)?; - if ctx.json { - render_json(&report) + CacheSubcommand::Verify(args) | CacheSubcommand::Inspect(args) => render_specialist( + ctx, + "Workspace cache layer", + &db.verify_workspace_layer(&args.layer)?, + ), + CacheSubcommand::Gc(args) => render_specialist( + ctx, + if args.dry_run { + "Workspace cache cleanup preview" } else { - if !ctx.quiet { - println!( - "cache gc {}: candidates={} reclaimable={} reclaimed={}", - if report.dry_run { - "dry-run" - } else { - "complete" - }, - report.candidates.len(), - report.reclaimable_bytes, - report.reclaimed_bytes - ); - for item in &report.candidates { - println!( - "{} {} bytes={} reason={}", - item.kind, item.id, item.physical_bytes, item.reason - ); - } - } - Ok(()) - } - } + "Workspace cache cleanup" + }, + &db.workspace_cache_gc(args.dry_run, args.retention_secs)?, + ), } } diff --git a/trail/src/cli/command/handler/acp.rs b/trail/src/cli/command/handler/acp.rs index 59ee6ea..b3a5f82 100644 --- a/trail/src/cli/command/handler/acp.rs +++ b/trail/src/cli/command/handler/acp.rs @@ -5,10 +5,6 @@ use trail::model::{AcpDoctorCheck, AcpDoctorReport, AcpProviderProfile}; pub(super) fn handle_acp_command(ctx: &RuntimeContext, acp: AcpCommand) -> Result<()> { match acp.command { - AcpSubcommand::Install(args) => handle_acp_install(ctx, args), - AcpSubcommand::Doctor(args) => handle_acp_doctor(ctx, args), - AcpSubcommand::List => handle_acp_list(ctx), - AcpSubcommand::Sessions(args) => handle_acp_sessions(ctx, args), AcpSubcommand::Relay(args) => handle_acp_relay(ctx, args), } } @@ -16,7 +12,7 @@ pub(super) fn handle_acp_command(ctx: &RuntimeContext, acp: AcpCommand) -> Resul pub(super) fn handle_transcript_command(ctx: &RuntimeContext, args: TranscriptArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.transcript(&args.selector)?; - render_transcript(&report, ctx.json, ctx.quiet) + render_transcript(&report, ctx.json, &ctx.render) } pub(super) fn handle_top_turn_command(ctx: &RuntimeContext, turn: TopTurnCommand) -> Result<()> { @@ -24,57 +20,104 @@ pub(super) fn handle_top_turn_command(ctx: &RuntimeContext, turn: TopTurnCommand TopTurnSubcommand::Show(args) => { let db = open_db(ctx)?; let details = db.show_lane_turn(&args.turn_id)?; - render_lane_turn_details(&details, ctx.json, ctx.quiet) + render_lane_turn_details(&details, ctx.json, &ctx.render) } } } -fn handle_acp_install(ctx: &RuntimeContext, args: AcpInstallArgs) -> Result<()> { +pub(super) fn handle_acp_status(ctx: &RuntimeContext, args: AgentAcpStatusArgs) -> Result<()> { let db = open_db(ctx).ok(); - let report = trail::acp::acp_install_report_with_registry( - &args.agent, - &args.editor, - args.dry_run, - db.as_ref().map(|db| db.db_dir()), - )?; - render_acp_install(&report, ctx.json, ctx.quiet, args.print_only) -} - -fn handle_acp_list(ctx: &RuntimeContext) -> Result<()> { - let db = open_db(ctx).ok(); - let profiles = + let mut profiles = trail::acp::acp_provider_profiles_with_registry(db.as_ref().map(|db| db.db_dir()))?; - render_acp_profiles(&profiles, ctx.json, ctx.quiet) + if args.provider.is_some() || args.provider_flag.is_some() { + let provider = resolve_agent_provider_argument(args.provider, args.provider_flag, None)?; + let canonical = trail::acp::acp_provider_profile_with_registry( + &provider, + db.as_ref().map(|db| db.db_dir()), + )? + .agent; + profiles.retain(|profile| profile.agent == canonical); + } + render_acp_profiles(&profiles, ctx.json, &ctx.render) } -fn handle_acp_sessions(ctx: &RuntimeContext, args: AcpSessionsArgs) -> Result<()> { +pub(super) fn handle_acp_sessions(ctx: &RuntimeContext, args: AgentAcpSessionsArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.list_lane_acp_sessions(args.lane.as_deref())?; - render_acp_sessions(&report, ctx.json, ctx.quiet) + render_acp_sessions(&report, ctx.json, &ctx.render) } -fn handle_acp_doctor(ctx: &RuntimeContext, args: AcpDoctorArgs) -> Result<()> { +pub(super) fn handle_acp_doctor(ctx: &RuntimeContext, args: AgentAcpDoctorArgs) -> Result<()> { let mut checks = Vec::new(); - let mut warnings = Vec::new(); + let warnings = Vec::new(); let mut status = "ok".to_string(); let db_result = open_db(ctx); match &db_result { - Ok(_) => checks.push(acp_check("workspace", "ok", "Trail workspace opened")), + Ok(db) => { + checks.push(acp_check("workspace", "ok", "Trail workspace opened")); + match db.ensure_live_path_invariant_indexes() { + Ok(()) => checks.push(acp_check( + "path_invariant_index", + "ok", + "all live branch and lane roots can accept safe mutations", + )), + Err(err) => { + status = "failed".to_string(); + checks.push(acp_check( + "path_invariant_index", + "failed", + &err.to_string(), + )); + } + } + let capture_journal = capture_journal_check(db); + if capture_journal.status == "failed" { + status = "failed".to_string(); + } + checks.push(capture_journal); + match trail::acp::validate_acp_path_mapping(db.workspace_root()) { + Ok(()) => checks.push(acp_check( + "path_mapping", + "ok", + "workspace paths isolate correctly and external roots are preserved", + )), + Err(err) => { + status = "failed".to_string(); + checks.push(acp_check("path_mapping", "failed", &err.to_string())); + } + } + } Err(err) => { status = "failed".to_string(); checks.push(acp_check("workspace", "failed", &format!("{err}"))); + checks.push(acp_check( + "capture_journal", + "skipped", + "workspace unavailable", + )); + checks.push(acp_check( + "path_invariant_index", + "skipped", + "workspace unavailable", + )); + checks.push(acp_check( + "path_mapping", + "skipped", + "workspace unavailable", + )); } } + let provider = resolve_agent_provider_argument(args.provider, args.provider_flag, None)?; let profile = match trail::acp::acp_provider_profile_with_registry( - &args.agent, + &provider, db_result.as_ref().ok().map(|db| db.db_dir()), ) { Ok(profile) => profile, Err(_) if !args.relay_command.is_empty() => AcpProviderProfile { - agent: args.agent.clone(), - display_name: args.agent.clone(), + agent: provider.clone(), + display_name: provider, available: true, relay_command: args.relay_command.clone(), notes: vec!["using caller-supplied ACP relay command".to_string()], @@ -151,15 +194,11 @@ fn handle_acp_doctor(ctx: &RuntimeContext, args: AcpDoctorArgs) -> Result<()> { )); } - if db_result.is_ok() { - checks.push(acp_check( - "capture", - "skipped", - "external provider launch is validated by command availability; run a real ACP prompt through an editor to verify capture", - )); - } else { - warnings.push("capture smoke skipped because workspace could not be opened".to_string()); - } + checks.push(acp_check( + "capture_smoke", + "skipped", + "run a real ACP prompt through an editor to add provider-specific capture evidence", + )); let report = AcpDoctorReport { status, @@ -167,14 +206,41 @@ fn handle_acp_doctor(ctx: &RuntimeContext, args: AcpDoctorArgs) -> Result<()> { relay_command, lane: None, session_id: None, + conformance: trail::acp::acp_v1_conformance_evidence(), checks, warnings, }; - render_acp_doctor(&report, ctx.json, ctx.quiet) + render_acp_doctor(&report, ctx.json, &ctx.render) +} + +fn capture_journal_check(db: &trail::Trail) -> AcpDoctorCheck { + let journal = db.db_dir().join("acp-ingress"); + if let Err(err) = std::fs::create_dir_all(&journal) { + return acp_check( + "capture_journal", + "failed", + &format!("cannot prepare durable ACP capture journal: {err}"), + ); + } + match std::fs::read_dir(&journal) { + Ok(entries) => { + let pending = entries.filter_map(|entry| entry.ok()).count(); + acp_check( + "capture_journal", + "ok", + &format!("durable capture journal is accessible ({pending} pending spill files)"), + ) + } + Err(err) => acp_check( + "capture_journal", + "failed", + &format!("cannot read durable ACP capture journal: {err}"), + ), + } } fn handle_acp_relay(ctx: &RuntimeContext, args: AcpRelayArgs) -> Result<()> { - let db = open_db(ctx)?; + let db = open_acp_relay_db(ctx)?; let materialize = if args.no_materialize { false } else if let Some(materialize) = args.materialize { @@ -201,6 +267,24 @@ fn handle_acp_relay(ctx: &RuntimeContext, args: AcpRelayArgs) -> Result<()> { }) } +fn open_acp_relay_db(ctx: &RuntimeContext) -> Result { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let mut delay = std::time::Duration::from_millis(2); + loop { + match open_db(ctx) { + Ok(db) => return Ok(db), + Err(Error::SchemaReinitializeRequired { ref found, .. }) + if found == "schema main/WAL/SHM generation changed during mutable handoff" + && std::time::Instant::now() < deadline => + { + std::thread::sleep(delay); + delay = (delay * 2).min(std::time::Duration::from_millis(50)); + } + Err(error) => return Err(error), + } + } +} + fn resolve_acp_relay_command( args: &AcpRelayArgs, cache_dir: Option<&std::path::Path>, diff --git a/trail/src/cli/command/handler/agent.rs b/trail/src/cli/command/handler/agent.rs index 9c3af1a..5032c1f 100644 --- a/trail/src/cli/command/handler/agent.rs +++ b/trail/src/cli/command/handler/agent.rs @@ -1,14 +1,29 @@ use super::*; use crate::cli::command::render::render_agent_timeline; +use std::path::Path; use std::process::{Command as ProcessCommand, Stdio}; +use trail::agent_hooks::{ + apply_agent_hook_install_plan, build_agent_hook_install_plan, inspect_agent_hook_installation, + redact_agent_hook_payload, remove_agent_hook_installation, rollback_agent_hook_install_plan, + AgentHookInstallRequest, AgentHookInstallScope, AgentProviderRegistry, +}; use trail::{ - AgentContinueReport, AgentReviewAction, AgentRunReport, LaneWorkdirMode, StatusSuggestion, + AgentCaptureTransport, AgentContinueReport, AgentHookReceiptInput, AgentReviewAction, + AgentRunReport, LaneWorkdirMode, StatusSuggestion, }; pub(super) fn handle_agent_command(ctx: &RuntimeContext, agent: AgentCommand) -> Result<()> { match agent.command { None => handle_agent_home(ctx), - Some(AgentSubcommand::Setup(args)) => handle_agent_setup(ctx, args), + Some(AgentSubcommand::Hook(args)) => handle_agent_hook(ctx, args), + Some(AgentSubcommand::Hooks(args)) => handle_agent_hooks(ctx, args), + Some(AgentSubcommand::Capture(args)) => handle_agent_capture(ctx, args), + Some(AgentSubcommand::Artifacts(args)) => handle_agent_artifacts(ctx, args), + Some(AgentSubcommand::Provenance(args)) => handle_agent_provenance(ctx, args), + Some(AgentSubcommand::Attest(args)) => handle_agent_attest(ctx, args), + Some(AgentSubcommand::Learnings(args)) => handle_agent_learnings(ctx, args), + Some(AgentSubcommand::Export(args)) => handle_agent_trace_export(ctx, args), + Some(AgentSubcommand::GitLink(args)) => handle_agent_git_link(ctx, args), Some(AgentSubcommand::Acp(args)) => handle_agent_acp(ctx, args), Some(AgentSubcommand::Start(args)) => handle_agent_start(ctx, args), Some(AgentSubcommand::Continue(args)) => handle_agent_continue(ctx, args), @@ -72,30 +87,1240 @@ pub(super) fn handle_agent_command(ctx: &RuntimeContext, agent: AgentCommand) -> } } +fn handle_agent_hook(ctx: &RuntimeContext, args: AgentHookCommand) -> Result<()> { + match args.command { + AgentHookSubcommand::Receive(args) => handle_agent_hook_receive(ctx, args), + } +} + +fn handle_agent_capture(ctx: &RuntimeContext, args: AgentCaptureCommand) -> Result<()> { + let mut db = open_db(ctx)?; + match args.command { + AgentCaptureSubcommand::Begin(args) => { + let workdir = args + .workdir + .unwrap_or(std::env::current_dir()?) + .canonicalize()?; + let report = db.begin_agent_capture_run(trail::AgentCaptureRunInput { + lane: args.lane, + workdir: workdir.to_string_lossy().into_owned(), + owner_agent: args.owner, + owner_session_id: args.session, + executor_agent: args.executor, + work_item_id: args.work_item, + lease_ms: args.ttl_ms, + metadata_json: Some("{\"source\":\"cli\"}".to_string()), + })?; + render_agent_hooks_value( + ctx, + serde_json::to_value(&report)?, + format!("capture run {} started", report.capture_run_id), + ) + } + AgentCaptureSubcommand::Renew(args) => { + let report = + db.renew_agent_capture_run(&args.run_id, &args.owner, &args.session, args.ttl_ms)?; + render_agent_hooks_value( + ctx, + serde_json::to_value(&report)?, + format!("capture run {} renewed", report.capture_run_id), + ) + } + AgentCaptureSubcommand::End(args) => { + let report = db.end_agent_capture_run(&args.run_id, &args.owner, &args.session)?; + render_agent_hooks_value( + ctx, + serde_json::to_value(&report)?, + format!("capture run {} ended", report.capture_run_id), + ) + } + AgentCaptureSubcommand::Status(args) => { + let reports = db.list_agent_capture_runs_page(!args.all, args.offset, args.limit)?; + let summary = format!("{} capture run(s)", reports.len()); + render_agent_hooks_value(ctx, serde_json::to_value(reports)?, summary) + } + AgentCaptureSubcommand::Reconcile => { + let report = db.reconcile_expired_agent_capture_runs()?; + let summary = format!( + "expired {} run(s); interrupted {} session mapping(s)", + report.expired_run_ids.len(), + report.interrupted_mapping_ids.len() + ); + render_agent_hooks_value(ctx, serde_json::to_value(report)?, summary) + } + } +} + +fn handle_agent_artifacts(ctx: &RuntimeContext, args: AgentArtifactsArgs) -> Result<()> { + let db = open_db(ctx)?; + let artifacts = + db.list_lane_artifacts_page(&args.session, args.turn.as_deref(), args.offset, args.limit)?; + let summary = format!("{} artifact(s) for {}", artifacts.len(), args.session); + render_agent_hooks_value(ctx, serde_json::to_value(artifacts)?, summary) +} + +fn handle_agent_provenance(ctx: &RuntimeContext, args: AgentProvenanceArgs) -> Result<()> { + let db = open_db(ctx)?; + let (nodes, edges) = db.list_session_provenance_page(&args.session, args.offset, args.limit)?; + let summary = format!( + "{} provenance node(s), {} edge(s) for {}", + nodes.len(), + edges.len(), + args.session + ); + render_agent_hooks_value( + ctx, + serde_json::json!({"session_id":args.session,"nodes":nodes,"edges":edges}), + summary, + ) +} + +fn handle_agent_attest(ctx: &RuntimeContext, args: AgentAttestCommand) -> Result<()> { + let mut db = open_db(ctx)?; + match args.command { + AgentAttestSubcommand::Create(args) => { + let report = db.create_session_attestation( + &args.session, + &args.policy, + Some(serde_json::json!({"source":"cli"})), + )?; + render_agent_hooks_value( + ctx, + serde_json::to_value(&report)?, + format!("attestation {} created", report.attestation_id), + ) + } + AgentAttestSubcommand::List(args) => { + let reports = + db.list_session_attestations_page(&args.session, args.offset, args.limit)?; + let summary = format!("{} attestation(s) for {}", reports.len(), args.session); + render_agent_hooks_value(ctx, serde_json::to_value(reports)?, summary) + } + AgentAttestSubcommand::Show(args) => { + let report = db.session_attestation(&args.attestation_id)?; + render_agent_hooks_value( + ctx, + serde_json::to_value(&report)?, + format!("attestation {} ({})", report.attestation_id, report.status), + ) + } + AgentAttestSubcommand::Verify(args) => { + let report = db.verify_session_attestation(&args.attestation_id)?; + render_agent_hooks_value( + ctx, + serde_json::to_value(&report)?, + format!( + "attestation {}: {}", + report.attestation_id, + if report.valid { "valid" } else { "invalid" } + ), + ) + } + } +} + +fn handle_agent_learnings(ctx: &RuntimeContext, args: AgentLearningsCommand) -> Result<()> { + let mut db = open_db(ctx)?; + match args.command { + AgentLearningsSubcommand::List(args) => { + let reports = db.list_learnings_page( + args.session.as_deref(), + args.status.as_deref(), + args.offset, + args.limit, + )?; + let summary = format!("{} learning(s)", reports.len()); + render_agent_hooks_value(ctx, serde_json::to_value(reports)?, summary) + } + AgentLearningsSubcommand::Accept(args) => { + let report = db.review_learning(&args.learning_id, true, &args.reviewer)?; + render_agent_hooks_value( + ctx, + serde_json::to_value(&report)?, + format!("learning {} accepted", report.learning_id), + ) + } + AgentLearningsSubcommand::Reject(args) => { + let report = db.review_learning(&args.learning_id, false, &args.reviewer)?; + render_agent_hooks_value( + ctx, + serde_json::to_value(&report)?, + format!("learning {} rejected", report.learning_id), + ) + } + } +} + +fn handle_agent_trace_export(ctx: &RuntimeContext, args: AgentTraceExportArgs) -> Result<()> { + use std::io::Write; + + let db = open_db(ctx)?; + let trace = db.export_agent_trace(&args.session, args.attachments)?; + let bytes = trace.to_canonical_json()?; + if let Some(output) = args.output { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&output)?; + file.write_all(&bytes)?; + file.sync_all()?; + render_document( + &TerminalDocument::new("Exported portable agent trace", UiTone::Success).block( + UiBlock::Metadata(vec![("Path".to_string(), output.display().to_string())]), + ), + &ctx.render, + )?; + } else { + std::io::stdout().write_all(&bytes)?; + } + Ok(()) +} + +fn handle_agent_git_link(ctx: &RuntimeContext, args: AgentGitLinkCommand) -> Result<()> { + let mut db = open_db(ctx)?; + match args.command { + AgentGitLinkSubcommand::Link(args) => { + let report = db.link_git_commit_to_agent(trail::GitAgentLinkInput { + session_id: args.session, + turn_id: args.turn, + git_commit: args.git_commit, + from_change: args.from_change, + through_change: args.through_change, + confidence: args.confidence, + source: args.source, + metadata: Some(serde_json::json!({"source":"cli"})), + })?; + render_agent_hooks_value( + ctx, + serde_json::to_value(&report)?, + format!( + "linked Git commit {} to session {}", + report.git_commit, report.session_id + ), + ) + } + AgentGitLinkSubcommand::List(args) => { + let reports = db.list_git_agent_links_page(&args.session, args.offset, args.limit)?; + let summary = format!("{} Git link(s) for {}", reports.len(), args.session); + render_agent_hooks_value(ctx, serde_json::to_value(reports)?, summary) + } + } +} + +fn handle_agent_hooks(ctx: &RuntimeContext, args: AgentHooksCommand) -> Result<()> { + match args.command { + AgentHooksSubcommand::Setup(args) => handle_agent_hooks_setup(ctx, args), + AgentHooksSubcommand::Remove(args) => handle_agent_hooks_remove(ctx, args), + AgentHooksSubcommand::List(args) => handle_agent_hooks_list(ctx, args), + AgentHooksSubcommand::Status(args) => handle_agent_hooks_status(ctx, args), + AgentHooksSubcommand::Doctor(args) => handle_agent_hooks_doctor(ctx, args), + AgentHooksSubcommand::Events(args) => handle_agent_hooks_events(ctx, args), + AgentHooksSubcommand::Replay(args) => handle_agent_hooks_replay(ctx, args), + AgentHooksSubcommand::Retry(args) => { + let mut db = open_db(ctx)?; + let report = db.retry_agent_hook_receipt(&args.receipt)?; + render_agent_hooks_value( + ctx, + serde_json::to_value(&report)?, + format!("receipt {} queued for retry", report.receipt_id), + ) + } + AgentHooksSubcommand::Discard(args) => { + let mut db = open_db(ctx)?; + let report = db.discard_agent_hook_receipt(&args.receipt)?; + render_agent_hooks_value( + ctx, + serde_json::to_value(&report)?, + format!("receipt {} discarded", report.receipt_id), + ) + } + } +} + +fn handle_agent_hooks_setup(ctx: &RuntimeContext, args: AgentHooksSetupArgs) -> Result<()> { + let provider = resolve_agent_provider_argument(args.provider, args.provider_flag, None)?; + let registry = AgentProviderRegistry::built_in()?; + registry.resolve(&provider)?; + let mut db = open_db(ctx)?; + let scope = agent_hook_scope(args.scope); + let home = agent_hooks_home_dir(); + let plan = build_agent_hook_install_plan(AgentHookInstallRequest { + registry: ®istry, + provider: &provider, + workspace_id: &db.config().workspace.id.0, + workspace_root: db.workspace_root(), + home_dir: home.as_deref(), + scope, + force: args.force, + })?; + let dry_run = !args.yes || args.print_only; + let report = apply_agent_hook_install_plan(&plan, dry_run)?; + if !dry_run { + if let Err(error) = db.record_agent_hook_installation(&plan, args.lane.as_deref()) { + rollback_agent_hook_install_plan(&plan).map_err(|rollback| { + Error::Conflict(format!( + "failed to record hook installation ({error}); rollback also failed: {rollback}" + )) + })?; + return Err(error); + } + } + render_agent_hooks_value( + ctx, + serde_json::to_value(&report)?, + format!( + "{} {} hooks at {} ({:?}){}", + if dry_run { + "Would configure" + } else { + "Configured" + }, + report.provider, + report.config_path.display(), + report.action, + if dry_run { " [dry run]" } else { "" } + ), + ) +} + +fn handle_agent_hooks_remove(ctx: &RuntimeContext, args: AgentHooksRemoveArgs) -> Result<()> { + let provider = resolve_agent_provider_argument(args.provider, args.provider_flag, None)?; + let registry = AgentProviderRegistry::built_in()?; + let provider = registry.resolve(&provider)?.provider.clone(); + let mut db = open_db(ctx)?; + let scope = agent_hook_scope(args.scope); + let record = db + .list_agent_hook_installations(Some(&provider))? + .into_iter() + .find(|record| record.scope == scope && record.status == "installed") + .ok_or_else(|| { + Error::InvalidInput(format!( + "no installed {provider} hooks were recorded for {} scope", + scope.as_str() + )) + })?; + let home = agent_hooks_home_dir(); + let plan = build_agent_hook_install_plan(AgentHookInstallRequest { + registry: ®istry, + provider: &provider, + workspace_id: &db.config().workspace.id.0, + workspace_root: db.workspace_root(), + home_dir: home.as_deref(), + scope, + force: true, + })?; + if plan.installation_id != record.installation_id || plan.config_path != record.config_path { + return Err(Error::Conflict( + "recorded hook ownership does not match the current provider install target" + .to_string(), + )); + } + let report = remove_agent_hook_installation(&plan, &record.config_after_digest, args.dry_run)?; + if !args.dry_run { + db.mark_agent_hook_installation_removed(&record.installation_id)?; + } + render_agent_hooks_value( + ctx, + serde_json::to_value(&report)?, + format!( + "{} {} hooks from {}{}", + if args.dry_run { + "Would remove" + } else { + "Removed" + }, + provider, + report.config_path.display(), + if args.dry_run { " [dry run]" } else { "" } + ), + ) +} + +fn handle_agent_hooks_list(ctx: &RuntimeContext, args: AgentHooksListArgs) -> Result<()> { + let registry = AgentProviderRegistry::built_in()?; + let db = open_db(ctx)?; + let installations = db.list_agent_hook_installations(None)?; + let rows = registry + .list() + .into_iter() + .filter_map(|manifest| { + let matching = installations + .iter() + .filter(|record| { + record.provider == manifest.provider && record.status == "installed" + }) + .collect::>(); + if args.installed && matching.is_empty() { + return None; + } + Some(serde_json::json!({ + "provider": manifest.provider, + "display_name": manifest.display_name, + "support": manifest.support, + "deployment": manifest.deployment, + "project_config_path": manifest.project_config_path, + "installations": matching, + })) + }) + .collect::>(); + if ctx.json { + render_json(&rows)?; + } else if !ctx.quiet { + let table = UiTable::new( + vec![ + UiColumn::left("PROVIDER", 0, 12), + UiColumn::left("SUPPORT", 1, 10), + UiColumn::right("INSTALLED", 0, 9), + ], + rows.iter() + .map(|row| { + vec![ + row["display_name"] + .as_str() + .or_else(|| row["provider"].as_str()) + .unwrap_or_default() + .to_string(), + state_label(row["support"].as_str().unwrap_or("known")), + row["installations"] + .as_array() + .map(Vec::len) + .unwrap_or_default() + .to_string(), + ] + }) + .collect(), + ); + render_document( + &TerminalDocument::new("Agent hook providers", UiTone::Neutral) + .block(UiBlock::Table(table)), + &ctx.render, + )?; + } + Ok(()) +} + +fn handle_agent_hooks_status(ctx: &RuntimeContext, args: AgentHooksProviderArgs) -> Result<()> { + let provider = resolve_agent_provider_argument(args.provider, args.provider_flag, None)?; + let registry = AgentProviderRegistry::built_in()?; + let provider = registry.resolve(&provider)?.provider.clone(); + let db = open_db(ctx)?; + let records = db.list_agent_hook_installations(Some(&provider))?; + let home = agent_hooks_home_dir(); + let mut statuses = Vec::new(); + for record in records { + let plan = build_agent_hook_install_plan(AgentHookInstallRequest { + registry: ®istry, + provider: &provider, + workspace_id: &db.config().workspace.id.0, + workspace_root: db.workspace_root(), + home_dir: home.as_deref(), + scope: record.scope, + force: true, + })?; + statuses.push(serde_json::json!({ + "record": record, + "filesystem_status": inspect_agent_hook_installation(&plan)?, + })); + } + let summary = if statuses.is_empty() { + format!("{provider}: not installed") + } else { + format!("{provider}: {} recorded installation(s)", statuses.len()) + }; + render_agent_hooks_value( + ctx, + serde_json::json!({"provider": provider, "installations": statuses}), + summary, + ) +} + +fn handle_agent_hooks_doctor(ctx: &RuntimeContext, args: AgentHooksDoctorArgs) -> Result<()> { + let registry = AgentProviderRegistry::built_in()?; + let requested = match (args.provider, args.provider_flag) { + (None, None) => None, + (positional, named) => Some(resolve_agent_provider_argument(positional, named, None)?), + }; + let providers = if args.all { + registry.list() + } else if let Some(requested) = requested { + vec![registry.resolve(&requested)?] + } else { + registry.list() + }; + let db = open_db(ctx)?; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| i64::try_from(duration.as_millis()).unwrap_or(i64::MAX)) + .unwrap_or(0); + let spool = agent_hook_spool_pressure(db.db_dir()); + let mut reports = Vec::new(); + for manifest in providers { + let installations = db.list_agent_hook_installations(Some(&manifest.provider))?; + let receipts = db.list_agent_hook_receipts(Some(&manifest.provider), None, 1_000)?; + let mappings = db.list_lane_agent_sessions(Some(&manifest.provider), None, 1_000)?; + let runs = db + .list_agent_capture_runs(false, 1_000)? + .into_iter() + .filter(|run| { + run.owner_agent == manifest.provider + || run.executor_agent.as_deref() == Some(manifest.provider.as_str()) + }) + .collect::>(); + let probe = registry.probe(&manifest.provider, args.probe)?; + let failed_receipts = receipts + .iter() + .filter(|receipt| matches!(receipt.status.as_str(), "retry" | "quarantined")) + .count(); + let stale_finalizers = mappings + .iter() + .filter(|mapping| { + mapping.status == trail::AgentCapturePhase::Finalizing + && mapping + .finalization_lease_expires_at + .is_none_or(|expires_at| expires_at <= now) + }) + .count(); + let home = agent_hooks_home_dir(); + let installation_checks = installations + .iter() + .map(|record| { + build_agent_hook_install_plan(AgentHookInstallRequest { + registry: ®istry, + provider: &manifest.provider, + workspace_id: &db.config().workspace.id.0, + workspace_root: db.workspace_root(), + home_dir: home.as_deref(), + scope: record.scope, + force: true, + }) + .and_then(|plan| inspect_agent_hook_installation(&plan)) + .map(|status| { + serde_json::json!({ + "installation_id": record.installation_id, + "status": status, + }) + }) + .unwrap_or_else(|error| { + serde_json::json!({ + "installation_id": record.installation_id, + "status": "malformed", + "diagnostic": error.to_string(), + }) + }) + }) + .collect::>(); + let checks = vec![ + serde_json::json!({ + "code": "HOOK_INSTALLATION_STATUS", + "status": if installations.is_empty() { "warning" } else { "ok" }, + "message": if installations.is_empty() { "no recorded native hook installation" } else { "native hook ownership records found" }, + "remediation": format!("trail agent hooks setup {}", manifest.provider), + "details": installation_checks, + }), + serde_json::json!({ + "code": "RECEIPT_REPLAY_FAILURES", + "status": if failed_receipts == 0 { "ok" } else { "warning" }, + "message": format!("{failed_receipts} receipt(s) require retry or operator review"), + "remediation": "trail agent hooks replay --pending", + }), + serde_json::json!({ + "code": "TURN_FINALIZATION_LEASE_STALE", + "status": if stale_finalizers == 0 { "ok" } else { "warning" }, + "message": format!("{stale_finalizers} stale finalization lease(s)"), + "remediation": "trail agent hooks replay --pending", + }), + serde_json::json!({ + "code": "CAPTURE_RECEIPT_SPOOLING", + "status": if spool.files == 0 { "ok" } else { "warning" }, + "message": format!("{} spooled receipt file(s), {} bytes", spool.files, spool.bytes), + "remediation": "trail agent hooks replay --pending", + }), + serde_json::json!({ + "code": "TRANSCRIPT_CAPABILITY", + "status": if manifest.transcript_location_hints.is_empty() && manifest.canonical_export_command.is_none() { "warning" } else { "ok" }, + "message": if manifest.transcript_location_hints.is_empty() && manifest.canonical_export_command.is_none() { "provider has no stable native transcript or canonical export contract" } else { "provider transcript/export collection is capability-gated" }, + "remediation": "inspect provider capability and fidelity diagnostics before relying on transcript evidence", + }), + ]; + reports.push(serde_json::json!({ + "provider": manifest.provider, + "support": manifest.support, + "adapter_version": manifest.adapter_version, + "provider_version_range": manifest.provider_version_range, + "contract_source": manifest.contract_source, + "installations": installations, + "last_receipt": receipts.first(), + "receipt_status_counts": receipt_status_counts(&receipts), + "capture_mappings": mappings, + "managed_runs": runs, + "spool": spool, + "checks": checks, + "probe": probe, + "probe_requested": args.probe, + "probe_status": if args.probe { "version-command" } else { "static-discovery" }, + })); + } + let summary = format!("checked {} agent hook provider(s)", reports.len()); + render_agent_hooks_value(ctx, serde_json::Value::Array(reports), summary) +} + +fn handle_agent_hooks_events(ctx: &RuntimeContext, args: AgentHooksEventsArgs) -> Result<()> { + let provider = resolve_agent_provider_argument(args.provider, args.provider_flag, None)?; + let registry = AgentProviderRegistry::built_in()?; + let provider = registry.resolve(&provider)?.provider.clone(); + let db = open_db(ctx)?; + let mut receipts = + db.list_agent_hook_receipts_page(Some(&provider), None, args.offset, args.last)?; + if args.failed { + receipts.retain(|receipt| { + matches!( + receipt.status.as_str(), + "retry" | "quarantined" | "discarded" + ) + }); + } + let summary = format!("{}: {} durable receipt(s)", provider, receipts.len()); + render_agent_hooks_value(ctx, serde_json::to_value(receipts)?, summary) +} + +fn handle_agent_hooks_replay(ctx: &RuntimeContext, args: AgentHooksReplayArgs) -> Result<()> { + if args.receipt.is_none() && !args.pending { + return Err(Error::InvalidInput( + "agent hooks replay requires --receipt or --pending".to_string(), + )); + } + let mut db = open_db(ctx)?; + let spool = if args.pending { + drain_agent_hook_spool(&mut db)? + } else { + AgentHookSpoolDrain::default() + }; + let batch = if let Some(receipt_id) = args.receipt { + match db.replay_agent_hook_receipt(&receipt_id) { + Ok(report) => trail::AgentHookReplayBatchReport { + recovered_stale_receipts: 0, + replayed: vec![report], + failures: Vec::new(), + }, + Err(error) => trail::AgentHookReplayBatchReport { + recovered_stale_receipts: 0, + replayed: Vec::new(), + failures: vec![trail::AgentHookReplayFailure { + receipt_id, + code: error.code().to_string(), + message: error.to_string(), + }], + }, + } + } else { + db.replay_pending_agent_hook_receipts(args.limit)? + }; + let value = serde_json::json!({ + "spool": spool, + "recovered_stale_receipts": batch.recovered_stale_receipts, + "replayed": batch.replayed, + "failures": batch.failures + }); + let summary = format!( + "replayed {} receipt(s); {} failed", + value["replayed"].as_array().map(Vec::len).unwrap_or(0), + value["failures"].as_array().map(Vec::len).unwrap_or(0) + ); + render_agent_hooks_value(ctx, value, summary) +} + +#[derive(Default, serde::Serialize)] +struct AgentHookSpoolDrain { + imported: usize, + duplicates: usize, + failed: Vec, +} + +#[derive(Clone, Copy, Default, serde::Serialize)] +struct AgentHookSpoolPressure { + files: usize, + bytes: u64, +} + +fn agent_hook_spool_pressure(db_dir: &std::path::Path) -> AgentHookSpoolPressure { + let directory = db_dir.join("runtime/agent-hooks-spool"); + if directory + .symlink_metadata() + .map(|metadata| metadata.file_type().is_symlink() || !metadata.is_dir()) + .unwrap_or(true) + { + return AgentHookSpoolPressure::default(); + } + let mut pressure = AgentHookSpoolPressure::default(); + if let Ok(entries) = std::fs::read_dir(directory) { + for entry in entries.flatten().take(10_000) { + if let Ok(metadata) = entry.metadata() { + if metadata.is_file() { + pressure.files += 1; + pressure.bytes = pressure.bytes.saturating_add(metadata.len()); + } + } + } + } + pressure +} + +fn receipt_status_counts( + receipts: &[trail::AgentHookReceipt], +) -> std::collections::BTreeMap { + let mut counts = std::collections::BTreeMap::new(); + for receipt in receipts { + *counts.entry(receipt.status.clone()).or_insert(0) += 1; + } + counts +} + +fn drain_agent_hook_spool(db: &mut trail::Trail) -> Result { + let directory = db.db_dir().join("runtime/agent-hooks-spool"); + if !directory.exists() { + return Ok(AgentHookSpoolDrain::default()); + } + if directory + .symlink_metadata() + .is_ok_and(|metadata| metadata.file_type().is_symlink()) + { + return Err(Error::InvalidPath { + path: directory.display().to_string(), + reason: "agent hook spool directory is a symlink".to_string(), + }); + } + let mut paths = std::fs::read_dir(&directory)? + .filter_map(|entry| entry.ok().map(|entry| entry.path())) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("receipt-") && name.ends_with(".json")) + }) + .collect::>(); + paths.sort(); + let mut report = AgentHookSpoolDrain::default(); + for path in paths.into_iter().take(1_000) { + let result = (|| -> Result { + let metadata = path.symlink_metadata()?; + if !metadata.is_file() + || metadata.file_type().is_symlink() + || metadata.len() > (trail::AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES + 16_384) as u64 + { + return Err(Error::InvalidPath { + path: path.display().to_string(), + reason: "invalid agent hook spool entry".to_string(), + }); + } + let envelope: AgentHookSpoolEnvelope = serde_json::from_slice(&std::fs::read(&path)?)?; + if envelope.version != 1 { + return Err(Error::InvalidInput(format!( + "unsupported agent hook spool version {}", + envelope.version + ))); + } + let ingested = db.persist_agent_hook_receipt(envelope.input)?; + std::fs::remove_file(&path)?; + Ok(ingested.duplicate) + })(); + match result { + Ok(true) => report.duplicates += 1, + Ok(false) => report.imported += 1, + Err(error) => report.failed.push(serde_json::json!({ + "path": path, + "code": error.code(), + "error": error.to_string(), + })), + } + } + Ok(report) +} + +fn agent_hook_scope(scope: AgentHooksScopeArg) -> AgentHookInstallScope { + match scope { + AgentHooksScopeArg::Project => AgentHookInstallScope::Project, + AgentHooksScopeArg::User => AgentHookInstallScope::User, + } +} + +fn agent_hooks_home_dir() -> Option { + std::env::var_os("HOME") + .or_else(|| std::env::var_os("USERPROFILE")) + .map(std::path::PathBuf::from) +} + +fn render_agent_hooks_value( + ctx: &RuntimeContext, + value: serde_json::Value, + human: String, +) -> Result<()> { + if ctx.json { + render_json(&value)?; + } else { + render_document( + &TerminalDocument::new("Agent hook", UiTone::Neutral).block(UiBlock::paragraph(human)), + &ctx.render, + )?; + } + Ok(()) +} + +fn handle_agent_hook_receive(ctx: &RuntimeContext, args: AgentHookReceiveArgs) -> Result<()> { + use std::io::Read; + + const MAX_STDIN_BYTES: u64 = (trail::AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES + 1) as u64; + let registry = AgentProviderRegistry::built_in(); + let provider = registry + .as_ref() + .ok() + .and_then(|registry| registry.resolve(&args.provider).ok()) + .map(|manifest| manifest.provider.clone()) + .unwrap_or_else(|| args.provider.trim().to_ascii_lowercase()); + let mut bytes = Vec::new(); + if let Err(error) = std::io::stdin() + .lock() + .take(MAX_STDIN_BYTES) + .read_to_end(&mut bytes) + { + render_native_receipt_diagnostic( + ctx, + "Native agent receipt could not be read", + error.to_string(), + ); + return acknowledge_agent_hook(&provider, &args.native_event); + } + if bytes.len() > trail::AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES { + render_native_receipt_diagnostic( + ctx, + "Native agent receipt was not recorded", + format!( + "The receipt exceeds {} bytes.", + trail::AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES + ), + ); + return acknowledge_agent_hook(&provider, &args.native_event); + } + let payload: serde_json::Value = + match if bytes.iter().all(u8::is_ascii_whitespace) && provider == "kiro" { + Ok(serde_json::json!({})) + } else { + serde_json::from_slice(&bytes) + } { + Ok(payload) => payload, + Err(error) => { + render_native_receipt_diagnostic( + ctx, + "Native agent receipt was not recorded", + format!("The receipt is not valid JSON: {error}"), + ); + return acknowledge_agent_hook(&provider, &args.native_event); + } + }; + let payload = enrich_kiro_hook_payload(payload, &provider, &args.native_event); + if serde_json::to_vec(&payload) + .map(|encoded| encoded.len()) + .unwrap_or(usize::MAX) + > trail::AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES + { + render_native_receipt_diagnostic( + ctx, + "Native agent receipt was not recorded", + format!( + "The enriched receipt exceeds {} bytes.", + trail::AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES + ), + ); + return acknowledge_agent_hook(&provider, &args.native_event); + } + if registry + .as_ref() + .ok() + .and_then(|registry| registry.resolve(&provider).ok()) + .is_none() + { + render_native_receipt_diagnostic( + ctx, + "Native agent provider is not registered", + format!("Provider `{provider}` is not registered."), + ); + return acknowledge_agent_hook(&provider, &args.native_event); + } + let native_session_id = hook_payload_string( + &payload, + &[ + "session_id", + "sessionId", + "sessionID", + "conversation_id", + "conversationId", + "thread_id", + "threadId", + ], + ); + let native_turn_id = hook_payload_string(&payload, &["turn_id", "turnId"]); + let occurred_at = hook_payload_i64(&payload, &["timestamp", "occurred_at", "occurredAt"]); + let dedupe_key = args.dedupe_key.unwrap_or_else(|| { + derive_hook_dedupe_key( + &provider, + &args.native_event, + native_session_id.as_deref(), + native_turn_id.as_deref(), + &payload, + ) + }); + let input = AgentHookReceiptInput { + installation_id: args.installation, + provider: provider.clone(), + native_event: args.native_event.clone(), + native_session_id, + native_turn_id, + transport: AgentCaptureTransport::NativeHooks, + connection_id: None, + direction: None, + connection_sequence: None, + dedupe_key, + payload: payload.clone(), + occurred_at, + }; + let capture_result = open_db(ctx).and_then(|mut db| { + let ingested = db.persist_agent_hook_receipt(input.clone())?; + if ingested.receipt.status != "processed" { + if let Err(error) = db.replay_agent_hook_receipt(&ingested.receipt.receipt_id) { + render_native_receipt_diagnostic( + ctx, + "Native receipt replay is deferred", + error.to_string(), + ); + } + } + Ok(()) + }); + if let Err(error) = capture_result { + let envelope = AgentHookSpoolEnvelope { + version: 1, + input: AgentHookReceiptInput { + payload: redact_agent_hook_payload(payload), + ..input + }, + }; + match spool_agent_hook_receipt(ctx, &envelope) { + Err(spool_error) => { + render_native_receipt_diagnostic( + ctx, + "Native receipt was not recorded", + format!("{error}; fallback spool also failed: {spool_error}"), + ); + } + _ => { + render_native_receipt_diagnostic( + ctx, + "Native receipt was spooled for later replay", + error.to_string(), + ); + } + } + } + acknowledge_agent_hook(&provider, &args.native_event) +} + +fn enrich_kiro_hook_payload( + mut payload: serde_json::Value, + provider: &str, + native_event: &str, +) -> serde_json::Value { + if provider != "kiro" { + return payload; + } + let Some(root) = payload.as_object_mut() else { + return payload; + }; + if native_event.eq_ignore_ascii_case("UserPromptSubmit") && !root.contains_key("prompt") { + if let Ok(prompt) = std::env::var("USER_PROMPT") { + if !prompt.is_empty() && prompt.len() <= trail::AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES { + root.insert("prompt".to_string(), serde_json::Value::String(prompt)); + } + } + } + if !root.contains_key("cwd") { + if let Ok(cwd) = std::env::current_dir() { + root.insert( + "cwd".to_string(), + serde_json::Value::String(cwd.to_string_lossy().into_owned()), + ); + } + } + payload +} + +#[derive(Clone, serde::Deserialize, serde::Serialize)] +struct AgentHookSpoolEnvelope { + version: u16, + input: AgentHookReceiptInput, +} + +fn acknowledge_agent_hook(provider: &str, native_event: &str) -> Result<()> { + if provider == "codex" && matches!(native_event, "Stop" | "SubagentStop") { + render_protocol_content("{\"continue\":true}\n")?; + } else if provider == "gemini" { + render_protocol_content("{}\n")?; + } + Ok(()) +} + +fn render_native_receipt_diagnostic(ctx: &RuntimeContext, summary: &str, cause: String) { + let diagnostic = UiDiagnostic { + code: "AGENT_HOOK_RECEIPT".to_string(), + summary: summary.to_string(), + cause: Some(cause), + consequence: Some( + "Trail acknowledged the native hook but did not apply this receipt.".to_string(), + ), + recovery: Some(UiNextAction { + command: "trail agent hooks replay --pending".to_string(), + reason: "Retry recorded native hook receipts after fixing the provider issue." + .to_string(), + }), + alternatives: Vec::new(), + }; + let _ = render_error_document( + &TerminalDocument::empty().block(UiBlock::Diagnostic(diagnostic)), + &ctx.render, + ); +} + +fn spool_agent_hook_receipt(ctx: &RuntimeContext, envelope: &AgentHookSpoolEnvelope) -> Result<()> { + use std::io::Write; + + let db_dir = agent_hook_spool_db_dir(ctx) + .ok_or_else(|| Error::WorkspaceNotFound(std::env::current_dir().unwrap_or_default()))?; + if db_dir + .symlink_metadata() + .is_ok_and(|metadata| metadata.file_type().is_symlink()) + { + return Err(Error::InvalidPath { + path: db_dir.display().to_string(), + reason: "agent hook spool database directory is a symlink".to_string(), + }); + } + let spool_dir = db_dir.join("runtime/agent-hooks-spool"); + std::fs::create_dir_all(&spool_dir)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&spool_dir, std::fs::Permissions::from_mode(0o700))?; + } + let mut spool_files = 0usize; + let mut spool_bytes = 0u64; + for entry in std::fs::read_dir(&spool_dir)? { + let entry = entry?; + let metadata = entry.path().symlink_metadata()?; + if metadata.is_file() && !metadata.file_type().is_symlink() { + spool_files = spool_files.saturating_add(1); + spool_bytes = spool_bytes.saturating_add(metadata.len()); + } + } + if spool_files >= 10_000 || spool_bytes >= 64 * 1024 * 1024 { + return Err(Error::InvalidInput(format!( + "agent hook spool quota exceeded ({spool_files} files, {spool_bytes} bytes)" + ))); + } + let bytes = serde_json::to_vec(envelope)?; + if bytes.len() > trail::AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES + 16_384 { + return Err(Error::InvalidInput( + "redacted agent hook spool envelope is too large".to_string(), + )); + } + let digest = hook_short_hash(&bytes, 24); + let target = spool_dir.join(format!("receipt-{digest}.json")); + if target.exists() { + return Ok(()); + } + let temp = spool_dir.join(format!(".receipt-{digest}-{}.tmp", std::process::id())); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&temp)?; + if let Err(error) = (|| -> std::io::Result<()> { + file.write_all(&bytes)?; + file.sync_all()?; + std::fs::rename(&temp, &target)?; + #[cfg(unix)] + std::fs::File::open(&spool_dir)?.sync_all()?; + Ok(()) + })() { + let _ = std::fs::remove_file(&temp); + return Err(error.into()); + } + Ok(()) +} + +fn agent_hook_spool_db_dir(ctx: &RuntimeContext) -> Option { + if let Some(db_dir) = ctx.db_dir.as_ref() { + return Some(db_dir.clone()); + } + if let Some(workspace) = ctx.workspace.as_ref() { + return Some(workspace.join(".trail")); + } + let mut cursor = std::env::current_dir().ok()?; + loop { + let candidate = cursor.join(".trail"); + if candidate.is_dir() { + return Some(candidate); + } + if !cursor.pop() { + return None; + } + } +} + +fn hook_payload_string(payload: &serde_json::Value, keys: &[&str]) -> Option { + keys.iter().find_map(|key| { + hook_payload_value(payload, key) + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) + .map(str::to_string) + }) +} + +fn hook_payload_i64(payload: &serde_json::Value, keys: &[&str]) -> Option { + keys.iter().find_map(|key| { + hook_payload_value(payload, key).and_then(|value| { + value + .as_i64() + .or_else(|| value.as_u64().and_then(|value| i64::try_from(value).ok())) + }) + }) +} + +fn hook_payload_value<'a>( + payload: &'a serde_json::Value, + key: &str, +) -> Option<&'a serde_json::Value> { + payload.get(key).or_else(|| { + ["properties", "input", "event", "data", "session"] + .iter() + .find_map(|container| payload.get(*container).and_then(|value| value.get(key))) + }) +} + +fn derive_hook_dedupe_key( + provider: &str, + native_event: &str, + native_session_id: Option<&str>, + native_turn_id: Option<&str>, + payload: &serde_json::Value, +) -> String { + let explicit_event_id = hook_payload_string( + payload, + &[ + "event_id", + "eventId", + "tool_use_id", + "toolUseId", + "toolCallId", + "agent_id", + "agentId", + ], + ); + let timestamp = hook_payload_i64(payload, &["timestamp", "occurred_at", "occurredAt"]); + let payload_digest = hook_short_hash( + serde_json::to_vec(payload).unwrap_or_default().as_slice(), + 24, + ); + let entropy = if explicit_event_id.is_none() && native_turn_id.is_none() && timestamp.is_none() + { + hook_short_hash( + format!( + "{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|value| value.as_nanos()) + .unwrap_or_default() + ) + .as_bytes(), + 16, + ) + } else { + String::new() + }; + format!( + "{}:{}:{}:{}:{}:{}:{}", + provider, + native_event, + native_session_id.unwrap_or("none"), + native_turn_id.unwrap_or("none"), + explicit_event_id.as_deref().unwrap_or("none"), + timestamp + .map(|value| value.to_string()) + .unwrap_or_else(|| "none".to_string()), + if entropy.is_empty() { + payload_digest + } else { + format!("{payload_digest}:{entropy}") + } + ) +} + +fn hook_short_hash(bytes: &[u8], digest_bytes: usize) -> String { + use sha2::{Digest, Sha256}; + + let digest = Sha256::digest(bytes); + digest + .iter() + .take(digest_bytes.min(digest.len())) + .map(|byte| format!("{byte:02x}")) + .collect() +} + fn handle_agent_home(ctx: &RuntimeContext) -> Result<()> { let mut db = open_db(ctx)?; let tasks = db.list_agent_tasks()?.tasks; if tasks.len() == 1 { let report = db.agent_dashboard(&tasks[0].lane)?; - render_agent_dashboard(&report, ctx.json, ctx.quiet) + render_agent_dashboard(&report, ctx.json, &ctx.render) } else { let report = db.agent_inbox()?; - render_agent_inbox(&report, ctx.json, ctx.quiet) + render_agent_inbox(&report, ctx.json, &ctx.render) } } -fn handle_agent_setup(ctx: &RuntimeContext, args: AgentSetupArgs) -> Result<()> { +fn handle_agent_acp(ctx: &RuntimeContext, args: AgentAcpCommand) -> Result<()> { + match args.command { + AgentAcpSubcommand::Setup(args) => handle_agent_acp_setup(ctx, args), + AgentAcpSubcommand::Run(args) => handle_agent_acp_run(ctx, args), + AgentAcpSubcommand::Status(args) => super::acp::handle_acp_status(ctx, args), + AgentAcpSubcommand::Doctor(args) => super::acp::handle_acp_doctor(ctx, args), + AgentAcpSubcommand::Sessions(args) => super::acp::handle_acp_sessions(ctx, args), + } +} + +fn handle_agent_acp_setup(ctx: &RuntimeContext, args: AgentAcpSetupArgs) -> Result<()> { + let provider = resolve_agent_provider_argument(args.provider, args.provider_flag, None)?; let db = open_db(ctx)?; - let report = db.agent_setup_report(&args.provider, &args.editor)?; - render_agent_setup(&report, ctx.json, ctx.quiet) + let plan = trail::acp::build_acp_setup_plan( + db.workspace_root(), + db.db_dir(), + &provider, + &args.editor, + )?; + let report = trail::acp::apply_acp_setup_plan(plan, args.yes && !args.print_only)?; + render_acp_setup(&report, ctx.json, &ctx.render, args.print_only) } -fn handle_agent_acp(ctx: &RuntimeContext, args: AgentAcpArgs) -> Result<()> { +fn handle_agent_acp_run(ctx: &RuntimeContext, args: AgentAcpRunArgs) -> Result<()> { + let provider = resolve_agent_provider_argument(args.provider, args.provider_flag, None)?; let db = open_db(ctx)?; - let lane = db.fresh_agent_lane_name(&args.provider, args.name.as_deref()); + let lane = db.fresh_agent_lane_name(&provider, args.name.as_deref()); let launch = if args.command.is_empty() { Some(trail::acp::resolve_acp_provider( - &args.provider, + &provider, Some(db.db_dir()), )?) } else { @@ -104,7 +1329,7 @@ fn handle_agent_acp(ctx: &RuntimeContext, args: AgentAcpArgs) -> Result<()> { let provider = launch .as_ref() .map(|launch| launch.profile.agent.clone()) - .unwrap_or_else(|| args.provider.clone()); + .unwrap_or(provider); let upstream_command = launch .as_ref() .map(|launch| launch.upstream_command.clone()) @@ -127,7 +1352,13 @@ fn handle_agent_acp(ctx: &RuntimeContext, args: AgentAcpArgs) -> Result<()> { fn handle_agent_start(ctx: &RuntimeContext, args: AgentStartArgs) -> Result<()> { let db = open_db(ctx)?; - let lane = db.fresh_agent_lane_name(&args.provider, args.name.as_deref()); + let configured_provider = db.config().agent.default_provider.clone(); + let provider = resolve_agent_provider_argument( + args.provider, + args.provider_flag, + configured_provider.as_deref(), + )?; + let lane = db.fresh_agent_lane_name(&provider, args.name.as_deref()); let workdir_mode = db.resolve_lane_spawn_workdir_mode( args.from.as_deref(), Some(&args.workdir_mode), @@ -140,12 +1371,12 @@ fn handle_agent_start(ctx: &RuntimeContext, args: AgentStartArgs) -> Result<()> ctx, db, lane, - args.provider, + provider, args.from, workdir_mode, args.command, )?; - render_agent_run(&report, ctx.json, ctx.quiet) + render_agent_run(&report, ctx.json, &ctx.render) } fn handle_agent_continue(ctx: &RuntimeContext, args: AgentContinueArgs) -> Result<()> { @@ -202,7 +1433,7 @@ fn handle_agent_continue(ctx: &RuntimeContext, args: AgentContinueArgs) -> Resul run, suggestions, }; - render_agent_continue(&report, ctx.json, ctx.quiet) + render_agent_continue(&report, ctx.json, &ctx.render) } fn run_terminal_agent_task( @@ -214,6 +1445,8 @@ fn run_terminal_agent_task( workdir_mode: LaneWorkdirMode, command: Vec, ) -> Result { + const TERMINAL_CAPTURE_LEASE_MS: u64 = 86_400_000; + let spawn = db.spawn_lane_with_workdir_mode_paths_and_neighbors( &lane, from.as_deref(), @@ -227,8 +1460,8 @@ fn run_terminal_agent_task( let workdir = spawn.workdir.clone().ok_or_else(|| { Error::InvalidInput("agent start requires a filesystem lane workdir".to_string()) })?; - let overlay_mount = if workdir_mode == LaneWorkdirMode::OverlayCow { - Some(db.mount_overlay_cow_workdir_for_lane(&lane)?) + let fuse_mount = if workdir_mode == LaneWorkdirMode::FuseCow { + Some(db.mount_fuse_cow_workdir_for_lane(&lane)?) } else { None }; @@ -237,9 +1470,25 @@ fn run_terminal_agent_task( } else { None }; + #[cfg(target_os = "windows")] + let dokan_mount = if workdir_mode == LaneWorkdirMode::DokanCow { + Some(db.mount_dokan_cow_workdir_for_lane(&lane)?) + } else { + None + }; let session = db .start_lane_session(&lane, Some(format!("Agent terminal {}", provider)), None)? .session; + let capture_run = db.begin_agent_capture_run(trail::AgentCaptureRunInput { + lane: Some(lane.clone()), + workdir: workdir.clone(), + owner_agent: provider.clone(), + owner_session_id: session.session_id.clone(), + executor_agent: Some(provider.clone()), + work_item_id: Some(lane.clone()), + lease_ms: TERMINAL_CAPTURE_LEASE_MS, + metadata_json: Some("{\"source\":\"agent-start\",\"mode\":\"terminal\"}".to_string()), + })?; db.add_lane_session_event( &lane, &session.session_id, @@ -252,24 +1501,83 @@ fn run_terminal_agent_task( "from": from })), )?; - let workspace_environment = db.lane_workspace_environment(&lane)?; + let mut workspace_environment = db.lane_workspace_environment(&lane)?; + workspace_environment.extend([ + ("TRAIL_CAPTURE_MODE".to_string(), "terminal".to_string()), + ( + "TRAIL_CAPTURE_RUN_ID".to_string(), + capture_run.capture_run_id.clone(), + ), + ( + "TRAIL_CAPTURE_WORKSPACE".to_string(), + db.workspace_root().to_string_lossy().into_owned(), + ), + ( + "TRAIL_CAPTURE_LANE".to_string(), + capture_run.lane_id.clone().unwrap_or_else(|| lane.clone()), + ), + ]); + let project_hook_settings = if provider == "claude-code" { + db.list_agent_hook_installations(Some(&provider))? + .into_iter() + .find(|installation| { + installation.status == "installed" + && installation.scope == AgentHookInstallScope::Project + && installation.config_path.is_file() + }) + .map(|installation| installation.config_path) + } else { + None + }; + let workspace_root = db.workspace_root().to_path_buf(); drop(db); - let command = if command.is_empty() { + let command_is_default = command.is_empty(); + let mut command = if command_is_default { default_terminal_agent_command(&provider)? } else { command }; - if !ctx.quiet && !ctx.json { - println!("Agent task: {lane}"); - println!("Workdir: {workdir}"); - println!("Command: {}", command.join(" ")); + if command_is_default { + if let Some(settings) = project_hook_settings { + command.push("--settings".to_string()); + command.push(settings.to_string_lossy().into_owned()); + } } - let mut process = ProcessCommand::new(&command[0]); + if !ctx.json { + render_document( + &TerminalDocument::new(format!("Launching agent task {lane}"), UiTone::Success).block( + UiBlock::Metadata(vec![ + ("Workdir".to_string(), workdir.clone()), + ("Command".to_string(), command.join(" ")), + ]), + ), + &ctx.render, + )?; + } + let (launch_program, launch_args) = + confined_terminal_agent_command(&command, &workspace_root, Path::new(&workdir))?; + let mut process = ProcessCommand::new(launch_program); + let mut git_ceiling_directories = std::env::var_os("GIT_CEILING_DIRECTORIES") + .map(|paths| std::env::split_paths(&paths).collect::>()) + .unwrap_or_default(); + if !git_ceiling_directories + .iter() + .any(|path| path == &workspace_root) + { + git_ceiling_directories.push(workspace_root.clone()); + } + let git_ceiling_directories = + std::env::join_paths(git_ceiling_directories).map_err(|error| { + Error::InvalidInput(format!("cannot construct Git discovery ceiling: {error}")) + })?; process - .args(&command[1..]) + .args(launch_args) .current_dir(&workdir) .envs(workspace_environment) + .env("PWD", &workdir) + .env("OLDPWD", &workdir) + .env("GIT_CEILING_DIRECTORIES", git_ceiling_directories) .stdin(Stdio::inherit()) .stderr(Stdio::inherit()); if ctx.json { @@ -312,7 +1620,10 @@ fn run_terminal_agent_task( "status": status })), )?; - db.end_lane_session(&session.session_id, status)?; + if db.show_lane_session(&session.session_id)?.session.status == "active" { + db.end_lane_session(&session.session_id, status)?; + } + db.end_agent_capture_run(&capture_run.capture_run_id, &provider, &session.session_id)?; let view = db.agent_task_view(&lane)?; let report = AgentRunReport { task: view.task, @@ -323,8 +1634,10 @@ fn run_terminal_agent_task( recorded: Some(recorded), status: status.to_string(), }; - drop(overlay_mount); + drop(fuse_mount); drop(nfs_mount); + #[cfg(target_os = "windows")] + drop(dokan_mount); Ok(report) } @@ -347,7 +1660,7 @@ fn push_agent_cli_suggestion( fn handle_agent_guide(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_guide(&args.selector)?; - render_agent_guide(&report, ctx.json, ctx.quiet) + render_agent_guide(&report, ctx.json, &ctx.render) } #[derive(Debug, Clone, PartialEq, Eq)] @@ -1320,19 +2633,19 @@ fn agent_ask_clean_path(token: &str) -> Option { fn handle_agent_status(ctx: &RuntimeContext) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_status()?; - render_agent_status(&report, ctx.json, ctx.quiet) + render_agent_status(&report, ctx.json, &ctx.render) } fn handle_agent_dashboard(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let mut db = open_db(ctx)?; let report = db.agent_dashboard(&args.selector)?; - render_agent_dashboard(&report, ctx.json, ctx.quiet) + render_agent_dashboard(&report, ctx.json, &ctx.render) } fn handle_agent_review_data(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let mut db = open_db(ctx)?; let report = db.agent_review_data(&args.selector)?; - render_agent_review_data(&report, ctx.json, ctx.quiet) + render_agent_review_data(&report, ctx.json, &ctx.render) } fn handle_agent_action(ctx: &RuntimeContext, args: AgentActionArgs) -> Result<()> { @@ -1344,12 +2657,12 @@ fn handle_agent_action(ctx: &RuntimeContext, args: AgentActionArgs) -> Result<() if let Some(action_id) = action_id { return handle_agent_empty_action(ctx, &action_id, &args); } - return render_agent_empty_action_palette(ctx.json, ctx.quiet); + return render_agent_empty_action_palette(ctx.json, &ctx.render); } Err(err) => return Err(err), }; let Some(action_id) = action_id else { - return render_agent_action_palette(&review_data, ctx.json, ctx.quiet); + return render_agent_action_palette(&review_data, ctx.json, &ctx.render); }; let action = review_data .actions @@ -1375,9 +2688,7 @@ fn handle_agent_action(ctx: &RuntimeContext, args: AgentActionArgs) -> Result<() "action": action })); } - if !ctx.quiet { - println!("{}", action.command); - } + render_raw_content(&format!("{}\n", action.command), &ctx.render)?; return Ok(()); } if !action.enabled { @@ -1402,11 +2713,11 @@ fn handle_agent_action(ctx: &RuntimeContext, args: AgentActionArgs) -> Result<() "open_focus_file" => run_agent_shell_action(ctx, &action.command), "inspect_focus_file" => { let report = db.agent_focus(&lane, action.path.as_deref(), false)?; - render_agent_focus(&report, ctx.json, ctx.quiet) + render_agent_focus(&report, ctx.json, &ctx.render) } "show_focus_patch" => { let report = db.agent_focus(&lane, action.path.as_deref(), true)?; - render_agent_focus(&report, ctx.json, ctx.quiet) + render_agent_focus(&report, ctx.json, &ctx.render) } "mark_focus_file_reviewed" => { let path = action.path.as_deref().ok_or_else(|| { @@ -1415,15 +2726,15 @@ fn handle_agent_action(ctx: &RuntimeContext, args: AgentActionArgs) -> Result<() ) })?; let report = db.agent_mark_file_reviewed(&lane, path, args.note)?; - render_agent_mark_file_reviewed(&report, ctx.json, ctx.quiet) + render_agent_mark_file_reviewed(&report, ctx.json, &ctx.render) } "show_review_map" => { let report = db.agent_review_map(&lane)?; - render_agent_review_map(&report, ctx.json, ctx.quiet) + render_agent_review_map(&report, ctx.json, &ctx.render) } "show_test_plan" => { let report = db.agent_test_plan(&lane)?; - render_agent_test_plan(&report, ctx.json, ctx.quiet) + render_agent_test_plan(&report, ctx.json, &ctx.render) } "validation_next" => { if ctx.json { @@ -1435,15 +2746,15 @@ fn handle_agent_action(ctx: &RuntimeContext, args: AgentActionArgs) -> Result<() } "apply_dry_run" => { let report = db.agent_apply(&lane, true, args.message)?; - render_agent_apply(&report, ctx.json, ctx.quiet) + render_agent_apply(&report, ctx.json, &ctx.render) } "apply_task" => { let report = db.agent_finish(&lane, false, args.message, args.note)?; - render_agent_finish(&report, ctx.json, ctx.quiet) + render_agent_finish(&report, ctx.json, &ctx.render) } "mark_task_reviewed" => { let report = db.agent_mark_reviewed(&lane, args.note)?; - render_agent_mark_reviewed(&report, ctx.json, ctx.quiet) + render_agent_mark_reviewed(&report, ctx.json, &ctx.render) } _ => Err(Error::InvalidInput(format!( "agent action `{}` is not executable by this Trail version; run `{}` directly", @@ -1474,9 +2785,7 @@ fn handle_agent_empty_action( "action": action })); } - if !ctx.quiet { - println!("{}", action.command); - } + render_raw_content(&format!("{}\n", action.command), &ctx.render)?; return Ok(()); } if action.requires_confirmation && !args.confirm { @@ -1487,62 +2796,76 @@ fn handle_agent_empty_action( } match action.id.as_str() { - "setup_vscode" => handle_agent_setup( + "acp_setup_vscode" => handle_agent_acp_setup( ctx, - AgentSetupArgs { - provider: "claude-code".to_string(), + AgentAcpSetupArgs { + provider: Some("claude-code".to_string()), + provider_flag: None, editor: "vscode".to_string(), + print_only: false, + yes: false, }, ), - "setup_codex_vscode" => handle_agent_setup( + "acp_setup_codex_vscode" => handle_agent_acp_setup( ctx, - AgentSetupArgs { - provider: "codex".to_string(), + AgentAcpSetupArgs { + provider: Some("codex".to_string()), + provider_flag: None, editor: "vscode".to_string(), + print_only: false, + yes: false, }, ), - "setup_cursor_vscode" => handle_agent_setup( + "acp_setup_cursor_vscode" => handle_agent_acp_setup( ctx, - AgentSetupArgs { - provider: "cursor".to_string(), + AgentAcpSetupArgs { + provider: Some("cursor".to_string()), + provider_flag: None, editor: "vscode".to_string(), + print_only: false, + yes: false, }, ), "doctor_claude_code" => handle_agent_doctor( ctx, AgentDoctorArgs { - provider: "claude-code".to_string(), + provider: Some("claude-code".to_string()), + provider_flag: None, }, ), "doctor_codex" => handle_agent_doctor( ctx, AgentDoctorArgs { - provider: "codex".to_string(), + provider: Some("codex".to_string()), + provider_flag: None, }, ), "doctor_cursor" => handle_agent_doctor( ctx, AgentDoctorArgs { - provider: "cursor".to_string(), + provider: Some("cursor".to_string()), + provider_flag: None, }, ), "start_terminal_task" => handle_agent_start( ctx, AgentStartArgs { - provider: "claude-code".to_string(), + provider: Some("claude-code".to_string()), + provider_flag: None, name: None, from: None, - workdir_mode: "full-cow".to_string(), + workdir_mode: "native-cow".to_string(), command: Vec::new(), }, ), "start_gemini_task" => handle_agent_start( ctx, AgentStartArgs { - provider: "gemini".to_string(), + provider: Some("gemini".to_string()), + provider_flag: None, name: None, from: None, - workdir_mode: "full-cow".to_string(), + workdir_mode: "native-cow".to_string(), command: Vec::new(), }, ), @@ -1634,88 +2957,91 @@ fn run_agent_shell_action(ctx: &RuntimeContext, command: &str) -> Result<()> { .unwrap_or_else(|| "terminated by signal".to_string()) ))); } - if !ctx.quiet { - println!("Agent action command completed: {command}"); - } + render_document( + &TerminalDocument::new("Agent action completed", UiTone::Success).block(UiBlock::Metadata( + vec![("Command".to_string(), command.to_string())], + )), + &ctx.render, + )?; Ok(()) } fn handle_agent_review_flow(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let mut db = open_db(ctx)?; let report = db.agent_review_flow(&args.selector)?; - render_agent_review_flow(&report, ctx.json, ctx.quiet) + render_agent_review_flow(&report, ctx.json, &ctx.render) } fn handle_agent_inbox(ctx: &RuntimeContext, args: AgentInboxArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_inbox_with_options(args.all)?; - render_agent_inbox(&report, ctx.json, ctx.quiet) + render_agent_inbox(&report, ctx.json, &ctx.render) } fn handle_agent_board(ctx: &RuntimeContext, args: AgentInboxArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_board_with_options(args.all)?; - render_agent_board(&report, ctx.json, ctx.quiet) + render_agent_board(&report, ctx.json, &ctx.render) } fn handle_agent_stack(ctx: &RuntimeContext, args: AgentInboxArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_stack_with_options(args.all)?; - render_agent_stack(&report, ctx.json, ctx.quiet) + render_agent_stack(&report, ctx.json, &ctx.render) } fn handle_agent_next(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_next(&args.selector)?; - render_agent_next(&report, ctx.json, ctx.quiet) + render_agent_next(&report, ctx.json, &ctx.render) } fn handle_agent_list(ctx: &RuntimeContext, args: AgentListArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.list_agent_tasks_with_options(args.all)?; - render_agent_list(&report, ctx.json, ctx.quiet) + render_agent_list(&report, ctx.json, &ctx.render) } fn handle_agent_brief(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_brief(&args.selector)?; - render_agent_brief(&report, ctx.json, ctx.quiet) + render_agent_brief(&report, ctx.json, &ctx.render) } fn handle_agent_summary(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let mut db = open_db(ctx)?; let report = db.agent_summary(&args.selector)?; - render_agent_summary(&report, ctx.json, ctx.quiet) + render_agent_summary(&report, ctx.json, &ctx.render) } fn handle_agent_validate(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_validate(&args.selector)?; - render_agent_validate(&report, ctx.json, ctx.quiet) + render_agent_validate(&report, ctx.json, &ctx.render) } fn handle_agent_test_plan(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_test_plan(&args.selector)?; - render_agent_test_plan(&report, ctx.json, ctx.quiet) + render_agent_test_plan(&report, ctx.json, &ctx.render) } fn handle_agent_report(ctx: &RuntimeContext, args: AgentReportArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_report(&args.selector)?; - render_agent_report(&report, ctx.json, ctx.quiet, args.markdown) + render_agent_report(&report, ctx.json, &ctx.render, args.markdown) } fn handle_agent_handoff(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_handoff(&args.selector)?; - render_agent_handoff(&report, ctx.json, ctx.quiet) + render_agent_handoff(&report, ctx.json, &ctx.render) } fn handle_agent_receipt(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_receipt(&args.selector)?; - render_agent_receipt(&report, ctx.json, ctx.quiet) + render_agent_receipt(&report, ctx.json, &ctx.render) } fn handle_agent_pr(ctx: &RuntimeContext, args: AgentPrArgs) -> Result<()> { @@ -1724,7 +3050,7 @@ fn handle_agent_pr(ctx: &RuntimeContext, args: AgentPrArgs) -> Result<()> { render_agent_pr( &report, ctx.json, - ctx.quiet, + &ctx.render, args.title_only, args.body_only, ) @@ -1733,55 +3059,55 @@ fn handle_agent_pr(ctx: &RuntimeContext, args: AgentPrArgs) -> Result<()> { fn handle_agent_story(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_story(&args.selector)?; - render_agent_story(&report, ctx.json, ctx.quiet) + render_agent_story(&report, ctx.json, &ctx.render) } fn handle_agent_tools(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_tools(&args.selector)?; - render_agent_tools(&report, ctx.json, ctx.quiet) + render_agent_tools(&report, ctx.json, &ctx.render) } fn handle_agent_impact(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_impact(&args.selector)?; - render_agent_impact(&report, ctx.json, ctx.quiet) + render_agent_impact(&report, ctx.json, &ctx.render) } fn handle_agent_review_map(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_review_map(&args.selector)?; - render_agent_review_map(&report, ctx.json, ctx.quiet) + render_agent_review_map(&report, ctx.json, &ctx.render) } fn handle_agent_risk(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_risk(&args.selector)?; - render_agent_risk(&report, ctx.json, ctx.quiet) + render_agent_risk(&report, ctx.json, &ctx.render) } fn handle_agent_confidence(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let mut db = open_db(ctx)?; let report = db.agent_confidence(&args.selector)?; - render_agent_confidence(&report, ctx.json, ctx.quiet) + render_agent_confidence(&report, ctx.json, &ctx.render) } fn handle_agent_ready(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let mut db = open_db(ctx)?; let report = db.agent_ready(&args.selector)?; - render_agent_ready(&report, ctx.json, ctx.quiet) + render_agent_ready(&report, ctx.json, &ctx.render) } fn handle_agent_diagnose(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let mut db = open_db(ctx)?; let report = db.agent_diagnose(&args.selector)?; - render_agent_diagnose(&report, ctx.json, ctx.quiet) + render_agent_diagnose(&report, ctx.json, &ctx.render) } fn handle_agent_compare(ctx: &RuntimeContext, args: AgentCompareArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_compare(&args.left, &args.right)?; - render_agent_compare(&report, ctx.json, ctx.quiet) + render_agent_compare(&report, ctx.json, &ctx.render) } enum AgentGateKind { @@ -1812,7 +3138,7 @@ fn handle_agent_gate(ctx: &RuntimeContext, args: AgentGateArgs, kind: AgentGateK options, )?, }; - let render_result = render_lane_test(&report, ctx.json, ctx.quiet); + let render_result = render_lane_test(&report, ctx.json, &ctx.render); if render_result.is_ok() && !report.success { std::process::exit(command_failure_exit_code(report.exit_code)); } @@ -1822,15 +3148,15 @@ fn handle_agent_gate(ctx: &RuntimeContext, args: AgentGateArgs, kind: AgentGateK fn handle_agent_workdir(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_workdir(&args.selector)?; - render_agent_workdir(&report, ctx.json, ctx.quiet) + render_agent_workdir(&report, ctx.json, &ctx.render) } fn handle_agent_view(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; match db.agent_task_view(&args.selector) { - Ok(report) => render_agent_view(&report, ctx.json, ctx.quiet), + Ok(report) => render_agent_view(&report, ctx.json, &ctx.render), Err(Error::InvalidInput(message)) if message.contains("no agent tasks") => { - render_agent_empty_task_hint("view", ctx.json, ctx.quiet) + render_agent_empty_task_hint("view", ctx.json, &ctx.render) } Err(err) => Err(err), } @@ -1840,9 +3166,9 @@ fn handle_agent_changes(ctx: &RuntimeContext, args: AgentChangesArgs) -> Result< let db = open_db(ctx)?; let _ = args.by_turn; match db.agent_changes_with_options(&args.selector, args.by_operation, args.by_file) { - Ok(report) => render_agent_changes(&report, ctx.json, ctx.quiet), + Ok(report) => render_agent_changes(&report, ctx.json, &ctx.render), Err(Error::InvalidInput(message)) if message.contains("no agent tasks") => { - render_agent_empty_task_hint("changes", ctx.json, ctx.quiet) + render_agent_empty_task_hint("changes", ctx.json, &ctx.render) } Err(err) => Err(err), } @@ -1857,19 +3183,19 @@ fn handle_agent_delta(ctx: &RuntimeContext, args: AgentDeltaArgs) -> Result<()> args.file.as_deref(), args.patch, )?; - render_agent_delta(&report, ctx.json, ctx.quiet) + render_agent_delta(&report, ctx.json, &ctx.render) } fn handle_agent_new(ctx: &RuntimeContext, args: AgentNewArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_new(&args.selector, args.file.as_deref(), args.patch)?; - render_agent_new(&report, ctx.json, ctx.quiet) + render_agent_new(&report, ctx.json, &ctx.render) } fn handle_agent_mark_reviewed(ctx: &RuntimeContext, args: AgentMarkReviewedArgs) -> Result<()> { let mut db = open_db(ctx)?; let report = db.agent_mark_reviewed(&args.selector, args.note)?; - render_agent_mark_reviewed(&report, ctx.json, ctx.quiet) + render_agent_mark_reviewed(&report, ctx.json, &ctx.render) } fn handle_agent_mark_file_reviewed( @@ -1879,7 +3205,7 @@ fn handle_agent_mark_file_reviewed( let mut db = open_db(ctx)?; let (selector, path) = agent_mark_file_reviewed_selector_args(&args); let report = db.agent_mark_file_reviewed(&selector, &path, args.note)?; - render_agent_mark_file_reviewed(&report, ctx.json, ctx.quiet) + render_agent_mark_file_reviewed(&report, ctx.json, &ctx.render) } fn handle_agent_archive( @@ -1889,40 +3215,40 @@ fn handle_agent_archive( ) -> Result<()> { let mut db = open_db(ctx)?; let report = db.agent_archive(&args.selector, archived, args.note)?; - render_agent_archive(&report, ctx.json, ctx.quiet) + render_agent_archive(&report, ctx.json, &ctx.render) } fn handle_agent_change(ctx: &RuntimeContext, args: AgentChangeArgs) -> Result<()> { let db = open_db(ctx)?; let (selector, change_selector) = agent_change_selector_args(&args); let report = db.agent_change_set(&selector, &change_selector, args.patch)?; - render_agent_change_set(&report, ctx.json, ctx.quiet) + render_agent_change_set(&report, ctx.json, &ctx.render) } fn handle_agent_timeline(ctx: &RuntimeContext, args: AgentTimelineArgs) -> Result<()> { let db = open_db(ctx)?; let _ = args.by_turn; let report = db.agent_timeline(&args.selector, args.by_operation)?; - render_agent_timeline(&report, ctx.json, ctx.quiet) + render_agent_timeline(&report, ctx.json, &ctx.render) } fn handle_agent_files(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_files(&args.selector)?; - render_agent_files(&report, ctx.json, ctx.quiet) + render_agent_files(&report, ctx.json, &ctx.render) } fn handle_agent_file(ctx: &RuntimeContext, args: AgentFileArgs) -> Result<()> { let db = open_db(ctx)?; let (selector, path) = agent_file_selector_args(&args); let report = db.agent_file(&selector, &path, args.patch)?; - render_agent_file(&report, ctx.json, ctx.quiet) + render_agent_file(&report, ctx.json, &ctx.render) } fn handle_agent_checkpoints(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_checkpoints(&args.selector)?; - render_agent_checkpoints(&report, ctx.json, ctx.quiet) + render_agent_checkpoints(&report, ctx.json, &ctx.render) } fn handle_agent_why(ctx: &RuntimeContext, args: AgentWhyArgs) -> Result<()> { @@ -1932,7 +3258,7 @@ fn handle_agent_why(ctx: &RuntimeContext, args: AgentWhyArgs) -> Result<()> { None => ("latest".to_string(), args.selector_or_path), }; let report = db.agent_why(&selector, &path)?; - render_agent_why(&report, ctx.json, ctx.quiet) + render_agent_why(&report, ctx.json, &ctx.render) } fn agent_change_selector_args(args: &AgentChangeArgs) -> (String, String) { @@ -1962,7 +3288,7 @@ fn handle_agent_turn(ctx: &RuntimeContext, args: AgentTurnArgs) -> Result<()> { let db = open_db(ctx)?; let (selector, turn) = resolve_agent_turn_cli_args(&db, &args); let report = db.agent_turn(&selector, &turn, args.file.as_deref(), args.patch)?; - render_agent_turn(&report, ctx.json, ctx.quiet) + render_agent_turn(&report, ctx.json, &ctx.render) } fn handle_agent_turn_diff(ctx: &RuntimeContext, args: AgentTurnDiffArgs) -> Result<()> { @@ -1976,7 +3302,7 @@ fn handle_agent_turn_diff(ctx: &RuntimeContext, args: AgentTurnDiffArgs) -> Resu args.file.as_deref(), args.patch, )?; - render_agent_diff(&report, ctx.json, ctx.quiet, args.stat) + render_agent_diff(&report, ctx.json, &ctx.render, args.stat) } fn resolve_agent_turn_cli_args(db: &trail::Trail, args: &AgentTurnArgs) -> (String, String) { @@ -2005,19 +3331,19 @@ fn handle_agent_diff(ctx: &RuntimeContext, args: AgentDiffArgs) -> Result<()> { args.file.as_deref(), args.patch, )?; - render_agent_diff(&report, ctx.json, ctx.quiet, args.stat) + render_agent_diff(&report, ctx.json, &ctx.render, args.stat) } fn handle_agent_review(ctx: &RuntimeContext, args: AgentSelectorArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_review(&args.selector)?; - render_agent_review(&report, ctx.json, ctx.quiet) + render_agent_review(&report, ctx.json, &ctx.render) } fn handle_agent_focus(ctx: &RuntimeContext, args: AgentFocusArgs) -> Result<()> { let db = open_db(ctx)?; let report = db.agent_focus(&args.selector, args.file.as_deref(), args.patch)?; - render_agent_focus(&report, ctx.json, ctx.quiet) + render_agent_focus(&report, ctx.json, &ctx.render) } fn handle_agent_open(ctx: &RuntimeContext, args: AgentOpenArgs) -> Result<()> { @@ -2036,12 +3362,10 @@ fn handle_agent_open(ctx: &RuntimeContext, args: AgentOpenArgs) -> Result<()> { )) })?; if ctx.json { - return render_agent_focus(&report, true, ctx.quiet); + return render_agent_focus(&report, true, &ctx.render); } if args.print { - if !ctx.quiet { - println!("{open_command}"); - } + render_raw_content(&format!("{open_command}\n"), &ctx.render)?; return Ok(()); } let shell = std::env::var_os("SHELL") @@ -2064,9 +3388,11 @@ fn handle_agent_open(ctx: &RuntimeContext, args: AgentOpenArgs) -> Result<()> { .unwrap_or_else(|| "terminated by signal".to_string()) ))); } - if !ctx.quiet { - println!("Opened: {open_path}"); - } + render_document( + &TerminalDocument::new("Opened agent focus", UiTone::Success) + .block(UiBlock::Metadata(vec![("Path".to_string(), open_path)])), + &ctx.render, + )?; Ok(()) } @@ -2074,9 +3400,9 @@ fn handle_agent_apply(ctx: &RuntimeContext, args: AgentApplyArgs) -> Result<()> let _ = args.into_current_git_branch; let mut db = open_db(ctx)?; match db.agent_apply(&args.selector, args.dry_run, args.message) { - Ok(report) => render_agent_apply(&report, ctx.json, ctx.quiet), + Ok(report) => render_agent_apply(&report, ctx.json, &ctx.render), Err(Error::InvalidInput(message)) if message.contains("no agent tasks") => { - render_agent_empty_task_hint("apply", ctx.json, ctx.quiet) + render_agent_empty_task_hint("apply", ctx.json, &ctx.render) } Err(err) => Err(err), } @@ -2086,13 +3412,13 @@ fn handle_agent_finish(ctx: &RuntimeContext, args: AgentFinishArgs) -> Result<() let _ = args.into_current_git_branch; let mut db = open_db(ctx)?; let report = db.agent_finish(&args.selector, args.dry_run, args.message, args.note)?; - render_agent_finish(&report, ctx.json, ctx.quiet) + render_agent_finish(&report, ctx.json, &ctx.render) } fn handle_agent_rewind(ctx: &RuntimeContext, args: AgentRewindArgs) -> Result<()> { let mut db = open_db(ctx)?; let report = db.agent_rewind(&args.selector, &args.target)?; - render_lane_rewind(&report, ctx.json, ctx.quiet) + render_lane_rewind(&report, ctx.json, &ctx.render) } fn handle_agent_undo(ctx: &RuntimeContext, args: AgentUndoArgs) -> Result<()> { @@ -2104,10 +3430,12 @@ fn handle_agent_undo(ctx: &RuntimeContext, args: AgentUndoArgs) -> Result<()> { args.prompt.as_deref(), args.last_operation, )?; - render_lane_rewind(&report, ctx.json, ctx.quiet) + render_lane_rewind(&report, ctx.json, &ctx.render) } fn handle_agent_doctor(ctx: &RuntimeContext, args: AgentDoctorArgs) -> Result<()> { + let provider = + resolve_agent_provider_argument(args.provider, args.provider_flag, Some("claude-code"))?; let mut checks = Vec::new(); let mut status = "ok"; let mut workspace_ok = true; @@ -2127,7 +3455,7 @@ fn handle_agent_doctor(ctx: &RuntimeContext, args: AgentDoctorArgs) -> Result<() })); } } - let profile = trail::acp::agent_provider_profile(&args.provider)?; + let profile = trail::acp::agent_provider_profile(&provider)?; checks.push(serde_json::json!({ "name": "provider", "status": "ok", @@ -2202,27 +3530,20 @@ fn handle_agent_doctor(ctx: &RuntimeContext, args: AgentDoctorArgs) -> Result<() if workspace_ok && !launch_ok { status = "failed"; } - let setup_command = if profile.agent == "claude-code" { - "trail agent setup".to_string() - } else { - format!( - "trail agent setup --provider {} --editor vscode", - profile.agent - ) - }; + let setup_command = format!("trail agent acp setup {} --editor vscode", profile.agent); let mut suggestions = vec![serde_json::json!({ "command": setup_command, "reason": "print the recommended Trail setup for this provider" })]; if profile.supports_terminal { suggestions.push(serde_json::json!({ - "command": format!("trail agent start --provider {}", profile.agent), + "command": format!("trail agent start {}", profile.agent), "reason": "launch a fresh materialized task lane from the terminal" })); } if profile.supports_acp { suggestions.push(serde_json::json!({ - "command": format!("trail acp doctor --agent {}", profile.agent), + "command": format!("trail agent acp doctor {}", profile.agent), "reason": "check the lower-level ACP relay command" })); } @@ -2249,24 +3570,87 @@ fn handle_agent_doctor(ctx: &RuntimeContext, args: AgentDoctorArgs) -> Result<() if ctx.json { return render_json(&report); } - if !ctx.quiet { - println!("Agent doctor: {status}"); - for check in checks { - println!( - "[{}] {}: {}", - check["status"].as_str().unwrap_or("-"), - check["name"].as_str().unwrap_or("-"), - check["message"].as_str().unwrap_or("") - ); - } - } - Ok(()) + let check_rows = checks + .iter() + .map(|check| { + UiCheck::new( + match check["status"].as_str().unwrap_or_default() { + "ok" | "pass" => UiCheckState::Pass, + "warn" | "warning" => UiCheckState::Warn, + "pending" => UiCheckState::Pending, + _ => UiCheckState::Fail, + }, + check["name"].as_str().unwrap_or("check"), + check["message"].as_str().unwrap_or_default(), + ) + }) + .collect(); + render_document( + &TerminalDocument::new( + format!("Agent diagnostics: {status}"), + if status == "ok" { + UiTone::Success + } else { + UiTone::Attention + }, + ) + .block(UiBlock::Checklist(check_rows)), + &ctx.render, + ) } fn default_terminal_agent_command(provider: &str) -> Result> { trail::acp::terminal_agent_command(provider) } +fn confined_terminal_agent_command( + command: &[String], + workspace_root: &Path, + workdir: &Path, +) -> Result<(std::ffi::OsString, Vec)> { + #[cfg(target_os = "macos")] + { + let sandbox = PathBuf::from("/usr/bin/sandbox-exec"); + if !sandbox.is_file() { + return Err(Error::InvalidInput( + "isolated terminal agents require `/usr/bin/sandbox-exec` on macOS".to_string(), + )); + } + let workspace_root = workspace_root.canonicalize()?; + let workdir = workdir.canonicalize()?; + let trail_internal = workspace_root.join(".trail").canonicalize()?; + let escape = |path: &Path| { + path.to_string_lossy() + .replace('\\', "\\\\") + .replace('"', "\\\"") + }; + let profile = format!( + "(version 1)\n\ + (allow default)\n\ + (deny file-write* (subpath \"{}\"))\n\ + (allow file-write* (subpath \"{}\") (subpath \"{}\"))", + escape(&workspace_root), + escape(&trail_internal), + escape(&workdir), + ); + let mut args = vec![ + std::ffi::OsString::from("-p"), + std::ffi::OsString::from(profile), + std::ffi::OsString::from(&command[0]), + ]; + args.extend(command[1..].iter().map(std::ffi::OsString::from)); + Ok((sandbox.into_os_string(), args)) + } + #[cfg(not(target_os = "macos"))] + { + let _ = (workspace_root, workdir); + Ok(( + std::ffi::OsString::from(&command[0]), + command[1..].iter().map(std::ffi::OsString::from).collect(), + )) + } +} + fn command_available(command: &str) -> bool { if command.contains(std::path::MAIN_SEPARATOR) { return std::path::Path::new(command).is_file(); diff --git a/trail/src/cli/command/handler/collaboration.rs b/trail/src/cli/command/handler/collaboration.rs index fb54ffc..502a466 100644 --- a/trail/src/cli/command/handler/collaboration.rs +++ b/trail/src/cli/command/handler/collaboration.rs @@ -1,42 +1,34 @@ use super::*; -pub(super) fn handle_merge_lane_command(ctx: &RuntimeContext, args: MergeLaneArgs) -> Result<()> { - let mut db = open_db(ctx)?; - validate_merge_strategy(args.strategy.as_deref())?; - let report = - db.merge_lane_user_with_options(&args.name, &args.into, args.dry_run, args.direct)?; - render_merge(&report, ctx.json, ctx.quiet) -} - -pub(super) fn handle_merge_queue_command( +pub(super) fn handle_lane_merge_queue_command( ctx: &RuntimeContext, - queue: MergeQueueCommand, + queue: LaneMergeQueueCommand, ) -> Result<()> { match queue.command { - MergeQueueSubcommand::Add(args) => { + LaneMergeQueueSubcommand::Add(args) => { let mut db = open_db(ctx)?; - let report = db.enqueue_merge(&args.source, &args.into, args.priority)?; - render_merge_queue_add(&report, ctx.json, ctx.quiet) + let report = db.enqueue_lane_merge(&args.lane, &args.into, args.priority)?; + render_lane_merge_queue_add(&report, ctx.json, &ctx.render) } - MergeQueueSubcommand::List => { + LaneMergeQueueSubcommand::List => { let db = open_db(ctx)?; - let entries = db.list_merge_queue()?; - render_merge_queue_list(&entries, ctx.json, ctx.quiet) + let entries = db.list_lane_merge_queue()?; + render_lane_merge_queue_list(&entries, ctx.json, &ctx.render) } - MergeQueueSubcommand::Explain(args) => { + LaneMergeQueueSubcommand::Explain(args) => { let mut db = open_db(ctx)?; - let report = db.explain_merge_queue(&args.selector)?; - render_merge_queue_explain(&report, ctx.json, ctx.quiet) + let report = db.explain_lane_merge_queue(&args.selector)?; + render_lane_merge_queue_explain(&report, ctx.json, &ctx.render) } - MergeQueueSubcommand::Run(args) => { + LaneMergeQueueSubcommand::Run(args) => { let mut db = open_db(ctx)?; - let report = db.run_merge_queue(args.limit)?; - render_merge_queue_run(&report, ctx.json, ctx.quiet) + let report = db.run_lane_merge_queue(args.limit)?; + render_lane_merge_queue_run(&report, ctx.json, &ctx.render) } - MergeQueueSubcommand::Remove(args) => { + LaneMergeQueueSubcommand::Remove(args) => { let mut db = open_db(ctx)?; - let report = db.remove_merge_queue(&args.selector)?; - render_merge_queue_remove(&report, ctx.json, ctx.quiet) + let report = db.remove_lane_merge_queue(&args.selector)?; + render_lane_merge_queue_remove(&report, ctx.json, &ctx.render) } } } @@ -49,12 +41,12 @@ pub(super) fn handle_conflicts_command( ConflictsSubcommand::List => { let db = open_db(ctx)?; let conflicts = db.list_conflicts()?; - render_conflicts(&conflicts, ctx.json, ctx.quiet) + render_conflicts(&conflicts, ctx.json, &ctx.render) } ConflictsSubcommand::Show(args) => { let db = open_db(ctx)?; let conflict = db.show_conflict_with_limit(&args.conflict_set_id, args.limit)?; - render_conflict(&conflict, ctx.json, ctx.quiet) + render_conflict(&conflict, ctx.json, &ctx.render) } ConflictsSubcommand::Resolve(args) => { let mut db = open_db(ctx)?; @@ -68,7 +60,7 @@ pub(super) fn handle_conflicts_command( "conflicts resolve requires `--take` or `--manual`".to_string(), )); }; - render_conflict_resolve(&report, ctx.json, ctx.quiet) + render_conflict_resolve(&report, ctx.json, &ctx.render) } } } @@ -78,22 +70,22 @@ pub(super) fn handle_anchor_command(ctx: &RuntimeContext, anchor: AnchorCommand) AnchorSubcommand::Create(args) => { let mut db = open_db(ctx)?; let report = db.create_anchor(&args.path_line, args.label, ctx.branch.as_deref())?; - render_anchor_create(&report, ctx.json, ctx.quiet) + render_anchor_create(&report, ctx.json, &ctx.render) } AnchorSubcommand::Resolve(args) => { let db = open_db(ctx)?; let report = db.resolve_anchor(&args.anchor_id, ctx.branch.as_deref())?; - render_anchor_resolve(&report, ctx.json, ctx.quiet) + render_anchor_resolve(&report, ctx.json, &ctx.render) } AnchorSubcommand::List => { let db = open_db(ctx)?; let anchors = db.list_anchors()?; - render_anchor_list(&anchors, ctx.json, ctx.quiet) + render_anchor_list(&anchors, ctx.json, &ctx.render) } AnchorSubcommand::Delete(args) => { let mut db = open_db(ctx)?; let report = db.delete_anchor(&args.anchor_id)?; - render_anchor_delete(&report, ctx.json, ctx.quiet) + render_anchor_delete(&report, ctx.json, &ctx.render) } } } @@ -108,17 +100,17 @@ pub(super) fn handle_lease_command(ctx: &RuntimeContext, lease: LeaseCommand) -> args.mode.as_str(), args.ttl_secs, )?; - render_lease_acquire(&report, ctx.json, ctx.quiet) + render_lease_acquire(&report, ctx.json, &ctx.render) } LeaseSubcommand::List(args) => { let db = open_db(ctx)?; let leases = db.list_leases(args.all)?; - render_lease_list(&leases, ctx.json, ctx.quiet) + render_lease_list(&leases, ctx.json, &ctx.render) } LeaseSubcommand::Release(args) => { let mut db = open_db(ctx)?; let report = db.release_lease(&args.lease_id)?; - render_lease_release(&report, ctx.json, ctx.quiet) + render_lease_release(&report, ctx.json, &ctx.render) } } } @@ -128,32 +120,32 @@ pub(super) fn handle_session_command(ctx: &RuntimeContext, session: SessionComma SessionSubcommand::Start(args) => { let mut db = open_db(ctx)?; let report = db.start_lane_session(&args.lane, args.title, args.id)?; - render_session_start(&report, ctx.json, ctx.quiet) + render_session_start(&report, ctx.json, &ctx.render) } SessionSubcommand::Current(args) => { let db = open_db(ctx)?; let reports = db.current_lane_sessions(args.lane.as_deref())?; - render_session_current(&reports, ctx.json, ctx.quiet) + render_session_current(&reports, ctx.json, &ctx.render) } SessionSubcommand::List(args) => { let db = open_db(ctx)?; let sessions = db.list_lane_sessions(args.lane.as_deref())?; - render_session_list(&sessions, ctx.json, ctx.quiet) + render_session_list(&sessions, ctx.json, &ctx.render) } SessionSubcommand::Show(args) => { let db = open_db(ctx)?; let details = db.show_lane_session(&args.session_id)?; - render_session_details(&details, ctx.json, ctx.quiet) + render_session_details(&details, ctx.json, &ctx.render) } SessionSubcommand::Context(args) => { let db = open_db(ctx)?; let report = db.lane_session_context(&args.session_id, args.limit)?; - render_session_context(&report, ctx.json, ctx.quiet) + render_session_context(&report, ctx.json, &ctx.render) } SessionSubcommand::End(args) => { let mut db = open_db(ctx)?; let report = db.end_lane_session(&args.session_id, &args.status)?; - render_session_end(&report, ctx.json, ctx.quiet) + render_session_end(&report, ctx.json, &ctx.render) } } } @@ -177,17 +169,17 @@ pub(super) fn handle_approvals_command( args.session.as_deref(), args.turn.as_deref(), )?; - render_approval_request(&report, ctx.json, ctx.quiet) + render_approval_request(&report, ctx.json, &ctx.render) } ApprovalsSubcommand::List(args) => { let db = open_db(ctx)?; let approvals = db.list_lane_approvals(args.lane.as_deref(), args.status.as_deref())?; - render_approval_list(&approvals, ctx.json, ctx.quiet) + render_approval_list(&approvals, ctx.json, &ctx.render) } ApprovalsSubcommand::Show(args) => { let db = open_db(ctx)?; let approval = db.show_lane_approval(&args.approval_id)?; - render_approval(&approval, ctx.json, ctx.quiet) + render_approval(&approval, ctx.json, &ctx.render) } ApprovalsSubcommand::Decide(args) => { let mut db = open_db(ctx)?; @@ -197,7 +189,7 @@ pub(super) fn handle_approvals_command( args.reviewer, args.note, )?; - render_approval_decision(&report, ctx.json, ctx.quiet) + render_approval_decision(&report, ctx.json, &ctx.render) } } } diff --git a/trail/src/cli/command/handler/daemon_rpc.rs b/trail/src/cli/command/handler/daemon_rpc.rs index ae70d19..739e134 100644 --- a/trail/src/cli/command/handler/daemon_rpc.rs +++ b/trail/src/cli/command/handler/daemon_rpc.rs @@ -1,5 +1,7 @@ use std::io::{ErrorKind, Read, Write}; use std::net::{IpAddr, SocketAddr, TcpStream}; +#[cfg(unix)] +use std::os::unix::net::UnixStream; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -10,6 +12,11 @@ use trail::model::*; use super::*; +const PERFORMANCE_METRICS_FILE_ENV: &str = "TRAIL_PERFORMANCE_METRICS_FILE"; +const OPERATION_METRICS_HEADER: &str = "x-trail-operation-metrics"; +const DAEMON_READ_REQUEST_TIMEOUT: Duration = Duration::from_secs(120); +const DAEMON_MUTATING_REQUEST_TIMEOUT: Duration = Duration::from_secs(15 * 60); + pub(super) fn try_handle_auto_daemon_command( ctx: &RuntimeContext, daemon_token: Option, @@ -18,6 +25,77 @@ pub(super) fn try_handle_auto_daemon_command( if !daemon_supports_command(command) { return Ok(false); } + if matches!( + command, + Command::Status(_) + | Command::Diff(_) + | Command::Record(_) + | Command::Index(IndexCommand { + command: IndexSubcommand::Reconcile(_), + }) + ) || matches!( + command, + Command::Lane(LaneCommand { + command: LaneSubcommand::Status(_) + | LaneSubcommand::Record(_) + | LaneSubcommand::ApplyPatch(_) + | LaneSubcommand::Diff(_), + }) + ) { + let workspace = daemon_start::workspace_from_context(ctx)?; + if !workspace.join(".trail").is_dir() { + return Err(Error::WorkspaceNotFound(workspace)); + } + let ready = + daemon_start::ensure_workspace_daemon_ready(&workspace, daemon_token.as_deref())?; + let result = try_handle_daemon_command( + ctx, + Some(ready.url.clone()), + Some(ready.auth_token.clone()), + command, + ); + match result { + Ok(handled) => return Ok(handled), + Err(error) => { + if daemon_command_requires_one_recovery_retry(&error) { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + match daemon_start::ensure_workspace_daemon_ready( + &workspace, + daemon_token.as_deref(), + ) { + Ok(recovered) => { + return try_handle_daemon_command( + ctx, + Some(recovered.url), + Some(recovered.auth_token), + command, + ); + } + Err(_) if std::time::Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(_) => return Err(error), + } + } + } + return Err(error); + } + } + } + let workspace = daemon_start::workspace_from_context(ctx)?; + if workspace.join(".trail").is_dir() { + if let Some(ready) = + daemon_start::existing_workspace_daemon_ready(&workspace, daemon_token.as_deref())? + { + return try_handle_daemon_command( + ctx, + Some(ready.url), + Some(ready.auth_token), + command, + ); + } + } let Some(daemon_url) = discover_daemon_url(ctx)? else { return Ok(false); }; @@ -28,6 +106,13 @@ pub(super) fn try_handle_auto_daemon_command( } } +fn daemon_command_requires_one_recovery_retry(error: &Error) -> bool { + matches!( + error, + Error::ChangeLedgerReconcileRequired { .. } | Error::DaemonError { exit_code: 16, .. } + ) +} + pub(super) fn try_handle_daemon_command( ctx: &RuntimeContext, daemon_url: Option, @@ -49,13 +134,21 @@ pub(super) fn try_handle_daemon_command( return Ok(false); } let report: StatusReport = client.get_json("/v1/status")?; - render_status(&report, ctx.json, ctx.quiet)?; + render_status(&report, ctx.json, &ctx.render)?; Ok(true) } Command::Diff(args) => { let path = diff_path(args)?; let summary: DiffSummary = client.get_json(&path)?; - render_diff(&summary, ctx.json, ctx.quiet, args.stat, ctx.color)?; + render_diff( + &summary, + ctx.json, + &ctx.render, + args.patch, + args.stat, + args.name_only, + args.name_status, + )?; Ok(true) } Command::Record(args) => { @@ -68,7 +161,20 @@ pub(super) fn try_handle_daemon_command( "allow_ignored": args.allow_ignored, }); let report: RecordReport = client.post_json("/v1/record", &body)?; - render_record(&report, ctx.json, ctx.quiet)?; + render_record(&report, ctx.json, &ctx.render)?; + Ok(true) + } + Command::Index(IndexCommand { + command: IndexSubcommand::Reconcile(args), + }) => { + let body = serde_json::json!({ "lane": args.lane }); + let report: ChangeLedgerReconcileReport = + client.post_json("/v1/index/reconcile", &body)?; + if matches!(ctx.format, OutputFormat::Ndjson) { + render_ndjson(&report)?; + } else { + render_change_ledger_reconcile(&report, ctx.json, &ctx.render)?; + } Ok(true) } Command::Timeline(args) => handle_timeline_command(ctx, &client, args), @@ -78,24 +184,10 @@ pub(super) fn try_handle_daemon_command( Command::Lane(lane) => handle_lane_command(ctx, &client, lane), Command::Session(session) => handle_session_command(ctx, &client, session), Command::Approvals(approvals) => handle_approvals_command(ctx, &client, approvals), - Command::MergeLane(args) => { - validate_merge_strategy(args.strategy.as_deref())?; - let body = serde_json::json!({ - "lane_id": args.name, - "strategy": args.strategy, - "dry_run": args.dry_run, - "direct": args.direct, - }); - let report: MergeReport = - client.post_json(&format!("/v1/branches/{}/merge-lane", args.into), &body)?; - render_merge(&report, ctx.json, ctx.quiet)?; - Ok(true) - } - Command::MergeQueue(queue) => handle_merge_queue_command(ctx, &client, queue), Command::Lease(lease) => handle_lease_command(ctx, &client, lease), Command::Doctor => { let report: DoctorReport = client.get_json("/v1/doctor")?; - render_doctor(&report, ctx.json, ctx.quiet)?; + render_doctor(&report, ctx.json, &ctx.render)?; Ok(true) } _ => Ok(false), @@ -106,6 +198,9 @@ fn daemon_supports_command(command: &Command) -> bool { match command { Command::Status(args) => args.branch.is_none(), Command::Record(_) + | Command::Index(IndexCommand { + command: IndexSubcommand::Reconcile(_), + }) | Command::Diff(_) | Command::Timeline(_) | Command::Why(_) @@ -113,19 +208,18 @@ fn daemon_supports_command(command: &Command) -> bool { | Command::CodeFrom(_) | Command::Session(_) | Command::Approvals(_) - | Command::MergeLane(_) - | Command::MergeQueue(_) | Command::Lease(_) | Command::Doctor => true, Command::Lane(lane) => match &lane.command { - LaneSubcommand::Spawn(_) - | LaneSubcommand::List + LaneSubcommand::List | LaneSubcommand::Show(_) | LaneSubcommand::Status(_) | LaneSubcommand::Review(_) | LaneSubcommand::Contribution(_) | LaneSubcommand::Gates(_) | LaneSubcommand::Readiness(_) + | LaneSubcommand::Merge(_) + | LaneSubcommand::MergeQueue(_) | LaneSubcommand::RefreshPreview(_) | LaneSubcommand::Handoff(_) | LaneSubcommand::Claim(_) @@ -167,6 +261,12 @@ fn handle_lane_command( } else if let Some(materialize) = args.materialize { body.insert("materialize".to_string(), Value::Bool(materialize)); } + if let Some(workdir_mode) = &args.workdir_mode { + body.insert( + "workdir_mode".to_string(), + Value::String(workdir_mode.clone()), + ); + } if let Some(workdir) = &args.workdir { body.insert( "workdir".to_string(), @@ -189,23 +289,23 @@ fn handle_lane_command( body.insert("model".to_string(), Value::String(model.clone())); } let report: LaneSpawnReport = client.post_json("/v1/lanes", &Value::Object(body))?; - render_lane_spawn(&report, ctx.json, ctx.quiet)?; + render_lane_spawn(&report, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::List => { let lanes: Vec = client.get_json("/v1/lanes")?; - render_lane_list(&lanes, ctx.json, ctx.quiet)?; + render_lane_list(&lanes, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Show(args) => { let details: LaneDetails = client.get_json(&format!("/v1/lanes/{}", args.name))?; - render_lane_details(&details, ctx.json, ctx.quiet)?; + render_lane_details(&details, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Status(args) => { let report: LaneStatusReport = client.get_json(&format!("/v1/lanes/{}/status", args.name))?; - render_lane_status(&report, ctx.json, ctx.quiet)?; + render_lane_status(&report, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Review(args) => { @@ -213,7 +313,7 @@ fn handle_lane_command( "/v1/lanes/{}/review?limit={}", args.name, args.limit ))?; - render_lane_review_packet(&report, ctx.json, ctx.quiet)?; + render_lane_review_packet(&report, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Contribution(args) => { @@ -221,7 +321,7 @@ fn handle_lane_command( "/v1/lanes/{}/contribution?limit={}", args.name, args.limit ))?; - render_lane_contribution(&report, ctx.json, ctx.quiet)?; + render_lane_contribution(&report, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Gates(args) => { @@ -231,22 +331,36 @@ fn handle_lane_command( } let path = append_query(&format!("/v1/lanes/{}/gates", args.name), params); let report: LaneGateHistoryReport = client.get_json(&path)?; - render_lane_gate_history(&report, ctx.json, ctx.quiet)?; + render_lane_gate_history(&report, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Readiness(args) => { let report: LaneReadinessReport = client.get_json(&format!("/v1/lanes/{}/readiness", args.name))?; - render_lane_readiness(&report, ctx.json, ctx.quiet)?; + render_lane_readiness(&report, ctx.json, &ctx.render)?; + Ok(true) + } + LaneSubcommand::Merge(args) => { + validate_merge_strategy(args.strategy.as_deref())?; + let body = serde_json::json!({ + "into": args.into, + "strategy": args.strategy, + "dry_run": args.dry_run, + "direct": args.direct, + }); + let report: MergeReport = + client.post_json(&format!("/v1/lanes/{}/merge", args.name), &body)?; + render_merge(&report, ctx.json, &ctx.render)?; Ok(true) } + LaneSubcommand::MergeQueue(queue) => handle_lane_merge_queue_command(ctx, client, queue), LaneSubcommand::RefreshPreview(args) => { let path = append_query( &format!("/v1/lanes/{}/refresh-preview", args.name), vec![format!("target={}", args.target)], ); let report: LaneRefreshPreviewReport = client.get_json(&path)?; - render_lane_refresh_preview(&report, ctx.json, ctx.quiet)?; + render_lane_refresh_preview(&report, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Handoff(args) => { @@ -254,7 +368,7 @@ fn handle_lane_command( "/v1/lanes/{}/handoff?limit={}", args.name, args.limit ))?; - render_lane_handoff(&report, ctx.json, ctx.quiet)?; + render_lane_handoff(&report, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Claim(args) => { @@ -264,7 +378,7 @@ fn handle_lane_command( }); let report: LaneClaimReport = client.post_json(&format!("/v1/lanes/{}/claims", args.name), &body)?; - render_lane_claim(&report, ctx.json, ctx.quiet)?; + render_lane_claim(&report, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Record(args) => { @@ -275,11 +389,11 @@ fn handle_lane_command( if args.preview { let report: LaneRecordPreviewReport = client.post_json(&format!("/v1/lanes/{}/record", args.name), &body)?; - render_lane_record_preview(&report, ctx.json, ctx.quiet)?; + render_lane_record_preview(&report, ctx.json, &ctx.render)?; } else { let report: LaneRecordReport = client.post_json(&format!("/v1/lanes/{}/record", args.name), &body)?; - render_lane_record(&report, ctx.json, ctx.quiet)?; + render_lane_record(&report, ctx.json, &ctx.render)?; } Ok(true) } @@ -291,7 +405,7 @@ fn handle_lane_command( }); let report: LaneRewindReport = client.post_json(&format!("/v1/lanes/{}/rewind", args.name), &body)?; - render_lane_rewind(&report, ctx.json, ctx.quiet)?; + render_lane_rewind(&report, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Events(args) => { @@ -310,7 +424,7 @@ fn handle_lane_command( } let path = append_query("/v1/lane/events", params); let events: Vec = client.get_json(&path)?; - render_lane_events(&events, ctx.json, ctx.quiet)?; + render_lane_events(&events, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::SyncWorkdir(args) => { @@ -321,7 +435,7 @@ fn handle_lane_command( }); let report: LaneWorkdirSyncReport = client.post_json(&format!("/v1/lanes/{}/sync-workdir", args.name), &body)?; - render_lane_workdir_sync(&report, ctx.json, ctx.quiet)?; + render_lane_workdir_sync(&report, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Read(args) => { @@ -341,13 +455,13 @@ fn handle_lane_command( &format!("/v1/lanes/{}/read-file", args.name), &Value::Object(body), )?; - render_lane_file_read(&report, ctx.json, ctx.quiet)?; + render_lane_file_read(&report, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Workdir(args) => { let report: LaneWorkdirReport = client.get_json(&format!("/v1/lanes/{}/workdir", args.name))?; - render_lane_workdir(&report, ctx.json, ctx.quiet)?; + render_lane_workdir(&report, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::ApplyPatch(args) => { @@ -362,10 +476,17 @@ fn handle_lane_command( let body = serde_json::to_value(&patch)?; let report: LanePatchReport = client.post_json(&format!("/v1/lanes/{}/patches", args.name), &body)?; - render_lane_patch(&report, ctx.json, ctx.quiet)?; + render_lane_patch(&report, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Diff(args) => { + validate_diff_view( + args.patch, + args.stat, + args.show_line_ids, + args.name_only, + args.name_status, + )?; let mut params = Vec::new(); if args.patch { params.push("patch=1".to_string()); @@ -379,9 +500,11 @@ fn handle_lane_command( render_diff_with_title( &summary, ctx.json, - ctx.quiet, + &ctx.render, + args.patch, args.stat, - ctx.color, + args.name_only, + args.name_status, Some(&title), )?; Ok(true) @@ -395,7 +518,7 @@ fn handle_lane_command( ], ); let entries: Vec = client.get_json(&path)?; - render_timeline(&entries, ctx.json, ctx.quiet)?; + render_timeline(&entries, ctx.json, &ctx.render)?; Ok(true) } LaneSubcommand::Turn(turn) => handle_lane_turn_command(ctx, client, turn), @@ -420,7 +543,7 @@ fn handle_timeline_command( params.push(format!("lane={lane}")); } let entries: Vec = client.get_json(&append_query("/v1/timeline", params))?; - render_timeline(&entries, ctx.json, ctx.quiet)?; + render_timeline(&entries, ctx.json, &ctx.render)?; Ok(true) } @@ -444,7 +567,7 @@ fn handle_why_command(ctx: &RuntimeContext, client: &DaemonClient, args: &WhyArg params.push(format!("at={at}")); } let result: WhyResult = client.get_json(&append_query("/v1/why", params))?; - render_why(&result, ctx.json, ctx.quiet)?; + render_why(&result, ctx.json, &ctx.render)?; Ok(true) } @@ -473,7 +596,7 @@ fn handle_history_command( } }; let result: HistoryResult = client.get_json(&append_query("/v1/history", params))?; - render_history(&result, ctx.json, ctx.quiet)?; + render_history(&result, ctx.json, &ctx.render)?; Ok(true) } @@ -486,7 +609,7 @@ fn handle_code_from_command( "/v1/code-from", vec![format!("selector={}", args.selector)], ))?; - render_code_from(&result, ctx.json, ctx.quiet)?; + render_code_from(&result, ctx.json, &ctx.render)?; Ok(true) } @@ -503,7 +626,7 @@ fn handle_session_command( "id": args.id, }); let report: LaneSessionStartReport = client.post_json("/v1/sessions", &body)?; - render_session_start(&report, ctx.json, ctx.quiet)?; + render_session_start(&report, ctx.json, &ctx.render)?; Ok(true) } SessionSubcommand::Current(args) => { @@ -512,7 +635,7 @@ fn handle_session_command( None => "/v1/sessions/current".to_string(), }; let reports: Vec = client.get_json(&path)?; - render_session_current(&reports, ctx.json, ctx.quiet)?; + render_session_current(&reports, ctx.json, &ctx.render)?; Ok(true) } SessionSubcommand::List(args) => { @@ -521,13 +644,13 @@ fn handle_session_command( None => "/v1/sessions".to_string(), }; let sessions: Vec = client.get_json(&path)?; - render_session_list(&sessions, ctx.json, ctx.quiet)?; + render_session_list(&sessions, ctx.json, &ctx.render)?; Ok(true) } SessionSubcommand::Show(args) => { let details: LaneSessionDetails = client.get_json(&format!("/v1/sessions/{}", args.session_id))?; - render_session_details(&details, ctx.json, ctx.quiet)?; + render_session_details(&details, ctx.json, &ctx.render)?; Ok(true) } SessionSubcommand::Context(args) => { @@ -536,7 +659,7 @@ fn handle_session_command( vec![format!("limit={}", args.limit)], ); let report: LaneSessionContextReport = client.get_json(&path)?; - render_session_context(&report, ctx.json, ctx.quiet)?; + render_session_context(&report, ctx.json, &ctx.render)?; Ok(true) } SessionSubcommand::End(args) => { @@ -545,7 +668,7 @@ fn handle_session_command( }); let report: LaneSessionEndReport = client.post_json(&format!("/v1/sessions/{}/end", args.session_id), &body)?; - render_session_end(&report, ctx.json, ctx.quiet)?; + render_session_end(&report, ctx.json, &ctx.render)?; Ok(true) } } @@ -573,7 +696,7 @@ fn handle_approvals_command( } let report: LaneApprovalRequestReport = client.post_json("/v1/approvals", &Value::Object(body))?; - render_approval_request(&report, ctx.json, ctx.quiet)?; + render_approval_request(&report, ctx.json, &ctx.render)?; Ok(true) } ApprovalsSubcommand::List(args) => { @@ -586,13 +709,13 @@ fn handle_approvals_command( } let approvals: Vec = client.get_json(&append_query("/v1/approvals", params))?; - render_approval_list(&approvals, ctx.json, ctx.quiet)?; + render_approval_list(&approvals, ctx.json, &ctx.render)?; Ok(true) } ApprovalsSubcommand::Show(args) => { let approval: LaneApproval = client.get_json(&format!("/v1/approvals/{}", args.approval_id))?; - render_approval(&approval, ctx.json, ctx.quiet)?; + render_approval(&approval, ctx.json, &ctx.render)?; Ok(true) } ApprovalsSubcommand::Decide(args) => { @@ -605,7 +728,7 @@ fn handle_approvals_command( &format!("/v1/approvals/{}/decision", args.approval_id), &body, )?; - render_approval_decision(&report, ctx.json, ctx.quiet)?; + render_approval_decision(&report, ctx.json, &ctx.render)?; Ok(true) } } @@ -634,13 +757,13 @@ fn handle_lane_turn_command( } let report: LaneTurnStartReport = client.post_json("/v1/lane/turns", &Value::Object(body))?; - render_lane_turn_start(&report, ctx.json, ctx.quiet)?; + render_lane_turn_start(&report, ctx.json, &ctx.render)?; Ok(true) } LaneTurnSubcommand::Show(args) => { let details: LaneTurnDetails = client.get_json(&format!("/v1/lane/turns/{}", args.turn_id))?; - render_lane_turn_details(&details, ctx.json, ctx.quiet)?; + render_lane_turn_details(&details, ctx.json, &ctx.render)?; Ok(true) } LaneTurnSubcommand::Message(args) => { @@ -650,7 +773,7 @@ fn handle_lane_turn_command( }); let report: LaneMessageReport = client.post_json(&format!("/v1/lane/turns/{}/messages", args.turn_id), &body)?; - render_lane_message(&report, ctx.json, ctx.quiet)?; + render_lane_message(&report, ctx.json, &ctx.render)?; Ok(true) } LaneTurnSubcommand::Event(args) => { @@ -672,7 +795,7 @@ fn handle_lane_turn_command( &format!("/v1/lane/turns/{}/events", args.turn_id), &Value::Object(body), )?; - render_lane_turn_event(&report, ctx.json, ctx.quiet)?; + render_lane_turn_event(&report, ctx.json, &ctx.render)?; Ok(true) } LaneTurnSubcommand::ApplyPatch(args) => { @@ -687,7 +810,7 @@ fn handle_lane_turn_command( let body = serde_json::to_value(&patch)?; let report: LanePatchReport = client.post_json(&format!("/v1/lane/turns/{}/patches", args.turn_id), &body)?; - render_lane_patch(&report, ctx.json, ctx.quiet)?; + render_lane_patch(&report, ctx.json, &ctx.render)?; Ok(true) } LaneTurnSubcommand::End(args) => { @@ -696,7 +819,7 @@ fn handle_lane_turn_command( }); let report: LaneTurnEndReport = client.post_json(&format!("/v1/lane/turns/{}/end", args.turn_id), &body)?; - render_lane_turn_end(&report, ctx.json, ctx.quiet)?; + render_lane_turn_end(&report, ctx.json, &ctx.render)?; Ok(true) } } @@ -728,7 +851,7 @@ fn handle_lane_trace_command( &format!("/v1/lane/turns/{}/spans", args.turn_id), &Value::Object(body), )?; - render_lane_trace_span_start(&report, ctx.json, ctx.quiet)?; + render_lane_trace_span_start(&report, ctx.json, &ctx.render)?; Ok(true) } LaneTraceSubcommand::End(args) => { @@ -741,7 +864,7 @@ fn handle_lane_trace_command( &format!("/v1/lane/spans/{}/end", args.span_id), &Value::Object(body), )?; - render_lane_trace_span_end(&report, ctx.json, ctx.quiet)?; + render_lane_trace_span_end(&report, ctx.json, &ctx.render)?; Ok(true) } LaneTraceSubcommand::List(args) => { @@ -754,7 +877,7 @@ fn handle_lane_trace_command( params.push(format!("limit={}", args.limit)); let path = append_query("/v1/lane/spans", params); let spans: Vec = client.get_json(&path)?; - render_lane_trace_spans(&spans, ctx.json, ctx.quiet)?; + render_lane_trace_spans(&spans, ctx.json, &ctx.render)?; Ok(true) } LaneTraceSubcommand::Summary(args) => { @@ -767,13 +890,13 @@ fn handle_lane_trace_command( params.push(format!("slowest={}", args.slowest_limit)); let path = append_query("/v1/lane/spans/summary", params); let report: LaneTraceSummaryReport = client.get_json(&path)?; - render_lane_trace_summary(&report, ctx.json, ctx.quiet)?; + render_lane_trace_summary(&report, ctx.json, &ctx.render)?; Ok(true) } LaneTraceSubcommand::Show(args) => { let span: LaneTraceSpan = client.get_json(&format!("/v1/lane/spans/{}", args.span_id))?; - render_lane_trace_span(&span, ctx.json, ctx.quiet)?; + render_lane_trace_span(&span, ctx.json, &ctx.render)?; Ok(true) } } @@ -801,48 +924,48 @@ fn trace_filter_params( params } -fn handle_merge_queue_command( +fn handle_lane_merge_queue_command( ctx: &RuntimeContext, client: &DaemonClient, - queue: &MergeQueueCommand, + queue: &LaneMergeQueueCommand, ) -> Result { match &queue.command { - MergeQueueSubcommand::Add(args) => { + LaneMergeQueueSubcommand::Add(args) => { let body = serde_json::json!({ - "source": args.source, - "target": args.into, + "lane": args.lane, + "into": args.into, "priority": args.priority, }); - let report: MergeQueueAddReport = client.post_json("/v1/merge-queue", &body)?; - render_merge_queue_add(&report, ctx.json, ctx.quiet)?; + let report: LaneMergeQueueAddReport = + client.post_json("/v1/lanes/merges/queue", &body)?; + render_lane_merge_queue_add(&report, ctx.json, &ctx.render)?; Ok(true) } - MergeQueueSubcommand::List => { - let entries: Vec = client.get_json("/v1/merge-queue")?; - render_merge_queue_list(&entries, ctx.json, ctx.quiet)?; + LaneMergeQueueSubcommand::List => { + let entries: Vec = client.get_json("/v1/lanes/merges/queue")?; + render_lane_merge_queue_list(&entries, ctx.json, &ctx.render)?; Ok(true) } - MergeQueueSubcommand::Explain(args) => { - let report: MergeQueueExplainReport = client.get_json(&format!( - "/v1/merge-queue/explain?selector={}", - args.selector - ))?; - render_merge_queue_explain(&report, ctx.json, ctx.quiet)?; + LaneMergeQueueSubcommand::Explain(args) => { + let report: LaneMergeQueueExplainReport = + client.get_json(&format!("/v1/lanes/merges/queue/{}/explain", args.selector))?; + render_lane_merge_queue_explain(&report, ctx.json, &ctx.render)?; Ok(true) } - MergeQueueSubcommand::Run(args) => { + LaneMergeQueueSubcommand::Run(args) => { let body = match args.limit { Some(limit) => serde_json::json!({ "limit": limit }), None => serde_json::json!({}), }; - let report: MergeQueueRunReport = client.post_json("/v1/merge-queue/run", &body)?; - render_merge_queue_run(&report, ctx.json, ctx.quiet)?; + let report: LaneMergeQueueRunReport = + client.post_json("/v1/lanes/merges/queue/run", &body)?; + render_lane_merge_queue_run(&report, ctx.json, &ctx.render)?; Ok(true) } - MergeQueueSubcommand::Remove(args) => { - let report: MergeQueueRemoveReport = - client.delete_json(&format!("/v1/merge-queue/{}", args.selector))?; - render_merge_queue_remove(&report, ctx.json, ctx.quiet)?; + LaneMergeQueueSubcommand::Remove(args) => { + let report: LaneMergeQueueRemoveReport = + client.delete_json(&format!("/v1/lanes/merges/queue/{}", args.selector))?; + render_lane_merge_queue_remove(&report, ctx.json, &ctx.render)?; Ok(true) } } @@ -862,7 +985,7 @@ fn handle_lease_command( "ttl_secs": args.ttl_secs, }); let report: LeaseAcquireReport = client.post_json("/v1/leases", &body)?; - render_lease_acquire(&report, ctx.json, ctx.quiet)?; + render_lease_acquire(&report, ctx.json, &ctx.render)?; Ok(true) } LeaseSubcommand::List(args) => { @@ -872,13 +995,13 @@ fn handle_lease_command( "/v1/leases".to_string() }; let leases: Vec = client.get_json(&path)?; - render_lease_list(&leases, ctx.json, ctx.quiet)?; + render_lease_list(&leases, ctx.json, &ctx.render)?; Ok(true) } LeaseSubcommand::Release(args) => { let report: LeaseReleaseReport = client.delete_json(&format!("/v1/leases/{}", args.lease_id))?; - render_lease_release(&report, ctx.json, ctx.quiet)?; + render_lease_release(&report, ctx.json, &ctx.render)?; Ok(true) } } @@ -919,15 +1042,15 @@ fn append_query(path: &str, params: Vec) -> String { } } -struct DaemonClient { - endpoint: DaemonEndpoint, +pub(super) struct DaemonClient { + endpoint: DaemonTransport, token: Option, } impl DaemonClient { - fn new(url: &str, token: Option) -> Result { + pub(super) fn new(url: &str, token: Option) -> Result { Ok(Self { - endpoint: DaemonEndpoint::parse(url)?, + endpoint: DaemonTransport::parse(url)?, token, }) } @@ -944,7 +1067,7 @@ impl DaemonClient { self.request_json("DELETE", path, None) } - fn request_json( + pub(super) fn request_json( &self, method: &str, path: &str, @@ -957,7 +1080,7 @@ impl DaemonClient { let request_path = self.endpoint.request_path(path); let mut request = format!( "{method} {request_path} HTTP/1.1\r\nHost: {}\r\nAccept: application/json\r\nContent-Length: {}\r\nConnection: close\r\n", - self.endpoint.authority, + self.endpoint.authority(), body_bytes.len() ); if body.is_some() { @@ -966,27 +1089,22 @@ impl DaemonClient { if let Some(token) = &self.token { request.push_str(&format!("Authorization: Bearer {token}\r\n")); } + let metrics_file = + std::env::var_os(PERFORMANCE_METRICS_FILE_ENV).filter(|path| !path.is_empty()); + if metrics_file.is_some() { + request.push_str("X-Trail-Operation-Metrics: 1\r\n"); + } request.push_str("\r\n"); - let mut stream = - TcpStream::connect((&*self.endpoint.host, self.endpoint.port)).map_err(|err| { - Error::DaemonUnavailable(format!( - "could not connect to {}: {err}", - self.endpoint.authority - )) - })?; - stream - .set_read_timeout(Some(Duration::from_secs(30))) - .map_err(Error::from)?; - stream.write_all(request.as_bytes())?; - if !body_bytes.is_empty() { - stream.write_all(&body_bytes)?; - } - stream.flush()?; - - let mut response = Vec::new(); - stream.read_to_end(&mut response)?; - let (status, response_body) = parse_http_response(&response)?; + let response = self.endpoint.exchange( + request.as_bytes(), + &body_bytes, + daemon_request_timeout(method, path), + )?; + let (status, operation_metrics, response_body) = parse_http_response(&response)?; + if let (Some(path), Some(report)) = (metrics_file.as_deref(), operation_metrics) { + emit_daemon_operation_metrics_report(Path::new(path), report)?; + } if !(200..300).contains(&status) { return Err(error_from_daemon_response(status, response_body)); } @@ -994,6 +1112,153 @@ impl DaemonClient { } } +fn daemon_request_timeout(method: &str, path: &str) -> Duration { + if !matches!(method, "POST" | "PUT" | "PATCH" | "DELETE") + || path == "/v1/record" + || path == "/v1/ledger/challenge" + { + DAEMON_READ_REQUEST_TIMEOUT + } else { + DAEMON_MUTATING_REQUEST_TIMEOUT + } +} + +#[derive(Serialize)] +struct LedgerFenceRequest<'a> { + protocol_version: u16, + owner_nonce: &'a str, + workspace_identity: &'a str, + executable_identity: &'a str, + scope_id: &'a str, + expected_epoch: u64, +} + +#[derive(Debug, Deserialize)] +pub(super) struct LedgerFenceProof { + pub(super) protocol_version: u16, + pub(super) pid: u32, + pub(super) process_start_identity: String, + pub(super) executable_identity: String, + pub(super) owner_nonce: String, + pub(super) workspace_identity: String, + pub(super) live_fence_sequence: u64, + pub(super) scope_id: String, + pub(super) epoch: u64, + pub(super) daemon_launch_nonce: String, + pub(super) durable_offset: u64, + pub(super) folded_offset: u64, +} + +pub(super) fn authenticated_ledger_fence( + endpoint: &daemon_start::WorkspaceDaemonEndpoint, +) -> Result { + let client = DaemonClient::new(&endpoint.url, Some(endpoint.auth_token.clone()))?; + let body = serde_json::to_value(LedgerFenceRequest { + protocol_version: endpoint.protocol_version, + owner_nonce: &endpoint.owner_nonce, + workspace_identity: &endpoint.workspace_identity, + executable_identity: &endpoint.executable_identity, + scope_id: &endpoint.scope_id, + expected_epoch: endpoint.epoch, + })?; + client.request_json("POST", "/v1/ledger/challenge", Some(&body)) +} + +enum DaemonTransport { + Tcp(DaemonEndpoint), + #[cfg(unix)] + Unix(PathBuf), +} + +impl DaemonTransport { + fn parse(url: &str) -> Result { + #[cfg(unix)] + if let Some(path) = url.strip_prefix("unix://") { + if path.is_empty() { + return Err(Error::InvalidInput( + "daemon Unix socket path is empty".into(), + )); + } + return Ok(Self::Unix(PathBuf::from(path))); + } + Ok(Self::Tcp(DaemonEndpoint::parse(url)?)) + } + + fn authority(&self) -> &str { + match self { + Self::Tcp(endpoint) => &endpoint.authority, + #[cfg(unix)] + Self::Unix(_) => "localhost", + } + } + + fn request_path(&self, path: &str) -> String { + match self { + Self::Tcp(endpoint) => endpoint.request_path(path), + #[cfg(unix)] + Self::Unix(_) => path.to_string(), + } + } + + fn exchange(&self, request: &[u8], body: &[u8], timeout: Duration) -> Result> { + match self { + Self::Tcp(endpoint) => { + let mut stream = + TcpStream::connect((&*endpoint.host, endpoint.port)).map_err(|error| { + Error::DaemonUnavailable(format!( + "could not connect to {}: {error}", + endpoint.authority + )) + })?; + stream.set_read_timeout(Some(timeout))?; + stream.set_write_timeout(Some(timeout))?; + exchange_stream(&mut stream, request, body) + .map_err(|error| map_daemon_exchange_error(error, timeout)) + } + #[cfg(unix)] + Self::Unix(path) => { + let mut stream = UnixStream::connect(path).map_err(|error| { + Error::DaemonUnavailable(format!( + "could not connect to workspace daemon socket {}: {error}", + path.display() + )) + })?; + stream.set_read_timeout(Some(timeout))?; + stream.set_write_timeout(Some(timeout))?; + exchange_stream(&mut stream, request, body) + .map_err(|error| map_daemon_exchange_error(error, timeout)) + } + } + } +} + +fn map_daemon_exchange_error(error: Error, timeout: Duration) -> Error { + match error { + Error::Io(error) if matches!(error.kind(), ErrorKind::TimedOut | ErrorKind::WouldBlock) => { + Error::DaemonUnavailable(format!( + "workspace daemon response timed out after {} seconds", + timeout.as_secs() + )) + } + error => error, + } +} + +fn exchange_stream( + stream: &mut S, + request: &[u8], + body: &[u8], +) -> Result> { + stream.write_all(request)?; + if !body.is_empty() { + stream.write_all(body)?; + } + stream.flush()?; + let mut response = Vec::new(); + stream.read_to_end(&mut response)?; + Ok(response) +} + struct DaemonEndpoint { host: String, port: u16, @@ -1026,7 +1291,7 @@ impl DaemonEndpoint { Some(_) => { return Err(Error::InvalidInput( "--daemon-url must include a non-empty host".to_string(), - )) + )); } }; let base_path = if path.is_empty() { @@ -1093,7 +1358,7 @@ fn discover_daemon_url(ctx: &RuntimeContext) -> Result> { Ok(Some(endpoint.url)) } -fn parse_http_response(response: &[u8]) -> Result<(u16, &[u8])> { +fn parse_http_response(response: &[u8]) -> Result<(u16, Option<&str>, &[u8])> { let Some(header_end) = response.windows(4).position(|window| window == b"\r\n\r\n") else { return Err(Error::DaemonUnavailable( "daemon returned a malformed HTTP response".to_string(), @@ -1116,7 +1381,54 @@ fn parse_http_response(response: &[u8]) -> Result<(u16, &[u8])> { "daemon response has invalid status `{status_line}`" )) })?; - Ok((status, &response[header_end + 4..])) + let mut operation_metrics = None; + for line in header.lines().skip(1) { + let Some((name, value)) = line.split_once(':') else { + return Err(Error::DaemonUnavailable( + "daemon response has a malformed HTTP header".into(), + )); + }; + if name.eq_ignore_ascii_case(OPERATION_METRICS_HEADER) { + if operation_metrics.is_some() { + return Err(Error::DaemonUnavailable( + "daemon response repeated its operation metrics report".into(), + )); + } + operation_metrics = Some(value.trim()); + } + } + Ok((status, operation_metrics, &response[header_end + 4..])) +} + +fn emit_daemon_operation_metrics_report(path: &Path, report: &str) -> Result<()> { + let report = serde_json::from_str::(report).map_err(|error| { + Error::DaemonUnavailable(format!( + "daemon returned malformed operation metrics JSON: {error}" + )) + })?; + if !report.is_object() { + return Err(Error::DaemonUnavailable( + "daemon returned a non-object operation metrics report".into(), + )); + } + let mut line = serde_json::to_vec(&report)?; + line.push(b'\n'); + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + let written = file.write(&line)?; + if written != line.len() { + return Err(Error::Io(std::io::Error::new( + ErrorKind::WriteZero, + format!( + "short performance metrics append: wrote {written} of {} bytes", + line.len() + ), + ))); + } + file.flush()?; + Ok(()) } fn error_from_daemon_response(status: u16, body: &[u8]) -> Error { @@ -1124,9 +1436,36 @@ fn error_from_daemon_response(status: u16, body: &[u8]) -> Error { if status == 401 { return Error::DaemonUnavailable(error.error.message); } + if error.error.code.as_ref().and_then(DaemonErrorCode::as_text) + == Some("CHANGE_LEDGER_RECONCILE_REQUIRED") + { + return Error::ChangeLedgerReconcileRequired { + scope: error.error.scope.unwrap_or_default(), + state: error.error.state.unwrap_or_default(), + reason: error + .error + .reason + .unwrap_or_else(|| error.error.message.clone()), + command: error + .error + .recovery + .map(|recovery| recovery.command) + .unwrap_or_else(|| "trail index reconcile".to_string()), + }; + } + let numeric_code = error + .error + .code + .as_ref() + .and_then(DaemonErrorCode::as_numeric); return Error::DaemonError { message: error.error.message, - exit_code: error.error.code.unwrap_or(1), + exit_code: error + .error + .exit + .or(error.error.exit_code) + .or(numeric_code) + .unwrap_or(1), }; } Error::DaemonError { @@ -1143,8 +1482,48 @@ struct DaemonErrorBody { #[derive(Deserialize)] struct DaemonErrorDetails { message: String, - #[serde(default, alias = "exit_code")] - code: Option, + #[serde(default)] + code: Option, + #[serde(default)] + exit: Option, + #[serde(default)] + exit_code: Option, + #[serde(default)] + scope: Option, + #[serde(default)] + state: Option, + #[serde(default)] + reason: Option, + #[serde(default)] + recovery: Option, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum DaemonErrorCode { + Text(String), + Numeric(i32), +} + +impl DaemonErrorCode { + fn as_text(&self) -> Option<&str> { + match self { + Self::Text(value) => Some(value), + Self::Numeric(_) => None, + } + } + + fn as_numeric(&self) -> Option { + match self { + Self::Text(_) => None, + Self::Numeric(value) => Some(*value), + } + } +} + +#[derive(Deserialize)] +struct DaemonErrorRecovery { + command: String, } fn resolve_daemon_token(ctx: &RuntimeContext, explicit: Option) -> Result> { @@ -1184,3 +1563,93 @@ fn discover_db_dir(ctx: &RuntimeContext) -> Option { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn daemon_response_carries_one_request_scoped_metrics_report() { + let response = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nX-Trail-Operation-Metrics: {\"generation\":7,\"operation\":\"status\"}\r\nContent-Length: 2\r\n\r\n{}"; + let (status, metrics, body) = parse_http_response(response).unwrap(); + assert_eq!(status, 200); + assert_eq!(metrics, Some("{\"generation\":7,\"operation\":\"status\"}")); + assert_eq!(body, b"{}"); + + let duplicate = b"HTTP/1.1 200 OK\r\nX-Trail-Operation-Metrics: {}\r\nx-trail-operation-metrics: {}\r\n\r\n{}"; + assert!(matches!( + parse_http_response(duplicate), + Err(Error::DaemonUnavailable(message)) + if message.contains("repeated") + )); + } + + #[test] + fn daemon_timeout_allows_large_mutations_without_extending_read_requests_indefinitely() { + assert_eq!( + daemon_request_timeout("GET", "/v1/status"), + Duration::from_secs(120) + ); + assert_eq!( + daemon_request_timeout("GET", "/v1/diff?dirty=1"), + Duration::from_secs(120) + ); + assert_eq!( + daemon_request_timeout("POST", "/v1/record"), + Duration::from_secs(120) + ); + assert_eq!( + daemon_request_timeout("POST", "/v1/lanes"), + Duration::from_secs(15 * 60) + ); + assert_eq!( + daemon_request_timeout("POST", "/v1/index/reconcile"), + Duration::from_secs(15 * 60) + ); + } + + #[test] + fn daemon_metrics_emission_appends_exactly_one_json_line() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("metrics.jsonl"); + emit_daemon_operation_metrics_report( + &path, + "{\"generation\":9,\"operation\":\"structured_patch\"}", + ) + .unwrap(); + let lines = std::fs::read_to_string(path).unwrap(); + assert_eq!(lines.lines().count(), 1); + let value: Value = serde_json::from_str(lines.trim()).unwrap(); + assert_eq!(value["generation"], 9); + assert_eq!(value["operation"], "structured_patch"); + } + + #[test] + fn daemon_error_parser_accepts_legacy_numeric_transport_codes() { + let error = error_from_daemon_response( + 429, + br#"{"error":{"message":"rate limit exceeded","code":2}}"#, + ); + match error { + Error::DaemonError { message, exit_code } => { + assert_eq!(message, "rate limit exceeded"); + assert_eq!(exit_code, 2); + } + error => panic!("unexpected daemon error: {error}"), + } + } + + #[test] + fn daemon_error_parser_preserves_structured_lane_recovery() { + let error = error_from_daemon_response( + 409, + br#"{"error":{"code":"CHANGE_LEDGER_RECONCILE_REQUIRED","status":409,"exit":16,"message":"reconcile","scope":"lane-scope","state":"untrusted_gap","reason":"overflow","recovery":{"command":"trail index reconcile --lane reconcile-bot"}}}"#, + ); + match error { + Error::ChangeLedgerReconcileRequired { command, .. } => { + assert_eq!(command, "trail index reconcile --lane reconcile-bot"); + } + error => panic!("unexpected daemon error: {error}"), + } + } +} diff --git a/trail/src/cli/command/handler/daemon_start.rs b/trail/src/cli/command/handler/daemon_start.rs new file mode 100644 index 0000000..d60e061 --- /dev/null +++ b/trail/src/cli/command/handler/daemon_start.rs @@ -0,0 +1,1687 @@ +use std::ffi::CString; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::mem::MaybeUninit; +use std::os::fd::{AsRawFd, FromRawFd, RawFd}; +use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt}; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::{Command as ProcessCommand, Stdio}; +use std::time::{Duration, Instant}; + +use getrandom::getrandom; +use rustix::fs::{flock, renameat_with, FlockOperation, RenameFlags}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use trail::{Error, Result, Trail}; + +use super::{daemon_rpc, RuntimeContext}; + +const PROTOCOL_VERSION: u16 = 2; +const LOCK_TIMEOUT: Duration = Duration::from_secs(60); +const ENDPOINT_FILE: &str = "daemon.json"; +const TOKEN_FILE: &str = "daemon.token"; +const LOCK_FILE: &str = "daemon.lock"; +const STARTING_FILE: &str = "daemon.starting.json"; +const SOCKET_FILE: &str = "changed-path.sock"; +const SOCKET_TOMBSTONE_PREFIX: &str = ".changed-path-socket-tombstone."; +const SOCKET_TOMBSTONE_SUFFIX: &str = ".removing"; +const SOCKET_CLEANUP_ARTIFACT_CAP: usize = 1024; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(super) struct WorkspaceDaemonEndpoint { + pub(super) protocol_version: u16, + pub(super) pid: u32, + pub(super) process_start_identity: String, + pub(super) executable_identity: String, + pub(super) workspace_identity: String, + pub(super) owner_nonce: String, + pub(super) auth_token: String, + pub(super) socket_path: PathBuf, + pub(super) socket_device: u64, + pub(super) socket_inode: u64, + pub(super) url: String, + pub(super) observer_ready: bool, + pub(super) recovery_complete: bool, + pub(super) reconciliation_complete: bool, + pub(super) live_fence_sequence: u64, + pub(super) scope_id: String, + pub(super) epoch: u64, + pub(super) daemon_launch_nonce: String, + pub(super) durable_offset: u64, + pub(super) folded_offset: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +struct WorkspaceDaemonStarting { + protocol_version: u16, + pid: u32, + process_start_identity: String, + executable_identity: String, + workspace_identity: String, + owner_nonce: String, + socket_path: PathBuf, + socket_device: u64, + socket_inode: u64, + #[serde(default)] + scope_id: Option, + #[serde(default)] + epoch: Option, + daemon_launch_nonce: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +struct VerifiedStaleOwnerHandoff { + stale_pid: u32, + process_start_identity: String, + daemon_launch_nonce: String, +} + +#[derive(Clone, Debug)] +pub(super) struct DaemonReady { + pub(super) url: String, + pub(super) auth_token: String, +} + +pub(super) fn ensure_workspace_daemon_ready( + workspace: &Path, + requested_token: Option<&str>, +) -> Result { + let workspace = workspace.canonicalize()?; + let db_dir = workspace.join(".trail"); + let authority = secure_authority_directory(&db_dir)?; + authority.verify_trail_identity(&db_dir)?; + let lock = secure_open_lock(&authority)?; + if let Some(ready) = + acquire_or_observe_published_daemon(&lock, &authority, &workspace, requested_token)? + { + return Ok(ready); + } + let mut verified_stale_owner = None; + if let Some(endpoint) = read_secure_endpoint(&authority)? { + match classify_endpoint(&workspace, &authority, &endpoint, requested_token)? { + EndpointState::Ready(ready) => return Ok(ready), + EndpointState::Stale(verified) => { + remove_stale_publication(&authority, &endpoint)?; + merge_verified_stale_owner(&mut verified_stale_owner, verified)?; + } + } + } + if let Some(verified) = recover_stale_starting_publication(&workspace, &authority)? { + merge_verified_stale_owner(&mut verified_stale_owner, verified)?; + } + + let token = match requested_token { + Some(token) if !token.trim().is_empty() => token.to_string(), + Some(_) => { + return Err(Error::InvalidInput( + "workspace daemon authentication token cannot be empty".into(), + )) + } + None => random_hex(32)?, + }; + #[cfg(debug_assertions)] + test_after_stale_verification_boundary(verified_stale_owner.as_ref())?; + spawn_workspace_daemon(&workspace, &token, verified_stale_owner.as_ref())?; + let endpoint = read_secure_endpoint(&authority)?.ok_or_else(|| { + Error::DaemonUnavailable("workspace daemon became ready without an endpoint".into()) + })?; + match classify_endpoint(&workspace, &authority, &endpoint, Some(&token))? { + EndpointState::Ready(ready) => Ok(ready), + EndpointState::Stale(_) => Err(Error::DaemonUnavailable( + "workspace daemon exited before readiness could be authenticated".into(), + )), + } +} + +pub(super) fn existing_workspace_daemon_ready( + workspace: &Path, + requested_token: Option<&str>, +) -> Result> { + let workspace = workspace.canonicalize()?; + let db_dir = workspace.join(".trail"); + let authority = secure_authority_directory(&db_dir)?; + authority.verify_trail_identity(&db_dir)?; + let lock = secure_open_lock(&authority)?; + if let Some(ready) = + acquire_or_observe_published_daemon(&lock, &authority, &workspace, requested_token)? + { + return Ok(Some(ready)); + } + if let Some(endpoint) = read_secure_endpoint(&authority)? { + return match classify_endpoint(&workspace, &authority, &endpoint, requested_token)? { + EndpointState::Ready(ready) => Ok(Some(ready)), + EndpointState::Stale(_) => Err(Error::DaemonUnavailable( + "workspace auto-daemon publication is stale; refusing local split-brain fallback" + .into(), + )), + }; + } + if read_secure_starting(&authority)?.is_some() { + return Err(Error::DaemonUnavailable( + "workspace auto-daemon publication exists but is not ready; refusing local split-brain fallback" + .into(), + )); + } + Ok(None) +} + +pub(super) fn retire_workspace_daemon_after_external_generation_change( + workspace: &Path, +) -> Result<()> { + let workspace = workspace.canonicalize()?; + let db_dir = workspace.join(".trail"); + let authority = secure_authority_directory(&db_dir)?; + authority.verify_trail_identity(&db_dir)?; + let Some(endpoint) = read_secure_endpoint(&authority)? else { + return Ok(()); + }; + match classify_endpoint(&workspace, &authority, &endpoint, None)? { + EndpointState::Stale(_) => return Ok(()), + EndpointState::Ready(_) => {} + } + let actual_start = process_start_identity(endpoint.pid).ok_or_else(|| { + Error::DaemonUnavailable( + "workspace daemon process identity disappeared before retirement".into(), + ) + })?; + if actual_start != endpoint.process_start_identity { + return Err(Error::DaemonUnavailable( + "workspace daemon PID was reused before retirement; refusing to signal it".into(), + )); + } + let result = unsafe { libc::kill(endpoint.pid as i32, libc::SIGTERM) }; + if result != 0 { + return Err(Error::Io(std::io::Error::last_os_error())); + } + let deadline = Instant::now() + Duration::from_secs(5); + while process_is_alive(endpoint.pid) { + if Instant::now() >= deadline { + return Err(Error::DaemonUnavailable( + "workspace daemon did not retire after its database generation changed".into(), + )); + } + std::thread::sleep(Duration::from_millis(10)); + } + Ok(()) +} + +enum EndpointState { + Ready(DaemonReady), + Stale(VerifiedStaleOwnerHandoff), +} + +fn classify_endpoint( + workspace: &Path, + authority: &SecureAuthority, + endpoint: &WorkspaceDaemonEndpoint, + requested_token: Option<&str>, +) -> Result { + let expected_workspace = workspace_identity(workspace)?; + let expected_executable = executable_identity(&std::env::current_exe()?)?; + let expected_socket = workspace.join(".trail").join("changed-path.sock"); + authority.verify_trail_identity(&workspace.join(".trail"))?; + if endpoint.socket_path != expected_socket + || endpoint.pid == 0 + || endpoint.process_start_identity.is_empty() + { + return Err(Error::DaemonUnavailable( + "workspace daemon endpoint lacks exact stale-cleanup identity; refusing replacement" + .into(), + )); + } + let mut invalid = Vec::new(); + if endpoint.protocol_version != PROTOCOL_VERSION { + invalid.push("protocol"); + } + if endpoint.workspace_identity != expected_workspace { + invalid.push("workspace"); + } + if endpoint.executable_identity != expected_executable { + invalid.push("executable"); + } + if endpoint.url != format!("unix://{}", expected_socket.display()) { + invalid.push("url"); + } + if endpoint.owner_nonce.len() != 64 { + invalid.push("owner_nonce"); + } + if endpoint.auth_token.len() != 64 { + invalid.push("auth_token"); + } + if requested_token.is_some_and(|token| token != endpoint.auth_token) { + invalid.push("requested_token"); + } + if !endpoint.observer_ready { + invalid.push("observer_ready"); + } + if !endpoint.recovery_complete { + invalid.push("recovery_complete"); + } + if !endpoint.reconciliation_complete { + invalid.push("reconciliation_complete"); + } + if endpoint.live_fence_sequence == 0 { + invalid.push("live_fence"); + } + if endpoint.scope_id.len() != 64 { + invalid.push("scope"); + } + if endpoint.epoch == 0 { + invalid.push("epoch"); + } + if endpoint.daemon_launch_nonce.len() != 64 { + invalid.push("daemon_launch_nonce"); + } + if endpoint.folded_offset > endpoint.durable_offset { + invalid.push("offsets"); + } + if !invalid.is_empty() { + return Err(Error::DaemonUnavailable(format!( + "workspace daemon endpoint identity is unverifiable ({}) ; refusing replacement", + invalid.join(",") + ))); + } + let alive = process_is_alive(endpoint.pid); + let actual_start = process_start_identity(endpoint.pid); + if !alive { + return Ok(EndpointState::Stale(verified_stale_handoff(endpoint)?)); + } + let Some(actual_start) = actual_start else { + return Err(Error::DaemonUnavailable(format!( + "live workspace daemon PID {} cannot be identity-verified; refusing replacement", + endpoint.pid + ))); + }; + let published_token = read_secure_owner_text(authority, TOKEN_FILE, 4096)?; + if published_token.trim_end() != endpoint.auth_token { + return Err(Error::DaemonUnavailable( + "workspace daemon token publication does not match the endpoint".into(), + )); + } + verify_secure_socket_leaf_identity( + &authority.trail_directory, + SOCKET_FILE, + endpoint.socket_device, + endpoint.socket_inode, + )?; + let proof = match daemon_rpc::authenticated_ledger_fence(endpoint) { + Ok(proof) => proof, + Err(_) if actual_start != endpoint.process_start_identity => { + return Ok(EndpointState::Stale(verified_stale_handoff(endpoint)?)) + } + Err(error) + if error + .to_string() + .contains("changed-path observer health no longer authorizes") => + { + let deadline = Instant::now() + Duration::from_secs(3); + while Instant::now() < deadline { + if !process_is_alive(endpoint.pid) + || process_start_identity(endpoint.pid).as_deref() + != Some(endpoint.process_start_identity.as_str()) + { + return Ok(EndpointState::Stale(verified_stale_handoff(endpoint)?)); + } + std::thread::sleep(Duration::from_millis(10)); + } + return Err(error); + } + Err(error) => return Err(error), + }; + let post_challenge_start = post_challenge_process_start_identity(endpoint.pid).ok_or_else(|| { + Error::DaemonUnavailable( + "workspace daemon process identity disappeared during authentication; refusing replacement" + .into(), + ) + })?; + if actual_start != endpoint.process_start_identity + || post_challenge_start != endpoint.process_start_identity + || post_challenge_start != actual_start + { + return Err(Error::DaemonUnavailable( + "an authenticated workspace daemon is live but its published process identity is unverifiable; refusing replacement".into(), + )); + } + if proof.protocol_version != endpoint.protocol_version + || proof.pid != endpoint.pid + || proof.process_start_identity != endpoint.process_start_identity + || proof.executable_identity != endpoint.executable_identity + || proof.owner_nonce != endpoint.owner_nonce + || proof.workspace_identity != endpoint.workspace_identity + || proof.scope_id != endpoint.scope_id + || proof.epoch != endpoint.epoch + || proof.daemon_launch_nonce != endpoint.daemon_launch_nonce + || proof.live_fence_sequence < endpoint.live_fence_sequence + || proof.durable_offset < endpoint.durable_offset + || proof.folded_offset > proof.durable_offset + { + return Err(Error::DaemonUnavailable( + "workspace daemon challenge-response identity mismatch".into(), + )); + } + Ok(EndpointState::Ready(DaemonReady { + url: endpoint.url.clone(), + auth_token: endpoint.auth_token.clone(), + })) +} + +fn verified_stale_handoff(endpoint: &WorkspaceDaemonEndpoint) -> Result { + if endpoint.daemon_launch_nonce.len() != 64 { + return Err(Error::DaemonUnavailable( + "stale workspace daemon endpoint lacks an exact ledger owner binding".into(), + )); + } + Ok(VerifiedStaleOwnerHandoff { + stale_pid: endpoint.pid, + process_start_identity: endpoint.process_start_identity.clone(), + daemon_launch_nonce: endpoint.daemon_launch_nonce.clone(), + }) +} + +fn merge_verified_stale_owner( + target: &mut Option, + additional: VerifiedStaleOwnerHandoff, +) -> Result<()> { + match target { + Some(existing) if *existing != additional => Err(Error::DaemonUnavailable( + "workspace daemon stale publications disagree on the exact ledger owner binding".into(), + )), + Some(_) => Ok(()), + None => { + *target = Some(additional); + Ok(()) + } + } +} + +#[cfg(debug_assertions)] +fn test_after_stale_verification_boundary( + verified: Option<&VerifiedStaleOwnerHandoff>, +) -> Result<()> { + let Some(barrier) = + std::env::var_os("TRAIL_TEST_WORKSPACE_DAEMON_AFTER_STALE_VERIFICATION_BARRIER") + else { + return Ok(()); + }; + let Some(verified) = verified else { + return Err(Error::DaemonUnavailable( + "stale-verification test boundary had no verified stale owner".into(), + )); + }; + let barrier = PathBuf::from(barrier); + fs::write(barrier.join("verified"), serde_json::to_vec(verified)?)?; + let deadline = Instant::now() + Duration::from_secs(10); + while !barrier.join("continue").exists() { + if Instant::now() >= deadline { + return Err(Error::DaemonUnavailable( + "stale-verification test boundary timed out".into(), + )); + } + std::thread::sleep(Duration::from_millis(1)); + } + Ok(()) +} + +fn spawn_workspace_daemon( + workspace: &Path, + token: &str, + verified_stale_owner: Option<&VerifiedStaleOwnerHandoff>, +) -> Result<()> { + // std::io::pipe creates close-on-exec descriptors while holding the + // standard library's process-spawn lock on platforms that need it. Keep + // every parent descriptor sealed; only the intended child clears its two + // inherited ends in pre_exec. + let (mut ready_reader, ready_writer) = std::io::pipe()?; + let (token_reader, mut token_writer) = std::io::pipe()?; + let write_fd = ready_writer.as_raw_fd(); + let token_read_fd = token_reader.as_raw_fd(); + let maximum_fd = unsafe { libc::getdtablesize() }; + if maximum_fd < 0 { + return Err(Error::Io(std::io::Error::last_os_error())); + } + + let owner_nonce = random_hex(32)?; + let daemon_launch_nonce = random_hex(32)?; + let mut command = ProcessCommand::new(std::env::current_exe()?); + command + .arg("--workspace") + .arg(workspace) + .arg("--quiet") + .arg("daemon") + .arg("--port") + .arg("0") + .env("TRAIL_WORKSPACE_DAEMON", "1") + .env("TRAIL_WORKSPACE_DAEMON_READY_FD", write_fd.to_string()) + .env("TRAIL_WORKSPACE_DAEMON_TOKEN_FD", token_read_fd.to_string()) + .env("TRAIL_WORKSPACE_DAEMON_OWNER_NONCE", owner_nonce) + .env("TRAIL_WORKSPACE_DAEMON_LAUNCH_NONCE", daemon_launch_nonce) + .env_remove("TRAIL_WORKSPACE_DAEMON_VERIFIED_STALE_OWNER") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()); + if let Some(verified) = verified_stale_owner { + command.env( + "TRAIL_WORKSPACE_DAEMON_VERIFIED_STALE_OWNER", + serde_json::to_string(verified)?, + ); + } + unsafe { + command.pre_exec(move || { + seal_daemon_exec_descriptors(maximum_fd)?; + clear_descriptor_cloexec(write_fd)?; + clear_descriptor_cloexec(token_read_fd)?; + Ok(()) + }); + } + let child = command.spawn(); + let mut child = match child { + Ok(child) => child, + Err(error) => return Err(Error::Io(error)), + }; + drop(ready_writer); + drop(token_reader); + token_writer.write_all(token.as_bytes())?; + drop(token_writer); + let deadline = Instant::now() + ready_timeout(); + let mut byte = [0_u8; 1]; + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + let _ = child.kill(); + let _ = child.wait(); + return Err(Error::DaemonUnavailable( + "workspace daemon readiness timed out".into(), + )); + } + let mut poll_fd = libc::pollfd { + fd: ready_reader.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + let timeout_ms = + i32::try_from(remaining.as_millis().min(i32::MAX as u128)).unwrap_or(i32::MAX); + let polled = unsafe { libc::poll(&mut poll_fd, 1, timeout_ms) }; + if polled < 0 { + return Err(Error::Io(std::io::Error::last_os_error())); + } + if polled == 0 { + continue; + } + match ready_reader.read(&mut byte) { + Ok(1) if byte[0] == 1 => return Ok(()), + Ok(_) => { + let status = child.try_wait()?.map(|status| status.to_string()); + let mut diagnostic = String::new(); + if let Some(stderr) = child.stderr.take() { + let _ = stderr.take(64 * 1024).read_to_string(&mut diagnostic); + } + return Err(Error::DaemonUnavailable(format!( + "workspace daemon exited before readiness{}{}", + status + .map(|value| format!(" ({value})")) + .unwrap_or_default(), + if diagnostic.trim().is_empty() { + String::new() + } else { + format!(": {}", diagnostic.trim()) + } + ))); + } + Err(error) => return Err(Error::Io(error)), + } + } +} + +fn seal_daemon_exec_descriptors(maximum_fd: RawFd) -> std::io::Result<()> { + #[cfg(target_os = "linux")] + { + let closed = unsafe { + libc::syscall( + libc::SYS_close_range, + 3_u32, + u32::MAX, + libc::CLOSE_RANGE_CLOEXEC, + ) + }; + if closed == 0 { + return Ok(()); + } + let error = std::io::Error::last_os_error(); + if !matches!( + error.raw_os_error(), + Some(libc::ENOSYS) | Some(libc::EINVAL) + ) { + return Err(error); + } + for fd in 3..maximum_fd { + mark_descriptor_cloexec_if_open(fd)?; + } + return Ok(()); + } + + #[cfg(target_os = "macos")] + { + for fd in 3..maximum_fd { + mark_descriptor_cloexec_if_open(fd)?; + } + Ok(()) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + for fd in 3..maximum_fd { + mark_descriptor_cloexec_if_open(fd)?; + } + Ok(()) + } +} + +fn mark_descriptor_cloexec_if_open(fd: RawFd) -> std::io::Result<()> { + if fd < 3 { + return Ok(()); + } + let flags = unsafe { libc::fcntl(fd, libc::F_GETFD) }; + if flags < 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::EBADF) { + return Ok(()); + } + return Err(error); + } + if flags & libc::FD_CLOEXEC == 0 + && unsafe { libc::fcntl(fd, libc::F_SETFD, flags | libc::FD_CLOEXEC) } < 0 + { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::EBADF) { + return Err(error); + } + } + Ok(()) +} + +fn clear_descriptor_cloexec(fd: RawFd) -> std::io::Result<()> { + let flags = unsafe { libc::fcntl(fd, libc::F_GETFD) }; + if flags < 0 || unsafe { libc::fcntl(fd, libc::F_SETFD, flags & !libc::FD_CLOEXEC) } < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +pub(super) fn is_auto_workspace_daemon() -> bool { + std::env::var_os("TRAIL_WORKSPACE_DAEMON").is_some() +} + +pub(super) fn run_auto_workspace_daemon(mut db: Trail) -> Result<()> { + let workspace = db.workspace_root().canonicalize()?; + let authority = secure_authority_directory(db.db_dir())?; + let token_fd = required_env("TRAIL_WORKSPACE_DAEMON_TOKEN_FD")? + .parse::() + .map_err(|_| Error::InvalidInput("workspace daemon token fd is invalid".into()))?; + let mut token_reader = unsafe { File::from_raw_fd(token_fd) }; + let mut token = String::new(); + (&mut token_reader).take(65).read_to_string(&mut token)?; + drop(token_reader); + let owner_nonce = required_env("TRAIL_WORKSPACE_DAEMON_OWNER_NONCE")?; + let daemon_launch_nonce = required_env("TRAIL_WORKSPACE_DAEMON_LAUNCH_NONCE")?; + if token.len() != 64 || owner_nonce.len() != 64 || daemon_launch_nonce.len() != 64 { + return Err(Error::InvalidInput( + "workspace daemon received malformed authentication identity".into(), + )); + } + let ready_fd = required_env("TRAIL_WORKSPACE_DAEMON_READY_FD")? + .parse::() + .map_err(|_| Error::InvalidInput("workspace daemon readiness fd is invalid".into()))?; + + let socket_path = db.db_dir().join(SOCKET_FILE); + authority.verify_trail_identity(db.db_dir())?; + // Keep the private bind leaf shorter than the stable leaf so workspaces + // whose final socket fits SUN_LEN do not fail only during publication. + ensure_socket_cleanup_artifact_capacity(&authority.trail_directory)?; + let socket_tmp_leaf = format!(".s{}", random_hex(6)?); + let socket_tmp_path = db.db_dir().join(&socket_tmp_leaf); + // This process was exec'd solely as the workspace daemon, and this bind + // precedes observer/server worker startup, so the process-global umask + // window cannot affect concurrent Trail-created files. + let socket = { + let _umask = ScopedUmask::owner_only(); + std::os::unix::net::UnixListener::bind(&socket_tmp_path) + }?; + #[cfg(debug_assertions)] + test_socket_bound_boundary(&socket_tmp_leaf)?; + let socket_identity = + verify_socket_leaf_owner(&authority.trail_directory, &socket_tmp_leaf, None)?; + let mut unpublished_socket = BoundSocketGuard { + authority: authority.try_clone()?, + leaf: socket_tmp_leaf.clone(), + device: socket_identity.0, + inode: socket_identity.1, + armed: true, + }; + verify_secure_socket_leaf_identity( + &authority.trail_directory, + &socket_tmp_leaf, + socket_identity.0, + socket_identity.1, + )?; + authority.verify_trail_identity(db.db_dir())?; + renameat_noreplace(&authority.trail_directory, &socket_tmp_leaf, SOCKET_FILE).map_err( + |error| { + if error.kind() == std::io::ErrorKind::AlreadyExists { + Error::DaemonUnavailable( + "workspace daemon socket pathname is already occupied".into(), + ) + } else { + Error::Io(error) + } + }, + )?; + unpublished_socket.leaf = SOCKET_FILE.to_string(); + authority.trail_directory.sync_all()?; + let socket_metadata = verify_secure_socket_leaf_identity( + &authority.trail_directory, + SOCKET_FILE, + socket_identity.0, + socket_identity.1, + )?; + let mut starting = WorkspaceDaemonStarting { + protocol_version: PROTOCOL_VERSION, + pid: std::process::id(), + process_start_identity: process_start_identity(std::process::id()).ok_or_else(|| { + Error::DaemonUnavailable("workspace daemon process identity is unavailable".into()) + })?, + executable_identity: executable_identity(&std::env::current_exe()?)?, + workspace_identity: workspace_identity(&workspace)?, + owner_nonce: owner_nonce.clone(), + socket_path: socket_path.clone(), + socket_device: socket_metadata.0, + socket_inode: socket_metadata.1, + scope_id: None, + epoch: None, + daemon_launch_nonce: daemon_launch_nonce.clone(), + }; + publish_owner_file( + &authority, + STARTING_FILE, + &serde_json::to_vec_pretty(&starting)?, + )?; + unpublished_socket.armed = false; + #[cfg(debug_assertions)] + if let Ok(delay) = std::env::var("TRAIL_TEST_WORKSPACE_DAEMON_DELAY_AFTER_INTENT_MS") { + if let Ok(delay) = delay.parse::() { + std::thread::sleep(Duration::from_millis(delay)); + } + } + + let ledger_ready = trail::server::prepare_workspace_changed_path_daemon(&mut db)?; + #[cfg(debug_assertions)] + if std::env::var_os( + "TRAIL_TEST_WORKSPACE_DAEMON_EXIT_AFTER_OWNER_ACQUIRE_BEFORE_BOUND_PUBLICATION", + ) + .is_some() + { + std::process::exit(87); + } + starting.scope_id = Some(ledger_ready.scope_id.clone()); + starting.epoch = Some(ledger_ready.epoch); + publish_owner_file( + &authority, + STARTING_FILE, + &serde_json::to_vec_pretty(&starting)?, + )?; + #[cfg(debug_assertions)] + if std::env::var_os("TRAIL_TEST_WORKSPACE_DAEMON_EXIT_AFTER_PREPARE").is_some() { + std::process::exit(86); + } + #[cfg(debug_assertions)] + if std::env::var_os("TRAIL_TEST_WORKSPACE_DAEMON_ERROR_AFTER_PREPARE").is_some() { + return Err(Error::DaemonUnavailable( + "injected ordinary readiness failure after observer ownership".into(), + )); + } + + let endpoint = WorkspaceDaemonEndpoint { + protocol_version: PROTOCOL_VERSION, + pid: std::process::id(), + process_start_identity: starting.process_start_identity.clone(), + executable_identity: starting.executable_identity.clone(), + workspace_identity: starting.workspace_identity.clone(), + owner_nonce, + auth_token: token.clone(), + socket_path: socket_path.clone(), + socket_device: socket_metadata.0, + socket_inode: socket_metadata.1, + url: format!("unix://{}", socket_path.display()), + observer_ready: true, + recovery_complete: true, + reconciliation_complete: true, + live_fence_sequence: ledger_ready.sequence, + scope_id: ledger_ready.scope_id, + epoch: ledger_ready.epoch, + daemon_launch_nonce: ledger_ready.daemon_launch_nonce, + durable_offset: ledger_ready.durable_offset, + folded_offset: ledger_ready.folded_offset, + }; + publish_owner_file( + &authority, + TOKEN_FILE, + format!("{}\n", endpoint.auth_token).as_bytes(), + )?; + authority.verify_trail_identity(db.db_dir())?; + publish_owner_file( + &authority, + ENDPOINT_FILE, + &serde_json::to_vec_pretty(&endpoint)?, + )?; + unlink_authority_file(&authority, STARTING_FILE)?; + let mut publication = PublicationGuard { + authority: authority.try_clone()?, + socket_device: endpoint.socket_device, + socket_inode: endpoint.socket_inode, + endpoint: endpoint.clone(), + preserve_stale_identity: false, + }; + let mut ready = unsafe { File::from_raw_fd(ready_fd) }; + ready.write_all(&[1])?; + ready.flush()?; + drop(ready); + + let result = trail::server::serve_unix_listener_with_auth_and_timeout( + &mut db, + socket, + trail::server::ServerAuth::bearer(token)?.with_daemon_identity( + trail::server::DaemonServerIdentity::new( + endpoint.owner_nonce.clone(), + endpoint.workspace_identity.clone(), + endpoint.executable_identity.clone(), + endpoint.process_start_identity.clone(), + ), + ), + Duration::from_secs(30), + ); + if result.as_ref().is_err_and(|error| { + error + .to_string() + .contains("workspace daemon observer health retirement requested") + }) { + publication.preserve_stale_identity = true; + } + result +} + +struct PublicationGuard { + authority: SecureAuthority, + socket_device: u64, + socket_inode: u64, + endpoint: WorkspaceDaemonEndpoint, + preserve_stale_identity: bool, +} + +struct BoundSocketGuard { + authority: SecureAuthority, + leaf: String, + device: u64, + inode: u64, + armed: bool, +} + +struct ScopedUmask(libc::mode_t); + +impl ScopedUmask { + fn owner_only() -> Self { + Self(unsafe { libc::umask(0o177) }) + } +} + +impl Drop for ScopedUmask { + fn drop(&mut self) { + unsafe { + libc::umask(self.0); + } + } +} + +impl Drop for BoundSocketGuard { + fn drop(&mut self) { + if self.armed { + let _ = remove_socket_leaf_if_identity( + &self.authority, + &self.leaf, + self.device, + self.inode, + true, + false, + ); + } + } +} + +impl Drop for PublicationGuard { + fn drop(&mut self) { + if self.preserve_stale_identity { + return; + } + if read_secure_endpoint(&self.authority) + .ok() + .flatten() + .as_ref() + == Some(&self.endpoint) + && remove_socket_leaf_if_identity( + &self.authority, + SOCKET_FILE, + self.socket_device, + self.socket_inode, + true, + false, + ) + .is_ok() + { + let _ = unlink_authority_file(&self.authority, ENDPOINT_FILE); + let _ = unlink_authority_file(&self.authority, TOKEN_FILE); + let _ = self.authority.directory.sync_all(); + } + } +} + +#[derive(Debug)] +struct SecureAuthority { + path: PathBuf, + directory: File, + trail_directory: File, + trail_identity: (u64, u64), +} + +impl SecureAuthority { + fn try_clone(&self) -> Result { + Ok(Self { + path: self.path.clone(), + directory: self.directory.try_clone()?, + trail_directory: self.trail_directory.try_clone()?, + trail_identity: self.trail_identity, + }) + } + + fn verify_trail_identity(&self, db_dir: &Path) -> Result<()> { + let pinned = self.trail_directory.metadata()?; + if (pinned.dev(), pinned.ino()) != self.trail_identity { + return Err(Error::DaemonUnavailable( + "workspace daemon pinned .trail authority changed identity".into(), + )); + } + let named = open_private_directory(db_dir)?; + let named = named.metadata()?; + if (named.dev(), named.ino()) != self.trail_identity { + return Err(Error::DaemonUnavailable( + "workspace .trail directory was replaced; refusing daemon pathname authority" + .into(), + )); + } + Ok(()) + } +} + +fn secure_authority_directory(db_dir: &Path) -> Result { + let trail_directory = open_private_directory(db_dir)?; + let trail_metadata = trail_directory.metadata()?; + let trail_identity = (trail_metadata.dev(), trail_metadata.ino()); + let mut directory = trail_directory.try_clone()?; + let mut current = db_dir.to_path_buf(); + for component in ["index", "change-ledger"] { + current.push(component); + directory = open_or_create_private_child(&directory, component, ¤t)?; + } + Ok(SecureAuthority { + path: current, + directory, + trail_directory, + trail_identity, + }) +} + +fn open_private_directory(path: &Path) -> Result { + let directory = OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path) + .map_err(|error| { + Error::DaemonUnavailable(format!( + "could not open pinned workspace authority directory {}: {error}", + path.display() + )) + })?; + verify_private_directory(&directory, path)?; + Ok(directory) +} + +fn verify_private_directory(directory: &File, path: &Path) -> Result<()> { + let metadata = directory.metadata()?; + if !metadata.is_dir() + || metadata.uid() != unsafe { libc::geteuid() } + || metadata.permissions().mode() & 0o777 != 0o700 + { + return Err(Error::DaemonUnavailable(format!( + "workspace daemon authority directory {} has unsafe owner or mode; reinitialize this workspace before using changed-path ledger commands", + path.display() + ))); + } + Ok(()) +} + +fn open_or_create_private_child(parent: &File, name: &str, path: &Path) -> Result { + match openat_file( + parent, + name, + libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC, + 0, + ) { + Ok(directory) => { + verify_private_directory(&directory, path)?; + Ok(directory) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let leaf = CString::new(name) + .map_err(|_| Error::InvalidInput("invalid authority leaf".into()))?; + let created = unsafe { libc::mkdirat(parent.as_raw_fd(), leaf.as_ptr(), 0o700) }; + if created != 0 { + let error = std::io::Error::last_os_error(); + if error.kind() != std::io::ErrorKind::AlreadyExists { + return Err(Error::Io(error)); + } + } + let directory = openat_file( + parent, + name, + libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC, + 0, + ) + .map_err(|error| { + Error::DaemonUnavailable(format!( + "could not reopen concurrently created authority directory {}: {error}", + path.display() + )) + })?; + verify_private_directory(&directory, path)?; + Ok(directory) + } + Err(error) => Err(Error::DaemonUnavailable(format!( + "could not open authority directory {}: {error}", + path.display() + ))), + } +} + +fn openat_file(parent: &File, name: &str, flags: i32, mode: libc::mode_t) -> std::io::Result { + let name = CString::new(name) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid file leaf"))?; + let fd = unsafe { + libc::openat( + parent.as_raw_fd(), + name.as_ptr(), + flags, + libc::c_uint::from(mode), + ) + }; + if fd < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(unsafe { File::from_raw_fd(fd) }) + } +} + +fn secure_open_lock(authority: &SecureAuthority) -> Result { + let deadline = Instant::now() + Duration::from_millis(250); + let file = loop { + match openat_file( + &authority.directory, + LOCK_FILE, + libc::O_RDWR | libc::O_CREAT | libc::O_NOFOLLOW | libc::O_CLOEXEC, + 0o600, + ) { + Ok(file) => break file, + Err(error) + if error.kind() == std::io::ErrorKind::NotFound && Instant::now() < deadline => + { + std::thread::sleep(Duration::from_millis(1)); + } + Err(error) => { + return Err(Error::DaemonUnavailable(format!( + "could not securely open workspace daemon lock: {error}" + ))) + } + } + }; + verify_owner_file(&file, &authority.path.join(LOCK_FILE))?; + Ok(file) +} + +fn acquire_or_observe_published_daemon( + lock: &File, + authority: &SecureAuthority, + workspace: &Path, + requested_token: Option<&str>, +) -> Result> { + let deadline = Instant::now() + LOCK_TIMEOUT; + loop { + match flock(lock, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => return Ok(None), + Err(error) if error == rustix::io::Errno::WOULDBLOCK => { + if let Some(endpoint) = read_secure_endpoint(authority)? { + if let EndpointState::Ready(ready) = + classify_endpoint(workspace, authority, &endpoint, requested_token)? + { + return Ok(Some(ready)); + } + } + if Instant::now() >= deadline { + return Err(Error::DaemonUnavailable( + "workspace daemon startup lock timed out".into(), + )); + } + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => return Err(Error::Io(error.into())), + } + } +} + +fn read_secure_endpoint(authority: &SecureAuthority) -> Result> { + let path = authority.path.join(ENDPOINT_FILE); + let file = match openat_file( + &authority.directory, + ENDPOINT_FILE, + libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC, + 0, + ) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(Error::Io(error)), + }; + verify_owner_file(&file, &path)?; + let mut bytes = Vec::new(); + file.take(64 * 1024).read_to_end(&mut bytes)?; + let endpoint = serde_json::from_slice(&bytes) + .map_err(|_| Error::DaemonUnavailable("workspace daemon endpoint is malformed".into()))?; + Ok(Some(endpoint)) +} + +fn read_secure_starting(authority: &SecureAuthority) -> Result> { + let path = authority.path.join(STARTING_FILE); + let file = match openat_file( + &authority.directory, + STARTING_FILE, + libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC, + 0, + ) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(Error::Io(error)), + }; + verify_owner_file(&file, &path)?; + let mut bytes = Vec::new(); + file.take(64 * 1024).read_to_end(&mut bytes)?; + serde_json::from_slice(&bytes).map(Some).map_err(|_| { + Error::DaemonUnavailable("workspace daemon startup intent is malformed".into()) + }) +} + +fn recover_stale_starting_publication( + workspace: &Path, + authority: &SecureAuthority, +) -> Result> { + let Some(starting) = read_secure_starting(authority)? else { + return Ok(None); + }; + let expected_socket = workspace.join(".trail/changed-path.sock"); + let mut invalid = Vec::new(); + if starting.protocol_version != PROTOCOL_VERSION { + invalid.push("protocol"); + } + if starting.pid == 0 { + invalid.push("pid"); + } + if starting.process_start_identity.is_empty() { + invalid.push("process_start_identity"); + } + if starting.executable_identity != executable_identity(&std::env::current_exe()?)? { + invalid.push("executable"); + } + if starting.workspace_identity != workspace_identity(workspace)? { + invalid.push("workspace"); + } + if starting.owner_nonce.len() != 64 { + invalid.push("owner_nonce"); + } + if starting.daemon_launch_nonce.len() != 64 { + invalid.push("daemon_launch_nonce"); + } + if starting.socket_path != expected_socket { + invalid.push("socket_path"); + } + if !invalid.is_empty() { + return Err(Error::DaemonUnavailable(format!( + "workspace daemon startup identity is unverifiable ({}); refusing replacement", + invalid.join(", ") + ))); + } + if process_is_alive(starting.pid) { + match process_start_identity(starting.pid) { + Some(identity) if identity == starting.process_start_identity => { + return Err(Error::DaemonUnavailable( + "workspace daemon startup owner is still live; refusing replacement".into(), + )); + } + Some(_) => {} + None => { + return Err(Error::DaemonUnavailable( + "live workspace daemon startup owner cannot be identity-verified; refusing replacement" + .into(), + )); + } + } + } + match (starting.scope_id.as_ref(), starting.epoch) { + (Some(scope_id), Some(epoch)) if scope_id.len() == 64 && epoch != 0 => {} + (None, None) => {} + _ => { + return Err(Error::DaemonUnavailable( + "workspace daemon startup publication has an incomplete ledger scope binding" + .into(), + )) + } + } + let handoff = VerifiedStaleOwnerHandoff { + stale_pid: starting.pid, + process_start_identity: starting.process_start_identity.clone(), + daemon_launch_nonce: starting.daemon_launch_nonce.clone(), + }; + remove_socket_leaf_if_identity( + authority, + SOCKET_FILE, + starting.socket_device, + starting.socket_inode, + true, + false, + )?; + unlink_authority_file(authority, STARTING_FILE)?; + authority.directory.sync_all()?; + Ok(Some(handoff)) +} + +fn read_secure_owner_text(authority: &SecureAuthority, name: &str, limit: u64) -> Result { + let path = authority.path.join(name); + let file = openat_file( + &authority.directory, + name, + libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC, + 0, + ) + .map_err(|error| { + Error::DaemonUnavailable(format!( + "could not securely open workspace daemon {name}: {error}" + )) + })?; + verify_owner_file(&file, &path)?; + let mut value = String::new(); + file.take(limit).read_to_string(&mut value)?; + Ok(value) +} + +fn verify_owner_file(file: &File, path: &Path) -> Result<()> { + let metadata = file.metadata()?; + if !metadata.is_file() + || metadata.uid() != unsafe { libc::geteuid() } + || metadata.permissions().mode() & 0o777 != 0o600 + { + return Err(Error::DaemonUnavailable(format!( + "workspace daemon file {} has unsafe owner or mode", + path.display() + ))); + } + Ok(()) +} + +fn socket_leaf_stat(parent: &File, leaf: &str) -> std::io::Result { + let leaf = CString::new(leaf).map_err(|_| { + std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid socket leaf") + })?; + let mut stat = MaybeUninit::::zeroed(); + if unsafe { + libc::fstatat( + parent.as_raw_fd(), + leaf.as_ptr(), + stat.as_mut_ptr(), + libc::AT_SYMLINK_NOFOLLOW, + ) + } != 0 + { + return Err(std::io::Error::last_os_error()); + } + Ok(unsafe { stat.assume_init() }) +} + +fn verify_socket_leaf_owner( + parent: &File, + leaf: &str, + required_mode: Option, +) -> Result<(u64, u64)> { + let metadata = socket_leaf_stat(parent, leaf).map_err(|error| { + Error::DaemonUnavailable(format!( + "could not inspect workspace daemon socket leaf {leaf}: {error}" + )) + })?; + if u32::from(metadata.st_mode) & u32::from(libc::S_IFMT) != u32::from(libc::S_IFSOCK) + || metadata.st_uid != unsafe { libc::geteuid() } + || required_mode.is_some_and(|mode| u32::from(metadata.st_mode) & 0o777 != mode) + { + return Err(Error::DaemonUnavailable( + "workspace daemon socket has unsafe type, owner, or mode".into(), + )); + } + Ok((metadata.st_dev as u64, metadata.st_ino as u64)) +} + +fn verify_secure_socket_leaf_identity( + parent: &File, + leaf: &str, + expected_device: u64, + expected_inode: u64, +) -> Result<(u64, u64)> { + let identity = verify_socket_leaf_owner(parent, leaf, Some(0o600))?; + if identity != (expected_device, expected_inode) { + return Err(Error::DaemonUnavailable( + "workspace daemon socket identity changed; refusing pathname authority".into(), + )); + } + Ok(identity) +} + +fn publish_owner_file(authority: &SecureAuthority, name: &str, bytes: &[u8]) -> Result<()> { + let tmp = format!(".{name}.{}.tmp", random_hex(12)?); + let mut file = openat_file( + &authority.directory, + &tmp, + libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_NOFOLLOW | libc::O_CLOEXEC, + 0o600, + )?; + file.write_all(bytes)?; + file.sync_all()?; + renameat_leaf(&authority.directory, &tmp, name)?; + authority.directory.sync_all()?; + Ok(()) +} + +fn remove_stale_publication( + authority: &SecureAuthority, + endpoint: &WorkspaceDaemonEndpoint, +) -> Result<()> { + remove_socket_leaf_if_identity( + authority, + SOCKET_FILE, + endpoint.socket_device, + endpoint.socket_inode, + true, + true, + )?; + for name in [ENDPOINT_FILE, TOKEN_FILE] { + unlink_authority_file(authority, name)?; + } + authority.directory.sync_all()?; + Ok(()) +} + +#[cfg(debug_assertions)] +fn test_socket_unlink_boundary() -> Result<()> { + let Some(barrier) = std::env::var_os("TRAIL_TEST_WORKSPACE_DAEMON_SOCKET_UNLINK_BARRIER") + else { + return Ok(()); + }; + let barrier = PathBuf::from(barrier); + fs::write(barrier.join("verified"), b"ready")?; + let deadline = Instant::now() + Duration::from_secs(10); + while !barrier.join("continue").exists() { + if Instant::now() >= deadline { + return Err(Error::DaemonUnavailable( + "socket unlink test barrier timed out".into(), + )); + } + std::thread::sleep(Duration::from_millis(1)); + } + Ok(()) +} + +#[cfg(debug_assertions)] +fn test_socket_quarantine_verified_boundary(quarantine: &str) -> Result<()> { + let Some(barrier) = std::env::var_os("TRAIL_TEST_WORKSPACE_DAEMON_SOCKET_QUARANTINE_BARRIER") + else { + return Ok(()); + }; + let barrier = PathBuf::from(barrier); + fs::write(barrier.join("verified"), quarantine.as_bytes())?; + let deadline = Instant::now() + Duration::from_secs(10); + while !barrier.join("continue").exists() { + if Instant::now() >= deadline { + return Err(Error::DaemonUnavailable( + "socket quarantine test barrier timed out".into(), + )); + } + std::thread::sleep(Duration::from_millis(1)); + } + Ok(()) +} + +#[cfg(debug_assertions)] +fn test_socket_quarantine_pre_rename_boundary(quarantine: &str) -> Result<()> { + let Some(barrier) = + std::env::var_os("TRAIL_TEST_WORKSPACE_DAEMON_SOCKET_QUARANTINE_PRE_RENAME_BARRIER") + else { + return Ok(()); + }; + let barrier = PathBuf::from(barrier); + fs::write(barrier.join("prepared"), quarantine.as_bytes())?; + let deadline = Instant::now() + Duration::from_secs(10); + while !barrier.join("continue").exists() { + if Instant::now() >= deadline { + return Err(Error::DaemonUnavailable( + "socket quarantine pre-rename test barrier timed out".into(), + )); + } + std::thread::sleep(Duration::from_millis(1)); + } + Ok(()) +} + +#[cfg(debug_assertions)] +fn test_socket_bound_boundary(leaf: &str) -> Result<()> { + let Some(barrier) = std::env::var_os("TRAIL_TEST_WORKSPACE_DAEMON_SOCKET_BOUND_BARRIER") else { + return Ok(()); + }; + let barrier = PathBuf::from(barrier); + fs::write(barrier.join("bound"), leaf.as_bytes())?; + fs::write(barrier.join("pid"), std::process::id().to_string())?; + let deadline = Instant::now() + Duration::from_secs(10); + while !barrier.join("continue").exists() { + if Instant::now() >= deadline { + return Err(Error::DaemonUnavailable( + "socket bind test barrier timed out".into(), + )); + } + std::thread::sleep(Duration::from_millis(1)); + } + Ok(()) +} + +fn remove_socket_leaf_if_identity( + authority: &SecureAuthority, + leaf: &str, + expected_device: u64, + expected_inode: u64, + missing_ok: bool, + run_test_boundary: bool, +) -> Result<()> { + match socket_leaf_stat(&authority.trail_directory, leaf) { + Ok(_) => { + verify_secure_socket_leaf_identity( + &authority.trail_directory, + leaf, + expected_device, + expected_inode, + )?; + } + Err(error) if missing_ok && error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(Error::Io(error)), + } + + #[cfg(debug_assertions)] + if run_test_boundary { + test_socket_unlink_boundary()?; + } + #[cfg(not(debug_assertions))] + let _ = run_test_boundary; + + ensure_socket_cleanup_artifact_capacity(&authority.trail_directory)?; + let quarantine = format!( + "{SOCKET_TOMBSTONE_PREFIX}{}{SOCKET_TOMBSTONE_SUFFIX}", + random_hex(12)? + ); + #[cfg(debug_assertions)] + test_socket_quarantine_pre_rename_boundary(&quarantine)?; + match renameat_noreplace(&authority.trail_directory, leaf, &quarantine) { + Ok(()) => {} + Err(error) if missing_ok && error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(Error::Io(error)), + } + authority.trail_directory.sync_all()?; + + let captured = verify_socket_leaf_owner(&authority.trail_directory, &quarantine, Some(0o600)); + if captured.as_ref().ok() != Some(&(expected_device, expected_inode)) { + let restore = renameat_noreplace(&authority.trail_directory, &quarantine, leaf); + let _ = authority.trail_directory.sync_all(); + return match restore { + Ok(()) => Err(Error::DaemonUnavailable( + "workspace daemon socket identity changed before removal; restored substituted socket and refused cleanup" + .into(), + )), + Err(error) => Err(Error::DaemonUnavailable(format!( + "workspace daemon socket identity changed before removal and could not be restored without replacing another leaf: {error}" + ))), + }; + } + + #[cfg(debug_assertions)] + test_socket_quarantine_verified_boundary(&quarantine)?; + + // The quarantine name is the last pathname boundary we can verify + // portably. Unlinking it after verification would reopen a same-user + // substitution window, so retain the detached socket inode as an inert + // tombstone. The 96-bit random suffix bounds collisions without requiring + // an unsafe pathname cleanup pass over previously verified tombstones. + authority.trail_directory.sync_all()?; + Ok(()) +} + +fn ensure_socket_cleanup_artifact_capacity(parent: &File) -> Result<()> { + // Foreground stale cleanup holds daemon.lock. A spawned child completes + // its unpublished guard while that caller still owns the lock, and a + // published child remains live until its publication guard runs at exit, + // so ordinary Trail lifecycle cleanup cannot independently overrun this + // count. Same-UID namespace races remain fail-closed at NOREPLACE. + let mut directory = rustix::fs::Dir::read_from(parent) + .map_err(|error| Error::Io(std::io::Error::from(error)))?; + let mut count = 0_usize; + while let Some(entry) = directory.read() { + let entry = entry.map_err(|error| Error::Io(std::io::Error::from(error)))?; + if is_socket_cleanup_artifact_name(entry.file_name().to_bytes()) { + count += 1; + if count >= SOCKET_CLEANUP_ARTIFACT_CAP { + return Err(Error::DaemonUnavailable(format!( + "workspace daemon retained socket cleanup artifact limit ({SOCKET_CLEANUP_ARTIFACT_CAP}) reached; reinitialize this workspace" + ))); + } + } + } + Ok(()) +} + +fn is_socket_cleanup_artifact_name(name: &[u8]) -> bool { + is_socket_tombstone_name(name) || is_private_socket_leaf_name(name) +} + +fn is_socket_tombstone_name(name: &[u8]) -> bool { + let prefix = SOCKET_TOMBSTONE_PREFIX.as_bytes(); + let suffix = SOCKET_TOMBSTONE_SUFFIX.as_bytes(); + if name.len() != prefix.len() + 24 + suffix.len() + || !name.starts_with(prefix) + || !name.ends_with(suffix) + { + return false; + } + name[prefix.len()..prefix.len() + 24] + .iter() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) +} + +fn is_private_socket_leaf_name(name: &[u8]) -> bool { + name.len() == 14 + && name.starts_with(b".s") + && name[2..] + .iter() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) +} + +fn renameat_noreplace(parent: &File, old: &str, new: &str) -> std::io::Result<()> { + renameat_with(parent, old, parent, new, RenameFlags::NOREPLACE).map_err(Into::into) +} + +fn renameat_leaf(parent: &File, old: &str, new: &str) -> Result<()> { + let old = + CString::new(old).map_err(|_| Error::InvalidInput("invalid publication leaf".into()))?; + let new = + CString::new(new).map_err(|_| Error::InvalidInput("invalid publication leaf".into()))?; + if unsafe { + libc::renameat( + parent.as_raw_fd(), + old.as_ptr(), + parent.as_raw_fd(), + new.as_ptr(), + ) + } != 0 + { + return Err(Error::Io(std::io::Error::last_os_error())); + } + Ok(()) +} + +fn unlink_authority_file(authority: &SecureAuthority, name: &str) -> Result<()> { + unlinkat_leaf(&authority.directory, name) +} + +fn unlinkat_leaf(parent: &File, name: &str) -> Result<()> { + let name = + CString::new(name).map_err(|_| Error::InvalidInput("invalid authority leaf".into()))?; + if unsafe { libc::unlinkat(parent.as_raw_fd(), name.as_ptr(), 0) } == 0 { + return Ok(()); + } + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::NotFound { + Ok(()) + } else { + Err(Error::Io(error)) + } +} + +fn random_hex(bytes: usize) -> Result { + let mut value = vec![0_u8; bytes]; + getrandom(&mut value).map_err(|error| { + Error::InvalidInput(format!("workspace daemon entropy failed: {error}")) + })?; + Ok(hex::encode(value)) +} + +fn workspace_identity(workspace: &Path) -> Result { + let canonical = workspace.canonicalize()?; + let metadata = fs::metadata(&canonical)?; + let mut digest = Sha256::new(); + digest.update(canonical.as_os_str().as_encoded_bytes()); + digest.update(metadata.dev().to_be_bytes()); + digest.update(metadata.ino().to_be_bytes()); + Ok(hex::encode(digest.finalize())) +} + +fn executable_identity(path: &Path) -> Result { + let canonical = path.canonicalize()?; + let mut file = File::open(&canonical)?; + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + digest.update(&buffer[..read]); + } + Ok(hex::encode(digest.finalize())) +} + +fn process_is_alive(pid: u32) -> bool { + if pid == 0 || pid > i32::MAX as u32 { + return false; + } + let result = unsafe { libc::kill(pid as i32, 0) }; + result == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) +} + +fn process_start_identity(pid: u32) -> Option { + #[cfg(debug_assertions)] + if std::env::var("TRAIL_TEST_WORKSPACE_DAEMON_UNVERIFIABLE_PID") + .ok() + .and_then(|value| value.parse::().ok()) + == Some(pid) + { + return None; + } + #[cfg(target_os = "linux")] + { + let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + let end = stat.rfind(')')?; + return stat + .get(end + 2..)? + .split_whitespace() + .nth(19) + .map(|value| format!("linux:{value}")); + } + #[cfg(target_os = "macos")] + { + let mut info = unsafe { std::mem::zeroed::() }; + let expected = std::mem::size_of::() as i32; + let read = unsafe { + libc::proc_pidinfo( + pid as i32, + libc::PROC_PIDTBSDINFO, + 0, + (&mut info as *mut libc::proc_bsdinfo).cast(), + expected, + ) + }; + if read != expected || info.pbi_pid != pid { + return None; + } + return Some(format!( + "macos:{}:{}:{}", + info.pbi_pid, info.pbi_start_tvsec, info.pbi_start_tvusec + )); + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = pid; + None + } +} + +fn post_challenge_process_start_identity(pid: u32) -> Option { + #[cfg(debug_assertions)] + if let Ok(value) = std::env::var("TRAIL_TEST_WORKSPACE_DAEMON_POST_CHALLENGE_START_IDENTITY") { + return Some(value); + } + process_start_identity(pid) +} + +fn required_env(name: &str) -> Result { + std::env::var(name).map_err(|_| Error::InvalidInput(format!("workspace daemon missing {name}"))) +} + +fn ready_timeout() -> Duration { + #[cfg(debug_assertions)] + if let Ok(value) = std::env::var("TRAIL_TEST_WORKSPACE_DAEMON_READY_TIMEOUT_MS") { + if let Ok(milliseconds) = value.parse::() { + return Duration::from_millis(milliseconds.max(1)); + } + } + Duration::from_secs(60) +} + +pub(super) fn workspace_from_context(ctx: &RuntimeContext) -> Result { + ctx.workspace + .clone() + .or_else(|| std::env::current_dir().ok()) + .ok_or_else(|| Error::InvalidInput("workspace path is unavailable".into())) +} diff --git a/trail/src/cli/command/handler/errors.rs b/trail/src/cli/command/handler/errors.rs index 7641bb0..d64551b 100644 --- a/trail/src/cli/command/handler/errors.rs +++ b/trail/src/cli/command/handler/errors.rs @@ -2,6 +2,9 @@ use clap::error::ErrorKind as ClapErrorKind; use super::*; +const CLI_PARSE_ERROR_FALLBACK: &str = r#"{"error":{"code":"INVALID_INPUT","status":400,"exit":2,"message":"invalid CLI input","scope":null,"state":null,"reason":null,"recovery":{"command":"trail --help"}}}"#; +const STRUCTURED_ERROR_FALLBACK: &str = r#"{"error":{"code":"SERIALIZATION_ERROR","status":500,"exit":1,"message":"failed to serialize structured error","scope":null,"state":null,"reason":null,"recovery":null}}"#; + pub(super) fn args_request_json_errors(args: I) -> bool where I: IntoIterator, @@ -10,13 +13,13 @@ where for arg in args { let arg = arg.to_string_lossy(); if expect_format { - if arg == "json" { + if arg == "json" || arg == "ndjson" { return true; } expect_format = false; continue; } - if arg == "--json" || arg == "--format=json" { + if arg == "--json" || arg == "--format=json" || arg == "--format=ndjson" { return true; } if arg == "--format" { @@ -28,7 +31,7 @@ where pub(super) fn env_requests_json_errors() -> bool { std::env::var("TRAIL_FORMAT") - .map(|value| value.eq_ignore_ascii_case("json")) + .map(|value| value.eq_ignore_ascii_case("json") || value.eq_ignore_ascii_case("ndjson")) .unwrap_or(false) } @@ -40,7 +43,17 @@ pub(super) fn handle_cli_parse_error(err: clap::Error, json: bool) -> ! { render_cli_parse_error(&err, exit_code); std::process::exit(exit_code); } - _ => err.exit(), + _ => { + let mut diagnostic = UiDiagnostic::new("INVALID_INPUT", "Invalid Trail command"); + diagnostic.cause = Some(err.to_string()); + diagnostic.recovery = Some(UiNextAction { + command: "trail --help".to_string(), + reason: "Show the available commands and global options.".to_string(), + }); + let document = TerminalDocument::empty().block(UiBlock::Diagnostic(diagnostic)); + let _ = render_error_document(&document, &default_error_options()); + std::process::exit(err.exit_code()); + } } } @@ -49,34 +62,379 @@ fn render_cli_parse_error(err: &clap::Error, exit_code: i32) { let value = serde_json::json!({ "error": { "code": "INVALID_INPUT", + "status": 400, + "exit": exit_code, "message": message.trim(), - "exit_code": exit_code + "scope": null, + "state": null, + "reason": null, + "recovery": { "command": "trail --help" } } }); - eprintln!( - "{}", - serde_json::to_string(&value).unwrap_or_else(|_| { - r#"{"error":{"code":"INVALID_INPUT","message":"invalid CLI input","exit_code":2}}"# - .to_string() - }) - ); + let rendered = + serde_json::to_string(&value).unwrap_or_else(|_| CLI_PARSE_ERROR_FALLBACK.to_string()); + let _ = render_structured_error(&rendered); } pub(super) fn render_error(err: &Error, json: bool) { if json { - let value = serde_json::json!({ - "error": { - "code": err.code(), - "message": err.to_string(), - "exit_code": err.exit_code() - } - }); - eprintln!( - "{}", - serde_json::to_string(&value) - .unwrap_or_else(|_| format!(r#"{{"error":{{"message":"{err}"}}}}"#)) + let value = StructuredErrorEnvelope::from_error(err); + let rendered = serde_json::to_string(&value) + .unwrap_or_else(|_| structured_error_fallback(err).to_string()); + let _ = render_structured_error(&rendered); + return; + } + let document = TerminalDocument::empty().block(UiBlock::Diagnostic(diagnostic_for_error(err))); + let _ = render_error_document(&document, &default_error_options()); +} + +fn structured_error_fallback(_err: &Error) -> &'static str { + STRUCTURED_ERROR_FALLBACK +} + +fn default_error_options() -> RenderOptions { + RenderOptions::from_environment( + RenderMode::Human, + ColorPolicy::Auto, + PagerPolicy::Never, + false, + false, + ) +} + +fn diagnostic_for_error(err: &Error) -> UiDiagnostic { + let mut diagnostic = match err { + Error::WorkspaceNotFound(_) => { + let mut diagnostic = UiDiagnostic::new(err.code(), "Trail workspace not found"); + diagnostic.consequence = Some( + "Trail cannot inspect or change work because this directory has no .trail workspace." + .to_string(), + ); + diagnostic.recovery = Some(UiNextAction { + command: "trail init --from-git".to_string(), + reason: "Initialize Trail from the Git-tracked files in this repository." + .to_string(), + }); + diagnostic + } + Error::WorkspaceExists(_) => { + let mut diagnostic = UiDiagnostic::new(err.code(), "Trail workspace already exists"); + diagnostic.consequence = + Some("Trail did not replace the existing workspace state.".to_string()); + diagnostic.recovery = Some(UiNextAction { + command: "trail status".to_string(), + reason: "Inspect the existing workspace before deciding whether to reuse it." + .to_string(), + }); + diagnostic + } + Error::DirtyWorktree | Error::DirtyWorktreeWithMessage(_) => { + let mut diagnostic = UiDiagnostic::new(err.code(), "Worktree has unrecorded changes"); + diagnostic.consequence = Some( + "Trail stopped the operation to protect files that could be overwritten." + .to_string(), + ); + diagnostic.recovery = Some(UiNextAction { + command: "trail status".to_string(), + reason: "Inspect the affected paths before recording, discarding, or moving them." + .to_string(), + }); + diagnostic.alternatives.push(UiNextAction { + command: "trail record -m \"save current work\"".to_string(), + reason: "Record the worktree changes as a Trail operation.".to_string(), + }); + diagnostic + } + Error::Conflict(_) | Error::PatchRejected(_) => { + let mut diagnostic = + UiDiagnostic::new(err.code(), "Patch or merge conflict requires resolution"); + diagnostic.recovery = Some(UiNextAction { + command: "trail conflicts".to_string(), + reason: "Inspect the conflict set and its recommended safe resolution.".to_string(), + }); + diagnostic + } + Error::WorkspaceLocked(_) => { + let mut diagnostic = + UiDiagnostic::new(err.code(), "Workspace is locked by another writer"); + diagnostic.consequence = Some( + "Trail will not risk concurrent writes to the same workspace state.".to_string(), + ); + diagnostic + } + Error::IgnoredPath(path) => { + let mut diagnostic = + UiDiagnostic::new(err.code(), "Path is protected by Trail ignore rules"); + diagnostic.recovery = Some(UiNextAction { + command: format!("trail ignore check {path}"), + reason: "Show the ignore rule that protects this path.".to_string(), + }); + diagnostic + } + Error::PathIndexRequired(_) => { + let mut diagnostic = + UiDiagnostic::new(err.code(), "Trail workspace upgrade is required"); + diagnostic.consequence = Some( + "Trail blocked this mutation until every live branch and lane head has a persistent path-invariant index." + .to_string(), + ); + diagnostic.recovery = Some(UiNextAction { + command: "trail index rebuild".to_string(), + reason: "Upgrade legacy live roots and retry the mutation.".to_string(), + }); + diagnostic + } + Error::SchemaReinitializeRequired { .. } => { + let mut diagnostic = + UiDiagnostic::new(err.code(), "Trail workspace schema must be reinitialized"); + diagnostic.consequence = Some( + "Trail rejected the workspace before opening any mutable storage.".to_string(), + ); + diagnostic.recovery = Some(UiNextAction { + command: "trail init --force".to_string(), + reason: "Back up the workspace, then create the required schema v18.".to_string(), + }); + diagnostic + } + Error::ChangeLedgerReconcileRequired { command, .. } => { + let mut diagnostic = UiDiagnostic::new( + err.code(), + "Changed-path ledger trust could not be established", + ); + diagnostic.recovery = Some(UiNextAction { + command: command.clone(), + reason: "Retry the operation that establishes a trusted changed-path snapshot." + .to_string(), + }); + diagnostic + } + Error::CommittedRepairRequired { .. } => { + let mut diagnostic = UiDiagnostic::new( + err.code(), + "Operation committed; a derived mirror still needs repair", + ); + diagnostic.consequence = Some( + "Do not retry the mutation: its authoritative database transaction already committed." + .to_string(), + ); + diagnostic.recovery = Some(UiNextAction { + command: "trail status".to_string(), + reason: "Reopen authoritative state and idempotently repair ref, marker, and runtime mirrors." + .to_string(), + }); + diagnostic + } + Error::InvalidInput(_) | Error::InvalidPath { .. } => { + let mut diagnostic = + UiDiagnostic::new(err.code(), "Trail cannot use the supplied input"); + diagnostic.recovery = Some(UiNextAction { + command: "trail --help".to_string(), + reason: "Review the command syntax and available options.".to_string(), + }); + diagnostic + } + Error::CloneUnsupported | Error::CloneCrossDevice | Error::NativeCowSourceUnavailable => { + let mut diagnostic = UiDiagnostic::new(err.code(), "Strict native COW is unavailable"); + diagnostic.consequence = Some( + "Trail did not publish a partially cloned workdir or copy bytes for the strict request." + .to_string(), + ); + diagnostic.recovery = Some(UiNextAction { + command: "trail lane spawn --workdir-mode portable-copy".to_string(), + reason: "Use portable materialization with truthful clone/copy reporting." + .to_string(), + }); + diagnostic + } + Error::RefNotFound(_) + | Error::OperationNotFound(_) + | Error::RootNotFound(_) + | Error::ObjectNotFound { .. } => { + let mut diagnostic = + UiDiagnostic::new(err.code(), "Trail could not resolve the requested selector"); + diagnostic.recovery = Some(UiNextAction { + command: "trail timeline --limit 20".to_string(), + reason: "Inspect recent operations and copy an available selector.".to_string(), + }); + diagnostic + } + Error::StaleBranch(_) => { + let mut diagnostic = + UiDiagnostic::new(err.code(), "Branch changed before Trail could apply work"); + diagnostic.consequence = Some( + "Trail did not apply work against a branch that may no longer match its review evidence." + .to_string(), + ); + diagnostic.recovery = Some(UiNextAction { + command: "trail status".to_string(), + reason: "Refresh branch state before reviewing or retrying the operation." + .to_string(), + }); + diagnostic + } + Error::Corrupt(_) => { + let mut diagnostic = + UiDiagnostic::new(err.code(), "Trail detected damaged workspace data"); + diagnostic.consequence = + Some("Trail stopped to avoid making damaged state worse.".to_string()); + diagnostic.recovery = Some(UiNextAction { + command: "trail fsck".to_string(), + reason: "Inspect workspace integrity before attempting repair or restore." + .to_string(), + }); + diagnostic + } + Error::Git(_) => { + let mut diagnostic = + UiDiagnostic::new(err.code(), "Git interoperability command failed"); + diagnostic.recovery = Some(UiNextAction { + command: "git status".to_string(), + reason: "Check the Git worktree and repository state Trail needs to interoperate safely." + .to_string(), + }); + diagnostic + } + Error::GitMappingRequired(_) => { + let mut diagnostic = UiDiagnostic::new(err.code(), "Git baseline mapping is required"); + diagnostic.recovery = Some(UiNextAction { + command: "trail git import-update".to_string(), + reason: "Import the current Git snapshot before retrying the handoff.".to_string(), + }); + diagnostic + } + Error::GitHeadChanged(_) => { + let mut diagnostic = UiDiagnostic::new(err.code(), "Git HEAD changed during handoff"); + diagnostic.recovery = Some(UiNextAction { + command: "git status --short".to_string(), + reason: "Inspect the current Git branch and worktree before retrying.".to_string(), + }); + diagnostic + } + Error::GitWorktreeDirty(_) => { + let mut diagnostic = UiDiagnostic::new(err.code(), "Git tracked worktree has changes"); + diagnostic.recovery = Some(UiNextAction { + command: "git status --short".to_string(), + reason: "Inspect tracked Git changes before retrying the handoff.".to_string(), + }); + diagnostic + } + Error::GitDeltaExportRequired(_) => { + UiDiagnostic::new(err.code(), "Mapped Git delta export is required") + } + Error::DaemonUnavailable(_) | Error::DaemonError { .. } => { + let mut diagnostic = UiDiagnostic::new(err.code(), "Trail daemon request failed"); + diagnostic.recovery = Some(UiNextAction { + command: "trail doctor".to_string(), + reason: "Check local workspace health before retrying the daemon-backed command." + .to_string(), + }); + diagnostic + } + Error::Io(_) + | Error::Sqlite(_) + | Error::Serialization(_) + | Error::Prolly(_) + | Error::ProllySqlite(_) + | Error::ProllySlateDb(_) + | Error::Json(_) + | Error::TomlSer(_) + | Error::TomlDe(_) => { + let mut diagnostic = + UiDiagnostic::new(err.code(), "Trail could not read or write workspace data"); + diagnostic.consequence = Some( + "No further action was taken after the failing storage or configuration operation." + .to_string(), + ); + diagnostic.recovery = Some(UiNextAction { + command: "trail doctor".to_string(), + reason: "Check workspace storage and configuration before retrying.".to_string(), + }); + diagnostic + } + }; + diagnostic.cause = Some(err.to_string()); + diagnostic +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_structured_error_formats() { + assert!(args_request_json_errors([std::ffi::OsString::from( + "--format=ndjson" + )])); + assert!(args_request_json_errors([ + std::ffi::OsString::from("--format"), + std::ffi::OsString::from("json"), + ])); + } + + #[test] + fn cli_parse_error_fallback_is_valid_structured_json() { + let value: serde_json::Value = serde_json::from_str(CLI_PARSE_ERROR_FALLBACK).unwrap(); + assert_eq!(value["error"]["code"], "INVALID_INPUT"); + assert_eq!(value["error"]["recovery"]["command"], "trail --help"); + } + + #[test] + fn general_error_fallback_is_valid_json_with_hostile_error_text() { + let hostile = Error::Serialization("quote: \" backslash: \\ newline:\n tab:\t".into()); + let rendered = structured_error_fallback(&hostile); + let value: serde_json::Value = serde_json::from_str(rendered).unwrap(); + assert_eq!(value["error"]["code"], "SERIALIZATION_ERROR"); + assert_eq!(value["error"]["status"], 500); + assert_eq!(value["error"]["exit"], 1); + assert!(!rendered.contains("quote:")); + } + + #[test] + fn dirty_worktree_has_safe_primary_recovery() { + let diagnostic = diagnostic_for_error(&Error::DirtyWorktree); + assert_eq!(diagnostic.code, "DIRTY_WORKTREE"); + assert_eq!(diagnostic.recovery.unwrap().command, "trail status"); + } + + #[test] + fn missing_git_mapping_recommends_explicit_reconciliation() { + let diagnostic = diagnostic_for_error(&Error::GitMappingRequired("missing".into())); + assert_eq!(diagnostic.code, "GIT_MAPPING_REQUIRED"); + assert_eq!( + diagnostic.recovery.unwrap().command, + "trail git import-update" ); - } else { - eprintln!("trail: {err}"); + } + + #[test] + fn path_index_required_recommends_workspace_upgrade() { + let diagnostic = diagnostic_for_error(&Error::PathIndexRequired( + "legacy root has no case-fold index".into(), + )); + assert_eq!(diagnostic.code, "PATH_INDEX_REQUIRED"); + assert_eq!(diagnostic.summary, "Trail workspace upgrade is required"); + assert!(diagnostic + .consequence + .as_deref() + .is_some_and(|value| value.contains("mutation"))); + assert_eq!(diagnostic.recovery.unwrap().command, "trail index rebuild"); + } + + #[test] + fn stable_error_categories_have_actionable_diagnostics() { + let errors = [ + Error::WorkspaceExists(std::path::PathBuf::from("/workspace")), + Error::PatchRejected("context changed".to_string()), + Error::StaleBranch("main".to_string()), + Error::Corrupt("bad object".to_string()), + Error::Git("git failed".to_string()), + Error::DaemonUnavailable("connection refused".to_string()), + ]; + for error in errors { + let diagnostic = diagnostic_for_error(&error); + assert_eq!(diagnostic.code, error.code()); + assert!(diagnostic.cause.is_some()); + assert!(diagnostic.recovery.is_some()); + } } } diff --git a/trail/src/cli/command/handler/inspect.rs b/trail/src/cli/command/handler/inspect.rs index 53359a4..bc432b9 100644 --- a/trail/src/cli/command/handler/inspect.rs +++ b/trail/src/cli/command/handler/inspect.rs @@ -9,13 +9,13 @@ pub(super) fn handle_timeline_command(ctx: &RuntimeContext, args: TimelineArgs) args.lane.as_deref(), args.limit, )?; - render_timeline(&entries, ctx.json, ctx.quiet) + render_timeline(&entries, ctx.json, &ctx.render) } pub(super) fn handle_show_command(ctx: &RuntimeContext, args: ShowArgs) -> Result<()> { let db = open_db(ctx)?; let result = db.show(&args.selector)?; - render_show(&result, ctx.json, ctx.quiet) + render_show(&result, ctx.json, &ctx.render) } pub(super) fn handle_object_command(ctx: &RuntimeContext, object: ObjectCommand) -> Result<()> { @@ -23,7 +23,7 @@ pub(super) fn handle_object_command(ctx: &RuntimeContext, object: ObjectCommand) ObjectSubcommand::Show(args) => { let db = open_db(ctx)?; let report = db.inspect_object(&args.object_id)?; - render_object_inspect(&report, ctx.json, ctx.quiet) + render_object_inspect(&report, ctx.json, &ctx.render) } } } @@ -33,7 +33,7 @@ pub(super) fn handle_root_command(ctx: &RuntimeContext, root: RootCommand) -> Re RootSubcommand::Show(args) => { let db = open_db(ctx)?; let report = db.inspect_root(&args.root_id)?; - render_root_inspect(&report, ctx.json, ctx.quiet) + render_root_inspect(&report, ctx.json, &ctx.render) } } } @@ -43,7 +43,7 @@ pub(super) fn handle_text_command(ctx: &RuntimeContext, text: TextCommand) -> Re TextSubcommand::Show(args) => { let db = open_db(ctx)?; let report = db.inspect_text(&args.text_id, args.limit)?; - render_text_inspect(&report, ctx.json, ctx.quiet) + render_text_inspect(&report, ctx.json, &ctx.render) } } } @@ -59,7 +59,7 @@ pub(super) fn handle_map_command(ctx: &RuntimeContext, map: MapCommand) -> Resul args.end.as_deref(), args.limit, )?; - render_map_range(&report, ctx.json, ctx.quiet) + render_map_range(&report, ctx.json, &ctx.render) } MapSubcommand::Diff(args) => { let db = open_db(ctx)?; @@ -71,7 +71,7 @@ pub(super) fn handle_map_command(ctx: &RuntimeContext, map: MapCommand) -> Resul args.end.as_deref(), args.limit, )?; - render_map_diff(&report, ctx.json, ctx.quiet) + render_map_diff(&report, ctx.json, &ctx.render) } } } @@ -93,7 +93,7 @@ pub(super) fn handle_why_command(ctx: &RuntimeContext, args: WhyArgs) -> Result< )); } }; - render_why(&result, ctx.json, ctx.quiet) + render_why(&result, ctx.json, &ctx.render) } pub(super) fn handle_history_command(ctx: &RuntimeContext, args: HistoryArgs) -> Result<()> { @@ -117,11 +117,11 @@ pub(super) fn handle_history_command(ctx: &RuntimeContext, args: HistoryArgs) -> )); } }; - render_history(&result, ctx.json, ctx.quiet) + render_history(&result, ctx.json, &ctx.render) } pub(super) fn handle_code_from_command(ctx: &RuntimeContext, args: CodeFromArgs) -> Result<()> { let db = open_db(ctx)?; let result = db.code_from(&args.selector)?; - render_code_from(&result, ctx.json, ctx.quiet) + render_code_from(&result, ctx.json, &ctx.render) } diff --git a/trail/src/cli/command/handler/lane.rs b/trail/src/cli/command/handler/lane.rs index 04081bc..5d591f5 100644 --- a/trail/src/cli/command/handler/lane.rs +++ b/trail/src/cli/command/handler/lane.rs @@ -17,7 +17,7 @@ pub(super) fn handle_lane_command(ctx: &RuntimeContext, lane: LaneCommand) -> Re args.workdir.is_some(), &args.paths, )?; - let report = db.spawn_lane_with_workdir_mode_paths_and_neighbors( + let report = db.spawn_lane_with_deferred_initial_ledger( &args.name, args.from.as_deref(), workdir_mode, @@ -27,67 +27,88 @@ pub(super) fn handle_lane_command(ctx: &RuntimeContext, lane: LaneCommand) -> Re &args.paths, args.include_neighbors, )?; - render_lane_spawn(&report, ctx.json, ctx.quiet) + let report = if report.workdir.is_some() && !report.workdir_mode.is_transparent_cow() { + drop(db); + let mut reopened = open_db(ctx)?; + let report = reopened.resume_deferred_initial_lane_ledger(&args.name)?; + let workspace = reopened.workspace_root().to_path_buf(); + drop(reopened); + daemon_start::retire_workspace_daemon_after_external_generation_change(&workspace)?; + report + } else { + report + }; + render_lane_spawn(&report, ctx.json, &ctx.render) } LaneSubcommand::List => { let db = open_db(ctx)?; let lanes = db.list_lanes()?; - render_lane_list(&lanes, ctx.json, ctx.quiet) + render_lane_list(&lanes, ctx.json, &ctx.render) } LaneSubcommand::Show(args) => { let db = open_db(ctx)?; let details = db.lane_details(&args.name)?; - render_lane_details(&details, ctx.json, ctx.quiet) + render_lane_details(&details, ctx.json, &ctx.render) } LaneSubcommand::Status(args) => { let db = open_db(ctx)?; let report = db.lane_status(&args.name)?; - render_lane_status(&report, ctx.json, ctx.quiet) + render_lane_status(&report, ctx.json, &ctx.render) } LaneSubcommand::Review(args) => { let db = open_db(ctx)?; let report = db.lane_review_packet(&args.name, args.limit)?; - render_lane_review_packet(&report, ctx.json, ctx.quiet) + render_lane_review_packet(&report, ctx.json, &ctx.render) } LaneSubcommand::Contribution(args) => { let db = open_db(ctx)?; let report = db.lane_contribution(&args.name, args.limit)?; - render_lane_contribution(&report, ctx.json, ctx.quiet) + render_lane_contribution(&report, ctx.json, &ctx.render) } LaneSubcommand::Gates(args) => { let db = open_db(ctx)?; let report = db.lane_gate_history(&args.name, args.kind.as_deref(), args.limit)?; - render_lane_gate_history(&report, ctx.json, ctx.quiet) + render_lane_gate_history(&report, ctx.json, &ctx.render) } LaneSubcommand::Readiness(args) => { let db = open_db(ctx)?; let report = db.lane_readiness(&args.name)?; - render_lane_readiness(&report, ctx.json, ctx.quiet) + render_lane_readiness(&report, ctx.json, &ctx.render) + } + LaneSubcommand::Merge(args) => { + let mut db = open_db(ctx)?; + validate_merge_strategy(args.strategy.as_deref())?; + let report = + db.merge_lane_user_with_options(&args.name, &args.into, args.dry_run, args.direct)?; + render_merge(&report, ctx.json, &ctx.render) + } + LaneSubcommand::MergeQueue(queue) => { + collaboration::handle_lane_merge_queue_command(ctx, queue) } LaneSubcommand::RefreshPreview(args) => { let db = open_db(ctx)?; let report = db.preview_lane_refresh(&args.name, &args.target)?; - render_lane_refresh_preview(&report, ctx.json, ctx.quiet) + render_lane_refresh_preview(&report, ctx.json, &ctx.render) } LaneSubcommand::Update(args) => { let mut db = open_db(ctx)?; let report = db.update_layered_lane_from(&args.name, &args.source, args.checkpoint)?; - render_merge(&report, ctx.json, ctx.quiet) + render_merge(&report, ctx.json, &ctx.render) } LaneSubcommand::Handoff(args) => { let db = open_db(ctx)?; let report = db.lane_handoff(&args.name, args.limit)?; - render_lane_handoff(&report, ctx.json, ctx.quiet) + render_lane_handoff(&report, ctx.json, &ctx.render) } LaneSubcommand::Claim(args) => { let mut db = open_db(ctx)?; let report = db.claim_lane_path(&args.name, &args.path, args.ttl_secs)?; - render_lane_claim(&report, ctx.json, ctx.quiet) + render_lane_claim(&report, ctx.json, &ctx.render) } LaneSubcommand::Message(args) => { let mut db = open_db(ctx)?; let report = db.add_lane_message(&args.name, &args.role, &args.text, args.session)?; - render_lane_message(&report, ctx.json, ctx.quiet) + render_lane_message(&report, ctx.json, &ctx.render) } LaneSubcommand::Turn(turn) => turns::handle_turn_command(ctx, turn), LaneSubcommand::Run(run) => runs::handle_run_command(ctx, run), @@ -100,56 +121,44 @@ pub(super) fn handle_lane_command(ctx: &RuntimeContext, lane: LaneCommand) -> Re args.event_type.as_deref(), args.limit, )?; - render_lane_events(&events, ctx.json, ctx.quiet) + render_lane_events(&events, ctx.json, &ctx.render) } LaneSubcommand::Trace(trace) => traces::handle_trace_command(ctx, trace), LaneSubcommand::Record(args) => { let mut db = open_db(ctx)?; if args.preview { let report = db.preview_lane_workdir_record(&args.name)?; - render_lane_record_preview(&report, ctx.json, ctx.quiet) + render_lane_record_preview(&report, ctx.json, &ctx.render) } else { let report = db.record_lane_workdir(&args.name, args.message)?; - render_lane_record(&report, ctx.json, ctx.quiet) + render_lane_record(&report, ctx.json, &ctx.render) } } LaneSubcommand::Checkpoint(args) => { let mut db = open_db(ctx)?; let report = db.checkpoint_lane_workspace(&args.name, args.message)?; - render_workspace_checkpoint(&report, ctx.json, ctx.quiet) + render_workspace_checkpoint(&report, ctx.json, &ctx.render) } LaneSubcommand::Space(args) => { let db = open_db(ctx)?; let report = db.lane_workspace_space(&args.name)?; - render_workspace_space(&report, ctx.json, ctx.quiet) + render_workspace_space(&report, ctx.json, &ctx.render) } LaneSubcommand::Exec(args) => { let db = open_db(ctx)?; let report = db.exec_lane_workspace(&args.name, &args.command)?; - render_workspace_exec(&report, ctx.json, ctx.quiet) + render_workspace_exec(&report, ctx.json, &ctx.render) } LaneSubcommand::Mount(args) => { let db = open_db(ctx)?; let _foreground = args.foreground; - if !ctx.json && !ctx.quiet { - let view = db.lane_workspace_view(&args.name)?.ok_or_else(|| { - Error::InvalidInput(format!( - "lane `{}` does not have a layered workspace view", - args.name - )) - })?; - println!( - "Mounting workspace view {} at {}; run `trail lane unmount {}` to stop it", - view.view_id, view.mountpoint, args.name - ); - } let report = db.mount_lane_workspace_until_requested(&args.name)?; - render_workspace_mount(&report, "unmounted", ctx.json, ctx.quiet) + render_workspace_mount(&report, "mounted", ctx.json, &ctx.render) } LaneSubcommand::Unmount(args) => { let db = open_db(ctx)?; let report = db.request_lane_workspace_unmount(&args.name)?; - render_workspace_mount(&report, "unmounted", ctx.json, ctx.quiet) + render_workspace_mount(&report, "unmounted", ctx.json, &ctx.render) } LaneSubcommand::Rewind(args) => { let mut db = open_db(ctx)?; @@ -159,7 +168,7 @@ pub(super) fn handle_lane_command(ctx: &RuntimeContext, lane: LaneCommand) -> Re args.record_current, args.sync_workdir, )?; - render_lane_rewind(&report, ctx.json, ctx.quiet) + render_lane_rewind(&report, ctx.json, &ctx.render) } LaneSubcommand::Watch(args) => work::handle_watch_command(ctx, args), LaneSubcommand::Test(args) => { @@ -184,7 +193,7 @@ pub(super) fn handle_lane_command(ctx: &RuntimeContext, lane: LaneCommand) -> Re args.force, args.include_neighbors, )?; - render_lane_file_read(&report, ctx.json, ctx.quiet) + render_lane_file_read(&report, ctx.json, &ctx.render) } LaneSubcommand::Hydrate(args) => { let mut db = open_db(ctx)?; @@ -194,12 +203,12 @@ pub(super) fn handle_lane_command(ctx: &RuntimeContext, lane: LaneCommand) -> Re &args.paths, args.include_neighbors, )?; - render_lane_workdir_sync(&report, ctx.json, ctx.quiet) + render_lane_workdir_sync(&report, ctx.json, &ctx.render) } LaneSubcommand::Workdir(args) => { let db = open_db(ctx)?; let report = db.lane_workdir(&args.name)?; - render_lane_workdir(&report, ctx.json, ctx.quiet) + render_lane_workdir(&report, ctx.json, &ctx.render) } LaneSubcommand::SyncWorkdir(args) => { let mut db = open_db(ctx)?; @@ -209,7 +218,7 @@ pub(super) fn handle_lane_command(ctx: &RuntimeContext, lane: LaneCommand) -> Re &args.paths, args.include_neighbors, )?; - render_lane_workdir_sync(&report, ctx.json, ctx.quiet) + render_lane_workdir_sync(&report, ctx.json, &ctx.render) } LaneSubcommand::ApplyPatch(args) => { let mut db = open_db(ctx)?; @@ -222,25 +231,34 @@ pub(super) fn handle_lane_command(ctx: &RuntimeContext, lane: LaneCommand) -> Re patch.allow_stale = true; } let report = db.apply_lane_patch(&args.name, patch)?; - render_lane_patch(&report, ctx.json, ctx.quiet) + render_lane_patch(&report, ctx.json, &ctx.render) } LaneSubcommand::Diff(args) => { let db = open_db(ctx)?; + validate_diff_view( + args.patch, + args.stat, + args.show_line_ids, + args.name_only, + args.name_status, + )?; let summary = db.diff_lane_with_options(&args.name, args.patch, args.show_line_ids)?; let title = format!("Lane diff: {}", args.name); render_diff_with_title( &summary, ctx.json, - ctx.quiet, + &ctx.render, + args.patch, args.stat, - ctx.color, + args.name_only, + args.name_status, Some(&title), ) } LaneSubcommand::Timeline(args) => { let db = open_db(ctx)?; let entries = db.lane_timeline(&args.name, args.limit)?; - render_timeline(&entries, ctx.json, ctx.quiet) + render_timeline(&entries, ctx.json, &ctx.render) } LaneSubcommand::Checkout(args) => { let mut db = open_db(ctx)?; @@ -250,12 +268,12 @@ pub(super) fn handle_lane_command(ctx: &RuntimeContext, lane: LaneCommand) -> Re args.dry_run, args.workdir.as_deref(), )?; - render_checkout(&report, ctx.json, ctx.quiet) + render_checkout(&report, ctx.json, &ctx.render) } LaneSubcommand::Rm(args) => { let mut db = open_db(ctx)?; let report = db.remove_lane(&args.name, args.force)?; - render_lane_remove(&report, ctx.json, ctx.quiet) + render_lane_remove(&report, ctx.json, &ctx.render) } } } diff --git a/trail/src/cli/command/handler/lane/runs.rs b/trail/src/cli/command/handler/lane/runs.rs index acd7bd7..ca9b052 100644 --- a/trail/src/cli/command/handler/lane/runs.rs +++ b/trail/src/cli/command/handler/lane/runs.rs @@ -15,23 +15,23 @@ pub(super) fn handle_run_command(ctx: &RuntimeContext, run: LaneRunCommand) -> R args.session.as_deref(), args.turn.as_deref(), )?; - render_lane_run_pause(&report, ctx.json, ctx.quiet) + render_lane_run_pause(&report, ctx.json, &ctx.render) } LaneRunSubcommand::List(args) => { let db = open_db(ctx)?; let run_states = db.list_lane_run_states(args.lane.as_deref(), args.status.as_deref())?; - render_lane_run_list(&run_states, ctx.json, ctx.quiet) + render_lane_run_list(&run_states, ctx.json, &ctx.render) } LaneRunSubcommand::Show(args) => { let db = open_db(ctx)?; let run_state = db.show_lane_run_state(&args.run_id)?; - render_lane_run_state(&run_state, ctx.json, ctx.quiet) + render_lane_run_state(&run_state, ctx.json, &ctx.render) } LaneRunSubcommand::Resume(args) => { let mut db = open_db(ctx)?; let report = db.resume_lane_run(&args.run_id, args.reviewer, args.note)?; - render_lane_run_resume(&report, ctx.json, ctx.quiet) + render_lane_run_resume(&report, ctx.json, &ctx.render) } } } diff --git a/trail/src/cli/command/handler/lane/traces.rs b/trail/src/cli/command/handler/lane/traces.rs index b454404..a7e8316 100644 --- a/trail/src/cli/command/handler/lane/traces.rs +++ b/trail/src/cli/command/handler/lane/traces.rs @@ -13,13 +13,13 @@ pub(super) fn handle_trace_command(ctx: &RuntimeContext, trace: LaneTraceCommand args.trace_id.as_deref(), attributes, )?; - render_lane_trace_span_start(&report, ctx.json, ctx.quiet) + render_lane_trace_span_start(&report, ctx.json, &ctx.render) } LaneTraceSubcommand::End(args) => { let mut db = open_db(ctx)?; let result = parse_optional_json(args.result_json.as_deref())?; let report = db.end_lane_trace_span(&args.span_id, &args.status, result)?; - render_lane_trace_span_end(&report, ctx.json, ctx.quiet) + render_lane_trace_span_end(&report, ctx.json, &ctx.render) } LaneTraceSubcommand::List(args) => { let db = open_db(ctx)?; @@ -30,7 +30,7 @@ pub(super) fn handle_trace_command(ctx: &RuntimeContext, trace: LaneTraceCommand args.trace_id.as_deref(), args.limit, )?; - render_lane_trace_spans(&spans, ctx.json, ctx.quiet) + render_lane_trace_spans(&spans, ctx.json, &ctx.render) } LaneTraceSubcommand::Summary(args) => { let db = open_db(ctx)?; @@ -41,12 +41,12 @@ pub(super) fn handle_trace_command(ctx: &RuntimeContext, trace: LaneTraceCommand args.trace_id.as_deref(), args.slowest_limit, )?; - render_lane_trace_summary(&report, ctx.json, ctx.quiet) + render_lane_trace_summary(&report, ctx.json, &ctx.render) } LaneTraceSubcommand::Show(args) => { let db = open_db(ctx)?; let span = db.show_lane_trace_span(&args.span_id)?; - render_lane_trace_span(&span, ctx.json, ctx.quiet) + render_lane_trace_span(&span, ctx.json, &ctx.render) } } } diff --git a/trail/src/cli/command/handler/lane/turns.rs b/trail/src/cli/command/handler/lane/turns.rs index e263659..68b8cf3 100644 --- a/trail/src/cli/command/handler/lane/turns.rs +++ b/trail/src/cli/command/handler/lane/turns.rs @@ -10,17 +10,17 @@ pub(super) fn handle_turn_command(ctx: &RuntimeContext, turn: LaneTurnCommand) - args.title, args.base_change.as_deref(), )?; - render_lane_turn_start(&report, ctx.json, ctx.quiet) + render_lane_turn_start(&report, ctx.json, &ctx.render) } LaneTurnSubcommand::Show(args) => { let db = open_db(ctx)?; let details = db.show_lane_turn(&args.turn_id)?; - render_lane_turn_details(&details, ctx.json, ctx.quiet) + render_lane_turn_details(&details, ctx.json, &ctx.render) } LaneTurnSubcommand::Message(args) => { let mut db = open_db(ctx)?; let report = db.add_lane_turn_message(&args.turn_id, &args.role, &args.text)?; - render_lane_message(&report, ctx.json, ctx.quiet) + render_lane_message(&report, ctx.json, &ctx.render) } LaneTurnSubcommand::Event(args) => { let mut db = open_db(ctx)?; @@ -32,7 +32,7 @@ pub(super) fn handle_turn_command(ctx: &RuntimeContext, turn: LaneTurnCommand) - args.change.as_deref(), args.message.as_deref(), )?; - render_lane_turn_event(&report, ctx.json, ctx.quiet) + render_lane_turn_event(&report, ctx.json, &ctx.render) } LaneTurnSubcommand::ApplyPatch(args) => { let mut db = open_db(ctx)?; @@ -45,12 +45,12 @@ pub(super) fn handle_turn_command(ctx: &RuntimeContext, turn: LaneTurnCommand) - patch.allow_stale = true; } let report = db.apply_lane_turn_patch(&args.turn_id, patch)?; - render_lane_patch(&report, ctx.json, ctx.quiet) + render_lane_patch(&report, ctx.json, &ctx.render) } LaneTurnSubcommand::End(args) => { let mut db = open_db(ctx)?; let report = db.end_lane_turn(&args.turn_id, &args.status)?; - render_lane_turn_end(&report, ctx.json, ctx.quiet) + render_lane_turn_end(&report, ctx.json, &ctx.render) } } } diff --git a/trail/src/cli/command/handler/lane/work.rs b/trail/src/cli/command/handler/lane/work.rs index fb14cf8..573c734 100644 --- a/trail/src/cli/command/handler/lane/work.rs +++ b/trail/src/cli/command/handler/lane/work.rs @@ -7,14 +7,14 @@ pub(super) fn handle_watch_command(ctx: &RuntimeContext, args: LaneWatchArgs) -> let _include_untracked = args.include_untracked; if args.once { let report = db.watch_lane_workdir(&args.name, args.message, interval, Some(1))?; - render_lane_watch(&report, ctx.json, ctx.quiet) + render_lane_watch(&report, ctx.json, &ctx.render) } else { loop { let report = db.record_lane_workdir(&args.name, args.message.clone())?; if matches!(ctx.format, OutputFormat::Ndjson) { - println!("{}", serde_json::to_string(&report)?); + render_ndjson(&report)?; } else if report.operation.is_some() { - render_lane_record(&report, ctx.json, ctx.quiet)?; + render_lane_record(&report, ctx.json, &ctx.render)?; } thread::sleep(interval); } @@ -53,7 +53,7 @@ pub(super) fn handle_gate_command( options, )?, }; - let render_result = render_lane_test(&report, ctx.json, ctx.quiet); + let render_result = render_lane_test(&report, ctx.json, &ctx.render); if render_result.is_ok() && !report.success { std::process::exit(command_failure_exit_code(report.exit_code)); } diff --git a/trail/src/cli/command/handler/maintenance.rs b/trail/src/cli/command/handler/maintenance.rs index b322e38..3466e3a 100644 --- a/trail/src/cli/command/handler/maintenance.rs +++ b/trail/src/cli/command/handler/maintenance.rs @@ -20,30 +20,31 @@ pub(super) fn handle_git_command(ctx: &RuntimeContext, git: GitCommand) -> Resul } let mut db = open_db(ctx)?; let report = db.git_export_commit(&args.range, &message)?; - render_git_export(&report, ctx.json, ctx.quiet) + render_git_export(&report, ctx.json, &ctx.render) } else if let Some(output) = args.output { let db = open_db(ctx)?; db.write_patch_to(&args.range, &output)?; - if !ctx.quiet { - println!("Wrote patch: {}", output.display()); - } - Ok(()) + render_document( + &TerminalDocument::new("Patch written", UiTone::Success).block( + UiBlock::Metadata(vec![("Path".to_string(), output.display().to_string())]), + ), + &ctx.render, + ) } else { let db = open_db(ctx)?; let patch = db.export_patch(&args.range)?; - print!("{patch}"); - Ok(()) + render_raw_content(&patch, &ctx.render) } } GitSubcommand::ImportUpdate(args) => { let mut db = open_db(ctx)?; let report = db.git_import_update(ctx.branch.as_deref(), args.message)?; - render_git_import_update(&report, ctx.json, ctx.quiet) + render_git_import_update(&report, ctx.json, &ctx.render) } GitSubcommand::Mappings(args) => { let db = open_db(ctx)?; let mappings = db.git_mappings(args.limit)?; - render_git_mappings(&mappings, ctx.json, ctx.quiet) + render_git_mappings(&mappings, ctx.json, &ctx.render) } } } @@ -56,7 +57,21 @@ pub(super) fn handle_api_command(_ctx: &RuntimeContext, api: ApiCommand) -> Resu if let Some(output) = args.output { fs::write(output, json)?; } else { - println!("{json}"); + render_raw_content( + &format!("{json}\n"), + &RenderOptions { + mode: RenderMode::Plain, + color: false, + glyphs: GlyphSet::Ascii, + width: 80, + height: 24, + stdout_is_terminal: false, + stderr_is_terminal: false, + verbose: false, + quiet: false, + pager: PagerPolicy::Never, + }, + )?; } Ok(()) } @@ -65,6 +80,9 @@ pub(super) fn handle_api_command(_ctx: &RuntimeContext, api: ApiCommand) -> Resu pub(super) fn handle_daemon_command(ctx: &RuntimeContext, args: DaemonArgs) -> Result<()> { let mut db = open_db(ctx)?; + if daemon_start::is_auto_workspace_daemon() { + return daemon_start::run_auto_workspace_daemon(db); + } let addr = daemon_listen_addr(&args)?; validate_daemon_listen_security(&args, addr)?; let rate_limit = daemon_rate_limit(&args)?; @@ -85,34 +103,66 @@ pub(super) fn handle_daemon_command(ctx: &RuntimeContext, args: DaemonArgs) -> R )?; let endpoint_writer = endpoint_registration.writer(); let quiet = ctx.quiet; + let error_options = ctx.render.clone(); thread::spawn(move || { if let Err(err) = cache_warmup .run() .and_then(|_| endpoint_writer.write_if_alive()) { if !quiet { - eprintln!("Trail daemon cache warmup failed: {err}"); + let diagnostic = UiDiagnostic { + code: err.code().to_string(), + summary: "Trail daemon cache warmup failed".to_string(), + cause: Some(err.to_string()), + consequence: Some( + "Requests may rebuild workspace cache entries before responding." + .to_string(), + ), + recovery: None, + alternatives: Vec::new(), + }; + let _ = render_error_document( + &TerminalDocument::empty().block(UiBlock::Diagnostic(diagnostic)), + &error_options, + ); } } }); - if !ctx.quiet { - println!("Trail API listening on {daemon_url}"); - } if args.no_auth { - eprintln!( - "WARNING: Trail daemon auth is disabled; any local process can mutate this workspace through {daemon_url}. Keep the listener on loopback and use this only for trusted local automation." - ); - } else if !ctx.quiet { + render_error_document( + &TerminalDocument::new("WARNING: daemon auth is disabled", UiTone::Attention) + .context("Any local process can mutate this workspace through the daemon.") + .block(UiBlock::Metadata(vec![( + "Endpoint".to_string(), + daemon_url.clone(), + )])) + .block(UiBlock::Notice( + "Keep the listener on loopback and use this only for trusted local automation." + .to_string(), + )), + &ctx.render, + )?; + } + let mut metadata = vec![("Endpoint".to_string(), daemon_url.clone())]; + if !args.no_auth { if let Some(path) = token_file { - println!("Daemon token file: {}", path.display()); - println!( - "Send requests with: Authorization: Bearer $(cat {})", - path.display() - ); + metadata.push(("Token file".to_string(), path.display().to_string())); + metadata.push(( + "Authorization".to_string(), + format!("Bearer token from {}", path.display()), + )); } else { - println!("Daemon token configured from flag or TRAIL_DAEMON_TOKEN"); + metadata.push(( + "Authorization".to_string(), + "token from --auth-token or TRAIL_DAEMON_TOKEN".to_string(), + )); } } + render_document( + &TerminalDocument::new("Trail API daemon listening", UiTone::Success) + .block(UiBlock::Metadata(metadata)), + &ctx.render, + )?; let max_requests = if args.once { Some(1) } else { @@ -233,7 +283,7 @@ pub(super) fn handle_mcp_command(ctx: &RuntimeContext) -> Result<()> { pub(super) fn handle_doctor_command(ctx: &RuntimeContext) -> Result<()> { let db = open_db(ctx)?; let report = db.doctor()?; - render_doctor(&report, ctx.json, ctx.quiet) + render_doctor(&report, ctx.json, &ctx.render) } pub(super) fn handle_backup_command(ctx: &RuntimeContext, backup: BackupCommand) -> Result<()> { @@ -241,11 +291,11 @@ pub(super) fn handle_backup_command(ctx: &RuntimeContext, backup: BackupCommand) BackupSubcommand::Create(args) => { let db = open_db(ctx)?; let report = db.create_backup(args.output, args.overwrite)?; - render_backup_create(&report, ctx.json, ctx.quiet) + render_backup_create(&report, ctx.json, &ctx.render) } BackupSubcommand::Verify(args) => { let report = Trail::verify_backup(args.path)?; - render_backup_verify(&report, ctx.json, ctx.quiet) + render_backup_verify(&report, ctx.json, &ctx.render) } BackupSubcommand::Restore(args) => { let workspace = ctx @@ -253,7 +303,7 @@ pub(super) fn handle_backup_command(ctx: &RuntimeContext, backup: BackupCommand) .clone() .unwrap_or(std::env::current_dir().map_err(Error::from)?); let report = Trail::restore_backup(workspace, args.path, args.force)?; - render_backup_restore(&report, ctx.json, ctx.quiet) + render_backup_restore(&report, ctx.json, &ctx.render) } } } @@ -261,11 +311,21 @@ pub(super) fn handle_backup_command(ctx: &RuntimeContext, backup: BackupCommand) pub(super) fn handle_fsck_command(ctx: &RuntimeContext) -> Result<()> { let db = open_db(ctx)?; let report = db.fsck()?; - render_fsck(&report, ctx.json, ctx.quiet) + render_fsck(&report, ctx.json, &ctx.render) } pub(super) fn handle_index_command(ctx: &RuntimeContext, index: IndexCommand) -> Result<()> { match index.command { + IndexSubcommand::Reconcile(args) => { + let mut db = open_db(ctx)?; + let report = + trail::server::reconcile_changed_path_ledger(&mut db, args.lane.as_deref())?; + if matches!(ctx.format, OutputFormat::Ndjson) { + render_ndjson(&report) + } else { + render_change_ledger_reconcile(&report, ctx.json, &ctx.render) + } + } IndexSubcommand::Rebuild(args) => { let mut db = open_db(ctx)?; let report = if args.rich_text { @@ -273,7 +333,7 @@ pub(super) fn handle_index_command(ctx: &RuntimeContext, index: IndexCommand) -> } else { db.rebuild_indexes()? }; - render_index_rebuild(&report, ctx.json, ctx.quiet) + render_index_rebuild(&report, ctx.json, &ctx.render) } IndexSubcommand::Watch(args) => { if args.interval_ms == 0 { @@ -288,9 +348,9 @@ pub(super) fn handle_index_command(ctx: &RuntimeContext, index: IndexCommand) -> let report = db.refresh_worktree_index()?; iterations += 1; if matches!(ctx.format, OutputFormat::Ndjson) { - println!("{}", serde_json::to_string(&report)?); + render_ndjson(&report)?; } else { - render_worktree_index(&report, ctx.json, ctx.quiet)?; + render_worktree_index(&report, ctx.json, &ctx.render)?; } if args.once || args.iterations.is_some_and(|max| iterations >= max) { break; @@ -305,7 +365,7 @@ pub(super) fn handle_index_command(ctx: &RuntimeContext, index: IndexCommand) -> pub(super) fn handle_gc_command(ctx: &RuntimeContext, args: GcArgs) -> Result<()> { let mut db = open_db(ctx)?; let report = db.gc(args.dry_run)?; - render_gc(&report, ctx.json, ctx.quiet) + render_gc(&report, ctx.json, &ctx.render) } fn daemon_auth( diff --git a/trail/src/cli/command/handler/runtime.rs b/trail/src/cli/command/handler/runtime.rs index 15110b5..3073183 100644 --- a/trail/src/cli/command/handler/runtime.rs +++ b/trail/src/cli/command/handler/runtime.rs @@ -6,8 +6,8 @@ pub(super) struct RuntimeContext { pub(super) branch: Option, pub(super) json: bool, pub(super) quiet: bool, - pub(super) color: bool, pub(super) format: OutputFormat, + pub(super) render: RenderOptions, } pub(super) fn resolve_output_format(cli_format: Option) -> Result { @@ -19,10 +19,11 @@ pub(super) fn resolve_output_format(cli_format: Option) -> Result< }; match value.trim().to_ascii_lowercase().as_str() { "" | "human" => Ok(OutputFormat::Human), + "plain" => Ok(OutputFormat::Plain), "json" => Ok(OutputFormat::Json), "ndjson" => Ok(OutputFormat::Ndjson), other => Err(Error::InvalidInput(format!( - "TRAIL_FORMAT must be human, json, or ndjson, got `{other}`" + "TRAIL_FORMAT must be human, plain, json, or ndjson, got `{other}`" ))), } } diff --git a/trail/src/cli/command/handler/workspace.rs b/trail/src/cli/command/handler/workspace.rs index 0ff710e..dd11f7e 100644 --- a/trail/src/cli/command/handler/workspace.rs +++ b/trail/src/cli/command/handler/workspace.rs @@ -5,17 +5,17 @@ pub(super) fn handle_config_command(ctx: &RuntimeContext, config: ConfigCommand) ConfigSubcommand::List => { let db = open_db(ctx)?; let entries = db.config_entries(); - render_config_list(&entries, ctx.json, ctx.quiet) + render_config_list(&entries, ctx.json, &ctx.render) } ConfigSubcommand::Get(args) => { let db = open_db(ctx)?; let entry = db.config_get(&args.key)?; - render_config_entry(&entry, ctx.json, ctx.quiet) + render_config_entry(&entry, ctx.json, &ctx.render) } ConfigSubcommand::Set(args) => { let mut db = open_db(ctx)?; let report = db.config_set(&args.key, &args.value)?; - render_config_set(&report, ctx.json, ctx.quiet) + render_config_set(&report, ctx.json, &ctx.render) } } } @@ -25,22 +25,22 @@ pub(super) fn handle_ignore_command(ctx: &RuntimeContext, ignore: IgnoreCommand) IgnoreSubcommand::List => { let db = open_db(ctx)?; let report = db.ignore_list()?; - render_ignore_list(&report, ctx.json, ctx.quiet) + render_ignore_list(&report, ctx.json, &ctx.render) } IgnoreSubcommand::Add(args) => { let mut db = open_db(ctx)?; let report = db.ignore_add(&args.pattern)?; - render_ignore_add(&report, ctx.json, ctx.quiet) + render_ignore_add(&report, ctx.json, &ctx.render) } IgnoreSubcommand::Remove(args) => { let mut db = open_db(ctx)?; let report = db.ignore_remove(&args.pattern)?; - render_ignore_remove(&report, ctx.json, ctx.quiet) + render_ignore_remove(&report, ctx.json, &ctx.render) } IgnoreSubcommand::Check(args) => { let db = open_db(ctx)?; let report = db.ignore_check(&args.path)?; - render_ignore_check(&report, ctx.json, ctx.quiet) + render_ignore_check(&report, ctx.json, &ctx.render) } } } @@ -60,7 +60,7 @@ pub(super) fn handle_guardrails_command( payload, &args.paths, )?; - render_guardrail_check(&report, ctx.json, ctx.quiet) + render_guardrail_check(&report, ctx.json, &ctx.render) } } } diff --git a/trail/src/cli/command/handler/worktree.rs b/trail/src/cli/command/handler/worktree.rs index a026f76..57fbd25 100644 --- a/trail/src/cli/command/handler/worktree.rs +++ b/trail/src/cli/command/handler/worktree.rs @@ -5,7 +5,7 @@ pub(super) fn handle_status_command(ctx: &RuntimeContext, args: StatusArgs) -> R let db = open_db(ctx)?; let branch = args.branch.as_deref().or(ctx.branch.as_deref()); let report = db.status(branch)?; - render_status(&report, ctx.json, ctx.quiet) + render_status(&report, ctx.json, &ctx.render) } pub(super) fn handle_record_command(ctx: &RuntimeContext, args: RecordArgs) -> Result<()> { @@ -26,7 +26,7 @@ pub(super) fn handle_record_command(ctx: &RuntimeContext, args: RecordArgs) -> R allow_ignored: args.allow_ignored, }, )?; - render_record(&report, ctx.json, ctx.quiet) + render_record(&report, ctx.json, &ctx.render) } pub(super) fn handle_watch_command(ctx: &RuntimeContext, args: WatchArgs) -> Result<()> { @@ -34,6 +34,7 @@ pub(super) fn handle_watch_command(ctx: &RuntimeContext, args: WatchArgs) -> Res let interval = watch_interval(args.interval_secs, args.debounce_ms)?; let _include_untracked = args.include_untracked; loop { + let mut progress = TransientProgress::start(&ctx.render, "Scanning worktree…"); let report = db.record_with_options( ctx.branch.as_deref(), args.message.clone(), @@ -44,10 +45,11 @@ pub(super) fn handle_watch_command(ctx: &RuntimeContext, args: WatchArgs) -> Res ..RecordOptions::default() }, )?; + progress.finish(); if matches!(ctx.format, OutputFormat::Ndjson) { - println!("{}", serde_json::to_string(&report)?); + render_ndjson(&report)?; } else if report.operation.is_some() { - render_record(&report, ctx.json, ctx.quiet)?; + render_record(&report, ctx.json, &ctx.render)?; } if args.once { break; @@ -60,7 +62,15 @@ pub(super) fn handle_watch_command(ctx: &RuntimeContext, args: WatchArgs) -> Res pub(super) fn handle_diff_command(ctx: &RuntimeContext, args: DiffArgs) -> Result<()> { let mut db = open_db(ctx)?; let summary = diff_from_args(&mut db, &args)?; - render_diff(&summary, ctx.json, ctx.quiet, args.stat, ctx.color) + render_diff( + &summary, + ctx.json, + &ctx.render, + args.patch, + args.stat, + args.name_only, + args.name_status, + ) } pub(super) fn handle_checkout_command(ctx: &RuntimeContext, args: CheckoutArgs) -> Result<()> { @@ -72,7 +82,7 @@ pub(super) fn handle_checkout_command(ctx: &RuntimeContext, args: CheckoutArgs) args.workdir.as_deref(), args.record_dirty, )?; - render_checkout(&report, ctx.json, ctx.quiet) + render_checkout(&report, ctx.json, &ctx.render) } pub(super) fn handle_branch_command(ctx: &RuntimeContext, args: BranchArgs) -> Result<()> { @@ -85,19 +95,19 @@ pub(super) fn handle_branch_command(ctx: &RuntimeContext, args: BranchArgs) -> R ) { (Some(name), None, None, None) => { let report = db.create_branch(name, args.from.as_deref())?; - render_branch(&report, ctx.json, ctx.quiet) + render_branch(&report, ctx.json, &ctx.render) } (None, Some(name), None, None) => { let report = db.delete_branch(name)?; - render_branch_delete(&report, ctx.json, ctx.quiet) + render_branch_delete(&report, ctx.json, &ctx.render) } (None, None, Some(old_name), Some(new_name)) => { let report = db.rename_branch(old_name, new_name)?; - render_branch_rename(&report, ctx.json, ctx.quiet) + render_branch_rename(&report, ctx.json, &ctx.render) } (None, None, None, None) => { let entries = db.list_branches()?; - render_branch_list(&entries, ctx.json, ctx.quiet) + render_branch_list(&entries, ctx.json, &ctx.render) } _ => Err(Error::InvalidInput( "branch accepts either NAME [--from REF], --delete NAME, --rename OLD --to NEW, or no arguments".to_string(), @@ -109,5 +119,5 @@ pub(super) fn handle_merge_command(ctx: &RuntimeContext, args: MergeArgs) -> Res let mut db = open_db(ctx)?; validate_merge_strategy(args.strategy.as_deref())?; let report = db.merge_branches_with_options(&args.source, &args.into, args.dry_run)?; - render_merge(&report, ctx.json, ctx.quiet) + render_merge(&report, ctx.json, &ctx.render) } diff --git a/trail/src/cli/command/lane_args.rs b/trail/src/cli/command/lane_args.rs index a7637f3..911509f 100644 --- a/trail/src/cli/command/lane_args.rs +++ b/trail/src/cli/command/lane_args.rs @@ -2,6 +2,8 @@ use std::path::PathBuf; use clap::{Args, Subcommand}; +use super::LaneMergeQueueCommand; + mod run; mod trace; mod turn; @@ -10,6 +12,19 @@ pub(super) use run::*; pub(super) use trace::*; pub(super) use turn::*; +#[derive(Args)] +pub(super) struct LaneMergeArgs { + pub(super) name: String, + #[arg(long, default_value = "main")] + pub(super) into: String, + #[arg(long)] + pub(super) strategy: Option, + #[arg(long)] + pub(super) dry_run: bool, + #[arg(long)] + pub(super) direct: bool, +} + #[derive(Subcommand)] pub(super) enum LaneSubcommand { /// Create a new lane branch and optional materialized workdir. @@ -28,6 +43,10 @@ pub(super) enum LaneSubcommand { Gates(LaneGatesArgs), /// Compute lane merge-readiness including blockers and warnings. Readiness(LaneReadinessArgs), + /// Merge a ready lane into a target branch, or preview the merge safely. + Merge(LaneMergeArgs), + /// Schedule and run serialized lane merges. + MergeQueue(LaneMergeQueueCommand), /// Preview refreshing a lane onto a target branch before merge. RefreshPreview(LaneRefreshPreviewArgs), /// Merge a branch into a layered lane and advance its pinned view generation. @@ -109,7 +128,7 @@ pub(super) struct LaneSpawnArgs { pub(super) no_materialize: bool, #[arg( long, - value_parser = ["auto", "virtual", "sparse", "full-cow", "overlay-cow", "nfs-cow"] + value_parser = ["auto", "virtual", "sparse", "native-cow", "portable-copy", "fuse-cow", "nfs-cow", "dokan-cow"] )] pub(super) workdir_mode: Option, #[arg(long)] @@ -347,6 +366,10 @@ pub(super) struct LaneDiffArgs { pub(super) patch: bool, #[arg(long)] pub(super) stat: bool, + #[arg(long = "name-only")] + pub(super) name_only: bool, + #[arg(long = "name-status")] + pub(super) name_status: bool, #[arg(long = "show-line-ids")] pub(super) show_line_ids: bool, } diff --git a/trail/src/cli/command/maintenance_args.rs b/trail/src/cli/command/maintenance_args.rs index ed8f035..f0a985a 100644 --- a/trail/src/cli/command/maintenance_args.rs +++ b/trail/src/cli/command/maintenance_args.rs @@ -83,12 +83,21 @@ pub(super) struct DaemonArgs { #[derive(Subcommand)] pub(super) enum IndexSubcommand { + /// Reconcile the changed-path ledger with a complete filesystem walk. + Reconcile(IndexReconcileArgs), /// Rebuild all derived indexes from current workspace state. Rebuild(IndexRebuildArgs), /// Continuously refresh the persisted worktree file index. Watch(IndexWatchArgs), } +#[derive(Args)] +pub(super) struct IndexReconcileArgs { + /// Reconcile a materialized lane instead of the workspace. + #[arg(long)] + pub(super) lane: Option, +} + #[derive(Args)] pub(super) struct IndexCommand { #[command(subcommand)] diff --git a/trail/src/cli/command/render.rs b/trail/src/cli/command/render.rs index d28cdc9..153d170 100644 --- a/trail/src/cli/command/render.rs +++ b/trail/src/cli/command/render.rs @@ -11,6 +11,7 @@ mod ignore; mod inspection; mod lane; mod maintenance; +mod ui; mod workspace; pub(crate) use acp::*; @@ -24,9 +25,35 @@ pub(crate) use ignore::*; pub(crate) use inspection::*; pub(crate) use lane::*; pub(crate) use maintenance::*; +pub(crate) use ui::*; pub(crate) use workspace::*; pub(crate) fn render_json(value: &T) -> Result<()> { - println!("{}", serde_json::to_string_pretty(value)?); - Ok(()) + render_structured_content(&serde_json::to_string_pretty(value)?) +} + +/// Emits one JSON record for a command whose documented contract is a record +/// stream. Human formatting must never use this path. +pub(crate) fn render_ndjson(value: &T) -> Result<()> { + render_structured_content(&serde_json::to_string(value)?) +} + +fn render_structured_content(content: &str) -> Result<()> { + let mut content = content.to_string(); + content.push('\n'); + render_raw_content( + &content, + &RenderOptions { + mode: RenderMode::Plain, + color: false, + glyphs: GlyphSet::Ascii, + width: 80, + height: 24, + stdout_is_terminal: false, + stderr_is_terminal: false, + verbose: false, + quiet: false, + pager: PagerPolicy::Never, + }, + ) } diff --git a/trail/src/cli/command/render/acp.rs b/trail/src/cli/command/render/acp.rs index 35035dd..6c77631 100644 --- a/trail/src/cli/command/render/acp.rs +++ b/trail/src/cli/command/render/acp.rs @@ -1,4 +1,4 @@ -use super::render_json; +use super::*; use trail::model::*; use trail::Result; @@ -6,141 +6,261 @@ use trail::Result; pub(crate) fn render_acp_profiles( profiles: &[AcpProviderProfile], json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(profiles); } - if !quiet { - for profile in profiles { - let status = if profile.available { - "available" - } else { - "missing" - }; - println!("{} ({status})", profile.agent); - println!( - " capabilities: acp={} mcp={} terminal={}", - profile.supports_acp, profile.supports_mcp, profile.supports_terminal - ); - println!(" {}", shell_join(&profile.relay_command)); - for note in &profile.notes { - println!(" note: {note}"); - } - } + if profiles.is_empty() { + return render_document( + &TerminalDocument::new("No ACP providers", UiTone::Neutral), + options, + ); } - Ok(()) + render_document( + &TerminalDocument::new( + format!("{} ACP provider(s)", profiles.len()), + UiTone::Neutral, + ) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("PROVIDER", 0, 10), + UiColumn::left("STATUS", 0, 9), + UiColumn::left("ACP", 1, 4), + UiColumn::left("MCP", 1, 4), + UiColumn::left("TERMINAL", 1, 8), + UiColumn::left("RELAY", 0, 18), + ], + profiles + .iter() + .map(|profile| { + vec![ + profile.agent.clone(), + if profile.available { + "available" + } else { + "missing" + } + .to_string(), + profile.supports_acp.to_string(), + profile.supports_mcp.to_string(), + profile.supports_terminal.to_string(), + shell_join(&profile.relay_command), + ] + }) + .collect(), + ))), + options, + ) } -pub(crate) fn render_acp_install( - report: &AcpInstallReport, +pub(crate) fn render_acp_setup( + report: &trail::acp::AcpSetupReport, json: bool, - quiet: bool, + options: &RenderOptions, print_only: bool, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - if print_only { - println!("{}", report.snippet); - return Ok(()); - } - let detected = if report.detected { "yes" } else { "no" }; - println!("ACP install plan: {}", report.agent); - println!("Editor: {}", report.editor); - println!("Detected: {detected}"); - println!("Relay command:"); - println!(" {}", shell_join(&report.relay_command)); - if !report.warnings.is_empty() { - println!("Warnings:"); - for warning in &report.warnings { - println!(" {warning}"); - } - } - println!("Snippet:"); - println!("{}", report.snippet); + if print_only { + return render_document( + &TerminalDocument::empty().block(UiBlock::Patch { + title: "ACP snippet".to_string(), + text: report.snippet.clone(), + }), + options, + ); + } + let mut document = TerminalDocument::new( + format!("ACP setup for {}", report.provider), + if report.applied { + UiTone::Success + } else { + UiTone::Attention + }, + ) + .block(UiBlock::Metadata(vec![ + ("Editor".to_string(), report.editor.clone()), + ("Action".to_string(), report.action.clone()), + ("Applied".to_string(), report.applied.to_string()), + ("Command".to_string(), shell_join(&report.command)), + ])) + .block(UiBlock::Patch { + title: "Configuration snippet".to_string(), + text: report.snippet.clone(), + }); + if !report.warnings.is_empty() { + document = document.block(UiBlock::Checklist( + report + .warnings + .iter() + .map(|warning| UiCheck::new(UiCheckState::Warn, "Install warning", warning)) + .collect(), + )); } - Ok(()) + render_document(&document.pager_eligible(), options) } -pub(crate) fn render_acp_doctor(report: &AcpDoctorReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_acp_doctor( + report: &AcpDoctorReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("ACP doctor: {}", report.status); - for check in &report.checks { - println!("[{}] {}: {}", check.status, check.name, check.message); - } - for warning in &report.warnings { - println!("warning: {warning}"); - } - } - Ok(()) + let mut checks: Vec<_> = report + .checks + .iter() + .map(|check| UiCheck::new(acp_check_state(&check.status), &check.name, &check.message)) + .collect(); + checks.extend( + report + .warnings + .iter() + .map(|warning| UiCheck::new(UiCheckState::Warn, "Warning", warning)), + ); + let evidence = &report.conformance; + let title = if evidence.evidence_status == "verified" { + "ACP v1 conformant".to_string() + } else { + format!("ACP diagnostics: {}", report.status) + }; + render_document( + &TerminalDocument::new( + title, + if report.status == "ok" { + UiTone::Success + } else { + UiTone::Attention + }, + ) + .block(UiBlock::Metadata(vec![ + ( + "Wire version".to_string(), + evidence.wire_version.to_string(), + ), + ("Schema commit".to_string(), evidence.schema_commit.clone()), + ("Schema SHA-256".to_string(), evidence.schema_sha256.clone()), + ("Metadata SHA-256".to_string(), evidence.meta_sha256.clone()), + ("Transport".to_string(), evidence.transport.clone()), + ("Methods".to_string(), evidence.method_count.to_string()), + ("Evidence".to_string(), evidence.evidence_status.clone()), + ("Build".to_string(), evidence.build_identifier.clone()), + ("Exclusions".to_string(), evidence.exclusions.join(", ")), + ])) + .block(UiBlock::Checklist(checks)), + options, + ) } pub(crate) fn render_acp_sessions( report: &AcpSessionListReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - if report.sessions.is_empty() { - println!("No ACP sessions"); - } - for session in &report.sessions { - let provider = session.provider.as_deref().unwrap_or("-"); - println!( - "{} {} {} {}", - session.acp_session_id, session.status, provider, session.trail_session_id - ); - } + if report.sessions.is_empty() { + return render_document( + &TerminalDocument::new("No ACP sessions", UiTone::Neutral), + options, + ); } - Ok(()) + render_document( + &TerminalDocument::new( + format!("{} ACP session(s)", report.sessions.len()), + UiTone::Neutral, + ) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("STATUS", 0, 8), + UiColumn::left("PROVIDER", 1, 10), + UiColumn::left("TRAIL SESSION", 1, 12), + UiColumn::left("ACP SESSION", 0, 14), + ], + report + .sessions + .iter() + .map(|session| { + vec![ + session.status.clone(), + session.provider.clone().unwrap_or_else(|| "—".to_string()), + session.trail_session_id.clone(), + session.acp_session_id.clone(), + ] + }) + .collect(), + ))), + options, + ) } -pub(crate) fn render_transcript(report: &TranscriptReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_transcript( + report: &TranscriptReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Transcript: {}", report.selector); - println!("Lane: {}", report.lane_name); - println!( - "Session: {} ({})", - report.session.session_id, report.session.status + let mut document = + TerminalDocument::new(format!("Transcript: {}", report.lane_name), UiTone::Neutral).block( + UiBlock::Metadata(vec![ + ("Session".to_string(), report.session.session_id.clone()), + ("Status".to_string(), report.session.status.clone()), + ("Turns".to_string(), report.turns.len().to_string()), + ]), ); - if let Some(acp) = &report.acp_session { - println!("ACP session: {} ({})", acp.acp_session_id, acp.status); + for turn in &report.turns { + let mut blocks = vec![UiBlock::Metadata(vec![ + ("Status".to_string(), turn.turn.status.clone()), + ( + "Checkpoint".to_string(), + turn.checkpoint + .as_ref() + .map(|value| value.0.clone()) + .unwrap_or_else(|| "—".to_string()), + ), + ])]; + if !turn.messages.is_empty() { + blocks.push(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("ROLE", 0, 8), + UiColumn::left("MESSAGE", 0, 28), + ], + turn.messages + .iter() + .map(|message| vec![message.role.clone(), preview(&message.body, 160)]) + .collect(), + ))); } - for turn in &report.turns { - let checkpoint = turn - .checkpoint - .as_ref() - .map(|change| change.0.as_str()) - .unwrap_or("-"); - println!(); - println!( - "Turn: {} {} checkpoint {}", - turn.turn.turn_id, turn.turn.status, checkpoint - ); - for message in &turn.messages { - println!( - "{}: {}", - message.role, - single_line_preview(&message.body, 160) - ); - } - for tool in &turn.tool_summaries { - println!("tool: {tool}"); - } + if !turn.tool_summaries.is_empty() { + blocks.push(UiBlock::Lines( + turn.tool_summaries + .iter() + .cloned() + .map(|tool| (tool, UiTone::Muted)) + .collect(), + )); } + document = document.block(UiBlock::section( + format!("Turn {}", turn.turn.turn_id), + blocks, + )); + } + render_document(&document.pager_eligible(), options) +} + +fn acp_check_state(status: &str) -> UiCheckState { + match status.to_ascii_lowercase().as_str() { + "ok" | "pass" | "healthy" => UiCheckState::Pass, + "warn" | "warning" => UiCheckState::Warn, + "pending" => UiCheckState::Pending, + _ => UiCheckState::Fail, } - Ok(()) } fn shell_join(parts: &[String]) -> String { @@ -158,12 +278,17 @@ fn shell_join(parts: &[String]) -> String { .collect::>() .join(" ") } - -fn single_line_preview(value: &str, limit: usize) -> String { - let mut preview = value.split_whitespace().collect::>().join(" "); - if preview.len() > limit { - preview.truncate(limit.saturating_sub(3)); - preview.push_str("..."); +fn preview(value: &str, limit: usize) -> String { + let compact = value.split_whitespace().collect::>().join(" "); + if compact.chars().count() <= limit { + compact + } else { + format!( + "{}…", + compact + .chars() + .take(limit.saturating_sub(1)) + .collect::() + ) } - preview } diff --git a/trail/src/cli/command/render/agent.rs b/trail/src/cli/command/render/agent.rs index 349edce..6e07d1e 100644 --- a/trail/src/cli/command/render/agent.rs +++ b/trail/src/cli/command/render/agent.rs @@ -1,2953 +1,667 @@ -use super::render_json; +use super::*; +use serde::Serialize; +use serde_json::Value; use trail::model::*; use trail::Result; -pub(crate) fn render_agent_setup(report: &AgentSetupReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent setup: {}", report.provider); - println!("Editor: {}", report.editor); - println!("Mode: {}", report.mode); - println!( - "Capabilities: acp={} mcp={} terminal={}", - report.supports_acp, report.supports_mcp, report.supports_terminal - ); - println!("Detected: {}", if report.detected { "yes" } else { "no" }); - println!("Command:"); - println!(" {}", shell_join(&report.command)); - if !report.warnings.is_empty() { - println!("Warnings:"); - for warning in &report.warnings { - println!(" {warning}"); - } - } - println!("Snippet:"); - println!("{}", report.snippet); - println!("Next steps:"); - if report.mode == "acp" { - println!(" Paste the snippet into your editor's ACP custom-agent settings."); - } else if report.supports_mcp { - println!(" Run the terminal command, and register the MCP command in the native agent if you want Trail context tools."); - } else { - println!(" Run the terminal command to launch a fresh Trail task lane."); - } - for suggestion in &report.suggestions { - println!(" {}", suggestion.command); - println!(" {}", suggestion.reason); - } - } - Ok(()) -} - -pub(crate) fn render_agent_status( - report: &AgentStatusReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent status: {:?}", report.status); - if let Some(task) = &report.latest { - print_agent_task_summary(task); - if let Some(risk) = &report.risk { - print_agent_risk_line(risk); - } - } else { - println!("No agent tasks recorded"); +macro_rules! agent_renderer { + ($name:ident, $report:ty, $title:literal) => { + pub(crate) fn $name(report: &$report, json: bool, options: &RenderOptions) -> Result<()> { + render_agent_value($title, report, json, options) } - print_suggestions(&report.suggestions); - } - Ok(()) + }; } -pub(crate) fn render_agent_guide(report: &AgentGuideReport, json: bool, quiet: bool) -> Result<()> { +agent_renderer!(render_agent_status, AgentStatusReport, "Agent status"); +agent_renderer!(render_agent_guide, AgentGuideReport, "Agent guide"); +agent_renderer!( + render_agent_dashboard, + AgentDashboardReport, + "Agent dashboard" +); +agent_renderer!( + render_agent_review_data, + AgentReviewDataReport, + "Agent review data" +); +agent_renderer!( + render_agent_action_palette, + AgentReviewDataReport, + "Agent actions" +); +agent_renderer!( + render_agent_review_flow, + AgentReviewFlowReport, + "Agent review flow" +); +agent_renderer!(render_agent_inbox, AgentInboxReport, "Agent inbox"); +agent_renderer!(render_agent_board, AgentBoardReport, "Agent board"); +agent_renderer!(render_agent_stack, AgentStackReport, "Agent stack"); +agent_renderer!(render_agent_next, AgentNextReport, "Agent next"); +agent_renderer!(render_agent_list, AgentTaskListReport, "Agent tasks"); +agent_renderer!(render_agent_brief, AgentBriefReport, "Agent brief"); +agent_renderer!(render_agent_summary, AgentSummaryReport, "Agent summary"); +agent_renderer!( + render_agent_validate, + AgentValidationReport, + "Agent validation" +); +agent_renderer!( + render_agent_test_plan, + AgentTestPlanReport, + "Agent test plan" +); +agent_renderer!(render_agent_receipt, AgentReceiptReport, "Agent receipt"); +agent_renderer!(render_agent_handoff, AgentHandoffReport, "Agent handoff"); +agent_renderer!(render_agent_story, AgentStoryReport, "Agent story"); +agent_renderer!(render_agent_tools, AgentToolsReport, "Agent tools"); +agent_renderer!(render_agent_impact, AgentImpactReport, "Agent impact"); +agent_renderer!( + render_agent_review_map, + AgentReviewMapReport, + "Agent review map" +); +agent_renderer!(render_agent_risk, AgentRiskReport, "Agent risk"); +agent_renderer!( + render_agent_confidence, + AgentConfidenceReport, + "Agent confidence" +); +agent_renderer!(render_agent_ready, AgentReadyReport, "Agent readiness"); +agent_renderer!( + render_agent_diagnose, + AgentDiagnosisReport, + "Agent diagnosis" +); +agent_renderer!(render_agent_compare, AgentCompareReport, "Agent comparison"); +agent_renderer!(render_agent_workdir, AgentWorkdirReport, "Agent workdir"); +agent_renderer!(render_agent_view, AgentTaskViewReport, "Agent task"); +agent_renderer!(render_agent_changes, AgentChangesReport, "Agent changes"); +agent_renderer!(render_agent_delta, AgentDeltaReport, "Agent delta"); +agent_renderer!(render_agent_new, AgentNewReport, "Created agent task"); +agent_renderer!( + render_agent_mark_reviewed, + AgentMarkReviewedReport, + "Recorded review" +); +agent_renderer!( + render_agent_mark_file_reviewed, + AgentMarkFileReviewedReport, + "Recorded file review" +); +agent_renderer!( + render_agent_archive, + AgentArchiveReport, + "Updated agent archive" +); +agent_renderer!(render_agent_timeline, AgentTimelineReport, "Agent timeline"); +agent_renderer!( + render_agent_change_set, + AgentChangeSetReport, + "Agent change set" +); +agent_renderer!(render_agent_files, AgentFilesReport, "Agent files"); +agent_renderer!(render_agent_file, AgentFileReport, "Agent file"); +agent_renderer!( + render_agent_checkpoints, + AgentCheckpointReport, + "Agent checkpoints" +); +agent_renderer!(render_agent_why, AgentWhyReport, "Agent provenance"); +agent_renderer!(render_agent_turn, AgentTurnReport, "Agent turn"); +agent_renderer!(render_agent_review, AgentReviewReport, "Agent review"); +agent_renderer!(render_agent_focus, AgentFocusReport, "Agent focus"); +agent_renderer!(render_agent_apply, AgentApplyReport, "Agent apply"); +agent_renderer!(render_agent_finish, AgentFinishReport, "Agent finish"); +agent_renderer!(render_agent_run, AgentRunReport, "Agent run"); +agent_renderer!( + render_agent_continue, + AgentContinueReport, + "Agent follow-up" +); + +pub(crate) fn render_agent_empty_action_palette(json: bool, options: &RenderOptions) -> Result<()> { + let actions = agent_empty_action_palette_actions(); if json { - return render_json(report); - } - if !quiet { - println!("Agent guide: {:?}", report.status); - println!("{}", report.headline); - println!("{}", report.current_state); - if let Some(task) = &report.task { - println!("Task: {}", agent_task_display_title(task)); - print_agent_task_id_if_needed(task); - print_agent_task_workdir(task); - } - println!("Do this next:"); - println!(" {}", report.primary.command); - println!(" {}", report.primary.reason); - if !report.steps.is_empty() { - println!("Workflow:"); - for (idx, step) in report.steps.iter().enumerate() { - println!(" {}. {}", idx + 1, step.label); - println!(" {}", step.command); - println!(" {}", step.reason); - println!(" When: {}", step.when); - } - } - if !report.concepts.is_empty() { - println!("Mental model:"); - for concept in &report.concepts { - println!(" {}: {}", concept.name, concept.meaning); - } - } + return render_json(&serde_json::json!({ + "status": "empty", + "task": null, + "summary": "No agent task is recorded yet. Set up an editor, verify the provider, or start a terminal task.", + "next": { + "command": "trail agent acp setup claude-code --editor vscode", + "reason": "print a stable editor config that creates fresh Trail tasks automatically" + }, + "actions": actions + })); } - Ok(()) + render_document( + &TerminalDocument::new("No agent task yet", UiTone::Neutral) + .context("Set up a provider or start a task to see review actions.") + .block(UiBlock::Table(actions_table(&actions))) + .next( + "trail agent acp setup claude-code --editor vscode", + "print a copyable provider configuration", + ), + options, + ) } -pub(crate) fn render_agent_dashboard( - report: &AgentDashboardReport, +pub(crate) fn render_agent_empty_task_hint( + requested: &str, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { - return render_json(report); - } - if !quiet { - println!("Agent dashboard: {:?}", report.status); - println!("{}", report.summary); - if let Some(task) = &report.task { - println!("Task: {}", agent_task_display_title(task)); - print_agent_task_id_if_needed(task); - print_agent_task_workdir(task); - println!( - "Changed files: {} Turns: {} Tool events: {}", - task.changed_paths.len(), - task.turns, - task.tool_events - ); - if let Some(checkpoint) = &task.latest_checkpoint { - println!("Last checkpoint: {}", checkpoint.0); - } - } - if let Some(ready) = &report.ready { - print_agent_risk_line(&ready.risk); - println!( - "Readiness: {} Apply: {} Ready: {}", - ready.readiness_status, ready.status, ready.ready - ); - } - if let Some(validation) = &report.validation { - println!("Validation: {}", validation.status); - if validation.needs_test || validation.needs_eval { - println!(" {}", validation.next.command); - println!(" {}", validation.next.reason); - } - } - if let Some(focus) = &report.focus { - println!("Focus:"); - println!(" {}", focus.path); - println!(" {}", focus.summary); - if let Some(command) = &focus.open_command { - println!(" Open: {command}"); - } else { - println!(" Inspect: trail agent focus {}", focus.task.lane); - } - } - if let Some(changes) = &report.changes { - if changes.total_changed_paths.is_empty() { - println!("Changed files: none"); - } else { - println!("Changed files:"); - for change in changes.total_changed_paths.iter().take(5) { - println!( - " {:?} {} (+{} -{})", - change.kind, change.path, change.additions, change.deletions - ); - } - if changes.total_changed_paths.len() > 5 { - println!( - " ... {} more; run `trail agent changes {} --by-file`", - changes.total_changed_paths.len() - 5, - changes.lane - ); - } - } - } - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - if let Some(task) = &report.task { - println!("Actions:"); - println!(" trail agent action {}", task.lane); - println!(" show runnable review, validation, apply, and recovery actions"); - } - if report.suggestions.len() > 1 { - println!("Useful commands:"); - for suggestion in report.suggestions.iter().skip(1) { - println!(" {}", suggestion.command); - println!(" {}", suggestion.reason); - } - } + return render_json(&serde_json::json!({ + "requested": requested, + "status": "empty", + "task": null, + "summary": "No agent task is recorded yet. Set up an editor, verify the provider, or start a terminal task.", + "next": { + "command": "trail agent acp setup claude-code --editor vscode", + "reason": "print a stable editor config that creates fresh Trail tasks automatically" + }, + "actions": agent_empty_action_palette_actions() + })); } - Ok(()) + render_document( + &TerminalDocument::new( + format!("No agent task available for {requested}"), + UiTone::Neutral, + ) + .next( + "trail agent start claude-code", + "start an isolated terminal task", + ), + options, + ) } -pub(crate) fn render_agent_review_data( - report: &AgentReviewDataReport, +pub(crate) fn render_agent_report( + report: &AgentReviewBundleReport, json: bool, - quiet: bool, + options: &RenderOptions, + markdown: bool, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Agent review data: {}", - agent_task_display_title(&report.task) - ); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("{}", report.summary); - println!( - "Files: {}/{} reviewed Needs review: {}", - report.reviewed_files, report.total_files, report.needs_review_files - ); - println!( - "Verdict: {} Confidence: {}/100 Validation: {} Apply: {}", - report.confidence_verdict, - report.confidence_score, - report.validation_status, - report.readiness_status - ); - println!("Risk: {:?}", report.risk_level); - if let Some(focus) = &report.focus { - println!("Focus:"); - println!(" {}", focus.path); - println!(" {}", focus.summary); - if let Some(command) = &focus.open_command { - println!(" Open: {command}"); - } - } - if !report.review_map.areas.is_empty() { - println!("Review areas:"); - for area in &report.review_map.areas { - println!( - " {}: {} file(s), {}", - area.label, - area.files.len(), - area.state - ); - } - } - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - if !report.actions.is_empty() { - println!("Actions:"); - for action in &report.actions { - let state = if action.enabled { - "enabled" - } else { - "disabled" - }; - let confirmation = if action.requires_confirmation { - ", confirmation" - } else { - "" - }; - println!( - " {} ({}) [{}: {}{}]", - action.label, action.id, state, action.safety, confirmation - ); - println!(" {}", action.command); - println!(" {}", action.reason); - if let Some(reason) = &action.disabled_reason { - println!(" Disabled: {reason}"); - } - } - } - print_suggestions(&report.suggestions); + if markdown { + return render_raw_content(&report.markdown, options); } - Ok(()) + render_agent_value("Agent report", report, false, options) } -pub(crate) fn render_agent_action_palette( - report: &AgentReviewDataReport, +pub(crate) fn render_agent_pr( + report: &AgentPrDraftReport, json: bool, - quiet: bool, + options: &RenderOptions, + title_only: bool, + body_only: bool, ) -> Result<()> { if json { - return render_json(&serde_json::json!({ - "task": &report.task, - "summary": &report.summary, - "next": &report.next, - "actions": &report.actions - })); - } - if !quiet { - println!("Agent actions: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - println!("{}", report.summary); - if report.actions.is_empty() { - println!("No actions are currently available."); - } else { - println!("Actions:"); - for action in &report.actions { - let state = if action.enabled { - "enabled" - } else { - "disabled" - }; - let confirmation = if action.requires_confirmation { - ", confirmation" - } else { - "" - }; - println!( - " {} {} [{}: {}{}]", - action.id, action.label, state, action.safety, confirmation - ); - println!( - " Run: trail agent action {} {}", - report.task.lane, action.id - ); - println!( - " Print: trail agent action {} {} --print", - report.task.lane, action.id - ); - println!(" {}", action.reason); - if let Some(reason) = &action.disabled_reason { - println!(" Disabled: {reason}"); - } - } - } + return render_json(report); } - Ok(()) -} - -pub(crate) fn render_agent_empty_action_palette(json: bool, quiet: bool) -> Result<()> { - let summary = "No agent task is recorded yet. Set up an editor, verify the provider, or start a terminal task."; - let next = StatusSuggestion { - command: "trail agent setup".to_string(), - reason: "print a stable editor config that creates fresh Trail tasks automatically" - .to_string(), - }; - let actions = agent_empty_action_palette_actions(); - if json { - return render_json(&serde_json::json!({ - "status": "empty", - "task": null, - "summary": summary, - "next": &next, - "actions": &actions - })); + if title_only { + return render_raw_content(&report.title, options); } - if !quiet { - println!("Agent actions: no agent task yet"); - println!("{summary}"); - println!("Next:"); - println!(" {}", next.command); - println!(" {}", next.reason); - println!("Actions:"); - for action in &actions { - let confirmation = if action.requires_confirmation { - ", confirmation" - } else { - "" - }; - println!( - " {} {} [enabled: {}{}]", - action.id, action.label, action.safety, confirmation - ); - println!(" Command: {}", action.command); - println!(" {}", action.reason); - } + if body_only { + return render_raw_content(&report.body, options); } - Ok(()) + render_agent_value("Agent pull request draft", report, false, options) } -pub(crate) fn render_agent_empty_task_hint(requested: &str, json: bool, quiet: bool) -> Result<()> { - let summary = match requested { - "changes" => "No agent task is recorded yet, so there are no agent changes to inspect. Set up an editor, verify the provider, or start a terminal task.".to_string(), - "apply" => "No agent task is recorded yet, so there is nothing to apply. Set up an editor, verify the provider, or start a terminal task.".to_string(), - "view" => "No agent task is recorded yet, so there is no task to view. Set up an editor, verify the provider, or start a terminal task.".to_string(), - _ => format!( - "No agent task is recorded yet. Set up an editor, verify the provider, or start a terminal task before running `{requested}`." - ), - }; - let next = StatusSuggestion { - command: "trail agent setup".to_string(), - reason: "print a stable editor config that creates fresh Trail tasks automatically" - .to_string(), - }; - let actions = agent_empty_action_palette_actions(); +pub(crate) fn render_agent_diff( + report: &AgentDiffReport, + json: bool, + options: &RenderOptions, + stat: bool, +) -> Result<()> { if json { - return render_json(&serde_json::json!({ - "status": "empty", - "task": null, - "requested": requested, - "summary": summary, - "next": &next, - "actions": &actions - })); - } - if !quiet { - println!("Agent {requested}: no agent task yet"); - println!("{summary}"); - println!("Next:"); - println!(" {}", next.command); - println!(" {}", next.reason); - println!("Other useful commands:"); - for action in &actions { - println!(" {}", action.command); - println!(" {}", action.reason); - } + return render_json(report); } - Ok(()) + render_diff_with_title( + &report.diff, + false, + options, + true, + stat, + false, + false, + Some("Agent diff"), + ) } pub(crate) fn agent_empty_action_palette_actions() -> Vec { - vec![ - agent_static_action( - "setup_vscode", - "Set up VS Code", + [ + ( + "acp_setup_vscode", + "Set up ACP in VS Code", "setup", - "trail agent setup", + "trail agent acp setup claude-code --editor vscode", "print a copyable ACP editor config that creates fresh task lanes automatically", "read_only", false, ), - agent_static_action( + ( "doctor_claude_code", "Check Claude Code", "doctor", - "trail agent doctor --provider claude-code", + "trail agent doctor claude-code", "verify Trail workspace readiness and provider availability", "read_only", false, ), - agent_static_action( - "setup_codex_vscode", - "Set up VS Code for Codex", + ( + "acp_setup_codex_vscode", + "Set up Codex ACP in VS Code", "setup", - "trail agent setup --provider codex", + "trail agent acp setup codex --editor vscode", "print a copyable Codex ACP editor config that creates fresh task lanes automatically", "read_only", false, ), - agent_static_action( + ( "doctor_codex", "Check Codex", "doctor", - "trail agent doctor --provider codex", + "trail agent doctor codex", "verify Trail workspace readiness and provider availability", "read_only", false, ), - agent_static_action( - "setup_cursor_vscode", - "Set up VS Code for Cursor", + ( + "acp_setup_cursor_vscode", + "Set up Cursor ACP in VS Code", "setup", - "trail agent setup --provider cursor", + "trail agent acp setup cursor --editor vscode", "print a copyable Cursor ACP editor config that creates fresh task lanes automatically", "read_only", false, ), - agent_static_action( + ( "doctor_cursor", "Check Cursor", "doctor", - "trail agent doctor --provider cursor", + "trail agent doctor cursor", "verify Trail workspace readiness and provider availability", "read_only", false, ), - agent_static_action( + ( "start_terminal_task", "Start terminal task", "start", - "trail agent start --provider claude-code", + "trail agent start claude-code", "launch a fresh materialized terminal task when you are not using an editor", "open_world", true, ), - agent_static_action( + ( "start_gemini_task", "Start Gemini task", "start", - "trail agent start --provider gemini", + "trail agent start gemini", "launch Gemini CLI in a fresh materialized Trail task lane", "open_world", true, ), ] -} - -fn agent_static_action( - id: &str, - label: &str, - kind: &str, - command: &str, - reason: &str, - safety: &str, - requires_confirmation: bool, -) -> AgentReviewAction { - AgentReviewAction { - id: id.to_string(), - label: label.to_string(), - kind: kind.to_string(), - command: command.to_string(), - reason: reason.to_string(), - enabled: true, - disabled_reason: None, - safety: safety.to_string(), - requires_confirmation, - path: None, - open_path: None, - mcp_tool: None, - mcp_arguments: None, - } -} - -pub(crate) fn render_agent_review_flow( - report: &AgentReviewFlowReport, + .into_iter() + .map( + |(id, label, kind, command, reason, safety, requires_confirmation)| AgentReviewAction { + id: id.to_string(), + label: label.to_string(), + kind: kind.to_string(), + command: command.to_string(), + reason: reason.to_string(), + enabled: true, + disabled_reason: None, + safety: safety.to_string(), + requires_confirmation, + path: None, + open_path: None, + mcp_tool: None, + mcp_arguments: None, + }, + ) + .collect() +} + +pub(crate) fn render_semantic_report( + title: &str, + report: &T, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Agent review flow: {}", - agent_task_display_title(&report.task) - ); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("{}", report.summary); - println!( - "Review: {} New files: {} New lines: {}", - report.review_status, report.new_changed_paths, report.new_changed_lines - ); - println!( - "Validation: {} Apply: {} Ready: {}", - report.validation.status, report.ready.status, report.ready.ready - ); - if let Some(checkpoint) = &report.task.latest_checkpoint { - println!("Last checkpoint: {}", checkpoint.0); - } - if let Some(marker) = &report.reviewed { - println!("Reviewed checkpoint: {}", marker.checkpoint.0); - } else { - println!("Reviewed checkpoint: none"); - } - if let Some(focus) = &report.focus { - println!("Focus: {}", focus.path); - } - println!("Checklist:"); - for (idx, step) in report.steps.iter().enumerate() { - println!(" {}. [{}] {}", idx + 1, step.state, step.label); - println!(" {}", step.command); - println!(" {}", step.reason); - } - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_inbox(report: &AgentInboxReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent inbox"); - println!( - "Tasks: {} Need attention: {} Archived: {}", - report.total, report.attention_count, report.archived_count - ); - if report.include_archived { - println!("Showing: active and archived tasks"); - } - if report.groups.is_empty() { - println!("No agent tasks recorded"); - } - for group in &report.groups { - print_agent_inbox_group(group); - } - println!("Next command:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - if report.suggestions.len() > 1 { - println!("Other useful commands:"); - for suggestion in report.suggestions.iter().skip(1) { - println!(" {}", suggestion.command); - println!(" {}", suggestion.reason); - } - } - } - Ok(()) + let value = serde_json::to_value(report)?; + let document = projected_document(title, &value); + render_document(&document.pager_eligible(), options) } -pub(crate) fn render_agent_board(report: &AgentBoardReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent board"); - println!( - "Tasks: {} Need attention: {} Ready: {} Running: {} Blocked: {} Conflicted: {} Applied: {} Archived: {}", - report.total, - report.attention_count, - report.ready_count, - report.active_count, - report.blocked_count, - report.conflicted_count, - report.applied_count, - report.archived_count - ); - if report.include_archived { - println!("Showing: active and archived tasks"); - } - if report.columns.is_empty() { - println!("No agent tasks recorded"); - } - for column in &report.columns { - print_agent_board_column(column); - } - println!("Next command:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - if report.suggestions.len() > 1 { - println!("Other useful commands:"); - for suggestion in report.suggestions.iter().skip(1) { - println!(" {}", suggestion.command); - println!(" {}", suggestion.reason); - } - } - } - Ok(()) +fn render_agent_value( + title: &str, + report: &T, + json: bool, + options: &RenderOptions, +) -> Result<()> { + render_semantic_report(title, report, json, options) } -pub(crate) fn render_agent_stack(report: &AgentStackReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent stack"); - println!("{}", report.summary); - println!( - "Tasks: {} Ready: {} Blocked: {} Shared paths: {}", - report.total, report.ready_count, report.blocked_count, report.overlap_count - ); - if report.include_archived { - println!("Showing: active and archived tasks"); - } - if !report.shared_paths.is_empty() { - println!("Shared files:"); - for shared in &report.shared_paths { - println!(" {} ({})", shared.path, shared.task_titles.join(", ")); - } - } - if !report.apply_order.is_empty() { - println!("Apply order:"); - for (idx, lane) in report.apply_order.iter().enumerate() { - if let Some(item) = report.items.iter().find(|item| &item.task.lane == lane) { - println!( - " {}. {} risk {:?} ({}/100), {} file(s)", - idx + 1, - agent_task_display_title(&item.task), - item.risk.level, - item.risk.score, - item.task.changed_paths.len() - ); - } - } - } - if !report.items.is_empty() { - println!("Tasks:"); - for item in &report.items { - let shared = if item.shared_paths.is_empty() { - "no shared files".to_string() +fn projected_document(title: &str, value: &Value) -> TerminalDocument { + let Some(object) = value.as_object() else { + return TerminalDocument::new(title, UiTone::Neutral).block(UiBlock::Metadata(vec![( + "Value".to_string(), + scalar(value), + )])); + }; + let mut metadata = Vec::new(); + let status = object + .get("status") + .or_else(|| object.get("state")) + .or_else(|| object.get("readiness_status")) + .or_else(|| object.get("validation_status")) + .or_else(|| object.get("decision")) + .and_then(Value::as_str); + let lead = status + .map(|status| format!("{title}: {}", state_label(status))) + .unwrap_or_else(|| title.to_string()); + let mut document = TerminalDocument::new(lead, document_tone(object)); + if let Some(context) = ["headline", "summary", "current_state"] + .into_iter() + .find_map(|key| object.get(key).and_then(Value::as_str)) + { + document = document.context(context); + } + if let Some(next) = ["primary", "next"] + .into_iter() + .find_map(|key| object.get(key).and_then(action_from_value)) + .or_else(|| { + object + .get("suggestions") + .and_then(Value::as_array) + .and_then(|suggestions| suggestions.first()) + .and_then(action_from_value) + }) + { + document.next = Some(next); + } + for (key, value) in object { + if matches!( + key.as_str(), + "headline" | "summary" | "current_state" | "primary" | "next" + ) { + continue; + } + match value { + Value::Null => {} + Value::Bool(_) | Value::Number(_) | Value::String(_) => { + let rendered = if matches!( + key.as_str(), + "status" | "state" | "readiness_status" | "validation_status" | "decision" + ) { + value + .as_str() + .map(state_label) + .unwrap_or_else(|| scalar(value)) } else { - format!("shared: {}", item.shared_paths.join(", ")) + scalar(value) }; - println!( - " {}. {} {} risk {:?} ({}/100), {}", - item.rank, - agent_task_display_title(&item.task), - item.status, - item.risk.level, - item.risk.score, - shared - ); - println!(" {}", item.next.command); - println!(" {}", item.next.reason); + metadata.push((label(key), rendered)); } - } - println!("Next command:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - if report.suggestions.len() > 1 { - println!("Other useful commands:"); - for suggestion in report.suggestions.iter().skip(1) { - println!(" {}", suggestion.command); - println!(" {}", suggestion.reason); + Value::Array(values) + if values + .iter() + .all(|value| value.is_string() || value.is_number() || value.is_boolean()) => + { + document = document.block(UiBlock::section( + label(key), + vec![UiBlock::Lines( + values + .iter() + .map(|value| (scalar(value), UiTone::Neutral)) + .collect(), + )], + )); + } + Value::Array(values) => { + document = document.block(UiBlock::section(label(key), vec![array_block(values)])); + } + Value::Object(values) => { + document = document.block(UiBlock::section(label(key), object_blocks(values))); } } } - Ok(()) + if !metadata.is_empty() { + document.blocks.insert(0, UiBlock::Metadata(metadata)); + } + document } -pub(crate) fn render_agent_next(report: &AgentNextReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent next: {:?}", report.status); - println!("{}", report.summary); - if let Some(task) = &report.task { - println!("Task: {}", agent_task_display_title(task)); - print_agent_task_id_if_needed(task); - print_agent_task_workdir(task); - println!("Changed files: {}", task.changed_paths.len()); - println!("Turns: {} Tool events: {}", task.turns, task.tool_events); - } - println!("Next command:"); - println!(" {}", report.primary.command); - println!(" {}", report.primary.reason); - if !report.suggestions.is_empty() { - println!("Other useful commands:"); - for suggestion in &report.suggestions { - println!(" {}", suggestion.command); - println!(" {}", suggestion.reason); - } - } +fn action_from_value(value: &Value) -> Option { + let object = value.as_object()?; + let command = object.get("command")?.as_str()?.trim(); + if command.is_empty() { + return None; } - Ok(()) + let reason = object + .get("reason") + .and_then(Value::as_str) + .unwrap_or("continue with the displayed workflow") + .to_string(); + Some(UiNextAction { + command: command.to_string(), + reason, + }) } -pub(crate) fn render_agent_list( - report: &AgentTaskListReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - if report.tasks.is_empty() { - println!("No agent tasks"); - } - for task in &report.tasks { - println!( - "{} {:?}{} {} changed path(s)", - agent_task_display_title(task), - task.status, - if task.archived { " archived" } else { "" }, - task.changed_paths.len() - ); - print_agent_task_id_if_needed(task); - print_agent_task_workdir(task); - } +fn document_tone(object: &serde_json::Map) -> UiTone { + let state = [ + "status", + "state", + "decision", + "readiness_status", + "validation_status", + ] + .into_iter() + .find_map(|key| object.get(key).and_then(Value::as_str)) + .unwrap_or_default() + .to_ascii_lowercase(); + if matches!( + state.as_str(), + "ok" | "pass" | "passed" | "ready" | "complete" | "completed" + ) { + UiTone::Success + } else if matches!( + state.as_str(), + "blocked" | "fail" | "failed" | "conflicted" | "conflict" + ) { + UiTone::Blocked + } else if matches!(state.as_str(), "warning" | "warn" | "pending" | "dirty") { + UiTone::Attention + } else { + UiTone::Neutral } - Ok(()) } -pub(crate) fn render_agent_brief(report: &AgentBriefReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent brief: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!( - "Status: {:?} Ready: {} Changed files: {}", - report.task.status, - report.ready_to_apply, - report.changed_paths.len() - ); - print_agent_risk_line(&report.risk); - println!("Next:"); - println!(" {}", report.next.primary.command); - println!(" {}", report.next.primary.reason); - if !report.blockers.is_empty() { - println!("Blockers:"); - for blocker in &report.blockers { - println!(" {}: {}", blocker.code, blocker.message); - } - } - if !report.warnings.is_empty() { - println!("Warnings:"); - for warning in &report.warnings { - println!(" {}: {}", warning.code, warning.message); - } - } - if !report.changed_paths.is_empty() { - println!("Changed files:"); - print_changed_paths(&report.changed_paths, " "); - } - if !report.groups.is_empty() { - println!("Changes:"); - for group in &report.groups { - print_agent_brief_group(group); - } - } - if let Some(diff) = &report.latest_change_diff { - print_agent_brief_diff(diff); - } - if !report.tool_summaries.is_empty() { - println!("Tools:"); - for tool in &report.tool_summaries { - println!(" {tool}"); - } +fn object_blocks(object: &serde_json::Map) -> Vec { + let scalar_entries = object + .iter() + .filter_map(|(key, value)| { + value + .is_boolean() + .then_some((key, value)) + .or_else(|| value.is_number().then_some((key, value))) + .or_else(|| value.is_string().then_some((key, value))) + }) + .map(|(key, value)| (label(key), scalar(value))) + .collect::>(); + let mut blocks = Vec::new(); + if !scalar_entries.is_empty() { + blocks.push(UiBlock::Metadata(scalar_entries)); + } + for (key, value) in object { + if !value.is_null() && !value.is_boolean() && !value.is_number() && !value.is_string() { + let nested = match value { + Value::Array(values) => array_block(values), + Value::Object(values) => UiBlock::section(label(key), object_blocks(values)), + _ => continue, + }; + blocks.push(UiBlock::section(label(key), vec![nested])); } - print_suggestions(&report.suggestions); } - Ok(()) + blocks } -pub(crate) fn render_agent_summary( - report: &AgentSummaryReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); +fn array_block(values: &[Value]) -> UiBlock { + if values.is_empty() { + return UiBlock::Notice("No items.".to_string()); } - if !quiet { - println!("Agent summary: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("{}", report.summary); - println!( - "Ready: {} Status: {} Readiness: {}", - report.ready, report.ready_status, report.readiness_status - ); - print_agent_risk_line(&report.risk); - println!( - "Changed files: {} Turns: {} Tool events: {}", - report.changed_paths.len(), - report.task.turns, - report.task.tool_events - ); - if let Some(checkpoint) = &report.latest_checkpoint { - println!("Last checkpoint: {}", checkpoint.0); - } - if report.validation.is_empty() { - println!("Validation: no test or eval gate recorded"); - } else { - println!("Validation:"); - for gate in &report.validation { - let suite = gate.suite.as_deref().unwrap_or("default"); - let result = if gate.success { "passed" } else { "failed" }; - println!( - " {} `{}` {} ({})", - gate.kind, - suite, - result, - shell_join(&gate.command) - ); - } - } - if !report.blockers.is_empty() { - println!("Blockers:"); - for blocker in &report.blockers { - println!(" {}: {}", blocker.code, blocker.message); - } - } - if !report.warnings.is_empty() { - println!("Warnings:"); - for warning in &report.warnings { - println!(" {}: {}", warning.code, warning.message); - } - } - if let Some(error) = &report.apply_error { - println!("Git preflight: failed"); - println!(" {error}"); - } else if let Some(preview) = &report.apply_preview { - println!("Git preflight: {}", preview.status); - if let Some(branch) = &preview.git_apply_plan.git_branch { - println!("Into Git branch: {branch}"); + let object_rows = values + .iter() + .filter_map(Value::as_object) + .collect::>(); + if object_rows.len() == values.len() { + let mut keys = Vec::new(); + for object in &object_rows { + for (key, value) in *object { + if (value.is_boolean() || value.is_number() || value.is_string()) + && !keys.contains(key) + { + keys.push(key.clone()); + } } } - println!("PR title:"); - println!(" {}", report.pr_title); - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - print_suggestions(&report.suggestions); - } - Ok(()) + if !keys.is_empty() { + let columns = keys + .iter() + .enumerate() + .map(|(index, key)| UiColumn::left(label(key), index.min(3) as u8, 10)) + .collect(); + let rows = object_rows + .iter() + .map(|object| { + keys.iter() + .map(|key| { + object + .get(key) + .map(scalar) + .unwrap_or_else(|| "-".to_string()) + }) + .collect() + }) + .collect(); + return UiBlock::Table(UiTable::new(columns, rows)); + } + } + UiBlock::Lines( + values + .iter() + .map(|value| (compact_value(value), UiTone::Neutral)) + .collect(), + ) } -pub(crate) fn render_agent_validate( - report: &AgentValidationReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!( - "Agent validation: {}", - agent_task_display_title(&report.task) - ); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Status: {}", report.status); - println!("{}", report.summary); - println!( - "Needs test: {} Needs eval: {} Changed files: {}", - report.needs_test, - report.needs_eval, - report.changed_paths.len() - ); - if let Some(test) = &report.latest_test { - println!("Latest test:"); - print_agent_gate_summary(test, " "); - } else { - println!("Latest test: none recorded"); - } - if let Some(eval) = &report.latest_eval { - println!("Latest eval:"); - print_agent_gate_summary(eval, " "); - } else { - println!("Latest eval: none recorded"); - } - if !report.recent_gates.is_empty() { - println!("Recent gates:"); - for gate in &report.recent_gates { - print_agent_gate_summary(gate, " "); - } - } - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - print_suggestions(&report.suggestions); +fn compact_value(value: &Value) -> String { + match value { + Value::Object(object) => object + .iter() + .filter(|(_, value)| value.is_boolean() || value.is_number() || value.is_string()) + .map(|(key, value)| format!("{}: {}", label(key), scalar(value))) + .collect::>() + .join("; "), + Value::Array(values) => format!("{} items", values.len()), + _ => scalar(value), } - Ok(()) } -pub(crate) fn render_agent_test_plan( - report: &AgentTestPlanReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!( - "Agent test plan: {}", - agent_task_display_title(&report.task) - ); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Status: {}", report.status); - println!("{}", report.summary); - print_agent_risk_line(&report.risk); - println!("Validation: {}", report.validation.status); - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - if report.steps.is_empty() { - println!("Steps: none"); - } else { - println!("Steps:"); - for step in &report.steps { - let required = if step.required { - "required" - } else { - "optional" - }; - println!( - " {}. {} [{}; {}]", - step.rank, step.label, step.state, required - ); - println!(" {}", step.command); - println!(" {}", step.reason); - if let Some(area) = &step.area_label { - println!(" Area: {area}"); - } - if !step.paths.is_empty() { - print_changed_paths(&step.paths, " "); - } - if let Some(gate) = &step.latest_gate { - println!(" Latest gate:"); - print_agent_gate_summary(gate, " "); - } - } - } - print_suggestions(&report.suggestions); +fn scalar(value: &Value) -> String { + match value { + Value::String(value) => value.clone(), + _ => value.to_string(), } - Ok(()) } +fn label(key: &str) -> String { + key.replace('_', " ") + .split_whitespace() + .map(|word| { + let mut chars = word.chars(); + chars + .next() + .map(|first| first.to_ascii_uppercase().to_string() + chars.as_str()) + .unwrap_or_default() + }) + .collect::>() + .join(" ") +} +fn actions_table(actions: &[AgentReviewAction]) -> UiTable { + UiTable::new( + vec![ + UiColumn::left("ACTION", 0, 14), + UiColumn::left("WHY", 0, 18), + UiColumn::left("COMMAND", 1, 20), + ], + actions + .iter() + .map(|a| vec![a.label.clone(), a.reason.clone(), a.command.clone()]) + .collect(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn agent_projection_prioritizes_state_focus_and_safe_next_action() { + let document = projected_document( + "Agent status", + &serde_json::json!({ + "status": "dirty", + "summary": "fix-login needs one recorded checkpoint", + "primary": { + "command": "trail agent mark-reviewed latest", + "reason": "record the reviewed checkpoint before applying work" + }, + "tasks": [ + {"title": "Fix login", "state": "dirty", "changed_paths": 2}, + {"title": "Update docs", "state": "ready", "changed_paths": 1} + ] + }), + ); + assert_eq!(document.lead.unwrap().text, "Agent status: needs record"); + assert_eq!( + document.context.as_deref(), + Some("fix-login needs one recorded checkpoint") + ); + assert_eq!( + document.next.unwrap().command, + "trail agent mark-reviewed latest" + ); + assert!(document + .blocks + .iter() + .any(|block| matches!(block, UiBlock::Section { title, .. } if title == "Tasks"))); + } -pub(crate) fn render_agent_report( - report: &AgentReviewBundleReport, - json: bool, - quiet: bool, - markdown: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if markdown { - print!("{}", report.markdown); - return Ok(()); - } - if !quiet { - println!("Agent report: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("{}", report.summary); - println!( - "Readiness: {} Ready: {}", - report.readiness_status, report.ready_to_apply - ); - print_agent_risk_line(&report.risk); - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - if !report.review.readiness.blockers.is_empty() { - println!("Blockers:"); - for blocker in &report.review.readiness.blockers { - println!(" {}: {}", blocker.code, blocker.message); - } - } - if !report.review.readiness.warnings.is_empty() { - println!("Warnings:"); - for warning in &report.review.readiness.warnings { - println!(" {}: {}", warning.code, warning.message); - } - } - if !report.story.turn_summaries.is_empty() { - println!("Turns:"); - for turn in &report.story.turn_summaries { - print_agent_story_turn(turn); - } - } - if !report.task.changed_paths.is_empty() { - println!("Changed files:"); - print_changed_paths(&report.task.changed_paths, " "); - } - println!("Markdown:"); - println!(" trail agent report {} --markdown", report.task.lane); - if report.suggestions.len() > 1 { - println!("Other useful commands:"); - for suggestion in report.suggestions.iter().skip(1) { - println!(" {}", suggestion.command); - println!(" {}", suggestion.reason); - } - } - } - Ok(()) -} - -pub(crate) fn render_agent_receipt( - report: &AgentReceiptReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - print!("{}", report.markdown); - } - Ok(()) -} - -pub(crate) fn render_agent_handoff( - report: &AgentHandoffReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - print!("{}", report.markdown); - } - Ok(()) -} - -pub(crate) fn render_agent_pr( - report: &AgentPrDraftReport, - json: bool, - quiet: bool, - title_only: bool, - body_only: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - if title_only { - println!("{}", report.title); - } else if body_only { - print!("{}", report.body); - } else { - println!("{}", report.title); - println!(); - print!("{}", report.body); - } - } - Ok(()) -} - -pub(crate) fn render_agent_story(report: &AgentStoryReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent story: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("{}", report.summary); - if !report.turn_summaries.is_empty() { - println!("Turns:"); - for turn in &report.turn_summaries { - print_agent_story_turn(turn); - } - } - if !report.changed_files.is_empty() { - println!("Changed files:"); - print_changed_paths(&report.changed_files, " "); - } - if !report.tool_summaries.is_empty() { - println!("Tools:"); - for tool in &report.tool_summaries { - println!(" {tool}"); - } - } - if !report.risk_notes.is_empty() { - println!("Notes:"); - for note in &report.risk_notes { - println!(" {note}"); - } - } - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - if report.suggestions.len() > 1 { - println!("Other useful commands:"); - for suggestion in report.suggestions.iter().skip(1) { - println!(" {}", suggestion.command); - println!(" {}", suggestion.reason); - } - } - } - Ok(()) -} - -pub(crate) fn render_agent_tools(report: &AgentToolsReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent tools: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("{}", report.summary); - println!( - "Tool events: {} Tools: {} Turns with tools: {}", - report.total_tool_events, report.unique_tools, report.turns_with_tools - ); - if report.available_commands.is_empty() { - println!("Available commands: none captured"); - } else { - println!("Available commands:"); - for command in report.available_commands.iter().take(20) { - println!(" {command}"); - } - if report.available_commands.len() > 20 { - println!(" ... {} more", report.available_commands.len() - 20); - } - } - if report.tools.is_empty() { - println!("Tool usage: none captured"); - } else { - println!("Tool usage:"); - for tool in &report.tools { - println!(); - println!( - " {}. {}{} {} event(s) across {} turn(s)", - tool.rank, - tool.name, - tool.kind - .as_ref() - .map(|kind| format!(" ({kind})")) - .unwrap_or_default(), - tool.event_count, - tool.turn_count - ); - if !tool.statuses.is_empty() { - let statuses = tool - .statuses - .iter() - .map(|(status, count)| format!("{status}:{count}")) - .collect::>() - .join(", "); - println!(" Statuses: {statuses}"); - } - if !tool.event_types.is_empty() { - println!(" Events: {}", tool.event_types.join(", ")); - } - println!( - " Changed files around this tool: {}", - tool.changed_paths.len() - ); - print_changed_paths(&tool.changed_paths, " "); - println!(" Turns:"); - for turn in &tool.turns { - let prompt = turn - .prompt_preview - .as_deref() - .map(|value| single_line_preview(value, 120)) - .unwrap_or_else(|| "no prompt preview".to_string()); - println!(" turn {} {} - {}", turn.index, turn.status, prompt); - if let Some(checkpoint) = &turn.checkpoint { - println!(" checkpoint: {}", checkpoint.0); - } - println!(" inspect: {}", turn.turn_command); - if let Some(command) = &turn.diff_command { - println!(" diff: {command}"); - } - } - } - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_impact( - report: &AgentImpactReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent impact: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("{}", report.summary); - println!( - "Changed files: {} Changed lines: {} Highest impact: {}", - report.changed_paths.len(), - report.changed_lines, - report.highest_impact - ); - print_agent_risk_line(&report.risk); - println!("Validation: {}", report.validation.status); - if report.areas.is_empty() { - println!("Impact areas: none"); - } else { - println!("Impact areas:"); - for area in &report.areas { - println!(); - println!( - " {} ({}) {} file(s), {} changed line(s)", - area.label, - area.severity, - area.changed_paths.len(), - area.changed_lines - ); - if !area.reasons.is_empty() { - println!(" Why: {}", area.reasons.join(", ")); - } - print_changed_paths(&area.changed_paths, " "); - println!(" Review: {}", area.review_command); - if let Some(command) = &area.diff_command { - println!(" Patch: {command}"); - } - } - } - if !report.recommendations.is_empty() { - println!("Recommended checks:"); - for recommendation in &report.recommendations { - println!(" {}", recommendation.command); - println!(" {}", recommendation.reason); - } - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_review_map( - report: &AgentReviewMapReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!( - "Agent review map: {}", - agent_task_display_title(&report.task) - ); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("{}", report.summary); - println!( - "Review: {} Changed files: {} Changed lines: {} Highest impact: {}", - report.review_status, - report.changed_paths.len(), - report.changed_lines, - report.highest_impact - ); - print_agent_risk_line(&report.risk); - println!("Validation: {}", report.validation.status); - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - if report.areas.is_empty() { - println!("Review areas: none"); - } else { - println!("Review areas:"); - for area in &report.areas { - println!(); - println!( - " {} ({}, {}) {} file(s), {} changed line(s)", - area.label, - area.severity, - area.state, - area.files.len(), - area.changed_lines - ); - if !area.reasons.is_empty() { - println!(" Why: {}", area.reasons.join(", ")); - } - println!(" Start: {}", area.review_command); - if let Some(command) = &area.patch_command { - println!(" Patch: {command}"); - } - for file in &area.files { - println!( - " {}. {} [{}] +{} -{} score {}", - file.rank, - file.path, - file.state, - file.change.additions, - file.change.deletions, - file.score - ); - if let Some(marker) = &file.reviewed { - println!(" Reviewed at: {}", marker.checkpoint.0); - } - if !file.reasons.is_empty() { - println!(" Reasons: {}", file.reasons.join(", ")); - } - println!(" Review: {}", file.review_command); - if let Some(command) = &file.open_command { - println!(" Open: {command}"); - } - println!(" Why: {}", file.why_command); - if let Some(command) = &file.diff_command { - println!(" Patch: {command}"); - } - } - } - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_risk(report: &AgentRiskReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent risk: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Risk: {:?} ({}/100)", report.level, report.score); - println!("{}", report.summary); - if report.reasons.is_empty() { - println!("Reasons: none"); - } else { - println!("Reasons:"); - for reason in &report.reasons { - println!( - " [{}] {}: {}", - reason.severity, reason.code, reason.message - ); - } - } - if !report.recommendations.is_empty() { - println!("Recommendations:"); - for recommendation in &report.recommendations { - println!(" {}", recommendation.command); - println!(" {}", recommendation.reason); - } - } - } - Ok(()) -} - -pub(crate) fn render_agent_confidence( - report: &AgentConfidenceReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!( - "Agent confidence: {}", - agent_task_display_title(&report.task) - ); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Verdict: {} Score: {}/100", report.verdict, report.score); - println!("{}", report.summary); - println!( - "Review: {} Validation: {} Apply: {}", - report.review_status, report.validation.status, report.ready.status - ); - print_agent_risk_line(&report.risk); - if let Some(marker) = &report.reviewed { - println!("Reviewed checkpoint: {}", marker.checkpoint.0); - } - if report.factors.is_empty() { - println!("Factors: none"); - } else { - println!("Factors:"); - for factor in &report.factors { - println!( - " [{}] {} ({})", - factor.state, factor.name, factor.score_delta - ); - println!(" {}", factor.message); - if let Some(command) = &factor.command { - println!(" {command}"); - } - } - } - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_ready(report: &AgentReadyReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent ready: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Ready: {}", report.ready); - println!("Status: {}", report.status); - println!("Readiness: {}", report.readiness_status); - print_agent_risk_line(&report.risk); - println!("Default commit message: {}", report.default_apply_message); - println!("{}", report.summary); - if !report.blockers.is_empty() { - println!("Blockers:"); - for blocker in &report.blockers { - println!(" {}: {}", blocker.code, blocker.message); - } - } - if !report.warnings.is_empty() { - println!("Warnings:"); - for warning in &report.warnings { - println!(" {}: {}", warning.code, warning.message); - } - } - if let Some(error) = &report.apply_error { - println!("Git preflight: failed"); - println!(" {error}"); - } else if let Some(preview) = &report.apply_preview { - println!("Git preflight: {}", preview.status); - if let Some(branch) = &preview.git_apply_plan.git_branch { - println!("Into Git branch: {branch}"); - } - if let Some(range) = &preview.git_apply_plan.range { - println!("Trail range: {range}"); - } - if preview.git_apply_plan.would_record { - println!("Would record task workdir before applying"); - } - if preview.git_apply_plan.would_create_git_commit { - println!("Would create Git commit"); - } - if preview.git_apply_plan.would_fast_forward { - println!("Would fast-forward current Git branch"); - } - } - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_diagnose( - report: &AgentDiagnosisReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent diagnose: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Status: {} Severity: {}", report.status, report.severity); - println!( - "Ready: {} Readiness: {}", - report.ready, report.readiness_status - ); - print_agent_risk_line(&report.risk); - println!("{}", report.summary); - println!("Likely issue:"); - println!(" {}", report.likely_issue); - if !report.evidence.is_empty() { - println!("Evidence:"); - for item in &report.evidence { - println!(" {item}"); - } - } - if !report.blockers.is_empty() { - println!("Blockers:"); - for blocker in &report.blockers { - println!(" {}: {}", blocker.code, blocker.message); - } - } - if !report.warnings.is_empty() { - println!("Warnings:"); - for warning in &report.warnings { - println!(" {}: {}", warning.code, warning.message); - } - } - if !report.checkpoints.is_empty() { - println!("Recent recovery targets:"); - for checkpoint in &report.checkpoints { - print_agent_checkpoint_entry(checkpoint); - } - } - if !report.recovery_options.is_empty() { - println!("Recovery options:"); - for option in &report.recovery_options { - println!(" {}", option.command); - println!(" {}", option.reason); - } - } - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_compare( - report: &AgentCompareReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent compare"); - println!("{}", report.summary); - println!(); - print_agent_compare_task("Left", &report.left, &report.left_risk); - print_agent_compare_task("Right", &report.right, &report.right_risk); - println!(); - println!("Shared changed files: {}", report.shared_paths.len()); - if !report.shared_paths.is_empty() { - for path in &report.shared_paths { - println!( - " {} left {:?} (+{} -{}) right {:?} (+{} -{})", - path.path, - path.left.kind, - path.left.additions, - path.left.deletions, - path.right.kind, - path.right.additions, - path.right.deletions - ); - } - } - println!("Left only: {}", report.left_only_paths.len()); - print_changed_paths(&report.left_only_paths, " "); - println!("Right only: {}", report.right_only_paths.len()); - print_changed_paths(&report.right_only_paths, " "); - println!("Recommendation:"); - println!(" {}", report.recommendation.command); - println!(" {}", report.recommendation.reason); - if report.suggestions.len() > 1 { - println!("Other useful commands:"); - for suggestion in report.suggestions.iter().skip(1) { - println!(" {}", suggestion.command); - println!(" {}", suggestion.reason); - } - } - } - Ok(()) -} - -pub(crate) fn render_agent_workdir( - report: &AgentWorkdirReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent workdir: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - if let Some(workdir) = &report.workdir { - println!("Workdir: {workdir}"); - if let Some(command) = &report.cd_command { - println!("Command:"); - println!(" {command}"); - } - } else { - println!("No materialized workdir recorded for this task"); - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_view( - report: &AgentTaskViewReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - print_agent_task_summary(&report.task); - if !report.task.changed_paths.is_empty() { - println!("Changed paths:"); - for path in &report.task.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } - } - if let Some(transcript) = &report.transcript { - println!("Transcript: {} turn(s)", transcript.turns.len()); - for turn in &transcript.turns { - println!(" Turn {} {}", turn.turn.turn_id, turn.turn.status); - for message in &turn.messages { - println!( - " {}: {}", - message.role, - single_line_preview(&message.body, 120) - ); - } - for tool in &turn.tool_summaries { - println!(" tool: {tool}"); - } - } - } - print_suggestions(&report.task.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_changes( - report: &AgentChangesReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent changes: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Status: {:?}", report.task.status); - println!("{}", report.summary); - println!("Range: {}..{}", report.base_change.0, report.head_change.0); - println!( - "Grouping: {} Total changed files: {}", - report.grouping, - report.total_changed_paths.len() - ); - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - if !report.cards.is_empty() { - println!("Change cards:"); - for card in &report.cards { - print_agent_change_card(card); - } - } - if report.groups.is_empty() { - println!("No recorded changes"); - } else { - println!("Details:"); - for group in &report.groups { - print_agent_change_group(group); - } - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_delta(report: &AgentDeltaReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent delta: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Mode: {} Matched: {}", report.mode, report.matched); - if let Some(path) = &report.file_filter { - println!("File: {path}"); - } - println!("{}", report.summary); - if let Some(group) = &report.group { - println!("Newest delta:"); - print_agent_change_group(group); - } else { - println!("Newest delta: none"); - } - println!("Changed files: {}", report.changed_paths.len()); - print_changed_paths(&report.changed_paths, " "); - if let Some(diff) = &report.diff { - if diff.diff.files.iter().any(|file| file.patch.is_some()) { - println!("Patch:"); - print_diff_files(&diff.diff.files, true); - } - } - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_new(report: &AgentNewReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent new: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Status: {} Matched: {}", report.status, report.matched); - if let Some(marker) = &report.reviewed { - println!( - "Reviewed: {} at {}", - marker.checkpoint.0, marker.reviewed_at - ); - } else { - println!("Reviewed: not marked yet"); - } - if let Some(path) = &report.file_filter { - println!("File: {path}"); - } - println!("Range: {}..{}", report.base_change.0, report.head_change.0); - println!("{}", report.summary); - if report.new_groups.is_empty() { - println!("New turns or operations: none"); - } else { - println!("New turns or operations:"); - for group in &report.new_groups { - print_agent_change_group(group); - } - } - println!("Changed files: {}", report.changed_paths.len()); - print_changed_paths(&report.changed_paths, " "); - if let Some(diff) = &report.diff { - if diff.diff.files.iter().any(|file| file.patch.is_some()) { - println!("Patch:"); - print_diff_files(&diff.diff.files, true); - } - } - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_mark_reviewed( - report: &AgentMarkReviewedReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent reviewed: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Checkpoint: {}", report.marker.checkpoint.0); - println!("Changed files: {}", report.marker.changed_paths); - if let Some(previous) = &report.previous { - println!("Previous reviewed: {}", previous.checkpoint.0); - } - if let Some(note) = &report.marker.note { - println!("Note: {note}"); - } - println!("{}", report.summary); - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_mark_file_reviewed( - report: &AgentMarkFileReviewedReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent file reviewed: {}", report.path); - println!("Task: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Checkpoint: {}", report.marker.checkpoint.0); - if let Some(note) = &report.marker.note { - println!("Note: {note}"); - } - if let Some(previous) = &report.previous { - println!("Previous file marker: {}", previous.checkpoint.0); - } - println!("{}", report.summary); - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_archive( - report: &AgentArchiveReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent archive: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!( - "Archived: {} Previous: {}", - report.archived, report.previous_archived - ); - if let Some(note) = &report.note { - println!("Note: {note}"); - } - println!("{}", report.summary); - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_timeline( - report: &AgentTimelineReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent timeline: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Status: {:?}", report.task.status); - println!("{}", report.summary); - println!("Range: {}..{}", report.base_change.0, report.head_change.0); - println!( - "Mode: {} Items: {} Changed files: {}", - report.mode, - report.items.len(), - report.task.changed_paths.len() - ); - if report.items.is_empty() { - println!("No recorded timeline items"); - } else { - println!("Timeline:"); - for item in &report.items { - print_agent_timeline_item(item); - } - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_change_set( - report: &AgentChangeSetReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!( - "Agent change set: {}", - agent_task_display_title(&report.task) - ); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!( - "Card: {}. {} ({}) {:?}", - report.card.rank, report.card.title, report.card.key, report.card.risk - ); - println!("{}", report.summary); - if !report.card.reasons.is_empty() { - println!("Review notes: {}", report.card.reasons.join(", ")); - } - println!("Files:"); - if report.files.is_empty() { - println!(" No changed files recorded"); - } else { - for file in &report.files { - print_agent_file_entry(file); - } - } - if !report.groups.is_empty() { - println!("Changed in:"); - for group in &report.groups { - print_agent_change_group(group); - } - } - if !report.diffs.is_empty() { - println!("Focused patches:"); - for diff in &report.diffs { - println!(); - println!( - " {} {}", - diff.target_kind, - diff.file_filter.as_deref().unwrap_or(&diff.target) - ); - print_diff_files(&diff.diff.files, true); - } - } - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_files(report: &AgentFilesReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent files: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!( - "Grouping: {} Changed files: {}", - report.grouping, - report.files.len() - ); - if report.files.is_empty() { - println!("No changed files recorded"); - } else { - for file in &report.files { - print_agent_file_entry(file); - } - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_file(report: &AgentFileReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent file: {}", report.path); - println!("Task: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Matched: {}", report.matched); - println!("{}", report.summary); - if let Some(file) = &report.file { - println!("File entry:"); - print_agent_file_entry(file); - } - if !report.change_cards.is_empty() { - println!("Change sets:"); - for card in &report.change_cards { - print_agent_change_card(card); - } - } - if !report.groups.is_empty() { - println!("Changed in:"); - for group in &report.groups { - print_agent_change_group(group); - } - } - if let Some(diff) = &report.diff { - println!("Focused patch:"); - print_diff_files(&diff.diff.files, true); - } - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_checkpoints( - report: &AgentCheckpointReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!( - "Agent checkpoints: {}", - agent_task_display_title(&report.task) - ); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Base: {}", report.base_change.0); - println!("Head: {}", report.head_change.0); - if report.entries.is_empty() { - println!("No turn or operation checkpoints recorded"); - } else { - println!("Targets:"); - for entry in &report.entries { - print_agent_checkpoint_entry(entry); - } - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_why(report: &AgentWhyReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent why: {}", report.path); - println!("Task: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("{}", report.summary); - if let Some(change) = &report.task_change { - println!( - "Task change: {:?} {} (+{} -{})", - change.kind, change.path, change.additions, change.deletions - ); - if let Some(old_path) = &change.old_path { - println!(" Old path: {old_path}"); - } - } - if report.groups.is_empty() { - println!("No prompt, turn, or operation changed this file in the selected task."); - } else { - println!("Changed in:"); - for group in &report.groups { - print_agent_change_group(group); - } - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_turn(report: &AgentTurnReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!( - "Agent turn {}: {}", - report.index, - agent_task_display_title(&report.task) - ); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Status: {}", report.status); - println!("Turn id: {}", report.turn_id); - if let Some(envelope) = &report.turn_envelope { - if let Some(provider) = &envelope.provider { - println!("Provider: {provider}"); - } - if let Some(model) = &envelope.model { - println!("Model: {model}"); - } - if envelope.outcome.no_changes { - println!("Outcome: no changes"); - } - if envelope.usage.used.is_some() || envelope.usage.size.is_some() { - println!( - "Usage: used {} / size {}", - envelope - .usage - .used - .map(|value| value.to_string()) - .unwrap_or_else(|| "-".to_string()), - envelope - .usage - .size - .map(|value| value.to_string()) - .unwrap_or_else(|| "-".to_string()) - ); - } - } - if let Some(prompt) = &report.prompt_preview { - println!("Prompt: {prompt}"); - } - if let Some(assistant) = &report.assistant_preview { - println!("Assistant: {assistant}"); - } - println!( - "Messages: {} Events: {} Tools: {}", - report.messages.len(), - report.event_count, - report.tool_summaries.len() - ); - println!("Before: {}", report.before_change.0); - if let Some(checkpoint) = &report.checkpoint { - println!("Checkpoint: {}", checkpoint.0); - } else { - println!("Checkpoint: none"); - } - println!("Changed files: {}", report.changed_paths.len()); - print_changed_paths(&report.changed_paths, " "); - for tool in &report.tool_summaries { - println!(" tool: {tool}"); - } - if let Some(diff) = &report.diff { - if diff.file_filter.is_some() || diff.diff.files.iter().any(|file| file.patch.is_some()) - { - println!("Diff:"); - if let Some(path) = &diff.file_filter { - println!(" File: {path}"); - } - print_diff_files(&diff.diff.files, false); - } - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_diff( - report: &AgentDiffReport, - json: bool, - quiet: bool, - stat: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent diff: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Target: {} {}", report.target_kind, report.target); - if let Some(turn_id) = &report.turn_id { - println!("Turn: {turn_id}"); - } - if let Some(operation_id) = &report.operation_id { - println!("Operation: {}", operation_id.0); - } - if let Some(path) = &report.file_filter { - println!("File: {path}"); - } - println!( - "Range: {}..{}", - report.before_change.0, report.after_change.0 - ); - print_diff_files(&report.diff.files, stat); - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_review( - report: &AgentReviewReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent review: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!( - "Status: {:?} Ready: {} Changed files: {}", - report.task.status, - report.ready_to_apply, - report.task.changed_paths.len() - ); - println!( - "Transcript: {} turn(s) Tool events: {}", - report.transcript_turns, report.tool_events - ); - if let Some(checkpoint) = &report.latest_checkpoint { - println!("Last checkpoint: {}", checkpoint.0); - } - print_agent_risk_line(&report.risk); - println!("{}", report.summary); - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - if !report.blockers.is_empty() { - println!("Blockers:"); - for blocker in &report.blockers { - println!(" {}: {}", blocker.code, blocker.message); - } - } - if !report.warnings.is_empty() { - println!("Warnings:"); - for warning in &report.warnings { - println!(" {}: {}", warning.code, warning.message); - } - } - if report.priorities.is_empty() { - println!("Review priority: no changed files recorded"); - } else { - println!("Review priority:"); - for priority in report.priorities.iter().take(8) { - print_agent_review_priority(priority); - } - if report.priorities.len() > 8 { - println!( - " ... {} more file(s); run `trail agent files {}`", - report.priorities.len() - 8, - report.task.lane - ); - } - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_focus(report: &AgentFocusReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent focus: {}", report.path); - println!("Task: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - if let Some(open_path) = &report.open_path { - println!("Open path: {open_path}"); - } - if let Some(open_command) = &report.open_command { - println!("Open command:"); - println!(" {open_command}"); - } - println!("Source: {}", report.source); - println!("{}", report.summary); - if let Some(priority) = &report.priority { - println!("Review priority:"); - print_agent_review_priority(priority); - } - if report.why.groups.is_empty() { - println!("Changed in: no captured turn or operation"); - } else { - println!("Changed in:"); - for group in &report.why.groups { - print_agent_change_group(group); - } - } - println!("Focused diff:"); - println!( - " Target: {} {}", - report.diff.target_kind, report.diff.target - ); - println!( - " Range: {}..{}", - report.diff.before_change.0, report.diff.after_change.0 - ); - print_diff_files(&report.diff.diff.files, true); - println!("Next:"); - println!(" {}", report.next.command); - println!(" {}", report.next.reason); - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_apply(report: &AgentApplyReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - if report.dry_run { - println!( - "Would apply agent task: {}", - agent_task_display_title(&report.task) - ); - } else { - println!( - "Applied agent task: {}", - agent_task_display_title(&report.task) - ); - } - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - if let Some(branch) = &report.git_apply_plan.git_branch { - println!("Into Git branch: {branch}"); - } - println!("Trail base: {}", report.git_apply_plan.crab_branch); - println!("Changed files: {}", report.task.changed_paths.len()); - println!("Result: {}", report.status); - if report.git_apply_plan.would_record { - println!("Would record lane workdir before applying"); - } - if let Some(range) = &report.git_apply_plan.range { - println!("Trail range: {range}"); - } - if let Some(export) = &report.git_export { - println!("Git commit: {}", export.commit); - } - if report.fast_forwarded { - println!("Fast-forwarded current Git branch"); - } - for warning in &report.warnings { - println!("warning: {warning}"); - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_finish( - report: &AgentFinishReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - if report.dry_run { - println!( - "Would finish agent task: {}", - agent_task_display_title(&report.task) - ); - } else { - println!( - "Finished agent task: {}", - agent_task_display_title(&report.task) - ); - } - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Apply: {}", report.apply.status); - if let Some(export) = &report.apply.git_export { - println!("Git commit: {}", export.commit); - } - if report.apply.fast_forwarded { - println!("Fast-forwarded current Git branch"); - } - if report.dry_run { - println!("Archive after apply: {}", report.would_archive); - } else if let Some(archive) = &report.archive { - println!("Archived: {}", archive.archived); - } else if report.task.archived { - println!("Archived: already"); - } else { - println!("Archived: no"); - } - println!("Result: {}", report.status); - for warning in &report.apply.warnings { - println!("warning: {warning}"); - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_run(report: &AgentRunReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!("Agent task: {}", agent_task_display_title(&report.task)); - print_agent_task_id_if_needed(&report.task); - print_agent_task_workdir(&report.task); - println!("Status: {}", report.status); - if let Some(code) = report.exit_code { - println!("Exit code: {code}"); - } - if let Some(recorded) = &report.recorded { - if let Some(operation) = &recorded.operation { - println!("Checkpoint: {}", operation.0); - } - } - print_suggestions(&report.task.suggestions); - } - Ok(()) -} - -pub(crate) fn render_agent_continue( - report: &AgentContinueReport, - json: bool, - quiet: bool, -) -> Result<()> { - if json { - return render_json(report); - } - if !quiet { - println!( - "Agent follow-up: {}", - agent_task_display_title(&report.run.task) - ); - println!( - "From: {} ({})", - agent_task_display_title(&report.source_task), - report.from_change.0 - ); - print_agent_task_id_if_needed(&report.run.task); - print_agent_task_workdir(&report.run.task); - println!("Status: {}", report.run.status); - if let Some(code) = report.run.exit_code { - println!("Exit code: {code}"); - } - if let Some(recorded) = &report.run.recorded { - if let Some(operation) = &recorded.operation { - println!("Checkpoint: {}", operation.0); - } - } - print_suggestions(&report.suggestions); - } - Ok(()) -} - -fn print_agent_task_summary(task: &AgentTaskReport) { - println!("Agent task: {}", agent_task_display_title(task)); - print_agent_task_id_if_needed(task); - print_agent_task_workdir(task); - println!("Status: {:?}", task.status); - if task.archived { - println!("Archived: yes"); - } - if let Some(provider) = &task.provider { - println!("Provider: {provider}"); - } - println!("Changed files: {}", task.changed_paths.len()); - println!("Turns: {} Tool events: {}", task.turns, task.tool_events); - if let Some(checkpoint) = &task.latest_checkpoint { - println!("Last checkpoint: {}", checkpoint.0); - } -} - -fn print_agent_inbox_group(group: &AgentInboxGroup) { - println!(); - println!("{}: {}", group.label, group.items.len()); - for item in &group.items { - let task = &item.task; - let checkpoint = task - .latest_checkpoint - .as_ref() - .map(|change| change.0.as_str()) - .unwrap_or("-"); - println!( - " {}{} {} file(s) {} turn(s) checkpoint {}", - agent_task_display_title(task), - if task.archived { " [archived]" } else { "" }, - task.changed_paths.len(), - task.turns, - checkpoint - ); - if agent_task_display_title(task) != task.name { - println!(" Task id: {}", task.name); + #[test] + fn agent_state_fixture_matrix_keeps_recovery_states_distinct() { + for (state, expected, tone) in [ + ("running", "running", UiTone::Neutral), + ("dirty", "needs record", UiTone::Attention), + ("blocked", "blocked", UiTone::Blocked), + ("conflicted", "conflicted", UiTone::Blocked), + ("ready", "ready", UiTone::Success), + ("archived", "archived", UiTone::Neutral), + ] { + let document = projected_document("Agent task", &serde_json::json!({"status": state})); + let lead = document.lead.unwrap(); + assert_eq!(lead.text, format!("Agent task: {expected}")); + assert_eq!(lead.tone, tone); } - if let Some(workdir) = &task.workdir { - println!(" Workdir: {workdir}"); - } - println!(" Attention: {} - {}", item.attention, item.detail); - if item.new_changed_paths > 0 { - println!( - " New since review: {} file(s), {} line(s)", - item.new_changed_paths, item.new_changed_lines - ); - } - if let Some(target) = &item.review_first { - println!(" Review first: {} ({})", target.path, target.reason); - println!(" {}", target.command); - } - println!(" Next: {}", item.next.command); - } - if let Some(next) = &group.next { - println!(" Group next: {}", next.command); - println!(" {}", next.reason); - } -} - -fn print_agent_board_column(column: &AgentBoardColumn) { - println!(); - println!("{}: {}", column.label, column.items.len()); - println!(" {}", column.summary); - for item in &column.items { - let task = &item.task; - let checkpoint = task - .latest_checkpoint - .as_ref() - .map(|change| change.0.as_str()) - .unwrap_or("-"); - println!( - " {}{} [{}] {} file(s) {} turn(s) checkpoint {}", - agent_task_display_title(task), - if task.archived { " [archived]" } else { "" }, - item.status_label, - item.changed_paths, - item.turns, - checkpoint - ); - if agent_task_display_title(task) != task.name { - println!(" Task id: {}", task.name); - } - println!(" {}", item.detail); - if let Some(target) = &item.review_first { - println!(" Review first: {} ({})", target.path, target.reason); - } - println!(" Next: {}", item.next.command); - } - if let Some(next) = &column.next { - println!(" Column next: {}", next.command); - println!(" {}", next.reason); - } -} - -fn agent_task_display_title(task: &AgentTaskReport) -> &str { - if task.title.trim().is_empty() { - &task.name - } else { - &task.title - } -} - -fn print_agent_task_id_if_needed(task: &AgentTaskReport) { - if agent_task_display_title(task) != task.name { - println!("Task id: {}", task.name); - } -} - -fn print_agent_task_workdir(task: &AgentTaskReport) { - if let Some(workdir) = &task.workdir { - println!("Workdir: {workdir}"); - } -} - -fn print_agent_risk_line(risk: &AgentRiskReport) { - let reason_codes = risk - .reasons - .iter() - .take(3) - .map(|reason| reason.code.as_str()) - .collect::>(); - if reason_codes.is_empty() { - println!("Risk: {:?} ({}/100)", risk.level, risk.score); - } else { - println!( - "Risk: {:?} ({}/100) {}", - risk.level, - risk.score, - reason_codes.join(", ") - ); - } -} - -fn print_agent_compare_task(label: &str, task: &AgentTaskReport, risk: &AgentRiskReport) { - println!( - "{label}: {} {:?} {} file(s) {} turn(s)", - agent_task_display_title(task), - task.status, - task.changed_paths.len(), - task.turns - ); - if agent_task_display_title(task) != task.name { - println!(" Task id: {}", task.name); - } - println!(" Lane: {}", task.lane); - println!(" Risk: {:?} ({}/100)", risk.level, risk.score); - if let Some(workdir) = &task.workdir { - println!(" Workdir: {workdir}"); - } -} - -fn print_agent_change_group(group: &AgentChangeGroup) { - match group.kind.as_str() { - "turn" => { - let prompt = group - .prompt_preview - .as_deref() - .unwrap_or("(no prompt captured)"); - println!(); - println!("Turn {}: {prompt}", group.index); - } - _ => { - println!(); - println!("Operation {}: {}", group.index, group.id); - } - } - if let Some(status) = &group.status { - println!(" Status: {status}"); - } - if let Some(checkpoint) = &group.checkpoint { - println!(" Checkpoint: {}", checkpoint.0); - } - if let Some(assistant) = &group.assistant_preview { - println!(" Assistant: {assistant}"); - } - if !group.operations.is_empty() { - let operations = group - .operations - .iter() - .map(|change| change.0.as_str()) - .collect::>() - .join(", "); - println!(" Operations: {operations}"); - } - println!(" Changed files: {}", group.changed_paths.len()); - print_changed_paths(&group.changed_paths, " "); - for tool in &group.tool_summaries { - println!(" tool: {tool}"); - } -} - -fn print_agent_change_card(card: &AgentChangeCard) { - println!(); - println!(" {}. {} {:?}", card.rank, card.title, card.risk); - println!(" {}", card.summary); - if !card.reasons.is_empty() { - println!(" Review notes: {}", card.reasons.join(", ")); - } - if !card.touched_by.is_empty() { - let touched_by = card - .touched_by - .iter() - .map(|touch| match touch.kind.as_str() { - "turn" => format!("turn {}", touch.index), - _ => format!("operation {}", touch.index), - }) - .collect::>() - .join(", "); - println!(" Touched by: {touched_by}"); - } - println!(" Files:"); - print_changed_paths(&card.changed_paths, " "); - for tool in &card.tool_summaries { - println!(" tool: {tool}"); - } - println!(" Review: {}", card.review_command); - if let Some(command) = &card.focus_command { - println!(" Focus: {command}"); - } - if let Some(command) = &card.why_command { - println!(" Why: {command}"); - } - if let Some(command) = &card.diff_command { - println!(" Diff: {command}"); - } -} - -fn print_agent_timeline_item(item: &AgentTimelineItem) { - println!(); - println!(" {}", item.title); - if let Some(status) = &item.status { - println!(" Status: {status}"); - } - if let Some(assistant) = &item.assistant_preview { - println!(" Assistant: {assistant}"); - } - println!( - " Messages: {} Events: {} Tools: {}", - item.message_count, - item.event_count, - item.tool_summaries.len() - ); - if let Some(before) = &item.before_change { - println!(" Before: {}", before.0); - } - if let Some(checkpoint) = &item.checkpoint { - println!(" Checkpoint: {}", checkpoint.0); - } - if !item.operations.is_empty() { - let operations = item - .operations - .iter() - .map(|change| change.0.as_str()) - .collect::>() - .join(", "); - println!(" Operations: {operations}"); - } - println!(" Changed files: {}", item.changed_paths.len()); - print_changed_paths(&item.changed_paths, " "); - for tool in &item.tool_summaries { - println!(" tool: {tool}"); - } - if let Some(command) = &item.view_command { - println!(" View: {command}"); - } - if let Some(command) = &item.diff_command { - println!(" Diff: {command}"); - } - if let Some(command) = &item.rewind_before_command { - println!(" Rewind before: {command}"); - } -} - -fn print_agent_checkpoint_entry(entry: &AgentCheckpointEntry) { - let label = entry - .prompt_preview - .as_deref() - .map(|prompt| format!("{}: {prompt}", entry.label)) - .unwrap_or_else(|| entry.label.clone()); - println!(); - println!(" {label}"); - if let Some(before) = &entry.before_change { - let target = entry.before_target.as_deref().unwrap_or(before.0.as_str()); - println!(" Before: {} target `{target}`", before.0); - } - if let Some(checkpoint) = &entry.checkpoint { - let target = entry - .checkpoint_target - .as_deref() - .unwrap_or(checkpoint.0.as_str()); - println!(" Checkpoint: {} target `{target}`", checkpoint.0); - } - println!(" Changed files: {}", entry.changed_paths.len()); - if let Some(command) = &entry.diff_command { - println!(" Diff: {command}"); - } - if let Some(command) = &entry.rewind_before_command { - println!(" Rewind before: {command}"); - } - if let Some(command) = &entry.rewind_checkpoint_command { - println!(" Rewind to checkpoint: {command}"); - } -} - -fn print_agent_file_entry(file: &AgentFileEntry) { - println!(); - println!( - " {:?} {} (+{} -{})", - file.change.kind, file.change.path, file.change.additions, file.change.deletions - ); - if let Some(old_path) = &file.change.old_path { - println!(" Old path: {old_path}"); - } - if file.touched_by.is_empty() { - println!(" Touched by: no captured turn or operation"); - } else { - println!(" Touched by:"); - for touch in &file.touched_by { - let label = match touch.kind.as_str() { - "turn" => format!("turn {}", touch.index), - _ => format!("operation {}", touch.index), - }; - if let Some(prompt) = &touch.prompt_preview { - println!(" {label}: {prompt}"); - } else { - println!(" {label}: {}", touch.id); - } - if let Some(checkpoint) = &touch.checkpoint { - println!(" checkpoint: {}", checkpoint.0); - } - } - } - println!(" Why: {}", file.why_command); - if let Some(command) = &file.diff_command { - println!(" Diff: {command}"); - } -} - -fn print_agent_review_priority(priority: &AgentReviewPriority) { - println!(); - println!( - " {}. {:?} {} (+{} -{}) score {}", - priority.rank, - priority.change.kind, - priority.change.path, - priority.change.additions, - priority.change.deletions, - priority.score - ); - if !priority.reasons.is_empty() { - println!(" Why review: {}", priority.reasons.join(", ")); - } - if !priority.touched_by.is_empty() { - let touches = priority - .touched_by - .iter() - .map(|touch| match touch.kind.as_str() { - "turn" => format!("turn {}", touch.index), - _ => format!("operation {}", touch.index), - }) - .collect::>() - .join(", "); - println!(" Touched by: {touches}"); - } - println!(" Why: {}", priority.why_command); - if let Some(command) = &priority.diff_command { - println!(" Diff: {command}"); - } -} - -fn print_agent_brief_group(group: &AgentChangeGroup) { - let label = match group.kind.as_str() { - "turn" => format!("Turn {}", group.index), - _ => format!("Operation {}", group.index), - }; - let summary = group - .prompt_preview - .as_deref() - .or(group.assistant_preview.as_deref()) - .unwrap_or(&group.id); - println!( - " {label}: {summary} ({} changed file(s))", - group.changed_paths.len() - ); -} - -fn print_agent_story_turn(turn: &AgentStoryTurn) { - let label = turn - .prompt_preview - .as_deref() - .or(turn.outcome_preview.as_deref()) - .unwrap_or(&turn.id); - println!( - " {}. {} ({} changed file(s))", - turn.index, - label, - turn.changed_paths.len() - ); - if let Some(checkpoint) = &turn.checkpoint { - println!(" Checkpoint: {}", checkpoint.0); - } - for tool in &turn.tool_summaries { - println!(" tool: {tool}"); - } -} - -fn print_agent_brief_diff(diff: &DiffSummary) { - let additions: u64 = diff.files.iter().map(|file| file.additions).sum(); - let deletions: u64 = diff.files.iter().map(|file| file.deletions).sum(); - println!( - "Latest change: {} file(s), +{} -{}", - diff.files.len(), - additions, - deletions - ); -} - -fn print_changed_paths(paths: &[FileDiffSummary], indent: &str) { - for path in paths { - println!( - "{}{:?} {} (+{} -{})", - indent, path.kind, path.path, path.additions, path.deletions - ); - } -} - -fn print_diff_files(files: &[FileDiffSummary], stat: bool) { - let mut total_additions = 0; - let mut total_deletions = 0; - if files.is_empty() { - println!("No file changes"); - } - for file in files { - total_additions += file.additions; - total_deletions += file.deletions; - println!( - " {:?} {} (+{} -{})", - file.kind, file.path, file.additions, file.deletions - ); - if let Some(patch) = &file.patch { - print!("{patch}"); - } - } - if stat { - println!( - "{} files changed, {} additions, {} deletions", - files.len(), - total_additions, - total_deletions - ); - } -} - -fn print_suggestions(suggestions: &[StatusSuggestion]) { - if !suggestions.is_empty() { - println!("Next step:"); - for suggestion in suggestions { - println!(" {}", suggestion.command); - println!(" {}", suggestion.reason); - } - } -} - -fn print_agent_gate_summary(gate: &LaneTestSummary, indent: &str) { - let suite = gate.suite.as_deref().unwrap_or("default"); - let result = if gate.success { "passed" } else { "failed" }; - println!( - "{indent}{} `{}` {} ({})", - gate.kind, - suite, - result, - shell_join(&gate.command) - ); -} - -fn shell_join(parts: &[String]) -> String { - parts - .iter() - .map(|part| { - if part.chars().all(|ch| { - ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '/' | '.' | '@' | ':') - }) { - part.clone() - } else { - format!("'{}'", part.replace('\'', "'\\''")) - } - }) - .collect::>() - .join(" ") -} - -fn single_line_preview(value: &str, limit: usize) -> String { - let mut preview = value.split_whitespace().collect::>().join(" "); - if preview.len() > limit { - preview.truncate(limit.saturating_sub(3)); - preview.push_str("..."); } - preview } diff --git a/trail/src/cli/command/render/collaboration.rs b/trail/src/cli/command/render/collaboration.rs index 9eabf2d..3318342 100644 --- a/trail/src/cli/command/render/collaboration.rs +++ b/trail/src/cli/command/render/collaboration.rs @@ -1,4 +1,4 @@ -use super::render_json; +use super::*; use trail::model::*; use trail::Result; @@ -9,57 +9,89 @@ mod merge; pub(crate) use coordination::*; pub(crate) use merge::*; -pub(crate) fn render_branch(report: &BranchReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_branch( + report: &BranchReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Created branch {} from {}", report.name, report.from.0); - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Created Trail branch {}", report.name), + UiTone::Success, + ) + .context(format!("From {}", report.from.0)), + options, + ) } pub(crate) fn render_branch_list( entries: &[BranchListEntry], json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(&entries); } - if !quiet { - for entry in entries { - let marker = if entry.is_current { "*" } else { " " }; - println!("{marker} {} {}", entry.name, entry.change_id.0); - } - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("{} Trail branch(es)", entries.len()), + UiTone::Neutral, + ) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("BRANCH", 0, 12), + UiColumn::left("STATE", 1, 7), + UiColumn::right("GEN", 2, 3), + ], + entries + .iter() + .map(|entry| { + vec![ + entry.name.clone(), + if entry.is_current { "current" } else { "" }.to_string(), + entry.generation.to_string(), + ] + }) + .collect(), + ))), + options, + ) } pub(crate) fn render_branch_delete( report: &BranchDeleteReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Deleted branch {}", report.name); - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Deleted Trail branch {}", report.name), + UiTone::Success, + ), + options, + ) } pub(crate) fn render_branch_rename( report: &BranchRenameReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Renamed branch {} to {}", report.old_name, report.new_name); - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Renamed Trail branch {}", report.old_name), + UiTone::Success, + ) + .context(format!("New name: {}", report.new_name)), + options, + ) } diff --git a/trail/src/cli/command/render/collaboration/coordination.rs b/trail/src/cli/command/render/collaboration/coordination.rs index 8430117..9b06205 100644 --- a/trail/src/cli/command/render/collaboration/coordination.rs +++ b/trail/src/cli/command/render/collaboration/coordination.rs @@ -1,4 +1,4 @@ -use super::super::render_json; +use crate::cli::command::render::*; use trail::model::*; use trail::Result; @@ -6,157 +6,297 @@ use trail::Result; pub(crate) fn render_anchor_create( report: &AnchorCreateReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Created anchor {} at {}:{}", - report.anchor.id.0, report.anchor.created_path, report.anchor.created_line - ); - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Created anchor {}", report.anchor.label), + UiTone::Success, + ) + .block(anchor_metadata(&report.anchor)) + .next( + format!("trail anchor resolve {}", report.anchor.id.0), + "confirm the anchor still points to the intended code", + ), + options, + ) } pub(crate) fn render_anchor_resolve( report: &AnchorResolveReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Anchor: {}", report.anchor.id.0); - println!("Label: {}", report.anchor.label); - println!("Status: {}", report.status); - if let (Some(path), Some(line_number)) = (&report.path, report.line_number) { - println!("Location: {path}:{line_number}"); - } else if let Some(path) = &report.path { - println!("Path: {path}"); - } - if let Some(text) = &report.text { - println!("{text}"); - } - } - Ok(()) + let resolved = matches!(report.status.as_str(), "resolved" | "ok" | "active"); + let mut metadata = vec![ + ("Status".to_string(), report.status.clone()), + ("Branch".to_string(), report.branch.clone()), + ]; + if let Some(path) = &report.path { + let location = report + .line_number + .map(|line| format!("{path}:{line}")) + .unwrap_or_else(|| path.clone()); + metadata.push(("Location".to_string(), location)); + } + let mut document = TerminalDocument::new( + format!("Anchor {}", report.anchor.label), + if resolved { + UiTone::Success + } else { + UiTone::Attention + }, + ) + .block(UiBlock::Metadata(metadata)); + if let Some(text) = &report.text { + document = document.block(UiBlock::Patch { + title: "Anchored text".to_string(), + text: text.clone(), + }); + } + render_document(&document, options) } -pub(crate) fn render_anchor_list(anchors: &[Anchor], json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_anchor_list( + anchors: &[Anchor], + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(&anchors); } - if !quiet { - for anchor in anchors { - println!( - "{} {} {}:{}", - anchor.id.0, anchor.label, anchor.created_path, anchor.created_line - ); - } + if anchors.is_empty() { + return render_document( + &TerminalDocument::new("No anchors", UiTone::Neutral), + options, + ); } - Ok(()) + render_document( + &TerminalDocument::new(format!("{} anchor(s)", anchors.len()), UiTone::Neutral).block( + UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("LABEL", 0, 10), + UiColumn::left("LOCATION", 0, 16), + UiColumn::left("ID", 2, 12), + ], + anchors + .iter() + .map(|anchor| { + vec![ + anchor.label.clone(), + format!("{}:{}", anchor.created_path, anchor.created_line), + anchor.id.0.clone(), + ] + }) + .collect(), + )), + ), + options, + ) } pub(crate) fn render_anchor_delete( report: &AnchorDeleteReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Deleted anchor {}", report.anchor_id.0); - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Deleted anchor {}", report.anchor_id.0), + UiTone::Success, + ), + options, + ) } pub(crate) fn render_lease_acquire( report: &LeaseAcquireReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Acquired lease {} {} {} {}", - report.lease.lease_id, - report.lease.mode, - report.lease.lane_id, - report.lease.path.as_deref().unwrap_or("") - ); - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Acquired {} lease", report.lease.mode), + UiTone::Success, + ) + .block(lease_metadata(&report.lease, options)) + .next( + format!("trail lease release {}", report.lease.lease_id), + "release the lease when this coordinated work is complete", + ), + options, + ) } -pub(crate) fn render_lane_claim(report: &LaneClaimReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_lane_claim( + report: &LaneClaimReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { + let tone = if report.claimed { + UiTone::Success + } else { + UiTone::Blocked + }; + let mut document = TerminalDocument::new( if report.claimed { - if let Some(lease) = &report.lease { - println!( - "Claimed {} for {} until {} ({})", - report.path, report.lane_id, lease.expires_at, lease.lease_id - ); - } else { - println!("Claimed {} for {}", report.path, report.lane_id); - } - if !report.hydrated_paths.is_empty() { - println!( - "Hydrated {} sparse workdir paths", - report.hydrated_paths.len() - ); - } - if let Some(warning) = &report.hydration_warning { - println!("Warning: {warning}"); - } - } else if let Some(warning) = &report.warning { - println!("Warning: {warning}"); + format!("Claimed {} for {}", report.path, report.lane_id) } else { - println!("Path {} is already claimed", report.path); - } + format!("Could not claim {}", report.path) + }, + tone, + ) + .block(UiBlock::Metadata(vec![ + ("Lane".to_string(), report.lane_id.clone()), + ("Ref".to_string(), report.ref_name.clone()), + ("Mode".to_string(), report.mode.clone()), + ("TTL".to_string(), format!("{} seconds", report.ttl_secs)), + ])); + if !report.conflicts.is_empty() { + document = document.block(UiBlock::section( + "Conflicting leases:", + vec![UiBlock::Table(lease_table(&report.conflicts, options))], + )); + } + if let Some(warning) = &report.warning { + document = document.block(UiBlock::Checklist(vec![UiCheck::new( + UiCheckState::Warn, + "Claim warning", + warning, + )])); + } + if let Some(warning) = &report.hydration_warning { + document = document.block(UiBlock::Checklist(vec![UiCheck::new( + UiCheckState::Warn, + "Hydration warning", + warning, + )])); } - Ok(()) + if !report.hydrated_paths.is_empty() { + document = document.block(UiBlock::Notice(format!( + "Hydrated {} sparse path(s)", + report.hydrated_paths.len() + ))); + } + if !report.claimed { + document = document.next( + "trail lease list", + "inspect ownership before retrying the claim", + ); + } + render_document(&document, options) } -pub(crate) fn render_lease_list(leases: &[LeaseRecord], json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_lease_list( + leases: &[LeaseRecord], + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(&leases); } - if !quiet { - if leases.is_empty() { - println!("No active leases"); - } - for lease in leases { - println!( - "{} {} {} {} expires_at={}", - lease.lease_id, - lease.mode, - lease.lane_id, - lease.path.as_deref().unwrap_or(""), - lease.expires_at - ); - } - } - Ok(()) + if leases.is_empty() { + return render_document( + &TerminalDocument::new("No active leases", UiTone::Success), + options, + ); + } + render_document( + &TerminalDocument::new(format!("{} active lease(s)", leases.len()), UiTone::Neutral) + .block(UiBlock::Table(lease_table(leases, options))), + options, + ) } pub(crate) fn render_lease_release( report: &LeaseReleaseReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Released lease {}", report.lease_id); - } - Ok(()) + render_document( + &TerminalDocument::new( + if report.released { + format!("Released lease {}", report.lease_id) + } else { + format!("Lease {} was already released", report.lease_id) + }, + UiTone::Success, + ), + options, + ) +} + +fn anchor_metadata(anchor: &Anchor) -> UiBlock { + UiBlock::Metadata(vec![ + ("ID".to_string(), anchor.id.0.clone()), + ( + "Location".to_string(), + format!("{}:{}", anchor.created_path, anchor.created_line), + ), + ("Change".to_string(), anchor.created_change.0.clone()), + ]) +} + +fn lease_metadata(lease: &LeaseRecord, options: &RenderOptions) -> UiBlock { + UiBlock::Metadata(vec![ + ("Lease".to_string(), lease.lease_id.clone()), + ("Lane".to_string(), lease.lane_id.clone()), + ("Ref".to_string(), lease.ref_name.clone()), + ( + "Path".to_string(), + lease + .path + .clone() + .unwrap_or_else(|| "workspace".to_string()), + ), + ( + "Expires".to_string(), + format_timestamp(lease.expires_at, options), + ), + ]) +} + +fn lease_table(leases: &[LeaseRecord], options: &RenderOptions) -> UiTable { + UiTable::new( + vec![ + UiColumn::left("LANE", 0, 10), + UiColumn::left("PATH", 0, 12), + UiColumn::left("MODE", 1, 7), + UiColumn::left("EXPIRES", 0, 10), + UiColumn::left("LEASE ID", 2, 12), + ], + leases + .iter() + .map(|lease| { + vec![ + lease.lane_id.clone(), + lease + .path + .clone() + .unwrap_or_else(|| "workspace".to_string()), + lease.mode.clone(), + format_timestamp(lease.expires_at, options), + lease.lease_id.clone(), + ] + }) + .collect(), + ) } diff --git a/trail/src/cli/command/render/collaboration/merge.rs b/trail/src/cli/command/render/collaboration/merge.rs index 6422110..7bc9f01 100644 --- a/trail/src/cli/command/render/collaboration/merge.rs +++ b/trail/src/cli/command/render/collaboration/merge.rs @@ -1,311 +1,431 @@ -use super::super::render_json; +use super::super::*; use trail::model::*; use trail::Result; -pub(crate) fn render_merge(report: &MergeReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_merge( + report: &MergeReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { + let conflicted = !report.conflicts.is_empty(); + let mut document = TerminalDocument::new( if report.dry_run { - println!( - "Would merge {} into {} as {}", - report.source_ref, report.target_ref, report.operation.0 - ); + format!( + "Merge preview: {} into {}", + report.source_ref, report.target_ref + ) } else { - println!( - "Merged {} into {} as {}", - report.source_ref, report.target_ref, report.operation.0 - ); - } - for conflict in &report.conflicts { - println!(" conflict {conflict}"); - } - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } + format!("Merged {} into {}", report.source_ref, report.target_ref) + }, + if conflicted { + UiTone::Blocked + } else { + UiTone::Success + }, + ) + .context(format!("{} changed path(s)", report.changed_paths.len())); + if conflicted { + document = document.block(UiBlock::Checklist( + report + .conflicts + .iter() + .map(|conflict| UiCheck::new(UiCheckState::Blocked, "Conflict", conflict)) + .collect(), + )); + document = document.next( + "trail conflicts list", + "inspect and resolve the merge conflicts", + ); + } else if report.dry_run { + document = document.next( + format!( + "trail merge {} --into {}", + report.source_ref, report.target_ref + ), + "apply this reviewed merge", + ); + } + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); } - Ok(()) + if options.verbose { + document = document.block(UiBlock::Metadata(vec![ + ("Operation".to_string(), report.operation.0.clone()), + ("Root".to_string(), report.root_id.0.clone()), + ])); + } + render_document(&document, options) } -pub(crate) fn render_merge_queue_add( - report: &MergeQueueAddReport, +pub(crate) fn render_lane_merge_queue_add( + report: &LaneMergeQueueAddReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Queued {} into {} as {}", - report.entry.source_ref, report.entry.target_ref, report.entry.queue_id - ); - } - Ok(()) + render_document( + &TerminalDocument::new( + format!( + "Queued {} into {}", + report.entry.lane, report.entry.target_ref + ), + UiTone::Success, + ) + .block(queue_entry_block(&report.entry)) + .next( + "trail lane merge-queue explain ", + "check readiness before processing this merge", + ), + options, + ) } -pub(crate) fn render_merge_queue_list( - entries: &[MergeQueueEntry], +pub(crate) fn render_lane_merge_queue_list( + entries: &[LaneMergeQueueEntry], json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(&entries); } - if !quiet { - for entry in entries { - println!( - "{} {} priority={} {} -> {}", - entry.queue_id, entry.status, entry.priority, entry.source_ref, entry.target_ref - ); - } + if entries.is_empty() { + return render_document( + &TerminalDocument::new("Merge queue is empty", UiTone::Neutral), + options, + ); } - Ok(()) + render_document( + &TerminalDocument::new( + format!("{} queued merge(s)", entries.len()), + UiTone::Neutral, + ) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("STATUS", 0, 8), + UiColumn::right("PRIORITY", 1, 8), + UiColumn::left("SOURCE", 0, 12), + UiColumn::left("TARGET", 0, 12), + UiColumn::left("ID", 2, 10), + ], + entries + .iter() + .map(|entry| { + vec![ + entry.status.clone(), + entry.priority.to_string(), + entry.lane.clone(), + entry.target_ref.clone(), + entry.queue_id.clone(), + ] + }) + .collect(), + ))) + .next( + "trail lane merge-queue explain ", + "inspect blockers and the dry-run before running the queue", + ), + options, + ) } -pub(crate) fn render_merge_queue_run( - report: &MergeQueueRunReport, +pub(crate) fn render_lane_merge_queue_run( + report: &LaneMergeQueueRunReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { + let tone = if report.stopped_on_conflict { + UiTone::Blocked + } else if report.stopped_on_failure { + UiTone::Failure + } else { + UiTone::Success + }; + let mut document = TerminalDocument::new( if report.processed.is_empty() { - println!("Merge queue is empty"); - } - for item in &report.processed { - match (&item.operation, &item.error) { - (Some(operation), _) => println!( - "{} {} as {} {} -> {}", - item.queue_id, item.status, operation.0, item.source_ref, item.target_ref - ), - (None, Some(error)) => println!( - "{} {} {} -> {}: {}", - item.queue_id, item.status, item.source_ref, item.target_ref, error - ), - (None, None) => println!( - "{} {} {} -> {}", - item.queue_id, item.status, item.source_ref, item.target_ref - ), - } - } - if report.stopped_on_conflict { - println!("Paused on conflict"); - } else if report.stopped_on_failure { - println!("Paused on failure"); - } + "Merge queue is empty".to_string() + } else { + format!("Processed {} queued merge(s)", report.processed.len()) + }, + tone, + ); + if !report.processed.is_empty() { + document = document.block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("STATUS", 0, 8), + UiColumn::left("SOURCE", 0, 12), + UiColumn::left("TARGET", 0, 12), + UiColumn::left("RESULT", 1, 16), + ], + report + .processed + .iter() + .map(|item| { + vec![ + item.status.clone(), + item.lane.clone(), + item.target_ref.clone(), + item.error + .clone() + .or_else(|| item.operation.as_ref().map(|id| format!("operation {id}"))) + .unwrap_or_else(|| "—".to_string()), + ] + }) + .collect(), + ))); } - Ok(()) + if report.stopped_on_conflict { + document = document.next( + "trail conflicts list", + "resolve the conflict before continuing the queue", + ); + } else if report.stopped_on_failure { + document = document.next( + "trail lane merge-queue list", + "inspect the failed queue item before retrying", + ); + } + render_document(&document, options) } -pub(crate) fn render_merge_queue_explain( - report: &MergeQueueExplainReport, +pub(crate) fn render_lane_merge_queue_explain( + report: &LaneMergeQueueExplainReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "{} {} priority={} {} -> {}", + let blocked = !report.blockers.is_empty() || report.error.is_some(); + let mut document = TerminalDocument::new( + format!( + "Queue item {} is {}", report.entry.queue_id, - report.entry.status, - report.entry.priority, - report.entry.source_ref, - report.entry.target_ref - ); - if report.blockers.is_empty() { - println!("Ready: true"); + if blocked { "blocked" } else { "ready" } + ), + if blocked { + UiTone::Blocked } else { - println!("Ready: false"); - for blocker in &report.blockers { - println!(" blocker {}: {}", blocker.code, blocker.message); - } - } - for warning in &report.warnings { - println!(" warning {}: {}", warning.code, warning.message); - } - if let Some(dry_run) = &report.dry_run { - if !dry_run.conflicts.is_empty() { - for conflict in &dry_run.conflicts { - println!(" conflict {conflict}"); - } - } else { - println!("Dry-run changed paths: {}", dry_run.changed_paths.len()); - } - } - if let Some(error) = &report.error { - println!("Preflight error: {error}"); - } - if !report.next_steps.is_empty() { - println!("Next steps:"); - for step in &report.next_steps { - println!(" - {step}"); - } + UiTone::Success + }, + ) + .block(queue_entry_block(&report.entry)); + let mut checks: Vec<_> = report + .blockers + .iter() + .map(|issue| UiCheck::new(UiCheckState::Blocked, &issue.code, &issue.message)) + .collect(); + checks.extend( + report + .warnings + .iter() + .map(|issue| UiCheck::new(UiCheckState::Warn, &issue.code, &issue.message)), + ); + if let Some(error) = &report.error { + checks.push(UiCheck::new(UiCheckState::Fail, "Preflight", error)); + } + if !checks.is_empty() { + document = document.block(UiBlock::Checklist(checks)); + } + if let Some(dry_run) = &report.dry_run { + if !dry_run.changed_paths.is_empty() { + document = document.block(UiBlock::section( + "Dry-run changes:", + vec![UiBlock::Changes(change_list(&dry_run.changed_paths))], + )); } } - Ok(()) + document = append_next_steps(document, &report.next_steps); + render_document(&document, options) } -pub(crate) fn render_merge_queue_remove( - report: &MergeQueueRemoveReport, +pub(crate) fn render_lane_merge_queue_remove( + report: &LaneMergeQueueRemoveReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Cancelled {}", report.entry.queue_id); - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Removed queued merge {}", report.entry.queue_id), + UiTone::Success, + ) + .block(queue_entry_block(&report.entry)), + options, + ) } pub(crate) fn render_conflicts( entries: &[ConflictSetSummary], json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(&entries); } - if !quiet { - if entries.is_empty() { - println!("No conflicts"); - } - for entry in entries { - println!( - "{} {} {} -> {}", - entry.conflict_set_id, - entry.status, - entry.source_ref.as_deref().unwrap_or("-"), - entry.target_ref.as_deref().unwrap_or("-") - ); - for detail in &entry.details { - println!(" {detail}"); - } - } + if entries.is_empty() { + return render_document( + &TerminalDocument::new("No unresolved conflicts", UiTone::Success), + options, + ); } - Ok(()) + render_document( + &TerminalDocument::new( + format!("{} unresolved conflict set(s)", entries.len()), + UiTone::Blocked, + ) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("STATUS", 0, 8), + UiColumn::left("SOURCE", 1, 12), + UiColumn::left("TARGET", 1, 12), + UiColumn::left("CONFLICT ID", 0, 14), + ], + entries + .iter() + .map(|entry| { + vec![ + entry.status.clone(), + entry.source_ref.clone().unwrap_or_else(|| "—".to_string()), + entry.target_ref.clone().unwrap_or_else(|| "—".to_string()), + entry.conflict_set_id.clone(), + ] + }) + .collect(), + ))) + .next( + "trail conflicts show ", + "review evidence and the proposed resolutions", + ), + options, + ) } -pub(crate) fn render_conflict(entry: &ConflictSetSummary, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_conflict( + entry: &ConflictSetSummary, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(entry); } - if !quiet { - println!("Conflict: {}", entry.conflict_set_id); - println!("Status: {}", entry.status); - if let Some(merge_id) = &entry.merge_id { - println!("Merge: {merge_id}"); - } - if let Some(source) = &entry.source_ref { - println!("Source: {source}"); - } - if let Some(target) = &entry.target_ref { - println!("Target: {target}"); - } - for detail in &entry.details { - println!(" {detail}"); - } - if let Some(explanation) = &entry.explanation { - println!(); - println!("Explanation:"); - println!( - " Merge: {} target={} source={} base={}", - explanation.merge.merge_id, - explanation.merge.target_change.0, - explanation.merge.source_change.0, - explanation.merge.base_change.0 - ); - for path in &explanation.paths { - println!(" Path: {}", path.path); - println!(" Class: {}", path.conflict_class); - println!(" {}", path.summary); - println!(" Reason: {}", path.reason); - if let Some(target) = &path.target { - println!(" Target: {}", side_summary(target)); - } - if let Some(source) = &path.source { - println!(" Source: {}", side_summary(source)); - } - for line in &path.lines { - println!(" Line {}: {}", line.line_id, line.reason); - if let Some(base) = &line.base { - println!(" base: {}", printable_line(base)); - } - if let Some(target) = &line.target { - println!(" target: {}", printable_line(target)); - } - if let Some(source) = &line.source { - println!(" source: {}", printable_line(source)); - } - } - println!( - " Recommend: {} ({}) - {}", - path.recommendation.resolution, - path.recommendation.confidence, - path.recommendation.reason - ); - } - if !explanation.next_steps.is_empty() { - println!(); - println!("Next steps:"); - for step in &explanation.next_steps { - println!(" - {step}"); - } - } - } + let mut document = TerminalDocument::new( + format!("Conflict {}", entry.conflict_set_id), + UiTone::Blocked, + ) + .block(UiBlock::Metadata(vec![ + ("Status".to_string(), entry.status.clone()), + ( + "Source".to_string(), + entry.source_ref.clone().unwrap_or_else(|| "—".to_string()), + ), + ( + "Target".to_string(), + entry.target_ref.clone().unwrap_or_else(|| "—".to_string()), + ), + ])); + if !entry.details.is_empty() { + document = document.block(UiBlock::section( + "Details:", + vec![UiBlock::Lines( + entry + .details + .iter() + .cloned() + .map(|detail| (detail, UiTone::Attention)) + .collect(), + )], + )); + } + if let Some(explanation) = &entry.explanation { + document = document.block(UiBlock::section( + "Paths:", + vec![UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("PATH", 0, 14), + UiColumn::left("CLASS", 1, 8), + UiColumn::left("RECOMMENDATION", 0, 14), + UiColumn::left("WHY", 0, 18), + ], + explanation + .paths + .iter() + .map(|path| { + vec![ + path.path.clone(), + path.conflict_class.clone(), + format!( + "{} ({})", + path.recommendation.resolution, path.recommendation.confidence + ), + path.recommendation.reason.clone(), + ] + }) + .collect(), + ))], + )); + document = append_next_steps(document, &explanation.next_steps); } - Ok(()) + document = document.next( + format!( + "trail conflicts resolve {} --take ", + entry.conflict_set_id + ), + "apply a reviewed resolution, or use --manual for edited content", + ); + render_document(&document.pager_eligible(), options) } pub(crate) fn render_conflict_resolve( report: &ConflictResolveReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - if report.resolution == "manual" { - println!( - "Resolved {} manually as {}", - report.conflict_set_id, report.operation.0 - ); - } else { - println!( - "Resolved {} by taking {} as {}", - report.conflict_set_id, report.resolution, report.operation.0 - ); - } - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } + let mut document = TerminalDocument::new( + format!("Resolved conflict {}", report.conflict_set_id), + UiTone::Success, + ) + .context(format!("using {}", report.resolution)); + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); } - Ok(()) + document = document.next( + "trail conflicts list", + "confirm no unresolved conflicts remain", + ); + render_document(&document, options) } -fn side_summary(side: &ConflictSideProvenance) -> String { - let mut value = format!( - "{} {} by {} on {}", - side.kind, side.change_id.0, side.actor_id, side.branch - ); - if let Some(session) = &side.session_id { - value.push_str(&format!(" session={session}")); - } - if let Some(message) = &side.message { - value.push_str(&format!(" - {message}")); - } - value +fn queue_entry_block(entry: &LaneMergeQueueEntry) -> UiBlock { + UiBlock::Metadata(vec![ + ("ID".to_string(), entry.queue_id.clone()), + ("Status".to_string(), entry.status.clone()), + ("Priority".to_string(), entry.priority.to_string()), + ("Lane".to_string(), entry.lane.clone()), + ("Target".to_string(), entry.target_ref.clone()), + ]) } -fn printable_line(value: &str) -> String { - value.replace('\r', "\\r").replace('\n', "\\n") +fn append_next_steps(mut document: TerminalDocument, steps: &[String]) -> TerminalDocument { + for (index, step) in steps.iter().enumerate() { + if index == 0 { + document = document.next(step, "recommended next step"); + } else { + document = document.more(step, "alternative next step"); + } + } + document } diff --git a/trail/src/cli/command/render/config.rs b/trail/src/cli/command/render/config.rs index c57ad5b..7b337ce 100644 --- a/trail/src/cli/command/render/config.rs +++ b/trail/src/cli/command/render/config.rs @@ -1,43 +1,101 @@ -use super::render_json; +use super::*; use trail::model::*; use trail::Result; -pub(crate) fn render_config_list(entries: &[ConfigEntry], json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_config_list( + entries: &[ConfigEntry], + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(&entries); } - if !quiet { - for entry in entries { - let read_only = if entry.read_only { " (read-only)" } else { "" }; - println!( - "{} = {} [{}]{}", - entry.key, entry.value, entry.value_type, read_only - ); - } + if entries.is_empty() { + return render_document( + &TerminalDocument::new("No configuration values", UiTone::Neutral), + options, + ); } - Ok(()) + let rows = entries + .iter() + .map(|entry| { + vec![ + entry.key.clone(), + entry.value.clone(), + entry.value_type.clone(), + if entry.read_only { + "read-only" + } else { + "editable" + } + .to_string(), + ] + }) + .collect(); + render_document( + &TerminalDocument::new( + format!("{} configuration value(s)", entries.len()), + UiTone::Neutral, + ) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("KEY", 0, 12), + UiColumn::left("VALUE", 0, 16), + UiColumn::left("TYPE", 2, 8), + UiColumn::left("ACCESS", 2, 8), + ], + rows, + ))), + options, + ) } -pub(crate) fn render_config_entry(entry: &ConfigEntry, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_config_entry( + entry: &ConfigEntry, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(entry); } - if !quiet { - println!("{}", entry.value); - } - Ok(()) + render_document( + &TerminalDocument::new(format!("Configuration {}", entry.key), UiTone::Neutral).block( + UiBlock::Metadata(vec![ + ("Value".to_string(), entry.value.clone()), + ("Type".to_string(), entry.value_type.clone()), + ( + "Access".to_string(), + if entry.read_only { + "read-only" + } else { + "editable" + } + .to_string(), + ), + ]), + ), + options, + ) } -pub(crate) fn render_config_set(report: &ConfigSetReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_config_set( + report: &ConfigSetReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "{}: {} -> {}", - report.key, report.old_value, report.new_value - ); - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Updated configuration {}", report.key), + UiTone::Success, + ) + .block(UiBlock::Metadata(vec![ + ("Previous".to_string(), report.old_value.clone()), + ("Current".to_string(), report.new_value.clone()), + ])), + options, + ) } diff --git a/trail/src/cli/command/render/diff.rs b/trail/src/cli/command/render/diff.rs index 91dafe8..462db1c 100644 --- a/trail/src/cli/command/render/diff.rs +++ b/trail/src/cli/command/render/diff.rs @@ -1,13 +1,19 @@ -use std::io::IsTerminal; use std::time::Duration; use super::super::DiffArgs; -use super::render_json; +use super::*; use trail::model::*; use trail::{Error, Result, Trail}; pub(crate) fn diff_from_args(db: &mut Trail, args: &DiffArgs) -> Result { + validate_diff_view( + args.patch, + args.stat, + args.show_line_ids, + args.name_only, + args.name_status, + )?; let forms = usize::from(args.range.is_some()) + usize::from(args.root.is_some()) + usize::from(args.dirty); @@ -25,6 +31,27 @@ pub(crate) fn diff_from_args(db: &mut Trail, args: &DiffArgs) -> Result Result<()> { + if name_only && name_status { + return Err(Error::InvalidInput( + "diff accepts only one of --name-only or --name-status".to_string(), + )); + } + if (name_only || name_status) && (patch || stat || show_line_ids) { + return Err(Error::InvalidInput( + "--name-only and --name-status cannot be combined with --patch, --stat, or --show-line-ids" + .to_string(), + )); + } + Ok(()) +} + pub(crate) fn watch_interval(interval_secs: u64, debounce_ms: Option) -> Result { if let Some(ms) = debounce_ms { if ms == 0 { @@ -45,282 +72,318 @@ pub(crate) fn watch_interval(interval_secs: u64, debounce_ms: Option) -> Re pub(crate) fn render_diff( summary: &DiffSummary, json: bool, - quiet: bool, + options: &RenderOptions, + patch_requested: bool, stat: bool, - color: bool, + name_only: bool, + name_status: bool, ) -> Result<()> { - render_diff_with_title(summary, json, quiet, stat, color, None) + render_diff_with_title( + summary, + json, + options, + patch_requested, + stat, + name_only, + name_status, + None, + ) } pub(crate) fn render_diff_with_title( summary: &DiffSummary, json: bool, - quiet: bool, + options: &RenderOptions, + patch_requested: bool, stat: bool, - color: bool, + name_only: bool, + name_status: bool, title: Option<&str>, ) -> Result<()> { if json { return render_json(summary); } - if !quiet { - let total_additions: u64 = summary.files.iter().map(|file| file.additions).sum(); - let total_deletions: u64 = summary.files.iter().map(|file| file.deletions).sum(); - println!("{}", title.unwrap_or("Diff")); - println!(" from: {}", summary.from); - println!(" to: {}", summary.to); - if summary.files.is_empty() { - println!(" No file changes"); - return Ok(()); - } - println!( - " {} file(s) changed, +{} -{}", - summary.files.len(), - total_additions, - total_deletions + if summary.files.is_empty() { + return render_document( + &TerminalDocument::new( + format!("No changes between {} and {}", summary.from, summary.to), + UiTone::Success, + ), + options, ); - println!(); - + } + if name_only { + return render_document( + &TerminalDocument::empty().block(UiBlock::Lines( + summary + .files + .iter() + .map(|file| (file.path.clone(), UiTone::Neutral)) + .collect(), + )), + options, + ); + } + if name_status { + return render_document( + &TerminalDocument::empty().block(UiBlock::Lines( + summary + .files + .iter() + .map(|file| { + ( + format!("{}\t{}", file_change_marker(&file.kind), file.path), + UiTone::Neutral, + ) + }) + .collect(), + )), + options, + ); + } + let lead = title + .map(str::to_string) + .unwrap_or_else(|| format!("Changes from {} to {}", summary.from, summary.to)); + let mut document = TerminalDocument::new(lead, UiTone::Neutral) + .block(UiBlock::Changes(diff_change_list(&summary.files))); + if patch_requested && !stat { for file in &summary.files { - println!(" {}", format_file_diff_line(file)); - print_line_change_summary(file); if let Some(patch) = &file.patch { - if !patch.starts_with('\n') { - println!(); - } - print_patch(patch, color && std::io::stdout().is_terminal()); - if !patch.ends_with('\n') { - println!(); - } + document = document.block(UiBlock::Patch { + title: display_path(file, options), + text: patch.clone(), + }); + } else { + document = document.block(UiBlock::Notice(format!( + "{}: no textual patch is available (binary, opaque, or missing source content).", + display_path(file, options) + ))); + } + if options.verbose && !file.line_changes.is_empty() { + document = document.block(UiBlock::Paragraph { + text: line_change_summary(file), + tone: UiTone::Muted, + }); } } - if stat { - println!( - "{} files changed, {} additions, {} deletions", - summary.files.len(), - total_additions, - total_deletions - ); - } - } - Ok(()) -} - -fn print_patch(patch: &str, color: bool) { - if !color { - print!("{patch}"); - return; - } - for line in patch.split_inclusive('\n') { - print!("{}", color_patch_line(line)); } -} - -fn color_patch_line(line: &str) -> String { - let color = if line.starts_with("@@") { - Some("\x1b[36m") - } else if line.starts_with("diff --git") - || line.starts_with("diff --trail") - || line.starts_with("index ") - { - Some("\x1b[1m") - } else if line.starts_with("--- ") || line.starts_with("+++ ") { - Some("\x1b[1m") - } else if line.starts_with('+') { - Some("\x1b[32m") - } else if line.starts_with('-') { - Some("\x1b[31m") - } else { - None - }; - match color { - Some(color) => format!("{color}{line}\x1b[0m"), - None => line.to_string(), + if patch_requested && !stat && summary.files.iter().any(|file| file.patch.is_some()) { + document = document.pager_eligible(); } + render_document(&document, options) } -fn format_file_diff_line(file: &FileDiffSummary) -> String { - let path = file - .old_path - .as_ref() - .map(|old_path| format!("{old_path} -> {}", file.path)) - .unwrap_or_else(|| file.path.clone()); - format!( - "{} {:<11} {} {}", - file_change_marker(&file.kind), - file_change_label(&file.kind), - path, - format_change_stat(file.additions, file.deletions) +fn diff_change_list(files: &[FileDiffSummary]) -> UiChangeList { + UiChangeList::new( + files + .iter() + .map(|file| UiChange { + marker: file_change_marker(&file.kind), + path: file.path.clone(), + old_path: file.old_path.clone(), + additions: file.additions, + deletions: file.deletions, + detail: if file.patch.is_none() { + Some("text patch unavailable".to_string()) + } else { + None + }, + }) + .collect(), ) } -fn file_change_marker(kind: &FileChangeKind) -> &'static str { - match kind { - FileChangeKind::Added => "A", - FileChangeKind::Modified => "M", - FileChangeKind::Deleted => "D", - FileChangeKind::Renamed => "R", - FileChangeKind::TypeChanged => "T", - } -} - -fn file_change_label(kind: &FileChangeKind) -> &'static str { - match kind { - FileChangeKind::Added => "added", - FileChangeKind::Modified => "modified", - FileChangeKind::Deleted => "deleted", - FileChangeKind::Renamed => "renamed", - FileChangeKind::TypeChanged => "type-changed", - } -} - -fn format_change_stat(additions: u64, deletions: u64) -> String { - let bar = format_change_bar(additions, deletions); - if bar.is_empty() { - format!("(+{additions} -{deletions})") - } else { - format!("(+{additions} -{deletions}) {bar}") - } -} - -fn format_change_bar(additions: u64, deletions: u64) -> String { - let total = additions + deletions; - if total == 0 { - return String::new(); - } - let width = total.min(24) as usize; - let mut plus = ((additions * width as u64) + total - 1) / total; - if additions == 0 { - plus = 0; +fn display_path(file: &FileDiffSummary, options: &RenderOptions) -> String { + match &file.old_path { + Some(old_path) => format!("{} {} {}", old_path, options.unicode("→", "->"), file.path), + None => file.path.clone(), } - let minus = width.saturating_sub(plus as usize); - format!("{}{}", "+".repeat(plus as usize), "-".repeat(minus)) } -fn print_line_change_summary(file: &FileDiffSummary) { - if file.line_changes.is_empty() { - return; - } +fn line_change_summary(file: &FileDiffSummary) -> String { let mut added = 0_u64; let mut modified = 0_u64; let mut deleted = 0_u64; let mut moved = 0_u64; for line in &file.line_changes { - match &line.kind { + match line.kind { LineChangeKind::Added => added += 1, LineChangeKind::Modified => modified += 1, LineChangeKind::Deleted => deleted += 1, LineChangeKind::Moved => moved += 1, } } - println!(" lines: +{added} ~{modified} -{deleted} moved {moved}"); - for line in file.line_changes.iter().take(8) { - println!( - " {} {} old={} new={}", - line_change_marker(&line.kind), - format_line_id(&line.line_id), - format_optional_line_number(line.old_line_number), - format_optional_line_number(line.new_line_number) - ); - } - if file.line_changes.len() > 8 { - println!( - " ... {} more line changes", - file.line_changes.len() - 8 - ); - } -} - -fn line_change_marker(kind: &LineChangeKind) -> &'static str { - match kind { - LineChangeKind::Added => "+", - LineChangeKind::Modified => "~", - LineChangeKind::Deleted => "-", - LineChangeKind::Moved => ">", - } + format!("Stable line changes: +{added} ~{modified} -{deleted} moved {moved}") } -fn format_line_id(line_id: &trail::LineId) -> String { - format!("{}:{}", line_id.origin_change.0, line_id.local_seq) -} - -fn format_optional_line_number(line: Option) -> String { - line.map(|line| line.to_string()) - .unwrap_or_else(|| "-".to_string()) -} - -pub(crate) fn render_history(result: &HistoryResult, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_history( + result: &HistoryResult, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(result); } - if !quiet { - println!("{}", result.selector); - for entry in &result.file_history { - let old_path = entry - .old_path - .as_ref() - .map(|path| format!(" from {path}")) - .unwrap_or_default(); - println!( - "{} {:?} {}{}", - entry.change_id.0, entry.kind, entry.path, old_path - ); - } - for entry in &result.line_history { - let line = entry - .line_number - .map(|line| format!(":{line}")) - .unwrap_or_default(); - println!( - "{} {:?} {}{}", - entry.change_id.0, entry.kind, entry.path, line - ); - } + if result.file_history.is_empty() && result.line_history.is_empty() { + return render_document( + &TerminalDocument::new( + format!("No history found for {}", result.selector), + UiTone::Neutral, + ), + options, + ); } - Ok(()) + let mut rows = Vec::new(); + rows.extend(result.file_history.iter().map(|entry| { + vec![ + entry.change_id.0.clone(), + format_timestamp(entry.created_at, options), + file_change_label(&entry.kind).to_string(), + entry.path.clone(), + ] + })); + rows.extend(result.line_history.iter().map(|entry| { + let location = entry + .line_number + .map(|line| format!("{}:{line}", entry.path)) + .unwrap_or_else(|| entry.path.clone()); + vec![ + entry.change_id.0.clone(), + format_timestamp(entry.created_at, options), + line_change_label(&entry.kind).to_string(), + location, + ] + })); + let document = + TerminalDocument::new(format!("History for {}", result.selector), UiTone::Neutral) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("OPERATION", 0, 16), + UiColumn::left("WHEN", 2, 20), + UiColumn::left("CHANGE", 1, 9), + UiColumn::left("PATH", 0, 12), + ], + rows, + ))) + .pager_eligible(); + render_document(&document, options) } -pub(crate) fn render_code_from(result: &CodeFromResult, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_code_from( + result: &CodeFromResult, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(result); } - if !quiet { - println!("{}", result.selector); - if result.operations.is_empty() { - println!("No operations found"); + if result.operations.is_empty() { + return render_document( + &TerminalDocument::new( + format!("No source operations found for {}", result.selector), + UiTone::Neutral, + ), + options, + ); + } + let mut blocks = Vec::new(); + for operation in &result.operations { + let paths = operation + .changed_paths + .iter() + .map(|path| UiChange { + marker: file_change_marker(&path.kind), + path: path.path.clone(), + old_path: path.old_path.clone(), + additions: path.additions, + deletions: path.deletions, + detail: None, + }) + .collect(); + let title = format!( + "{} · {} · {}", + operation.change_id.0, + operation_kind_label(&operation.kind), + operation.branch + ); + let mut operation_blocks = Vec::new(); + if let Some(message) = &operation.message { + operation_blocks.push(UiBlock::paragraph(message)); } - for operation in &result.operations { - let message = operation.message.as_deref().unwrap_or(""); - println!( - "{} {:?} {} {}", - operation.change_id.0, operation.kind, operation.branch, message - ); - for path in &operation.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } + if !operation.changed_paths.is_empty() { + operation_blocks.push(UiBlock::Changes(UiChangeList::new(paths))); } + blocks.push(UiBlock::section(title, operation_blocks)); } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Source operations for {}", result.selector), + UiTone::Neutral, + ) + .block(UiBlock::section("Operations:", blocks)) + .pager_eligible(), + options, + ) } -pub(crate) fn render_why(result: &WhyResult, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_why(result: &WhyResult, json: bool, options: &RenderOptions) -> Result<()> { if json { return render_json(result); } - if !quiet { - println!( - "{}:{} {}", - result.path, result.line_number, result.current_text - ); - println!( - "Line ID: {}:{}", - result.line_id.origin_change.0, result.line_id.local_seq - ); - println!("Introduced by: {}", result.introduced_by.0); - println!("Last content change: {}", result.last_content_change.0); - for item in &result.history { - println!(" {:?} {} {}", item.kind, item.change_id.0, item.path); - } + let mut document = TerminalDocument::new( + format!("Why {}:{}", result.path, result.line_number), + UiTone::Neutral, + ) + .block(UiBlock::paragraph(&result.current_text)) + .block(UiBlock::Metadata(vec![ + ("Line".to_string(), result.line_id.alias()), + ("Introduced by".to_string(), result.introduced_by.0.clone()), + ( + "Last content change".to_string(), + result.last_content_change.0.clone(), + ), + ])); + if !result.history.is_empty() { + let rows = result + .history + .iter() + .map(|entry| { + vec![ + entry.change_id.0.clone(), + line_change_label(&entry.kind).to_string(), + entry.path.clone(), + entry + .line_number + .map(|line| line.to_string()) + .unwrap_or_else(|| "—".to_string()), + ] + }) + .collect(); + document = document.block(UiBlock::section( + "Line history:", + vec![UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("OPERATION", 0, 16), + UiColumn::left("CHANGE", 1, 9), + UiColumn::left("PATH", 0, 12), + UiColumn::right("LINE", 2, 4), + ], + rows, + ))], + )); + } + render_document(&document, options) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn diff_projections_are_mutually_exclusive() { + assert!(validate_diff_view(false, false, false, true, false).is_ok()); + assert!(validate_diff_view(false, false, false, false, true).is_ok()); + assert!(validate_diff_view(false, false, false, true, true).is_err()); + assert!(validate_diff_view(true, false, false, true, false).is_err()); } - Ok(()) } diff --git a/trail/src/cli/command/render/git.rs b/trail/src/cli/command/render/git.rs index 930b314..6b1e30b 100644 --- a/trail/src/cli/command/render/git.rs +++ b/trail/src/cli/command/render/git.rs @@ -1,4 +1,4 @@ -use super::render_json; +use super::*; use trail::model::*; use trail::Result; @@ -6,76 +6,102 @@ use trail::Result; pub(crate) fn render_git_import_update( report: &GitImportReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - match &report.operation { - Some(change) => { - println!("Imported Git update {}", change.0); - println!( - "Imported: {} files ({} text, {} opaque, {} binary)", - report.imported.files, - report.imported.text, - report.imported.opaque, - report.imported.binary - ); - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } - } - None => println!("No Git-tracked changes to import"), - } + let Some(operation) = &report.operation else { + return render_document( + &TerminalDocument::new("No Git-tracked changes to import", UiTone::Neutral), + options, + ); + }; + let mut document = TerminalDocument::new( + format!("Imported Git update {}", operation.0), + UiTone::Success, + ) + .block(UiBlock::Metadata(vec![ + ("Files".to_string(), report.imported.files.to_string()), + ("Text".to_string(), report.imported.text.to_string()), + ("Opaque".to_string(), report.imported.opaque.to_string()), + ("Binary".to_string(), report.imported.binary.to_string()), + ])); + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); } - Ok(()) + render_document(&document, options) } -pub(crate) fn render_git_export(report: &GitExportReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_git_export( + report: &GitExportReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Created Git commit: {}", report.commit); - println!("Range: {}", report.range); - println!("Trail operation: {}", report.operation.0); - println!("Root: {}", report.root_id.0); - if let Some(parent) = &report.parent { - println!("Parent: {parent}"); - } - if let Some(mapping) = &report.mapping { - println!("Mapping: {}", mapping.mapping_id); - } + let mut metadata = vec![ + ("Commit".to_string(), report.commit.clone()), + ("Range".to_string(), report.range.clone()), + ("Trail operation".to_string(), report.operation.0.clone()), + ("Root".to_string(), report.root_id.0.clone()), + ]; + if let Some(parent) = &report.parent { + metadata.push(("Parent".to_string(), parent.clone())); + } + if let Some(mapping) = &report.mapping { + metadata.push(("Mapping".to_string(), mapping.mapping_id.clone())); } - Ok(()) + render_document( + &TerminalDocument::new("Created Git commit", UiTone::Success) + .block(UiBlock::Metadata(metadata)), + options, + ) } -pub(crate) fn render_git_mappings(entries: &[GitMapping], json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_git_mappings( + entries: &[GitMapping], + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(&entries); } - if !quiet { - if entries.is_empty() { - println!("No Git mappings"); - } - for entry in entries { - let git_head = entry - .git_head - .as_deref() - .map(|head| head.get(..12).unwrap_or(head)) - .unwrap_or("unborn"); - let dirty = if entry.git_dirty { " dirty" } else { "" }; - println!( - "{} {}{} {} {} {}", - entry.direction, - git_head, - dirty, - entry.branch, - entry.crab_change.0, - entry.crab_root.0 - ); - } + if entries.is_empty() { + return render_document( + &TerminalDocument::new("No Git mappings", UiTone::Neutral), + options, + ); } - Ok(()) + render_document( + &TerminalDocument::new(format!("{} Git mapping(s)", entries.len()), UiTone::Neutral).block( + UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("DIRECTION", 0, 9), + UiColumn::left("HEAD", 1, 12), + UiColumn::left("BRANCH", 0, 12), + UiColumn::left("STATE", 1, 7), + UiColumn::left("TRAIL CHANGE", 2, 12), + ], + entries + .iter() + .map(|entry| { + vec![ + entry.direction.clone(), + entry + .git_head + .clone() + .map(|head| head.chars().take(12).collect()) + .unwrap_or_else(|| "unborn".to_string()), + entry.branch.clone(), + if entry.git_dirty { "dirty" } else { "clean" }.to_string(), + entry.crab_change.0.clone(), + ] + }) + .collect(), + )), + ), + options, + ) } diff --git a/trail/src/cli/command/render/guardrails.rs b/trail/src/cli/command/render/guardrails.rs index 1db47ad..c229a6d 100644 --- a/trail/src/cli/command/render/guardrails.rs +++ b/trail/src/cli/command/render/guardrails.rs @@ -1,4 +1,4 @@ -use super::render_json; +use super::*; use trail::model::*; use trail::Result; @@ -6,45 +6,89 @@ use trail::Result; pub(crate) fn render_guardrail_check( report: &GuardrailCheckReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Guardrail decision: {}", report.decision); - println!("Action: {}", report.action); - if let Some(lane) = &report.lane { - println!("Lane: {}", lane.record.name); - } - if !report.reasons.is_empty() { - println!("Reasons:"); - for reason in &report.reasons { - println!( - " {} [{}]: {}", - reason.code, reason.severity, reason.message - ); - } - } - if !report.path_checks.is_empty() { - println!("Paths:"); - for check in &report.path_checks { - let status = if check.ignored { "ignored" } else { "allowed" }; - match &check.source { - Some(source) => println!(" {}: {} ({})", check.path, status, source), - None => println!(" {}: {}", check.path, status), - } - } - } - if let Some(approval) = &report.approval_request { - println!("Approval suggested: {}", approval.summary); - } - if !report.satisfied_approvals.is_empty() { - println!("Satisfied approvals:"); - for approval in &report.satisfied_approvals { - println!(" {} {}", approval.approval_id, approval.action); - } - } + let state = check_state_from_status(&report.decision); + let mut document = TerminalDocument::new( + format!("Guardrail decision: {}", report.decision), + check_tone(state), + ) + .context(format!("Action: {}", report.action)); + if let Some(lane) = &report.lane { + document = document.block(UiBlock::Metadata(vec![( + "Lane".to_string(), + lane.record.name.clone(), + )])); + } + if !report.reasons.is_empty() { + document = document.block(UiBlock::section( + "Checks:", + vec![UiBlock::Checklist( + report + .reasons + .iter() + .map(|reason| check_for_status(&reason.severity, &reason.code, &reason.message)) + .collect(), + )], + )); + } + if !report.path_checks.is_empty() { + document = document.block(UiBlock::section( + "Paths:", + vec![UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("PATH", 0, 12), + UiColumn::left("DECISION", 1, 8), + UiColumn::left("RULE", 2, 12), + ], + report + .path_checks + .iter() + .map(|check| { + vec![ + check.path.clone(), + if check.ignored { "ignored" } else { "allowed" }.to_string(), + check.source.clone().unwrap_or_else(|| "—".to_string()), + ] + }) + .collect(), + ))], + )); + } + if let Some(approval) = &report.approval_request { + document = document.block(UiBlock::Notice(format!( + "Approval required: {}", + approval.summary + ))); + } + if !report.satisfied_approvals.is_empty() { + document = document.block(UiBlock::section( + "Satisfied approvals:", + vec![UiBlock::Lines( + report + .satisfied_approvals + .iter() + .map(|approval| { + ( + format!("{} {}", approval.approval_id, approval.action), + UiTone::Success, + ) + }) + .collect(), + )], + )); + } + render_document(&document, options) +} + +fn check_tone(state: UiCheckState) -> UiTone { + match state { + UiCheckState::Pass => UiTone::Success, + UiCheckState::Warn | UiCheckState::Pending => UiTone::Attention, + UiCheckState::Blocked | UiCheckState::Fail => UiTone::Blocked, + UiCheckState::Skip => UiTone::Neutral, } - Ok(()) } diff --git a/trail/src/cli/command/render/ignore.rs b/trail/src/cli/command/render/ignore.rs index 1258cc0..248a43e 100644 --- a/trail/src/cli/command/render/ignore.rs +++ b/trail/src/cli/command/render/ignore.rs @@ -1,71 +1,99 @@ -use super::render_json; +use super::*; use trail::model::*; use trail::Result; -pub(crate) fn render_ignore_list(report: &IgnoreListReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_ignore_list( + report: &IgnoreListReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Ignore file: {}", report.path); - if report.patterns.is_empty() { - println!("No ignore patterns"); - } else { - for pattern in &report.patterns { - println!("{}: {}", pattern.line, pattern.pattern); - } - } + let mut document = + TerminalDocument::new("Trail ignore rules", UiTone::Neutral).context(report.path.clone()); + if report.patterns.is_empty() { + document = document.block(UiBlock::paragraph("No ignore patterns are configured.")); + } else { + document = document.block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::right("LINE", 0, 4), + UiColumn::left("PATTERN", 0, 16), + ], + report + .patterns + .iter() + .map(|pattern| vec![pattern.line.to_string(), pattern.pattern.clone()]) + .collect(), + ))); } - Ok(()) + render_document(&document, options) } -pub(crate) fn render_ignore_add(report: &IgnoreAddReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_ignore_add( + report: &IgnoreAddReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - if report.added { - println!("Added ignore pattern: {}", report.pattern); - } else { - println!("Ignore pattern already present: {}", report.pattern); - } - } - Ok(()) + let (lead, tone) = if report.added { + ("Added ignore pattern", UiTone::Success) + } else { + ("Ignore pattern already present", UiTone::Neutral) + }; + render_document( + &TerminalDocument::new(lead, tone).block(UiBlock::paragraph(&report.pattern)), + options, + ) } pub(crate) fn render_ignore_remove( report: &IgnoreRemoveReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - if report.removed { - println!("Removed ignore pattern: {}", report.pattern); - } else { - println!("Ignore pattern not present: {}", report.pattern); - } - } - Ok(()) + let (lead, tone) = if report.removed { + ("Removed ignore pattern", UiTone::Success) + } else { + ("Ignore pattern not present", UiTone::Neutral) + }; + render_document( + &TerminalDocument::new(lead, tone).block(UiBlock::paragraph(&report.pattern)), + options, + ) } pub(crate) fn render_ignore_check( report: &IgnoreCheckReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - match (&report.ignored, &report.source) { - (true, Some(source)) => println!("{}: ignored ({})", report.path, source), - (true, None) => println!("{}: ignored", report.path), - (false, _) => println!("{}: not ignored", report.path), - } + let mut document = TerminalDocument::new( + if report.ignored { + format!("{} is ignored", report.path) + } else { + format!("{} is not ignored", report.path) + }, + if report.ignored { + UiTone::Attention + } else { + UiTone::Success + }, + ); + if let Some(source) = &report.source { + document = document.block(UiBlock::Metadata(vec![( + "Rule".to_string(), + source.clone(), + )])); } - Ok(()) + render_document(&document, options) } diff --git a/trail/src/cli/command/render/inspection.rs b/trail/src/cli/command/render/inspection.rs index b4ccb6f..b524e45 100644 --- a/trail/src/cli/command/render/inspection.rs +++ b/trail/src/cli/command/render/inspection.rs @@ -1,223 +1,338 @@ -use super::render_json; +use super::*; use trail::model::*; use trail::Result; -pub(crate) fn render_show(result: &ShowResult, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_show(result: &ShowResult, json: bool, options: &RenderOptions) -> Result<()> { if json { return render_json(result); } - if quiet { - return Ok(()); - } - match result { + let document = match result { ShowResult::Operation { value } => { - let op = &value.operation; - println!("Operation: {}", op.change_id.0); - println!("Kind: {:?}", op.kind); - println!("Branch: {}", op.branch); - println!("Actor: {}", op.actor.id); - if let Some(message) = &op.message { - println!("Message: {message}"); - } - if !op.parents.is_empty() { - println!("Parents:"); - for parent in &op.parents { - println!(" {}", parent.0); - } + let operation = &value.operation; + let mut document = TerminalDocument::new( + format!("Operation {}", operation.change_id.0), + UiTone::Neutral, + ) + .context(operation_kind_label(&operation.kind)) + .block(UiBlock::Metadata(vec![ + ("Branch".to_string(), operation.branch.clone()), + ("Actor".to_string(), operation.actor.id.clone()), + ])); + if let Some(message) = &operation.message { + document = document.block(UiBlock::paragraph(message)); } - if let Some(before) = &op.before_root { - println!("Before root: {}", before.0); + if !value.changed_paths.is_empty() { + document = document.block(UiBlock::section( + "Changes:", + vec![UiBlock::Changes(crate::cli::command::render::change_list( + &value.changed_paths, + ))], + )); } - println!("After root: {}", op.after_root.0); - for path in &value.changed_paths { - println!( - " {:?} {} (+{} -{})", - path.kind, path.path, path.additions, path.deletions - ); - } - for message in &value.messages { - println!("Message object: {} {}", message.id.0, message.body); + if options.verbose { + let mut metadata = vec![("After root".to_string(), operation.after_root.0.clone())]; + if let Some(before_root) = &operation.before_root { + metadata.push(("Before root".to_string(), before_root.0.clone())); + } + document = document.block(UiBlock::Metadata(metadata)); + if !operation.parents.is_empty() { + document = document.block(UiBlock::section( + "Parents:", + vec![UiBlock::Lines( + operation + .parents + .iter() + .map(|parent| (parent.0.clone(), UiTone::Muted)) + .collect(), + )], + )); + } } + document } ShowResult::Message { value } => { - println!("Message: {}", value.id.0); - println!("Role: {}", value.role); - if let Some(lane_id) = &value.lane_id { - println!("Lane: {lane_id}"); - } - if let Some(session_id) = &value.session_id { - println!("Session: {session_id}"); - } - if let Some(change_id) = &value.change_id { - println!("Change: {}", change_id.0); - } - println!("{}", value.body); + TerminalDocument::new(format!("Message {}", value.id.0), UiTone::Neutral) + .block(UiBlock::paragraph(&value.body)) + .block(UiBlock::Metadata(message_metadata(value))) } ShowResult::Ref { value } => { - println!("Ref: {}", value.name); - println!("Change: {}", value.change_id.0); - println!("Root: {}", value.root_id.0); - println!("Generation: {}", value.generation); - } - ShowResult::Lane { value } => { - println!("Lane: {}", value.lane_id); - println!("Ref: {}", value.ref_name); - println!("Status: {}", value.status); - println!("Base: {}", value.base_change.0); - println!("Head: {}", value.head_change.0); - if let Some(workdir) = &value.workdir { - println!("Workdir: {workdir}"); - } + TerminalDocument::new(format!("Ref {}", value.name), UiTone::Neutral).block( + UiBlock::Metadata(vec![ + ("Change".to_string(), value.change_id.0.clone()), + ("Generation".to_string(), value.generation.to_string()), + ("Root".to_string(), value.root_id.0.clone()), + ]), + ) } + ShowResult::Lane { value } => TerminalDocument::new( + format!("Lane {}", value.lane_id), + status_tone(&value.status), + ) + .context(value.status.clone()) + .block(UiBlock::Metadata(lane_metadata(value, options))), ShowResult::Object { value } => { - println!("Object: {}", value.object_id.0); - println!("Kind: {}", value.kind); - println!("Version: {}", value.version); - println!("Size: {}", value.size_bytes); + TerminalDocument::new(format!("Object {}", value.object_id.0), UiTone::Neutral).block( + UiBlock::Metadata(vec![ + ("Kind".to_string(), value.kind.clone()), + ("Version".to_string(), value.version.to_string()), + ("Size".to_string(), format!("{} bytes", value.size_bytes)), + ]), + ) } - } - Ok(()) + }; + render_document(&document, options) } pub(crate) fn render_object_inspect( report: &ObjectInspectReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if quiet { - return Ok(()); - } - println!("Object: {}", report.info.object_id.0); - println!("Kind: {}", report.info.kind); - println!("Version: {}", report.info.version); - println!("Size: {}", report.info.size_bytes); - println!("Created at: {}", report.info.created_at); + let mut document = TerminalDocument::new( + format!("Object {}", report.info.object_id.0), + UiTone::Neutral, + ) + .block(UiBlock::Metadata(vec![ + ("Kind".to_string(), report.info.kind.clone()), + ("Version".to_string(), report.info.version.to_string()), + ( + "Size".to_string(), + format!("{} bytes", report.info.size_bytes), + ), + ( + "Created".to_string(), + format_timestamp(report.info.created_at, options), + ), + ])); if report .summary .as_object() .map(|summary| !summary.is_empty()) .unwrap_or(true) { - println!("Summary:"); - let rendered = serde_json::to_string_pretty(&report.summary)?; - for line in rendered.lines() { - println!(" {line}"); - } + let summary = serde_json::to_string_pretty(&report.summary)?; + document = document + .block(UiBlock::section( + "Summary:", + vec![UiBlock::Patch { + title: "Object metadata".to_string(), + text: summary, + }], + )) + .pager_eligible(); } - Ok(()) + render_document(&document, options) } pub(crate) fn render_root_inspect( report: &RootInspectReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if quiet { - return Ok(()); - } - println!("Root: {}", report.root_id.0); - println!("Created by: {}", report.root.created_by.0); - println!("Files: {}", report.root.file_count); - println!("Total text bytes: {}", report.root.total_text_bytes); + let mut metadata = vec![ + ("Created by".to_string(), report.root.created_by.0.clone()), + ("Files".to_string(), report.root.file_count.to_string()), + ( + "Text".to_string(), + format!("{} bytes", report.root.total_text_bytes), + ), + ]; if let Some(path_root) = &report.root.path_map_root { - println!("Path map: {path_root}"); + metadata.push(("Path map".to_string(), path_root.clone())); } if let Some(file_root) = &report.root.file_index_map_root { - println!("File index: {file_root}"); - } - for file in &report.files { - println!( - " {:?} {} {} -> {} ({} bytes)", - file.kind, file.path, file.file_id, file.content_object.0, file.size_bytes - ); + metadata.push(("File index".to_string(), file_root.clone())); } - Ok(()) + let rows = report + .files + .iter() + .map(|file| { + vec![ + file_kind_label(&file.kind).to_string(), + file.path.clone(), + file.file_id.clone(), + format!("{} bytes", file.size_bytes), + ] + }) + .collect(); + render_document( + &TerminalDocument::new(format!("Root {}", report.root_id.0), UiTone::Neutral) + .block(UiBlock::Metadata(metadata)) + .block(UiBlock::section( + "Files:", + vec![UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("KIND", 2, 6), + UiColumn::left("PATH", 0, 12), + UiColumn::left("FILE", 1, 12), + UiColumn::right("SIZE", 2, 8), + ], + rows, + ))], + )) + .pager_eligible(), + options, + ) } pub(crate) fn render_text_inspect( report: &TextInspectReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if quiet { - return Ok(()); - } - println!("Text: {}", report.text_id.0); - println!("Content hash: {}", report.content.content_hash); - println!( - "Lines: {} (showing {})", - report.content.line_count, - report.lines.len() - ); - println!("Bytes: {}", report.content.byte_count); - for line in &report.lines { - let text = serde_json::to_string(&line.text)?; - println!( - " {} {} {:?} {}", - line.line_number, line.line_id, line.newline, text - ); - } + let rows = report + .lines + .iter() + .map(|line| { + Ok(vec![ + line.line_number.to_string(), + line.line_id.clone(), + newline_kind_label(&line.newline).to_string(), + serde_json::to_string(&line.text)?, + ]) + }) + .collect::>>()?; + let mut document = TerminalDocument::new(format!("Text {}", report.text_id.0), UiTone::Neutral) + .block(UiBlock::Metadata(vec![ + ( + "Content hash".to_string(), + report.content.content_hash.clone(), + ), + ( + "Lines".to_string(), + format!( + "{} (showing {})", + report.content.line_count, + report.lines.len() + ), + ), + ("Bytes".to_string(), report.content.byte_count.to_string()), + ])) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::right("LINE", 0, 4), + UiColumn::left("LINE ID", 1, 12), + UiColumn::left("ENDING", 2, 5), + UiColumn::left("TEXT", 0, 16), + ], + rows, + ))); if report.truncated { - println!(" ... truncated; pass --limit 0 to show all lines"); + document = document.block(UiBlock::Notice( + "Output truncated; pass --limit 0 to show all lines.".to_string(), + )); } - Ok(()) + render_document(&document.pager_eligible(), options) } -pub(crate) fn render_map_range(report: &MapRangeReport, json: bool, quiet: bool) -> Result<()> { - if json { - return render_json(report); +fn file_kind_label(kind: &FileKind) -> &'static str { + match kind { + FileKind::Text => "text", + FileKind::OpaqueText => "opaque text", + FileKind::Binary => "binary", } - if quiet { - return Ok(()); +} + +fn newline_kind_label(kind: &NewlineKind) -> &'static str { + match kind { + NewlineKind::None => "none", + NewlineKind::Lf => "LF", + NewlineKind::Crlf => "CRLF", } - println!("Map: {}", report.map_id); - println!("Type: {}", report.map_type); - println!("Entries: {}", report.entries.len()); - for entry in &report.entries { - let key = render_map_key(&entry.key); - let value = render_map_value_summary(&entry.value)?; - println!(" {key} -> {value}"); +} + +pub(crate) fn render_map_range( + report: &MapRangeReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { + if json { + return render_json(report); } + let rows = report + .entries + .iter() + .map(|entry| { + Ok(vec![ + render_map_key(&entry.key), + render_map_value_summary(&entry.value)?, + ]) + }) + .collect::>>()?; + let mut document = TerminalDocument::new(format!("Map {}", report.map_id), UiTone::Neutral) + .context(report.map_type.clone()) + .block(UiBlock::Table(UiTable::new( + vec![UiColumn::left("KEY", 0, 12), UiColumn::left("VALUE", 0, 20)], + rows, + ))); if report.truncated { - println!(" ... truncated; pass --limit 0 to show all entries"); + document = document.block(UiBlock::Notice( + "Output truncated; pass --limit 0 to show all entries.".to_string(), + )); } - Ok(()) + render_document(&document.pager_eligible(), options) } -pub(crate) fn render_map_diff(report: &MapDiffReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_map_diff( + report: &MapDiffReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if quiet { - return Ok(()); - } - println!("Map diff {}..{}", report.left_map_id, report.right_map_id); - println!("Type: {}", report.map_type); - println!("Changes: {}", report.changes.len()); - for change in &report.changes { - let key = render_map_key(&change.key); - println!(" {} {key}", change.kind); - if let Some(old_value) = &change.old_value { - println!(" old: {}", render_map_value_summary(old_value)?); - } - if let Some(new_value) = &change.new_value { - println!(" new: {}", render_map_value_summary(new_value)?); - } - } + let rows = report + .changes + .iter() + .map(|change| { + Ok(vec![ + change.kind.clone(), + render_map_key(&change.key), + change + .old_value + .as_ref() + .map(render_map_value_summary) + .transpose()? + .unwrap_or_else(|| "—".to_string()), + change + .new_value + .as_ref() + .map(render_map_value_summary) + .transpose()? + .unwrap_or_else(|| "—".to_string()), + ]) + }) + .collect::>>()?; + let mut document = TerminalDocument::new( + format!("Map diff {} to {}", report.left_map_id, report.right_map_id), + UiTone::Neutral, + ) + .context(report.map_type.clone()) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("CHANGE", 1, 6), + UiColumn::left("KEY", 0, 12), + UiColumn::left("OLD", 2, 12), + UiColumn::left("NEW", 0, 12), + ], + rows, + ))); if report.truncated { - println!(" ... truncated; pass --limit 0 to show all changes"); + document = document.block(UiBlock::Notice( + "Output truncated; pass --limit 0 to show all changes.".to_string(), + )); } - Ok(()) + render_document(&document.pager_eligible(), options) } pub(crate) fn render_map_key(key: &MapKeyInspect) -> String { @@ -242,3 +357,38 @@ pub(crate) fn render_map_value_summary(value: &MapValueInspect) -> Result Vec<(String, String)> { + let mut metadata = vec![("Role".to_string(), value.role.clone())]; + if let Some(lane_id) = &value.lane_id { + metadata.push(("Lane".to_string(), lane_id.clone())); + } + if let Some(session_id) = &value.session_id { + metadata.push(("Session".to_string(), session_id.clone())); + } + if let Some(change_id) = &value.change_id { + metadata.push(("Change".to_string(), change_id.0.clone())); + } + metadata +} + +fn lane_metadata(value: &LaneBranch, options: &RenderOptions) -> Vec<(String, String)> { + let mut metadata = vec![("Ref".to_string(), value.ref_name.clone())]; + if options.verbose { + metadata.push(("Base".to_string(), value.base_change.0.clone())); + metadata.push(("Head".to_string(), value.head_change.0.clone())); + } + if let Some(workdir) = &value.workdir { + metadata.push(("Workdir".to_string(), workdir.clone())); + } + metadata +} + +fn status_tone(status: &str) -> UiTone { + match status.to_ascii_lowercase().as_str() { + "ready" | "active" | "open" => UiTone::Success, + "blocked" | "conflicted" | "failed" => UiTone::Blocked, + "paused" | "stale" | "pending" => UiTone::Attention, + _ => UiTone::Neutral, + } +} diff --git a/trail/src/cli/command/render/lane/approvals.rs b/trail/src/cli/command/render/lane/approvals.rs index 31b6c23..4c65c9b 100644 --- a/trail/src/cli/command/render/lane/approvals.rs +++ b/trail/src/cli/command/render/lane/approvals.rs @@ -1,4 +1,4 @@ -use super::render_json; +use crate::cli::command::render::*; use trail::model::*; use trail::Result; @@ -6,89 +6,138 @@ use trail::Result; pub(crate) fn render_approval_request( report: &LaneApprovalRequestReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Requested approval {} {}", - report.approval.approval_id, report.approval.action - ); - println!("{}", report.approval.summary); - if let Some(run_state) = &report.run_state { - println!("Paused run: {}", run_state.run_id); - } + let mut document = TerminalDocument::new( + format!("Approval requested: {}", report.approval.action), + UiTone::Attention, + ) + .block(approval_metadata(&report.approval)); + if let Some(run) = &report.run_state { + document = document.block(UiBlock::Notice(format!("Paused run: {}", run.run_id))); } - Ok(()) + document = document.next( + format!("trail approvals show {}", report.approval.approval_id), + "review the evidence before recording a decision", + ); + render_document(&document, options) } pub(crate) fn render_approval_list( approvals: &[LaneApproval], json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(&approvals); } - if !quiet { - if approvals.is_empty() { - println!("No approvals"); - } - for approval in approvals { - println!( - "{} {} {} {}", - approval.approval_id, approval.status, approval.lane_id, approval.action - ); - println!(" {}", approval.summary); - } + if approvals.is_empty() { + return render_document( + &TerminalDocument::new("No approvals", UiTone::Success), + options, + ); } - Ok(()) + render_document( + &TerminalDocument::new(format!("{} approval(s)", approvals.len()), UiTone::Neutral).block( + UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("STATUS", 0, 8), + UiColumn::left("LANE", 0, 10), + UiColumn::left("ACTION", 0, 12), + UiColumn::left("SUMMARY", 0, 18), + UiColumn::left("APPROVAL", 2, 12), + ], + approvals + .iter() + .map(|approval| { + vec![ + approval.status.clone(), + approval.lane_id.clone(), + approval.action.clone(), + approval.summary.clone(), + approval.approval_id.clone(), + ] + }) + .collect(), + )), + ), + options, + ) } -pub(crate) fn render_approval(approval: &LaneApproval, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_approval( + approval: &LaneApproval, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(approval); } - if !quiet { - println!("Approval: {}", approval.approval_id); - println!("Lane: {}", approval.lane_id); - println!("Status: {}", approval.status); - println!("Action: {}", approval.action); - println!("Summary: {}", approval.summary); - if let Some(session_id) = &approval.session_id { - println!("Session: {session_id}"); - } - if let Some(turn_id) = &approval.turn_id { - println!("Turn: {turn_id}"); - } - if let Some(reviewer) = &approval.reviewer { - println!("Reviewer: {reviewer}"); - } - if let Some(note) = &approval.note { - println!("Note: {note}"); - } + let pending = approval.status.eq_ignore_ascii_case("pending"); + let mut document = TerminalDocument::new( + format!("Approval {} is {}", approval.approval_id, approval.status), + if pending { + UiTone::Attention + } else { + UiTone::Neutral + }, + ) + .block(approval_metadata(approval)); + if pending { + document = document.next( + format!("trail approvals decide {} ", approval.approval_id), + "record a reviewed decision for this confirmation-sensitive action", + ); } - Ok(()) + render_document(&document, options) } pub(crate) fn render_approval_decision( report: &LaneApprovalDecisionReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Decision {} for {}", + let mut document = TerminalDocument::new( + format!( + "Recorded {} for approval {}", report.decision, report.approval.approval_id - ); - if !report.run_states.is_empty() { - println!("Linked run states: {}", report.run_states.len()); - } + ), + UiTone::Success, + ) + .block(approval_metadata(&report.approval)); + if !report.run_states.is_empty() { + document = document.block(UiBlock::Notice(format!( + "{} linked run(s) updated", + report.run_states.len() + ))); + } + render_document(&document, options) +} + +fn approval_metadata(approval: &LaneApproval) -> UiBlock { + let mut metadata = vec![ + ("Lane".to_string(), approval.lane_id.clone()), + ("Status".to_string(), approval.status.clone()), + ("Action".to_string(), approval.action.clone()), + ("Summary".to_string(), approval.summary.clone()), + ]; + if let Some(session) = &approval.session_id { + metadata.push(("Session".to_string(), session.clone())); + } + if let Some(turn) = &approval.turn_id { + metadata.push(("Turn".to_string(), turn.clone())); + } + if let Some(reviewer) = &approval.reviewer { + metadata.push(("Reviewer".to_string(), reviewer.clone())); + } + if let Some(note) = &approval.note { + metadata.push(("Note".to_string(), note.clone())); } - Ok(()) + UiBlock::Metadata(metadata) } diff --git a/trail/src/cli/command/render/lane/identity/basic.rs b/trail/src/cli/command/render/lane/identity/basic.rs index 6d7088e..9759e6f 100644 --- a/trail/src/cli/command/render/lane/identity/basic.rs +++ b/trail/src/cli/command/render/lane/identity/basic.rs @@ -1,79 +1,147 @@ use super::render_json; +use crate::cli::command::render::*; use trail::model::*; use trail::Result; -pub(crate) fn render_lane_spawn(report: &LaneSpawnReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_lane_spawn( + report: &LaneSpawnReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Spawned {} at {}", report.lane_id, report.base_change.0); - println!("Workdir mode: {}", report.workdir_mode.as_str()); - if let Some(cow_backend) = &report.cow_backend { - println!("COW backend: {cow_backend}"); - } - if !report.sparse_paths.is_empty() { - println!("Sparse paths: {}", report.sparse_paths.join(", ")); - } - if let Some(workdir) = &report.workdir { - println!("Workdir: {workdir}"); + let mut metadata = vec![ + ("Base".to_string(), report.base_change.0.clone()), + ( + "Requested mode".to_string(), + report.requested_workdir_mode.as_str().to_string(), + ), + ( + "Resolved mode".to_string(), + report.workdir_mode.as_str().to_string(), + ), + ( + "Backend".to_string(), + report + .workdir_backend + .map(WorkdirBackend::as_str) + .unwrap_or("unverified") + .to_string(), + ), + ]; + if let Some(materialization) = &report.materialization { + metadata.push(( + "Materialized".to_string(), + format!( + "{} cloned ({} bytes), {} copied ({} bytes)", + materialization.cloned_files, + materialization.cloned_bytes, + materialization.copied_files, + materialization.copied_bytes + ), + )); + if let Some(reason) = materialization.fallback_reason { + metadata.push(("Fallback".to_string(), reason.as_str().to_string())); } } - Ok(()) + let mut document = + TerminalDocument::new(format!("Created lane {}", report.lane_id), UiTone::Success) + .block(UiBlock::Metadata(metadata)); + if let Some(workdir) = &report.workdir { + document = document.block(UiBlock::Notice(format!("Workdir: {workdir}"))); + } + if !report.sparse_paths.is_empty() { + document = document.block(UiBlock::Metadata(vec![( + "Sparse paths".to_string(), + report.sparse_paths.join(", "), + )])); + } + document = document.next( + format!("trail lane status {}", report.lane_id), + "inspect the lane before beginning work", + ); + render_document(&document, options) } -pub(crate) fn render_lane_list(entries: &[LaneDetails], json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_lane_list( + entries: &[LaneDetails], + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(&entries); } - if !quiet { - for entry in entries { - println!( - "{} {} {} {}", - entry.record.name, - entry.branch.status, - entry.branch.head_change.0, - entry.branch.ref_name - ); - } + if entries.is_empty() { + return render_document( + &TerminalDocument::new("No lanes", UiTone::Neutral).next( + "trail lane spawn ", + "create an isolated lane for new work", + ), + options, + ); } - Ok(()) + render_document( + &TerminalDocument::new(format!("{} lane(s)", entries.len()), UiTone::Neutral).block( + UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("NAME", 0, 10), + UiColumn::left("STATUS", 0, 8), + UiColumn::left("REF", 1, 12), + UiColumn::left("HEAD", 2, 10), + ], + entries + .iter() + .map(|entry| { + vec![ + entry.record.name.clone(), + entry.branch.status.clone(), + entry.branch.ref_name.clone(), + entry.branch.head_change.0.clone(), + ] + }) + .collect(), + )), + ), + options, + ) } -pub(crate) fn render_lane_details(details: &LaneDetails, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_lane_details( + details: &LaneDetails, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(details); } - if !quiet { - println!("Lane: {}", details.record.name); - println!("ID: {}", details.record.lane_id); - println!("Ref: {}", details.branch.ref_name); - println!("Status: {}", details.branch.status); - println!("Base: {}", details.branch.base_change.0); - println!("Head: {}", details.branch.head_change.0); - if let Some(provider) = &details.record.provider { - println!("Provider: {provider}"); - } - if let Some(model) = &details.record.model { - println!("Model: {model}"); - } - if let Some(session_id) = &details.branch.session_id { - println!("Session: {session_id}"); - } - if let Some(workdir) = &details.branch.workdir { - println!("Workdir: {workdir}"); - } - if let Some(metadata_json) = &details.record.metadata_json { - if let Ok(metadata) = serde_json::from_str::(metadata_json) { - if let Some(mode) = metadata - .get("workdir_mode") - .and_then(serde_json::Value::as_str) - { - println!("Workdir mode: {mode}"); - } - } - } + let mut metadata = vec![ + ("ID".to_string(), details.record.lane_id.clone()), + ("Ref".to_string(), details.branch.ref_name.clone()), + ("Status".to_string(), details.branch.status.clone()), + ("Base".to_string(), details.branch.base_change.0.clone()), + ("Head".to_string(), details.branch.head_change.0.clone()), + ]; + if let Some(provider) = &details.record.provider { + metadata.push(("Provider".to_string(), provider.clone())); + } + if let Some(model) = &details.record.model { + metadata.push(("Model".to_string(), model.clone())); + } + if let Some(session_id) = &details.branch.session_id { + metadata.push(("Session".to_string(), session_id.clone())); + } + if let Some(workdir) = &details.branch.workdir { + metadata.push(("Workdir".to_string(), workdir.clone())); } - Ok(()) + render_document( + &TerminalDocument::new(format!("Lane {}", details.record.name), UiTone::Neutral) + .block(UiBlock::Metadata(metadata)) + .next( + format!("trail lane status {}", details.record.name), + "inspect lane state and merge readiness", + ), + options, + ) } diff --git a/trail/src/cli/command/render/lane/identity/reports.rs b/trail/src/cli/command/render/lane/identity/reports.rs index 9fc8b43..14e831a 100644 --- a/trail/src/cli/command/render/lane/identity/reports.rs +++ b/trail/src/cli/command/render/lane/identity/reports.rs @@ -1,392 +1,523 @@ use super::render_json; +use crate::cli::command::render::*; use trail::model::*; use trail::Result; -pub(crate) fn render_lane_status(report: &LaneStatusReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_lane_status( + report: &LaneStatusReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "{} {} ({} changed paths, {} queued merges)", - report.lane.record.name, - report.lane.branch.status, - report.changed_paths.len(), - report.queued_merges + let stale = report + .base_status + .as_ref() + .is_some_and(|base| base.stale || base.operations_behind.unwrap_or_default() > 0); + let tone = if stale { + UiTone::Attention + } else { + UiTone::Success + }; + let mut document = TerminalDocument::new( + format!( + "Lane {} is {}", + report.lane.record.name, report.lane.branch.status + ), + tone, + ) + .context(format!( + "{} changed path(s) · {} queued merge(s)", + report.changed_paths.len(), + report.queued_merges + )) + .block(lane_metadata(&report.lane)); + if let Some(base) = &report.base_status { + let freshness = match base.operations_behind { + Some(0) | None if !base.stale => "up to date".to_string(), + Some(behind) => format!("{behind} operation(s) behind {}", base.target_branch), + None => format!("stale against {}", base.target_branch), + }; + document = document.block(UiBlock::Notice(format!("Base: {freshness}"))); + } + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); + } + document = append_workdir( + document, + report.workdir_state.as_ref(), + &report.workdir_changed_paths, + ); + document = append_gate_summaries( + document, + report.latest_test.as_ref(), + report.latest_eval.as_ref(), + ); + if stale { + document = document.next( + format!( + "trail lane refresh-preview {} ", + report.lane.record.name + ), + "review the base update before refreshing this lane", + ); + } else { + document = document.next( + format!("trail lane readiness {}", report.lane.record.name), + "check merge readiness and any remaining gates", ); - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } - if let Some(base_status) = &report.base_status { - if let Some(behind) = base_status.operations_behind.filter(|behind| *behind > 0) { - let plural = if behind == 1 { - "operation" - } else { - "operations" - }; - println!( - "Lane started {behind} {plural} behind {}", - base_status.target_branch - ); - } - } - if let Some(state) = &report.workdir_state { - println!("Workdir: {:?}", state); - for path in &report.workdir_changed_paths { - println!(" workdir {:?} {}", path.kind, path.path); - } - } - if let Some(test) = &report.latest_test { - let command = if test.command.is_empty() { - String::new() - } else { - format!(" {}", test.command.join(" ")) - }; - println!( - "Latest test: {}{} ({} ms)", - test.status, command, test.duration_ms - ); - } - if let Some(eval) = &report.latest_eval { - let command = if eval.command.is_empty() { - String::new() - } else { - format!(" {}", eval.command.join(" ")) - }; - println!( - "Latest eval: {}{} ({} ms)", - eval.status, command, eval.duration_ms - ); - } } - Ok(()) + render_document(&document, options) } pub(crate) fn render_lane_contribution( report: &LaneContributionReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - let status = &report.status; - println!( - "Lane contribution: {} ({})", - status.lane.record.name, status.lane.branch.status - ); - println!("Ref: {}", status.lane.branch.ref_name); - println!( - "Base: {} Head: {}", - status.lane.branch.base_change.0, status.lane.branch.head_change.0 - ); - println!( - "Changed paths: {} Operations: {} Sessions: {} Events: {} Approvals: {}", - status.changed_paths.len(), - report.operations.len(), - report.sessions.len(), - report.recent_events.len(), - report.approvals.len() - ); - for path in &status.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } - if let Some(test) = &status.latest_test { - println!("Latest test: {} ({})", test.status, test.command.join(" ")); - } - if let Some(eval) = &status.latest_eval { - println!("Latest eval: {} ({})", eval.status, eval.command.join(" ")); - } - if !report.operations.is_empty() { - println!("Recent operations:"); - for operation in &report.operations { - println!( - " {} {:?} {} path(s) {}", - operation.change_id.0, - operation.kind, - operation.path_count, - operation.message.as_deref().unwrap_or("") - ); - } - } - let pending_approvals = report - .approvals - .iter() - .filter(|approval| approval.status == "pending") - .count(); - if pending_approvals > 0 { - println!("Pending approvals: {pending_approvals}"); - } + let status = &report.status; + let pending_approvals = report + .approvals + .iter() + .filter(|approval| approval.status == "pending") + .count(); + let mut document = TerminalDocument::new( + format!("Contribution for {}", status.lane.record.name), + UiTone::Neutral, + ) + .context(format!( + "{} operation(s) · {} session(s) · {} event(s)", + report.operations.len(), + report.sessions.len(), + report.recent_events.len() + )) + .block(lane_metadata(&status.lane)); + if pending_approvals > 0 { + document = document.block(UiBlock::Checklist(vec![UiCheck::new( + UiCheckState::Pending, + "Approvals", + format!("{pending_approvals} approval(s) pending"), + )])); + } + if !status.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&status.changed_paths))); } - Ok(()) + document = append_gate_summaries( + document, + status.latest_test.as_ref(), + status.latest_eval.as_ref(), + ); + if !report.operations.is_empty() { + document = document.block(UiBlock::section( + "Recent operations:", + vec![UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("WHEN", 2, 8), + UiColumn::left("KIND", 1, 8), + UiColumn::right("PATHS", 2, 5), + UiColumn::left("MESSAGE", 0, 16), + ], + report + .operations + .iter() + .map(|operation| { + vec![ + format_timestamp(operation.created_at, options), + operation_kind_label(&operation.kind).to_string(), + operation.path_count.to_string(), + operation.message.clone().unwrap_or_else(|| "—".to_string()), + ] + }) + .collect(), + ))], + )); + } + render_document(&document, options) } pub(crate) fn render_lane_review_packet( report: &LaneReviewPacketReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Lane review: {} ({})", - report.lane.record.name, report.readiness.status - ); - println!("Ref: {}", report.lane.branch.ref_name); - println!( - "Ready: {} Changed paths: {} Blockers: {} Warnings: {}", - report.readiness.ready, - report.changed_paths.len(), - report.readiness.blockers.len(), - report.readiness.warnings.len() - ); - println!( - "Evidence: {} operation(s), {} session(s), {} event(s), {} span(s), {} approval(s), {} gate(s), {} conflict(s), {} queued merge(s)", - report.evidence_summary.operations, - report.evidence_summary.sessions, - report.evidence_summary.events, - report.evidence_summary.spans, - report.evidence_summary.approvals, - report.evidence_summary.gates, - report.evidence_summary.conflicts, - report.evidence_summary.queued_merges - ); - if !report.readiness.blockers.is_empty() { - println!("Blockers:"); - for blocker in &report.readiness.blockers { - println!(" {}: {}", blocker.code, blocker.message); - } - } - if !report.readiness.warnings.is_empty() { - println!("Warnings:"); - for warning in &report.readiness.warnings { - println!(" {}: {}", warning.code, warning.message); - } - } - if let Some(test) = &report.latest_test { - println!("Latest test: {} ({})", test.status, test.command.join(" ")); - } - if let Some(eval) = &report.latest_eval { - println!("Latest eval: {} ({})", eval.status, eval.command.join(" ")); - } - if report.evidence_summary.pending_approvals > 0 { - println!( - "Pending approvals: {}", - report.evidence_summary.pending_approvals - ); - } - if !report.conflicts.is_empty() { - println!("Conflicts:"); - for conflict in &report.conflicts { - println!(" {} {}", conflict.conflict_set_id, conflict.status); - for detail in &conflict.details { - println!(" {detail}"); - } - } - } - if !report.changed_paths.is_empty() { - println!("Changed paths:"); - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } - } - if !report.recent_operations.is_empty() { - println!("Recent operations:"); - for operation in &report.recent_operations { - println!( - " {} {:?} {} path(s) {}", - operation.change_id.0, - operation.kind, - operation.path_count, - operation.message.as_deref().unwrap_or("") - ); - } - } - if !report.next_steps.is_empty() { - println!("Next steps:"); - for step in &report.next_steps { - println!(" {step}"); + let readiness = &report.readiness; + let mut document = TerminalDocument::new( + format!( + "Lane {} is {} for review", + report.lane.record.name, + if readiness.ready { + "ready" + } else { + "not ready" } - } + ), + readiness_tone(readiness.ready), + ) + .context(format!( + "{} blocker(s) · {} warning(s) · {} changed path(s)", + readiness.blockers.len(), + readiness.warnings.len(), + report.changed_paths.len() + )) + .block(lane_metadata(&report.lane)); + document = append_readiness(document, readiness); + document = append_gate_summaries( + document, + report.latest_test.as_ref(), + report.latest_eval.as_ref(), + ); + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); } - Ok(()) + document = document.block(UiBlock::Metadata(vec![ + ( + "Operations".to_string(), + report.evidence_summary.operations.to_string(), + ), + ( + "Sessions".to_string(), + report.evidence_summary.sessions.to_string(), + ), + ( + "Approvals".to_string(), + report.evidence_summary.approvals.to_string(), + ), + ( + "Gates".to_string(), + report.evidence_summary.gates.to_string(), + ), + ( + "Conflicts".to_string(), + report.evidence_summary.conflicts.to_string(), + ), + ])); + if !report.conflicts.is_empty() { + document = document.block(UiBlock::section( + "Conflicts:", + vec![UiBlock::Lines( + report + .conflicts + .iter() + .map(|conflict| { + ( + format!("{} {}", conflict.conflict_set_id, conflict.status), + UiTone::Blocked, + ) + }) + .collect(), + )], + )); + } + document = append_next_steps(document, &report.next_steps); + render_document(&document.pager_eligible(), options) } pub(crate) fn render_lane_gate_history( report: &LaneGateHistoryReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Lane gates for {} ({}, limit {})", - report.lane.record.name, report.kind, report.limit - ); - for gate in &report.gates { - let suite = gate.suite.as_deref().unwrap_or("-"); - let score = gate - .score - .map(|value| value.to_string()) - .unwrap_or_else(|| "-".to_string()); - let threshold = gate - .threshold - .map(|value| value.to_string()) - .unwrap_or_else(|| "-".to_string()); - println!( - " {} {} {} suite={} score={} threshold={} {}", - gate.created_at, - gate.kind, - gate.status, - suite, - score, - threshold, - gate.command.join(" ") - ); - } - } - Ok(()) + let document = TerminalDocument::new( + format!( + "{} gate history for {}", + report.kind, report.lane.record.name + ), + UiTone::Neutral, + ) + .context(format!( + "{} result(s), limit {}", + report.gates.len(), + report.limit + )) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("WHEN", 1, 8), + UiColumn::left("STATUS", 0, 8), + UiColumn::left("SUITE", 2, 8), + UiColumn::left("SCORE", 2, 6), + UiColumn::right("TIME", 1, 6), + UiColumn::left("COMMAND", 0, 16), + ], + report + .gates + .iter() + .map(|gate| { + vec![ + format_timestamp(gate.created_at, options), + gate.status.clone(), + gate.suite.clone().unwrap_or_else(|| "—".to_string()), + score_summary(gate), + format!("{} ms", gate.duration_ms), + gate.command.join(" "), + ] + }) + .collect(), + ))) + .next( + format!("trail lane readiness {}", report.lane.record.name), + "see whether the most recent gates satisfy merge readiness", + ) + .pager_eligible(); + render_document(&document, options) } pub(crate) fn render_lane_readiness( report: &LaneReadinessReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Lane readiness: {} ({})", - report.lane.record.name, report.status - ); - println!("Ref: {}", report.lane.branch.ref_name); - println!( - "Ready: {} Changed paths: {} Blockers: {} Warnings: {}", - report.ready, - report.changed_paths.len(), - report.blockers.len(), - report.warnings.len() - ); - if !report.blockers.is_empty() { - println!("Blockers:"); - for blocker in &report.blockers { - println!(" {}: {}", blocker.code, blocker.message); - } - } - if !report.warnings.is_empty() { - println!("Warnings:"); - for warning in &report.warnings { - println!(" {}: {}", warning.code, warning.message); + let mut document = TerminalDocument::new( + format!( + "Lane {} is {}", + report.lane.record.name, + if report.ready { + "ready to merge" + } else { + "not ready to merge" } - } - if let Some(test) = &report.latest_test { - println!("Latest test: {} ({})", test.status, test.command.join(" ")); - } - if let Some(eval) = &report.latest_eval { - println!("Latest eval: {} ({})", eval.status, eval.command.join(" ")); - } + ), + readiness_tone(report.ready), + ) + .context(format!( + "{} blocker(s) · {} warning(s) · {} pending approval(s)", + report.blockers.len(), + report.warnings.len(), + report.pending_approvals.len() + )) + .block(lane_metadata(&report.lane)); + document = append_readiness(document, report); + document = append_gate_summaries( + document, + report.latest_test.as_ref(), + report.latest_eval.as_ref(), + ); + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); + } + document = append_workdir( + document, + report.workdir_state.as_ref(), + &report.workdir_changed_paths, + ); + if report.ready { + document = document.next( + format!("trail lane merge {}", report.lane.record.name), + "merge this lane, or use --dry-run to preview it", + ); } - Ok(()) + render_document(&document, options) } pub(crate) fn render_lane_refresh_preview( report: &LaneRefreshPreviewReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Lane refresh preview: {} onto {}", - report.ref_name, report.target_ref - ); - if let Some(behind) = report.operations_behind { - let plural = if behind == 1 { - "operation" - } else { - "operations" - }; - println!("Lane started {behind} {plural} behind target"); - } - println!( - "Clean: {} Conflicted: {} Changed paths: {}", - report.clean, - report.conflicted, - report.changed_paths.len() - ); - for conflict in &report.conflicts { - println!(" conflict {conflict}"); - } - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } - if !report.next_steps.is_empty() { - println!("Next steps:"); - for step in &report.next_steps { - println!(" - {step}"); - } - } + let tone = if report.conflicted { + UiTone::Blocked + } else if report.clean { + UiTone::Success + } else { + UiTone::Attention + }; + let mut document = + TerminalDocument::new(format!("Refresh preview for {}", report.ref_name), tone) + .context(format!("onto {}", report.target_ref)) + .block(UiBlock::Metadata(vec![ + ("Clean".to_string(), report.clean.to_string()), + ("Conflicted".to_string(), report.conflicted.to_string()), + ( + "Behind".to_string(), + report + .operations_behind + .map(|behind| format!("{behind} operation(s)")) + .unwrap_or_else(|| "unknown".to_string()), + ), + ])); + if !report.conflicts.is_empty() { + document = document.block(UiBlock::Checklist( + report + .conflicts + .iter() + .map(|conflict| UiCheck::new(UiCheckState::Blocked, "Conflict", conflict)) + .collect(), + )); + } + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); } - Ok(()) + document = append_next_steps(document, &report.next_steps); + render_document(&document, options) } pub(crate) fn render_lane_handoff( report: &LaneHandoffReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Lane handoff: {} ({})", - report.lane.record.name, report.readiness.status - ); - println!("Ref: {}", report.lane.branch.ref_name); - println!( - "Ready: {} Sessions: {} Events: {} Spans: {} Operations: {}", - report.readiness.ready, - report.recent_sessions.len(), - report.recent_events.len(), - report.recent_spans.len(), - report.recent_operations.len() - ); - if let Some(session) = &report.current_session { - println!( - "Current session: {} ({})", - session.session.session_id, session.session.status - ); - println!( - "Session context: {} turn(s), {} message(s), {} event(s), {} operation(s)", - session.turns.len(), - session.messages.len(), - session.events.len(), - session.operations.len() - ); - } - if !report.readiness.blockers.is_empty() { - println!("Blockers:"); - for blocker in &report.readiness.blockers { - println!(" {}: {}", blocker.code, blocker.message); - } - } - if !report.next_steps.is_empty() { - println!("Next steps:"); - for step in &report.next_steps { - println!(" {step}"); - } + let mut document = TerminalDocument::new( + format!("Handoff for {}", report.lane.record.name), + readiness_tone(report.readiness.ready), + ) + .context(format!( + "{} session(s) · {} event(s) · {} operation(s)", + report.recent_sessions.len(), + report.recent_events.len(), + report.recent_operations.len() + )) + .block(lane_metadata(&report.lane)); + document = append_readiness(document, &report.readiness); + if let Some(session) = &report.current_session { + document = document.block(UiBlock::section( + "Current session:", + vec![UiBlock::Metadata(vec![ + ("ID".to_string(), session.session.session_id.clone()), + ("Status".to_string(), session.session.status.clone()), + ("Turns".to_string(), session.turns.len().to_string()), + ("Messages".to_string(), session.messages.len().to_string()), + ("Events".to_string(), session.events.len().to_string()), + ])], + )); + } + document = append_next_steps(document, &report.next_steps); + render_document(&document.pager_eligible(), options) +} + +fn lane_metadata(lane: &LaneDetails) -> UiBlock { + UiBlock::Metadata(vec![ + ("Ref".to_string(), lane.branch.ref_name.clone()), + ("Base".to_string(), lane.branch.base_change.0.clone()), + ("Head".to_string(), lane.branch.head_change.0.clone()), + ]) +} + +fn append_readiness( + mut document: TerminalDocument, + report: &LaneReadinessReport, +) -> TerminalDocument { + let mut checks: Vec<_> = report + .blockers + .iter() + .map(|issue| UiCheck::new(UiCheckState::Blocked, &issue.code, &issue.message)) + .collect(); + checks.extend( + report + .warnings + .iter() + .map(|issue| UiCheck::new(UiCheckState::Warn, &issue.code, &issue.message)), + ); + checks.extend(report.pending_approvals.iter().map(|approval| { + UiCheck::new( + UiCheckState::Pending, + "Approval", + format!("{} ({})", approval.approval_id, approval.action), + ) + })); + if !checks.is_empty() { + document = document.block(UiBlock::Checklist(checks)); + } + document +} + +fn append_workdir( + mut document: TerminalDocument, + state: Option<&WorktreeState>, + paths: &[FileDiffSummary], +) -> TerminalDocument { + if let Some(state) = state { + document = document.block(UiBlock::Notice(format!( + "Workdir: {}", + worktree_state_label(state) + ))); + } + if !paths.is_empty() { + document = document.block(UiBlock::section( + "Unrecorded workdir changes:", + vec![UiBlock::Changes(change_list(paths))], + )); + } + document +} + +fn append_gate_summaries( + mut document: TerminalDocument, + test: Option<&LaneTestSummary>, + eval: Option<&LaneTestSummary>, +) -> TerminalDocument { + let mut checks = Vec::new(); + if let Some(test) = test { + checks.push(gate_check("Test", test)); + } + if let Some(eval) = eval { + checks.push(gate_check("Eval", eval)); + } + if !checks.is_empty() { + document = document.block(UiBlock::Checklist(checks)); + } + document +} + +fn gate_check(label: &str, gate: &LaneTestSummary) -> UiCheck { + UiCheck::new( + if gate.success { + UiCheckState::Pass + } else { + UiCheckState::Fail + }, + label, + format!( + "{} · {} ms · {}", + gate.status, + gate.duration_ms, + gate.command.join(" ") + ), + ) +} + +fn append_next_steps(mut document: TerminalDocument, steps: &[String]) -> TerminalDocument { + for (index, step) in steps.iter().enumerate() { + if index == 0 { + document = document.next(step, "recommended next step"); + } else { + document = document.more(step, "alternative next step"); } } - Ok(()) + document +} + +fn readiness_tone(ready: bool) -> UiTone { + if ready { + UiTone::Success + } else { + UiTone::Blocked + } +} + +fn score_summary(gate: &LaneTestSummary) -> String { + match (gate.score, gate.threshold) { + (Some(score), Some(threshold)) => format!("{score} / {threshold}"), + (Some(score), None) => score.to_string(), + (None, Some(threshold)) => format!("threshold {threshold}"), + (None, None) => "—".to_string(), + } } diff --git a/trail/src/cli/command/render/lane/runs.rs b/trail/src/cli/command/render/lane/runs.rs index 434864c..e905c06 100644 --- a/trail/src/cli/command/render/lane/runs.rs +++ b/trail/src/cli/command/render/lane/runs.rs @@ -1,4 +1,4 @@ -use super::render_json; +use crate::cli::command::render::*; use trail::model::*; use trail::Result; @@ -6,98 +6,144 @@ use trail::Result; pub(crate) fn render_lane_run_pause( report: &LaneRunPauseReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Paused run {} for {}", - report.run_state.run_id, report.run_state.lane_id + let mut document = TerminalDocument::new( + format!("Paused run {}", report.run_state.run_id), + UiTone::Attention, + ) + .block(run_metadata(&report.run_state)); + if let Some(approval) = &report.run_state.approval_id { + document = document.next( + format!("trail approvals show {approval}"), + "review or decide the approval required to resume", + ); + } else { + document = document.next( + format!("trail lane run resume {}", report.run_state.run_id), + "resume when the interruption has been handled", ); - println!("Reason: {}", report.run_state.reason); - println!("Summary: {}", report.run_state.summary); - if let Some(approval_id) = &report.run_state.approval_id { - println!("Approval: {approval_id}"); - } } - Ok(()) + render_document(&document, options) } pub(crate) fn render_lane_run_resume( report: &LaneRunResumeReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Resumed run {} for {}", - report.run_state.run_id, report.run_state.lane_id - ); - if let Some(resumed_at) = report.run_state.resumed_at { - println!("Resumed at: {resumed_at}"); - } - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Resumed run {}", report.run_state.run_id), + UiTone::Success, + ) + .block(run_metadata(&report.run_state)), + options, + ) } pub(crate) fn render_lane_run_list( run_states: &[LaneRunState], json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(run_states); } - if !quiet { - if run_states.is_empty() { - println!("No lane run states"); - } - for run_state in run_states { - let approval = run_state.approval_id.as_deref().unwrap_or("-"); - println!( - "{} {} lane={} reason={} approval={}", - run_state.run_id, run_state.status, run_state.lane_id, run_state.reason, approval - ); - println!(" {}", run_state.summary); - } + if run_states.is_empty() { + return render_document( + &TerminalDocument::new("No lane runs", UiTone::Neutral), + options, + ); } - Ok(()) + render_document( + &TerminalDocument::new(format!("{} lane run(s)", run_states.len()), UiTone::Neutral).block( + UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("STATUS", 0, 8), + UiColumn::left("LANE", 0, 10), + UiColumn::left("REASON", 0, 12), + UiColumn::left("APPROVAL", 2, 12), + UiColumn::left("RUN", 2, 12), + ], + run_states + .iter() + .map(|run| { + vec![ + run.status.clone(), + run.lane_id.clone(), + run.reason.clone(), + run.approval_id.clone().unwrap_or_else(|| "—".to_string()), + run.run_id.clone(), + ] + }) + .collect(), + )), + ), + options, + ) } pub(crate) fn render_lane_run_state( run_state: &LaneRunState, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(run_state); } - if !quiet { - println!("Lane run: {}", run_state.run_id); - println!("Lane: {}", run_state.lane_id); - println!("Status: {}", run_state.status); - println!("Reason: {}", run_state.reason); - println!("Summary: {}", run_state.summary); - if let Some(session_id) = &run_state.session_id { - println!("Session: {session_id}"); - } - if let Some(turn_id) = &run_state.turn_id { - println!("Turn: {turn_id}"); - } - if let Some(approval_id) = &run_state.approval_id { - println!("Approval: {approval_id}"); - } - if let Some(reviewer) = &run_state.reviewer { - println!("Reviewer: {reviewer}"); - } - if let Some(note) = &run_state.note { - println!("Note: {note}"); + let paused = run_state.status.eq_ignore_ascii_case("paused"); + let mut document = TerminalDocument::new( + format!("Run {} is {}", run_state.run_id, run_state.status), + if paused { + UiTone::Attention + } else { + UiTone::Neutral + }, + ) + .block(run_metadata(run_state)); + if paused { + if let Some(approval) = &run_state.approval_id { + document = document.next( + format!("trail approvals show {approval}"), + "resolve the dependent approval before resuming", + ); + } else { + document = document.next( + format!("trail lane run resume {}", run_state.run_id), + "resume this paused run when ready", + ); } } - Ok(()) + render_document(&document, options) +} + +fn run_metadata(run: &LaneRunState) -> UiBlock { + let mut metadata = vec![ + ("Lane".to_string(), run.lane_id.clone()), + ("Reason".to_string(), run.reason.clone()), + ("Summary".to_string(), run.summary.clone()), + ]; + if let Some(session) = &run.session_id { + metadata.push(("Session".to_string(), session.clone())); + } + if let Some(turn) = &run.turn_id { + metadata.push(("Turn".to_string(), turn.clone())); + } + if let Some(approval) = &run.approval_id { + metadata.push(("Approval".to_string(), approval.clone())); + } + if let Some(reviewer) = &run.reviewer { + metadata.push(("Reviewer".to_string(), reviewer.clone())); + } + if let Some(note) = &run.note { + metadata.push(("Note".to_string(), note.clone())); + } + UiBlock::Metadata(metadata) } diff --git a/trail/src/cli/command/render/lane/sessions.rs b/trail/src/cli/command/render/lane/sessions.rs index 70d8cea..93f935c 100644 --- a/trail/src/cli/command/render/lane/sessions.rs +++ b/trail/src/cli/command/render/lane/sessions.rs @@ -1,4 +1,4 @@ -use super::render_json; +use crate::cli::command::render::*; use trail::model::*; use trail::Result; @@ -6,175 +6,256 @@ use trail::Result; pub(crate) fn render_session_start( report: &LaneSessionStartReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Started session {} for {}", - report.session.session_id, report.session.lane_id - ); - if let Some(title) = &report.session.title { - println!("Title: {title}"); - } - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Started session {}", report.session.session_id), + UiTone::Success, + ) + .block(session_metadata(&report.session)) + .next( + format!("trail session context {}", report.session.session_id), + "inspect the growing session context", + ), + options, + ) } pub(crate) fn render_session_current( reports: &[LaneSessionCurrentReport], json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(&reports); } - if !quiet { - if reports.is_empty() { - println!("No active sessions"); - } - for report in reports { - match &report.session { - Some(session) => { - let title = session.title.as_deref().unwrap_or(""); - println!( - "{} {} {} {}", - report.lane_name, session.session_id, session.status, title - ); - } - None => println!("{} has no active session", report.lane_name), - } - } + let active: Vec<_> = reports + .iter() + .filter_map(|report| report.session.as_ref().map(|session| (report, session))) + .collect(); + if active.is_empty() { + return render_document( + &TerminalDocument::new("No active sessions", UiTone::Neutral), + options, + ); } - Ok(()) + render_document( + &TerminalDocument::new( + format!("{} active session(s)", active.len()), + UiTone::Success, + ) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("LANE", 0, 10), + UiColumn::left("STATUS", 0, 8), + UiColumn::left("TITLE", 0, 14), + UiColumn::left("SESSION", 2, 12), + ], + active + .iter() + .map(|(report, session)| { + vec![ + report.lane_name.clone(), + session.status.clone(), + session.title.clone().unwrap_or_else(|| "—".to_string()), + session.session_id.clone(), + ] + }) + .collect(), + ))), + options, + ) } -pub(crate) fn render_session_list(sessions: &[LaneSession], json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_session_list( + sessions: &[LaneSession], + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(&sessions); } - if !quiet { - if sessions.is_empty() { - println!("No sessions"); - } - for session in sessions { - let title = session.title.as_deref().unwrap_or(""); - println!( - "{} {} {} {}", - session.session_id, session.status, session.lane_id, title - ); - } + if sessions.is_empty() { + return render_document( + &TerminalDocument::new("No sessions", UiTone::Neutral), + options, + ); } - Ok(()) + render_document( + &TerminalDocument::new(format!("{} session(s)", sessions.len()), UiTone::Neutral).block( + UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("STATUS", 0, 8), + UiColumn::left("LANE", 0, 10), + UiColumn::left("TITLE", 0, 14), + UiColumn::left("STARTED", 1, 10), + UiColumn::left("SESSION", 2, 12), + ], + sessions + .iter() + .map(|session| { + vec![ + session.status.clone(), + session.lane_id.clone(), + session.title.clone().unwrap_or_else(|| "—".to_string()), + format_timestamp(session.started_at, options), + session.session_id.clone(), + ] + }) + .collect(), + )), + ), + options, + ) } pub(crate) fn render_session_details( details: &LaneSessionDetails, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(details); } - if !quiet { - println!("Session: {}", details.session.session_id); - println!("Lane: {}", details.session.lane_id); - println!("Status: {}", details.session.status); - if let Some(title) = &details.session.title { - println!("Title: {title}"); - } - println!("Turns: {}", details.turns.len()); - println!("Messages: {}", details.messages.len()); - println!("Operations: {}", details.operations.len()); - for turn in &details.turns { - let after = turn - .after_change - .as_ref() - .map(|change| change.0.as_str()) - .unwrap_or("-"); - println!(" {} {} {}", turn.turn_id, turn.status, after); - } - for operation in &details.operations { - let message = operation.message.as_deref().unwrap_or(""); - println!( - " op {} {:?} {}", - operation.change_id.0, operation.kind, message - ); - } + let mut document = TerminalDocument::new( + format!("Session {}", details.session.session_id), + UiTone::Neutral, + ) + .block(session_metadata(&details.session)) + .block(UiBlock::Metadata(vec![ + ("Turns".to_string(), details.turns.len().to_string()), + ("Messages".to_string(), details.messages.len().to_string()), + ("Events".to_string(), details.events.len().to_string()), + ( + "Operations".to_string(), + details.operations.len().to_string(), + ), + ])); + if !details.turns.is_empty() { + document = document.block(UiBlock::section( + "Turns:", + vec![UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("STATUS", 0, 8), + UiColumn::left("CHECKPOINT", 1, 12), + UiColumn::left("TURN", 0, 12), + ], + details + .turns + .iter() + .map(|turn| { + vec![ + turn.status.clone(), + turn.after_change + .as_ref() + .map(|change| change.0.clone()) + .unwrap_or_else(|| "—".to_string()), + turn.turn_id.clone(), + ] + }) + .collect(), + ))], + )); } - Ok(()) + document = document.next( + format!("trail session context {}", details.session.session_id), + "review recent messages, turns, and operations", + ); + render_document(&document.pager_eligible(), options) } pub(crate) fn render_session_context( report: &LaneSessionContextReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Session context: {}", report.session.session_id); - println!("Lane: {}", report.session.lane_id); - println!("Status: {}", report.session.status); - if let Some(title) = &report.session.title { - println!("Title: {title}"); - } - println!( - "Totals: {} messages, {} events, {} turns, {} operations", - report.message_count, report.event_count, report.turn_count, report.operation_count - ); - if !report.recent_messages.is_empty() { - println!("Recent messages:"); - for message in &report.recent_messages { - let preview = single_line_preview(&message.body, 80); - println!(" {} {} {}", message.id.0, message.role, preview); - } - } - if !report.recent_turns.is_empty() { - println!("Recent turns:"); - for turn in &report.recent_turns { - println!(" {} {}", turn.turn_id, turn.status); - } - } - if !report.recent_operations.is_empty() { - println!("Recent operations:"); - for operation in &report.recent_operations { - let message = operation.message.as_deref().unwrap_or(""); - println!( - " {} {:?} {}", - operation.change_id.0, operation.kind, message - ); - } - } + let mut document = TerminalDocument::new( + format!("Session context: {}", report.session.session_id), + UiTone::Neutral, + ) + .block(session_metadata(&report.session)) + .block(UiBlock::Metadata(vec![ + ("Messages".to_string(), report.message_count.to_string()), + ("Events".to_string(), report.event_count.to_string()), + ("Turns".to_string(), report.turn_count.to_string()), + ("Operations".to_string(), report.operation_count.to_string()), + ])); + if !report.recent_messages.is_empty() { + document = document.block(UiBlock::section( + "Recent messages:", + vec![UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("ROLE", 0, 8), + UiColumn::left("MESSAGE", 0, 28), + ], + report + .recent_messages + .iter() + .map(|message| vec![message.role.clone(), preview(&message.body, 100)]) + .collect(), + ))], + )); + } + if !report.recent_turns.is_empty() { + document = document.block(UiBlock::section( + "Recent turns:", + vec![UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("STATUS", 0, 8), + UiColumn::left("TURN", 0, 12), + ], + report + .recent_turns + .iter() + .map(|turn| vec![turn.status.clone(), turn.turn_id.clone()]) + .collect(), + ))], + )); } - Ok(()) + render_document(&document.pager_eligible(), options) } pub(crate) fn render_session_end( report: &LaneSessionEndReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Ended session {} as {}", - report.session.session_id, report.session.status - ); + render_document( + &TerminalDocument::new( + format!("Ended session {}", report.session.session_id), + UiTone::Success, + ) + .block(session_metadata(&report.session)), + options, + ) +} + +fn session_metadata(session: &LaneSession) -> UiBlock { + let mut metadata = vec![ + ("Lane".to_string(), session.lane_id.clone()), + ("Status".to_string(), session.status.clone()), + ]; + if let Some(title) = &session.title { + metadata.push(("Title".to_string(), title.clone())); } - Ok(()) + UiBlock::Metadata(metadata) } -fn single_line_preview(value: &str, limit: usize) -> String { +fn preview(value: &str, limit: usize) -> String { let mut preview = value.split_whitespace().collect::>().join(" "); - if preview.len() > limit { - preview.truncate(limit.saturating_sub(3)); - preview.push_str("..."); + if preview.chars().count() > limit { + preview = preview.chars().take(limit.saturating_sub(1)).collect(); + preview.push('…'); } preview } diff --git a/trail/src/cli/command/render/lane/traces.rs b/trail/src/cli/command/render/lane/traces.rs index 05059ba..9c66da7 100644 --- a/trail/src/cli/command/render/lane/traces.rs +++ b/trail/src/cli/command/render/lane/traces.rs @@ -1,4 +1,4 @@ -use super::render_json; +use crate::cli::command::render::*; use trail; use trail::model::*; @@ -7,160 +7,192 @@ use trail::Result; pub(crate) fn render_lane_trace_span_start( report: &LaneTraceSpanStartReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Started span {} {} {}", - report.span.span_id, report.span.span_type, report.span.name - ); - println!("Trace: {}", report.span.trace_id); - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Started {} span", report.span.span_type), + UiTone::Success, + ) + .block(span_metadata(&report.span)), + options, + ) } pub(crate) fn render_lane_trace_span_end( report: &LaneTraceSpanEndReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Ended span {} {}", report.span.span_id, report.span.status); - if let Some(duration_ms) = report.span.duration_ms { - println!("Duration: {duration_ms} ms"); - } - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Ended span {}", report.span.span_id), + UiTone::Success, + ) + .block(span_metadata(&report.span)), + options, + ) } pub(crate) fn render_lane_trace_spans( spans: &[LaneTraceSpan], json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(spans); } - if !quiet { - for span in spans { - let parent = span.parent_span_id.as_deref().unwrap_or("-"); - let turn = span.turn_id.as_deref().unwrap_or("-"); - let duration = span - .duration_ms - .map(|duration_ms| format!("{duration_ms}ms")) - .unwrap_or_else(|| "-".to_string()); - println!( - "{} {} {} status={} trace={} parent={} turn={} duration={}", - span.span_id, - span.span_type, - span.name, - span.status, - span.trace_id, - parent, - turn, - duration - ); - } + if spans.is_empty() { + return render_document( + &TerminalDocument::new("No trace spans", UiTone::Neutral), + options, + ); } - Ok(()) + render_document( + &TerminalDocument::new(format!("{} trace span(s)", spans.len()), UiTone::Neutral) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("STATUS", 0, 8), + UiColumn::left("TYPE", 0, 8), + UiColumn::left("NAME", 0, 14), + UiColumn::left("TIME", 1, 7), + UiColumn::left("TRACE", 2, 12), + ], + spans + .iter() + .map(|span| { + vec![ + span.status.clone(), + span.span_type.clone(), + span.name.clone(), + duration(span.duration_ms), + span.trace_id.clone(), + ] + }) + .collect(), + ))) + .pager_eligible(), + options, + ) } pub(crate) fn render_lane_trace_summary( report: &LaneTraceSummaryReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Trace summary: {} spans ({} open, {} ended, {} failed)", - report.span_count, - report.open_span_count, - report.ended_span_count, - report.failed_span_count - ); - if let Some(trace_id) = &report.trace_id { - println!("Trace: {trace_id}"); - } - if let Some(lane_id) = &report.lane_id { - println!("Lane: {lane_id}"); - } - if let Some(turn_id) = &report.turn_id { - println!("Turn: {turn_id}"); - } - if report.total_duration_ms > 0 { - let average = report - .average_duration_ms - .map(|duration| format!("{duration:.1}")) - .unwrap_or_else(|| "-".to_string()); - println!( - "Duration: total={}ms max={}ms avg={}ms", - report.total_duration_ms, report.max_duration_ms, average - ); - } - println!("Statuses: {}", render_named_counts(&report.status_counts)); - println!("Types: {}", render_named_counts(&report.span_type_counts)); - println!("Traces: {}", render_named_counts(&report.trace_counts)); - if !report.slowest_spans.is_empty() { - println!("Slowest spans:"); - for span in &report.slowest_spans { - println!( - " {} {} {} {}ms", - span.span_id, - span.span_type, - span.status, - span.duration_ms.unwrap_or(0) - ); - } - } - if !report.open_spans.is_empty() { - println!("Open spans:"); - for span in &report.open_spans { - println!(" {} {} {}", span.span_id, span.span_type, span.name); - } - } + let mut metadata = vec![ + ("Spans".to_string(), report.span_count.to_string()), + ("Open".to_string(), report.open_span_count.to_string()), + ("Ended".to_string(), report.ended_span_count.to_string()), + ("Failed".to_string(), report.failed_span_count.to_string()), + ( + "Total duration".to_string(), + duration(Some(report.total_duration_ms)), + ), + ( + "Max duration".to_string(), + duration(Some(report.max_duration_ms)), + ), + ]; + if let Some(avg) = report.average_duration_ms { + metadata.push(("Average duration".to_string(), format!("{avg:.1} ms"))); } - Ok(()) -} - -pub(crate) fn render_named_counts(counts: &[trail::model::NamedCount]) -> String { - if counts.is_empty() { - return "-".to_string(); + if let Some(trace) = &report.trace_id { + metadata.push(("Trace".to_string(), trace.clone())); + } + if let Some(lane) = &report.lane_id { + metadata.push(("Lane".to_string(), lane.clone())); + } + let mut document = TerminalDocument::new( + "Trace summary", + if report.failed_span_count > 0 { + UiTone::Attention + } else { + UiTone::Success + }, + ) + .block(UiBlock::Metadata(metadata)); + if !report.slowest_spans.is_empty() { + document = document.block(UiBlock::section( + "Slowest spans:", + vec![UiBlock::Table(span_table(&report.slowest_spans))], + )); + } + if !report.open_spans.is_empty() { + document = document.block(UiBlock::section( + "Open spans:", + vec![UiBlock::Table(span_table(&report.open_spans))], + )); } - counts - .iter() - .map(|count| format!("{}={}", count.name, count.count)) - .collect::>() - .join(", ") + render_document(&document, options) } -pub(crate) fn render_lane_trace_span(span: &LaneTraceSpan, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_lane_trace_span( + span: &LaneTraceSpan, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(span); } - if !quiet { - println!("Span: {}", span.span_id); - println!("Trace: {}", span.trace_id); - println!("Type: {}", span.span_type); - println!("Name: {}", span.name); - println!("Status: {}", span.status); - if let Some(parent) = &span.parent_span_id { - println!("Parent: {parent}"); - } - if let Some(turn) = &span.turn_id { - println!("Turn: {turn}"); - } - if let Some(duration_ms) = span.duration_ms { - println!("Duration: {duration_ms} ms"); - } + render_document( + &TerminalDocument::new(format!("Span {}", span.name), UiTone::Neutral) + .block(span_metadata(span)), + options, + ) +} + +fn span_metadata(span: &LaneTraceSpan) -> UiBlock { + let mut metadata = vec![ + ("Span".to_string(), span.span_id.clone()), + ("Trace".to_string(), span.trace_id.clone()), + ("Type".to_string(), span.span_type.clone()), + ("Status".to_string(), span.status.clone()), + ("Duration".to_string(), duration(span.duration_ms)), + ]; + if let Some(parent) = &span.parent_span_id { + metadata.push(("Parent".to_string(), parent.clone())); + } + if let Some(turn) = &span.turn_id { + metadata.push(("Turn".to_string(), turn.clone())); } - Ok(()) + UiBlock::Metadata(metadata) +} + +fn span_table(spans: &[LaneTraceSpan]) -> UiTable { + UiTable::new( + vec![ + UiColumn::left("STATUS", 0, 8), + UiColumn::left("TYPE", 0, 8), + UiColumn::left("NAME", 0, 14), + UiColumn::left("TIME", 1, 7), + ], + spans + .iter() + .map(|span| { + vec![ + span.status.clone(), + span.span_type.clone(), + span.name.clone(), + duration(span.duration_ms), + ] + }) + .collect(), + ) +} + +fn duration(duration: Option) -> String { + duration + .map(|value| format!("{value} ms")) + .unwrap_or_else(|| "—".to_string()) } diff --git a/trail/src/cli/command/render/lane/turns.rs b/trail/src/cli/command/render/lane/turns.rs index 9008383..3ea0c65 100644 --- a/trail/src/cli/command/render/lane/turns.rs +++ b/trail/src/cli/command/render/lane/turns.rs @@ -1,4 +1,4 @@ -use super::render_json; +use crate::cli::command::render::*; use trail; use trail::model::*; @@ -7,135 +7,207 @@ use trail::Result; pub(crate) fn render_lane_message( report: &LaneMessageReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Added message {} ({})", report.message_id.0, report.role); - } - Ok(()) + render_document( + &TerminalDocument::new(format!("Added {} message", report.role), UiTone::Success).block( + UiBlock::Metadata(vec![("Message".to_string(), report.message_id.0.clone())]), + ), + options, + ) } pub(crate) fn render_lane_turn_start( report: &trail::LaneTurnStartReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Started turn {} for {}", - report.turn.turn_id, report.turn.lane_id - ); - println!("Session: {}", report.session.session_id); - println!("Base: {}", report.turn.before_change.0); - println!("Root: {}", report.base_root.0); - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Started turn {}", report.turn.turn_id), + UiTone::Success, + ) + .block(turn_metadata(&report.turn)) + .block(UiBlock::Metadata(vec![ + ("Session".to_string(), report.session.session_id.clone()), + ("Base root".to_string(), report.base_root.0.clone()), + ])), + options, + ) } pub(crate) fn render_lane_turn_details( details: &trail::LaneTurnDetails, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(details); } - if !quiet { - println!("Turn: {}", details.turn.turn_id); - println!("Lane: {}", details.turn.lane_id); - println!("Status: {}", details.turn.status); - if let Some(session) = &details.session { - println!("Session: {}", session.session_id); - } - println!("Base: {}", details.turn.before_change.0); - if let Some(after) = &details.turn.after_change { - println!("After: {}", after.0); + let mut document = + TerminalDocument::new(format!("Turn {}", details.turn.turn_id), UiTone::Neutral) + .block(turn_metadata(&details.turn)) + .block(UiBlock::Metadata(vec![ + ("Messages".to_string(), details.messages.len().to_string()), + ("Events".to_string(), details.events.len().to_string()), + ( + "Operations".to_string(), + details.operations.len().to_string(), + ), + ])); + if let Some(envelope) = &details.turn_envelope { + let mut metadata = Vec::new(); + if let Some(provider) = &envelope.provider { + metadata.push(("Provider".to_string(), provider.clone())); } - if let Some(envelope) = &details.turn_envelope { - if let Some(provider) = &envelope.provider { - println!("Provider: {provider}"); - } - if let Some(model) = &envelope.model { - println!("Model: {model}"); - } - if envelope.outcome.no_changes { - println!("Outcome: no changes"); - } + if let Some(model) = &envelope.model { + metadata.push(("Model".to_string(), model.clone())); } - println!("Messages: {}", details.messages.len()); - println!("Events: {}", details.events.len()); - println!("Operations: {}", details.operations.len()); - for event in &details.events { - println!(" event {} {}", event.event_id, event.event_type); + if envelope.outcome.no_changes { + metadata.push(("Outcome".to_string(), "no changes".to_string())); } - for operation in &details.operations { - let message = operation.message.as_deref().unwrap_or(""); - println!( - " op {} {:?} {}", - operation.change_id.0, operation.kind, message - ); + if !metadata.is_empty() { + document = document.block(UiBlock::section( + "Agent outcome:", + vec![UiBlock::Metadata(metadata)], + )); } } - Ok(()) + if !details.events.is_empty() { + document = document.block(UiBlock::section( + "Events:", + vec![UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("WHEN", 1, 8), + UiColumn::left("TYPE", 0, 12), + UiColumn::left("EVENT", 2, 12), + ], + details + .events + .iter() + .map(|event| { + vec![ + format_timestamp(event.created_at, options), + event.event_type.clone(), + event.event_id.clone(), + ] + }) + .collect(), + ))], + )); + } + render_document(&document.pager_eligible(), options) } pub(crate) fn render_lane_turn_event( report: &trail::LaneTurnEventReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Added event {} {}", - report.event.event_id, report.event.event_type - ); - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Recorded {} event", report.event.event_type), + UiTone::Success, + ) + .block(event_metadata(&report.event)), + options, + ) } pub(crate) fn render_lane_events( events: &[LaneEventRecord], json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(events); } - if !quiet { - for event in events { - let session = event.session_id.as_deref().unwrap_or("-"); - let turn = event.turn_id.as_deref().unwrap_or("-"); - println!( - "{} {} lane={} session={} turn={}", - event.event_id, event.event_type, event.lane_id, session, turn - ); - } + if events.is_empty() { + return render_document( + &TerminalDocument::new("No events", UiTone::Neutral), + options, + ); } - Ok(()) + render_document( + &TerminalDocument::new(format!("{} event(s)", events.len()), UiTone::Neutral) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("WHEN", 1, 8), + UiColumn::left("TYPE", 0, 12), + UiColumn::left("LANE", 0, 10), + UiColumn::left("SESSION", 2, 12), + UiColumn::left("TURN", 2, 12), + ], + events + .iter() + .map(|event| { + vec![ + format_timestamp(event.created_at, options), + event.event_type.clone(), + event.lane_id.clone(), + event.session_id.clone().unwrap_or_else(|| "—".to_string()), + event.turn_id.clone().unwrap_or_else(|| "—".to_string()), + ] + }) + .collect(), + ))) + .pager_eligible(), + options, + ) } pub(crate) fn render_lane_turn_end( report: &trail::LaneTurnEndReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Ended turn {} as {}", - report.turn.turn_id, report.turn.status - ); + render_document( + &TerminalDocument::new( + format!( + "Ended turn {} as {}", + report.turn.turn_id, report.turn.status + ), + UiTone::Success, + ) + .block(turn_metadata(&report.turn)), + options, + ) +} + +fn turn_metadata(turn: &LaneTurn) -> UiBlock { + let mut metadata = vec![ + ("Lane".to_string(), turn.lane_id.clone()), + ("Status".to_string(), turn.status.clone()), + ("Before".to_string(), turn.before_change.0.clone()), + ]; + if let Some(after) = &turn.after_change { + metadata.push(("After".to_string(), after.0.clone())); } - Ok(()) + if let Some(session) = &turn.session_id { + metadata.push(("Session".to_string(), session.clone())); + } + UiBlock::Metadata(metadata) +} + +fn event_metadata(event: &LaneEventRecord) -> UiBlock { + UiBlock::Metadata(vec![ + ("Event".to_string(), event.event_id.clone()), + ("Lane".to_string(), event.lane_id.clone()), + ( + "Turn".to_string(), + event.turn_id.clone().unwrap_or_else(|| "—".to_string()), + ), + ]) } diff --git a/trail/src/cli/command/render/lane/work.rs b/trail/src/cli/command/render/lane/work.rs index 50d519d..ba5b981 100644 --- a/trail/src/cli/command/render/lane/work.rs +++ b/trail/src/cli/command/render/lane/work.rs @@ -1,338 +1,590 @@ -use super::render_json; +use crate::cli::command::render::*; -use std::io::Write; use trail::model::*; use trail::Result; -pub(crate) fn render_lane_record(report: &LaneRecordReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_lane_record( + report: &LaneRecordReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - match &report.operation { - Some(operation) => { - println!("Recorded lane workdir {}", operation.0); - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } - } - None => println!("No lane workdir changes to record"), - } + let Some(operation) = &report.operation else { + return render_document( + &TerminalDocument::new("No lane workdir changes to record", UiTone::Neutral), + options, + ); + }; + let mut document = + TerminalDocument::new(format!("Recorded lane {}", report.lane_id), UiTone::Success) + .context(format!("{} changed path(s)", report.changed_paths.len())); + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); } - Ok(()) + if options.verbose { + document = document.block(UiBlock::Metadata(vec![ + ("Operation".to_string(), operation.0.clone()), + ("Root".to_string(), report.root_id.0.clone()), + ])); + } + render_document(&document, options) } pub(crate) fn render_workspace_checkpoint( report: &WorkspaceCheckpointReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Checkpointed view {} at {}", - report.view_id, report.root_id.0 - ); - println!("Journal sequence: {}", report.journal_sequence); - println!("Source paths: {}", report.source_paths.len()); - println!("Generated dirty paths: {}", report.generated_dirty_paths); + let mut document = TerminalDocument::new( + format!("Checkpointed workspace view {}", report.view_id), + UiTone::Success, + ) + .block(UiBlock::Metadata(vec![ + ( + "Journal sequence".to_string(), + report.journal_sequence.to_string(), + ), + ( + "Source paths".to_string(), + report.source_paths.len().to_string(), + ), + ( + "Generated dirty paths".to_string(), + report.generated_dirty_paths.to_string(), + ), + ])); + if options.verbose { + document = document.block(UiBlock::Metadata(vec![ + ("Root".to_string(), report.root_id.0.clone()), + ( + "Operation".to_string(), + report + .operation + .as_ref() + .map(|operation| operation.0.clone()) + .unwrap_or_else(|| "—".to_string()), + ), + ])); } - Ok(()) + render_document(&document, options) } pub(crate) fn render_workspace_space( report: &WorkspaceSpaceReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Workspace view: {}", report.view_id); - println!("Logical visible bytes: {}", report.logical_visible_bytes); - println!("Shared physical bytes: {}", report.shared_physical_bytes); - println!( - "Lane-exclusive physical bytes: {}", - report.lane_exclusive_physical_bytes - ); - println!( - "Uncheckpointed source bytes: {}", - report.uncheckpointed_source_bytes - ); - println!("Accounting: {}", report.physical_accounting); - } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Workspace space for view {}", report.view_id), + UiTone::Neutral, + ) + .block(UiBlock::Metadata(vec![ + ( + "Logical visible".to_string(), + byte_count(report.logical_visible_bytes), + ), + ( + "Shared physical".to_string(), + byte_count(report.shared_physical_bytes), + ), + ( + "Lane-exclusive".to_string(), + byte_count(report.lane_exclusive_physical_bytes), + ), + ( + "Uncheckpointed".to_string(), + byte_count(report.uncheckpointed_source_bytes), + ), + ("Accounting".to_string(), report.physical_accounting.clone()), + ])), + options, + ) } pub(crate) fn render_workspace_exec( report: &WorkspaceExecReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "View {} command exited with {} (root {}, generation {})", - report.view_id, report.exit_code, report.source_root.0, report.generation - ); + let success = report.exit_code == 0; + let mut document = TerminalDocument::new( + if success { + format!("Workspace command completed for {}", report.lane_id) + } else { + format!("Workspace command exited with {}", report.exit_code) + }, + if success { + UiTone::Success + } else { + UiTone::Failure + }, + ) + .block(UiBlock::Metadata(vec![ + ("Command".to_string(), report.command.join(" ")), + ("View".to_string(), report.view_id.clone()), + ("Backend".to_string(), report.backend.clone()), + ("Generation".to_string(), report.generation.to_string()), + ])); + if options.verbose { + document = document.block(UiBlock::Metadata(vec![( + "Source root".to_string(), + report.source_root.0.clone(), + )])); } - Ok(()) + render_document(&document, options) } pub(crate) fn render_workspace_mount( report: &WorkspaceMountReport, state: &str, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Workspace view {} {state} at {} (backend {}, generation {})", - report.view_id, report.mountpoint, report.backend, report.generation - ); - } - Ok(()) + let tone = if report.healthy { + UiTone::Success + } else { + UiTone::Attention + }; + render_document( + &TerminalDocument::new(format!("Workspace view {} {}", report.view_id, state), tone).block( + UiBlock::Metadata(vec![ + ("Mountpoint".to_string(), report.mountpoint.clone()), + ("Backend".to_string(), report.backend.clone()), + ("Generation".to_string(), report.generation.to_string()), + ( + "Health".to_string(), + if report.healthy { + "healthy" + } else { + "needs attention" + } + .to_string(), + ), + ]), + ), + options, + ) } pub(crate) fn render_lane_record_preview( report: &LaneRecordPreviewReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Lane workdir record preview: {}", report.workdir); - println!("Head: {}", report.head_change.0); - println!("Policy allowed: {}", report.policy.allowed); - if let Some(error) = &report.policy.error { - println!("Policy error: {error}"); - } - for warning in &report.policy.warnings { - println!("Policy warning: {warning}"); - } + let tone = if !report.policy.allowed { + UiTone::Blocked + } else if report.clean { + UiTone::Neutral + } else { + UiTone::Attention + }; + let mut document = TerminalDocument::new( if report.clean { - println!("No lane workdir changes to record"); + format!("No lane workdir changes to record for {}", report.lane_id) } else { - println!("Changed paths:"); - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } - } - if !report.oversized_files.is_empty() { - println!("Oversized files:"); - for file in &report.oversized_files { - println!( - " {} ({} bytes > {} bytes)", - file.path, file.size_bytes, file.limit_bytes - ); - } - } - if !report.ignored_paths.is_empty() { - println!("Ignored paths:"); - for path in &report.ignored_paths { - println!(" {} ({})", path.path, path.source); - } - } - if !report.risky_paths.is_empty() { - println!("Risky paths:"); - for path in &report.risky_paths { - println!(" {} [{}] {}", path.path, path.kind, path.message); + format!("Record preview for lane {}", report.lane_id) + }, + tone, + ) + .block(UiBlock::Metadata(vec![ + ("Workdir".to_string(), report.workdir.clone()), + ( + "Policy".to_string(), + if report.policy.allowed { + "allowed" + } else { + "blocked" } - } + .to_string(), + ), + ( + "Changed paths".to_string(), + report.changed_paths.len().to_string(), + ), + ])); + let mut checks = Vec::new(); + if let Some(error) = &report.policy.error { + checks.push(UiCheck::new(UiCheckState::Blocked, "Record policy", error)); + } + checks.extend( + report + .policy + .warnings + .iter() + .map(|warning| UiCheck::new(UiCheckState::Warn, "Record policy", warning)), + ); + checks.extend(report.risky_paths.iter().map(|path| { + UiCheck::new( + UiCheckState::Warn, + format!("Risky {}", path.kind), + format!("{}: {}", path.path, path.message), + ) + })); + if !checks.is_empty() { + document = document.block(UiBlock::Checklist(checks)); + } + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); + } + if !report.oversized_files.is_empty() { + document = document.block(UiBlock::section( + "Oversized files:", + vec![UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("PATH", 0, 16), + UiColumn::right("SIZE", 1, 8), + UiColumn::right("LIMIT", 1, 8), + ], + report + .oversized_files + .iter() + .map(|file| { + vec![ + file.path.clone(), + byte_count(file.size_bytes), + byte_count(file.limit_bytes), + ] + }) + .collect(), + ))], + )); } - Ok(()) + if !report.policy.allowed { + document = document.next( + format!("trail lane record {} --preview", report.lane_id), + "resolve the listed policy blockers before recording", + ); + } else if !report.clean { + document = document.next( + format!("trail lane record {}", report.lane_id), + "record the reviewed lane changes", + ); + } + render_document(&document, options) } -pub(crate) fn render_lane_rewind(report: &LaneRewindReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_lane_rewind( + report: &LaneRewindReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Rewound lane {} from {} to {}", - report.lane_id, report.previous_change.0, report.operation.0 - ); - println!("Target: {} ({})", report.target, report.target_change.0); - if let Some(operation) = &report.recorded_current { - println!("Recorded current workdir: {}", operation.0); - } - if let Some(branch) = &report.preserved_branch { - println!("Preserved previous head: {branch}"); - } - if report.workdir_synced { - if let Some(workdir) = &report.workdir { - println!("Synced workdir: {workdir}"); + let mut document = TerminalDocument::new( + format!("Rewound lane {} to {}", report.lane_id, report.target), + UiTone::Success, + ) + .block(UiBlock::Metadata(vec![ + ("Target change".to_string(), report.target_change.0.clone()), + ( + "Workdir".to_string(), + if report.workdir_synced { + "synced" + } else { + "not synced" } - } else if report.workdir.is_some() { - println!("Workdir not synced"); - } - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } - } - Ok(()) + .to_string(), + ), + ])); + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); + } + if let Some(recorded) = &report.recorded_current { + document = document.block(UiBlock::Notice(format!( + "Recorded current workdir as {} before rewinding.", + recorded.0 + ))); + } + render_document(&document, options) } -pub(crate) fn render_lane_watch(report: &LaneWatchReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_lane_watch( + report: &LaneWatchReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Watched {} for {} iteration(s); recorded {} operation(s)", - report.lane_id, - report.iterations, - report.recorded_operations.len() + let mut document = + TerminalDocument::new(format!("Watched lane {}", report.lane_id), UiTone::Success).context( + format!( + "{} iteration(s) · {} recorded operation(s)", + report.iterations, + report.recorded_operations.len() + ), ); - for operation in &report.recorded_operations { - println!(" {operation}"); - } - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } - } - Ok(()) + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); + } + render_document(&document, options) } -pub(crate) fn render_lane_test(report: &LaneTestReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_lane_test( + report: &LaneTestReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Lane {} {} for {}", - report.kind, report.status, report.lane_id - ); - println!("Turn: {}", report.turn_id); - println!("Command: {}", report.command.join(" ")); - if let Some(suite) = &report.suite { - println!("Suite: {suite}"); - } - if report.score.is_some() || report.threshold.is_some() { - let score = report - .score - .map(|value| value.to_string()) - .unwrap_or_else(|| "n/a".to_string()); - let threshold = report - .threshold - .map(|value| value.to_string()) - .unwrap_or_else(|| "n/a".to_string()); - println!("Score: {score} / threshold {threshold}"); - } - match report.exit_code { - Some(code) => println!("Exit: {code}"), - None if report.timed_out => println!("Exit: timed out"), - None => println!("Exit: unavailable"), - } - println!("Duration: {} ms", report.duration_ms); - println!("Stdout object: {}", report.stdout_object.0); - println!("Stderr object: {}", report.stderr_object.0); - if !report.stdout_preview.is_empty() { - println!("Stdout:"); - print!("{}", report.stdout_preview); - if !report.stdout_preview.ends_with('\n') { - println!(); + let mut document = TerminalDocument::new( + format!( + "{} {} for lane {}", + report.kind, + if report.success { "passed" } else { "failed" }, + report.lane_id + ), + if report.success { + UiTone::Success + } else { + UiTone::Failure + }, + ) + .block(UiBlock::Metadata(vec![ + ("Command".to_string(), report.command.join(" ")), + ("Turn".to_string(), report.turn_id.clone()), + ("Status".to_string(), report.status.clone()), + ( + "Duration".to_string(), + super::super::format_duration(report.duration_ms, options), + ), + ( + "Exit".to_string(), + report + .exit_code + .map(|code| code.to_string()) + .unwrap_or_else(|| { + if report.timed_out { + "timed out" + } else { + "unavailable" + } + .to_string() + }), + ), + ])); + if let Some(suite) = &report.suite { + document = document.block(UiBlock::Metadata(vec![( + "Suite".to_string(), + suite.clone(), + )])); + } + if report.score.is_some() || report.threshold.is_some() { + document = document.block(UiBlock::Metadata(vec![( + "Score".to_string(), + format!( + "{} / threshold {}", + report + .score + .map(|value| value.to_string()) + .unwrap_or_else(|| "n/a".to_string()), + report + .threshold + .map(|value| value.to_string()) + .unwrap_or_else(|| "n/a".to_string()) + ), + )])); + } + if !report.stdout_preview.is_empty() { + document = document.block(UiBlock::Patch { + title: if report.stdout_truncated { + "Stdout (truncated)" + } else { + "Stdout" } - } - if !report.stderr_preview.is_empty() { - println!("Stderr:"); - eprint!("{}", report.stderr_preview); - if !report.stderr_preview.ends_with('\n') { - eprintln!(); + .to_string(), + text: report.stdout_preview.clone(), + }); + } + if !report.stderr_preview.is_empty() { + document = document.block(UiBlock::Patch { + title: if report.stderr_truncated { + "Stderr (truncated)" + } else { + "Stderr" } - } + .to_string(), + text: report.stderr_preview.clone(), + }); } - Ok(()) + render_document(&document.pager_eligible(), options) } pub(crate) fn render_lane_workdir( report: &LaneWorkdirReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - if let Some(workdir) = &report.workdir { - println!("{workdir}"); - } else { - println!("Lane {} has no materialized workdir", report.lane_id); - } + let Some(workdir) = &report.workdir else { + return render_document( + &TerminalDocument::new( + format!("Lane {} has no materialized workdir", report.lane_id), + UiTone::Neutral, + ), + options, + ); + }; + let mut metadata = vec![ + ("Path".to_string(), workdir.clone()), + ( + "Requested mode".to_string(), + report.requested_workdir_mode.as_str().to_string(), + ), + ( + "Resolved mode".to_string(), + report.workdir_mode.as_str().to_string(), + ), + ( + "Backend".to_string(), + report + .workdir_backend + .map(WorkdirBackend::as_str) + .unwrap_or("unverified") + .to_string(), + ), + ( + "Transparent COW available".to_string(), + report.transparent_cow_available.to_string(), + ), + ]; + if let Some(materialization) = &report.materialization { + metadata.push(( + "Materialized".to_string(), + format!( + "{} cloned ({} bytes), {} copied ({} bytes)", + materialization.cloned_files, + materialization.cloned_bytes, + materialization.copied_files, + materialization.copied_bytes + ), + )); } - Ok(()) + render_document( + &TerminalDocument::new( + format!("Workdir for lane {}", report.lane_id), + UiTone::Success, + ) + .block(UiBlock::Metadata(metadata)), + options, + ) } pub(crate) fn render_lane_file_read( report: &LaneFileReadReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - std::io::stdout().write_all(report.content.as_bytes())?; - } - Ok(()) + // File reads are intentionally raw so callers can inspect or pipe exact + // content. This is the documented raw-content exception for the CLI. + render_raw_content(&report.content, options) } pub(crate) fn render_lane_workdir_sync( report: &LaneWorkdirSyncReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Synced lane workdir: {}", report.workdir); - println!("Head: {}", report.head_change.0); - if report.forced { - println!("Forced: true"); - } - if let Some(rescue_workdir) = &report.rescue_workdir { - println!("Rescue workdir: {rescue_workdir}"); - } - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } - } - Ok(()) + let mut document = TerminalDocument::new( + format!("Synced workdir for lane {}", report.lane_id), + UiTone::Success, + ) + .block(UiBlock::Metadata(vec![ + ("Workdir".to_string(), report.workdir.clone()), + ("Head".to_string(), report.head_change.0.clone()), + ("Forced".to_string(), report.forced.to_string()), + ])); + if let Some(rescue) = &report.rescue_workdir { + document = document.block(UiBlock::Notice(format!("Rescue workdir: {rescue}"))); + } + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); + } + render_document(&document, options) } -pub(crate) fn render_lane_patch(report: &LanePatchReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_lane_patch( + report: &LanePatchReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Applied lane patch {}", report.operation.0); - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } + let mut document = TerminalDocument::new( + format!("Applied patch to lane {}", report.lane_id), + UiTone::Success, + ) + .context(format!("{} changed path(s)", report.changed_paths.len())); + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); + } + if options.verbose { + document = document.block(UiBlock::Metadata(vec![ + ("Operation".to_string(), report.operation.0.clone()), + ("Root".to_string(), report.root_id.0.clone()), + ])); } - Ok(()) + render_document(&document, options) } -pub(crate) fn render_lane_remove(report: &LaneRemoveReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_lane_remove( + report: &LaneRemoveReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Removed lane {} ({})", report.lane_id, report.ref_name); - if let Some(workdir) = &report.removed_workdir { - println!("Removed workdir: {workdir}"); - } + let mut document = + TerminalDocument::new(format!("Removed lane {}", report.lane_id), UiTone::Success).block( + UiBlock::Metadata(vec![ + ("Ref".to_string(), report.ref_name.clone()), + ("Forced".to_string(), report.forced.to_string()), + ]), + ); + if let Some(workdir) = &report.removed_workdir { + document = document.block(UiBlock::Notice(format!("Removed workdir: {workdir}"))); + } + render_document(&document, options) +} + +fn byte_count(value: u64) -> String { + const KIB: u64 = 1024; + const MIB: u64 = KIB * 1024; + const GIB: u64 = MIB * 1024; + match value { + value if value >= GIB => format!("{:.1} GiB", value as f64 / GIB as f64), + value if value >= MIB => format!("{:.1} MiB", value as f64 / MIB as f64), + value if value >= KIB => format!("{:.1} KiB", value as f64 / KIB as f64), + _ => format!("{value} B"), } - Ok(()) } diff --git a/trail/src/cli/command/render/maintenance.rs b/trail/src/cli/command/render/maintenance.rs index bd50cb9..a3bf275 100644 --- a/trail/src/cli/command/render/maintenance.rs +++ b/trail/src/cli/command/render/maintenance.rs @@ -1,179 +1,392 @@ -use super::render_json; +use super::*; use trail::model::*; use trail::Result; -pub(crate) fn render_doctor(report: &DoctorReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_doctor( + report: &DoctorReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Doctor: {}", report.status); - for check in &report.checks { - println!("[{}] {}: {}", check.status, check.name, check.message); - } + let checks = report + .checks + .iter() + .map(|check| { + UiCheck::new( + doctor_check_state(&check.status), + &check.name, + &check.message, + ) + }) + .collect(); + let tone = if report.status.eq_ignore_ascii_case("ok") + || report.status.eq_ignore_ascii_case("healthy") + || report.status.eq_ignore_ascii_case("pass") + { + UiTone::Success + } else { + UiTone::Attention + }; + render_document( + &TerminalDocument::new(format!("Trail diagnostics: {}", report.status), tone) + .block(UiBlock::Checklist(checks)), + options, + ) +} + +fn doctor_check_state(status: &str) -> UiCheckState { + match status.to_ascii_lowercase().as_str() { + "ok" | "pass" | "healthy" | "ready" => UiCheckState::Pass, + "warning" | "warn" => UiCheckState::Warn, + "blocked" => UiCheckState::Blocked, + "pending" | "running" => UiCheckState::Pending, + "skip" | "skipped" => UiCheckState::Skip, + _ => UiCheckState::Fail, } - Ok(()) } pub(crate) fn render_backup_create( report: &BackupCreateReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Created backup: {}", report.path); - println!("Branch: {}", report.branch); - println!("Refs: {}", report.ref_count); - println!("Operations: {}", report.operation_count); - println!("SQLite bytes: {}", report.sqlite_bytes); - println!("SQLite SHA-256: {}", report.sqlite_sha256); - if !report.fsck_errors.is_empty() { - println!("FSCK warnings:"); - for error in &report.fsck_errors { - println!(" {error}"); - } - } + let mut document = + TerminalDocument::new("Created backup", UiTone::Success).block(UiBlock::Metadata(vec![ + ("Path".to_string(), report.path.clone()), + ("Branch".to_string(), report.branch.clone()), + ("Refs".to_string(), report.ref_count.to_string()), + ("Operations".to_string(), report.operation_count.to_string()), + ( + "SQLite".to_string(), + format!("{} bytes", report.sqlite_bytes), + ), + ])); + if !report.fsck_errors.is_empty() { + document = document.block(UiBlock::Checklist( + report + .fsck_errors + .iter() + .map(|error| UiCheck::new(UiCheckState::Warn, "Backup integrity", error)) + .collect(), + )); } - Ok(()) + document = document.next( + format!("trail backup verify {}", report.path), + "verify the backup before relying on it", + ); + render_document(&document, options) } pub(crate) fn render_backup_verify( report: &BackupVerifyReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - let status = if report.valid { "valid" } else { "invalid" }; - println!("Backup {status}: {}", report.path); - if let Some(branch) = &report.branch { - println!("Branch: {branch}"); - } - println!( - "Checked {} refs, {} roots, {} text objects", - report.checked_refs, report.checked_roots, report.checked_texts - ); - for error in &report.errors { - println!(" {error}"); - } + let mut document = TerminalDocument::new( + if report.valid { + "Backup verification passed" + } else { + "Backup verification failed" + }, + if report.valid { + UiTone::Success + } else { + UiTone::Failure + }, + ) + .block(UiBlock::Metadata(vec![ + ("Path".to_string(), report.path.clone()), + ("Refs".to_string(), report.checked_refs.to_string()), + ("Roots".to_string(), report.checked_roots.to_string()), + ("Text objects".to_string(), report.checked_texts.to_string()), + ])); + if !report.errors.is_empty() { + document = document.block(UiBlock::Checklist( + report + .errors + .iter() + .map(|error| UiCheck::new(UiCheckState::Fail, "Verification error", error)) + .collect(), + )); } - Ok(()) + render_document(&document, options) } pub(crate) fn render_backup_restore( report: &BackupRestoreReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Restored backup: {}", report.backup_path); - println!("Workspace: {}", report.workspace); - println!("Branch: {}", report.branch); - println!("Replaced existing DB: {}", report.replaced_existing); - println!("Rewritten lane workdirs: {}", report.rewritten_workdirs); - println!( - "Checked {} refs, {} roots, {} text objects", - report.checked_refs, report.checked_roots, report.checked_texts - ); - } - Ok(()) + render_document( + &TerminalDocument::new("Restored backup", UiTone::Success).block(UiBlock::Metadata(vec![ + ("Backup".to_string(), report.backup_path.clone()), + ("Workspace".to_string(), report.workspace.clone()), + ("Branch".to_string(), report.branch.clone()), + ( + "Replaced existing DB".to_string(), + report.replaced_existing.to_string(), + ), + ( + "Rewritten workdirs".to_string(), + report.rewritten_workdirs.to_string(), + ), + ])), + options, + ) } -pub(crate) fn render_fsck(report: &FsckReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_fsck(report: &FsckReport, json: bool, options: &RenderOptions) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Checked {} refs, {} roots, {} text objects", - report.checked_refs, report.checked_roots, report.checked_texts - ); - if report.errors.is_empty() { - println!("No errors"); + let valid = report.errors.is_empty(); + let mut document = TerminalDocument::new( + if valid { + "Integrity check passed".to_string() } else { - for error in &report.errors { - println!(" {error}"); - } - } + format!("Integrity check found {} error(s)", report.errors.len()) + }, + if valid { + UiTone::Success + } else { + UiTone::Failure + }, + ) + .block(UiBlock::Metadata(vec![ + ("Refs".to_string(), report.checked_refs.to_string()), + ("Roots".to_string(), report.checked_roots.to_string()), + ("Text objects".to_string(), report.checked_texts.to_string()), + ])); + if !valid { + document = document.block(UiBlock::Checklist( + report + .errors + .iter() + .map(|error| UiCheck::new(UiCheckState::Fail, "Integrity error", error)) + .collect(), + )); } - Ok(()) + render_document(&document, options) } pub(crate) fn render_index_rebuild( report: &IndexRebuildReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Rebuilt indexes: {} operations, {} parents, {} file rows, {} line rows, {} messages", - report.operations, - report.operation_parents, - report.file_history_rows, - report.line_history_rows, - report.messages - ); - if report.rich_text_hydrated > 0 { - println!("Hydrated {} lazy text object(s)", report.rich_text_hydrated); - } - for error in &report.errors { - println!(" warning: {error}"); - } - } - Ok(()) + render_document(&index_rebuild_document(report), options) +} + +pub(crate) fn render_change_ledger_reconcile( + report: &ChangeLedgerReconcileReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { + if json { + return render_json(report); + } + render_document( + &TerminalDocument::new("Reconciled changed-path ledger", UiTone::Success).block( + UiBlock::Metadata(vec![ + ("Scope".to_string(), report.scope_id.clone()), + ("Scope kind".to_string(), report.scope_kind.clone()), + ("Previous state".to_string(), report.previous_state.clone()), + ("Reason".to_string(), report.reason.clone()), + ( + "Observed paths".to_string(), + report.observed_paths.to_string(), + ), + ("Candidates".to_string(), report.candidates.to_string()), + ( + "Resulting epoch".to_string(), + report.resulting_epoch.to_string(), + ), + ( + "Resulting state".to_string(), + report.resulting_state.clone(), + ), + ]), + ), + options, + ) +} + +fn index_rebuild_document(report: &IndexRebuildReport) -> TerminalDocument { + let mut document = + TerminalDocument::new("Rebuilt indexes", UiTone::Success).block(UiBlock::Metadata(vec![ + ("Operations".to_string(), report.operations.to_string()), + ("Parents".to_string(), report.operation_parents.to_string()), + ( + "File rows".to_string(), + report.file_history_rows.to_string(), + ), + ( + "Line rows".to_string(), + report.line_history_rows.to_string(), + ), + ("Messages".to_string(), report.messages.to_string()), + ( + "Path roots repaired".to_string(), + report.path_index_repaired_roots.len().to_string(), + ), + ( + "Live refs repaired".to_string(), + report.path_index_repaired_refs.len().to_string(), + ), + ])); + if !report.path_index_repaired_refs.is_empty() { + let refs = report + .path_index_repaired_refs + .iter() + .map(|repair| repair.name.as_str()) + .collect::>() + .join(", "); + document = document.block(UiBlock::Notice(format!( + "Upgraded persistent path-invariant indexes for: {refs}" + ))); + } + if report.rich_text_hydrated > 0 { + document = document.block(UiBlock::Notice(format!( + "Hydrated {} lazy text object(s)", + report.rich_text_hydrated + ))); + } + if !report.errors.is_empty() { + document = document.block(UiBlock::Checklist( + report + .errors + .iter() + .map(|error| UiCheck::new(UiCheckState::Warn, "Index warning", error)) + .collect(), + )); + } + document } pub(crate) fn render_worktree_index( report: &WorktreeIndexReport, json: bool, - quiet: bool, + options: &RenderOptions, ) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!( - "Worktree index refreshed: {} files, {} cached entries in {}ms", - report.files, report.indexed_entries, report.duration_ms - ); - } - Ok(()) + render_document( + &TerminalDocument::new("Refreshed worktree index", UiTone::Success).block( + UiBlock::Metadata(vec![ + ("Files".to_string(), report.files.to_string()), + ( + "Cached entries".to_string(), + report.indexed_entries.to_string(), + ), + ("Duration".to_string(), format!("{} ms", report.duration_ms)), + ]), + ), + options, + ) } -pub(crate) fn render_gc(report: &GcReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_gc(report: &GcReport, json: bool, options: &RenderOptions) -> Result<()> { if json { return render_json(report); } - if !quiet { + let mut document = TerminalDocument::new( if report.dry_run { - println!( - "GC dry run: {} prunable of {} known objects ({} reachable, {} unknown preserved)", - report.prunable_objects, - report.total_known_objects, - report.reachable_objects, - report.preserved_unknown_objects - ); + "GC dry run" } else { - println!( - "GC pruned {} objects ({} reachable, {} unknown preserved)", - report.pruned_objects, report.reachable_objects, report.preserved_unknown_objects - ); - } - for error in &report.errors { - println!(" warning: {error}"); - } - } - Ok(()) + "GC complete" + }, + UiTone::Success, + ) + .block(UiBlock::Metadata(vec![ + ( + if report.dry_run { "Prunable" } else { "Pruned" }.to_string(), + if report.dry_run { + report.prunable_objects + } else { + report.pruned_objects + } + .to_string(), + ), + ( + "Reachable".to_string(), + report.reachable_objects.to_string(), + ), + ( + "Unknown preserved".to_string(), + report.preserved_unknown_objects.to_string(), + ), + ])); + if !report.errors.is_empty() { + document = document.block(UiBlock::Checklist( + report + .errors + .iter() + .map(|error| UiCheck::new(UiCheckState::Warn, "GC warning", error)) + .collect(), + )); + } + if report.dry_run { + document = document.next( + "trail gc", + "remove the objects identified by this reviewed dry run", + ); + } + render_document(&document, options) +} + +#[cfg(test)] +mod tests { + use super::*; + use trail::{ChangeId, ObjectId}; + + #[test] + fn index_rebuild_document_reports_path_index_repairs() { + let report = IndexRebuildReport { + path_index_repaired_roots: vec![PathIndexRootRepair { + old_root: ObjectId("old-root".to_string()), + new_root: ObjectId("new-root".to_string()), + case_fold_map_root: "fold-root".to_string(), + }], + path_index_repaired_refs: vec![PathIndexRefRepair { + name: "refs/branches/main".to_string(), + old_change: ChangeId("old-change".to_string()), + new_change: ChangeId("new-change".to_string()), + old_root: ObjectId("old-root".to_string()), + new_root: ObjectId("new-root".to_string()), + }], + ..IndexRebuildReport::default() + }; + + let document = index_rebuild_document(&report); + + let metadata = document + .blocks + .iter() + .find_map(|block| match block { + UiBlock::Metadata(values) => Some(values), + _ => None, + }) + .unwrap(); + assert!(metadata.contains(&("Path roots repaired".to_string(), "1".to_string()))); + assert!(metadata.contains(&("Live refs repaired".to_string(), "1".to_string()))); + assert!(document.blocks.iter().any(|block| { + matches!(block, UiBlock::Notice(text) if text.contains("refs/branches/main")) + })); + } } diff --git a/trail/src/cli/command/render/ui.rs b/trail/src/cli/command/render/ui.rs new file mode 100644 index 0000000..f8dc1cd --- /dev/null +++ b/trail/src/cli/command/render/ui.rs @@ -0,0 +1,1573 @@ +use std::fmt::Write as _; +use std::io::{self, BufWriter, IsTerminal, Write}; +use std::process::{Command, Stdio}; + +use anstyle::{AnsiColor, Color, Style}; +use terminal_size::{terminal_size, Height, Width}; +use time::format_description::well_known::Rfc3339; +use time::OffsetDateTime; +use trail::model::{FileChangeKind, LineChangeKind, OperationKind, WorktreeState}; +use trail::{Error, Result}; +use unicode_width::UnicodeWidthStr; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum RenderMode { + Human, + Plain, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ColorPolicy { + Auto, + Always, + Never, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum PagerPolicy { + Auto, + Always, + Never, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum GlyphSet { + Unicode, + Ascii, +} + +#[derive(Clone, Debug)] +pub(crate) struct RenderOptions { + pub(crate) mode: RenderMode, + pub(crate) color: bool, + pub(crate) glyphs: GlyphSet, + pub(crate) width: usize, + pub(crate) height: usize, + pub(crate) stdout_is_terminal: bool, + pub(crate) stderr_is_terminal: bool, + pub(crate) verbose: bool, + pub(crate) quiet: bool, + pub(crate) pager: PagerPolicy, +} + +impl RenderOptions { + pub(crate) fn from_environment( + mode: RenderMode, + policy: ColorPolicy, + pager: PagerPolicy, + verbose: bool, + quiet: bool, + ) -> Self { + let stdout_is_terminal = io::stdout().is_terminal(); + let stderr_is_terminal = io::stderr().is_terminal(); + let term_is_dumb = std::env::var("TERM") + .map(|term| term.eq_ignore_ascii_case("dumb")) + .unwrap_or(false); + let color = resolve_color( + mode, + policy, + stdout_is_terminal, + term_is_dumb, + std::env::var_os("NO_COLOR").is_some(), + ); + let glyphs = if mode == RenderMode::Human && stdout_is_terminal && !term_is_dumb { + GlyphSet::Unicode + } else { + GlyphSet::Ascii + }; + let (width, height) = terminal_size() + .map(|(Width(width), Height(height))| (usize::from(width), usize::from(height))) + .unwrap_or((80, 24)); + Self { + mode, + color, + glyphs, + width: width.max(24), + height: height.max(8), + stdout_is_terminal, + stderr_is_terminal, + verbose, + quiet, + pager, + } + } + + #[cfg(test)] + pub(crate) fn test(mode: RenderMode, width: usize) -> Self { + Self { + mode, + color: false, + glyphs: if mode == RenderMode::Human { + GlyphSet::Unicode + } else { + GlyphSet::Ascii + }, + width, + height: 24, + stdout_is_terminal: mode == RenderMode::Human, + stderr_is_terminal: mode == RenderMode::Human, + verbose: false, + quiet: false, + pager: PagerPolicy::Never, + } + } + + pub(crate) fn unicode(&self, unicode: &'static str, ascii: &'static str) -> &'static str { + match self.glyphs { + GlyphSet::Unicode => unicode, + GlyphSet::Ascii => ascii, + } + } + + pub(crate) fn progress_allowed(&self) -> bool { + self.mode == RenderMode::Human && self.stderr_is_terminal && !self.quiet + } +} + +fn resolve_color( + mode: RenderMode, + policy: ColorPolicy, + stdout_is_terminal: bool, + term_is_dumb: bool, + no_color: bool, +) -> bool { + match policy { + ColorPolicy::Always => mode == RenderMode::Human, + ColorPolicy::Never => false, + ColorPolicy::Auto => { + mode == RenderMode::Human && stdout_is_terminal && !term_is_dumb && !no_color + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum UiTone { + Success, + Attention, + Blocked, + Failure, + Neutral, + Info, + Muted, +} + +#[derive(Clone, Debug)] +pub(crate) struct TerminalDocument { + pub(crate) lead: Option, + pub(crate) context: Option, + pub(crate) blocks: Vec, + pub(crate) next: Option, + pub(crate) more: Vec, + pub(crate) pager_eligible: bool, +} + +impl TerminalDocument { + pub(crate) fn new(lead: impl Into, tone: UiTone) -> Self { + Self { + lead: Some(UiLead { + text: lead.into(), + tone, + }), + context: None, + blocks: Vec::new(), + next: None, + more: Vec::new(), + pager_eligible: false, + } + } + + pub(crate) fn empty() -> Self { + Self { + lead: None, + context: None, + blocks: Vec::new(), + next: None, + more: Vec::new(), + pager_eligible: false, + } + } + + pub(crate) fn context(mut self, context: impl Into) -> Self { + self.context = Some(context.into()); + self + } + + pub(crate) fn block(mut self, block: UiBlock) -> Self { + self.blocks.push(block); + self + } + + pub(crate) fn next(mut self, command: impl Into, reason: impl Into) -> Self { + self.next = Some(UiNextAction { + command: command.into(), + reason: reason.into(), + }); + self + } + + pub(crate) fn more(mut self, command: impl Into, reason: impl Into) -> Self { + self.more.push(UiNextAction { + command: command.into(), + reason: reason.into(), + }); + self + } + + pub(crate) fn pager_eligible(mut self) -> Self { + self.pager_eligible = true; + self + } +} + +#[derive(Clone, Debug)] +pub(crate) struct UiLead { + pub(crate) text: String, + pub(crate) tone: UiTone, +} + +#[derive(Clone, Debug)] +pub(crate) struct UiNextAction { + pub(crate) command: String, + pub(crate) reason: String, +} + +#[derive(Clone, Debug)] +pub(crate) enum UiBlock { + Paragraph { text: String, tone: UiTone }, + Metadata(Vec<(String, String)>), + Section { title: String, blocks: Vec }, + Table(UiTable), + Changes(UiChangeList), + Checklist(Vec), + Diagnostic(UiDiagnostic), + Notice(String), + Lines(Vec<(String, UiTone)>), + Patch { title: String, text: String }, +} + +impl UiBlock { + pub(crate) fn paragraph(text: impl Into) -> Self { + Self::Paragraph { + text: text.into(), + tone: UiTone::Neutral, + } + } + + pub(crate) fn section(title: impl Into, blocks: Vec) -> Self { + Self::Section { + title: title.into(), + blocks, + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct UiTable { + pub(crate) columns: Vec, + pub(crate) rows: Vec>, +} + +impl UiTable { + pub(crate) fn new(columns: Vec, rows: Vec>) -> Self { + Self { columns, rows } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct UiColumn { + pub(crate) header: String, + pub(crate) priority: u8, + pub(crate) min_width: usize, + pub(crate) align: UiAlign, +} + +impl UiColumn { + pub(crate) fn left(header: impl Into, priority: u8, min_width: usize) -> Self { + Self { + header: header.into(), + priority, + min_width, + align: UiAlign::Left, + } + } + + pub(crate) fn right(header: impl Into, priority: u8, min_width: usize) -> Self { + Self { + header: header.into(), + priority, + min_width, + align: UiAlign::Right, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum UiAlign { + Left, + Right, +} + +#[derive(Clone, Debug)] +pub(crate) struct UiChangeList { + pub(crate) changes: Vec, + pub(crate) additions: u64, + pub(crate) deletions: u64, + pub(crate) omitted: Option, +} + +impl UiChangeList { + pub(crate) fn new(changes: Vec) -> Self { + let additions = changes.iter().map(|change| change.additions).sum(); + let deletions = changes.iter().map(|change| change.deletions).sum(); + Self { + changes, + additions, + deletions, + omitted: None, + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct UiChange { + pub(crate) marker: char, + pub(crate) path: String, + pub(crate) old_path: Option, + pub(crate) additions: u64, + pub(crate) deletions: u64, + pub(crate) detail: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct UiOmitted { + pub(crate) count: usize, + pub(crate) command: String, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum UiCheckState { + Pass, + Warn, + Fail, + Blocked, + Pending, + Skip, +} + +#[derive(Clone, Debug)] +pub(crate) struct UiCheck { + pub(crate) state: UiCheckState, + pub(crate) label: String, + pub(crate) detail: String, +} + +impl UiCheck { + pub(crate) fn new( + state: UiCheckState, + label: impl Into, + detail: impl Into, + ) -> Self { + Self { + state, + label: label.into(), + detail: detail.into(), + } + } +} + +/// Creates a checklist entry from a stable domain status while keeping the +/// renderer's blocker-before-warning ordering shared across command families. +pub(crate) fn check_for_status( + status: &str, + label: impl Into, + detail: impl Into, +) -> UiCheck { + UiCheck::new(check_state_from_status(status), label, detail) +} + +pub(crate) fn check_state_from_status(status: &str) -> UiCheckState { + match status.to_ascii_lowercase().as_str() { + "ok" | "allow" | "allowed" | "pass" | "passed" | "healthy" | "ready" | "info" => { + UiCheckState::Pass + } + "warning" | "warn" | "stale" => UiCheckState::Warn, + "pending" | "approval_required" | "waiting" => UiCheckState::Pending, + "deny" | "denied" | "blocked" | "conflict" | "conflicted" | "error" => { + UiCheckState::Blocked + } + "skip" | "skipped" | "ignored" => UiCheckState::Skip, + _ => UiCheckState::Fail, + } +} + +#[derive(Clone, Debug)] +pub(crate) struct UiDiagnostic { + pub(crate) code: String, + pub(crate) summary: String, + pub(crate) cause: Option, + pub(crate) consequence: Option, + pub(crate) recovery: Option, + pub(crate) alternatives: Vec, +} + +impl UiDiagnostic { + pub(crate) fn new(code: impl Into, summary: impl Into) -> Self { + Self { + code: code.into(), + summary: summary.into(), + cause: None, + consequence: None, + recovery: None, + alternatives: Vec::new(), + } + } +} + +pub(crate) fn render_document(document: &TerminalDocument, options: &RenderOptions) -> Result<()> { + if options.quiet { + return Ok(()); + } + if should_page(document, options) { + let rendered = render_document_bytes(document, options)?; + if should_send_to_pager(&rendered, options) { + match page(&rendered) { + Ok(()) => return Ok(()), + Err(error) if options.verbose => { + let notice = TerminalDocument::empty().block(UiBlock::Notice(format!( + "Pager unavailable ({}); wrote review content directly.", + error + ))); + let _ = render_error_document(¬ice, options); + } + Err(_) => {} + } + } + return write_stdout(&rendered); + } + let stdout = io::stdout(); + let mut writer = BufWriter::new(stdout.lock()); + let result = Renderer::new(&mut writer, options).render(document); + match result { + Ok(()) => match writer.flush() { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::BrokenPipe => Ok(()), + Err(error) => Err(Error::from(error)), + }, + Err(error) if error.kind() == io::ErrorKind::BrokenPipe => Ok(()), + Err(error) => Err(Error::from(error)), + } +} + +fn render_document_bytes(document: &TerminalDocument, options: &RenderOptions) -> Result> { + let mut output = Vec::new(); + Renderer::new(&mut output, options) + .render(document) + .map_err(Error::from)?; + Ok(output) +} + +fn should_page(document: &TerminalDocument, options: &RenderOptions) -> bool { + document.pager_eligible + && options.mode == RenderMode::Human + && options.stdout_is_terminal + && !options.quiet + && !matches!(options.pager, PagerPolicy::Never) +} + +fn should_send_to_pager(output: &[u8], options: &RenderOptions) -> bool { + const MAX_PAGED_BYTES: usize = 16 * 1024 * 1024; + if output.len() > MAX_PAGED_BYTES { + return false; + } + matches!(options.pager, PagerPolicy::Always) + || output.iter().filter(|byte| **byte == b'\n').count() > options.height +} + +fn page(output: &[u8]) -> io::Result<()> { + let pager = std::env::var("PAGER").unwrap_or_else(|_| "less -FRX".to_string()); + let mut parts = pager.split_whitespace(); + let Some(program) = parts.next() else { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "empty PAGER")); + }; + let mut child = Command::new(program) + .args(parts) + .stdin(Stdio::piped()) + .spawn()?; + if let Some(stdin) = child.stdin.as_mut() { + match stdin.write_all(output) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::BrokenPipe => return Ok(()), + Err(error) => return Err(error), + } + } + let _ = child.wait()?; + Ok(()) +} + +fn write_stdout(output: &[u8]) -> Result<()> { + let stdout = io::stdout(); + let mut writer = BufWriter::new(stdout.lock()); + match writer.write_all(output).and_then(|()| writer.flush()) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::BrokenPipe => Ok(()), + Err(error) => Err(Error::from(error)), + } +} + +/// Writes an explicitly raw command payload (for example `lane read`) while +/// retaining the renderer's quiet and broken-pipe behavior. Command adapters +/// must use a semantic document for every other successful result. +pub(crate) fn render_raw_content(content: &str, options: &RenderOptions) -> Result<()> { + if options.quiet { + return Ok(()); + } + write_stdout(content.as_bytes()) +} + +/// Emits a machine protocol acknowledgement. Unlike user-facing raw content, +/// this intentionally ignores `--quiet`: a native agent is waiting for it. +pub(crate) fn render_protocol_content(content: &str) -> Result<()> { + write_stdout(content.as_bytes()) +} + +/// Writes a documented structured diagnostic to stderr without routing it +/// through the human renderer. +pub(crate) fn render_structured_error(content: &str) -> Result<()> { + let stderr = io::stderr(); + let mut writer = BufWriter::new(stderr.lock()); + let result = writer + .write_all(content.as_bytes()) + .and_then(|()| writer.write_all(b"\n")); + match result.and_then(|()| writer.flush()) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::BrokenPipe => Ok(()), + Err(error) => Err(Error::from(error)), + } +} + +pub(crate) fn render_error_document( + document: &TerminalDocument, + options: &RenderOptions, +) -> Result<()> { + let stderr = io::stderr(); + let mut writer = BufWriter::new(stderr.lock()); + let mut error_options = options.clone(); + error_options.quiet = false; + let result = Renderer::new(&mut writer, &error_options).render(document); + match result { + Ok(()) => match writer.flush() { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::BrokenPipe => Ok(()), + Err(error) => Err(Error::from(error)), + }, + Err(error) if error.kind() == io::ErrorKind::BrokenPipe => Ok(()), + Err(error) => Err(Error::from(error)), + } +} + +#[cfg(test)] +pub(crate) fn render_document_to_string( + document: &TerminalDocument, + options: &RenderOptions, +) -> String { + let mut output = Vec::new(); + Renderer::new(&mut output, options) + .render(document) + .expect("writing to Vec cannot fail"); + String::from_utf8(output).expect("renderer emits UTF-8") +} + +pub(crate) fn sanitize_inline(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for character in value.chars() { + match character { + '\u{1b}' => out.push_str("\\x1b"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + character if character.is_control() => { + let _ = write!(out, "\\u{{{:04x}}}", character as u32); + } + character => out.push(character), + } + } + out +} + +pub(crate) fn sanitize_patch(value: &str) -> String { + value + .lines() + .map(|line| sanitize_patch_line(line)) + .collect::>() + .join("\n") +} + +pub(crate) fn format_timestamp(timestamp: i64, options: &RenderOptions) -> String { + format_timestamp_at( + timestamp, + OffsetDateTime::now_utc().unix_timestamp(), + options, + ) +} + +pub(crate) fn format_timestamp_at(timestamp: i64, now: i64, options: &RenderOptions) -> String { + if options.mode == RenderMode::Plain { + return OffsetDateTime::from_unix_timestamp(timestamp) + .ok() + .and_then(|value| value.format(&Rfc3339).ok()) + .unwrap_or_else(|| format!("unix:{timestamp}")); + } + let elapsed = now.saturating_sub(timestamp); + match elapsed { + i64::MIN..=-1 => "just now".to_string(), + 0..=4 => "just now".to_string(), + 5..=59 => format!("{elapsed}s ago"), + 60..=3_599 => format!("{}m ago", elapsed / 60), + 3_600..=86_399 => format!("{}h ago", elapsed / 3_600), + 86_400..=604_799 => format!("{}d ago", elapsed / 86_400), + _ => OffsetDateTime::from_unix_timestamp(timestamp) + .ok() + .and_then(|value| value.format(&Rfc3339).ok()) + .unwrap_or_else(|| format!("unix:{timestamp}")), + } +} + +pub(crate) fn format_duration(duration_ms: u64, options: &RenderOptions) -> String { + if options.mode == RenderMode::Plain { + return format!("{duration_ms} ms"); + } + match duration_ms { + 0..=999 => format!("{duration_ms} ms"), + 1_000..=59_999 => format!("{:.1} s", duration_ms as f64 / 1_000.0), + _ => format!("{:.1} m", duration_ms as f64 / 60_000.0), + } +} + +pub(crate) fn file_change_marker(kind: &FileChangeKind) -> char { + match kind { + FileChangeKind::Added => 'A', + FileChangeKind::Modified => 'M', + FileChangeKind::Deleted => 'D', + FileChangeKind::Renamed => 'R', + FileChangeKind::TypeChanged => 'T', + } +} + +pub(crate) fn file_change_label(kind: &FileChangeKind) -> &'static str { + match kind { + FileChangeKind::Added => "added", + FileChangeKind::Modified => "modified", + FileChangeKind::Deleted => "deleted", + FileChangeKind::Renamed => "renamed", + FileChangeKind::TypeChanged => "type changed", + } +} + +pub(crate) fn line_change_label(kind: &LineChangeKind) -> &'static str { + match kind { + LineChangeKind::Added => "added", + LineChangeKind::Modified => "modified", + LineChangeKind::Deleted => "deleted", + LineChangeKind::Moved => "moved", + } +} + +pub(crate) fn operation_kind_label(kind: &OperationKind) -> &'static str { + match kind { + OperationKind::Init => "initialize", + OperationKind::GitImport => "import from Git", + OperationKind::FileEdit => "edit file", + OperationKind::MultiFileEdit => "edit files", + OperationKind::Format => "format", + OperationKind::ManualCheckpoint => "checkpoint", + OperationKind::ManualRecord => "record", + OperationKind::WatchRecord => "watch record", + OperationKind::Checkout => "checkout", + OperationKind::Branch => "branch", + OperationKind::Merge => "merge", + OperationKind::LaneSpawn => "create lane", + OperationKind::LanePatch => "apply lane patch", + OperationKind::LaneRecord => "record lane", + OperationKind::LaneRewind => "rewind lane", + OperationKind::LaneMerge => "merge lane", + OperationKind::GitExport => "export to Git", + } +} + +pub(crate) fn worktree_state_label(state: &WorktreeState) -> &'static str { + match state { + WorktreeState::Clean => "clean", + WorktreeState::DirtyTracked => "unrecorded changes", + WorktreeState::DirtyUntracked => "unrecorded changes including untracked paths", + } +} + +/// Converts externally stored status strings into stable user-facing phrases. +pub(crate) fn state_label(value: &str) -> String { + match value.to_ascii_lowercase().as_str() { + "ok" | "pass" | "passed" | "healthy" => "passed".to_string(), + "fail" | "failed" | "failure" => "failed".to_string(), + "warn" | "warning" => "warning".to_string(), + "pending" | "approval_required" => "pending approval".to_string(), + "blocked" | "deny" | "denied" => "blocked".to_string(), + "conflicted" | "conflict" => "conflicted".to_string(), + "dirty" => "needs record".to_string(), + "dirty_tracked" => "unrecorded changes".to_string(), + "dirty_untracked" => "unrecorded changes including untracked paths".to_string(), + value => value.replace(['_', '-'], " "), + } +} + +fn sanitize_patch_line(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for character in value.chars() { + match character { + '\u{1b}' => out.push_str("\\x1b"), + '\r' => out.push_str("\\r"), + character if character.is_control() && character != '\t' => { + let _ = write!(out, "\\u{{{:04x}}}", character as u32); + } + character => out.push(character), + } + } + out +} + +struct Renderer<'a, W: Write> { + writer: &'a mut W, + options: &'a RenderOptions, +} + +impl<'a, W: Write> Renderer<'a, W> { + fn new(writer: &'a mut W, options: &'a RenderOptions) -> Self { + Self { writer, options } + } + + fn render(&mut self, document: &TerminalDocument) -> io::Result<()> { + let mut wrote = false; + if let Some(lead) = &document.lead { + self.line(&sanitize_inline(&lead.text), lead.tone, true)?; + wrote = true; + } + if let Some(context) = &document.context { + self.line(&sanitize_inline(context), UiTone::Muted, false)?; + wrote = true; + } + for block in &document.blocks { + if wrote { + self.blank()?; + } + self.block(block, 0)?; + wrote = true; + } + if let Some(next) = &document.next { + if wrote { + self.blank()?; + } + self.heading("Next:")?; + self.command(next, 2)?; + wrote = true; + } + if !document.more.is_empty() && self.options.verbose { + if wrote { + self.blank()?; + } + self.heading("More:")?; + for action in &document.more { + self.command(action, 2)?; + } + } + Ok(()) + } + + fn block(&mut self, block: &UiBlock, indent: usize) -> io::Result<()> { + match block { + UiBlock::Paragraph { text, tone } => { + self.line_indented(&sanitize_inline(text), *tone, false, indent) + } + UiBlock::Metadata(entries) => self.metadata(entries, indent), + UiBlock::Section { title, blocks } => { + self.heading_indented(title, indent)?; + for (index, block) in blocks.iter().enumerate() { + if index > 0 { + self.blank()?; + } + self.block(block, indent + 2)?; + } + Ok(()) + } + UiBlock::Table(table) => self.table(table, indent), + UiBlock::Changes(changes) => self.changes(changes, indent), + UiBlock::Checklist(checks) => self.checklist(checks, indent), + UiBlock::Diagnostic(diagnostic) => self.diagnostic(diagnostic, indent), + UiBlock::Notice(text) => { + self.line_indented(&sanitize_inline(text), UiTone::Attention, false, indent) + } + UiBlock::Lines(lines) => { + for (line, tone) in lines { + self.line_indented(&sanitize_inline(line), *tone, false, indent)?; + } + Ok(()) + } + UiBlock::Patch { title, text } => self.patch(title, text, indent), + } + } + + fn metadata(&mut self, entries: &[(String, String)], indent: usize) -> io::Result<()> { + let label_width = entries + .iter() + .map(|(label, _)| display_width(&sanitize_inline(label))) + .max() + .unwrap_or_default(); + for (label, value) in entries { + let label = sanitize_inline(label); + let value = sanitize_inline(value); + let padding = " ".repeat(label_width.saturating_sub(display_width(&label))); + let line = format!("{label}{padding}: {value}"); + self.line_indented(&line, UiTone::Neutral, false, indent)?; + } + Ok(()) + } + + fn table(&mut self, table: &UiTable, indent: usize) -> io::Result<()> { + if table.columns.is_empty() || table.rows.is_empty() { + return Ok(()); + } + let columns = visible_columns(table, self.options.width.saturating_sub(indent)); + if columns.len() < 2 { + return self.stacked_table(table, &[], indent); + } + let available = self.options.width.saturating_sub(indent); + let widths = column_widths(table, &columns, available); + if widths.iter().sum::() + widths.len().saturating_sub(1) * 2 > available { + return self.stacked_table(table, &columns, indent); + } + let headers = columns + .iter() + .zip(&widths) + .map(|(index, width)| { + pad( + &sanitize_inline(&table.columns[*index].header), + *width, + table.columns[*index].align, + ) + }) + .collect::>() + .join(" "); + self.line_indented(&headers, UiTone::Muted, false, indent)?; + for row in &table.rows { + let rendered = columns + .iter() + .zip(&widths) + .map(|(index, width)| { + let value = row.get(*index).map(String::as_str).unwrap_or(""); + let value = truncate(&sanitize_inline(value), *width, self.options); + pad(&value, *width, table.columns[*index].align) + }) + .collect::>() + .join(" "); + self.line_indented(&rendered, UiTone::Neutral, false, indent)?; + } + Ok(()) + } + + fn stacked_table( + &mut self, + table: &UiTable, + columns: &[usize], + indent: usize, + ) -> io::Result<()> { + let columns = if columns.is_empty() { + (0..table.columns.len()).collect::>() + } else { + columns.to_vec() + }; + for (row_index, row) in table.rows.iter().enumerate() { + if row_index > 0 { + self.blank()?; + } + for index in &columns { + let value = row.get(*index).map(String::as_str).unwrap_or(""); + let line = format!( + "{}: {}", + sanitize_inline(&table.columns[*index].header), + sanitize_inline(value) + ); + self.line_indented(&line, UiTone::Neutral, false, indent)?; + } + } + Ok(()) + } + + fn changes(&mut self, changes: &UiChangeList, indent: usize) -> io::Result<()> { + for change in &changes.changes { + let mut line = format!("{} ", change.marker); + let path = match &change.old_path { + Some(old_path) => format!( + "{} {} {}", + sanitize_inline(old_path), + self.options.unicode("→", "->"), + sanitize_inline(&change.path) + ), + None => sanitize_inline(&change.path), + }; + line.push_str(&path); + if change.additions > 0 || change.deletions > 0 { + let _ = write!(line, " +{} -{}", change.additions, change.deletions); + if self.options.width >= indent + 48 { + line.push_str(" "); + line.push_str(&change_bar(change.additions, change.deletions)); + } + } + if let Some(detail) = &change.detail { + let _ = write!(line, " {}", sanitize_inline(detail)); + } + self.line_indented( + &truncate( + &line, + self.options.width.saturating_sub(indent), + self.options, + ), + UiTone::Neutral, + false, + indent, + )?; + } + let summary = format!( + "{} file{} changed, {} insertion{}, {} deletion{}", + changes.changes.len(), + if changes.changes.len() == 1 { "" } else { "s" }, + changes.additions, + if changes.additions == 1 { "" } else { "s" }, + changes.deletions, + if changes.deletions == 1 { "" } else { "s" } + ); + self.line_indented(&summary, UiTone::Muted, false, indent)?; + if let Some(omitted) = &changes.omitted { + let message = format!( + "{} {} more; run `{}`", + self.options.unicode("…", "..."), + omitted.count, + sanitize_inline(&omitted.command) + ); + self.line_indented(&message, UiTone::Attention, false, indent)?; + } + Ok(()) + } + + fn checklist(&mut self, checks: &[UiCheck], indent: usize) -> io::Result<()> { + let mut checks = checks.to_vec(); + checks.sort_by(|left, right| check_rank(left.state).cmp(&check_rank(right.state))); + let label_width = checks + .iter() + .map(|check| display_width(&sanitize_inline(&check.label))) + .max() + .unwrap_or_default(); + for check in checks { + let status = check_state_label(check.state); + let label = sanitize_inline(&check.label); + let padding = " ".repeat(label_width.saturating_sub(display_width(&label))); + let line = format!( + "{status:<7} {label}{padding} {}", + sanitize_inline(&check.detail) + ); + self.line_indented(&line, check_state_tone(check.state), false, indent)?; + } + Ok(()) + } + + fn diagnostic(&mut self, diagnostic: &UiDiagnostic, indent: usize) -> io::Result<()> { + self.line_indented( + &format!( + "error [{}]: {}", + sanitize_inline(&diagnostic.code), + sanitize_inline(&diagnostic.summary) + ), + UiTone::Failure, + true, + indent, + )?; + if let Some(cause) = &diagnostic.cause { + self.multiline_indented(cause, UiTone::Neutral, indent)?; + } + if let Some(consequence) = &diagnostic.consequence { + self.multiline_indented(consequence, UiTone::Attention, indent)?; + } + if let Some(recovery) = &diagnostic.recovery { + self.blank()?; + self.heading_indented("Next:", indent)?; + self.command_indented(recovery, indent + 2)?; + } + for alternative in &diagnostic.alternatives { + self.command_indented(alternative, indent + 2)?; + } + Ok(()) + } + + fn patch(&mut self, title: &str, text: &str, indent: usize) -> io::Result<()> { + self.line_indented(&sanitize_inline(title), UiTone::Info, true, indent)?; + let rule = self + .options + .unicode("─", "-") + .repeat(self.options.width.saturating_sub(indent).min(72).max(8)); + self.line_indented(&rule, UiTone::Muted, false, indent)?; + for line in sanitize_patch(text).lines() { + let tone = if line.starts_with('+') && !line.starts_with("+++") { + UiTone::Success + } else if line.starts_with('-') && !line.starts_with("---") { + UiTone::Failure + } else if line.starts_with("@@") { + UiTone::Info + } else { + UiTone::Neutral + }; + self.line_indented(line, tone, false, indent)?; + } + Ok(()) + } + + fn heading(&mut self, value: &str) -> io::Result<()> { + self.line(value, UiTone::Info, true) + } + + fn heading_indented(&mut self, value: &str, indent: usize) -> io::Result<()> { + self.line_indented(value, UiTone::Info, true, indent) + } + + fn command(&mut self, action: &UiNextAction, indent: usize) -> io::Result<()> { + self.command_indented(action, indent) + } + + fn command_indented(&mut self, action: &UiNextAction, indent: usize) -> io::Result<()> { + self.line_indented( + &sanitize_inline(&action.command), + UiTone::Info, + false, + indent, + )?; + self.line_indented( + &sanitize_inline(&action.reason), + UiTone::Neutral, + false, + indent + 2, + ) + } + + fn blank(&mut self) -> io::Result<()> { + self.writer.write_all(b"\n") + } + + fn line(&mut self, value: &str, tone: UiTone, strong: bool) -> io::Result<()> { + self.line_indented(value, tone, strong, 0) + } + + fn line_indented( + &mut self, + value: &str, + tone: UiTone, + strong: bool, + indent: usize, + ) -> io::Result<()> { + self.writer.write_all(" ".repeat(indent).as_bytes())?; + let rendered = self.style(value, tone, strong); + self.writer.write_all(rendered.as_bytes())?; + self.writer.write_all(b"\n") + } + + fn multiline_indented(&mut self, value: &str, tone: UiTone, indent: usize) -> io::Result<()> { + for line in value.split('\n') { + self.line_indented(&sanitize_inline(line), tone, false, indent)?; + } + Ok(()) + } + + fn style(&self, value: &str, tone: UiTone, strong: bool) -> String { + if !self.options.color { + return value.to_string(); + } + let color = match tone { + UiTone::Success => AnsiColor::Green, + UiTone::Attention => AnsiColor::Yellow, + UiTone::Blocked | UiTone::Failure => AnsiColor::Red, + UiTone::Info => AnsiColor::Cyan, + UiTone::Muted => AnsiColor::BrightBlack, + UiTone::Neutral => { + return if strong { + format!( + "{}{}{}", + Style::new().bold().render(), + value, + Style::new().bold().render_reset() + ) + } else { + value.to_string() + }; + } + }; + let style = if strong { + Style::new().bold().fg_color(Some(Color::Ansi(color))) + } else { + Style::new().fg_color(Some(Color::Ansi(color))) + }; + format!("{}{}{}", style.render(), value, style.render_reset()) + } +} + +fn visible_columns(table: &UiTable, available: usize) -> Vec { + let mut columns = (0..table.columns.len()).collect::>(); + loop { + let minimum = columns + .iter() + .map(|index| table.columns[*index].min_width) + .sum::() + + columns.len().saturating_sub(1) * 2; + if minimum <= available || columns.len() <= 1 { + return columns; + } + let removable = columns + .iter() + .enumerate() + .filter(|(_, index)| table.columns[**index].priority > 0) + .max_by_key(|(_, index)| table.columns[**index].priority) + .map(|(position, _)| position); + match removable { + Some(position) => { + columns.remove(position); + } + None => return columns, + } + } +} + +fn column_widths(table: &UiTable, columns: &[usize], available: usize) -> Vec { + let gaps = columns.len().saturating_sub(1) * 2; + let mut widths = columns + .iter() + .map(|index| { + let header = display_width(&sanitize_inline(&table.columns[*index].header)); + let content = table + .rows + .iter() + .map(|row| row.get(*index).map(String::as_str).unwrap_or("")) + .map(sanitize_inline) + .map(|value| display_width(&value)) + .max() + .unwrap_or_default(); + header.max(content).max(table.columns[*index].min_width) + }) + .collect::>(); + let target = available.saturating_sub(gaps); + let current = widths.iter().sum::(); + if current > target { + let mut positions = (0..widths.len()).collect::>(); + positions.sort_by(|left, right| { + table.columns[columns[*right]] + .priority + .cmp(&table.columns[columns[*left]].priority) + .then_with(|| widths[*right].cmp(&widths[*left])) + }); + let mut excess = current - target; + for position in positions { + let minimum = table.columns[columns[position]].min_width; + let reducible = widths[position].saturating_sub(minimum); + let reduction = reducible.min(excess); + widths[position] -= reduction; + excess -= reduction; + if excess == 0 { + break; + } + } + } + widths +} + +fn pad(value: &str, width: usize, align: UiAlign) -> String { + let value_width = display_width(value); + let padding = " ".repeat(width.saturating_sub(value_width)); + match align { + UiAlign::Left => format!("{value}{padding}"), + UiAlign::Right => format!("{padding}{value}"), + } +} + +fn truncate(value: &str, max_width: usize, options: &RenderOptions) -> String { + if display_width(value) <= max_width { + return value.to_string(); + } + let marker = options.unicode("…", "..."); + let target = max_width.saturating_sub(display_width(marker)); + let mut output = String::new(); + let mut width = 0; + for character in value.chars() { + let character_width = unicode_width::UnicodeWidthChar::width(character).unwrap_or(0); + if width + character_width > target { + break; + } + output.push(character); + width += character_width; + } + output.push_str(marker); + output +} + +fn display_width(value: &str) -> usize { + UnicodeWidthStr::width(value) +} + +fn change_bar(additions: u64, deletions: u64) -> String { + let total = additions + deletions; + if total == 0 { + return String::new(); + } + let width = total.min(20) as usize; + let plus = usize::try_from((additions * width as u64 + total - 1) / total).unwrap_or(width); + format!( + "{}{}", + "+".repeat(plus), + "-".repeat(width.saturating_sub(plus)) + ) +} + +fn check_rank(state: UiCheckState) -> u8 { + match state { + UiCheckState::Blocked => 0, + UiCheckState::Fail => 1, + UiCheckState::Warn => 2, + UiCheckState::Pending => 3, + UiCheckState::Pass => 4, + UiCheckState::Skip => 5, + } +} + +fn check_state_label(state: UiCheckState) -> &'static str { + match state { + UiCheckState::Pass => "PASS", + UiCheckState::Warn => "WARN", + UiCheckState::Fail => "FAIL", + UiCheckState::Blocked => "BLOCKED", + UiCheckState::Pending => "PENDING", + UiCheckState::Skip => "SKIP", + } +} + +fn check_state_tone(state: UiCheckState) -> UiTone { + match state { + UiCheckState::Pass => UiTone::Success, + UiCheckState::Warn | UiCheckState::Pending => UiTone::Attention, + UiCheckState::Fail => UiTone::Failure, + UiCheckState::Blocked => UiTone::Blocked, + UiCheckState::Skip => UiTone::Muted, + } +} + +pub(crate) struct TransientProgress { + active: bool, +} + +impl TransientProgress { + pub(crate) fn start(options: &RenderOptions, message: &str) -> Self { + if !options.progress_allowed() { + return Self { active: false }; + } + let mut stderr = io::stderr().lock(); + let _ = write!(stderr, "\r{}", sanitize_inline(message)); + let _ = stderr.flush(); + Self { active: true } + } + + pub(crate) fn finish(&mut self) { + if !self.active { + return; + } + let mut stderr = io::stderr().lock(); + let _ = write!(stderr, "\r\x1b[2K"); + let _ = stderr.flush(); + self.active = false; + } +} + +impl Drop for TransientProgress { + fn drop(&mut self) { + self.finish(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sanitizes_terminal_controls() { + assert_eq!( + sanitize_inline("one\u{1b}[31m\ntwo\t"), + "one\\x1b[31m\\ntwo\\t" + ); + assert_eq!( + sanitize_patch("+ one\u{1b}[0m\n- two"), + "+ one\\x1b[0m\n- two" + ); + } + + #[test] + fn table_stacks_when_narrow() { + let document = + TerminalDocument::new("Timeline", UiTone::Neutral).block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("OPERATION", 0, 12), + UiColumn::left("BRANCH", 1, 8), + UiColumn::left("MESSAGE", 2, 16), + ], + vec![vec![ + "change_alpha".into(), + "main".into(), + "Improve rendering".into(), + ]], + ))); + let output = + render_document_to_string(&document, &RenderOptions::test(RenderMode::Plain, 21)); + assert!(output.contains("OPERATION: change_alpha")); + assert!(output.contains("BRANCH: main")); + } + + #[test] + fn checklist_orders_blockers_before_warnings() { + let document = TerminalDocument::empty().block(UiBlock::Checklist(vec![ + UiCheck::new(UiCheckState::Pass, "tests", "cargo test"), + UiCheck::new(UiCheckState::Warn, "freshness", "behind main"), + UiCheck::new(UiCheckState::Blocked, "worktree", "dirty"), + ])); + let output = + render_document_to_string(&document, &RenderOptions::test(RenderMode::Plain, 80)); + assert!(output.find("BLOCKED").unwrap() < output.find("WARN").unwrap()); + assert!(output.find("WARN").unwrap() < output.find("PASS").unwrap()); + } + + #[test] + fn terminal_layout_goldens_cover_widths_and_policies() { + let document = TerminalDocument::new("Lane review is blocked", UiTone::Blocked) + .context("2 blockers · 1 warning") + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("LANE", 0, 10), + UiColumn::left("STATE", 0, 9), + UiColumn::left("DETAIL", 2, 18), + ], + vec![ + vec!["fix-login".into(), "blocked".into(), "dirty workdir".into()], + vec!["docs".into(), "warning".into(), "base is stale".into()], + ], + ))) + .block(UiBlock::Checklist(vec![ + UiCheck::new(UiCheckState::Warn, "base", "3 operations behind"), + UiCheck::new(UiCheckState::Blocked, "workdir", "record local changes"), + ])) + .next( + "trail lane record fix-login -m \"save local work\"", + "record the highest-priority blocker before merging", + ); + let narrow = + render_document_to_string(&document, &RenderOptions::test(RenderMode::Human, 30)); + let standard = + render_document_to_string(&document, &RenderOptions::test(RenderMode::Human, 70)); + let wide = + render_document_to_string(&document, &RenderOptions::test(RenderMode::Human, 110)); + let plain = + render_document_to_string(&document, &RenderOptions::test(RenderMode::Plain, 70)); + let narrow_expected = concat!( + "Lane review is blocked\n", + "2 blockers · 1 warning\n\n", + "LANE STATE \n", + "fix-login blocked \n", + "docs warning \n\n", + "BLOCKED workdir record local changes\n", + "WARN base 3 operations behind\n\n", + "Next:\n", + " trail lane record fix-login -m \"save local work\"\n", + " record the highest-priority blocker before merging\n", + ); + let standard_expected = concat!( + "Lane review is blocked\n", + "2 blockers · 1 warning\n\n", + "LANE STATE DETAIL \n", + "fix-login blocked dirty workdir \n", + "docs warning base is stale \n\n", + "BLOCKED workdir record local changes\n", + "WARN base 3 operations behind\n\n", + "Next:\n", + " trail lane record fix-login -m \"save local work\"\n", + " record the highest-priority blocker before merging\n", + ); + assert_eq!(narrow, narrow_expected); + assert_eq!(standard, standard_expected); + assert_eq!(wide, standard_expected); + assert_eq!(plain, standard_expected); + + let glyph_document = TerminalDocument::new("Rename", UiTone::Neutral).block( + UiBlock::Changes(UiChangeList::new(vec![UiChange { + marker: 'R', + path: "new-name.rs".to_string(), + old_path: Some("old-name.rs".to_string()), + additions: 1, + deletions: 1, + detail: None, + }])), + ); + let mut colored = RenderOptions::test(RenderMode::Human, 80); + colored.color = true; + let mut ascii = RenderOptions::test(RenderMode::Human, 80); + ascii.glyphs = GlyphSet::Ascii; + assert!(render_document_to_string(&document, &colored).contains("\u{1b}[")); + assert!(render_document_to_string(&glyph_document, &ascii) + .contains("old-name.rs -> new-name.rs")); + assert!(render_document_to_string( + &glyph_document, + &RenderOptions::test(RenderMode::Human, 80), + ) + .contains("old-name.rs → new-name.rs")); + } + + #[test] + fn next_action_reason_uses_normal_text_color() { + let document = TerminalDocument::new("Invalid input", UiTone::Failure).next( + "trail --help", + "Review the command syntax and available options.", + ); + let mut options = RenderOptions::test(RenderMode::Human, 80); + options.color = true; + let output = render_document_to_string(&document, &options); + assert!(output.contains("Review the command syntax and available options.")); + assert!(!output.contains("\u{1b}[90mReview the command syntax and available options.")); + } + + /// An opt-in release benchmark for the renderer cutover. It deliberately + /// stays dependency-free so it measures the production renderer rather + /// than a benchmark framework's setup costs. + #[test] + #[ignore = "run with scripts/terminal-output-bench.sh before release"] + fn terminal_render_baseline() { + use std::time::{Duration, Instant}; + + let table = UiTable::new( + vec![ + UiColumn::left("TASK", 0, 12), + UiColumn::left("STATE", 0, 10), + UiColumn::left("DETAIL", 1, 18), + ], + (0..10_000) + .map(|index| { + vec![ + format!("task-{index:05}"), + "needs review".to_string(), + "rendered table benchmark row".to_string(), + ] + }) + .collect(), + ); + let table_document = + TerminalDocument::new("Table benchmark", UiTone::Neutral).block(UiBlock::Table(table)); + let options = RenderOptions::test(RenderMode::Plain, 120); + let table_start = Instant::now(); + let table_output = render_document_to_string(&table_document, &options); + let table_elapsed = table_start.elapsed(); + assert!(table_output.lines().count() >= 10_000); + assert!( + table_elapsed < Duration::from_millis(750), + "10,000-row render took {table_elapsed:?}; threshold is 750ms" + ); + + let patch = + "+ benchmark source line with enough content to exercise sanitization\n".repeat(32_768); + let patch_document = + TerminalDocument::new("Patch benchmark", UiTone::Neutral).block(UiBlock::Patch { + title: "benchmark.rs".to_string(), + text: patch, + }); + let patch_start = Instant::now(); + let patch_output = render_document_to_string(&patch_document, &options); + let patch_elapsed = patch_start.elapsed(); + assert!(patch_output.len() > 1_000_000); + assert!( + patch_elapsed < Duration::from_millis(1_500), + "large patch render took {patch_elapsed:?}; threshold is 1.5s" + ); + eprintln!( + "terminal rendering baseline: 10k table={table_elapsed:?}, large patch={patch_elapsed:?}" + ); + } + + #[test] + fn plain_timestamps_are_rfc3339_and_human_timestamps_are_relative() { + let plain = RenderOptions::test(RenderMode::Plain, 80); + let human = RenderOptions::test(RenderMode::Human, 80); + assert_eq!(format_timestamp_at(0, 60, &plain), "1970-01-01T00:00:00Z"); + assert_eq!(format_timestamp_at(0, 60, &human), "1m ago"); + } + + #[test] + fn plain_durations_are_integer_milliseconds() { + let plain = RenderOptions::test(RenderMode::Plain, 80); + let human = RenderOptions::test(RenderMode::Human, 80); + assert_eq!(format_duration(1_250, &plain), "1250 ms"); + assert_eq!(format_duration(1_250, &human), "1.2 s"); + } + + #[test] + fn maps_model_states_to_user_facing_labels() { + assert_eq!( + worktree_state_label(&WorktreeState::DirtyTracked), + "unrecorded changes" + ); + assert_eq!(state_label("dirty"), "needs record"); + assert_eq!(state_label("approval_required"), "pending approval"); + } + + #[test] + fn color_policy_honors_terminal_capabilities() { + assert!(resolve_color( + RenderMode::Human, + ColorPolicy::Auto, + true, + false, + false + )); + assert!(!resolve_color( + RenderMode::Human, + ColorPolicy::Auto, + false, + false, + false + )); + assert!(!resolve_color( + RenderMode::Human, + ColorPolicy::Auto, + true, + true, + false + )); + assert!(!resolve_color( + RenderMode::Human, + ColorPolicy::Auto, + true, + false, + true + )); + assert!(resolve_color( + RenderMode::Human, + ColorPolicy::Always, + false, + true, + true + )); + } + + #[test] + fn pager_policy_requires_long_interactive_output() { + let mut options = RenderOptions::test(RenderMode::Human, 80); + options.height = 3; + options.pager = PagerPolicy::Auto; + assert!(should_send_to_pager(b"one\ntwo\nthree\nfour\n", &options)); + assert!(!should_send_to_pager(b"one\ntwo\n", &options)); + options.pager = PagerPolicy::Always; + assert!(should_send_to_pager(b"one\n", &options)); + } +} diff --git a/trail/src/cli/command/render/workspace.rs b/trail/src/cli/command/render/workspace.rs index 2f100eb..6311df0 100644 --- a/trail/src/cli/command/render/workspace.rs +++ b/trail/src/cli/command/render/workspace.rs @@ -1,118 +1,330 @@ -use super::render_json; +use super::*; use trail::model::*; use trail::{Result, WorktreeState}; -pub(crate) fn render_init(report: &InitReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_init(report: &InitReport, json: bool, options: &RenderOptions) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Initialized Trail workspace"); - println!("Workspace: {}", report.workspace_id.0); - println!("Branch: {}", report.branch); - println!("Initial operation: {}", report.operation.0); - println!( - "Imported: {} files ({} text, {} opaque, {} binary)", + let mut document = TerminalDocument::new("Initialized Trail workspace", UiTone::Success) + .context(format!("Trail branch {}", report.branch)) + .block(UiBlock::paragraph(format!( + "Imported {} file{}: {} text, {} opaque, {} binary", report.imported.files, + plural(report.imported.files), report.imported.text, report.imported.opaque, report.imported.binary - ); + ))); + if options.verbose { + document = document.block(UiBlock::Metadata(vec![ + ("Workspace".to_string(), report.workspace_id.0.clone()), + ("Initial operation".to_string(), report.operation.0.clone()), + ("Root".to_string(), report.root_id.0.clone()), + ])); } - Ok(()) + render_document(&document, options) } -pub(crate) fn render_status(report: &StatusReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_status( + report: &StatusReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - println!("Branch: {}", report.branch); - println!("Head: {}", report.head.change_id.0); - println!("Root: {}", report.head.root_id.0); - println!( - "Worktree: {}", - match report.worktree_state { - WorktreeState::Clean => "clean", - WorktreeState::DirtyTracked => "dirty", - WorktreeState::DirtyUntracked => "dirty with untracked paths", - } - ); - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } - if !report.suggestions.is_empty() { - println!("Suggested next steps:"); - for suggestion in &report.suggestions { - println!(" {}", suggestion.command); - println!(" {}", suggestion.reason); - } + render_document(&status_document(report, options), options) +} + +fn status_document(report: &StatusReport, options: &RenderOptions) -> TerminalDocument { + let changed = report.changed_paths.len(); + let (lead, tone, context) = match report.worktree_state { + WorktreeState::Clean => ( + "Worktree clean".to_string(), + UiTone::Success, + format!("Trail branch {}", report.branch), + ), + WorktreeState::DirtyTracked => ( + format!( + "Worktree has {changed} unrecorded change{}", + plural(changed as u64) + ), + UiTone::Attention, + format!("Trail branch {}", report.branch), + ), + WorktreeState::DirtyUntracked => ( + format!( + "Worktree has {changed} unrecorded change{}, including untracked paths", + plural(changed as u64) + ), + UiTone::Attention, + format!("Trail branch {}", report.branch), + ), + }; + let mut document = TerminalDocument::new(lead, tone).context(context); + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::section( + "Changes:", + vec![UiBlock::Changes(change_list(&report.changed_paths))], + )); + } + if options.verbose { + document = document.block(UiBlock::Metadata(vec![ + ("Head".to_string(), report.head.change_id.0.clone()), + ("Root".to_string(), report.head.root_id.0.clone()), + ("Ref".to_string(), report.head.name.clone()), + ])); + } + if !matches!(report.worktree_state, WorktreeState::Clean) { + let primary = report + .suggestions + .first() + .map(|suggestion| (suggestion.command.clone(), suggestion.reason.clone())); + let (command, reason) = primary.unwrap_or_else(|| { + ( + "trail diff --dirty".to_string(), + "Review the unrecorded changes.".to_string(), + ) + }); + document = document.next(command, reason); + for suggestion in report.suggestions.iter().skip(1) { + document = document.more(suggestion.command.clone(), suggestion.reason.clone()); } } - Ok(()) + document } -pub(crate) fn render_record(report: &RecordReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_record( + report: &RecordReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { - match &report.operation { - Some(change) => { - println!("Recorded {}", change.0); - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); - } + let document = match &report.operation { + Some(operation) => { + let mut document = TerminalDocument::new( + format!( + "Recorded {} change{} on {}", + report.changed_paths.len(), + plural(report.changed_paths.len() as u64), + report.branch + ), + UiTone::Success, + ) + .block(UiBlock::Changes(change_list(&report.changed_paths))); + if options.verbose { + document = document.block(UiBlock::Metadata(vec![ + ("Operation".to_string(), operation.0.clone()), + ("Root".to_string(), report.root_id.0.clone()), + ])); } - None => println!("No changes to record"), + document } - } - Ok(()) + None => TerminalDocument::new("No changes to record", UiTone::Neutral), + }; + render_document(&document, options) } -pub(crate) fn render_timeline(entries: &[TimelineEntry], json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_timeline( + entries: &[TimelineEntry], + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(&entries); } - if !quiet { - for entry in entries { - let message = entry.message.as_deref().unwrap_or(""); - println!( - "{} {:?} {} {}", - entry.change_id.0, entry.kind, entry.branch, message - ); - } + if entries.is_empty() { + return render_document( + &TerminalDocument::new("No operations recorded", UiTone::Neutral), + options, + ); + } + let rows = entries + .iter() + .map(|entry| { + vec![ + format_timestamp(entry.created_at, options), + operation_kind_label(&entry.kind).to_string(), + entry.branch.clone(), + entry.path_count.to_string(), + entry.message.clone().unwrap_or_else(|| "—".to_string()), + ] + }) + .collect(); + let mut document = TerminalDocument::new( + format!( + "{} recent operation{}", + entries.len(), + plural(entries.len() as u64) + ), + UiTone::Neutral, + ) + .block(UiBlock::Table(UiTable::new( + vec![ + UiColumn::left("WHEN", 2, 20), + UiColumn::left("KIND", 0, 15), + UiColumn::left("BRANCH", 1, 8), + UiColumn::right("PATHS", 2, 5), + UiColumn::left("MESSAGE", 3, 16), + ], + rows, + ))); + if options.verbose { + document = document.block(UiBlock::section( + "Operation selectors:", + vec![UiBlock::Lines( + entries + .iter() + .map(|entry| (entry.change_id.0.clone(), UiTone::Muted)) + .collect(), + )], + )); } - Ok(()) + render_document(&document, options) } -pub(crate) fn render_checkout(report: &CheckoutReport, json: bool, quiet: bool) -> Result<()> { +pub(crate) fn render_checkout( + report: &CheckoutReport, + json: bool, + options: &RenderOptions, +) -> Result<()> { if json { return render_json(report); } - if !quiet { + let lead = if report.dry_run { + format!( + "Checkout preview: {} path{} would change", + report.changed_paths.len(), + plural(report.changed_paths.len() as u64) + ) + } else { + format!( + "Checked out {} file{}", + report.written_files, + plural(report.written_files) + ) + }; + let mut document = TerminalDocument::new( + lead, if report.dry_run { - println!( - "Would check out {} ({} changed paths)", - report.change_id.0, - report.changed_paths.len() - ); + UiTone::Attention } else { - println!( - "Checked out {} ({} files)", - report.change_id.0, report.written_files - ); - } - if let Some(output_root) = &report.output_root { - println!("Output: {output_root}"); - } - if let Some(recorded) = &report.recorded_dirty { - println!("Recorded dirty worktree: {}", recorded.0); + UiTone::Success + }, + ); + if !report.changed_paths.is_empty() { + document = document.block(UiBlock::Changes(change_list(&report.changed_paths))); + } + if let Some(output_root) = &report.output_root { + document = document.block(UiBlock::Metadata(vec![( + "Output".to_string(), + output_root.clone(), + )])); + } + if let Some(recorded) = &report.recorded_dirty { + document = document.block(UiBlock::Notice(format!( + "Recorded existing worktree changes as {} before checkout.", + recorded.0 + ))); + } + if options.verbose { + document = document.block(UiBlock::Metadata(vec![ + ("Operation".to_string(), report.change_id.0.clone()), + ("Root".to_string(), report.root_id.0.clone()), + ])); + } + render_document(&document, options) +} + +pub(crate) fn change_list(paths: &[FileDiffSummary]) -> UiChangeList { + UiChangeList::new( + paths + .iter() + .map(|path| UiChange { + marker: file_change_marker(&path.kind), + path: path.path.clone(), + old_path: path.old_path.clone(), + additions: path.additions, + deletions: path.deletions, + detail: None, + }) + .collect(), + ) +} + +fn plural(value: u64) -> &'static str { + if value == 1 { + "" + } else { + "s" + } +} + +#[cfg(test)] +mod tests { + use super::*; + use trail::{ChangeId, ObjectId}; + + fn status_fixture(state: WorktreeState, paths: Vec) -> StatusReport { + StatusReport { + branch: "main".to_string(), + head: RefRecord { + name: "refs/heads/main".to_string(), + change_id: ChangeId("change_fixture".to_string()), + root_id: ObjectId("root_fixture".to_string()), + operation_id: ObjectId("operation_fixture".to_string()), + generation: 1, + updated_at: 0, + }, + worktree_state: state, + changed_paths: paths, + suggestions: vec![StatusSuggestion { + command: "trail diff --dirty".to_string(), + reason: "Review the unrecorded changes.".to_string(), + }], } - for path in &report.changed_paths { - println!(" {:?} {}", path.kind, path.path); + } + + fn changed_path(kind: FileChangeKind) -> FileDiffSummary { + FileDiffSummary { + path: "src/login.rs".to_string(), + old_path: None, + kind, + before_hash: None, + after_hash: None, + additions: 3, + deletions: 1, + line_changes: Vec::new(), + patch: None, } } - Ok(()) + + #[test] + fn typed_status_fixtures_cover_clean_and_dirty_workspace_states() { + let options = RenderOptions::test(RenderMode::Plain, 80); + let clean = status_document(&status_fixture(WorktreeState::Clean, Vec::new()), &options); + let dirty = status_document( + &status_fixture( + WorktreeState::DirtyTracked, + vec![changed_path(FileChangeKind::Modified)], + ), + &options, + ); + let untracked = status_document( + &status_fixture( + WorktreeState::DirtyUntracked, + vec![changed_path(FileChangeKind::Added)], + ), + &options, + ); + assert_eq!(clean.lead.as_ref().unwrap().text, "Worktree clean"); + assert!(dirty.lead.as_ref().unwrap().text.contains("unrecorded")); + assert!(untracked.lead.as_ref().unwrap().text.contains("untracked")); + assert!(dirty.next.is_some()); + assert!(untracked.next.is_some()); + } } diff --git a/trail/src/cli/command/worktree_args.rs b/trail/src/cli/command/worktree_args.rs index 90c22cc..9a5f413 100644 --- a/trail/src/cli/command/worktree_args.rs +++ b/trail/src/cli/command/worktree_args.rs @@ -93,6 +93,10 @@ pub(super) struct DiffArgs { pub(super) patch: bool, #[arg(long)] pub(super) stat: bool, + #[arg(long = "name-only")] + pub(super) name_only: bool, + #[arg(long = "name-status")] + pub(super) name_status: bool, #[arg(long)] pub(super) dirty: bool, #[arg(long)] diff --git a/trail/src/cli/environment_sandbox.rs b/trail/src/cli/environment_sandbox.rs new file mode 100644 index 0000000..f2328e6 --- /dev/null +++ b/trail/src/cli/environment_sandbox.rs @@ -0,0 +1,1140 @@ +#[cfg(target_os = "linux")] +mod linux { + use std::ffi::{OsStr, OsString}; + use std::fs; + use std::io; + use std::os::unix::process::CommandExt; + use std::path::{Path, PathBuf}; + use std::process::Command; + + use landlock::{ + Access, AccessFs, CompatLevel, Compatible, PathBeneath, PathFd, Ruleset, RulesetAttr, + RulesetCreatedAttr, ABI, + }; + + #[derive(Debug)] + struct SandboxInvocation { + root: PathBuf, + reads: Vec, + outputs: Vec, + caches: Vec, + home: PathBuf, + temporary: PathBuf, + program: PathBuf, + args: Vec, + } + + pub(super) fn run(arguments: impl Iterator) -> Result { + let invocation = parse(arguments)?; + apply_landlock(&invocation)?; + apply_network_and_privilege_seccomp()?; + let error = Command::new(&invocation.program) + .args(&invocation.args) + .exec(); + Err(format!( + "failed to execute sandboxed program `{}`: {error}", + invocation.program.display() + )) + } + + fn parse(mut arguments: impl Iterator) -> Result { + fn value( + arguments: &mut impl Iterator, + expected: &str, + ) -> Result { + let flag = arguments + .next() + .ok_or_else(|| format!("missing internal sandbox flag `{expected}`"))?; + if flag != OsStr::new(expected) { + return Err(format!( + "expected internal sandbox flag `{expected}`, got `{}`", + flag.to_string_lossy() + )); + } + arguments + .next() + .map(PathBuf::from) + .ok_or_else(|| format!("missing value for internal sandbox flag `{expected}`")) + } + + let root = value(&mut arguments, "--root")?; + let mut reads = Vec::new(); + let mut outputs = Vec::new(); + let mut caches = Vec::new(); + let home = loop { + let flag = arguments + .next() + .ok_or_else(|| "missing internal sandbox output or home flag".to_string())?; + let path = arguments + .next() + .map(PathBuf::from) + .ok_or_else(|| format!("missing value for `{}`", flag.to_string_lossy()))?; + if flag == OsStr::new("--read") { + reads.push(path); + } else if flag == OsStr::new("--output") { + outputs.push(path); + } else if flag == OsStr::new("--cache") { + caches.push(path); + } else if flag == OsStr::new("--home") { + break path; + } else { + return Err(format!( + "expected internal sandbox flag `--read`, `--output`, `--cache`, or `--home`, got `{}`", + flag.to_string_lossy() + )); + } + }; + let temporary = value(&mut arguments, "--tmp")?; + let program = value(&mut arguments, "--program")?; + let separator = arguments + .next() + .ok_or_else(|| "missing internal sandbox argument separator".to_string())?; + if separator != OsStr::new("--") { + return Err("invalid internal sandbox argument separator".to_string()); + } + for (name, path) in [ + ("root", &root), + ("home", &home), + ("tmp", &temporary), + ("program", &program), + ] { + if !path.is_absolute() || !path.exists() { + return Err(format!( + "internal sandbox {name} path `{}` must be an existing absolute path", + path.display() + )); + } + } + for cache in &caches { + if !cache.is_absolute() || !cache.is_dir() { + return Err(format!( + "internal sandbox cache path `{}` must be an existing absolute directory", + cache.display() + )); + } + } + let root = fs::canonicalize(&root) + .map_err(|error| format!("cannot canonicalize sandbox root: {error}"))?; + let reads = reads + .into_iter() + .map(|read| { + fs::canonicalize(&read).map_err(|error| { + format!( + "cannot canonicalize sandbox input `{}`: {error}", + read.display() + ) + }) + }) + .collect::, _>>()?; + let outputs = outputs + .into_iter() + .map(|output| { + fs::canonicalize(&output).map_err(|error| { + format!( + "cannot canonicalize sandbox output `{}`: {error}", + output.display() + ) + }) + }) + .collect::, _>>()?; + let caches = caches + .into_iter() + .map(|cache| { + fs::canonicalize(&cache).map_err(|error| { + format!( + "cannot canonicalize sandbox cache `{}`: {error}", + cache.display() + ) + }) + }) + .collect::, _>>()?; + let home = fs::canonicalize(&home) + .map_err(|error| format!("cannot canonicalize sandbox home: {error}"))?; + let temporary = fs::canonicalize(&temporary) + .map_err(|error| format!("cannot canonicalize sandbox temporary directory: {error}"))?; + let program = fs::canonicalize(&program) + .map_err(|error| format!("cannot canonicalize sandbox program: {error}"))?; + for (kind, paths) in [("input", &reads), ("output", &outputs)] { + for path in paths { + if path.starts_with(&root) { + continue; + } + return Err(format!( + "internal sandbox {kind} `{}` must be an existing absolute path under its root", + path.display() + )); + } + } + // HOME and temporary storage may live beside an ephemeral mounted + // candidate rather than beneath its read root. They remain explicit + // allow-list entries and receive no access to their parent directory. + Ok(SandboxInvocation { + root, + reads, + outputs, + caches, + home, + temporary, + program, + args: arguments.collect(), + }) + } + + fn apply_landlock(invocation: &SandboxInvocation) -> Result<(), String> { + let abi = ABI::V4; + let all = AccessFs::from_all(abi); + let read_without_execute = AccessFs::from_read(abi) & !AccessFs::Execute; + let read_file = read_without_execute & AccessFs::from_file(abi); + let read_execute_file = read_file | AccessFs::Execute; + let writable = all & !AccessFs::Execute; + let writable_file = writable & AccessFs::from_file(abi); + let mut ruleset = Ruleset::default() + .set_compatibility(CompatLevel::HardRequirement) + .handle_access(all) + .map_err(|error| format!("Landlock cannot handle required filesystem rights: {error}"))? + .create() + .map_err(|error| format!("Landlock ruleset creation failed: {error}"))? + .set_compatibility(CompatLevel::HardRequirement); + + for path in ["/usr", "/bin", "/lib", "/lib64", "/nix/store"] { + if Path::new(path).exists() { + ruleset = ruleset + .add_rule(PathBeneath::new( + PathFd::new(path).map_err(|error| { + format!("cannot open sandbox system path `{path}`: {error}") + })?, + read_without_execute, + )) + .map_err(|error| { + format!("cannot allow sandbox system path `{path}`: {error}") + })?; + } + } + for path in ["/etc/ld.so.cache", "/etc/ld.so.preload"] { + if Path::new(path).is_file() { + ruleset = ruleset + .add_rule(PathBeneath::new( + PathFd::new(path).map_err(|error| { + format!("cannot open loader file `{path}`: {error}") + })?, + read_file, + )) + .map_err(|error| format!("cannot allow loader file `{path}`: {error}"))?; + } + } + // Grant directory enumeration/metadata for path traversal without + // granting ReadFile across the repository. Exact pinned files and + // declared outputs are added separately below. + ruleset = ruleset + .add_rule(PathBeneath::new( + PathFd::new(&invocation.root).map_err(|error| { + format!( + "cannot open sandbox root `{}`: {error}", + invocation.root.display() + ) + })?, + AccessFs::ReadDir, + )) + .map_err(|error| format!("cannot allow sandbox root reads: {error}"))?; + for path in &invocation.reads { + ruleset = ruleset + .add_rule(PathBeneath::new( + PathFd::new(path).map_err(|error| { + format!("cannot open sandbox input `{}`: {error}", path.display()) + })?, + if path.is_dir() { + read_without_execute + } else { + read_file + }, + )) + .map_err(|error| { + format!("cannot allow sandbox input `{}`: {error}", path.display()) + })?; + } + for path in invocation.outputs.iter().chain(&invocation.caches) { + ruleset = ruleset + .add_rule(PathBeneath::new( + PathFd::new(path).map_err(|error| { + format!( + "cannot open sandbox writable path `{}`: {error}", + path.display() + ) + })?, + writable, + )) + .map_err(|error| { + format!( + "cannot allow sandbox writable path `{}`: {error}", + path.display() + ) + })?; + } + for path in [&invocation.home, &invocation.temporary] { + ruleset = ruleset + .add_rule(PathBeneath::new( + PathFd::new(path).map_err(|error| { + format!( + "cannot open sandbox writable path `{}`: {error}", + path.display() + ) + })?, + writable, + )) + .map_err(|error| { + format!( + "cannot allow sandbox writable path `{}`: {error}", + path.display() + ) + })?; + } + ruleset = ruleset + .add_rule(PathBeneath::new( + PathFd::new(&invocation.program).map_err(|error| { + format!( + "cannot open sandbox executable `{}`: {error}", + invocation.program.display() + ) + })?, + read_execute_file, + )) + .map_err(|error| format!("cannot allow selected sandbox executable: {error}"))?; + for path in ["/dev/null", "/dev/zero", "/dev/random", "/dev/urandom"] { + if Path::new(path).exists() { + ruleset = ruleset + .add_rule(PathBeneath::new( + PathFd::new(path).map_err(|error| { + format!("cannot open sandbox device `{path}`: {error}") + })?, + writable_file, + )) + .map_err(|error| format!("cannot allow sandbox device `{path}`: {error}"))?; + } + } + ruleset + .restrict_self() + .map_err(|error| format!("Landlock restriction was not fully enforced: {error}"))?; + Ok(()) + } + + fn apply_network_and_privilege_seccomp() -> Result<(), String> { + // A deny-list is appropriate here because Landlock supplies the + // filesystem allow-list and exact executable policy. Seccomp closes + // every socket-family entry point plus kernel/namespace escape hatches. + let denied = [ + libc::SYS_socket, + libc::SYS_socketpair, + libc::SYS_connect, + libc::SYS_bind, + libc::SYS_listen, + libc::SYS_accept, + libc::SYS_accept4, + libc::SYS_sendto, + libc::SYS_recvfrom, + libc::SYS_sendmsg, + libc::SYS_recvmsg, + libc::SYS_shutdown, + libc::SYS_setsockopt, + libc::SYS_getsockopt, + libc::SYS_ptrace, + libc::SYS_process_vm_readv, + libc::SYS_process_vm_writev, + libc::SYS_bpf, + libc::SYS_perf_event_open, + libc::SYS_mount, + libc::SYS_umount2, + libc::SYS_pivot_root, + libc::SYS_chroot, + libc::SYS_setns, + libc::SYS_unshare, + libc::SYS_kexec_load, + libc::SYS_init_module, + libc::SYS_finit_module, + libc::SYS_delete_module, + libc::SYS_open_by_handle_at, + libc::SYS_userfaultfd, + libc::SYS_memfd_create, + libc::SYS_io_uring_setup, + libc::SYS_io_uring_enter, + libc::SYS_io_uring_register, + ]; + let mut filter = Vec::::with_capacity(20 + denied.len() * 2); + filter.push(stmt(BPF_LD | BPF_W | BPF_ABS, 4)); + filter.push(jump(BPF_JMP | BPF_JEQ | BPF_K, audit_arch(), 1, 0)); + filter.push(stmt(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS)); + filter.push(stmt(BPF_LD | BPF_W | BPF_ABS, 0)); + filter.push(jump(BPF_JMP | BPF_JGE | BPF_K, 0x4000_0000, 0, 1)); + filter.push(stmt(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS)); + for syscall in denied { + filter.push(jump(BPF_JMP | BPF_JEQ | BPF_K, syscall as u32, 0, 1)); + filter.push(stmt( + BPF_RET | BPF_K, + SECCOMP_RET_ERRNO | libc::EPERM as u32, + )); + } + #[cfg(target_arch = "x86_64")] + for syscall in [libc::SYS_fork, libc::SYS_vfork] { + filter.push(jump(BPF_JMP | BPF_JEQ | BPF_K, syscall as u32, 0, 1)); + filter.push(stmt( + BPF_RET | BPF_K, + SECCOMP_RET_ERRNO | libc::EPERM as u32, + )); + } + // Modern pthread implementations may probe clone3. Report it as + // unavailable so they can fall back to clone, whose flags we can + // inspect in classic BPF without dereferencing user memory. + filter.push(jump( + BPF_JMP | BPF_JEQ | BPF_K, + libc::SYS_clone3 as u32, + 0, + 1, + )); + filter.push(stmt( + BPF_RET | BPF_K, + SECCOMP_RET_ERRNO | libc::ENOSYS as u32, + )); + // Permit threads but reject process creation. clone's first argument + // is the flags word on every Linux architecture Trail supports. + filter.push(jump( + BPF_JMP | BPF_JEQ | BPF_K, + libc::SYS_clone as u32, + 0, + 4, + )); + filter.push(stmt(BPF_LD | BPF_W | BPF_ABS, 16)); + filter.push(jump( + BPF_JMP | BPF_JSET | BPF_K, + libc::CLONE_THREAD as u32, + 1, + 0, + )); + filter.push(stmt( + BPF_RET | BPF_K, + SECCOMP_RET_ERRNO | libc::EPERM as u32, + )); + filter.push(stmt(BPF_RET | BPF_K, SECCOMP_RET_ALLOW)); + filter.push(stmt(BPF_RET | BPF_K, SECCOMP_RET_ALLOW)); + let mut program = libc::sock_fprog { + len: u16::try_from(filter.len()) + .map_err(|_| "seccomp filter is too large".to_string())?, + filter: filter.as_mut_ptr(), + }; + // SAFETY: `prctl` receives the documented scalar values and a valid + // sock_fprog whose backing vector remains alive for the syscall. + unsafe { + if libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0 { + return Err(format!( + "cannot enable no_new_privs: {}", + io::Error::last_os_error() + )); + } + if libc::prctl( + libc::PR_SET_SECCOMP, + libc::SECCOMP_MODE_FILTER, + &mut program as *mut libc::sock_fprog, + ) != 0 + { + return Err(format!( + "cannot install network-denying seccomp filter: {}", + io::Error::last_os_error() + )); + } + } + Ok(()) + } + + const BPF_LD: u16 = 0x00; + const BPF_W: u16 = 0x00; + const BPF_ABS: u16 = 0x20; + const BPF_JMP: u16 = 0x05; + const BPF_JEQ: u16 = 0x10; + const BPF_JGE: u16 = 0x30; + const BPF_JSET: u16 = 0x40; + const BPF_K: u16 = 0x00; + const BPF_RET: u16 = 0x06; + const SECCOMP_RET_KILL_PROCESS: u32 = 0x8000_0000; + const SECCOMP_RET_ERRNO: u32 = 0x0005_0000; + const SECCOMP_RET_ALLOW: u32 = 0x7fff_0000; + + fn stmt(code: u16, value: u32) -> libc::sock_filter { + libc::sock_filter { + code, + jt: 0, + jf: 0, + k: value, + } + } + + fn jump(code: u16, value: u32, jt: u8, jf: u8) -> libc::sock_filter { + libc::sock_filter { + code, + jt, + jf, + k: value, + } + } + + #[cfg(target_arch = "x86_64")] + fn audit_arch() -> u32 { + 0xc000_003e + } + + #[cfg(target_arch = "aarch64")] + fn audit_arch() -> u32 { + 0xc000_00b7 + } + + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] + compile_error!("Trail's Linux recipe sandbox supports x86_64 and aarch64 only"); +} + +#[cfg(target_os = "windows")] +mod windows { + use std::ffi::{OsStr, OsString}; + use std::fs; + use std::io; + use std::mem::{size_of, zeroed}; + use std::os::windows::ffi::OsStrExt; + use std::path::{Path, PathBuf}; + use std::process::Command; + use std::ptr::{null, null_mut}; + use std::time::{SystemTime, UNIX_EPOCH}; + + use winapi::shared::basetsd::{DWORD_PTR, SIZE_T}; + use winapi::shared::minwindef::{DWORD, FALSE}; + use winapi::shared::sddl::ConvertSidToStringSidW; + use winapi::um::errhandlingapi::GetLastError; + use winapi::um::handleapi::{CloseHandle, INVALID_HANDLE_VALUE}; + use winapi::um::jobapi2::{ + AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject, + }; + use winapi::um::processthreadsapi::{ + CreateProcessW, DeleteProcThreadAttributeList, GetExitCodeProcess, + InitializeProcThreadAttributeList, ResumeThread, TerminateProcess, + UpdateProcThreadAttribute, LPPROC_THREAD_ATTRIBUTE_LIST, PROCESS_INFORMATION, + }; + use winapi::um::securitybaseapi::FreeSid; + use winapi::um::synchapi::WaitForSingleObject; + use winapi::um::userenv::{CreateAppContainerProfile, DeleteAppContainerProfile}; + use winapi::um::winbase::{ + LocalFree, CREATE_SUSPENDED, EXTENDED_STARTUPINFO_PRESENT, INFINITE, STARTUPINFOEXW, + WAIT_FAILED, WAIT_OBJECT_0, + }; + use winapi::um::winnt::{ + JobObjectExtendedLimitInformation, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, + JOB_OBJECT_LIMIT_ACTIVE_PROCESS, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, + JOB_OBJECT_LIMIT_PROCESS_MEMORY, PSID, SECURITY_CAPABILITIES, + }; + + const PROC_THREAD_ATTRIBUTE_SECURITY_CAPABILITIES: DWORD_PTR = 0x0002_0009; + + #[derive(Debug)] + struct SandboxInvocation { + root: PathBuf, + reads: Vec, + outputs: Vec, + caches: Vec, + home: PathBuf, + temporary: PathBuf, + program: PathBuf, + args: Vec, + } + + struct AppContainerProfile { + name: Vec, + sid: PSID, + } + + impl Drop for AppContainerProfile { + fn drop(&mut self) { + // SAFETY: both values were returned by the AppContainer APIs and + // remain owned by this guard until process completion. + unsafe { + let _ = DeleteAppContainerProfile(self.name.as_ptr()); + if !self.sid.is_null() { + let _ = FreeSid(self.sid); + } + } + } + } + + struct OwnedHandle(winapi::shared::ntdef::HANDLE); + + impl OwnedHandle { + fn new(handle: winapi::shared::ntdef::HANDLE, what: &str) -> Result { + if handle.is_null() || handle == INVALID_HANDLE_VALUE { + Err(last_error(what)) + } else { + Ok(Self(handle)) + } + } + } + + impl Drop for OwnedHandle { + fn drop(&mut self) { + // SAFETY: this guard uniquely owns a valid Win32 handle. + unsafe { + let _ = CloseHandle(self.0); + } + } + } + + /// A process created suspended must never escape if one of the Job Object + /// setup steps fails. Closing a process handle does not terminate the + /// process, so the ordinary handle guard is insufficient for this case. + struct OwnedProcessHandle { + handle: winapi::shared::ntdef::HANDLE, + terminate_on_drop: bool, + } + + impl OwnedProcessHandle { + fn new(handle: winapi::shared::ntdef::HANDLE) -> Result { + if handle.is_null() || handle == INVALID_HANDLE_VALUE { + Err(last_error("invalid Windows sandbox process handle")) + } else { + Ok(Self { + handle, + terminate_on_drop: true, + }) + } + } + + fn process_exited(&mut self) { + self.terminate_on_drop = false; + } + } + + impl Drop for OwnedProcessHandle { + fn drop(&mut self) { + // SAFETY: this guard uniquely owns a valid process handle. A failed + // setup must terminate the still-suspended child before closing it. + unsafe { + if self.terminate_on_drop { + let _ = TerminateProcess(self.handle, 126); + // Termination is asynchronous. Give the process a bounded + // opportunity to leave the AppContainer before its profile + // is deleted by the outer guard. + let _ = WaitForSingleObject(self.handle, 5_000); + } + let _ = CloseHandle(self.handle); + } + } + } + + struct AttributeList { + storage: Vec, + list: LPPROC_THREAD_ATTRIBUTE_LIST, + } + + impl AttributeList { + fn one() -> Result { + let mut bytes: SIZE_T = 0; + // SAFETY: the documented sizing call accepts a null list. + unsafe { + let _ = InitializeProcThreadAttributeList(null_mut(), 1, 0, &mut bytes); + } + if bytes == 0 { + return Err(last_error("cannot size Windows process attribute list")); + } + let words = bytes.div_ceil(size_of::()); + let mut storage = vec![0usize; words]; + let list = storage.as_mut_ptr().cast(); + // SAFETY: `storage` is aligned, writable, and at least the size + // returned by the sizing call. + if unsafe { InitializeProcThreadAttributeList(list, 1, 0, &mut bytes) } == 0 { + return Err(last_error( + "cannot initialize Windows process attribute list", + )); + } + Ok(Self { storage, list }) + } + } + + impl Drop for AttributeList { + fn drop(&mut self) { + // SAFETY: initialization succeeded and storage outlives this call. + unsafe { DeleteProcThreadAttributeList(self.list) }; + let _ = self.storage.len(); + } + } + + pub(super) fn run(arguments: impl Iterator) -> Result { + let invocation = parse(arguments)?; + let profile = create_profile()?; + let sid = sid_string(profile.sid)?; + grant_appcontainer_access(&invocation.root, &sid, "RX", false)?; + for read in &invocation.reads { + grant_appcontainer_access(read, &sid, "R", false)?; + let mut ancestor = read.parent(); + while let Some(path) = ancestor { + if !path.starts_with(&invocation.root) { + break; + } + grant_appcontainer_access(path, &sid, "RX", false)?; + if path == invocation.root { + break; + } + ancestor = path.parent(); + } + } + for output in invocation.outputs.iter().chain(&invocation.caches) { + grant_appcontainer_access(output, &sid, "M", true)?; + } + grant_appcontainer_access(&invocation.home, &sid, "M", true)?; + grant_appcontainer_access(&invocation.temporary, &sid, "M", true)?; + launch_in_appcontainer(&invocation, &profile) + } + + fn parse(mut arguments: impl Iterator) -> Result { + fn value( + arguments: &mut impl Iterator, + expected: &str, + ) -> Result { + let flag = arguments + .next() + .ok_or_else(|| format!("missing internal sandbox flag `{expected}`"))?; + if flag != OsStr::new(expected) { + return Err(format!( + "expected internal sandbox flag `{expected}`, got `{}`", + flag.to_string_lossy() + )); + } + arguments + .next() + .map(PathBuf::from) + .ok_or_else(|| format!("missing value for internal sandbox flag `{expected}`")) + } + + let root = value(&mut arguments, "--root")?; + let mut reads = Vec::new(); + let mut outputs = Vec::new(); + let mut caches = Vec::new(); + let home = loop { + let flag = arguments + .next() + .ok_or_else(|| "missing internal sandbox output or home flag".to_string())?; + let path = arguments + .next() + .map(PathBuf::from) + .ok_or_else(|| format!("missing value for `{}`", flag.to_string_lossy()))?; + if flag == OsStr::new("--read") { + reads.push(path); + } else if flag == OsStr::new("--output") { + outputs.push(path); + } else if flag == OsStr::new("--cache") { + caches.push(path); + } else if flag == OsStr::new("--home") { + break path; + } else { + return Err(format!( + "expected internal sandbox flag `--read`, `--output`, `--cache`, or `--home`, got `{}`", + flag.to_string_lossy() + )); + } + }; + let temporary = value(&mut arguments, "--tmp")?; + let program = value(&mut arguments, "--program")?; + let separator = arguments + .next() + .ok_or_else(|| "missing internal sandbox argument separator".to_string())?; + if separator != OsStr::new("--") { + return Err("invalid internal sandbox argument separator".to_string()); + } + for (name, path) in [ + ("root", &root), + ("home", &home), + ("tmp", &temporary), + ("program", &program), + ] { + if !path.is_absolute() || !path.exists() { + return Err(format!( + "internal sandbox {name} path `{}` must be an existing absolute path", + path.display() + )); + } + } + for cache in &caches { + if !cache.is_absolute() || !cache.is_dir() { + return Err(format!( + "internal sandbox cache path `{}` must be an existing absolute directory", + cache.display() + )); + } + } + let root = fs::canonicalize(&root) + .map_err(|error| format!("cannot canonicalize sandbox root: {error}"))?; + let reads = reads + .into_iter() + .map(|read| { + fs::canonicalize(&read).map_err(|error| { + format!( + "cannot canonicalize sandbox input `{}`: {error}", + read.display() + ) + }) + }) + .collect::, _>>()?; + let outputs = outputs + .into_iter() + .map(|output| { + fs::canonicalize(&output).map_err(|error| { + format!( + "cannot canonicalize sandbox output `{}`: {error}", + output.display() + ) + }) + }) + .collect::, _>>()?; + let caches = caches + .into_iter() + .map(|cache| { + fs::canonicalize(&cache).map_err(|error| { + format!( + "cannot canonicalize sandbox cache `{}`: {error}", + cache.display() + ) + }) + }) + .collect::, _>>()?; + let home = fs::canonicalize(&home) + .map_err(|error| format!("cannot canonicalize sandbox home: {error}"))?; + let temporary = fs::canonicalize(&temporary) + .map_err(|error| format!("cannot canonicalize sandbox temporary directory: {error}"))?; + let program = fs::canonicalize(&program) + .map_err(|error| format!("cannot canonicalize sandbox program: {error}"))?; + for (kind, paths) in [("input", &reads), ("output", &outputs)] { + for path in paths { + if path.starts_with(&root) { + continue; + } + return Err(format!( + "internal sandbox {kind} `{}` must be an existing absolute path under its root", + path.display() + )); + } + } + // HOME and temporary storage may live beside an ephemeral mounted + // candidate rather than beneath its read root. The AppContainer ACLs + // grant only these directories, never their parent. + Ok(SandboxInvocation { + root, + reads, + outputs, + caches, + home, + temporary, + program, + args: arguments.collect(), + }) + } + + fn create_profile() -> Result { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| format!("system clock is before the Unix epoch: {error}"))? + .as_nanos(); + let name = format!("Trail.Recipe.{}.{nonce:x}", std::process::id()); + let name = wide_null(OsStr::new(&name)); + let display = wide_null(OsStr::new("Trail restricted recipe")); + let description = wide_null(OsStr::new("Ephemeral Trail command recipe sandbox")); + let mut sid: PSID = null_mut(); + // SAFETY: all strings are NUL-terminated and `sid` is an out pointer. + let result = unsafe { + CreateAppContainerProfile( + name.as_ptr(), + display.as_ptr(), + description.as_ptr(), + null_mut(), + 0, + &mut sid, + ) + }; + if result < 0 || sid.is_null() { + return Err(format!( + "cannot create capability-free Windows AppContainer (HRESULT 0x{:08x})", + result as u32 + )); + } + Ok(AppContainerProfile { name, sid }) + } + + fn sid_string(sid: PSID) -> Result { + let mut string_sid = null_mut(); + // SAFETY: `sid` is valid for the profile lifetime and the API allocates + // a NUL-terminated string released with LocalFree. + if unsafe { ConvertSidToStringSidW(sid, &mut string_sid) } == 0 { + return Err(last_error("cannot format Windows AppContainer SID")); + } + let mut length = 0usize; + // SAFETY: the conversion API returned a valid NUL-terminated string. + unsafe { + while *string_sid.add(length) != 0 { + length += 1; + } + } + // SAFETY: `length` was measured within the allocated string. + let value = String::from_utf16(unsafe { std::slice::from_raw_parts(string_sid, length) }) + .map_err(|error| format!("AppContainer SID is not valid UTF-16: {error}")); + // SAFETY: ownership of the allocation belongs to this function. + unsafe { + let _ = LocalFree(string_sid.cast()); + } + value + } + + fn grant_appcontainer_access( + path: &Path, + sid: &str, + rights: &str, + recursive: bool, + ) -> Result<(), String> { + let system_root = std::env::var_os("SystemRoot") + .ok_or_else(|| "SystemRoot is unavailable to the Windows sandbox helper".to_string())?; + let icacls = PathBuf::from(system_root).join("System32/icacls.exe"); + if !icacls.is_file() { + return Err(format!( + "Windows ACL utility `{}` is unavailable", + icacls.display() + )); + } + let principal = if recursive { + format!("*{sid}:(OI)(CI){rights}") + } else { + format!("*{sid}:{rights}") + }; + let mut command = Command::new(&icacls); + command.arg(path).args(["/grant", &principal]); + if recursive { + command.arg("/T"); + } + let status = command + .arg("/Q") + .status() + .map_err(|error| format!("cannot launch `{}`: {error}", icacls.display()))?; + if !status.success() { + return Err(format!( + "`{}` could not grant AppContainer {rights} access to `{}` ({status})", + icacls.display(), + path.display() + )); + } + Ok(()) + } + + fn launch_in_appcontainer( + invocation: &SandboxInvocation, + profile: &AppContainerProfile, + ) -> Result { + let attributes = AttributeList::one()?; + let mut capabilities = SECURITY_CAPABILITIES { + AppContainerSid: profile.sid, + Capabilities: null_mut(), + CapabilityCount: 0, + Reserved: 0, + }; + // SAFETY: the attribute list is initialized and both it and the + // capabilities structure remain alive through CreateProcessW. + if unsafe { + UpdateProcThreadAttribute( + attributes.list, + 0, + PROC_THREAD_ATTRIBUTE_SECURITY_CAPABILITIES, + (&mut capabilities as *mut SECURITY_CAPABILITIES).cast(), + size_of::(), + null_mut(), + null_mut(), + ) + } == 0 + { + return Err(last_error( + "cannot attach AppContainer capabilities to Windows process", + )); + } + + let program = wide_null(invocation.program.as_os_str()); + let mut command_line = windows_command_line(&invocation.program, &invocation.args); + let current_directory = std::env::current_dir() + .and_then(std::fs::canonicalize) + .map_err(|error| format!("cannot canonicalize sandbox working directory: {error}"))?; + if !current_directory.starts_with(&invocation.root) { + return Err(format!( + "sandbox working directory `{}` escapes root `{}`", + current_directory.display(), + invocation.root.display() + )); + } + let current_directory = wide_null(current_directory.as_os_str()); + // SAFETY: zero is the documented initialization for these structures; + // required fields are assigned below. + let mut startup: STARTUPINFOEXW = unsafe { zeroed() }; + startup.StartupInfo.cb = size_of::() as DWORD; + startup.lpAttributeList = attributes.list; + let mut process: PROCESS_INFORMATION = unsafe { zeroed() }; + // SAFETY: all pointers reference live, writable, NUL-terminated buffers + // for the duration of CreateProcessW. + if unsafe { + CreateProcessW( + program.as_ptr(), + command_line.as_mut_ptr(), + null_mut(), + null_mut(), + FALSE, + EXTENDED_STARTUPINFO_PRESENT | CREATE_SUSPENDED, + null_mut(), + current_directory.as_ptr(), + (&mut startup as *mut STARTUPINFOEXW).cast(), + &mut process, + ) + } == 0 + { + return Err(last_error(&format!( + "cannot launch `{}` in Windows AppContainer", + invocation.program.display() + ))); + } + let mut process_handle = OwnedProcessHandle::new(process.hProcess)?; + let thread_handle = OwnedHandle::new(process.hThread, "invalid thread handle")?; + let job = OwnedHandle::new( + // SAFETY: null security/name pointers request a private job object. + unsafe { CreateJobObjectW(null_mut(), null()) }, + "cannot create Windows sandbox Job Object", + )?; + // SAFETY: zero initialization followed by the documented limit fields. + let mut limits: JOBOBJECT_EXTENDED_LIMIT_INFORMATION = unsafe { zeroed() }; + limits.BasicLimitInformation.LimitFlags = + JOB_OBJECT_LIMIT_ACTIVE_PROCESS | JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + limits.BasicLimitInformation.ActiveProcessLimit = 1; + if invocation.outputs.is_empty() { + limits.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PROCESS_MEMORY; + limits.ProcessMemoryLimit = 512 * 1024 * 1024; + } + // SAFETY: the job and structure are valid for the call. + if unsafe { + SetInformationJobObject( + job.0, + JobObjectExtendedLimitInformation, + (&mut limits as *mut JOBOBJECT_EXTENDED_LIMIT_INFORMATION).cast(), + size_of::() as DWORD, + ) + } == 0 + { + return Err(last_error("cannot constrain Windows sandbox Job Object")); + } + // SAFETY: both handles are valid and the process is still suspended. + if unsafe { AssignProcessToJobObject(job.0, process_handle.handle) } == 0 { + return Err(last_error( + "cannot assign restricted process to Windows sandbox Job Object", + )); + } + // SAFETY: the primary thread is suspended exactly once by CreateProcessW. + if unsafe { ResumeThread(thread_handle.0) } == u32::MAX { + return Err(last_error("cannot resume Windows sandbox process")); + } + // SAFETY: process_handle remains live and an infinite wait is intended. + let wait_result = unsafe { WaitForSingleObject(process_handle.handle, INFINITE) }; + if wait_result == WAIT_FAILED { + return Err(last_error("cannot wait for Windows sandbox process")); + } + if wait_result != WAIT_OBJECT_0 { + return Err(format!( + "Windows sandbox process wait returned unexpected result 0x{wait_result:08x}" + )); + } + process_handle.process_exited(); + let mut exit_code = 0u32; + // SAFETY: the process has terminated and the output pointer is valid. + if unsafe { GetExitCodeProcess(process_handle.handle, &mut exit_code) } == 0 { + return Err(last_error("cannot read Windows sandbox process exit code")); + } + drop(attributes); + Ok(exit_code as i32) + } + + fn windows_command_line(program: &Path, arguments: &[OsString]) -> Vec { + let mut command = quote_windows_argument(program.as_os_str()); + for argument in arguments { + command.push(u16::from(b' ')); + command.extend(quote_windows_argument(argument)); + } + command.push(0); + command + } + + fn quote_windows_argument(argument: &OsStr) -> Vec { + let value = argument.encode_wide().collect::>(); + if !value.is_empty() + && !value + .iter() + .any(|unit| [u16::from(b' '), u16::from(b'\t'), u16::from(b'"')].contains(unit)) + { + return value; + } + let mut quoted = vec![u16::from(b'"')]; + let mut backslashes = 0usize; + for unit in value { + if unit == u16::from(b'\\') { + backslashes += 1; + } else if unit == u16::from(b'"') { + quoted.extend(std::iter::repeat(u16::from(b'\\')).take(backslashes * 2 + 1)); + quoted.push(u16::from(b'"')); + backslashes = 0; + } else { + quoted.extend(std::iter::repeat(u16::from(b'\\')).take(backslashes)); + backslashes = 0; + quoted.push(unit); + } + } + quoted.extend(std::iter::repeat(u16::from(b'\\')).take(backslashes * 2)); + quoted.push(u16::from(b'"')); + quoted + } + + fn wide_null(value: &OsStr) -> Vec { + value.encode_wide().chain(std::iter::once(0)).collect() + } + + fn last_error(context: &str) -> String { + // SAFETY: GetLastError has no preconditions. + let code = unsafe { GetLastError() }; + format!("{context}: {}", io::Error::from_raw_os_error(code as i32)) + } + + #[cfg(test)] + mod tests { + use super::*; + use std::os::windows::ffi::OsStringExt; + + fn quoted(value: &str) -> String { + String::from_utf16("e_windows_argument(OsStr::new(value))).unwrap() + } + + #[test] + fn windows_arguments_follow_command_line_to_argv_w_quoting() { + assert_eq!(quoted("plain"), "plain"); + assert_eq!(quoted(""), "\"\""); + assert_eq!(quoted("two words"), "\"two words\""); + assert_eq!(quoted("say \"hello\""), "\"say \\\"hello\\\"\""); + assert_eq!( + quoted("C:\\path with space\\"), + "\"C:\\path with space\\\\\"" + ); + + let unpaired_surrogate = OsString::from_wide(&[0xd800]); + assert_eq!( + quote_windows_argument(&unpaired_surrogate), + vec![0xd800], + "native Windows paths must never pass through lossy UTF-8 conversion" + ); + } + } +} + +#[cfg(target_os = "linux")] +pub(crate) fn run(arguments: impl Iterator) -> Result { + linux::run(arguments) +} + +#[cfg(target_os = "windows")] +pub(crate) fn run(arguments: impl Iterator) -> Result { + windows::run(arguments) +} diff --git a/trail/src/cli/mod.rs b/trail/src/cli/mod.rs index 74e00d6..c9fddcd 100644 --- a/trail/src/cli/mod.rs +++ b/trail/src/cli/mod.rs @@ -1,5 +1,40 @@ pub mod command; +#[cfg(any(target_os = "linux", target_os = "windows"))] +mod environment_sandbox; pub(crate) fn run() { + if std::env::args_os().nth(1).as_deref() == Some(std::ffi::OsStr::new("__process-watchdog")) { + let arguments = std::env::args().skip(2).collect::>(); + let result = (|| -> std::result::Result<(), String> { + if arguments.len() != 3 { + return Err("expected parent PID, child PID, and child start token".to_string()); + } + let parent_pid = arguments[0] + .parse::() + .map_err(|error| format!("invalid watchdog parent PID: {error}"))?; + let child_pid = arguments[1] + .parse::() + .map_err(|error| format!("invalid watchdog child PID: {error}"))?; + trail::db::run_internal_process_watchdog(parent_pid, child_pid, &arguments[2]) + })(); + match result { + Ok(()) => std::process::exit(0), + Err(error) => { + eprintln!("trail process watchdog: {error}"); + std::process::exit(126); + } + } + } + #[cfg(any(target_os = "linux", target_os = "windows"))] + if std::env::args_os().nth(1).as_deref() == Some(std::ffi::OsStr::new("__environment-sandbox")) + { + match environment_sandbox::run(std::env::args_os().skip(2)) { + Ok(code) => std::process::exit(code), + Err(error) => { + eprintln!("trail restricted environment sandbox: {error}"); + std::process::exit(126); + } + } + } command::run(); } diff --git a/trail/src/db/agent.rs b/trail/src/db/agent.rs index 4edc2fa..9b6fc49 100644 --- a/trail/src/db/agent.rs +++ b/trail/src/db/agent.rs @@ -267,85 +267,6 @@ impl Trail { ) } - pub fn agent_setup_report(&self, provider: &str, editor: &str) -> Result { - let profile = crate::acp::agent_provider_profile(provider)?; - let editor = match editor { - "generic" | "vscode" | "zed" => editor, - other => { - return Err(Error::InvalidInput(format!( - "unsupported agent editor `{other}`; supported editors: generic, vscode, zed" - ))) - } - }; - let mode = if profile.supports_acp { - "acp" - } else { - "terminal" - }; - let mut command = vec![ - "trail".to_string(), - "--workspace".to_string(), - self.workspace_root.to_string_lossy().to_string(), - "agent".to_string(), - ]; - if profile.supports_acp { - command.push("acp".to_string()); - } else { - command.push("start".to_string()); - } - command.push("--provider".to_string()); - command.push(profile.agent.clone()); - let mcp_command = vec![ - "trail".to_string(), - "--workspace".to_string(), - self.workspace_root.to_string_lossy().to_string(), - "mcp".to_string(), - ]; - let snippet = if profile.supports_acp { - agent_editor_snippet(editor, &profile.agent, &command) - } else { - agent_terminal_setup_snippet(&profile, &command, &mcp_command) - }; - let mut suggestions = vec![StatusSuggestion { - command: format!("trail agent doctor --provider {}", profile.agent), - reason: "verify the workspace and provider before the first agent session".to_string(), - }]; - if profile.supports_terminal { - suggestions.push(StatusSuggestion { - command: format!("trail agent start --provider {}", profile.agent), - reason: "launch a fresh materialized task lane with this provider".to_string(), - }); - } - suggestions.extend([ - StatusSuggestion { - command: "trail agent".to_string(), - reason: "after one prompt, show the task inbox and next useful action".to_string(), - }, - StatusSuggestion { - command: "trail agent action".to_string(), - reason: "show runnable setup, review, validation, apply, and recovery actions" - .to_string(), - }, - ]); - Ok(AgentSetupReport { - provider: profile.agent, - editor: editor.to_string(), - mode: mode.to_string(), - command, - snippet, - detected: profile.available, - supports_acp: profile.supports_acp, - supports_mcp: profile.supports_mcp, - supports_terminal: profile.supports_terminal, - warnings: if profile.available { - Vec::new() - } else { - profile.notes - }, - suggestions, - }) - } - pub fn list_agent_tasks(&self) -> Result { self.list_agent_tasks_with_options(false) } @@ -449,7 +370,7 @@ impl Trail { if inbox.total == 0 { agent_push_suggestion( &mut suggestions, - "trail agent start --provider claude-code".to_string(), + "trail agent start claude-code".to_string(), "start a terminal agent task without configuring an editor", ); } @@ -475,7 +396,7 @@ impl Trail { let tasks = self.list_agent_tasks_with_options(include_archived)?.tasks; if tasks.is_empty() { let next = StatusSuggestion { - command: "trail agent setup".to_string(), + command: "trail agent acp setup claude-code --editor vscode".to_string(), reason: "configure an editor once, then start an agent task".to_string(), }; return Ok(AgentStackReport { @@ -491,7 +412,7 @@ impl Trail { suggestions: vec![ next.clone(), StatusSuggestion { - command: "trail agent start --provider claude-code".to_string(), + command: "trail agent start claude-code".to_string(), reason: "start a terminal agent task without configuring an editor" .to_string(), }, @@ -590,7 +511,7 @@ impl Trail { let tasks = self.list_agent_tasks_with_options(include_archived)?.tasks; if tasks.is_empty() { let next = StatusSuggestion { - command: "trail agent setup".to_string(), + command: "trail agent acp setup claude-code --editor vscode".to_string(), reason: "configure an editor once, then start an agent task".to_string(), }; return Ok(AgentInboxReport { @@ -604,12 +525,12 @@ impl Trail { suggestions: vec![ next.clone(), StatusSuggestion { - command: "trail agent doctor --provider claude-code".to_string(), + command: "trail agent doctor claude-code".to_string(), reason: "verify the provider and workspace before connecting an editor" .to_string(), }, StatusSuggestion { - command: "trail agent start --provider claude-code".to_string(), + command: "trail agent start claude-code".to_string(), reason: "start a terminal agent task instead of using an editor" .to_string(), }, @@ -649,7 +570,7 @@ impl Trail { }) .or_else(|| items.first().map(|item| item.next.clone())) .unwrap_or_else(|| StatusSuggestion { - command: "trail agent setup".to_string(), + command: "trail agent acp setup claude-code --editor vscode".to_string(), reason: "configure an editor once, then start an agent task".to_string(), }); let mut suggestions = vec![next.clone()]; @@ -694,7 +615,7 @@ impl Trail { latest: None, risk: None, suggestions: vec![StatusSuggestion { - command: "trail agent setup".to_string(), + command: "trail agent acp setup claude-code --editor vscode".to_string(), reason: "configure an editor once, then start an agent task".to_string(), }], }); @@ -715,7 +636,7 @@ impl Trail { pub fn agent_next(&self, selector: &str) -> Result { let Some(lane) = self.resolve_agent_selector(selector)? else { let primary = StatusSuggestion { - command: "trail agent setup".to_string(), + command: "trail agent acp setup claude-code --editor vscode".to_string(), reason: "configure an editor once; Trail will create fresh task lanes automatically" .to_string(), @@ -728,12 +649,12 @@ impl Trail { primary, suggestions: vec![ StatusSuggestion { - command: "trail agent doctor --provider claude-code".to_string(), + command: "trail agent doctor claude-code".to_string(), reason: "verify the provider and workspace before connecting an editor" .to_string(), }, StatusSuggestion { - command: "trail agent start --provider claude-code".to_string(), + command: "trail agent start claude-code".to_string(), reason: "start a terminal agent task instead of using an editor" .to_string(), }, @@ -1047,7 +968,10 @@ impl Trail { .resolve_agent_selector(selector)? .ok_or_else(|| Error::InvalidInput("no agent tasks have been recorded".to_string()))?; let review = self.lane_review_packet(&lane, 50)?; - let transcript = self.transcript(&lane).ok(); + let transcript = self + .preferred_agent_session_id(&review)? + .and_then(|session_id| self.transcript(&session_id).ok()) + .or_else(|| self.transcript(&lane).ok()); let task = self.agent_task_from_review(&review, transcript.as_ref())?; Ok(AgentTaskViewReport { task, @@ -2332,7 +2256,7 @@ impl Trail { target_kind: "reviewed".to_string(), target: reviewed .as_ref() - .map(|marker| format!("since {}", marker.checkpoint.0)) + .map(|marker| format!("since {}", marker.checkpoint.checkpoint_alias())) .unwrap_or_else(|| "whole task".to_string()), turn_id: None, operation_id: None, @@ -2901,6 +2825,7 @@ impl Trail { dry_run: bool, message: Option, ) -> Result { + self.reset_git_handoff_metrics(); let lane = self .resolve_agent_selector(selector)? .ok_or_else(|| Error::InvalidInput("no agent tasks have been recorded".to_string()))?; @@ -2927,6 +2852,7 @@ impl Trail { merge: None, git_export: None, fast_forwarded: false, + performance: self.git_handoff_metrics_report(), warnings: vec![ "agent task has already been applied; use `trail agent continue` for follow-up work to avoid reusing old lane history" .to_string(), @@ -2934,29 +2860,24 @@ impl Trail { suggestions: agent_already_applied_suggestions(&view.task), }); } - let git_branch = self.current_git_branch()?; - let git_state = self.current_git_state()?.ok_or_else(|| { + let git_identity = self.current_git_identity()?.ok_or_else(|| { Error::Git(format!( - "agent apply requires a Git working tree at {}; Trail branch `{crab_branch}` is the internal apply base", + "agent apply requires a Git working tree with a HEAD commit at {}; Trail branch `{crab_branch}` is the internal apply base", self.workspace_root.display() )) })?; - if git_state.dirty { - return Err(Error::Git( - "current Git worktree has tracked changes; commit, stash, or revert them before `trail agent apply`".to_string(), - )); - } - self.ensure_git_head_matches_root( - &target_ref.change_id, - &target_ref.root_id, - git_state.head.as_deref(), - &crab_branch, - )?; + let git_branch = git_identity.branch.clone(); + self.ensure_git_head_matches_root(&target_ref.root_id, &git_identity.head)?; + self.set_git_export_mode(GitExportMode::MappedDelta); - let _overlay_mount = self.maybe_mount_overlay_cow_workdir_for_lane(&lane)?; + let _fuse_mount = self.maybe_mount_fuse_cow_workdir_for_lane(&lane)?; let _nfs_mount = self.maybe_mount_nfs_cow_workdir_for_lane(&lane)?; + #[cfg(target_os = "windows")] + let _dokan_mount = self.maybe_mount_dokan_cow_workdir_for_lane(&lane)?; let would_record = self.lane_workdir_dirty(&lane)?; if dry_run && would_record { + let git_state = self.tracked_git_state(&git_identity)?; + validate_git_publication_state(&git_identity.head, &git_state)?; let view = self.agent_task_view(&lane)?; let plan = AgentGitApplyPlan { crab_branch, @@ -2977,6 +2898,7 @@ impl Trail { merge: None, git_export: None, fast_forwarded: false, + performance: self.git_handoff_metrics_report(), warnings: vec![ "lane workdir has unrecorded changes; actual apply will record them first" .to_string(), @@ -2995,32 +2917,40 @@ impl Trail { }; self.ensure_agent_checkpoint_reviewed(&lane)?; - let merge = self.merge_lane_user_with_options(&lane, &crab_branch, dry_run, true)?; - let range = if merge.changed_paths.is_empty() { + let merge_preview = self.merge_lane_user_with_options(&lane, &crab_branch, true, true)?; + self.set_git_changed_path_count(merge_preview.changed_paths.len() as u64); + let preview_range = if merge_preview.changed_paths.is_empty() { None } else { - Some(format!("{}..{}", target_ref.change_id.0, merge.operation.0)) + Some(format!( + "{}..{}", + target_ref.change_id.0, merge_preview.operation.0 + )) }; - let plan = AgentGitApplyPlan { + let preview_plan = AgentGitApplyPlan { crab_branch: crab_branch.clone(), git_branch: git_branch.clone(), base_change: target_ref.change_id.clone(), - result_change: range.as_ref().map(|_| merge.operation.clone()), - range: range.clone(), + result_change: preview_range + .as_ref() + .map(|_| merge_preview.operation.clone()), + range: preview_range.clone(), would_record, - would_create_git_commit: range.is_some(), - would_fast_forward: range.is_some(), + would_create_git_commit: preview_range.is_some(), + would_fast_forward: preview_range.is_some(), }; if dry_run { + let git_state = self.tracked_git_state(&git_identity)?; + validate_git_publication_state(&git_identity.head, &git_state)?; let view = self.agent_task_view(&lane)?; - let (status, suggestions) = if merge.conflicts.is_empty() { + let (status, suggestions) = if merge_preview.conflicts.is_empty() { let mut suggestions = vec![StatusSuggestion { - command: format!("trail agent land {lane}"), - reason: format!( - "create a Git commit using default message `{}` and fast-forward the current branch", - default_agent_apply_message_for_task(&view.task) - ), - }]; + command: format!("trail agent land {lane}"), + reason: format!( + "create a Git commit using default message `{}` and fast-forward the current branch", + default_agent_apply_message_for_task(&view.task) + ), + }]; agent_push_suggestion( &mut suggestions, format!("trail agent finish {lane}"), @@ -3040,26 +2970,49 @@ impl Trail { task: view.task, status, dry_run, - git_apply_plan: plan, + git_apply_plan: preview_plan, recorded, - merge: Some(merge), + merge: Some(merge_preview), git_export: None, fast_forwarded: false, + performance: self.git_handoff_metrics_report(), warnings: Vec::new(), suggestions, }); } + let git_state = self.tracked_git_state(&git_identity)?; + validate_git_publication_state(&git_identity.head, &git_state)?; + self.ensure_git_identity_unchanged(&git_identity)?; + + let merge = self.merge_lane_user_with_options(&lane, &crab_branch, false, true)?; + self.set_git_changed_path_count(merge.changed_paths.len() as u64); + let range = if merge.changed_paths.is_empty() { + None + } else { + Some(format!("{}..{}", target_ref.change_id.0, merge.operation.0)) + }; + let plan = AgentGitApplyPlan { + crab_branch: crab_branch.clone(), + git_branch, + base_change: target_ref.change_id.clone(), + result_change: range.as_ref().map(|_| merge.operation.clone()), + range: range.clone(), + would_record, + would_create_git_commit: range.is_some(), + would_fast_forward: range.is_some(), + }; let git_export = if let Some(range) = &range { let view = self.agent_task_view(&lane)?; let message = message .filter(|value| !value.trim().is_empty()) .unwrap_or_else(|| default_agent_apply_message_for_task(&view.task)); - Some(self.git_export_commit(range, &message)?) + Some(self.git_export_commit_mapped(range, &message, Some(git_state))?) } else { None }; if let Some(export) = &git_export { + self.ensure_git_identity_unchanged(&git_identity)?; self.git_fast_forward(&export.commit)?; } let view = self.agent_task_view(&lane)?; @@ -3076,6 +3029,7 @@ impl Trail { merge: Some(merge), git_export, fast_forwarded: range.is_some(), + performance: self.git_handoff_metrics_report(), warnings: Vec::new(), suggestions: vec![StatusSuggestion { command: format!("trail agent view {lane}"), @@ -3187,7 +3141,10 @@ impl Trail { limit: usize, ) -> Result { let review = self.lane_review_packet(&lane.record.name, limit)?; - let transcript = self.transcript(&lane.record.name).ok(); + let transcript = self + .preferred_agent_session_id(&review)? + .and_then(|session_id| self.transcript(&session_id).ok()) + .or_else(|| self.transcript(&lane.record.name).ok()); self.agent_task_from_review(&review, transcript.as_ref()) } @@ -3205,35 +3162,28 @@ impl Trail { let session_id = acp_session .as_ref() .map(|session| session.trail_session_id.clone()) - .or_else(|| { - review - .recent_sessions - .first() - .map(|session| session.session_id.clone()) - }); - let latest_checkpoint = transcript - .and_then(|report| { - report - .turns - .iter() - .rev() - .find_map(|turn| turn.checkpoint.clone()) - }) - .or_else(|| { - review - .recent_operations - .iter() - .find(|operation| { - matches!( - operation.kind, - OperationKind::LaneRecord - | OperationKind::LanePatch - | OperationKind::LaneMerge - | OperationKind::LaneRewind - ) + .or(self.preferred_agent_session_id(review)?); + let latest_checkpoint = + review + .recent_operations + .iter() + .find(|operation| { + matches!( + operation.kind, + OperationKind::LaneRecord + | OperationKind::LanePatch + | OperationKind::LaneMerge + | OperationKind::LaneRewind + ) + }) + .map(|operation| operation.change_id.clone()) + .or_else(|| { + transcript.and_then(|report| { + report.turns.iter().rev().find_map(|turn| { + turn.turn.after_change.clone().or(turn.checkpoint.clone()) + }) }) - .map(|operation| operation.change_id.clone()) - }); + }); let turns = transcript .map(|report| report.turns.len()) .unwrap_or(review.recent_sessions.len()); @@ -3285,6 +3235,39 @@ impl Trail { }) } + fn preferred_agent_session_id( + &self, + review: &LaneReviewPacketReport, + ) -> Result> { + let mut native_sessions = self + .list_lane_agent_sessions(None, None, 10_000)? + .into_iter() + .filter(|session| session.lane_id == review.lane.record.lane_id) + .collect::>(); + native_sessions.sort_by(|left, right| { + right + .status + .eq(&AgentCapturePhase::Active) + .cmp(&left.status.eq(&AgentCapturePhase::Active)) + .then_with(|| right.updated_at.cmp(&left.updated_at)) + .then_with(|| right.created_at.cmp(&left.created_at)) + }); + if let Some(session) = native_sessions.first() { + return Ok(Some(session.trail_session_id.clone())); + } + + let mut sessions = review.recent_sessions.clone(); + sessions.sort_by(|left, right| { + right + .status + .eq("active") + .cmp(&left.status.eq("active")) + .then_with(|| right.started_at.cmp(&left.started_at)) + .then_with(|| right.session_id.cmp(&left.session_id)) + }); + Ok(sessions.first().map(|session| session.session_id.clone())) + } + fn resolve_agent_selector(&self, selector: &str) -> Result> { if selector == "latest" { return self.latest_agent_lane(); @@ -3295,6 +3278,9 @@ impl Trail { if let Some(acp) = self.try_lane_acp_session(selector)? { return self.resolve_lane_handle(&acp.lane_id).map(Some); } + if let Some(native) = self.try_lane_agent_session_by_native_id(selector)? { + return self.resolve_lane_handle(&native.lane_id).map(Some); + } if let Some(session) = self.try_lane_session(selector)? { return self.resolve_lane_handle(&session.lane_id).map(Some); } @@ -3975,17 +3961,20 @@ impl Trail { view: &AgentTaskViewReport, change_id: &str, ) -> Result { + let canonical_change_id = ChangeId::from_checkpoint_alias(change_id) + .map(|change| change.0) + .unwrap_or_else(|| change_id.to_string()); if let Some(transcript) = &view.transcript { if let Some(turn) = transcript.turns.iter().find(|turn| { turn.checkpoint .as_ref() .or(turn.turn.after_change.as_ref()) - .is_some_and(|checkpoint| checkpoint.0 == change_id) + .is_some_and(|checkpoint| checkpoint.0 == canonical_change_id) }) { return self.agent_diff_target_from_turn(turn, "checkpoint", change_id.to_string()); } } - self.agent_diff_target_for_operation(view, change_id, "checkpoint") + self.agent_diff_target_for_operation(view, &canonical_change_id, "checkpoint") } fn resolve_agent_rewind_target(&self, lane: &str, target: &str) -> Result { @@ -3995,6 +3984,9 @@ impl Trail { "agent rewind target cannot be empty".to_string(), )); } + if let Some(change_id) = ChangeId::from_checkpoint_alias(target) { + return Ok(change_id.0); + } let normalized = target.to_ascii_lowercase(); if !agent_rewind_target_needs_resolution(&normalized) { return Ok(target.to_string()); @@ -4123,46 +4115,35 @@ impl Trail { Ok(normalized) } - fn current_git_branch(&self) -> Result> { - if self.current_git_state()?.is_none() { - return Ok(None); + fn ensure_git_head_matches_root(&self, root_id: &ObjectId, git_head: &str) -> Result<()> { + if self.git_clean_head_matches_root_mapping(git_head, root_id)? { + return Ok(()); } - let branch = self.git_output(&["branch".to_string(), "--show-current".to_string()])?; - Ok((!branch.trim().is_empty()).then_some(branch)) + Err(Error::GitMappingRequired(format!( + "clean Git HEAD `{git_head}` has no mapping for Trail root `{}`", + root_id.0 + ))) } - fn ensure_git_head_matches_root( - &self, - change_id: &ChangeId, - root_id: &ObjectId, - git_head: Option<&str>, - crab_branch: &str, - ) -> Result<()> { - let Some(git_head) = git_head else { - return Err(Error::Git( - "agent apply requires a Git HEAD commit before it can fast-forward".to_string(), - )); - }; - if self.ensure_git_clean_head_root_mapping(crab_branch, change_id, root_id, git_head)? { + fn ensure_git_identity_unchanged(&self, expected: &GitIdentity) -> Result<()> { + let current = self.current_git_identity()?; + if current.as_ref().is_some_and(|identity| { + identity.head == expected.head && identity.branch == expected.branch + }) { return Ok(()); } - let files = self.load_root_files(root_id)?; - let crab_tree = self.git_write_tree(&files)?; - let git_tree = - self.git_output(&["rev-parse".to_string(), format!("{git_head}^{{tree}}")])?; - if crab_tree == git_tree { - self.insert_git_mapping_for_state( - "verify", - crab_branch, - change_id, - root_id, - Some(git_head.to_string()), - false, - )?; - return Ok(()); - } - Err(Error::Git(format!( - "current Git HEAD does not match Trail branch `{crab_branch}`; run `trail git import-update --branch {crab_branch}` or apply from a Git branch that matches Trail `{crab_branch}`" + Err(Error::GitHeadChanged(format!( + "expected Git HEAD `{}` on branch `{}`, found `{}` on branch `{}`", + expected.head, + expected.branch.as_deref().unwrap_or(""), + current + .as_ref() + .map(|identity| identity.head.as_str()) + .unwrap_or(""), + current + .as_ref() + .and_then(|identity| identity.branch.as_deref()) + .unwrap_or("") ))) } @@ -5196,7 +5177,7 @@ fn agent_empty_action_palette_value() -> serde_json::Value { "task": null, "summary": "No agent task is recorded yet. Set up an editor, verify the provider, or start a terminal task.", "next": { - "command": "trail agent setup", + "command": "trail agent acp setup claude-code --editor vscode", "reason": "print a stable editor config that creates fresh Trail tasks automatically" }, "actions": agent_empty_action_palette_actions() @@ -5206,10 +5187,10 @@ fn agent_empty_action_palette_value() -> serde_json::Value { fn agent_empty_action_palette_actions() -> Vec { vec![ agent_static_action( - "setup_vscode", + "acp_setup_vscode", "Set up VS Code", "setup", - "trail agent setup", + "trail agent acp setup claude-code --editor vscode", "print a copyable ACP editor config that creates fresh task lanes automatically", "read_only", false, @@ -5218,16 +5199,16 @@ fn agent_empty_action_palette_actions() -> Vec { "doctor_claude_code", "Check Claude Code", "doctor", - "trail agent doctor --provider claude-code", + "trail agent doctor claude-code", "verify Trail workspace readiness and provider availability", "read_only", false, ), agent_static_action( - "setup_codex_vscode", + "acp_setup_codex_vscode", "Set up VS Code for Codex", "setup", - "trail agent setup --provider codex", + "trail agent acp setup codex --editor vscode", "print a copyable Codex ACP editor config that creates fresh task lanes automatically", "read_only", false, @@ -5236,16 +5217,16 @@ fn agent_empty_action_palette_actions() -> Vec { "doctor_codex", "Check Codex", "doctor", - "trail agent doctor --provider codex", + "trail agent doctor codex", "verify Trail workspace readiness and provider availability", "read_only", false, ), agent_static_action( - "setup_cursor_vscode", + "acp_setup_cursor_vscode", "Set up VS Code for Cursor", "setup", - "trail agent setup --provider cursor", + "trail agent acp setup cursor --editor vscode", "print a copyable Cursor ACP editor config that creates fresh task lanes automatically", "read_only", false, @@ -5254,7 +5235,7 @@ fn agent_empty_action_palette_actions() -> Vec { "doctor_cursor", "Check Cursor", "doctor", - "trail agent doctor --provider cursor", + "trail agent doctor cursor", "verify Trail workspace readiness and provider availability", "read_only", false, @@ -5263,7 +5244,7 @@ fn agent_empty_action_palette_actions() -> Vec { "start_terminal_task", "Start terminal task", "start", - "trail agent start --provider claude-code", + "trail agent start claude-code", "launch a fresh materialized terminal task when you are not using an editor", "open_world", true, @@ -5272,7 +5253,7 @@ fn agent_empty_action_palette_actions() -> Vec { "start_gemini_task", "Start Gemini task", "start", - "trail agent start --provider gemini", + "trail agent start gemini", "launch Gemini CLI in a fresh materialized Trail task lane", "open_world", true, @@ -5572,7 +5553,7 @@ fn agent_inbox_next_for_task(task: &AgentTaskReport) -> StatusSuggestion { let lane = &task.lane; match &task.status { AgentTaskStatus::Empty => StatusSuggestion { - command: "trail agent setup".to_string(), + command: "trail agent acp setup claude-code --editor vscode".to_string(), reason: "configure an editor once, then start an agent task".to_string(), }, AgentTaskStatus::Active => StatusSuggestion { @@ -5858,7 +5839,10 @@ fn agent_guide_current_state(task: Option<&AgentTaskReport>, next: &AgentNextRep task.tool_events, next.primary.reason ), - None => format!("No agent task is recorded yet. Next: {}.", next.primary.reason), + None => format!( + "No agent task is recorded yet. Next: {}.", + next.primary.reason + ), } } @@ -5870,19 +5854,19 @@ fn agent_guide_steps( return vec![ agent_guide_step( "Connect an editor", - "trail agent setup", + "trail agent acp setup claude-code --editor vscode", "print a stable ACP editor config that creates fresh Trail tasks automatically", "once per editor setup", ), agent_guide_step( "Check the provider", - "trail agent doctor --provider claude-code", + "trail agent doctor claude-code", "verify Trail and the provider before the first real session", "before or after setup when something does not connect", ), agent_guide_step( "Start in terminal", - "trail agent start --provider claude-code", + "trail agent start claude-code", "launch a fresh materialized terminal task when you are not using an ACP editor", "when you want to work from the shell", ), @@ -7433,33 +7417,35 @@ fn agent_confidence_factors( ) -> Vec { let mut factors = Vec::new(); - let (review_state, review_delta, review_message, review_command) = - match progress.status.as_str() { - "up_to_date" => ( - "pass", - 0, - "current checkpoint has been marked reviewed".to_string(), - Some(format!("trail agent review-flow {lane}")), - ), - "new_changes" => ( - "warn", - -18, - format!( - "{} changed file(s) and {} changed line(s) have not been reviewed since the last marker", - progress.changed_paths, progress.changed_lines - ), - Some(format!("trail agent review-flow {lane}")), + let (review_state, review_delta, review_message, review_command) = match progress + .status + .as_str() + { + "up_to_date" => ( + "pass", + 0, + "current checkpoint has been marked reviewed".to_string(), + Some(format!("trail agent review-flow {lane}")), + ), + "new_changes" => ( + "warn", + -18, + format!( + "{} changed file(s) and {} changed line(s) have not been reviewed since the last marker", + progress.changed_paths, progress.changed_lines ), - _ => ( - "warn", - -22, - format!( - "{} changed file(s) and {} changed line(s) have not been marked reviewed yet", - progress.changed_paths, progress.changed_lines - ), - Some(format!("trail agent review-flow {lane}")), + Some(format!("trail agent review-flow {lane}")), + ), + _ => ( + "warn", + -22, + format!( + "{} changed file(s) and {} changed line(s) have not been marked reviewed yet", + progress.changed_paths, progress.changed_lines ), - }; + Some(format!("trail agent review-flow {lane}")), + ), + }; factors.push(agent_confidence_factor( "review", review_state, @@ -8716,7 +8702,10 @@ fn agent_diagnosis_assessment( evidence.push(format!("Git preflight failed: {error}")); } if let Some(checkpoint) = &ready.task.latest_checkpoint { - evidence.push(format!("latest checkpoint `{}`", checkpoint.0)); + evidence.push(format!( + "latest checkpoint `{}`", + checkpoint.checkpoint_alias() + )); } evidence.push(format!( "{} friendly checkpoint target(s) available", @@ -10105,7 +10094,7 @@ fn agent_new_summary( return match reviewed { Some(marker) => format!( "No new changes touched `{path}` since reviewed checkpoint `{}`.", - marker.checkpoint.0 + marker.checkpoint.checkpoint_alias() ), None => format!( "`{path}` has no recorded changes in {}.", @@ -10127,7 +10116,7 @@ fn agent_new_summary( format!( "{} has no new changes{scope} since reviewed checkpoint `{}`.", agent_task_label(task), - marker.checkpoint.0 + marker.checkpoint.checkpoint_alias() ) }) .unwrap_or_else(|| { @@ -10138,8 +10127,8 @@ fn agent_new_summary( }), "new_changes" => { let checkpoint = reviewed - .map(|marker| marker.checkpoint.0.as_str()) - .unwrap_or("task start"); + .map(|marker| marker.checkpoint.checkpoint_alias()) + .unwrap_or_else(|| "task start".to_string()); format!( "{} has {} new changed file(s){scope} and {} changed line(s) since `{checkpoint}`. {}", agent_task_label(task), @@ -10266,18 +10255,18 @@ fn agent_mark_reviewed_summary( Some(previous) if previous.checkpoint == marker.checkpoint => format!( "{} was already reviewed at checkpoint `{}`; refreshed the reviewed marker.", agent_task_label(task), - marker.checkpoint.0 + marker.checkpoint.checkpoint_alias() ), Some(previous) => format!( "{} marked reviewed at checkpoint `{}`; previous reviewed checkpoint was `{}`.", agent_task_label(task), - marker.checkpoint.0, - previous.checkpoint.0 + marker.checkpoint.checkpoint_alias(), + previous.checkpoint.checkpoint_alias() ), None => format!( "{} marked reviewed at checkpoint `{}`.", agent_task_label(task), - marker.checkpoint.0 + marker.checkpoint.checkpoint_alias() ), } } @@ -10305,18 +10294,18 @@ fn agent_mark_file_reviewed_summary( Some(previous) if previous.checkpoint == marker.checkpoint => format!( "`{path}` in {} was already marked reviewed at checkpoint `{}`; refreshed the file marker.", agent_task_label(task), - marker.checkpoint.0 + marker.checkpoint.checkpoint_alias() ), Some(previous) => format!( "`{path}` in {} marked reviewed at checkpoint `{}`; previous file marker was `{}`.", agent_task_label(task), - marker.checkpoint.0, - previous.checkpoint.0 + marker.checkpoint.checkpoint_alias(), + previous.checkpoint.checkpoint_alias() ), None => format!( "`{path}` in {} marked reviewed at checkpoint `{}`.", agent_task_label(task), - marker.checkpoint.0 + marker.checkpoint.checkpoint_alias() ), } } @@ -10345,10 +10334,18 @@ fn agent_archive_summary( ) -> String { let label = agent_task_label(task); match (archived, previous_archived) { - (true, true) => format!("{label} was already archived; it remains hidden from the default agent inbox."), - (true, false) => format!("{label} archived; it is hidden from the default agent inbox and can be restored with `agent unarchive`."), - (false, true) => format!("{label} restored; it will appear in the default agent inbox again."), - (false, false) => format!("{label} was not archived; it remains visible in the default agent inbox."), + (true, true) => { + format!("{label} was already archived; it remains hidden from the default agent inbox.") + } + (true, false) => format!( + "{label} archived; it is hidden from the default agent inbox and can be restored with `agent unarchive`." + ), + (false, true) => { + format!("{label} restored; it will appear in the default agent inbox again.") + } + (false, false) => { + format!("{label} was not archived; it remains visible in the default agent inbox.") + } } } @@ -10850,7 +10847,10 @@ fn agent_report_markdown( view.task.turns, view.task.tool_events )); if let Some(checkpoint) = &view.task.latest_checkpoint { - out.push_str(&format!("- Latest checkpoint: `{}`\n", checkpoint.0)); + out.push_str(&format!( + "- Latest checkpoint: `{}`\n", + checkpoint.checkpoint_alias() + )); } out.push_str(&format!( "- Risk: {:?} ({}/100)\n\n", @@ -10916,7 +10916,10 @@ fn agent_report_markdown( turn.changed_paths.len() )); if let Some(checkpoint) = &turn.checkpoint { - out.push_str(&format!(" - Checkpoint: `{}`\n", checkpoint.0)); + out.push_str(&format!( + " - Checkpoint: `{}`\n", + checkpoint.checkpoint_alias() + )); } for tool in &turn.tool_summaries { out.push_str(&format!(" - Tool: {tool}\n")); @@ -10965,7 +10968,10 @@ fn agent_receipt_markdown( report.task.turns, report.task.tool_events )); if let Some(checkpoint) = &report.task.latest_checkpoint { - out.push_str(&format!("- Latest checkpoint: `{}`\n", checkpoint.0)); + out.push_str(&format!( + "- Latest checkpoint: `{}`\n", + checkpoint.checkpoint_alias() + )); } out.push_str(&format!( "- Risk: {:?} ({}/100)\n\n", @@ -11059,7 +11065,10 @@ fn agent_receipt_markdown( turn.changed_paths.len() )); if let Some(checkpoint) = &turn.checkpoint { - out.push_str(&format!(" - Checkpoint: `{}`\n", checkpoint.0)); + out.push_str(&format!( + " - Checkpoint: `{}`\n", + checkpoint.checkpoint_alias() + )); } for tool in &turn.tool_summaries { out.push_str(&format!(" - Tool: {tool}\n")); @@ -11132,7 +11141,10 @@ fn agent_handoff_markdown( report.task.tool_events )); if let Some(checkpoint) = &report.task.latest_checkpoint { - out.push_str(&format!("- Latest checkpoint: `{}`\n", checkpoint.0)); + out.push_str(&format!( + "- Latest checkpoint: `{}`\n", + checkpoint.checkpoint_alias() + )); } if let Some(workdir) = &report.task.workdir { out.push_str(&format!("- Workdir: `{workdir}`\n")); @@ -11236,7 +11248,10 @@ fn agent_handoff_markdown( turn.changed_paths.len() )); if let Some(checkpoint) = &turn.checkpoint { - out.push_str(&format!(" - Checkpoint: `{}`\n", checkpoint.0)); + out.push_str(&format!( + " - Checkpoint: `{}`\n", + checkpoint.checkpoint_alias() + )); } for tool in &turn.tool_summaries { out.push_str(&format!(" - Tool: {tool}\n")); @@ -11291,7 +11306,10 @@ fn agent_pr_body(receipt: &AgentReceiptReport) -> String { receipt.risk.level, receipt.risk.score )); if let Some(checkpoint) = &receipt.latest_checkpoint { - out.push_str(&format!("- Trail checkpoint: `{}`\n", checkpoint.0)); + out.push_str(&format!( + "- Trail checkpoint: `{}`\n", + checkpoint.checkpoint_alias() + )); } out.push_str(&format!("- Agent task: `{}`\n\n", receipt.task.name)); @@ -11362,7 +11380,10 @@ fn agent_pr_body(receipt: &AgentReceiptReport) -> String { turn.changed_paths.len() )); if let Some(checkpoint) = &turn.checkpoint { - out.push_str(&format!(" - Checkpoint: `{}`\n", checkpoint.0)); + out.push_str(&format!( + " - Checkpoint: `{}`\n", + checkpoint.checkpoint_alias() + )); } } out.push('\n'); @@ -11397,7 +11418,7 @@ fn agent_checkpoint_entry(lane: &str, group: &AgentChangeGroup) -> AgentCheckpoi if is_turn { format!("turn:{}", group.index) } else { - checkpoint.0.clone() + checkpoint.checkpoint_alias() } }); let rewind_before_command = before_target @@ -11692,7 +11713,7 @@ fn agent_risk_summary(level: &AgentRiskLevel, score: u8, task: &AgentTaskReport) fn agent_task_suggestions(lane: &str, status: &AgentTaskStatus) -> Vec { match status { AgentTaskStatus::Empty => vec![StatusSuggestion { - command: "trail agent setup".to_string(), + command: "trail agent acp setup claude-code --editor vscode".to_string(), reason: "configure an editor once, then start an agent task".to_string(), }], AgentTaskStatus::Dirty => vec![StatusSuggestion { @@ -11735,7 +11756,7 @@ fn agent_next_report_from_view( "setup", "No agent task is available.", StatusSuggestion { - command: "trail agent setup".to_string(), + command: "trail agent acp setup claude-code --editor vscode".to_string(), reason: "configure an editor once, then start an agent task".to_string(), }, ), @@ -11985,66 +12006,6 @@ fn agent_brief_suggestions(lane: &str, next: &AgentNextReport) -> Vec String { - match editor { - "vscode" => serde_json::to_string_pretty(&serde_json::json!({ - (format!("Trail {}", provider_display_name(provider))): { - "command": command.first().cloned().unwrap_or_default(), - "args": command.iter().skip(1).cloned().collect::>(), - "env": {} - } - })) - .unwrap_or_else(|_| "{}".to_string()), - "zed" => serde_json::to_string_pretty(&serde_json::json!({ - "agent_servers": { - (format!("trail-{}", provider)): { - "type": "custom", - "command": command.first().cloned().unwrap_or_default(), - "args": command.iter().skip(1).cloned().collect::>() - } - } - })) - .unwrap_or_else(|_| "{}".to_string()), - _ => format!("ACP command:\n{}", shell_join(command)), - } -} - -fn agent_terminal_setup_snippet( - profile: &AcpProviderProfile, - command: &[String], - mcp_command: &[String], -) -> String { - let mut snippet = format!( - "Terminal command:\n{}\n\nTrail MCP server command:\n{}", - shell_join(command), - shell_join(mcp_command) - ); - if profile.supports_mcp { - snippet.push_str(&format!( - "\n\nRegister the MCP command in {} if you want direct Trail context tools inside the agent.", - profile.display_name - )); - } else { - snippet.push_str(&format!( - "\n\n{} is configured here as a terminal agent; use the terminal command to run it in an isolated Trail task lane.", - profile.display_name - )); - } - snippet -} - -fn provider_display_name(provider: &str) -> String { - match provider { - "claude-code" | "claude" => "Claude Code".to_string(), - "codex" | "codex-cli" | "openai-codex" => "Codex".to_string(), - "cursor" | "cursor-agent" => "Cursor".to_string(), - "gemini" | "gemini-cli" => "Gemini CLI".to_string(), - "aider" => "Aider".to_string(), - "opencode" | "open-code" => "OpenCode".to_string(), - other => other.to_string(), - } -} - fn sanitize_agent_ref_component(value: &str) -> String { let mut out = String::new(); for ch in value.chars() { diff --git a/trail/src/db/change_ledger/activation.rs b/trail/src/db/change_ledger/activation.rs new file mode 100644 index 0000000..3f4a2b0 --- /dev/null +++ b/trail/src/db/change_ledger/activation.rs @@ -0,0 +1,130 @@ +use serde::Serialize; +use sha2::{Digest, Sha256}; + +const APPROVED_PRODUCER_INVENTORY_SHA256: &str = + "67027c4bcfba0f3105833978637fe5e81c9cbdb43ba51cbd1a58026a2e067185"; +const APPROVED_RAW_MUTATION_INVENTORY_SHA256: &str = + "668a39497a58335e57b02a0c6fff3e0a0c127b06e0aa63d4cf93255f3942c943"; +const APPROVED_ACTIVATION_AUDIT_SHA256: &str = + "f538c5750f234ed5b164536ce0603dc92df17324429c7487e225f789a0e27c70"; +const ACTIVATION_AUDIT_MANIFEST: &str = concat!( + "trail-changed-path-activation-v1\n", + "schema=18\n", + "producer=67027c4bcfba0f3105833978637fe5e81c9cbdb43ba51cbd1a58026a2e067185\n", + "raw=668a39497a58335e57b02a0c6fff3e0a0c127b06e0aa63d4cf93255f3942c943\n", + "linux_suite=changed_path_ledger_linux\n", + "macos_suite=changed_path_ledger_macos\n", + "recovery_suite=changed_path_ledger_recovery\n", + "activation_suite=changed_path_ledger_activation\n", + "scale_workflow=CLI Scale Benchmark/changed-path-ledger\n", + "native_workflow=Changed-path Ledger Native Gates/native\n", + "exact_sha_tag_gate=Release Automation/exact-sha-native-ledger\n", + "exact_sha_publish_gate=Release/custom-changed-path-ledger-native\n", + "gate_schema=changed-path-thresholds-v1\n", + "metrics_schema=operation-metrics-jsonl-v1\n", +); + +// These bytes are compiled into the checked binary. Activation never trusts +// a mutable report, CI artifact, environment variable, or file discovered at +// runtime to decide whether command authority is enabled. +const PRODUCER_INVENTORY: &[u8] = + include_bytes!("../../../tests/fixtures/changed_path_producers.v1"); +const RAW_MUTATION_INVENTORY: &[u8] = + include_bytes!("../../../tests/fixtures/changed_path_raw_mutations.v1"); + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub(crate) struct ActivationEvidence { + pub(crate) schema_hard_cutover: bool, + pub(crate) producer_inventory_complete: bool, + pub(crate) linux_native_suite: bool, + pub(crate) macos_native_suite: bool, + pub(crate) crash_matrix: bool, + pub(crate) corruption_matrix: bool, + pub(crate) scale_gates: bool, + pub(crate) metrics_jsonl: bool, + pub(crate) exact_sha_tag_gate: bool, + pub(crate) exact_sha_publish_gate: bool, + pub(crate) producer_inventory_sha256: String, + pub(crate) raw_mutation_inventory_sha256: String, + pub(crate) activation_audit_sha256: String, +} + +impl ActivationEvidence { + pub(crate) fn from_checked_build() -> std::result::Result { + let producer_inventory_sha256 = sha256(PRODUCER_INVENTORY); + let raw_mutation_inventory_sha256 = sha256(RAW_MUTATION_INVENTORY); + let activation_audit_sha256 = sha256(ACTIVATION_AUDIT_MANIFEST.as_bytes()); + if producer_inventory_sha256 != APPROVED_PRODUCER_INVENTORY_SHA256 { + return Err(format!( + "controlled producer inventory changed: expected {APPROVED_PRODUCER_INVENTORY_SHA256}, compiled {producer_inventory_sha256}" + )); + } + if raw_mutation_inventory_sha256 != APPROVED_RAW_MUTATION_INVENTORY_SHA256 { + return Err(format!( + "raw mutation inventory changed: expected {APPROVED_RAW_MUTATION_INVENTORY_SHA256}, compiled {raw_mutation_inventory_sha256}" + )); + } + if activation_audit_sha256 != APPROVED_ACTIVATION_AUDIT_SHA256 { + return Err(format!( + "compiled activation audit changed: expected {APPROVED_ACTIVATION_AUDIT_SHA256}, compiled {activation_audit_sha256}" + )); + } + let checked_activation_audit = activation_audit_sha256 == APPROVED_ACTIVATION_AUDIT_SHA256; + Ok(Self { + schema_hard_cutover: checked_activation_audit + && super::super::TRAIL_SCHEMA_VERSION == 18, + producer_inventory_complete: checked_activation_audit, + // These fields declare the checked build contract. Exact-SHA + // workflow dependencies, rather than this self-hash, authorize a + // release tag and every cargo-dist publication phase. + linux_native_suite: checked_activation_audit, + macos_native_suite: checked_activation_audit, + crash_matrix: checked_activation_audit, + corruption_matrix: checked_activation_audit, + scale_gates: checked_activation_audit, + metrics_jsonl: checked_activation_audit, + exact_sha_tag_gate: checked_activation_audit, + exact_sha_publish_gate: checked_activation_audit, + producer_inventory_sha256, + raw_mutation_inventory_sha256, + activation_audit_sha256, + }) + } + + pub(crate) fn is_complete(&self) -> bool { + self.schema_hard_cutover + && self.producer_inventory_complete + && self.linux_native_suite + && self.macos_native_suite + && self.crash_matrix + && self.corruption_matrix + && self.scale_gates + && self.metrics_jsonl + && self.exact_sha_tag_gate + && self.exact_sha_publish_gate + } +} + +pub(crate) fn ledger_authority_enabled_for(platform: &str, evidence: &ActivationEvidence) -> bool { + matches!(platform, "linux" | "macos") && evidence.is_complete() +} + +fn sha256(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_platform_and_incomplete_evidence_are_hard_off() { + let complete = ActivationEvidence::from_checked_build().unwrap(); + assert!(!ledger_authority_enabled_for("windows", &complete)); + assert!(!ledger_authority_enabled_for("freebsd", &complete)); + let mut incomplete = complete; + incomplete.corruption_matrix = false; + assert!(!ledger_authority_enabled_for("linux", &incomplete)); + assert!(!ledger_authority_enabled_for("macos", &incomplete)); + } +} diff --git a/trail/src/db/change_ledger/daemon.rs b/trail/src/db/change_ledger/daemon.rs new file mode 100644 index 0000000..ba9971b --- /dev/null +++ b/trail/src/db/change_ledger/daemon.rs @@ -0,0 +1,3275 @@ +use std::collections::HashMap; +use std::ffi::OsString; +use std::fs; +use std::fs::OpenOptions; +use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; +use std::path::Path; +use std::time::Duration; + +use getrandom::getrandom; +use rusqlite::{named_params, params, OptionalExtension, Transaction, TransactionBehavior}; +use sha2::{Digest, Sha256}; + +use super::FencedCandidateSnapshot; +use super::{ + compile_policy, fold_observer_interval, raw_event_invalidates_policy, reconcile_full_with_tail, + revalidate_compiled_policy, BaselineIdentity, CompiledPolicy, DaemonLaunchBinding, DurableCut, + EvidenceCut, EvidenceSource, ExpectedScope, FilesystemIdentity, IntentEvidence, IntentId, + ObserverEvent, ObserverFence, ObserverLease, PersistedLogLimits, PolicyCompileContext, + PolicyDependencyMetrics, PolicyIdentity, ProviderCapabilities, ProviderIdentity, + QualifiedFilesystemProof, QualifiedObserver, RecoveredTail, RecoveryScope, ScopeId, + ScopeIdentity, ScopeKind, SegmentWriter, +}; +use crate::error::{Error, Result}; +use crate::Trail; + +pub(crate) struct WorkspaceDaemonProof { + pub(crate) scope_id: String, + pub(crate) epoch: u64, + pub(crate) observer_owner_token: String, + pub(crate) daemon_launch_nonce: Option, + pub(crate) cut: EvidenceCut, + pub(crate) reconcile_report: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct VerifiedStaleWorkspaceOwner { + pub(crate) stale_pid: u32, + pub(crate) process_start_identity: String, + pub(crate) scope_id: String, + pub(crate) epoch: u64, + pub(crate) observer_owner_token: String, + pub(crate) daemon_launch_nonce: String, +} + +#[derive(Clone, Debug)] +pub(crate) struct WorkspaceDaemonLaunchIdentity { + pub(crate) nonce: String, + pub(crate) pid: u32, + pub(crate) process_start_identity: String, +} + +#[derive(Default)] +pub(crate) struct ChangedPathDaemonRegistry { + pub(super) workspace: Option, + pub(super) materialized_lanes: HashMap, +} + +struct DaemonScopeTarget { + root: std::path::PathBuf, + identity: ScopeIdentity, + baseline: BaselineIdentity, +} + +const MAX_STARTUP_POLICY_RETRIES: usize = 2; + +#[cfg(test)] +type WorkspaceRetryBoundaryHook = Box Result<()> + Send>; +#[cfg(test)] +static WORKSPACE_RETRY_BOUNDARY_HOOK: std::sync::OnceLock< + std::sync::Mutex>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +fn install_workspace_retry_boundary_hook( + scope_id: ScopeId, + hook: impl FnOnce(&Trail) -> Result<()> + Send + 'static, +) { + WORKSPACE_RETRY_BOUNDARY_HOOK + .get_or_init(|| std::sync::Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .insert(scope_id, Box::new(hook)); +} + +#[cfg(test)] +fn run_workspace_retry_boundary_hook(db: &Trail) -> Result<()> { + let scope_id = workspace_daemon_target(db)?.identity.scope_id; + let hook = WORKSPACE_RETRY_BOUNDARY_HOOK + .get_or_init(|| std::sync::Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .remove(&scope_id); + match hook { + Some(hook) => hook(db), + None => Ok(()), + } +} + +pub(crate) fn prepare_workspace_daemon( + db: &Trail, + replace_stale_owner: bool, +) -> Result { + prepare_workspace_daemon_inner(db, replace_stale_owner, None, None) +} + +pub(crate) fn prepare_workspace_daemon_verified_replacement( + db: &Trail, + verified_stale_owner: VerifiedStaleWorkspaceOwner, + launch: Option, +) -> Result { + prepare_workspace_daemon_inner(db, true, Some(verified_stale_owner), launch) +} + +fn prepare_workspace_daemon_inner( + db: &Trail, + replace_stale_owner: bool, + verified_stale_owner: Option, + launch: Option, +) -> Result { + if daemon_registry(db).workspace.is_some() { + return workspace_daemon_fence(db, None, None); + } + let mut retries = 0_usize; + let mut verified_stale_owner = verified_stale_owner; + loop { + let mut runtime = match WorkspaceDaemonRuntime::start( + db, + workspace_daemon_target(db)?, + replace_stale_owner || retries > 0, + verified_stale_owner.is_none(), + verified_stale_owner.as_ref(), + launch.as_ref(), + ) { + Ok(runtime) => runtime, + Err(error) + if startup_policy_retryable(&error) && retries < MAX_STARTUP_POLICY_RETRIES => + { + verified_stale_owner = workspace_retry_owner_capability(db, launch.as_ref())?; + retries += 1; + #[cfg(test)] + run_workspace_retry_boundary_hook(db)?; + startup_policy_retry_delay(retries); + continue; + } + Err(error) => return Err(error), + }; + match runtime.reconcile(db, "daemon_initial_full_reconciliation") { + Ok(proof) => { + daemon_registry(db).workspace = Some(runtime); + return Ok(proof); + } + Err(error) + if startup_policy_retryable(&error) && retries < MAX_STARTUP_POLICY_RETRIES => + { + verified_stale_owner = workspace_retry_owner_capability(db, launch.as_ref())?; + drop(runtime); + retries += 1; + #[cfg(test)] + run_workspace_retry_boundary_hook(db)?; + startup_policy_retry_delay(retries); + } + Err(error) => return Err(error), + } + } +} + +fn workspace_retry_owner_capability( + db: &Trail, + launch: Option<&WorkspaceDaemonLaunchIdentity>, +) -> Result> { + let Some(launch) = launch else { + return Ok(None); + }; + verified_stale_workspace_owner_for_launch( + db, + launch.pid, + &launch.process_start_identity, + &launch.nonce, + ) +} + +pub(crate) fn prepare_workspace_daemon_launch( + db: &Trail, + launch: WorkspaceDaemonLaunchIdentity, + verified_stale_owner: Option, +) -> Result { + prepare_workspace_daemon_inner( + db, + verified_stale_owner.is_some(), + verified_stale_owner, + Some(launch), + ) +} + +pub(crate) fn verified_stale_workspace_owner_for_launch( + db: &Trail, + stale_pid: u32, + process_start_identity: &str, + daemon_launch_nonce: &str, +) -> Result> { + if stale_pid == 0 || process_start_identity.is_empty() || daemon_launch_nonce.len() != 64 { + return Err(Error::DaemonUnavailable( + "stale workspace daemon launch identity is malformed".into(), + )); + } + let binding = db + .conn + .query_row( + "SELECT scope_id,epoch,owner_token,daemon_pid,daemon_process_start_identity + FROM changed_path_observer_owners + WHERE daemon_launch_nonce=?1", + params![daemon_launch_nonce], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, String>(4)?, + )) + }, + ) + .optional()?; + let Some((scope_id, epoch, owner_token, bound_pid, bound_start)) = binding else { + return Ok(None); + }; + if bound_pid != i64::from(stale_pid) || bound_start != process_start_identity { + return Err(Error::DaemonUnavailable( + "stale workspace daemon publication does not match its atomically persisted launch identity" + .into(), + )); + } + let epoch = u64::try_from(epoch) + .map_err(|_| Error::Corrupt("negative stale daemon owner epoch".into()))?; + Ok(Some(VerifiedStaleWorkspaceOwner { + stale_pid, + process_start_identity: process_start_identity.to_string(), + scope_id, + epoch, + observer_owner_token: owner_token, + daemon_launch_nonce: daemon_launch_nonce.to_string(), + })) +} + +pub(crate) fn prepare_workspace_controlled_projection(db: &mut Trail) -> Result { + prepare_workspace_daemon(db, true)?; + with_workspace_runtime(db, |runtime, db| { + runtime.fence_and_seal(db)?; + Ok(runtime.expected.clone()) + }) +} + +pub(crate) fn workspace_daemon_fence( + db: &Trail, + scope_id: Option<&str>, + epoch: Option, +) -> Result { + let mut runtime = daemon_registry(db).workspace.take().ok_or_else(|| { + Error::DaemonUnavailable("changed-path observer runtime is unavailable".into()) + })?; + runtime.validate_request(scope_id, epoch)?; + let result = runtime.fence(db); + match result { + Ok(proof) => { + daemon_registry(db).workspace = Some(runtime); + Ok(proof) + } + Err(error) if startup_policy_retryable(&error) => { + // A fenced policy dependency changed outside native observer + // coverage. The failed observer has already revoked its durable + // owner; discard it, recompile policy, and establish a fresh full + // reconciliation before continuing the command. + drop(runtime); + prepare_workspace_daemon_inner(db, true, None, None) + } + Err(error) => { + daemon_registry(db).workspace = Some(runtime); + Err(error) + } + } +} + +pub(crate) fn workspace_daemon_reconcile( + db: &Trail, + scope_id: Option<&str>, + epoch: Option, +) -> Result { + let mut runtime = daemon_registry(db).workspace.take().ok_or_else(|| { + Error::DaemonUnavailable("changed-path observer runtime is unavailable".into()) + })?; + runtime.validate_request(scope_id, epoch)?; + let result = runtime.reconcile(db, "daemon_requested_full_reconciliation"); + daemon_registry(db).workspace = Some(runtime); + result +} + +pub(crate) fn workspace_daemon_full_reconcile( + db: &Trail, +) -> Result { + let running = daemon_registry(db).workspace.is_some(); + let proof = if running { + workspace_daemon_reconcile(db, None, None)? + } else { + prepare_workspace_daemon(db, true)? + }; + proof.reconcile_report.ok_or_else(|| { + Error::DaemonUnavailable("workspace daemon returned no reconciliation report".into()) + }) +} + +pub(crate) fn workspace_daemon_ready_proof(db: &Trail) -> Result { + daemon_registry(db) + .workspace + .as_ref() + .ok_or_else(|| { + Error::DaemonUnavailable("changed-path observer runtime is unavailable".into()) + })? + .current_proof() +} + +pub(crate) fn prepare_materialized_lane_daemon( + db: &Trail, + lane: &str, + replace_verified_stale_owner: bool, +) -> Result { + prepare_materialized_lane_daemon_inner(db, lane, replace_verified_stale_owner, true, true) +} + +pub(super) fn prepare_materialized_lane_daemon_verified_replacement( + db: &Trail, + lane: &str, +) -> Result { + prepare_materialized_lane_daemon_inner(db, lane, true, false, false) +} + +fn prepare_materialized_lane_daemon_inner( + db: &Trail, + lane: &str, + replace_verified_stale_owner: bool, + refuse_live_unverified_owner: bool, + publish_marker: bool, +) -> Result { + let lane_id = db.lane_branch(lane)?.lane_id; + if daemon_registry(db) + .materialized_lanes + .contains_key(&lane_id) + { + let proof = with_materialized_lane_runtime(db, &lane_id, |runtime, db| runtime.fence(db))?; + if publish_marker { + db.publish_lane_marker_if_materialized(&lane_id)?; + } + return Ok(proof); + } + let mut retries = 0_usize; + loop { + let target = materialized_lane_daemon_target(db, &lane_id)?; + let mut runtime = match WorkspaceDaemonRuntime::start( + db, + target, + replace_verified_stale_owner, + refuse_live_unverified_owner, + None, + None, + ) { + Ok(runtime) => runtime, + Err(error) + if startup_policy_retryable(&error) && retries < MAX_STARTUP_POLICY_RETRIES => + { + retries += 1; + startup_policy_retry_delay(retries); + continue; + } + Err(error) => { + return Err(Error::DaemonUnavailable(format!( + "materialized-lane observer startup failed: {error}" + ))) + } + }; + match runtime.reconcile(db, "materialized_lane_initial_full_reconciliation") { + Ok(proof) => { + daemon_registry(db) + .materialized_lanes + .insert(lane_id.clone(), runtime); + if publish_marker { + db.publish_lane_marker_if_materialized(&lane_id)?; + } + return Ok(proof); + } + Err(error) + if startup_policy_retryable(&error) && retries < MAX_STARTUP_POLICY_RETRIES => + { + drop(runtime); + retries += 1; + startup_policy_retry_delay(retries); + } + Err(error) => { + return Err(Error::DaemonUnavailable(format!( + "materialized-lane initial full reconciliation failed: {error}" + ))) + } + } + } +} + +fn startup_policy_retryable(error: &Error) -> bool { + match error { + Error::ChangeLedgerReconcileRequired { reason, .. } => { + reason.contains("policy_dependency_invalidated") + || reason.contains("recording policy changed") + } + Error::DaemonUnavailable(reason) => { + reason.contains("recording policy changed during observer startup") + } + _ => false, + } +} + +pub(crate) fn policy_runtime_restart_required(error: &Error) -> bool { + startup_policy_retryable(error) +} + +fn startup_policy_retry_delay(retry: usize) { + let millis = if retry <= 1 { 100 } else { 400 }; + std::thread::sleep(Duration::from_millis(millis)); +} + +pub(crate) fn materialized_lane_daemon_fence( + db: &Trail, + lane: &str, +) -> Result { + let proof = with_materialized_lane_runtime(db, lane, |runtime, db| runtime.fence(db))?; + db.publish_lane_marker_if_materialized(lane)?; + Ok(proof) +} + +/// Establish an exact sidecar boundary immediately before preparing a +/// controlled materialized-lane intent. The returned scope's provider cursor +/// is the start cursor of the newly-opened segment, so the subsequent intent +/// cannot begin ambiguously in the middle of an observer segment. +pub(crate) fn prepare_materialized_lane_controlled_projection( + db: &mut Trail, + lane: &str, +) -> Result { + // Controlled preparation must not write the marker inside the watched + // workdir. Such a write is native observer traffic in the fence-to-seal + // gap. The projection protocol repairs the marker after its terminal SQL + // publication through its existing repair callback. + let lane_id = db.lane_branch(lane)?.lane_id; + let mut reconciled = false; + loop { + prepare_materialized_lane_daemon_verified_replacement(db, &lane_id)?; + match with_materialized_lane_runtime(db, &lane_id, |runtime, db| { + runtime.fence_and_seal(db)?; + Ok(runtime.expected.clone()) + }) { + Ok(expected) => return Ok(expected), + Err(error) + if !reconciled + && requires_reconciliation(&error) + && !startup_policy_retryable(&error) => + { + daemon_registry(db).materialized_lanes.remove(&lane_id); + prepare_materialized_lane_daemon(db, &lane_id, true)?; + reconciled = true; + } + Err(error) => return Err(error), + } + } +} + +pub(crate) fn materialized_lane_daemon_matches_target(db: &Trail, lane: &str) -> Result { + let lane_id = db.lane_branch(lane)?.lane_id; + let registry = daemon_registry(db); + let Some(runtime) = registry.materialized_lanes.get(&lane_id) else { + return Ok(false); + }; + runtime.matches_target(&materialized_lane_daemon_target(db, &lane_id)?) +} + +pub(crate) fn materialized_lane_daemon_reconcile( + db: &Trail, + lane: &str, + reason: &str, +) -> Result { + let proof = + with_materialized_lane_runtime(db, lane, |runtime, db| runtime.reconcile(db, reason))?; + db.publish_lane_marker_if_materialized(lane)?; + Ok(proof) +} + +pub(crate) fn materialized_lane_daemon_full_reconcile( + db: &Trail, + lane: &str, +) -> Result { + let lane_id = db.lane_branch(lane)?.lane_id; + let running = daemon_registry(db) + .materialized_lanes + .contains_key(&lane_id); + let proof = if running { + materialized_lane_daemon_reconcile(db, &lane_id, "user_requested_full_reconciliation")? + } else { + prepare_materialized_lane_daemon(db, &lane_id, true)? + }; + proof.reconcile_report.ok_or_else(|| { + Error::DaemonUnavailable( + "materialized-lane daemon returned no reconciliation report".into(), + ) + }) +} + +pub(crate) fn accept_materialized_lane_daemon_baseline( + db: &Trail, + lane: &str, + expected: &ExpectedScope, + target: &BaselineIdentity, +) -> Result<()> { + with_materialized_lane_runtime(db, lane, |runtime, _| { + runtime.accept_observed_baseline(expected, target) + })?; + db.publish_lane_marker_if_materialized(lane) +} + +/// Run one controlled materialized-lane filesystem interval. The intent must +/// already be durably prepared at the runtime's retained cut. `apply_and_sync` +/// mutates and durably syncs bytes; c1 is then fenced before `pinned_verify` +/// compares the intended paths through descriptor-relative reads. A final c2 +/// retains every post-c1 race and the returned sidecar-backed proof is suitable +/// for `mark_filesystem_applied`. +pub(crate) fn with_materialized_lane_controlled_interval( + db: &mut Trail, + lane: &str, + intent_id: &IntentId, + evidence: &IntentEvidence, + apply_and_sync: A, + pinned_verify: V, +) -> Result +where + A: FnOnce(&mut Trail) -> Result<()>, + V: FnOnce(&mut Trail, &CompiledPolicy, &super::CandidateSnapshot) -> Result<()>, +{ + let lane_id = db.lane_branch(lane)?.lane_id; + with_controlled_interval( + db, + ControlledRuntimeKey::MaterializedLane(lane_id), + intent_id, + evidence, + apply_and_sync, + pinned_verify, + ) +} + +pub(crate) fn with_workspace_controlled_interval( + db: &mut Trail, + intent_id: &IntentId, + evidence: &IntentEvidence, + apply_and_sync: A, + pinned_verify: V, +) -> Result +where + A: FnOnce(&mut Trail) -> Result<()>, + V: FnOnce(&mut Trail, &CompiledPolicy, &super::CandidateSnapshot) -> Result<()>, +{ + with_controlled_interval( + db, + ControlledRuntimeKey::Workspace, + intent_id, + evidence, + apply_and_sync, + pinned_verify, + ) +} + +#[derive(Clone)] +enum ControlledRuntimeKey { + Workspace, + MaterializedLane(String), +} + +fn with_controlled_interval( + db: &mut Trail, + runtime_key: ControlledRuntimeKey, + intent_id: &IntentId, + evidence: &IntentEvidence, + apply_and_sync: A, + pinned_verify: V, +) -> Result +where + A: FnOnce(&mut Trail) -> Result<()>, + V: FnOnce(&mut Trail, &CompiledPolicy, &super::CandidateSnapshot) -> Result<()>, +{ + // Establish/recover the exact lane scope before inspecting the prepared + // intent. A no-op snapshot supplies the authenticated retained c1 cut; it + // must happen before prepare_intent, so callers normally invoke this only + // after an earlier status/producer preparation fence. + ensure_controlled_runtime(db, &runtime_key)?; + let command = controlled_status_command(&runtime_key); + let expected = { + let registry = daemon_registry(db); + controlled_runtime(®istry, &runtime_key) + .ok_or_else(|| Error::DaemonUnavailable("lane runtime disappeared".into()))? + .expected + .clone() + }; + let (intent_scope, expected_root_id, start_cursor): (String, String, Option>) = + db.conn.query_row( + "SELECT scope_id,expected_root_id,start_cursor FROM changed_path_intents + WHERE intent_id=?1 AND lifecycle_state='prepared'", + [&intent_id.0], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + if intent_scope != expected.scope_id.to_text() { + return Err(Error::Conflict(format!( + "intent `{}` belongs to a different changed-path scope", + intent_id.0 + ))); + } + apply_and_sync(db)?; + if !controlled_runtime_matches_target(db, &runtime_key)? { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "controlled lane projection replaced or rebound the pinned root".into(), + command: command.clone(), + }); + } + // c1 closes the controlled write interval before any comparison. An + // external same-path write after this fence can never be acknowledged by + // the intent, even if it races the pinned verification below. + let c1 = + with_controlled_runtime(db, &runtime_key, |runtime, db| runtime.fence_and_seal(db))?.cut; + let (policy, mut candidates) = { + let registry = daemon_registry(db); + let runtime = controlled_runtime(®istry, &runtime_key) + .ok_or_else(|| Error::DaemonUnavailable("lane runtime disappeared".into()))?; + ( + runtime.policy.clone(), + db.changed_path_ledger() + .snapshot_candidates_for_controlled_intent( + &runtime.expected, + &intent_id.0, + start_cursor.as_deref(), + )?, + ) + }; + candidates.cut = c1.clone(); + db.filter_controlled_internal_candidates(&policy, &mut candidates, evidence)?; + let sparse_selection = match &runtime_key { + ControlledRuntimeKey::Workspace => None, + ControlledRuntimeKey::MaterializedLane(lane) => { + db.filter_controlled_lane_sparse_candidates(lane, &policy, &mut candidates)? + } + }; + pinned_verify(db, &policy, &candidates)?; + if !controlled_runtime_matches_target(db, &runtime_key)? { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "controlled lane verification lost its pinned root".into(), + command: command.clone(), + }); + } + // c2 closes the post-c1 comparison interval without another rotation. + // c1 already sealed the controlled segment and advanced to a fresh + // anchor; this ordinary fence leaves the scope exactly at publication_cut + // while every event in (c1,c2] remains pending. + let (c2, publication_durable) = with_controlled_runtime(db, &runtime_key, |runtime, db| { + let proof = runtime.fence(db)?; + let anchor = runtime.tail_anchor.as_ref().ok_or_else(|| { + Error::DaemonUnavailable( + "controlled runtime lost its authenticated publication anchor".into(), + ) + })?; + let durable = runtime + .observer + .authenticated_cut(&runtime.expected, anchor)?; + if durable.last_sequence != proof.cut.sequence + || durable.durable_end_offset != proof.cut.durable_offset + { + return Err(Error::ChangeLedgerReconcileRequired { + scope: runtime.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "publication fence lost its exact sidecar boundary".into(), + command: controlled_status_command(&runtime_key), + }); + } + Ok((proof.cut, durable)) + })?; + if let ControlledRuntimeKey::MaterializedLane(lane) = &runtime_key { + let branch = db.lane_branch(lane)?; + let workdir = std::path::PathBuf::from(branch.workdir.as_deref().ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{}` does not have a materialized workdir", + branch.lane_id + )) + })?); + if db.authenticated_lane_sparse_selection(&workdir)? != sparse_selection { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "sparse lane selection changed during controlled projection".into(), + command: command.clone(), + }); + } + } + + let ( + scope_root_identity, + filesystem_identity, + provider_id, + provider_identity, + owner_token, + fence_nonce, + max_log_bytes, + max_segment_bytes, + max_tail_records, + ): ( + String, + String, + String, + String, + String, + Option>, + i64, + i64, + i64, + ) = db.conn.query_row( + "SELECT scope.scope_root_identity,scope.filesystem_identity, + scope.provider_id,scope.provider_identity,owner.owner_token,owner.fence_nonce, + scope.max_observer_log_bytes,scope.max_segment_bytes, + scope.max_unfolded_tail_records + FROM changed_path_scopes scope + JOIN changed_path_observer_owners owner + ON owner.scope_id=scope.scope_id AND owner.epoch=scope.epoch + WHERE scope.scope_id=?1 AND scope.epoch=?2 AND scope.trust_state='trusted' + AND owner.lease_state='active' AND owner.error_state IS NULL + AND owner.expires_at>strftime('%s','now')", + params![ + expected.scope_id.to_text(), + i64::try_from(expected.epoch) + .map_err(|_| Error::InvalidInput("lane epoch overflow".into()))? + ], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + row.get(6)?, + row.get(7)?, + row.get(8)?, + )) + }, + )?; + let owner_bytes: [u8; 32] = hex::decode(&owner_token) + .map_err(|_| Error::Corrupt("invalid lane observer owner token".into()))? + .try_into() + .map_err(|_| Error::Corrupt("invalid lane observer owner token length".into()))?; + let trail_directory = super::secure_fs::SecureDirectory::open_absolute(&db.db_dir)?; + let observer_directory = trail_directory.open_dir("observer-segments")?; + let secure_segment_directory = observer_directory.open_dir(&expected.scope_id.to_text())?; + let recovery_scope = RecoveryScope { + scope_id: expected.scope_id, + epoch: expected.epoch, + owner_token: owner_bytes, + }; + let limits = PersistedLogLimits { + max_log_bytes: u64::try_from(max_log_bytes) + .map_err(|_| Error::Corrupt("negative observer log limit".into()))?, + max_segment_bytes: u64::try_from(max_segment_bytes) + .map_err(|_| Error::Corrupt("negative observer segment limit".into()))?, + max_unfolded_tail_records: usize::try_from(max_tail_records) + .map_err(|_| Error::Corrupt("invalid observer tail limit".into()))?, + }; + let mut recovered = None; + for attempt in 0..16 { + let candidate = super::recover_segments_from_connection( + &db.conn, + &secure_segment_directory, + &recovery_scope, + limits, + ) + .map_err(|error| Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: format!("controlled interval sidecar authentication failed: {error}"), + command: command.clone(), + })?; + if !candidate.requires_reconciliation || attempt == 15 { + recovered = Some(candidate); + break; + } + std::thread::sleep(Duration::from_millis(1)); + } + let recovered = recovered.ok_or_else(|| { + Error::Corrupt("controlled interval sidecar recovery did not produce a result".into()) + })?; + if recovered.requires_reconciliation { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "controlled interval sidecar chain requires reconciliation".into(), + command: command.clone(), + }); + } + let segment = recovered + .segments + .iter() + .find(|segment| { + segment.state == "sealed" + && segment.last_sequence == c1.sequence + && segment.durable_end_offset == c1.durable_offset + && segment.folded_end_offset >= c1.folded_offset + && Some(segment.start_cursor.as_slice()) == start_cursor.as_deref() + }) + .ok_or_else(|| Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "prepared intent cut does not start the authenticated controlled interval" + .into(), + command, + })?; + let publication_boundary = + authenticated_publication_boundary(&recovered, &c2, &publication_durable).ok_or_else( + || Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "publication cut is absent from the authenticated observer chain".into(), + command: controlled_status_command(&runtime_key), + }, + )?; + Ok(QualifiedFilesystemProof { + scope_id: expected.scope_id, + epoch: expected.epoch, + expected_root_id: crate::ObjectId(expected_root_id), + scope_root_identity: hex::decode(scope_root_identity) + .map_err(|_| Error::Corrupt("invalid lane scope root identity".into()))?, + filesystem_identity: hex::decode(filesystem_identity) + .map_err(|_| Error::Corrupt("invalid lane filesystem identity".into()))?, + provider_id, + provider_identity: hex::decode(provider_identity) + .map_err(|_| Error::Corrupt("invalid lane provider identity".into()))?, + observer_owner_token: owner_token, + owner_fence_nonce: fence_nonce, + durable_segment_id: segment.segment_id.clone(), + durable_segment_hash: segment.segment_hash, + segment_directory: format!("observer-segments/{}", expected.scope_id.to_text()), + segment_path: segment.segment_path.clone(), + start_cursor, + end_cursor: segment.end_cursor.clone(), + publication_segment_id: publication_boundary.0, + publication_cursor: publication_boundary.1, + start_sequence: segment.first_sequence, + end_cut: c1.clone(), + publication_cut: c2, + segment_durable_offset: c1.durable_offset, + segment_folded_offset: c1.folded_offset, + verified_paths: evidence.exact_paths.len().try_into().unwrap_or(u64::MAX), + verified_prefixes: evidence + .complete_prefixes + .len() + .try_into() + .unwrap_or(u64::MAX), + complete_root_interval: true, + complete_policy_interval: true, + persisted_evidence_through_end: true, + }) +} + +fn authenticated_publication_boundary( + recovered: &RecoveredTail, + publication_cut: &EvidenceCut, + publication_durable: &DurableCut, +) -> Option<(String, Vec)> { + recovered + .record_boundaries + .iter() + .find(|boundary| { + boundary.segment_id == publication_durable.segment_id + && boundary.sequence == publication_cut.sequence + && boundary.durable_end_offset == publication_cut.durable_offset + && boundary.provider_cursor == publication_durable.provider_cursor + }) + .map(|boundary| { + ( + boundary.segment_id.clone(), + boundary.provider_cursor.clone(), + ) + }) + .or_else(|| { + let first_sequence = publication_cut.sequence.checked_add(1)?; + recovered + .segments + .iter() + .find(|candidate| { + candidate.segment_id == publication_durable.segment_id + && matches!(candidate.state.as_str(), "open" | "sealed") + && candidate.start_cursor == publication_durable.provider_cursor + && candidate.first_sequence == first_sequence + && candidate.last_sequence >= publication_cut.sequence + && candidate.header_end_offset == publication_cut.durable_offset + && candidate.durable_end_offset >= publication_cut.durable_offset + && candidate.folded_end_offset >= publication_cut.folded_offset + }) + .map(|candidate| (candidate.segment_id.clone(), candidate.start_cursor.clone())) + }) +} + +pub(crate) fn materialized_lane_daemon_ready_proof( + db: &Trail, + lane: &str, +) -> Result { + let lane_id = db.lane_branch(lane)?.lane_id; + daemon_registry(db) + .materialized_lanes + .get(&lane_id) + .ok_or_else(|| { + Error::DaemonUnavailable(format!( + "changed-path observer runtime for materialized lane `{lane_id}` is unavailable" + )) + })? + .current_proof() +} + +pub(crate) fn materialized_lane_daemon_marker_cut( + db: &Trail, + lane: &str, +) -> Result> { + let lane_id = db.lane_branch(lane)?.lane_id; + let registry = daemon_registry(db); + let Some(runtime) = registry.materialized_lanes.get(&lane_id) else { + return Ok(None); + }; + let proof = runtime.current_proof()?; + let anchor = runtime.tail_anchor.as_ref().ok_or_else(|| { + Error::DaemonUnavailable("lane runtime has no authenticated marker anchor".into()) + })?; + let durable = runtime + .observer + .authenticated_cut(&runtime.expected, anchor)?; + if durable.last_sequence != proof.cut.sequence + || durable.durable_end_offset != proof.cut.durable_offset + { + return Err(Error::ChangeLedgerReconcileRequired { + scope: runtime.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "lane marker runtime cut lost its exact sidecar boundary".into(), + command: format!("trail lane status {lane_id}"), + }); + } + Ok(Some((proof.cut, durable.segment_id))) +} + +pub(crate) fn materialized_lane_daemon_expected_scope( + db: &Trail, + lane: &str, +) -> Result { + let lane_id = db.lane_branch(lane)?.lane_id; + if !daemon_registry(db) + .materialized_lanes + .contains_key(&lane_id) + { + prepare_materialized_lane_daemon(db, &lane_id, true)?; + } + daemon_registry(db) + .materialized_lanes + .get(&lane_id) + .map(|runtime| runtime.expected.clone()) + .ok_or_else(|| Error::DaemonUnavailable("lane runtime disappeared".into())) +} + +fn with_materialized_lane_runtime( + db: &Trail, + lane: &str, + operation: impl FnOnce(&mut WorkspaceDaemonRuntime, &Trail) -> Result, +) -> Result { + let lane_id = db.lane_branch(lane)?.lane_id; + let mut runtime = daemon_registry(db) + .materialized_lanes + .remove(&lane_id) + .ok_or_else(|| { + Error::DaemonUnavailable(format!( + "changed-path observer runtime for materialized lane `{lane_id}` is unavailable" + )) + })?; + let result = operation(&mut runtime, db); + daemon_registry(db) + .materialized_lanes + .insert(lane_id, runtime); + result +} + +fn with_workspace_runtime( + db: &Trail, + operation: impl FnOnce(&mut WorkspaceDaemonRuntime, &Trail) -> Result, +) -> Result { + let mut runtime = daemon_registry(db).workspace.take().ok_or_else(|| { + Error::DaemonUnavailable("changed-path workspace runtime is unavailable".into()) + })?; + let result = operation(&mut runtime, db); + daemon_registry(db).workspace = Some(runtime); + result +} + +fn controlled_runtime<'a>( + registry: &'a ChangedPathDaemonRegistry, + key: &ControlledRuntimeKey, +) -> Option<&'a WorkspaceDaemonRuntime> { + match key { + ControlledRuntimeKey::Workspace => registry.workspace.as_ref(), + ControlledRuntimeKey::MaterializedLane(lane) => registry.materialized_lanes.get(lane), + } +} + +fn with_controlled_runtime( + db: &Trail, + key: &ControlledRuntimeKey, + operation: impl FnOnce(&mut WorkspaceDaemonRuntime, &Trail) -> Result, +) -> Result { + match key { + ControlledRuntimeKey::Workspace => with_workspace_runtime(db, operation), + ControlledRuntimeKey::MaterializedLane(lane) => { + with_materialized_lane_runtime(db, lane, operation) + } + } +} + +fn ensure_controlled_runtime(db: &Trail, key: &ControlledRuntimeKey) -> Result<()> { + match key { + ControlledRuntimeKey::Workspace => { + if daemon_registry(db).workspace.is_none() { + prepare_workspace_daemon(db, true)?; + } + } + ControlledRuntimeKey::MaterializedLane(lane) => { + if !daemon_registry(db).materialized_lanes.contains_key(lane) { + prepare_materialized_lane_daemon(db, lane, true)?; + } + } + } + Ok(()) +} + +fn controlled_runtime_matches_target(db: &Trail, key: &ControlledRuntimeKey) -> Result { + let registry = daemon_registry(db); + let Some(runtime) = controlled_runtime(®istry, key) else { + return Ok(false); + }; + let target = match key { + ControlledRuntimeKey::Workspace => workspace_daemon_target(db)?, + ControlledRuntimeKey::MaterializedLane(lane) => materialized_lane_daemon_target(db, lane)?, + }; + runtime.matches_target(&target) +} + +fn controlled_status_command(key: &ControlledRuntimeKey) -> String { + match key { + ControlledRuntimeKey::Workspace => "trail status".into(), + ControlledRuntimeKey::MaterializedLane(lane) => format!("trail lane status {lane}"), + } +} + +fn daemon_registry(db: &Trail) -> std::sync::MutexGuard<'_, ChangedPathDaemonRegistry> { + db.changed_path_daemon_registry + .lock() + .unwrap_or_else(|poison| poison.into_inner()) +} + +pub(crate) struct WorkspaceDaemonRuntime { + root: std::path::PathBuf, + scope_kind: ScopeKind, + expected: ExpectedScope, + policy: CompiledPolicy, + observer: PlatformObserver, + tail_anchor: Option, + last_cut: Option, + daemon_launch_nonce: Option, + #[cfg(all(test, target_os = "macos"))] + inject_policy_drift_after_end: bool, +} + +impl WorkspaceDaemonRuntime { + fn start( + db: &Trail, + target: DaemonScopeTarget, + replace_stale_owner: bool, + refuse_live_unverified_owner: bool, + verified_stale_owner: Option<&VerifiedStaleWorkspaceOwner>, + launch: Option<&WorkspaceDaemonLaunchIdentity>, + ) -> Result { + let DaemonScopeTarget { + root: scope_root, + identity: scope_identity, + baseline, + } = target; + let scope_id = scope_identity.scope_id; + let scope_kind = scope_identity.kind; + let segment_directory = db.db_dir.join("observer-segments").join(scope_id.to_text()); + fs::create_dir_all(&segment_directory)?; + let filesystem_identity = root_identity(&scope_root)?; + let provider_identity = platform_provider_identity(); + let capabilities = platform_capabilities(); + let ledger = db.changed_path_ledger(); + let existing = load_existing_scope(db, scope_id).map_err(|error| { + Error::DaemonUnavailable(format!( + "changed-path observer could not load its persisted scope: {error}" + )) + })?; + let case_sensitive = match existing + .as_ref() + .filter(|stored| stored.filesystem_identity == filesystem_identity) + .map(|stored| stored.case_sensitive) + { + Some(0) => false, + Some(1) => true, + Some(_) => { + return Err(Error::Corrupt( + "changed-path scope has invalid case-sensitivity metadata".into(), + )) + } + None => !crate::db::util::is_case_insensitive_filesystem(&scope_root)?, + }; + #[cfg(debug_assertions)] + if existing.is_some() { + test_daemon_transition_after_load_boundary()?; + } + let (epoch, policy_generation, mut resume_cursor) = match existing { + None => { + ledger + .begin_scope( + &scope_identity, + &baseline, + &PolicyIdentity { + fingerprint: [0; 32], + generation: 1, + }, + &FilesystemIdentity(filesystem_identity.clone()), + &ProviderIdentity { + identity: provider_identity.clone(), + capabilities: capabilities.clone(), + }, + ) + .map_err(|error| { + Error::DaemonUnavailable(format!( + "changed-path observer could not initialize its scope: {error}" + )) + })?; + (1, 1, None) + } + Some(stored) => { + let policy_generation = stored.policy_generation; + let current_filesystem_identity = hex::encode(&filesystem_identity); + let current_provider_identity = hex::encode(&provider_identity); + let identity_changed = stored.filesystem_identity != filesystem_identity + || stored.scope_root_identity != current_filesystem_identity + || stored.provider_identity != provider_identity + || stored.provider_id_text.as_deref() + != Some(current_provider_identity.as_str()); + if identity_changed && !replace_stale_owner { + return Err(Error::ChangeLedgerReconcileRequired { + scope: scope_id.to_text(), + state: "stale_baseline".into(), + reason: "persisted daemon scope does not match the exact current baseline" + .into(), + command: "trail status".into(), + }); + } + let baseline_changed = stored.ref_name != baseline.ref_name + || stored.ref_generation != baseline.ref_generation + || stored.change_id != baseline.change_id.0 + || stored.baseline_root != baseline.root_id.0; + let old_expected = ExpectedScope { + scope_id, + epoch: stored.epoch, + ref_name: stored.ref_name.clone(), + ref_generation: stored.ref_generation, + baseline_root: crate::ObjectId(stored.baseline_root.clone()), + policy_fingerprint: stored.policy_fingerprint, + policy_generation: stored.policy_generation, + filesystem_identity: stored.filesystem_identity.clone(), + provider_identity: stored.provider_identity.clone(), + }; + if !identity_changed { + ledger.recover_scope(&old_expected)?; + } + if !replace_stale_owner { + return Err(Error::DaemonUnavailable( + "persisted workspace daemon owner exists without verified stale process identity" + .into(), + )); + } + if let Some(verified) = verified_stale_owner { + let mut invalid = Vec::new(); + if verified.scope_id != scope_id.to_text() { + invalid.push("scope"); + } + if verified.epoch != stored.epoch { + invalid.push("scope_epoch"); + } + match stored.observer_owner.as_ref() { + Some(owner) => { + if owner.daemon_launch_nonce.as_deref() + != Some(verified.daemon_launch_nonce.as_str()) + { + invalid.push("launch_nonce"); + } + if owner.daemon_pid != Some(i64::from(verified.stale_pid)) { + invalid.push("pid"); + } + if owner.daemon_process_start_identity.as_deref() + != Some(verified.process_start_identity.as_str()) + { + invalid.push("process_start_identity"); + } + if owner.epoch != verified.epoch { + invalid.push("owner_epoch"); + } + if owner.owner_token != verified.observer_owner_token { + invalid.push("owner_token"); + } + } + None => invalid.push("owner"), + } + if stored.observer_owner_token.as_deref() + != Some(verified.observer_owner_token.as_str()) + { + invalid.push("scope_owner_token"); + } + if !invalid.is_empty() { + return Err(Error::DaemonUnavailable(format!( + "verified stale daemon PID {} ({}) does not match the exact persisted observer scope/epoch/owner token ({})", + verified.stale_pid, verified.process_start_identity, invalid.join(", ") + ))); + } + } else if refuse_live_unverified_owner + && stored.observer_owner.as_ref().is_some_and(|owner| { + owner.lease_state == "active" + && owner.error_state.is_none() + && owner.expires_at > crate::db::util::now_ts() + }) + { + return Err(Error::DaemonUnavailable( + "changed-path observer owner is still live; refusing unverified authority replacement" + .into(), + )); + } + let old_epoch = stored.epoch; + let next = old_epoch.checked_add(1).ok_or_else(|| { + Error::InvalidInput("changed-path daemon scope epoch overflow".into()) + })?; + let owner_authority_consistent = match stored.observer_owner.as_ref() { + Some(owner) => { + let provider_binding_matches_loaded_scope = + stored.provider_id_text.as_deref() == Some(owner.provider_id.as_str()) + && stored.provider_identity_text.as_deref() + == Some(owner.provider_identity.as_str()); + let provider_binding_is_verified_drift_target = identity_changed + && owner.provider_id == current_provider_identity + && owner.provider_identity == current_provider_identity; + owner.epoch == stored.epoch + && stored.observer_owner_token.as_deref() + == Some(owner.owner_token.as_str()) + && (provider_binding_matches_loaded_scope + || provider_binding_is_verified_drift_target) + } + None => stored.observer_owner_token.is_none() || !refuse_live_unverified_owner, + }; + if !owner_authority_consistent { + return Err(daemon_owner_authority_inconsistent(scope_id)); + } + let tx = db.conn.unchecked_transaction()?; + let owner_changed = match stored.observer_owner.as_ref() { + Some(owner) => tx.execute( + "UPDATE changed_path_observer_owners + SET lease_state='revoked', error_state='daemon_owner_replaced', + error_at=strftime('%s','now'), updated_at=strftime('%s','now') + WHERE scope_id=:scope_id AND epoch=:epoch + AND owner_token=:owner_token AND provider_id=:provider_id + AND provider_identity=:provider_identity + AND lease_state=:lease_state AND fence_nonce IS :fence_nonce + AND acquired_at=:acquired_at AND heartbeat_at=:heartbeat_at + AND expires_at=:expires_at AND error_state IS :error_state + AND error_at IS :error_at", + named_params! { + ":scope_id": scope_id.to_text(), + ":epoch": i64::try_from(owner.epoch).map_err(|_| Error::InvalidInput("observer epoch overflow".into()))?, + ":owner_token": &owner.owner_token, + ":provider_id": &owner.provider_id, + ":provider_identity": &owner.provider_identity, + ":lease_state": &owner.lease_state, + ":fence_nonce": &owner.fence_nonce, + ":acquired_at": owner.acquired_at, + ":heartbeat_at": owner.heartbeat_at, + ":expires_at": owner.expires_at, + ":error_state": &owner.error_state, + ":error_at": owner.error_at, + }, + )?, + None => { + let count = tx.query_row( + "SELECT COUNT(*) FROM changed_path_observer_owners WHERE scope_id=?1", + [scope_id.to_text()], + |row| row.get::<_, i64>(0), + )?; + usize::from(count == 0) + } + }; + if owner_changed != 1 { + tx.rollback()?; + return Err(daemon_authority_transition_lost(scope_id)); + } + let owner_revocation_triggered = stored + .observer_owner + .as_ref() + .is_some_and(|owner| owner.lease_state == "active"); + let old_trust_state_after_revocation = if owner_revocation_triggered + && matches!(stored.trust_state.as_str(), "trusted" | "reconciling") + { + "untrusted_gap" + } else { + stored.trust_state.as_str() + }; + let old_trust_reason_after_revocation = if owner_revocation_triggered + && matches!(stored.trust_state.as_str(), "trusted" | "reconciling") + { + "observer_owner_revoked" + } else { + stored.trust_reason.as_str() + }; + let old_continuity_after_revocation = stored + .continuity_generation + .checked_add(u64::from(owner_revocation_triggered)) + .ok_or_else(|| Error::InvalidInput("continuity generation overflow".into()))?; + let changed = tx.execute( + "UPDATE changed_path_scopes + SET epoch=:next_epoch, + ref_name=:next_ref_name, ref_generation=:next_ref_generation, + change_id=:next_change_id, baseline_root_id=:next_baseline_root, + scope_root_identity=:next_filesystem_identity, + filesystem_identity=:next_filesystem_identity, + provider_id=:next_provider_identity, + provider_identity=:next_provider_identity, + durable_cursor=:next_durable_cursor, + linearizable_fence=:next_linearizable_fence, + rename_pairing=:next_rename_pairing, + overflow_scope=:next_overflow_scope, + filesystem_supported=:next_filesystem_supported, + clean_proof_allowed=:next_clean_proof_allowed, + power_loss_durability=:next_power_loss_durability, + trust_state='untrusted_gap', trust_reason=:next_trust_reason, + observer_owner_token=NULL, provider_cursor=NULL, provider_fence=NULL, + observer_heartbeat_at=NULL, observer_error_state=NULL, + observer_error_at=NULL, + durable_offset=0, folded_offset=0, + continuity_generation=continuity_generation+1, + updated_at=strftime('%s','now') + WHERE scope_id=:scope_id AND scope_kind=:old_scope_kind + AND schema_version=:old_schema_version + AND owner_id=:old_owner_id AND scope_root=:old_scope_root + AND scope_root_identity=:old_scope_root_identity + AND filesystem_identity=:old_filesystem_identity + AND filesystem_kind=:old_filesystem_kind + AND case_sensitive=:old_case_sensitive + AND epoch=:old_epoch AND ref_name=:old_ref_name + AND ref_generation=:old_ref_generation + AND change_id=:old_change_id + AND baseline_root_id=:old_baseline_root + AND policy_fingerprint=:old_policy_fingerprint + AND policy_dependency_generation=:old_policy_generation + AND trust_state=:old_trust_state + AND trust_reason=:old_trust_reason + AND continuity_generation=:old_continuity_generation + AND provider_id IS :old_provider_id + AND provider_identity IS :old_provider_identity + AND durable_cursor=:old_durable_cursor + AND linearizable_fence=:old_linearizable_fence + AND rename_pairing=:old_rename_pairing + AND overflow_scope=:old_overflow_scope + AND filesystem_supported=:old_filesystem_supported + AND clean_proof_allowed=:old_clean_proof_allowed + AND power_loss_durability=:old_power_loss_durability + AND provider_cursor IS :old_provider_cursor + AND provider_fence IS :old_provider_fence + AND durable_offset=:old_durable_offset + AND folded_offset=:old_folded_offset + AND observer_owner_token IS :old_observer_owner_token + AND observer_heartbeat_at IS :old_observer_heartbeat_at + AND observer_error_state IS :old_observer_error_state + AND observer_error_at IS :old_observer_error_at + AND max_candidate_rows=:old_max_candidate_rows + AND max_prefix_rows=:old_max_prefix_rows + AND max_observer_log_bytes=:old_max_observer_log_bytes + AND max_segment_bytes=:old_max_segment_bytes + AND max_unfolded_tail_records=:old_max_unfolded_tail_records + AND retired_at IS :old_retired_at", + named_params! { + ":next_epoch": i64::try_from(next).map_err(|_| Error::InvalidInput("epoch overflow".into()))?, + ":next_ref_name": &baseline.ref_name, + ":next_ref_generation": i64::try_from(baseline.ref_generation).map_err(|_| Error::InvalidInput("ref generation overflow".into()))?, + ":next_change_id": &baseline.change_id.0, + ":next_baseline_root": &baseline.root_id.0, + ":next_filesystem_identity": hex::encode(&filesystem_identity), + ":next_provider_identity": hex::encode(&provider_identity), + ":next_durable_cursor": i64::from(capabilities.durable_cursor), + ":next_linearizable_fence": i64::from(capabilities.linearizable_fence), + ":next_rename_pairing": i64::from(capabilities.rename_pairing), + ":next_overflow_scope": i64::from(capabilities.overflow_scope), + ":next_filesystem_supported": i64::from(capabilities.filesystem_supported), + ":next_clean_proof_allowed": i64::from(capabilities.clean_proof_allowed), + ":next_power_loss_durability": i64::from(capabilities.power_loss_durability), + ":next_trust_reason": if identity_changed { "daemon_identity_transition" } else { "daemon_owner_restarted" }, + ":scope_id": scope_id.to_text(), + ":old_scope_kind": &stored.scope_kind, + ":old_schema_version": stored.schema_version, + ":old_owner_id": &stored.owner_id, + ":old_scope_root": &stored.scope_root, + ":old_scope_root_identity": &stored.scope_root_identity, + ":old_filesystem_identity": hex::encode(&stored.filesystem_identity), + ":old_filesystem_kind": &stored.filesystem_kind, + ":old_case_sensitive": stored.case_sensitive, + ":old_epoch": i64::try_from(old_epoch).map_err(|_| Error::InvalidInput("epoch overflow".into()))?, + ":old_ref_name": &stored.ref_name, + ":old_ref_generation": i64::try_from(stored.ref_generation).map_err(|_| Error::InvalidInput("ref generation overflow".into()))?, + ":old_change_id": &stored.change_id, + ":old_baseline_root": &stored.baseline_root, + ":old_policy_fingerprint": hex::encode(stored.policy_fingerprint), + ":old_policy_generation": i64::try_from(stored.policy_generation).map_err(|_| Error::InvalidInput("policy generation overflow".into()))?, + ":old_trust_state": old_trust_state_after_revocation, + ":old_trust_reason": old_trust_reason_after_revocation, + ":old_continuity_generation": i64::try_from(old_continuity_after_revocation).map_err(|_| Error::InvalidInput("continuity generation overflow".into()))?, + ":old_provider_id": &stored.provider_id_text, + ":old_provider_identity": &stored.provider_identity_text, + ":old_durable_cursor": stored.capabilities[0], + ":old_linearizable_fence": stored.capabilities[1], + ":old_rename_pairing": stored.capabilities[2], + ":old_overflow_scope": stored.capabilities[3], + ":old_filesystem_supported": stored.capabilities[4], + ":old_clean_proof_allowed": stored.capabilities[5], + ":old_power_loss_durability": stored.capabilities[6], + ":old_provider_cursor": &stored.provider_cursor, + ":old_provider_fence": &stored.provider_fence, + ":old_durable_offset": i64::try_from(stored.durable_offset).map_err(|_| Error::InvalidInput("durable offset overflow".into()))?, + ":old_folded_offset": i64::try_from(stored.folded_offset).map_err(|_| Error::InvalidInput("folded offset overflow".into()))?, + ":old_observer_owner_token": &stored.observer_owner_token, + ":old_observer_heartbeat_at": stored.observer_heartbeat_at, + ":old_observer_error_state": &stored.observer_error_state, + ":old_observer_error_at": stored.observer_error_at, + ":old_retired_at": stored.retired_at, + ":old_max_candidate_rows": stored.limits[0], + ":old_max_prefix_rows": stored.limits[1], + ":old_max_observer_log_bytes": stored.limits[2], + ":old_max_segment_bytes": stored.limits[3], + ":old_max_unfolded_tail_records": stored.limits[4], + }, + )?; + if changed != 1 { + tx.rollback()?; + return Err(daemon_authority_transition_lost(scope_id)); + } + tx.commit()?; + ( + next, + policy_generation, + if baseline_changed || identity_changed { + None + } else { + stored.provider_cursor + }, + ) + } + }; + + let mut expected = ExpectedScope { + scope_id, + epoch, + ref_name: baseline.ref_name, + ref_generation: baseline.ref_generation, + baseline_root: baseline.root_id, + policy_fingerprint: [0; 32], + policy_generation, + filesystem_identity, + provider_identity, + }; + let stored_fingerprint: String = db + .conn + .query_row( + "SELECT policy_fingerprint FROM changed_path_scopes WHERE scope_id=?1 AND epoch=?2", + params![scope_id.to_text(), i64::try_from(epoch).unwrap_or(i64::MAX)], + |row| row.get(0), + ) + .map_err(|error| { + Error::DaemonUnavailable(format!( + "changed-path observer could not read its policy identity: {error}" + )) + })?; + expected.policy_fingerprint = decode_fingerprint(&stored_fingerprint)?; + let git_environment = std::env::vars_os().collect::>(); + let mut metrics = PolicyDependencyMetrics::default(); + let mut policy = compile_policy( + &db.conn, + &expected, + &PolicyCompileContext { + workspace_root: &scope_root, + db_dir: &db.db_dir, + recording: &db.config.recording, + case_sensitive, + git_environment: &git_environment, + }, + &mut metrics, + ) + .map_err(|error| { + Error::DaemonUnavailable(format!( + "changed-path observer could not compile its recording policy: {error}" + )) + })?; + if expected.policy_fingerprint == [0; 32] { + expected.policy_fingerprint = policy.fingerprint(); + db.conn.execute( + "UPDATE changed_path_scopes SET policy_fingerprint=?1, updated_at=strftime('%s','now') + WHERE scope_id=?2 AND epoch=?3 AND policy_fingerprint=?4", + params![ + hex::encode(expected.policy_fingerprint), + scope_id.to_text(), + i64::try_from(epoch).unwrap_or(i64::MAX), + stored_fingerprint, + ], + )?; + } else if expected.policy_fingerprint != policy.fingerprint() { + let previous_fingerprint = expected.policy_fingerprint; + let previous_generation = expected.policy_generation; + let next_generation = previous_generation.checked_add(1).ok_or_else(|| { + Error::InvalidInput("changed-path policy generation overflow".into()) + })?; + let next_fingerprint = policy.fingerprint(); + let tx = db.conn.unchecked_transaction()?; + tx.execute( + "UPDATE changed_path_policy_dependencies SET generation=?1, updated_at=strftime('%s','now') + WHERE scope_id=?2 AND generation=?3", + params![ + i64::try_from(next_generation) + .map_err(|_| Error::InvalidInput("policy generation overflow".into()))?, + scope_id.to_text(), + i64::try_from(previous_generation) + .map_err(|_| Error::InvalidInput("policy generation overflow".into()))?, + ], + )?; + let changed = tx.execute( + "UPDATE changed_path_scopes + SET policy_fingerprint=?1, policy_dependency_generation=?2, + trust_state='untrusted_gap', trust_reason='daemon_policy_transition', + provider_cursor=NULL, provider_fence=NULL, + continuity_generation=continuity_generation+1, + updated_at=strftime('%s','now') + WHERE scope_id=?3 AND epoch=?4 AND policy_fingerprint=?5 + AND policy_dependency_generation=?6", + params![ + hex::encode(next_fingerprint), + i64::try_from(next_generation) + .map_err(|_| Error::InvalidInput("policy generation overflow".into()))?, + scope_id.to_text(), + i64::try_from(epoch) + .map_err(|_| Error::InvalidInput("epoch overflow".into()))?, + hex::encode(previous_fingerprint), + i64::try_from(previous_generation) + .map_err(|_| Error::InvalidInput("policy generation overflow".into()))?, + ], + )?; + if changed != 1 { + return Err(Error::ChangeLedgerReconcileRequired { + scope: scope_id.to_text(), + state: "stale_baseline".into(), + reason: "daemon policy transition lost exact scope authority".into(), + command: "trail status".into(), + }); + } + tx.commit()?; + expected.policy_fingerprint = next_fingerprint; + expected.policy_generation = next_generation; + resume_cursor = None; + } + + let mut owner = [0_u8; 32]; + let mut fence_nonce = [0_u8; 24]; + getrandom(&mut owner) + .map_err(|error| Error::InvalidInput(format!("observer owner entropy: {error}")))?; + getrandom(&mut fence_nonce) + .map_err(|error| Error::InvalidInput(format!("observer fence entropy: {error}")))?; + let primary_autocommit = db.conn.is_autocommit(); + let writer = match launch { + Some(launch) => SegmentWriter::acquire_for_daemon( + &db.sqlite_path, + &segment_directory, + scope_id, + epoch, + owner, + &hex::encode(&expected.provider_identity), + resume_cursor.clone().unwrap_or_default(), + Duration::from_secs(30), + DaemonLaunchBinding { + nonce: launch.nonce.clone(), + pid: launch.pid, + process_start_identity: launch.process_start_identity.clone(), + }, + ), + None => SegmentWriter::acquire( + &db.sqlite_path, + &segment_directory, + scope_id, + epoch, + owner, + &hex::encode(&expected.provider_identity), + resume_cursor.clone().unwrap_or_default(), + Duration::from_secs(30), + ), + } + .map_err(|error| { + Error::DaemonUnavailable(format!( + "changed-path observer could not acquire its durable segment writer (primary_autocommit={primary_autocommit}): {error}" + )) + })?; + let observer = PlatformObserver::start( + &scope_root, + writer, + expected.provider_identity.clone(), + fence_nonce.to_vec(), + policy.dependency_files(), + resume_cursor.take(), + ) + .map_err(|error| { + Error::DaemonUnavailable(format!( + "changed-path observer could not start native coverage: {error}" + )) + })?; + let compile_start = observer.begin_observation(&expected).map_err(|error| { + Error::DaemonUnavailable(format!( + "changed-path observer could not establish its policy-compile start fence: {error}" + )) + })?; + let mut verified_metrics = PolicyDependencyMetrics::default(); + let verified_policy = revalidate_compiled_policy( + &policy, + &PolicyCompileContext { + workspace_root: &scope_root, + db_dir: &db.db_dir, + recording: &db.config.recording, + case_sensitive, + git_environment: &git_environment, + }, + &mut verified_metrics, + ) + .map_err(|error| { + Error::DaemonUnavailable(format!( + "changed-path observer could not verify its recording policy: {error}" + )) + })?; + db.note_operation_metrics(crate::db::OperationMetricsDelta { + policy_dependency_full_discovery: metrics + .policy_dependency_full_discovery + .saturating_add(verified_metrics.policy_dependency_full_discovery), + ..crate::db::OperationMetricsDelta::default() + }); + let compile_end = observer + .end_fence(&expected, &compile_start) + .map_err(|error| { + Error::DaemonUnavailable(format!( + "changed-path observer could not establish its policy-compile end fence: {error}" + )) + })?; + let lease = observer.lease().map_err(|error| { + Error::DaemonUnavailable(format!( + "changed-path observer could not authenticate its startup lease: {error}" + )) + })?; + observer + .drain_through( + &expected, + &lease.root_identity, + &compile_start, + &compile_end, + &mut |_event| Ok(()), + ) + .map_err(|error| { + Error::DaemonUnavailable(format!( + "changed-path observer could not drain its policy-compile interval: {error}" + )) + })?; + if verified_policy.fingerprint() != policy.fingerprint() + || verified_policy.dependency_files() != policy.dependency_files() + { + return Err(Error::DaemonUnavailable( + "recording policy changed during observer startup; retrying requires a fresh full reconciliation" + .into(), + )); + } + policy = verified_policy; + policy.authorize_native_reconciliation(&expected, &lease)?; + Ok(Self { + root: scope_root, + scope_kind, + expected, + policy, + observer, + tail_anchor: None, + last_cut: None, + daemon_launch_nonce: launch.map(|launch| launch.nonce.clone()), + #[cfg(all(test, target_os = "macos"))] + inject_policy_drift_after_end: false, + }) + } + + fn matches_target(&self, target: &DaemonScopeTarget) -> Result { + Ok(self.root == target.root + && self.expected.scope_id == target.identity.scope_id + && self.expected.ref_name == target.baseline.ref_name + && self.expected.ref_generation == target.baseline.ref_generation + && self.expected.baseline_root == target.baseline.root_id + && self.expected.filesystem_identity == root_identity(&target.root)?) + } + + fn validate_request(&self, scope_id: Option<&str>, epoch: Option) -> Result<()> { + if scope_id.is_some_and(|scope| scope != self.expected.scope_id.to_text()) + || epoch.is_some_and(|epoch| epoch != self.expected.epoch) + { + return Err(Error::DaemonUnavailable( + "changed-path daemon RPC scope or epoch mismatch".into(), + )); + } + Ok(()) + } + + fn reconcile(&mut self, db: &Trail, reason: &str) -> Result { + db.note_operation_metrics(crate::db::OperationMetricsDelta { + reconciliation_run_count: 1, + ..crate::db::OperationMetricsDelta::default() + }); + let ledger = db.changed_path_ledger(); + let previous_state = ledger + .conn + .query_row( + "SELECT trust_state FROM changed_path_scopes WHERE scope_id=?1", + [self.expected.scope_id.to_text()], + |row| row.get::<_, String>(0), + ) + .unwrap_or_else(|_| "uninitialized".to_string()); + ledger + .recover_scope(&self.expected) + .map_err(|error| match error { + Error::Sqlite(_) => Error::DaemonUnavailable(format!( + "changed-path reconciliation could not recover its durable sidecar state: {error}" + )), + other => other, + })?; + let (mut report, tail_anchor) = reconcile_full_with_tail( + db, + &ledger, + &self.observer, + &self.expected, + &self.policy, + reason, + ) + .map_err(|error| match error { + Error::Sqlite(_) => Error::DaemonUnavailable(format!( + "changed-path reconciliation could not complete its scan and observer-tail fold: {error}" + )), + other => other, + })?; + let cut = EvidenceCut { + source: EvidenceSource::Observer, + sequence: tail_anchor.sequence, + durable_offset: tail_anchor.durable_offset, + folded_offset: tail_anchor.durable_offset, + }; + self.tail_anchor = Some(tail_anchor); + self.last_cut = Some(cut.clone()); + report.scope_id = self.expected.scope_id.to_text(); + report.scope_kind = self.scope_kind.as_str().to_string(); + report.previous_state = previous_state; + report.observed_paths = report.observed_files; + report.candidates = report.candidate_rows; + report.resulting_epoch = self.expected.epoch; + report.resulting_state = report.trust_state.clone(); + let observer_owner_token = self.observer.lease()?.owner_token; + Ok(WorkspaceDaemonProof { + scope_id: self.expected.scope_id.to_text(), + epoch: self.expected.epoch, + observer_owner_token, + daemon_launch_nonce: self.daemon_launch_nonce.clone(), + cut, + reconcile_report: Some(report), + }) + } + + fn fence(&mut self, db: &Trail) -> Result { + self.fence_with_activity(db).map(|(proof, _)| proof) + } + + fn fence_with_activity(&mut self, db: &Trail) -> Result<(WorkspaceDaemonProof, usize)> { + let start = self.tail_anchor.clone().ok_or_else(|| { + Error::DaemonUnavailable("workspace daemon has no continuous observer anchor".into()) + })?; + let fence = self.observer.end_fence(&self.expected, &start)?; + self.fold_end_fence_with_activity(db, start, fence) + } + + fn fold_end_fence_with_activity( + &mut self, + db: &Trail, + start: ObserverFence, + fence: ObserverFence, + ) -> Result<(WorkspaceDaemonProof, usize)> { + #[cfg(all(test, target_os = "macos"))] + if self.inject_policy_drift_after_end { + self.inject_policy_drift_after_end = false; + self.observer.fail_next_direct_policy_fence_for_test(); + } + // Preserve typed policy invalidation so the command wrapper can + // revoke/restart/recompile and automatically full-reconcile. Wrapping + // it as generic daemon unavailability would strand a revoked runtime. + let lease = self.observer.lease()?; + let mut events = Vec::::new(); + let mut qualification = self.observer.drain_through_retaining_end( + &self.expected, + &lease.root_identity, + &start, + &fence, + &mut |event| { + if raw_event_invalidates_policy( + &self.policy, + std::path::Path::new(event.path.as_str()), + ) { + return Err(Error::ChangeLedgerReconcileRequired { + scope: self.expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "recording policy changed during continuous observer interval" + .into(), + command: "trail status".into(), + }); + } + events.push(event); + Ok(()) + }, + )?; + let cut = fold_observer_interval( + &db.changed_path_ledger(), + &self.expected, + &lease.root_identity, + &start, + &fence, + &mut qualification, + &events, + )?; + let authenticated_end = self.observer.authenticated_cut(&self.expected, &fence)?; + let cursor_changed = db.conn.execute( + "UPDATE changed_path_scopes SET provider_cursor=?1,updated_at=?2 + WHERE scope_id=?3 AND epoch=?4 AND durable_offset=?5 AND folded_offset=?5 + AND observer_owner_token=?6 AND trust_state='trusted'", + params![ + authenticated_end.provider_cursor, + crate::db::util::now_ts(), + self.expected.scope_id.to_text(), + i64::try_from(self.expected.epoch) + .map_err(|_| Error::InvalidInput("observer epoch overflow".into()))?, + i64::try_from(fence.durable_offset) + .map_err(|_| Error::InvalidInput("observer fence offset overflow".into()))?, + lease.owner_token, + ], + )?; + if cursor_changed != 1 { + return Err(Error::ChangeLedgerReconcileRequired { + scope: self.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "continuous observer cursor lost its exact scope CAS".into(), + command: "trail status".into(), + }); + } + let observed_activity = events + .iter() + .filter(|event| !event.path.as_str().starts_with(".trail/observer-fences/")) + .count(); + db.note_operation_metrics(crate::db::OperationMetricsDelta { + observer_tail_record_fold_count: events.len().try_into().unwrap_or(u64::MAX), + ledger_row_touch_count: events.len().try_into().unwrap_or(u64::MAX), + ..crate::db::OperationMetricsDelta::default() + }); + self.tail_anchor = Some(fence); + self.last_cut = Some(cut.clone()); + Ok(( + WorkspaceDaemonProof { + scope_id: self.expected.scope_id.to_text(), + epoch: self.expected.epoch, + observer_owner_token: lease.owner_token, + daemon_launch_nonce: self.daemon_launch_nonce.clone(), + cut, + reconcile_report: None, + }, + observed_activity, + )) + } + + /// Fence, fold, and seal an exact observer segment boundary. The native + /// observer appends its controlled fence and rotates in one durability + /// worker turn, so traffic ordered after the fence starts in the linked + /// segment instead of starving the boundary. + fn fence_and_seal(&mut self, db: &Trail) -> Result { + let start = self.tail_anchor.clone().ok_or_else(|| { + Error::DaemonUnavailable("workspace daemon has no continuous observer anchor".into()) + })?; + let (end, sealed, anchor_cut) = + self.observer.controlled_end_fence(&self.expected, &start)?; + let (proof, _) = self.fold_end_fence_with_activity(db, start, end.clone())?; + if sealed.last_sequence != proof.cut.sequence + || sealed.durable_end_offset != proof.cut.durable_offset + || sealed.last_sequence != anchor_cut.last_sequence + || sealed.provider_cursor != anchor_cut.provider_cursor + { + return Err(Error::ChangeLedgerReconcileRequired { + scope: self.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "controlled observer fence did not equal its atomically sealed cut".into(), + command: "trail lane status".into(), + }); + } + let anchor = + self.observer + .install_rotation_anchor(&self.expected, &end, anchor_cut.clone())?; + self.advance_rotation_anchor(db, &sealed, &anchor_cut)?; + self.tail_anchor = Some(anchor); + self.last_cut = Some(EvidenceCut { + source: EvidenceSource::Observer, + sequence: anchor_cut.last_sequence, + durable_offset: anchor_cut.durable_end_offset, + folded_offset: anchor_cut.durable_end_offset, + }); + Ok(proof) + } + + fn advance_rotation_anchor( + &self, + db: &Trail, + sealed: &DurableCut, + anchor: &DurableCut, + ) -> Result<()> { + if sealed.last_sequence != anchor.last_sequence + || sealed.provider_cursor != anchor.provider_cursor + { + return Err(Error::ChangeLedgerReconcileRequired { + scope: self.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "observer rotation did not preserve an exact linked anchor".into(), + command: "trail status".into(), + }); + } + let tx = Transaction::new_unchecked(&db.conn, TransactionBehavior::Immediate)?; + let owner = self.observer.lease()?.owner_token; + let now = crate::db::util::now_ts(); + let anchor_offset = i64::try_from(anchor.durable_end_offset) + .map_err(|_| Error::InvalidInput("rotation anchor offset overflow".into()))?; + let sealed_offset = i64::try_from(sealed.durable_end_offset) + .map_err(|_| Error::InvalidInput("sealed offset overflow".into()))?; + let epoch = i64::try_from(self.expected.epoch) + .map_err(|_| Error::InvalidInput("rotation epoch overflow".into()))?; + if sealed.segment_id == anchor.segment_id { + let first_sequence = anchor + .last_sequence + .checked_add(1) + .ok_or_else(|| Error::InvalidInput("rotation anchor sequence overflow".into()))?; + let anchor_is_installed: bool = tx.query_row( + "SELECT EXISTS( + SELECT 1 + FROM changed_path_observer_segments segment + JOIN changed_path_scopes scope + ON scope.scope_id=segment.scope_id AND scope.epoch=segment.epoch + JOIN changed_path_observer_owners owner + ON owner.scope_id=scope.scope_id AND owner.epoch=scope.epoch + WHERE segment.scope_id=?1 AND segment.epoch=?2 + AND segment.segment_id=?3 AND segment.owner_token=?4 + AND segment.first_sequence=?5 AND segment.last_sequence IS NULL + AND segment.durable_end_offset=?6 AND segment.folded_end_offset=?6 + AND segment.state='open' + AND scope.durable_offset=?6 AND scope.folded_offset=?6 + AND scope.provider_cursor=?7 AND scope.observer_owner_token=?4 + AND scope.trust_state='trusted' + AND owner.owner_token=?4 AND owner.lease_state='active' + AND owner.error_state IS NULL AND owner.expires_at>?8 + )", + params![ + self.expected.scope_id.to_text(), + epoch, + anchor.segment_id, + owner, + i64::try_from(first_sequence) + .map_err(|_| Error::InvalidInput("rotation sequence overflow".into()))?, + anchor_offset, + anchor.provider_cursor, + now, + ], + |row| row.get(0), + )?; + if !anchor_is_installed { + return Err(Error::ChangeLedgerReconcileRequired { + scope: self.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "header-only observer rotation lost its exact installed anchor".into(), + command: "trail status".into(), + }); + } + tx.commit()?; + return Ok(()); + } + let sealed_segment_hash: String = tx.query_row( + "SELECT segment_hash FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=?2 AND segment_id=?3 AND owner_token=?4 + AND COALESCE(last_sequence,0)=?5 AND durable_end_offset=?6 AND state='sealed' + AND segment_hash IS NOT NULL", + params![ + self.expected.scope_id.to_text(), + epoch, + sealed.segment_id, + owner, + i64::try_from(sealed.last_sequence) + .map_err(|_| Error::InvalidInput("sealed sequence overflow".into()))?, + sealed_offset, + ], + |row| row.get(0), + )?; + let segment_changed = tx.execute( + "UPDATE changed_path_observer_segments SET folded_end_offset=?1,updated_at=?2 + WHERE scope_id=?3 AND epoch=?4 AND segment_id=?5 AND owner_token=?6 + AND previous_segment_id=?7 AND previous_segment_hash=?8 + AND durable_end_offset>=?1 AND folded_end_offset=0 AND state='open'", + params![ + anchor_offset, + now, + self.expected.scope_id.to_text(), + epoch, + anchor.segment_id, + owner, + sealed.segment_id, + sealed_segment_hash, + ], + )?; + let scope_changed = tx.execute( + "UPDATE changed_path_scopes SET durable_offset=?1,folded_offset=?1, + provider_cursor=?2,trust_reason='observer_rotation_anchor',updated_at=?3 + WHERE scope_id=?4 AND epoch=?5 AND durable_offset=?6 AND folded_offset=?6 + AND observer_owner_token=?7 AND trust_state='trusted'", + params![ + anchor_offset, + anchor.provider_cursor, + now, + self.expected.scope_id.to_text(), + epoch, + sealed_offset, + owner, + ], + )?; + if segment_changed != 1 || scope_changed != 1 { + let segment_state: Option<(i64, i64, Option, Option, String, String)> = + tx.query_row( + "SELECT durable_end_offset,folded_end_offset,previous_segment_id, + previous_segment_hash,state,owner_token + FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=?2 AND segment_id=?3", + params![self.expected.scope_id.to_text(), epoch, anchor.segment_id], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + )) + }, + ) + .optional()?; + return Err(Error::ChangeLedgerReconcileRequired { + scope: self.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: format!( + "observer rotation anchor lost its exact sidecar CAS (segment={segment_changed}, scope={scope_changed}, expected_anchor_offset={anchor_offset}, expected_previous_id={}, expected_previous_hash={sealed_segment_hash}, actual={segment_state:?})", + sealed.segment_id, + ), + command: "trail status".into(), + }); + } + tx.commit()?; + Ok(()) + } + + fn current_proof(&self) -> Result { + let lease = self.observer.lease().map_err(|error| { + Error::DaemonUnavailable(format!( + "changed-path observer health no longer authorizes the ready proof: {error}" + )) + })?; + if !self.policy.observer_lease_matches(&self.expected, &lease) + || !self.policy.authorizes_reconciliation(&self.expected) + { + return Err(Error::DaemonUnavailable( + "changed-path observer health no longer authorizes the ready proof".into(), + )); + } + Ok(WorkspaceDaemonProof { + scope_id: self.expected.scope_id.to_text(), + epoch: self.expected.epoch, + observer_owner_token: lease.owner_token, + daemon_launch_nonce: self.daemon_launch_nonce.clone(), + cut: self.last_cut.clone().ok_or_else(|| { + Error::DaemonUnavailable("workspace daemon has no authenticated ready cut".into()) + })?, + reconcile_report: None, + }) + } + + pub(crate) fn with_authoritative_snapshot( + &mut self, + db: &Trail, + consume: &mut F, + ) -> Result<(T, FencedCandidateSnapshot)> + where + F: FnMut(&Trail, &CompiledPolicy, &super::CandidateSnapshot) -> Result, + { + let mut retried = false; + loop { + let c1 = match self.fence(db) { + Ok(proof) => proof.cut, + Err(error) + if !retried + && requires_reconciliation(&error) + && !startup_policy_retryable(&error) => + { + self.reconcile(db, "authoritative_snapshot_retry")?; + retried = true; + continue; + } + Err(error) => return Err(error), + }; + let mut candidates = match db.changed_path_ledger().snapshot_candidates(&self.expected) + { + Ok(snapshot) => snapshot, + Err(error) + if !retried + && requires_reconciliation(&error) + && !startup_policy_retryable(&error) => + { + self.reconcile(db, "authoritative_snapshot_retry")?; + retried = true; + continue; + } + Err(error) => return Err(error), + }; + candidates.cut = c1; + let consumed = consume(db, &self.policy, &candidates); + // c2 is mandatory even when comparison/build fails, so no command + // can strand an unconsumed interval behind c1. + let c2 = self.fence(db); + match (consumed, c2) { + (Ok(value), Ok(c2)) => { + return Ok(( + value, + FencedCandidateSnapshot { + candidates, + c2: c2.cut, + }, + )); + } + (Err(error), _) + if !retried + && requires_reconciliation(&error) + && !startup_policy_retryable(&error) => + { + self.reconcile(db, "authoritative_snapshot_retry")?; + retried = true; + } + (Err(error), _) => return Err(error), + (_, Err(error)) + if !retried + && requires_reconciliation(&error) + && !startup_policy_retryable(&error) => + { + self.reconcile(db, "authoritative_snapshot_retry")?; + retried = true; + } + (_, Err(error)) => return Err(error), + } + } + } + + pub(crate) fn accept_observed_baseline( + &mut self, + expected: &ExpectedScope, + target: &BaselineIdentity, + ) -> Result<()> { + // Post-commit repair may be retried after a caller loses the result. + // Treat an already-rebound runtime as success, while still rejecting a + // different target or scope. + if self.expected.scope_id == expected.scope_id + && self.expected.epoch == expected.epoch + && self.expected.ref_name == target.ref_name + && self.expected.ref_generation == target.ref_generation + && self.expected.baseline_root == target.root_id + { + return Ok(()); + } + if self.expected != *expected + || self + .last_cut + .as_ref() + .is_none_or(|cut| cut.durable_offset != cut.folded_offset) + { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "daemon baseline transition did not match the committed observed scope" + .into(), + command: "trail status".into(), + }); + } + let mut next = self.expected.clone(); + next.ref_name = target.ref_name.clone(); + next.ref_generation = target.ref_generation; + next.baseline_root = target.root_id.clone(); + let anchor = self.tail_anchor.as_ref().ok_or_else(|| { + Error::DaemonUnavailable("daemon has no retained tail to rebind".into()) + })?; + self.observer + .rebind_retained_tail(&self.expected, &next, anchor)?; + self.policy + .rebind_observed_baseline(&self.expected, &next)?; + self.expected = next; + Ok(()) + } +} + +fn requires_reconciliation(error: &Error) -> bool { + matches!(error, Error::ChangeLedgerReconcileRequired { .. }) +} + +#[cfg(debug_assertions)] +fn test_daemon_transition_after_load_boundary() -> Result<()> { + let Some(barrier) = std::env::var_os("TRAIL_TEST_DAEMON_TRANSITION_AFTER_LOAD_BARRIER") else { + return Ok(()); + }; + let barrier = std::path::PathBuf::from(barrier); + fs::write(barrier.join("loaded"), b"ready")?; + let deadline = std::time::Instant::now() + Duration::from_secs(60); + while !barrier.join("continue").exists() { + if std::time::Instant::now() >= deadline { + return Err(Error::DaemonUnavailable( + "daemon transition after-load test barrier timed out".into(), + )); + } + std::thread::sleep(Duration::from_millis(1)); + } + Ok(()) +} + +enum PlatformObserver { + #[cfg(target_os = "linux")] + Linux(super::observer::linux::LinuxInotifyObserver), + #[cfg(target_os = "macos")] + MacOs(super::observer::macos::MacOsFseventsObserver), +} + +impl PlatformObserver { + fn start( + root: &Path, + writer: SegmentWriter, + provider_identity: Vec, + fence_nonce: Vec, + dependencies: &[std::path::PathBuf], + resume_cursor: Option>, + ) -> Result { + #[cfg(target_os = "linux")] + { + let _ = resume_cursor; + let durability = super::observer::linux::SegmentWriterDurability::new( + writer, + provider_identity, + fence_nonce, + )?; + return Ok(Self::Linux( + super::observer::linux::LinuxInotifyObserver::start( + root, + Box::new(durability), + dependencies, + )?, + )); + } + #[cfg(target_os = "macos")] + { + let resume = resume_cursor + .as_deref() + .map(super::observer::macos::MacOsProviderCursor::decode) + .transpose()?; + let durability = super::observer::macos::MacSegmentWriterDurability::new( + writer, + provider_identity, + fence_nonce, + )?; + return Ok(Self::MacOs( + super::observer::macos::MacOsFseventsObserver::start( + root, + Box::new(durability), + resume, + dependencies, + )?, + )); + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = ( + root, + writer, + provider_identity, + fence_nonce, + dependencies, + resume_cursor, + ); + Err(Error::DaemonUnavailable( + "changed-path workspace daemon requires Linux or macOS".into(), + )) + } + } + + fn lease(&self) -> Result { + match self { + #[cfg(target_os = "linux")] + Self::Linux(observer) => observer.lease(), + #[cfg(target_os = "macos")] + Self::MacOs(observer) => observer.lease(), + } + } + + #[cfg(all(test, target_os = "macos"))] + fn fail_next_direct_policy_fence_for_test(&self) { + match self { + Self::MacOs(observer) => observer.fail_next_direct_policy_fence_for_test(), + } + } + + fn seal_after_fence( + &self, + expected: &ExpectedScope, + fence: &ObserverFence, + ) -> Result> { + match self { + #[cfg(target_os = "linux")] + Self::Linux(observer) => observer.seal_after_fence(expected, fence), + #[cfg(target_os = "macos")] + Self::MacOs(observer) => observer.seal_after_fence(expected, fence), + } + } + + fn controlled_end_fence( + &self, + expected: &ExpectedScope, + start: &ObserverFence, + ) -> Result<(ObserverFence, DurableCut, DurableCut)> { + match self { + #[cfg(target_os = "linux")] + Self::Linux(observer) => observer.controlled_end_fence(expected, start), + #[cfg(target_os = "macos")] + Self::MacOs(observer) => observer.controlled_end_fence(expected, start), + } + } + + fn install_rotation_anchor( + &self, + expected: &ExpectedScope, + end: &ObserverFence, + anchor: DurableCut, + ) -> Result { + match self { + #[cfg(target_os = "linux")] + Self::Linux(observer) => observer.install_rotation_anchor(expected, end, anchor), + #[cfg(target_os = "macos")] + Self::MacOs(observer) => observer.install_rotation_anchor(expected, end, anchor), + } + } + + fn authenticated_cut( + &self, + expected: &ExpectedScope, + fence: &ObserverFence, + ) -> Result { + match self { + #[cfg(target_os = "linux")] + Self::Linux(observer) => observer.authenticated_cut(expected, fence), + #[cfg(target_os = "macos")] + Self::MacOs(observer) => observer.authenticated_cut(expected, fence), + } + } +} + +impl QualifiedObserver for PlatformObserver { + fn begin_observation(&self, expected: &ExpectedScope) -> Result { + match self { + #[cfg(target_os = "linux")] + Self::Linux(observer) => observer.begin_observation(expected), + #[cfg(target_os = "macos")] + Self::MacOs(observer) => observer.begin_observation(expected), + } + } + + fn end_fence(&self, expected: &ExpectedScope, start: &ObserverFence) -> Result { + match self { + #[cfg(target_os = "linux")] + Self::Linux(observer) => observer.end_fence(expected, start), + #[cfg(target_os = "macos")] + Self::MacOs(observer) => observer.end_fence(expected, start), + } + } + + fn drain_through( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(super::ObserverEvent) -> Result<()>, + ) -> Result { + match self { + #[cfg(target_os = "linux")] + Self::Linux(observer) => { + observer.drain_through(expected, root_handle_identity, start, end, sink) + } + #[cfg(target_os = "macos")] + Self::MacOs(observer) => { + observer.drain_through(expected, root_handle_identity, start, end, sink) + } + } + } + + fn drain_through_retaining_end( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(super::ObserverEvent) -> Result<()>, + ) -> Result { + match self { + #[cfg(target_os = "linux")] + Self::Linux(observer) => observer.drain_through_retaining_end( + expected, + root_handle_identity, + start, + end, + sink, + ), + #[cfg(target_os = "macos")] + Self::MacOs(observer) => observer.drain_through_retaining_end( + expected, + root_handle_identity, + start, + end, + sink, + ), + } + } + + fn rebind_retained_tail( + &self, + previous: &ExpectedScope, + next: &ExpectedScope, + anchor: &ObserverFence, + ) -> Result<()> { + match self { + #[cfg(target_os = "linux")] + Self::Linux(observer) => observer.rebind_retained_tail(previous, next, anchor), + #[cfg(target_os = "macos")] + Self::MacOs(observer) => observer.rebind_retained_tail(previous, next, anchor), + } + } +} + +fn workspace_scope_id(db: &Trail) -> ScopeId { + let mut digest = Sha256::new(); + digest.update(b"trail-workspace-changed-path-scope-v1\0"); + digest.update(db.config.workspace.id.0.as_bytes()); + ScopeId(digest.finalize().into()) +} + +fn workspace_daemon_target(db: &Trail) -> Result { + let branch = db.current_branch()?; + let head = db.resolve_branch_ref(&branch)?; + Ok(DaemonScopeTarget { + root: db.workspace_root().to_path_buf(), + identity: ScopeIdentity { + scope_id: workspace_scope_id(db), + kind: ScopeKind::Workspace, + owner_id: db.config.workspace.id.0.clone(), + }, + baseline: BaselineIdentity { + ref_name: head.name, + ref_generation: u64::try_from(head.generation) + .map_err(|_| Error::Corrupt("negative daemon ref generation".into()))?, + change_id: head.change_id, + root_id: head.root_id, + }, + }) +} + +fn materialized_lane_daemon_target(db: &Trail, lane: &str) -> Result { + let branch = db.lane_branch(lane)?; + let workdir = branch.workdir.as_deref().ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{}` does not have a materialized workdir", + branch.lane_id + )) + })?; + let root = std::path::PathBuf::from(workdir); + if !root.is_dir() { + return Err(Error::InvalidInput(format!( + "materialized lane workdir `{}` is unavailable", + root.display() + ))); + } + let head = db.get_ref(&branch.ref_name)?; + if head.change_id != branch.head_change || head.root_id != branch.head_root { + return Err(Error::StaleBranch(branch.ref_name)); + } + let scope_id = materialized_lane_scope_id(&db.config.workspace.id.0, &branch.lane_id); + Ok(DaemonScopeTarget { + root, + identity: ScopeIdentity { + scope_id, + kind: ScopeKind::MaterializedLane, + owner_id: branch.lane_id, + }, + baseline: BaselineIdentity { + ref_name: head.name, + ref_generation: u64::try_from(head.generation) + .map_err(|_| Error::Corrupt("negative lane ref generation".into()))?, + change_id: head.change_id, + root_id: head.root_id, + }, + }) +} + +pub(crate) fn materialized_lane_scope_id(workspace_id: &str, lane_id: &str) -> ScopeId { + let mut digest = Sha256::new(); + digest.update(b"trail-materialized-lane-changed-path-scope-v2\0"); + digest.update(workspace_id.as_bytes()); + digest.update([0]); + digest.update(lane_id.as_bytes()); + ScopeId(digest.finalize().into()) +} + +fn daemon_authority_transition_lost(scope_id: ScopeId) -> Error { + Error::ChangeLedgerReconcileRequired { + scope: scope_id.to_text(), + state: "stale_baseline".into(), + reason: "daemon authority transition lost exact loaded authority".into(), + command: "trail status".into(), + } +} + +fn daemon_owner_authority_inconsistent(scope_id: ScopeId) -> Error { + Error::ChangeLedgerReconcileRequired { + scope: scope_id.to_text(), + state: "corrupt".into(), + reason: "persisted daemon scope and observer owner authority are inconsistent".into(), + command: "trail status".into(), + } +} + +fn platform_provider_identity() -> Vec { + #[cfg(target_os = "linux")] + return b"linux-inotify-native-v1".to_vec(); + #[cfg(target_os = "macos")] + return b"macos-fsevents-segment-writer-v1".to_vec(); + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + Vec::new() +} + +fn platform_capabilities() -> ProviderCapabilities { + ProviderCapabilities { + durable_cursor: true, + linearizable_fence: true, + rename_pairing: cfg!(target_os = "linux"), + overflow_scope: true, + filesystem_supported: cfg!(any(target_os = "linux", target_os = "macos")), + clean_proof_allowed: cfg!(any(target_os = "linux", target_os = "macos")), + power_loss_durability: true, + } +} + +fn root_identity(path: &Path) -> Result> { + let file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path)?; + let metadata = file.metadata()?; + let prefix = "root-v1"; + Ok(format!( + "{prefix}:dev={};ino={};mode={};uid={};gid={}", + metadata.dev(), + metadata.ino(), + metadata.mode(), + metadata.uid(), + metadata.gid() + ) + .into_bytes()) +} + +fn decode_fingerprint(value: &str) -> Result<[u8; 32]> { + let bytes = hex::decode(value) + .map_err(|_| Error::Corrupt("invalid changed-path policy fingerprint".into()))?; + bytes + .try_into() + .map_err(|_| Error::Corrupt("invalid changed-path policy fingerprint length".into())) +} + +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] +mod tests { + use super::*; + + #[test] + fn publication_header_anchor_accepts_only_its_own_contiguous_later_suffix() { + let publication_cut = EvidenceCut { + source: EvidenceSource::Observer, + sequence: 7, + durable_offset: 101, + folded_offset: 101, + }; + let publication_durable = DurableCut { + segment_id: "publication-segment".into(), + durable_end_offset: 101, + last_sequence: 7, + last_hash: [7; 32], + provider_cursor: b"cursor-7".to_vec(), + }; + let segment = |segment_id: &str| super::super::AuthenticatedSegment { + segment_id: segment_id.into(), + segment_path: format!("{segment_id}.wal"), + state: "open".into(), + start_cursor: b"cursor-7".to_vec(), + end_cursor: b"cursor-8".to_vec(), + first_sequence: 8, + last_sequence: 8, + header_end_offset: 101, + durable_end_offset: 180, + folded_end_offset: 101, + segment_hash: [8; 32], + }; + let recovered = RecoveredTail { + records: Vec::new(), + record_boundaries: Vec::new(), + durable_end: 360, + last_sequence: 8, + last_hash: [8; 32], + requires_reconciliation: false, + segments: vec![segment("decoy-segment"), segment("publication-segment")], + }; + + assert_eq!( + authenticated_publication_boundary(&recovered, &publication_cut, &publication_durable,), + Some(("publication-segment".into(), b"cursor-7".to_vec(),)) + ); + } + + #[test] + fn explicit_full_reconcile_repairs_an_existing_runtime_without_a_ready_proof() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("tracked.txt"), b"baseline\n").unwrap(); + crate::Trail::init( + temp.path(), + "main", + crate::InitImportMode::WorkingTree, + false, + ) + .unwrap(); + let db = crate::Trail::open(temp.path()).unwrap(); + prepare_workspace_daemon(&db, true).unwrap(); + + daemon_registry(&db).workspace.as_mut().unwrap().last_cut = None; + assert!(workspace_daemon_ready_proof(&db).is_err()); + + let report = workspace_daemon_full_reconcile(&db).unwrap(); + assert_eq!(report.scope_kind, "workspace"); + assert_eq!(report.resulting_state, "trusted"); + assert!(workspace_daemon_ready_proof(&db).is_ok()); + } + + #[test] + fn repeated_controlled_preparation_reuses_an_authenticated_header_only_anchor() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("tracked.txt"), b"baseline\n").unwrap(); + crate::Trail::init( + temp.path(), + "main", + crate::InitImportMode::WorkingTree, + false, + ) + .unwrap(); + let mut db = crate::Trail::open(temp.path()).unwrap(); + + let first = prepare_workspace_controlled_projection(&mut db).unwrap(); + let second = prepare_workspace_controlled_projection(&mut db).unwrap(); + let third = prepare_workspace_controlled_projection(&mut db).unwrap(); + assert_eq!(first.scope_id, second.scope_id); + assert_eq!(second.scope_id, third.scope_id); + assert_eq!(first.epoch, second.epoch); + assert_eq!(second.epoch, third.epoch); + assert!(workspace_daemon_ready_proof(&db).is_ok()); + } + + #[test] + fn initial_materialized_lane_policy_invalidation_reconciles_and_retries_transparently() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("tracked.txt"), b"baseline\n").unwrap(); + crate::Trail::init( + temp.path(), + "main", + crate::InitImportMode::WorkingTree, + false, + ) + .unwrap(); + let mut db = crate::Trail::open(temp.path()).unwrap(); + let lane = db + .spawn_lane("startup-policy-retry", Some("main"), true, None, None) + .unwrap(); + let lane_id = lane.lane_id; + let workdir = std::path::PathBuf::from(lane.workdir.unwrap()); + let scope_id = materialized_lane_scope_id(&db.config.workspace.id.0, &lane_id); + super::super::install_initial_scan_hook(scope_id, move || { + std::fs::write(workdir.join(".gitignore"), b"generated/**\n")?; + Ok(()) + }); + + let report = materialized_lane_daemon_full_reconcile(&db, &lane_id).unwrap(); + assert_eq!(report.scope_kind, "materialized_lane"); + assert_eq!(report.resulting_state, "trusted"); + assert_eq!(report.resulting_epoch, 2); + assert!(materialized_lane_daemon_ready_proof(&db, &lane_id).is_ok()); + } + + #[test] + fn initial_workspace_policy_invalidation_retries_with_the_same_daemon_launch() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("tracked.txt"), b"baseline\n").unwrap(); + crate::Trail::init( + temp.path(), + "main", + crate::InitImportMode::WorkingTree, + false, + ) + .unwrap(); + let db = crate::Trail::open(temp.path()).unwrap(); + let scope_id = workspace_daemon_target(&db).unwrap().identity.scope_id; + let root = temp.path().to_path_buf(); + super::super::install_initial_scan_hook(scope_id, move || { + std::fs::write(root.join(".gitignore"), b"generated/**\n")?; + Ok(()) + }); + let launch = WorkspaceDaemonLaunchIdentity { + nonce: "9a".repeat(32), + pid: std::process::id(), + process_start_identity: "workspace-startup-retry-test".into(), + }; + + let proof = prepare_workspace_daemon_launch(&db, launch.clone(), None).unwrap(); + assert_eq!(proof.epoch, 2); + assert_eq!(proof.daemon_launch_nonce, Some(launch.nonce.clone())); + assert_eq!(proof.reconcile_report.unwrap().resulting_state, "trusted"); + assert!(workspace_daemon_ready_proof(&db).is_ok()); + let binding: (i64, String, i64, String, String) = db + .conn + .query_row( + "SELECT epoch,daemon_launch_nonce,daemon_pid, + daemon_process_start_identity,lease_state + FROM changed_path_observer_owners WHERE scope_id=?1", + [scope_id.to_text()], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, + ) + .unwrap(); + assert_eq!(binding.0, 2); + assert_eq!(binding.1, launch.nonce); + assert_eq!(binding.2, i64::from(launch.pid)); + assert_eq!(binding.3, launch.process_start_identity); + assert_eq!(binding.4, "active"); + } + + #[test] + fn workspace_startup_retry_cannot_replace_an_owner_swapped_after_capability_capture() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("tracked.txt"), b"baseline\n").unwrap(); + crate::Trail::init( + temp.path(), + "main", + crate::InitImportMode::WorkingTree, + false, + ) + .unwrap(); + let db = crate::Trail::open(temp.path()).unwrap(); + let scope_id = workspace_daemon_target(&db).unwrap().identity.scope_id; + let root = temp.path().to_path_buf(); + super::super::install_initial_scan_hook(scope_id, move || { + std::fs::write(root.join(".gitignore"), b"generated/**\n")?; + Ok(()) + }); + let replacement_token = "bc".repeat(32); + let replacement_nonce = "cd".repeat(32); + let expected_token = replacement_token.clone(); + let expected_nonce = replacement_nonce.clone(); + install_workspace_retry_boundary_hook(scope_id, move |db| { + let tx = db.conn.unchecked_transaction()?; + let owner_changed = tx.execute( + "UPDATE changed_path_observer_owners + SET owner_token=?1,daemon_launch_nonce=?2,daemon_pid=?3, + daemon_process_start_identity='different-live-owner', + lease_state='active',error_state=NULL,error_at=NULL, + heartbeat_at=strftime('%s','now'),expires_at=strftime('%s','now')+30, + updated_at=strftime('%s','now') + WHERE scope_id=?4", + params![ + replacement_token, + replacement_nonce, + i64::from(std::process::id()), + scope_id.to_text(), + ], + )?; + let scope_changed = tx.execute( + "UPDATE changed_path_scopes SET observer_owner_token=?1 WHERE scope_id=?2", + params![replacement_token, scope_id.to_text()], + )?; + if owner_changed != 1 || scope_changed != 1 { + return Err(Error::Corrupt( + "retry-race fixture could not install replacement owner".into(), + )); + } + tx.commit()?; + Ok(()) + }); + let launch = WorkspaceDaemonLaunchIdentity { + nonce: "ab".repeat(32), + pid: std::process::id(), + process_start_identity: "original-startup-owner".into(), + }; + + let error = prepare_workspace_daemon_launch(&db, launch, None) + .err() + .expect("swapped owner unexpectedly authorized retry"); + assert!(error + .to_string() + .contains("does not match the exact persisted observer scope/epoch/owner token")); + assert!(daemon_registry(&db).workspace.is_none()); + let owner: (String, String, String, String) = db + .conn + .query_row( + "SELECT owner.owner_token,owner.daemon_launch_nonce,owner.lease_state, + scope.observer_owner_token + FROM changed_path_observer_owners owner + JOIN changed_path_scopes scope ON scope.scope_id=owner.scope_id + WHERE owner.scope_id=?1", + [scope_id.to_text()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .unwrap(); + assert_eq!( + owner, + ( + expected_token.clone(), + expected_nonce, + "active".into(), + expected_token + ) + ); + } + + #[test] + fn startup_retry_classification_keeps_overflow_fail_closed() { + assert!(startup_policy_retryable( + &Error::ChangeLedgerReconcileRequired { + scope: "scope".into(), + state: "stale_baseline".into(), + reason: "fsevents_policy_dependency_invalidated:.gitignore".into(), + command: "trail index reconcile".into(), + } + )); + assert!(!startup_policy_retryable( + &Error::ChangeLedgerReconcileRequired { + scope: "scope".into(), + state: "untrusted_gap".into(), + reason: "fsevents_history_overflow".into(), + command: "trail index reconcile".into(), + } + )); + } + + #[cfg(target_os = "macos")] + #[test] + fn direct_policy_drift_between_end_fence_and_lease_restarts_and_reconciles() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("tracked.txt"), b"baseline\n").unwrap(); + crate::Trail::init( + temp.path(), + "main", + crate::InitImportMode::WorkingTree, + false, + ) + .unwrap(); + let db = crate::Trail::open(temp.path()).unwrap(); + let first = prepare_workspace_daemon(&db, true).unwrap(); + daemon_registry(&db) + .workspace + .as_mut() + .unwrap() + .inject_policy_drift_after_end = true; + + let restarted = workspace_daemon_fence(&db, None, None).unwrap(); + + assert!(restarted.epoch > first.epoch); + assert_eq!( + restarted.reconcile_report.unwrap().resulting_state, + "trusted" + ); + assert!(workspace_daemon_ready_proof(&db).is_ok()); + } + + #[test] + fn persistent_startup_policy_churn_exhausts_the_bound_and_fails_closed() { + fn install_churn(scope_id: ScopeId, ignore: std::path::PathBuf, remaining: usize) { + super::super::install_initial_scan_hook(scope_id, move || { + std::fs::write(&ignore, format!("generated-{remaining}/**\n").as_bytes())?; + if remaining > 1 { + install_churn(scope_id, ignore, remaining - 1); + } + Ok(()) + }); + } + + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("tracked.txt"), b"baseline\n").unwrap(); + crate::Trail::init( + temp.path(), + "main", + crate::InitImportMode::WorkingTree, + false, + ) + .unwrap(); + let mut db = crate::Trail::open(temp.path()).unwrap(); + let lane = db + .spawn_lane("persistent-policy-churn", Some("main"), true, None, None) + .unwrap(); + let lane_id = lane.lane_id; + let scope_id = materialized_lane_scope_id(&db.config.workspace.id.0, &lane_id); + install_churn( + scope_id, + std::path::PathBuf::from(lane.workdir.unwrap()).join(".gitignore"), + 3, + ); + + let error = materialized_lane_daemon_full_reconcile(&db, &lane_id).unwrap_err(); + assert!( + matches!(&error, Error::ChangeLedgerReconcileRequired { .. }), + "persistent churn failed with the wrong error: {error:?}" + ); + assert!(!daemon_registry(&db) + .materialized_lanes + .contains_key(&lane_id)); + let state: String = db + .conn + .query_row( + "SELECT trust_state FROM changed_path_scopes WHERE scope_id=?1", + [scope_id.to_text()], + |row| row.get(0), + ) + .unwrap(); + assert_ne!(state, "trusted"); + } +} + +struct ExistingScope { + scope_kind: String, + owner_id: String, + scope_root: String, + scope_root_identity: String, + filesystem_kind: String, + case_sensitive: i64, + schema_version: i64, + limits: [i64; 5], + epoch: u64, + ref_name: String, + ref_generation: u64, + change_id: String, + baseline_root: String, + policy_fingerprint: [u8; 32], + policy_generation: u64, + filesystem_identity: Vec, + provider_identity: Vec, + provider_id_text: Option, + provider_identity_text: Option, + provider_cursor: Option>, + provider_fence: Option>, + capabilities: [i64; 7], + trust_state: String, + trust_reason: String, + continuity_generation: u64, + durable_offset: u64, + folded_offset: u64, + observer_owner_token: Option, + observer_heartbeat_at: Option, + observer_error_state: Option, + observer_error_at: Option, + retired_at: Option, + observer_owner: Option, +} + +struct ExistingObserverOwner { + epoch: u64, + owner_token: String, + provider_id: String, + provider_identity: String, + lease_state: String, + fence_nonce: Option>, + acquired_at: i64, + heartbeat_at: i64, + expires_at: i64, + error_state: Option, + error_at: Option, + daemon_launch_nonce: Option, + daemon_pid: Option, + daemon_process_start_identity: Option, +} + +fn load_existing_scope(db: &Trail, scope_id: ScopeId) -> Result> { + let row = db + .conn + .query_row( + "SELECT scope.scope_kind,scope.owner_id,scope.scope_root, + scope.scope_root_identity,scope.filesystem_kind,scope.case_sensitive, + scope.epoch,scope.ref_name,scope.ref_generation,scope.change_id, + scope.baseline_root_id,scope.policy_fingerprint, + scope.policy_dependency_generation,scope.filesystem_identity, + scope.provider_id,scope.provider_identity,scope.provider_cursor, + scope.provider_fence,scope.durable_cursor,scope.linearizable_fence, + scope.rename_pairing,scope.overflow_scope,scope.filesystem_supported, + scope.clean_proof_allowed,scope.power_loss_durability, + scope.trust_state,scope.trust_reason,scope.continuity_generation, + scope.durable_offset,scope.folded_offset,scope.observer_owner_token, + scope.observer_heartbeat_at,scope.observer_error_state, + scope.observer_error_at,scope.retired_at,scope.schema_version, + scope.max_candidate_rows,scope.max_prefix_rows, + scope.max_observer_log_bytes,scope.max_segment_bytes, + scope.max_unfolded_tail_records, + owner.epoch,owner.owner_token,owner.provider_id, + owner.provider_identity,owner.lease_state,owner.fence_nonce, + owner.acquired_at,owner.heartbeat_at,owner.expires_at, + owner.error_state,owner.error_at,owner.daemon_launch_nonce, + owner.daemon_pid,owner.daemon_process_start_identity + FROM changed_path_scopes scope + LEFT JOIN changed_path_observer_owners owner ON owner.scope_id=scope.scope_id + WHERE scope.scope_id=?1", + [scope_id.to_text()], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, i64>(6)?, + row.get::<_, String>(7)?, + row.get::<_, i64>(8)?, + row.get::<_, String>(9)?, + row.get::<_, String>(10)?, + row.get::<_, String>(11)?, + row.get::<_, i64>(12)?, + row.get::<_, String>(13)?, + row.get::<_, Option>(14)?, + row.get::<_, Option>(15)?, + row.get::<_, Option>>(16)?, + row.get::<_, Option>>(17)?, + [ + row.get::<_, i64>(18)?, + row.get::<_, i64>(19)?, + row.get::<_, i64>(20)?, + row.get::<_, i64>(21)?, + row.get::<_, i64>(22)?, + row.get::<_, i64>(23)?, + row.get::<_, i64>(24)?, + ], + row.get::<_, String>(25)?, + row.get::<_, String>(26)?, + row.get::<_, i64>(27)?, + row.get::<_, i64>(28)?, + row.get::<_, i64>(29)?, + row.get::<_, Option>(30)?, + row.get::<_, Option>(31)?, + row.get::<_, Option>(32)?, + row.get::<_, Option>(33)?, + row.get::<_, Option>(34)?, + row.get::<_, i64>(35)?, + [ + row.get::<_, i64>(36)?, + row.get::<_, i64>(37)?, + row.get::<_, i64>(38)?, + row.get::<_, i64>(39)?, + row.get::<_, i64>(40)?, + ], + row.get::<_, Option>(41)?, + row.get::<_, Option>(42)?, + row.get::<_, Option>(43)?, + row.get::<_, Option>(44)?, + row.get::<_, Option>(45)?, + row.get::<_, Option>>(46)?, + row.get::<_, Option>(47)?, + row.get::<_, Option>(48)?, + row.get::<_, Option>(49)?, + row.get::<_, Option>(50)?, + row.get::<_, Option>(51)?, + row.get::<_, Option>(52)?, + row.get::<_, Option>(53)?, + row.get::<_, Option>(54)?, + )) + }, + ) + .optional()?; + row.map( + |( + scope_kind, + owner_id, + scope_root, + scope_root_identity, + filesystem_kind, + case_sensitive, + epoch, + ref_name, + ref_generation, + change_id, + baseline_root, + policy_fingerprint, + policy_generation, + filesystem_identity, + provider_id_text, + provider_identity_text, + provider_cursor, + provider_fence, + capabilities, + trust_state, + trust_reason, + continuity_generation, + durable_offset, + folded_offset, + observer_owner_token, + observer_heartbeat_at, + observer_error_state, + observer_error_at, + retired_at, + schema_version, + limits, + owner_epoch, + owner_token, + owner_provider_id, + owner_provider_identity, + owner_lease_state, + owner_fence_nonce, + owner_acquired_at, + owner_heartbeat_at, + owner_expires_at, + owner_error_state, + owner_error_at, + owner_daemon_launch_nonce, + owner_daemon_pid, + owner_daemon_process_start_identity, + )| { + let provider_identity_text = provider_identity_text.ok_or_else(|| { + Error::Corrupt("daemon scope is missing provider identity".into()) + })?; + let observer_owner = match ( + owner_epoch, + owner_token, + owner_provider_id, + owner_provider_identity, + owner_lease_state, + owner_acquired_at, + owner_heartbeat_at, + owner_expires_at, + ) { + ( + Some(epoch), + Some(token), + Some(provider_id), + Some(provider_identity), + Some(lease_state), + Some(acquired_at), + Some(heartbeat_at), + Some(expires_at), + ) => Some(ExistingObserverOwner { + epoch: u64::try_from(epoch) + .map_err(|_| Error::Corrupt("negative observer owner epoch".into()))?, + owner_token: token, + provider_id, + provider_identity, + lease_state, + fence_nonce: owner_fence_nonce, + acquired_at, + heartbeat_at, + expires_at, + error_state: owner_error_state, + error_at: owner_error_at, + daemon_launch_nonce: owner_daemon_launch_nonce, + daemon_pid: owner_daemon_pid, + daemon_process_start_identity: owner_daemon_process_start_identity, + }), + (None, None, None, None, None, None, None, None) => None, + _ => { + return Err(Error::Corrupt( + "partial daemon observer owner authority".into(), + )) + } + }; + Ok(ExistingScope { + scope_kind, + owner_id, + scope_root, + scope_root_identity, + filesystem_kind, + case_sensitive, + schema_version, + limits, + epoch: u64::try_from(epoch) + .map_err(|_| Error::Corrupt("negative changed-path scope epoch".into()))?, + ref_name, + ref_generation: u64::try_from(ref_generation) + .map_err(|_| Error::Corrupt("negative changed-path ref generation".into()))?, + change_id, + baseline_root, + policy_fingerprint: decode_fingerprint(&policy_fingerprint)?, + policy_generation: u64::try_from(policy_generation).map_err(|_| { + Error::Corrupt("negative changed-path policy generation".into()) + })?, + filesystem_identity: hex::decode(filesystem_identity).map_err(|_| { + Error::Corrupt("invalid changed-path filesystem identity".into()) + })?, + provider_identity: hex::decode(&provider_identity_text) + .map_err(|_| Error::Corrupt("invalid changed-path provider identity".into()))?, + provider_id_text, + provider_identity_text: Some(provider_identity_text), + provider_cursor, + provider_fence, + capabilities, + trust_state, + trust_reason, + continuity_generation: u64::try_from(continuity_generation) + .map_err(|_| Error::Corrupt("negative continuity generation".into()))?, + durable_offset: u64::try_from(durable_offset) + .map_err(|_| Error::Corrupt("negative durable offset".into()))?, + folded_offset: u64::try_from(folded_offset) + .map_err(|_| Error::Corrupt("negative folded offset".into()))?, + observer_owner_token, + observer_heartbeat_at, + observer_error_state, + observer_error_at, + retired_at, + observer_owner, + }) + }, + ) + .transpose() +} diff --git a/trail/src/db/change_ledger/intent.rs b/trail/src/db/change_ledger/intent.rs new file mode 100644 index 0000000..0696817 --- /dev/null +++ b/trail/src/db/change_ledger/intent.rs @@ -0,0 +1,1112 @@ +use std::collections::BTreeSet; +use std::path::Path; +use std::time::{Duration, Instant}; + +use getrandom::getrandom; +use rusqlite::{params, OptionalExtension, Transaction, TransactionBehavior}; +use serde::{Deserialize, Serialize}; + +use super::{DirtyPrefix, EvidenceCut, EvidenceFlags, ExpectedScope, LedgerPath}; +use crate::db::util::now_ts; +use crate::error::{Error, Result}; +use crate::model::{OPERATION_KIND, WORKTREE_ROOT_KIND}; +use crate::{ChangeId, ObjectId}; + +#[cfg(debug_assertions)] +thread_local! { + static SIDECAR_ANCESTOR_SUBSTITUTION_HOOK: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(test)] +thread_local! { + static SIDECAR_RETRY_HOOK: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; + static SIDECAR_POST_VALIDATION_HOOK: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(test)] +pub(super) fn install_sidecar_retry_hook(hook: impl FnOnce() + 'static) { + SIDECAR_RETRY_HOOK.with(|slot| { + *slot.borrow_mut() = Some(Box::new(hook)); + }); +} + +#[cfg(test)] +fn run_sidecar_retry_hook() { + SIDECAR_RETRY_HOOK.with(|slot| { + if let Some(hook) = slot.borrow_mut().take() { + hook(); + } + }); +} + +#[cfg(test)] +pub(super) fn install_sidecar_post_validation_hook(hook: impl FnOnce() + 'static) { + SIDECAR_POST_VALIDATION_HOOK.with(|slot| { + *slot.borrow_mut() = Some(Box::new(hook)); + }); +} + +#[cfg(test)] +fn run_sidecar_post_validation_hook() { + SIDECAR_POST_VALIDATION_HOOK.with(|slot| { + if let Some(hook) = slot.borrow_mut().take() { + hook(); + } + }); +} + +#[cfg(debug_assertions)] +pub(super) fn install_sidecar_ancestor_substitution_hook(hook: impl FnOnce() + 'static) { + SIDECAR_ANCESTOR_SUBSTITUTION_HOOK.with(|slot| { + *slot.borrow_mut() = Some(Box::new(hook)); + }); +} + +#[cfg(debug_assertions)] +fn run_sidecar_ancestor_substitution_hook() { + SIDECAR_ANCESTOR_SUBSTITUTION_HOOK.with(|slot| { + if let Some(hook) = slot.borrow_mut().take() { + hook(); + } + }); +} + +const SIDECAR_RECOVERY_HARD_LIMIT: Duration = Duration::from_secs(15); +pub(super) const SIDECAR_RECOVERY_IDLE_LIMIT: Duration = Duration::from_secs(2); +const SIDECAR_RECOVERY_RETRY_DELAY: Duration = Duration::from_millis(2); + +#[derive(Default)] +struct SidecarRecoveryBudget { + last_progress_at: Duration, + last_progress: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct SidecarRecoveryProgress { + durable_end: u64, + last_sequence: u64, + last_hash: [u8; 32], + record_count: usize, + boundary_count: usize, + segment_count: usize, + tail_segment: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct SidecarTailSegmentProgress { + segment_id: String, + state: String, + last_sequence: u64, + durable_end_offset: u64, + folded_end_offset: u64, + segment_hash: [u8; 32], +} + +impl From<&super::RecoveredTail> for SidecarRecoveryProgress { + fn from(recovered: &super::RecoveredTail) -> Self { + Self { + durable_end: recovered.durable_end, + last_sequence: recovered.last_sequence, + last_hash: recovered.last_hash, + record_count: recovered.records.len(), + boundary_count: recovered.record_boundaries.len(), + segment_count: recovered.segments.len(), + tail_segment: recovered + .segments + .last() + .map(|segment| SidecarTailSegmentProgress { + segment_id: segment.segment_id.clone(), + state: segment.state.clone(), + last_sequence: segment.last_sequence, + durable_end_offset: segment.durable_end_offset, + folded_end_offset: segment.folded_end_offset, + segment_hash: segment.segment_hash, + }), + } + } +} + +impl SidecarRecoveryBudget { + fn retry_after(&mut self, elapsed: Duration, recovered: &super::RecoveredTail) -> bool { + if elapsed >= SIDECAR_RECOVERY_HARD_LIMIT { + return false; + } + let progress = SidecarRecoveryProgress::from(recovered); + if self.last_progress.as_ref() != Some(&progress) { + self.last_progress = Some(progress); + self.last_progress_at = elapsed; + } + elapsed.saturating_sub(self.last_progress_at) < SIDECAR_RECOVERY_IDLE_LIMIT + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub(crate) struct IntentId(pub(crate) String); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum IntentProducer { + Checkout, + LaneSync, + Materialize, + StructuredPatchProjection, + RestoreProjection, + CowPublication, + ObservedCheckpoint, +} + +impl IntentProducer { + pub(super) const fn as_str(self) -> &'static str { + match self { + Self::Checkout => "checkout", + Self::LaneSync => "lane_sync", + Self::Materialize => "materialize", + Self::StructuredPatchProjection => "structured_patch_projection", + Self::RestoreProjection => "restore_projection", + Self::CowPublication => "cow_publication", + Self::ObservedCheckpoint => "observed_checkpoint", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum IntentState { + Prepared, + FilesystemApplied, + Published, + Acknowledged, + Aborted, +} + +impl IntentState { + pub(super) const fn as_str(self) -> &'static str { + match self { + Self::Prepared => "prepared", + Self::FilesystemApplied => "filesystem_applied", + Self::Published => "published", + Self::Acknowledged => "acknowledged", + Self::Aborted => "aborted", + } + } + + pub(super) fn parse(value: &str) -> Result { + match value { + "prepared" => Ok(Self::Prepared), + "filesystem_applied" => Ok(Self::FilesystemApplied), + "published" => Ok(Self::Published), + "acknowledged" => Ok(Self::Acknowledged), + "aborted" => Ok(Self::Aborted), + other => Err(Error::Corrupt(format!( + "unknown changed-path intent state `{other}`" + ))), + } + } + + pub(super) const fn is_terminal(self) -> bool { + matches!(self, Self::Acknowledged | Self::Aborted) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct IntentTarget { + pub(crate) change_id: ChangeId, + pub(crate) root_id: ObjectId, + pub(crate) operation_id: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct IntentEvidence { + pub(crate) exact_paths: Vec, + pub(crate) complete_prefixes: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct QualifiedFilesystemProof { + pub(crate) scope_id: super::ScopeId, + pub(crate) epoch: u64, + pub(crate) expected_root_id: ObjectId, + pub(crate) scope_root_identity: Vec, + pub(crate) filesystem_identity: Vec, + pub(crate) provider_id: String, + pub(crate) provider_identity: Vec, + pub(crate) observer_owner_token: String, + pub(crate) owner_fence_nonce: Option>, + pub(crate) durable_segment_id: String, + pub(crate) durable_segment_hash: [u8; 32], + pub(crate) segment_directory: String, + pub(crate) segment_path: String, + pub(crate) start_cursor: Option>, + pub(crate) end_cursor: Vec, + pub(crate) publication_segment_id: String, + /// Provider cursor at `publication_cut`. This can differ from + /// `end_cursor` when the observer advances after the verified c1 cut. + pub(crate) publication_cursor: Vec, + pub(crate) start_sequence: u64, + /// Cut through which the controlled apply was pinned and may be + /// acknowledged. Events after this cut are never consumed by publication. + pub(crate) end_cut: EvidenceCut, + /// Final folded cut used only for the scope baseline CAS. Evidence in + /// `(end_cut, publication_cut]` remains pending for the next snapshot. + pub(crate) publication_cut: EvidenceCut, + pub(crate) segment_durable_offset: u64, + pub(crate) segment_folded_offset: u64, + pub(crate) verified_paths: u64, + pub(crate) verified_prefixes: u64, + pub(crate) complete_root_interval: bool, + pub(crate) complete_policy_interval: bool, + pub(crate) persisted_evidence_through_end: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct PersistedIntent { + pub(super) id: IntentId, + pub(super) scope_id: String, + pub(super) state: IntentState, + pub(super) expected_epoch: u64, + pub(super) expected_ref_name: String, + pub(super) expected_ref_generation: u64, + pub(super) expected_change_id: ChangeId, + pub(super) expected_root_id: ObjectId, + pub(super) target: IntentTarget, + pub(super) start_cursor: Option>, + pub(super) verified_cut: Option, +} + +pub(crate) fn prepare_intent( + ledger: &super::ChangedPathLedger<'_>, + expected: &ExpectedScope, + producer: IntentProducer, + target: &IntentTarget, + evidence: &IntentEvidence, +) -> Result { + validate_evidence(evidence)?; + ledger.recover_scope(expected)?; + let tx = Transaction::new_unchecked(ledger.conn, TransactionBehavior::Immediate)?; + exact_scope_guard(&tx, expected, true)?; + let pending: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM changed_path_intents + WHERE scope_id=?1 AND lifecycle_state IN ('prepared','filesystem_applied','published'))", + [expected.scope_id.to_text()], + |row| row.get(0), + )?; + if pending { + return Err(Error::Conflict( + "changed-path scope still has a nonterminal intent".into(), + )); + } + let (expected_change_id, start_cursor) = tx.query_row( + "SELECT change_id,provider_cursor FROM changed_path_scopes WHERE scope_id=?1", + [expected.scope_id.to_text()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, Option>>(1)?)), + )?; + let id = IntentId(new_intent_id()?); + let now = now_ts(); + tx.execute( + "INSERT INTO changed_path_intents( + intent_id,schema_version,scope_id,producer,expected_scope_epoch, + expected_ref_name,expected_ref_generation,expected_change_id,expected_root_id, + target_change_id,target_root_id,target_operation_id,start_cursor,lifecycle_state, + verified_cut,failure_reason,created_at,updated_at + ) VALUES(?1,1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,'prepared',NULL,NULL,?13,?13)", + params![ + id.0, + expected.scope_id.to_text(), + producer.as_str(), + sql_u64(expected.epoch, "scope epoch")?, + expected.ref_name, + sql_u64(expected.ref_generation, "ref generation")?, + expected_change_id, + expected.baseline_root.0, + target.change_id.0, + target.root_id.0, + target.operation_id.as_ref().map(|id| id.0.as_str()), + start_cursor, + now, + ], + )?; + for path in &evidence.exact_paths { + tx.execute( + "INSERT INTO changed_path_intent_paths(intent_id,normalized_path,event_flags) + VALUES(?1,?2,?3)", + params![id.0, path.as_str(), EvidenceFlags::ANY_MUTATION.0], + )?; + } + for prefix in &evidence.complete_prefixes { + tx.execute( + "INSERT INTO changed_path_intent_prefixes( + intent_id,normalized_prefix,completeness_reason,event_flags + ) VALUES(?1,?2,?3,?4)", + params![ + id.0, + prefix.path.as_str(), + prefix.reason, + EvidenceFlags::ANY_MUTATION.0 + ], + )?; + } + tx.commit()?; + durable_intent_barrier(ledger.conn)?; + crate::db::util::test_crash_point("changed_path_after_intent_prepare"); + Ok(id) +} + +pub(crate) fn mark_filesystem_applied( + ledger: &super::ChangedPathLedger<'_>, + expected: &ExpectedScope, + intent_id: &IntentId, + proof: &QualifiedFilesystemProof, +) -> Result<()> { + // Do not hold a SQLite writer reservation while authenticating the live + // sidecar. The observer publishes a just-fsynced open tail to SQLite; an + // IMMEDIATE transaction here would prevent that publication and turn the + // normal file-before-database window into a self-sustaining mismatch. + exact_scope_guard(ledger.conn, expected, true)?; + let intent = load_intent(ledger.conn, intent_id)?.ok_or_else(|| { + Error::InvalidInput(format!("unknown changed-path intent `{}`", intent_id.0)) + })?; + if intent.scope_id != expected.scope_id.to_text() || intent.state != IntentState::Prepared { + return Err(Error::Conflict(format!( + "intent `{}` is not in prepared state for this scope", + intent_id.0 + ))); + } + validate_qualified_filesystem_proof( + ledger.conn, + ledger.database_path()?, + expected, + &intent, + proof, + )?; + #[cfg(test)] + run_sidecar_post_validation_hook(); + + let tx = Transaction::new_unchecked(ledger.conn, TransactionBehavior::Immediate)?; + exact_scope_guard(&tx, expected, true)?; + let current = load_intent(&tx, intent_id)?.ok_or_else(|| { + Error::InvalidInput(format!("unknown changed-path intent `{}`", intent_id.0)) + })?; + // Intent path/prefix evidence is insert-only during prepare_intent; no + // lifecycle path mutates those child rows. Reloading the full parent here + // therefore binds the same prepared intent whose ordered evidence was + // authenticated above, while the joined authority query binds its live + // scope/owner/fence/segment at the publication transaction boundary. + if current != intent + || current.scope_id != expected.scope_id.to_text() + || current.state != IntentState::Prepared + || !filesystem_proof_authority_matches(&tx, expected, proof)? + { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "filesystem proof authority changed before intent publication".into(), + command: "trail status".into(), + }); + } + let changed = tx.execute( + "UPDATE changed_path_intents SET lifecycle_state='filesystem_applied',verified_cut=?1, + updated_at=?2 + WHERE intent_id=?3 AND scope_id=?4 AND lifecycle_state='prepared'", + params![ + serde_json::to_vec(proof)?, + now_ts(), + intent_id.0, + expected.scope_id.to_text() + ], + )?; + if changed != 1 { + return Err(Error::Conflict(format!( + "intent `{}` is not in prepared state", + intent_id.0 + ))); + } + tx.commit()?; + durable_intent_barrier(ledger.conn)?; + crate::db::util::test_crash_point("changed_path_after_filesystem_applied"); + Ok(()) +} + +pub(crate) fn publish_intent( + ledger: &super::ChangedPathLedger<'_>, + expected: &ExpectedScope, + intent_id: &IntentId, +) -> Result<()> { + let tx = Transaction::new_unchecked(ledger.conn, TransactionBehavior::Immediate)?; + exact_scope_guard(&tx, expected, true)?; + let intent = load_intent(&tx, intent_id)?.ok_or_else(|| { + Error::InvalidInput(format!("unknown changed-path intent `{}`", intent_id.0)) + })?; + if intent.scope_id != expected.scope_id.to_text() + || intent.state != IntentState::FilesystemApplied + { + return Err(Error::Conflict(format!( + "intent `{}` is not filesystem-applied for this scope", + intent_id.0 + ))); + } + if !authoritative_ref_matches_target(&tx, &intent)? { + return Err(Error::StaleBranch(intent.expected_ref_name)); + } + stage_intent_evidence(&tx, &intent)?; + let changed = tx.execute( + "UPDATE changed_path_intents SET lifecycle_state='published',updated_at=?1 + WHERE intent_id=?2 AND lifecycle_state='filesystem_applied'", + params![now_ts(), intent_id.0], + )?; + if changed != 1 { + return Err(Error::Conflict( + "intent publication raced another transition".into(), + )); + } + tx.commit()?; + durable_intent_barrier(ledger.conn)?; + crate::db::util::test_crash_point("changed_path_after_intent_publish"); + Ok(()) +} + +pub(super) fn load_intent( + conn: &rusqlite::Connection, + id: &IntentId, +) -> Result> { + conn.query_row( + "SELECT intent_id,scope_id,lifecycle_state,expected_scope_epoch,expected_ref_name, + expected_ref_generation,expected_change_id,expected_root_id,target_change_id, + target_root_id,target_operation_id,start_cursor,verified_cut + FROM changed_path_intents WHERE intent_id=?1", + [&id.0], + |row| { + let state = row.get::<_, String>(2)?; + let verified = row.get::<_, Option>>(12)?; + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + state, + row.get::<_, i64>(3)?, + row.get::<_, String>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, String>(6)?, + row.get::<_, String>(7)?, + row.get::<_, String>(8)?, + row.get::<_, String>(9)?, + row.get::<_, Option>(10)?, + row.get::<_, Option>>(11)?, + verified, + )) + }, + ) + .optional()? + .map(|row| { + Ok(PersistedIntent { + id: IntentId(row.0), + scope_id: row.1, + state: IntentState::parse(&row.2)?, + expected_epoch: db_u64(row.3, "intent epoch")?, + expected_ref_name: row.4, + expected_ref_generation: db_u64(row.5, "intent ref generation")?, + expected_change_id: ChangeId(row.6), + expected_root_id: ObjectId(row.7), + target: IntentTarget { + change_id: ChangeId(row.8), + root_id: ObjectId(row.9), + operation_id: row.10.map(ObjectId), + }, + start_cursor: row.11, + verified_cut: row + .12 + .map(|bytes| serde_json::from_slice(&bytes)) + .transpose()?, + }) + }) + .transpose() +} + +pub(super) fn stage_intent_evidence(tx: &Transaction<'_>, intent: &PersistedIntent) -> Result<()> { + let sequence = intent + .verified_cut + .as_ref() + .map(|proof| proof.end_cut.sequence) + .unwrap_or_default(); + let sequence = sql_u64(sequence, "intent sequence")?; + let now = now_ts(); + tx.execute( + "INSERT INTO changed_path_entries(scope_id,normalized_path,event_flags,source_mask, + first_sequence,last_sequence,provider_id,provider_sequence,intent_id,created_at,updated_at) + SELECT i.scope_id,p.normalized_path,p.event_flags,2,?1,?1,'intent',NULL,i.intent_id,?2,?2 + FROM changed_path_intent_paths p JOIN changed_path_intents i ON i.intent_id=p.intent_id + WHERE p.intent_id=?3 + ON CONFLICT(scope_id,normalized_path) DO UPDATE SET + event_flags=changed_path_entries.event_flags|excluded.event_flags, + source_mask=changed_path_entries.source_mask|excluded.source_mask, + first_sequence=MIN(changed_path_entries.first_sequence,excluded.first_sequence), + last_sequence=MAX(changed_path_entries.last_sequence,excluded.last_sequence), + intent_id=excluded.intent_id,updated_at=excluded.updated_at", + params![sequence, now, intent.id.0], + )?; + tx.execute( + "INSERT INTO changed_path_prefixes(scope_id,normalized_prefix,completeness_reason, + event_flags,source_mask,first_sequence,last_sequence,provider_id,provider_sequence, + intent_id,created_at,updated_at) + SELECT i.scope_id,p.normalized_prefix,p.completeness_reason,p.event_flags,2,?1,?1, + 'intent',NULL,i.intent_id,?2,?2 + FROM changed_path_intent_prefixes p JOIN changed_path_intents i ON i.intent_id=p.intent_id + WHERE p.intent_id=?3 + ON CONFLICT(scope_id,normalized_prefix) DO UPDATE SET + event_flags=changed_path_prefixes.event_flags|excluded.event_flags, + source_mask=changed_path_prefixes.source_mask|excluded.source_mask, + first_sequence=MIN(changed_path_prefixes.first_sequence,excluded.first_sequence), + last_sequence=MAX(changed_path_prefixes.last_sequence,excluded.last_sequence), + intent_id=excluded.intent_id,updated_at=excluded.updated_at", + params![sequence, now, intent.id.0], + )?; + Ok(()) +} + +fn filesystem_proof_authority_matches( + conn: &rusqlite::Connection, + expected: &ExpectedScope, + proof: &QualifiedFilesystemProof, +) -> Result { + conn.query_row( + "SELECT EXISTS( + SELECT 1 + FROM changed_path_scopes scope + JOIN changed_path_observer_owners owner + ON owner.scope_id=scope.scope_id AND owner.epoch=scope.epoch + JOIN changed_path_observer_segments segment + ON segment.scope_id=scope.scope_id AND segment.epoch=scope.epoch + WHERE scope.scope_id=?1 AND scope.epoch=?2 + AND scope.observer_owner_token=?3 AND scope.trust_state='trusted' + AND owner.owner_token=?3 AND owner.provider_id=?4 + AND owner.provider_identity=?5 AND owner.fence_nonce IS ?6 + AND owner.lease_state='active' AND owner.error_state IS NULL + AND owner.error_at IS NULL AND owner.expires_at>?7 + AND segment.segment_id=?8 AND segment.owner_token=?3 + AND segment.provider_id=?4 AND segment.first_sequence<=?9 + AND segment.last_sequence=?10 AND segment.durable_end_offset=?11 + AND segment.folded_end_offset=?12 AND segment.segment_hash=?13 + AND segment.segment_path=?14 AND segment.state='sealed' + )", + params![ + expected.scope_id.to_text(), + sql_u64(expected.epoch, "scope epoch")?, + proof.observer_owner_token, + proof.provider_id, + hex::encode(&proof.provider_identity), + proof.owner_fence_nonce, + now_ts(), + proof.durable_segment_id, + sql_u64(proof.start_sequence, "proof start sequence")?, + sql_u64(proof.end_cut.sequence, "proof end sequence")?, + sql_u64(proof.segment_durable_offset, "segment durable offset")?, + sql_u64(proof.segment_folded_offset, "segment folded offset")?, + hex::encode(proof.durable_segment_hash), + proof.segment_path, + ], + |row| row.get(0), + ) + .map_err(Error::from) +} + +pub(super) fn validate_qualified_filesystem_proof( + conn: &rusqlite::Connection, + database_path: &Path, + expected: &ExpectedScope, + intent: &PersistedIntent, + proof: &QualifiedFilesystemProof, +) -> Result<()> { + let invalid = |reason: &str| Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: reason.into(), + command: "trail status".into(), + }; + if proof.scope_id != expected.scope_id + || proof.epoch != expected.epoch + || proof.expected_root_id != intent.expected_root_id + || proof.filesystem_identity != expected.filesystem_identity + || proof.provider_identity != expected.provider_identity + || proof.start_cursor != intent.start_cursor + || proof.end_cut.source != super::EvidenceSource::Observer + || proof.publication_cut.source != super::EvidenceSource::Observer + || proof.start_sequence > proof.end_cut.sequence + || proof.end_cut.sequence > proof.publication_cut.sequence + || proof.end_cut.durable_offset != proof.end_cut.folded_offset + || proof.publication_cut.durable_offset != proof.publication_cut.folded_offset + || proof.segment_folded_offset != proof.segment_durable_offset + || proof.observer_owner_token.is_empty() + || proof.provider_id.is_empty() + || proof.durable_segment_id.is_empty() + || proof.publication_segment_id.is_empty() + || proof.segment_directory.is_empty() + || proof.segment_path.is_empty() + || !proof.complete_root_interval + || !proof.complete_policy_interval + || !proof.persisted_evidence_through_end + { + return Err(invalid( + "filesystem proof identity or completeness is invalid", + )); + } + + let scope = conn + .query_row( + "SELECT scope_root_identity,filesystem_identity,provider_id,provider_identity, + provider_cursor,durable_offset,folded_offset,clean_proof_allowed, + linearizable_fence,filesystem_supported,power_loss_durability,trust_state + FROM changed_path_scopes WHERE scope_id=?1 AND epoch=?2", + params![ + expected.scope_id.to_text(), + sql_u64(expected.epoch, "scope epoch")? + ], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, i64>(6)?, + row.get::<_, bool>(7)?, + row.get::<_, bool>(8)?, + row.get::<_, bool>(9)?, + row.get::<_, bool>(10)?, + row.get::<_, String>(11)?, + )) + }, + ) + .optional()?; + let Some(( + scope_root_identity, + filesystem_identity, + provider_id, + provider_identity, + _provider_cursor, + durable_offset, + folded_offset, + clean_proof_allowed, + linearizable_fence, + filesystem_supported, + power_loss_durability, + trust_state, + )) = scope + else { + return Err(invalid("filesystem proof scope disappeared")); + }; + if trust_state != "trusted" + || !clean_proof_allowed + || !linearizable_fence + || !filesystem_supported + || !power_loss_durability + || hex::decode(scope_root_identity).ok().as_deref() + != Some(proof.scope_root_identity.as_slice()) + || hex::decode(filesystem_identity).ok().as_deref() + != Some(proof.filesystem_identity.as_slice()) + || provider_id.as_deref() != Some(proof.provider_id.as_str()) + || provider_identity + .as_deref() + .and_then(|identity| hex::decode(identity).ok()) + .as_deref() + != Some(proof.provider_identity.as_slice()) + // The verified intent cut may be an authenticated prefix of a newer + // observer suffix. Current scope offsets must contain that cut; exact + // boundary identity is checked against the recovered record below. + || db_u64(durable_offset, "scope durable offset")? < proof.publication_cut.durable_offset + || db_u64(folded_offset, "scope folded offset")? < proof.publication_cut.folded_offset + { + return Err(invalid( + "filesystem proof no longer matches the trusted scope boundary", + )); + } + + if !filesystem_proof_authority_matches(conn, expected, proof)? { + return Err(invalid( + "filesystem proof owner, fence, or sealed segment changed", + )); + } + + let expected_directory = format!("observer-segments/{}", expected.scope_id.to_text()); + if proof.segment_directory != expected_directory { + return Err(invalid( + "filesystem proof segment directory is not the confined scope directory", + )); + } + let database_root = database_path + .parent() + .and_then(Path::parent) + .ok_or_else(|| invalid("filesystem proof database has no Trail root"))?; + let trail_directory = super::secure_fs::SecureDirectory::open_absolute(database_root) + .map_err(|error| invalid(&format!("open Trail root securely: {error}")))?; + let observer_directory = trail_directory + .open_dir("observer-segments") + .map_err(|error| invalid(&format!("open observer root securely: {error}")))?; + #[cfg(debug_assertions)] + run_sidecar_ancestor_substitution_hook(); + let segment_directory = observer_directory + .open_dir(&expected.scope_id.to_text()) + .map_err(|error| invalid(&format!("open observer scope securely: {error}")))?; + let owner_token: [u8; 32] = hex::decode(&proof.observer_owner_token) + .ok() + .and_then(|bytes| bytes.try_into().ok()) + .ok_or_else(|| invalid("filesystem proof owner token is not canonical"))?; + let raw_limits: (i64, i64, i64) = conn.query_row( + "SELECT max_observer_log_bytes,max_segment_bytes,max_unfolded_tail_records + FROM changed_path_scopes WHERE scope_id=?1 AND epoch=?2", + params![ + expected.scope_id.to_text(), + sql_u64(expected.epoch, "scope epoch")? + ], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + let limits = super::PersistedLogLimits { + max_log_bytes: db_u64(raw_limits.0, "observer log byte limit")?, + max_segment_bytes: db_u64(raw_limits.1, "segment byte limit")?, + max_unfolded_tail_records: usize::try_from(db_u64( + raw_limits.2, + "unfolded observer record limit", + )?) + .map_err(|_| Error::Corrupt("unfolded observer record limit exceeds usize".into()))?, + }; + let recovery_scope = super::RecoveryScope { + scope_id: expected.scope_id, + epoch: expected.epoch, + owner_token, + }; + let started = Instant::now(); + let mut budget = SidecarRecoveryBudget::default(); + let recovered = loop { + if !filesystem_proof_authority_matches(conn, expected, proof)? { + return Err(invalid( + "filesystem proof owner, fence, or sealed segment changed during sidecar verification", + )); + } + let candidate = super::recover_segments_from_connection( + conn, + &segment_directory, + &recovery_scope, + limits, + ) + .map_err(|error| invalid(&format!("observer sidecar verification failed: {error}")))?; + if !filesystem_proof_authority_matches(conn, expected, proof)? { + return Err(invalid( + "filesystem proof owner, fence, or sealed segment changed during sidecar verification", + )); + } + if !candidate.requires_reconciliation { + break candidate; + } + if !budget.retry_after(started.elapsed(), &candidate) { + break candidate; + } + #[cfg(test)] + run_sidecar_retry_hook(); + std::thread::sleep(SIDECAR_RECOVERY_RETRY_DELAY); + }; + if recovered.requires_reconciliation { + return Err(invalid( + "observer sidecar chain is incomplete or contains unpublished entries", + )); + } + let authenticated = recovered + .segments + .iter() + .find(|segment| segment.segment_id == proof.durable_segment_id) + .ok_or_else(|| invalid("filesystem proof segment is absent from verified chain"))?; + let publication_boundary = recovered.record_boundaries.iter().find(|boundary| { + boundary.segment_id == proof.publication_segment_id + && boundary.sequence == proof.publication_cut.sequence + && boundary.durable_end_offset == proof.publication_cut.durable_offset + && proof.publication_cursor == boundary.provider_cursor + }); + let publication_authenticated = publication_boundary.is_some_and(|boundary| { + recovered.segments.iter().any(|segment| { + segment.segment_id == boundary.segment_id + && matches!(segment.state.as_str(), "open" | "sealed") + && segment.first_sequence <= boundary.sequence + && segment.last_sequence >= boundary.sequence + && segment.durable_end_offset >= boundary.durable_end_offset + && segment.folded_end_offset >= proof.publication_cut.folded_offset + }) + }); + let publication_anchor_authenticated = proof + .publication_cut + .sequence + .checked_add(1) + .is_some_and(|next_sequence| { + recovered.segments.iter().any(|segment| { + segment.segment_id == proof.publication_segment_id + && matches!(segment.state.as_str(), "open" | "sealed") + && segment.start_cursor == proof.publication_cursor + && segment.first_sequence == next_sequence + && segment.last_sequence >= proof.publication_cut.sequence + && segment.header_end_offset == proof.publication_cut.durable_offset + && segment.durable_end_offset >= proof.publication_cut.durable_offset + && segment.folded_end_offset >= proof.publication_cut.folded_offset + }) + }); + if authenticated.state != "sealed" + || authenticated.segment_path != proof.segment_path + || authenticated.start_cursor != proof.start_cursor.clone().unwrap_or_default() + || authenticated.end_cursor != proof.end_cursor + || authenticated.first_sequence != proof.start_sequence + || authenticated.last_sequence != proof.end_cut.sequence + || authenticated.durable_end_offset != proof.segment_durable_offset + || authenticated.folded_end_offset < proof.segment_folded_offset + || authenticated.segment_hash != proof.durable_segment_hash + || proof.end_cut.durable_offset != proof.segment_durable_offset + || proof.end_cut.folded_offset != proof.segment_folded_offset + || recovered.last_sequence < proof.publication_cut.sequence + || !(publication_authenticated || publication_anchor_authenticated) + { + return Err(invalid( + "filesystem proof is not an authenticated prefix of the observer chain", + )); + } + + let intent_paths = conn + .prepare( + "SELECT normalized_path,event_flags FROM changed_path_intent_paths + WHERE intent_id=?1 ORDER BY normalized_path COLLATE BINARY", + )? + .query_map([&intent.id.0], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })? + .collect::, _>>()?; + let intent_prefixes = conn + .prepare( + "SELECT normalized_prefix,completeness_reason,event_flags + FROM changed_path_intent_prefixes + WHERE intent_id=?1 ORDER BY normalized_prefix COLLATE BINARY", + )? + .query_map([&intent.id.0], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + })? + .collect::, _>>()?; + let mut exact_flags = std::collections::BTreeMap::<&str, i64>::new(); + let mut prefix_flags = std::collections::BTreeMap::<&str, i64>::new(); + for record in recovered.records.iter().filter(|record| { + record.source == super::EvidenceSource::Observer + && record.sequence >= proof.start_sequence + && record.sequence <= proof.end_cut.sequence + }) { + let flags = record.flags.0 & !super::EvidenceFlags::PROVIDER_COMPLETE_PREFIX.0; + *exact_flags.entry(record.path.as_str()).or_default() |= flags; + if record.flags.0 & super::EvidenceFlags::PROVIDER_COMPLETE_PREFIX.0 != 0 { + *prefix_flags.entry(record.path.as_str()).or_default() |= flags; + } + } + let paths_covered = intent_paths.iter().all(|(path, required)| { + exact_flags + .get(path.as_str()) + .is_some_and(|observed| observed & required != 0) + }); + let prefixes_covered = intent_prefixes.iter().all(|(prefix, reason, required)| { + reason == "provider_complete" + && prefix_flags + .get(prefix.as_str()) + .is_some_and(|observed| observed & required != 0) + }); + if !paths_covered + || !prefixes_covered + || u64::try_from(intent_paths.len()).ok() != Some(proof.verified_paths) + || u64::try_from(intent_prefixes.len()).ok() != Some(proof.verified_prefixes) + { + return Err(invalid( + "filesystem proof interval does not contain all authenticated intent evidence", + )); + } + if !filesystem_proof_authority_matches(conn, expected, proof)? { + return Err(invalid( + "filesystem proof owner, fence, or sealed segment changed after sidecar verification", + )); + } + Ok(()) +} + +pub(super) fn authoritative_ref_matches_target( + conn: &rusqlite::Connection, + intent: &PersistedIntent, +) -> Result { + let observed = conn + .query_row( + "SELECT change_id,root_id,operation_id,generation FROM refs WHERE name=?1", + [&intent.expected_ref_name], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + )) + }, + ) + .optional()?; + let Some((change, root, operation, generation)) = observed else { + return Ok(false); + }; + let root_exists: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM objects WHERE object_id=?1 AND kind=?2)", + params![intent.target.root_id.0, WORKTREE_ROOT_KIND], + |row| row.get(0), + )?; + let operation_exists = match &intent.target.operation_id { + Some(operation_id) => conn.query_row( + "SELECT EXISTS( + SELECT 1 FROM objects o JOIN operations p ON p.operation_id=o.object_id + WHERE o.object_id=?1 AND o.kind=?2 AND p.change_id=?3 AND p.after_root=?4 + )", + params![ + operation_id.0, + OPERATION_KIND, + intent.target.change_id.0, + intent.target.root_id.0 + ], + |row| row.get(0), + )?, + None => true, + }; + Ok(root_exists + && operation_exists + && change == intent.target.change_id.0 + && root == intent.target.root_id.0 + && intent + .target + .operation_id + .as_ref() + .is_none_or(|id| id.0 == operation) + && db_u64(generation, "authoritative ref generation")? + == intent.expected_ref_generation.saturating_add(1)) +} + +pub(super) fn exact_scope_guard( + conn: &rusqlite::Connection, + expected: &ExpectedScope, + trusted: bool, +) -> Result<()> { + let exists: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM changed_path_scopes WHERE scope_id=?1 AND epoch=?2 + AND ref_name=?3 AND ref_generation=?4 AND baseline_root_id=?5 + AND policy_fingerprint=?6 AND policy_dependency_generation=?7 + AND filesystem_identity=?8 AND provider_identity=?9 + AND (?10=0 OR trust_state='trusted'))", + params![ + expected.scope_id.to_text(), + sql_u64(expected.epoch, "scope epoch")?, + expected.ref_name, + sql_u64(expected.ref_generation, "ref generation")?, + expected.baseline_root.0, + hex::encode(expected.policy_fingerprint), + sql_u64(expected.policy_generation, "policy generation")?, + hex::encode(&expected.filesystem_identity), + hex::encode(&expected.provider_identity), + i64::from(trusted) + ], + |row| row.get(0), + )?; + if !exists { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "stale intent scope CAS".into(), + command: "trail status".into(), + }); + } + Ok(()) +} + +fn validate_evidence(evidence: &IntentEvidence) -> Result<()> { + let mut paths = BTreeSet::new(); + for path in &evidence.exact_paths { + if !paths.insert(path.as_str()) { + return Err(Error::InvalidInput("duplicate intent path".into())); + } + } + let mut prefixes = BTreeSet::new(); + for prefix in &evidence.complete_prefixes { + if !prefix.complete || prefix.reason.is_empty() { + return Err(Error::InvalidInput( + "intent prefixes require complete nonempty evidence".into(), + )); + } + if !prefixes.insert(prefix.path.as_str()) { + return Err(Error::InvalidInput("duplicate intent prefix".into())); + } + } + Ok(()) +} + +fn new_intent_id() -> Result { + let mut bytes = [0_u8; 32]; + getrandom(&mut bytes).map_err(|error| Error::Io(std::io::Error::other(error.to_string())))?; + Ok(format!("intent-{}", hex::encode(bytes))) +} + +pub(super) fn sql_u64(value: u64, label: &str) -> Result { + value + .try_into() + .map_err(|_| Error::InvalidInput(format!("{label} exceeds SQLite INTEGER range"))) +} + +pub(super) fn db_u64(value: i64, label: &str) -> Result { + value + .try_into() + .map_err(|_| Error::Corrupt(format!("{label} is negative"))) +} + +pub(super) fn durable_intent_barrier(conn: &rusqlite::Connection) -> Result<()> { + let busy: i64 = conn.query_row("PRAGMA wal_checkpoint(FULL)", [], |row| row.get(0))?; + if busy != 0 { + return Err(Error::Conflict( + "changed-path intent durability checkpoint remained busy".into(), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn recovered_progress(sequence: u64) -> super::super::RecoveredTail { + super::super::RecoveredTail { + records: Vec::new(), + record_boundaries: Vec::new(), + durable_end: sequence, + last_sequence: sequence, + last_hash: [sequence as u8; 32], + requires_reconciliation: true, + segments: Vec::new(), + } + } + + #[test] + fn static_unpublished_sidecar_tail_exhausts_idle_budget() { + let mut budget = SidecarRecoveryBudget::default(); + let static_tail = recovered_progress(7); + + assert!(budget.retry_after(Duration::ZERO, &static_tail)); + assert!(budget.retry_after(Duration::from_millis(1_999), &static_tail)); + assert!(!budget.retry_after(SIDECAR_RECOVERY_IDLE_LIMIT, &static_tail)); + } + + #[test] + fn authenticated_sidecar_progress_extends_idle_but_never_hard_budget() { + let mut budget = SidecarRecoveryBudget::default(); + + for (index, elapsed) in [0_u64, 1_900, 3_800, 5_700, 7_600, 9_500, 11_400, 13_300] + .into_iter() + .enumerate() + { + assert!(budget.retry_after( + Duration::from_millis(elapsed), + &recovered_progress(index as u64 + 1), + )); + } + assert!(!budget.retry_after(SIDECAR_RECOVERY_HARD_LIMIT, &recovered_progress(9),)); + } +} diff --git a/trail/src/db/change_ledger/log.rs b/trail/src/db/change_ledger/log.rs new file mode 100644 index 0000000..076a760 --- /dev/null +++ b/trail/src/db/change_ledger/log.rs @@ -0,0 +1,214 @@ +use std::fmt; + +use super::types::{EvidenceFlags, EvidenceSource, LedgerPath, ScopeId}; +use crate::error::{Error, Result}; + +mod codec; +mod writer; + +pub(crate) use codec::authenticate_segment_for_deletion; +#[cfg(test)] +pub(crate) use codec::recover_segments; +pub(crate) use codec::{recover_segments_from_connection, recover_segments_from_directory}; +pub(crate) use writer::{DaemonLaunchBinding, SegmentWriter}; + +#[cfg(all(test, target_os = "linux"))] +use codec::open_segment_no_follow; +#[cfg(test)] +use codec::{ + decode_header, encode_header, encode_record, encoded_segment, header_end, recover_bytes, +}; +#[cfg(test)] +use writer::{ + install_append_flush_boundary_hook, segment_filename, segment_id, sync_directory, FaultPoint, + FaultScript, +}; + +#[cfg(test)] +mod tests; + +const SEGMENT_MAGIC: &[u8; 8] = b"TRAILCPL"; +const LOG_FORMAT_VERSION: u16 = 1; +const MAX_HEADER_BYTES: usize = 1024 * 1024; +const MAX_RECORD_PAYLOAD_BYTES: usize = 1024 * 1024; +const RECORD_FIXED_BYTES: usize = 8 + 1 + 32 + 32; +const LENGTH_PREFIX_BYTES: usize = 4; +const MAX_SEGMENT_FILENAME_BYTES: usize = 128; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ObserverRecord { + pub(crate) sequence: u64, + pub(crate) source: EvidenceSource, + pub(crate) path: LedgerPath, + pub(crate) flags: EvidenceFlags, + pub(crate) provider_cursor: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct DurableCut { + pub(crate) segment_id: String, + pub(crate) durable_end_offset: u64, + pub(crate) last_sequence: u64, + pub(crate) last_hash: [u8; 32], + pub(crate) provider_cursor: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ObserverWriterBinding { + pub(crate) owner_token: String, + pub(crate) provider_id: String, + pub(crate) provider_identity: Vec, + pub(crate) fence_nonce: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct PersistedLogLimits { + pub(crate) max_log_bytes: u64, + pub(crate) max_segment_bytes: u64, + pub(crate) max_unfolded_tail_records: usize, +} + +impl Default for PersistedLogLimits { + fn default() -> Self { + Self { + max_log_bytes: 268_435_456, + max_segment_bytes: 16_777_216, + max_unfolded_tail_records: 65_536, + } + } +} + +impl PersistedLogLimits { + fn validate(self) -> std::result::Result { + if self.max_log_bytes == 0 + || self.max_segment_bytes == 0 + || self.max_segment_bytes > self.max_log_bytes + || self.max_unfolded_tail_records == 0 + { + return Err(RecoveryError::new("invalid persisted observer log limits")); + } + Ok(self) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RecoveredTail { + pub(crate) records: Vec, + pub(crate) record_boundaries: Vec, + pub(crate) durable_end: u64, + pub(crate) last_sequence: u64, + pub(crate) last_hash: [u8; 32], + pub(crate) requires_reconciliation: bool, + pub(crate) segments: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AuthenticatedRecordBoundary { + pub(crate) segment_id: String, + pub(crate) sequence: u64, + pub(crate) durable_end_offset: u64, + pub(crate) provider_cursor: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AuthenticatedSegment { + pub(crate) segment_id: String, + pub(crate) segment_path: String, + pub(crate) state: String, + pub(crate) start_cursor: Vec, + pub(crate) end_cursor: Vec, + pub(crate) first_sequence: u64, + pub(crate) last_sequence: u64, + pub(crate) header_end_offset: u64, + pub(crate) durable_end_offset: u64, + pub(crate) folded_end_offset: u64, + pub(crate) segment_hash: [u8; 32], +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct DeletionSegmentExpectation { + pub(crate) scope_id: ScopeId, + pub(crate) epoch: u64, + pub(crate) segment_id: String, + pub(crate) owner_token: [u8; 32], + pub(crate) first_sequence: u64, + pub(crate) last_sequence: Option, + pub(crate) durable_end_offset: u64, + pub(crate) previous_segment_hash: [u8; 32], + pub(crate) stored_segment_hash: Option<[u8; 32]>, + pub(crate) state: String, + pub(crate) limits: PersistedLogLimits, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AuthenticatedDeletionSegment { + pub(crate) file_length: u64, + pub(crate) file_hash: [u8; 32], + pub(crate) durable_hash: [u8; 32], +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RecoveryScope { + pub(crate) scope_id: ScopeId, + pub(crate) epoch: u64, + pub(crate) owner_token: [u8; 32], +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RecoveryError { + pub(crate) message: String, + pub(crate) requires_reconciliation: bool, +} + +impl RecoveryError { + fn new(message: impl Into) -> Self { + Self { + message: message.into(), + requires_reconciliation: true, + } + } +} + +impl fmt::Display for RecoveryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for RecoveryError {} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct SegmentIdentity { + scope_id: ScopeId, + epoch: u64, + owner_token: [u8; 32], + provider_cursor: Vec, + previous_segment_hash: [u8; 32], +} + +impl SegmentIdentity { + #[cfg(test)] + fn test(scope_id: ScopeId, epoch: u64, owner_token: [u8; 32]) -> Self { + Self { + scope_id, + epoch, + owner_token, + provider_cursor: Vec::new(), + previous_segment_hash: [0; 32], + } + } + + fn recovery_scope(&self) -> RecoveryScope { + RecoveryScope { + scope_id: self.scope_id, + epoch: self.epoch, + owner_token: self.owner_token, + } + } +} + +fn sql_i64(value: u64, label: &str) -> Result { + value + .try_into() + .map_err(|_| Error::InvalidInput(format!("{label} exceeds SQLite range"))) +} diff --git a/trail/src/db/change_ledger/log/codec.rs b/trail/src/db/change_ledger/log/codec.rs new file mode 100644 index 0000000..c01c0e1 --- /dev/null +++ b/trail/src/db/change_ledger/log/codec.rs @@ -0,0 +1,1305 @@ +use std::fs::{self, File}; +use std::io::{Read, Seek}; +use std::path::Path; + +use rusqlite::types::ValueRef; +use rusqlite::{params, Connection, OpenFlags, Row}; +use serde_cbor::Value; +use sha2::{Digest, Sha256}; + +use super::*; + +fn source_tag(source: EvidenceSource) -> u8 { + match source { + EvidenceSource::Observer => 1, + EvidenceSource::Intent => 2, + EvidenceSource::Reconciliation => 3, + EvidenceSource::GitAdvisory => 4, + } +} + +fn parse_source(tag: u8) -> std::result::Result { + match tag { + 1 => Ok(EvidenceSource::Observer), + 2 => Ok(EvidenceSource::Intent), + 3 => Ok(EvidenceSource::Reconciliation), + 4 => Ok(EvidenceSource::GitAdvisory), + _ => Err(RecoveryError::new("unknown observer record source")), + } +} + +fn cbor_bytes(value: Value) -> std::result::Result, RecoveryError> { + serde_cbor::to_vec(&value).map_err(|error| RecoveryError::new(error.to_string())) +} + +pub(super) fn encode_header( + identity: &SegmentIdentity, +) -> std::result::Result, RecoveryError> { + if identity.epoch == 0 { + return Err(RecoveryError::new( + "observer segment epoch must be positive", + )); + } + let payload = cbor_bytes(Value::Array(vec![ + Value::Bytes(identity.scope_id.0.to_vec()), + Value::Integer(identity.epoch.into()), + Value::Bytes(identity.owner_token.to_vec()), + Value::Bytes(identity.provider_cursor.clone()), + Value::Bytes(identity.previous_segment_hash.to_vec()), + ]))?; + if payload.len() > MAX_HEADER_BYTES { + return Err(RecoveryError::new("observer segment header exceeds limit")); + } + let mut bytes = Vec::with_capacity(SEGMENT_MAGIC.len() + 2 + 4 + payload.len()); + bytes.extend_from_slice(SEGMENT_MAGIC); + bytes.extend_from_slice(&LOG_FORMAT_VERSION.to_be_bytes()); + bytes.extend_from_slice(&(payload.len() as u32).to_be_bytes()); + bytes.extend_from_slice(&payload); + Ok(bytes) +} + +pub(super) fn decode_header( + bytes: &[u8], +) -> std::result::Result<(SegmentIdentity, usize), RecoveryError> { + const PREFIX: usize = 8 + 2 + 4; + if bytes.len() < PREFIX { + return Err(RecoveryError::new("partial observer segment header")); + } + if &bytes[..8] != SEGMENT_MAGIC { + return Err(RecoveryError::new("invalid observer segment magic")); + } + let format = u16::from_be_bytes([bytes[8], bytes[9]]); + if format != LOG_FORMAT_VERSION { + return Err(RecoveryError::new("unsupported observer segment format")); + } + let length = u32::from_be_bytes(bytes[10..14].try_into().unwrap()) as usize; + if length > MAX_HEADER_BYTES { + return Err(RecoveryError::new("observer segment header exceeds limit")); + } + let end = PREFIX + .checked_add(length) + .ok_or_else(|| RecoveryError::new("observer segment header length overflow"))?; + if end > bytes.len() { + return Err(RecoveryError::new("partial observer segment header")); + } + let payload = &bytes[PREFIX..end]; + let value: Value = serde_cbor::from_slice(payload) + .map_err(|error| RecoveryError::new(format!("invalid observer header CBOR: {error}")))?; + if cbor_bytes(value.clone())? != payload { + return Err(RecoveryError::new("non-canonical observer header CBOR")); + } + let Value::Array(fields) = value else { + return Err(RecoveryError::new("invalid observer header shape")); + }; + if fields.len() != 5 { + return Err(RecoveryError::new("invalid observer header field count")); + } + let scope_id = fixed_bytes::<32>(&fields[0], "scope identity")?; + let epoch = unsigned(&fields[1], "epoch")?; + if epoch == 0 { + return Err(RecoveryError::new( + "observer segment epoch must be positive", + )); + } + let owner_token = fixed_bytes::<32>(&fields[2], "owner token")?; + let provider_cursor = byte_string(&fields[3], "provider cursor")?; + let previous_segment_hash = fixed_bytes::<32>(&fields[4], "previous segment hash")?; + Ok(( + SegmentIdentity { + scope_id: ScopeId(scope_id), + epoch, + owner_token, + provider_cursor, + previous_segment_hash, + }, + end, + )) +} + +fn fixed_bytes( + value: &Value, + label: &str, +) -> std::result::Result<[u8; N], RecoveryError> { + let bytes = byte_string(value, label)?; + bytes + .try_into() + .map_err(|_| RecoveryError::new(format!("invalid {label} length"))) +} + +fn byte_string(value: &Value, label: &str) -> std::result::Result, RecoveryError> { + match value { + Value::Bytes(bytes) => Ok(bytes.clone()), + _ => Err(RecoveryError::new(format!("invalid {label} encoding"))), + } +} + +fn unsigned(value: &Value, label: &str) -> std::result::Result { + match value { + Value::Integer(value) => (*value) + .try_into() + .map_err(|_| RecoveryError::new(format!("invalid {label}"))), + _ => Err(RecoveryError::new(format!("invalid {label} encoding"))), + } +} + +fn encode_payload(record: &ObserverRecord) -> std::result::Result, RecoveryError> { + if record.flags.0 < 0 { + return Err(RecoveryError::new("observer flags cannot be negative")); + } + let payload = cbor_bytes(Value::Array(vec![ + Value::Text(record.path.as_str().to_owned()), + Value::Integer(record.flags.0.into()), + Value::Bytes(record.provider_cursor.clone()), + ]))?; + if payload.len() > MAX_RECORD_PAYLOAD_BYTES { + return Err(RecoveryError::new("observer record payload exceeds 1 MiB")); + } + Ok(payload) +} + +pub(super) fn encode_record( + record: &ObserverRecord, + previous_hash: [u8; 32], +) -> std::result::Result<(Vec, [u8; 32]), RecoveryError> { + if record.sequence == 0 { + return Err(RecoveryError::new("observer sequence must be positive")); + } + let payload = encode_payload(record)?; + let body_len = RECORD_FIXED_BYTES + .checked_add(payload.len()) + .ok_or_else(|| RecoveryError::new("observer record length overflow"))?; + let mut body = Vec::with_capacity(body_len); + body.extend_from_slice(&record.sequence.to_be_bytes()); + body.push(source_tag(record.source)); + body.extend_from_slice(&payload); + body.extend_from_slice(&previous_hash); + let checksum: [u8; 32] = Sha256::digest(&body).into(); + body.extend_from_slice(&checksum); + let mut framed = Vec::with_capacity(LENGTH_PREFIX_BYTES + body.len()); + framed.extend_from_slice(&(body.len() as u32).to_be_bytes()); + framed.extend_from_slice(&body); + Ok((framed, checksum)) +} + +fn decode_record( + body: &[u8], + expected_previous_hash: [u8; 32], +) -> std::result::Result<(ObserverRecord, [u8; 32]), RecoveryError> { + if body.len() < RECORD_FIXED_BYTES { + return Err(RecoveryError::new( + "observer record is shorter than fixed fields", + )); + } + let sequence = u64::from_be_bytes(body[..8].try_into().unwrap()); + if sequence == 0 { + return Err(RecoveryError::new("observer sequence must be positive")); + } + let source = parse_source(body[8])?; + let payload_len = body.len() - RECORD_FIXED_BYTES; + if payload_len > MAX_RECORD_PAYLOAD_BYTES { + return Err(RecoveryError::new("observer record payload exceeds 1 MiB")); + } + let expected_len = RECORD_FIXED_BYTES + .checked_add(payload_len) + .ok_or_else(|| RecoveryError::new("observer record length overflow"))?; + if body.len() != expected_len { + return Err(RecoveryError::new( + "observer record length does not match payload", + )); + } + let payload_end = 9 + payload_len; + let previous_hash: [u8; 32] = body[payload_end..payload_end + 32].try_into().unwrap(); + if previous_hash != expected_previous_hash { + return Err(RecoveryError::new("broken observer record hash linkage")); + } + let checksum: [u8; 32] = body[payload_end + 32..].try_into().unwrap(); + let calculated: [u8; 32] = Sha256::digest(&body[..payload_end + 32]).into(); + if checksum != calculated { + return Err(RecoveryError::new("observer record checksum mismatch")); + } + let payload = &body[9..payload_end]; + let value: Value = serde_cbor::from_slice(payload) + .map_err(|error| RecoveryError::new(format!("invalid observer record CBOR: {error}")))?; + if cbor_bytes(value.clone())? != payload { + return Err(RecoveryError::new("non-canonical observer record CBOR")); + } + let Value::Array(fields) = value else { + return Err(RecoveryError::new("invalid observer payload shape")); + }; + if fields.len() != 3 { + return Err(RecoveryError::new("invalid observer payload field count")); + } + let Value::Text(path) = &fields[0] else { + return Err(RecoveryError::new("invalid observer path encoding")); + }; + let path = LedgerPath::parse(path) + .map_err(|error| RecoveryError::new(format!("invalid observer path: {error}")))?; + let flags = unsigned(&fields[1], "observer flags")?; + let flags = i64::try_from(flags) + .map(EvidenceFlags) + .map_err(|_| RecoveryError::new("observer flags exceed supported range"))?; + let provider_cursor = byte_string(&fields[2], "provider cursor")?; + Ok(( + ObserverRecord { + sequence, + source, + path, + flags, + provider_cursor, + }, + checksum, + )) +} + +pub(super) fn recover_bytes( + bytes: &[u8], + expected: &SegmentIdentity, + limits: PersistedLogLimits, +) -> std::result::Result { + let limits = limits.validate()?; + if bytes.len() as u64 > limits.max_segment_bytes || bytes.len() as u64 > limits.max_log_bytes { + return Err(RecoveryError::new( + "observer segment exceeds persisted byte limit", + )); + } + let (actual, mut offset) = decode_header(bytes)?; + if actual != *expected { + return Err(RecoveryError::new( + "observer segment identity does not match expected lease", + )); + } + let durable_start = offset as u64; + let mut records = Vec::new(); + let mut record_boundaries = Vec::new(); + let mut previous_hash = [0; 32]; + let mut last_sequence = 0; + while offset < bytes.len() { + let remaining = bytes.len() - offset; + if remaining < LENGTH_PREFIX_BYTES { + return Ok(RecoveredTail { + records, + record_boundaries, + durable_end: offset as u64, + last_sequence, + last_hash: previous_hash, + requires_reconciliation: true, + segments: Vec::new(), + }); + } + let body_len = u32::from_be_bytes(bytes[offset..offset + 4].try_into().unwrap()) as usize; + if !(RECORD_FIXED_BYTES..=RECORD_FIXED_BYTES + MAX_RECORD_PAYLOAD_BYTES).contains(&body_len) + { + return Err(RecoveryError::new( + "observer record length exceeds protocol bound", + )); + } + let end = offset + .checked_add(LENGTH_PREFIX_BYTES) + .and_then(|value| value.checked_add(body_len)) + .ok_or_else(|| RecoveryError::new("observer record length overflow"))?; + if end > bytes.len() { + return Ok(RecoveredTail { + records, + record_boundaries, + durable_end: offset as u64, + last_sequence, + last_hash: previous_hash, + requires_reconciliation: true, + segments: Vec::new(), + }); + } + if records.len() == limits.max_unfolded_tail_records { + return Err(RecoveryError::new( + "observer tail record count exceeds persisted limit", + )); + } + let (record, hash) = decode_record(&bytes[offset + 4..end], previous_hash)?; + if record.sequence <= last_sequence { + return Err(RecoveryError::new( + "observer sequence is not strictly monotonic", + )); + } + last_sequence = record.sequence; + previous_hash = hash; + record_boundaries.push(AuthenticatedRecordBoundary { + segment_id: String::new(), + sequence: record.sequence, + durable_end_offset: end as u64, + provider_cursor: record.provider_cursor.clone(), + }); + records.push(record); + offset = end; + } + Ok(RecoveredTail { + records, + record_boundaries, + durable_end: if offset as u64 == durable_start { + durable_start + } else { + offset as u64 + }, + last_sequence, + last_hash: previous_hash, + requires_reconciliation: false, + segments: Vec::new(), + }) +} + +#[cfg(test)] +pub(super) fn encoded_segment( + identity: &SegmentIdentity, + records: &[ObserverRecord], +) -> std::result::Result, RecoveryError> { + let mut bytes = encode_header(identity)?; + let mut previous_hash = [0; 32]; + for record in records { + let (encoded, hash) = encode_record(record, previous_hash)?; + bytes.extend_from_slice(&encoded); + previous_hash = hash; + } + Ok(bytes) +} + +#[cfg(test)] +pub(super) fn header_end(bytes: &[u8]) -> std::result::Result { + decode_header(bytes).map(|(_, end)| end) +} + +#[cfg(test)] +pub(crate) fn recover_segments( + database_path: &Path, + segment_directory: &Path, + expected: &RecoveryScope, + limits: PersistedLogLimits, +) -> std::result::Result { + let canonical_directory = segment_directory.canonicalize().map_err(|error| { + RecoveryError::new(format!("canonicalize observer test directory: {error}")) + })?; + let segment_directory = super::super::secure_fs::SecureDirectory::open_absolute( + &canonical_directory, + ) + .map_err(|error| RecoveryError::new(format!("open observer directory securely: {error}")))?; + recover_segments_from_directory(database_path, &segment_directory, expected, limits) +} + +pub(crate) fn recover_segments_from_directory( + database_path: &Path, + segment_directory: &super::super::secure_fs::SecureDirectory, + expected: &RecoveryScope, + limits: PersistedLogLimits, +) -> std::result::Result { + let (connection, immutable_snapshot) = open_recovery_database(database_path)?; + connection + .pragma_update(None, "foreign_keys", true) + .map_err(|error| RecoveryError::new(format!("enable observer foreign keys: {error}")))?; + let immutable_wal = immutable_snapshot && database_header_uses_wal(database_path)?; + recover_segments_with_connection( + &connection, + segment_directory, + expected, + limits, + true, + immutable_wal, + ) +} + +/// Authenticate sidecars against the caller's already-current SQLite view. +/// Controlled projections and intent publication use this path so a second +/// connection can never select an older checkpoint while the writer's exact +/// segment metadata is visible in the authoritative transaction. +pub(crate) fn recover_segments_from_connection( + connection: &Connection, + segment_directory: &super::super::secure_fs::SecureDirectory, + expected: &RecoveryScope, + limits: PersistedLogLimits, +) -> std::result::Result { + recover_segments_with_connection( + connection, + segment_directory, + expected, + limits, + false, + false, + ) +} + +fn recover_segments_with_connection( + connection: &Connection, + segment_directory: &super::super::secure_fs::SecureDirectory, + expected: &RecoveryScope, + limits: PersistedLogLimits, + manage_snapshot: bool, + immutable_wal: bool, +) -> std::result::Result { + let limits = limits.validate()?; + let journal_mode: String = connection + .query_row("PRAGMA journal_mode", [], |row| row.get(0)) + .map_err(|error| RecoveryError::new(format!("read observer journal mode: {error}")))?; + let foreign_keys: i64 = connection + .query_row("PRAGMA foreign_keys", [], |row| row.get(0)) + .map_err(|error| RecoveryError::new(format!("read observer foreign keys: {error}")))?; + if !(journal_mode.eq_ignore_ascii_case("wal") || immutable_wal) || foreign_keys != 1 { + return Err(RecoveryError::new( + "observer recovery requires WAL journal mode and foreign keys", + )); + } + if manage_snapshot { + connection + .execute_batch("BEGIN DEFERRED") + .map_err(|error| { + RecoveryError::new(format!("begin observer recovery snapshot: {error}")) + })?; + } + let epoch = sql_i64(expected.epoch, "observer epoch") + .map_err(|error| RecoveryError::new(error.to_string()))?; + validate_recovery_owner(&connection, expected, epoch)?; + let segment_count: i64 = connection + .query_row( + "SELECT COUNT(*) FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=?2", + params![expected.scope_id.to_text(), epoch], + |row| row.get(0), + ) + .map_err(|error| RecoveryError::new(format!("count observer metadata: {error}")))?; + let segment_count = usize::try_from(segment_count) + .map_err(|_| RecoveryError::new("negative observer segment count"))?; + if segment_count > limits.max_unfolded_tail_records { + return Err(RecoveryError::new( + "observer segment count exceeds unfolded-tail limit", + )); + } + if segment_count == 0 { + if manage_snapshot { + connection.execute_batch("COMMIT").map_err(|error| { + RecoveryError::new(format!("commit observer recovery snapshot: {error}")) + })?; + } + validate_recovery_owner(&connection, expected, epoch)?; + return Ok(RecoveredTail { + records: Vec::new(), + record_boundaries: Vec::new(), + durable_end: 0, + last_sequence: 0, + last_hash: [0; 32], + requires_reconciliation: true, + segments: Vec::new(), + }); + } + let mut statement = connection + .prepare( + "SELECT segment_id, owner_token, first_sequence, last_sequence, + durable_end_offset, folded_end_offset, previous_segment_id, + previous_segment_hash, segment_hash, segment_path, state + FROM changed_path_observer_segments + WHERE scope_id = ?1 AND epoch = ?2 + ORDER BY first_sequence, segment_id", + ) + .map_err(|error| RecoveryError::new(format!("read observer metadata: {error}")))?; + let mut rows = statement + .query(params![expected.scope_id.to_text(), epoch]) + .map_err(|error| RecoveryError::new(format!("query observer metadata: {error}")))?; + let expected_owner = hex::encode(expected.owner_token); + let mut total_bytes = 0_u64; + let mut records = Vec::new(); + let mut record_boundaries = Vec::new(); + let mut durable_end = 0_u64; + let mut last_sequence = 0_u64; + let mut last_hash = [0; 32]; + let mut previous_segment_id: Option = None; + let mut previous_segment_hash = [0; 32]; + let mut requires_reconciliation = false; + let mut authenticated_segments = Vec::with_capacity(segment_count); + let mut index = 0_usize; + while let Some(sql_row) = rows + .next() + .map_err(|error| RecoveryError::new(format!("stream observer metadata: {error}")))? + { + let row = decode_segment_metadata(sql_row)?; + index = index + .checked_add(1) + .ok_or_else(|| RecoveryError::new("observer segment count overflow"))?; + if index > segment_count { + return Err(RecoveryError::new( + "observer segment count changed during recovery", + )); + } + if row.owner_token != expected_owner { + return Err(RecoveryError::new( + "observer segment metadata has wrong owner", + )); + } + let first_sequence = db_u64(row.first_sequence, "first observer sequence")?; + let metadata_token = decode_owner_token(&row.owner_token)?; + let expected_id = super::writer::segment_id(expected.epoch, first_sequence, metadata_token); + if row.segment_id != expected_id { + return Err(RecoveryError::new( + "observer segment id does not match its exact derived identity", + )); + } + let expected_filename = super::writer::segment_filename(&expected_id) + .map_err(|error| RecoveryError::new(error.to_string()))?; + if row.segment_path != expected_filename { + return Err(RecoveryError::new( + "observer segment path is not the exact derived relative filename", + )); + } + let durable = db_u64(row.durable_end_offset, "durable observer offset")?; + if durable > limits.max_segment_bytes { + return Err(RecoveryError::new( + "observer segment exceeds persisted byte limit", + )); + } + total_bytes = total_bytes + .checked_add(durable) + .ok_or_else(|| RecoveryError::new("observer log byte count overflow"))?; + if total_bytes > limits.max_log_bytes { + return Err(RecoveryError::new( + "observer log exceeds persisted byte limit", + )); + } + if row.state == "open" && index != segment_count { + return Err(RecoveryError::new( + "open observer segment appears before the recovered tail", + )); + } + let file = segment_directory + .open_regular(&expected_filename) + .map_err(|error| { + RecoveryError::new(format!("open observer segment securely: {error}")) + })?; + let metadata = file.metadata().map_err(|error| { + RecoveryError::new(format!("read observer segment metadata: {error}")) + })?; + if metadata.len() > limits.max_segment_bytes || metadata.len() > limits.max_log_bytes { + return Err(RecoveryError::new( + "observer segment file exceeds persisted byte limit", + )); + } + if durable > metadata.len() { + return Err(RecoveryError::new( + "durable observer offset exceeds segment length", + )); + } + if metadata.len() > durable && row.state == "sealed" { + return Err(RecoveryError::new( + "sealed observer segment contains unpublished trailing bytes", + )); + } + if metadata.len() > durable { + requires_reconciliation = true; + } + let durable_usize = usize::try_from(durable) + .map_err(|_| RecoveryError::new("durable observer offset cannot fit memory"))?; + let mut bytes = Vec::with_capacity(durable_usize); + file.take(durable) + .read_to_end(&mut bytes) + .map_err(|error| RecoveryError::new(format!("read observer segment: {error}")))?; + if bytes.len() != durable_usize { + return Err(RecoveryError::new( + "observer segment shortened during recovery", + )); + } + let (identity, header_end_offset) = decode_header(&bytes)?; + if identity.recovery_scope() != *expected { + return Err(RecoveryError::new( + "observer segment identity does not match expected lease", + )); + } + if identity.previous_segment_hash != previous_segment_hash { + return Err(RecoveryError::new("broken observer segment header lineage")); + } + if row.previous_segment_id != previous_segment_id { + return Err(RecoveryError::new( + "broken observer segment metadata lineage", + )); + } + let metadata_previous_hash = decode_optional_hash(row.previous_segment_hash.as_deref())?; + if metadata_previous_hash != previous_segment_hash { + return Err(RecoveryError::new("broken observer segment metadata hash")); + } + let recovered = recover_bytes(&bytes, &identity, limits)?; + if recovered.requires_reconciliation && index != segment_count { + return Err(RecoveryError::new( + "partial observer record before final segment", + )); + } + if recovered + .records + .first() + .is_some_and(|record| record.sequence <= last_sequence) + { + return Err(RecoveryError::new( + "observer sequence is not monotonic across segments", + )); + } + let next_record_count = records + .len() + .checked_add(recovered.records.len()) + .ok_or_else(|| RecoveryError::new("observer tail record count overflow"))?; + if next_record_count > limits.max_unfolded_tail_records { + return Err(RecoveryError::new( + "observer tail record count exceeds persisted limit", + )); + } + if let Some(first) = recovered.records.first() { + if first_sequence != first.sequence { + return Err(RecoveryError::new( + "observer segment first sequence metadata mismatch", + )); + } + } + if let Some(metadata_last) = row.last_sequence { + if db_u64(metadata_last, "last observer sequence")? != recovered.last_sequence { + return Err(RecoveryError::new( + "observer segment last sequence metadata mismatch", + )); + } + } + let segment_hash: [u8; 32] = Sha256::digest(&bytes).into(); + if row.state == "sealed" { + let stored = decode_required_hash(row.segment_hash.as_deref(), "sealed segment hash")?; + if stored != segment_hash { + return Err(RecoveryError::new("sealed observer segment hash mismatch")); + } + } else if row.state != "open" { + return Err(RecoveryError::new( + "observer segment metadata is not recoverable", + )); + } + let end_cursor = recovered + .records + .last() + .map(|record| record.provider_cursor.clone()) + .unwrap_or_else(|| identity.provider_cursor.clone()); + let authenticated_last_sequence = recovered + .records + .last() + .map(|record| record.sequence) + .unwrap_or_else(|| first_sequence.saturating_sub(1)); + authenticated_segments.push(super::AuthenticatedSegment { + segment_id: row.segment_id.clone(), + segment_path: row.segment_path.clone(), + state: row.state.clone(), + start_cursor: identity.provider_cursor.clone(), + end_cursor, + first_sequence, + last_sequence: authenticated_last_sequence, + header_end_offset: header_end_offset as u64, + durable_end_offset: durable, + folded_end_offset: db_u64(row.folded_end_offset, "folded observer offset")?, + segment_hash, + }); + record_boundaries.extend(recovered.record_boundaries.into_iter().map(|mut boundary| { + boundary.segment_id = row.segment_id.clone(); + boundary + })); + records.extend(recovered.records); + durable_end = durable_end + .checked_add(recovered.durable_end) + .ok_or_else(|| RecoveryError::new("observer durable offset overflow"))?; + last_sequence = recovered.last_sequence.max(last_sequence); + last_hash = recovered.last_hash; + requires_reconciliation |= recovered.requires_reconciliation; + previous_segment_id = Some(row.segment_id); + previous_segment_hash = segment_hash; + } + if index != segment_count { + return Err(RecoveryError::new( + "observer segment count changed during recovery", + )); + } + + for filename in segment_directory.entry_names().map_err(|error| { + RecoveryError::new(format!("list observer segment directory securely: {error}")) + })? { + if Path::new(&filename) + .extension() + .is_some_and(|extension| extension == "cpl") + { + let published = filename.to_str().is_some_and(|filename| { + filename.len() <= MAX_SEGMENT_FILENAME_BYTES + && is_strictly_published_filename( + &connection, + &expected.scope_id, + filename, + segment_directory, + ) + .unwrap_or(false) + }); + if published { + continue; + } + // A completed authoritative epoch advance permanently excludes + // every earlier epoch from the current chain. A crash may leave + // the newly-created header of an old rotation without a SQLite + // row. Once its canonical header proves that it belongs to a + // superseded epoch, it cannot create a gap in the current owner; + // continuing to treat it as current would make full + // reconciliation unable to recover automatically. Malformed, + // current-epoch, and future-epoch files still fail closed. + if filename.to_str().is_some_and(|filename| { + is_authenticated_superseded_epoch_file(segment_directory, filename, expected) + .unwrap_or(false) + }) { + continue; + } + requires_reconciliation = true; + } + } + let recovered = RecoveredTail { + records, + record_boundaries, + durable_end, + last_sequence, + last_hash, + requires_reconciliation, + segments: authenticated_segments, + }; + drop(rows); + drop(statement); + if manage_snapshot { + connection.execute_batch("COMMIT").map_err(|error| { + RecoveryError::new(format!("commit observer recovery snapshot: {error}")) + })?; + } + validate_recovery_owner(&connection, expected, epoch)?; + Ok(recovered) +} + +fn is_authenticated_superseded_epoch_file( + directory: &super::super::secure_fs::SecureDirectory, + filename: &str, + current: &RecoveryScope, +) -> std::result::Result { + let Some(segment_id) = filename.strip_suffix(".cpl") else { + return Ok(false); + }; + if super::writer::segment_filename(segment_id).ok().as_deref() != Some(filename) { + return Ok(false); + } + let bytes = segment_id.as_bytes(); + let Some(epoch) = std::str::from_utf8(&bytes[..20]) + .ok() + .and_then(|value| value.parse::().ok()) + else { + return Ok(false); + }; + if epoch >= current.epoch { + return Ok(false); + } + let owner_token: [u8; 32] = match hex::decode(&segment_id[42..]) { + Ok(bytes) => match bytes.try_into() { + Ok(token) => token, + Err(_) => return Ok(false), + }, + Err(_) => return Ok(false), + }; + published_header_matches( + directory, + filename, + &RecoveryScope { + scope_id: current.scope_id, + epoch, + owner_token, + }, + ) +} + +pub(crate) fn authenticate_segment_for_deletion( + file: &File, + expected: &DeletionSegmentExpectation, +) -> std::result::Result { + let limits = expected.limits.validate()?; + let metadata = file + .metadata() + .map_err(|error| RecoveryError::new(format!("read deletion segment metadata: {error}")))?; + if metadata.len() > limits.max_segment_bytes || metadata.len() > limits.max_log_bytes { + return Err(RecoveryError::new( + "deletion segment exceeds persisted byte limit", + )); + } + if expected.durable_end_offset > metadata.len() { + return Err(RecoveryError::new( + "deletion segment durable offset exceeds file length", + )); + } + let file_length = usize::try_from(metadata.len()) + .map_err(|_| RecoveryError::new("deletion segment length cannot fit memory"))?; + let mut bytes = Vec::with_capacity(file_length); + let mut reader = file + .try_clone() + .map_err(|error| RecoveryError::new(format!("clone deletion segment: {error}")))?; + reader + .rewind() + .map_err(|error| RecoveryError::new(format!("rewind deletion segment: {error}")))?; + reader + .read_to_end(&mut bytes) + .map_err(|error| RecoveryError::new(format!("read deletion segment: {error}")))?; + if bytes.len() != file_length { + return Err(RecoveryError::new( + "deletion segment changed length while authenticating", + )); + } + let durable_length = usize::try_from(expected.durable_end_offset) + .map_err(|_| RecoveryError::new("durable deletion length cannot fit memory"))?; + let durable_bytes = &bytes[..durable_length]; + let (identity, _) = decode_header(durable_bytes)?; + let expected_scope = RecoveryScope { + scope_id: expected.scope_id, + epoch: expected.epoch, + owner_token: expected.owner_token, + }; + if identity.recovery_scope() != expected_scope + || identity.previous_segment_hash != expected.previous_segment_hash + { + return Err(RecoveryError::new( + "deletion segment header lineage does not match persisted metadata", + )); + } + let derived_id = super::writer::segment_id( + expected.epoch, + expected.first_sequence, + expected.owner_token, + ); + if expected.segment_id != derived_id { + return Err(RecoveryError::new( + "deletion segment id does not match exact derived identity", + )); + } + let recovered = recover_bytes(durable_bytes, &identity, limits)?; + if recovered.requires_reconciliation { + return Err(RecoveryError::new( + "deletion segment durable prefix is not record-complete", + )); + } + match (expected.last_sequence, recovered.records.first()) { + (Some(last), Some(first)) + if first.sequence == expected.first_sequence && recovered.last_sequence == last => {} + (None, None) if recovered.last_sequence == 0 => {} + _ => { + return Err(RecoveryError::new( + "deletion segment sequence metadata does not match durable bytes", + )); + } + } + let durable_hash: [u8; 32] = Sha256::digest(durable_bytes).into(); + match expected.state.as_str() { + "sealed" + if metadata.len() == expected.durable_end_offset + && expected.stored_segment_hash == Some(durable_hash) => {} + "open" if expected.stored_segment_hash.is_none() => {} + "sealed" => { + return Err(RecoveryError::new( + "sealed deletion segment hash or length does not match metadata", + )); + } + _ => { + return Err(RecoveryError::new( + "deletion segment source state is not authenticatable", + )); + } + } + Ok(AuthenticatedDeletionSegment { + file_length: metadata.len(), + file_hash: Sha256::digest(&bytes).into(), + durable_hash, + }) +} + +fn validate_recovery_owner( + connection: &Connection, + expected: &RecoveryScope, + expected_epoch_sql: i64, +) -> std::result::Result<(), RecoveryError> { + let mut statement = connection + .prepare( + "SELECT owner.epoch, owner.owner_token, owner.lease_state, owner.expires_at, + owner.error_state, owner.error_at, scope.epoch + FROM changed_path_observer_owners owner + JOIN changed_path_scopes scope ON scope.scope_id=owner.scope_id + WHERE owner.scope_id=?1", + ) + .map_err(|error| RecoveryError::new(format!("read observer owner: {error}")))?; + let mut rows = statement + .query([expected.scope_id.to_text()]) + .map_err(|error| RecoveryError::new(format!("query observer owner: {error}")))?; + let Some(row) = rows + .next() + .map_err(|error| RecoveryError::new(format!("stream observer owner: {error}")))? + else { + return Err(RecoveryError::new("current observer owner row is missing")); + }; + let owner_epoch: i64 = row + .get(0) + .map_err(|error| RecoveryError::new(format!("decode observer owner epoch: {error}")))?; + let owner_token = bounded_text(row, 1, "observer owner token", 64)?; + let lease_state = bounded_text(row, 2, "observer owner lease state", 16)?; + let expires_at: i64 = row + .get(3) + .map_err(|error| RecoveryError::new(format!("decode observer owner expiry: {error}")))?; + let error_state = bounded_optional_text(row, 4, "observer owner error", 128)?; + let error_at: Option = row.get(5).map_err(|error| { + RecoveryError::new(format!("decode observer owner error time: {error}")) + })?; + let scope_epoch: i64 = row + .get(6) + .map_err(|error| RecoveryError::new(format!("decode observer scope epoch: {error}")))?; + if rows + .next() + .map_err(|error| RecoveryError::new(format!("stream observer owner: {error}")))? + .is_some() + { + return Err(RecoveryError::new("multiple current observer owner rows")); + } + if owner_epoch != expected_epoch_sql || scope_epoch != expected_epoch_sql { + return Err(RecoveryError::new("current observer owner epoch mismatch")); + } + if owner_token != hex::encode(expected.owner_token) { + return Err(RecoveryError::new("current observer owner token mismatch")); + } + decode_owner_token(&owner_token)?; + if lease_state != "active" { + return Err(RecoveryError::new(format!( + "current observer owner is {lease_state}" + ))); + } + if error_state.is_some() || error_at.is_some() { + return Err(RecoveryError::new( + "current observer owner has an error state", + )); + } + if expires_at <= crate::db::util::now_ts() { + return Err(RecoveryError::new( + "current observer owner lease is expired", + )); + } + Ok(()) +} + +fn is_strictly_published_filename( + connection: &Connection, + scope_id: &ScopeId, + filename: &str, + directory: &super::super::secure_fs::SecureDirectory, +) -> std::result::Result { + let mut statement = connection + .prepare( + "SELECT epoch, segment_id, first_sequence, owner_token, segment_path + FROM changed_path_observer_segments + WHERE scope_id=?1 AND segment_path=?2", + ) + .map_err(|error| RecoveryError::new(format!("read published segment names: {error}")))?; + let mut rows = statement + .query(params![scope_id.to_text(), filename]) + .map_err(|error| RecoveryError::new(format!("query published segment names: {error}")))?; + while let Some(row) = rows + .next() + .map_err(|error| RecoveryError::new(format!("stream published segment names: {error}")))? + { + let epoch = match row + .get::<_, i64>(0) + .ok() + .and_then(|value| value.try_into().ok()) + { + Some(epoch) => epoch, + None => continue, + }; + let segment_id = + match bounded_text(row, 1, "published segment id", MAX_SEGMENT_FILENAME_BYTES) { + Ok(value) => value, + Err(_) => continue, + }; + let first_sequence = match row + .get::<_, i64>(2) + .ok() + .and_then(|value| value.try_into().ok()) + { + Some(sequence) => sequence, + None => continue, + }; + let owner = match bounded_text(row, 3, "published owner token", 64) + .and_then(|owner| decode_owner_token(&owner).map(|token| (owner, token))) + { + Ok(value) => value, + Err(_) => continue, + }; + let segment_path = + match bounded_text(row, 4, "published segment path", MAX_SEGMENT_FILENAME_BYTES) { + Ok(value) => value, + Err(_) => continue, + }; + let derived_id = super::writer::segment_id(epoch, first_sequence, owner.1); + let Ok(derived_filename) = super::writer::segment_filename(&derived_id) else { + continue; + }; + if owner.0 == hex::encode(owner.1) + && segment_id == derived_id + && segment_path == derived_filename + && filename == derived_filename + && published_header_matches( + directory, + filename, + &RecoveryScope { + scope_id: *scope_id, + epoch, + owner_token: owner.1, + }, + )? + { + return Ok(true); + } + } + Ok(false) +} + +fn published_header_matches( + directory: &super::super::secure_fs::SecureDirectory, + filename: &str, + expected: &RecoveryScope, +) -> std::result::Result { + const PREFIX_BYTES: usize = 8 + 2 + 4; + let mut file = directory.open_regular(filename).map_err(|error| { + RecoveryError::new(format!("open published segment header securely: {error}")) + })?; + let mut prefix = [0_u8; PREFIX_BYTES]; + file.read_exact(&mut prefix) + .map_err(|error| RecoveryError::new(format!("read published segment header: {error}")))?; + let payload_length = u32::from_be_bytes(prefix[10..14].try_into().unwrap()) as usize; + if payload_length > MAX_HEADER_BYTES { + return Err(RecoveryError::new( + "published observer segment header exceeds limit", + )); + } + let total = PREFIX_BYTES + .checked_add(payload_length) + .ok_or_else(|| RecoveryError::new("published observer header length overflow"))?; + let mut bytes = Vec::with_capacity(total); + bytes.extend_from_slice(&prefix); + bytes.resize(total, 0); + file.read_exact(&mut bytes[PREFIX_BYTES..]) + .map_err(|error| RecoveryError::new(format!("read published segment header: {error}")))?; + let (identity, consumed) = decode_header(&bytes)?; + Ok(consumed == total && identity.recovery_scope() == *expected) +} + +#[derive(Debug)] +struct SegmentMetadata { + segment_id: String, + owner_token: String, + first_sequence: i64, + last_sequence: Option, + durable_end_offset: i64, + folded_end_offset: i64, + previous_segment_id: Option, + previous_segment_hash: Option, + segment_hash: Option, + segment_path: String, + state: String, +} + +fn decode_segment_metadata(row: &Row<'_>) -> std::result::Result { + Ok(SegmentMetadata { + segment_id: bounded_text(row, 0, "segment id", MAX_SEGMENT_FILENAME_BYTES)?, + owner_token: bounded_text(row, 1, "owner token", 64)?, + first_sequence: row + .get(2) + .map_err(|error| RecoveryError::new(format!("decode first sequence: {error}")))?, + last_sequence: row + .get(3) + .map_err(|error| RecoveryError::new(format!("decode last sequence: {error}")))?, + durable_end_offset: row + .get(4) + .map_err(|error| RecoveryError::new(format!("decode durable offset: {error}")))?, + folded_end_offset: row + .get(5) + .map_err(|error| RecoveryError::new(format!("decode folded offset: {error}")))?, + previous_segment_id: bounded_optional_text( + row, + 6, + "previous segment id", + MAX_SEGMENT_FILENAME_BYTES, + )?, + previous_segment_hash: bounded_optional_text(row, 7, "previous segment hash", 64)?, + segment_hash: bounded_optional_text(row, 8, "segment hash", 64)?, + segment_path: bounded_text(row, 9, "segment path", MAX_SEGMENT_FILENAME_BYTES)?, + state: bounded_text(row, 10, "segment state", 16)?, + }) +} + +fn bounded_text( + row: &Row<'_>, + index: usize, + label: &str, + max: usize, +) -> std::result::Result { + match row + .get_ref(index) + .map_err(|error| RecoveryError::new(format!("decode {label}: {error}")))? + { + ValueRef::Text(bytes) if bytes.len() <= max => std::str::from_utf8(bytes) + .map(str::to_owned) + .map_err(|_| RecoveryError::new(format!("invalid {label} text"))), + ValueRef::Text(_) => Err(RecoveryError::new(format!( + "{label} exceeds bounded length" + ))), + _ => Err(RecoveryError::new(format!("invalid {label} type"))), + } +} + +fn bounded_optional_text( + row: &Row<'_>, + index: usize, + label: &str, + max: usize, +) -> std::result::Result, RecoveryError> { + if matches!( + row.get_ref(index) + .map_err(|error| RecoveryError::new(format!("decode {label}: {error}")))?, + ValueRef::Null + ) { + Ok(None) + } else { + bounded_text(row, index, label, max).map(Some) + } +} + +#[cfg(unix)] +pub(super) fn open_segment_no_follow(path: &Path) -> std::result::Result { + use std::os::unix::fs::OpenOptionsExt; + + let mut options = fs::OpenOptions::new(); + options.read(true); + options.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC); + let file = options.open(path).map_err(|error| { + RecoveryError::new(format!("open observer segment without symlinks: {error}")) + })?; + if !file + .metadata() + .map_err(|error| RecoveryError::new(format!("inspect observer segment: {error}")))? + .is_file() + { + return Err(RecoveryError::new("observer segment is not a regular file")); + } + Ok(file) +} + +#[cfg(not(unix))] +pub(super) fn open_segment_no_follow(path: &Path) -> std::result::Result { + let before = fs::symlink_metadata(path) + .map_err(|error| RecoveryError::new(format!("inspect observer segment: {error}")))?; + if before.file_type().is_symlink() || !before.is_file() { + return Err(RecoveryError::new( + "observer segment symlink metadata is rejected", + )); + } + let file = File::open(path) + .map_err(|error| RecoveryError::new(format!("open observer segment: {error}")))?; + let after = fs::symlink_metadata(path) + .map_err(|error| RecoveryError::new(format!("reinspect observer segment: {error}")))?; + if after.file_type().is_symlink() || !after.is_file() { + return Err(RecoveryError::new("observer segment changed to a symlink")); + } + Ok(file) +} + +fn decode_optional_hash(value: Option<&str>) -> std::result::Result<[u8; 32], RecoveryError> { + match value { + Some(value) => decode_required_hash(Some(value), "segment hash"), + None => Ok([0; 32]), + } +} + +fn decode_owner_token(value: &str) -> std::result::Result<[u8; 32], RecoveryError> { + if value.len() != 64 || value != value.to_ascii_lowercase() { + return Err(RecoveryError::new("non-canonical observer owner token")); + } + let bytes = hex::decode(value) + .map_err(|_| RecoveryError::new("invalid observer owner token encoding"))?; + bytes + .try_into() + .map_err(|_| RecoveryError::new("invalid observer owner token length")) +} + +fn decode_required_hash( + value: Option<&str>, + label: &str, +) -> std::result::Result<[u8; 32], RecoveryError> { + let value = value.ok_or_else(|| RecoveryError::new(format!("missing {label}")))?; + if value != value.to_ascii_lowercase() { + return Err(RecoveryError::new(format!("non-canonical {label}"))); + } + let bytes = hex::decode(value).map_err(|_| RecoveryError::new(format!("invalid {label}")))?; + bytes + .try_into() + .map_err(|_| RecoveryError::new(format!("invalid {label} length"))) +} + +fn db_u64(value: i64, label: &str) -> std::result::Result { + value + .try_into() + .map_err(|_| RecoveryError::new(format!("negative {label}"))) +} + +fn open_recovery_database(path: &Path) -> std::result::Result<(Connection, bool), RecoveryError> { + let mut wal_name = path.as_os_str().to_os_string(); + wal_name.push("-wal"); + let wal = std::path::PathBuf::from(wal_name); + let mut shm_name = path.as_os_str().to_os_string(); + shm_name.push("-shm"); + let shm = std::path::PathBuf::from(shm_name); + let flags = OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX; + let (connection, immutable) = match (wal.exists(), shm.exists()) { + (true, true) => (Connection::open_with_flags(path, flags), false), + (false, false) => ( + Connection::open_with_flags( + immutable_sqlite_uri(path), + flags | OpenFlags::SQLITE_OPEN_URI, + ), + true, + ), + _ => { + return Err(RecoveryError::new( + "observer database has incomplete WAL sidecar state", + )); + } + }; + connection + .map(|connection| (connection, immutable)) + .map_err(|error| RecoveryError::new(format!("open observer control database: {error}"))) +} + +fn database_header_uses_wal(path: &Path) -> std::result::Result { + let mut file = File::open(path) + .map_err(|error| RecoveryError::new(format!("open observer database header: {error}")))?; + let mut header = [0_u8; 20]; + std::io::Read::read_exact(&mut file, &mut header) + .map_err(|error| RecoveryError::new(format!("read observer database header: {error}")))?; + Ok(&header[..16] == b"SQLite format 3\0" && header[18] == 2 && header[19] == 2) +} + +#[cfg(unix)] +fn immutable_sqlite_uri(path: &Path) -> String { + use std::os::unix::ffi::OsStrExt; + + let mut uri = String::from("file:"); + for byte in path.as_os_str().as_bytes() { + match byte { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' | b'/' => { + uri.push(char::from(*byte)); + } + _ => uri.push_str(&format!("%{byte:02x}")), + } + } + uri.push_str("?immutable=1"); + uri +} + +#[cfg(not(unix))] +fn immutable_sqlite_uri(path: &Path) -> String { + let encoded = path + .to_string_lossy() + .replace('%', "%25") + .replace('?', "%3f") + .replace('#', "%23"); + format!("file:{encoded}?immutable=1") +} diff --git a/trail/src/db/change_ledger/log/tests.rs b/trail/src/db/change_ledger/log/tests.rs new file mode 100644 index 0000000..f48f63d --- /dev/null +++ b/trail/src/db/change_ledger/log/tests.rs @@ -0,0 +1,1765 @@ +use super::*; +use crate::db::change_ledger::{ + BaselineIdentity, ChangedPathLedger, EvidenceFlags, EvidenceSource, FilesystemIdentity, + LedgerPath, PolicyIdentity, ProviderCapabilities, ProviderIdentity, ScopeId, ScopeIdentity, + ScopeKind, +}; +use crate::{ChangeId, InitImportMode, ObjectId, Trail}; +use rusqlite::{params, Connection}; +use sha2::{Digest, Sha256}; +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +const TEST_SCHEMA: &str = " + PRAGMA foreign_keys = ON; + CREATE TABLE changed_path_scopes ( + scope_id TEXT PRIMARY KEY, + epoch INTEGER NOT NULL, + trust_state TEXT NOT NULL DEFAULT 'untrusted_gap', + trust_reason TEXT NOT NULL DEFAULT 'observer fixture', + continuity_generation INTEGER NOT NULL DEFAULT 1, + max_observer_log_bytes INTEGER NOT NULL, + max_segment_bytes INTEGER NOT NULL, + max_unfolded_tail_records INTEGER NOT NULL + ); + CREATE TABLE changed_path_observer_owners ( + scope_id TEXT PRIMARY KEY REFERENCES changed_path_scopes(scope_id), + epoch INTEGER NOT NULL, + owner_token TEXT NOT NULL UNIQUE, + provider_id TEXT NOT NULL, + provider_identity TEXT NOT NULL, + lease_state TEXT NOT NULL, + fence_nonce BLOB, + acquired_at INTEGER NOT NULL, + heartbeat_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + error_state TEXT, + error_at INTEGER, + daemon_launch_nonce TEXT, + daemon_pid INTEGER, + daemon_process_start_identity TEXT, + updated_at INTEGER NOT NULL + ); + CREATE TABLE changed_path_observer_segments ( + scope_id TEXT NOT NULL, + epoch INTEGER NOT NULL, + segment_id TEXT NOT NULL, + log_format_version INTEGER NOT NULL, + owner_token TEXT NOT NULL, + provider_id TEXT NOT NULL, + first_sequence INTEGER NOT NULL, + last_sequence INTEGER, + durable_end_offset INTEGER NOT NULL, + folded_end_offset INTEGER NOT NULL, + previous_segment_id TEXT, + previous_segment_hash TEXT, + segment_hash TEXT, + segment_path TEXT NOT NULL, + state TEXT NOT NULL, + created_at INTEGER NOT NULL, + sealed_at INTEGER, + updated_at INTEGER NOT NULL, + PRIMARY KEY(scope_id, epoch, segment_id) + );"; + +struct Fixture { + _temp: tempfile::TempDir, + database: std::path::PathBuf, + segments: std::path::PathBuf, + scope: ScopeId, +} + +impl Fixture { + fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("trail.db"); + let segments = temp.path().join("observer"); + let scope = ScopeId([0x44; 32]); + let connection = Connection::open(&database).unwrap(); + crate::db::util::apply_sqlite_pragmas(&connection).unwrap(); + connection.execute_batch(TEST_SCHEMA).unwrap(); + connection + .execute( + "INSERT INTO changed_path_scopes( + scope_id,epoch,max_observer_log_bytes,max_segment_bytes, + max_unfolded_tail_records + ) VALUES(?1,3,?2,?3,?4)", + params![scope.to_text(), 268_435_456_i64, 16_777_216_i64, 65_536_i64], + ) + .unwrap(); + Self { + _temp: temp, + database, + segments, + scope, + } + } + + fn acquire(&self, token: [u8; 32]) -> SegmentWriter { + self.acquire_epoch(3, token).unwrap() + } + + fn acquire_epoch(&self, epoch: u64, token: [u8; 32]) -> Result { + SegmentWriter::acquire( + &self.database, + &self.segments, + self.scope, + epoch, + token, + "test-provider", + b"cursor-0".to_vec(), + Duration::from_secs(60), + ) + } + + fn full_v18() -> Self { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let scope = ScopeId([0x44; 32]); + ChangedPathLedger::new(&db.conn) + .begin_scope( + &ScopeIdentity { + scope_id: scope, + kind: ScopeKind::Workspace, + owner_id: "observer-review-fixture".into(), + }, + &BaselineIdentity { + ref_name: "refs/branches/main".into(), + ref_generation: 1, + change_id: ChangeId("observer-review-change".into()), + root_id: ObjectId("observer-review-root".into()), + }, + &PolicyIdentity { + fingerprint: [0x45; 32], + generation: 1, + }, + &FilesystemIdentity(vec![0x46]), + &ProviderIdentity { + identity: vec![0x47], + capabilities: ProviderCapabilities { + durable_cursor: false, + linearizable_fence: false, + rename_pairing: true, + overflow_scope: true, + filesystem_supported: true, + clean_proof_allowed: false, + power_loss_durability: false, + }, + }, + ) + .unwrap(); + Self { + database: db.db_dir.join(crate::db::DB_RELATIVE_PATH), + segments: db.db_dir.join("observer-review"), + scope, + _temp: temp, + } + } + + fn durable_offset(&self) -> u64 { + Connection::open(&self.database) + .unwrap() + .query_row( + "SELECT durable_end_offset FROM changed_path_observer_segments + ORDER BY first_sequence DESC LIMIT 1", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap() as u64 + } +} + +#[test] +fn observer_writer_acquisition_obeys_workspace_exclusion_without_mutation() { + let fixture = Fixture::new(); + let workspace_db_dir = fixture.database.parent().unwrap(); + let _workspace_lock = + crate::db::acquire_workspace_lock_for_database(workspace_db_dir, &fixture.database) + .unwrap(); + + assert!(matches!( + fixture.acquire_epoch(3, [0x19; 32]), + Err(crate::Error::WorkspaceLocked(_)) + )); + + let connection = Connection::open(&fixture.database).unwrap(); + let owner_count: i64 = connection + .query_row( + "SELECT COUNT(*) FROM changed_path_observer_owners", + [], + |row| row.get(0), + ) + .unwrap(); + let segment_count: i64 = connection + .query_row( + "SELECT COUNT(*) FROM changed_path_observer_segments", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!((owner_count, segment_count), (0, 0)); + assert!(!fixture.segments.exists()); +} + +fn database_snapshot(fixture: &Fixture) -> (Vec, Vec) { + let connection = Connection::open(&fixture.database).unwrap(); + let mut owners = connection + .prepare( + "SELECT printf('%s|%d|%s|%s|%d|%d', scope_id, epoch, owner_token, + lease_state, expires_at, updated_at) + FROM changed_path_observer_owners ORDER BY scope_id", + ) + .unwrap(); + let owners = owners + .query_map([], |row| row.get(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + let mut segments = connection + .prepare( + "SELECT printf('%s|%d|%s|%d|%s|%s|%s', scope_id, epoch, segment_id, + first_sequence, owner_token, segment_path, state) + FROM changed_path_observer_segments + ORDER BY epoch, first_sequence, segment_id", + ) + .unwrap(); + let segments = segments + .query_map([], |row| row.get(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + (owners, segments) +} + +fn file_snapshot(fixture: &Fixture) -> Vec<(String, Vec)> { + let mut files = fs::read_dir(&fixture.segments) + .unwrap() + .map(|entry| { + let entry = entry.unwrap(); + ( + entry.file_name().to_string_lossy().into_owned(), + fs::read(entry.path()).unwrap(), + ) + }) + .collect::>(); + files.sort_by(|left, right| left.0.cmp(&right.0)); + files +} + +fn event(sequence: u64) -> ObserverRecord { + ObserverRecord { + sequence, + source: EvidenceSource::Observer, + path: LedgerPath::parse(&format!("src/{sequence}.rs")).unwrap(), + flags: EvidenceFlags::CONTENT, + provider_cursor: sequence.to_be_bytes().to_vec(), + } +} + +#[test] +fn header_only_post_rotation_flush_keeps_database_last_sequence_null() { + let fixture = Fixture::full_v18(); + let mut writer = fixture.acquire_epoch(1, [0xa1; 32]).unwrap(); + writer.append(&[event(1)]).unwrap(); + let sealed = writer.flush_durable().unwrap(); + writer.rotate().unwrap(); + + let anchor = writer.flush_durable().unwrap(); + assert_eq!(anchor.last_sequence, sealed.last_sequence); + assert_eq!(anchor.provider_cursor, sealed.provider_cursor); + assert_ne!(anchor.segment_id, sealed.segment_id); + let (empty_sealed, empty_anchor) = writer.rotate_and_cuts().unwrap(); + assert_eq!(empty_sealed, anchor); + assert_eq!(empty_anchor, anchor); + assert_eq!(writer.flush_durable().unwrap(), anchor); + + let stored: (i64, Option, i64) = Connection::open(&fixture.database) + .unwrap() + .query_row( + "SELECT first_sequence,last_sequence,durable_end_offset + FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=1 AND segment_id=?2 AND state='open'", + params![fixture.scope.to_text(), anchor.segment_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(stored.0, 2); + assert_eq!(stored.1, None); + assert_eq!(u64::try_from(stored.2).unwrap(), anchor.durable_end_offset); + + let recovered = recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 1, + owner_token: [0xa1; 32], + }, + PersistedLogLimits::default(), + ) + .unwrap(); + let open = recovered + .segments + .iter() + .find(|segment| segment.segment_id == anchor.segment_id) + .unwrap(); + assert_eq!(open.first_sequence, 2); + assert_eq!(open.last_sequence, 1); + assert_eq!(open.header_end_offset, anchor.durable_end_offset); +} + +#[test] +fn append_flush_and_rotate_seals_the_boundary_and_opens_its_linked_anchor() { + let fixture = Fixture::full_v18(); + let mut writer = fixture.acquire_epoch(1, [0xa2; 32]).unwrap(); + + let (sealed, anchor) = writer.append_flush_and_rotate(&[event(1)]).unwrap(); + + assert_eq!(sealed.last_sequence, 1); + assert_eq!(anchor.last_sequence, sealed.last_sequence); + assert_eq!(anchor.provider_cursor, sealed.provider_cursor); + assert_ne!(anchor.segment_id, sealed.segment_id); + let connection = Connection::open(&fixture.database).unwrap(); + let stored = connection + .prepare( + "SELECT segment_id,state,last_sequence,durable_end_offset,previous_segment_id + FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=1 ORDER BY created_at,segment_id", + ) + .unwrap() + .query_map([fixture.scope.to_text()], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, Option>(4)?, + )) + }) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(stored.len(), 2); + let old = stored + .iter() + .find(|segment| segment.0 == sealed.segment_id) + .unwrap(); + assert_eq!(old.1, "sealed"); + assert_eq!(old.2, Some(1)); + assert_eq!(u64::try_from(old.3).unwrap(), sealed.durable_end_offset); + let next = stored + .iter() + .find(|segment| segment.0 == anchor.segment_id) + .unwrap(); + assert_eq!(next.1, "open"); + assert_eq!(next.2, None); + assert_eq!(next.4.as_deref(), Some(sealed.segment_id.as_str())); + + let later = writer.append_and_flush(&[event(2)]).unwrap(); + assert_eq!(later.segment_id, anchor.segment_id); + assert_eq!(later.last_sequence, 2); +} + +#[test] +fn non_daemon_owner_replacement_clears_prior_daemon_launch_binding() { + let fixture = Fixture::full_v18(); + let old_launch_nonce = "ab".repeat(32); + let old_pid = 4242; + let old_start = "old-daemon-start"; + let first = SegmentWriter::acquire_for_daemon( + &fixture.database, + &fixture.segments, + fixture.scope, + 1, + [0xb1; 32], + "test-provider", + Vec::new(), + Duration::from_secs(60), + DaemonLaunchBinding { + nonce: old_launch_nonce.clone(), + pid: old_pid, + process_start_identity: old_start.into(), + }, + ) + .unwrap(); + drop(first); + + Connection::open(&fixture.database) + .unwrap() + .execute( + "UPDATE changed_path_scopes SET epoch=2 WHERE scope_id=?1", + [fixture.scope.to_text()], + ) + .unwrap(); + let replacement_token = [0xb2; 32]; + let replacement = fixture.acquire_epoch(2, replacement_token).unwrap(); + + let db = Trail::open(fixture._temp.path()).unwrap(); + assert!(crate::db::verified_stale_workspace_owner_for_launch( + &db, + old_pid, + old_start, + &old_launch_nonce, + ) + .unwrap() + .is_none()); + let owner: ( + i64, + String, + String, + Option, + Option, + Option, + ) = db + .conn + .query_row( + "SELECT epoch,owner_token,lease_state,daemon_launch_nonce,daemon_pid, + daemon_process_start_identity + FROM changed_path_observer_owners WHERE scope_id=?1", + [fixture.scope.to_text()], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + )) + }, + ) + .unwrap(); + assert_eq!(owner.0, 2); + assert_eq!(owner.1, hex::encode(replacement_token)); + assert_eq!(owner.2, "active"); + assert_eq!((owner.3, owner.4, owner.5), (None, None, None)); + drop(replacement); +} + +#[test] +fn dropping_an_unbound_short_lived_writer_only_closes_its_file() { + let fixture = Fixture::full_v18(); + let token = [0xb3; 32]; + let writer = fixture.acquire_epoch(1, token).unwrap(); + drop(writer); + + let owner: (String, String, Option) = Connection::open(&fixture.database) + .unwrap() + .query_row( + "SELECT owner_token,lease_state,error_state + FROM changed_path_observer_owners WHERE scope_id=?1", + [fixture.scope.to_text()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(owner.0, hex::encode(token)); + assert_eq!(owner.1, "active"); + assert_eq!(owner.2, None); +} + +#[test] +fn dropping_a_native_bound_writer_revokes_its_exact_owner() { + let fixture = Fixture::full_v18(); + let token = [0xb4; 32]; + let connection = Connection::open(&fixture.database).unwrap(); + let (provider_id, provider_identity): (String, String) = connection + .query_row( + "SELECT provider_id,provider_identity FROM changed_path_scopes WHERE scope_id=?1", + [fixture.scope.to_text()], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + drop(connection); + let mut writer = SegmentWriter::acquire( + &fixture.database, + &fixture.segments, + fixture.scope, + 1, + token, + &provider_id, + b"cursor-0".to_vec(), + Duration::from_secs(60), + ) + .unwrap(); + writer + .bind_native_observer(hex::decode(provider_identity).unwrap(), vec![0x55; 32]) + .unwrap(); + drop(writer); + + let owner: (String, String, Option) = Connection::open(&fixture.database) + .unwrap() + .query_row( + "SELECT owner_token,lease_state,error_state + FROM changed_path_observer_owners WHERE scope_id=?1", + [fixture.scope.to_text()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(owner.0, hex::encode(token)); + assert_eq!(owner.1, "error"); + assert_eq!(owner.2.as_deref(), Some("observer_writer_dropped")); +} + +#[test] +fn append_and_flush_keeps_command_lock_out_until_sqlite_publication() { + let fixture = Fixture::full_v18(); + let mut writer = fixture.acquire_epoch(1, [0xc5; 32]).unwrap(); + let boundary_observed = Arc::new(AtomicBool::new(false)); + let observed = Arc::clone(&boundary_observed); + install_append_flush_boundary_hook(move |workspace_db_dir, database_path| { + let acquisition = + crate::db::acquire_workspace_lock_for_database(workspace_db_dir, database_path); + assert!(matches!(acquisition, Err(Error::WorkspaceLocked(_)))); + let unpublished: Option = Connection::open(database_path) + .unwrap() + .query_row( + "SELECT last_sequence FROM changed_path_observer_segments WHERE state='open'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(unpublished, None); + observed.store(true, Ordering::Release); + }); + + let cut = writer.append_and_flush(&[event(1)]).unwrap(); + + assert!(boundary_observed.load(Ordering::Acquire)); + let database_parent = fixture.database.parent().unwrap(); + let workspace_db_dir = if database_parent + .file_name() + .is_some_and(|name| name == "index") + { + database_parent.parent().unwrap() + } else { + database_parent + }; + let command_lock = + crate::db::acquire_workspace_lock_for_database(workspace_db_dir, &fixture.database) + .unwrap(); + drop(command_lock); + let published: (i64, i64) = Connection::open(&fixture.database) + .unwrap() + .query_row( + "SELECT last_sequence,durable_end_offset + FROM changed_path_observer_segments WHERE state='open'", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(published.0, 1); + assert_eq!(u64::try_from(published.1).unwrap(), cut.durable_end_offset); +} + +#[test] +fn heartbeat_remains_live_while_a_command_holds_the_workspace_lock() { + let fixture = Fixture::full_v18(); + let token = [0xc6; 32]; + let mut writer = fixture.acquire_epoch(1, token).unwrap(); + let database_parent = fixture.database.parent().unwrap(); + let workspace_db_dir = if database_parent + .file_name() + .is_some_and(|name| name == "index") + { + database_parent.parent().unwrap() + } else { + database_parent + }; + let command_lock = + crate::db::acquire_workspace_lock_for_database(workspace_db_dir, &fixture.database) + .unwrap(); + + writer.heartbeat().unwrap(); + + let owner: (String, String, Option) = Connection::open(&fixture.database) + .unwrap() + .query_row( + "SELECT owner_token,lease_state,error_state + FROM changed_path_observer_owners WHERE scope_id=?1", + [fixture.scope.to_text()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(owner.0, hex::encode(token)); + assert_eq!(owner.1, "active"); + assert_eq!(owner.2, None); + drop(command_lock); +} + +#[test] +fn dropping_a_daemon_owned_writer_revokes_its_exact_owner() { + let fixture = Fixture::full_v18(); + let token = [0xb5; 32]; + let provider_id: String = Connection::open(&fixture.database) + .unwrap() + .query_row( + "SELECT provider_id FROM changed_path_scopes WHERE scope_id=?1", + [fixture.scope.to_text()], + |row| row.get(0), + ) + .unwrap(); + let writer = SegmentWriter::acquire_for_daemon( + &fixture.database, + &fixture.segments, + fixture.scope, + 1, + token, + &provider_id, + b"cursor-0".to_vec(), + Duration::from_secs(60), + DaemonLaunchBinding { + nonce: "b6".repeat(32), + pid: std::process::id(), + process_start_identity: "daemon-drop-test".into(), + }, + ) + .unwrap(); + drop(writer); + + let owner: (String, String, Option) = Connection::open(&fixture.database) + .unwrap() + .query_row( + "SELECT owner_token,lease_state,error_state + FROM changed_path_observer_owners WHERE scope_id=?1", + [fixture.scope.to_text()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(owner.0, hex::encode(token)); + assert_eq!(owner.1, "error"); + assert_eq!(owner.2.as_deref(), Some("observer_writer_dropped")); +} + +#[test] +fn torn_tail_recovers_only_through_last_checked_record() { + let identity = SegmentIdentity::test(ScopeId([7; 32]), 3, [9; 32]); + let mut bytes = encoded_segment(&identity, &[event(1), event(2)]).unwrap(); + bytes.truncate(bytes.len() - 7); + + let recovered = recover_bytes(&bytes, &identity, PersistedLogLimits::default()).unwrap(); + + assert_eq!(recovered.records, vec![event(1)]); + assert!(recovered.requires_reconciliation); +} + +#[test] +fn corrupt_middle_record_fails_closed() { + let identity = SegmentIdentity::test(ScopeId([7; 32]), 3, [9; 32]); + let mut bytes = encoded_segment(&identity, &[event(1), event(2)]).unwrap(); + let first = header_end(&bytes).unwrap(); + bytes[first + 16] ^= 0x40; + + let error = recover_bytes(&bytes, &identity, PersistedLogLimits::default()).unwrap_err(); + + assert!(error.requires_reconciliation); +} + +#[test] +fn payload_over_one_mib_is_rejected_before_append() { + let mut record = event(1); + record.provider_cursor = vec![0; MAX_RECORD_PAYLOAD_BYTES + 1]; + + assert!(encode_record(&record, [0; 32]).is_err()); +} + +#[test] +fn recovery_rejects_wrong_identity_non_monotonic_sequences_and_count_caps() { + let identity = SegmentIdentity::test(ScopeId([7; 32]), 3, [9; 32]); + let bytes = encoded_segment(&identity, &[event(1), event(2)]).unwrap(); + let wrong = SegmentIdentity::test(ScopeId([8; 32]), 3, [9; 32]); + assert!(recover_bytes(&bytes, &wrong, PersistedLogLimits::default()).is_err()); + + let non_monotonic = encoded_segment(&identity, &[event(2), event(1)]).unwrap(); + assert!(recover_bytes(&non_monotonic, &identity, PersistedLogLimits::default()).is_err()); + + let limits = PersistedLogLimits { + max_unfolded_tail_records: 1, + ..PersistedLogLimits::default() + }; + assert!(recover_bytes(&bytes, &identity, limits).is_err()); +} + +#[test] +fn global_epoch_fences_same_epoch_replacement_without_any_mutation() { + let fixture = Fixture::full_v18(); + let mut first = fixture.acquire_epoch(1, [1; 32]).unwrap(); + assert!(SegmentWriter::acquire( + &fixture.database, + &fixture.segments, + fixture.scope, + 1, + [2; 32], + "test-provider", + Vec::new(), + Duration::from_secs(60), + ) + .is_err()); + + let connection = Connection::open(&fixture.database).unwrap(); + for (state, error_state, error_at) in [ + ("revoked", None, None), + ("expired", None, None), + ("error", Some("injected-terminal-state"), Some(1_i64)), + ] { + connection + .execute( + "UPDATE changed_path_observer_owners + SET lease_state=?1, error_state=?2, error_at=?3 WHERE scope_id=?4", + params![state, error_state, error_at, fixture.scope.to_text()], + ) + .unwrap(); + let database_before = database_snapshot(&fixture); + let files_before = file_snapshot(&fixture); + + let error = fixture.acquire_epoch(1, [2; 32]).err().unwrap(); + + assert!( + error.to_string().contains("reconciliation"), + "state {state}" + ); + assert_eq!( + database_snapshot(&fixture), + database_before, + "state {state}" + ); + assert_eq!(file_snapshot(&fixture), files_before, "state {state}"); + } + assert!(first.append(&[event(1)]).is_err()); + + connection + .execute( + "UPDATE changed_path_scopes SET epoch=2 WHERE scope_id=?1", + [fixture.scope.to_text()], + ) + .unwrap(); + let replacement = fixture.acquire_epoch(2, [2; 32]).unwrap(); + drop(replacement); + let epochs: Vec<(i64, i64)> = Connection::open(&fixture.database) + .unwrap() + .prepare( + "SELECT epoch, first_sequence FROM changed_path_observer_segments + ORDER BY epoch, first_sequence", + ) + .unwrap() + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!(epochs, vec![(1, 1), (2, 1)]); + + let recovered = recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 2, + owner_token: [2; 32], + }, + PersistedLogLimits::default(), + ) + .unwrap(); + assert!(recovered.records.is_empty()); + assert!(!recovered.requires_reconciliation); + + assert!(!first.is_authorized()); +} + +#[test] +fn segment_id_is_exactly_derived_from_epoch_sequence_and_full_owner_token() { + let mut token_b = [0x11; 32]; + token_b[31] = 0x12; + let first = segment_id(1, 1, [0x11; 32]); + let different_epoch = segment_id(2, 1, [0x11; 32]); + let different_token_tail = segment_id(1, 1, token_b); + + assert_eq!( + first, + format!("{:020}-{:020}-{}", 1, 1, hex::encode([0x11; 32])) + ); + assert_ne!(first, different_epoch); + assert_ne!(first, different_token_tail); + assert_eq!(segment_filename(&first).unwrap(), format!("{first}.cpl")); + assert!(segment_filename(&format!("{:020}-{:020}-{}", 0, 1, hex::encode([0x11; 32]))).is_err()); + assert!(segment_filename(&segment_id(1, 1, [0xab; 32]).to_ascii_uppercase()).is_err()); +} + +#[test] +fn epoch_advance_makes_filesystem_name_unique_even_with_the_same_full_token() { + let fixture = Fixture::full_v18(); + let token = [0x12; 32]; + drop(fixture.acquire_epoch(1, token).unwrap()); + Connection::open(&fixture.database) + .unwrap() + .execute( + "UPDATE changed_path_scopes SET epoch=2 WHERE scope_id=?1", + [fixture.scope.to_text()], + ) + .unwrap(); + drop(fixture.acquire_epoch(2, token).unwrap()); + + let files = file_snapshot(&fixture); + assert_eq!(files.len(), 2); + assert_ne!(files[0].0, files[1].0); + assert!(files + .iter() + .any(|file| file.0.starts_with("00000000000000000001-"))); + assert!(files + .iter() + .any(|file| file.0.starts_with("00000000000000000002-"))); +} + +#[test] +fn invalid_other_epoch_filename_metadata_cannot_suppress_an_orphan() { + let fixture = Fixture::new(); + let token = [0x16; 32]; + let writer = fixture.acquire(token); + let other_token = [0x17; 32]; + let other_id = segment_id(2, 1, other_token); + let orphan_filename = segment_filename(&other_id).unwrap(); + fs::write(fixture.segments.join(&orphan_filename), b"orphan").unwrap(); + Connection::open(&fixture.database) + .unwrap() + .execute( + "INSERT INTO changed_path_observer_segments( + scope_id, epoch, segment_id, log_format_version, owner_token, + provider_id, first_sequence, last_sequence, durable_end_offset, + folded_end_offset, previous_segment_id, previous_segment_hash, + segment_hash, segment_path, state, created_at, sealed_at, updated_at) + SELECT scope_id, 2, ?1, log_format_version, ?2, + provider_id, first_sequence, last_sequence, durable_end_offset, + folded_end_offset, previous_segment_id, previous_segment_hash, + segment_hash, ?3, state, created_at, sealed_at, updated_at + FROM changed_path_observer_segments WHERE epoch=3", + params![other_id, hex::encode(other_token), orphan_filename], + ) + .unwrap(); + + let recovered = recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: token, + }, + PersistedLogLimits::default(), + ) + .unwrap(); + assert!(recovered.requires_reconciliation); + drop(writer); +} + +#[test] +fn authenticated_orphan_from_superseded_epoch_does_not_poison_current_epoch() { + let fixture = Fixture::new(); + let current_token = [0x18; 32]; + let writer = fixture.acquire(current_token); + let stale_token = [0x19; 32]; + let stale_id = segment_id(2, 1, stale_token); + let stale_filename = segment_filename(&stale_id).unwrap(); + let stale_header = encode_header(&SegmentIdentity { + scope_id: fixture.scope, + epoch: 2, + owner_token: stale_token, + provider_cursor: b"stale-cursor".to_vec(), + previous_segment_hash: [0; 32], + }) + .unwrap(); + fs::write(fixture.segments.join(stale_filename), stale_header).unwrap(); + + let recovered = recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: current_token, + }, + PersistedLogLimits::default(), + ) + .unwrap(); + + assert!(!recovered.requires_reconciliation); + drop(writer); +} + +#[test] +fn coordinated_segment_id_path_and_file_rename_cannot_defeat_derivation() { + let fixture = Fixture::new(); + let token = [0x13; 32]; + let writer = fixture.acquire(token); + let forged_id = segment_id(3, 1, [0x14; 32]); + let forged_filename = segment_filename(&forged_id).unwrap(); + fs::rename(&writer.path, fixture.segments.join(&forged_filename)).unwrap(); + Connection::open(&fixture.database) + .unwrap() + .execute( + "UPDATE changed_path_observer_segments SET segment_id=?1, segment_path=?2", + params![forged_id, forged_filename], + ) + .unwrap(); + + let error = recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: token, + }, + PersistedLogLimits::default(), + ) + .unwrap_err(); + assert!(error.message.contains("derived")); +} + +#[test] +fn recovery_fails_closed_for_every_invalid_current_owner_state() { + let fixture = Fixture::new(); + let token = [0x15; 32]; + let writer = fixture.acquire(token); + drop(writer); + let connection = Connection::open(&fixture.database).unwrap(); + let original_expiry: i64 = connection + .query_row( + "SELECT expires_at FROM changed_path_observer_owners", + [], + |row| row.get(0), + ) + .unwrap(); + let expected = RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: token, + }; + for mutation in [ + "UPDATE changed_path_observer_owners SET owner_token=lower(hex(zeroblob(32)))", + "UPDATE changed_path_observer_owners SET epoch=2", + "UPDATE changed_path_observer_owners SET lease_state='revoked'", + "UPDATE changed_path_observer_owners SET lease_state='expired'", + "UPDATE changed_path_observer_owners SET expires_at=heartbeat_at", + "UPDATE changed_path_observer_owners SET error_state='owner-error', error_at=1", + ] { + connection.execute_batch(mutation).unwrap(); + let error = recover_segments( + &fixture.database, + &fixture.segments, + &expected, + PersistedLogLimits::default(), + ) + .unwrap_err(); + assert!(error.message.contains("owner"), "mutation: {mutation}"); + connection + .execute( + "UPDATE changed_path_observer_owners + SET epoch=3, owner_token=?1, lease_state='active', expires_at=?2, + error_state=NULL, error_at=NULL", + params![hex::encode(token), original_expiry], + ) + .unwrap(); + } + connection + .execute("DELETE FROM changed_path_observer_owners", []) + .unwrap(); + let error = recover_segments( + &fixture.database, + &fixture.segments, + &expected, + PersistedLogLimits::default(), + ) + .unwrap_err(); + assert!(error.message.contains("owner")); +} + +#[test] +fn append_failure_revokes_writer_and_flush_never_publishes_ahead_of_sync() { + let fixture = Fixture::new(); + let append_fault = Arc::new(FaultScript::new([FaultPoint::AppendWrite])); + let mut append_writer = SegmentWriter::acquire_with_faults( + &fixture.database, + &fixture.segments, + fixture.scope, + 3, + [3; 32], + "test-provider", + Vec::new(), + Duration::from_secs(60), + append_fault, + ) + .unwrap(); + assert!(append_writer.append(&[event(1)]).is_err()); + assert!(!append_writer.is_authorized()); + + let fixture = Fixture::new(); + let sync_fault = Arc::new(FaultScript::new([FaultPoint::FileSync])); + let mut sync_writer = SegmentWriter::acquire_with_faults( + &fixture.database, + &fixture.segments, + fixture.scope, + 3, + [4; 32], + "test-provider", + Vec::new(), + Duration::from_secs(60), + sync_fault, + ) + .unwrap(); + sync_writer.append(&[event(1)]).unwrap(); + let durable_before_failed_sync = fixture.durable_offset(); + assert!(sync_writer.flush_durable().is_err()); + assert_eq!(fixture.durable_offset(), durable_before_failed_sync); + assert!(!sync_writer.is_authorized()); +} + +#[test] +fn clean_rotation_publishes_hash_lineage_and_recovers_all_records() { + let fixture = Fixture::new(); + let mut writer = fixture.acquire([5; 32]); + writer.append(&[event(1)]).unwrap(); + writer.flush_durable().unwrap(); + writer.rotate().unwrap(); + writer.append(&[event(2)]).unwrap(); + writer.flush_durable().unwrap(); + + let recovered = recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: [5; 32], + }, + PersistedLogLimits::default(), + ) + .unwrap(); + assert_eq!(recovered.records, vec![event(1), event(2)]); + assert!(!recovered.requires_reconciliation); +} + +#[test] +fn every_rotation_publication_fault_retires_the_writer() { + for point in [ + FaultPoint::RotationOldSync, + FaultPoint::SealPublication, + FaultPoint::FirstDirectorySync, + FaultPoint::NextHeaderCreate, + FaultPoint::NextHeaderWrite, + FaultPoint::NextHeaderSync, + FaultPoint::SecondDirectorySync, + FaultPoint::NextMetadataPublication, + ] { + let fixture = Fixture::full_v18(); + let faults = Arc::new(FaultScript::new([point])); + let mut writer = SegmentWriter::acquire_with_faults( + &fixture.database, + &fixture.segments, + fixture.scope, + 1, + [point as u8 + 10; 32], + "test-provider", + Vec::new(), + Duration::from_secs(60), + faults, + ) + .unwrap(); + writer.append(&[event(1)]).unwrap(); + writer.flush_durable().unwrap(); + let durable_before = fixture.durable_offset(); + let metadata_before = database_snapshot(&fixture).1; + + assert!(writer.rotate().is_err(), "fault {point:?} did not fail"); + assert!( + !writer.is_authorized(), + "fault {point:?} did not retire writer" + ); + assert!(writer.append(&[event(2)]).is_err()); + assert_eq!(fixture.durable_offset(), durable_before, "fault {point:?}"); + assert_eq!( + database_snapshot(&fixture).1, + metadata_before, + "fault {point:?}" + ); + let error = recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 1, + owner_token: [point as u8 + 10; 32], + }, + PersistedLogLimits::default(), + ) + .unwrap_err(); + assert!( + error.requires_reconciliation && error.message.contains("owner"), + "fault {point:?} did not fail for retired owner: {error:?}" + ); + } +} + +#[test] +fn writer_retirement_advances_scope_continuity_generation() { + let fixture = Fixture::full_v18(); + let faults = Arc::new(FaultScript::new([FaultPoint::FileSync])); + let mut writer = SegmentWriter::acquire_with_faults( + &fixture.database, + &fixture.segments, + fixture.scope, + 1, + [0x5a; 32], + "test-provider", + Vec::new(), + Duration::from_secs(60), + faults, + ) + .unwrap(); + writer.append(&[event(1)]).unwrap(); + let connection = Connection::open(&fixture.database).unwrap(); + let before: i64 = connection + .query_row( + "SELECT continuity_generation FROM changed_path_scopes WHERE scope_id=?1", + [fixture.scope.to_text()], + |row| row.get(0), + ) + .unwrap(); + + assert!(writer.flush_durable().is_err()); + + let after: (String, i64, String) = connection + .query_row( + "SELECT s.trust_state,s.continuity_generation,o.lease_state + FROM changed_path_scopes s + JOIN changed_path_observer_owners o ON o.scope_id=s.scope_id + WHERE s.scope_id=?1", + [fixture.scope.to_text()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(after, ("untrusted_gap".into(), before + 1, "error".into())); +} + +#[test] +fn owner_terminal_transition_preserves_existing_stronger_fail_closed_state() { + let fixture = Fixture::full_v18(); + let _writer = fixture.acquire_epoch(1, [0x5b; 32]).unwrap(); + let connection = Connection::open(&fixture.database).unwrap(); + connection + .execute( + "UPDATE changed_path_scopes SET trust_state='corrupt',trust_reason='corrupt log' + WHERE scope_id=?1", + [fixture.scope.to_text()], + ) + .unwrap(); + let before: i64 = connection + .query_row( + "SELECT continuity_generation FROM changed_path_scopes WHERE scope_id=?1", + [fixture.scope.to_text()], + |row| row.get(0), + ) + .unwrap(); + + connection + .execute( + "UPDATE changed_path_observer_owners SET lease_state='revoked',updated_at=?1 + WHERE scope_id=?2", + params![crate::db::util::now_ts(), fixture.scope.to_text()], + ) + .unwrap(); + + let after: (String, String, i64) = connection + .query_row( + "SELECT trust_state,trust_reason,continuity_generation + FROM changed_path_scopes WHERE scope_id=?1", + [fixture.scope.to_text()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(after, ("corrupt".into(), "corrupt log".into(), before + 1)); +} + +#[test] +fn append_batch_capacity_never_exceeds_persisted_remaining_bytes() { + let fixture = Fixture::new(); + let token = [0x33; 32]; + let identity = SegmentIdentity::test(fixture.scope, 3, token); + let header = encode_header(&identity).unwrap().len() as u64; + let (first, _) = encode_record(&event(1), [0; 32]).unwrap(); + let remaining = first.len() as u64 + 8; + let cap = header + remaining; + Connection::open(&fixture.database) + .unwrap() + .execute( + "UPDATE changed_path_scopes SET max_observer_log_bytes=?1, max_segment_bytes=?1", + [cap as i64], + ) + .unwrap(); + let faults = Arc::new(FaultScript::default()); + let mut writer = SegmentWriter::acquire_with_faults( + &fixture.database, + &fixture.segments, + fixture.scope, + 3, + token, + "test-provider", + Vec::new(), + Duration::from_secs(60), + Arc::clone(&faults), + ) + .unwrap(); + + assert!(writer.append(&[event(1), event(2)]).is_err()); + assert!(faults.max_batch_capacity() > 0); + assert!(faults.max_batch_capacity() as u64 <= remaining); +} + +#[test] +fn acquisition_and_rotation_count_headers_against_both_byte_caps() { + let fixture = Fixture::new(); + Connection::open(&fixture.database) + .unwrap() + .execute( + "UPDATE changed_path_scopes SET max_observer_log_bytes=16, max_segment_bytes=16", + [], + ) + .unwrap(); + assert!(fixture.acquire_epoch(3, [0x31; 32]).is_err()); + assert!(database_snapshot(&fixture).0.is_empty()); + assert!(!fixture.segments.exists()); + + let fixture = Fixture::new(); + let token = [0x32; 32]; + let first_identity = SegmentIdentity::test(fixture.scope, 3, token); + let first_header = encode_header(&first_identity).unwrap().len() as u64; + let (record, _) = encode_record(&event(1), [0; 32]).unwrap(); + let mut next_identity = first_identity; + next_identity.provider_cursor = event(1).provider_cursor; + next_identity.previous_segment_hash = [1; 32]; + let next_header = encode_header(&next_identity).unwrap().len() as u64; + let cap = first_header + record.len() as u64 + next_header - 1; + Connection::open(&fixture.database) + .unwrap() + .execute( + "UPDATE changed_path_scopes SET max_observer_log_bytes=?1, max_segment_bytes=?1", + [cap as i64], + ) + .unwrap(); + let mut writer = fixture.acquire(token); + writer.append(&[event(1)]).unwrap(); + let before = database_snapshot(&fixture).1; + assert!(writer.rotate().is_err()); + assert_eq!(database_snapshot(&fixture).1, before); +} + +#[test] +fn version_one_header_is_canonical_and_identity_is_lossless() { + let fixture = Fixture::new(); + let token = [0xab; 32]; + let writer = fixture.acquire(token); + let bytes = fs::read(&writer.path).unwrap(); + assert_eq!(&bytes[..8], b"TRAILCPL"); + assert_eq!(u16::from_be_bytes(bytes[8..10].try_into().unwrap()), 1); + let (header, end) = decode_header(&bytes).unwrap(); + assert_eq!(header.scope_id, fixture.scope); + assert_eq!(header.owner_token, token); + assert_eq!(encode_header(&header).unwrap(), bytes[..end]); + + let stored: String = Connection::open(&fixture.database) + .unwrap() + .query_row( + "SELECT owner_token FROM changed_path_observer_owners", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(stored, hex::encode(token)); + assert_eq!(stored, stored.to_ascii_lowercase()); + + let mut unsupported = bytes; + unsupported[9] = 2; + assert!(decode_header(&unsupported).is_err()); +} + +#[test] +fn independently_parsed_version_one_record_has_exactly_six_fields_and_checksum() { + let identity = SegmentIdentity::test(ScopeId([7; 32]), 3, [9; 32]); + let bytes = encoded_segment(&identity, &[event(1)]).unwrap(); + let record_start = header_end(&bytes).unwrap(); + let body_len = + u32::from_be_bytes(bytes[record_start..record_start + 4].try_into().unwrap()) as usize; + + assert_eq!(body_len + 4, bytes.len() - record_start); + assert_eq!(bytes[record_start + 4 + 8], 1); + let body = &bytes[record_start + 4..]; + let sequence = u64::from_be_bytes(body[..8].try_into().unwrap()); + let source = body[8]; + let payload_end = body.len() - 64; + let payload = &body[9..payload_end]; + let previous_hash: [u8; 32] = body[payload_end..payload_end + 32].try_into().unwrap(); + let checksum: [u8; 32] = body[payload_end + 32..].try_into().unwrap(); + let independently_calculated: [u8; 32] = Sha256::digest(&body[..payload_end + 32]).into(); + assert_eq!(sequence, 1); + assert_eq!(source, 1); + assert_eq!(payload[0], 0x83); + assert_eq!(previous_hash, [0; 32]); + assert_eq!(checksum, independently_calculated); + assert_eq!( + 4 + 8 + 1 + payload.len() + 32 + 32, + bytes.len() - record_start + ); +} + +#[test] +fn independently_crafted_noncanonical_cbor_is_rejected() { + let identity = SegmentIdentity::test(ScopeId([7; 32]), 3, [9; 32]); + let canonical = encode_header(&identity).unwrap(); + let payload_start = 14; + let epoch_offset = payload_start + 1 + 2 + 32; + assert_eq!(canonical[epoch_offset], 3); + let mut noncanonical = canonical; + let canonical_length = u32::from_be_bytes(noncanonical[10..14].try_into().unwrap()); + noncanonical[10..14].copy_from_slice(&(canonical_length + 1).to_be_bytes()); + noncanonical[epoch_offset] = 0x18; + noncanonical.insert(epoch_offset + 1, 3); + assert!(decode_header(&noncanonical).is_err()); +} + +#[test] +fn metadata_paths_are_derived_relative_bounded_and_never_followed() { + let fixture = Fixture::new(); + let token = [0x71; 32]; + let writer = fixture.acquire(token); + let filename = writer + .path + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_owned(); + let stored: String = Connection::open(&fixture.database) + .unwrap() + .query_row( + "SELECT segment_path FROM changed_path_observer_segments", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(stored, filename); + + let connection = Connection::open(&fixture.database).unwrap(); + connection + .execute( + "UPDATE changed_path_observer_segments SET segment_path='../escape.cpl'", + [], + ) + .unwrap(); + assert!(recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: token, + }, + PersistedLogLimits::default(), + ) + .is_err()); + + connection + .execute( + "UPDATE changed_path_observer_segments SET segment_path=?1", + ["x".repeat(MAX_SEGMENT_FILENAME_BYTES + 1)], + ) + .unwrap(); + let error = recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: token, + }, + PersistedLogLimits::default(), + ) + .unwrap_err(); + assert!(error.message.contains("path") || error.message.contains("filename")); +} + +#[cfg(unix)] +#[test] +fn recovery_rejects_symlink_segment_final_component_with_no_follow() { + use std::os::unix::fs::symlink; + + let fixture = Fixture::new(); + let writer = fixture.acquire([0x72; 32]); + let path = writer.path.clone(); + let target = fixture._temp.path().join("outside.cpl"); + fs::copy(&path, &target).unwrap(); + drop(writer); + fs::remove_file(&path).unwrap(); + symlink(&target, &path).unwrap(); + let error = recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: [0x72; 32], + }, + PersistedLogLimits::default(), + ) + .unwrap_err(); + assert!(error.message.contains("symlink") || error.message.contains("segment")); +} + +#[cfg(target_os = "linux")] +#[test] +fn linux_recovery_open_is_compiled_with_no_follow() { + let fixture = Fixture::new(); + let writer = fixture.acquire([0x73; 32]); + assert!(open_segment_no_follow(&writer.path).is_ok()); +} + +#[test] +fn recovery_open_is_read_only_and_does_not_create_a_missing_database() { + let temp = tempfile::tempdir().unwrap(); + let missing = temp.path().join("missing.sqlite"); + assert!(recover_segments( + &missing, + temp.path(), + &RecoveryScope { + scope_id: ScopeId([1; 32]), + epoch: 1, + owner_token: [2; 32], + }, + PersistedLogLimits::default(), + ) + .is_err()); + assert!(!missing.exists()); +} + +#[test] +fn read_only_recovery_does_not_create_wal_or_shm_sidecars_to_inspect() { + let fixture = Fixture::new(); + let writer = fixture.acquire([0x7a; 32]); + let expected = RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: [0x7a; 32], + }; + drop(writer); + let wal = PathBuf::from(format!("{}-wal", fixture.database.display())); + let shm = PathBuf::from(format!("{}-shm", fixture.database.display())); + if wal.exists() { + fs::remove_file(&wal).unwrap(); + } + if shm.exists() { + fs::remove_file(&shm).unwrap(); + } + + recover_segments( + &fixture.database, + &fixture.segments, + &expected, + PersistedLogLimits::default(), + ) + .unwrap(); + + assert!(!wal.exists()); + assert!(!shm.exists()); +} + +#[test] +fn recovery_rejects_segment_count_before_streaming_row_strings_or_records() { + let fixture = Fixture::new(); + let mut writer = fixture.acquire([0x7b; 32]); + writer.append(&[event(1)]).unwrap(); + writer.rotate().unwrap(); + drop(writer); + let error = recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: [0x7b; 32], + }, + PersistedLogLimits { + max_unfolded_tail_records: 1, + ..PersistedLogLimits::default() + }, + ) + .unwrap_err(); + assert!(error.message.contains("segment count")); +} + +#[test] +fn writer_applies_required_sqlite_runtime_pragmas() { + let fixture = Fixture::new(); + let writer = fixture.acquire([0x74; 32]); + let (journal_mode, foreign_keys, synchronous, temp_store) = writer.runtime_pragmas(); + assert_eq!(journal_mode, "wal"); + assert_eq!(foreign_keys, 1); + assert!(synchronous >= 1, "runtime connection weakened durability"); + assert_eq!(temp_store, 2); +} + +#[test] +fn append_post_write_expiry_retires_without_publishing_memory_or_durability() { + let fixture = Fixture::new(); + let faults = Arc::new(FaultScript::new([FaultPoint::AppendPostWriteLeaseExpiry])); + let mut writer = SegmentWriter::acquire_with_faults( + &fixture.database, + &fixture.segments, + fixture.scope, + 3, + [0x75; 32], + "test-provider", + Vec::new(), + Duration::from_secs(60), + faults, + ) + .unwrap(); + let offset = fixture.durable_offset(); + assert!(writer.append(&[event(1)]).is_err()); + assert_eq!(fixture.durable_offset(), offset); + let owner_state: (String, Option) = Connection::open(&fixture.database) + .unwrap() + .query_row( + "SELECT lease_state,error_state FROM changed_path_observer_owners", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(owner_state, ("error".into(), Some("append_failed".into()))); + let error = recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: [0x75; 32], + }, + PersistedLogLimits::default(), + ) + .unwrap_err(); + assert!(error.requires_reconciliation); + assert!(error.message.contains("owner")); +} + +#[test] +fn flush_rejects_a_synchronized_file_length_different_from_claimed_offset() { + let fixture = Fixture::new(); + let mut writer = fixture.acquire([0x76; 32]); + writer.append(&[event(1)]).unwrap(); + OpenOptions::new() + .append(true) + .open(&writer.path) + .unwrap() + .write_all(b"unclaimed") + .unwrap(); + let durable = fixture.durable_offset(); + assert!(writer.flush_durable().is_err()); + assert_eq!(fixture.durable_offset(), durable); +} + +#[test] +fn directory_sync_runs_on_the_current_host() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("entry"), b"durable-name").unwrap(); + sync_directory(temp.path()).unwrap(); +} + +#[test] +fn heartbeat_failure_retires_writer_immediately() { + let fixture = Fixture::new(); + let faults = Arc::new(FaultScript::new([FaultPoint::Heartbeat])); + let mut writer = SegmentWriter::acquire_with_faults( + &fixture.database, + &fixture.segments, + fixture.scope, + 3, + [0x55; 32], + "test-provider", + Vec::new(), + Duration::from_secs(60), + faults, + ) + .unwrap(); + + assert!(writer.heartbeat().is_err()); + assert!(!writer.is_authorized()); + assert!(writer.append(&[event(1)]).is_err()); +} + +#[test] +fn orphan_next_header_requires_reconciliation() { + let fixture = Fixture::new(); + let mut writer = fixture.acquire([0x66; 32]); + writer.append(&[event(1)]).unwrap(); + writer.flush_durable().unwrap(); + fs::write(fixture.segments.join("orphan.cpl"), b"TRAILCPL").unwrap(); + + let recovered = recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: [0x66; 32], + }, + PersistedLogLimits::default(), + ) + .unwrap(); + + assert_eq!(recovered.records, vec![event(1)]); + assert!(recovered.requires_reconciliation); +} + +#[test] +fn broken_segment_lineage_fails_closed() { + let fixture = Fixture::new(); + let mut writer = fixture.acquire([0x77; 32]); + writer.append(&[event(1)]).unwrap(); + writer.rotate().unwrap(); + writer.append(&[event(2)]).unwrap(); + writer.flush_durable().unwrap(); + Connection::open(&fixture.database) + .unwrap() + .execute( + "UPDATE changed_path_observer_segments SET previous_segment_hash=?1 + WHERE previous_segment_id IS NOT NULL", + [hex::encode([0xff; 32])], + ) + .unwrap(); + + assert!(recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: [0x77; 32], + }, + PersistedLogLimits::default(), + ) + .is_err()); +} + +#[test] +fn trailing_bytes_in_a_sealed_middle_segment_fail_closed() { + let fixture = Fixture::new(); + let mut writer = fixture.acquire([0x79; 32]); + writer.append(&[event(1)]).unwrap(); + writer.rotate().unwrap(); + let sealed_path: String = Connection::open(&fixture.database) + .unwrap() + .query_row( + "SELECT segment_path FROM changed_path_observer_segments WHERE state='sealed'", + [], + |row| row.get(0), + ) + .unwrap(); + OpenOptions::new() + .append(true) + .open(fixture.segments.join(sealed_path)) + .unwrap() + .write_all(b"corrupt-middle") + .unwrap(); + + assert!(recover_segments( + &fixture.database, + &fixture.segments, + &RecoveryScope { + scope_id: fixture.scope, + epoch: 3, + owner_token: [0x79; 32], + }, + PersistedLogLimits::default(), + ) + .is_err()); +} + +#[test] +fn append_enforces_persisted_total_log_byte_cap() { + let fixture = Fixture::new(); + Connection::open(&fixture.database) + .unwrap() + .execute( + "UPDATE changed_path_scopes + SET max_observer_log_bytes=12000, max_segment_bytes=8000", + [], + ) + .unwrap(); + let mut writer = fixture.acquire([0x88; 32]); + let mut first = event(1); + first.path = LedgerPath::parse(&format!("first-{}", "a".repeat(6000))).unwrap(); + writer.append(&[first]).unwrap(); + writer.rotate().unwrap(); + let mut second = event(2); + second.path = LedgerPath::parse(&format!("second-{}", "b".repeat(6000))).unwrap(); + + assert!(writer.append(&[second]).is_err()); + assert!(!writer.is_authorized()); +} + +#[test] +fn writer_sql_matches_the_fresh_v18_schema() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::Empty, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let scope = ScopeId([0x91; 32]); + ChangedPathLedger::new(&db.conn) + .begin_scope( + &ScopeIdentity { + scope_id: scope, + kind: ScopeKind::Workspace, + owner_id: "writer-full-schema".into(), + }, + &BaselineIdentity { + ref_name: "refs/branches/main".into(), + ref_generation: 1, + change_id: ChangeId("change-writer-schema".into()), + root_id: ObjectId("root-writer-schema".into()), + }, + &PolicyIdentity { + fingerprint: [0x92; 32], + generation: 1, + }, + &FilesystemIdentity(vec![0x93]), + &ProviderIdentity { + identity: vec![0x94], + capabilities: ProviderCapabilities { + durable_cursor: false, + linearizable_fence: false, + rename_pairing: true, + overflow_scope: true, + filesystem_supported: true, + clean_proof_allowed: false, + power_loss_durability: false, + }, + }, + ) + .unwrap(); + let database = db.db_dir.join(crate::db::DB_RELATIVE_PATH); + let segments = db.db_dir.join("observer-test"); + let mut writer = SegmentWriter::acquire( + &database, + &segments, + scope, + 1, + [0x95; 32], + "full-schema-provider", + Vec::new(), + Duration::from_secs(60), + ) + .unwrap(); + + writer.append(&[event(1)]).unwrap(); + writer.flush_durable().unwrap(); + writer.rotate().unwrap(); +} diff --git a/trail/src/db/change_ledger/log/writer.rs b/trail/src/db/change_ledger/log/writer.rs new file mode 100644 index 0000000..f7584c0 --- /dev/null +++ b/trail/src/db/change_ledger/log/writer.rs @@ -0,0 +1,1255 @@ +use std::collections::VecDeque; +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use rusqlite::{params, Connection, OptionalExtension, TransactionBehavior}; +use sha2::{Digest, Sha256}; + +use super::codec::{encode_header, encode_record}; +use super::*; +use crate::db::util::{apply_sqlite_runtime_pragmas, now_ts}; + +#[cfg(test)] +thread_local! { + static APPEND_FLUSH_BOUNDARY_HOOK: std::cell::RefCell< + Option>, + > = const { std::cell::RefCell::new(None) }; +} + +#[cfg(test)] +pub(super) fn install_append_flush_boundary_hook(hook: impl FnOnce(&Path, &Path) + 'static) { + APPEND_FLUSH_BOUNDARY_HOOK.with(|slot| { + *slot.borrow_mut() = Some(Box::new(hook)); + }); +} + +#[cfg(test)] +fn run_append_flush_boundary_hook(workspace_db_dir: &Path, database_path: &Path) { + APPEND_FLUSH_BOUNDARY_HOOK.with(|slot| { + if let Some(hook) = slot.borrow_mut().take() { + hook(workspace_db_dir, database_path); + } + }); +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u8)] +pub(super) enum FaultPoint { + AppendWrite, + AppendPostWriteLeaseExpiry, + FileSync, + RotationOldSync, + SealPublication, + FirstDirectorySync, + NextHeaderCreate, + NextHeaderWrite, + NextHeaderSync, + SecondDirectorySync, + NextMetadataPublication, + Heartbeat, +} + +#[derive(Debug, Default)] +pub(super) struct FaultScript { + points: Mutex>, + max_batch_capacity: Mutex, +} + +impl FaultScript { + #[cfg(test)] + pub(super) fn new(points: impl IntoIterator) -> Self { + Self { + points: Mutex::new(points.into_iter().collect()), + max_batch_capacity: Mutex::new(0), + } + } + + fn check(&self, point: FaultPoint) -> std::io::Result<()> { + if self.take(point) { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + format!("injected observer I/O fault at {point:?}"), + )); + } + Ok(()) + } + + fn take(&self, point: FaultPoint) -> bool { + let mut points = self + .points + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if points.front() == Some(&point) { + points.pop_front(); + return true; + } + false + } + + fn observe_batch_capacity(&self, capacity: usize) { + let mut maximum = self + .max_batch_capacity + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + *maximum = (*maximum).max(capacity); + } + + #[cfg(test)] + pub(super) fn max_batch_capacity(&self) -> usize { + *self + .max_batch_capacity + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } +} + +pub(crate) struct SegmentWriter { + control: Connection, + workspace_db_dir: PathBuf, + database_path: PathBuf, + segment_directory: PathBuf, + file: File, + pub(super) path: PathBuf, + segment_id: String, + previous_segment_id: Option, + identity: SegmentIdentity, + provider_id: String, + limits: PersistedLogLimits, + lease_duration: Duration, + authorized: bool, + revoke_owner_on_drop: bool, + current_offset: u64, + /// Whether the currently open segment contains a record after its header. + /// `last_sequence` is scope-global and intentionally survives rotation, so + /// it cannot by itself distinguish a header-only segment. + current_segment_has_records: bool, + last_sequence: u64, + last_hash: [u8; 32], + last_cursor: Vec, + faults: Arc, +} + +#[derive(Clone, Debug)] +pub(crate) struct DaemonLaunchBinding { + pub(crate) nonce: String, + pub(crate) pid: u32, + pub(crate) process_start_identity: String, +} + +impl SegmentWriter { + pub(crate) fn bind_native_observer( + &mut self, + provider_identity: Vec, + fence_nonce: Vec, + ) -> Result { + if provider_identity.is_empty() || fence_nonce.len() < 16 { + return Err(Error::InvalidInput( + "native observer binding requires provider identity and an unguessable fence nonce" + .into(), + )); + } + if self.provider_id != hex::encode(&provider_identity) { + return Err(Error::InvalidInput( + "native observer provider identity does not match the acquired writer".into(), + )); + } + let _workspace_lock = crate::db::acquire_workspace_lock_for_database( + &self.workspace_db_dir, + &self.database_path, + )?; + self.ensure_authorized()?; + let transaction = self + .control + .transaction_with_behavior(TransactionBehavior::Immediate)?; + validate_lease_on(&transaction, &self.identity)?; + let owner_token = hex::encode(self.identity.owner_token); + let provider_identity_text = hex::encode(&provider_identity); + let owner_changed = transaction.execute( + "UPDATE changed_path_observer_owners + SET provider_identity=?1,fence_nonce=?2,updated_at=?3 + WHERE scope_id=?4 AND epoch=?5 AND owner_token=?6 + AND provider_id=?7 AND lease_state='active' AND expires_at>?3", + params![ + provider_identity_text, + fence_nonce, + now_ts(), + self.identity.scope_id.to_text(), + sql_i64(self.identity.epoch, "observer epoch")?, + owner_token, + self.provider_id, + ], + )?; + let scope_changed = transaction.execute( + "UPDATE changed_path_scopes + SET observer_owner_token=?1,updated_at=?2 + WHERE scope_id=?3 AND epoch=?4 AND provider_id=?5 + AND provider_identity=?6", + params![ + owner_token, + now_ts(), + self.identity.scope_id.to_text(), + sql_i64(self.identity.epoch, "observer epoch")?, + self.provider_id, + provider_identity_text, + ], + )?; + if owner_changed != 1 || scope_changed != 1 { + return Err(Error::WorkspaceLocked( + "native observer binding lost its exact writer lease".into(), + )); + } + transaction.commit()?; + // A writer becomes runtime authority only after its native provider + // identity and fence are durably bound. From this point onward an + // ordinary runtime shutdown must revoke that exact owner and force a + // full reconciliation before the scope can be trusted again. + self.revoke_owner_on_drop = true; + Ok(super::ObserverWriterBinding { + owner_token, + provider_id: self.provider_id.clone(), + provider_identity, + fence_nonce, + }) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn acquire( + database_path: &Path, + segment_directory: &Path, + scope_id: ScopeId, + epoch: u64, + owner_token: [u8; 32], + provider_id: &str, + provider_cursor: Vec, + lease_duration: Duration, + ) -> Result { + Self::acquire_inner( + database_path, + segment_directory, + scope_id, + epoch, + owner_token, + provider_id, + provider_cursor, + lease_duration, + None, + Arc::new(FaultScript::default()), + ) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn acquire_for_daemon( + database_path: &Path, + segment_directory: &Path, + scope_id: ScopeId, + epoch: u64, + owner_token: [u8; 32], + provider_id: &str, + provider_cursor: Vec, + lease_duration: Duration, + daemon_launch: DaemonLaunchBinding, + ) -> Result { + Self::acquire_inner( + database_path, + segment_directory, + scope_id, + epoch, + owner_token, + provider_id, + provider_cursor, + lease_duration, + Some(daemon_launch), + Arc::new(FaultScript::default()), + ) + } + + #[cfg(test)] + #[allow(clippy::too_many_arguments)] + pub(super) fn acquire_with_faults( + database_path: &Path, + segment_directory: &Path, + scope_id: ScopeId, + epoch: u64, + owner_token: [u8; 32], + provider_id: &str, + provider_cursor: Vec, + lease_duration: Duration, + faults: Arc, + ) -> Result { + Self::acquire_inner( + database_path, + segment_directory, + scope_id, + epoch, + owner_token, + provider_id, + provider_cursor, + lease_duration, + None, + faults, + ) + } + + #[allow(clippy::too_many_arguments)] + fn acquire_inner( + database_path: &Path, + segment_directory: &Path, + scope_id: ScopeId, + epoch: u64, + owner_token: [u8; 32], + provider_id: &str, + provider_cursor: Vec, + lease_duration: Duration, + daemon_launch: Option, + faults: Arc, + ) -> Result { + if epoch == 0 || provider_id.is_empty() || lease_duration.is_zero() { + return Err(Error::InvalidInput( + "observer lease requires positive epoch/duration and provider id".into(), + )); + } + let workspace_db_dir = workspace_db_dir_for_database(database_path)?; + let _workspace_lock = + crate::db::acquire_workspace_lock_for_database(&workspace_db_dir, database_path)?; + let mut control = Connection::open(database_path).map_err(|error| { + Error::DaemonUnavailable(format!( + "observer segment writer could not open its control connection: {error}" + )) + })?; + apply_sqlite_runtime_pragmas(&control).map_err(|error| { + Error::DaemonUnavailable(format!( + "observer segment writer could not configure its control connection: {error}" + )) + })?; + control + .busy_timeout(Duration::from_secs(5)) + .map_err(|error| { + Error::DaemonUnavailable(format!( + "observer segment writer could not configure its busy timeout: {error}" + )) + })?; + let now = now_ts(); + let expires_at = lease_expiry(now, lease_duration)?; + let scope_text = scope_id.to_text(); + let epoch_sql = sql_i64(epoch, "observer epoch")?; + let owner_text = hex::encode(owner_token); + let transaction = control.transaction_with_behavior(TransactionBehavior::Immediate)?; + let limits = transaction + .query_row( + "SELECT max_observer_log_bytes, max_segment_bytes, + max_unfolded_tail_records + FROM changed_path_scopes WHERE scope_id = ?1 AND epoch = ?2", + params![scope_text, epoch_sql], + |row| { + Ok(PersistedLogLimits { + max_log_bytes: row.get::<_, i64>(0)?.try_into().map_err(|_| { + rusqlite::Error::IntegralValueOutOfRange(0, row.get(0).unwrap_or(-1)) + })?, + max_segment_bytes: row.get::<_, i64>(1)?.try_into().map_err(|_| { + rusqlite::Error::IntegralValueOutOfRange(1, row.get(1).unwrap_or(-1)) + })?, + max_unfolded_tail_records: row.get::<_, i64>(2)?.try_into().map_err( + |_| { + rusqlite::Error::IntegralValueOutOfRange( + 2, + row.get(2).unwrap_or(-1), + ) + }, + )?, + }) + }, + ) + .optional() + .map_err(|error| { + Error::DaemonUnavailable(format!( + "observer segment writer could not read its persisted limits: {error}" + )) + })? + .ok_or_else(|| Error::InvalidInput("observer scope/epoch is stale".into()))?; + limits + .validate() + .map_err(|error| Error::Corrupt(error.to_string()))?; + let existing_owner = transaction + .query_row( + "SELECT epoch, owner_token, lease_state, expires_at + FROM changed_path_observer_owners WHERE scope_id=?1", + params![scope_text], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + )) + }, + ) + .optional() + .map_err(|error| { + Error::DaemonUnavailable(format!( + "observer segment writer could not read its existing owner: {error}" + )) + })?; + if let Some((owner_epoch, _token, state, owner_expiry)) = existing_owner { + let owner_epoch = u64::try_from(owner_epoch) + .map_err(|_| Error::Corrupt("negative observer owner epoch".into()))?; + if owner_epoch == epoch { + if state == "active" && owner_expiry > now { + return Err(Error::WorkspaceLocked( + "changed-path observer lease is already active".into(), + )); + } + return Err(Error::WorkspaceLocked( + "same-epoch observer owner replacement requires reconciliation and an authoritative epoch advance".into(), + )); + } + if owner_epoch > epoch { + return Err(Error::WorkspaceLocked( + "observer owner epoch is ahead of the requested epoch; reconciliation required" + .into(), + )); + } + } + let identity = SegmentIdentity { + scope_id, + epoch, + owner_token, + provider_cursor: provider_cursor.clone(), + previous_segment_hash: [0; 32], + }; + let header = encode_header(&identity).map_err(|error| Error::Corrupt(error.to_string()))?; + let header_len = header.len() as u64; + if header_len > limits.max_segment_bytes || header_len > limits.max_log_bytes { + return Err(Error::InvalidInput( + "observer segment header exceeds persisted byte cap".into(), + )); + } + let segment_id = segment_id(epoch, 1, owner_token); + let filename = segment_filename(&segment_id)?; + let path = segment_directory.join(&filename); + fs::create_dir_all(segment_directory)?; + sync_directory(segment_directory)?; + let mut file = OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&path)?; + super::super::secure_fs::lock_observer_writer(&file)?; + file.write_all(&header)?; + file.sync_all()?; + sync_directory(segment_directory)?; + let publish_now = now_ts(); + if publish_now >= expires_at { + return Err(Error::WorkspaceLocked( + "observer lease expired before initial publication".into(), + )); + } + transaction.execute( + "INSERT INTO changed_path_observer_owners( + scope_id, epoch, owner_token, provider_id, provider_identity, + lease_state, fence_nonce, acquired_at, heartbeat_at, expires_at, + error_state, error_at, updated_at + ) VALUES(?1, ?2, ?3, ?4, '', 'active', NULL, ?5, ?5, ?6, NULL, NULL, ?5) + ON CONFLICT(scope_id) DO UPDATE SET + epoch=excluded.epoch, owner_token=excluded.owner_token, + provider_id=excluded.provider_id, provider_identity=excluded.provider_identity, + lease_state='active', fence_nonce=NULL, acquired_at=excluded.acquired_at, + heartbeat_at=excluded.heartbeat_at, expires_at=excluded.expires_at, + error_state=NULL, error_at=NULL, + daemon_launch_nonce=NULL, daemon_pid=NULL, + daemon_process_start_identity=NULL, + updated_at=excluded.updated_at", + params![ + scope_text, + epoch_sql, + owner_text, + provider_id, + now, + expires_at + ], + )?; + if let Some(daemon_launch) = &daemon_launch { + if daemon_launch.nonce.len() != 64 + || daemon_launch.pid == 0 + || daemon_launch.process_start_identity.is_empty() + { + return Err(Error::InvalidInput( + "daemon observer launch binding is malformed".into(), + )); + } + let bound = transaction.execute( + "UPDATE changed_path_observer_owners + SET daemon_launch_nonce=?1,daemon_pid=?2,daemon_process_start_identity=?3 + WHERE scope_id=?4 AND epoch=?5 AND owner_token=?6 AND lease_state='active'", + params![ + daemon_launch.nonce, + i64::from(daemon_launch.pid), + daemon_launch.process_start_identity, + scope_text, + epoch_sql, + owner_text, + ], + )?; + if bound != 1 { + return Err(Error::WorkspaceLocked( + "daemon observer owner launch binding lost exact authority".into(), + )); + } + } + let inserted = transaction.execute( + "INSERT INTO changed_path_observer_segments( + scope_id, epoch, segment_id, log_format_version, owner_token, + provider_id, first_sequence, last_sequence, durable_end_offset, + folded_end_offset, previous_segment_id, previous_segment_hash, + segment_hash, segment_path, state, created_at, sealed_at, updated_at + ) SELECT ?1, ?2, ?3, 1, ?4, ?5, 1, NULL, ?6, 0, + NULL, NULL, NULL, ?7, 'open', ?8, NULL, ?8 + WHERE EXISTS( + SELECT 1 FROM changed_path_observer_owners + WHERE scope_id=?1 AND epoch=?2 AND owner_token=?4 + AND lease_state='active' AND expires_at>?8 + )", + params![ + scope_text, + epoch_sql, + segment_id, + owner_text, + provider_id, + sql_i64(header_len, "durable observer header offset")?, + filename, + publish_now + ], + )?; + if inserted != 1 { + return Err(Error::WorkspaceLocked( + "observer lease changed before publication".into(), + )); + } + transaction.commit()?; + let current_offset = header_len; + Ok(Self { + control, + workspace_db_dir, + database_path: database_path.to_path_buf(), + segment_directory: segment_directory.to_path_buf(), + file, + path, + segment_id, + previous_segment_id: None, + identity, + provider_id: provider_id.to_owned(), + limits, + lease_duration, + authorized: true, + revoke_owner_on_drop: daemon_launch.is_some(), + current_offset, + current_segment_has_records: false, + last_sequence: 0, + last_hash: [0; 32], + last_cursor: provider_cursor, + faults, + }) + } + + pub(crate) fn append(&mut self, records: &[ObserverRecord]) -> Result<()> { + let result = match crate::db::acquire_workspace_lock_for_observer( + &self.workspace_db_dir, + &self.database_path, + ) { + Ok(_workspace_lock) => self.append_inner(records), + Err(error) => Err(error), + }; + if result.is_err() { + self.retire("append_failed"); + } + result + } + + /// Append and publish one durable batch without exposing the + /// file-before-SQLite window to a command workspace lock holder. + pub(crate) fn append_and_flush(&mut self, records: &[ObserverRecord]) -> Result { + let result = (|| { + let _workspace_lock = crate::db::acquire_workspace_lock_for_observer( + &self.workspace_db_dir, + &self.database_path, + )?; + self.append_inner(records)?; + #[cfg(test)] + run_append_flush_boundary_hook(&self.workspace_db_dir, &self.database_path); + self.flush_inner() + })(); + if result.is_err() { + self.retire("append_and_flush_failed"); + } + result + } + + /// Append and publish an authenticated boundary record, then rotate that + /// exact durable cut while retaining one observer workspace lock. The + /// native durability worker calls this as one queue operation, so records + /// ordered after the boundary cannot advance the segment before it seals. + pub(crate) fn append_flush_and_rotate( + &mut self, + records: &[ObserverRecord], + ) -> Result<(DurableCut, DurableCut)> { + let result = (|| { + let _workspace_lock = crate::db::acquire_workspace_lock_for_observer( + &self.workspace_db_dir, + &self.database_path, + )?; + self.append_inner(records)?; + let sealed = self.flush_inner()?; + self.rotate_inner()?; + let anchor = DurableCut { + segment_id: self.segment_id.clone(), + durable_end_offset: self.current_offset, + last_sequence: self.last_sequence, + last_hash: self.last_hash, + provider_cursor: self.last_cursor.clone(), + }; + Ok((sealed, anchor)) + })(); + if result.is_err() { + self.retire("append_flush_and_rotate_failed"); + } + result + } + + fn append_inner(&mut self, records: &[ObserverRecord]) -> Result<()> { + self.ensure_authorized()?; + let transaction = self + .control + .transaction_with_behavior(TransactionBehavior::Immediate)?; + validate_lease_on(&transaction, &self.identity)?; + if records.is_empty() { + transaction.commit()?; + return Ok(()); + } + let other_durable = transaction.query_row( + "SELECT COALESCE(SUM(durable_end_offset), 0) + FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=?2 AND segment_id<>?3", + params![ + self.identity.scope_id.to_text(), + sql_i64(self.identity.epoch, "observer epoch")?, + self.segment_id, + ], + |row| row.get::<_, i64>(0), + )?; + let other_durable = u64::try_from(other_durable) + .map_err(|_| Error::Corrupt("negative observer log byte total".into()))?; + let segment_remaining = self + .limits + .max_segment_bytes + .checked_sub(self.current_offset) + .ok_or_else(|| Error::Corrupt("observer segment already exceeds byte cap".into()))?; + let current_total = other_durable + .checked_add(self.current_offset) + .ok_or_else(|| Error::InvalidInput("observer log byte total overflow".into()))?; + let log_remaining = self + .limits + .max_log_bytes + .checked_sub(current_total) + .ok_or_else(|| Error::Corrupt("observer log already exceeds byte cap".into()))?; + let remaining = segment_remaining.min(log_remaining); + let remaining_usize = usize::try_from(remaining).map_err(|_| { + Error::InvalidInput("observer append capacity cannot fit memory".into()) + })?; + let mut batch = Vec::new(); + let mut sequence = self.last_sequence; + let mut hash = self.last_hash; + for record in records { + if record.sequence != sequence.saturating_add(1) { + return Err(Error::InvalidInput( + "observer append sequence is not exactly monotonic".into(), + )); + } + let (encoded, next_hash) = encode_record(record, hash) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + let next_batch_len = batch + .len() + .checked_add(encoded.len()) + .ok_or_else(|| Error::InvalidInput("observer append batch overflow".into()))?; + if next_batch_len > remaining_usize { + return Err(Error::InvalidInput( + "observer append batch exceeds persisted remaining byte cap".into(), + )); + } + batch.try_reserve_exact(encoded.len()).map_err(|_| { + Error::InvalidInput("observer append batch allocation failed".into()) + })?; + if batch.capacity() > remaining_usize { + return Err(Error::InvalidInput( + "observer append batch allocation exceeds persisted remaining byte cap".into(), + )); + } + batch.extend_from_slice(&encoded); + self.faults.observe_batch_capacity(batch.capacity()); + sequence = record.sequence; + hash = next_hash; + } + let next_offset = self + .current_offset + .checked_add(batch.len() as u64) + .ok_or_else(|| Error::InvalidInput("observer segment offset overflow".into()))?; + validate_lease_on(&transaction, &self.identity)?; + self.faults.check(FaultPoint::AppendWrite)?; + self.file.write_all(&batch)?; + if self.faults.take(FaultPoint::AppendPostWriteLeaseExpiry) { + transaction.execute( + "UPDATE changed_path_observer_owners + SET expires_at=heartbeat_at + WHERE scope_id=?1 AND epoch=?2 AND owner_token=?3", + params![ + self.identity.scope_id.to_text(), + sql_i64(self.identity.epoch, "observer epoch")?, + hex::encode(self.identity.owner_token) + ], + )?; + } + validate_lease_on(&transaction, &self.identity)?; + transaction.commit()?; + self.current_offset = next_offset; + self.current_segment_has_records = true; + self.last_sequence = sequence; + self.last_hash = hash; + self.last_cursor = records.last().unwrap().provider_cursor.clone(); + Ok(()) + } + + pub(crate) fn flush_durable(&mut self) -> Result { + let result = match crate::db::acquire_workspace_lock_for_observer( + &self.workspace_db_dir, + &self.database_path, + ) { + Ok(_workspace_lock) => self.flush_inner(), + Err(error) => Err(error), + }; + if result.is_err() { + self.retire("flush_failed"); + } + result + } + + fn flush_inner(&mut self) -> Result { + self.ensure_authorized()?; + let transaction = self + .control + .transaction_with_behavior(TransactionBehavior::Immediate)?; + validate_lease_on(&transaction, &self.identity)?; + self.faults.check(FaultPoint::FileSync)?; + self.file.sync_data()?; + if self.file.metadata()?.len() != self.current_offset { + return Err(Error::Corrupt( + "observer segment length differs from the claimed durable offset".into(), + )); + } + validate_lease_on(&transaction, &self.identity)?; + let changed = transaction.execute( + "UPDATE changed_path_observer_segments + SET durable_end_offset=?1, last_sequence=?2, updated_at=?3 + WHERE scope_id=?4 AND epoch=?5 AND segment_id=?6 AND owner_token=?7 + AND state='open' + AND EXISTS( + SELECT 1 FROM changed_path_observer_owners + WHERE scope_id=?4 AND epoch=?5 AND owner_token=?7 + AND lease_state='active' AND expires_at>?3 + )", + params![ + sql_i64(self.current_offset, "durable observer offset")?, + if !self.current_segment_has_records { + None + } else { + Some(sql_i64(self.last_sequence, "observer sequence")?) + }, + now_ts(), + self.identity.scope_id.to_text(), + sql_i64(self.identity.epoch, "observer epoch")?, + self.segment_id, + hex::encode(self.identity.owner_token), + ], + )?; + if changed != 1 { + return Err(Error::WorkspaceLocked( + "observer lease changed before durable publication".into(), + )); + } + transaction.commit()?; + Ok(DurableCut { + segment_id: self.segment_id.clone(), + durable_end_offset: self.current_offset, + last_sequence: self.last_sequence, + last_hash: self.last_hash, + provider_cursor: self.last_cursor.clone(), + }) + } + + pub(crate) fn heartbeat(&mut self) -> Result<()> { + // Lease renewal publishes no filesystem evidence and changes no + // segment boundary. It must remain live while an ordinary command + // holds the high-level workspace lock for a long materialization or + // checkpoint. SQLite serialization plus the exact owner CAS below are + // the authority boundary for this owner-only heartbeat row. + let result = self.heartbeat_inner(); + if result.is_err() { + self.retire("heartbeat_failed"); + } + result + } + + /// Durably revokes the exact active observer owner after its terminal + /// invalidation marker has been flushed. + pub(crate) fn revoke(&mut self, reason: &str) -> Result<()> { + self.ensure_authorized()?; + let _workspace_lock = crate::db::acquire_workspace_lock_for_observer( + &self.workspace_db_dir, + &self.database_path, + )?; + let transaction = self + .control + .transaction_with_behavior(TransactionBehavior::Immediate)?; + validate_lease_on(&transaction, &self.identity)?; + let changed = transaction.execute( + "UPDATE changed_path_observer_owners + SET lease_state='error', error_state=?1, error_at=?2, updated_at=?2, + expires_at=?2 + WHERE scope_id=?3 AND epoch=?4 AND owner_token=?5 + AND lease_state='active'", + params![ + reason, + now_ts(), + self.identity.scope_id.to_text(), + sql_i64(self.identity.epoch, "observer epoch")?, + hex::encode(self.identity.owner_token), + ], + )?; + if changed != 1 { + return Err(Error::WorkspaceLocked( + "observer owner changed before durable revocation".into(), + )); + } + transaction.commit()?; + self.authorized = false; + Ok(()) + } + + fn heartbeat_inner(&mut self) -> Result<()> { + self.validate_lease()?; + self.faults.check(FaultPoint::Heartbeat)?; + let now = now_ts(); + let expiry = lease_expiry(now, self.lease_duration)?; + let changed = self.control.execute( + "UPDATE changed_path_observer_owners + SET heartbeat_at=?1, expires_at=?2, updated_at=?1 + WHERE scope_id=?3 AND epoch=?4 AND owner_token=?5 + AND lease_state='active' AND expires_at>?1", + params![ + now, + expiry, + self.identity.scope_id.to_text(), + sql_i64(self.identity.epoch, "observer epoch")?, + hex::encode(self.identity.owner_token) + ], + )?; + if changed != 1 { + return Err(Error::WorkspaceLocked( + "observer heartbeat lost its lease".into(), + )); + } + Ok(()) + } + + pub(crate) fn rotate(&mut self) -> Result<()> { + self.rotate_and_cut().map(|_| ()) + } + + /// Seal the current append-only segment and return the exact durable cut + /// authenticated by that segment. Controlled filesystem producers use + /// this to prove that an intent interval starts and ends on unambiguous + /// sidecar boundaries. + pub(crate) fn rotate_and_cut(&mut self) -> Result { + self.rotate_and_cuts().map(|(sealed, _anchor)| sealed) + } + + pub(crate) fn rotate_and_cuts(&mut self) -> Result<(DurableCut, DurableCut)> { + let sealed = DurableCut { + segment_id: self.segment_id.clone(), + durable_end_offset: self.current_offset, + last_sequence: self.last_sequence, + last_hash: self.last_hash, + provider_cursor: self.last_cursor.clone(), + }; + let result = match crate::db::acquire_workspace_lock_for_observer( + &self.workspace_db_dir, + &self.database_path, + ) { + Ok(_workspace_lock) => self.rotate_inner().map(|_| { + let anchor = DurableCut { + segment_id: self.segment_id.clone(), + durable_end_offset: self.current_offset, + last_sequence: self.last_sequence, + last_hash: self.last_hash, + provider_cursor: self.last_cursor.clone(), + }; + (sealed, anchor) + }), + Err(error) => Err(error), + }; + if result.is_err() { + self.retire("rotation_failed"); + } + result + } + + pub(crate) fn rotate_if_cut( + &mut self, + expected: &DurableCut, + ) -> Result> { + let current = DurableCut { + segment_id: self.segment_id.clone(), + durable_end_offset: self.current_offset, + last_sequence: self.last_sequence, + last_hash: self.last_hash, + provider_cursor: self.last_cursor.clone(), + }; + if ¤t != expected { + return Ok(None); + } + self.rotate_and_cuts().map(Some) + } + + fn rotate_inner(&mut self) -> Result<()> { + self.ensure_authorized()?; + let transaction = self + .control + .transaction_with_behavior(TransactionBehavior::Immediate)?; + validate_lease_on(&transaction, &self.identity)?; + self.faults.check(FaultPoint::RotationOldSync)?; + self.file.sync_data()?; + let bytes = fs::read(&self.path)?; + if bytes.len() as u64 != self.current_offset { + return Err(Error::Corrupt( + "observer segment length changed during rotation".into(), + )); + } + // A header-only segment already represents an exact durable boundary. + // Rotating it cannot advance `first_sequence`, and both the segment-id + // derivation and schema intentionally reject a second segment for the + // same first sequence. Keep the existing open anchor after rechecking + // its durability and owner authority. + if !self.current_segment_has_records { + validate_lease_on(&transaction, &self.identity)?; + transaction.commit()?; + return Ok(()); + } + let old_segment_hash: [u8; 32] = Sha256::digest(&bytes).into(); + self.faults.check(FaultPoint::FirstDirectorySync)?; + sync_directory(&self.segment_directory)?; + + let first_sequence = self.last_sequence.saturating_add(1).max(1); + let next_segment_id = segment_id( + self.identity.epoch, + first_sequence, + self.identity.owner_token, + ); + let next_path = self + .segment_directory + .join(segment_filename(&next_segment_id)?); + let next_identity = SegmentIdentity { + scope_id: self.identity.scope_id, + epoch: self.identity.epoch, + owner_token: self.identity.owner_token, + provider_cursor: self.last_cursor.clone(), + previous_segment_hash: old_segment_hash, + }; + let header = + encode_header(&next_identity).map_err(|error| Error::Corrupt(error.to_string()))?; + let header_len = header.len() as u64; + if header_len > self.limits.max_segment_bytes { + return Err(Error::InvalidInput( + "observer rotation header exceeds segment byte cap".into(), + )); + } + let other_durable = transaction.query_row( + "SELECT COALESCE(SUM(durable_end_offset), 0) + FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=?2 AND segment_id<>?3", + params![ + self.identity.scope_id.to_text(), + sql_i64(self.identity.epoch, "observer epoch")?, + self.segment_id, + ], + |row| row.get::<_, i64>(0), + )?; + let total_after_rotation = u64::try_from(other_durable) + .map_err(|_| Error::Corrupt("negative observer log byte total".into()))? + .checked_add(self.current_offset) + .and_then(|total| total.checked_add(header_len)) + .ok_or_else(|| Error::InvalidInput("observer log byte total overflow".into()))?; + if total_after_rotation > self.limits.max_log_bytes { + return Err(Error::InvalidInput( + "observer rotation header exceeds total log byte cap".into(), + )); + } + self.faults.check(FaultPoint::NextHeaderCreate)?; + let mut next_file = OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&next_path)?; + super::super::secure_fs::lock_observer_writer(&next_file)?; + self.faults.check(FaultPoint::NextHeaderWrite)?; + next_file.write_all(&header)?; + self.faults.check(FaultPoint::NextHeaderSync)?; + next_file.sync_all()?; + self.faults.check(FaultPoint::SecondDirectorySync)?; + sync_directory(&self.segment_directory)?; + validate_lease_on(&transaction, &self.identity)?; + let now = now_ts(); + self.faults.check(FaultPoint::SealPublication)?; + let sealed = transaction.execute( + "UPDATE changed_path_observer_segments + SET state='sealed', last_sequence=?1, durable_end_offset=?2, + segment_hash=?3, sealed_at=?4, updated_at=?4 + WHERE scope_id=?5 AND epoch=?6 AND segment_id=?7 AND owner_token=?8 + AND state='open' + AND EXISTS( + SELECT 1 FROM changed_path_observer_owners + WHERE scope_id=?5 AND epoch=?6 AND owner_token=?8 + AND lease_state='active' AND expires_at>?4 + )", + params![ + if !self.current_segment_has_records { + None + } else { + Some(sql_i64(self.last_sequence, "observer sequence")?) + }, + sql_i64(self.current_offset, "durable observer offset")?, + hex::encode(old_segment_hash), + now, + self.identity.scope_id.to_text(), + sql_i64(self.identity.epoch, "observer epoch")?, + self.segment_id, + hex::encode(self.identity.owner_token), + ], + )?; + if sealed != 1 { + return Err(Error::WorkspaceLocked( + "observer lease changed before sealing".into(), + )); + } + self.faults.check(FaultPoint::NextMetadataPublication)?; + let next_filename = segment_filename(&next_segment_id)?; + let published = transaction.execute( + "INSERT INTO changed_path_observer_segments( + scope_id, epoch, segment_id, log_format_version, owner_token, + provider_id, first_sequence, last_sequence, durable_end_offset, + folded_end_offset, previous_segment_id, previous_segment_hash, + segment_hash, segment_path, state, created_at, sealed_at, updated_at + ) SELECT ?1, ?2, ?3, 1, ?4, ?5, ?6, NULL, ?7, 0, + ?8, ?9, NULL, ?10, 'open', ?11, NULL, ?11 + WHERE EXISTS( + SELECT 1 FROM changed_path_observer_owners + WHERE scope_id=?1 AND epoch=?2 AND owner_token=?4 + AND lease_state='active' AND expires_at>?11 + )", + params![ + self.identity.scope_id.to_text(), + sql_i64(self.identity.epoch, "observer epoch")?, + next_segment_id, + hex::encode(self.identity.owner_token), + self.provider_id, + sql_i64(first_sequence, "observer sequence")?, + sql_i64(header_len, "durable observer header offset")?, + self.segment_id, + hex::encode(old_segment_hash), + next_filename, + now, + ], + )?; + if published != 1 { + return Err(Error::WorkspaceLocked( + "observer lease changed before rotation publication".into(), + )); + } + transaction.commit()?; + self.file = next_file; + self.path = next_path; + self.previous_segment_id = Some(self.segment_id.clone()); + self.segment_id = next_segment_id; + self.identity = next_identity; + self.current_offset = header_len; + self.current_segment_has_records = false; + self.last_hash = [0; 32]; + Ok(()) + } + + fn ensure_authorized(&self) -> Result<()> { + if !self.authorized { + return Err(Error::WorkspaceLocked("observer writer is retired".into())); + } + Ok(()) + } + + fn validate_lease(&self) -> Result<()> { + self.ensure_authorized()?; + validate_lease_on(&self.control, &self.identity) + } + + fn retire(&mut self, reason: &str) { + if self.authorized { + self.authorized = false; + if let Ok(_workspace_lock) = crate::db::acquire_workspace_lock_for_observer( + &self.workspace_db_dir, + &self.database_path, + ) { + revoke_owner( + &self.control, + &self.identity.scope_id.to_text(), + self.identity.epoch, + &hex::encode(self.identity.owner_token), + reason, + ); + } + } + } + + #[cfg(test)] + pub(super) fn is_authorized(&self) -> bool { + self.authorized + } + + #[cfg(test)] + pub(super) fn runtime_pragmas(&self) -> (String, i64, i64, i64) { + ( + self.control + .query_row("PRAGMA journal_mode", [], |row| row.get(0)) + .unwrap(), + self.control + .query_row("PRAGMA foreign_keys", [], |row| row.get(0)) + .unwrap(), + self.control + .query_row("PRAGMA synchronous", [], |row| row.get(0)) + .unwrap(), + self.control + .query_row("PRAGMA temp_store", [], |row| row.get(0)) + .unwrap(), + ) + } +} + +impl Drop for SegmentWriter { + fn drop(&mut self) { + if self.revoke_owner_on_drop { + self.retire("observer_writer_dropped"); + } + } +} + +fn workspace_db_dir_for_database(database_path: &Path) -> Result { + let parent = database_path.parent().ok_or_else(|| { + Error::InvalidInput("observer database path has no workspace directory".into()) + })?; + if parent.file_name().is_some_and(|name| name == "index") { + return parent.parent().map(Path::to_path_buf).ok_or_else(|| { + Error::InvalidInput("observer database index has no workspace directory".into()) + }); + } + Ok(parent.to_path_buf()) +} + +fn validate_lease_on(connection: &Connection, identity: &SegmentIdentity) -> Result<()> { + let exists = connection.query_row( + "SELECT EXISTS( + SELECT 1 FROM changed_path_observer_owners owner + JOIN changed_path_scopes scope ON scope.scope_id=owner.scope_id + WHERE owner.scope_id=?1 AND owner.epoch=?2 AND scope.epoch=?2 + AND owner.owner_token=?3 AND owner.lease_state='active' + AND owner.expires_at>?4 + )", + params![ + identity.scope_id.to_text(), + sql_i64(identity.epoch, "observer epoch")?, + hex::encode(identity.owner_token), + now_ts() + ], + |row| row.get::<_, bool>(0), + )?; + if !exists { + return Err(Error::WorkspaceLocked( + "observer lease is stale or expired".into(), + )); + } + Ok(()) +} + +fn lease_expiry(now: i64, duration: Duration) -> Result { + let seconds = i64::try_from(duration.as_secs().max(1)) + .map_err(|_| Error::InvalidInput("observer lease duration exceeds range".into()))?; + now.checked_add(seconds) + .ok_or_else(|| Error::InvalidInput("observer lease expiry overflow".into())) +} + +pub(super) fn segment_id(epoch: u64, first_sequence: u64, owner_token: [u8; 32]) -> String { + format!( + "{epoch:020}-{first_sequence:020}-{}", + hex::encode(owner_token) + ) +} + +pub(super) fn segment_filename(segment_id: &str) -> Result { + let bytes = segment_id.as_bytes(); + let epoch = std::str::from_utf8(bytes.get(..20).unwrap_or_default()) + .ok() + .and_then(|value| value.parse::().ok()); + let first_sequence = std::str::from_utf8(bytes.get(21..41).unwrap_or_default()) + .ok() + .and_then(|value| value.parse::().ok()); + let valid = bytes.len() == 106 + && bytes[..20].iter().all(u8::is_ascii_digit) + && bytes[20] == b'-' + && bytes[21..41].iter().all(u8::is_ascii_digit) + && bytes[41] == b'-' + && bytes[42..] + .iter() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte)) + && epoch.is_some_and(|epoch| epoch > 0 && format!("{epoch:020}") == segment_id[..20]) + && first_sequence.is_some_and(|sequence| { + sequence > 0 && format!("{sequence:020}") == segment_id[21..41] + }); + if !valid { + return Err(Error::Corrupt("invalid observer segment id".into())); + } + let filename = format!("{segment_id}.cpl"); + if filename.len() > MAX_SEGMENT_FILENAME_BYTES { + return Err(Error::Corrupt( + "observer segment filename exceeds bound".into(), + )); + } + Ok(filename) +} + +pub(super) fn sync_directory(path: &Path) -> std::io::Result<()> { + File::open(path)?.sync_all() +} + +fn revoke_owner( + connection: &Connection, + scope_id: &str, + epoch: u64, + owner_token: &str, + reason: &str, +) { + let now = now_ts(); + let Ok(epoch) = i64::try_from(epoch) else { + return; + }; + let _ = connection.execute( + "UPDATE changed_path_observer_owners + SET lease_state='error', error_state=?1, error_at=?2, updated_at=?2 + WHERE scope_id=?3 AND epoch=?4 AND owner_token=?5 AND lease_state='active'", + params![reason, now, scope_id, epoch, owner_token], + ); +} diff --git a/trail/src/db/change_ledger/mod.rs b/trail/src/db/change_ledger/mod.rs new file mode 100644 index 0000000..3bb6775 --- /dev/null +++ b/trail/src/db/change_ledger/mod.rs @@ -0,0 +1,158 @@ +// This module is intentionally dormant until the final activation task wires +// authoritative readers and native producers to it. +#![allow(dead_code)] + +mod activation; +mod daemon; +mod intent; +mod log; +mod observer; +mod policy; +mod projection; +mod reconcile; +mod recovery; +pub(crate) mod secure_fs; +mod snapshot; +mod store; +mod types; + +#[allow(unused_imports)] +pub(crate) use activation::{ledger_authority_enabled_for, ActivationEvidence}; + +#[allow(unused_imports)] +pub(crate) use snapshot::{ + command_authority_enabled, CandidateComparison, CandidateMaterialization, + FencedCandidateSnapshot, ObservedRecordCut, LEDGER_AUTHORITY_ENABLED, +}; +#[cfg(debug_assertions)] +#[allow(unused_imports)] +pub(crate) use snapshot::{ + run_command_flow, run_command_long_lock_flow, run_materialized_candidate_lifecycle_flow, + run_materialized_lane_snapshot_flow, set_command_authority_override, +}; + +#[allow(unused_imports)] +pub(crate) use daemon::{ + accept_materialized_lane_daemon_baseline, materialized_lane_daemon_expected_scope, + materialized_lane_daemon_fence, materialized_lane_daemon_full_reconcile, + materialized_lane_daemon_marker_cut, materialized_lane_daemon_matches_target, + materialized_lane_daemon_ready_proof, materialized_lane_daemon_reconcile, + materialized_lane_scope_id, policy_runtime_restart_required, + prepare_materialized_lane_controlled_projection, prepare_materialized_lane_daemon, + prepare_workspace_controlled_projection, prepare_workspace_daemon, + prepare_workspace_daemon_launch, prepare_workspace_daemon_verified_replacement, + verified_stale_workspace_owner_for_launch, with_materialized_lane_controlled_interval, + with_workspace_controlled_interval, workspace_daemon_fence, workspace_daemon_full_reconcile, + workspace_daemon_ready_proof, workspace_daemon_reconcile, ChangedPathDaemonRegistry, + VerifiedStaleWorkspaceOwner, WorkspaceDaemonLaunchIdentity, WorkspaceDaemonProof, + WorkspaceDaemonRuntime, +}; +#[allow(unused_imports)] +pub(crate) use intent::{ + mark_filesystem_applied, prepare_intent, publish_intent, IntentEvidence, IntentId, + IntentProducer, IntentState, IntentTarget, QualifiedFilesystemProof, +}; +#[allow(unused_imports)] +pub(crate) use projection::{ + commit_lane_operation_atomic, publish_alignment_in_transaction, + publish_ref_advancing_projection, run_projection_alignment, run_ref_advancing_projection, + ProjectionAlignmentMode, ProjectionPublication, RefAdvancingProjectionMode, + WorkspaceViewCheckpointPublication, +}; + +#[allow(unused_imports)] +pub(crate) use log::{ + recover_segments_from_connection, recover_segments_from_directory, AuthenticatedSegment, + DaemonLaunchBinding, DurableCut, ObserverRecord, ObserverWriterBinding, PersistedLogLimits, + RecoveredTail, RecoveryError, RecoveryScope, SegmentWriter, +}; +#[cfg(all(debug_assertions, target_os = "linux"))] +pub(crate) use observer::linux::{ + run_authenticated_fence_rejections, run_complete_prefix_publication_races, + run_content_mode_create_delete, run_controlled_fence_queue_ordering, run_delayed_backlog, + run_fault_revocation_matrix, run_fence_ordering, run_owner_death_and_root_replacement, + run_policy_dependency_observation, run_process_owner_child, run_raw_decoder_faults, + run_reconciliation_interval_qualification, run_recursive_coverage, run_rename_matrix, + run_rename_storm_and_cookie_expiry, run_segment_writer_reconcile_publication, + run_unsupported_filesystem_rejection, +}; +#[cfg(all(debug_assertions, target_os = "macos"))] +pub(crate) use observer::macos::{ + run_continuity_fault_matrix as run_macos_continuity_fault_matrix, + run_fence_ordering as run_macos_fence_ordering, + run_gap_flag_matrix as run_macos_gap_flag_matrix, + run_history_authority as run_macos_history_authority, + run_malformed_callbacks as run_macos_malformed_callbacks, + run_null_context_generation as run_macos_null_context_generation, + run_paused_callback_fence as run_macos_paused_callback_fence, + run_real_apfs_file_events as run_macos_real_apfs_file_events, + run_root_revalidation_failures as run_macos_root_revalidation_failures, + run_startup_cancellation as run_macos_startup_cancellation, + run_unsupported_filesystem_rejection as run_macos_unsupported_filesystem_rejection, + run_uuid_revalidation as run_macos_uuid_revalidation, +}; +#[allow(unused_imports)] +pub(crate) use observer::{select_observer, ObserverFence, ObserverLease, QualifiedObserver}; +#[allow(unused_imports)] +pub(crate) use policy::{ + capture_policy_dependency_fence_identity, + capture_policy_dependency_fence_identity_at_observed_path, compile_policy, + raw_event_invalidates_policy, raw_path_may_invalidate_policy, revalidate_compiled_policy, + validate_policy_manifest, AdapterEquivalence, CompiledPolicy, CompiledRecordingMatcher, + PolicyCompileContext, PolicyDependency, PolicyDependencyFenceIdentity, PolicyDependencyKind, + PolicyDependencyMetrics, PolicyInvalidationIndex, PolicyManifest, PolicyManifestValidation, + RecordingPolicySnapshot, +}; +#[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] +#[allow(unused_imports)] +pub(crate) use reconcile::install_initial_scan_hook; +#[allow(unused_imports)] +pub(crate) use reconcile::{ + begin_reconciliation, fold_observer_interval, persisted_proven_prefixes, reconcile_full, + reconcile_full_with_tail, ObserverEvent, ProvenPrefixSet, ReconcileMode, ReconciliationAttempt, +}; +#[cfg(debug_assertions)] +pub(crate) use reconcile::{run_callback_spool, run_oracle, run_races}; +#[cfg(all(debug_assertions, unix))] +pub(crate) use recovery::run_non_utf_database_path_mark_recover_and_retire; +#[allow(unused_imports)] +pub(crate) use recovery::{ + ledger_gc_roots, mark_backup_scopes_untrusted, recover_scope, remove_retired_segments, + retire_deletion_scopes, retire_scope, rotate_restored_scopes, IntentGcRoot, RecoveryDecision, + SegmentDeletionToken, +}; +#[cfg(debug_assertions)] +pub(crate) use recovery::{ + run_acknowledgement_race, run_advanced_prefix_recovery, run_ambiguous_recovery_gate, + run_backup_overwrite_rollback, run_backup_restore_rotation, run_crash_matrix, + run_deletion_normal_retry_idempotence, run_deletion_parent_substitution_rejection, + run_deletion_post_quarantine_verification_substitution_rejection, + run_deletion_post_verification_substitution_rejection, + run_deletion_quiesced_missing_quarantine_rejection, + run_deletion_quiesced_reappeared_original_rejection, + run_deletion_retry_hostile_quarantine_replacement_rejection, + run_exact_interval_bridge_rejection, run_gc_root_lifecycle, run_lane_deletion_retirement, + run_missing_sidecar_rejection, run_prefix_interval_bridge_rejection, + run_qualified_proof_revalidation, run_restored_nullable_provider_lane_deletion, + run_retained_writer_quiescence, run_retirement_barrier, run_valid_prefix_interval_recovery, +}; +#[cfg(all(debug_assertions, unix))] +pub(crate) use recovery::{ + run_deletion_leaf_substitution_rejection, run_mark_ancestor_substitution_rejection, + run_recovery_ancestor_substitution_rejection, +}; +#[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] +pub(crate) use recovery::{ + run_empty_orphan_quarantine_rejection, run_no_orphan_quarantine_allocation, + run_orphan_quarantine_substitution_rejection, +}; +#[allow(unused_imports)] +pub(crate) use store::ChangedPathLedger; +#[allow(unused_imports)] +pub(crate) use types::{ + BaselineIdentity, CandidateSnapshot, DirtyPrefix, EvidenceAcknowledgementToken, EvidenceCut, + EvidenceFlags, EvidenceRowKind, EvidenceSource, ExpectedScope, FilesystemIdentity, + GitEvidenceQualification, GitStructuralMetrics, LedgerPath, OwnedEvidence, PolicyIdentity, + ProviderCapabilities, ProviderIdentity, QualifiedGitCandidates, ScopeId, ScopeIdentity, + ScopeKind, TrustState, +}; diff --git a/trail/src/db/change_ledger/observer/linux.rs b/trail/src/db/change_ledger/observer/linux.rs new file mode 100644 index 0000000..f3ac56d --- /dev/null +++ b/trail/src/db/change_ledger/observer/linux.rs @@ -0,0 +1,3370 @@ +//! Qualified Linux inotify observer. + +use std::collections::{BTreeSet, HashMap}; +use std::ffi::OsStr; +#[cfg(debug_assertions)] +use std::ffi::OsString; +use std::fs::{self, File, OpenOptions}; +use std::io::{ErrorKind, Write}; +use std::os::fd::AsRawFd; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Component, Path, PathBuf}; +#[cfg(debug_assertions)] +use std::sync::atomic::AtomicU64; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{self, Receiver, SyncSender, TrySendError}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use inotify::{EventMask, Inotify, WatchDescriptor, WatchMask}; +use rustix::fs::{fstat, fsync, openat, unlinkat, AtFlags, Mode, OFlags}; +use sha2::{Digest, Sha256}; + +use super::{ObserverFence, ObserverLease, QualifiedObserver}; +use crate::db::change_ledger::reconcile::{ObserverEvent, ObserverQualification}; +#[cfg(debug_assertions)] +use crate::db::change_ledger::ScopeId; +#[cfg(debug_assertions)] +use crate::db::change_ledger::{ + begin_reconciliation, install_initial_scan_hook, reconcile_full, BaselineIdentity, + CompiledPolicy, FilesystemIdentity, PolicyIdentity, ProviderIdentity, ReconcileMode, + RecordingPolicySnapshot, ScopeIdentity, ScopeKind, +}; +use crate::db::change_ledger::{ + DurableCut, EvidenceFlags, EvidenceSource, ExpectedScope, LedgerPath, ObserverRecord, + ObserverWriterBinding, ProviderCapabilities, SegmentWriter, +}; +use crate::error::{Error, Result}; +#[cfg(debug_assertions)] +use crate::{InitImportMode, Trail}; + +const READ_BUFFER_BYTES: usize = 256 * 1024; +const MAX_RETAINED_EVENTS: usize = 65_536; +const MAX_PENDING_RECORDS: usize = 8_192; +const COOKIE_EXPIRY: Duration = Duration::from_millis(75); +const FENCE_TIMEOUT: Duration = Duration::from_secs(10); +const LOOP_PAUSE: Duration = Duration::from_millis(2); + +const WATCH_MASK: WatchMask = WatchMask::CREATE + .union(WatchMask::DELETE) + .union(WatchMask::MODIFY) + .union(WatchMask::CLOSE_WRITE) + .union(WatchMask::ATTRIB) + .union(WatchMask::MOVED_FROM) + .union(WatchMask::MOVED_TO) + .union(WatchMask::DELETE_SELF) + .union(WatchMask::MOVE_SELF) + .union(WatchMask::DONT_FOLLOW) + .union(WatchMask::ONLYDIR) + .union(WatchMask::EXCL_UNLINK); + +pub(crate) trait ObserverDurability: Send { + fn binding(&self) -> ObserverWriterBinding; + fn append_and_flush(&mut self, record: ObserverRecord) -> Result; + fn append_flush_and_rotate( + &mut self, + record: ObserverRecord, + ) -> Result<(DurableCut, DurableCut)>; + fn rotate_if_cut(&mut self, expected: &DurableCut) -> Result>; + fn heartbeat(&mut self) -> Result<()> { + Ok(()) + } + fn revoke_owner(&mut self, _reason: &str) -> Result<()> { + Ok(()) + } +} + +/// Segment writes run on the observer worker, never in an inotify callback. +/// The direct inotify adapter has no callback that could acquire the workspace +/// lock or primary SQLite connection. +pub(crate) struct SegmentWriterDurability { + writer: SegmentWriter, + binding: ObserverWriterBinding, +} + +impl SegmentWriterDurability { + pub(crate) fn new( + mut writer: SegmentWriter, + provider_identity: Vec, + fence_nonce: Vec, + ) -> Result { + let binding = writer.bind_native_observer(provider_identity, fence_nonce)?; + Ok(Self { writer, binding }) + } +} + +impl ObserverDurability for SegmentWriterDurability { + fn binding(&self) -> ObserverWriterBinding { + self.binding.clone() + } + + fn append_and_flush(&mut self, record: ObserverRecord) -> Result { + self.writer.append_and_flush(&[record]) + } + + fn append_flush_and_rotate( + &mut self, + record: ObserverRecord, + ) -> Result<(DurableCut, DurableCut)> { + self.writer.append_flush_and_rotate(&[record]) + } + + fn rotate_if_cut(&mut self, expected: &DurableCut) -> Result> { + self.writer.rotate_if_cut(expected) + } + + fn heartbeat(&mut self) -> Result<()> { + self.writer.heartbeat() + } + + fn revoke_owner(&mut self, reason: &str) -> Result<()> { + self.writer.revoke(reason) + } +} + +#[derive(Clone)] +struct DurableEvent { + event: ObserverEvent, + cut: DurableCut, + internal_fence: bool, +} + +struct PendingRename { + path: LedgerPath, + is_dir: bool, + observed_at: Instant, +} + +struct State { + active: bool, + revoked: Option, + events: Vec, + next_sequence: u64, + pending_renames: HashMap, + issued_fences: HashMap, IssuedFence>, + internal_fence_paths: BTreeSet, + fail_next_watch_add: bool, + policy_invalidation_pending: bool, +} + +#[derive(Clone)] +enum IssuedFenceKind { + Start, + TailAnchor, + ControlledTailAnchor, + RotationAnchor, + End { start_nonce: Vec }, +} + +#[derive(Clone)] +struct IssuedFence { + public: ObserverFence, + expected: ExpectedScope, + root_identity: Vec, + owner_token: String, + provider_id: String, + provider_identity: Vec, + owner_fence_nonce: Vec, + sentinel_path: LedgerPath, + create_sequence: u64, + delete_sequence: u64, + segment_id: String, + durable_cut: DurableCut, + kind: IssuedFenceKind, +} + +struct Shared { + state: Mutex, + changed: Condvar, + shutdown: AtomicBool, +} + +impl Shared { + fn lock(&self) -> MutexGuard<'_, State> { + self.state + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + } + + fn revoke(&self, reason: impl Into) { + let mut state = self.lock(); + if state.revoked.is_none() { + state.revoked = Some(reason.into()); + } + state.active = false; + self.changed.notify_all(); + } +} + +struct InternalFenceRegistration { + shared: Arc, + path: LedgerPath, +} + +impl InternalFenceRegistration { + fn new(shared: Arc, path: LedgerPath) -> Result { + if !shared.lock().internal_fence_paths.insert(path.clone()) { + shared.revoke("inotify_internal_fence_registration_collision"); + return Err(reconcile_error( + "inotify_internal_fence_registration_collision", + )); + } + Ok(Self { shared, path }) + } +} + +impl Drop for InternalFenceRegistration { + fn drop(&mut self) { + self.shared.lock().internal_fence_paths.remove(&self.path); + } +} + +pub(crate) struct LinuxInotifyObserver { + root_path: PathBuf, + root: File, + root_identity: Vec, + provider_identity: Vec, + provider_id: String, + owner_token: String, + owner_fence_nonce: Vec, + policy_dependencies: Vec, + policy_directories: Vec, + records: SyncSender, + shared: Arc, + workers: Mutex>>, +} + +struct PlannedRecord { + path: LedgerPath, + flags: EvidenceFlags, + internal_fence: bool, +} + +enum DurabilityCommand { + Record(PlannedRecord), + ControlledFence { + nonce: Vec, + response: SyncSender>, + }, + Rotate { + expected: DurableCut, + response: SyncSender>>, + }, + PolicyInvalidation { + dependency: PathBuf, + reason: String, + response: SyncSender>, + }, +} + +struct PolicyDirectoryAuthority { + named_path: PathBuf, + canonical_path: PathBuf, + directory: File, + identity: Vec, +} + +impl PolicyDirectoryAuthority { + fn try_clone(&self) -> Result { + Ok(Self { + named_path: self.named_path.clone(), + canonical_path: self.canonical_path.clone(), + directory: self.directory.try_clone()?, + identity: self.identity.clone(), + }) + } +} + +#[derive(Clone)] +struct PolicyDependencyWatch { + dependency: PathBuf, + observed_path: PathBuf, + directory_index: usize, +} + +struct WorkerPolicyDirectory { + authority: PolicyDirectoryAuthority, + watch_descriptor: WatchDescriptor, +} + +impl LinuxInotifyObserver { + pub(crate) fn authenticated_cut( + &self, + expected: &ExpectedScope, + fence: &ObserverFence, + ) -> Result { + Ok(self.issued_fence(expected, fence)?.durable_cut) + } + + pub(crate) fn install_rotation_anchor( + &self, + expected: &ExpectedScope, + end: &ObserverFence, + anchor_cut: DurableCut, + ) -> Result { + self.issued_fence(expected, end)?; + if anchor_cut.last_sequence != end.sequence { + return Err(reconcile_error("inotify_rotation_anchor_sequence_mismatch")); + } + let mut nonce = vec![0_u8; 24]; + getrandom::getrandom(&mut nonce) + .map_err(|error| Error::InvalidInput(format!("rotation anchor entropy: {error}")))?; + let public = ObserverFence { + sequence: anchor_cut.last_sequence, + durable_offset: anchor_cut.durable_end_offset, + nonce: nonce.clone(), + }; + let issued = IssuedFence { + public: public.clone(), + expected: expected.clone(), + root_identity: self.root_identity.clone(), + owner_token: self.owner_token.clone(), + provider_id: self.provider_id.clone(), + provider_identity: self.provider_identity.clone(), + owner_fence_nonce: self.owner_fence_nonce.clone(), + sentinel_path: LedgerPath::parse(".trail/rotation-anchor")?, + create_sequence: public.sequence, + delete_sequence: public.sequence, + segment_id: anchor_cut.segment_id.clone(), + durable_cut: anchor_cut, + kind: IssuedFenceKind::RotationAnchor, + }; + self.shared.lock().issued_fences.insert(nonce, issued); + Ok(public) + } + + pub(crate) fn seal_after_fence( + &self, + expected: &ExpectedScope, + fence: &ObserverFence, + ) -> Result> { + let issued = self.issued_fence(expected, fence)?; + let (response_tx, response_rx) = mpsc::sync_channel(1); + self.records + .send(DurabilityCommand::Rotate { + expected: issued.durable_cut, + response: response_tx, + }) + .map_err(|_| reconcile_error("inotify_rotation_worker_unavailable"))?; + match response_rx.recv_timeout(FENCE_TIMEOUT) { + Ok(result) => result, + Err(_) => { + self.shared.revoke("inotify_rotation_timeout"); + Err(reconcile_error("inotify_rotation_timeout")) + } + } + } + + pub(crate) fn controlled_end_fence( + &self, + expected: &ExpectedScope, + start: &ObserverFence, + ) -> Result<(ObserverFence, DurableCut, DurableCut)> { + let issued_start = self.issued_fence(expected, start)?; + if !matches!( + issued_start.kind, + IssuedFenceKind::Start + | IssuedFenceKind::TailAnchor + | IssuedFenceKind::ControlledTailAnchor + | IssuedFenceKind::RotationAnchor + ) { + self.shared.revoke("inotify_controlled_start_kind_mismatch"); + return Err(reconcile_error("inotify_controlled_start_kind_mismatch")); + } + let barrier = self.sentinel_fence( + expected, + IssuedFenceKind::End { + start_nonce: start.nonce.clone(), + }, + )?; + let mut nonce = vec![0_u8; 24]; + getrandom::getrandom(&mut nonce).map_err(|error| { + Error::InvalidInput(format!("controlled inotify fence entropy failed: {error}")) + })?; + let (response_tx, response_rx) = mpsc::sync_channel(1); + self.records + .send(DurabilityCommand::ControlledFence { + nonce: nonce.clone(), + response: response_tx, + }) + .map_err(|_| reconcile_error("inotify_controlled_fence_worker_unavailable"))?; + let (public, sealed, anchor) = match response_rx.recv_timeout(FENCE_TIMEOUT) { + Ok(result) => result?, + Err(_) => { + self.shared.revoke("inotify_controlled_fence_timeout"); + return Err(reconcile_error("inotify_controlled_fence_timeout")); + } + }; + if public.nonce != nonce + || public.sequence != sealed.last_sequence + || public.durable_offset != sealed.durable_end_offset + || sealed.last_sequence != anchor.last_sequence + || sealed.provider_cursor != anchor.provider_cursor + { + self.shared.revoke("inotify_controlled_fence_cut_mismatch"); + return Err(reconcile_error("inotify_controlled_fence_cut_mismatch")); + } + let path = LedgerPath::parse(&format!(".trail-controlled-fence-{}", hex::encode(&nonce)))?; + let issued = IssuedFence { + public: public.clone(), + expected: expected.clone(), + root_identity: self.root_identity.clone(), + owner_token: self.owner_token.clone(), + provider_id: self.provider_id.clone(), + provider_identity: self.provider_identity.clone(), + owner_fence_nonce: self.owner_fence_nonce.clone(), + sentinel_path: path, + create_sequence: public.sequence, + delete_sequence: public.sequence, + segment_id: sealed.segment_id.clone(), + durable_cut: sealed.clone(), + kind: IssuedFenceKind::End { + start_nonce: start.nonce.clone(), + }, + }; + let mut state = self.shared.lock(); + state.issued_fences.remove(&barrier.nonce); + state.issued_fences.insert(nonce, issued); + drop(state); + if public.sequence <= start.sequence + || (issued_start.durable_cut.segment_id == sealed.segment_id + && public.durable_offset < start.durable_offset) + { + self.shared.revoke("inotify_controlled_fence_non_monotonic"); + return Err(reconcile_error("inotify_controlled_fence_non_monotonic")); + } + Ok((public, sealed, anchor)) + } + + fn rebind_tail_anchor( + &self, + previous: &ExpectedScope, + next: &ExpectedScope, + anchor: &ObserverFence, + ) -> Result<()> { + let mut state = self.shared.lock(); + let retained = state + .issued_fences + .get_mut(&anchor.nonce) + .ok_or_else(|| reconcile_error("inotify_retained_tail_rebind_missing"))?; + if retained.public != *anchor + || retained.expected != *previous + || !matches!( + retained.kind, + IssuedFenceKind::TailAnchor | IssuedFenceKind::ControlledTailAnchor + ) + || !stable_observer_binding(previous, next) + { + return Err(reconcile_error("inotify_retained_tail_rebind_mismatch")); + } + retained.expected = next.clone(); + Ok(()) + } + + pub(crate) fn start( + root_path: &Path, + durability: Box, + policy_dependencies: &[PathBuf], + ) -> Result { + ensure_supported_linux_filesystem(root_path)?; + let root = open_root_no_follow(root_path)?; + let root_identity = root_identity(&root)?; + let lease_policy_dependencies = policy_dependencies.to_vec(); + let binding = durability.binding(); + if binding.owner_token.is_empty() + || binding.provider_id.is_empty() + || binding.provider_identity.is_empty() + || binding.fence_nonce.len() < 16 + || binding.provider_id != hex::encode(&binding.provider_identity) + { + return Err(Error::InvalidInput( + "native observer durability binding is incomplete or inconsistent".into(), + )); + } + let mut inotify = Inotify::init()?; + let mut watches = HashMap::new(); + add_tree(&mut inotify, root_path, Path::new(""), &mut watches, false)?; + let (policy_watches, policy_directories) = + build_policy_coverage(&mut inotify, root_path, policy_dependencies, &mut watches)?; + let observer_policy_directories = policy_directories + .iter() + .map(|directory| directory.authority.try_clone()) + .collect::>>()?; + + let owner_token = binding.owner_token.clone(); + let provider_id = binding.provider_id.clone(); + let provider_identity = binding.provider_identity.clone(); + let owner_fence_nonce = binding.fence_nonce.clone(); + let shared = Arc::new(Shared { + state: Mutex::new(State { + active: true, + revoked: None, + events: Vec::new(), + next_sequence: 1, + pending_renames: HashMap::new(), + issued_fences: HashMap::new(), + internal_fence_paths: BTreeSet::new(), + fail_next_watch_add: false, + policy_invalidation_pending: false, + }), + changed: Condvar::new(), + shutdown: AtomicBool::new(false), + }); + let worker_shared = Arc::clone(&shared); + let worker_root_path = root_path.to_path_buf(); + let worker_root = root.try_clone()?; + let expected_identity = root_identity.clone(); + let (records_tx, records_rx) = mpsc::sync_channel(MAX_PENDING_RECORDS); + let observer_records = records_tx.clone(); + let durability_shared = Arc::clone(&shared); + let durability_worker = thread::Builder::new() + .name("trail-linux-observer-durability".into()) + .spawn(move || run_durability_worker(records_rx, durability, durability_shared))?; + let worker = thread::Builder::new() + .name("trail-linux-inotify".into()) + .spawn(move || { + run_worker( + inotify, + watches, + worker_root_path, + worker_root, + expected_identity, + policy_watches, + policy_directories, + records_tx, + worker_shared, + ) + })?; + Ok(Self { + root_path: root_path.to_path_buf(), + root, + root_identity, + provider_identity, + provider_id, + owner_token, + owner_fence_nonce, + policy_dependencies: lease_policy_dependencies, + policy_directories: observer_policy_directories, + records: observer_records, + shared, + workers: Mutex::new(vec![worker, durability_worker]), + }) + } + + pub(crate) fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + durable_cursor: true, + linearizable_fence: true, + rename_pairing: true, + overflow_scope: true, + filesystem_supported: true, + clean_proof_allowed: true, + power_loss_durability: true, + } + } + + pub(crate) fn root_identity(&self) -> Result> { + self.ensure_available()?; + if root_identity(&self.root)? != self.root_identity + || root_identity(&open_root_no_follow(&self.root_path)?)? != self.root_identity + { + self.shared.revoke("inotify_root_replaced"); + return Err(reconcile_error("inotify_root_replaced")); + } + if verify_policy_directories(&self.policy_directories).is_err() { + let dependency = self.policy_dependencies.first().cloned().ok_or_else(|| { + reconcile_error("inotify_policy_parent_identity_revalidation_failure") + })?; + request_policy_invalidation( + &self.shared, + &self.records, + dependency, + "inotify_policy_parent_replaced", + )?; + return Err(reconcile_error( + "inotify_policy_parent_identity_revalidation_failure", + )); + } + Ok(self.root_identity.clone()) + } + + pub(crate) fn lease(&self) -> Result { + Ok(ObserverLease { + owner_token: self.owner_token.clone(), + root_identity: self.root_identity()?, + provider_identity: self.provider_identity.clone(), + policy_dependencies: self.policy_dependencies.clone(), + direct_policy_fences: Vec::new(), + capabilities: self.capabilities(), + }) + } + + fn ensure_available(&self) -> Result<()> { + let state = self.shared.lock(); + if let Some(reason) = &state.revoked { + return Err(reconcile_error(reason)); + } + if !state.active { + return Err(reconcile_error("inotify_observer_unavailable")); + } + Ok(()) + } + + fn sentinel_fence( + &self, + expected: &ExpectedScope, + kind: IssuedFenceKind, + ) -> Result { + self.ensure_available()?; + self.root_identity()?; + if expected.provider_identity != self.provider_identity { + self.shared.revoke("inotify_provider_identity_mismatch"); + return Err(reconcile_error("inotify_provider_identity_mismatch")); + } + let mut nonce = [0_u8; 24]; + getrandom::getrandom(&mut nonce).map_err(|error| { + Error::InvalidInput(format!("observer fence nonce failed: {error}")) + })?; + let nonce_hex = hex::encode(nonce); + let name = format!(".trail-observer-fence-{nonce_hex}"); + let path = LedgerPath::parse(&name)?; + // Registration, not a filename pattern, authenticates observer-owned + // sentinel events. A user path that merely resembles a sentinel must + // remain an ordinary changed-path candidate. + let _internal_fence = + InternalFenceRegistration::new(Arc::clone(&self.shared), path.clone())?; + + let fd = openat( + &self.root, + Path::new(&name), + OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::RUSR | Mode::WUSR, + ) + .map_err(|error| Error::Io(error.into()))?; + let mut sentinel = File::from(fd); + sentinel.write_all(nonce_hex.as_bytes())?; + sentinel.sync_all()?; + fsync(&self.root).map_err(|error| Error::Io(error.into()))?; + let create = self.wait_for(&path, EvidenceFlags::CREATE, 0)?; + + unlinkat(&self.root, Path::new(&name), AtFlags::empty()) + .map_err(|error| Error::Io(error.into()))?; + fsync(&self.root).map_err(|error| Error::Io(error.into()))?; + let delete = self.wait_for(&path, EvidenceFlags::DELETE, create.event.sequence)?; + let public = ObserverFence { + sequence: delete.event.sequence, + durable_offset: delete.cut.durable_end_offset, + nonce: nonce.to_vec(), + }; + let issued = IssuedFence { + public: public.clone(), + expected: expected.clone(), + root_identity: self.root_identity.clone(), + owner_token: self.owner_token.clone(), + provider_id: self.provider_id.clone(), + provider_identity: self.provider_identity.clone(), + owner_fence_nonce: self.owner_fence_nonce.clone(), + sentinel_path: path, + create_sequence: create.event.sequence, + delete_sequence: delete.event.sequence, + segment_id: delete.cut.segment_id.clone(), + durable_cut: delete.cut, + kind, + }; + self.shared + .lock() + .issued_fences + .insert(nonce.to_vec(), issued); + Ok(public) + } + + fn issued_fence(&self, expected: &ExpectedScope, fence: &ObserverFence) -> Result { + let state = self.shared.lock(); + let Some(issued) = state.issued_fences.get(&fence.nonce) else { + drop(state); + self.shared.revoke("inotify_fence_unknown_or_replayed"); + return Err(reconcile_error("inotify_fence_unknown_or_replayed")); + }; + let rotation_anchor = matches!(issued.kind, IssuedFenceKind::RotationAnchor); + let controlled_end = matches!(issued.kind, IssuedFenceKind::End { .. }) + && issued.create_sequence == issued.delete_sequence + && issued + .sentinel_path + .as_str() + .starts_with(".trail-controlled-fence-"); + let controlled_tail = matches!(issued.kind, IssuedFenceKind::ControlledTailAnchor) + && issued.create_sequence == issued.delete_sequence + && issued + .sentinel_path + .as_str() + .starts_with(".trail-controlled-fence-"); + let mut mismatches = Vec::new(); + if issued.public != *fence { + mismatches.push("public_fence"); + } + if issued.expected != *expected { + mismatches.push("expected_scope"); + } + if issued.root_identity != self.root_identity { + mismatches.push("root_identity"); + } + if issued.owner_token != self.owner_token { + mismatches.push("owner_token"); + } + if issued.provider_id != self.provider_id { + mismatches.push("provider_id"); + } + if issued.provider_identity != self.provider_identity { + mismatches.push("provider_identity"); + } + if issued.owner_fence_nonce != self.owner_fence_nonce { + mismatches.push("owner_fence_nonce"); + } + if issued.delete_sequence != fence.sequence { + mismatches.push("delete_sequence"); + } + if issued.durable_cut.last_sequence != fence.sequence { + mismatches.push("durable_sequence"); + } + if issued.durable_cut.durable_end_offset != fence.durable_offset { + mismatches.push("durable_offset"); + } + if issued.durable_cut.segment_id != issued.segment_id { + mismatches.push("segment_id"); + } + if !rotation_anchor && !controlled_end && !controlled_tail { + if issued.create_sequence >= issued.delete_sequence { + mismatches.push("sentinel_order"); + } + if issued.sentinel_path.as_str() + != format!(".trail-observer-fence-{}", hex::encode(&fence.nonce)) + { + mismatches.push("sentinel_path"); + } + } + if !mismatches.is_empty() { + drop(state); + let reason = format!( + "inotify_fence_authentication_mismatch:{}", + mismatches.join(",") + ); + self.shared.revoke(reason.clone()); + return Err(reconcile_error(&reason)); + } + Ok(issued.clone()) + } + + fn wait_for( + &self, + path: &LedgerPath, + required: EvidenceFlags, + after: u64, + ) -> Result { + let deadline = Instant::now() + FENCE_TIMEOUT; + let mut state = self.shared.lock(); + loop { + if let Some(reason) = &state.revoked { + return Err(reconcile_error(reason)); + } + if let Some(found) = state.events.iter().find(|item| { + item.event.sequence > after + && item.event.path == *path + && item.event.flags.0 & required.0 == required.0 + }) { + return Ok(found.clone()); + } + let now = Instant::now(); + if now >= deadline { + drop(state); + self.shared.revoke("inotify_fence_delivery_timeout"); + return Err(reconcile_error("inotify_fence_delivery_timeout")); + } + let duration = deadline.saturating_duration_since(now); + let waited = self + .shared + .changed + .wait_timeout(state, duration) + .unwrap_or_else(|poison| poison.into_inner()); + state = waited.0; + } + } + + fn shutdown_inner(&self) -> Result<()> { + self.shared.shutdown.store(true, Ordering::Release); + self.shared.changed.notify_all(); + let workers = std::mem::take( + &mut *self + .workers + .lock() + .unwrap_or_else(|poison| poison.into_inner()), + ); + for worker in workers { + worker + .join() + .map_err(|_| Error::InvalidInput("inotify observer worker panicked".into()))?; + } + let mut state = self.shared.lock(); + state.active = false; + Ok(()) + } + + #[cfg(debug_assertions)] + fn test_fail_next_watch_add(&self) { + self.shared.lock().fail_next_watch_add = true; + } +} + +fn ensure_supported_linux_filesystem(path: &Path) -> Result<()> { + let path_c = std::ffi::CString::new(path.as_os_str().as_bytes()) + .map_err(|_| Error::InvalidInput("observer root contained NUL".into()))?; + let mut stat: libc::statfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statfs(path_c.as_ptr(), &mut stat) } != 0 { + return Err(Error::Io(std::io::Error::last_os_error())); + } + if !linux_filesystem_magic_is_supported(stat.f_type as u64) { + return Err(Error::ChangeLedgerReconcileRequired { + scope: path.display().to_string(), + state: "stale_baseline".into(), + reason: format!( + "native changed-path observer does not qualify filesystem type 0x{:x}", + stat.f_type + ), + command: "trail index reconcile".into(), + }); + } + Ok(()) +} + +fn linux_filesystem_magic_is_supported(magic: u64) -> bool { + // Activation evidence currently covers the ext-family native gate. New + // filesystems must be added only with their own real observer gate; an + // unknown magic is never silently treated as qualified. + magic == 0x0000_ef53 +} + +#[cfg(debug_assertions)] +pub(crate) fn run_unsupported_filesystem_rejection() -> std::result::Result<(), String> { + for unsupported in [ + 0x0000_6969, + 0x0000_517b, + 0xff53_4d42, + 0x6573_5546, + 0x0102_1997, + 0x5346_414f, + 0x7375_7245, + 0x0000_564c, + 0x00c3_6400, + ] { + if linux_filesystem_magic_is_supported(unsupported) { + return Err(format!( + "unsupported Linux filesystem magic 0x{unsupported:x} was qualified" + )); + } + } + if !linux_filesystem_magic_is_supported(0xef53) { + return Err("ext-family filesystem was rejected".into()); + } + for unqualified_local in [0x9123_683e, 0x5846_5342, 0x2fc1_2fc1] { + if linux_filesystem_magic_is_supported(unqualified_local) { + return Err(format!( + "ungated Linux filesystem magic 0x{unqualified_local:x} was qualified" + )); + } + } + Ok(()) +} + +impl QualifiedObserver for LinuxInotifyObserver { + fn begin_observation(&self, expected: &ExpectedScope) -> Result { + self.sentinel_fence(expected, IssuedFenceKind::Start) + } + + fn end_fence(&self, expected: &ExpectedScope, start: &ObserverFence) -> Result { + if expected.provider_identity != self.provider_identity || start.nonce.is_empty() { + return Err(reconcile_error( + "inotify_reconciliation_start_not_qualified", + )); + } + let issued_start = self.issued_fence(expected, start)?; + if !matches!( + issued_start.kind, + IssuedFenceKind::Start + | IssuedFenceKind::TailAnchor + | IssuedFenceKind::ControlledTailAnchor + | IssuedFenceKind::RotationAnchor + ) { + self.shared + .revoke("inotify_reconciliation_start_not_qualified"); + return Err(reconcile_error( + "inotify_reconciliation_start_not_qualified", + )); + } + let end = self.sentinel_fence( + expected, + IssuedFenceKind::End { + start_nonce: start.nonce.clone(), + }, + )?; + let issued_end = self.issued_fence(expected, &end)?; + if end.sequence <= start.sequence + || (issued_start.durable_cut.segment_id == issued_end.durable_cut.segment_id + && end.durable_offset < start.durable_offset) + { + self.shared.revoke("inotify_non_monotonic_fence"); + return Err(reconcile_error("inotify_non_monotonic_fence")); + } + Ok(end) + } + + fn drain_through( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + ) -> Result { + self.drain_interval(expected, root_handle_identity, start, end, sink, false) + } + + fn drain_through_retaining_end( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + ) -> Result { + self.drain_interval(expected, root_handle_identity, start, end, sink, true) + } + + fn rebind_retained_tail( + &self, + previous: &ExpectedScope, + next: &ExpectedScope, + anchor: &ObserverFence, + ) -> Result<()> { + self.rebind_tail_anchor(previous, next, anchor) + } +} + +fn stable_observer_binding(previous: &ExpectedScope, next: &ExpectedScope) -> bool { + previous.scope_id == next.scope_id + && previous.epoch == next.epoch + && previous.policy_fingerprint == next.policy_fingerprint + && previous.policy_generation == next.policy_generation + && previous.filesystem_identity == next.filesystem_identity + && previous.provider_identity == next.provider_identity +} + +impl LinuxInotifyObserver { + fn drain_interval( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + retain_end: bool, + ) -> Result { + self.ensure_available()?; + if self.root_identity()? != root_handle_identity { + self.shared.revoke("inotify_root_identity_mismatch"); + return Err(reconcile_error("inotify_root_identity_mismatch")); + } + let issued_start = self.issued_fence(expected, start)?; + let issued_end = self.issued_fence(expected, end)?; + if !matches!( + issued_start.kind, + IssuedFenceKind::Start + | IssuedFenceKind::TailAnchor + | IssuedFenceKind::ControlledTailAnchor + | IssuedFenceKind::RotationAnchor + ) || !matches!( + &issued_end.kind, + IssuedFenceKind::End { start_nonce } if *start_nonce == start.nonce + ) { + self.shared.revoke("inotify_fence_interval_mismatch"); + return Err(reconcile_error("inotify_fence_interval_mismatch")); + } + let (events, end_cut) = { + let state = self.shared.lock(); + let events = state + .events + .iter() + .filter(|item| { + item.event.sequence > start.sequence + && item.event.sequence <= end.sequence + && !item.internal_fence + }) + .map(|item| item.event.clone()) + .collect::>(); + let end_cut = state + .events + .iter() + .find(|item| item.event.sequence == end.sequence) + .map(|item| item.cut.clone()); + (events, end_cut) + }; + for event in events { + sink(event)?; + } + let end_cut = end_cut.ok_or_else(|| reconcile_error("inotify_end_fence_not_retained"))?; + if end_cut != issued_end.durable_cut { + self.shared.revoke("inotify_end_fence_durable_cut_mismatch"); + return Err(reconcile_error("inotify_end_fence_durable_cut_mismatch")); + } + let qualification = ObserverQualification::native( + expected, + root_handle_identity.to_vec(), + start.clone(), + end.clone(), + self.owner_token.clone(), + self.owner_fence_nonce.clone(), + end_cut.segment_id, + end_cut.durable_end_offset, + end_cut.durable_end_offset, + ); + let mut state = self.shared.lock(); + state + .events + .retain(|item| item.event.sequence > end.sequence); + state.issued_fences.remove(&start.nonce); + if retain_end { + let retained = state + .issued_fences + .get_mut(&end.nonce) + .ok_or_else(|| reconcile_error("inotify_end_fence_not_retained_for_rotation"))?; + retained.kind = if retained.create_sequence == retained.delete_sequence + && retained + .sentinel_path + .as_str() + .starts_with(".trail-controlled-fence-") + { + IssuedFenceKind::ControlledTailAnchor + } else { + IssuedFenceKind::TailAnchor + }; + } else { + state.issued_fences.remove(&end.nonce); + } + Ok(qualification) + } +} + +impl Drop for LinuxInotifyObserver { + fn drop(&mut self) { + let _ = self.shutdown_inner(); + } +} + +fn run_worker( + mut inotify: Inotify, + mut watches: HashMap, + root_path: PathBuf, + root: File, + root_identity_expected: Vec, + policy_watches: Vec, + policy_directories: Vec, + records: SyncSender, + shared: Arc, +) { + let mut buffer = vec![0_u8; READ_BUFFER_BYTES]; + while !shared.shutdown.load(Ordering::Acquire) { + if shared.lock().revoked.is_some() { + break; + } + if verify_root(&root_path, &root, &root_identity_expected).is_err() { + shared.revoke("inotify_root_replaced"); + break; + } + if verify_worker_policy_directories(&policy_directories).is_err() { + let Some(dependency) = policy_watches.first().map(|watch| watch.dependency.clone()) + else { + shared.revoke("inotify_policy_parent_identity_revalidation_failure"); + break; + }; + let _ = request_policy_invalidation( + &shared, + &records, + dependency, + "inotify_policy_parent_replaced", + ); + break; + } + let events = match inotify.read_events(&mut buffer) { + Ok(events) => events + .map(|event| { + ( + event.wd, + event.mask, + event.cookie, + event.name.map(OsStr::to_os_string), + ) + }) + .collect::>(), + Err(error) if error.kind() == ErrorKind::WouldBlock => Vec::new(), + Err(_) => { + shared.revoke("inotify_decode_or_read_failure"); + break; + } + }; + for (wd, mask, cookie, name) in events { + let policy_directory_index = policy_directories + .iter() + .position(|directory| directory.watch_descriptor == wd); + if let Some(dependency) = policy_directory_invalidation_dependency( + mask, + policy_directory_index, + &policy_watches, + ) { + let _ = request_policy_invalidation( + &shared, + &records, + dependency, + "inotify_policy_parent_replaced", + ); + return; + } + if classify_raw_authority_event( + &shared, + mask, + watches.contains_key(&wd) || policy_directory_index.is_some(), + ) + .is_err() + { + break; + } + let ledger_parent = watches.get(&wd).cloned(); + if ledger_parent + .as_ref() + .is_some_and(|parent| parent.as_os_str().is_empty()) + && (mask.contains(EventMask::DELETE_SELF) || mask.contains(EventMask::MOVE_SELF)) + { + shared.revoke("inotify_root_deleted_or_moved"); + break; + } + let Some(name) = name else { + continue; + }; + let Some(name) = name.to_str() else { + shared.revoke("inotify_path_decode_ambiguity"); + break; + }; + if name.is_empty() || name == "." || name == ".." || name.contains('/') { + shared.revoke("inotify_path_decode_ambiguity"); + break; + } + let candidate = match (ledger_parent.as_ref(), policy_directory_index) { + (Some(parent), _) => root_path.join(parent).join(name), + (None, Some(index)) => policy_directories[index] + .authority + .canonical_path + .join(name), + (None, None) => { + shared.revoke("inotify_unknown_watch_descriptor"); + break; + } + }; + if let Some(dependency) = policy_watches + .iter() + .find(|watch| policy_dependency_triggered(&candidate, &watch.observed_path, mask)) + .map(|watch| watch.dependency.clone()) + { + let _ = request_policy_invalidation( + &shared, + &records, + dependency, + "inotify_policy_dependency_invalidated", + ); + return; + } + let Some(parent) = ledger_parent else { + // External policy coverage is classification-only. Paths that + // are not exact dependencies or their ancestors never become + // ledger candidates. + continue; + }; + let relative = parent.join(name); + let is_dir = mask.contains(EventMask::ISDIR); + if observer_internal_path(&relative) + && !policy_watches.iter().any(|watch| { + watch + .observed_path + .strip_prefix(&root_path) + .is_ok_and(|dependency| dependency == relative) + }) + { + continue; + } + let Some(relative_text) = relative.to_str() else { + shared.revoke("inotify_path_decode_ambiguity"); + break; + }; + let path = match LedgerPath::parse(relative_text) { + Ok(path) => path, + Err(_) => { + shared.revoke("inotify_path_decode_ambiguity"); + break; + } + }; + if is_dir && (mask.contains(EventMask::CREATE) || mask.contains(EventMask::MOVED_TO)) { + let fail = { + let mut state = shared.lock(); + let fail = state.fail_next_watch_add; + state.fail_next_watch_add = false; + fail + }; + if add_tree(&mut inotify, &root_path, &relative, &mut watches, fail).is_err() { + // A just-created directory can be renamed again before its + // CREATE is drained. If the old endpoint is already gone, + // the complete parent prefix plus the later MOVED_TO + // watch-before-enumerate closes that race. Every other + // watch-add failure revokes continuity globally. + if mask.contains(EventMask::CREATE) && !root_path.join(&relative).exists() { + if enqueue( + &shared, + &records, + complete_parent(&path), + EvidenceFlags::PROVIDER_COMPLETE_PREFIX, + ) + .is_err() + { + break; + } + } else { + shared.revoke("inotify_watch_add_failure"); + break; + } + } + if enqueue( + &shared, + &records, + complete_parent(&path), + EvidenceFlags::PROVIDER_COMPLETE_PREFIX, + ) + .is_err() + { + break; + } + } + let flags = event_flags(mask); + if flags.0 != 0 && enqueue(&shared, &records, path.clone(), flags).is_err() { + break; + } + if mask.contains(EventMask::MOVED_FROM) && cookie != 0 { + shared.lock().pending_renames.insert( + cookie, + PendingRename { + path: path.clone(), + is_dir, + observed_at: Instant::now(), + }, + ); + } + if mask.contains(EventMask::MOVED_TO) && cookie != 0 { + let paired = shared.lock().pending_renames.remove(&cookie); + if let Some(from) = paired { + if from.is_dir && is_dir { + if enqueue( + &shared, + &records, + from.path.clone(), + EvidenceFlags::PROVIDER_COMPLETE_PREFIX, + ) + .is_err() + { + break; + } + remap_watches(&mut watches, Path::new(from.path.as_str()), &relative); + } + } + } + } + if expire_rename_cookies(&shared, &records).is_err() { + break; + } + thread::sleep(LOOP_PAUSE); + } + let mut state = shared.lock(); + state.active = false; + shared.changed.notify_all(); +} + +fn classify_raw_authority_event( + shared: &Shared, + mask: EventMask, + known_watch_descriptor: bool, +) -> Result<()> { + if mask.contains(EventMask::Q_OVERFLOW) { + shared.revoke("inotify_queue_overflow"); + return Err(reconcile_error("inotify_queue_overflow")); + } + if mask.contains(EventMask::IGNORED) { + shared.revoke("inotify_watch_ignored"); + return Err(reconcile_error("inotify_watch_ignored")); + } + if !known_watch_descriptor { + shared.revoke("inotify_unknown_watch_descriptor"); + return Err(reconcile_error("inotify_unknown_watch_descriptor")); + } + Ok(()) +} + +fn enqueue( + shared: &Shared, + records: &SyncSender, + path: LedgerPath, + flags: EvidenceFlags, +) -> Result<()> { + let internal_fence = { + let mut state = shared.lock(); + let registered = state.internal_fence_paths.contains(&path); + if registered && flags.0 & EvidenceFlags::DELETE.0 != 0 { + state.internal_fence_paths.remove(&path); + } + registered + }; + match records.try_send(DurabilityCommand::Record(PlannedRecord { + path, + flags, + internal_fence, + })) { + Ok(()) => Ok(()), + Err(TrySendError::Full(_)) => { + shared.revoke("inotify_bounded_queue_overflow"); + Err(reconcile_error("inotify_bounded_queue_overflow")) + } + Err(TrySendError::Disconnected(_)) => { + shared.revoke("inotify_durability_worker_unavailable"); + Err(reconcile_error("inotify_durability_worker_unavailable")) + } + } +} + +fn request_policy_invalidation( + shared: &Shared, + records: &SyncSender, + dependency: PathBuf, + reason: &str, +) -> Result<()> { + { + let mut state = shared.lock(); + if state.policy_invalidation_pending { + drop(state); + return Err(reconcile_error("inotify_policy_invalidation_pending")); + } + state.policy_invalidation_pending = true; + } + let (response, result) = mpsc::sync_channel(1); + match records.try_send(DurabilityCommand::PolicyInvalidation { + dependency, + reason: reason.to_string(), + response, + }) { + Ok(()) => {} + Err(TrySendError::Full(_)) => { + shared.revoke("inotify_policy_invalidation_queue_overflow"); + return Err(reconcile_error( + "inotify_policy_invalidation_queue_overflow", + )); + } + Err(TrySendError::Disconnected(_)) => { + shared.revoke("inotify_policy_invalidation_worker_unavailable"); + return Err(reconcile_error( + "inotify_policy_invalidation_worker_unavailable", + )); + } + } + match result.recv_timeout(FENCE_TIMEOUT) { + Ok(result) => result, + Err(_) => { + shared.revoke("inotify_policy_invalidation_durability_timeout"); + Err(reconcile_error( + "inotify_policy_invalidation_durability_timeout", + )) + } + } +} + +fn run_durability_worker( + records: Receiver, + mut durability: Box, + shared: Arc, +) { + let mut last_heartbeat = Instant::now(); + loop { + if shared.shutdown.load(Ordering::Acquire) { + break; + } + match records.recv_timeout(Duration::from_millis(10)) { + Ok(command) => match command { + DurabilityCommand::Record(record) => { + if persist(&shared, durability.as_mut(), record).is_err() { + break; + } + } + DurabilityCommand::ControlledFence { nonce, response } => { + let path = LedgerPath::parse(&format!( + ".trail-controlled-fence-{}", + hex::encode(&nonce) + )); + let result = path.and_then(|path| { + persist_and_rotate(&shared, durability.as_mut(), path, nonce) + }); + if let Err(error) = &result { + shared.revoke(format!("inotify_controlled_fence_failure:{error}")); + } + let failed = result.is_err(); + let _ = response.send(result); + if failed { + break; + } + } + DurabilityCommand::Rotate { expected, response } => { + let result = durability.rotate_if_cut(&expected); + if let Err(error) = &result { + shared.revoke(format!("inotify_rotation_failure:{error}")); + } + let failed = result.is_err(); + let _ = response.send(result); + if failed { + break; + } + } + DurabilityCommand::PolicyInvalidation { + dependency, + reason, + response, + } => { + let digest = Sha256::digest(dependency.as_os_str().as_encoded_bytes()); + let marker = LedgerPath::parse(&format!( + ".trail/policy-invalidations/{}", + hex::encode(digest) + )); + let result = match marker { + Ok(marker) => persist( + &shared, + durability.as_mut(), + PlannedRecord { + path: marker, + flags: EvidenceFlags::CONTENT, + internal_fence: true, + }, + ) + .and_then(|()| { + let exact_reason = format!("{reason}:{}", dependency.display()); + durability.revoke_owner(&exact_reason)?; + shared.revoke(exact_reason); + Ok(()) + }), + Err(error) => Err(error), + }; + if let Err(error) = &result { + shared.revoke(format!( + "inotify_policy_invalidation_durability_failure:{error}" + )); + } + let _ = response.send(result); + break; + } + }, + Err(mpsc::RecvTimeoutError::Timeout) if !shared.shutdown.load(Ordering::Acquire) => { + if last_heartbeat.elapsed() >= Duration::from_secs(1) { + if durability.heartbeat().is_err() { + shared.revoke("inotify_observer_heartbeat_failed"); + break; + } + last_heartbeat = Instant::now(); + } + } + Err(mpsc::RecvTimeoutError::Timeout) => break, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + } + } + shared.changed.notify_all(); +} + +fn persist( + shared: &Shared, + durability: &mut dyn ObserverDurability, + record: PlannedRecord, +) -> Result<()> { + let PlannedRecord { + path, + flags, + internal_fence, + } = record; + let sequence = { + let mut state = shared.lock(); + if state.events.len() >= MAX_RETAINED_EVENTS { + drop(state); + shared.revoke("inotify_bounded_queue_overflow"); + return Err(reconcile_error("inotify_bounded_queue_overflow")); + } + let sequence = state.next_sequence; + state.next_sequence = state.next_sequence.saturating_add(1); + sequence + }; + let cut = match durability.append_and_flush(ObserverRecord { + sequence, + source: EvidenceSource::Observer, + path: path.clone(), + flags, + provider_cursor: sequence.to_be_bytes().to_vec(), + }) { + Ok(cut) if cut.last_sequence == sequence => cut, + Ok(_) => { + shared.revoke("inotify_durability_cut_mismatch"); + return Err(reconcile_error("inotify_durability_cut_mismatch")); + } + Err(_) => { + shared.revoke("inotify_durability_failure"); + return Err(reconcile_error("inotify_durability_failure")); + } + }; + let mut state = shared.lock(); + state.events.push(DurableEvent { + event: ObserverEvent { + path, + flags, + sequence, + }, + cut, + internal_fence, + }); + shared.changed.notify_all(); + Ok(()) +} + +fn persist_and_rotate( + shared: &Shared, + durability: &mut dyn ObserverDurability, + path: LedgerPath, + nonce: Vec, +) -> Result<(ObserverFence, DurableCut, DurableCut)> { + let sequence = { + let mut state = shared.lock(); + if state.events.len() >= MAX_RETAINED_EVENTS { + drop(state); + shared.revoke("inotify_bounded_queue_overflow"); + return Err(reconcile_error("inotify_bounded_queue_overflow")); + } + let sequence = state.next_sequence; + state.next_sequence = state.next_sequence.saturating_add(1); + sequence + }; + let (sealed, anchor) = durability.append_flush_and_rotate(ObserverRecord { + sequence, + source: EvidenceSource::Observer, + path: path.clone(), + flags: EvidenceFlags::default(), + provider_cursor: sequence.to_be_bytes().to_vec(), + })?; + if sealed.last_sequence != sequence + || sealed.last_sequence != anchor.last_sequence + || sealed.provider_cursor != anchor.provider_cursor + { + return Err(reconcile_error("inotify_controlled_fence_cut_mismatch")); + } + let public = ObserverFence { + sequence, + durable_offset: sealed.durable_end_offset, + nonce, + }; + let mut state = shared.lock(); + state.events.push(DurableEvent { + event: ObserverEvent { + path, + flags: EvidenceFlags::default(), + sequence, + }, + cut: sealed.clone(), + internal_fence: true, + }); + shared.changed.notify_all(); + Ok((public, sealed, anchor)) +} + +fn expire_rename_cookies(shared: &Shared, records: &SyncSender) -> Result<()> { + let expired = { + let mut state = shared.lock(); + let now = Instant::now(); + let cookies = state + .pending_renames + .iter() + .filter_map(|(cookie, pending)| { + (now.duration_since(pending.observed_at) >= COOKIE_EXPIRY).then_some(*cookie) + }) + .collect::>(); + cookies + .into_iter() + .filter_map(|cookie| state.pending_renames.remove(&cookie)) + .collect::>() + }; + for pending in expired { + enqueue( + shared, + records, + complete_parent(&pending.path), + EvidenceFlags::PROVIDER_COMPLETE_PREFIX, + )?; + } + Ok(()) +} + +fn add_tree( + inotify: &mut Inotify, + root: &Path, + relative: &Path, + watches: &mut HashMap, + inject_failure: bool, +) -> Result<()> { + if observer_internal_path(relative) { + return Ok(()); + } + if inject_failure { + return Err(Error::InvalidInput("injected watch-add failure".into())); + } + let absolute = root.join(relative); + let metadata = fs::symlink_metadata(&absolute)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(Error::InvalidInput( + "inotify recursive watch target is not a no-follow directory".into(), + )); + } + #[allow(deprecated)] + let wd = inotify.add_watch(&absolute, WATCH_MASK)?; + watches.insert(wd, relative.to_path_buf()); + let entries = fs::read_dir(&absolute)?.collect::>>()?; + for entry in entries { + let metadata = entry.file_type()?; + if metadata.is_dir() && !metadata.is_symlink() { + let child = relative.join(entry.file_name()); + if !observer_internal_path(&child) { + add_tree(inotify, root, &child, watches, false)?; + } + } + } + Ok(()) +} + +fn build_policy_coverage( + inotify: &mut Inotify, + root: &Path, + dependencies: &[PathBuf], + watches: &mut HashMap, +) -> Result<(Vec, Vec)> { + let root = root.canonicalize()?; + let mut policy_watches = Vec::with_capacity(dependencies.len()); + let mut authorities = Vec::::new(); + for dependency in dependencies { + let dependency = normalize_absolute_policy_dependency(&root, dependency)?; + let (named_parent, canonical_parent, directory) = + nearest_existing_policy_parent(&dependency)?; + let suffix = dependency + .strip_prefix(&named_parent) + .map_err(|_| reconcile_error("inotify_policy_dependency_coverage_mapping_failure"))?; + let observed_path = canonical_parent.join(suffix); + let directory_index = match authorities + .iter() + .position(|authority| authority.canonical_path == canonical_parent) + { + Some(index) => index, + None => { + let index = authorities.len(); + authorities.push(PolicyDirectoryAuthority { + named_path: named_parent, + canonical_path: canonical_parent, + identity: root_identity(&directory)?, + directory, + }); + index + } + }; + policy_watches.push(PolicyDependencyWatch { + dependency, + observed_path, + directory_index, + }); + } + + let mut policy_directories = Vec::with_capacity(authorities.len()); + for authority in authorities { + let pinned_watch_path = PathBuf::from(format!( + "/proc/self/fd/{}/.", + authority.directory.as_raw_fd() + )); + #[allow(deprecated)] + let watch_descriptor = inotify.add_watch(&pinned_watch_path, WATCH_MASK)?; + if let Ok(relative) = authority.canonical_path.strip_prefix(&root) { + watches.insert(watch_descriptor.clone(), relative.to_path_buf()); + } + policy_directories.push(WorkerPolicyDirectory { + authority, + watch_descriptor, + }); + } + Ok((policy_watches, policy_directories)) +} + +fn normalize_absolute_policy_dependency(root: &Path, dependency: &Path) -> Result { + let absolute = if dependency.is_absolute() { + dependency.to_path_buf() + } else { + root.join(dependency) + }; + let mut normalized = PathBuf::new(); + for component in absolute.components() { + match component { + Component::RootDir => normalized.push(Path::new("/")), + Component::CurDir => {} + Component::ParentDir => { + if !normalized.pop() { + return Err(reconcile_error( + "inotify_policy_dependency_path_is_unsafe_or_ambiguous", + )); + } + } + Component::Normal(component) => normalized.push(component), + Component::Prefix(_) => { + return Err(reconcile_error( + "inotify_policy_dependency_path_is_unsafe_or_ambiguous", + )); + } + } + } + if !normalized.is_absolute() || normalized == Path::new("/") || normalized.to_str().is_none() { + return Err(reconcile_error( + "inotify_policy_dependency_path_is_unsafe_or_ambiguous", + )); + } + Ok(normalized) +} + +fn nearest_existing_policy_parent(dependency: &Path) -> Result<(PathBuf, PathBuf, File)> { + match fs::symlink_metadata(dependency) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { + return Err(reconcile_error( + "inotify_policy_dependency_is_symlink_or_unsafe", + )); + } + Ok(_) => {} + Err(error) if error.kind() == ErrorKind::NotFound => {} + Err(_) => { + return Err(reconcile_error( + "inotify_policy_dependency_is_symlink_or_unsafe", + )); + } + } + let mut candidate = dependency + .parent() + .ok_or_else(|| reconcile_error("inotify_policy_dependency_has_no_parent"))?; + loop { + match fs::symlink_metadata(candidate) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => { + return Err(reconcile_error( + "inotify_policy_dependency_parent_is_symlink_or_unsafe", + )); + } + Ok(_) => { + let directory = open_absolute_directory_no_follow(candidate).map_err(|_| { + reconcile_error("inotify_policy_dependency_parent_is_symlink_or_unsafe") + })?; + let canonical = candidate.canonicalize().map_err(|_| { + reconcile_error("inotify_policy_dependency_parent_identity_unavailable") + })?; + return Ok((candidate.to_path_buf(), canonical, directory)); + } + Err(error) if error.kind() == ErrorKind::NotFound => { + candidate = candidate.parent().ok_or_else(|| { + reconcile_error("inotify_policy_dependency_parent_unobservable") + })?; + } + Err(_) => { + return Err(reconcile_error( + "inotify_policy_dependency_parent_is_symlink_or_unsafe", + )); + } + } + } +} + +fn open_absolute_directory_no_follow(path: &Path) -> Result { + if !path.is_absolute() { + return Err(reconcile_error( + "inotify_policy_dependency_parent_is_not_absolute", + )); + } + let mut directory = open_root_no_follow(Path::new("/"))?; + for component in path.components() { + match component { + Component::RootDir => {} + Component::Normal(component) => { + let child = openat( + &directory, + Path::new(component), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(|error| Error::Io(error.into()))?; + directory = File::from(child); + } + _ => { + return Err(reconcile_error( + "inotify_policy_dependency_parent_is_symlink_or_unsafe", + )); + } + } + } + Ok(directory) +} + +fn verify_policy_directories(directories: &[PolicyDirectoryAuthority]) -> Result<()> { + for authority in directories { + if authority.named_path.canonicalize()? != authority.canonical_path + || root_identity(&authority.directory)? != authority.identity + || root_identity(&open_absolute_directory_no_follow(&authority.named_path)?)? + != authority.identity + { + return Err(reconcile_error( + "inotify_policy_parent_identity_revalidation_failure", + )); + } + } + Ok(()) +} + +fn verify_worker_policy_directories(directories: &[WorkerPolicyDirectory]) -> Result<()> { + for directory in directories { + verify_policy_directories(std::slice::from_ref(&directory.authority))?; + } + Ok(()) +} + +fn policy_dependency_triggered(candidate: &Path, dependency: &Path, mask: EventMask) -> bool { + if candidate == dependency { + return true; + } + dependency.starts_with(candidate) + && mask.intersects( + EventMask::CREATE + | EventMask::DELETE + | EventMask::MOVED_FROM + | EventMask::MOVED_TO + | EventMask::ATTRIB, + ) +} + +fn policy_directory_invalidation_dependency( + mask: EventMask, + directory_index: Option, + watches: &[PolicyDependencyWatch], +) -> Option { + if !mask.intersects(EventMask::IGNORED | EventMask::DELETE_SELF | EventMask::MOVE_SELF) { + return None; + } + let directory_index = directory_index?; + watches + .iter() + .find(|watch| watch.directory_index == directory_index) + .map(|watch| watch.dependency.clone()) +} + +fn observer_internal_path(relative: &Path) -> bool { + relative.components().next().is_some_and(|component| { + component.as_os_str() == ".trail" || component.as_os_str() == ".git" + }) +} + +fn remap_watches(watches: &mut HashMap, from: &Path, to: &Path) { + for relative in watches.values_mut() { + if let Ok(suffix) = relative.strip_prefix(from) { + *relative = to.join(suffix); + } + } +} + +fn event_flags(mask: EventMask) -> EvidenceFlags { + let mut flags = EvidenceFlags::default(); + if mask.contains(EventMask::CREATE) { + flags |= EvidenceFlags::CREATE; + } + if mask.intersects(EventMask::MODIFY | EventMask::CLOSE_WRITE) { + flags |= EvidenceFlags::CONTENT; + } + if mask.contains(EventMask::ATTRIB) { + flags |= EvidenceFlags::MODE; + } + if mask.contains(EventMask::DELETE) { + flags |= EvidenceFlags::DELETE; + } + if mask.contains(EventMask::MOVED_FROM) { + flags |= EvidenceFlags::RENAME_FROM; + } + if mask.contains(EventMask::MOVED_TO) { + flags |= EvidenceFlags::RENAME_TO; + } + flags +} + +fn complete_parent(path: &LedgerPath) -> LedgerPath { + let text = path.as_str(); + match text.rsplit_once('/') { + Some((parent, _)) => LedgerPath::parse(parent).unwrap_or_else(|_| path.clone()), + None => path.clone(), + } +} + +fn open_root_no_follow(path: &Path) -> Result { + Ok(OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path)?) +} + +fn root_identity(file: &File) -> Result> { + let stat = fstat(file).map_err(|error| Error::Io(error.into()))?; + Ok(format!( + "root-v1:dev={};ino={};mode={};uid={};gid={}", + stat.st_dev, stat.st_ino, stat.st_mode, stat.st_uid, stat.st_gid + ) + .into_bytes()) +} + +fn verify_root(path: &Path, root: &File, expected: &[u8]) -> Result<()> { + if root_identity(root)? != expected || root_identity(&open_root_no_follow(path)?)? != expected { + return Err(reconcile_error("inotify_root_replaced")); + } + Ok(()) +} + +fn reconcile_error(reason: &str) -> Error { + Error::ChangeLedgerReconcileRequired { + scope: "native-linux-observer".into(), + state: "untrusted_gap".into(), + reason: reason.into(), + command: "trail status".into(), + } +} + +#[cfg(debug_assertions)] +struct MemoryDurability { + offset: u64, + fail_after: Option, + binding: ObserverWriterBinding, + trace: Option>>>, +} + +#[cfg(debug_assertions)] +impl ObserverDurability for MemoryDurability { + fn binding(&self) -> ObserverWriterBinding { + self.binding.clone() + } + + fn append_and_flush(&mut self, record: ObserverRecord) -> Result { + if self.fail_after == Some(self.offset) { + return Err(Error::InvalidInput( + "injected observer durability failure".into(), + )); + } + if let Some(trace) = &self.trace { + trace + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .push(format!("append:{}", record.path.as_str())); + } + self.offset = self.offset.saturating_add(1); + Ok(DurableCut { + segment_id: "linux-native-test".into(), + durable_end_offset: self.offset, + last_sequence: record.sequence, + last_hash: [0; 32], + provider_cursor: record.provider_cursor, + }) + } + + fn append_flush_and_rotate( + &mut self, + record: ObserverRecord, + ) -> Result<(DurableCut, DurableCut)> { + let sealed = self.append_and_flush(record)?; + Ok((sealed.clone(), sealed)) + } + + fn rotate_if_cut(&mut self, expected: &DurableCut) -> Result> { + let cut = DurableCut { + segment_id: "linux-native-test".into(), + durable_end_offset: self.offset, + last_sequence: self.offset, + last_hash: [0; 32], + provider_cursor: self.offset.to_be_bytes().to_vec(), + }; + Ok((&cut == expected).then_some((cut.clone(), cut))) + } + + fn revoke_owner(&mut self, reason: &str) -> Result<()> { + if let Some(trace) = &self.trace { + trace + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .push(format!("revoke:{reason}")); + } + Ok(()) + } +} + +#[cfg(debug_assertions)] +fn memory_durability(fail_after: Option) -> MemoryDurability { + let mut owner = [0_u8; 32]; + let mut fence = [0_u8; 24]; + getrandom::getrandom(&mut owner).expect("test observer owner entropy"); + getrandom::getrandom(&mut fence).expect("test observer fence entropy"); + let provider_identity = b"linux-inotify-memory-test-v1".to_vec(); + MemoryDurability { + offset: 0, + fail_after, + trace: None, + binding: ObserverWriterBinding { + owner_token: hex::encode(owner), + provider_id: hex::encode(&provider_identity), + provider_identity, + fence_nonce: fence.to_vec(), + }, + } +} + +#[cfg(debug_assertions)] +struct NativeFixture { + _temp: tempfile::TempDir, + db: Trail, + expected: ExpectedScope, + policy: CompiledPolicy, + segment_directory: PathBuf, +} + +#[cfg(debug_assertions)] +static NEXT_NATIVE_FIXTURE_SCOPE: AtomicU64 = AtomicU64::new(1); + +#[cfg(debug_assertions)] +impl NativeFixture { + fn new(setup: impl FnOnce(&Path) -> Result<()>) -> Result { + Self::new_with_scope(setup, ScopeId([0x91; 32])) + } + + fn new_unique(setup: impl FnOnce(&Path) -> Result<()>) -> Result { + let sequence = NEXT_NATIVE_FIXTURE_SCOPE.fetch_add(1, Ordering::Relaxed); + let mut scope_id = [0x93_u8; 32]; + scope_id[24..].copy_from_slice(&sequence.to_be_bytes()); + Self::new_with_scope(setup, ScopeId(scope_id)) + } + + fn new_with_scope(setup: impl FnOnce(&Path) -> Result<()>, scope_id: ScopeId) -> Result { + let temp = tempfile::tempdir()?; + setup(temp.path())?; + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false)?; + let db = Trail::open(temp.path())?; + fs::create_dir_all(db.workspace_root.join(".git/info"))?; + for path in [ + db.workspace_root.join(".git/config"), + db.workspace_root.join(".git/config.worktree"), + db.workspace_root.join(".git/info/exclude"), + db.workspace_root.join(".trailignore"), + ] { + if !path.exists() { + fs::write(path, b"# native observer policy fixture\n")?; + } + } + let branch = db.current_branch()?; + let head = db.resolve_branch_ref(&branch)?; + let scope = ScopeIdentity { + scope_id, + kind: ScopeKind::Workspace, + owner_id: "linux-native-reconciliation".into(), + }; + let fingerprint = [0x92; 32]; + let filesystem_identity = root_identity(&open_root_no_follow(temp.path())?)?; + let provider_identity = b"linux-inotify-native-v1".to_vec(); + let baseline = BaselineIdentity { + ref_name: head.name.clone(), + ref_generation: u64::try_from(head.generation) + .map_err(|_| Error::Corrupt("negative native ref generation".into()))?, + change_id: head.change_id, + root_id: head.root_id, + }; + db.changed_path_ledger().begin_scope( + &scope, + &baseline, + &PolicyIdentity { + fingerprint, + generation: 1, + }, + &FilesystemIdentity(filesystem_identity.clone()), + &ProviderIdentity { + identity: provider_identity.clone(), + capabilities: ProviderCapabilities { + durable_cursor: true, + linearizable_fence: true, + rename_pairing: true, + overflow_scope: true, + filesystem_supported: true, + clean_proof_allowed: true, + power_loss_durability: true, + }, + }, + )?; + let expected = ExpectedScope { + scope_id: scope.scope_id, + epoch: 1, + ref_name: baseline.ref_name, + ref_generation: baseline.ref_generation, + baseline_root: baseline.root_id, + policy_fingerprint: fingerprint, + policy_generation: 1, + filesystem_identity, + provider_identity, + }; + let dependency_files = vec![ + db.db_dir.join("config.toml"), + db.workspace_root.join(".git/info/exclude"), + db.workspace_root.join(".git/config"), + db.workspace_root.join(".git/config.worktree"), + db.workspace_root.join(".trailignore"), + ]; + let policy = CompiledPolicy::for_reconciliation_test( + RecordingPolicySnapshot { + workspace_root: db.workspace_root.clone(), + ignore_gitignored: true, + dependency_files, + case_sensitive: true, + rule_sources: Vec::new(), + }, + fingerprint, + &expected, + ); + let segment_directory = db.db_dir.join("change-observer-segments"); + Ok(Self { + _temp: temp, + db, + expected, + policy, + segment_directory, + }) + } + + fn observer(&self) -> Result { + let mut owner = [0_u8; 32]; + let mut fence = [0_u8; 24]; + getrandom::getrandom(&mut owner).map_err(|error| Error::InvalidInput(error.to_string()))?; + getrandom::getrandom(&mut fence).map_err(|error| Error::InvalidInput(error.to_string()))?; + let writer = SegmentWriter::acquire( + &self.db.sqlite_path, + &self.segment_directory, + self.expected.scope_id, + self.expected.epoch, + owner, + &hex::encode(&self.expected.provider_identity), + Vec::new(), + Duration::from_secs(3_600), + )?; + let durability = SegmentWriterDurability::new( + writer, + self.expected.provider_identity.clone(), + fence.to_vec(), + )?; + LinuxInotifyObserver::start( + &self.db.workspace_root, + Box::new(durability), + self.policy.dependency_files(), + ) + } + + fn published_paths(&self) -> Result> { + let mut statement = self.db.conn.prepare( + "SELECT normalized_path FROM changed_path_entries + WHERE scope_id=?1 ORDER BY normalized_path COLLATE BINARY", + )?; + let paths = statement + .query_map([self.expected.scope_id.to_text()], |row| row.get(0))? + .collect::, _>>()?; + Ok(paths) + } +} + +#[cfg(debug_assertions)] +struct SlowDurability { + inner: MemoryDurability, +} + +#[cfg(debug_assertions)] +impl ObserverDurability for SlowDurability { + fn binding(&self) -> ObserverWriterBinding { + self.inner.binding() + } + + fn append_and_flush(&mut self, record: ObserverRecord) -> Result { + thread::sleep(Duration::from_millis(2)); + self.inner.append_and_flush(record) + } + + fn append_flush_and_rotate( + &mut self, + record: ObserverRecord, + ) -> Result<(DurableCut, DurableCut)> { + thread::sleep(Duration::from_millis(2)); + self.inner.append_flush_and_rotate(record) + } + + fn rotate_if_cut(&mut self, expected: &DurableCut) -> Result> { + self.inner.rotate_if_cut(expected) + } +} + +#[cfg(debug_assertions)] +fn fixture() -> std::result::Result<(tempfile::TempDir, LinuxInotifyObserver), String> { + let temp = tempfile::tempdir().map_err(|error| error.to_string())?; + let observer = LinuxInotifyObserver::start(temp.path(), Box::new(memory_durability(None)), &[]) + .map_err(|error| error.to_string())?; + Ok((temp, observer)) +} + +#[cfg(debug_assertions)] +fn expected_for(observer: &LinuxInotifyObserver, scope_byte: u8) -> ExpectedScope { + ExpectedScope { + scope_id: ScopeId([scope_byte; 32]), + epoch: 1, + ref_name: "refs/branches/main".into(), + ref_generation: 1, + baseline_root: crate::ObjectId(format!("object_linux_observer_{scope_byte}")), + policy_fingerprint: [8; 32], + policy_generation: 1, + filesystem_identity: observer.root_identity.clone(), + provider_identity: observer.provider_identity.clone(), + } +} + +#[cfg(debug_assertions)] +fn events_through( + observer: &LinuxInotifyObserver, +) -> std::result::Result, String> { + observer + .begin_observation(&expected_for(observer, 1)) + .map_err(|error| error.to_string())?; + Ok(observer + .shared + .lock() + .events + .iter() + .map(|item| item.event.clone()) + .collect()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_recursive_coverage() -> std::result::Result<(), String> { + let (temp, observer) = fixture()?; + fs::create_dir(temp.path().join("a")).map_err(|error| error.to_string())?; + fs::create_dir(temp.path().join("a/b")).map_err(|error| error.to_string())?; + fs::write(temp.path().join("a/b/file"), b"covered").map_err(|error| error.to_string())?; + let events = events_through(&observer)?; + if !events.iter().any(|event| { + event.path.as_str() == "a" && event.flags.0 & EvidenceFlags::PROVIDER_COMPLETE_PREFIX.0 != 0 + }) { + return Err("recursive directory add did not emit a complete dirty prefix".into()); + } + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_reconciliation_interval_qualification() -> std::result::Result<(), String> { + let (temp, observer) = fixture()?; + let expected = expected_for(&observer, 7); + let start = observer + .begin_observation(&expected) + .map_err(|error| error.to_string())?; + fs::write(temp.path().join("during-reconcile"), b"changed") + .map_err(|error| error.to_string())?; + let end = observer + .end_fence(&expected, &start) + .map_err(|error| error.to_string())?; + let mut drained = Vec::new(); + observer + .drain_through( + &expected, + &observer.root_identity, + &start, + &end, + &mut |event| { + drained.push(event); + Ok(()) + }, + ) + .map_err(|error| error.to_string())?; + if !has_event(&drained, "during-reconcile", EvidenceFlags::CREATE) { + return Err("qualified reconciliation interval omitted an observed change".into()); + } + Ok(()) +} + +#[cfg(debug_assertions)] +fn has_event(events: &[ObserverEvent], path: &str, flag: EvidenceFlags) -> bool { + events + .iter() + .any(|event| event.path.as_str() == path && event.flags.0 & flag.0 != 0) +} + +#[cfg(debug_assertions)] +fn expect_revoked( + observer: &LinuxInotifyObserver, + reason: &str, +) -> std::result::Result<(), String> { + let deadline = Instant::now() + FENCE_TIMEOUT; + loop { + match observer.ensure_available() { + Err(error) if error.to_string().contains(reason) => return Ok(()), + Err(error) => return Err(format!("expected {reason}, got {error}")), + Ok(()) if Instant::now() < deadline => thread::sleep(LOOP_PAUSE), + Ok(()) => return Err(format!("observer was not revoked for {reason}")), + } + } +} + +#[cfg(debug_assertions)] +pub(crate) fn run_content_mode_create_delete() -> std::result::Result<(), String> { + use std::os::unix::fs::PermissionsExt; + + let (temp, observer) = fixture()?; + let path = temp.path().join("tracked.txt"); + fs::write(&path, b"one").map_err(|error| error.to_string())?; + fs::write(&path, b"two").map_err(|error| error.to_string())?; + fs::set_permissions(&path, fs::Permissions::from_mode(0o700)) + .map_err(|error| error.to_string())?; + fs::remove_file(&path).map_err(|error| error.to_string())?; + let events = events_through(&observer)?; + for flag in [ + EvidenceFlags::CREATE, + EvidenceFlags::CONTENT, + EvidenceFlags::MODE, + EvidenceFlags::DELETE, + ] { + if !has_event(&events, "tracked.txt", flag) { + return Err(format!("missing {:?} evidence for tracked.txt", flag)); + } + } + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_rename_matrix() -> std::result::Result<(), String> { + let (temp, observer) = fixture()?; + fs::write(temp.path().join("file-a"), b"file").map_err(|error| error.to_string())?; + fs::rename(temp.path().join("file-a"), temp.path().join("file-b")) + .map_err(|error| error.to_string())?; + fs::create_dir(temp.path().join("dir-a")).map_err(|error| error.to_string())?; + fs::write(temp.path().join("dir-a/child"), b"child").map_err(|error| error.to_string())?; + fs::rename(temp.path().join("dir-a"), temp.path().join("dir-b")) + .map_err(|error| error.to_string())?; + fs::write(temp.path().join("case"), b"case").map_err(|error| error.to_string())?; + fs::rename(temp.path().join("case"), temp.path().join("CASE")) + .map_err(|error| error.to_string())?; + fs::write(temp.path().join("dir-b/after"), b"after").map_err(|error| error.to_string())?; + let events = events_through(&observer)?; + for (path, flag) in [ + ("file-a", EvidenceFlags::RENAME_FROM), + ("file-b", EvidenceFlags::RENAME_TO), + ("dir-a", EvidenceFlags::RENAME_FROM), + ("dir-b", EvidenceFlags::RENAME_TO), + ("case", EvidenceFlags::RENAME_FROM), + ("CASE", EvidenceFlags::RENAME_TO), + ] { + if !has_event(&events, path, flag) { + return Err(format!("missing rename endpoint {path}")); + } + } + if !events + .iter() + .any(|event| event.path.as_str() == "dir-b/after") + && !has_event(&events, "dir-b", EvidenceFlags::PROVIDER_COMPLETE_PREFIX) + { + return Err("directory rename was covered by neither remapped watch nor prefix".into()); + } + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_rename_storm_and_cookie_expiry() -> std::result::Result<(), String> { + let (temp, observer) = fixture()?; + fs::write(temp.path().join("storm-0"), b"storm").map_err(|error| error.to_string())?; + for index in 0..128 { + fs::rename( + temp.path().join(format!("storm-{index}")), + temp.path().join(format!("storm-{}", index + 1)), + ) + .map_err(|error| error.to_string())?; + } + let outside = tempfile::tempdir().map_err(|error| error.to_string())?; + fs::write(temp.path().join("departing"), b"departing").map_err(|error| error.to_string())?; + fs::rename( + temp.path().join("departing"), + outside.path().join("departed"), + ) + .map_err(|error| error.to_string())?; + thread::sleep(COOKIE_EXPIRY + Duration::from_millis(50)); + let events = events_through(&observer)?; + if !has_event(&events, "storm-0", EvidenceFlags::RENAME_FROM) + || !has_event(&events, "storm-128", EvidenceFlags::RENAME_TO) + { + return Err("rename storm lost an endpoint".into()); + } + if !has_event( + &events, + "departing", + EvidenceFlags::PROVIDER_COMPLETE_PREFIX, + ) { + return Err("expired rename cookie did not conservatively dirty its prefix".into()); + } + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_delayed_backlog() -> std::result::Result<(), String> { + let (temp, observer) = fixture()?; + for index in 0..512 { + fs::write(temp.path().join(format!("backlog-{index}")), b"queued") + .map_err(|error| error.to_string())?; + } + thread::sleep(Duration::from_millis(20)); + let events = events_through(&observer)?; + if !has_event(&events, "backlog-511", EvidenceFlags::CREATE) { + return Err("durable fence returned before delayed backlog".into()); + } + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_controlled_fence_queue_ordering() -> std::result::Result<(), String> { + let shared = Arc::new(Shared { + state: Mutex::new(State { + active: true, + revoked: None, + events: Vec::new(), + next_sequence: 1, + pending_renames: HashMap::new(), + issued_fences: HashMap::new(), + internal_fence_paths: BTreeSet::new(), + fail_next_watch_add: false, + policy_invalidation_pending: false, + }), + changed: Condvar::new(), + shutdown: AtomicBool::new(false), + }); + let (commands, receiver) = mpsc::sync_channel(MAX_PENDING_RECORDS); + let worker_shared = Arc::clone(&shared); + let worker = thread::spawn(move || { + run_durability_worker(receiver, Box::new(memory_durability(None)), worker_shared) + }); + commands + .send(DurabilityCommand::Record(PlannedRecord { + path: LedgerPath::parse("before-controlled-fence").map_err(|e| e.to_string())?, + flags: EvidenceFlags::CONTENT, + internal_fence: false, + })) + .map_err(|_| "linux durability worker rejected initial record".to_string())?; + let nonce = vec![7; 24]; + let (response_tx, response_rx) = mpsc::sync_channel(1); + commands + .send(DurabilityCommand::ControlledFence { + nonce: nonce.clone(), + response: response_tx, + }) + .map_err(|_| "linux durability worker rejected controlled fence".to_string())?; + const LATER_RECORDS: usize = 256; + for index in 0..LATER_RECORDS { + commands + .send(DurabilityCommand::Record(PlannedRecord { + path: LedgerPath::parse(&format!("later/{index:03}")).map_err(|e| e.to_string())?, + flags: EvidenceFlags::CONTENT, + internal_fence: false, + })) + .map_err(|_| "linux durability worker rejected later record".to_string())?; + } + let (public, sealed, anchor) = response_rx + .recv_timeout(FENCE_TIMEOUT) + .map_err(|_| "linux controlled fence response timed out".to_string())? + .map_err(|error| error.to_string())?; + if public.nonce != nonce + || public.sequence != sealed.last_sequence + || public.durable_offset != sealed.durable_end_offset + || anchor.last_sequence != sealed.last_sequence + || anchor.provider_cursor != sealed.provider_cursor + { + return Err("linux controlled fence did not return its exact sealed cut".into()); + } + let deadline = Instant::now() + FENCE_TIMEOUT; + loop { + let state = shared.lock(); + if state.events.len() >= LATER_RECORDS + 2 { + let boundary = state + .events + .iter() + .find(|event| event.event.sequence == public.sequence) + .ok_or_else(|| "linux controlled boundary event is missing".to_string())?; + if !boundary.internal_fence || boundary.cut != sealed { + return Err("linux controlled boundary lost internal provenance or cut".into()); + } + let later = state + .events + .iter() + .filter(|event| event.event.path.as_str().starts_with("later/")) + .collect::>(); + if later.len() != LATER_RECORDS + || later + .iter() + .any(|event| event.internal_fence || event.event.sequence <= public.sequence) + { + return Err( + "linux post-boundary records were suppressed or entered the sealed cut".into(), + ); + } + break; + } + if let Some(reason) = &state.revoked { + return Err(format!("linux controlled fence worker revoked: {reason}")); + } + drop(state); + if Instant::now() >= deadline { + return Err("linux later-record durability timed out".into()); + } + thread::sleep(Duration::from_millis(1)); + } + shared.shutdown.store(true, Ordering::Release); + drop(commands); + worker + .join() + .map_err(|_| "linux controlled fence worker panicked".to_string())?; + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_fence_ordering() -> std::result::Result<(), String> { + let (temp, observer) = fixture()?; + fs::write(temp.path().join("before"), b"before").map_err(|error| error.to_string())?; + let fence = observer + .begin_observation(&expected_for(&observer, 2)) + .map_err(|error| error.to_string())?; + let state = observer.shared.lock(); + let sentinel = state + .events + .iter() + .filter(|item| { + item.event + .path + .as_str() + .starts_with(".trail-observer-fence-") + }) + .collect::>(); + let create = sentinel + .iter() + .find(|item| item.event.flags.0 & EvidenceFlags::CREATE.0 != 0) + .ok_or_else(|| "fence create was not durably observed".to_string())?; + let delete = sentinel + .iter() + .find(|item| item.event.flags.0 & EvidenceFlags::DELETE.0 != 0) + .ok_or_else(|| "fence delete was not durably observed".to_string())?; + if create.event.sequence >= delete.event.sequence + || create.cut.durable_end_offset >= delete.cut.durable_end_offset + || fence.sequence != delete.event.sequence + { + return Err("sentinel durable create/delete ordering is invalid".into()); + } + drop(state); + if fs::read_dir(temp.path()) + .map_err(|error| error.to_string())? + .any(|entry| { + entry + .ok() + .and_then(|entry| entry.file_name().to_str().map(str::to_owned)) + .is_some_and(|name| name.starts_with(".trail-observer-fence-")) + }) + { + return Err("sentinel remained after the delete fence".into()); + } + + // A controlled fence is a single synthetic durability record. Retaining + // it as the next continuous tail must preserve that provenance instead of + // relabeling it as an ordinary create/delete sentinel. + let (_temp, observer) = fixture()?; + let expected = expected_for(&observer, 3); + let start = observer + .begin_observation(&expected) + .map_err(|error| error.to_string())?; + let end = observer + .end_fence(&expected, &start) + .map_err(|error| error.to_string())?; + let lease = observer.lease().map_err(|error| error.to_string())?; + observer + .drain_through_retaining_end(&expected, &lease.root_identity, &start, &end, &mut |_| { + Ok(()) + }) + .map_err(|error| error.to_string())?; + let (controlled, _, _) = observer + .controlled_end_fence(&expected, &end) + .map_err(|error| error.to_string())?; + observer + .drain_through_retaining_end( + &expected, + &lease.root_identity, + &end, + &controlled, + &mut |_| Ok(()), + ) + .map_err(|error| error.to_string())?; + observer + .authenticated_cut(&expected, &controlled) + .map_err(|error| error.to_string())?; + observer + .controlled_end_fence(&expected, &controlled) + .map_err(|error| error.to_string())?; + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_fault_revocation_matrix() -> std::result::Result<(), String> { + let temp = tempfile::tempdir().map_err(|error| error.to_string())?; + let observer = LinuxInotifyObserver::start( + temp.path(), + Box::new(SlowDurability { + inner: memory_durability(None), + }), + &[], + ) + .map_err(|error| error.to_string())?; + for index in 0..6_000 { + fs::write(temp.path().join(format!("overflow-{index}")), b"overflow") + .map_err(|error| error.to_string())?; + } + expect_revoked(&observer, "overflow")?; + + let (_temp, observer) = fixture()?; + if classify_raw_authority_event(&observer.shared, EventMask::CREATE, false).is_ok() { + return Err("raw unknown watch descriptor passed the authority classifier".into()); + } + expect_revoked(&observer, "inotify_unknown_watch_descriptor")?; + + let (temp, observer) = fixture()?; + observer.test_fail_next_watch_add(); + fs::create_dir(temp.path().join("watch-add-fails")).map_err(|error| error.to_string())?; + expect_revoked(&observer, "inotify_watch_add_failure")?; + + let (temp, observer) = fixture()?; + fs::create_dir(temp.path().join("ignored")).map_err(|error| error.to_string())?; + observer + .begin_observation(&expected_for(&observer, 3)) + .map_err(|error| error.to_string())?; + fs::remove_dir(temp.path().join("ignored")).map_err(|error| error.to_string())?; + expect_revoked(&observer, "inotify_watch_ignored")?; + + use std::os::unix::ffi::OsStringExt; + let (temp, observer) = fixture()?; + let bad = OsString::from_vec(vec![b'b', b'a', b'd', 0xff]); + fs::write(temp.path().join(bad), b"ambiguous").map_err(|error| error.to_string())?; + expect_revoked(&observer, "inotify_path_decode_ambiguity")?; + + let temp = tempfile::tempdir().map_err(|error| error.to_string())?; + let observer = + LinuxInotifyObserver::start(temp.path(), Box::new(memory_durability(Some(0))), &[]) + .map_err(|error| error.to_string())?; + fs::write(temp.path().join("durability-fails"), b"fail").map_err(|error| error.to_string())?; + expect_revoked(&observer, "inotify_durability_failure")?; + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_owner_death_and_root_replacement() -> std::result::Result<(), String> { + use std::io::BufRead; + use std::process::{Command, Stdio}; + + let native = NativeFixture::new(|_| Ok(())).map_err(|error| error.to_string())?; + let executable = std::env::current_exe().map_err(|error| error.to_string())?; + let mut child = Command::new(executable) + .arg("linux_observer_process_owner_child") + .arg("--exact") + .arg("--nocapture") + .env("TRAIL_LINUX_OBSERVER_CHILD_ROOT", &native.db.workspace_root) + .env("TRAIL_LINUX_OBSERVER_CHILD_SQLITE", "1") + .stdout(Stdio::piped()) + .spawn() + .map_err(|error| error.to_string())?; + let stdout = child + .stdout + .take() + .ok_or_else(|| "observer owner child stdout was unavailable".to_string())?; + let mut ready = false; + for line in std::io::BufReader::new(stdout).lines() { + let line = line.map_err(|error| error.to_string())?; + if line.contains("TRAIL_LINUX_OBSERVER_OWNER_READY") { + ready = true; + break; + } + } + if !ready { + let _ = child.kill(); + return Err("observer owner child exited before readiness".into()); + } + child.kill().map_err(|error| error.to_string())?; + let status = child.wait().map_err(|error| error.to_string())?; + if status.success() { + return Err("observer owner child was not killed".into()); + } + let persisted_owner: (i64, String, String) = native + .db + .conn + .query_row( + "SELECT epoch,owner_token,lease_state FROM changed_path_observer_owners + WHERE scope_id=?1", + [native.expected.scope_id.to_text()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .map_err(|error| error.to_string())?; + if persisted_owner.0 != 1 || persisted_owner.1.is_empty() || persisted_owner.2 != "active" { + return Err("killed observer owner was not persisted as the active epoch owner".into()); + } + let replacement_owner = [0xa5; 32]; + if SegmentWriter::acquire( + &native.db.sqlite_path, + &native.segment_directory, + native.expected.scope_id, + native.expected.epoch, + replacement_owner, + &hex::encode(&native.expected.provider_identity), + Vec::new(), + Duration::from_secs(3_600), + ) + .is_ok() + { + return Err("same-epoch owner replacement succeeded after SIGKILL".into()); + } + native + .db + .conn + .execute( + "UPDATE changed_path_scopes + SET epoch=2,trust_state='reconciling',trust_reason='authoritative_epoch_advance', + continuity_generation=continuity_generation+1,observer_owner_token=NULL, + durable_offset=0,folded_offset=0 + WHERE scope_id=?1 AND epoch=1", + [native.expected.scope_id.to_text()], + ) + .map_err(|error| error.to_string())?; + let writer = SegmentWriter::acquire( + &native.db.sqlite_path, + &native.segment_directory, + native.expected.scope_id, + 2, + replacement_owner, + &hex::encode(&native.expected.provider_identity), + Vec::new(), + Duration::from_secs(3_600), + ) + .map_err(|error| error.to_string())?; + let durability = SegmentWriterDurability::new( + writer, + native.expected.provider_identity.clone(), + vec![0x5a; 24], + ) + .map_err(|error| error.to_string())?; + let replacement = LinuxInotifyObserver::start( + &native.db.workspace_root, + Box::new(durability), + native.policy.dependency_files(), + ) + .map_err(|error| error.to_string())?; + let mut advanced = native.expected.clone(); + advanced.epoch = 2; + replacement + .begin_observation(&advanced) + .map_err(|error| error.to_string())?; + + let (temp, observer) = fixture()?; + let root = temp.path().to_path_buf(); + let displaced = root.with_extension("displaced"); + fs::rename(&root, &displaced).map_err(|error| error.to_string())?; + fs::create_dir(&root).map_err(|error| error.to_string())?; + expect_revoked(&observer, "inotify_root")?; + fs::remove_dir_all(&root).map_err(|error| error.to_string())?; + fs::rename(&displaced, &root).map_err(|error| error.to_string())?; + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_complete_prefix_publication_races() -> std::result::Result<(), String> { + fn run() -> Result<()> { + let fixture = NativeFixture::new_unique(|_| Ok(()))?; + let outside = tempfile::tempdir()?; + fs::create_dir_all(outside.path().join("populated/deep"))?; + fs::write(outside.path().join("populated/one"), b"one")?; + fs::write(outside.path().join("populated/deep/two"), b"two")?; + let source = outside.path().join("populated"); + let destination = fixture.db.workspace_root.join("incoming"); + let (start_move, receive_move) = mpsc::channel(); + let (moved, receive_moved) = mpsc::channel(); + let mover = thread::spawn(move || -> Result<()> { + receive_move + .recv() + .map_err(|_| Error::InvalidInput("prefix race start signal lost".into()))?; + fs::rename(source, destination)?; + moved + .send(()) + .map_err(|_| Error::InvalidInput("prefix race completion signal lost".into()))?; + Ok(()) + }); + install_initial_scan_hook(fixture.expected.scope_id, move || { + start_move + .send(()) + .map_err(|_| Error::InvalidInput("prefix race mover unavailable".into()))?; + receive_moved + .recv() + .map_err(|_| Error::InvalidInput("prefix race move acknowledgement lost".into())) + }); + let observer = fixture.observer()?; + let report = reconcile_full( + &fixture.db, + &fixture.db.changed_path_ledger(), + &observer, + &fixture.expected, + &fixture.policy, + "linux_complete_prefix_move_in", + )?; + mover + .join() + .map_err(|_| Error::InvalidInput("prefix race mover panicked".into()))??; + if !report.published { + return Err(Error::Corrupt( + "move-in prefix reconciliation did not publish".into(), + )); + } + let paths = fixture.published_paths()?; + if !paths.contains(&"incoming/one".to_string()) + || !paths.contains(&"incoming/deep/two".to_string()) + { + return Err(Error::Corrupt(format!( + "move-in prefix reconciliation omitted descendants: {paths:?}" + ))); + } + + let fixture = NativeFixture::new_unique(|root| { + fs::create_dir_all(root.join("old/deep"))?; + fs::write(root.join("old/one"), b"one")?; + fs::write(root.join("old/deep/two"), b"two")?; + Ok(()) + })?; + let old = fixture.db.workspace_root.join("old"); + let new = fixture.db.workspace_root.join("new"); + let (start_move, receive_move) = mpsc::channel(); + let (moved, receive_moved) = mpsc::channel(); + let mover = thread::spawn(move || -> Result<()> { + receive_move + .recv() + .map_err(|_| Error::InvalidInput("rename race start signal lost".into()))?; + fs::rename(old, new)?; + moved + .send(()) + .map_err(|_| Error::InvalidInput("rename race completion signal lost".into()))?; + Ok(()) + }); + install_initial_scan_hook(fixture.expected.scope_id, move || { + start_move + .send(()) + .map_err(|_| Error::InvalidInput("rename race mover unavailable".into()))?; + receive_moved + .recv() + .map_err(|_| Error::InvalidInput("rename race acknowledgement lost".into())) + }); + let observer = fixture.observer()?; + let report = reconcile_full( + &fixture.db, + &fixture.db.changed_path_ledger(), + &observer, + &fixture.expected, + &fixture.policy, + "linux_complete_prefix_directory_rename", + )?; + mover + .join() + .map_err(|_| Error::InvalidInput("rename race mover panicked".into()))??; + if !report.published { + return Err(Error::Corrupt( + "directory rename reconciliation did not publish".into(), + )); + } + let paths = fixture.published_paths()?; + for required in ["old/one", "old/deep/two", "new/one", "new/deep/two"] { + if !paths.contains(&required.to_string()) { + return Err(Error::Corrupt(format!( + "directory rename prefix reconciliation omitted {required}: {paths:?}" + ))); + } + } + Ok(()) + } + run().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_authenticated_fence_rejections() -> std::result::Result<(), String> { + fn must_reject(mutate: impl FnOnce(&mut ObserverFence)) -> std::result::Result<(), String> { + let (_temp, observer) = fixture()?; + let expected = expected_for(&observer, 0x31); + let mut start = observer + .begin_observation(&expected) + .map_err(|error| error.to_string())?; + mutate(&mut start); + if observer.end_fence(&expected, &start).is_ok() { + return Err("forged issued fence was accepted".into()); + } + Ok(()) + } + must_reject(|fence| fence.sequence = fence.sequence.saturating_add(1))?; + must_reject(|fence| fence.durable_offset = fence.durable_offset.saturating_add(1))?; + must_reject(|fence| fence.nonce[0] ^= 0xff)?; + + let (_temp, observer) = fixture()?; + let expected = expected_for(&observer, 0x32); + let other = expected_for(&observer, 0x33); + let start = observer + .begin_observation(&expected) + .map_err(|error| error.to_string())?; + if observer.end_fence(&other, &start).is_ok() { + return Err("cross-scope issued fence was accepted".into()); + } + + let (_temp, observer) = fixture()?; + let expected = expected_for(&observer, 0x34); + let start = observer + .begin_observation(&expected) + .map_err(|error| error.to_string())?; + let end = observer + .end_fence(&expected, &start) + .map_err(|error| error.to_string())?; + observer + .drain_through( + &expected, + &observer.root_identity, + &start, + &end, + &mut |_| Ok(()), + ) + .map_err(|error| error.to_string())?; + if observer + .drain_through( + &expected, + &observer.root_identity, + &start, + &end, + &mut |_| Ok(()), + ) + .is_ok() + { + return Err("consumed issued fence interval was replayed".into()); + } + + let native = NativeFixture::new(|_| Ok(())).map_err(|error| error.to_string())?; + let observer = native.observer().map_err(|error| error.to_string())?; + let start = observer + .begin_observation(&native.expected) + .map_err(|error| error.to_string())?; + native + .db + .conn + .execute( + "UPDATE changed_path_observer_owners SET lease_state='revoked' + WHERE scope_id=?1", + [native.expected.scope_id.to_text()], + ) + .map_err(|error| error.to_string())?; + if observer.end_fence(&native.expected, &start).is_ok() { + return Err("persisted observer owner replacement retained fence authority".into()); + } + + let native = NativeFixture::new(|_| Ok(())).map_err(|error| error.to_string())?; + let observer = native.observer().map_err(|error| error.to_string())?; + let start = observer + .begin_observation(&native.expected) + .map_err(|error| error.to_string())?; + native + .db + .conn + .execute( + "UPDATE changed_path_observer_segments SET owner_token='replacement-owner' + WHERE scope_id=?1", + [native.expected.scope_id.to_text()], + ) + .map_err(|error| error.to_string())?; + if observer.end_fence(&native.expected, &start).is_ok() { + return Err("persisted observer segment replacement retained fence authority".into()); + } + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_segment_writer_reconcile_publication() -> std::result::Result<(), String> { + fn run() -> Result<()> { + let fixture = NativeFixture::new(|root| { + fs::write(root.join("tracked"), b"before")?; + Ok(()) + })?; + let observer = fixture.observer()?; + fs::write(fixture.db.workspace_root.join("tracked"), b"after")?; + let report = reconcile_full( + &fixture.db, + &fixture.db.changed_path_ledger(), + &observer, + &fixture.expected, + &fixture.policy, + "linux_native_segment_writer", + )?; + if !report.published || !fixture.published_paths()?.contains(&"tracked".into()) { + return Err(Error::Corrupt( + "native SegmentWriter reconciliation did not publish".into(), + )); + } + let (scope_folded, segment_folded, owner_matches): (i64, i64, bool) = + fixture.db.conn.query_row( + "SELECT scope.folded_offset,segment.folded_end_offset, + owner.owner_token=scope.observer_owner_token + AND owner.provider_identity=scope.provider_identity + AND owner.fence_nonce IS NOT NULL + FROM changed_path_scopes scope + JOIN changed_path_observer_owners owner ON owner.scope_id=scope.scope_id + JOIN changed_path_observer_segments segment + ON segment.scope_id=scope.scope_id AND segment.owner_token=owner.owner_token + WHERE scope.scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + if scope_folded != i64::try_from(report.end_durable_offset).unwrap_or(-1) + || segment_folded != scope_folded + || !owner_matches + { + return Err(Error::Corrupt( + "native owner/fence/fold binding was not exact".into(), + )); + } + + let fixture = NativeFixture::new(|root| { + fs::write(root.join("rollback"), b"before")?; + fs::write(root.join("rollback-two"), b"before")?; + Ok(()) + })?; + let observer = fixture.observer()?; + fs::write(fixture.db.workspace_root.join("rollback"), b"after")?; + fs::write(fixture.db.workspace_root.join("rollback-two"), b"after")?; + let ledger = fixture.db.changed_path_ledger(); + let mut attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::Full, + "linux_atomic_fold_rollback", + )?; + attempt.observe(&fixture.db, &ledger, &observer, &fixture.policy)?; + let before_fold: i64 = fixture.db.conn.query_row( + "SELECT folded_end_offset FROM changed_path_observer_segments + WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + )?; + if before_fold != 0 { + return Err(Error::Corrupt( + "drain persisted folded offset before publication".into(), + )); + } + fixture.db.conn.execute( + "UPDATE changed_path_scopes SET max_candidate_rows=1 WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + )?; + if attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .is_ok() + { + return Err(Error::Corrupt( + "candidate-cap failure unexpectedly published".into(), + )); + } + let after_fold: i64 = fixture.db.conn.query_row( + "SELECT folded_end_offset FROM changed_path_observer_segments + WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + )?; + if after_fold != 0 { + return Err(Error::Corrupt( + "failed publication did not roll back folded offset".into(), + )); + } + Ok(()) + } + run().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_raw_decoder_faults() -> std::result::Result<(), String> { + let (_temp, observer) = fixture()?; + if classify_raw_authority_event(&observer.shared, EventMask::Q_OVERFLOW, false).is_ok() { + return Err("raw IN_Q_OVERFLOW passed the authority classifier".into()); + } + expect_revoked(&observer, "inotify_queue_overflow")?; + let (_temp, observer) = fixture()?; + if classify_raw_authority_event(&observer.shared, EventMask::CREATE, false).is_ok() { + return Err("raw unknown watch descriptor passed the authority classifier".into()); + } + expect_revoked(&observer, "inotify_unknown_watch_descriptor") +} + +#[cfg(debug_assertions)] +pub(crate) fn run_policy_dependency_observation() -> std::result::Result<(), String> { + let dependency_paths = [ + ".trail/config.toml", + ".git/info/exclude", + ".git/config", + ".git/config.worktree", + ".trailignore", + ]; + for relative in dependency_paths { + let fixture = NativeFixture::new_unique(|_| Ok(())).map_err(|error| error.to_string())?; + let observer = fixture.observer().map_err(|error| error.to_string())?; + let changed = fixture.db.workspace_root.join(relative); + install_initial_scan_hook(fixture.expected.scope_id, move || { + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(&changed)?; + writeln!(file, "# changed after native observation began")?; + file.sync_all()?; + Ok(()) + }); + let result = reconcile_full( + &fixture.db, + &fixture.db.changed_path_ledger(), + &observer, + &fixture.expected, + &fixture.policy, + "native_policy_dependency_change", + ); + let error = result + .err() + .ok_or_else(|| format!("policy dependency `{relative}` published false clean"))?; + if !error + .to_string() + .contains("inotify_policy_dependency_invalidated") + { + return Err(format!( + "policy dependency `{relative}` failed for the wrong reason: {error}" + )); + } + } + + let fixture = NativeFixture::new(|_| Ok(())).map_err(|error| error.to_string())?; + let observer = fixture.observer().map_err(|error| error.to_string())?; + let start = observer + .begin_observation(&fixture.expected) + .map_err(|error| error.to_string())?; + let internal_noise = fixture.db.db_dir.join("observer-internal-noise"); + for index in 0_u64..2_000 { + fs::write(&internal_noise, index.to_be_bytes()).map_err(|error| error.to_string())?; + } + fs::remove_file(&internal_noise).map_err(|error| error.to_string())?; + fixture + .db + .conn + .execute_batch("PRAGMA wal_checkpoint(PASSIVE);") + .map_err(|error| error.to_string())?; + let end = observer + .end_fence(&fixture.expected, &start) + .map_err(|error| error.to_string())?; + let mut observed = Vec::new(); + observer + .drain_through( + &fixture.expected, + &observer + .root_identity() + .map_err(|error| error.to_string())?, + &start, + &end, + &mut |event| { + observed.push(event.path.as_str().to_string()); + Ok(()) + }, + ) + .map_err(|error| error.to_string())?; + if observed + .iter() + .any(|path| path.starts_with(".trail/") || path.starts_with(".git/")) + { + return Err(format!( + "internal storage activity self-fed the durable observer: {observed:?}" + )); + } + + for parent in [".trail", ".git", ".git/info"] { + let root = tempfile::tempdir().map_err(|error| error.to_string())?; + fs::create_dir_all(root.path().join(".trail")).map_err(|error| error.to_string())?; + fs::create_dir_all(root.path().join(".git/info")).map_err(|error| error.to_string())?; + let dependencies = [ + root.path().join(".trail/config.toml"), + root.path().join(".git/config"), + root.path().join(".git/config.worktree"), + root.path().join(".git/info/exclude"), + ]; + let observer = LinuxInotifyObserver::start( + root.path(), + Box::new(memory_durability(None)), + &dependencies, + ) + .map_err(|error| error.to_string())?; + let watched_parent = root.path().join(parent); + let moved_parent = root + .path() + .join(format!("{}.replaced", parent.replace('/', "-"))); + fs::rename(&watched_parent, &moved_parent).map_err(|error| error.to_string())?; + fs::create_dir_all(&watched_parent).map_err(|error| error.to_string())?; + expect_revoked(&observer, "inotify_policy_parent_replaced")?; + } + + let root = tempfile::tempdir().map_err(|error| error.to_string())?; + let external = tempfile::tempdir().map_err(|error| error.to_string())?; + let dependency = external.path().join("policy"); + let trace = Arc::new(Mutex::new(Vec::new())); + let mut durability = memory_durability(None); + durability.trace = Some(Arc::clone(&trace)); + let observer = LinuxInotifyObserver::start( + root.path(), + Box::new(durability), + std::slice::from_ref(&dependency), + ) + .map_err(|error| error.to_string())?; + let ignored_dependency = policy_directory_invalidation_dependency( + EventMask::IGNORED, + Some(0), + &[PolicyDependencyWatch { + dependency: dependency.clone(), + observed_path: dependency.clone(), + directory_index: 0, + }], + ) + .ok_or_else(|| "policy watch IN_IGNORED was not terminal".to_string())?; + request_policy_invalidation( + &observer.shared, + &observer.records, + ignored_dependency, + "inotify_policy_parent_replaced", + ) + .map_err(|error| error.to_string())?; + let trace = trace.lock().unwrap_or_else(|poison| poison.into_inner()); + let marker = trace + .iter() + .position(|action| action.starts_with("append:.trail/policy-invalidations/")) + .ok_or_else(|| format!("IN_IGNORED omitted durable policy marker: {trace:?}"))?; + let revoke = trace + .iter() + .position(|action| action.starts_with("revoke:inotify_policy_parent_replaced")) + .ok_or_else(|| format!("IN_IGNORED omitted owner revocation: {trace:?}"))?; + if marker >= revoke { + return Err(format!( + "IN_IGNORED revoked before its durable marker: {trace:?}" + )); + } + drop(trace); + drop(observer); + + let root = tempfile::tempdir().map_err(|error| error.to_string())?; + let external = tempfile::tempdir().map_err(|error| error.to_string())?; + let dependency = external.path().join("policy"); + let observer = LinuxInotifyObserver::start( + root.path(), + Box::new(memory_durability(None)), + std::slice::from_ref(&dependency), + ) + .map_err(|error| error.to_string())?; + fs::write(&dependency, b"created").map_err(|error| error.to_string())?; + expect_revoked(&observer, "inotify_policy_dependency_invalidated")?; + + let root = tempfile::tempdir().map_err(|error| error.to_string())?; + let external = tempfile::Builder::new() + .prefix("trail-cross-device-policy-") + .tempdir_in("/dev/shm") + .map_err(|error| format!("cross-device policy fixture unavailable: {error}"))?; + let dependency = external.path().join("policy"); + let observer = LinuxInotifyObserver::start( + root.path(), + Box::new(memory_durability(None)), + std::slice::from_ref(&dependency), + ) + .map_err(|error| format!("cross-device policy dependency was not observable: {error}"))?; + fs::write(&dependency, b"created").map_err(|error| error.to_string())?; + expect_revoked(&observer, "inotify_policy_dependency_invalidated")?; + + let root = tempfile::tempdir().map_err(|error| error.to_string())?; + let external = tempfile::tempdir().map_err(|error| error.to_string())?; + let target = external.path().join("target"); + fs::create_dir(&target).map_err(|error| error.to_string())?; + std::os::unix::fs::symlink(&target, external.path().join("alias")) + .map_err(|error| error.to_string())?; + let error = LinuxInotifyObserver::start( + root.path(), + Box::new(memory_durability(None)), + &[external.path().join("alias/policy")], + ) + .err() + .ok_or_else(|| "symlinked policy parent was accepted".to_string())?; + if !error.to_string().contains("symlink_or_unsafe") { + return Err(format!( + "symlinked policy dependency failed for the wrong reason: {error}" + )); + } + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_process_owner_child(root: &str) -> std::result::Result<(), String> { + let _database; + let _observer = if std::env::var_os("TRAIL_LINUX_OBSERVER_CHILD_SQLITE").is_some() { + let database = Trail::open(Path::new(root)).map_err(|error| error.to_string())?; + let provider_identity = b"linux-inotify-native-v1".to_vec(); + let mut owner = [0_u8; 32]; + let mut fence = [0_u8; 24]; + getrandom::getrandom(&mut owner).map_err(|error| error.to_string())?; + getrandom::getrandom(&mut fence).map_err(|error| error.to_string())?; + let writer = SegmentWriter::acquire( + &database.sqlite_path, + &database.db_dir.join("change-observer-segments"), + ScopeId([0x91; 32]), + 1, + owner, + &hex::encode(&provider_identity), + Vec::new(), + Duration::from_secs(3_600), + ) + .map_err(|error| error.to_string())?; + let durability = SegmentWriterDurability::new(writer, provider_identity, fence.to_vec()) + .map_err(|error| error.to_string())?; + let observer = LinuxInotifyObserver::start(Path::new(root), Box::new(durability), &[]) + .map_err(|error| error.to_string())?; + _database = Some(database); + observer + } else { + _database = None; + LinuxInotifyObserver::start(Path::new(root), Box::new(memory_durability(None)), &[]) + .map_err(|error| error.to_string())? + }; + println!("TRAIL_LINUX_OBSERVER_OWNER_READY"); + std::io::stdout() + .flush() + .map_err(|error| error.to_string())?; + loop { + thread::sleep(Duration::from_secs(60)); + } +} diff --git a/trail/src/db/change_ledger/observer/macos.rs b/trail/src/db/change_ledger/observer/macos.rs new file mode 100644 index 0000000..56476c5 --- /dev/null +++ b/trail/src/db/change_ledger/observer/macos.rs @@ -0,0 +1,5030 @@ +//! Qualified macOS FSEvents observer. +//! +//! The callback is deliberately small: it validates the native flags and +//! path, then performs one bounded `try_send`. Segment I/O and SQLite lease +//! validation belong exclusively to the durability worker. + +use std::collections::{BTreeMap, HashMap}; +use std::ffi::CStr; +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::os::raw::c_void; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; +use std::path::{Component, Path, PathBuf}; +#[cfg(debug_assertions)] +use std::ptr; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, SyncSender, TrySendError}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use core_foundation_sys::base::{CFRelease, CFTypeRef}; +use core_foundation_sys::uuid::{CFUUIDGetUUIDBytes, CFUUIDRef}; +use fsevent_sys as fs_events; +use rustix::fs::{fsync, openat, unlinkat, AtFlags, Mode, OFlags}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::{ObserverFence, ObserverLease, QualifiedObserver}; +use crate::db::change_ledger::reconcile::{ObserverEvent, ObserverQualification}; +use crate::db::change_ledger::secure_fs::SecureDirectory; +use crate::db::change_ledger::{ + capture_policy_dependency_fence_identity, + capture_policy_dependency_fence_identity_at_observed_path, DurableCut, EvidenceFlags, + EvidenceSource, ExpectedScope, LedgerPath, ObserverRecord, ObserverWriterBinding, + PolicyDependencyFenceIdentity, ProviderCapabilities, SegmentWriter, +}; +#[cfg(debug_assertions)] +use crate::db::change_ledger::{ + BaselineIdentity, FilesystemIdentity, PolicyIdentity, ProviderIdentity, ScopeId, ScopeIdentity, + ScopeKind, +}; +use crate::error::{Error, Result}; +#[cfg(debug_assertions)] +use crate::{InitImportMode, Trail}; + +const MAX_PENDING_RECORDS: usize = 8_192; +const MAX_RETAINED_EVENTS: usize = 65_536; +const MAX_DIRECT_POLICY_DEPENDENCIES: usize = 1_024; +const FENCE_TIMEOUT: Duration = Duration::from_secs(10); + +const CAPABILITY_VERSION: u16 = 4; +const STREAM_FLAGS: u32 = fs_events::kFSEventStreamCreateFlagFileEvents + | fs_events::kFSEventStreamCreateFlagNoDefer + | fs_events::kFSEventStreamCreateFlagWatchRoot; +const GAP_FLAGS: u32 = fs_events::kFSEventStreamEventFlagMustScanSubDirs + | fs_events::kFSEventStreamEventFlagUserDropped + | fs_events::kFSEventStreamEventFlagKernelDropped + | fs_events::kFSEventStreamEventFlagEventIdsWrapped + | fs_events::kFSEventStreamEventFlagRootChanged + | fs_events::kFSEventStreamEventFlagUnmount; + +static NULL_CONTEXT_GENERATION: AtomicU64 = AtomicU64::new(0); + +#[link(name = "CoreServices", kind = "framework")] +unsafe extern "C" { + fn FSEventsCopyUUIDForDevice(device: libc::dev_t) -> CFUUIDRef; +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct HistoryAuthority { + device: u64, + database_uuid: [u8; 16], + device_relative_root: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +struct CursorCoverageRoot { + device_relative_root: String, + root_identity: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +struct SystemAliasBinding { + alias_path: PathBuf, + canonical_target: PathBuf, + alias_identity: Vec, + target_identity: Vec, +} + +struct CoverageRoot { + absolute_root: PathBuf, + device_relative_root: String, + root_identity: Vec, + root: File, +} + +#[derive(Clone)] +struct CallbackCoverageRoot { + absolute_root: PathBuf, + device_relative_root: PathBuf, +} + +#[derive(Clone, Debug)] +struct PolicyWatch { + observed_path: PathBuf, + dependency: PathBuf, + identity: PolicyDependencyFenceIdentity, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +struct ExternalPolicyFence { + dependency: PathBuf, + observed_path: PathBuf, + identity: PolicyDependencyFenceIdentity, +} + +struct CoveragePlan { + roots: Vec, + stream_roots: Vec, + system_aliases: Vec, + policy_dependencies: Vec, + policy_watches: Vec, + external_policy_fences: Vec, +} + +pub(crate) trait MacObserverDurability: Send { + fn binding(&self) -> ObserverWriterBinding; + fn append_and_flush(&mut self, record: ObserverRecord) -> Result; + fn append_flush_and_rotate( + &mut self, + record: ObserverRecord, + ) -> Result<(DurableCut, DurableCut)>; + fn rotate_if_cut(&mut self, expected: &DurableCut) -> Result>; + fn heartbeat(&mut self) -> Result<()> { + Ok(()) + } + fn revoke_owner(&mut self, reason: &str) -> Result<()>; +} + +pub(crate) struct MacSegmentWriterDurability { + writer: SegmentWriter, + binding: ObserverWriterBinding, +} + +impl MacSegmentWriterDurability { + pub(crate) fn new( + mut writer: SegmentWriter, + provider_identity: Vec, + fence_nonce: Vec, + ) -> Result { + let binding = writer.bind_native_observer(provider_identity, fence_nonce)?; + Ok(Self { writer, binding }) + } +} + +impl MacObserverDurability for MacSegmentWriterDurability { + fn binding(&self) -> ObserverWriterBinding { + self.binding.clone() + } + + fn append_and_flush(&mut self, record: ObserverRecord) -> Result { + self.writer.append_and_flush(&[record]) + } + + fn append_flush_and_rotate( + &mut self, + record: ObserverRecord, + ) -> Result<(DurableCut, DurableCut)> { + self.writer.append_flush_and_rotate(&[record]) + } + + fn rotate_if_cut(&mut self, expected: &DurableCut) -> Result> { + self.writer.rotate_if_cut(expected) + } + + fn heartbeat(&mut self) -> Result<()> { + self.writer.heartbeat() + } + + fn revoke_owner(&mut self, reason: &str) -> Result<()> { + self.writer.revoke(reason) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct MacOsProviderCursor { + version: u16, + event_id: u64, + device: u64, + history_database_uuid: [u8; 16], + device_relative_root: String, + coverage_roots: Vec, + system_aliases: Vec, + policy_dependencies: Vec, + external_policy_fences: Vec, + root_identity: Vec, + lineage_identity: Vec, + provider_identity: Vec, + stream_flags: u32, + capabilities: ProviderCapabilities, +} + +impl MacOsProviderCursor { + pub(crate) fn decode(bytes: &[u8]) -> Result { + crate::error::from_cbor(bytes) + } + + fn encode(&self) -> Result> { + crate::error::cbor(self) + } + + fn validate_resume( + &self, + root_identity: &[u8], + authority: &HistoryAuthority, + coverage: &CoveragePlan, + provider_identity: &[u8], + ) -> Result<()> { + let coverage_roots = cursor_coverage_roots(coverage); + if self.version != CAPABILITY_VERSION + || self.event_id == fs_events::kFSEventStreamEventIdSinceNow + || self.device != authority.device + || self.history_database_uuid != authority.database_uuid + || self.device_relative_root != authority.device_relative_root + || self.coverage_roots != coverage_roots + || self.system_aliases != coverage.system_aliases + || self.policy_dependencies != coverage.policy_dependencies + || self.external_policy_fences != coverage.external_policy_fences + || self.root_identity != root_identity + || self.lineage_identity.len() < 16 + || self.provider_identity != provider_identity + || self.stream_flags != STREAM_FLAGS + || self.capabilities != native_capabilities() + { + return Err(reconcile_error( + "fsevents_resume_identity_or_capability_mismatch", + )); + } + // The per-device-before-time query is a conservative journal lookup + // and may lag IDs already delivered to a live stream. Event IDs are a + // host-wide clock, so the host current ID is the safe future bound; + // device/history authority comes from the exact UUID/device/path above. + let current = unsafe { fs_events::FSEventsGetCurrentEventId() }; + if self.event_id > current { + return Err(reconcile_error(&format!( + "fsevents_resume_cursor_is_from_replaced_history: cursor_event_id={} device_event_id={current}", + self.event_id, + ))); + } + Ok(()) + } +} + +#[derive(Clone)] +struct DurableEvent { + event: ObserverEvent, + provider_event_id: u64, + cut: DurableCut, + internal_fence: bool, +} + +#[derive(Clone)] +enum IssuedFenceKind { + Start, + TailAnchor, + RotationAnchor, + End { start_nonce: Vec }, +} + +#[derive(Clone)] +struct IssuedFence { + public: ObserverFence, + expected: ExpectedScope, + root_identity: Vec, + owner_token: String, + owner_fence_nonce: Vec, + provider_event_id: u64, + durable_cut: DurableCut, + kind: IssuedFenceKind, +} + +struct State { + active: bool, + revoked: Option, + history_pending: usize, + events: Vec, + next_sequence: u64, + last_provider_event_id: u64, + last_cursor: Option, + issued_fences: HashMap, IssuedFence>, +} + +struct Shared { + state: Mutex, + changed: Condvar, + shutdown: AtomicBool, +} + +impl Shared { + fn lock(&self) -> MutexGuard<'_, State> { + self.state + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + } + + fn revoke(&self, reason: impl Into) { + let mut state = self.lock(); + if state.revoked.is_none() { + state.revoked = Some(reason.into()); + } + state.active = false; + self.changed.notify_all(); + } +} + +#[derive(Clone)] +struct CallbackContext { + root_path: PathBuf, + device_relative_root: PathBuf, + coverage_roots: Vec, + policy_watches: Vec, + records: SyncSender, + shared: Arc, +} + +enum DurabilityCommand { + Record { + path: LedgerPath, + flags: EvidenceFlags, + provider_event_id: u64, + }, + PolicyInvalidation { + dependency: PathBuf, + provider_event_id: u64, + }, + PolicyEvent { + watch: PolicyWatch, + provider_event_id: u64, + }, + DirectPolicyInvalidation { + dependency: PathBuf, + response: SyncSender>, + }, + Fence { + minimum_provider_event_id: u64, + nonce: Vec, + response: SyncSender>, + }, + ControlledFence { + minimum_provider_event_id: u64, + nonce: Vec, + response: SyncSender>, + }, + Rotate { + expected: DurableCut, + response: SyncSender>>, + }, + #[cfg(debug_assertions)] + StopForTest, + Shutdown, +} + +#[derive(Clone)] +struct StreamHandle { + streams: Vec, + run_loop: usize, +} + +struct WorkerHandle { + name: &'static str, + join: JoinHandle<()>, + done: Receiver<()>, +} + +enum StartupDecision { + Publish, + Cancel, +} + +#[derive(Clone)] +struct StartOptions { + timeout: Duration, + authority_override: Option, + post_start_database_uuid_override: Option<[u8; 16]>, + delay_after_native_start: Duration, + cleanup_observed: Option>, +} + +impl StartOptions { + fn production() -> Self { + Self { + timeout: FENCE_TIMEOUT, + authority_override: None, + post_start_database_uuid_override: None, + delay_after_native_start: Duration::ZERO, + cleanup_observed: None, + } + } +} + +pub(crate) struct MacOsFseventsObserver { + root_path: PathBuf, + root: File, + root_identity: Vec, + fence_directory: SecureDirectory, + fence_directory_identity: (u64, u64), + device: u64, + provider_identity: Vec, + owner_token: String, + owner_fence_nonce: Vec, + policy_dependencies: Vec, + external_policy_fences: Vec, + lineage_identity: Vec, + history_authority: HistoryAuthority, + coverage_roots: Vec, + system_aliases: Vec, + null_context_generation: u64, + shared: Arc, + commands: SyncSender, + stream: StreamHandle, + workers: Mutex>, + #[cfg(debug_assertions)] + next_test_fence_nonce: Mutex>>, + #[cfg(debug_assertions)] + fail_next_fence_sync: Mutex, + #[cfg(debug_assertions)] + fail_next_root_descriptor: Mutex, + #[cfg(debug_assertions)] + fail_next_coverage_descriptor: Mutex, + #[cfg(test)] + fail_next_direct_policy_fence: Mutex, + #[cfg(debug_assertions)] + next_history_authority_override: Mutex>, +} + +impl MacOsFseventsObserver { + pub(crate) fn authenticated_cut( + &self, + expected: &ExpectedScope, + fence: &ObserverFence, + ) -> Result { + Ok(self.issued_fence(expected, fence)?.durable_cut) + } + + pub(crate) fn install_rotation_anchor( + &self, + expected: &ExpectedScope, + end: &ObserverFence, + anchor_cut: DurableCut, + ) -> Result { + self.issued_fence(expected, end)?; + if anchor_cut.last_sequence != end.sequence { + return Err(reconcile_error( + "fsevents_rotation_anchor_sequence_mismatch", + )); + } + let mut nonce = vec![0_u8; 24]; + getrandom::getrandom(&mut nonce) + .map_err(|error| Error::InvalidInput(format!("rotation anchor entropy: {error}")))?; + let public = ObserverFence { + sequence: anchor_cut.last_sequence, + durable_offset: anchor_cut.durable_end_offset, + nonce: nonce.clone(), + }; + let provider_event_id = self.shared.lock().last_provider_event_id; + self.shared.lock().issued_fences.insert( + nonce, + IssuedFence { + public: public.clone(), + expected: expected.clone(), + root_identity: self.root_identity.clone(), + owner_token: self.owner_token.clone(), + owner_fence_nonce: self.owner_fence_nonce.clone(), + provider_event_id, + durable_cut: anchor_cut, + kind: IssuedFenceKind::RotationAnchor, + }, + ); + Ok(public) + } + + pub(crate) fn seal_after_fence( + &self, + expected: &ExpectedScope, + fence: &ObserverFence, + ) -> Result> { + let issued = self.issued_fence(expected, fence)?; + let (response_tx, response_rx) = mpsc::sync_channel(1); + self.commands + .send(DurabilityCommand::Rotate { + expected: issued.durable_cut, + response: response_tx, + }) + .map_err(|_| reconcile_error("fsevents_rotation_worker_unavailable"))?; + match response_rx.recv_timeout(FENCE_TIMEOUT) { + Ok(result) => result, + Err(_) => { + self.shared.revoke("fsevents_rotation_timeout"); + Err(reconcile_error("fsevents_rotation_timeout")) + } + } + } + + /// End an authenticated native interval and seal its exact durable tail + /// in one durability-worker turn. Native records queued before the + /// controlled fence are included in the sealed segment; records queued + /// after it can only be appended to the linked rotation anchor. + pub(crate) fn controlled_end_fence( + &self, + expected: &ExpectedScope, + start: &ObserverFence, + ) -> Result<(ObserverFence, DurableCut, DurableCut)> { + let issued_start = self.issued_fence(expected, start)?; + if !matches!( + issued_start.kind, + IssuedFenceKind::Start | IssuedFenceKind::TailAnchor | IssuedFenceKind::RotationAnchor + ) { + self.shared + .revoke("fsevents_controlled_start_kind_mismatch"); + return Err(reconcile_error("fsevents_controlled_start_kind_mismatch")); + } + let barrier = self.flush_fence( + expected, + IssuedFenceKind::End { + start_nonce: start.nonce.clone(), + }, + )?; + let issued_barrier = self.issued_fence(expected, &barrier)?; + let mut nonce = vec![0_u8; 24]; + getrandom::getrandom(&mut nonce).map_err(|error| { + Error::InvalidInput(format!("controlled FSEvents fence entropy failed: {error}")) + })?; + let (response_tx, response_rx) = mpsc::sync_channel(1); + self.commands + .send(DurabilityCommand::ControlledFence { + minimum_provider_event_id: issued_barrier.provider_event_id, + nonce: nonce.clone(), + response: response_tx, + }) + .map_err(|_| reconcile_error("fsevents_controlled_fence_worker_unavailable"))?; + let (public, sealed, anchor, provider_event_id) = + match response_rx.recv_timeout(FENCE_TIMEOUT) { + Ok(result) => result?, + Err(_) => { + self.shared.revoke("fsevents_controlled_fence_timeout"); + return Err(reconcile_error("fsevents_controlled_fence_timeout")); + } + }; + if public.nonce != nonce + || public.sequence != sealed.last_sequence + || public.durable_offset != sealed.durable_end_offset + || sealed.last_sequence != anchor.last_sequence + || sealed.provider_cursor != anchor.provider_cursor + { + self.shared.revoke("fsevents_controlled_fence_cut_mismatch"); + return Err(reconcile_error("fsevents_controlled_fence_cut_mismatch")); + } + self.ensure_history_authority()?; + self.root_identity()?; + let issued = IssuedFence { + public: public.clone(), + expected: expected.clone(), + root_identity: self.root_identity.clone(), + owner_token: self.owner_token.clone(), + owner_fence_nonce: self.owner_fence_nonce.clone(), + provider_event_id, + durable_cut: sealed.clone(), + kind: IssuedFenceKind::End { + start_nonce: start.nonce.clone(), + }, + }; + let mut state = self.shared.lock(); + state.issued_fences.remove(&barrier.nonce); + state.issued_fences.insert(nonce, issued); + drop(state); + if public.sequence <= start.sequence + || (issued_start.durable_cut.segment_id == sealed.segment_id + && public.durable_offset < start.durable_offset) + { + self.shared + .revoke("fsevents_controlled_fence_non_monotonic"); + return Err(reconcile_error("fsevents_controlled_fence_non_monotonic")); + } + Ok((public, sealed, anchor)) + } + + fn rebind_tail_anchor( + &self, + previous: &ExpectedScope, + next: &ExpectedScope, + anchor: &ObserverFence, + ) -> Result<()> { + let mut state = self.shared.lock(); + let retained = state + .issued_fences + .get_mut(&anchor.nonce) + .ok_or_else(|| reconcile_error("fsevents_retained_tail_rebind_missing"))?; + if retained.public != *anchor + || retained.expected != *previous + || !matches!(retained.kind, IssuedFenceKind::TailAnchor) + || !stable_observer_binding(previous, next) + { + return Err(reconcile_error("fsevents_retained_tail_rebind_mismatch")); + } + retained.expected = next.clone(); + Ok(()) + } + + pub(crate) fn start( + root_path: &Path, + durability: Box, + resume: Option, + policy_dependencies: &[PathBuf], + ) -> Result { + ensure_apfs(root_path)?; + Self::start_inner( + root_path, + durability, + resume, + policy_dependencies, + StartOptions::production(), + ) + } + + fn start_inner( + root_path: &Path, + durability: Box, + resume: Option, + policy_dependencies: &[PathBuf], + options: StartOptions, + ) -> Result { + let requested_root = root_path.to_path_buf(); + let root_path = root_path.canonicalize()?; + let root = open_root_no_follow(&root_path)?; + let root_identity = root_identity(&root)?; + let lease_policy_dependencies = policy_dependencies + .iter() + .map(|dependency| normalize_absolute_dependency(&requested_root, dependency)) + .collect::>>()?; + let secure_root = SecureDirectory::open_absolute(&root_path)?; + let trail_directory = secure_root + .open_dir(".trail") + .map_err(|_| reconcile_error("fsevents_workspace_storage_absent_or_unsafe"))?; + let fence_directory = match trail_directory.open_private_dir("observer-fences") { + Ok(directory) => directory, + Err(_) => trail_directory.create_private_dir("observer-fences")?, + }; + let fence_directory_identity = fence_directory.identity()?; + let device = root.metadata()?.dev(); + let authority = options + .authority_override + .clone() + .unwrap_or(actual_history_authority(&root_path, device)?); + if authority.device != device { + return Err(reconcile_error("fsevents_actual_history_device_mismatch")); + } + let coverage = build_coverage_plan( + &root_path, + &requested_root, + device, + &lease_policy_dependencies, + )?; + let null_context_generation = NULL_CONTEXT_GENERATION.load(Ordering::Acquire); + let binding = durability.binding(); + if binding.owner_token.is_empty() + || binding.provider_id != hex::encode(&binding.provider_identity) + || binding.provider_identity.is_empty() + || binding.fence_nonce.len() < 16 + { + return Err(Error::InvalidInput( + "native macOS observer durability binding is incomplete".into(), + )); + } + if let Some(cursor) = &resume { + cursor.validate_resume( + &root_identity, + &authority, + &coverage, + &binding.provider_identity, + )?; + } + let mut lineage_identity = resume + .as_ref() + .map(|cursor| cursor.lineage_identity.clone()) + .unwrap_or_else(|| vec![0_u8; 24]); + if resume.is_none() { + getrandom::getrandom(&mut lineage_identity).map_err(|error| { + Error::InvalidInput(format!("FSEvents stream identity entropy failed: {error}")) + })?; + } + let since_when = resume + .as_ref() + .map(|cursor| cursor.event_id) + .unwrap_or(fs_events::kFSEventStreamEventIdSinceNow); + let shared = Arc::new(Shared { + state: Mutex::new(State { + active: true, + revoked: None, + history_pending: if resume.is_some() { + coverage.stream_roots.len() + } else { + 0 + }, + events: Vec::new(), + next_sequence: 1, + last_provider_event_id: resume.as_ref().map_or(0, |cursor| cursor.event_id), + last_cursor: resume.clone(), + issued_fences: HashMap::new(), + }), + changed: Condvar::new(), + shutdown: AtomicBool::new(false), + }); + let (commands, records) = mpsc::sync_channel(MAX_PENDING_RECORDS); + let durability_shared = Arc::clone(&shared); + let cursor_template = MacOsProviderCursor { + version: CAPABILITY_VERSION, + event_id: 0, + device, + history_database_uuid: authority.database_uuid, + device_relative_root: authority.device_relative_root.clone(), + coverage_roots: cursor_coverage_roots(&coverage), + system_aliases: coverage.system_aliases.clone(), + policy_dependencies: coverage.policy_dependencies.clone(), + external_policy_fences: coverage.external_policy_fences.clone(), + root_identity: root_identity.clone(), + lineage_identity: lineage_identity.clone(), + provider_identity: binding.provider_identity.clone(), + stream_flags: STREAM_FLAGS, + capabilities: native_capabilities(), + }; + let (durability_done_tx, durability_done_rx) = mpsc::sync_channel(1); + let durability_worker = thread::Builder::new() + .name("trail-macos-observer-durability".into()) + .spawn(move || { + run_durability_worker(records, durability, durability_shared, cursor_template); + let _ = durability_done_tx.send(()); + })?; + + let callback = CallbackContext { + root_path: root_path.clone(), + device_relative_root: PathBuf::from(&authority.device_relative_root), + coverage_roots: coverage + .roots + .iter() + .map(|root| CallbackCoverageRoot { + absolute_root: root.absolute_root.clone(), + device_relative_root: PathBuf::from(&root.device_relative_root), + }) + .collect(), + policy_watches: coverage.policy_watches.clone(), + records: commands.clone(), + shared: Arc::clone(&shared), + }; + let (ready_tx, ready_rx) = mpsc::sync_channel(1); + let (decision_tx, decision_rx) = mpsc::sync_channel(1); + let startup_cancelled = Arc::new(AtomicBool::new(false)); + let stream_shared = Arc::clone(&shared); + let stream_authority = authority.clone(); + let stream_cancelled = Arc::clone(&startup_cancelled); + let delay_after_native_start = options.delay_after_native_start; + let post_start_database_uuid_override = options.post_start_database_uuid_override; + let cleanup_observed = options.cleanup_observed.clone(); + let watched_roots = coverage.stream_roots.clone(); + let (stream_done_tx, stream_done_rx) = mpsc::sync_channel(1); + let stream_worker = match thread::Builder::new() + .name("trail-macos-fsevents".into()) + .spawn(move || { + run_stream( + stream_authority, + watched_roots, + since_when, + callback, + ready_tx, + decision_rx, + stream_cancelled, + post_start_database_uuid_override, + delay_after_native_start, + cleanup_observed, + stream_shared, + ); + let _ = stream_done_tx.send(()); + }) { + Ok(worker) => worker, + Err(error) => { + shared.shutdown.store(true, Ordering::Release); + let _ = commands.send(DurabilityCommand::Shutdown); + let _ = durability_worker.join(); + return Err(Error::Io(error)); + } + }; + let stream = match ready_rx.recv_timeout(options.timeout) { + Ok(Ok(stream)) => { + if decision_tx.send(StartupDecision::Publish).is_err() { + shared.revoke("fsevents_startup_publish_handshake_lost"); + shared.shutdown.store(true, Ordering::Release); + let _ = commands.try_send(DurabilityCommand::Shutdown); + let _ = stream_worker.join(); + let _ = durability_worker.join(); + return Err(reconcile_error("fsevents_startup_publish_handshake_lost")); + } + stream + } + Ok(Err(error)) => { + startup_cancelled.store(true, Ordering::Release); + let _ = decision_tx.try_send(StartupDecision::Cancel); + shared.revoke("fsevents_stream_start_failure"); + shared.shutdown.store(true, Ordering::Release); + let _ = commands.send(DurabilityCommand::Shutdown); + let _ = stream_worker.join(); + let _ = durability_worker.join(); + return Err(error); + } + Err(_) => { + startup_cancelled.store(true, Ordering::Release); + let _ = decision_tx.try_send(StartupDecision::Cancel); + shared.revoke("fsevents_stream_start_timeout"); + shared.shutdown.store(true, Ordering::Release); + let _ = commands.try_send(DurabilityCommand::Shutdown); + drop(stream_worker); + drop(durability_worker); + return Err(reconcile_error("fsevents_stream_start_timeout")); + } + }; + let observer = Self { + root_path, + root, + root_identity, + fence_directory, + fence_directory_identity, + device, + provider_identity: binding.provider_identity, + owner_token: binding.owner_token, + owner_fence_nonce: binding.fence_nonce, + policy_dependencies: lease_policy_dependencies, + external_policy_fences: coverage.external_policy_fences, + lineage_identity, + history_authority: authority, + coverage_roots: coverage.roots, + system_aliases: coverage.system_aliases, + null_context_generation, + shared, + commands, + stream, + workers: Mutex::new(vec![ + WorkerHandle { + name: "FSEvents run loop", + join: stream_worker, + done: stream_done_rx, + }, + WorkerHandle { + name: "observer durability", + join: durability_worker, + done: durability_done_rx, + }, + ]), + #[cfg(debug_assertions)] + next_test_fence_nonce: Mutex::new(None), + #[cfg(debug_assertions)] + fail_next_fence_sync: Mutex::new(false), + #[cfg(debug_assertions)] + fail_next_root_descriptor: Mutex::new(false), + #[cfg(debug_assertions)] + fail_next_coverage_descriptor: Mutex::new(false), + #[cfg(test)] + fail_next_direct_policy_fence: Mutex::new(false), + #[cfg(debug_assertions)] + next_history_authority_override: Mutex::new(None), + }; + observer.ensure_null_context_generation()?; + observer.wait_for_history()?; + observer.ensure_history_authority()?; + observer.root_identity()?; + Ok(observer) + } + + pub(crate) fn capabilities(&self) -> ProviderCapabilities { + native_capabilities() + } + + pub(crate) fn lease(&self) -> Result { + self.ensure_history_authority()?; + let direct_policy_fences = self + .external_policy_fences + .iter() + .map(|fence| (fence.dependency.clone(), fence.identity.clone())) + .collect::>(); + let direct_paths = direct_policy_fences + .iter() + .map(|(path, _)| path) + .collect::>(); + Ok(ObserverLease { + owner_token: self.owner_token.clone(), + root_identity: self.root_identity()?, + provider_identity: self.provider_identity.clone(), + policy_dependencies: self + .policy_dependencies + .iter() + .filter(|path| !direct_paths.contains(path)) + .cloned() + .collect(), + direct_policy_fences, + capabilities: self.capabilities(), + }) + } + + pub(crate) fn resume_cursor(&self) -> Result> { + self.ensure_available()?; + Ok(self.shared.lock().last_cursor.clone()) + } + + fn ensure_available(&self) -> Result<()> { + self.ensure_null_context_generation()?; + let state = self.shared.lock(); + if let Some(reason) = &state.revoked { + return Err(reconcile_error(reason)); + } + if !state.active { + return Err(reconcile_error("fsevents_observer_unavailable")); + } + if state.history_pending != 0 { + return Err(reconcile_error("fsevents_history_not_complete")); + } + Ok(()) + } + + fn ensure_null_context_generation(&self) -> Result<()> { + if NULL_CONTEXT_GENERATION.load(Ordering::Acquire) != self.null_context_generation { + self.shared + .revoke("fsevents_null_callback_context_generation_changed"); + return Err(reconcile_error( + "fsevents_null_callback_context_generation_changed", + )); + } + Ok(()) + } + + fn ensure_history_authority(&self) -> Result<()> { + self.ensure_available()?; + #[cfg(debug_assertions)] + let injected = self + .next_history_authority_override + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .take(); + #[cfg(not(debug_assertions))] + let injected: Option = None; + let observed = match injected { + Some(authority) => authority, + None => match actual_history_authority(&self.root_path, self.device) { + Ok(authority) => authority, + Err(_) => { + self.shared + .revoke("fsevents_history_authority_revalidation_failure"); + return Err(reconcile_error( + "fsevents_history_authority_revalidation_failure", + )); + } + }, + }; + self.ensure_available()?; + if observed != self.history_authority { + self.shared + .revoke("fsevents_history_authority_revalidation_mismatch"); + return Err(reconcile_error( + "fsevents_history_authority_revalidation_mismatch", + )); + } + self.ensure_coverage_roots() + } + + fn ensure_coverage_roots(&self) -> Result<()> { + #[cfg(test)] + { + let mut fail = self + .fail_next_direct_policy_fence + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if *fail { + *fail = false; + drop(fail); + return self.revoke_direct_policy_dependency(Path::new("test-after-end-fence")); + } + } + #[cfg(debug_assertions)] + { + let mut fail = self + .fail_next_coverage_descriptor + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if *fail { + *fail = false; + drop(fail); + self.shared + .revoke("fsevents_policy_coverage_descriptor_failure"); + return Err(reconcile_error( + "fsevents_policy_coverage_descriptor_failure", + )); + } + } + for pinned in &self.system_aliases { + let observed = match pinned.alias_path.as_path() { + path if path == Path::new("/etc") => exact_system_alias_binding( + path, + Path::new("/private/etc"), + Path::new("private/etc"), + ), + path if path == Path::new("/var") => exact_system_alias_binding( + path, + Path::new("/private/var"), + Path::new("private/var"), + ), + path if path == Path::new("/tmp") => exact_system_alias_binding( + path, + Path::new("/private/tmp"), + Path::new("private/tmp"), + ), + _ => Err(reconcile_error( + "fsevents_system_policy_alias_is_not_allowlisted", + )), + } + .map_err(|_| { + self.shared + .revoke("fsevents_system_policy_alias_revalidation_failure"); + reconcile_error("fsevents_system_policy_alias_revalidation_failure") + })?; + if &observed != pinned { + self.shared + .revoke("fsevents_system_policy_alias_revalidation_mismatch"); + return Err(reconcile_error( + "fsevents_system_policy_alias_revalidation_mismatch", + )); + } + } + for coverage in &self.coverage_roots { + let descriptor = root_identity(&coverage.root).map_err(|_| { + self.shared + .revoke("fsevents_policy_coverage_descriptor_failure"); + reconcile_error("fsevents_policy_coverage_descriptor_failure") + })?; + let named = open_root_no_follow(&coverage.absolute_root) + .and_then(|root| root_identity(&root)) + .map_err(|_| { + self.shared.revoke("fsevents_policy_coverage_named_failure"); + reconcile_error("fsevents_policy_coverage_named_failure") + })?; + if descriptor != coverage.root_identity || named != coverage.root_identity { + self.shared.revoke("fsevents_policy_coverage_replaced"); + return Err(reconcile_error("fsevents_policy_coverage_replaced")); + } + } + for pinned in &self.external_policy_fences { + let observed = match capture_policy_dependency_fence_identity_at_observed_path( + &pinned.dependency, + &pinned.observed_path, + ) { + Ok(observed) => observed, + Err(_) => return self.revoke_direct_policy_dependency(&pinned.dependency), + }; + if observed != pinned.identity { + return self.revoke_direct_policy_dependency(&pinned.dependency); + } + } + Ok(()) + } + + fn revoke_direct_policy_dependency(&self, dependency: &Path) -> Result<()> { + let reason = format!( + "fsevents_policy_dependency_invalidated:{}", + dependency.display() + ); + let (response_tx, response_rx) = mpsc::sync_channel(1); + self.commands + .send(DurabilityCommand::DirectPolicyInvalidation { + dependency: dependency.to_path_buf(), + response: response_tx, + }) + .map_err(|_| reconcile_error("fsevents_durability_worker_disconnected"))?; + match response_rx.recv_timeout(FENCE_TIMEOUT) { + Ok(Ok(())) => Err(reconcile_error(&reason)), + Ok(Err(error)) => Err(error), + Err(_) => { + self.shared + .revoke("fsevents_direct_policy_revocation_timeout"); + Err(reconcile_error("fsevents_direct_policy_revocation_timeout")) + } + } + } + + fn wait_for_history(&self) -> Result<()> { + let deadline = Instant::now() + FENCE_TIMEOUT; + let mut state = self.shared.lock(); + while state.history_pending != 0 && state.revoked.is_none() { + if NULL_CONTEXT_GENERATION.load(Ordering::Acquire) != self.null_context_generation { + drop(state); + self.shared + .revoke("fsevents_null_callback_context_generation_changed"); + return Err(reconcile_error( + "fsevents_null_callback_context_generation_changed", + )); + } + let now = Instant::now(); + if now >= deadline { + drop(state); + self.shared.revoke("fsevents_history_done_timeout"); + return Err(reconcile_error("fsevents_history_done_timeout")); + } + let waited = self + .shared + .changed + .wait_timeout( + state, + deadline + .saturating_duration_since(now) + .min(Duration::from_millis(25)), + ) + .unwrap_or_else(|poison| poison.into_inner()); + state = waited.0; + } + if let Some(reason) = &state.revoked { + return Err(reconcile_error(reason)); + } + drop(state); + self.ensure_null_context_generation() + } + + pub(crate) fn root_identity(&self) -> Result> { + self.ensure_available()?; + #[cfg(debug_assertions)] + { + let mut fail = self + .fail_next_root_descriptor + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if *fail { + *fail = false; + drop(fail); + self.shared + .revoke("fsevents_root_descriptor_revalidation_failure"); + return Err(reconcile_error( + "fsevents_root_descriptor_revalidation_failure", + )); + } + } + let descriptor_identity = match root_identity(&self.root) { + Ok(identity) => identity, + Err(_) => { + self.shared + .revoke("fsevents_root_descriptor_revalidation_failure"); + return Err(reconcile_error( + "fsevents_root_descriptor_revalidation_failure", + )); + } + }; + let named_identity = + match open_root_no_follow(&self.root_path).and_then(|root| root_identity(&root)) { + Ok(identity) => identity, + Err(_) => { + self.shared + .revoke("fsevents_root_named_revalidation_failure"); + return Err(reconcile_error("fsevents_root_named_revalidation_failure")); + } + }; + if descriptor_identity != self.root_identity || named_identity != self.root_identity { + self.shared.revoke("fsevents_root_replaced"); + return Err(reconcile_error("fsevents_root_replaced")); + } + Ok(self.root_identity.clone()) + } + + fn flush_fence( + &self, + expected: &ExpectedScope, + kind: IssuedFenceKind, + ) -> Result { + self.ensure_history_authority()?; + self.root_identity()?; + if expected.provider_identity != self.provider_identity + || expected.filesystem_identity != self.root_identity + { + self.shared.revoke("fsevents_expected_identity_mismatch"); + return Err(reconcile_error("fsevents_expected_identity_mismatch")); + } + #[cfg(debug_assertions)] + let injected_nonce = self + .next_test_fence_nonce + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .take(); + #[cfg(not(debug_assertions))] + let injected_nonce: Option> = None; + let mut nonce = injected_nonce.unwrap_or_else(|| vec![0_u8; 24]); + if nonce.iter().all(|byte| *byte == 0) { + getrandom::getrandom(&mut nonce).map_err(|error| { + Error::InvalidInput(format!("FSEvents fence entropy failed: {error}")) + })?; + } + if nonce.len() < 16 { + self.shared.revoke("fsevents_fence_nonce_invalid"); + return Err(reconcile_error("fsevents_fence_nonce_invalid")); + } + self.fence_directory + .verify_identity(self.fence_directory_identity) + .map_err(|_| { + self.shared.revoke("fsevents_fence_directory_replaced"); + reconcile_error("fsevents_fence_directory_replaced") + })?; + let sentinel_name = hex::encode(&nonce); + let sentinel_path = LedgerPath::parse(&format!(".trail/observer-fences/{sentinel_name}"))?; + let fd = match openat( + self.fence_directory.file(), + Path::new(&sentinel_name), + OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::RUSR | Mode::WUSR, + ) { + Ok(fd) => fd, + Err(_) => { + self.shared + .revoke("fsevents_fence_collision_or_create_failure"); + return Err(reconcile_error( + "fsevents_fence_collision_or_create_failure", + )); + } + }; + let mut sentinel = File::from(fd); + if sentinel.write_all(hex::encode(&nonce).as_bytes()).is_err() { + return Err(self.fail_fence(&sentinel_name, "fsevents_fence_write_failure")); + } + #[cfg(debug_assertions)] + let injected_sync_failure = { + let mut fail = self + .fail_next_fence_sync + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + let injected = *fail; + *fail = false; + injected + }; + #[cfg(not(debug_assertions))] + let injected_sync_failure = false; + if injected_sync_failure || sentinel.sync_all().is_err() { + return Err(self.fail_fence(&sentinel_name, "fsevents_fence_file_sync_failure")); + } + if fsync(self.fence_directory.file()).is_err() { + return Err(self.fail_fence(&sentinel_name, "fsevents_fence_parent_sync_failure")); + } + // FlushSync alone cannot order a write which fseventsd has not yet + // ingested. The private create is the first journal barrier; its + // durable callback proves every earlier journal entry was ingested. + for stream in &self.stream.streams { + unsafe { + fs_events::FSEventStreamFlushSync(*stream as fs_events::FSEventStreamRef); + } + } + if let Err(error) = self.wait_for_sentinel(&sentinel_path, EvidenceFlags::CREATE) { + let _ = unlinkat( + self.fence_directory.file(), + Path::new(&sentinel_name), + AtFlags::empty(), + ); + let _ = fsync(self.fence_directory.file()); + return Err(error); + } + if unlinkat( + self.fence_directory.file(), + Path::new(&sentinel_name), + AtFlags::empty(), + ) + .is_err() + { + self.shared.revoke("fsevents_fence_cleanup_failure"); + return Err(reconcile_error("fsevents_fence_cleanup_failure")); + } + if fsync(self.fence_directory.file()).is_err() { + self.shared + .revoke("fsevents_fence_post_unlink_parent_sync_failure"); + return Err(reconcile_error( + "fsevents_fence_post_unlink_parent_sync_failure", + )); + } + for stream in &self.stream.streams { + unsafe { + fs_events::FSEventStreamFlushSync(*stream as fs_events::FSEventStreamRef); + } + } + let sentinel_event = self.wait_for_sentinel(&sentinel_path, EvidenceFlags::DELETE)?; + let (response_tx, response_rx) = mpsc::sync_channel(1); + if self + .commands + .send(DurabilityCommand::Fence { + minimum_provider_event_id: sentinel_event.provider_event_id, + nonce: nonce.clone(), + response: response_tx, + }) + .is_err() + { + self.shared + .revoke("fsevents_durability_worker_disconnected"); + return Err(reconcile_error("fsevents_durability_worker_disconnected")); + } + let (public, durable_cut, provider_event_id) = match response_rx.recv_timeout(FENCE_TIMEOUT) + { + Ok(Ok(result)) => result, + Ok(Err(_)) => { + self.shared.revoke("fsevents_durable_fence_failure"); + return Err(reconcile_error("fsevents_durable_fence_failure")); + } + Err(_) => { + self.shared.revoke("fsevents_durable_fence_timeout"); + return Err(reconcile_error("fsevents_durable_fence_timeout")); + } + }; + self.ensure_history_authority()?; + self.root_identity()?; + let issued = IssuedFence { + public: public.clone(), + expected: expected.clone(), + root_identity: self.root_identity.clone(), + owner_token: self.owner_token.clone(), + owner_fence_nonce: self.owner_fence_nonce.clone(), + provider_event_id, + durable_cut, + kind, + }; + self.shared.lock().issued_fences.insert(nonce, issued); + Ok(public) + } + + fn fail_fence(&self, sentinel_name: &str, reason: &'static str) -> Error { + let _ = unlinkat( + self.fence_directory.file(), + Path::new(sentinel_name), + AtFlags::empty(), + ); + let _ = fsync(self.fence_directory.file()); + self.shared.revoke(reason); + reconcile_error(reason) + } + + fn wait_for_sentinel( + &self, + path: &LedgerPath, + required: EvidenceFlags, + ) -> Result { + let deadline = Instant::now() + FENCE_TIMEOUT; + let mut state = self.shared.lock(); + loop { + if let Some(reason) = &state.revoked { + return Err(reconcile_error(reason)); + } + if let Some(event) = state.events.iter().find(|event| { + event.event.path == *path && event.event.flags.0 & required.0 == required.0 + }) { + return Ok(event.clone()); + } + let now = Instant::now(); + if now >= deadline { + drop(state); + self.shared.revoke("fsevents_sentinel_delivery_timeout"); + return Err(reconcile_error("fsevents_sentinel_delivery_timeout")); + } + let waited = self + .shared + .changed + .wait_timeout(state, deadline.saturating_duration_since(now)) + .unwrap_or_else(|poison| poison.into_inner()); + state = waited.0; + } + } + + fn issued_fence(&self, expected: &ExpectedScope, fence: &ObserverFence) -> Result { + self.ensure_history_authority()?; + let state = self.shared.lock(); + let Some(issued) = state.issued_fences.get(&fence.nonce) else { + drop(state); + self.shared.revoke("fsevents_fence_unknown_or_replayed"); + return Err(reconcile_error("fsevents_fence_unknown_or_replayed")); + }; + if issued.public != *fence + || issued.expected != *expected + || issued.root_identity != self.root_identity + || issued.owner_token != self.owner_token + || issued.owner_fence_nonce != self.owner_fence_nonce + || issued.durable_cut.last_sequence != fence.sequence + || issued.durable_cut.durable_end_offset != fence.durable_offset + { + drop(state); + self.shared.revoke("fsevents_fence_authentication_mismatch"); + return Err(reconcile_error("fsevents_fence_authentication_mismatch")); + } + Ok(issued.clone()) + } + + fn shutdown_inner(&self) -> Result<()> { + self.shared.shutdown.store(true, Ordering::Release); + let _ = self.commands.try_send(DurabilityCommand::Shutdown); + unsafe { + fs_events::core_foundation::CFRunLoopStop( + self.stream.run_loop as fs_events::core_foundation::CFRunLoopRef, + ); + } + let workers = std::mem::take( + &mut *self + .workers + .lock() + .unwrap_or_else(|poison| poison.into_inner()), + ); + let deadline = Instant::now() + FENCE_TIMEOUT; + let mut failure = None; + for worker in workers { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() || worker.done.recv_timeout(remaining).is_err() { + failure = Some(reconcile_error(&format!( + "fsevents_bounded_shutdown_timeout:{}", + worker.name + ))); + drop(worker.join); + continue; + } + if worker.join.join().is_err() { + failure = Some(Error::InvalidInput(format!( + "macOS {} worker panicked", + worker.name + ))); + } + } + self.shared.lock().active = false; + failure.map_or(Ok(()), Err) + } + + #[cfg(debug_assertions)] + fn inject_flags(&self, flags: u32) -> Result<()> { + classify_authority_flags(&self.shared, flags) + } + + #[cfg(debug_assertions)] + fn stop_durability_worker_for_test(&self) -> Result<()> { + self.commands + .send(DurabilityCommand::StopForTest) + .map_err(|_| reconcile_error("fsevents_durability_worker_disconnected")) + } + + #[cfg(debug_assertions)] + fn set_next_fence_nonce_for_test(&self, nonce: Vec) { + *self + .next_test_fence_nonce + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Some(nonce); + } + + #[cfg(debug_assertions)] + fn fail_next_fence_sync_for_test(&self) { + *self + .fail_next_fence_sync + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = true; + } + + #[cfg(debug_assertions)] + fn fail_next_root_descriptor_for_test(&self) { + *self + .fail_next_root_descriptor + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = true; + } + + #[cfg(debug_assertions)] + fn fail_next_coverage_descriptor_for_test(&self) { + *self + .fail_next_coverage_descriptor + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = true; + } + + #[cfg(test)] + pub(crate) fn fail_next_direct_policy_fence_for_test(&self) { + *self + .fail_next_direct_policy_fence + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = true; + } + + #[cfg(debug_assertions)] + fn set_next_history_authority_for_test(&self, authority: HistoryAuthority) { + *self + .next_history_authority_override + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Some(authority); + } +} + +impl QualifiedObserver for MacOsFseventsObserver { + fn begin_observation(&self, expected: &ExpectedScope) -> Result { + self.flush_fence(expected, IssuedFenceKind::Start) + } + + fn end_fence(&self, expected: &ExpectedScope, start: &ObserverFence) -> Result { + let issued = self.issued_fence(expected, start)?; + if !matches!( + issued.kind, + IssuedFenceKind::Start | IssuedFenceKind::TailAnchor | IssuedFenceKind::RotationAnchor + ) { + self.shared.revoke("fsevents_start_fence_kind_mismatch"); + return Err(reconcile_error("fsevents_start_fence_kind_mismatch")); + } + let end = self.flush_fence( + expected, + IssuedFenceKind::End { + start_nonce: start.nonce.clone(), + }, + )?; + let issued_end = self.issued_fence(expected, &end)?; + if end.sequence <= start.sequence + || (issued.durable_cut.segment_id == issued_end.durable_cut.segment_id + && end.durable_offset < start.durable_offset) + { + self.shared.revoke("fsevents_non_monotonic_fence"); + return Err(reconcile_error("fsevents_non_monotonic_fence")); + } + Ok(end) + } + + fn drain_through( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + ) -> Result { + self.drain_interval(expected, root_handle_identity, start, end, sink, false) + } + + fn drain_through_retaining_end( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + ) -> Result { + self.drain_interval(expected, root_handle_identity, start, end, sink, true) + } + + fn rebind_retained_tail( + &self, + previous: &ExpectedScope, + next: &ExpectedScope, + anchor: &ObserverFence, + ) -> Result<()> { + self.rebind_tail_anchor(previous, next, anchor) + } +} + +fn stable_observer_binding(previous: &ExpectedScope, next: &ExpectedScope) -> bool { + previous.scope_id == next.scope_id + && previous.epoch == next.epoch + && previous.policy_fingerprint == next.policy_fingerprint + && previous.policy_generation == next.policy_generation + && previous.filesystem_identity == next.filesystem_identity + && previous.provider_identity == next.provider_identity +} + +impl MacOsFseventsObserver { + fn drain_interval( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + retain_end: bool, + ) -> Result { + self.ensure_history_authority()?; + if self.root_identity()? != root_handle_identity { + self.shared.revoke("fsevents_root_identity_mismatch"); + return Err(reconcile_error("fsevents_root_identity_mismatch")); + } + let issued_start = self.issued_fence(expected, start)?; + let issued_end = self.issued_fence(expected, end)?; + if !matches!( + issued_start.kind, + IssuedFenceKind::Start | IssuedFenceKind::TailAnchor | IssuedFenceKind::RotationAnchor + ) || !matches!( + &issued_end.kind, + IssuedFenceKind::End { start_nonce } if *start_nonce == start.nonce + ) || issued_end.provider_event_id < issued_start.provider_event_id + { + self.shared.revoke("fsevents_fence_interval_mismatch"); + return Err(reconcile_error("fsevents_fence_interval_mismatch")); + } + let events = { + let state = self.shared.lock(); + state + .events + .iter() + .filter(|item| { + !item.internal_fence + && item.event.sequence > start.sequence + && item.event.sequence <= end.sequence + && item.provider_event_id <= issued_end.provider_event_id + }) + .map(|item| item.event.clone()) + .collect::>() + }; + for event in events { + sink(event)?; + } + self.ensure_history_authority()?; + let qualification = ObserverQualification::native( + expected, + root_handle_identity.to_vec(), + start.clone(), + end.clone(), + self.owner_token.clone(), + self.owner_fence_nonce.clone(), + issued_end.durable_cut.segment_id.clone(), + issued_end.durable_cut.durable_end_offset, + issued_end.durable_cut.durable_end_offset, + ); + let mut state = self.shared.lock(); + state + .events + .retain(|item| item.event.sequence > end.sequence); + state.issued_fences.remove(&start.nonce); + if retain_end { + let retained = state + .issued_fences + .get_mut(&end.nonce) + .ok_or_else(|| reconcile_error("fsevents_end_fence_not_retained_for_rotation"))?; + retained.kind = IssuedFenceKind::TailAnchor; + } else { + state.issued_fences.remove(&end.nonce); + } + Ok(qualification) + } +} + +impl Drop for MacOsFseventsObserver { + fn drop(&mut self) { + let _ = self.shutdown_inner(); + } +} + +extern "C" fn release_callback_context(info: *const c_void) { + if !info.is_null() { + unsafe { + drop(Box::from_raw(info as *mut CallbackContext)); + } + } +} + +extern "C" fn callback( + _stream: fs_events::FSEventStreamRef, + info: *mut c_void, + count: usize, + event_paths: *mut c_void, + event_flags: *const u32, + event_ids: *const u64, +) { + if info.is_null() { + if count != 0 { + NULL_CONTEXT_GENERATION.fetch_add(1, Ordering::Release); + } + return; + } + let context = unsafe { &*(info as *const CallbackContext) }; + if count == 0 { + return; + } + if event_paths.is_null() || event_flags.is_null() || event_ids.is_null() { + context + .shared + .revoke("fsevents_malformed_nonempty_callback_batch"); + return; + } + let paths = unsafe { std::slice::from_raw_parts(event_paths as *const *const i8, count) }; + let flags = unsafe { std::slice::from_raw_parts(event_flags, count) }; + let ids = unsafe { std::slice::from_raw_parts(event_ids, count) }; + for index in 0..count { + if classify_authority_flags(&context.shared, flags[index]).is_err() { + return; + } + if flags[index] & fs_events::kFSEventStreamEventFlagHistoryDone != 0 { + continue; + } + let Some(path_ptr) = paths.get(index).copied().filter(|path| !path.is_null()) else { + context.shared.revoke("fsevents_path_decode_ambiguity"); + return; + }; + let Ok(path_text) = unsafe { CStr::from_ptr(path_ptr) }.to_str() else { + context.shared.revoke("fsevents_path_decode_ambiguity"); + return; + }; + let disposition = classify_callback_path(context, Path::new(path_text), flags[index]); + let Ok(disposition) = disposition else { + context.shared.revoke(format!( + "fsevents_path_escaped_or_ambiguous: callback={path_text:?} root={:?} device_relative_root={:?}", + context.root_path, context.device_relative_root, + )); + return; + }; + let CallbackDisposition::Ledger(path) = disposition else { + match disposition { + CallbackDisposition::PolicyDependency(watch) => { + match context.records.try_send(DurabilityCommand::PolicyEvent { + watch, + provider_event_id: ids[index], + }) { + Ok(()) => {} + Err(TrySendError::Full(_)) => { + context + .shared + .revoke("fsevents_bounded_callback_queue_overflow"); + return; + } + Err(TrySendError::Disconnected(_)) => { + context + .shared + .revoke("fsevents_durability_worker_disconnected"); + return; + } + } + } + CallbackDisposition::Ignore => {} + CallbackDisposition::Ledger(_) => unreachable!(), + } + continue; + }; + let Some(path) = path else { + // Directory metadata events for the watched root have no ledger + // path. Root replacement and loss are still handled above by the + // authoritative WatchRoot flags. + continue; + }; + if observer_internal_path(&path) && !observer_fence_path(&path) { + continue; + } + let evidence = evidence_flags(flags[index]); + if evidence.0 == 0 { + continue; + } + match context.records.try_send(DurabilityCommand::Record { + path, + flags: evidence, + provider_event_id: ids[index], + }) { + Ok(()) => {} + Err(TrySendError::Full(_)) => { + context + .shared + .revoke("fsevents_bounded_callback_queue_overflow"); + return; + } + Err(TrySendError::Disconnected(_)) => { + context + .shared + .revoke("fsevents_durability_worker_disconnected"); + return; + } + } + } +} + +enum CallbackDisposition { + Ledger(Option), + PolicyDependency(PolicyWatch), + Ignore, +} + +fn classify_callback_path( + context: &CallbackContext, + event_path: &Path, + native_flags: u32, +) -> Result { + let candidates = if event_path.is_absolute() { + vec![lexical_normalize_path(event_path)] + } else { + let mut candidates = Vec::new(); + for coverage in &context.coverage_roots { + if let Ok(relative) = event_path.strip_prefix(&coverage.device_relative_root) { + candidates.push(lexical_normalize_path( + &coverage.absolute_root.join(relative), + )); + } + } + candidates + }; + if candidates.is_empty() { + return Err(reconcile_error( + "fsevents_callback_outside_exact_watch_coverage", + )); + } + for candidate in &candidates { + if let Some(watch) = context + .policy_watches + .iter() + .find(|watch| dependency_triggered_by(candidate, &watch.observed_path, native_flags)) + { + return Ok(CallbackDisposition::PolicyDependency(watch.clone())); + } + } + if let Some(candidate) = candidates + .iter() + .find(|candidate| candidate.starts_with(&context.root_path)) + { + return normalize_callback_path( + &context.root_path, + &context.device_relative_root, + candidate, + ) + .map(CallbackDisposition::Ledger); + } + if candidates.iter().any(|candidate| { + context + .coverage_roots + .iter() + .any(|coverage| candidate.starts_with(&coverage.absolute_root)) + }) { + return Ok(CallbackDisposition::Ignore); + } + Err(reconcile_error( + "fsevents_callback_outside_exact_watch_coverage", + )) +} + +fn dependency_triggered_by(event: &Path, dependency: &Path, native_flags: u32) -> bool { + if event == dependency { + return true; + } + let ancestor_identity_flags = fs_events::kFSEventStreamEventFlagItemCreated + | fs_events::kFSEventStreamEventFlagItemRemoved + | fs_events::kFSEventStreamEventFlagItemRenamed + | fs_events::kFSEventStreamEventFlagItemInodeMetaMod + | fs_events::kFSEventStreamEventFlagItemChangeOwner; + dependency.strip_prefix(event).is_ok() && native_flags & ancestor_identity_flags != 0 +} + +fn classify_authority_flags(shared: &Shared, flags: u32) -> Result<()> { + if flags & GAP_FLAGS != 0 { + let reason = if flags & fs_events::kFSEventStreamEventFlagMustScanSubDirs != 0 { + "fsevents_must_scan_subdirs" + } else if flags & fs_events::kFSEventStreamEventFlagUserDropped != 0 { + "fsevents_user_dropped" + } else if flags & fs_events::kFSEventStreamEventFlagKernelDropped != 0 { + "fsevents_kernel_dropped" + } else if flags & fs_events::kFSEventStreamEventFlagEventIdsWrapped != 0 { + "fsevents_event_ids_wrapped" + } else if flags & fs_events::kFSEventStreamEventFlagRootChanged != 0 { + "fsevents_root_changed" + } else { + "fsevents_unmount" + }; + shared.revoke(reason); + return Err(reconcile_error(reason)); + } + if flags & fs_events::kFSEventStreamEventFlagHistoryDone != 0 { + let mut state = shared.lock(); + if state.history_pending == 0 { + drop(state); + shared.revoke("fsevents_inconsistent_history_done"); + return Err(reconcile_error("fsevents_inconsistent_history_done")); + } + state.history_pending -= 1; + shared.changed.notify_all(); + } + Ok(()) +} + +fn normalize_callback_path( + root: &Path, + device_relative_root: &Path, + event_path: &Path, +) -> Result> { + let relative = if event_path.is_absolute() { + event_path + .strip_prefix(root) + .map_err(|_| Error::InvalidInput("FSEvents path escaped pinned root".into()))? + } else { + event_path.strip_prefix(device_relative_root).map_err(|_| { + Error::InvalidInput("device-relative FSEvents path escaped pinned root".into()) + })? + }; + if relative.as_os_str().is_empty() { + return Ok(None); + } + if relative + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(Error::InvalidInput( + "FSEvents path was not a normalized descendant".into(), + )); + } + let text = relative + .to_str() + .ok_or_else(|| Error::InvalidInput("FSEvents path was not UTF-8".into()))?; + LedgerPath::parse(text).map(Some) +} + +fn evidence_flags(flags: u32) -> EvidenceFlags { + let mut evidence = EvidenceFlags::default(); + if flags & fs_events::kFSEventStreamEventFlagItemCreated != 0 { + evidence |= EvidenceFlags::CREATE; + } + if flags & fs_events::kFSEventStreamEventFlagItemRemoved != 0 { + evidence |= EvidenceFlags::DELETE; + } + if flags & fs_events::kFSEventStreamEventFlagItemModified != 0 { + evidence |= EvidenceFlags::CONTENT; + } + if flags + & (fs_events::kFSEventStreamEventFlagItemInodeMetaMod + | fs_events::kFSEventStreamEventFlagItemFinderInfoMod + | fs_events::kFSEventStreamEventFlagItemChangeOwner + | fs_events::kFSEventStreamEventFlagItemXattrMod) + != 0 + { + evidence |= EvidenceFlags::MODE; + } + if flags & fs_events::kFSEventStreamEventFlagItemRenamed != 0 { + // FSEvents does not provide a rename cookie. Marking each delivered + // endpoint both ways is conservative and retains every endpoint. + evidence |= EvidenceFlags::RENAME_FROM | EvidenceFlags::RENAME_TO; + } + if flags & fs_events::kFSEventStreamEventFlagItemIsDir != 0 + && flags + & (fs_events::kFSEventStreamEventFlagItemCreated + | fs_events::kFSEventStreamEventFlagItemRenamed) + != 0 + { + evidence |= EvidenceFlags::PROVIDER_COMPLETE_PREFIX; + } + evidence +} + +fn observer_internal_path(path: &LedgerPath) -> bool { + let path = path.as_str(); + path == ".trail" || path.starts_with(".trail/") || path == ".git" || path.starts_with(".git/") +} + +fn observer_fence_path(path: &LedgerPath) -> bool { + path.as_str().starts_with(".trail/observer-fences/") +} + +fn normalize_absolute_dependency(requested_root: &Path, dependency: &Path) -> Result { + let absolute = if dependency.is_absolute() { + dependency.to_path_buf() + } else { + requested_root.join(dependency) + }; + let normalized = lexical_normalize_path(&absolute); + if !normalized.is_absolute() { + return Err(reconcile_error( + "fsevents_policy_dependency_is_not_absolute", + )); + } + normalized + .to_str() + .ok_or_else(|| reconcile_error("fsevents_policy_dependency_path_decode_ambiguity"))?; + Ok(normalized) +} + +fn lexical_normalize_path(path: &Path) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), + Component::RootDir => normalized.push(Path::new("/")), + Component::CurDir => {} + Component::ParentDir => { + normalized.pop(); + } + Component::Normal(component) => normalized.push(component), + } + } + normalized +} + +fn build_coverage_plan( + root: &Path, + requested_root: &Path, + device: u64, + dependencies: &[PathBuf], +) -> Result { + let mut roots = BTreeMap::::new(); + add_coverage_root(&mut roots, root, device)?; + let mut system_aliases = BTreeMap::::new(); + let mut policy_watches = Vec::with_capacity(dependencies.len()); + let mut external_policy_fences = Vec::new(); + for dependency in dependencies { + let dependency = normalize_absolute_dependency(requested_root, dependency)?; + if let Ok(relative) = dependency.strip_prefix(requested_root) { + let observed_path = lexical_normalize_path(&root.join(relative)); + let identity = capture_policy_dependency_fence_identity_at_observed_path( + &dependency, + &observed_path, + ) + .map_err(|_| reconcile_error("fsevents_policy_dependency_leaf_is_symlink_or_unsafe"))?; + policy_watches.push(PolicyWatch { + observed_path, + dependency, + identity, + }); + continue; + } + let (observed_dependency, alias) = canonicalize_known_system_alias(&dependency)?; + if let Some(alias) = alias { + system_aliases + .entry(alias.alias_path.clone()) + .or_insert(alias); + } + // Validate every ordinary component from `/` to the final leaf with + // descriptor-relative O_NOFOLLOW traversal. The only symlink already + // removed from this observed path is an exact authenticated platform + // alias above. + capture_policy_dependency_fence_identity(&observed_dependency) + .map_err(|_| reconcile_error("fsevents_policy_dependency_leaf_is_symlink_or_unsafe"))?; + if observed_dependency.starts_with(root) { + validate_policy_dependency_leaf(&observed_dependency)?; + let identity = capture_policy_dependency_fence_identity_at_observed_path( + &dependency, + &observed_dependency, + ) + .map_err(|_| reconcile_error("fsevents_policy_dependency_leaf_is_symlink_or_unsafe"))?; + policy_watches.push(PolicyWatch { + observed_path: observed_dependency, + dependency, + identity, + }); + continue; + } + let (named_root, canonical, coverage_root) = nearest_existing_parent(&observed_dependency)?; + let metadata = coverage_root.metadata()?; + if metadata.dev() != device { + let identity = capture_policy_dependency_fence_identity_at_observed_path( + &dependency, + &observed_dependency, + ) + .map_err(|_| reconcile_error("fsevents_policy_dependency_leaf_is_symlink_or_unsafe"))?; + external_policy_fences.push(ExternalPolicyFence { + dependency, + observed_path: observed_dependency, + identity, + }); + if external_policy_fences.len() > MAX_DIRECT_POLICY_DEPENDENCIES { + return Err(reconcile_error( + "fsevents_direct_policy_dependency_bound_exceeded", + )); + } + continue; + } + validate_policy_dependency_leaf(&observed_dependency)?; + let relative = observed_dependency + .strip_prefix(&named_root) + .map_err(|_| reconcile_error("fsevents_policy_dependency_coverage_mapping_failure"))?; + let identity = capture_policy_dependency_fence_identity_at_observed_path( + &dependency, + &observed_dependency, + ) + .map_err(|_| reconcile_error("fsevents_policy_dependency_leaf_is_symlink_or_unsafe"))?; + policy_watches.push(PolicyWatch { + observed_path: lexical_normalize_path(&canonical.join(relative)), + dependency, + identity, + }); + add_open_coverage_root(&mut roots, canonical, coverage_root, device)?; + } + let roots = roots.into_values().collect::>(); + let stream_roots = roots + .iter() + .filter(|candidate| { + !roots.iter().any(|other| { + other.absolute_root != candidate.absolute_root + && candidate.absolute_root.starts_with(&other.absolute_root) + }) + }) + .map(|root| root.device_relative_root.clone()) + .collect(); + Ok(CoveragePlan { + roots, + stream_roots, + system_aliases: system_aliases.into_values().collect(), + policy_dependencies: dependencies.to_vec(), + policy_watches, + external_policy_fences, + }) +} + +fn canonicalize_known_system_alias( + dependency: &Path, +) -> Result<(PathBuf, Option)> { + for (alias, target, link_target) in [ + ("/etc", "/private/etc", "private/etc"), + ("/var", "/private/var", "private/var"), + ("/tmp", "/private/tmp", "private/tmp"), + ] { + let alias = Path::new(alias); + let Ok(relative) = dependency.strip_prefix(alias) else { + continue; + }; + let binding = exact_system_alias_binding(alias, Path::new(target), Path::new(link_target))?; + return Ok(( + lexical_normalize_path(&binding.canonical_target.join(relative)), + Some(binding), + )); + } + Ok((dependency.to_path_buf(), None)) +} + +fn exact_system_alias_binding( + alias: &Path, + expected_target: &Path, + expected_link_target: &Path, +) -> Result { + let metadata = std::fs::symlink_metadata(alias) + .map_err(|_| reconcile_error("fsevents_system_policy_alias_identity_unavailable"))?; + let link_target = std::fs::read_link(alias) + .map_err(|_| reconcile_error("fsevents_system_policy_alias_identity_unavailable"))?; + let canonical_target = alias + .canonicalize() + .map_err(|_| reconcile_error("fsevents_system_policy_alias_target_unavailable"))?; + if !metadata.file_type().is_symlink() + || metadata.uid() != 0 + || link_target != expected_link_target + || canonical_target != expected_target + { + return Err(reconcile_error( + "fsevents_system_policy_alias_identity_mismatch", + )); + } + let target = open_root_no_follow(&canonical_target) + .map_err(|_| reconcile_error("fsevents_system_policy_alias_target_unavailable"))?; + let alias_identity = format!( + "system-alias-v1:dev={};ino={};mode={};uid={};gid={};target={}", + metadata.dev(), + metadata.ino(), + metadata.mode(), + metadata.uid(), + metadata.gid(), + link_target.display(), + ) + .into_bytes(); + Ok(SystemAliasBinding { + alias_path: alias.to_path_buf(), + canonical_target, + alias_identity, + target_identity: root_identity(&target)?, + }) +} + +fn validate_policy_dependency_leaf(dependency: &Path) -> Result<()> { + let named = match std::fs::symlink_metadata(dependency) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(_) => { + return Err(reconcile_error( + "fsevents_policy_dependency_leaf_is_symlink_or_unsafe", + )); + } + }; + if named.file_type().is_symlink() || !named.is_file() { + return Err(reconcile_error( + "fsevents_policy_dependency_leaf_is_symlink_or_unsafe", + )); + } + let opened = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(dependency) + .map_err(|_| reconcile_error("fsevents_policy_dependency_leaf_is_symlink_or_unsafe"))?; + let descriptor = opened + .metadata() + .map_err(|_| reconcile_error("fsevents_policy_dependency_leaf_is_symlink_or_unsafe"))?; + let renamed = std::fs::symlink_metadata(dependency) + .map_err(|_| reconcile_error("fsevents_policy_dependency_leaf_is_symlink_or_unsafe"))?; + if !descriptor.is_file() + || renamed.file_type().is_symlink() + || !renamed.is_file() + || descriptor.dev() != renamed.dev() + || descriptor.ino() != renamed.ino() + { + return Err(reconcile_error( + "fsevents_policy_dependency_leaf_is_symlink_or_unsafe", + )); + } + Ok(()) +} + +fn nearest_existing_parent(dependency: &Path) -> Result<(PathBuf, PathBuf, File)> { + let mut candidate = dependency + .parent() + .ok_or_else(|| reconcile_error("fsevents_policy_dependency_has_no_parent"))?; + loop { + match std::fs::symlink_metadata(candidate) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => { + return Err(reconcile_error( + "fsevents_policy_dependency_parent_is_symlink_or_unsafe", + )); + } + Ok(_) => { + let root = open_root_no_follow(candidate).map_err(|_| { + reconcile_error("fsevents_policy_dependency_parent_is_symlink_or_unsafe") + })?; + let canonical = candidate.canonicalize().map_err(|_| { + reconcile_error("fsevents_policy_coverage_root_identity_unavailable") + })?; + return Ok((candidate.to_path_buf(), canonical, root)); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + candidate = candidate.parent().ok_or_else(|| { + reconcile_error("fsevents_policy_dependency_parent_unobservable") + })?; + } + Err(_) => { + return Err(reconcile_error( + "fsevents_policy_dependency_parent_is_symlink_or_unsafe", + )); + } + } + } +} + +fn add_coverage_root( + roots: &mut BTreeMap, + path: &Path, + device: u64, +) -> Result<()> { + let root = open_root_no_follow(path)?; + add_open_coverage_root(roots, path.to_path_buf(), root, device) +} + +fn add_open_coverage_root( + roots: &mut BTreeMap, + absolute_root: PathBuf, + root: File, + device: u64, +) -> Result<()> { + let metadata = root.metadata()?; + if metadata.dev() != device { + return Err(reconcile_error("fsevents_policy_dependency_crosses_device")); + } + let relative = device_relative_root(&absolute_root)?; + let identity = root_identity(&root)?; + roots.entry(absolute_root.clone()).or_insert(CoverageRoot { + absolute_root, + device_relative_root: relative, + root_identity: identity, + root, + }); + Ok(()) +} + +fn cursor_coverage_roots(coverage: &CoveragePlan) -> Vec { + coverage + .roots + .iter() + .map(|root| CursorCoverageRoot { + device_relative_root: root.device_relative_root.clone(), + root_identity: root.root_identity.clone(), + }) + .collect() +} + +fn run_stream( + authority: HistoryAuthority, + watched_roots: Vec, + since_when: u64, + callback_context: CallbackContext, + ready: SyncSender>, + decision: Receiver, + cancelled: Arc, + post_start_database_uuid_override: Option<[u8; 16]>, + delay_after_native_start: Duration, + cleanup_observed: Option>, + shared: Arc, +) { + if cancelled.load(Ordering::Acquire) { + return; + } + let mut streams = Vec::with_capacity(watched_roots.len()); + for root in &watched_roots { + let paths = unsafe { + fs_events::core_foundation::CFArrayCreateMutable( + fs_events::core_foundation::kCFAllocatorDefault, + 1, + &fs_events::core_foundation::kCFTypeArrayCallBacks, + ) + }; + if paths.is_null() { + unsafe { cleanup_streams(&streams, cleanup_observed.as_ref()) }; + let _ = ready.send(Err(reconcile_error("fsevents_paths_array_failure"))); + return; + } + let Ok(relative_root) = std::ffi::CString::new(root.as_bytes()) else { + unsafe { fs_events::core_foundation::CFRelease(paths) }; + unsafe { cleanup_streams(&streams, cleanup_observed.as_ref()) }; + let _ = ready.send(Err(reconcile_error("fsevents_relative_root_contains_nul"))); + return; + }; + let cf_path = unsafe { + fs_events::core_foundation::CFStringCreateWithCString( + fs_events::core_foundation::kCFAllocatorDefault, + relative_root.as_ptr(), + fs_events::core_foundation::kCFStringEncodingUTF8, + ) + }; + if cf_path.is_null() { + unsafe { fs_events::core_foundation::CFRelease(paths) }; + unsafe { cleanup_streams(&streams, cleanup_observed.as_ref()) }; + let _ = ready.send(Err(reconcile_error("fsevents_root_cfstring_failure"))); + return; + } + unsafe { + fs_events::core_foundation::CFArrayAppendValue(paths, cf_path); + fs_events::core_foundation::CFRelease(cf_path); + } + let raw_context = Box::into_raw(Box::new(callback_context.clone())); + let context = fs_events::FSEventStreamContext { + version: 0, + info: raw_context.cast(), + retain: None, + release: Some(release_callback_context), + copy_description: None, + }; + let stream = unsafe { + fs_events::FSEventStreamCreateRelativeToDevice( + fs_events::core_foundation::kCFAllocatorDefault, + callback, + &context, + authority.device as libc::dev_t, + paths, + since_when, + 0.01, + STREAM_FLAGS, + ) + }; + unsafe { fs_events::core_foundation::CFRelease(paths) }; + if stream.is_null() { + unsafe { + drop(Box::from_raw(raw_context)); + cleanup_streams(&streams, cleanup_observed.as_ref()); + } + let _ = ready.send(Err(reconcile_error("fsevents_stream_create_failure"))); + return; + } + let actual_device = unsafe { fs_events::FSEventStreamGetDeviceBeingWatched(stream) }; + let actual_roots = copy_watched_roots(stream); + if actual_device as u64 != authority.device + || !matches!(actual_roots.as_ref(), Ok(paths) if paths == std::slice::from_ref(root)) + { + let reason = format!( + "fsevents_native_device_or_relative_root_mismatch: expected_device={} actual_device={} expected_root={:?} actual_roots={:?}", + authority.device, + actual_device, + root, + actual_roots.as_ref().map_err(ToString::to_string), + ); + streams.push(stream); + unsafe { cleanup_streams(&streams, cleanup_observed.as_ref()) }; + let _ = ready.send(Err(reconcile_error(&reason))); + return; + } + streams.push(stream); + } + unsafe { + let run_loop = fs_events::core_foundation::CFRunLoopGetCurrent(); + for stream in &streams { + fs_events::FSEventStreamScheduleWithRunLoop( + *stream, + run_loop, + fs_events::core_foundation::kCFRunLoopDefaultMode, + ); + if fs_events::FSEventStreamStart(*stream) == 0 { + cleanup_streams(&streams, cleanup_observed.as_ref()); + let _ = ready.send(Err(reconcile_error(&format!( + "fsevents_stream_start_failure:watched_roots={watched_roots:?}" + )))); + return; + } + } + let post_start_database_uuid = match post_start_database_uuid_override { + Some(uuid) => Ok(uuid), + None => copy_history_database_uuid(authority.device), + }; + match post_start_database_uuid { + Ok(uuid) if uuid == authority.database_uuid => {} + Ok(_) => { + cleanup_streams(&streams, cleanup_observed.as_ref()); + let _ = ready.send(Err(reconcile_error( + "fsevents_post_start_history_database_uuid_mismatch", + ))); + return; + } + Err(_) => { + cleanup_streams(&streams, cleanup_observed.as_ref()); + let _ = ready.send(Err(reconcile_error( + "fsevents_post_start_history_database_uuid_unavailable", + ))); + return; + } + } + if !delay_after_native_start.is_zero() { + thread::sleep(delay_after_native_start); + } + if cancelled.load(Ordering::Acquire) { + cleanup_streams(&streams, cleanup_observed.as_ref()); + return; + } + if ready + .send(Ok(StreamHandle { + streams: streams.iter().map(|stream| *stream as usize).collect(), + run_loop: run_loop as usize, + })) + .is_err() + { + cleanup_streams(&streams, cleanup_observed.as_ref()); + return; + } + match decision.recv_timeout(FENCE_TIMEOUT) { + Ok(StartupDecision::Publish) if !cancelled.load(Ordering::Acquire) => {} + Ok(StartupDecision::Publish | StartupDecision::Cancel) + | Err(RecvTimeoutError::Timeout | RecvTimeoutError::Disconnected) => { + cleanup_streams(&streams, cleanup_observed.as_ref()); + return; + } + } + fs_events::core_foundation::CFRunLoopRun(); + cleanup_streams(&streams, cleanup_observed.as_ref()); + } + if !shared.shutdown.load(Ordering::Acquire) { + shared.revoke("fsevents_run_loop_stopped"); + } +} + +fn copy_watched_roots(stream: fs_events::ConstFSEventStreamRef) -> Result> { + let paths = unsafe { fs_events::FSEventStreamCopyPathsBeingWatched(stream) }; + if paths.is_null() { + return Err(reconcile_error("fsevents_copy_watched_paths_failure")); + } + let result = (|| { + let count = unsafe { fs_events::core_foundation::CFArrayGetCount(paths) }; + if count <= 0 { + return Err(reconcile_error("fsevents_watched_path_count_mismatch")); + } + let mut result = Vec::with_capacity(count as usize); + for index in 0..count { + let value = unsafe { fs_events::core_foundation::CFArrayGetValueAtIndex(paths, index) }; + if value.is_null() { + return Err(reconcile_error("fsevents_watched_path_is_null")); + } + let mut buffer = vec![0_i8; 16 * 1024]; + let copied = unsafe { + fs_events::core_foundation::CFStringGetCString( + value, + buffer.as_mut_ptr(), + buffer.len() as i64, + fs_events::core_foundation::kCFStringEncodingUTF8, + ) + }; + if !copied { + return Err(reconcile_error("fsevents_watched_path_decode_failure")); + } + result.push( + unsafe { CStr::from_ptr(buffer.as_ptr()) } + .to_str() + .map(str::to_owned) + .map_err(|_| reconcile_error("fsevents_watched_path_not_utf8"))?, + ); + } + result.sort(); + Ok(result) + })(); + unsafe { fs_events::core_foundation::CFRelease(paths) }; + result +} + +unsafe fn cleanup_streams( + streams: &[fs_events::FSEventStreamRef], + cleanup_observed: Option<&Arc>, +) { + for stream in streams { + unsafe { + fs_events::FSEventStreamStop(*stream); + fs_events::FSEventStreamInvalidate(*stream); + fs_events::FSEventStreamRelease(*stream); + } + } + if let Some(observed) = cleanup_observed { + observed.store(true, Ordering::Release); + } +} + +fn run_durability_worker( + receiver: Receiver, + mut durability: Box, + shared: Arc, + cursor_template: MacOsProviderCursor, +) { + let mut last_heartbeat = Instant::now(); + loop { + if shared.shutdown.load(Ordering::Acquire) { + return; + } + let command = match receiver.recv_timeout(Duration::from_millis(25)) { + Ok(command) => command, + Err(RecvTimeoutError::Timeout) => { + if last_heartbeat.elapsed() >= Duration::from_secs(1) { + if durability.heartbeat().is_err() { + shared.revoke("fsevents_observer_heartbeat_failed"); + return; + } + last_heartbeat = Instant::now(); + } + continue; + } + Err(RecvTimeoutError::Disconnected) => { + if !shared.shutdown.load(Ordering::Acquire) { + shared.revoke("fsevents_durability_worker_disconnected"); + } + return; + } + }; + if matches!(command, DurabilityCommand::Shutdown) { + return; + } + #[cfg(debug_assertions)] + if matches!(command, DurabilityCommand::StopForTest) { + shared.revoke("fsevents_durability_worker_stopped"); + return; + } + let command = match command { + DurabilityCommand::PolicyEvent { + watch, + provider_event_id, + } => { + let unchanged = capture_policy_dependency_fence_identity_at_observed_path( + &watch.dependency, + &watch.observed_path, + ) + .is_ok_and(|identity| identity == watch.identity); + if unchanged { + let mut state = shared.lock(); + state.last_provider_event_id = + state.last_provider_event_id.max(provider_event_id); + shared.changed.notify_all(); + continue; + } + DurabilityCommand::PolicyInvalidation { + dependency: watch.dependency, + provider_event_id, + } + } + command => command, + }; + let (command, direct_response) = match command { + DurabilityCommand::DirectPolicyInvalidation { + dependency, + response, + } => ( + DurabilityCommand::PolicyInvalidation { + dependency, + provider_event_id: shared.lock().last_provider_event_id, + }, + Some(response), + ), + command => (command, None), + }; + let ( + path, + flags, + provider_event_id, + internal, + fence_nonce, + response, + controlled_response, + revocation, + ) = match command { + DurabilityCommand::Record { + path, + flags, + provider_event_id, + } => { + let internal = path.as_str().starts_with(".trail/observer-fences/"); + ( + path, + flags, + provider_event_id, + internal, + Vec::new(), + None, + None, + None, + ) + } + DurabilityCommand::PolicyInvalidation { + dependency, + provider_event_id, + } => { + let digest = Sha256::digest(dependency.as_os_str().as_bytes()); + let path = match LedgerPath::parse(&format!( + ".trail/policy-invalidations/{}", + hex::encode(digest) + )) { + Ok(path) => path, + Err(_) => { + shared.revoke("fsevents_policy_invalidation_marker_failure"); + return; + } + }; + ( + path, + EvidenceFlags::CONTENT, + provider_event_id, + true, + Vec::new(), + None, + None, + Some(format!( + "fsevents_policy_dependency_invalidated:{}", + dependency.display() + )), + ) + } + DurabilityCommand::Fence { + minimum_provider_event_id, + nonce, + response, + } => { + let provider_event_id = shared.lock().last_provider_event_id; + if provider_event_id < minimum_provider_event_id { + let _ = response.send(Err(reconcile_error( + "fsevents_fence_precedes_authenticated_sentinel", + ))); + shared.revoke("fsevents_fence_precedes_authenticated_sentinel"); + return; + } + let name = format!(".trail-fsevents-fence-{}", hex::encode(&nonce)); + let Ok(path) = LedgerPath::parse(&name) else { + let _ = response.send(Err(reconcile_error("fsevents_fence_path_failure"))); + shared.revoke("fsevents_fence_path_failure"); + return; + }; + ( + path, + EvidenceFlags::default(), + provider_event_id, + true, + nonce, + Some(response), + None, + None, + ) + } + DurabilityCommand::ControlledFence { + minimum_provider_event_id, + nonce, + response, + } => { + let provider_event_id = shared.lock().last_provider_event_id; + if provider_event_id < minimum_provider_event_id { + let _ = response.send(Err(reconcile_error( + "fsevents_controlled_fence_precedes_authenticated_sentinel", + ))); + shared.revoke("fsevents_controlled_fence_precedes_authenticated_sentinel"); + return; + } + let name = format!(".trail-fsevents-controlled-fence-{}", hex::encode(&nonce)); + let Ok(path) = LedgerPath::parse(&name) else { + let _ = response.send(Err(reconcile_error( + "fsevents_controlled_fence_path_failure", + ))); + shared.revoke("fsevents_controlled_fence_path_failure"); + return; + }; + ( + path, + EvidenceFlags::default(), + provider_event_id, + true, + nonce, + None, + Some(response), + None, + ) + } + DurabilityCommand::Rotate { expected, response } => { + let result = durability.rotate_if_cut(&expected); + if let Err(error) = &result { + shared.revoke(format!("fsevents_rotation_failure:{error}")); + } + let failed = result.is_err(); + let _ = response.send(result); + if failed { + return; + } + continue; + } + DurabilityCommand::DirectPolicyInvalidation { .. } => unreachable!(), + DurabilityCommand::PolicyEvent { .. } => unreachable!(), + #[cfg(debug_assertions)] + DurabilityCommand::StopForTest => unreachable!(), + DurabilityCommand::Shutdown => unreachable!(), + }; + let sequence = shared.lock().next_sequence; + // Device event IDs are global, but callbacks from separate one-root + // streams are not promised to be delivered in global-ID order. The + // resume cursor is therefore the durable high-water mark, while the + // individual event retains its exact native ID for fence filtering. + let cursor_event_id = shared.lock().last_provider_event_id.max(provider_event_id); + let mut cursor = cursor_template.clone(); + cursor.event_id = cursor_event_id; + let provider_cursor = match cursor.encode() { + Ok(cursor) => cursor, + Err(error) => { + shared.revoke(format!("fsevents_cursor_encode_failure:{error}")); + return; + } + }; + let record = ObserverRecord { + sequence, + source: EvidenceSource::Observer, + path: path.clone(), + flags, + provider_cursor, + }; + let (cut, controlled_rotation) = match controlled_response.is_some() { + true => match durability.append_flush_and_rotate(record) { + Ok((sealed, anchor)) => (sealed.clone(), Some((sealed, anchor))), + Err(error) => { + if let Some(response) = controlled_response { + let _ = response.send(Err(reconcile_error( + "fsevents_controlled_fence_durability_failure", + ))); + } + shared.revoke(format!( + "fsevents_controlled_fence_durability_failure:{error}" + )); + return; + } + }, + false => match durability.append_and_flush(record) { + Ok(cut) => (cut, None), + Err(error) => { + if let Some(response) = response { + let _ = response.send(Err(reconcile_error("fsevents_durability_failure"))); + } + if let Some(response) = direct_response { + let _ = response.send(Err(reconcile_error("fsevents_durability_failure"))); + } + shared.revoke(format!("fsevents_durability_failure:{error}")); + return; + } + }, + }; + let public = ObserverFence { + sequence, + durable_offset: cut.durable_end_offset, + nonce: fence_nonce, + }; + { + let mut state = shared.lock(); + state.next_sequence = sequence.saturating_add(1); + state.last_provider_event_id = state.last_provider_event_id.max(provider_event_id); + state.last_cursor = Some(cursor); + state.events.push(DurableEvent { + event: ObserverEvent { + path, + flags, + sequence, + }, + provider_event_id, + cut: cut.clone(), + internal_fence: internal, + }); + if state.events.len() > MAX_RETAINED_EVENTS { + drop(state); + shared.revoke("fsevents_retained_event_overflow"); + return; + } + shared.changed.notify_all(); + } + if let Some(response) = response { + let _ = response.send(Ok((public.clone(), cut.clone(), provider_event_id))); + } + if let (Some(response), Some((sealed, anchor))) = (controlled_response, controlled_rotation) + { + let _ = response.send(Ok((public, sealed, anchor, provider_event_id))); + } + if let Some(reason) = revocation { + match durability.revoke_owner(&reason) { + Ok(()) => { + shared.revoke(reason); + if let Some(response) = direct_response { + let _ = response.send(Ok(())); + } + } + Err(error) => { + shared.revoke(format!("fsevents_policy_owner_revocation_failure:{error}")); + if let Some(response) = direct_response { + let _ = response.send(Err(reconcile_error( + "fsevents_policy_owner_revocation_failure", + ))); + } + } + } + return; + } + } +} + +fn native_capabilities() -> ProviderCapabilities { + ProviderCapabilities { + durable_cursor: true, + linearizable_fence: true, + rename_pairing: false, + overflow_scope: true, + filesystem_supported: true, + clean_proof_allowed: true, + power_loss_durability: true, + } +} + +fn open_root_no_follow(path: &Path) -> Result { + Ok(OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path)?) +} + +fn actual_history_authority(root: &Path, device: u64) -> Result { + let observed_device = open_root_no_follow(root)?.metadata()?.dev(); + if observed_device != device { + return Err(reconcile_error( + "fsevents_history_authority_device_mismatch", + )); + } + let database_uuid = copy_history_database_uuid(device)?; + let device_relative_root = device_relative_root(root)?; + Ok(HistoryAuthority { + device, + database_uuid, + device_relative_root, + }) +} + +fn device_relative_root(root: &Path) -> Result { + let root_text = root + .to_str() + .ok_or_else(|| reconcile_error("fsevents_non_utf8_root"))?; + let root_c = std::ffi::CString::new(root.as_os_str().as_bytes()) + .map_err(|_| reconcile_error("fsevents_root_contains_nul"))?; + let mut stat: libc::statfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statfs(root_c.as_ptr(), &mut stat) } != 0 { + return Err(reconcile_error("fsevents_root_statfs_failure")); + } + let mount = unsafe { CStr::from_ptr(stat.f_mntonname.as_ptr()) } + .to_str() + .map_err(|_| reconcile_error("fsevents_mount_path_not_utf8"))?; + let device_relative_root = root + .strip_prefix(mount) + .ok() + .and_then(Path::to_str) + .map(|path| path.trim_start_matches('/').to_owned()) + // The writable APFS root-data volume is exposed through firmlinks + // such as /Users and /private while its mount point is + // /System/Volumes/Data. Those paths are nevertheless present at the + // same relative location in the device namespace. + .unwrap_or_else(|| root_text.trim_start_matches('/').to_owned()); + if !device_relative_root.is_empty() + && device_relative_root + .split('/') + .any(|component| component.is_empty() || component == "." || component == "..") + { + return Err(reconcile_error( + "fsevents_device_relative_root_is_not_normal", + )); + } + Ok(device_relative_root) +} + +fn copy_history_database_uuid(device: u64) -> Result<[u8; 16]> { + let uuid = unsafe { FSEventsCopyUUIDForDevice(device as libc::dev_t) }; + if uuid.is_null() { + return Err(reconcile_error( + "fsevents_history_database_uuid_unavailable", + )); + } + let bytes = unsafe { CFUUIDGetUUIDBytes(uuid) }; + unsafe { CFRelease(uuid as CFTypeRef) }; + Ok([ + bytes.byte0, + bytes.byte1, + bytes.byte2, + bytes.byte3, + bytes.byte4, + bytes.byte5, + bytes.byte6, + bytes.byte7, + bytes.byte8, + bytes.byte9, + bytes.byte10, + bytes.byte11, + bytes.byte12, + bytes.byte13, + bytes.byte14, + bytes.byte15, + ]) +} + +fn root_identity(file: &File) -> Result> { + let metadata = file.metadata()?; + Ok(format!( + "root-v1:dev={};ino={};mode={};uid={};gid={}", + metadata.dev(), + metadata.ino(), + metadata.mode(), + metadata.uid(), + metadata.gid() + ) + .into_bytes()) +} + +fn ensure_apfs(path: &Path) -> Result<()> { + let path_c = std::ffi::CString::new(path.as_os_str().as_bytes()) + .map_err(|_| Error::InvalidInput("observer root contained NUL".into()))?; + let mut stat: libc::statfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statfs(path_c.as_ptr(), &mut stat) } != 0 { + return Err(Error::Io(std::io::Error::last_os_error())); + } + let filesystem = unsafe { CStr::from_ptr(stat.f_fstypename.as_ptr()) } + .to_str() + .map_err(|_| Error::InvalidInput("filesystem type was not UTF-8".into()))?; + if !macos_filesystem_is_supported(filesystem) { + return Err(Error::ChangeLedgerReconcileRequired { + scope: path.display().to_string(), + state: "stale_baseline".into(), + reason: format!("native macOS changed-path observer requires APFS, found {filesystem}"), + command: "trail index reconcile".into(), + }); + } + Ok(()) +} + +fn macos_filesystem_is_supported(filesystem: &str) -> bool { + filesystem == "apfs" +} + +#[cfg(debug_assertions)] +pub(crate) fn run_unsupported_filesystem_rejection() -> std::result::Result<(), String> { + if !macos_filesystem_is_supported("apfs") { + return Err("APFS was rejected".into()); + } + for unsupported in ["hfs", "nfs", "smbfs", "osxfuse"] { + if macos_filesystem_is_supported(unsupported) { + return Err(format!( + "unsupported macOS filesystem `{unsupported}` was qualified" + )); + } + } + Ok(()) +} + +fn reconcile_error(reason: &str) -> Error { + Error::ChangeLedgerReconcileRequired { + scope: "native-macos-observer".into(), + state: "untrusted_gap".into(), + reason: reason.into(), + command: "trail status".into(), + } +} + +#[cfg(debug_assertions)] +struct MemoryDurability { + binding: ObserverWriterBinding, + offset: u64, + delay: Duration, + records: Arc>>, +} + +#[cfg(debug_assertions)] +impl MacObserverDurability for MemoryDurability { + fn binding(&self) -> ObserverWriterBinding { + self.binding.clone() + } + + fn append_and_flush(&mut self, record: ObserverRecord) -> Result { + if record.sequence != self.offset.saturating_add(1) { + return Err(Error::InvalidInput( + "test durability received non-contiguous sequence".into(), + )); + } + if !self.delay.is_zero() { + thread::sleep(self.delay); + } + self.offset = self.offset.saturating_add(1); + let provider_cursor = record.provider_cursor.clone(); + self.records + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .push(record.clone()); + Ok(DurableCut { + segment_id: "macos-native-test".into(), + durable_end_offset: self.offset, + last_sequence: record.sequence, + last_hash: [0; 32], + provider_cursor, + }) + } + + fn append_flush_and_rotate( + &mut self, + record: ObserverRecord, + ) -> Result<(DurableCut, DurableCut)> { + let sealed = self.append_and_flush(record)?; + Ok((sealed.clone(), sealed)) + } + + fn rotate_if_cut(&mut self, expected: &DurableCut) -> Result> { + let provider_cursor = self + .records + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .last() + .map(|record| record.provider_cursor.clone()) + .unwrap_or_default(); + let cut = DurableCut { + segment_id: "macos-native-test".into(), + durable_end_offset: self.offset, + last_sequence: self.offset, + last_hash: [0; 32], + provider_cursor, + }; + Ok((&cut == expected).then_some((cut.clone(), cut))) + } + + fn revoke_owner(&mut self, _reason: &str) -> Result<()> { + Ok(()) + } +} + +#[cfg(debug_assertions)] +fn memory_durability( + provider_identity: Vec, + delay: Duration, +) -> (MemoryDurability, Arc>>) { + let records = Arc::new(Mutex::new(Vec::new())); + ( + MemoryDurability { + binding: ObserverWriterBinding { + owner_token: hex::encode([0x71; 32]), + provider_id: hex::encode(&provider_identity), + provider_identity, + fence_nonce: vec![0x72; 24], + }, + offset: 0, + delay, + records: Arc::clone(&records), + }, + records, + ) +} + +#[cfg(debug_assertions)] +struct TestFixture { + temp: tempfile::TempDir, + observer: MacOsFseventsObserver, + expected: ExpectedScope, + records: Arc>>, +} + +#[cfg(debug_assertions)] +struct NativeSegmentFixture { + temp: tempfile::TempDir, + db: Trail, + expected: ExpectedScope, + segment_directory: PathBuf, +} + +#[cfg(debug_assertions)] +impl NativeSegmentFixture { + fn new() -> Result { + let temp = tempfile::tempdir()?; + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false)?; + let db = Trail::open(temp.path())?; + let branch = db.current_branch()?; + let head = db.resolve_branch_ref(&branch)?; + let scope = ScopeIdentity { + scope_id: ScopeId([0xb8; 32]), + kind: ScopeKind::Workspace, + owner_id: "macos-native-segment".into(), + }; + let provider_identity = b"macos-fsevents-segment-writer-v1".to_vec(); + let filesystem_identity = root_identity(&open_root_no_follow(temp.path())?)?; + let baseline = BaselineIdentity { + ref_name: head.name.clone(), + ref_generation: u64::try_from(head.generation) + .map_err(|_| Error::Corrupt("negative native ref generation".into()))?, + change_id: head.change_id, + root_id: head.root_id, + }; + db.changed_path_ledger().begin_scope( + &scope, + &baseline, + &PolicyIdentity { + fingerprint: [0xb9; 32], + generation: 1, + }, + &FilesystemIdentity(filesystem_identity.clone()), + &ProviderIdentity { + identity: provider_identity.clone(), + capabilities: native_capabilities(), + }, + )?; + let expected = ExpectedScope { + scope_id: scope.scope_id, + epoch: 1, + ref_name: baseline.ref_name, + ref_generation: baseline.ref_generation, + baseline_root: baseline.root_id, + policy_fingerprint: [0xb9; 32], + policy_generation: 1, + filesystem_identity, + provider_identity, + }; + let segment_directory = db.db_dir.join("change-observer-segments"); + Ok(Self { + temp, + db, + expected, + segment_directory, + }) + } + + fn observer(&self) -> Result { + let writer = SegmentWriter::acquire( + &self.db.sqlite_path, + &self.segment_directory, + self.expected.scope_id, + self.expected.epoch, + [0xba; 32], + &hex::encode(&self.expected.provider_identity), + Vec::new(), + Duration::from_secs(3_600), + )?; + let durability = MacSegmentWriterDurability::new( + writer, + self.expected.provider_identity.clone(), + vec![0xbb; 24], + )?; + MacOsFseventsObserver::start( + self.temp.path(), + Box::new(durability), + None, + &[self.temp.path().join(".trail/config.toml")], + ) + } +} + +#[cfg(debug_assertions)] +impl TestFixture { + fn new() -> Result { + let temp = tempfile::tempdir()?; + std::fs::create_dir(temp.path().join(".trail"))?; + Self::at(temp, None, Duration::ZERO) + } + + fn at( + temp: tempfile::TempDir, + resume: Option, + delay: Duration, + ) -> Result { + let provider_identity = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, records) = memory_durability(provider_identity.clone(), delay); + let observer = MacOsFseventsObserver::start( + temp.path(), + Box::new(durability), + resume, + &[temp.path().join(".trail/config.toml")], + )?; + let expected = ExpectedScope { + scope_id: crate::db::change_ledger::ScopeId([0xa8; 32]), + epoch: 1, + ref_name: "refs/branches/main".into(), + ref_generation: 1, + baseline_root: crate::ObjectId("object_macos_observer".into()), + policy_fingerprint: [0xa9; 32], + policy_generation: 1, + filesystem_identity: observer.root_identity.clone(), + provider_identity, + }; + Ok(Self { + temp, + observer, + expected, + records, + }) + } + + fn interval(&self, action: impl FnOnce(&Path) -> Result<()>) -> Result> { + let start = self.observer.begin_observation(&self.expected)?; + action(self.temp.path())?; + let end = self.observer.end_fence(&self.expected, &start)?; + let mut events = Vec::new(); + self.observer.drain_through( + &self.expected, + &self.observer.root_identity, + &start, + &end, + &mut |event| { + events.push(event); + Ok(()) + }, + )?; + Ok(events) + } +} + +#[cfg(debug_assertions)] +fn has_event(events: &[ObserverEvent], path: &str, flag: EvidenceFlags) -> bool { + events + .iter() + .any(|event| event.path.as_str() == path && event.flags.0 & flag.0 != 0) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_real_apfs_file_events() -> std::result::Result<(), String> { + fn run() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let fixture = TestFixture::new()?; + ensure_apfs(fixture.temp.path())?; + let events = fixture.interval(|root| { + std::fs::write(root.join("tracked.txt"), b"one")?; + Ok(()) + })?; + if !has_event(&events, "tracked.txt", EvidenceFlags::CREATE) { + return Err(Error::Corrupt("FSEvents omitted file create".into())); + } + let events = fixture.interval(|root| { + std::fs::write(root.join("tracked.txt"), b"two")?; + std::fs::set_permissions( + root.join("tracked.txt"), + std::fs::Permissions::from_mode(0o700), + )?; + Ok(()) + })?; + if !has_event(&events, "tracked.txt", EvidenceFlags::CONTENT) + || !has_event(&events, "tracked.txt", EvidenceFlags::MODE) + { + return Err(Error::Corrupt( + "FSEvents omitted content or mode evidence".into(), + )); + } + let events = fixture.interval(|root| { + std::fs::rename(root.join("tracked.txt"), root.join("renamed.txt"))?; + Ok(()) + })?; + for path in ["tracked.txt", "renamed.txt"] { + if !has_event(&events, path, EvidenceFlags::RENAME_FROM) + || !has_event(&events, path, EvidenceFlags::RENAME_TO) + { + return Err(Error::Corrupt(format!( + "FSEvents omitted conservative rename endpoint {path}" + ))); + } + } + let events = fixture.interval(|root| { + std::fs::create_dir(root.join("dir-a"))?; + std::fs::write(root.join("dir-a/child"), b"child")?; + std::fs::rename(root.join("dir-a"), root.join("dir-b"))?; + std::fs::write(root.join("case"), b"case")?; + std::fs::rename(root.join("case"), root.join("CASE"))?; + Ok(()) + })?; + for path in ["dir-a", "dir-b", "case", "CASE"] { + if !has_event(&events, path, EvidenceFlags::RENAME_FROM) { + return Err(Error::Corrupt(format!( + "FSEvents omitted directory/case rename endpoint {path}" + ))); + } + } + let events = fixture.interval(|root| { + std::fs::remove_file(root.join("renamed.txt"))?; + Ok(()) + })?; + if !has_event(&events, "renamed.txt", EvidenceFlags::DELETE) { + return Err(Error::Corrupt("FSEvents omitted file delete".into())); + } + let events = fixture.interval(|root| { + for index in 0..256 { + std::fs::write(root.join(format!("batch-{index}")), b"batch")?; + } + Ok(()) + })?; + if !has_event(&events, "batch-255", EvidenceFlags::CREATE) { + return Err(Error::Corrupt( + "synchronous flush omitted delayed batch tail".into(), + )); + } + let events = fixture.interval(|root| { + std::fs::write(root.join(".trail/internal-noise"), b"noise")?; + Ok(()) + })?; + if events + .iter() + .any(|event| event.path.as_str() == ".trail/internal-noise") + { + return Err(Error::Corrupt( + "internal storage noise leaked into the ledger".into(), + )); + } + if fixture.records.lock().unwrap().is_empty() { + return Err(Error::Corrupt( + "durability worker appended no records".into(), + )); + } + Ok(()) + } + run().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_gap_flag_matrix() -> std::result::Result<(), String> { + fn run() -> Result<()> { + for (flag, reason) in [ + ( + fs_events::kFSEventStreamEventFlagMustScanSubDirs, + "fsevents_must_scan_subdirs", + ), + ( + fs_events::kFSEventStreamEventFlagUserDropped, + "fsevents_user_dropped", + ), + ( + fs_events::kFSEventStreamEventFlagKernelDropped, + "fsevents_kernel_dropped", + ), + ( + fs_events::kFSEventStreamEventFlagEventIdsWrapped, + "fsevents_event_ids_wrapped", + ), + ( + fs_events::kFSEventStreamEventFlagRootChanged, + "fsevents_root_changed", + ), + ( + fs_events::kFSEventStreamEventFlagUnmount, + "fsevents_unmount", + ), + ] { + let fixture = TestFixture::new()?; + let error = fixture.observer.inject_flags(flag).unwrap_err().to_string(); + if !error.contains(reason) + || fixture.observer.ensure_available().unwrap_err().code() + != "CHANGE_LEDGER_RECONCILE_REQUIRED" + { + return Err(Error::Corrupt(format!( + "gap flag did not globally revoke qualification: {reason}" + ))); + } + } + let fixture = TestFixture::new()?; + let error = fixture + .observer + .inject_flags(fs_events::kFSEventStreamEventFlagHistoryDone) + .unwrap_err(); + if !error + .to_string() + .contains("fsevents_inconsistent_history_done") + { + return Err(Error::Corrupt( + "unexpected HistoryDone did not fail closed".into(), + )); + } + Ok(()) + } + run().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +fn callback_overflow_or_disconnect(disconnect: bool) -> Result<()> { + let temp = tempfile::tempdir()?; + let shared = Arc::new(Shared { + state: Mutex::new(State { + active: true, + revoked: None, + history_pending: 0, + events: Vec::new(), + next_sequence: 1, + last_provider_event_id: 0, + last_cursor: None, + issued_fences: HashMap::new(), + }), + changed: Condvar::new(), + shutdown: AtomicBool::new(false), + }); + let (tx, rx) = mpsc::sync_channel(1); + if disconnect { + drop(rx); + } else { + tx.send(DurabilityCommand::Shutdown) + .map_err(|_| Error::InvalidInput("could not prime bounded queue".into()))?; + } + let context = CallbackContext { + root_path: temp.path().to_path_buf(), + device_relative_root: PathBuf::new(), + coverage_roots: vec![CallbackCoverageRoot { + absolute_root: temp.path().to_path_buf(), + device_relative_root: PathBuf::new(), + }], + policy_watches: Vec::new(), + records: tx, + shared: Arc::clone(&shared), + }; + let path = std::ffi::CString::new(temp.path().join("overflow").to_string_lossy().as_bytes()) + .map_err(|_| Error::InvalidInput("test callback path contained NUL".into()))?; + let paths = [path.as_ptr()]; + let flags = [fs_events::kFSEventStreamEventFlagItemCreated + | fs_events::kFSEventStreamEventFlagItemIsFile]; + let ids = [1_u64]; + callback( + ptr::null_mut(), + (&context as *const CallbackContext).cast_mut().cast(), + 1, + paths.as_ptr().cast_mut().cast(), + flags.as_ptr(), + ids.as_ptr(), + ); + let reason = shared + .lock() + .revoked + .clone() + .ok_or_else(|| Error::Corrupt("bounded callback failure did not revoke".into()))?; + let expected = if disconnect { + "fsevents_durability_worker_disconnected" + } else { + "fsevents_bounded_callback_queue_overflow" + }; + if reason != expected { + return Err(Error::Corrupt(format!( + "bounded callback revoked for {reason}, expected {expected}" + ))); + } + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_continuity_fault_matrix() -> std::result::Result<(), String> { + if let Ok(root) = std::env::var("TRAIL_MACOS_OBSERVER_OWNER_CHILD_ROOT") { + return run_owner_process_child(Path::new(&root)).map_err(|error| error.to_string()); + } + fn run() -> Result<()> { + use std::os::unix::fs::{symlink, PermissionsExt}; + + callback_overflow_or_disconnect(false)?; + callback_overflow_or_disconnect(true)?; + + let external_root = tempfile::tempdir()?; + std::fs::create_dir(external_root.path().join(".trail"))?; + let external_policy_home = tempfile::tempdir()?; + let external_dependency = external_policy_home.path().join("missing-global.gitconfig"); + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, external_records) = memory_durability(provider, Duration::ZERO); + let external_observer = MacOsFseventsObserver::start( + external_root.path(), + Box::new(durability), + None, + std::slice::from_ref(&external_dependency), + )?; + std::fs::write(external_policy_home.path().join("unrelated"), b"unrelated")?; + for stream in &external_observer.stream.streams { + unsafe { + fs_events::FSEventStreamFlushSync(*stream as fs_events::FSEventStreamRef); + } + } + if external_observer.ensure_available().is_err() + || !external_records + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .is_empty() + { + return Err(Error::Corrupt( + "unrelated covered sibling leaked into policy invalidation".into(), + )); + } + std::fs::write(&external_dependency, b"[core]\n\texcludesFile = ignored\n")?; + let deadline = Instant::now() + FENCE_TIMEOUT; + while external_observer.ensure_available().is_ok() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + if external_observer.ensure_available().is_ok() { + return Err(Error::Corrupt( + "same-device external policy creation did not revoke observation".into(), + )); + } + let records = external_records + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + if !records.iter().any(|record| { + record + .path + .as_str() + .starts_with(".trail/policy-invalidations/") + }) { + return Err(Error::Corrupt( + "external policy invalidation was not durable before revocation".into(), + )); + } + drop(records); + drop(external_observer); + + let durable_fixture = NativeSegmentFixture::new()?; + let durable_policy_home = tempfile::tempdir()?; + let durable_dependency = durable_policy_home.path().join("missing-global.gitconfig"); + let writer = SegmentWriter::acquire( + &durable_fixture.db.sqlite_path, + &durable_fixture.segment_directory, + durable_fixture.expected.scope_id, + durable_fixture.expected.epoch, + [0xda; 32], + &hex::encode(&durable_fixture.expected.provider_identity), + Vec::new(), + Duration::from_secs(3_600), + )?; + let durability = MacSegmentWriterDurability::new( + writer, + durable_fixture.expected.provider_identity.clone(), + vec![0xdb; 24], + )?; + let durable_observer = MacOsFseventsObserver::start( + durable_fixture.temp.path(), + Box::new(durability), + None, + std::slice::from_ref(&durable_dependency), + )?; + std::fs::write(&durable_dependency, b"[core]\n")?; + let deadline = Instant::now() + FENCE_TIMEOUT; + let persisted = loop { + let persisted = durable_fixture.db.conn.query_row( + "SELECT owner.lease_state,owner.error_state, + segment.last_sequence,segment.durable_end_offset + FROM changed_path_observer_owners owner + JOIN changed_path_observer_segments segment + ON segment.scope_id=owner.scope_id AND segment.epoch=owner.epoch + WHERE owner.scope_id=?1", + [durable_fixture.expected.scope_id.to_text()], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, i64>(3)?, + )) + }, + )?; + if persisted.0 == "error" || Instant::now() >= deadline { + break persisted; + } + thread::sleep(Duration::from_millis(5)); + }; + if persisted.0 != "error" + || !persisted + .1 + .as_deref() + .is_some_and(|reason| reason.contains("policy_dependency_invalidated")) + || persisted.2.unwrap_or(0) < 1 + || persisted.3 <= 0 + { + return Err(Error::Corrupt(format!( + "policy invalidation did not flush marker before durable owner revoke: {persisted:?}" + ))); + } + drop(durable_observer); + + let overlapping_home = tempfile::tempdir()?; + let nested_workspace = overlapping_home.path().join("workspace"); + std::fs::create_dir_all(nested_workspace.join(".trail"))?; + let overlapping_dependency = overlapping_home.path().join(".gitconfig"); + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider.clone(), Duration::ZERO); + let overlapping_observer = MacOsFseventsObserver::start( + &nested_workspace, + Box::new(durability), + None, + std::slice::from_ref(&overlapping_dependency), + )?; + if overlapping_observer.stream.streams.len() != 1 { + return Err(Error::Corrupt( + "overlapping policy/workspace roots created duplicate native streams".into(), + )); + } + let overlapping_expected = ExpectedScope { + scope_id: ScopeId([0xd8; 32]), + epoch: 1, + ref_name: "refs/branches/main".into(), + ref_generation: 1, + baseline_root: crate::ObjectId("overlapping-root".into()), + policy_fingerprint: [0xd9; 32], + policy_generation: 1, + filesystem_identity: overlapping_observer.root_identity.clone(), + provider_identity: provider, + }; + let start = overlapping_observer.begin_observation(&overlapping_expected)?; + std::fs::write(nested_workspace.join("one-event"), b"one")?; + let end = overlapping_observer.end_fence(&overlapping_expected, &start)?; + let mut duplicate_count = 0; + overlapping_observer.drain_through( + &overlapping_expected, + &overlapping_observer.root_identity, + &start, + &end, + &mut |event| { + if event.path.as_str() == "one-event" { + duplicate_count += 1; + } + Ok(()) + }, + )?; + if duplicate_count != 1 { + return Err(Error::Corrupt(format!( + "overlapping policy/workspace roots produced {duplicate_count} events" + ))); + } + + // When a writable second volume is available, prove that a repository + // there can use a home-volume Git policy dependency without claiming + // a second FSEvents history. The dependency is instead bound into the + // direct authenticated fence and any drift revokes the lease. + let policy_volume = tempfile::tempdir()?; + let cross_device_dependency = policy_volume.path().join("global.gitconfig"); + std::fs::write(&cross_device_dependency, b"[core]\n")?; + let policy_device = std::fs::metadata(policy_volume.path())?.dev(); + let mut cross_device_workspace = None; + if let Ok(volumes) = std::fs::read_dir("/Volumes") { + for volume in volumes.flatten() { + let path = volume.path(); + if std::fs::metadata(&path) + .ok() + .is_none_or(|metadata| !metadata.is_dir() || metadata.dev() == policy_device) + { + continue; + } + if let Ok(temp) = tempfile::Builder::new() + .prefix("trail-cross-device-policy-") + .tempdir_in(&path) + { + cross_device_workspace = Some(temp); + break; + } + } + } + if let Some(cross_device_workspace) = cross_device_workspace { + std::fs::create_dir(cross_device_workspace.path().join(".trail"))?; + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + let observer = MacOsFseventsObserver::start( + cross_device_workspace.path(), + Box::new(durability), + None, + std::slice::from_ref(&cross_device_dependency), + )?; + let lease = observer.lease()?; + if observer.stream.streams.len() != 1 + || observer.coverage_roots.len() != 1 + || !lease.policy_dependencies.is_empty() + || lease.direct_policy_fences.len() != 1 + || lease.direct_policy_fences[0].0 != cross_device_dependency + { + return Err(Error::Corrupt( + "cross-device policy dependency was not exactly direct-fenced".into(), + )); + } + std::fs::write( + &cross_device_dependency, + b"[core]\n\texcludesFile = ignored\n", + )?; + let error = observer + .lease() + .err() + .ok_or_else(|| Error::Corrupt("cross-device policy drift was accepted".into()))?; + if !error + .to_string() + .contains("fsevents_policy_dependency_invalidated") + { + return Err(Error::Corrupt( + "cross-device policy drift did not revoke exact authority".into(), + )); + } + + let nested_target = policy_volume.path().join("nested-target"); + let nested_parent = policy_volume.path().join("nested-parent"); + std::fs::create_dir_all(nested_target.join("existing"))?; + std::fs::create_dir(&nested_parent)?; + symlink(&nested_target, nested_parent.join("link"))?; + std::fs::write(nested_target.join("existing/global.gitconfig"), b"[core]\n")?; + let nested_dependency = nested_parent.join("link/existing/global.gitconfig"); + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + let nested_error = MacOsFseventsObserver::start( + cross_device_workspace.path(), + Box::new(durability), + None, + std::slice::from_ref(&nested_dependency), + ) + .err() + .ok_or_else(|| { + Error::Corrupt("cross-device nested policy symlink was accepted".into()) + })?; + if !nested_error + .to_string() + .contains("fsevents_policy_dependency_leaf_is_symlink_or_unsafe") + { + return Err(Error::Corrupt( + "cross-device nested policy symlink did not fail closed".into(), + )); + } + } + + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + let unsafe_error = MacOsFseventsObserver::start( + external_root.path(), + Box::new(durability), + None, + &[PathBuf::from("/dev/null")], + ) + .err() + .ok_or_else(|| Error::Corrupt("unsafe cross-device policy leaf was accepted".into()))?; + if !unsafe_error + .to_string() + .contains("fsevents_policy_dependency_leaf_is_symlink_or_unsafe") + { + return Err(Error::Corrupt( + "unsafe cross-device policy leaf did not fail closed explicitly".into(), + )); + } + + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + let system_dependency = PathBuf::from("/etc/gitconfig"); + let system_observer = MacOsFseventsObserver::start( + external_root.path(), + Box::new(durability), + None, + std::slice::from_ref(&system_dependency), + )?; + let system_lease = system_observer.lease()?; + if system_lease.policy_dependencies != [system_dependency] + || system_observer.system_aliases.len() != 1 + || system_observer.system_aliases[0].alias_path != Path::new("/etc") + || !system_observer + .coverage_roots + .iter() + .any(|root| root.absolute_root == Path::new("/private/etc")) + { + return Err(Error::Corrupt( + "normal real-Git /etc system policy was not canonically covered".into(), + )); + } + drop(system_observer); + + let symlink_policy_home = tempfile::tempdir()?; + let symlink_target = tempfile::tempdir()?; + let symlink_parent = symlink_policy_home.path().join("linked-config-home"); + symlink(symlink_target.path(), &symlink_parent)?; + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + let error = MacOsFseventsObserver::start( + external_root.path(), + Box::new(durability), + None, + &[symlink_parent.join("global.gitconfig")], + ) + .err() + .ok_or_else(|| Error::Corrupt("symlinked policy parent was accepted".into()))?; + if !error + .to_string() + .contains("fsevents_policy_dependency_leaf_is_symlink_or_unsafe") + { + return Err(Error::Corrupt( + "symlinked policy parent did not fail closed explicitly".into(), + )); + } + + let in_root_symlink_target = external_root.path().join("actual-in-root.gitconfig"); + let in_root_symlink_dependency = external_root.path().join("linked-in-root.gitconfig"); + std::fs::write(&in_root_symlink_target, b"[core]\n")?; + symlink(&in_root_symlink_target, &in_root_symlink_dependency)?; + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + let in_root_error = MacOsFseventsObserver::start( + external_root.path(), + Box::new(durability), + None, + std::slice::from_ref(&in_root_symlink_dependency), + ) + .err(); + + let nested_target = external_root.path().join("in-root-nested-target"); + let nested_parent = external_root.path().join("in-root-nested-parent"); + std::fs::create_dir_all(nested_target.join("existing"))?; + std::fs::create_dir(&nested_parent)?; + symlink(&nested_target, nested_parent.join("link"))?; + std::fs::write(nested_target.join("existing/config"), b"[core]\n")?; + let in_root_nested_dependency = nested_parent.join("link/existing/config"); + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + let in_root_nested_error = MacOsFseventsObserver::start( + external_root.path(), + Box::new(durability), + None, + std::slice::from_ref(&in_root_nested_dependency), + ) + .err(); + + let external_leaf_home = tempfile::tempdir()?; + let external_symlink_target = external_leaf_home.path().join("actual-global.gitconfig"); + let external_symlink_dependency = external_leaf_home.path().join("linked-global.gitconfig"); + std::fs::write(&external_symlink_target, b"[core]\n")?; + symlink(&external_symlink_target, &external_symlink_dependency)?; + let external_error = if std::fs::metadata(external_leaf_home.path())?.dev() + == std::fs::metadata(external_root.path())?.dev() + { + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + MacOsFseventsObserver::start( + external_root.path(), + Box::new(durability), + None, + std::slice::from_ref(&external_symlink_dependency), + ) + .err() + } else { + None + }; + for (kind, error) in [ + ("in-root", in_root_error), + ("in-root nested-ancestor", in_root_nested_error), + ("same-device external", external_error), + ] { + let error = error.ok_or_else(|| { + Error::Corrupt(format!( + "{kind} symlink policy dependency leaf was accepted" + )) + })?; + if !error + .to_string() + .contains("fsevents_policy_dependency_leaf_is_symlink_or_unsafe") + { + return Err(Error::Corrupt(format!( + "{kind} symlink policy dependency leaf did not fail closed explicitly" + ))); + } + } + + let lease_root = tempfile::tempdir()?; + std::fs::create_dir(lease_root.path().join(".trail"))?; + let lease_policy_home = tempfile::tempdir()?; + let lease_dependency = lease_policy_home.path().join("global.gitconfig"); + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + let lease_observer = MacOsFseventsObserver::start( + lease_root.path(), + Box::new(durability), + None, + std::slice::from_ref(&lease_dependency), + )?; + lease_observer.fail_next_coverage_descriptor_for_test(); + if lease_observer.lease().is_ok() { + return Err(Error::Corrupt( + "coverage descriptor failure remained lease-authoritative".into(), + )); + } + drop(lease_observer); + + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + let lease_observer = MacOsFseventsObserver::start( + lease_root.path(), + Box::new(durability), + None, + std::slice::from_ref(&lease_dependency), + )?; + let policy_home = lease_policy_home.path().to_path_buf(); + let displaced_policy_home = policy_home.with_extension("displaced-policy-home"); + std::fs::rename(&policy_home, &displaced_policy_home)?; + std::fs::create_dir(&policy_home)?; + let lease_result = lease_observer.lease(); + std::fs::remove_dir(&policy_home)?; + std::fs::rename(&displaced_policy_home, &policy_home)?; + if lease_result.is_ok() { + return Err(Error::Corrupt( + "replaced external coverage parent remained lease-authoritative".into(), + )); + } + + let leftover_temp = tempfile::tempdir()?; + std::fs::create_dir(leftover_temp.path().join(".trail"))?; + let leftover_dir = leftover_temp.path().join(".trail/observer-fences"); + std::fs::create_dir(&leftover_dir)?; + std::fs::set_permissions(&leftover_dir, std::fs::Permissions::from_mode(0o700))?; + std::fs::write(leftover_dir.join("crash-leftover"), b"old")?; + let leftover = TestFixture::at(leftover_temp, None, Duration::ZERO)?; + let events = leftover.interval(|root| { + std::fs::write(root.join("visible"), b"visible")?; + Ok(()) + })?; + if events + .iter() + .any(|event| event.path.as_str().starts_with(".trail/observer-fences/")) + || !has_event(&events, "visible", EvidenceFlags::CREATE) + { + return Err(Error::Corrupt( + "crash-leftover fence leaked into user-visible evidence".into(), + )); + } + + let collision = TestFixture::new()?; + let collision_nonce = vec![0xc1; 24]; + std::fs::write( + collision.temp.path().join(format!( + ".trail/observer-fences/{}", + hex::encode(&collision_nonce) + )), + b"hostile", + )?; + collision + .observer + .set_next_fence_nonce_for_test(collision_nonce); + let error = collision + .observer + .begin_observation(&collision.expected) + .unwrap_err() + .to_string(); + if !error.contains("fsevents_fence_collision_or_create_failure") + || collision.observer.ensure_available().is_ok() + { + return Err(Error::Corrupt( + "fence collision did not globally revoke qualification".into(), + )); + } + + let sync_failure = TestFixture::new()?; + sync_failure.observer.fail_next_fence_sync_for_test(); + let error = sync_failure + .observer + .begin_observation(&sync_failure.expected) + .unwrap_err() + .to_string(); + if !error.contains("fsevents_fence_file_sync_failure") + || sync_failure.observer.ensure_available().is_ok() + || std::fs::read_dir(sync_failure.temp.path().join(".trail/observer-fences"))? + .next() + .is_some() + { + return Err(Error::Corrupt( + "fence sync failure did not revoke and clean up".into(), + )); + } + + let fixture = TestFixture::new()?; + fixture.interval(|root| { + std::fs::write(root.join("restart"), b"before")?; + Ok(()) + })?; + let cursor = fixture + .observer + .resume_cursor()? + .ok_or_else(|| Error::Corrupt("durable resume cursor was absent".into()))?; + let persisted_cursor = { + let records = fixture + .records + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + MacOsProviderCursor::decode( + &records + .last() + .ok_or_else(|| Error::Corrupt("persisted cursor record was absent".into()))? + .provider_cursor, + )? + }; + if persisted_cursor != cursor { + return Err(Error::Corrupt( + "persisted provider cursor did not round-trip exactly".into(), + )); + } + let root = fixture.temp.path().to_path_buf(); + let temp = fixture.temp; + drop(fixture.observer); + let resumed = TestFixture::at(temp, Some(cursor.clone()), Duration::ZERO)?; + let events = resumed.interval(|root| { + std::fs::write(root.join("restart"), b"after")?; + Ok(()) + })?; + if !has_event(&events, "restart", EvidenceFlags::CONTENT) { + return Err(Error::Corrupt( + "resumed stream omitted post-restart content".into(), + )); + } + let mut forged = cursor; + forged.device = forged.device.saturating_add(1); + let bogus_temp = tempfile::tempdir()?; + std::fs::create_dir(bogus_temp.path().join(".trail"))?; + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + if MacOsFseventsObserver::start(bogus_temp.path(), Box::new(durability), Some(forged), &[]) + .is_ok() + { + return Err(Error::Corrupt("forged resume identity was accepted".into())); + } + + let displaced = root.with_extension("displaced-macos-observer"); + std::fs::rename(&root, &displaced)?; + std::fs::create_dir(&root)?; + let error = resumed.observer.root_identity().unwrap_err(); + if error.code() != "CHANGE_LEDGER_RECONCILE_REQUIRED" + || (!error.to_string().contains("fsevents_root_replaced") + && !error.to_string().contains("fsevents_root_changed")) + { + return Err(Error::Corrupt("root replacement did not revoke".into())); + } + std::fs::remove_dir(&root)?; + std::fs::rename(&displaced, &root)?; + + let fixture = TestFixture::new()?; + fixture.observer.stop_durability_worker_for_test()?; + let deadline = Instant::now() + FENCE_TIMEOUT; + loop { + if fixture.observer.ensure_available().is_err() { + break; + } + if Instant::now() >= deadline { + return Err(Error::Corrupt( + "durability worker death did not revoke observer".into(), + )); + } + thread::sleep(Duration::from_millis(1)); + } + + run_owner_process_death()?; + + Ok(()) + } + run().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +fn run_owner_process_child(root: &Path) -> Result<()> { + use std::io::Write as _; + + let db = Trail::open(root)?; + let branch = db.current_branch()?; + let head = db.resolve_branch_ref(&branch)?; + let expected = ExpectedScope { + scope_id: ScopeId([0xb8; 32]), + epoch: 1, + ref_name: head.name, + ref_generation: u64::try_from(head.generation) + .map_err(|_| Error::Corrupt("negative child ref generation".into()))?, + baseline_root: head.root_id, + policy_fingerprint: [0xb9; 32], + policy_generation: 1, + filesystem_identity: root_identity(&open_root_no_follow(root)?)?, + provider_identity: b"macos-fsevents-segment-writer-v1".to_vec(), + }; + let segment_directory = db.db_dir.join("change-observer-segments"); + let writer = SegmentWriter::acquire( + &db.sqlite_path, + &segment_directory, + expected.scope_id, + expected.epoch, + [0xbc; 32], + &hex::encode(&expected.provider_identity), + Vec::new(), + Duration::from_secs(3_600), + )?; + let durability = MacSegmentWriterDurability::new( + writer, + expected.provider_identity.clone(), + vec![0xbd; 24], + )?; + let observer = MacOsFseventsObserver::start( + root, + Box::new(durability), + None, + &[root.join(".trail/config.toml")], + )?; + observer.begin_observation(&expected)?; + println!("TRAIL_MACOS_OBSERVER_OWNER_READY"); + std::io::stdout().flush()?; + loop { + thread::sleep(Duration::from_secs(60)); + } +} + +#[cfg(debug_assertions)] +fn run_owner_process_death() -> Result<()> { + use std::io::{BufRead, BufReader}; + use std::process::{Command, Stdio}; + + let fixture = NativeSegmentFixture::new()?; + let executable = std::env::current_exe()?; + let mut child = Command::new(executable) + .arg("fsevents_restart_root_cursor_overflow_and_worker_death_fail_closed") + .arg("--nocapture") + .arg("--test-threads=1") + .env("TRAIL_MACOS_OBSERVER_OWNER_CHILD_ROOT", fixture.temp.path()) + .stdout(Stdio::piped()) + .spawn()?; + let stdout = child.stdout.take().ok_or_else(|| { + Error::InvalidInput("macOS observer owner child stdout was unavailable".into()) + })?; + let mut ready = false; + for line in BufReader::new(stdout).lines() { + let line = line?; + if line.contains("TRAIL_MACOS_OBSERVER_OWNER_READY") { + ready = true; + break; + } + } + if !ready { + let _ = child.kill(); + let _ = child.wait(); + return Err(Error::Corrupt( + "macOS observer owner child exited before readiness".into(), + )); + } + child.kill()?; + let status = child.wait()?; + if status.success() { + return Err(Error::Corrupt( + "macOS observer owner child was not killed".into(), + )); + } + let replacement = SegmentWriter::acquire( + &fixture.db.sqlite_path, + &fixture.segment_directory, + fixture.expected.scope_id, + fixture.expected.epoch, + [0xbe; 32], + &hex::encode(&fixture.expected.provider_identity), + Vec::new(), + Duration::from_secs(3_600), + ); + if replacement.is_ok() { + return Err(Error::Corrupt( + "same-epoch macOS observer owner replacement succeeded after SIGKILL".into(), + )); + } + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_fence_ordering() -> std::result::Result<(), String> { + fn run() -> Result<()> { + let fixture = NativeSegmentFixture::new()?; + let observer = fixture.observer()?; + let start = observer.begin_observation(&fixture.expected)?; + std::fs::write(fixture.temp.path().join("ordered"), b"ordered")?; + let end = observer.end_fence(&fixture.expected, &start)?; + let issued = observer.issued_fence(&fixture.expected, &end)?; + let state = observer.shared.lock(); + let changed = state + .events + .iter() + .find(|event| event.event.path.as_str() == "ordered") + .ok_or_else(|| Error::Corrupt("ordered callback was not durably retained".into()))?; + if changed.event.sequence >= end.sequence + || changed.provider_event_id > issued.provider_event_id + || changed.cut.durable_end_offset >= end.durable_offset + || issued.durable_cut + != state + .events + .iter() + .find(|event| event.event.sequence == end.sequence) + .ok_or_else(|| Error::Corrupt("durable fence marker was absent".into()))? + .cut + { + return Err(Error::Corrupt( + "FlushSync fence was not ordered after durable callbacks".into(), + )); + } + drop(state); + let persisted: (i64, i64) = fixture.db.conn.query_row( + "SELECT last_sequence,durable_end_offset + FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=1 AND state='open'", + [fixture.expected.scope_id.to_text()], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + if persisted.0 != i64::try_from(end.sequence).unwrap_or(-1) + || persisted.1 != i64::try_from(end.durable_offset).unwrap_or(-1) + { + return Err(Error::Corrupt( + "SegmentWriter did not persist the exact synchronous fence cut".into(), + )); + } + Ok(()) + } + run().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +fn test_shared(last_provider_event_id: u64) -> Arc { + Arc::new(Shared { + state: Mutex::new(State { + active: true, + revoked: None, + history_pending: 0, + events: Vec::new(), + next_sequence: 1, + last_provider_event_id, + last_cursor: None, + issued_fences: HashMap::new(), + }), + changed: Condvar::new(), + shutdown: AtomicBool::new(false), + }) +} + +#[cfg(debug_assertions)] +fn wait_for_test_events(shared: &Shared, count: usize) -> Result<()> { + let deadline = Instant::now() + FENCE_TIMEOUT; + let mut state = shared.lock(); + while state.events.len() < count && state.revoked.is_none() { + if Instant::now() >= deadline { + return Err(Error::Corrupt(format!( + "timed out waiting for {count} durable test events" + ))); + } + let waited = shared + .changed + .wait_timeout(state, Duration::from_millis(10)) + .unwrap_or_else(|poison| poison.into_inner()); + state = waited.0; + } + if let Some(reason) = &state.revoked { + return Err(reconcile_error(reason)); + } + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_paused_callback_fence() -> std::result::Result<(), String> { + fn run() -> Result<()> { + use std::sync::Barrier; + + let provider = b"macos-paused-callback-v1".to_vec(); + let (durability, _) = memory_durability(provider.clone(), Duration::ZERO); + let shared = test_shared(0); + let (commands, receiver) = mpsc::sync_channel(MAX_PENDING_RECORDS); + let worker_shared = Arc::clone(&shared); + let worker = thread::spawn(move || { + run_durability_worker( + receiver, + Box::new(durability), + worker_shared, + MacOsProviderCursor { + version: CAPABILITY_VERSION, + event_id: 0, + device: 1, + history_database_uuid: [1; 16], + device_relative_root: "tmp/workspace".into(), + coverage_roots: vec![CursorCoverageRoot { + device_relative_root: "tmp/workspace".into(), + root_identity: b"root".to_vec(), + }], + system_aliases: Vec::new(), + policy_dependencies: Vec::new(), + external_policy_fences: Vec::new(), + root_identity: b"root".to_vec(), + lineage_identity: vec![2; 24], + provider_identity: provider, + stream_flags: STREAM_FLAGS, + capabilities: native_capabilities(), + }, + ) + }); + commands + .send(DurabilityCommand::Record { + path: LedgerPath::parse("sentinel-delete")?, + flags: EvidenceFlags::DELETE, + provider_event_id: 10, + }) + .map_err(|_| Error::Corrupt("could not enqueue sentinel record".into()))?; + wait_for_test_events(&shared, 1)?; + + let paused = Arc::new(Barrier::new(2)); + let paused_sender = commands.clone(); + let paused_gate = Arc::clone(&paused); + let callback_thread = thread::spawn(move || { + paused_gate.wait(); + paused_sender.send(DurabilityCommand::Record { + path: LedgerPath::parse("post-sentinel-paused").expect("valid test path"), + flags: EvidenceFlags::CONTENT, + provider_event_id: 20, + }) + }); + let (response_tx, response_rx) = mpsc::sync_channel(1); + commands + .send(DurabilityCommand::Fence { + minimum_provider_event_id: 10, + nonce: vec![3; 24], + response: response_tx, + }) + .map_err(|_| Error::Corrupt("could not enqueue durability barrier".into()))?; + let (fence, cut, cursor_id) = response_rx + .recv_timeout(FENCE_TIMEOUT) + .map_err(|_| Error::Corrupt("durability barrier did not respond".into()))??; + if cursor_id != 10 || MacOsProviderCursor::decode(&cut.provider_cursor)?.event_id != 10 { + return Err(Error::Corrupt( + "fence overclaimed the paused callback provider ID".into(), + )); + } + paused.wait(); + callback_thread + .join() + .map_err(|_| Error::Corrupt("paused callback thread panicked".into()))? + .map_err(|_| Error::Corrupt("paused callback enqueue failed".into()))?; + wait_for_test_events(&shared, 3)?; + let state = shared.lock(); + let delayed = state + .events + .iter() + .find(|event| event.event.path.as_str() == "post-sentinel-paused") + .ok_or_else(|| Error::Corrupt("paused callback was not retained".into()))?; + if delayed.event.sequence <= fence.sequence || delayed.provider_event_id != 20 { + return Err(Error::Corrupt( + "post-barrier callback was discarded or folded into the fence".into(), + )); + } + drop(state); + shared.shutdown.store(true, Ordering::Release); + let _ = commands.try_send(DurabilityCommand::Shutdown); + worker + .join() + .map_err(|_| Error::Corrupt("durability worker panicked".into()))?; + Ok(()) + } + run().map_err(|error| error.to_string()) +} + +#[cfg(all(test, target_os = "macos"))] +#[test] +fn controlled_fence_seals_before_a_continuous_later_queue() { + let provider = b"macos-controlled-fence-v1".to_vec(); + let (durability, _) = memory_durability(provider.clone(), Duration::ZERO); + let shared = test_shared(0); + let (commands, receiver) = mpsc::sync_channel(MAX_PENDING_RECORDS); + let worker_shared = Arc::clone(&shared); + let worker = thread::spawn(move || { + run_durability_worker( + receiver, + Box::new(durability), + worker_shared, + MacOsProviderCursor { + version: CAPABILITY_VERSION, + event_id: 0, + device: 1, + history_database_uuid: [1; 16], + device_relative_root: "tmp/workspace".into(), + coverage_roots: vec![CursorCoverageRoot { + device_relative_root: "tmp/workspace".into(), + root_identity: b"root".to_vec(), + }], + system_aliases: Vec::new(), + policy_dependencies: Vec::new(), + external_policy_fences: Vec::new(), + root_identity: b"root".to_vec(), + lineage_identity: vec![2; 24], + provider_identity: provider, + stream_flags: STREAM_FLAGS, + capabilities: native_capabilities(), + }, + ) + }); + commands + .send(DurabilityCommand::Record { + path: LedgerPath::parse("before-controlled-fence").unwrap(), + flags: EvidenceFlags::CONTENT, + provider_event_id: 10, + }) + .unwrap(); + wait_for_test_events(&shared, 1).unwrap(); + + let nonce = vec![7; 24]; + let (response_tx, response_rx) = mpsc::sync_channel(1); + commands + .send(DurabilityCommand::ControlledFence { + minimum_provider_event_id: 10, + nonce: nonce.clone(), + response: response_tx, + }) + .unwrap(); + const LATER_RECORDS: usize = 256; + for index in 0..LATER_RECORDS { + commands + .send(DurabilityCommand::Record { + path: LedgerPath::parse(&format!("later/{index:03}")).unwrap(), + flags: EvidenceFlags::CONTENT, + provider_event_id: 20 + index as u64, + }) + .unwrap(); + } + let (public, sealed, anchor, _) = response_rx + .recv_timeout(FENCE_TIMEOUT) + .expect("controlled fence response") + .expect("controlled fence durability"); + assert_eq!(public.nonce, nonce); + assert_eq!(public.sequence, sealed.last_sequence); + assert_eq!(public.durable_offset, sealed.durable_end_offset); + assert_eq!(anchor.last_sequence, sealed.last_sequence); + assert_eq!(anchor.provider_cursor, sealed.provider_cursor); + + wait_for_test_events(&shared, LATER_RECORDS + 2).unwrap(); + let state = shared.lock(); + let boundary = state + .events + .iter() + .find(|event| event.event.sequence == public.sequence) + .expect("controlled boundary event"); + assert!(boundary.internal_fence); + assert_eq!(boundary.cut, sealed); + let later = state + .events + .iter() + .filter(|event| event.event.path.as_str().starts_with("later/")) + .collect::>(); + assert_eq!(later.len(), LATER_RECORDS); + assert!(later + .iter() + .all(|event| event.event.sequence > public.sequence)); + drop(state); + shared.shutdown.store(true, Ordering::Release); + let _ = commands.try_send(DurabilityCommand::Shutdown); + worker.join().expect("durability worker"); +} + +#[cfg(debug_assertions)] +pub(crate) fn run_history_authority() -> std::result::Result<(), String> { + fn run() -> Result<()> { + let fixture = TestFixture::new()?; + let events = fixture.interval(|root| { + std::fs::write(root.join("device-relative-alias"), b"native")?; + Ok(()) + })?; + if !has_event(&events, "device-relative-alias", EvidenceFlags::CREATE) { + return Err(Error::Corrupt( + "real device-relative callback did not normalize through APFS firmlink aliases" + .into(), + )); + } + let cursor = fixture + .observer + .resume_cursor()? + .ok_or_else(|| Error::Corrupt("history-bound cursor was absent".into()))?; + let actual = actual_history_authority(&fixture.observer.root_path, cursor.device)?; + if cursor.history_database_uuid != actual.database_uuid + || cursor.device_relative_root != actual.device_relative_root + || cursor.device != actual.device + { + return Err(Error::Corrupt( + format!( + "persisted cursor did not bind genuine CoreServices history authority: cursor_device={} actual_device={} cursor_uuid={} actual_uuid={} cursor_root={:?} actual_root={:?}", + cursor.device, + actual.device, + hex::encode(cursor.history_database_uuid), + hex::encode(actual.database_uuid), + cursor.device_relative_root, + actual.device_relative_root, + ), + )); + } + + let reject = |forged: MacOsProviderCursor, + override_authority: Option| + -> Result<()> { + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + let mut options = StartOptions::production(); + options.authority_override = override_authority; + if MacOsFseventsObserver::start_inner( + fixture.temp.path(), + Box::new(durability), + Some(forged), + &[fixture.temp.path().join(".trail/config.toml")], + options, + ) + .is_ok() + { + return Err(Error::Corrupt( + "independently substituted history cursor was accepted".into(), + )); + } + Ok(()) + }; + + let mut forged = cursor.clone(); + forged.device = forged.device.saturating_add(1); + reject(forged, None)?; + let mut forged = cursor.clone(); + forged.event_id = unsafe { fs_events::FSEventsGetCurrentEventId() }.saturating_add(1); + reject(forged, None)?; + let mut forged = cursor.clone(); + forged.history_database_uuid[0] ^= 0xff; + reject(forged, None)?; + let mut forged = cursor.clone(); + forged.device_relative_root.push_str("-replacement"); + reject(forged, None)?; + let mut forged = cursor.clone(); + forged.coverage_roots[0].root_identity.push(0xff); + reject(forged, None)?; + let mut forged = cursor.clone(); + forged.system_aliases.push(SystemAliasBinding { + alias_path: PathBuf::from("/etc"), + canonical_target: PathBuf::from("/private/etc"), + alias_identity: vec![0xff], + target_identity: vec![0xff], + }); + reject(forged, None)?; + let mut forged = cursor.clone(); + forged + .policy_dependencies + .push(fixture.temp.path().join("uncovered-policy")); + reject(forged, None)?; + let mut forged = cursor.clone(); + forged.root_identity.push(0xff); + reject(forged, None)?; + let mut forged = cursor.clone(); + forged.lineage_identity.clear(); + reject(forged, None)?; + let mut forged = cursor.clone(); + forged.provider_identity.push(0xff); + reject(forged, None)?; + let mut forged = cursor.clone(); + forged.stream_flags ^= fs_events::kFSEventStreamCreateFlagNoDefer; + reject(forged, None)?; + let mut forged = cursor.clone(); + forged.capabilities.power_loss_durability = false; + reject(forged, None)?; + + let mut replaced_authority = actual; + replaced_authority.database_uuid[1] ^= 0xff; + reject(cursor, Some(replaced_authority))?; + Ok(()) + } + run().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_startup_cancellation() -> std::result::Result<(), String> { + fn run() -> Result<()> { + let temp = tempfile::tempdir()?; + std::fs::create_dir(temp.path().join(".trail"))?; + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + let cleanup = Arc::new(AtomicBool::new(false)); + let options = StartOptions { + timeout: Duration::from_millis(20), + authority_override: None, + post_start_database_uuid_override: None, + delay_after_native_start: Duration::from_millis(200), + cleanup_observed: Some(Arc::clone(&cleanup)), + }; + let started = Instant::now(); + let result = MacOsFseventsObserver::start_inner( + temp.path(), + Box::new(durability), + None, + &[], + options, + ); + if result.is_ok() || started.elapsed() >= Duration::from_millis(150) { + return Err(Error::Corrupt( + "startup timeout waited for a deliberately late native start".into(), + )); + } + let deadline = Instant::now() + Duration::from_secs(2); + while !cleanup.load(Ordering::Acquire) && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + if !cleanup.load(Ordering::Acquire) { + return Err(Error::Corrupt( + "late native readiness did not stop/invalidate/release its stream".into(), + )); + } + Ok(()) + } + run().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_malformed_callbacks() -> std::result::Result<(), String> { + fn run() -> Result<()> { + fn new_context() -> (tempfile::TempDir, Arc, CallbackContext) { + let temp = tempfile::tempdir().expect("callback fixture tempdir"); + let shared = test_shared(0); + let (records, _receiver) = mpsc::sync_channel(1); + let context = CallbackContext { + root_path: temp.path().to_path_buf(), + device_relative_root: PathBuf::from("tmp/callback"), + coverage_roots: vec![CallbackCoverageRoot { + absolute_root: temp.path().to_path_buf(), + device_relative_root: PathBuf::from("tmp/callback"), + }], + policy_watches: Vec::new(), + records, + shared: Arc::clone(&shared), + }; + (temp, shared, context) + } + + let (_temp, shared, zero_context) = new_context(); + callback( + ptr::null_mut(), + (&zero_context as *const CallbackContext).cast_mut().cast(), + 0, + ptr::null_mut(), + ptr::null(), + ptr::null(), + ); + if shared.lock().revoked.is_some() { + return Err(Error::Corrupt( + "zero-count callback revoked authority".into(), + )); + } + for missing in 0..3 { + let (temp, shared, context) = new_context(); + let path = + std::ffi::CString::new(temp.path().join("event").to_string_lossy().as_bytes()) + .map_err(|_| Error::Corrupt("callback test path contained NUL".into()))?; + let paths = [path.as_ptr()]; + let flags = [fs_events::kFSEventStreamEventFlagItemCreated]; + let ids = [1_u64]; + callback( + ptr::null_mut(), + (&context as *const CallbackContext).cast_mut().cast(), + 1, + if missing == 0 { + ptr::null_mut() + } else { + paths.as_ptr().cast_mut().cast() + }, + if missing == 1 { + ptr::null() + } else { + flags.as_ptr() + }, + if missing == 2 { + ptr::null() + } else { + ids.as_ptr() + }, + ); + if shared.lock().revoked.as_deref() + != Some("fsevents_malformed_nonempty_callback_batch") + { + return Err(Error::Corrupt(format!( + "malformed callback array {missing} did not revoke globally" + ))); + } + } + let before = NULL_CONTEXT_GENERATION.load(Ordering::Acquire); + callback( + ptr::null_mut(), + ptr::null_mut(), + 1, + ptr::null_mut(), + ptr::null(), + ptr::null(), + ); + if NULL_CONTEXT_GENERATION.load(Ordering::Acquire) != before.saturating_add(1) { + return Err(Error::Corrupt( + "structurally impossible null callback context was not observable".into(), + )); + } + Ok(()) + } + run().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_root_revalidation_failures() -> std::result::Result<(), String> { + fn globally_revoked(observer: &MacOsFseventsObserver) -> Result<()> { + let error = observer.root_identity().unwrap_err(); + if error.code() != "CHANGE_LEDGER_RECONCILE_REQUIRED" || observer.ensure_available().is_ok() + { + return Err(Error::Corrupt( + "root revalidation failure left observer reusable".into(), + )); + } + Ok(()) + } + fn run() -> Result<()> { + use std::os::unix::fs::{symlink, PermissionsExt}; + + let fixture = TestFixture::new()?; + fixture.observer.fail_next_root_descriptor_for_test(); + globally_revoked(&fixture.observer)?; + + let fixture = TestFixture::new()?; + let root = fixture.temp.path().to_path_buf(); + let displaced = root.with_extension("absent-root"); + std::fs::rename(&root, &displaced)?; + globally_revoked(&fixture.observer)?; + std::fs::rename(&displaced, &root)?; + + let fixture = TestFixture::new()?; + let root = fixture.temp.path().to_path_buf(); + let displaced = root.with_extension("inaccessible-root"); + std::fs::rename(&root, &displaced)?; + std::fs::create_dir(&root)?; + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o000))?; + globally_revoked(&fixture.observer)?; + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700))?; + std::fs::remove_dir(&root)?; + std::fs::rename(&displaced, &root)?; + + let fixture = TestFixture::new()?; + let root = fixture.temp.path().to_path_buf(); + let displaced = root.with_extension("symlink-root"); + std::fs::rename(&root, &displaced)?; + symlink(&displaced, &root)?; + globally_revoked(&fixture.observer)?; + std::fs::remove_file(&root)?; + std::fs::rename(&displaced, &root)?; + Ok(()) + } + run().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_null_context_generation() -> std::result::Result<(), String> { + fn run() -> Result<()> { + let ensure_fixture = TestFixture::new()?; + let begin_fixture = TestFixture::new()?; + let end_fixture = TestFixture::new()?; + let end_start = end_fixture + .observer + .begin_observation(&end_fixture.expected)?; + let drain_fixture = TestFixture::new()?; + let drain_start = drain_fixture + .observer + .begin_observation(&drain_fixture.expected)?; + let drain_end = drain_fixture + .observer + .end_fence(&drain_fixture.expected, &drain_start)?; + let baseline = NULL_CONTEXT_GENERATION.load(Ordering::Acquire); + callback( + ptr::null_mut(), + ptr::null_mut(), + 0, + ptr::null_mut(), + ptr::null(), + ptr::null(), + ); + if NULL_CONTEXT_GENERATION.load(Ordering::Acquire) != baseline + || ensure_fixture.observer.ensure_available().is_err() + || begin_fixture.observer.ensure_available().is_err() + || end_fixture.observer.ensure_available().is_err() + || drain_fixture.observer.ensure_available().is_err() + { + return Err(Error::Corrupt( + "zero-count null-context callback changed live authority".into(), + )); + } + callback( + ptr::null_mut(), + ptr::null_mut(), + 1, + ptr::null_mut(), + ptr::null(), + ptr::null(), + ); + let unavailable = ensure_fixture.observer.ensure_available().unwrap_err(); + if unavailable.code() != "CHANGE_LEDGER_RECONCILE_REQUIRED" + || !unavailable + .to_string() + .contains("fsevents_null_callback_context_generation_changed") + { + return Err(Error::Corrupt( + "nonempty null-context callback did not revoke the live observer".into(), + )); + } + if begin_fixture + .observer + .begin_observation(&begin_fixture.expected) + .is_ok() + || end_fixture + .observer + .end_fence(&end_fixture.expected, &end_start) + .is_ok() + { + return Err(Error::Corrupt( + "observer issued or authenticated a fence after null-context generation changed" + .into(), + )); + } + let mut sink = |_event: ObserverEvent| Ok(()); + let drain_root_identity = drain_fixture.observer.root_identity.clone(); + if drain_fixture + .observer + .drain_through( + &drain_fixture.expected, + &drain_root_identity, + &drain_start, + &drain_end, + &mut sink, + ) + .is_ok() + { + return Err(Error::Corrupt( + "observer qualified a drain after null-context generation changed".into(), + )); + } + Ok(()) + } + run().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_uuid_revalidation() -> std::result::Result<(), String> { + fn run() -> Result<()> { + let resume_fixture = TestFixture::new()?; + resume_fixture.interval(|root| { + std::fs::write(root.join("post-start-uuid-resume"), b"history")?; + Ok(()) + })?; + let cursor = resume_fixture + .observer + .resume_cursor()? + .ok_or_else(|| Error::Corrupt("post-start UUID test cursor was absent".into()))?; + let canonical_root = resume_fixture.temp.path().canonicalize()?; + let expected_authority = actual_history_authority(&canonical_root, cursor.device)?; + let mut replacement_uuid = expected_authority.database_uuid; + replacement_uuid[0] ^= 0xff; + let TestFixture { temp, observer, .. } = resume_fixture; + drop(observer); + let cleanup = Arc::new(AtomicBool::new(false)); + let provider = b"macos-fsevents-file-events-v1".to_vec(); + let (durability, _) = memory_durability(provider, Duration::ZERO); + let options = StartOptions { + timeout: FENCE_TIMEOUT, + authority_override: None, + post_start_database_uuid_override: Some(replacement_uuid), + delay_after_native_start: Duration::ZERO, + cleanup_observed: Some(Arc::clone(&cleanup)), + }; + let error = match MacOsFseventsObserver::start_inner( + temp.path(), + Box::new(durability), + Some(cursor), + &[temp.path().join(".trail/config.toml")], + options, + ) { + Ok(_) => { + return Err(Error::Corrupt( + "post-start UUID replacement was accepted".into(), + )) + } + Err(error) => error, + }; + if !error + .to_string() + .contains("fsevents_post_start_history_database_uuid_mismatch") + || !cleanup.load(Ordering::Acquire) + { + return Err(Error::Corrupt( + "post-start UUID replacement did not reject and clean the native stream".into(), + )); + } + + let fixture = TestFixture::new()?; + let mut replacement = fixture.observer.history_authority.clone(); + replacement.database_uuid[0] ^= 0xff; + fixture + .observer + .set_next_history_authority_for_test(replacement); + let error = fixture + .observer + .begin_observation(&fixture.expected) + .unwrap_err(); + if !error + .to_string() + .contains("fsevents_history_authority_revalidation_mismatch") + || fixture.observer.ensure_available().is_ok() + { + return Err(Error::Corrupt( + "proof-boundary UUID replacement did not revoke clean authority".into(), + )); + } + Ok(()) + } + run().map_err(|error| error.to_string()) +} + +#[cfg(test)] +mod policy_event_tests { + use super::*; + use std::fs; + + fn test_cursor(provider_identity: Vec) -> MacOsProviderCursor { + MacOsProviderCursor { + version: CAPABILITY_VERSION, + event_id: 0, + device: 1, + history_database_uuid: [1; 16], + device_relative_root: "tmp/workspace".into(), + coverage_roots: vec![CursorCoverageRoot { + device_relative_root: "tmp/workspace".into(), + root_identity: b"root".to_vec(), + }], + system_aliases: Vec::new(), + policy_dependencies: Vec::new(), + external_policy_fences: Vec::new(), + root_identity: b"root".to_vec(), + lineage_identity: vec![2; 24], + provider_identity, + stream_flags: STREAM_FLAGS, + capabilities: native_capabilities(), + } + } + + fn policy_watch(path: &Path) -> PolicyWatch { + PolicyWatch { + dependency: path.to_path_buf(), + observed_path: path.to_path_buf(), + identity: capture_policy_dependency_fence_identity(path).unwrap(), + } + } + + fn spawn_test_worker(shared: &Arc) -> (SyncSender, JoinHandle<()>) { + let provider = b"macos-policy-event-revalidation-v1".to_vec(); + let (durability, _) = memory_durability(provider.clone(), Duration::ZERO); + let (commands, receiver) = mpsc::sync_channel(MAX_PENDING_RECORDS); + let worker_shared = Arc::clone(shared); + let worker = thread::spawn(move || { + run_durability_worker( + receiver, + Box::new(durability), + worker_shared, + test_cursor(provider), + ) + }); + (commands, worker) + } + + fn wait_for_revocation(shared: &Shared) -> String { + let deadline = Instant::now() + FENCE_TIMEOUT; + let mut state = shared.lock(); + while state.revoked.is_none() && Instant::now() < deadline { + let waited = shared + .changed + .wait_timeout(state, Duration::from_millis(10)) + .unwrap_or_else(|poison| poison.into_inner()); + state = waited.0; + } + state + .revoked + .clone() + .expect("policy identity mismatch did not revoke the observer") + } + + #[test] + fn queued_policy_event_revalidates_exact_identity_before_durable_revocation() { + let temp = tempfile::tempdir().unwrap(); + let config = temp.path().canonicalize().unwrap().join("config.toml"); + fs::write(&config, b"[recording]\nmode = 'content'\n").unwrap(); + let watch = policy_watch(&config); + let shared = test_shared(0); + let (commands, worker) = spawn_test_worker(&shared); + + commands + .send(DurabilityCommand::PolicyEvent { + watch: watch.clone(), + provider_event_id: 10, + }) + .unwrap(); + let (response_tx, response_rx) = mpsc::sync_channel(1); + commands + .send(DurabilityCommand::Fence { + minimum_provider_event_id: 10, + nonce: vec![3; 24], + response: response_tx, + }) + .unwrap(); + let (_, cut, provider_event_id) = response_rx + .recv_timeout(FENCE_TIMEOUT) + .expect("unchanged policy event blocked the fence") + .expect("unchanged policy event revoked the observer"); + assert_eq!(provider_event_id, 10); + assert_eq!( + MacOsProviderCursor::decode(&cut.provider_cursor) + .unwrap() + .event_id, + 10 + ); + assert!(shared.lock().revoked.is_none()); + + fs::write(&config, b"[recording]\nmode = 'metadata'\n").unwrap(); + commands + .send(DurabilityCommand::PolicyEvent { + watch, + provider_event_id: 20, + }) + .unwrap(); + let reason = wait_for_revocation(&shared); + assert!(reason.contains("fsevents_policy_dependency_invalidated")); + worker.join().unwrap(); + + fs::remove_file(&config).unwrap(); + fs::write(&config, b"[recording]\nmode = 'content'\n").unwrap(); + let watch = policy_watch(&config); + let shared = test_shared(0); + let (commands, worker) = spawn_test_worker(&shared); + fs::remove_file(&config).unwrap(); + fs::create_dir(&config).unwrap(); + commands + .send(DurabilityCommand::PolicyEvent { + watch, + provider_event_id: 30, + }) + .unwrap(); + let reason = wait_for_revocation(&shared); + assert!(reason.contains("fsevents_policy_dependency_invalidated")); + worker.join().unwrap(); + } +} diff --git a/trail/src/db/change_ledger/observer/mod.rs b/trail/src/db/change_ledger/observer/mod.rs new file mode 100644 index 0000000..84d0215 --- /dev/null +++ b/trail/src/db/change_ledger/observer/mod.rs @@ -0,0 +1,124 @@ +//! Native observer qualification contract. +//! +//! This is the single observer authority consumed by reconciliation. Platform +//! adapters may collect advisory evidence before qualification, but they can +//! only prove continuity through this trait and a durable end fence. + +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +use super::reconcile::{ObserverEvent, ObserverQualification}; +use super::{ExpectedScope, PolicyDependencyFenceIdentity, ProviderCapabilities}; +use crate::error::Result; + +#[cfg(target_os = "linux")] +pub(crate) mod linux; +#[cfg(target_os = "macos")] +pub(crate) mod macos; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ObserverFence { + pub(crate) sequence: u64, + pub(crate) durable_offset: u64, + pub(crate) nonce: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ObserverLease { + pub(crate) owner_token: String, + pub(crate) root_identity: Vec, + pub(crate) provider_identity: Vec, + /// Dependencies continuously covered by the native observer. + pub(crate) policy_dependencies: Vec, + /// Dependencies on other filesystem-history authorities, covered by an + /// authenticated direct pre/post fingerprint fence instead. + pub(crate) direct_policy_fences: Vec<(PathBuf, PolicyDependencyFenceIdentity)>, + pub(crate) capabilities: ProviderCapabilities, +} + +/// Generic `notify` delivery is useful as a reconciliation hint only. It has +/// no durable native cursor or linearizable fence and therefore can never +/// authorize a clean result. +pub(crate) struct AdvisoryObserver; + +impl AdvisoryObserver { + pub(crate) fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + durable_cursor: false, + linearizable_fence: false, + rename_pairing: false, + overflow_scope: false, + filesystem_supported: false, + clean_proof_allowed: false, + power_loss_durability: false, + } + } +} + +pub(crate) trait QualifiedObserver: Send + Sync { + fn begin_observation(&self, expected: &ExpectedScope) -> Result; + + fn end_fence(&self, expected: &ExpectedScope, start: &ObserverFence) -> Result; + + fn drain_through( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + ) -> Result; + + /// Consume `(start, end]` while retaining `end` as the authenticated + /// anchor for the next rotation. Native adapters override this method; + /// the default preserves compatibility for reconciliation-only test + /// observers which do not provide continuous command authority. + fn drain_through_retaining_end( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + ) -> Result { + self.drain_through(expected, root_handle_identity, start, end, sink) + } + + fn rebind_retained_tail( + &self, + _previous: &ExpectedScope, + _next: &ExpectedScope, + _anchor: &ObserverFence, + ) -> Result<()> { + Err(crate::Error::DaemonUnavailable( + "observer does not support retained-tail baseline rebinding".into(), + )) + } +} + +pub(crate) enum SelectedObserver { + #[cfg(target_os = "linux")] + Linux, + #[cfg(target_os = "macos")] + MacOs, + Advisory, +} + +/// Platform selection and production authority share the same compile-time +/// Linux/macOS boundary. Runtime capability, filesystem, owner, cursor, and +/// fence qualification remain mandatory before any scope can become trusted. +pub(crate) fn select_observer() -> SelectedObserver { + #[cfg(target_os = "linux")] + { + SelectedObserver::Linux + } + #[cfg(not(target_os = "linux"))] + #[cfg(target_os = "macos")] + { + SelectedObserver::MacOs + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + SelectedObserver::Advisory + } +} diff --git a/trail/src/db/change_ledger/policy.rs b/trail/src/db/change_ledger/policy.rs new file mode 100644 index 0000000..84c66ee --- /dev/null +++ b/trail/src/db/change_ledger/policy.rs @@ -0,0 +1,3846 @@ +#[cfg(test)] +mod tests { + use super::*; + use crate::db::change_ledger::{ + BaselineIdentity, ChangedPathLedger, ExpectedScope, FilesystemIdentity, PolicyIdentity, + ProviderCapabilities, ProviderIdentity, ScopeId, ScopeIdentity, ScopeKind, + }; + use crate::ids::{ChangeId, ObjectId}; + use crate::{InitImportMode, Trail}; + use std::ffi::OsString; + use std::fs; + use std::path::{Path, PathBuf}; + use std::process::Command; + + struct Fixture { + _temp: tempfile::TempDir, + db: Trail, + expected: ExpectedScope, + git_env: Vec<(OsString, OsString)>, + } + + impl Fixture { + fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + run_git(root, &["init", "--quiet"]); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write(root.join("src/.gitignore"), "generated\n").unwrap(); + fs::write(root.join("src/.trailignore"), "private\n").unwrap(); + Trail::init(root, "main", InitImportMode::Empty, false).unwrap(); + let db = Trail::open(root).unwrap(); + let scope = ScopeIdentity { + scope_id: ScopeId([7; 32]), + kind: ScopeKind::Workspace, + owner_id: "policy-test".to_string(), + }; + let baseline = BaselineIdentity { + ref_name: "main".to_string(), + ref_generation: 1, + change_id: ChangeId("change".to_string()), + root_id: ObjectId("root".to_string()), + }; + let policy = PolicyIdentity { + fingerprint: [0; 32], + generation: 1, + }; + let filesystem = FilesystemIdentity(vec![1]); + let provider = ProviderIdentity { + identity: vec![2], + capabilities: ProviderCapabilities { + durable_cursor: false, + linearizable_fence: false, + rename_pairing: false, + overflow_scope: false, + filesystem_supported: false, + clean_proof_allowed: false, + power_loss_durability: false, + }, + }; + ChangedPathLedger::new(&db.conn) + .begin_scope(&scope, &baseline, &policy, &filesystem, &provider) + .unwrap(); + let expected = ExpectedScope { + scope_id: scope.scope_id, + epoch: 1, + ref_name: baseline.ref_name, + ref_generation: baseline.ref_generation, + baseline_root: baseline.root_id, + policy_fingerprint: policy.fingerprint, + policy_generation: policy.generation, + filesystem_identity: filesystem.0, + provider_identity: provider.identity, + }; + Self { + _temp: temp, + db, + expected, + git_env: vec![("GIT_CONFIG_NOSYSTEM".into(), "1".into())], + } + } + + fn root(&self) -> &Path { + &self.db.workspace_root + } + + fn context(&self) -> PolicyCompileContext<'_> { + PolicyCompileContext { + workspace_root: &self.db.workspace_root, + db_dir: &self.db.db_dir, + recording: &self.db.config.recording, + case_sensitive: true, + git_environment: &self.git_env, + } + } + + fn compile(&self, metrics: &mut PolicyDependencyMetrics) -> CompiledPolicy { + compile_policy(&self.db.conn, &self.expected, &self.context(), metrics).unwrap() + } + } + + #[test] + fn raw_nested_ignore_event_stales_scope_before_ignore_filtering() { + let fixture = Fixture::new(); + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + + assert!(raw_event_invalidates_policy( + &policy, + &fixture.root().join("src/.gitignore") + )); + assert!(raw_event_invalidates_policy( + &policy, + &fixture.root().join("src/.trailignore") + )); + assert!(raw_path_may_invalidate_policy(Path::new( + ".trail/config.toml" + ))); + let nested_rules = policy + .snapshot + .rule_sources + .iter() + .find(|source| source.path == fixture.root().join("src/.gitignore")) + .expect("compiled snapshot pins semantic rule bytes"); + assert_eq!(nested_rules.bytes, b"generated\n"); + assert!(policy + .dependencies + .iter() + .filter(|dependency| dependency_is_file(dependency)) + .all(|dependency| !dependency.observable && dependency.last_source_sequence == 0)); + } + + #[test] + fn unchanged_manifest_reuses_policy_without_tree_discovery() { + let fixture = Fixture::new(); + let mut metrics = PolicyDependencyMetrics::default(); + + let first = fixture.compile(&mut metrics); + let second = fixture.compile(&mut metrics); + + assert_eq!(first.fingerprint, second.fingerprint); + assert_eq!(metrics.policy_dependency_full_discovery, 1); + assert!(second.reused_manifest); + assert!(second.stale_baseline); + assert_eq!(second.adapter_equivalence, AdapterEquivalence::Conservative); + let persisted: i64 = fixture + .db + .conn + .query_row( + "SELECT COUNT(*) FROM changed_path_policy_dependencies WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(persisted as usize, second.dependencies.len()); + } + + #[test] + fn no_observer_proof_never_returns_equivalent_even_when_direct_checks_reuse_manifest() { + let fixture = Fixture::new(); + let mut metrics = PolicyDependencyMetrics::default(); + let _first = fixture.compile(&mut metrics); + fs::create_dir_all(fixture.root().join("new/nested")).unwrap(); + fs::write(fixture.root().join("new/nested/.gitignore"), "late\n").unwrap(); + + let reused = fixture.compile(&mut metrics); + + assert!( + reused.reused_manifest, + "bounded direct reuse remains permitted" + ); + assert!(reused.stale_baseline); + assert_eq!(reused.adapter_equivalence, AdapterEquivalence::Conservative); + assert_eq!(metrics.policy_dependency_full_discovery, 1); + } + + #[test] + fn fabricated_observer_cut_cannot_promote_synthetic_only_manifest() { + let fixture = Fixture::new(); + let context = fixture.context(); + let observer_cut = QualifiedPolicyObserverCut { + scope_id: fixture.expected.scope_id, + provider_identity: fixture.expected.provider_identity.clone(), + discovery_started_sequence: 1, + through_sequence: 1, + covered_roots: vec![fixture.root().to_path_buf()], + case_sensitive: true, + }; + assert!(observer_cut.validate_for( + &fixture.expected, + context.workspace_root, + context.case_sensitive, + )); + let manifest = PolicyManifest { + dependencies: synthetic_dependencies(fixture.expected.policy_generation, &context) + .unwrap(), + generation: fixture.expected.policy_generation, + rule_sources: Vec::new(), + }; + let policy = finish_compiled_policy(&context, manifest.clone(), false).unwrap(); + + assert_eq!(policy.adapter_equivalence, AdapterEquivalence::Conservative); + assert!(policy.stale_baseline); + + persist_policy_manifest_rows(&fixture.db.conn, &fixture.expected, &manifest).unwrap(); + let compiled = fixture.compile(&mut PolicyDependencyMetrics::default()); + assert!(compiled.reused_manifest); + assert_eq!( + compiled.adapter_equivalence, + AdapterEquivalence::Conservative + ); + assert!(compiled.stale_baseline); + let (trust_state, continuity_generation): (String, i64) = fixture + .db + .conn + .query_row( + "SELECT trust_state,continuity_generation + FROM changed_path_scopes WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(trust_state, "stale_baseline"); + assert!(continuity_generation > 1); + } + + #[test] + fn production_compiled_policy_cannot_authorize_reconciliation() { + let fixture = Fixture::new(); + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + + assert!(!policy.authorizes_reconciliation(&fixture.expected)); + assert_eq!(policy.adapter_equivalence, AdapterEquivalence::Conservative); + assert!(policy.stale_baseline); + } + + #[test] + fn exact_invalidation_index_matches_arbitrary_dependency_with_case_policy() { + let fixture = Fixture::new(); + let mut policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + let arbitrary = fixture.root().join("config/Arbitrary.Rules"); + fs::create_dir_all(arbitrary.parent().unwrap()).unwrap(); + fs::write(&arbitrary, "*.generated\n").unwrap(); + policy.dependencies.push( + file_dependency( + &arbitrary, + PolicyDependencyKind::GitExcludesFile, + policy.manifest().generation, + &fixture.context(), + ) + .unwrap(), + ); + policy.invalidation_index = + PolicyInvalidationIndex::from_dependencies(fixture.root(), false, &policy.dependencies) + .unwrap(); + + assert!(raw_event_invalidates_policy( + &policy, + Path::new("CONFIG/arbitrary.rules") + )); + assert!(!raw_event_invalidates_policy( + &policy, + Path::new("config/unrelated.rules") + )); + } + + #[test] + fn default_missing_git_candidates_and_missing_include_target_are_authoritative() { + let mut fixture = Fixture::new(); + let home = fixture.root().parent().unwrap().join("home"); + let xdg = fixture.root().parent().unwrap().join("xdg"); + fs::create_dir_all(home.join(".config/git")).unwrap(); + fs::create_dir_all(xdg.join("git")).unwrap(); + let global = home.join(".gitconfig"); + let missing_include = home.join("not-created-yet.gitconfig"); + fs::write( + &global, + format!("[include]\n\tpath = {}\n", missing_include.display()), + ) + .unwrap(); + fixture.git_env = vec![ + ("HOME".into(), home.as_os_str().to_owned()), + ("XDG_CONFIG_HOME".into(), xdg.as_os_str().to_owned()), + ("GIT_CONFIG_NOSYSTEM".into(), "0".into()), + ( + "GIT_CONFIG_SYSTEM".into(), + fixture + .root() + .parent() + .unwrap() + .join("system.gitconfig") + .into_os_string(), + ), + ]; + + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + for candidate in [ + home.join(".gitconfig"), + xdg.join("git/config"), + fixture.root().parent().unwrap().join("system.gitconfig"), + xdg.join("git/ignore"), + missing_include.clone(), + ] { + assert!( + policy + .dependencies + .iter() + .any(|dependency| dependency.identity + == dependency_path_identity_with_case(&candidate, true)), + "missing candidate {}", + candidate.display() + ); + } + fs::write(&missing_include, "[core]\n\texcludesFile = ignored\n").unwrap(); + assert_eq!( + validate_policy_manifest(&fixture.context(), &policy.manifest()).unwrap(), + PolicyManifestValidation::Changed + ); + } + + #[test] + fn config_selector_environment_change_invalidates_direct_reuse() { + let mut fixture = Fixture::new(); + let first = fixture.compile(&mut PolicyDependencyMetrics::default()); + fixture.git_env.push(( + "XDG_CONFIG_HOME".into(), + fixture.root().join("different-xdg").into_os_string(), + )); + + assert_eq!( + validate_policy_manifest(&fixture.context(), &first.manifest()).unwrap(), + PolicyManifestValidation::Changed + ); + } + + #[test] + fn command_scope_missing_includes_are_authoritative_dependencies() { + let mut fixture = Fixture::new(); + let counted = fixture.root().join("counted/missing.gitconfig"); + let parameter = fixture.root().join("parameter/missing.gitconfig"); + fixture.git_env.extend([ + ("GIT_CONFIG_COUNT".into(), "2".into()), + ("GIT_CONFIG_KEY_0".into(), "include.path".into()), + ("GIT_CONFIG_VALUE_0".into(), counted.as_os_str().to_owned()), + ( + "GIT_CONFIG_KEY_1".into(), + "includeIf.gitdir:/**.path".into(), + ), + ( + "GIT_CONFIG_VALUE_1".into(), + fixture + .root() + .join("counted/conditional.gitconfig") + .into_os_string(), + ), + ( + "GIT_CONFIG_PARAMETERS".into(), + OsString::from(format!("'include.path'='{}'", parameter.display())), + ), + ]); + + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + for target in [ + counted, + fixture.root().join("counted/conditional.gitconfig"), + parameter, + ] { + let dependency = policy + .dependencies + .iter() + .find(|dependency| { + dependency.identity == dependency_path_identity_with_case(&target, true) + }) + .unwrap_or_else(|| panic!("missing injected include {}", target.display())); + assert!(!dependency.observable); + assert!(policy.invalidation_index.matches(fixture.root(), &target)); + fs::create_dir_all(target.parent().unwrap()).unwrap(); + fs::write(&target, "[core]\n\tignoreCase = false\n").unwrap(); + assert_eq!( + validate_policy_manifest(&fixture.context(), &policy.manifest()).unwrap(), + PolicyManifestValidation::Changed + ); + fs::remove_file(&target).unwrap(); + } + } + + #[test] + fn empty_and_ascii_padded_git_config_count_keep_indexed_includes() { + let mut fixture = Fixture::new(); + let target = fixture.root().join("indexed/missing.gitconfig"); + fixture.git_env.extend([ + ("GIT_CONFIG_COUNT".into(), " 1\t".into()), + ("GIT_CONFIG_KEY_0".into(), "include.path".into()), + ("GIT_CONFIG_VALUE_0".into(), target.as_os_str().to_owned()), + ]); + + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + assert!(policy.dependencies.iter().any(|dependency| { + dependency.identity == dependency_path_identity_with_case(&target, true) + })); + + fixture.git_env.retain(|(key, _)| { + !matches!( + key.to_str(), + Some("GIT_CONFIG_KEY_0" | "GIT_CONFIG_VALUE_0") + ) + }); + fixture.git_env.iter_mut().for_each(|(key, value)| { + if key == OsStr::new("GIT_CONFIG_COUNT") { + *value = OsString::new(); + } + }); + assert!(discover_git_dependencies(1, &fixture.context()).is_ok()); + } + + #[test] + fn git_selector_paths_follow_git_cwd_and_empty_semantics() { + let mut fixture = Fixture::new(); + let home = fixture.root().join("home"); + fixture.git_env = vec![ + ("HOME".into(), home.as_os_str().to_owned()), + ("XDG_CONFIG_HOME".into(), OsString::new()), + ("GIT_CONFIG_GLOBAL".into(), OsString::new()), + ("GIT_CONFIG_SYSTEM".into(), "config/system.gitconfig".into()), + ]; + + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + let identities = policy + .dependencies + .iter() + .map(|dependency| dependency.identity.as_str()) + .collect::>(); + + assert!(identities.contains( + dependency_path_identity_with_case(&home.join(".config/git/ignore"), true).as_str() + )); + assert!(identities.contains( + dependency_path_identity_with_case( + &fixture.root().join("config/system.gitconfig"), + true, + ) + .as_str() + )); + assert!(!identities + .contains(dependency_path_identity_with_case(&home.join(".gitconfig"), true).as_str())); + assert!(!identities.contains( + dependency_path_identity_with_case(&home.join(".config/git/config"), true).as_str() + )); + } + + #[test] + fn empty_home_uses_root_based_global_and_xdg_candidates() { + let mut fixture = Fixture::new(); + fixture.git_env.extend([ + ("HOME".into(), OsString::new()), + ("XDG_CONFIG_HOME".into(), OsString::new()), + ]); + + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + for target in [ + PathBuf::from("/.gitconfig"), + PathBuf::from("/.config/git/config"), + PathBuf::from("/.config/git/ignore"), + ] { + assert!(policy.dependencies.iter().any(|dependency| { + dependency.identity == dependency_path_identity_with_case(&target, true) + })); + } + } + + #[test] + fn relative_xdg_and_global_paths_resolve_from_git_cwd() { + let mut fixture = Fixture::new(); + fixture.git_env = vec![ + ("XDG_CONFIG_HOME".into(), "xdg-relative".into()), + ("GIT_CONFIG_GLOBAL".into(), "config/global.gitconfig".into()), + ("GIT_CONFIG_NOSYSTEM".into(), "1".into()), + ]; + + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + for target in [ + fixture.root().join("xdg-relative/git/ignore"), + fixture.root().join("config/global.gitconfig"), + ] { + assert!(policy.dependencies.iter().any(|dependency| { + dependency.identity == dependency_path_identity_with_case(&target, true) + })); + } + } + + #[test] + fn git_subprocess_environment_drops_every_ambient_repository_selector() { + let mut fixture = Fixture::new(); + fixture.git_env.extend([ + ("HOME".into(), fixture.root().join("home").into_os_string()), + ( + "GIT_DIR".into(), + fixture.root().join(".git").into_os_string(), + ), + ]); + let ambient = vec![ + ("PATH".into(), "/usr/bin:/bin".into()), + ("GIT_CONFIG".into(), "/tmp/hostile-config".into()), + ("GIT_COMMON_DIR".into(), "/tmp/hostile-common".into()), + ("GIT_WORK_TREE".into(), "/tmp/hostile-worktree".into()), + ("GIT_OBJECT_DIRECTORY".into(), "/tmp/hostile-objects".into()), + ("HOME".into(), "/tmp/hostile-home".into()), + ("XDG_CONFIG_HOME".into(), "/tmp/hostile-xdg".into()), + ("UNRELATED_AMBIENT".into(), "not-needed".into()), + ]; + + let environment = git_command_environment(&fixture.context(), ambient); + let keys = environment + .iter() + .map(|(key, _)| key.as_os_str()) + .collect::>(); + + assert_eq!( + keys, + BTreeSet::from([ + OsStr::new("PATH"), + OsStr::new("HOME"), + OsStr::new("GIT_DIR"), + OsStr::new("GIT_CONFIG_NOSYSTEM"), + ]) + ); + let expected_home = fixture.root().join("home").into_os_string(); + assert_eq!( + environment + .iter() + .find(|(key, _)| key == OsStr::new("HOME")) + .map(|(_, value)| value), + Some(&expected_home) + ); + } + + #[test] + fn canonical_fingerprint_is_framed_and_rejects_duplicate_identities() { + let fixture = Fixture::new(); + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + let mut duplicate = policy.manifest(); + duplicate + .dependencies + .push(duplicate.dependencies[0].clone()); + assert!(validate_policy_manifest(&fixture.context(), &duplicate).is_err()); + + let mut left = policy.dependencies[0].clone(); + let mut right = policy.dependencies[0].clone(); + left.identity = "ab".into(); + right.identity = "a".into(); + let left_fingerprint = policy_fingerprint(&[left.clone(), { + let mut x = right.clone(); + x.identity = "c".into(); + x + }]) + .unwrap(); + let right_fingerprint = policy_fingerprint(&[ + { + left.identity = "a".into(); + left + }, + { + right.identity = "bc".into(); + right + }, + ]) + .unwrap(); + assert_ne!(left_fingerprint, right_fingerprint); + } + + #[test] + fn stale_expected_scope_cannot_delete_concurrent_manifest_replacement() { + let fixture = Fixture::new(); + let first = fixture.compile(&mut PolicyDependencyMetrics::default()); + fixture + .db + .conn + .execute( + "UPDATE changed_path_scopes SET epoch=epoch+1 WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + ) + .unwrap(); + let mut replacement_expected = fixture.expected.clone(); + replacement_expected.epoch += 1; + let replacement = + discover_policy_manifest(replacement_expected.policy_generation, &fixture.context()) + .unwrap(); + persist_policy_manifest_and_stale(&fixture.db.conn, &replacement_expected, &replacement) + .unwrap(); + + let stale_result = persist_policy_manifest_and_stale( + &fixture.db.conn, + &fixture.expected, + &first.manifest(), + ); + assert!(stale_result.is_err()); + let count: i64 = fixture + .db + .conn + .query_row( + "SELECT COUNT(*) FROM changed_path_policy_dependencies WHERE scope_id=?1", + [replacement_expected.scope_id.to_text()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count as usize, replacement.dependencies.len()); + } + + #[test] + fn direct_nested_dependency_change_is_detected_and_marks_scope_stale() { + let fixture = Fixture::new(); + let mut metrics = PolicyDependencyMetrics::default(); + let first = fixture.compile(&mut metrics); + fs::write(fixture.root().join("src/.gitignore"), "different\n").unwrap(); + + assert_eq!( + validate_policy_manifest(&fixture.context(), &first.manifest()).unwrap(), + PolicyManifestValidation::Changed + ); + let second = fixture.compile(&mut metrics); + + assert_ne!(first.fingerprint, second.fingerprint); + assert!(second.stale_baseline); + assert_eq!(metrics.policy_dependency_full_discovery, 2); + let state: String = fixture + .db + .conn + .query_row( + "SELECT trust_state FROM changed_path_scopes WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(state, "stale_baseline"); + } + + #[test] + fn git_config_includes_core_excludes_and_global_origins_are_dependencies() { + let mut fixture = Fixture::new(); + let global = fixture.root().parent().unwrap().join("global.gitconfig"); + let included = fixture.root().parent().unwrap().join("included.gitconfig"); + let excludes = fixture.root().parent().unwrap().join("global-excludes"); + fs::write(&excludes, "*.cache\n").unwrap(); + fs::write( + &included, + format!("[core]\n\texcludesFile = {}\n", excludes.display()), + ) + .unwrap(); + fs::write( + &global, + format!("[include]\n\tpath = {}\n", included.display()), + ) + .unwrap(); + fixture + .git_env + .push(("GIT_CONFIG_GLOBAL".into(), global.as_os_str().to_owned())); + + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + + for path in [&global, &included, &excludes] { + let identity = dependency_path_identity(path); + let dependency = policy + .dependencies + .iter() + .find(|dependency| dependency.identity == identity) + .unwrap_or_else(|| panic!("missing dependency {}", path.display())); + assert!(!dependency.observable); + } + assert!(policy.stale_baseline); + assert_eq!(policy.adapter_equivalence, AdapterEquivalence::Conservative); + } + + #[test] + fn git_decodes_include_values_and_resolves_them_from_the_origin_file() { + let mut fixture = Fixture::new(); + let config_root = fixture.root().parent().unwrap().join("git-config-grammar"); + fs::create_dir_all(&config_root).unwrap(); + let global = config_root.join("global.gitconfig"); + fs::write( + &global, + concat!( + "[include]\n", + "\tpath = \"quoted dir/missing include.conf\" # trailing comment\n", + "[include]\n", + "\tpath = continued-\\\n", + "target.conf\n", + "[includeIf \"gitdir:/**\"]\n", + "\tpath = \"tab\\tmissing.conf\"\n", + ), + ) + .unwrap(); + fixture + .git_env + .push(("GIT_CONFIG_GLOBAL".into(), global.into_os_string())); + + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + + for target in [ + config_root.join("quoted dir/missing include.conf"), + config_root.join("continued-target.conf"), + config_root.join("tab\tmissing.conf"), + ] { + assert!( + policy.dependencies.iter().any(|dependency| { + dependency.identity == dependency_path_identity_with_case(&target, true) + }), + "Git-decoded missing include was not persisted: {}", + target.display() + ); + } + } + + #[test] + fn git_expands_policy_paths_before_origin_or_cwd_resolution() { + let mut fixture = Fixture::new(); + let home = fixture.root().parent().unwrap().join("path-expansion-home"); + let config_root = fixture + .root() + .parent() + .unwrap() + .join("path-expansion-config"); + fs::create_dir_all(&home).unwrap(); + fs::create_dir_all(&config_root).unwrap(); + let global = config_root.join("global.gitconfig"); + fs::write( + &global, + concat!( + "[include]\n", + "\tpath = \"%(prefix)/share/trail approval/include.conf\"\n", + "[includeIf \"gitdir:/**\"]\n", + "\tpath = \"~/quoted dir/conditional.conf\"\n", + "[core]\n", + "\texcludesFile = \"~/excluded-\\\n", + "rules\"\n", + ), + ) + .unwrap(); + fixture.git_env.extend([ + ("HOME".into(), home.into_os_string()), + ("GIT_CONFIG_GLOBAL".into(), global.as_os_str().to_owned()), + ]); + + let git_path = |key: &str| { + let output = Command::new("git") + .args(["config", "--file"]) + .arg(&global) + .args(["--path", "--get-all", key]) + .current_dir(fixture.root()) + .env_clear() + .envs(git_command_environment( + &fixture.context(), + std::env::vars_os(), + )) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + PathBuf::from(os_string_from_bytes( + output.stdout.strip_suffix(b"\n").unwrap_or(&output.stdout), + )) + }; + let expected = [ + git_path("include.path"), + git_path("includeIf.gitdir:/**.path"), + git_path("core.excludesFile"), + ]; + + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + + for target in expected { + let target = if target.is_absolute() { + target + } else { + config_root.join(target) + }; + assert!( + policy.dependencies.iter().any(|dependency| { + dependency.identity == dependency_path_identity_with_case(&target, true) + }), + "missing Git-expanded path {}", + target.display() + ); + } + } + + #[test] + fn legacy_git_config_file_and_its_policy_keys_are_dependencies() { + let mut fixture = Fixture::new(); + let config_dir = fixture.root().join("config"); + fs::create_dir_all(&config_dir).unwrap(); + let legacy = config_dir.join("legacy.gitconfig"); + let included = config_dir.join("included.gitconfig"); + fs::write( + &legacy, + "[include]\n\tpath = included.gitconfig\n[core]\n\texcludesFile = legacy.ignore\n", + ) + .unwrap(); + fs::write(&included, "[core]\n\texcludesFile = included.ignore\n").unwrap(); + fixture + .git_env + .push(("GIT_CONFIG".into(), "config/legacy.gitconfig".into())); + + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + + for target in [ + legacy, + included, + fixture.root().join("legacy.ignore"), + fixture.root().join("included.ignore"), + ] { + assert!( + policy.dependencies.iter().any(|dependency| { + dependency.identity == dependency_path_identity_with_case(&target, true) + }), + "missing legacy GIT_CONFIG dependency {}", + target.display() + ); + } + } + + #[test] + fn missing_and_empty_legacy_git_config_selectors_are_persisted() { + for (value, target) in [ + ( + OsString::from("config/missing-legacy.gitconfig"), + PathBuf::from("config/missing-legacy.gitconfig"), + ), + (OsString::new(), PathBuf::new()), + ] { + let mut fixture = Fixture::new(); + fixture.git_env.push(("GIT_CONFIG".into(), value)); + + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + let target = lexical_normalize(&fixture.root().join(target)); + + assert!( + policy.dependencies.iter().any(|dependency| { + dependency.kind == PolicyDependencyKind::GitConfig + && dependency.identity == dependency_path_identity_with_case(&target, true) + }), + "missing selected config dependency {}", + target.display() + ); + } + } + + #[test] + fn every_forwarded_repository_selector_invalidates_direct_reuse() { + for key in ["GIT_CONFIG", "GIT_DIR", "GIT_COMMON_DIR", "GIT_WORK_TREE"] { + let mut fixture = Fixture::new(); + let first = fixture.compile(&mut PolicyDependencyMetrics::default()); + fixture.git_env.push(( + key.into(), + fixture + .root() + .join(format!("changed-{key}")) + .into_os_string(), + )); + + assert_eq!( + validate_policy_manifest(&fixture.context(), &first.manifest()).unwrap(), + PolicyManifestValidation::Changed, + "selector {key} did not invalidate reuse", + ); + } + } + + #[test] + fn repository_selector_change_forces_full_policy_discovery() { + let mut fixture = Fixture::new(); + let mut metrics = PolicyDependencyMetrics::default(); + let first = fixture.compile(&mut metrics); + fixture.git_env.push(( + "GIT_DIR".into(), + fixture.root().join(".git").into_os_string(), + )); + + let second = fixture.compile(&mut metrics); + + assert_eq!(metrics.policy_dependency_full_discovery, 2); + assert!(!second.reused_manifest); + assert_ne!(first.fingerprint, second.fingerprint); + } + + #[test] + fn missing_external_global_config_is_persisted_and_creation_is_detected() { + let mut fixture = Fixture::new(); + let global = fixture + .root() + .parent() + .unwrap() + .join("missing-global.gitconfig"); + fixture + .git_env + .push(("GIT_CONFIG_GLOBAL".into(), global.as_os_str().to_owned())); + + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + let dependency = policy + .dependencies + .iter() + .find(|dependency| dependency.identity == dependency_path_identity(&global)) + .expect("missing global config must remain an authoritative dependency"); + assert!(!dependency.observable); + + fs::write(&global, "[core]\n\texcludesFile = /tmp/excludes\n").unwrap(); + assert_eq!( + validate_policy_manifest(&fixture.context(), &policy.manifest()).unwrap(), + PolicyManifestValidation::Changed + ); + } + + #[cfg(unix)] + #[test] + fn symlinked_policy_dependency_is_unobservable_and_mode_identity_is_validated() { + use std::os::unix::fs::{symlink, PermissionsExt}; + + let fixture = Fixture::new(); + let external = fixture.root().parent().unwrap().join("external-ignore"); + fs::write(&external, "secret\n").unwrap(); + let nested = fixture.root().join("src/.gitignore"); + fs::remove_file(&nested).unwrap(); + symlink(&external, &nested).unwrap(); + + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + let dependency = policy + .dependencies + .iter() + .find(|dependency| dependency.identity == dependency_path_identity(&nested)) + .unwrap(); + assert!(!dependency.observable); + assert!(policy.stale_baseline); + + fs::remove_file(&nested).unwrap(); + fs::write(&nested, "secret\n").unwrap(); + let mut permissions = fs::metadata(&nested).unwrap().permissions(); + permissions.set_mode(0o600); + fs::set_permissions(&nested, permissions).unwrap(); + assert_eq!( + validate_policy_manifest(&fixture.context(), &policy.manifest()).unwrap(), + PolicyManifestValidation::Changed + ); + } + + #[cfg(unix)] + #[test] + fn final_symlink_retarget_and_removal_change_identity_without_target_bytes() { + use std::os::unix::fs::symlink; + + let fixture = Fixture::new(); + let first_target = fixture.root().parent().unwrap().join("first-secret-ignore"); + let second_target = fixture + .root() + .parent() + .unwrap() + .join("second-secret-ignore"); + fs::write(&first_target, b"first secret bytes\n").unwrap(); + fs::write(&second_target, b"second secret bytes\n").unwrap(); + let dependency_path = fixture.root().join("final-symlink-ignore"); + symlink(&first_target, &dependency_path).unwrap(); + let (dependency, bytes) = read_file_dependency( + &dependency_path, + PolicyDependencyKind::GitExcludesFile, + 1, + &fixture.context(), + ) + .unwrap(); + assert!(bytes.is_empty()); + assert_eq!(dependency.content_identity, digest(b"")); + assert_ne!(dependency.content_identity, digest(b"first secret bytes\n")); + let mut manifest_dependencies = synthetic_dependencies(1, &fixture.context()).unwrap(); + manifest_dependencies.push(dependency.clone()); + manifest_dependencies.sort_by(|left, right| { + (left.kind, left.identity.as_str()).cmp(&(right.kind, right.identity.as_str())) + }); + let manifest = PolicyManifest { + dependencies: manifest_dependencies, + generation: 1, + rule_sources: Vec::new(), + }; + let (_, sources) = validate_policy_manifest_and_pin(&fixture.context(), &manifest).unwrap(); + let source = sources + .iter() + .find(|source| source.path == dependency_path) + .expect("final symlink remains a semantic rule source without target bytes"); + assert!(source.bytes.is_empty()); + fs::remove_file(&dependency_path).unwrap(); + symlink(&second_target, &dependency_path).unwrap(); + let (retargeted, retargeted_bytes) = read_file_dependency( + &dependency_path, + PolicyDependencyKind::GitExcludesFile, + 1, + &fixture.context(), + ) + .unwrap(); + assert!(retargeted_bytes.is_empty()); + assert_ne!(retargeted.metadata_identity, dependency.metadata_identity); + fs::remove_file(&dependency_path).unwrap(); + let (removed, removed_bytes) = read_file_dependency( + &dependency_path, + PolicyDependencyKind::GitExcludesFile, + 1, + &fixture.context(), + ) + .unwrap(); + assert!(removed_bytes.is_empty()); + assert_ne!(removed.metadata_identity, dependency.metadata_identity); + } + + #[cfg(unix)] + #[test] + fn symlinked_directory_ancestor_never_pins_external_policy_bytes() { + use std::os::unix::fs::symlink; + + let fixture = Fixture::new(); + let external = fixture.root().parent().unwrap().join("external-config-dir"); + fs::create_dir_all(&external).unwrap(); + fs::write( + external.join("policy.gitconfig"), + b"[include]\npath = should-never-be-decoded\n", + ) + .unwrap(); + let linked_dir = fixture.root().join("linked-config-dir"); + symlink(&external, &linked_dir).unwrap(); + let linked_policy = linked_dir.join("policy.gitconfig"); + + let (dependency, bytes) = read_file_dependency( + &linked_policy, + PolicyDependencyKind::GitConfig, + 1, + &fixture.context(), + ) + .unwrap(); + + assert!(bytes.is_empty()); + assert_eq!(dependency.content_identity, digest(b"")); + assert!(!dependency.observable); + } + + #[cfg(unix)] + #[test] + fn ancestor_symlink_retarget_and_removal_change_identity_and_invalidate_raw_events() { + use std::os::unix::fs::symlink; + + let fixture = Fixture::new(); + let first = fixture + .root() + .parent() + .unwrap() + .join("first-secret-config-dir"); + let second = fixture + .root() + .parent() + .unwrap() + .join("second-secret-config-dir"); + fs::create_dir_all(&first).unwrap(); + fs::create_dir_all(&second).unwrap(); + fs::write(first.join("policy.gitconfig"), b"first ancestor secret\n").unwrap(); + fs::write(second.join("policy.gitconfig"), b"second ancestor secret\n").unwrap(); + let ancestor = fixture.root().join("linked-policy-dir"); + let dependency_path = ancestor.join("policy.gitconfig"); + symlink(&first, &ancestor).unwrap(); + let (dependency, bytes) = read_file_dependency( + &dependency_path, + PolicyDependencyKind::GitConfig, + 1, + &fixture.context(), + ) + .unwrap(); + let index = PolicyInvalidationIndex::from_dependencies( + fixture.root(), + true, + std::slice::from_ref(&dependency), + ) + .unwrap(); + + assert!(bytes.is_empty()); + assert_eq!(dependency.content_identity, digest(b"")); + assert_ne!( + dependency.content_identity, + digest(b"first ancestor secret\n") + ); + assert!(index.matches(fixture.root(), &ancestor)); + let policy = finish_compiled_policy( + &fixture.context(), + PolicyManifest { + dependencies: vec![dependency.clone()], + generation: 1, + rule_sources: Vec::new(), + }, + false, + ) + .unwrap(); + assert!(raw_event_invalidates_policy(&policy, &ancestor)); + fs::remove_file(&ancestor).unwrap(); + symlink(&second, &ancestor).unwrap(); + let (retargeted, retargeted_bytes) = read_file_dependency( + &dependency_path, + PolicyDependencyKind::GitConfig, + 1, + &fixture.context(), + ) + .unwrap(); + assert!(retargeted_bytes.is_empty()); + assert_ne!(retargeted.metadata_identity, dependency.metadata_identity); + fs::remove_file(&ancestor).unwrap(); + let (removed, removed_bytes) = read_file_dependency( + &dependency_path, + PolicyDependencyKind::GitConfig, + 1, + &fixture.context(), + ) + .unwrap(); + assert!(removed_bytes.is_empty()); + assert_ne!(removed.metadata_identity, dependency.metadata_identity); + } + + #[test] + fn builtins_normalization_mode_and_case_policy_are_authoritative_identities() { + let fixture = Fixture::new(); + let policy = fixture.compile(&mut PolicyDependencyMetrics::default()); + for kind in [ + PolicyDependencyKind::Builtin, + PolicyDependencyKind::TrailConfig, + PolicyDependencyKind::Ignore, + PolicyDependencyKind::Normalization, + PolicyDependencyKind::Mode, + PolicyDependencyKind::CasePolicy, + ] { + assert!(policy + .dependencies + .iter() + .any(|dependency| dependency.kind == kind)); + } + + let changed_context = PolicyCompileContext { + case_sensitive: false, + ..fixture.context() + }; + assert_eq!( + validate_policy_manifest(&changed_context, &policy.manifest()).unwrap(), + PolicyManifestValidation::Changed + ); + } + + #[test] + fn direct_policy_fence_detects_content_replacement_and_missing_creation() { + let temp = tempfile::tempdir().unwrap(); + let dependency = temp.path().canonicalize().unwrap().join("global.gitconfig"); + let missing = capture_policy_dependency_fence_identity(&dependency).unwrap(); + assert_eq!( + missing, + capture_policy_dependency_fence_identity(&dependency).unwrap() + ); + + fs::write(&dependency, b"[core]\n").unwrap(); + let created = capture_policy_dependency_fence_identity(&dependency).unwrap(); + assert_ne!(missing, created); + + let replacement = temp.path().join("replacement"); + fs::write(&replacement, b"[core]\n").unwrap(); + let replace_target = dependency.clone(); + install_policy_fence_after_hash_hook(dependency.clone(), move || { + fs::rename(&replacement, &replace_target).unwrap(); + }); + let replaced = capture_policy_dependency_fence_identity(&dependency).unwrap(); + assert_eq!(created.content_identity, replaced.content_identity); + assert_ne!(created.metadata_identity, replaced.metadata_identity); + + fs::write(&dependency, b"[core]\n\texcludesFile = ignored\n").unwrap(); + let changed = capture_policy_dependency_fence_identity(&dependency).unwrap(); + assert_ne!(replaced.content_identity, changed.content_identity); + } + + #[cfg(unix)] + #[test] + fn authenticated_alias_policy_manifest_binds_target_creation_and_retarget() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let first_target = root.join("first-target"); + let second_target = root.join("second-target"); + fs::create_dir(&first_target).unwrap(); + fs::create_dir(&second_target).unwrap(); + let alias = root.join("alias"); + symlink(&first_target, &alias).unwrap(); + let dependency = alias.join("gitconfig"); + let first_observed = first_target.join("gitconfig"); + let included_dependency = alias.join("included.gitconfig"); + let included_observed = first_target.join("included.gitconfig"); + + assert!(capture_policy_dependency_fence_identity(&dependency).is_err()); + let alias_binding = + install_test_authenticated_alias(dependency.clone(), first_observed.clone()); + let included_alias_binding = install_test_authenticated_alias( + included_dependency.clone(), + included_observed.clone(), + ); + let missing = + capture_policy_dependency_fence_identity_at_observed_path(&dependency, &first_observed) + .unwrap(); + assert!(missing + .metadata_identity + .starts_with(b"unsafe-component-v1:path=")); + assert_eq!(missing.observed_metadata_identity, b"missing-v1"); + + let mut fixture = Fixture::new(); + fixture.git_env.push(( + OsString::from("GIT_CONFIG_GLOBAL"), + dependency.clone().into(), + )); + let mut compile_metrics = PolicyDependencyMetrics::default(); + let missing_policy = fixture.compile(&mut compile_metrics); + assert!(!missing_policy.reused_manifest); + let (missing_manifest, missing_bytes) = read_file_dependency( + &dependency, + PolicyDependencyKind::GitConfig, + 1, + &fixture.context(), + ) + .unwrap(); + assert!(missing_bytes.is_empty()); + assert_eq!(missing_manifest.content_identity, missing.content_identity); + assert_eq!( + missing_manifest.metadata_identity, + missing.metadata_identity + ); + + let excludes = first_target.join("global-ignore"); + fs::write(&excludes, b"ignored-from-alias\n").unwrap(); + let included_excludes = first_target.join("included-ignore"); + fs::write(&included_excludes, b"ignored-from-include\n").unwrap(); + fs::write( + &included_observed, + format!( + "[core]\n\texcludesFile = {}\n", + included_excludes.to_string_lossy() + ), + ) + .unwrap(); + let config_bytes = format!( + "[include]\n\tpath = included.gitconfig\n[core]\n\texcludesFile = {}\n", + excludes.to_string_lossy() + ) + .into_bytes(); + fs::write(&first_observed, &config_bytes).unwrap(); + let created = + capture_policy_dependency_fence_identity_at_observed_path(&dependency, &first_observed) + .unwrap(); + let pinned_before_observer = missing_policy + .dependencies + .iter() + .find(|candidate| { + dependency_identity_path(&candidate.identity).as_deref() + == Some(dependency.as_path()) + }) + .unwrap(); + assert_ne!( + pinned_before_observer.content_identity, + created.content_identity + ); + assert_ne!( + pinned_before_observer.metadata_identity, + created.metadata_identity + ); + assert_ne!(missing.content_identity, created.content_identity); + assert_ne!(missing.metadata_identity, created.metadata_identity); + assert_ne!( + missing.observed_content_identity, + created.observed_content_identity + ); + assert_ne!( + missing.observed_metadata_identity, + created.observed_metadata_identity + ); + let (created_manifest, created_bytes) = read_file_dependency( + &dependency, + PolicyDependencyKind::GitConfig, + 1, + &fixture.context(), + ) + .unwrap(); + assert_eq!(created_bytes, config_bytes); + assert_ne!( + missing_manifest.content_identity, + created_manifest.content_identity + ); + assert_ne!( + missing_manifest.metadata_identity, + created_manifest.metadata_identity + ); + let created_policy = fixture.compile(&mut compile_metrics); + assert!(!created_policy.reused_manifest); + assert_eq!(compile_metrics.policy_dependency_full_discovery, 2); + assert_ne!(missing_policy.fingerprint, created_policy.fingerprint); + assert!(created_policy.dependencies.iter().any(|candidate| { + candidate.kind == PolicyDependencyKind::GitExcludesFile + && dependency_identity_path(&candidate.identity).as_deref() + == Some(excludes.as_path()) + })); + assert!(created_policy.dependencies.iter().any(|candidate| { + candidate.kind == PolicyDependencyKind::GitConfig + && dependency_identity_path(&candidate.identity).as_deref() + == Some(included_dependency.as_path()) + })); + assert!(created_policy.dependencies.iter().any(|candidate| { + candidate.kind == PolicyDependencyKind::GitExcludesFile + && dependency_identity_path(&candidate.identity).as_deref() + == Some(included_excludes.as_path()) + })); + assert!(created_policy.snapshot.rule_sources.iter().any(|source| { + source.kind == PolicyDependencyKind::GitExcludesFile + && source.path == excludes + && source.bytes == b"ignored-from-alias\n" + })); + assert!(created_policy.snapshot.rule_sources.iter().any(|source| { + source.kind == PolicyDependencyKind::GitExcludesFile + && source.path == included_excludes + && source.bytes == b"ignored-from-include\n" + })); + + let race_a_excludes = first_target.join("race-a-ignore"); + let race_b_excludes = first_target.join("race-b-ignore"); + fs::write(&race_a_excludes, b"race-a\n").unwrap(); + fs::write(&race_b_excludes, b"race-b\n").unwrap(); + fs::write( + &first_observed, + format!("[core]\n\texcludesFile = {}\n", race_a_excludes.display()), + ) + .unwrap(); + let race_target = first_observed.clone(); + let race_b_config = format!("[core]\n\texcludesFile = {}\n", race_b_excludes.display()); + install_git_semantic_after_parse_hook(fixture.root().to_path_buf(), move || { + fs::write(race_target, race_b_config).unwrap(); + }); + let race_error = compile_policy( + &fixture.db.conn, + &fixture.expected, + &fixture.context(), + &mut compile_metrics, + ) + .err() + .unwrap(); + assert!(race_error + .to_string() + .contains("changed between semantic discovery and manifest capture")); + + let retried_policy = fixture.compile(&mut compile_metrics); + assert!(!retried_policy.reused_manifest); + assert!(retried_policy.dependencies.iter().any(|candidate| { + candidate.kind == PolicyDependencyKind::GitExcludesFile + && dependency_identity_path(&candidate.identity).as_deref() + == Some(race_b_excludes.as_path()) + })); + assert!(!retried_policy.dependencies.iter().any(|candidate| { + candidate.kind == PolicyDependencyKind::GitExcludesFile + && dependency_identity_path(&candidate.identity).as_deref() + == Some(race_a_excludes.as_path()) + })); + + drop(alias_binding); + drop(included_alias_binding); + fs::remove_file(&alias).unwrap(); + symlink(&second_target, &alias).unwrap(); + let second_observed = second_target.join("gitconfig"); + let _retargeted_binding = + install_test_authenticated_alias(dependency.clone(), second_observed.clone()); + let retargeted = capture_policy_dependency_fence_identity_at_observed_path( + &dependency, + &second_observed, + ) + .unwrap(); + assert_ne!(created.metadata_identity, retargeted.metadata_identity); + assert_ne!( + created.observed_content_identity, + retargeted.observed_content_identity + ); + } + + fn run_git(root: &Path, args: &[&str]) { + let output = Command::new("git") + .args(args) + .current_dir(root) + .output() + .unwrap(); + assert!( + output.status.success(), + "git {:?} failed: {}", + args, + String::from_utf8_lossy(&output.stderr) + ); + } + + #[allow(dead_code)] + fn path(value: impl Into) -> PathBuf { + value.into() + } +} +use super::ExpectedScope; +use crate::db::util::{is_default_ignored, normalize_relative_path, path_from_rel}; +use crate::error::{Error, Result}; +use crate::model::RecordingConfig; +use rusqlite::{params, Connection}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::io::{Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::process::{Command, Stdio}; +use unicode_normalization::UnicodeNormalization; +use walkdir::WalkDir; + +const BUILTIN_POLICY_VERSION: &[u8] = b"trail-recording-policy-v1"; +const NORMALIZATION_POLICY: &[u8] = b"relative-forward-slash-unicode-nfc-v1"; +const MODE_POLICY: &[u8] = b"regular-files-only-no-follow-executable-bit-v1"; +const MAX_POLICY_DEPENDENCY_BYTES: u64 = 16 * 1024 * 1024; + +#[cfg(test)] +type PolicyFenceAfterHashHook = Box; +#[cfg(test)] +static POLICY_FENCE_AFTER_HASH_HOOK: std::sync::OnceLock< + std::sync::Mutex>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +fn install_policy_fence_after_hash_hook(path: PathBuf, hook: impl FnOnce() + Send + 'static) { + *POLICY_FENCE_AFTER_HASH_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Some((path, Box::new(hook))); +} + +#[cfg(test)] +fn run_policy_fence_after_hash_hook(path: &Path) { + let hook = POLICY_FENCE_AFTER_HASH_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .take(); + match hook { + Some((expected, hook)) if expected == path => hook(), + Some(other) => { + *POLICY_FENCE_AFTER_HASH_HOOK + .get() + .unwrap() + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Some(other); + } + None => {} + } +} + +#[cfg(test)] +type GitSemanticAfterParseHook = Box; +#[cfg(test)] +static GIT_SEMANTIC_AFTER_PARSE_HOOKS: std::sync::OnceLock< + std::sync::Mutex>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +fn install_git_semantic_after_parse_hook( + workspace_root: PathBuf, + hook: impl FnOnce() + Send + 'static, +) { + GIT_SEMANTIC_AFTER_PARSE_HOOKS + .get_or_init(|| std::sync::Mutex::new(BTreeMap::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .insert(workspace_root, Box::new(hook)); +} + +#[cfg(test)] +fn run_git_semantic_after_parse_hook(workspace_root: &Path) { + let hook = GIT_SEMANTIC_AFTER_PARSE_HOOKS + .get_or_init(|| std::sync::Mutex::new(BTreeMap::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .remove(workspace_root); + if let Some(hook) = hook { + hook(); + } +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) enum PolicyDependencyKind { + Builtin, + TrailConfig, + Ignore, + Trailignore, + Gitignore, + GitInfoExclude, + GitExcludesFile, + GitConfig, + Normalization, + Mode, + CasePolicy, +} + +impl PolicyDependencyKind { + const fn as_str(self) -> &'static str { + match self { + Self::Builtin => "builtin", + Self::TrailConfig => "trail_config", + Self::Ignore => "ignore", + Self::Trailignore => "trailignore", + Self::Gitignore => "gitignore", + Self::GitInfoExclude => "git_info_exclude", + Self::GitExcludesFile => "git_excludes_file", + Self::GitConfig => "git_config", + Self::Normalization => "normalization", + Self::Mode => "mode", + Self::CasePolicy => "case_policy", + } + } + + fn parse(value: &str) -> Result { + match value { + "builtin" => Ok(Self::Builtin), + "trail_config" => Ok(Self::TrailConfig), + "ignore" => Ok(Self::Ignore), + "trailignore" => Ok(Self::Trailignore), + "gitignore" => Ok(Self::Gitignore), + "git_info_exclude" => Ok(Self::GitInfoExclude), + "git_excludes_file" => Ok(Self::GitExcludesFile), + "git_config" => Ok(Self::GitConfig), + "normalization" => Ok(Self::Normalization), + "mode" => Ok(Self::Mode), + "case_policy" => Ok(Self::CasePolicy), + other => Err(Error::Corrupt(format!( + "unknown changed-path policy dependency kind `{other}`" + ))), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PolicyDependency { + pub(crate) identity: String, + pub(crate) kind: PolicyDependencyKind, + pub(crate) content_identity: [u8; 32], + pub(crate) metadata_identity: Vec, + pub(crate) observable: bool, + pub(crate) generation: u64, + pub(crate) last_source_sequence: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PolicyManifest { + pub(crate) dependencies: Vec, + pub(crate) generation: u64, + rule_sources: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PolicyRuleSource { + pub(crate) kind: PolicyDependencyKind, + pub(crate) path: PathBuf, + pub(crate) bytes: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct RecordingPolicySnapshot { + pub(crate) workspace_root: PathBuf, + pub(crate) ignore_gitignored: bool, + pub(crate) dependency_files: Vec, + pub(crate) case_sensitive: bool, + pub(crate) rule_sources: Vec, +} + +#[cfg(test)] +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct QualifiedPolicyObserverCut { + pub(crate) scope_id: super::ScopeId, + pub(crate) provider_identity: Vec, + pub(crate) discovery_started_sequence: u64, + pub(crate) through_sequence: u64, + pub(crate) covered_roots: Vec, + pub(crate) case_sensitive: bool, +} + +#[cfg(test)] +impl QualifiedPolicyObserverCut { + pub(crate) fn validate_for( + &self, + expected: &ExpectedScope, + workspace_root: &Path, + case_sensitive: bool, + ) -> bool { + self.scope_id == expected.scope_id + && self.provider_identity == expected.provider_identity + && self.discovery_started_sequence > 0 + && self.through_sequence >= self.discovery_started_sequence + && self.case_sensitive == case_sensitive + && self.covered_roots.iter().any(|root| { + normalized_path_key(root, case_sensitive) + == normalized_path_key(workspace_root, case_sensitive) + }) + } + + fn covers(&self, path: &Path) -> bool { + let path = normalized_path_key(path, self.case_sensitive); + self.covered_roots.iter().any(|root| { + let root = normalized_path_key(root, self.case_sensitive); + path == root || path.strip_prefix(&format!("{root}/")).is_some() + }) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PolicyInvalidationIndex { + case_sensitive: bool, + exact_paths: BTreeSet, +} + +impl PolicyInvalidationIndex { + pub(crate) fn from_paths<'a>( + workspace_root: &Path, + case_sensitive: bool, + paths: impl IntoIterator, + ) -> Self { + let exact_paths = paths + .into_iter() + .map(|path| { + let path = if path.is_absolute() { + path.clone() + } else { + workspace_root.join(path) + }; + normalized_path_key(&path, case_sensitive) + }) + .collect(); + Self { + case_sensitive, + exact_paths, + } + } + + pub(crate) fn from_dependencies( + workspace_root: &Path, + case_sensitive: bool, + dependencies: &[PolicyDependency], + ) -> Result { + let root = lexical_normalize(workspace_root); + let mut exact_paths = BTreeSet::new(); + for dependency in dependencies + .iter() + .filter(|dependency| dependency_is_file(dependency)) + { + let path = dependency_identity_path(&dependency.identity) + .ok_or_else(|| Error::Corrupt("non-canonical policy path identity".into()))?; + let path = if path.is_absolute() { + path + } else { + root.join(path) + }; + exact_paths.insert(normalized_path_key(&path, case_sensitive)); + if let Some(unsafe_component) = + unsafe_component_path_from_metadata_identity(&dependency.metadata_identity) + { + exact_paths.insert(normalized_path_key(&unsafe_component, case_sensitive)); + } + } + Ok(Self { + case_sensitive, + exact_paths, + }) + } + + pub(crate) fn matches(&self, workspace_root: &Path, path: &Path) -> bool { + let path = if path.is_absolute() { + path.to_path_buf() + } else { + workspace_root.join(path) + }; + self.exact_paths + .contains(&normalized_path_key(&path, self.case_sensitive)) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum AdapterEquivalence { + Equivalent, + Conservative, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CompiledPolicy { + snapshot: RecordingPolicySnapshot, + fingerprint: [u8; 32], + dependencies: Vec, + adapter_equivalence: AdapterEquivalence, + stale_baseline: bool, + reused_manifest: bool, + invalidation_index: PolicyInvalidationIndex, + reconciliation_authorization: Option, +} + +pub(crate) struct CompiledRecordingMatcher { + workspace_root: PathBuf, + matcher: ::ignore::gitignore::Gitignore, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PolicyReconciliationAuthorization { + expected: ExpectedScope, +} + +impl CompiledPolicy { + pub(crate) fn manifest(&self) -> PolicyManifest { + PolicyManifest { + generation: self + .dependencies + .first() + .map(|dependency| dependency.generation) + .unwrap_or(0), + dependencies: self.dependencies.clone(), + rule_sources: self.snapshot.rule_sources.clone(), + } + } + + pub(crate) fn workspace_root(&self) -> &Path { + &self.snapshot.workspace_root + } + + pub(crate) fn fingerprint(&self) -> [u8; 32] { + self.fingerprint + } + + pub(crate) fn case_sensitive(&self) -> bool { + self.snapshot.case_sensitive + } + + pub(crate) fn dependency_files(&self) -> &[PathBuf] { + &self.snapshot.dependency_files + } + + pub(crate) fn recording_matcher(&self) -> Result { + let mut builder = ::ignore::gitignore::GitignoreBuilder::new(&self.snapshot.workspace_root); + for source in &self.snapshot.rule_sources { + let applies = matches!(source.kind, PolicyDependencyKind::Trailignore) + || (self.snapshot.ignore_gitignored + && matches!( + source.kind, + PolicyDependencyKind::Gitignore + | PolicyDependencyKind::GitInfoExclude + | PolicyDependencyKind::GitExcludesFile + )); + if !applies { + continue; + } + let text = std::str::from_utf8(&source.bytes).map_err(|_| { + Error::InvalidInput(format!( + "compiled recording rule `{}` is not UTF-8", + source.path.display() + )) + })?; + for line in text.lines() { + builder + .add_line(Some(source.path.clone()), line) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + } + } + Ok(CompiledRecordingMatcher { + workspace_root: self.snapshot.workspace_root.clone(), + matcher: builder + .build() + .map_err(|error| Error::InvalidInput(error.to_string()))?, + }) + } + + pub(crate) fn authorizes_reconciliation(&self, expected: &ExpectedScope) -> bool { + self.adapter_equivalence == AdapterEquivalence::Equivalent + && !self.stale_baseline + && self + .reconciliation_authorization + .as_ref() + .is_some_and(|authorization| authorization.expected == *expected) + } + + pub(crate) fn authorize_native_reconciliation( + &mut self, + expected: &ExpectedScope, + lease: &super::ObserverLease, + ) -> Result<()> { + if !self.observer_lease_matches(expected, lease) { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: super::TrustState::StaleBaseline.as_str().into(), + reason: "native observer lease cannot authorize the compiled recording policy" + .into(), + command: "trail status".into(), + }); + } + self.adapter_equivalence = AdapterEquivalence::Equivalent; + self.stale_baseline = false; + self.reconciliation_authorization = Some(PolicyReconciliationAuthorization { + expected: expected.clone(), + }); + Ok(()) + } + + pub(crate) fn observer_lease_matches( + &self, + expected: &ExpectedScope, + lease: &super::ObserverLease, + ) -> bool { + let mut covered_dependencies = lease.policy_dependencies.clone(); + covered_dependencies.extend( + lease + .direct_policy_fences + .iter() + .map(|(path, _)| path.clone()), + ); + covered_dependencies.sort(); + covered_dependencies.dedup(); + let mut observed_dependencies = lease.policy_dependencies.clone(); + observed_dependencies.sort(); + observed_dependencies.dedup(); + let direct_paths = lease + .direct_policy_fences + .iter() + .map(|(path, _)| path.clone()) + .collect::>(); + let direct_fences_match = lease.direct_policy_fences.iter().all(|(path, fence)| { + self.dependencies.iter().any(|dependency| { + dependency_identity_path(&dependency.identity).as_deref() == Some(path.as_path()) + && dependency.content_identity == fence.content_identity + && dependency.metadata_identity == fence.metadata_identity + }) + }); + self.fingerprint == expected.policy_fingerprint + && lease.root_identity == expected.filesystem_identity + && lease.provider_identity == expected.provider_identity + && covered_dependencies == self.snapshot.dependency_files + && observed_dependencies.len() == lease.policy_dependencies.len() + && direct_paths.len() == lease.direct_policy_fences.len() + && !lease + .policy_dependencies + .iter() + .any(|path| direct_paths.contains(path)) + && direct_fences_match + && lease.capabilities.durable_cursor + && lease.capabilities.linearizable_fence + && lease.capabilities.overflow_scope + && lease.capabilities.filesystem_supported + && lease.capabilities.clean_proof_allowed + && lease.capabilities.power_loss_durability + } + + pub(crate) fn rebind_observed_baseline( + &mut self, + previous: &ExpectedScope, + next: &ExpectedScope, + ) -> Result<()> { + if previous.scope_id != next.scope_id + || previous.epoch != next.epoch + || previous.policy_fingerprint != next.policy_fingerprint + || previous.policy_generation != next.policy_generation + || previous.filesystem_identity != next.filesystem_identity + || previous.provider_identity != next.provider_identity + || self.fingerprint != next.policy_fingerprint + || self + .reconciliation_authorization + .as_ref() + .is_none_or(|authorization| authorization.expected != *previous) + { + return Err(Error::ChangeLedgerReconcileRequired { + scope: previous.scope_id.to_text(), + state: super::TrustState::StaleBaseline.as_str().into(), + reason: + "compiled recording policy could not rebind the committed observed baseline" + .into(), + command: "trail status".into(), + }); + } + self.reconciliation_authorization = Some(PolicyReconciliationAuthorization { + expected: next.clone(), + }); + Ok(()) + } + + #[cfg(any(test, debug_assertions))] + pub(crate) fn authorize_reconciliation_for_test(&mut self, expected: &ExpectedScope) { + self.adapter_equivalence = AdapterEquivalence::Equivalent; + self.stale_baseline = false; + self.reconciliation_authorization = Some(PolicyReconciliationAuthorization { + expected: expected.clone(), + }); + } + + #[cfg(test)] + pub(crate) fn set_ignore_gitignored_for_test(&mut self, ignore_gitignored: bool) { + self.snapshot.ignore_gitignored = ignore_gitignored; + } + + #[cfg(test)] + pub(crate) fn set_gitignore_rule_for_test(&mut self, path: PathBuf, bytes: Vec) { + self.snapshot.rule_sources = vec![PolicyRuleSource { + kind: PolicyDependencyKind::Gitignore, + path, + bytes, + }]; + } + + #[cfg(any(test, debug_assertions))] + pub(crate) fn for_reconciliation_test( + snapshot: RecordingPolicySnapshot, + fingerprint: [u8; 32], + expected: &ExpectedScope, + ) -> Self { + let invalidation_index = PolicyInvalidationIndex::from_paths( + &snapshot.workspace_root, + snapshot.case_sensitive, + snapshot.dependency_files.iter(), + ); + let mut policy = Self { + snapshot, + fingerprint, + dependencies: Vec::new(), + adapter_equivalence: AdapterEquivalence::Conservative, + stale_baseline: true, + reused_manifest: false, + invalidation_index, + reconciliation_authorization: None, + }; + policy.authorize_reconciliation_for_test(expected); + policy + } +} + +impl CompiledRecordingMatcher { + pub(crate) fn is_ignored(&self, path: &str, is_dir: bool) -> Result { + let normalized = normalize_relative_path(path)?; + if is_default_ignored(&normalized) { + return Ok(true); + } + let absolute = self.workspace_root.join(path_from_rel(&normalized)); + Ok(self + .matcher + .matched_path_or_any_parents(absolute, is_dir) + .is_ignore()) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum PolicyManifestValidation { + Current, + Changed, + Unobservable, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct PolicyDependencyMetrics { + pub(crate) policy_dependency_full_discovery: u64, + pub(crate) policy_dependency_direct_checks: u64, +} + +pub(crate) struct PolicyCompileContext<'a> { + pub(crate) workspace_root: &'a Path, + pub(crate) db_dir: &'a Path, + pub(crate) recording: &'a RecordingConfig, + pub(crate) case_sensitive: bool, + pub(crate) git_environment: &'a [(OsString, OsString)], +} + +pub(crate) fn compile_policy( + conn: &Connection, + expected: &ExpectedScope, + context: &PolicyCompileContext<'_>, + metrics: &mut PolicyDependencyMetrics, +) -> Result { + conn.execute_batch("SAVEPOINT changed_path_policy_compile;")?; + let result = compile_policy_guarded(conn, expected, context, metrics); + match result { + Ok(policy) => { + conn.execute_batch("RELEASE changed_path_policy_compile;")?; + Ok(policy) + } + Err(err) => { + let _ = conn.execute_batch( + "ROLLBACK TO changed_path_policy_compile; RELEASE changed_path_policy_compile;", + ); + Err(err) + } + } +} + +pub(crate) fn revalidate_compiled_policy( + policy: &CompiledPolicy, + context: &PolicyCompileContext<'_>, + metrics: &mut PolicyDependencyMetrics, +) -> Result { + let mut manifest = policy.manifest(); + metrics.policy_dependency_direct_checks = metrics + .policy_dependency_direct_checks + .saturating_add(manifest.dependencies.len() as u64); + let (validation, rule_sources) = validate_policy_manifest_and_pin(context, &manifest)?; + if validation == PolicyManifestValidation::Changed { + return Err(Error::InvalidInput( + "policy dependency changed during observer startup".into(), + )); + } + manifest.rule_sources = rule_sources; + finish_compiled_policy(context, manifest, true) +} + +fn compile_policy_guarded( + conn: &Connection, + expected: &ExpectedScope, + context: &PolicyCompileContext<'_>, + metrics: &mut PolicyDependencyMetrics, +) -> Result { + policy_scope_cas_guard(conn, expected)?; + let stored = load_policy_manifest(conn, expected)?; + if let Some(mut manifest) = stored { + metrics.policy_dependency_direct_checks = metrics + .policy_dependency_direct_checks + .saturating_add(manifest.dependencies.len() as u64); + let (validation, rule_sources) = validate_policy_manifest_and_pin(context, &manifest)?; + manifest.rule_sources = rule_sources; + match validation { + PolicyManifestValidation::Current => { + let policy = finish_compiled_policy(context, manifest, true)?; + mark_policy_stale_guarded(conn, expected, "policy_observer_cut_unavailable")?; + return Ok(policy); + } + PolicyManifestValidation::Unobservable => { + let policy = finish_compiled_policy(context, manifest, true)?; + mark_policy_stale_guarded(conn, expected, "policy_observer_cut_unavailable")?; + return Ok(policy); + } + PolicyManifestValidation::Changed => {} + } + } + + metrics.policy_dependency_full_discovery = + metrics.policy_dependency_full_discovery.saturating_add(1); + let mut manifest = discover_policy_manifest(expected.policy_generation, context)?; + let (validation, rule_sources) = validate_policy_manifest_and_pin(context, &manifest)?; + if validation == PolicyManifestValidation::Changed { + return Err(Error::InvalidInput( + "policy dependency changed during discovery; retry reconciliation".into(), + )); + } + manifest.rule_sources = rule_sources; + persist_policy_manifest_rows(conn, expected, &manifest)?; + let policy = finish_compiled_policy(context, manifest, false)?; + mark_policy_stale_guarded(conn, expected, "policy_observer_cut_unavailable")?; + Ok(policy) +} + +fn finish_compiled_policy( + context: &PolicyCompileContext<'_>, + manifest: PolicyManifest, + reused_manifest: bool, +) -> Result { + let fingerprint = policy_fingerprint(&manifest.dependencies)?; + let mut dependency_files = manifest + .dependencies + .iter() + .filter(|dependency| dependency_is_file(dependency)) + .filter_map(|dependency| dependency_identity_path(&dependency.identity)) + .collect::>(); + dependency_files.sort(); + dependency_files.dedup(); + let invalidation_index = PolicyInvalidationIndex::from_dependencies( + context.workspace_root, + context.case_sensitive, + &manifest.dependencies, + )?; + Ok(CompiledPolicy { + snapshot: RecordingPolicySnapshot { + workspace_root: context.workspace_root.to_path_buf(), + ignore_gitignored: context.recording.ignore_gitignored, + dependency_files, + case_sensitive: context.case_sensitive, + rule_sources: manifest.rule_sources.clone(), + }, + fingerprint, + dependencies: manifest.dependencies, + // Task 4 persists dependency evidence but has no authorized trust + // promotion. Even a future crate-local observer proof cannot enter + // this compile context or change the result. + adapter_equivalence: AdapterEquivalence::Conservative, + stale_baseline: true, + reused_manifest, + invalidation_index, + reconciliation_authorization: None, + }) +} + +pub(crate) fn validate_policy_manifest( + context: &PolicyCompileContext<'_>, + manifest: &PolicyManifest, +) -> Result { + validate_policy_manifest_and_pin(context, manifest).map(|(validation, _)| validation) +} + +fn validate_policy_manifest_and_pin( + context: &PolicyCompileContext<'_>, + manifest: &PolicyManifest, +) -> Result<(PolicyManifestValidation, Vec)> { + validate_manifest_canonical(manifest)?; + let synthetic = synthetic_dependencies(manifest.generation, context)? + .into_iter() + .map(|dependency| ((dependency.kind, dependency.identity.clone()), dependency)) + .collect::>(); + let mut unobservable = false; + let mut rule_sources = Vec::new(); + for dependency in &manifest.dependencies { + let (current, bytes) = if dependency_is_file(dependency) { + let Some(path) = dependency_identity_path(&dependency.identity) else { + return Ok((PolicyManifestValidation::Changed, Vec::new())); + }; + read_file_dependency(&path, dependency.kind, dependency.generation, context)? + } else { + let Some(current) = synthetic.get(&(dependency.kind, dependency.identity.clone())) + else { + return Ok((PolicyManifestValidation::Changed, Vec::new())); + }; + (current.clone(), Vec::new()) + }; + if dependency.content_identity != current.content_identity + || dependency.metadata_identity != current.metadata_identity + || dependency.observable != current.observable + || dependency.generation != current.generation + { + return Ok((PolicyManifestValidation::Changed, Vec::new())); + } + if dependency_is_rule_file(dependency.kind) { + let path = dependency_identity_path(&dependency.identity) + .ok_or_else(|| Error::Corrupt("invalid policy rule identity".into()))?; + rule_sources.push(PolicyRuleSource { + kind: dependency.kind, + path, + bytes, + }); + } + unobservable |= !dependency.observable; + } + if manifest + .dependencies + .iter() + .filter(|dependency| !dependency_is_file(dependency)) + .count() + != synthetic.len() + { + return Ok((PolicyManifestValidation::Changed, Vec::new())); + } + Ok(( + if unobservable { + PolicyManifestValidation::Unobservable + } else { + PolicyManifestValidation::Current + }, + rule_sources, + )) +} + +fn dependency_is_file(dependency: &PolicyDependency) -> bool { + dependency.identity.starts_with("path:") +} + +fn dependency_is_rule_file(kind: PolicyDependencyKind) -> bool { + matches!( + kind, + PolicyDependencyKind::Ignore + | PolicyDependencyKind::Trailignore + | PolicyDependencyKind::Gitignore + | PolicyDependencyKind::GitInfoExclude + | PolicyDependencyKind::GitExcludesFile + ) +} + +pub(crate) fn raw_event_invalidates_policy(policy: &CompiledPolicy, path: &Path) -> bool { + raw_path_may_invalidate_policy_with_case(path, policy.snapshot.case_sensitive) + || policy + .invalidation_index + .matches(&policy.snapshot.workspace_root, path) +} + +pub(crate) fn raw_path_may_invalidate_policy(path: &Path) -> bool { + raw_path_may_invalidate_policy_with_case(path, platform_default_case_sensitive()) +} + +fn raw_path_may_invalidate_policy_with_case(path: &Path, case_sensitive: bool) -> bool { + let mut normalized = path.to_string_lossy().replace('\\', "/"); + if !case_sensitive { + normalized = normalized.to_lowercase(); + } + let file_name = normalized.rsplit('/').next(); + matches!(file_name, Some(".ignore" | ".trailignore" | ".gitignore")) + || normalized.ends_with("/.trail/config.toml") + || normalized == ".trail/config.toml" + || normalized.ends_with("/.git/info/exclude") + || normalized == ".git/info/exclude" + || normalized.ends_with("/.git/config") + || normalized == ".git/config" + || normalized.ends_with("/.git/config.worktree") + || normalized == ".git/config.worktree" +} + +fn discover_policy_manifest( + generation: u64, + context: &PolicyCompileContext<'_>, +) -> Result { + let mut dependencies = synthetic_dependencies(generation, context)?; + let trail_config = lexical_normalize(&context.db_dir.join("config.toml")); + dependencies.push(file_dependency( + &trail_config, + PolicyDependencyKind::TrailConfig, + generation, + context, + )?); + + let root = lexical_normalize(context.workspace_root); + for (name, kind) in [ + (".ignore", PolicyDependencyKind::Ignore), + (".trailignore", PolicyDependencyKind::Trailignore), + (".gitignore", PolicyDependencyKind::Gitignore), + ] { + dependencies.push(file_dependency( + &root.join(name), + kind, + generation, + context, + )?); + } + let walker = WalkDir::new(&root) + .follow_links(false) + .into_iter() + .filter_entry(|entry| { + if entry.depth() == 0 { + return true; + } + let Ok(relative) = entry.path().strip_prefix(&root) else { + return false; + }; + let first = relative.components().next(); + !matches!( + first, + Some(Component::Normal(name)) if name == OsStr::new(".git") || name == OsStr::new(".trail") + ) + }); + for entry in walker { + let entry = entry.map_err(|err| Error::InvalidInput(err.to_string()))?; + if entry.depth() == 1 { + continue; + } + if !entry.file_type().is_file() && !entry.file_type().is_symlink() { + continue; + } + let kind = match entry.file_name().to_str() { + Some(".ignore") => PolicyDependencyKind::Ignore, + Some(".trailignore") => PolicyDependencyKind::Trailignore, + Some(".gitignore") => PolicyDependencyKind::Gitignore, + _ => continue, + }; + dependencies.push(file_dependency(entry.path(), kind, generation, context)?); + } + + dependencies.extend(discover_git_dependencies(generation, context)?); + dependencies.sort_by(|left, right| { + (left.kind, left.identity.as_str()).cmp(&(right.kind, right.identity.as_str())) + }); + let manifest = PolicyManifest { + dependencies, + generation, + rule_sources: Vec::new(), + }; + validate_manifest_canonical(&manifest)?; + Ok(manifest) +} + +fn synthetic_dependencies( + generation: u64, + context: &PolicyCompileContext<'_>, +) -> Result> { + let recording = serde_json::to_vec(context.recording) + .map_err(|err| Error::InvalidInput(err.to_string()))?; + let mut dependencies = vec![ + synthetic_dependency( + "builtin:recording-policy", + PolicyDependencyKind::Builtin, + BUILTIN_POLICY_VERSION, + generation, + ), + synthetic_dependency( + "trail-config:recording", + PolicyDependencyKind::TrailConfig, + &recording, + generation, + ), + synthetic_dependency( + "normalization:path", + PolicyDependencyKind::Normalization, + NORMALIZATION_POLICY, + generation, + ), + synthetic_dependency( + "mode:filesystem-entry", + PolicyDependencyKind::Mode, + MODE_POLICY, + generation, + ), + synthetic_dependency( + "case-policy:scope", + PolicyDependencyKind::CasePolicy, + if context.case_sensitive { + b"case-sensitive-v1" + } else { + b"case-insensitive-v1" + }, + generation, + ), + ]; + let mut selector_keys = [ + OsString::from("HOME"), + OsString::from("XDG_CONFIG_HOME"), + OsString::from("GIT_CONFIG"), + OsString::from("GIT_DIR"), + OsString::from("GIT_COMMON_DIR"), + OsString::from("GIT_WORK_TREE"), + OsString::from("GIT_CONFIG_GLOBAL"), + OsString::from("GIT_CONFIG_SYSTEM"), + OsString::from("GIT_CONFIG_NOSYSTEM"), + ] + .into_iter() + .collect::>(); + selector_keys.extend( + context + .git_environment + .iter() + .map(|(key, _)| key.clone()) + .filter(|key| key.to_string_lossy().starts_with("GIT_")), + ); + for key in selector_keys { + let value = git_environment_value_os(context, &key) + .map(|value| os_str_bytes(&value).to_vec()) + .unwrap_or_else(|| b"".to_vec()); + dependencies.push(synthetic_dependency( + &format!("git-env:{}", hex::encode(os_str_bytes(&key))), + PolicyDependencyKind::GitConfig, + &value, + generation, + )); + } + Ok(dependencies) +} + +fn synthetic_dependency( + identity: &str, + kind: PolicyDependencyKind, + content: &[u8], + generation: u64, +) -> PolicyDependency { + PolicyDependency { + identity: identity.to_string(), + kind, + content_identity: digest(content), + metadata_identity: b"synthetic-v1".to_vec(), + observable: true, + generation, + last_source_sequence: 0, + } +} + +fn discover_git_dependencies( + generation: u64, + context: &PolicyCompileContext<'_>, +) -> Result> { + if !context.recording.ignore_gitignored { + return Ok(Vec::new()); + } + let Some(git_dirs) = run_git_optional_repository( + context, + &[ + "rev-parse", + "--path-format=absolute", + "--git-dir", + "--git-common-dir", + ], + )? + else { + return Ok(Vec::new()); + }; + let mut paths = BTreeMap::<(PolicyDependencyKind, String), PathBuf>::new(); + let cwd = lexical_normalize(context.workspace_root); + let home = git_environment_value(context, "HOME").map(|value| { + if value.is_empty() { + PathBuf::from("/") + } else { + resolve_git_cwd_path(&cwd, PathBuf::from(value)) + } + }); + let xdg = git_environment_value(context, "XDG_CONFIG_HOME") + .filter(|value| !value.is_empty()) + .map(PathBuf::from) + .map(|path| resolve_git_cwd_path(&cwd, path)) + .or_else(|| home.as_ref().map(|home| home.join(".config"))); + if let Some(selected) = git_environment_value(context, "GIT_CONFIG") { + insert_git_dependency_path( + &mut paths, + context.case_sensitive, + resolve_git_cwd_path(&cwd, PathBuf::from(selected)), + PolicyDependencyKind::GitConfig, + ); + } + if let Some(global) = git_environment_value(context, "GIT_CONFIG_GLOBAL") { + if !global.is_empty() { + insert_git_dependency_path( + &mut paths, + context.case_sensitive, + resolve_git_cwd_path(&cwd, PathBuf::from(global)), + PolicyDependencyKind::GitConfig, + ); + } + } else { + if let Some(home) = &home { + insert_git_dependency_path( + &mut paths, + context.case_sensitive, + home.join(".gitconfig"), + PolicyDependencyKind::GitConfig, + ); + } + if let Some(xdg) = &xdg { + insert_git_dependency_path( + &mut paths, + context.case_sensitive, + xdg.join("git/config"), + PolicyDependencyKind::GitConfig, + ); + } + } + if !git_environment_value(context, "GIT_CONFIG_NOSYSTEM").is_some_and(|v| git_truthy(&v)) { + let system = git_environment_value(context, "GIT_CONFIG_SYSTEM") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("/etc/gitconfig")); + if !system.as_os_str().is_empty() { + insert_git_dependency_path( + &mut paths, + context.case_sensitive, + resolve_git_cwd_path(&cwd, system), + PolicyDependencyKind::GitConfig, + ); + } + } + if let Some(xdg) = &xdg { + insert_git_dependency_path( + &mut paths, + context.case_sensitive, + xdg.join("git/ignore"), + PolicyDependencyKind::GitExcludesFile, + ); + } + let mut lines = git_dirs.split(|byte| *byte == b'\n'); + let git_dir = lines + .next() + .filter(|line| !line.is_empty()) + .map(|line| PathBuf::from(os_string_from_bytes(line))); + if let Some(git_dir) = &git_dir { + insert_git_dependency_path( + &mut paths, + context.case_sensitive, + git_dir.join("config"), + PolicyDependencyKind::GitConfig, + ); + insert_git_dependency_path( + &mut paths, + context.case_sensitive, + git_dir.join("config.worktree"), + PolicyDependencyKind::GitConfig, + ); + } + if let Some(common) = lines.next().filter(|line| !line.is_empty()) { + insert_git_dependency_path( + &mut paths, + context.case_sensitive, + PathBuf::from(os_string_from_bytes(common)).join("info/exclude"), + PolicyDependencyKind::GitInfoExclude, + ); + } + + for entry in injected_git_config_entries(context)? { + if git_config_key_is_include(&entry.key) { + if let Some(path) = resolve_git_include_path(&entry.value, &cwd, home.as_deref()) { + insert_git_dependency_path( + &mut paths, + context.case_sensitive, + path, + PolicyDependencyKind::GitConfig, + ); + } + } else if entry.key.eq_ignore_ascii_case(b"core.excludesfile") { + if let Some(path) = resolve_git_config_path(&entry.value, &cwd, home.as_deref()) { + insert_git_dependency_path( + &mut paths, + context.case_sensitive, + path, + PolicyDependencyKind::GitExcludesFile, + ); + } + } + } + + // Ask Git to decode each safely pinned config independently. Keeping + // includes disabled makes missing and inactive include/includeIf targets + // visible without allowing Git to reopen or recursively follow paths. + let mut pending = paths + .iter() + .filter(|((kind, _), _)| *kind == PolicyDependencyKind::GitConfig) + .map(|(_, path)| path.clone()) + .collect::>(); + let mut parsed = BTreeSet::new(); + let mut semantic_evidence = BTreeMap::new(); + while let Some(config) = pending.pop() { + let key = normalized_path_key(&config, context.case_sensitive); + if !parsed.insert(key) { + continue; + } + let read = read_policy_path_for_semantic_discovery(&config)?; + semantic_evidence.insert( + normalized_path_key(&config, context.case_sensitive), + read.identity, + ); + let Some(bytes) = read.bytes else { + continue; + }; + for entry in git_config_entries_from_bytes(context, &bytes)? { + if git_config_key_is_include(&entry.key) { + if let Some(included) = resolve_git_include_path( + &entry.value, + config.parent().unwrap_or(Path::new("/")), + home.as_deref(), + ) { + let included_key = ( + PolicyDependencyKind::GitConfig, + normalized_path_key(&included, context.case_sensitive), + ); + if !paths.contains_key(&included_key) { + paths.insert(included_key, included.clone()); + pending.push(included); + } + } + } else if entry.key.eq_ignore_ascii_case(b"core.excludesfile") { + if let Some(path) = resolve_git_config_path(&entry.value, &cwd, home.as_deref()) { + insert_git_dependency_path( + &mut paths, + context.case_sensitive, + path, + PolicyDependencyKind::GitExcludesFile, + ); + } + } + } + } + + #[cfg(test)] + run_git_semantic_after_parse_hook(context.workspace_root); + + paths + .into_iter() + .map(|((kind, key), path)| { + let dependency = file_dependency(&path, kind, generation, context)?; + if kind == PolicyDependencyKind::GitConfig { + let parsed = semantic_evidence.get(&key).ok_or_else(|| { + Error::InvalidInput(format!( + "Git config `{}` was not pinned during semantic discovery", + path.display() + )) + })?; + if dependency.content_identity != parsed.content_identity + || dependency.metadata_identity != parsed.metadata_identity + { + return Err(Error::InvalidInput(format!( + "Git config `{}` changed between semantic discovery and manifest capture; retry reconciliation", + path.display() + ))); + } + } + Ok(dependency) + }) + .collect() +} + +fn insert_git_dependency_path( + paths: &mut BTreeMap<(PolicyDependencyKind, String), PathBuf>, + case_sensitive: bool, + path: PathBuf, + kind: PolicyDependencyKind, +) { + let path = lexical_normalize(&path); + paths.insert((kind, normalized_path_key(&path, case_sensitive)), path); +} + +#[derive(Debug)] +struct GitConfigEntry { + key: Vec, + value: OsString, +} + +const GIT_POLICY_KEY_PATTERN: &str = r"^(include\.path|include[Ii]f\..*\.path|core\.excludesfile)$"; + +fn git_config_entries_from_bytes( + context: &PolicyCompileContext<'_>, + bytes: &[u8], +) -> Result> { + let output = run_git_with_stdin( + context, + &[ + "config", + "--file", + "-", + "--no-includes", + "--show-origin", + "--null", + "--get-regexp", + GIT_POLICY_KEY_PATTERN, + ], + false, + bytes, + )?; + expand_git_config_entry_paths(context, parse_git_config_entries(&output, false)?) +} + +fn injected_git_config_entries(context: &PolicyCompileContext<'_>) -> Result> { + let output = run_injected_git_config(context)?; + expand_git_config_entry_paths(context, parse_git_config_entries(&output, true)?) +} + +fn expand_git_config_entry_paths( + context: &PolicyCompileContext<'_>, + entries: Vec, +) -> Result> { + entries + .into_iter() + .map(|entry| { + Ok(GitConfigEntry { + key: entry.key, + value: git_expand_path_value(context, &entry.value)?, + }) + }) + .collect() +} + +fn parse_git_config_entries(output: &[u8], with_scope: bool) -> Result> { + let fields = output + .split(|byte| *byte == 0) + .filter(|field| !field.is_empty()) + .collect::>(); + let width = if with_scope { 3 } else { 2 }; + if fields.len() % width != 0 { + return Err(Error::InvalidInput( + "git config returned malformed origin output".into(), + )); + } + let mut entries = Vec::new(); + for record in fields.chunks_exact(width) { + if with_scope && record[0] != b"command" { + continue; + } + let key_value = record[width - 1]; + let Some(separator) = key_value.iter().position(|byte| *byte == b'\n') else { + return Err(Error::InvalidInput( + "git config returned an entry without a key/value separator".into(), + )); + }; + entries.push(GitConfigEntry { + key: key_value[..separator].to_vec(), + value: os_string_from_bytes(&key_value[separator + 1..]), + }); + } + Ok(entries) +} + +fn git_config_key_is_include(key: &[u8]) -> bool { + let key = String::from_utf8_lossy(key).to_ascii_lowercase(); + key == "include.path" || (key.starts_with("includeif.") && key.ends_with(".path")) +} + +fn normalize_git_config_count(value: &OsStr) -> OsString { + let bytes = os_str_bytes(value); + let start = bytes + .iter() + .position(|byte| !byte.is_ascii_whitespace()) + .unwrap_or(bytes.len()); + let end = bytes + .iter() + .rposition(|byte| !byte.is_ascii_whitespace()) + .map(|index| index + 1) + .unwrap_or(start); + if start == end { + OsString::from("0") + } else { + os_string_from_bytes(&bytes[start..end]) + } +} + +fn resolve_git_cwd_path(cwd: &Path, path: PathBuf) -> PathBuf { + if path.is_absolute() { + lexical_normalize(&path) + } else { + lexical_normalize(&cwd.join(path)) + } +} + +fn resolve_git_include_path(value: &OsStr, base: &Path, _home: Option<&Path>) -> Option { + resolve_git_config_path(value, base, None) +} + +fn resolve_git_config_path(value: &OsStr, cwd: &Path, _home: Option<&Path>) -> Option { + if value.is_empty() { + return None; + } + Some(resolve_git_cwd_path(cwd, PathBuf::from(value))) +} + +fn git_truthy(value: &OsStr) -> bool { + !matches!( + value.to_string_lossy().to_ascii_lowercase().as_str(), + "" | "0" | "false" | "no" | "off" + ) +} + +fn git_environment_value(context: &PolicyCompileContext<'_>, key: &str) -> Option { + git_environment_value_os(context, OsStr::new(key)) +} + +fn git_environment_value_os(context: &PolicyCompileContext<'_>, key: &OsStr) -> Option { + context + .git_environment + .iter() + .rev() + .find(|(candidate, _)| candidate == key) + .map(|(_, value)| value.clone()) +} + +fn git_command_environment( + context: &PolicyCompileContext<'_>, + ambient: impl IntoIterator, +) -> Vec<(OsString, OsString)> { + let mut environment = BTreeMap::new(); + for (key, value) in ambient { + if key == OsStr::new("PATH") { + environment.insert(key, value); + } + } + for (key, value) in context.git_environment { + environment.insert( + key.clone(), + if key == OsStr::new("GIT_CONFIG_COUNT") { + normalize_git_config_count(value) + } else { + value.clone() + }, + ); + } + environment.into_iter().collect() +} + +fn git_expand_path_value(context: &PolicyCompileContext<'_>, value: &OsStr) -> Result { + let mut assignment = OsString::from("trail.policyPath="); + assignment.push(value); + let ambient = git_command_environment(context, std::env::vars_os()); + let mut environment = BTreeMap::new(); + for (key, value) in ambient { + if key == OsStr::new("PATH") || key == OsStr::new("HOME") { + environment.insert(key, value); + } + } + environment.insert( + OsString::from("XDG_CONFIG_HOME"), + OsString::from("/dev/null"), + ); + environment.insert( + OsString::from("GIT_CONFIG_GLOBAL"), + OsString::from("/dev/null"), + ); + environment.insert(OsString::from("GIT_CONFIG_NOSYSTEM"), OsString::from("1")); + let output = Command::new("git") + .arg("-c") + .arg(assignment) + .args(["config", "--path", "--get", "trail.policyPath"]) + .current_dir(context.workspace_root) + .env_clear() + .envs(environment) + .output() + .map_err(Error::Io)?; + let output = git_output( + &["-c", "trail.policyPath=", "config", "--path"], + true, + output, + )?; + Ok(os_string_from_bytes( + output.strip_suffix(b"\n").unwrap_or(&output), + )) +} + +fn run_git(context: &PolicyCompileContext<'_>, args: &[&str], required: bool) -> Result> { + let mut command = Command::new("git"); + command.args(args).current_dir(context.workspace_root); + command + .env_clear() + .envs(git_command_environment(context, std::env::vars_os())); + let output = command.output().map_err(Error::Io)?; + git_output(args, required, output) +} + +fn run_git_optional_repository( + context: &PolicyCompileContext<'_>, + args: &[&str], +) -> Result>> { + let mut command = Command::new("git"); + command + .args(args) + .current_dir(context.workspace_root) + .env_clear() + .envs(git_command_environment(context, std::env::vars_os())) + .env("LC_ALL", "C"); + let output = command.output().map_err(Error::Io)?; + if output.status.success() { + return Ok(Some(output.stdout)); + } + let stderr = String::from_utf8_lossy(&output.stderr); + if output.status.code() == Some(128) && stderr.contains("not a git repository") { + return Ok(None); + } + git_output(args, true, output).map(Some) +} + +fn run_git_with_stdin( + context: &PolicyCompileContext<'_>, + args: &[&str], + required: bool, + stdin: &[u8], +) -> Result> { + let mut command = Command::new("git"); + command + .args(args) + .current_dir(context.workspace_root) + .env_clear() + .envs(git_command_environment(context, std::env::vars_os())) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = command.spawn().map_err(Error::Io)?; + child + .stdin + .take() + .ok_or_else(|| Error::InvalidInput("git config stdin was unavailable".into()))? + .write_all(stdin) + .map_err(Error::Io)?; + let output = child.wait_with_output().map_err(Error::Io)?; + git_output(args, required, output) +} + +fn run_injected_git_config(context: &PolicyCompileContext<'_>) -> Result> { + let args = [ + "config", + "--no-includes", + "--show-origin", + "--show-scope", + "--null", + "--get-regexp", + GIT_POLICY_KEY_PATTERN, + ]; + let ambient = git_command_environment(context, std::env::vars_os()); + let mut environment = BTreeMap::new(); + for (key, value) in ambient { + if key == OsStr::new("PATH") + || key == OsStr::new("GIT_CONFIG_PARAMETERS") + || key == OsStr::new("GIT_CONFIG_COUNT") + || key.to_string_lossy().starts_with("GIT_CONFIG_KEY_") + || key.to_string_lossy().starts_with("GIT_CONFIG_VALUE_") + { + environment.insert(key, value); + } + } + environment.insert(OsString::from("HOME"), OsString::from("/")); + environment.insert( + OsString::from("XDG_CONFIG_HOME"), + OsString::from("/dev/null"), + ); + environment.insert( + OsString::from("GIT_CONFIG_GLOBAL"), + OsString::from("/dev/null"), + ); + environment.insert(OsString::from("GIT_CONFIG_NOSYSTEM"), OsString::from("1")); + + let output = Command::new("git") + .args(args) + .current_dir("/") + .env_clear() + .envs(environment) + .output() + .map_err(Error::Io)?; + git_output(&args, false, output) +} + +fn git_output(args: &[&str], required: bool, output: std::process::Output) -> Result> { + if output.status.success() { + return Ok(output.stdout); + } + if !required && output.status.code() == Some(1) { + return Ok(Vec::new()); + } + Err(Error::InvalidInput(format!( + "git {} failed while compiling recording policy: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + ))) +} + +fn file_dependency( + path: &Path, + kind: PolicyDependencyKind, + generation: u64, + context: &PolicyCompileContext<'_>, +) -> Result { + read_file_dependency(path, kind, generation, context).map(|(dependency, _)| dependency) +} + +fn read_file_dependency( + path: &Path, + kind: PolicyDependencyKind, + generation: u64, + _context: &PolicyCompileContext<'_>, +) -> Result<(PolicyDependency, Vec)> { + let path = lexical_normalize(path); + if let Some(observed_path) = authenticated_system_alias_observed_path(&path)? { + let fence = + capture_policy_dependency_fence_identity_at_observed_path(&path, &observed_path)?; + let observed = read_file_state_no_follow(&observed_path)?; + if observed.unsafe_metadata_identity.is_some() + || observed + .metadata + .as_ref() + .is_some_and(|metadata| !metadata.is_file()) + || digest(&observed.bytes) != fence.observed_content_identity + || metadata_identity(observed.metadata.as_ref(), &observed_path)? + != fence.observed_metadata_identity + { + return Err(Error::InvalidInput(format!( + "system policy dependency `{}` changed during authenticated alias discovery", + path.display() + ))); + } + let dependency = PolicyDependency { + identity: dependency_path_identity(&path), + kind, + content_identity: fence.content_identity, + metadata_identity: fence.metadata_identity, + observable: false, + generation, + last_source_sequence: 0, + }; + return Ok((dependency, observed.bytes)); + } + let state = read_file_state_no_follow(&path)?; + // Task 4 has no native-observer handoff yet. Direct filesystem checks can + // validate reuse, but they cannot establish observer continuity, so file + // dependencies remain explicitly uncovered until a later task wires a + // qualified discovery cut into this compiler. + let observable = false; + let dependency = PolicyDependency { + identity: dependency_path_identity(&path), + kind, + content_identity: digest(&state.bytes), + metadata_identity: match state.unsafe_metadata_identity { + Some(identity) => identity, + None => metadata_identity(state.metadata.as_ref(), &path)?, + }, + observable, + generation, + last_source_sequence: 0, + }; + Ok((dependency, state.bytes)) +} + +struct PolicySemanticRead { + bytes: Option>, + identity: PolicyDependencyFenceIdentity, +} + +fn read_policy_path_for_semantic_discovery(path: &Path) -> Result { + let path = lexical_normalize(path); + if let Some(observed_path) = authenticated_system_alias_observed_path(&path)? { + let fence = + capture_policy_dependency_fence_identity_at_observed_path(&path, &observed_path)?; + let observed = read_file_state_no_follow(&observed_path)?; + if observed.unsafe_metadata_identity.is_some() + || observed + .metadata + .as_ref() + .is_some_and(|metadata| !metadata.is_file()) + || digest(&observed.bytes) != fence.observed_content_identity + || metadata_identity(observed.metadata.as_ref(), &observed_path)? + != fence.observed_metadata_identity + { + return Err(Error::InvalidInput(format!( + "system policy dependency `{}` changed during authenticated semantic discovery", + path.display() + ))); + } + return Ok(PolicySemanticRead { + bytes: observed.metadata.map(|_| observed.bytes), + identity: fence, + }); + } + let fence = capture_policy_dependency_fence_identity(&path)?; + let state = read_file_state_no_follow(&path)?; + if state.unsafe_metadata_identity.is_some() + || state + .metadata + .as_ref() + .is_some_and(|metadata| !metadata.is_file()) + || digest(&state.bytes) != fence.content_identity + || metadata_identity(state.metadata.as_ref(), &path)? != fence.metadata_identity + { + return Err(Error::InvalidInput(format!( + "policy dependency `{}` changed during semantic discovery", + path.display() + ))); + } + Ok(PolicySemanticRead { + bytes: state.metadata.map(|_| state.bytes), + identity: fence, + }) +} + +struct NoFollowFileState { + metadata: Option, + bytes: Vec, + unsafe_metadata_identity: Option>, +} + +/// A bounded, authenticated identity for policy files that cannot share the +/// workspace observer's native filesystem history (for example a global Git +/// config in `$HOME` while the repository is on an external APFS volume). +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +pub(crate) struct PolicyDependencyFenceIdentity { + /// Identity of the dependency exactly as named by the compiled manifest. + pub(crate) content_identity: [u8; 32], + pub(crate) metadata_identity: Vec, + /// Identity of the canonical path whose bytes Git actually consumes. + /// These differ only for an authenticated platform alias such as the + /// macOS `/etc -> /private/etc` binding. + pub(crate) observed_content_identity: [u8; 32], + pub(crate) observed_metadata_identity: Vec, +} + +pub(crate) fn capture_policy_dependency_fence_identity( + path: &Path, +) -> Result { + capture_policy_dependency_fence_identity_at_observed_path(path, path) +} + +pub(crate) fn capture_policy_dependency_fence_identity_at_observed_path( + dependency: &Path, + observed_path: &Path, +) -> Result { + let dependency = lexical_normalize(dependency); + let observed_path = lexical_normalize(observed_path); + let alias_observed = if dependency == observed_path { + false + } else { + authenticated_system_alias_observed_path(&dependency)?.as_deref() + == Some(observed_path.as_path()) + }; + if dependency != observed_path && !alias_observed { + return Err(Error::InvalidInput(format!( + "policy dependency `{}` does not use an authenticated system alias for `{}`", + dependency.display(), + observed_path.display() + ))); + } + let capture = || -> Result { + let named = read_file_state_no_follow(&dependency)?; + let named_metadata_identity = match named.unsafe_metadata_identity { + Some(identity) if alias_observed => identity, + Some(_) => { + return Err(Error::InvalidInput(format!( + "policy dependency `{}` is not a safely traversable regular file", + dependency.display() + ))) + } + None if named + .metadata + .as_ref() + .is_some_and(|metadata| !metadata.is_file()) => + { + return Err(Error::InvalidInput(format!( + "policy dependency `{}` is not a safely traversable regular file", + dependency.display() + ))) + } + None => metadata_identity(named.metadata.as_ref(), &dependency)?, + }; + let observed = read_file_state_no_follow(&observed_path)?; + if observed.unsafe_metadata_identity.is_some() + || observed + .metadata + .as_ref() + .is_some_and(|metadata| !metadata.is_file()) + { + return Err(Error::InvalidInput(format!( + "policy dependency `{}` is not a safely traversable regular file", + observed_path.display() + ))); + } + let observed_content_identity = digest(&observed.bytes); + let observed_metadata_identity = + metadata_identity(observed.metadata.as_ref(), &observed_path)?; + let (content_identity, metadata_identity) = if alias_observed { + ( + observed_content_identity, + authenticated_alias_metadata_identity( + &named_metadata_identity, + &observed_content_identity, + &observed_metadata_identity, + ), + ) + } else { + (digest(&named.bytes), named_metadata_identity) + }; + Ok(PolicyDependencyFenceIdentity { + content_identity, + metadata_identity, + observed_content_identity, + observed_metadata_identity, + }) + }; + let first = capture()?; + let second = capture()?; + if first != second { + return Err(Error::InvalidInput(format!( + "policy dependency `{}` changed or was rebound while its direct fence was captured", + dependency.display() + ))); + } + Ok(second) +} + +fn authenticated_alias_metadata_identity( + named: &[u8], + observed_content: &[u8; 32], + observed_metadata: &[u8], +) -> Vec { + let mut identity = named.to_vec(); + identity.extend_from_slice(b"authenticated-alias-v1:observed-content="); + identity.extend_from_slice(hex::encode(observed_content).as_bytes()); + identity.extend_from_slice(b";observed-metadata="); + identity.extend_from_slice(hex::encode(observed_metadata).as_bytes()); + identity.push(b';'); + identity +} + +#[cfg(target_os = "macos")] +fn authenticated_system_alias_observed_path(path: &Path) -> Result> { + use std::os::unix::fs::MetadataExt; + + #[cfg(test)] + if let Some(observed) = test_authenticated_alias_observed_path(path) { + return Ok(Some(observed)); + } + + for (alias, target, link_target) in [ + ("/etc", "/private/etc", "private/etc"), + ("/var", "/private/var", "private/var"), + ("/tmp", "/private/tmp", "private/tmp"), + ] { + let alias = Path::new(alias); + let Ok(relative) = path.strip_prefix(alias) else { + continue; + }; + let metadata = fs::symlink_metadata(alias).map_err(Error::Io)?; + let named_target = fs::read_link(alias).map_err(Error::Io)?; + let canonical_target = alias.canonicalize().map_err(Error::Io)?; + if !metadata.file_type().is_symlink() + || metadata.uid() != 0 + || named_target != Path::new(link_target) + || canonical_target != Path::new(target) + { + return Err(Error::InvalidInput(format!( + "system policy alias `{}` does not match its authenticated platform binding", + alias.display() + ))); + } + return Ok(Some(lexical_normalize(&canonical_target.join(relative)))); + } + Ok(None) +} + +#[cfg(not(target_os = "macos"))] +fn authenticated_system_alias_observed_path(_path: &Path) -> Result> { + #[cfg(test)] + if let Some(observed) = test_authenticated_alias_observed_path(_path) { + return Ok(Some(observed)); + } + Ok(None) +} + +#[cfg(test)] +static TEST_AUTHENTICATED_ALIASES: std::sync::OnceLock< + std::sync::Mutex>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +fn test_authenticated_alias_observed_path(path: &Path) -> Option { + TEST_AUTHENTICATED_ALIASES + .get_or_init(|| std::sync::Mutex::new(BTreeMap::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .get(path) + .cloned() +} + +#[cfg(test)] +struct TestAuthenticatedAlias { + dependency: PathBuf, +} + +#[cfg(test)] +fn install_test_authenticated_alias( + dependency: PathBuf, + observed: PathBuf, +) -> TestAuthenticatedAlias { + TEST_AUTHENTICATED_ALIASES + .get_or_init(|| std::sync::Mutex::new(BTreeMap::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .insert(dependency.clone(), observed); + TestAuthenticatedAlias { dependency } +} + +#[cfg(test)] +impl Drop for TestAuthenticatedAlias { + fn drop(&mut self) { + TEST_AUTHENTICATED_ALIASES + .get_or_init(|| std::sync::Mutex::new(BTreeMap::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .remove(&self.dependency); + } +} + +impl NoFollowFileState { + fn missing() -> Self { + Self { + metadata: None, + bytes: Vec::new(), + unsafe_metadata_identity: None, + } + } + + fn unsafe_component(metadata_identity: Vec) -> Self { + Self { + metadata: None, + bytes: Vec::new(), + unsafe_metadata_identity: Some(metadata_identity), + } + } +} + +fn read_file_state_no_follow(path: &Path) -> Result { + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + return read_file_state_openat(path); + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = path; + Ok(NoFollowFileState::missing()) + } +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn read_file_state_openat(path: &Path) -> Result { + use rustix::fs::{openat, Mode, OFlags, CWD}; + + let path = lexical_normalize(path); + if !path.is_absolute() { + return Err(Error::InvalidInput(format!( + "policy dependency `{}` is not absolute", + path.display() + ))); + } + let components = path + .strip_prefix(Path::new("/")) + .ok() + .map(|relative| relative.components().collect::>()) + .unwrap_or_default(); + if components.is_empty() + || components + .iter() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(Error::InvalidInput(format!( + "policy dependency `{}` cannot be traversed safely", + path.display() + ))); + } + + for _ in 0..2 { + let mut directory = match openat( + CWD, + Path::new("/"), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) { + Ok(directory) => directory, + Err(err) => return Err(Error::Io(err.into())), + }; + let mut traversed = PathBuf::from("/"); + for component in &components[..components.len() - 1] { + let Component::Normal(name) = component else { + return Err(Error::InvalidInput(format!( + "policy dependency `{}` cannot be traversed safely", + path.display() + ))); + }; + traversed.push(name); + directory = match openat( + &directory, + Path::new(name), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) { + Ok(directory) => directory, + Err(open_error) => { + return unsafe_open_failure_state( + &directory, + Path::new(name), + &traversed, + open_error, + ) + } + }; + } + let Component::Normal(file_name) = components[components.len() - 1] else { + return Err(Error::InvalidInput(format!( + "policy dependency `{}` cannot be traversed safely", + path.display() + ))); + }; + let descriptor = match openat( + &directory, + Path::new(file_name), + OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) { + Ok(descriptor) => descriptor, + Err(open_error) => { + return unsafe_open_failure_state( + &directory, + Path::new(file_name), + &path, + open_error, + ) + } + }; + let mut file = fs::File::from(descriptor); + let before = file.metadata().map_err(Error::Io)?; + if !before.is_file() { + return Ok(NoFollowFileState { + metadata: Some(before), + bytes: Vec::new(), + unsafe_metadata_identity: None, + }); + } + let mut bytes = Vec::new(); + (&mut file) + .take(MAX_POLICY_DEPENDENCY_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(Error::Io)?; + if bytes.len() as u64 > MAX_POLICY_DEPENDENCY_BYTES { + return Err(Error::InvalidInput(format!( + "policy dependency `{}` exceeds the {} byte safety bound", + path.display(), + MAX_POLICY_DEPENDENCY_BYTES + ))); + } + #[cfg(test)] + run_policy_fence_after_hash_hook(&path); + let after = file.metadata().map_err(Error::Io)?; + if metadata_identity(Some(&before), &path)? == metadata_identity(Some(&after), &path)? + && revalidate_named_regular_file_no_follow(&path, &after)? + { + return Ok(NoFollowFileState { + metadata: Some(after), + bytes, + unsafe_metadata_identity: None, + }); + } + } + Err(Error::InvalidInput(format!( + "policy dependency `{}` changed while it was read", + path.display() + ))) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn revalidate_named_regular_file_no_follow(path: &Path, expected: &fs::Metadata) -> Result { + use rustix::fs::{openat, Mode, OFlags, CWD}; + + let components = path + .strip_prefix(Path::new("/")) + .ok() + .map(|relative| relative.components().collect::>()) + .unwrap_or_default(); + if components.is_empty() + || components + .iter() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Ok(false); + } + let mut directory = match openat( + CWD, + Path::new("/"), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) { + Ok(directory) => directory, + Err(_) => return Ok(false), + }; + for component in &components[..components.len() - 1] { + let Component::Normal(name) = component else { + return Ok(false); + }; + directory = match openat( + &directory, + Path::new(name), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) { + Ok(directory) => directory, + Err(_) => return Ok(false), + }; + } + let Component::Normal(leaf) = components[components.len() - 1] else { + return Ok(false); + }; + let named = match openat( + &directory, + Path::new(leaf), + OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) { + Ok(named) => fs::File::from(named), + Err(_) => return Ok(false), + }; + let named = named.metadata().map_err(Error::Io)?; + Ok(named.is_file() + && metadata_identity(Some(&named), path)? == metadata_identity(Some(expected), path)?) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn unsafe_open_failure_state( + directory: Fd, + name: &Path, + component_path: &Path, + open_error: rustix::io::Errno, +) -> Result { + use rustix::fs::{readlinkat, statat, AtFlags, FileType}; + + let stat = match statat(&directory, name, AtFlags::SYMLINK_NOFOLLOW) { + Ok(stat) => stat, + Err(err) if err == rustix::io::Errno::NOENT => { + if open_error == rustix::io::Errno::NOENT { + return Ok(NoFollowFileState::missing()); + } + return Err(Error::Io(open_error.into())); + } + Err(err) => return Err(Error::Io(err.into())), + }; + let file_type = FileType::from_raw_mode(stat.st_mode); + let mut identity = format!( + "unsafe-component-v1:path={};kind={file_type:?};mode={};dev={};ino={};len={};uid={};gid={};", + hex::encode(os_str_bytes(component_path.as_os_str())), + stat.st_mode, + stat.st_dev, + stat.st_ino, + stat.st_size, + stat.st_uid, + stat.st_gid, + ) + .into_bytes(); + identity.extend_from_slice( + format!( + "mtime={};mtime_nsec={};ctime={};ctime_nsec={};", + stat.st_mtime, stat.st_mtime_nsec, stat.st_ctime, stat.st_ctime_nsec, + ) + .as_bytes(), + ); + if file_type == FileType::Symlink { + let target = readlinkat(&directory, name, Vec::new()).map_err(|err| { + Error::InvalidInput(format!( + "policy dependency component `{}` changed while its symlink identity was read: {err}", + component_path.display() + )) + })?; + identity.extend_from_slice(b"target="); + identity.extend_from_slice(hex::encode(target.as_bytes()).as_bytes()); + identity.push(b';'); + } + Ok(NoFollowFileState::unsafe_component(identity)) +} + +fn unsafe_component_path_from_metadata_identity(identity: &[u8]) -> Option { + let encoded = identity.strip_prefix(b"unsafe-component-v1:path=")?; + let encoded = encoded.split(|byte| *byte == b';').next()?; + let bytes = hex::decode(encoded).ok()?; + Some(PathBuf::from(os_string_from_bytes(&bytes))) +} + +fn metadata_identity(metadata: Option<&fs::Metadata>, _path: &Path) -> Result> { + let Some(metadata) = metadata else { + return Ok(b"missing-v1".to_vec()); + }; + let kind = if metadata.file_type().is_symlink() { + "symlink" + } else if metadata.is_file() { + "file" + } else if metadata.is_dir() { + "directory" + } else { + "other" + }; + let mut identity = format!("kind={kind};len={};", metadata.len()).into_bytes(); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + identity.extend_from_slice( + format!( + "mode={};dev={};ino={};mtime={};mtime_nsec={};ctime={};ctime_nsec={};", + metadata.mode(), + metadata.dev(), + metadata.ino(), + metadata.mtime(), + metadata.mtime_nsec(), + metadata.ctime(), + metadata.ctime_nsec() + ) + .as_bytes(), + ); + } + Ok(identity) +} + +fn digest(bytes: &[u8]) -> [u8; 32] { + Sha256::digest(bytes).into() +} + +fn validate_manifest_canonical(manifest: &PolicyManifest) -> Result<()> { + let mut identities = BTreeSet::new(); + for dependency in &manifest.dependencies { + if dependency.identity.is_empty() + || dependency.generation != manifest.generation + || !identities.insert((dependency.kind, dependency.identity.clone())) + { + return Err(Error::Corrupt( + "duplicate or non-canonical policy dependency identity".into(), + )); + } + if dependency_is_file(dependency) { + let path = dependency_identity_path(&dependency.identity) + .ok_or_else(|| Error::Corrupt("invalid policy path identity".into()))?; + if !path.is_absolute() + || lexical_normalize(&path) != path + || dependency_path_identity(&path) != dependency.identity + { + return Err(Error::Corrupt("non-canonical policy path identity".into())); + } + } + } + Ok(()) +} + +fn policy_fingerprint(dependencies: &[PolicyDependency]) -> Result<[u8; 32]> { + let mut ordered = dependencies.iter().collect::>(); + ordered.sort_by_key(|dependency| (dependency.kind, dependency.identity.as_str())); + let mut hash = Sha256::new(); + hash.update(b"trail-policy-fingerprint-v2"); + hash.update((ordered.len() as u64).to_be_bytes()); + for dependency in ordered { + for field in [ + dependency.kind.as_str().as_bytes(), + dependency.identity.as_bytes(), + dependency.content_identity.as_slice(), + dependency.metadata_identity.as_slice(), + ] { + let len = u64::try_from(field.len()) + .map_err(|_| Error::InvalidInput("policy fingerprint field too large".into()))?; + hash.update(len.to_be_bytes()); + hash.update(field); + } + } + Ok(hash.finalize().into()) +} + +fn load_policy_manifest( + conn: &Connection, + expected: &ExpectedScope, +) -> Result> { + let scope_id = expected.scope_id.to_text(); + let mut statement = conn.prepare( + "SELECT dependency_identity, dependency_kind, content_identity, metadata_identity, + observable, generation, last_source_sequence + FROM changed_path_policy_dependencies + WHERE scope_id=?1 AND generation=?2 + ORDER BY dependency_kind COLLATE BINARY, dependency_identity COLLATE BINARY", + )?; + let mut rows = statement.query(params![scope_id, expected.policy_generation as i64])?; + let mut dependencies = Vec::new(); + while let Some(row) = rows.next()? { + let content = row.get::<_, Vec>(2)?; + let content_identity: [u8; 32] = content.try_into().map_err(|content: Vec| { + Error::Corrupt(format!( + "changed-path policy content identity has {} bytes; expected 32", + content.len() + )) + })?; + dependencies.push(PolicyDependency { + identity: row.get(0)?, + kind: PolicyDependencyKind::parse(&row.get::<_, String>(1)?)?, + content_identity, + metadata_identity: row.get(3)?, + observable: row.get::<_, i64>(4)? != 0, + generation: row.get::<_, i64>(5)?.try_into().map_err(|_| { + Error::Corrupt("negative changed-path policy generation".to_string()) + })?, + last_source_sequence: row.get::<_, i64>(6)?.try_into().map_err(|_| { + Error::Corrupt("negative changed-path policy source sequence".to_string()) + })?, + }); + } + if dependencies.is_empty() { + Ok(None) + } else { + Ok(Some(PolicyManifest { + dependencies, + generation: expected.policy_generation, + rule_sources: Vec::new(), + })) + } +} + +fn persist_policy_manifest_and_stale( + conn: &Connection, + expected: &ExpectedScope, + manifest: &PolicyManifest, +) -> Result<()> { + conn.execute_batch("SAVEPOINT changed_path_policy_manifest;")?; + let result = (|| -> Result<()> { + policy_scope_cas_guard(conn, expected)?; + persist_policy_manifest_rows(conn, expected, manifest)?; + mark_policy_stale_guarded(conn, expected, "policy_observer_cut_unavailable")?; + Ok(()) + })(); + match result { + Ok(()) => { + conn.execute_batch("RELEASE changed_path_policy_manifest;")?; + Ok(()) + } + Err(err) => { + let _ = conn.execute_batch( + "ROLLBACK TO changed_path_policy_manifest; RELEASE changed_path_policy_manifest;", + ); + Err(err) + } + } +} + +fn policy_scope_cas_guard(conn: &Connection, expected: &ExpectedScope) -> Result<()> { + let changed = conn.execute( + "UPDATE changed_path_scopes SET updated_at=updated_at + WHERE scope_id=?1 AND epoch=?2 AND ref_name=?3 AND ref_generation=?4 + AND baseline_root_id=?5 AND policy_fingerprint=?6 + AND policy_dependency_generation=?7 + AND filesystem_identity=?8 AND provider_identity=?9", + params![ + expected.scope_id.to_text(), + expected.epoch as i64, + expected.ref_name, + expected.ref_generation as i64, + expected.baseline_root.0, + hex::encode(expected.policy_fingerprint), + expected.policy_generation as i64, + hex::encode(&expected.filesystem_identity), + hex::encode(&expected.provider_identity), + ], + )?; + if changed == 1 { + Ok(()) + } else { + Err(policy_stale_cas_error(expected)) + } +} + +fn persist_policy_manifest_rows( + conn: &Connection, + expected: &ExpectedScope, + manifest: &PolicyManifest, +) -> Result<()> { + validate_manifest_canonical(manifest)?; + conn.execute( + "DELETE FROM changed_path_policy_dependencies WHERE scope_id=?1", + [expected.scope_id.to_text()], + )?; + let now = crate::db::util::now_ts(); + let mut insert = conn.prepare( + "INSERT INTO changed_path_policy_dependencies( + scope_id, dependency_identity, dependency_kind, content_identity, + metadata_identity, observable, generation, last_source_sequence, + created_at, updated_at + ) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?9)", + )?; + for dependency in &manifest.dependencies { + insert.execute(params![ + expected.scope_id.to_text(), + dependency.identity, + dependency.kind.as_str(), + dependency.content_identity.as_slice(), + dependency.metadata_identity, + i64::from(dependency.observable), + dependency.generation as i64, + dependency.last_source_sequence as i64, + now, + ])?; + } + Ok(()) +} + +fn mark_policy_stale_guarded( + conn: &Connection, + expected: &ExpectedScope, + reason: &str, +) -> Result<()> { + let changed = conn.execute( + "UPDATE changed_path_scopes + SET trust_state='stale_baseline', trust_reason=?1, + continuity_generation=continuity_generation+1, updated_at=?2 + WHERE scope_id=?3 AND epoch=?4 AND ref_name=?5 AND ref_generation=?6 + AND baseline_root_id=?7 AND policy_fingerprint=?8 + AND policy_dependency_generation=?9 + AND filesystem_identity=?10 AND provider_identity=?11", + params![ + reason, + crate::db::util::now_ts(), + expected.scope_id.to_text(), + expected.epoch as i64, + expected.ref_name, + expected.ref_generation as i64, + expected.baseline_root.0, + hex::encode(expected.policy_fingerprint), + expected.policy_generation as i64, + hex::encode(&expected.filesystem_identity), + hex::encode(&expected.provider_identity), + ], + )?; + if changed == 1 { + Ok(()) + } else { + Err(policy_stale_cas_error(expected)) + } +} + +fn policy_stale_cas_error(expected: &ExpectedScope) -> Error { + Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".to_string(), + reason: "policy_full_expected_scope_cas_mismatch".to_string(), + command: "trail index reconcile".to_string(), + } +} + +pub(crate) fn dependency_path_identity(path: &Path) -> String { + format!("path:{}", hex::encode(os_str_bytes(path.as_os_str()))) +} + +fn dependency_path_identity_with_case(path: &Path, _case_sensitive: bool) -> String { + dependency_path_identity(&lexical_normalize(path)) +} + +fn dependency_identity_path(identity: &str) -> Option { + let bytes = hex::decode(identity.strip_prefix("path:")?).ok()?; + Some(PathBuf::from(os_string_from_bytes(&bytes))) +} + +#[cfg(unix)] +fn os_str_bytes(value: &OsStr) -> Vec { + use std::os::unix::ffi::OsStrExt; + value.as_bytes().to_vec() +} + +#[cfg(not(unix))] +fn os_str_bytes(value: &OsStr) -> Vec { + value.to_string_lossy().as_bytes().to_vec() +} + +#[cfg(unix)] +fn os_string_from_bytes(value: &[u8]) -> OsString { + use std::os::unix::ffi::OsStringExt; + OsString::from_vec(value.to_vec()) +} + +#[cfg(not(unix))] +fn os_string_from_bytes(value: &[u8]) -> OsString { + OsString::from(String::from_utf8_lossy(value).into_owned()) +} + +fn lexical_normalize(path: &Path) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + if !normalized.pop() { + normalized.push(component.as_os_str()); + } + } + other => normalized.push(other.as_os_str()), + } + } + normalized +} + +fn normalized_path_key(path: &Path, case_sensitive: bool) -> String { + let mut key = lexical_normalize(path) + .to_string_lossy() + .replace('\\', "/") + .nfc() + .collect::(); + if !case_sensitive { + key = key.to_lowercase(); + } + key +} + +const fn platform_default_case_sensitive() -> bool { + !cfg!(any(target_os = "windows", target_os = "macos")) +} diff --git a/trail/src/db/change_ledger/projection.rs b/trail/src/db/change_ledger/projection.rs new file mode 100644 index 0000000..fc87df3 --- /dev/null +++ b/trail/src/db/change_ledger/projection.rs @@ -0,0 +1,954 @@ +//! Shared publication state machines for controlled filesystem producers. +//! +//! Applying filesystem bytes and proving their observer cut happens before +//! these functions are entered. Publication is deliberately one SQLite +//! transaction: an indexed operation, ref generation, lane head, ledger +//! baseline, intent terminal state, and acknowledgement can never become +//! visible at different cuts. Ref and marker mirrors are repaired only after +//! commit by the caller. + +use rusqlite::{params, Transaction, TransactionBehavior}; +use std::collections::BTreeSet; +use std::time::Duration; + +use super::intent::{ + authoritative_ref_matches_target, exact_scope_guard, load_intent, stage_intent_evidence, + IntentEvidence, IntentId, IntentProducer, IntentState, IntentTarget, +}; +use super::types::{trail_atomic_temp_target, trail_case_probe_token}; +use super::{ + BaselineIdentity, EvidenceSource, ExpectedScope, ObservedRecordCut, QualifiedFilesystemProof, +}; +use super::{EvidenceAcknowledgementToken, EvidenceFlags, EvidenceRowKind, LedgerPath}; +use crate::db::util::now_ts; +use crate::error::{Error, Result}; +use crate::model::{Operation, RefRecord}; +use crate::{ChangeId, ObjectId}; + +#[derive(Clone, Debug)] +pub(crate) struct ProjectionPublication { + pub(crate) operation_id: ObjectId, + pub(crate) baseline: BaselineIdentity, +} + +#[derive(Clone, Debug)] +pub(crate) struct WorkspaceViewCheckpointPublication { + pub(crate) view_id: String, + pub(crate) expected_generation: u64, + pub(crate) journal_sequence: u64, + pub(crate) next_generation: u64, + pub(crate) journal_qualified: bool, +} + +fn acquire_projection_publication_lock(db: &crate::Trail) -> Result { + // The exclusion is activated by every caller immediately before this + // handoff. New observer durability writers are therefore blocked; a + // writer that acquired the lock just before exclusion gets a short, + // bounded interval to finish and release it. External or stuck ownership + // still fails closed at the deadline. + crate::Trail::with_write_lock_wait(Duration::from_secs(2), || db.acquire_write_lock()) +} + +pub(crate) enum ProjectionAlignmentMode { + Aligned, + RetainDirty { target: IntentTarget }, +} + +pub(crate) enum RefAdvancingProjectionMode<'a> { + ControlledIntent, + UnalignedReconcileRequired { + reason: &'a str, + }, + LayeredUpdateReconcileRequired { + reason: &'a str, + lane_base_change: &'a ChangeId, + lane_base_root: &'a ObjectId, + view_id: &'a str, + checkpoint_sequence: u64, + }, + ObservedCut { + cut: &'a ObservedRecordCut, + acknowledge_complete_prefixes: bool, + }, +} + +/// Complete projection-only protocol: recover and prepare against the exact +/// unchanged ref/root, apply and durably verify bytes, qualify the final +/// observer cut, publish terminal intent state atomically, then repair the +/// compact marker mirror. The apply closure must include filesystem sync and +/// pinned verification before returning its proof. +pub(crate) fn run_projection_alignment( + db: &mut crate::Trail, + expected: &ExpectedScope, + producer: IntentProducer, + evidence: &IntentEvidence, + mode: ProjectionAlignmentMode, + apply_sync_verify_and_fence: A, + repair_marker_after_commit: M, +) -> Result +where + A: FnOnce(&mut crate::Trail, &IntentId) -> Result, + M: FnOnce(&crate::Trail) -> Result<()>, +{ + if !matches!( + producer, + IntentProducer::Checkout | IntentProducer::LaneSync | IntentProducer::Materialize + ) { + return Err(Error::InvalidInput( + "ref-advancing producer cannot use projection alignment".into(), + )); + } + db.changed_path_ledger().recover_scope(expected)?; + let current: (String, ObjectId) = db.conn.query_row( + "SELECT change_id,root_id FROM refs WHERE name=?1 AND generation=?2", + params![ + expected.ref_name, + sql_u64(expected.ref_generation, "ref generation")? + ], + |row| Ok((row.get(0)?, ObjectId(row.get(1)?))), + )?; + if current.1 != expected.baseline_root { + return Err(Error::StaleBranch(expected.ref_name.clone())); + } + let target = match mode { + ProjectionAlignmentMode::Aligned => IntentTarget { + change_id: crate::ChangeId(current.0), + root_id: current.1, + operation_id: None, + }, + ProjectionAlignmentMode::RetainDirty { target } => target, + }; + let retain_dirty = target.root_id != expected.baseline_root; + let intent = super::prepare_intent( + &db.changed_path_ledger(), + expected, + producer, + &target, + evidence, + )?; + let proof = apply_sync_verify_and_fence(db, &intent)?; + let _observer_exclusion = crate::db::begin_authorized_observer_write_exclusion(&db.db_dir); + let _write_lock = acquire_projection_publication_lock(db)?; + super::mark_filesystem_applied(&db.changed_path_ledger(), expected, &intent, &proof)?; + if retain_dirty { + publish_dirty_alignment_in_transaction(db, expected, &intent)?; + } else { + publish_alignment_in_transaction(db, expected, &intent)?; + } + repair_marker_after_commit(db).map_err(|error| Error::CommittedRepairRequired { + operation: intent.0.clone(), + repair: "projection marker".into(), + reason: error.to_string(), + })?; + Ok(intent) +} + +fn publish_dirty_alignment_in_transaction( + db: &crate::Trail, + expected: &ExpectedScope, + intent_id: &IntentId, +) -> Result<()> { + let conn = &db.conn; + let tx = Transaction::new_unchecked(conn, TransactionBehavior::Immediate)?; + exact_scope_guard(&tx, expected, true)?; + let intent = load_intent(&tx, intent_id)?.ok_or_else(|| { + Error::InvalidInput(format!("unknown changed-path intent `{}`", intent_id.0)) + })?; + if intent.scope_id != expected.scope_id.to_text() + || intent.state != IntentState::FilesystemApplied + || intent.target.operation_id.is_some() + || intent.target.root_id == expected.baseline_root + { + return Err(Error::Conflict(format!( + "intent `{}` is not a dirty projection", + intent_id.0 + ))); + } + let proof = intent + .verified_cut + .as_ref() + .ok_or_else(|| Error::Conflict(format!("intent `{}` has no qualified cut", intent_id.0)))?; + super::intent::validate_qualified_filesystem_proof( + &tx, + ledger_database_path(&tx)?, + expected, + &intent, + proof, + )?; + acknowledge_controlled_observer_evidence_in_transaction(db, &tx, expected, &intent, proof)?; + // Restage intent evidence after removing only the verified controlled + // observer rows. These rows deliberately remain pending against the + // unchanged baseline, making checkout-to-another-root dirty. + stage_intent_evidence(&tx, &intent)?; + terminal_intent(&tx, intent_id)?; + tx.commit()?; + super::intent::durable_intent_barrier(conn) +} + +/// Complete ref-advancing protocol: prebuild immutable operation/object, +/// recover and durably prepare, apply+sync+verify under a qualified observer +/// interval, and atomically publish operation/ref/lane/ledger/ack/terminal +/// state. Ref and marker mirrors are repaired only after SQLite commit. +pub(crate) fn run_ref_advancing_projection( + db: &mut crate::Trail, + expected: &ExpectedScope, + expected_ref: &RefRecord, + lane_id: &str, + producer: IntentProducer, + operation: &Operation, + evidence: &IntentEvidence, + mode: RefAdvancingProjectionMode<'_>, + apply_sync_verify_and_fence: A, + repair_marker_after_commit: M, +) -> Result +where + A: FnOnce(&mut crate::Trail, &IntentId) -> Result, + M: FnOnce(&crate::Trail, &ProjectionPublication) -> Result<()>, +{ + if matches!( + mode, + RefAdvancingProjectionMode::UnalignedReconcileRequired { .. } + | RefAdvancingProjectionMode::LayeredUpdateReconcileRequired { .. } + ) { + let reason = match &mode { + RefAdvancingProjectionMode::UnalignedReconcileRequired { reason } + | RefAdvancingProjectionMode::LayeredUpdateReconcileRequired { reason, .. } => *reason, + _ => unreachable!(), + }; + let _observer_exclusion = crate::db::begin_authorized_observer_write_exclusion(&db.db_dir); + let _write_lock = acquire_projection_publication_lock(db)?; + if operation.before_root.as_ref() != Some(&expected_ref.root_id) + || operation.parents.first() != Some(&expected_ref.change_id) + || operation.branch != expected_ref.name + { + return Err(Error::StaleBranch(expected_ref.name.clone())); + } + let (operation, operation_id) = db.store_operation_object_unindexed(operation)?; + let tx = Transaction::new_unchecked(&db.conn, TransactionBehavior::Immediate)?; + db.index_operation_in_transaction(&operation, &operation_id)?; + db.advance_ref_cas_in_transaction( + expected_ref, + &operation.change_id, + &operation.after_root, + &operation_id, + )?; + let lane_changed = tx.execute( + "UPDATE lane_branches SET head_change=?1,head_root=?2,updated_at=?3 + WHERE lane_id=?4 AND ref_name=?5 AND head_change=?6 AND head_root=?7", + params![ + operation.change_id.0, + operation.after_root.0, + now_ts(), + lane_id, + expected_ref.name, + expected_ref.change_id.0, + expected_ref.root_id.0, + ], + )?; + if lane_changed != 1 { + return Err(Error::StaleBranch(expected_ref.name.clone())); + } + if let RefAdvancingProjectionMode::LayeredUpdateReconcileRequired { + lane_base_change, + lane_base_root, + view_id, + checkpoint_sequence, + .. + } = &mode + { + let base_changed = tx.execute( + "UPDATE lane_branches + SET base_change=?1,base_root=?2,status='active',updated_at=?3 + WHERE lane_id=?4 AND head_change=?5 AND head_root=?6", + params![ + lane_base_change.0, + lane_base_root.0, + now_ts(), + lane_id, + operation.change_id.0, + operation.after_root.0, + ], + )?; + let view_changed = tx.execute( + "UPDATE workspace_views + SET base_change=?1,base_root=?2,generation=generation+1, + checkpoint_seq=?3,checkpoint_root=?2,updated_at=?4 + WHERE view_id=?5", + params![ + operation.change_id.0, + operation.after_root.0, + i64::try_from(*checkpoint_sequence).map_err(|_| { + Error::InvalidInput("workspace checkpoint sequence overflow".into()) + })?, + now_ts(), + view_id, + ], + )?; + if base_changed != 1 || view_changed != 1 { + return Err(Error::StaleBranch(expected_ref.name.clone())); + } + } + tx.execute( + "UPDATE changed_path_scopes + SET trust_state='stale_baseline',trust_reason=?1,updated_at=?2 + WHERE scope_id=?3 AND retired_at IS NULL", + params![reason, now_ts(), expected.scope_id.to_text()], + )?; + tx.commit()?; + db.repair_ref_mirror( + expected_ref, + &operation.change_id, + &operation.after_root, + &operation_id, + ) + .map_err(|error| Error::CommittedRepairRequired { + operation: operation_id.0.clone(), + repair: "ref mirror".into(), + reason: error.to_string(), + })?; + let publication = ProjectionPublication { + operation_id, + baseline: BaselineIdentity { + ref_name: expected_ref.name.clone(), + ref_generation: u64::try_from(expected_ref.generation + 1) + .map_err(|_| Error::InvalidInput("ref generation overflow".into()))?, + change_id: operation.change_id.clone(), + root_id: operation.after_root.clone(), + }, + }; + repair_marker_after_commit(db, &publication).map_err(|error| { + Error::CommittedRepairRequired { + operation: publication.operation_id.0.clone(), + repair: "projection marker/runtime".into(), + reason: error.to_string(), + } + })?; + return Ok(publication); + } + if let RefAdvancingProjectionMode::ObservedCut { + cut, + acknowledge_complete_prefixes, + } = &mode + { + if producer != IntentProducer::ObservedCheckpoint || cut.expected != *expected { + return Err(Error::InvalidInput( + "observed ref projection requires its exact fenced scope".into(), + )); + } + let _observer_exclusion = crate::db::begin_authorized_observer_write_exclusion(&db.db_dir); + let _write_lock = acquire_projection_publication_lock(db)?; + let operation_id = db.commit_observed_record( + operation, + expected_ref, + cut, + *acknowledge_complete_prefixes, + Some(lane_id), + )?; + let publication = ProjectionPublication { + operation_id, + baseline: BaselineIdentity { + ref_name: expected_ref.name.clone(), + ref_generation: u64::try_from(expected_ref.generation + 1) + .map_err(|_| Error::InvalidInput("ref generation overflow".into()))?, + change_id: operation.change_id.clone(), + root_id: operation.after_root.clone(), + }, + }; + repair_marker_after_commit(db, &publication).map_err(|error| { + Error::CommittedRepairRequired { + operation: publication.operation_id.0.clone(), + repair: "projection marker/runtime".into(), + reason: error.to_string(), + } + })?; + return Ok(publication); + } + if !matches!( + producer, + IntentProducer::StructuredPatchProjection + | IntentProducer::RestoreProjection + | IntentProducer::CowPublication + | IntentProducer::ObservedCheckpoint + ) { + return Err(Error::InvalidInput( + "projection-only producer cannot advance a ref".into(), + )); + } + db.changed_path_ledger().recover_scope(expected)?; + let (operation, operation_id) = db.store_operation_object_unindexed(operation)?; + let target = IntentTarget { + change_id: operation.change_id.clone(), + root_id: operation.after_root.clone(), + operation_id: Some(operation_id.clone()), + }; + let intent = super::prepare_intent( + &db.changed_path_ledger(), + expected, + producer, + &target, + evidence, + )?; + let proof = apply_sync_verify_and_fence(db, &intent)?; + let _observer_exclusion = crate::db::begin_authorized_observer_write_exclusion(&db.db_dir); + let _write_lock = acquire_projection_publication_lock(db)?; + super::mark_filesystem_applied(&db.changed_path_ledger(), expected, &intent, &proof)?; + let publication = publish_ref_advancing_projection( + db, + expected, + expected_ref, + lane_id, + &intent, + &operation, + &operation_id, + )?; + db.repair_ref_mirror( + expected_ref, + &operation.change_id, + &operation.after_root, + &operation_id, + ) + .map_err(|error| Error::CommittedRepairRequired { + operation: publication.operation_id.0.clone(), + repair: "ref mirror".into(), + reason: error.to_string(), + })?; + repair_marker_after_commit(db, &publication).map_err(|error| { + Error::CommittedRepairRequired { + operation: publication.operation_id.0.clone(), + repair: "projection marker/runtime".into(), + reason: error.to_string(), + } + })?; + Ok(publication) +} + +/// The non-authoritative fallback used while ledger command authority remains +/// disabled. It still removes the historic ref-before-lane-row visibility +/// window. Task 15 selects the intent-qualified publisher above once native +/// lane observers are platform-qualified. +pub(crate) fn commit_lane_operation_atomic( + db: &crate::Trail, + expected_ref: &RefRecord, + lane_id: &str, + operation: &Operation, + workspace_checkpoint: Option<&WorkspaceViewCheckpointPublication>, +) -> Result { + if operation.before_root.as_ref() != Some(&expected_ref.root_id) + || operation.parents.first() != Some(&expected_ref.change_id) + || operation.branch != expected_ref.name + { + return Err(Error::StaleBranch(expected_ref.name.clone())); + } + let (operation, operation_id) = db.store_operation_object_unindexed(operation)?; + let tx = Transaction::new_unchecked(&db.conn, TransactionBehavior::Immediate)?; + db.index_operation_in_transaction(&operation, &operation_id)?; + db.advance_ref_cas_in_transaction( + expected_ref, + &operation.change_id, + &operation.after_root, + &operation_id, + )?; + let changed = tx.execute( + "UPDATE lane_branches SET head_change=?1,head_root=?2,updated_at=?3 + WHERE lane_id=?4 AND ref_name=?5 AND head_change=?6 AND head_root=?7", + params![ + operation.change_id.0, + operation.after_root.0, + now_ts(), + lane_id, + expected_ref.name, + expected_ref.change_id.0, + expected_ref.root_id.0, + ], + )?; + if changed != 1 { + return Err(Error::StaleBranch(expected_ref.name.clone())); + } + if let Some(checkpoint) = workspace_checkpoint { + let view_changed = tx.execute( + "UPDATE workspace_views + SET checkpoint_seq=?1,checkpoint_root=?2,generation=?3,updated_at=?4 + WHERE view_id=?5 AND lane_id=?6 AND generation=?7", + params![ + sql_u64(checkpoint.journal_sequence, "workspace journal sequence")?, + operation.after_root.0, + sql_u64(checkpoint.next_generation, "workspace view generation")?, + now_ts(), + checkpoint.view_id, + lane_id, + sql_u64(checkpoint.expected_generation, "workspace view generation")?, + ], + )?; + if view_changed != 1 { + return Err(Error::StaleBranch(expected_ref.name.clone())); + } + } + tx.commit()?; + db.repair_ref_mirror( + expected_ref, + &operation.change_id, + &operation.after_root, + &operation_id, + ) + .map_err(|error| Error::CommittedRepairRequired { + operation: operation_id.0.clone(), + repair: "lane operation ref mirror".into(), + reason: error.to_string(), + })?; + Ok(operation_id) +} + +/// Publish an unchanged-ref projection after the filesystem proof has moved +/// the intent to `filesystem_applied`. This is used by checkout alignment, +/// materialization, hydration, and sync. The target must be the exact current +/// authoritative ref/root; projection-only work never fabricates history. +pub(crate) fn publish_alignment_in_transaction( + db: &crate::Trail, + expected: &ExpectedScope, + intent_id: &IntentId, +) -> Result<()> { + let conn = &db.conn; + let tx = Transaction::new_unchecked(conn, TransactionBehavior::Immediate)?; + exact_scope_guard(&tx, expected, true)?; + let intent = load_intent(&tx, intent_id)?.ok_or_else(|| { + Error::InvalidInput(format!("unknown changed-path intent `{}`", intent_id.0)) + })?; + if intent.scope_id != expected.scope_id.to_text() + || intent.state != IntentState::FilesystemApplied + || intent.target.operation_id.is_some() + || intent.target.change_id != intent.expected_change_id + || intent.target.root_id != intent.expected_root_id + || intent.target.root_id != expected.baseline_root + { + return Err(Error::Conflict(format!( + "intent `{}` is not an unchanged-ref filesystem alignment", + intent_id.0 + ))); + } + let current_ref: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM refs WHERE name=?1 AND generation=?2 AND change_id=?3 AND root_id=?4)", + params![ + expected.ref_name, + sql_u64(expected.ref_generation, "ref generation")?, + intent.expected_change_id.0, + intent.expected_root_id.0, + ], + |row| row.get(0), + )?; + if !current_ref { + return Err(Error::StaleBranch(expected.ref_name.clone())); + } + let proof = intent.verified_cut.as_ref().ok_or_else(|| { + Error::Conflict(format!( + "intent `{}` has no qualified final cut", + intent_id.0 + )) + })?; + super::intent::validate_qualified_filesystem_proof( + &tx, + ledger_database_path(&tx)?, + expected, + &intent, + proof, + )?; + let cut_matches: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM changed_path_scopes WHERE scope_id=?1 AND epoch=?2 + AND durable_offset=?3 AND folded_offset=?3)", + params![ + expected.scope_id.to_text(), + sql_u64(expected.epoch, "scope epoch")?, + sql_u64(proof.publication_cut.durable_offset, "provider cut")?, + ], + |row| row.get(0), + )?; + if !cut_matches { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "alignment lost the exact qualified provider cut".into(), + command: "trail status".into(), + }); + } + acknowledge_controlled_observer_evidence_in_transaction(db, &tx, expected, &intent, proof)?; + stage_intent_evidence(&tx, &intent)?; + acknowledge_intent_owned_evidence_in_transaction(&tx, expected, &intent, proof)?; + terminal_intent(&tx, intent_id)?; + tx.commit()?; + super::intent::durable_intent_barrier(conn) +} + +/// Atomically publish a ref-advancing projection. The immutable operation +/// object must already exist but must not yet be indexed. On failure it stays +/// unreachable and is reclaimed by ordinary object GC. +pub(crate) fn publish_ref_advancing_projection( + db: &crate::Trail, + expected: &ExpectedScope, + expected_ref: &RefRecord, + lane_id: &str, + intent_id: &IntentId, + operation: &Operation, + operation_id: &ObjectId, +) -> Result { + if operation.before_root.as_ref() != Some(&expected.baseline_root) + || operation.parents.first() != Some(&expected_ref.change_id) + || expected_ref.name != expected.ref_name + || u64::try_from(expected_ref.generation).ok() != Some(expected.ref_generation) + { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "ref-advancing projection does not match its prepared scope".into(), + command: "trail status".into(), + }); + } + let tx = Transaction::new_unchecked(&db.conn, TransactionBehavior::Immediate)?; + exact_scope_guard(&tx, expected, true)?; + let intent = load_intent(&tx, intent_id)?.ok_or_else(|| { + Error::InvalidInput(format!("unknown changed-path intent `{}`", intent_id.0)) + })?; + if intent.scope_id != expected.scope_id.to_text() + || intent.state != IntentState::FilesystemApplied + || intent.target.change_id != operation.change_id + || intent.target.root_id != operation.after_root + || intent.target.operation_id.as_ref() != Some(operation_id) + { + return Err(Error::Conflict(format!( + "intent `{}` target is not the immutable projection operation", + intent_id.0 + ))); + } + let proof = intent.verified_cut.as_ref().ok_or_else(|| { + Error::Conflict(format!( + "intent `{}` has no qualified final cut", + intent_id.0 + )) + })?; + super::intent::validate_qualified_filesystem_proof( + &tx, + ledger_database_path(&tx)?, + expected, + &intent, + proof, + )?; + + db.index_operation_in_transaction(operation, operation_id)?; + db.advance_ref_cas_in_transaction( + expected_ref, + &operation.change_id, + &operation.after_root, + operation_id, + )?; + let lane_changed = tx.execute( + "UPDATE lane_branches SET head_change=?1,head_root=?2,updated_at=?3 + WHERE lane_id=?4 AND ref_name=?5 AND head_change=?6 AND head_root=?7", + params![ + operation.change_id.0, + operation.after_root.0, + now_ts(), + lane_id, + expected_ref.name, + expected_ref.change_id.0, + expected_ref.root_id.0, + ], + )?; + if lane_changed != 1 { + return Err(Error::StaleBranch(expected_ref.name.clone())); + } + if !authoritative_ref_matches_target(&tx, &intent)? { + return Err(Error::StaleBranch(expected_ref.name.clone())); + } + acknowledge_controlled_observer_evidence_in_transaction(db, &tx, expected, &intent, proof)?; + stage_intent_evidence(&tx, &intent)?; + acknowledge_intent_owned_evidence_in_transaction(&tx, expected, &intent, proof)?; + let next_generation = expected_ref + .generation + .checked_add(1) + .ok_or_else(|| Error::InvalidInput("ref generation overflow".into()))?; + let scope_changed = tx.execute( + "UPDATE changed_path_scopes SET ref_generation=?1,change_id=?2,baseline_root_id=?3, + updated_at=?4 + WHERE scope_id=?5 AND epoch=?6 AND ref_name=?7 AND ref_generation=?8 + AND baseline_root_id=?9 AND policy_fingerprint=?10 + AND policy_dependency_generation=?11 AND filesystem_identity=?12 + AND provider_identity=?13 AND trust_state='trusted' + AND durable_offset=?14 AND folded_offset=?14", + params![ + next_generation, + operation.change_id.0, + operation.after_root.0, + now_ts(), + expected.scope_id.to_text(), + sql_u64(expected.epoch, "scope epoch")?, + expected.ref_name, + expected_ref.generation, + expected.baseline_root.0, + hex::encode(expected.policy_fingerprint), + sql_u64(expected.policy_generation, "policy generation")?, + hex::encode(&expected.filesystem_identity), + hex::encode(&expected.provider_identity), + sql_u64(proof.publication_cut.durable_offset, "provider cut")?, + ], + )?; + if scope_changed != 1 { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "projection publication lost the exact scope CAS".into(), + command: "trail status".into(), + }); + } + terminal_intent(&tx, intent_id)?; + tx.commit()?; + super::intent::durable_intent_barrier(&db.conn)?; + Ok(ProjectionPublication { + operation_id: operation_id.clone(), + baseline: BaselineIdentity { + ref_name: expected_ref.name.clone(), + ref_generation: u64::try_from(next_generation) + .map_err(|_| Error::InvalidInput("ref generation overflow".into()))?, + change_id: operation.change_id.clone(), + root_id: operation.after_root.clone(), + }, + }) +} + +/// Remove immutable observer-only evidence for intended paths only when the +/// entire row falls inside the authenticated controlled interval. Pre-existing, +/// mixed-source, and post-c1 rows remain pending. +fn acknowledge_controlled_observer_evidence_in_transaction( + db: &crate::Trail, + tx: &rusqlite::Connection, + expected: &ExpectedScope, + intent: &super::intent::PersistedIntent, + proof: &QualifiedFilesystemProof, +) -> Result<()> { + let mut tokens = Vec::new(); + let mut exact = tx.prepare( + "SELECT entry.normalized_path,entry.event_flags,entry.source_mask, + entry.first_sequence,entry.last_sequence,entry.provider_id, + entry.provider_sequence,entry.intent_id + FROM changed_path_entries entry + JOIN changed_path_intent_paths path + ON path.intent_id=?1 AND path.normalized_path=entry.normalized_path + WHERE entry.scope_id=?2 AND entry.source_mask=?3 + AND entry.first_sequence>=?4 AND entry.last_sequence<=?5 + AND entry.provider_sequence IS NOT NULL + AND (entry.event_flags & path.event_flags)!=0 + ORDER BY entry.normalized_path COLLATE BINARY", + )?; + let rows = exact.query_map( + params![ + intent.id.0, + expected.scope_id.to_text(), + EvidenceSource::Observer.mask(), + sql_u64(proof.start_sequence, "controlled start sequence")?, + sql_u64(proof.end_cut.sequence, "controlled end sequence")?, + ], + |row| acknowledgement_token(row, EvidenceRowKind::Exact), + )?; + tokens.extend(rows.collect::, _>>()?); + drop(exact); + let mut prefixes = tx.prepare( + "SELECT entry.normalized_prefix,entry.event_flags,entry.source_mask, + entry.first_sequence,entry.last_sequence,entry.provider_id, + entry.provider_sequence,entry.intent_id + FROM changed_path_prefixes entry + JOIN changed_path_intent_prefixes prefix + ON prefix.intent_id=?1 AND prefix.normalized_prefix=entry.normalized_prefix + WHERE entry.scope_id=?2 AND entry.source_mask=?3 + AND entry.first_sequence>=?4 AND entry.last_sequence<=?5 + AND entry.provider_sequence IS NOT NULL + AND (entry.event_flags & prefix.event_flags)!=0 + AND prefix.completeness_reason='provider_complete' + ORDER BY entry.normalized_prefix COLLATE BINARY", + )?; + let rows = prefixes.query_map( + params![ + intent.id.0, + expected.scope_id.to_text(), + EvidenceSource::Observer.mask(), + sql_u64(proof.start_sequence, "controlled start sequence")?, + sql_u64(proof.end_cut.sequence, "controlled end sequence")?, + ], + |row| acknowledgement_token(row, EvidenceRowKind::CompletePrefix), + )?; + tokens.extend(rows.collect::, _>>()?); + drop(prefixes); + let intended = tx + .prepare( + "SELECT normalized_path FROM changed_path_intent_paths + WHERE intent_id=?1 ORDER BY normalized_path COLLATE BINARY", + )? + .query_map([&intent.id.0], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + let mut internal = tx.prepare( + "SELECT normalized_path,event_flags,source_mask,first_sequence,last_sequence, + provider_id,provider_sequence,intent_id + FROM changed_path_entries + WHERE scope_id=?1 AND source_mask=?2 AND intent_id IS NULL + AND first_sequence>=?3 AND last_sequence<=?4 + AND provider_sequence IS NOT NULL + ORDER BY normalized_path COLLATE BINARY", + )?; + let rows = internal.query_map( + params![ + expected.scope_id.to_text(), + EvidenceSource::Observer.mask(), + sql_u64(proof.start_sequence, "controlled start sequence")?, + sql_u64(proof.end_cut.sequence, "controlled end sequence")?, + ], + |row| acknowledgement_token(row, EvidenceRowKind::Exact), + )?; + let mut internal_tokens = Vec::new(); + for token in rows { + let token = token?; + if trail_case_probe_token(&token) + || trail_atomic_temp_target(&token) + .as_ref() + .is_some_and(|target| intended.contains(target)) + { + internal_tokens.push(token); + } + } + drop(internal); + let baseline_internal = db.load_root_files_for_paths( + &expected.baseline_root, + &internal_tokens + .iter() + .map(|token| token.path.as_str().to_string()) + .collect::>(), + )?; + tokens.extend( + internal_tokens + .into_iter() + .filter(|token| !baseline_internal.contains_key(token.path.as_str())), + ); + tokens.sort_by(|left, right| left.path.as_str().cmp(right.path.as_str())); + tokens.dedup(); + super::ChangedPathLedger::acknowledge_immutable_tokens_in_transaction( + tx, + expected, + &proof.publication_cut, + proof.end_cut.sequence, + &tokens, + true, + )?; + Ok(()) +} + +/// Remove the immutable intent-only rows after controlled observer evidence +/// has been acknowledged. Any merge makes the row mixed-source and preserves +/// it for the next snapshot. +pub(super) fn acknowledge_intent_owned_evidence_in_transaction( + tx: &rusqlite::Connection, + expected: &ExpectedScope, + intent: &super::intent::PersistedIntent, + proof: &QualifiedFilesystemProof, +) -> Result<()> { + let mut tokens = Vec::new(); + let mut exact = tx.prepare( + "SELECT entry.normalized_path,entry.event_flags,entry.source_mask, + entry.first_sequence,entry.last_sequence,entry.provider_id, + entry.provider_sequence,entry.intent_id + FROM changed_path_entries entry + JOIN changed_path_intent_paths path + ON path.intent_id=?1 AND path.normalized_path=entry.normalized_path + WHERE entry.scope_id=?2 AND entry.source_mask=?3 AND entry.intent_id=?1 + AND entry.first_sequence=?4 AND entry.last_sequence=?4 + AND entry.provider_sequence IS NULL + AND (entry.event_flags & path.event_flags)=path.event_flags + ORDER BY entry.normalized_path COLLATE BINARY", + )?; + let rows = exact.query_map( + params![ + intent.id.0, + expected.scope_id.to_text(), + EvidenceSource::Intent.mask(), + sql_u64(proof.end_cut.sequence, "intent end sequence")?, + ], + |row| acknowledgement_token(row, EvidenceRowKind::Exact), + )?; + tokens.extend(rows.collect::, _>>()?); + drop(exact); + let mut prefixes = tx.prepare( + "SELECT entry.normalized_prefix,entry.event_flags,entry.source_mask, + entry.first_sequence,entry.last_sequence,entry.provider_id, + entry.provider_sequence,entry.intent_id + FROM changed_path_prefixes entry + JOIN changed_path_intent_prefixes prefix + ON prefix.intent_id=?1 AND prefix.normalized_prefix=entry.normalized_prefix + WHERE entry.scope_id=?2 AND entry.source_mask=?3 AND entry.intent_id=?1 + AND entry.first_sequence=?4 AND entry.last_sequence=?4 + AND entry.provider_sequence IS NULL + AND (entry.event_flags & prefix.event_flags)=prefix.event_flags + AND prefix.completeness_reason='provider_complete' + ORDER BY entry.normalized_prefix COLLATE BINARY", + )?; + let rows = prefixes.query_map( + params![ + intent.id.0, + expected.scope_id.to_text(), + EvidenceSource::Intent.mask(), + sql_u64(proof.end_cut.sequence, "intent end sequence")?, + ], + |row| acknowledgement_token(row, EvidenceRowKind::CompletePrefix), + )?; + tokens.extend(rows.collect::, _>>()?); + drop(prefixes); + super::ChangedPathLedger::acknowledge_immutable_tokens_in_transaction( + tx, + expected, + &proof.publication_cut, + proof.end_cut.sequence, + &tokens, + true, + )?; + Ok(()) +} + +fn acknowledgement_token( + row: &rusqlite::Row<'_>, + kind: EvidenceRowKind, +) -> rusqlite::Result { + let first = row.get::<_, i64>(3)?; + let last = row.get::<_, i64>(4)?; + let provider_sequence = row.get::<_, Option>(6)?; + Ok(EvidenceAcknowledgementToken { + kind, + path: LedgerPath(row.get(0)?), + flags: EvidenceFlags(row.get(1)?), + source_mask: row.get(2)?, + first_sequence: u64::try_from(first).unwrap_or(u64::MAX), + last_sequence: u64::try_from(last).unwrap_or(u64::MAX), + provider_id: row.get(5)?, + provider_sequence: provider_sequence.and_then(|value| u64::try_from(value).ok()), + intent_id: row.get(7)?, + }) +} + +fn terminal_intent(tx: &rusqlite::Connection, intent_id: &IntentId) -> Result<()> { + let changed = tx.execute( + "UPDATE changed_path_intents SET lifecycle_state='acknowledged',updated_at=?1 + WHERE intent_id=?2 AND lifecycle_state='filesystem_applied'", + params![now_ts(), intent_id.0], + )?; + if changed != 1 { + return Err(Error::Conflict( + "projection intent terminal transition raced".into(), + )); + } + Ok(()) +} + +fn sql_u64(value: u64, label: &str) -> Result { + value + .try_into() + .map_err(|_| Error::InvalidInput(format!("{label} exceeds SQLite INTEGER range"))) +} + +fn ledger_database_path(conn: &rusqlite::Connection) -> Result<&std::path::Path> { + let path = conn.path().ok_or_else(|| { + Error::InvalidInput("changed-path publication database path is unavailable".into()) + })?; + Ok(std::path::Path::new(path)) +} diff --git a/trail/src/db/change_ledger/reconcile.rs b/trail/src/db/change_ledger/reconcile.rs new file mode 100644 index 0000000..e6ce44b --- /dev/null +++ b/trail/src/db/change_ledger/reconcile.rs @@ -0,0 +1,5239 @@ +#[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] +use std::collections::HashMap; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::sync::atomic::{AtomicU64, Ordering}; +#[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] +use std::sync::{Mutex, OnceLock}; + +use rusqlite::{params, OptionalExtension, Transaction, TransactionBehavior}; +use serde::{Deserialize, Serialize}; + +use super::{ + raw_event_invalidates_policy, ChangedPathLedger, CompiledPolicy, EvidenceCut, EvidenceFlags, + ExpectedScope, LedgerPath, ObserverFence, QualifiedObserver, ScopeId, TrustState, +}; +use crate::db::storage::{ + PinnedWorktreeRoot, ReconciliationDirectory, ReconciliationFile, ReconciliationScanEntry, +}; +use crate::db::util::now_ts; +use crate::error::{Error, Result}; +use crate::model::{ChangeLedgerReconcileReport, FileEntry, FileKind}; +use crate::Trail; + +const STAGING_BATCH_ROWS: usize = 256; +const MAX_IDENTITY_RACE_RETRIES: u64 = 2; +const MAX_OBSERVER_SPOOL_EVENTS: u64 = 1_000_000; +const MAX_OBSERVER_SPOOL_BYTES: u64 = 256 * 1024 * 1024; +const MAX_OBSERVER_SPOOL_PATH_BYTES: usize = 1024 * 1024; +const OBSERVER_SPOOL_HEADER_BYTES: usize = 4 + 8 + 8; +// Do not linearize against a lease that can expire in the same instant as the +// final CAS and SQLite commit. +const MIN_PUBLICATION_LEASE_HORIZON_SECS: i64 = 5; +static NEXT_ATTEMPT_ID: AtomicU64 = AtomicU64::new(1); +#[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] +type InitialScanHook = Box Result<()> + Send>; +#[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] +static INITIAL_SCAN_HOOKS: OnceLock>> = OnceLock::new(); + +#[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] +pub(crate) fn install_initial_scan_hook(scope_id: ScopeId, hook: F) +where + F: FnOnce() -> Result<()> + Send + 'static, +{ + INITIAL_SCAN_HOOKS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .insert(scope_id, Box::new(hook)); +} + +#[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] +fn run_initial_scan_hook(scope_id: ScopeId) -> Result<()> { + let hook = INITIAL_SCAN_HOOKS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .remove(&scope_id); + if let Some(hook) = hook { + hook()?; + } + Ok(()) +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ObserverEvent { + pub(crate) path: LedgerPath, + pub(crate) flags: EvidenceFlags, + pub(crate) sequence: u64, +} + +struct ObserverEventSpool { + file: std::fs::File, + events: u64, + bytes: u64, +} + +impl ObserverEventSpool { + fn new() -> Result { + Ok(Self { + file: tempfile::tempfile()?, + events: 0, + bytes: 0, + }) + } + + fn push(&mut self, event: ObserverEvent) -> Result<()> { + let path = event.path.as_str().as_bytes(); + if path.len() > MAX_OBSERVER_SPOOL_PATH_BYTES { + return Err(Error::InvalidInput( + "observer event path exceeds reconciliation spool limit".into(), + )); + } + let next_events = self.events.saturating_add(1); + let record_bytes = OBSERVER_SPOOL_HEADER_BYTES + .checked_add(path.len()) + .ok_or_else(|| Error::InvalidInput("observer spool record size overflow".into()))?; + let next_bytes = self + .bytes + .checked_add(record_bytes as u64) + .ok_or_else(|| Error::InvalidInput("observer spool size overflow".into()))?; + if next_events > MAX_OBSERVER_SPOOL_EVENTS || next_bytes > MAX_OBSERVER_SPOOL_BYTES { + return Err(Error::InvalidInput( + "observer evidence exceeds bounded reconciliation spool".into(), + )); + } + let path_len = u32::try_from(path.len()) + .map_err(|_| Error::InvalidInput("observer path length overflow".into()))?; + self.file.write_all(&path_len.to_be_bytes())?; + self.file.write_all(&event.flags.0.to_be_bytes())?; + self.file.write_all(&event.sequence.to_be_bytes())?; + self.file.write_all(path)?; + self.events = next_events; + self.bytes = next_bytes; + Ok(()) + } + + fn finish(&mut self) -> Result<()> { + self.file.sync_all()?; + self.file.seek(SeekFrom::Start(0))?; + Ok(()) + } + + fn next(&mut self) -> Result> { + let mut header = [0_u8; OBSERVER_SPOOL_HEADER_BYTES]; + let read = self.file.read(&mut header[..1])?; + if read == 0 { + return Ok(None); + } + self.file.read_exact(&mut header[1..])?; + let path_len = u32::from_be_bytes(header[..4].try_into().expect("four byte length")); + let path_len = usize::try_from(path_len) + .map_err(|_| Error::Corrupt("observer spool path length cannot fit memory".into()))?; + if path_len > MAX_OBSERVER_SPOOL_PATH_BYTES { + return Err(Error::Corrupt( + "observer spool path exceeds bounded record limit".into(), + )); + } + let flags = i64::from_be_bytes(header[4..12].try_into().expect("eight byte flags")); + let sequence = u64::from_be_bytes(header[12..20].try_into().expect("eight byte sequence")); + let mut path = vec![0_u8; path_len]; + self.file.read_exact(&mut path)?; + let path = String::from_utf8(path) + .map_err(|_| Error::Corrupt("observer spool path is not UTF-8".into()))?; + Ok(Some(ObserverEvent { + path: LedgerPath::parse(&path)?, + flags: EvidenceFlags(flags), + sequence, + })) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ObserverQualification { + scope_id: ScopeId, + provider_identity: Vec, + filesystem_identity: Vec, + root_handle_identity: Vec, + policy_fingerprint: [u8; 32], + policy_generation: u64, + start_fence: ObserverFence, + end_fence: ObserverFence, + observer_owner_token: String, + owner_fence_nonce: Option>, + durable_segment_id: String, + segment_durable_offset: u64, + segment_consumed_offset: u64, + segment_initial_folded_offset: u64, + complete_root_interval: bool, + complete_policy_interval: bool, + persisted_evidence_through_end: bool, +} + +impl ObserverQualification { + #[allow(clippy::too_many_arguments)] + pub(super) fn native( + expected: &ExpectedScope, + root_handle_identity: Vec, + start_fence: ObserverFence, + end_fence: ObserverFence, + observer_owner_token: String, + owner_fence_nonce: Vec, + durable_segment_id: String, + segment_durable_offset: u64, + segment_consumed_offset: u64, + ) -> Self { + Self { + scope_id: expected.scope_id, + provider_identity: expected.provider_identity.clone(), + filesystem_identity: expected.filesystem_identity.clone(), + root_handle_identity, + policy_fingerprint: expected.policy_fingerprint, + policy_generation: expected.policy_generation, + start_fence, + end_fence, + observer_owner_token, + owner_fence_nonce: Some(owner_fence_nonce), + durable_segment_id, + segment_durable_offset, + segment_consumed_offset, + segment_initial_folded_offset: 0, + complete_root_interval: true, + complete_policy_interval: true, + persisted_evidence_through_end: true, + } + } + + #[cfg(any(test, debug_assertions))] + fn seal_for_test( + expected: &ExpectedScope, + root_handle_identity: Vec, + start_fence: ObserverFence, + end_fence: ObserverFence, + ) -> Self { + Self { + scope_id: expected.scope_id, + provider_identity: expected.provider_identity.clone(), + filesystem_identity: expected.filesystem_identity.clone(), + root_handle_identity, + policy_fingerprint: expected.policy_fingerprint, + policy_generation: expected.policy_generation, + start_fence, + end_fence, + observer_owner_token: "full-test-owner".into(), + owner_fence_nonce: Some(b"full-test-fence".to_vec()), + durable_segment_id: "full-test-segment".into(), + segment_durable_offset: 100, + segment_consumed_offset: 100, + segment_initial_folded_offset: 100, + complete_root_interval: true, + complete_policy_interval: true, + persisted_evidence_through_end: true, + } + } + + fn validates( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + ) -> bool { + self.scope_id == expected.scope_id + && self.provider_identity == expected.provider_identity + && self.filesystem_identity == expected.filesystem_identity + && self.root_handle_identity == root_handle_identity + && self.policy_fingerprint == expected.policy_fingerprint + && self.policy_generation == expected.policy_generation + && self.start_fence == *start + && self.end_fence == *end + && !self.observer_owner_token.is_empty() + && !self.durable_segment_id.is_empty() + && self.segment_initial_folded_offset <= self.segment_consumed_offset + && self.segment_consumed_offset <= self.segment_durable_offset + && end.sequence >= start.sequence + && self.complete_root_interval + && self.complete_policy_interval + && self.persisted_evidence_through_end + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ProvenPrefixSet { + prefixes: Vec, + epoch: u64, + continuity_generation: u64, + owner_token: String, + owner_fence_nonce: Option>, + provider_id: String, + provider_identity: String, + provider_cursor: Option>, + provider_fence: Option>, + durable_offset: u64, + folded_offset: u64, + rows: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct ProvenPrefixRow { + prefix: LedgerPath, + event_flags: i64, + source_mask: i64, + first_sequence: u64, + last_sequence: u64, + provider_sequence: u64, + created_at: i64, + updated_at: i64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ReconcileMode { + Full, + ProvenPrefixes(ProvenPrefixSet), +} + +impl ReconcileMode { + fn label(&self) -> &'static str { + match self { + Self::Full => "full", + Self::ProvenPrefixes(_) => "prefix", + } + } + + fn completeness(&self) -> &'static str { + match self { + Self::Full => "complete", + Self::ProvenPrefixes(_) => "provider_complete_prefix", + } + } + + fn prefixes(&self) -> Vec { + match self { + Self::Full => Vec::new(), + Self::ProvenPrefixes(proof) => proof + .prefixes + .iter() + .map(|path| path.as_str().to_string()) + .collect(), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +struct StoredAttemptIdentity { + scope_id: String, + scope_root: String, + scope_root_identity: String, + case_sensitive: bool, + epoch: u64, + ref_name: String, + ref_generation: u64, + change_id: String, + baseline_root_id: String, + policy_fingerprint: String, + policy_generation: u64, + trust_state: String, + continuity_generation: u64, + filesystem_identity: String, + provider_id: Option, + provider_identity: Option, + observer_owner_token: Option, + initial_durable_offset: u64, + initial_folded_offset: u64, + start_fence: ObserverFence, + root_handle_identity: Vec, +} + +pub(crate) struct ReconciliationAttempt { + attempt_id: String, + expected: ExpectedScope, + mode: ReconcileMode, + reason: String, + start_fence: ObserverFence, + end_fence: Option, + qualification: Option, + stored_identity: StoredAttemptIdentity, + encoded_identity: Vec, + root: PinnedWorktreeRoot, + report: ChangeLedgerReconcileReport, + #[cfg(test)] + final_publication_hook: Option>, +} + +pub(crate) fn persisted_proven_prefixes( + ledger: &ChangedPathLedger<'_>, + expected: &ExpectedScope, + requested: &[LedgerPath], +) -> Result { + if requested.is_empty() { + return Err(Error::InvalidInput( + "provider-proven prefix reconciliation requires at least one prefix".into(), + )); + } + exact_scope_guard(ledger.conn, expected)?; + let scope_id = expected.scope_id.to_text(); + let ( + state, + reason, + provider_id, + provider_identity, + provider_cursor, + provider_fence, + durable_offset, + folded_offset, + continuity_generation, + ): ( + String, + String, + String, + String, + Option>, + Option>, + i64, + i64, + i64, + ) = ledger.conn.query_row( + "SELECT trust_state,trust_reason,provider_id,provider_identity, + provider_cursor,provider_fence,durable_offset,folded_offset, + continuity_generation + FROM changed_path_scopes WHERE scope_id=?1", + [&scope_id], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + row.get(6)?, + row.get(7)?, + row.get(8)?, + )) + }, + )?; + if !matches!(state.as_str(), "trusted" | "reconciling") { + return Err(reconcile_required( + expected, + &state, + &format!("full reconciliation required: {reason}"), + )); + } + if provider_identity != hex::encode(&expected.provider_identity) { + return Err(reconcile_required( + expected, + &state, + "provider identity changed; full reconciliation required", + )); + } + let owner = ledger + .conn + .query_row( + "SELECT owner_token,fence_nonce FROM changed_path_observer_owners + WHERE scope_id=?1 AND epoch=?2 AND provider_id=?3 + AND provider_identity=?4 AND lease_state='active' AND expires_at>=?5", + params![ + scope_id, + sql_u64(expected.epoch)?, + provider_id, + provider_identity, + now_ts(), + ], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, Option>>(1)?)), + ) + .optional()?; + let Some((owner_token, owner_fence_nonce)) = owner else { + return Err(reconcile_required( + expected, + &state, + "qualified provider owner is unavailable; full reconciliation required", + )); + }; + let mut prefixes = requested.to_vec(); + prefixes.sort(); + prefixes.dedup(); + let mut proof_rows = Vec::with_capacity(prefixes.len()); + for prefix in &prefixes { + let row = ledger + .conn + .query_row( + "SELECT event_flags,source_mask,first_sequence,last_sequence, + provider_sequence,created_at,updated_at + FROM changed_path_prefixes + WHERE scope_id=?1 AND normalized_prefix=?2 COLLATE BINARY + AND completeness_reason='provider_complete' + AND source_mask=?3 AND provider_id=?4 + AND provider_sequence IS NOT NULL AND intent_id IS NULL", + params![ + scope_id, + prefix.as_str(), + super::EvidenceSource::Observer.mask(), + provider_id, + ], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, i64>(6)?, + )) + }, + ) + .optional()?; + let Some((event_flags, source_mask, first, last, sequence, created_at, updated_at)) = row + else { + return Err(reconcile_required( + expected, + &state, + "prefix was not persisted by the qualified provider", + )); + }; + proof_rows.push(ProvenPrefixRow { + prefix: prefix.clone(), + event_flags, + source_mask, + first_sequence: db_u64(first)?, + last_sequence: db_u64(last)?, + provider_sequence: db_u64(sequence)?, + created_at, + updated_at, + }); + } + let durable_offset = db_u64(durable_offset)?; + let folded_offset = db_u64(folded_offset)?; + let continuity_generation = db_u64(continuity_generation)?; + if folded_offset > durable_offset { + return Err(Error::Corrupt( + "prefix proof folded cut exceeds durable cut".into(), + )); + } + Ok(ProvenPrefixSet { + prefixes, + epoch: expected.epoch, + continuity_generation, + owner_token, + owner_fence_nonce, + provider_id, + provider_identity, + provider_cursor, + provider_fence, + durable_offset, + folded_offset, + rows: proof_rows, + }) +} + +pub(crate) fn begin_reconciliation( + trail: &Trail, + ledger: &ChangedPathLedger<'_>, + observer: &dyn QualifiedObserver, + expected: &ExpectedScope, + policy: &CompiledPolicy, + mode: ReconcileMode, + reason: &str, +) -> Result { + if !policy.authorizes_reconciliation(expected) { + return Err(reconcile_required( + expected, + TrustState::StaleBaseline.as_str(), + "compiled recording policy has no authenticated reconciliation authorization", + )); + } + ledger.recover_scope(expected)?; + // The observation cut is deliberately acquired before the root is opened + // and before any enumeration can begin. + let start_fence = observer.begin_observation(expected)?; + let root = trail.open_pinned_worktree_root(policy)?; + let root_handle_identity = trail.pinned_worktree_root_identity(&root); + let attempt_id = format!( + "reconcile-{}-{}", + now_ts(), + NEXT_ATTEMPT_ID.fetch_add(1, Ordering::Relaxed) + ); + let tx = Transaction::new_unchecked(ledger.conn, TransactionBehavior::Immediate)?; + exact_scope_guard(&tx, expected)?; + validate_mode_start(&tx, expected, &mode)?; + if matches!(mode, ReconcileMode::Full) { + exact_scope_update_state(&tx, expected, TrustState::Reconciling, reason)?; + } + let stored_identity = + capture_attempt_identity(&tx, expected, start_fence.clone(), root_handle_identity)?; + let encoded_identity = serde_json::to_vec(&stored_identity)?; + let scope_id = stored_identity.scope_id.clone(); + tx.execute( + "UPDATE changed_path_reconciliations + SET state='abandoned', updated_at=?1 + WHERE scope_id=?2 AND state IN ('prepared','staging','ready')", + params![now_ts(), scope_id], + )?; + let start_cursor = start_fence.sequence.to_be_bytes(); + tx.execute( + "INSERT INTO changed_path_reconciliations( + attempt_id, scope_id, expected_scope_epoch, expected_ref_name, + expected_ref_generation, expected_change_id, expected_root_id, + filesystem_identity, policy_fingerprint, + policy_dependency_generation, provider_id, provider_identity, + start_cursor, start_fence, mode, reason, completeness_class, + staged_store_location, state, created_at, updated_at + ) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14, + ?15,?16,?17,'sqlite','staging',?18,?18)", + params![ + attempt_id, + scope_id, + sql_u64(expected.epoch)?, + expected.ref_name, + sql_u64(expected.ref_generation)?, + stored_identity.change_id, + stored_identity.baseline_root_id, + stored_identity.filesystem_identity, + stored_identity.policy_fingerprint, + sql_u64(stored_identity.policy_generation)?, + stored_identity.provider_id, + stored_identity.provider_identity, + start_cursor.as_slice(), + encoded_identity, + mode.label(), + reason, + mode.completeness(), + now_ts(), + ], + )?; + tx.commit()?; + + Ok(ReconciliationAttempt { + attempt_id, + expected: expected.clone(), + mode: mode.clone(), + reason: reason.to_string(), + start_fence: start_fence.clone(), + end_fence: None, + qualification: None, + stored_identity, + encoded_identity, + root, + report: ChangeLedgerReconcileReport { + mode: mode.label().to_string(), + reason: reason.to_string(), + start_sequence: start_fence.sequence, + start_durable_offset: start_fence.durable_offset, + trust_state: TrustState::Reconciling.as_str().to_string(), + ..ChangeLedgerReconcileReport::default() + }, + #[cfg(test)] + final_publication_hook: None, + }) +} + +pub(crate) fn reconcile_full( + trail: &Trail, + ledger: &ChangedPathLedger<'_>, + observer: &dyn QualifiedObserver, + expected: &ExpectedScope, + policy: &CompiledPolicy, + reason: &str, +) -> Result { + reconcile_full_with_tail(trail, ledger, observer, expected, policy, reason) + .map(|(report, _tail)| report) +} + +/// Full reconciliation plus the authenticated end fence retained by native +/// observers as the start of continuous post-reconciliation coverage. +pub(crate) fn reconcile_full_with_tail( + trail: &Trail, + ledger: &ChangedPathLedger<'_>, + observer: &dyn QualifiedObserver, + expected: &ExpectedScope, + policy: &CompiledPolicy, + reason: &str, +) -> Result<(ChangeLedgerReconcileReport, ObserverFence)> { + let mut retries = 0; + loop { + let mut attempt = begin_reconciliation( + trail, + ledger, + observer, + expected, + policy, + ReconcileMode::Full, + reason, + ) + .map_err(|error| sqlite_reconciliation_stage(error, "begin"))?; + if let Err(error) = attempt.observe(trail, ledger, observer, policy) { + let error = sqlite_reconciliation_stage(error, "observe"); + if retries < MAX_IDENTITY_RACE_RETRIES && is_retryable_identity_race(&error) { + retries += 1; + continue; + } + return Err(error); + } + let retained_tail = attempt.end_fence.clone().ok_or_else(|| { + Error::InvalidInput("reconciliation produced no retained observer tail".into()) + })?; + match attempt + .publish(trail, ledger, policy) + .map_err(|error| sqlite_reconciliation_stage(error, "publish")) + { + Ok(mut report) => { + report.retries = retries; + return Ok((report, retained_tail)); + } + Err(error) + if retries < MAX_IDENTITY_RACE_RETRIES && is_retryable_identity_race(&error) => + { + retries += 1; + } + Err(error) => return Err(error), + } + } +} + +fn sqlite_reconciliation_stage(error: Error, stage: &str) -> Error { + match error { + Error::Sqlite(_) => Error::DaemonUnavailable(format!( + "changed-path reconciliation {stage} stage failed: {error}" + )), + other => other, + } +} + +/// Atomically folds one authenticated continuous observer interval. The +/// caller must have drained the interval with `drain_through_retaining_end`; +/// no evidence row becomes visible unless the exact owner, segment, scope and +/// cut transition all commit together. +pub(crate) fn fold_observer_interval( + ledger: &ChangedPathLedger<'_>, + expected: &ExpectedScope, + root_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + qualification: &mut ObserverQualification, + events: &[ObserverEvent], +) -> Result { + if end.sequence <= start.sequence { + return Err(reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "continuous observer tail regressed", + )); + } + if !qualification.validates(expected, root_identity, start, end) + || events + .iter() + .any(|event| event.sequence <= start.sequence || event.sequence > end.sequence) + { + return Err(reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "continuous observer interval did not authenticate the exact scope and cut", + )); + } + + let tx = Transaction::new_unchecked(ledger.conn, TransactionBehavior::Immediate)?; + exact_scope_guard(&tx, expected)?; + let (trust_state, durable, folded, owner): (String, i64, i64, Option) = tx.query_row( + "SELECT trust_state,durable_offset,folded_offset,observer_owner_token + FROM changed_path_scopes WHERE scope_id=?1", + [expected.scope_id.to_text()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + )?; + if trust_state != TrustState::Trusted.as_str() + || db_u64(durable)? != start.durable_offset + || db_u64(folded)? != start.durable_offset + || owner.as_deref() != Some(qualification.observer_owner_token.as_str()) + { + return Err(reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "continuous observer anchor does not match the persisted trusted cut", + )); + } + qualification.segment_initial_folded_offset = + qualification_segment_folded_offset(&tx, expected, qualification)?; + if qualification.segment_initial_folded_offset != start.durable_offset { + return Err(reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "continuous observer segment was not folded through the previous anchor", + )); + } + validate_observer_owner_and_segment(&tx, expected, qualification, end)?; + + let scope_id = expected.scope_id.to_text(); + let now = now_ts(); + tx.execute_batch("SAVEPOINT changed_path_continuous_fold_rows;")?; + for event in events { + let sequence = sql_u64(event.sequence)?; + if event.flags.0 & EvidenceFlags::PROVIDER_COMPLETE_PREFIX.0 != 0 { + tx.execute( + "INSERT INTO changed_path_prefixes( + scope_id,normalized_prefix,completeness_reason,event_flags,source_mask, + first_sequence,last_sequence,provider_id,provider_sequence,intent_id, + created_at,updated_at + ) VALUES(?1,?2,'provider_complete',?3,?4,?5,?5,?6,?5,NULL,?7,?7) + ON CONFLICT(scope_id,normalized_prefix) DO UPDATE SET + event_flags=(changed_path_prefixes.event_flags | excluded.event_flags), + source_mask=(changed_path_prefixes.source_mask | excluded.source_mask), + first_sequence=MIN(changed_path_prefixes.first_sequence,excluded.first_sequence), + last_sequence=MAX(changed_path_prefixes.last_sequence,excluded.last_sequence), + provider_id=CASE WHEN changed_path_prefixes.source_mask=excluded.source_mask + THEN excluded.provider_id ELSE NULL END, + provider_sequence=MAX(COALESCE(changed_path_prefixes.provider_sequence,0),excluded.provider_sequence), + updated_at=excluded.updated_at", + params![ + scope_id, + event.path.as_str(), + event.flags.0, + super::EvidenceSource::Observer.mask(), + sequence, + hex::encode(&expected.provider_identity), + now, + ], + )?; + } else { + tx.execute( + "INSERT INTO changed_path_entries( + scope_id,normalized_path,event_flags,source_mask,first_sequence,last_sequence, + provider_id,provider_sequence,intent_id,created_at,updated_at + ) VALUES(?1,?2,?3,?4,?5,?5,?6,?5,NULL,?7,?7) + ON CONFLICT(scope_id,normalized_path) DO UPDATE SET + event_flags=(changed_path_entries.event_flags | excluded.event_flags), + source_mask=(changed_path_entries.source_mask | excluded.source_mask), + first_sequence=MIN(changed_path_entries.first_sequence,excluded.first_sequence), + last_sequence=MAX(changed_path_entries.last_sequence,excluded.last_sequence), + provider_id=CASE WHEN changed_path_entries.source_mask=excluded.source_mask + THEN excluded.provider_id ELSE NULL END, + provider_sequence=MAX(COALESCE(changed_path_entries.provider_sequence,0),excluded.provider_sequence), + updated_at=excluded.updated_at", + params![ + scope_id, + event.path.as_str(), + event.flags.0, + super::EvidenceSource::Observer.mask(), + sequence, + hex::encode(&expected.provider_identity), + now, + ], + )?; + } + } + if evidence_cap_exceeded(&tx, &scope_id)? { + tx.execute_batch( + "ROLLBACK TO changed_path_continuous_fold_rows; + RELEASE changed_path_continuous_fold_rows;", + )?; + tx.execute( + "UPDATE changed_path_scopes SET trust_state='overflow', + trust_reason='persisted evidence cap exceeded', + continuity_generation=continuity_generation+1,updated_at=?1 + WHERE scope_id=?2", + params![now, scope_id], + )?; + tx.commit()?; + return Err(reconcile_required( + expected, + TrustState::Overflow.as_str(), + "persisted evidence cap exceeded", + )); + } + tx.execute_batch("RELEASE changed_path_continuous_fold_rows;")?; + persist_consumed_observer_fold(&tx, expected, qualification, end)?; + let changed = tx.execute( + "UPDATE changed_path_scopes SET durable_offset=?1,folded_offset=?1, + trust_reason='continuous_tail_folded',updated_at=?2 + WHERE scope_id=?3 AND epoch=?4 AND durable_offset=?5 AND folded_offset=?5 + AND trust_state='trusted' AND observer_owner_token=?6", + params![ + sql_u64(end.durable_offset)?, + now, + scope_id, + sql_u64(expected.epoch)?, + sql_u64(start.durable_offset)?, + qualification.observer_owner_token, + ], + )?; + if changed != 1 { + return Err(reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "continuous observer fold lost its exact cut CAS", + )); + } + tx.commit()?; + Ok(EvidenceCut { + source: super::EvidenceSource::Observer, + sequence: end.sequence, + durable_offset: end.durable_offset, + folded_offset: end.durable_offset, + }) +} + +fn validate_observer_owner_and_segment( + conn: &rusqlite::Connection, + expected: &ExpectedScope, + qualification: &ObserverQualification, + end: &ObserverFence, +) -> Result<()> { + let owner_ok = conn.query_row( + "SELECT COUNT(*) FROM changed_path_observer_owners + WHERE scope_id=?1 AND epoch=?2 AND owner_token=?3 + AND provider_id=?4 AND provider_identity=?4 + AND fence_nonce=?5 AND lease_state='active' AND expires_at>=?6", + params![ + expected.scope_id.to_text(), + sql_u64(expected.epoch)?, + qualification.observer_owner_token, + hex::encode(&expected.provider_identity), + qualification.owner_fence_nonce, + publication_lease_deadline()?, + ], + |row| row.get::<_, i64>(0), + )? == 1; + let segment_ok = conn.query_row( + "SELECT COUNT(*) FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=?2 AND segment_id=?3 AND owner_token=?4 + AND provider_id=?5 AND state IN ('open','sealed') + AND last_sequence>=?6 AND durable_end_offset>=?7 + AND folded_end_offset=?8", + params![ + expected.scope_id.to_text(), + sql_u64(expected.epoch)?, + qualification.durable_segment_id, + qualification.observer_owner_token, + hex::encode(&expected.provider_identity), + sql_u64(end.sequence)?, + sql_u64(end.durable_offset)?, + sql_u64(qualification.segment_initial_folded_offset)?, + ], + |row| row.get::<_, i64>(0), + )? == 1; + if !owner_ok || !segment_ok { + return Err(reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "continuous observer owner or segment authority changed", + )); + } + Ok(()) +} + +fn is_retryable_identity_race(error: &Error) -> bool { + let message = match error { + Error::InvalidInput(message) => message.as_str(), + Error::ChangeLedgerReconcileRequired { reason, .. } => reason.as_str(), + _ => return false, + }; + message.contains("identity race") + || message.contains("changed while it was read") + || message.contains("workspace root identity changed") + || message.contains("directory identity changed") +} + +impl ReconciliationAttempt { + #[cfg(test)] + fn set_final_publication_hook(&mut self, hook: F) + where + F: FnOnce(&rusqlite::Connection) + 'static, + { + self.final_publication_hook = Some(Box::new(hook)); + } + + #[cfg(test)] + fn run_final_publication_hook(&mut self, conn: &rusqlite::Connection) { + if let Some(hook) = self.final_publication_hook.take() { + hook(conn); + } + } + + pub(crate) fn observe( + &mut self, + trail: &Trail, + ledger: &ChangedPathLedger<'_>, + observer: &dyn QualifiedObserver, + policy: &CompiledPolicy, + ) -> Result<()> { + let result = self.observe_inner(trail, ledger, observer, policy); + if let Err(error) = result { + mark_attempt_failed(ledger.conn, &self.attempt_id, &error.to_string())?; + return Err(error); + } + Ok(()) + } + + fn observe_inner( + &mut self, + trail: &Trail, + ledger: &ChangedPathLedger<'_>, + observer: &dyn QualifiedObserver, + policy: &CompiledPolicy, + ) -> Result<()> { + if self.end_fence.is_some() { + return Err(Error::InvalidInput( + "reconciliation attempt was already observed".into(), + )); + } + let prefixes = self.mode.prefixes(); + let mut writer = StagingWriter::new(ledger.conn, &self.attempt_id); + trail.visit_pinned_worktree_files(&self.root, policy, &prefixes, |entry| match entry { + ReconciliationScanEntry::Directory(directory) => { + writer.stage_directory_guard(directory) + } + ReconciliationScanEntry::File(file) => writer.stage_filesystem(file), + })?; + writer.flush()?; + self.validate_directory_guards(trail, ledger)?; + #[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] + run_initial_scan_hook(self.expected.scope_id)?; + trail.visit_root_file_entries( + &self.expected.baseline_root, + &prefixes, + |path, baseline| writer.compare_baseline(path, baseline), + )?; + writer.flush()?; + + let end = observer.end_fence(&self.expected, &self.start_fence)?; + if end.sequence < self.start_fence.sequence { + return self.fail( + ledger, + "observer end fence regressed behind reconciliation start fence", + ); + } + let root_identity = trail.pinned_worktree_root_identity(&self.root); + let mut spool = ObserverEventSpool::new()?; + let mut qualification = observer.drain_through_retaining_end( + &self.expected, + &root_identity, + &self.start_fence, + &end, + &mut |event| { + if event.sequence <= self.start_fence.sequence || event.sequence > end.sequence { + return Err(Error::InvalidInput( + "observer returned evidence outside requested fence interval".into(), + )); + } + spool.push(event) + }, + )?; + qualification.segment_initial_folded_offset = + qualification_segment_folded_offset(ledger.conn, &self.expected, &qualification)?; + spool.finish()?; + while let Some(event) = spool.next()? { + if raw_event_invalidates_policy(policy, std::path::Path::new(event.path.as_str())) { + return self.fail( + ledger, + "recording policy changed during reconciliation interval", + ); + } + if event.flags.0 & EvidenceFlags::PROVIDER_COMPLETE_PREFIX.0 != 0 { + writer.flush()?; + writer.clear_prefix(event.path.as_str())?; + if !trail.verify_pinned_worktree_root(&self.root)? { + return self.fail( + ledger, + "workspace root identity changed during prefix replay", + ); + } + let prefix = vec![event.path.as_str().to_string()]; + trail.visit_pinned_worktree_files( + &self.root, + policy, + &prefix, + |entry| match entry { + ReconciliationScanEntry::Directory(directory) => { + writer.stage_directory_guard(directory) + } + ReconciliationScanEntry::File(file) => { + writer.stage_prefix_filesystem(file, event.sequence) + } + }, + )?; + writer.flush()?; + trail.visit_root_file_entries( + &self.expected.baseline_root, + &prefix, + |path, baseline| writer.compare_baseline(path, baseline), + )?; + writer.flush()?; + self.validate_directory_guards(trail, ledger)?; + if !trail.verify_pinned_worktree_root(&self.root)? { + return self.fail( + ledger, + "workspace root identity changed after prefix replay", + ); + } + } else { + let current = + trail.read_pinned_worktree_path(&self.root, policy, event.path.as_str())?; + let baseline = + trail.root_file_entry(&self.expected.baseline_root, event.path.as_str())?; + writer.stage_observer_result(event, current, baseline)?; + } + } + writer.flush()?; + let candidate_rows = ledger.conn.query_row( + "SELECT COUNT(*) FROM changed_path_reconciliation_rows + WHERE attempt_id=?1 AND before_identity LIKE 'flags:%'", + [&self.attempt_id], + |row| row.get::<_, i64>(0), + )?; + let changed = ledger.conn.execute( + "UPDATE changed_path_reconciliations SET state='ready', updated_at=?1 + WHERE attempt_id=?2 AND state='staging'", + params![now_ts(), self.attempt_id], + )?; + if changed != 1 { + return Err(reconcile_required( + &self.expected, + TrustState::Reconciling.as_str(), + "reconciliation staging attempt was replaced", + )); + } + self.report.observed_files = writer.observed_files; + self.report.staged_rows = writer.staged_rows; + self.report.observed_candidates = db_u64(candidate_rows)?; + self.report.candidate_rows = db_u64(candidate_rows)?; + self.report.hashed_bytes = writer.hashed_bytes; + self.report.peak_batch_rows = writer.peak_batch_rows; + self.report.peak_buffer_bytes = writer.peak_buffer_bytes; + self.report.end_sequence = end.sequence; + self.report.end_durable_offset = end.durable_offset; + self.end_fence = Some(end); + self.qualification = Some(qualification); + Ok(()) + } + + fn validate_directory_guards( + &self, + trail: &Trail, + ledger: &ChangedPathLedger<'_>, + ) -> Result<()> { + let mut statement = ledger.conn.prepare( + "SELECT relative_path,directory_identity + FROM changed_path_reconciliation_guards + WHERE attempt_id=?1 + ORDER BY relative_path", + )?; + let mut rows = statement.query([&self.attempt_id])?; + while let Some(row) = rows.next()? { + let path = String::from_utf8(row.get::<_, Vec>(0)?) + .map_err(|_| Error::Corrupt("invalid staged directory guard path".into()))?; + let identity = row.get::<_, Vec>(1)?; + if !trail.verify_pinned_worktree_directory(&self.root, &path, &identity)? { + return Err(Error::InvalidInput(format!( + "directory identity changed during reconciliation: `{path}`" + ))); + } + } + Ok(()) + } + + pub(crate) fn publish( + mut self, + trail: &Trail, + ledger: &ChangedPathLedger<'_>, + policy: &CompiledPolicy, + ) -> Result { + let Some(end) = self.end_fence.clone() else { + return Err(Error::InvalidInput( + "reconciliation attempt has not completed observation".into(), + )); + }; + let Some(qualification) = self.qualification.clone() else { + return Err(reconcile_required( + &self.expected, + TrustState::Reconciling.as_str(), + "qualified observer proof is unavailable", + )); + }; + if policy.fingerprint() != self.expected.policy_fingerprint + || !policy.authorizes_reconciliation(&self.expected) + { + return self.fail_report( + ledger, + "compiled recording policy fingerprint changed before publication", + ); + } + let root_identity = trail.pinned_worktree_root_identity(&self.root); + if !qualification.validates(&self.expected, &root_identity, &self.start_fence, &end) { + return self.fail_report(ledger, "observer qualification did not cover exact scope"); + } + + if matches!(self.mode, ReconcileMode::ProvenPrefixes(_)) { + return self.publish_prefix(ledger, trail, &root_identity, &end); + } + + let tx = Transaction::new_unchecked(ledger.conn, TransactionBehavior::Immediate)?; + let (current_durable, current_folded) = + match validate_stored_scope(&tx, &self.expected, &self.stored_identity) { + Ok(cuts) => cuts, + Err(_) => { + return self.fail_publication_transaction( + tx, + "scope changed during reconciliation publication", + ) + } + }; + if current_durable < self.stored_identity.initial_durable_offset + || current_folded < self.stored_identity.initial_folded_offset + { + return self.fail_publication_transaction( + tx, + "scope evidence cuts regressed during reconciliation publication", + ); + } + if validate_observer_continuity( + &tx, + &self.expected, + &self.stored_identity, + &qualification, + &end, + ) + .is_err() + { + return self.fail_observer_continuity_transaction( + tx, + "observer owner or durable segment changed before publication", + ); + } + if !matches!(trail.verify_pinned_worktree_root(&self.root), Ok(true)) { + return self.fail_publication_transaction( + tx, + "workspace root identity changed before publication", + ); + } + if self.validate_directory_guards(trail, ledger).is_err() { + return self.fail_publication_transaction( + tx, + "directory identity changed before reconciliation publication", + ); + } + if validate_ready_attempt(&tx, &self, &root_identity).is_err() { + return self + .fail_publication_transaction(tx, "reconciliation attempt identity changed"); + } + if validate_scope_capabilities(&tx, &self.expected).is_err() { + return self.fail_publication_transaction( + tx, + "provider qualification changed before publication", + ); + } + let scope_id = self.expected.scope_id.to_text(); + tx.execute_batch("SAVEPOINT changed_path_reconciliation_candidates;")?; + if persist_consumed_observer_fold(&tx, &self.expected, &qualification, &end).is_err() { + return self.fail_observer_continuity_candidate_transaction( + tx, + "observer consumed cut could not be atomically folded", + ); + } + tx.execute( + "DELETE FROM changed_path_entries + WHERE scope_id=?1 AND ( + source_mask=?2 OR + (source_mask=?3 AND provider_sequence IS NOT NULL AND provider_sequence<=?4) + )", + params![ + scope_id, + super::EvidenceSource::Reconciliation.mask(), + super::EvidenceSource::Observer.mask(), + sql_u64(end.sequence)?, + ], + )?; + tx.execute( + "DELETE FROM changed_path_prefixes + WHERE scope_id=?1 AND ( + source_mask=?2 OR + (source_mask=?3 AND provider_sequence IS NOT NULL AND provider_sequence<=?4) + )", + params![ + scope_id, + super::EvidenceSource::Reconciliation.mask(), + super::EvidenceSource::Observer.mask(), + sql_u64(end.sequence)?, + ], + )?; + let now = now_ts(); + tx.execute( + "INSERT INTO changed_path_entries( + scope_id, normalized_path, event_flags, source_mask, + first_sequence, last_sequence, provider_id, provider_sequence, + intent_id, created_at, updated_at + ) + SELECT ?1, normalized_path, + CAST(substr(before_identity, 7) AS INTEGER), ?2, + ?3, ?3, 'reconciliation', NULL, NULL, ?4, ?4 + FROM changed_path_reconciliation_rows + WHERE attempt_id=?5 AND before_identity LIKE 'flags:%' + ON CONFLICT(scope_id, normalized_path) DO UPDATE SET + event_flags=(changed_path_entries.event_flags | excluded.event_flags), + source_mask=(changed_path_entries.source_mask | excluded.source_mask), + first_sequence=MIN(changed_path_entries.first_sequence, excluded.first_sequence), + last_sequence=MAX(changed_path_entries.last_sequence, excluded.last_sequence), + updated_at=excluded.updated_at", + params![ + scope_id, + super::EvidenceSource::Reconciliation.mask(), + sql_u64(end.sequence)?, + now, + self.attempt_id, + ], + )?; + if evidence_cap_exceeded(&tx, &scope_id)? { + return self.fail_candidate_cap_transaction(tx); + } + #[cfg(test)] + self.run_final_publication_hook(&tx); + if validate_observer_continuity_at( + &tx, + &self.expected, + &self.stored_identity, + &qualification, + &end, + publication_lease_deadline()?, + ) + .is_err() + { + return self.fail_observer_continuity_candidate_transaction( + tx, + "observer lease or durable segment became unavailable at publication boundary", + ); + } + tx.execute_batch("RELEASE changed_path_reconciliation_candidates;")?; + let merged_durable = current_durable.max(end.durable_offset); + let merged_folded = current_folded.max(end.durable_offset); + debug_assert!(merged_folded <= merged_durable); + let changed = tx.execute( + "UPDATE changed_path_scopes + SET trust_state='trusted', trust_reason='reconciliation_published', + durable_offset=?1, folded_offset=?2, updated_at=?3 + WHERE scope_id=?4 AND scope_root=?5 AND scope_root_identity=?6 + AND case_sensitive=?7 AND epoch=?8 AND ref_name=?9 + AND ref_generation=?10 AND change_id=?11 AND baseline_root_id=?12 + AND policy_fingerprint=?13 AND policy_dependency_generation=?14 + AND filesystem_identity=?15 AND provider_id IS ?16 + AND provider_identity IS ?17 AND observer_owner_token IS ?18 + AND durable_offset=?19 AND folded_offset=?20 + AND trust_state='reconciling' AND continuity_generation=?21", + params![ + sql_u64(merged_durable)?, + sql_u64(merged_folded)?, + now, + self.stored_identity.scope_id, + self.stored_identity.scope_root, + self.stored_identity.scope_root_identity, + self.stored_identity.case_sensitive, + sql_u64(self.stored_identity.epoch)?, + self.stored_identity.ref_name, + sql_u64(self.stored_identity.ref_generation)?, + self.stored_identity.change_id, + self.stored_identity.baseline_root_id, + self.stored_identity.policy_fingerprint, + sql_u64(self.stored_identity.policy_generation)?, + self.stored_identity.filesystem_identity, + self.stored_identity.provider_id, + self.stored_identity.provider_identity, + self.stored_identity.observer_owner_token, + sql_u64(current_durable)?, + sql_u64(current_folded)?, + sql_u64(self.stored_identity.continuity_generation)?, + ], + )?; + if changed != 1 { + tx.rollback()?; + return self.fail_report(ledger, "scope changed during reconciliation publication"); + } + tx.execute( + "UPDATE changed_path_reconciliations SET state='published', updated_at=?1 + WHERE attempt_id=?2 AND state='ready'", + params![now, self.attempt_id], + )?; + tx.commit()?; + self.report.published = true; + self.report.refreshed = true; + self.report.trust_state = TrustState::Trusted.as_str().to_string(); + let candidate_rows = ledger.conn.query_row( + "SELECT COUNT(*) FROM changed_path_entries WHERE scope_id=?1", + [self.expected.scope_id.to_text()], + |row| row.get::<_, i64>(0), + )?; + self.report.candidate_rows = db_u64(candidate_rows)?; + Ok(self.report) + } + + fn publish_prefix( + mut self, + ledger: &ChangedPathLedger<'_>, + trail: &Trail, + root_identity: &[u8], + end: &ObserverFence, + ) -> Result { + let ReconcileMode::ProvenPrefixes(proof) = self.mode.clone() else { + unreachable!("prefix publication requires a prefix proof"); + }; + let tx = Transaction::new_unchecked(ledger.conn, TransactionBehavior::Immediate)?; + if validate_stored_scope(&tx, &self.expected, &self.stored_identity).is_err() { + return self.fail_publication_transaction( + tx, + "scope changed during prefix reconciliation publication", + ); + } + let qualification = self + .qualification + .clone() + .expect("publication checked observer qualification"); + if validate_observer_continuity( + &tx, + &self.expected, + &self.stored_identity, + &qualification, + end, + ) + .is_err() + { + return self.fail_observer_continuity_transaction( + tx, + "observer owner or durable segment changed before prefix publication", + ); + } + if validate_prefix_proof(&tx, &self.expected, &proof).is_err() { + return self.fail_publication_transaction( + tx, + "provider owner, cut, or persisted prefix changed before publication", + ); + } + if !matches!(trail.verify_pinned_worktree_root(&self.root), Ok(true)) { + return self.fail_publication_transaction( + tx, + "workspace root identity changed before prefix publication", + ); + } + if self.validate_directory_guards(trail, ledger).is_err() { + return self.fail_publication_transaction( + tx, + "directory identity changed before prefix publication", + ); + } + if validate_ready_attempt(&tx, &self, root_identity).is_err() { + return self.fail_publication_transaction(tx, "prefix reconciliation attempt changed"); + } + + let scope_id = self.stored_identity.scope_id.clone(); + tx.execute_batch("SAVEPOINT changed_path_reconciliation_candidates;")?; + if persist_consumed_observer_fold(&tx, &self.expected, &qualification, end).is_err() { + return self.fail_observer_continuity_candidate_transaction( + tx, + "observer consumed prefix cut could not be atomically folded", + ); + } + for proof_row in &proof.rows { + let prefix = &proof_row.prefix; + let lower = format!("{}/", prefix.as_str()); + let upper = format!("{}0", prefix.as_str()); + tx.execute( + "DELETE FROM changed_path_entries + WHERE scope_id=?1 + AND (normalized_path=?2 COLLATE BINARY OR + (normalized_path>=?3 COLLATE BINARY AND normalized_path=?3 COLLATE BINARY AND normalized_prefix(0), + )?; + tx.commit()?; + self.report.refreshed = true; + self.report.published = false; + self.report.trust_state = trust_state; + self.report.candidate_rows = db_u64(candidate_rows)?; + Ok(self.report) + } + + fn fail_candidate_cap_transaction( + &self, + tx: Transaction<'_>, + ) -> Result { + tx.execute_batch( + "ROLLBACK TO changed_path_reconciliation_candidates; + RELEASE changed_path_reconciliation_candidates;", + )?; + let reason = "reconciliation candidate row cap exceeded"; + tx.execute( + "UPDATE changed_path_scopes + SET trust_state='overflow',trust_reason=?1, + continuity_generation=continuity_generation+1,updated_at=?2 + WHERE scope_id=?3", + params![reason, now_ts(), self.stored_identity.scope_id], + )?; + tx.execute( + "UPDATE changed_path_reconciliations + SET state='failed',reason=?1,updated_at=?2 + WHERE attempt_id=?3 AND state!='published'", + params![reason, now_ts(), self.attempt_id], + )?; + tx.commit()?; + Err(reconcile_required( + &self.expected, + TrustState::Overflow.as_str(), + reason, + )) + } + + fn fail_observer_continuity_transaction( + &self, + tx: Transaction<'_>, + reason: &str, + ) -> Result { + self.commit_observer_continuity_failure(tx, reason) + } + + fn fail_observer_continuity_candidate_transaction( + &self, + tx: Transaction<'_>, + reason: &str, + ) -> Result { + tx.execute_batch( + "ROLLBACK TO changed_path_reconciliation_candidates; + RELEASE changed_path_reconciliation_candidates;", + )?; + self.commit_observer_continuity_failure(tx, reason) + } + + fn commit_observer_continuity_failure( + &self, + tx: Transaction<'_>, + reason: &str, + ) -> Result { + let now = now_ts(); + tx.execute( + "UPDATE changed_path_scopes + SET trust_state='untrusted_gap',trust_reason=?1, + continuity_generation=continuity_generation+1,updated_at=?2 + WHERE scope_id=?3 AND continuity_generation=?4 + AND trust_state IN ('trusted','reconciling')", + params![ + reason, + now, + self.stored_identity.scope_id, + sql_u64(self.stored_identity.continuity_generation)?, + ], + )?; + tx.execute( + "UPDATE changed_path_reconciliations + SET state='failed',reason=?1,updated_at=?2 + WHERE attempt_id=?3 AND state!='published'", + params![reason, now, self.attempt_id], + )?; + tx.commit()?; + Err(reconcile_required( + &self.expected, + TrustState::UntrustedGap.as_str(), + reason, + )) + } + + fn fail(&self, ledger: &ChangedPathLedger<'_>, reason: &str) -> Result { + mark_attempt_failed(ledger.conn, &self.attempt_id, reason)?; + Err(reconcile_required( + &self.expected, + TrustState::Reconciling.as_str(), + reason, + )) + } + + fn fail_report( + &self, + ledger: &ChangedPathLedger<'_>, + reason: &str, + ) -> Result { + self.fail(ledger, reason) + } + + fn fail_publication_transaction( + &self, + tx: Transaction<'_>, + reason: &str, + ) -> Result { + tx.execute( + "UPDATE changed_path_reconciliations + SET state='failed', reason=?1, updated_at=?2 + WHERE attempt_id=?3 AND state!='published'", + params![reason, now_ts(), self.attempt_id], + )?; + tx.commit()?; + Err(reconcile_required( + &self.expected, + TrustState::Reconciling.as_str(), + reason, + )) + } +} + +#[derive(Clone)] +struct StagedRow { + path: String, + row_kind: &'static str, + file_kind: Option, + content_hash: Option, + executable: Option, + size_bytes: Option, + flags: EvidenceFlags, + identity: Option>, + source_sequence: Option, +} + +struct StagedDirectoryGuard { + path: Vec, + identity: Vec, +} + +struct StagingWriter<'a> { + conn: &'a rusqlite::Connection, + attempt_id: &'a str, + pending: Vec, + pending_guards: Vec, + observed_files: u64, + staged_rows: u64, + hashed_bytes: u64, + peak_batch_rows: u64, + peak_buffer_bytes: u64, +} + +impl<'a> StagingWriter<'a> { + fn new(conn: &'a rusqlite::Connection, attempt_id: &'a str) -> Self { + Self { + conn, + attempt_id, + pending: Vec::with_capacity(STAGING_BATCH_ROWS), + pending_guards: Vec::with_capacity(STAGING_BATCH_ROWS), + observed_files: 0, + staged_rows: 0, + hashed_bytes: 0, + peak_batch_rows: 0, + peak_buffer_bytes: 0, + } + } + + fn stage_filesystem(&mut self, file: ReconciliationFile) -> Result<()> { + self.observed_files = self.observed_files.saturating_add(1); + self.hashed_bytes = self.hashed_bytes.saturating_add(file.size_bytes); + self.peak_buffer_bytes = self.peak_buffer_bytes.max(file.peak_buffer_bytes); + self.push(file_row(file, EvidenceFlags::CREATE, None)) + } + + fn stage_prefix_filesystem(&mut self, file: ReconciliationFile, sequence: u64) -> Result<()> { + self.observed_files = self.observed_files.saturating_add(1); + self.hashed_bytes = self.hashed_bytes.saturating_add(file.size_bytes); + self.peak_buffer_bytes = self.peak_buffer_bytes.max(file.peak_buffer_bytes); + self.push(file_row( + file, + EvidenceFlags::CREATE | EvidenceFlags::PROVIDER_COMPLETE_PREFIX, + Some(sequence), + )) + } + + fn clear_prefix(&mut self, prefix: &str) -> Result<()> { + self.flush()?; + let prefix_len = i64::try_from(prefix.len()) + .map_err(|_| Error::InvalidInput("observer prefix length exceeds range".into()))?; + self.conn.execute( + "DELETE FROM changed_path_reconciliation_rows + WHERE attempt_id=?1 AND ( + normalized_path=?2 COLLATE BINARY OR + (substr(normalized_path,1,?3)=?2 COLLATE BINARY + AND substr(normalized_path,?3+1,1)='/') + )", + params![self.attempt_id, prefix, prefix_len], + )?; + self.conn.execute( + "DELETE FROM changed_path_reconciliation_guards + WHERE attempt_id=?1 AND ( + CAST(relative_path AS TEXT)=?2 COLLATE BINARY OR + (substr(CAST(relative_path AS TEXT),1,?3)=?2 COLLATE BINARY + AND substr(CAST(relative_path AS TEXT),?3+1,1)='/') + )", + params![self.attempt_id, prefix, prefix_len], + )?; + Ok(()) + } + + fn stage_directory_guard(&mut self, directory: ReconciliationDirectory) -> Result<()> { + self.pending_guards.push(StagedDirectoryGuard { + path: directory.path.into_bytes(), + identity: directory.identity, + }); + self.after_push() + } + + fn compare_baseline(&mut self, path: String, baseline: FileEntry) -> Result<()> { + let staged = self + .conn + .query_row( + "SELECT file_kind, content_hash, executable, size_bytes, after_identity + FROM changed_path_reconciliation_rows + WHERE attempt_id=?1 AND normalized_path=?2 COLLATE BINARY", + params![self.attempt_id, path], + |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + )) + }, + ) + .optional()?; + match staged { + None => self.push(StagedRow { + path, + row_kind: "deletion", + file_kind: None, + content_hash: None, + executable: None, + size_bytes: None, + flags: EvidenceFlags::DELETE, + identity: None, + source_sequence: None, + }), + Some((kind, hash, executable, size, identity)) => { + let current_kind = kind.unwrap_or_default(); + let current_hash = hash.unwrap_or_default(); + let current_executable = executable.unwrap_or_default() != 0; + let mut flags = EvidenceFlags::default(); + if current_hash != baseline.content_hash + || current_kind != file_kind_label(&baseline.kind) + || size.unwrap_or_default().max(0) as u64 != baseline.size_bytes + { + flags |= EvidenceFlags::CONTENT; + } + if current_executable != baseline.executable { + flags |= EvidenceFlags::MODE; + } + if flags == EvidenceFlags::default() { + self.conn.execute( + "DELETE FROM changed_path_reconciliation_rows + WHERE attempt_id=?1 AND normalized_path=?2 COLLATE BINARY", + params![self.attempt_id, path], + )?; + Ok(()) + } else { + self.push(StagedRow { + path, + row_kind: "entry", + file_kind: Some(current_kind), + content_hash: Some(current_hash), + executable: Some(current_executable), + size_bytes: size.map(|value| value.max(0) as u64), + flags, + identity: identity.and_then(|value| hex::decode(value).ok()), + source_sequence: None, + }) + } + } + } + } + + fn stage_observer_result( + &mut self, + event: ObserverEvent, + current: Option, + baseline: Option, + ) -> Result<()> { + match (current, baseline) { + (None, None) => { + self.conn.execute( + "DELETE FROM changed_path_reconciliation_rows + WHERE attempt_id=?1 AND normalized_path=?2 COLLATE BINARY", + params![self.attempt_id, event.path.as_str()], + )?; + Ok(()) + } + (None, Some(_)) => self.push(StagedRow { + path: event.path.as_str().to_string(), + row_kind: "deletion", + file_kind: None, + content_hash: None, + executable: None, + size_bytes: None, + flags: EvidenceFlags::DELETE | event.flags, + identity: None, + source_sequence: Some(event.sequence), + }), + (Some(file), None) => { + self.observed_files = self.observed_files.saturating_add(1); + self.hashed_bytes = self.hashed_bytes.saturating_add(file.size_bytes); + self.peak_buffer_bytes = self.peak_buffer_bytes.max(file.peak_buffer_bytes); + self.push(file_row( + file, + EvidenceFlags::CREATE | event.flags, + Some(event.sequence), + )) + } + (Some(file), Some(baseline)) => { + let mut flags = EvidenceFlags::default(); + if file.content_hash != baseline.content_hash + || file.file_kind != file_kind_label(&baseline.kind) + || file.size_bytes != baseline.size_bytes + { + flags |= EvidenceFlags::CONTENT; + } + if file.executable != baseline.executable { + flags |= EvidenceFlags::MODE; + } + if flags == EvidenceFlags::default() { + self.conn.execute( + "DELETE FROM changed_path_reconciliation_rows + WHERE attempt_id=?1 AND normalized_path=?2 COLLATE BINARY", + params![self.attempt_id, event.path.as_str()], + )?; + Ok(()) + } else { + self.push(file_row(file, flags | event.flags, Some(event.sequence))) + } + } + } + } + + fn push(&mut self, row: StagedRow) -> Result<()> { + self.pending.push(row); + self.after_push() + } + + fn after_push(&mut self) -> Result<()> { + let pending_rows = self.pending.len().saturating_add(self.pending_guards.len()); + self.peak_batch_rows = self.peak_batch_rows.max(pending_rows as u64); + if pending_rows >= STAGING_BATCH_ROWS { + self.flush()?; + } + Ok(()) + } + + fn flush(&mut self) -> Result<()> { + if self.pending.is_empty() && self.pending_guards.is_empty() { + return Ok(()); + } + let tx = Transaction::new_unchecked(self.conn, TransactionBehavior::Immediate)?; + for row in self.pending.drain(..) { + tx.execute( + "INSERT INTO changed_path_reconciliation_rows( + attempt_id, normalized_path, row_kind, file_kind, + content_hash, executable, size_bytes, before_identity, + after_identity, source_sequence, staged_at + ) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11) + ON CONFLICT(attempt_id,normalized_path) DO UPDATE SET + row_kind=excluded.row_kind, file_kind=excluded.file_kind, + content_hash=excluded.content_hash, executable=excluded.executable, + size_bytes=excluded.size_bytes, before_identity=excluded.before_identity, + after_identity=excluded.after_identity, + source_sequence=excluded.source_sequence, staged_at=excluded.staged_at", + params![ + self.attempt_id, + row.path, + row.row_kind, + row.file_kind, + row.content_hash, + row.executable.map(i64::from), + row.size_bytes.map(sql_u64).transpose()?, + format!("flags:{}", row.flags.0), + row.identity.map(hex::encode), + row.source_sequence.map(sql_u64).transpose()?, + now_ts(), + ], + )?; + self.staged_rows = self.staged_rows.saturating_add(1); + } + for guard in self.pending_guards.drain(..) { + tx.execute( + "INSERT INTO changed_path_reconciliation_guards( + attempt_id,relative_path,directory_identity,staged_at + ) VALUES(?1,?2,?3,?4) + ON CONFLICT(attempt_id,relative_path) DO UPDATE SET + directory_identity=excluded.directory_identity, + staged_at=excluded.staged_at", + params![self.attempt_id, guard.path, guard.identity, now_ts()], + )?; + self.staged_rows = self.staged_rows.saturating_add(1); + } + tx.commit()?; + Ok(()) + } +} + +fn file_row( + file: ReconciliationFile, + flags: EvidenceFlags, + source_sequence: Option, +) -> StagedRow { + StagedRow { + path: file.path, + row_kind: "entry", + file_kind: Some(file.file_kind), + content_hash: Some(file.content_hash), + executable: Some(file.executable), + size_bytes: Some(file.size_bytes), + flags, + identity: Some(file.identity), + source_sequence, + } +} + +fn validate_mode_start( + tx: &Transaction<'_>, + expected: &ExpectedScope, + mode: &ReconcileMode, +) -> Result<()> { + if let ReconcileMode::ProvenPrefixes(proof) = mode { + let state = current_trust_state(tx, expected)?; + if !matches!(state.as_str(), "trusted" | "reconciling") { + return Err(reconcile_required( + expected, + &state, + "continuity failure forces full reconciliation", + )); + } + if proof.prefixes.is_empty() { + return Err(Error::InvalidInput( + "empty provider prefix proof is not authoritative".into(), + )); + } + validate_prefix_proof(tx, expected, proof)?; + } + Ok(()) +} + +fn validate_prefix_proof( + conn: &rusqlite::Connection, + expected: &ExpectedScope, + proof: &ProvenPrefixSet, +) -> Result<()> { + let scope_id = expected.scope_id.to_text(); + let scope_matches = conn.query_row( + "SELECT COUNT(*) FROM changed_path_scopes + WHERE scope_id=?1 AND epoch=?2 AND provider_id=?3 + AND provider_identity=?4 AND provider_cursor IS ?5 + AND provider_fence IS ?6 AND durable_offset=?7 AND folded_offset=?8 + AND trust_state IN ('trusted','reconciling') + AND continuity_generation=?9", + params![ + scope_id, + sql_u64(proof.epoch)?, + proof.provider_id, + proof.provider_identity, + proof.provider_cursor, + proof.provider_fence, + sql_u64(proof.durable_offset)?, + sql_u64(proof.folded_offset)?, + sql_u64(proof.continuity_generation)?, + ], + |row| row.get::<_, i64>(0), + )? == 1; + let owner_matches = conn.query_row( + "SELECT COUNT(*) FROM changed_path_observer_owners + WHERE scope_id=?1 AND epoch=?2 AND owner_token=?3 + AND provider_id=?4 AND provider_identity=?5 + AND fence_nonce IS ?6 AND lease_state='active' AND expires_at>=?7", + params![ + scope_id, + sql_u64(proof.epoch)?, + proof.owner_token, + proof.provider_id, + proof.provider_identity, + proof.owner_fence_nonce, + now_ts(), + ], + |row| row.get::<_, i64>(0), + )? == 1; + if !scope_matches || !owner_matches { + return Err(reconcile_required( + expected, + TrustState::Reconciling.as_str(), + "provider owner or exact cut changed; full reconciliation required", + )); + } + for row in &proof.rows { + let matched = conn.query_row( + "SELECT COUNT(*) FROM changed_path_prefixes + WHERE scope_id=?1 AND normalized_prefix=?2 COLLATE BINARY + AND completeness_reason='provider_complete' AND event_flags=?3 + AND source_mask=?4 AND first_sequence=?5 AND last_sequence=?6 + AND provider_id=?7 AND provider_sequence=?8 AND intent_id IS NULL + AND created_at=?9 AND updated_at=?10", + params![ + scope_id, + row.prefix.as_str(), + row.event_flags, + row.source_mask, + sql_u64(row.first_sequence)?, + sql_u64(row.last_sequence)?, + proof.provider_id, + sql_u64(row.provider_sequence)?, + row.created_at, + row.updated_at, + ], + |query| query.get::<_, i64>(0), + )?; + if matched != 1 { + return Err(reconcile_required( + expected, + TrustState::Reconciling.as_str(), + "persisted provider-complete prefix changed; full reconciliation required", + )); + } + } + Ok(()) +} + +fn validate_scope_capabilities(tx: &Transaction<'_>, expected: &ExpectedScope) -> Result<()> { + let qualified = tx.query_row( + "SELECT clean_proof_allowed=1 AND linearizable_fence=1 + AND filesystem_supported=1 + FROM changed_path_scopes WHERE scope_id=?1", + [expected.scope_id.to_text()], + |row| row.get::<_, bool>(0), + )?; + if !qualified { + return Err(reconcile_required( + expected, + TrustState::Reconciling.as_str(), + "provider capabilities do not permit a clean proof", + )); + } + Ok(()) +} + +fn validate_observer_continuity( + tx: &Transaction<'_>, + expected: &ExpectedScope, + identity: &StoredAttemptIdentity, + qualification: &ObserverQualification, + end: &ObserverFence, +) -> Result<()> { + validate_observer_continuity_at( + tx, + expected, + identity, + qualification, + end, + now_ts().saturating_add(1), + ) +} + +fn persist_consumed_observer_fold( + tx: &Transaction<'_>, + expected: &ExpectedScope, + qualification: &ObserverQualification, + end: &ObserverFence, +) -> Result<()> { + if qualification.segment_consumed_offset != end.durable_offset { + return Err(reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "observer consumed cut does not equal the authenticated end fence", + )); + } + let changed = tx.execute( + "UPDATE changed_path_observer_segments + SET folded_end_offset=?1,updated_at=?2 + WHERE scope_id=?3 AND epoch=?4 AND segment_id=?5 + AND owner_token=?6 AND state IN ('open','sealed') + AND durable_end_offset>=?1 AND last_sequence>=?7 + AND folded_end_offset>=?8 AND folded_end_offset<=?1", + params![ + sql_u64(qualification.segment_consumed_offset)?, + now_ts(), + expected.scope_id.to_text(), + sql_u64(expected.epoch)?, + qualification.durable_segment_id, + qualification.observer_owner_token, + sql_u64(end.sequence)?, + sql_u64(qualification.segment_initial_folded_offset)?, + ], + )?; + if changed != 1 { + return Err(reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "observer consumed cut did not match the durable owner segment", + )); + } + Ok(()) +} + +fn qualification_segment_folded_offset( + conn: &rusqlite::Connection, + expected: &ExpectedScope, + qualification: &ObserverQualification, +) -> Result { + let folded = conn + .query_row( + "SELECT folded_end_offset FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=?2 AND segment_id=?3 AND owner_token=?4 + AND provider_id=?5 AND state IN ('open','sealed')", + params![ + expected.scope_id.to_text(), + sql_u64(expected.epoch)?, + qualification.durable_segment_id, + qualification.observer_owner_token, + hex::encode(&expected.provider_identity), + ], + |row| row.get::<_, i64>(0), + ) + .optional()? + .ok_or_else(|| { + reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "observer durable segment was unavailable after drain", + ) + })?; + db_u64(folded) +} + +fn validate_observer_continuity_at( + tx: &Transaction<'_>, + expected: &ExpectedScope, + identity: &StoredAttemptIdentity, + qualification: &ObserverQualification, + end: &ObserverFence, + minimum_lease_expiry: i64, +) -> Result<()> { + if identity.observer_owner_token.as_deref() != Some(qualification.observer_owner_token.as_str()) + { + return Err(reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "sealed observer owner does not match the reconciliation scope", + )); + } + let owner_matches = tx.query_row( + "SELECT COUNT(*) FROM changed_path_observer_owners + WHERE scope_id=?1 AND epoch=?2 AND owner_token=?3 + AND provider_id IS ?4 AND provider_identity IS ?5 + AND fence_nonce IS ?6 AND lease_state='active' AND expires_at>=?7", + params![ + identity.scope_id, + sql_u64(identity.epoch)?, + qualification.observer_owner_token, + identity.provider_id, + identity.provider_identity, + qualification.owner_fence_nonce, + minimum_lease_expiry, + ], + |row| row.get::<_, i64>(0), + )? == 1; + let segment_matches = tx.query_row( + "SELECT COUNT(*) FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=?2 AND segment_id=?3 + AND owner_token=?4 AND provider_id IS ?5 + AND state IN ('open','sealed') AND last_sequence IS NOT NULL + AND last_sequence>=?6 AND durable_end_offset>=?7", + params![ + identity.scope_id, + sql_u64(identity.epoch)?, + qualification.durable_segment_id, + qualification.observer_owner_token, + identity.provider_id, + sql_u64(end.sequence)?, + sql_u64(qualification.segment_durable_offset)?, + ], + |row| row.get::<_, i64>(0), + )? == 1; + if !owner_matches || !segment_matches { + return Err(reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "sealed observer owner or durable segment continuity is unavailable", + )); + } + Ok(()) +} + +fn publication_lease_deadline() -> Result { + now_ts() + .checked_add(MIN_PUBLICATION_LEASE_HORIZON_SECS) + .ok_or_else(|| Error::Corrupt("observer publication lease deadline overflowed".into())) +} + +fn validate_ready_attempt( + tx: &Transaction<'_>, + attempt: &ReconciliationAttempt, + root_identity: &[u8], +) -> Result<()> { + if attempt.stored_identity.root_handle_identity != root_identity + || serde_json::to_vec(&attempt.stored_identity)? != attempt.encoded_identity + { + return Err(reconcile_required( + &attempt.expected, + TrustState::Reconciling.as_str(), + "reconciliation attempt's pinned identity changed", + )); + } + let matched = tx.query_row( + "SELECT COUNT(*) FROM changed_path_reconciliations + WHERE attempt_id=?1 AND scope_id=?2 AND expected_scope_epoch=?3 + AND expected_ref_name=?4 AND expected_ref_generation=?5 + AND expected_change_id=?6 AND expected_root_id=?7 + AND filesystem_identity=?8 AND policy_fingerprint=?9 + AND policy_dependency_generation=?10 AND provider_id IS ?11 + AND provider_identity IS ?12 AND start_fence=?13 AND state='ready'", + params![ + attempt.attempt_id, + attempt.stored_identity.scope_id, + sql_u64(attempt.stored_identity.epoch)?, + attempt.stored_identity.ref_name, + sql_u64(attempt.stored_identity.ref_generation)?, + attempt.stored_identity.change_id, + attempt.stored_identity.baseline_root_id, + attempt.stored_identity.filesystem_identity, + attempt.stored_identity.policy_fingerprint, + sql_u64(attempt.stored_identity.policy_generation)?, + attempt.stored_identity.provider_id, + attempt.stored_identity.provider_identity, + attempt.encoded_identity, + ], + |row| row.get::<_, i64>(0), + )?; + if matched != 1 { + return Err(reconcile_required( + &attempt.expected, + TrustState::Reconciling.as_str(), + "reconciliation attempt identity changed", + )); + } + Ok(()) +} + +fn capture_attempt_identity( + conn: &rusqlite::Connection, + expected: &ExpectedScope, + start_fence: ObserverFence, + root_handle_identity: Vec, +) -> Result { + let identity = conn.query_row( + "SELECT scope_id,scope_root,scope_root_identity,case_sensitive,epoch, + ref_name,ref_generation,change_id,baseline_root_id, + policy_fingerprint,policy_dependency_generation, + trust_state,continuity_generation,filesystem_identity, + provider_id,provider_identity,observer_owner_token, + durable_offset,folded_offset + FROM changed_path_scopes WHERE scope_id=?1", + [expected.scope_id.to_text()], + |row| { + let epoch = row.get::<_, i64>(4)?; + let ref_generation = row.get::<_, i64>(6)?; + let policy_generation = row.get::<_, i64>(10)?; + let continuity_generation = row.get::<_, i64>(12)?; + let durable_offset = row.get::<_, i64>(17)?; + let folded_offset = row.get::<_, i64>(18)?; + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, bool>(3)?, + epoch, + row.get::<_, String>(5)?, + ref_generation, + row.get::<_, String>(7)?, + row.get::<_, String>(8)?, + row.get::<_, String>(9)?, + policy_generation, + row.get::<_, String>(11)?, + continuity_generation, + row.get::<_, String>(13)?, + row.get::<_, Option>(14)?, + row.get::<_, Option>(15)?, + row.get::<_, Option>(16)?, + durable_offset, + folded_offset, + )) + }, + )?; + let ( + scope_id, + scope_root, + scope_root_identity, + case_sensitive, + epoch, + ref_name, + ref_generation, + change_id, + baseline_root_id, + policy_fingerprint, + policy_generation, + trust_state, + continuity_generation, + filesystem_identity, + provider_id, + provider_identity, + observer_owner_token, + durable_offset, + folded_offset, + ) = identity; + let identity = StoredAttemptIdentity { + scope_id, + scope_root, + scope_root_identity, + case_sensitive, + epoch: db_u64(epoch)?, + ref_name, + ref_generation: db_u64(ref_generation)?, + change_id, + baseline_root_id, + policy_fingerprint, + policy_generation: db_u64(policy_generation)?, + trust_state, + continuity_generation: db_u64(continuity_generation)?, + filesystem_identity, + provider_id, + provider_identity, + observer_owner_token, + initial_durable_offset: db_u64(durable_offset)?, + initial_folded_offset: db_u64(folded_offset)?, + start_fence, + root_handle_identity, + }; + if identity.initial_folded_offset > identity.initial_durable_offset + || identity.scope_id != expected.scope_id.to_text() + || identity.epoch != expected.epoch + || identity.ref_name != expected.ref_name + || identity.ref_generation != expected.ref_generation + || identity.baseline_root_id != expected.baseline_root.0 + || identity.policy_fingerprint != hex::encode(expected.policy_fingerprint) + || identity.policy_generation != expected.policy_generation + || !matches!(identity.trust_state.as_str(), "trusted" | "reconciling") + || identity.continuity_generation == 0 + || identity.filesystem_identity != hex::encode(&expected.filesystem_identity) + || identity.provider_identity.as_deref() + != Some(hex::encode(&expected.provider_identity).as_str()) + { + return Err(reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "full scope identity or initial cuts changed before reconciliation began", + )); + } + Ok(identity) +} + +fn validate_stored_scope( + conn: &rusqlite::Connection, + expected: &ExpectedScope, + identity: &StoredAttemptIdentity, +) -> Result<(u64, u64)> { + let cuts = conn + .query_row( + "SELECT durable_offset,folded_offset FROM changed_path_scopes + WHERE scope_id=?1 AND scope_root=?2 AND scope_root_identity=?3 + AND case_sensitive=?4 AND epoch=?5 AND ref_name=?6 + AND ref_generation=?7 AND change_id=?8 AND baseline_root_id=?9 + AND policy_fingerprint=?10 AND policy_dependency_generation=?11 + AND filesystem_identity=?12 AND provider_id IS ?13 + AND provider_identity IS ?14 AND observer_owner_token IS ?15 + AND trust_state=?16 AND continuity_generation=?17", + params![ + identity.scope_id, + identity.scope_root, + identity.scope_root_identity, + identity.case_sensitive, + sql_u64(identity.epoch)?, + identity.ref_name, + sql_u64(identity.ref_generation)?, + identity.change_id, + identity.baseline_root_id, + identity.policy_fingerprint, + sql_u64(identity.policy_generation)?, + identity.filesystem_identity, + identity.provider_id, + identity.provider_identity, + identity.observer_owner_token, + identity.trust_state, + sql_u64(identity.continuity_generation)?, + ], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)), + ) + .optional()?; + let Some((durable, folded)) = cuts else { + return Err(reconcile_required( + expected, + TrustState::Reconciling.as_str(), + "stored reconciliation scope identity changed", + )); + }; + let durable = db_u64(durable)?; + let folded = db_u64(folded)?; + if folded > durable { + return Err(Error::Corrupt( + "changed-path scope folded cut exceeds durable cut".into(), + )); + } + Ok((durable, folded)) +} + +fn exact_scope_guard(conn: &rusqlite::Connection, expected: &ExpectedScope) -> Result<()> { + let matched = conn.query_row( + "SELECT COUNT(*) FROM changed_path_scopes + WHERE scope_id=?1 AND epoch=?2 AND ref_name=?3 AND ref_generation=?4 + AND baseline_root_id=?5 AND policy_fingerprint=?6 + AND policy_dependency_generation=?7 AND filesystem_identity=?8 + AND provider_identity=?9", + params![ + expected.scope_id.to_text(), + sql_u64(expected.epoch)?, + expected.ref_name, + sql_u64(expected.ref_generation)?, + expected.baseline_root.0, + hex::encode(expected.policy_fingerprint), + sql_u64(expected.policy_generation)?, + hex::encode(&expected.filesystem_identity), + hex::encode(&expected.provider_identity), + ], + |row| row.get::<_, i64>(0), + )?; + if matched != 1 { + return Err(reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "expected scope identity changed", + )); + } + Ok(()) +} + +fn exact_scope_update_state( + conn: &rusqlite::Connection, + expected: &ExpectedScope, + state: TrustState, + reason: &str, +) -> Result<()> { + let changed = conn.execute( + "UPDATE changed_path_scopes SET trust_state=?1, trust_reason=?2, + continuity_generation=continuity_generation+1, updated_at=?3 + WHERE scope_id=?4 AND epoch=?5 AND ref_name=?6 AND ref_generation=?7 + AND baseline_root_id=?8 AND policy_fingerprint=?9 + AND policy_dependency_generation=?10 AND filesystem_identity=?11 + AND provider_identity=?12", + params![ + state.as_str(), + reason, + now_ts(), + expected.scope_id.to_text(), + sql_u64(expected.epoch)?, + expected.ref_name, + sql_u64(expected.ref_generation)?, + expected.baseline_root.0, + hex::encode(expected.policy_fingerprint), + sql_u64(expected.policy_generation)?, + hex::encode(&expected.filesystem_identity), + hex::encode(&expected.provider_identity), + ], + )?; + if changed != 1 { + return Err(reconcile_required( + expected, + TrustState::UntrustedGap.as_str(), + "expected scope changed while reconciliation started", + )); + } + Ok(()) +} + +fn current_trust_state(conn: &rusqlite::Connection, expected: &ExpectedScope) -> Result { + exact_scope_guard(conn, expected)?; + conn.query_row( + "SELECT trust_state FROM changed_path_scopes WHERE scope_id=?1", + [expected.scope_id.to_text()], + |row| row.get(0), + ) + .map_err(Error::from) +} + +fn mark_attempt_failed(conn: &rusqlite::Connection, attempt_id: &str, reason: &str) -> Result<()> { + conn.execute( + "UPDATE changed_path_reconciliations + SET state='failed', reason=?1, updated_at=?2 + WHERE attempt_id=?3 AND state!='published'", + params![reason, now_ts(), attempt_id], + )?; + Ok(()) +} + +fn evidence_cap_exceeded(conn: &rusqlite::Connection, scope_id: &str) -> Result { + let (candidate_count, candidate_cap, prefix_count, prefix_cap): (i64, i64, i64, i64) = conn + .query_row( + "SELECT (SELECT COUNT(*) FROM changed_path_entries WHERE scope_id=?1), + max_candidate_rows, + (SELECT COUNT(*) FROM changed_path_prefixes WHERE scope_id=?1), + max_prefix_rows + FROM changed_path_scopes WHERE scope_id=?1", + [scope_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + )?; + Ok(db_u64(candidate_count)? > db_u64(candidate_cap)? + || db_u64(prefix_count)? > db_u64(prefix_cap)?) +} + +fn reconcile_required(expected: &ExpectedScope, state: &str, reason: &str) -> Error { + Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: state.to_string(), + reason: reason.to_string(), + command: "trail status".to_string(), + } +} + +fn file_kind_label(kind: &FileKind) -> &'static str { + match kind { + FileKind::Text => "Text", + FileKind::OpaqueText => "OpaqueText", + FileKind::Binary => "Binary", + } +} + +fn sql_u64(value: u64) -> Result { + i64::try_from(value).map_err(|_| Error::InvalidInput("value exceeds SQLite INTEGER".into())) +} + +fn db_u64(value: i64) -> Result { + u64::try_from(value).map_err(|_| Error::Corrupt("negative reconciliation count".into())) +} + +#[cfg(debug_assertions)] +mod compiled_harness { + use super::*; + use crate::db::change_ledger::{ + BaselineIdentity, FilesystemIdentity, PolicyIdentity, ProviderCapabilities, + ProviderIdentity, RecordingPolicySnapshot, ScopeIdentity, ScopeKind, + }; + use crate::InitImportMode; + use sha2::{Digest, Sha256}; + use std::fs; + + struct HarnessObserver { + end_sequence: u64, + } + + impl QualifiedObserver for HarnessObserver { + fn begin_observation(&self, _expected: &ExpectedScope) -> Result { + Ok(ObserverFence { + sequence: 10, + durable_offset: 100, + nonce: b"compiled-harness-start".to_vec(), + }) + } + + fn end_fence( + &self, + _expected: &ExpectedScope, + _start: &ObserverFence, + ) -> Result { + Ok(ObserverFence { + sequence: self.end_sequence, + durable_offset: 100, + nonce: b"compiled-harness-end".to_vec(), + }) + } + + fn drain_through( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + _sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + ) -> Result { + Ok(ObserverQualification::seal_for_test( + expected, + root_handle_identity.to_vec(), + start.clone(), + end.clone(), + )) + } + } + + struct CallbackObserver<'a> { + conn: &'a rusqlite::Connection, + path: std::path::PathBuf, + } + + // The compiled harness invokes this observer synchronously on its creating + // thread; the unsafe marker only lets it satisfy the production daemon + // contract without pretending rusqlite itself is thread-safe. + unsafe impl Send for CallbackObserver<'_> {} + unsafe impl Sync for CallbackObserver<'_> {} + + impl QualifiedObserver for CallbackObserver<'_> { + fn begin_observation(&self, _expected: &ExpectedScope) -> Result { + Ok(ObserverFence { + sequence: 10, + durable_offset: 100, + nonce: b"compiled-callback-start".to_vec(), + }) + } + + fn end_fence( + &self, + _expected: &ExpectedScope, + _start: &ObserverFence, + ) -> Result { + Ok(ObserverFence { + sequence: 266, + durable_offset: 100, + nonce: b"compiled-callback-end".to_vec(), + }) + } + + fn drain_through( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + ) -> Result { + let transaction = + Transaction::new_unchecked(self.conn, TransactionBehavior::Immediate)?; + fs::write(&self.path, b"during callback\n")?; + for sequence in 11..=266 { + sink(ObserverEvent { + path: LedgerPath::parse("modify.txt")?, + flags: EvidenceFlags::CONTENT, + sequence, + })?; + } + fs::write(&self.path, b"after callback\n")?; + transaction.commit()?; + Ok(ObserverQualification::seal_for_test( + expected, + root_handle_identity.to_vec(), + start.clone(), + end.clone(), + )) + } + } + + struct HarnessFixture { + _temp: tempfile::TempDir, + db: Trail, + expected: ExpectedScope, + policy: CompiledPolicy, + } + + impl HarnessFixture { + fn new() -> Result { + let temp = tempfile::tempdir()?; + fs::write(temp.path().join("modify.txt"), b"before\n")?; + fs::write(temp.path().join("delete.txt"), b"delete\n")?; + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false)?; + let db = Trail::open(temp.path())?; + let branch = db.current_branch()?; + let head = db.resolve_branch_ref(&branch)?; + let scope = ScopeIdentity { + scope_id: ScopeId([0x6b; 32]), + kind: ScopeKind::Workspace, + owner_id: "compiled-reconciliation-harness".into(), + }; + let fingerprint = [0x6c; 32]; + let baseline = BaselineIdentity { + ref_name: head.name.clone(), + ref_generation: u64::try_from(head.generation) + .map_err(|_| Error::Corrupt("negative harness ref generation".into()))?, + change_id: head.change_id, + root_id: head.root_id.clone(), + }; + let filesystem = FilesystemIdentity(vec![0x6d]); + let provider = ProviderIdentity { + identity: vec![0x6e], + capabilities: ProviderCapabilities { + durable_cursor: true, + linearizable_fence: true, + rename_pairing: true, + overflow_scope: true, + filesystem_supported: true, + clean_proof_allowed: true, + power_loss_durability: false, + }, + }; + db.changed_path_ledger().begin_scope( + &scope, + &baseline, + &PolicyIdentity { + fingerprint, + generation: 1, + }, + &filesystem, + &provider, + )?; + let expected = ExpectedScope { + scope_id: scope.scope_id, + epoch: 1, + ref_name: baseline.ref_name, + ref_generation: baseline.ref_generation, + baseline_root: baseline.root_id, + policy_fingerprint: fingerprint, + policy_generation: 1, + filesystem_identity: filesystem.0, + provider_identity: provider.identity, + }; + install_continuity(&db.conn, &expected)?; + let policy = CompiledPolicy::for_reconciliation_test( + RecordingPolicySnapshot { + workspace_root: db.workspace_root.clone(), + ignore_gitignored: true, + dependency_files: Vec::new(), + case_sensitive: true, + rule_sources: Vec::new(), + }, + fingerprint, + &expected, + ); + Ok(Self { + _temp: temp, + db, + expected, + policy, + }) + } + + fn begin_observed( + &self, + observer: &dyn QualifiedObserver, + ) -> Result { + let ledger = self.db.changed_path_ledger(); + let mut attempt = begin_reconciliation( + &self.db, + &ledger, + observer, + &self.expected, + &self.policy, + ReconcileMode::Full, + "compiled_harness", + )?; + attempt.observe(&self.db, &ledger, observer, &self.policy)?; + Ok(attempt) + } + } + + fn install_continuity(conn: &rusqlite::Connection, expected: &ExpectedScope) -> Result<()> { + let scope_id = expected.scope_id.to_text(); + let provider_id = hex::encode(&expected.provider_identity); + let now = now_ts(); + conn.execute( + "UPDATE changed_path_scopes + SET observer_owner_token='full-test-owner',durable_offset=100,folded_offset=100 + WHERE scope_id=?1", + [&scope_id], + )?; + conn.execute( + "INSERT INTO changed_path_observer_owners( + scope_id,epoch,owner_token,provider_id,provider_identity, + lease_state,fence_nonce,acquired_at,heartbeat_at,expires_at,updated_at + ) VALUES(?1,?2,'full-test-owner',?3,?4,'active',?5,?6,?6,?7,?6)", + params![ + scope_id, + sql_u64(expected.epoch)?, + provider_id, + hex::encode(&expected.provider_identity), + b"full-test-fence".as_slice(), + now, + now + 3_600, + ], + )?; + conn.execute( + "INSERT INTO changed_path_observer_segments( + scope_id,epoch,segment_id,owner_token,provider_id, + first_sequence,last_sequence,durable_end_offset,folded_end_offset, + segment_path,state,created_at,updated_at + ) VALUES(?1,?2,'full-test-segment','full-test-owner',?3, + 1,1000,100,100,'full-test-segment.cpl','open',?4,?4)", + params![scope_id, sql_u64(expected.epoch)?, provider_id, now], + )?; + Ok(()) + } + + fn require(condition: bool, message: &str) -> Result<()> { + if condition { + Ok(()) + } else { + Err(Error::Corrupt(message.into())) + } + } + + fn oracle() -> Result<()> { + let fixture = HarnessFixture::new()?; + fs::write(fixture.db.workspace_root.join("modify.txt"), b"after\n")?; + fs::remove_file(fixture.db.workspace_root.join("delete.txt"))?; + fs::write(fixture.db.workspace_root.join("add.txt"), b"added\n")?; + let observer = HarnessObserver { end_sequence: 10 }; + let ledger = fixture.db.changed_path_ledger(); + let report = reconcile_full( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + "compiled_oracle", + )?; + let rows = fixture + .db + .conn + .prepare( + "SELECT normalized_path,event_flags FROM changed_path_entries + WHERE scope_id=?1 ORDER BY normalized_path COLLATE BINARY", + )? + .query_map([fixture.expected.scope_id.to_text()], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })? + .collect::, _>>()?; + require(report.published, "compiled oracle did not publish")?; + require( + rows == vec![ + ("add.txt".into(), EvidenceFlags::CREATE.0), + ("delete.txt".into(), EvidenceFlags::DELETE.0), + ("modify.txt".into(), EvidenceFlags::CONTENT.0), + ], + "compiled reconciliation oracle mismatch", + ) + } + + fn races() -> Result<()> { + let fixture = HarnessFixture::new()?; + fs::write(fixture.db.workspace_root.join("add.txt"), b"added\n")?; + let observer = HarnessObserver { end_sequence: 10 }; + let attempt = fixture.begin_observed(&observer)?; + let ledger = fixture.db.changed_path_ledger(); + ledger.mark_untrusted( + &fixture.expected, + TrustState::StaleBaseline, + "compiled concurrent invalidation", + )?; + require( + attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .is_err(), + "compiled fail-closed state race promoted trust", + )?; + + let fixture = HarnessFixture::new()?; + let attempt = fixture.begin_observed(&observer)?; + fixture.db.conn.execute( + "UPDATE changed_path_observer_owners SET lease_state='revoked' + WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + )?; + let ledger = fixture.db.changed_path_ledger(); + require( + attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .is_err(), + "compiled owner-loss race promoted trust", + ) + } + + fn callback_spool() -> Result<()> { + let fixture = HarnessFixture::new()?; + let observer = CallbackObserver { + conn: &fixture.db.conn, + path: fixture.db.workspace_root.join("modify.txt"), + }; + let attempt = fixture.begin_observed(&observer)?; + let staged_hash: String = fixture.db.conn.query_row( + "SELECT content_hash FROM changed_path_reconciliation_rows + WHERE attempt_id=?1 AND normalized_path='modify.txt'", + [&attempt.attempt_id], + |row| row.get(0), + )?; + require( + staged_hash == hex::encode(Sha256::digest(b"after callback\n")), + "compiled callback harness replayed before drain returned", + ) + } + + pub(crate) fn run_oracle() -> std::result::Result<(), String> { + oracle().map_err(|error| error.to_string()) + } + + pub(crate) fn run_races() -> std::result::Result<(), String> { + races().map_err(|error| error.to_string()) + } + + pub(crate) fn run_callback_spool() -> std::result::Result<(), String> { + callback_spool().map_err(|error| error.to_string()) + } +} + +#[cfg(debug_assertions)] +pub(crate) use compiled_harness::{run_callback_spool, run_oracle, run_races}; + +#[cfg(test)] +mod tests { + use super::*; + use sha2::{Digest, Sha256}; + use std::fs; + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt; + use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + use std::sync::Mutex; + + use crate::db::change_ledger::{ + BaselineIdentity, FilesystemIdentity, PolicyIdentity, ProviderCapabilities, + ProviderIdentity, RecordingPolicySnapshot, ScopeIdentity, ScopeKind, + }; + use crate::{InitImportMode, ObjectId}; + + struct FakeQualifiedObserver { + began: AtomicBool, + drains: AtomicU64, + sequence: AtomicU64, + corrupt_proof: bool, + } + + struct EventDuringFenceObserver { + event: ObserverEvent, + mutation: Mutex>>, + } + + #[derive(Clone, Copy)] + enum ObserverFailurePoint { + EndFence, + Drain, + Callback, + } + + struct FailingObserver { + point: ObserverFailurePoint, + callback_path: Option, + } + + impl QualifiedObserver for FailingObserver { + fn begin_observation(&self, _expected: &ExpectedScope) -> Result { + Ok(ObserverFence { + sequence: 10, + durable_offset: 100, + nonce: b"failure-start".to_vec(), + }) + } + + fn end_fence( + &self, + _expected: &ExpectedScope, + _start: &ObserverFence, + ) -> Result { + if matches!(self.point, ObserverFailurePoint::EndFence) { + return Err(Error::InvalidInput("injected end fence failure".into())); + } + Ok(ObserverFence { + sequence: 11, + durable_offset: 100, + nonce: b"failure-end".to_vec(), + }) + } + + fn drain_through( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + ) -> Result { + if matches!(self.point, ObserverFailurePoint::Drain) { + return Err(Error::InvalidInput("injected drain failure".into())); + } + if matches!(self.point, ObserverFailurePoint::Callback) { + sink(ObserverEvent { + path: self.callback_path.clone().unwrap(), + flags: EvidenceFlags::CONTENT, + sequence: 11, + })?; + } + Ok(ObserverQualification::seal_for_test( + expected, + root_handle_identity.to_vec(), + start.clone(), + end.clone(), + )) + } + } + + struct RetryOnceObserver { + attempts: AtomicU64, + } + + struct PrimaryTransactionObserver<'a> { + conn: &'a rusqlite::Connection, + path: std::path::PathBuf, + } + + // This test double is only invoked synchronously by reconciliation. It + // deliberately holds the primary connection to prove callbacks merely + // spool while that connection has an open transaction. + unsafe impl Send for PrimaryTransactionObserver<'_> {} + unsafe impl Sync for PrimaryTransactionObserver<'_> {} + + impl QualifiedObserver for PrimaryTransactionObserver<'_> { + fn begin_observation(&self, _expected: &ExpectedScope) -> Result { + Ok(ObserverFence { + sequence: 10, + durable_offset: 100, + nonce: b"spool-start".to_vec(), + }) + } + + fn end_fence( + &self, + _expected: &ExpectedScope, + _start: &ObserverFence, + ) -> Result { + Ok(ObserverFence { + sequence: 266, + durable_offset: 100, + nonce: b"spool-end".to_vec(), + }) + } + + fn drain_through( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + ) -> Result { + let tx = Transaction::new_unchecked(self.conn, TransactionBehavior::Immediate)?; + fs::write(&self.path, b"during callback\n")?; + for sequence in 11..=266 { + sink(ObserverEvent { + path: LedgerPath::parse("modify.txt")?, + flags: EvidenceFlags::CONTENT, + sequence, + })?; + } + fs::write(&self.path, b"after callback\n")?; + tx.commit()?; + Ok(ObserverQualification::seal_for_test( + expected, + root_handle_identity.to_vec(), + start.clone(), + end.clone(), + )) + } + } + + impl QualifiedObserver for RetryOnceObserver { + fn begin_observation(&self, _expected: &ExpectedScope) -> Result { + let attempt = self.attempts.fetch_add(1, Ordering::SeqCst) + 1; + Ok(ObserverFence { + sequence: 10, + durable_offset: 100, + nonce: format!("retry-start-{attempt}").into_bytes(), + }) + } + + fn end_fence( + &self, + _expected: &ExpectedScope, + _start: &ObserverFence, + ) -> Result { + if self.attempts.load(Ordering::SeqCst) == 1 { + return Err(Error::InvalidInput( + "workspace root identity race; retry reconciliation".into(), + )); + } + Ok(ObserverFence { + sequence: 10, + durable_offset: 100, + nonce: b"retry-end".to_vec(), + }) + } + + fn drain_through( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + _sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + ) -> Result { + Ok(ObserverQualification::seal_for_test( + expected, + root_handle_identity.to_vec(), + start.clone(), + end.clone(), + )) + } + } + + impl QualifiedObserver for EventDuringFenceObserver { + fn begin_observation(&self, _expected: &ExpectedScope) -> Result { + Ok(ObserverFence { + sequence: self.event.sequence - 1, + durable_offset: 100, + nonce: b"event-start".to_vec(), + }) + } + + fn end_fence( + &self, + _expected: &ExpectedScope, + _start: &ObserverFence, + ) -> Result { + if let Some(mutation) = self + .mutation + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .take() + { + mutation(); + } + Ok(ObserverFence { + sequence: self.event.sequence, + durable_offset: 100, + nonce: b"event-end".to_vec(), + }) + } + + fn drain_through( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + ) -> Result { + sink(self.event.clone())?; + Ok(ObserverQualification::seal_for_test( + expected, + root_handle_identity.to_vec(), + start.clone(), + end.clone(), + )) + } + } + + impl FakeQualifiedObserver { + fn new() -> Self { + Self { + began: AtomicBool::new(false), + drains: AtomicU64::new(0), + sequence: AtomicU64::new(10), + corrupt_proof: false, + } + } + + fn with_corrupt_proof() -> Self { + Self { + began: AtomicBool::new(false), + drains: AtomicU64::new(0), + sequence: AtomicU64::new(10), + corrupt_proof: true, + } + } + } + + impl QualifiedObserver for FakeQualifiedObserver { + fn begin_observation(&self, _expected: &ExpectedScope) -> Result { + assert!(!self.began.swap(true, Ordering::SeqCst)); + Ok(ObserverFence { + sequence: self.sequence.load(Ordering::SeqCst), + durable_offset: 100, + nonce: b"start".to_vec(), + }) + } + + fn end_fence( + &self, + _expected: &ExpectedScope, + _start: &ObserverFence, + ) -> Result { + assert!(self.began.load(Ordering::SeqCst)); + Ok(ObserverFence { + sequence: self.sequence.load(Ordering::SeqCst), + durable_offset: 100, + nonce: b"end".to_vec(), + }) + } + + fn drain_through( + &self, + expected: &ExpectedScope, + root_handle_identity: &[u8], + start: &ObserverFence, + end: &ObserverFence, + _sink: &mut dyn FnMut(ObserverEvent) -> Result<()>, + ) -> Result { + self.drains.fetch_add(1, Ordering::SeqCst); + let mut qualification = ObserverQualification::seal_for_test( + expected, + root_handle_identity.to_vec(), + start.clone(), + end.clone(), + ); + if self.corrupt_proof { + qualification.provider_identity.push(0xff); + } + Ok(qualification) + } + } + + struct Fixture { + _temp: tempfile::TempDir, + db: Trail, + expected: ExpectedScope, + policy: CompiledPolicy, + } + + fn install_test_observer_continuity(conn: &rusqlite::Connection, expected: &ExpectedScope) { + let scope_id = expected.scope_id.to_text(); + let provider_id = hex::encode(&expected.provider_identity); + let now = now_ts(); + conn.execute( + "UPDATE changed_path_scopes + SET observer_owner_token='full-test-owner',durable_offset=100,folded_offset=100 + WHERE scope_id=?1", + [&scope_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO changed_path_observer_owners( + scope_id,epoch,owner_token,provider_id,provider_identity, + lease_state,fence_nonce,acquired_at,heartbeat_at,expires_at,updated_at + ) VALUES(?1,?2,'full-test-owner',?3,?4,'active',?5,?6,?6,?7,?6)", + params![ + scope_id, + sql_u64(expected.epoch).unwrap(), + provider_id, + hex::encode(&expected.provider_identity), + b"full-test-fence".as_slice(), + now, + now + 3_600, + ], + ) + .unwrap(); + conn.execute( + "INSERT INTO changed_path_observer_segments( + scope_id,epoch,segment_id,owner_token,provider_id, + first_sequence,last_sequence,durable_end_offset,folded_end_offset, + segment_path,state,created_at,updated_at + ) VALUES(?1,?2,'full-test-segment','full-test-owner',?3, + 1,1000,100,100,'full-test-segment.cpl','open',?4,?4)", + params![scope_id, sql_u64(expected.epoch).unwrap(), provider_id, now,], + ) + .unwrap(); + } + + impl Fixture { + fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + fs::write(root.join("modify.txt"), b"before\n").unwrap(); + fs::write(root.join("mode.txt"), b"mode\n").unwrap(); + fs::write(root.join("delete.txt"), b"delete\n").unwrap(); + fs::write(root.join("rename-old.txt"), b"rename\n").unwrap(); + Trail::init(root, "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(root).unwrap(); + let branch = db.current_branch().unwrap(); + let head = db.resolve_branch_ref(&branch).unwrap(); + let scope = ScopeIdentity { + scope_id: ScopeId([51; 32]), + kind: ScopeKind::Workspace, + owner_id: "reconcile-test".into(), + }; + let fingerprint = [63; 32]; + let baseline = BaselineIdentity { + ref_name: head.name.clone(), + ref_generation: u64::try_from(head.generation).unwrap(), + change_id: head.change_id, + root_id: head.root_id.clone(), + }; + let policy_identity = PolicyIdentity { + fingerprint, + generation: 1, + }; + let filesystem = FilesystemIdentity(vec![7, 8]); + let provider = ProviderIdentity { + identity: vec![9, 10], + capabilities: ProviderCapabilities { + durable_cursor: true, + linearizable_fence: true, + rename_pairing: true, + overflow_scope: true, + filesystem_supported: true, + clean_proof_allowed: true, + power_loss_durability: false, + }, + }; + db.changed_path_ledger() + .begin_scope(&scope, &baseline, &policy_identity, &filesystem, &provider) + .unwrap(); + let expected = ExpectedScope { + scope_id: scope.scope_id, + epoch: 1, + ref_name: baseline.ref_name, + ref_generation: baseline.ref_generation, + baseline_root: baseline.root_id, + policy_fingerprint: fingerprint, + policy_generation: 1, + filesystem_identity: filesystem.0, + provider_identity: provider.identity, + }; + let policy_root = db.workspace_root.clone(); + let policy = CompiledPolicy::for_reconciliation_test( + RecordingPolicySnapshot { + workspace_root: policy_root.clone(), + ignore_gitignored: true, + dependency_files: Vec::new(), + case_sensitive: true, + rule_sources: Vec::new(), + }, + fingerprint, + &expected, + ); + let fixture = Self { + _temp: temp, + db, + expected, + policy, + }; + fixture.install_full_observer_continuity(); + fixture + } + + fn root(&self) -> &std::path::Path { + &self.db.workspace_root + } + + fn observed_paths(&self) -> Vec { + let observer = FakeQualifiedObserver::new(); + let ledger = ChangedPathLedger::new(&self.db.conn); + let mut attempt = begin_reconciliation( + &self.db, + &ledger, + &observer, + &self.expected, + &self.policy, + ReconcileMode::Full, + "scan_paths", + ) + .unwrap(); + attempt + .observe(&self.db, &ledger, &observer, &self.policy) + .unwrap(); + self.db + .conn + .prepare( + "SELECT normalized_path FROM changed_path_reconciliation_rows + WHERE attempt_id=?1 AND before_identity LIKE 'flags:%' + ORDER BY normalized_path COLLATE BINARY", + ) + .unwrap() + .query_map([attempt.attempt_id], |row| row.get(0)) + .unwrap() + .collect::, _>>() + .unwrap() + } + + fn ledger_rows(&self) -> Vec<(String, i64)> { + self.db + .conn + .prepare( + "SELECT normalized_path,event_flags FROM changed_path_entries + WHERE scope_id=?1 ORDER BY normalized_path COLLATE BINARY", + ) + .unwrap() + .query_map([self.expected.scope_id.to_text()], |row| { + Ok((row.get(0)?, row.get(1)?)) + }) + .unwrap() + .collect::, _>>() + .unwrap() + } + + fn ledger_prefixes(&self) -> Vec { + self.db + .conn + .prepare( + "SELECT normalized_prefix FROM changed_path_prefixes + WHERE scope_id=?1 ORDER BY normalized_prefix COLLATE BINARY", + ) + .unwrap() + .query_map([self.expected.scope_id.to_text()], |row| row.get(0)) + .unwrap() + .collect::, _>>() + .unwrap() + } + + fn latest_attempt_state(&self) -> String { + self.db + .conn + .query_row( + "SELECT state FROM changed_path_reconciliations + ORDER BY created_at DESC, attempt_id DESC LIMIT 1", + [], + |row| row.get(0), + ) + .unwrap() + } + + fn begin_observed(&self) -> ReconciliationAttempt { + let observer = FakeQualifiedObserver::new(); + let ledger = ChangedPathLedger::new(&self.db.conn); + let mut attempt = begin_reconciliation( + &self.db, + &ledger, + &observer, + &self.expected, + &self.policy, + ReconcileMode::Full, + "race_test", + ) + .unwrap(); + attempt + .observe(&self.db, &ledger, &observer, &self.policy) + .unwrap(); + attempt + } + + fn install_full_observer_continuity(&self) { + install_test_observer_continuity(&self.db.conn, &self.expected); + } + + fn persist_live_provider_prefix(&self, prefix: &str) { + let scope_id = self.expected.scope_id.to_text(); + let provider_id = hex::encode(&self.expected.provider_identity); + self.db + .conn + .execute( + "UPDATE changed_path_scopes + SET trust_state='reconciling', trust_reason='provider_prefix' + WHERE scope_id=?1", + [&scope_id], + ) + .unwrap(); + self.db + .conn + .execute( + "INSERT INTO changed_path_prefixes( + scope_id,normalized_prefix,completeness_reason,event_flags, + source_mask,first_sequence,last_sequence,provider_id, + provider_sequence,created_at,updated_at + ) VALUES(?1,?2,'provider_complete',0,?3,1,2,?4,2,?5,?5)", + params![ + scope_id, + prefix, + super::super::EvidenceSource::Observer.mask(), + provider_id, + now_ts(), + ], + ) + .unwrap(); + } + } + + #[test] + fn reconciliation_contract_starts_observation_before_streaming() { + let _ = ReconcileMode::Full; + let _ = begin_reconciliation; + let _ = reconcile_full; + } + + #[test] + fn reconciliation_matches_add_modify_mode_delete_and_rename_oracle() { + let fixture = Fixture::new(); + fs::write(fixture.root().join("add.txt"), b"added\n").unwrap(); + fs::write(fixture.root().join("modify.txt"), b"after and larger\n").unwrap(); + fs::remove_file(fixture.root().join("delete.txt")).unwrap(); + fs::rename( + fixture.root().join("rename-old.txt"), + fixture.root().join("rename-new.txt"), + ) + .unwrap(); + #[cfg(unix)] + { + let mut permissions = fs::metadata(fixture.root().join("mode.txt")) + .unwrap() + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(fixture.root().join("mode.txt"), permissions).unwrap(); + } + + let observer = FakeQualifiedObserver::new(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let report = reconcile_full( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + "test_full_scan", + ) + .unwrap(); + + let mut oracle = vec![ + ("add.txt".to_string(), EvidenceFlags::CREATE.0), + ("delete.txt".to_string(), EvidenceFlags::DELETE.0), + ("modify.txt".to_string(), EvidenceFlags::CONTENT.0), + ("rename-new.txt".to_string(), EvidenceFlags::CREATE.0), + ("rename-old.txt".to_string(), EvidenceFlags::DELETE.0), + ]; + #[cfg(unix)] + oracle.push(("mode.txt".to_string(), EvidenceFlags::MODE.0)); + oracle.sort(); + assert_eq!(fixture.ledger_rows(), oracle); + assert_eq!(report.observed_candidates, oracle.len() as u64); + assert!(report.published); + assert_eq!(report.trust_state, "trusted"); + assert!(report.peak_batch_rows <= STAGING_BATCH_ROWS as u64); + assert!(observer.began.load(Ordering::SeqCst)); + assert_eq!( + fixture.expected.baseline_root, + ObjectId(fixture.expected.baseline_root.0.clone()) + ); + } + + #[test] + fn reconciliation_does_not_flatten_nested_gitignore_authority() { + let mut fixture = Fixture::new(); + fs::create_dir_all(fixture.root().join("src/generated")).unwrap(); + fs::create_dir_all(fixture.root().join("other/generated")).unwrap(); + fs::write(fixture.root().join("src/.gitignore"), "generated\n").unwrap(); + fs::write(fixture.root().join("src/generated/ignored.txt"), b"src\n").unwrap(); + fs::write( + fixture.root().join("other/generated/must-scan.txt"), + b"other\n", + ) + .unwrap(); + let rule_path = fixture.root().join("src/.gitignore"); + fixture + .policy + .set_gitignore_rule_for_test(rule_path, b"generated\n".to_vec()); + + let paths = fixture.observed_paths(); + + assert!(paths + .iter() + .any(|path| path == "other/generated/must-scan.txt")); + assert!(paths.iter().any(|path| path == "src/generated/ignored.txt")); + } + + #[test] + fn reconciliation_scans_gitignored_files_when_recording_keeps_them() { + let mut fixture = Fixture::new(); + fixture.policy.set_ignore_gitignored_for_test(false); + fs::write(fixture.root().join(".gitignore"), "kept.txt\n").unwrap(); + fs::write(fixture.root().join("kept.txt"), b"kept\n").unwrap(); + + assert!(fixture + .observed_paths() + .iter() + .any(|path| path == "kept.txt")); + } + + #[test] + fn stale_scope_cas_fails_attempt_without_replacing_candidates() { + for column in [ + "ref_generation", + "filesystem_identity", + "provider_identity", + "policy_dependency_generation", + ] { + let fixture = Fixture::new(); + fs::write(fixture.root().join("add.txt"), b"added\n").unwrap(); + let attempt = fixture.begin_observed(); + fixture + .db + .conn + .execute( + &format!("UPDATE changed_path_scopes SET {column}=?1 WHERE scope_id=?2"), + params!["replacement", fixture.expected.scope_id.to_text()], + ) + .unwrap(); + + let ledger = ChangedPathLedger::new(&fixture.db.conn); + assert!(attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .is_err()); + let (attempt_state, trust, candidates): (String, String, i64) = fixture + .db + .conn + .query_row( + "SELECT r.state,s.trust_state, + (SELECT COUNT(*) FROM changed_path_entries e WHERE e.scope_id=s.scope_id) + FROM changed_path_reconciliations r + JOIN changed_path_scopes s ON s.scope_id=r.scope_id + ORDER BY r.created_at DESC LIMIT 1", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(attempt_state, "failed", "column {column}"); + assert_ne!(trust, "trusted", "column {column}"); + assert_eq!(candidates, 0, "column {column}"); + } + } + + #[test] + fn fail_closed_transition_after_ready_never_promotes_full_trust() { + for state in [ + TrustState::Overflow, + TrustState::Corrupt, + TrustState::UntrustedGap, + TrustState::StaleBaseline, + ] { + let fixture = Fixture::new(); + fs::write(fixture.root().join("add.txt"), b"added\n").unwrap(); + let attempt = fixture.begin_observed(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + ledger + .mark_untrusted(&fixture.expected, state, "concurrent invalidation") + .unwrap(); + + assert!(attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .is_err()); + let (scope_state, attempt_state): (String, String) = fixture + .db + .conn + .query_row( + "SELECT s.trust_state,r.state + FROM changed_path_scopes s + JOIN changed_path_reconciliations r ON r.scope_id=s.scope_id + ORDER BY r.created_at DESC,r.attempt_id DESC LIMIT 1", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(scope_state, state.as_str()); + assert_eq!(attempt_state, "failed"); + } + } + + #[test] + fn observer_owner_loss_after_drain_forces_full_reconciliation() { + for mutation in [ + "UPDATE changed_path_observer_owners SET lease_state='revoked'", + "UPDATE changed_path_observer_owners SET lease_state='expired'", + "UPDATE changed_path_observer_owners SET expires_at=heartbeat_at", + "UPDATE changed_path_observer_owners SET fence_nonce=x'00'", + ] { + let fixture = Fixture::new(); + let attempt = fixture.begin_observed(); + fixture.db.conn.execute(mutation, []).unwrap(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + + assert!( + attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .is_err(), + "mutation {mutation}" + ); + let state: String = fixture + .db + .conn + .query_row("SELECT trust_state FROM changed_path_scopes", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(state, "untrusted_gap", "mutation {mutation}"); + assert_eq!(fixture.latest_attempt_state(), "failed"); + } + } + + #[test] + fn durable_segment_loss_or_cut_regression_forces_full_reconciliation() { + for mutation in [ + "DELETE FROM changed_path_observer_segments", + "UPDATE changed_path_observer_segments SET durable_end_offset=99,folded_end_offset=99", + "UPDATE changed_path_observer_segments SET folded_end_offset=99", + "UPDATE changed_path_observer_segments SET last_sequence=9", + ] { + let fixture = Fixture::new(); + let attempt = fixture.begin_observed(); + fixture.db.conn.execute(mutation, []).unwrap(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + + assert!( + attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .is_err(), + "mutation {mutation}" + ); + assert_eq!(fixture.latest_attempt_state(), "failed"); + } + } + + #[test] + fn full_publication_expiry_at_final_boundary_rolls_back_candidates() { + let fixture = Fixture::new(); + fs::write(fixture.root().join("late.txt"), b"late\n").unwrap(); + fixture + .db + .conn + .execute( + "INSERT INTO changed_path_prefixes( + scope_id,normalized_prefix,completeness_reason,event_flags, + source_mask,first_sequence,last_sequence,created_at,updated_at + ) VALUES(?1,'old-prefix','reconciliation',0,?2,1,1,?3,?3)", + params![ + fixture.expected.scope_id.to_text(), + super::super::EvidenceSource::Reconciliation.mask(), + now_ts(), + ], + ) + .unwrap(); + let before_rows = fixture.ledger_rows(); + let before_prefixes = fixture.ledger_prefixes(); + let mut attempt = fixture.begin_observed(); + let before_generation: i64 = fixture + .db + .conn + .query_row( + "SELECT continuity_generation FROM changed_path_scopes", + [], + |row| row.get(0), + ) + .unwrap(); + attempt.set_final_publication_hook(|conn| { + conn.execute( + "UPDATE changed_path_observer_owners SET expires_at=?1", + [now_ts()], + ) + .unwrap(); + }); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + + assert!(attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .is_err()); + + assert_eq!(fixture.ledger_rows(), before_rows); + assert_eq!(fixture.ledger_prefixes(), before_prefixes); + let (state, generation): (String, i64) = fixture + .db + .conn + .query_row( + "SELECT trust_state,continuity_generation FROM changed_path_scopes", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(state, "untrusted_gap"); + assert_eq!(generation, before_generation + 1); + assert_eq!(fixture.latest_attempt_state(), "failed"); + } + + #[test] + fn prefix_publication_insufficient_final_horizon_rolls_back_candidates() { + let fixture = Fixture::new(); + fs::create_dir_all(fixture.root().join("src")).unwrap(); + fs::write(fixture.root().join("src/current.txt"), b"current\n").unwrap(); + fixture.persist_live_provider_prefix("src"); + fixture + .db + .conn + .execute( + "INSERT INTO changed_path_entries( + scope_id,normalized_path,event_flags,source_mask, + first_sequence,last_sequence,created_at,updated_at + ) VALUES(?1,'src/stale.txt',?2,?3,1,1,?4,?4)", + params![ + fixture.expected.scope_id.to_text(), + EvidenceFlags::CONTENT.0, + super::super::EvidenceSource::Reconciliation.mask(), + now_ts(), + ], + ) + .unwrap(); + fixture + .db + .conn + .execute( + "INSERT INTO changed_path_prefixes( + scope_id,normalized_prefix,completeness_reason,event_flags, + source_mask,first_sequence,last_sequence,created_at,updated_at + ) VALUES(?1,'src/stale','reconciliation',0,?2,1,1,?3,?3)", + params![ + fixture.expected.scope_id.to_text(), + super::super::EvidenceSource::Reconciliation.mask(), + now_ts(), + ], + ) + .unwrap(); + let before_rows = fixture.ledger_rows(); + let before_prefixes = fixture.ledger_prefixes(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let proof = persisted_proven_prefixes( + &ledger, + &fixture.expected, + &[LedgerPath::parse("src").unwrap()], + ) + .unwrap(); + let observer = FakeQualifiedObserver::new(); + let mut attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::ProvenPrefixes(proof), + "final_lease_horizon", + ) + .unwrap(); + attempt + .observe(&fixture.db, &ledger, &observer, &fixture.policy) + .unwrap(); + let before_generation: i64 = fixture + .db + .conn + .query_row( + "SELECT continuity_generation FROM changed_path_scopes", + [], + |row| row.get(0), + ) + .unwrap(); + attempt.set_final_publication_hook(|conn| { + conn.execute( + "UPDATE changed_path_observer_owners SET expires_at=?1", + [now_ts() + 1], + ) + .unwrap(); + }); + + assert!(attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .is_err()); + + assert_eq!(fixture.ledger_rows(), before_rows); + assert_eq!(fixture.ledger_prefixes(), before_prefixes); + let (state, generation): (String, i64) = fixture + .db + .conn + .query_row( + "SELECT trust_state,continuity_generation FROM changed_path_scopes", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(state, "untrusted_gap"); + assert_eq!(generation, before_generation + 1); + assert_eq!(fixture.latest_attempt_state(), "failed"); + } + + #[test] + fn full_scope_identity_changes_fail_the_ready_attempt() { + for (column, replacement) in [ + ("change_id", "replacement-change"), + ("provider_id", "replacement-provider"), + ("scope_root", "/replacement/root"), + ("scope_root_identity", "replacement-root-identity"), + ("case_sensitive", "0"), + ] { + let fixture = Fixture::new(); + fs::write(fixture.root().join("add.txt"), b"added\n").unwrap(); + let attempt = fixture.begin_observed(); + fixture + .db + .conn + .execute( + &format!("UPDATE changed_path_scopes SET {column}=?1 WHERE scope_id=?2"), + params![replacement, fixture.expected.scope_id.to_text()], + ) + .unwrap(); + + let ledger = ChangedPathLedger::new(&fixture.db.conn); + assert!( + attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .is_err(), + "column {column}" + ); + let state: String = fixture + .db + .conn + .query_row( + "SELECT state FROM changed_path_reconciliations ORDER BY created_at DESC LIMIT 1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(state, "failed", "column {column}"); + } + } + + #[test] + fn publication_never_regresses_later_durable_or_folded_cuts() { + let fixture = Fixture::new(); + fs::write(fixture.root().join("add.txt"), b"added\n").unwrap(); + let attempt = fixture.begin_observed(); + fixture + .db + .conn + .execute( + "UPDATE changed_path_scopes SET durable_offset=150, folded_offset=140 + WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + ) + .unwrap(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + + attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .unwrap(); + + let cuts: (i64, i64) = fixture + .db + .conn + .query_row( + "SELECT durable_offset,folded_offset FROM changed_path_scopes WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(cuts, (150, 140)); + } + + #[test] + fn continuous_fold_prefix_cap_rolls_back_rows_and_marks_overflow_atomically() { + let fixture = Fixture::new(); + let scope_id = fixture.expected.scope_id.to_text(); + let now = now_ts(); + fixture + .db + .conn + .execute( + "UPDATE changed_path_scopes + SET max_prefix_rows=1,trust_state='trusted',trust_reason='test_ready' + WHERE scope_id=?1", + [&scope_id], + ) + .unwrap(); + fixture + .db + .conn + .execute( + "INSERT INTO changed_path_prefixes( + scope_id,normalized_prefix,completeness_reason,event_flags,source_mask, + first_sequence,last_sequence,provider_id,provider_sequence, + created_at,updated_at + ) VALUES(?1,'existing','provider_complete',?2,?3,99,99,?4,99,?5,?5)", + params![ + scope_id, + EvidenceFlags::PROVIDER_COMPLETE_PREFIX.0, + super::super::EvidenceSource::Observer.mask(), + hex::encode(&fixture.expected.provider_identity), + now, + ], + ) + .unwrap(); + let start = ObserverFence { + sequence: 100, + durable_offset: 100, + nonce: b"continuous-prefix-cap-start".to_vec(), + }; + let end = ObserverFence { + sequence: 101, + durable_offset: 100, + nonce: b"continuous-prefix-cap-end".to_vec(), + }; + let root_identity = b"continuous-prefix-cap-root".to_vec(); + let mut qualification = ObserverQualification::seal_for_test( + &fixture.expected, + root_identity.clone(), + start.clone(), + end.clone(), + ); + let ledger = fixture.db.changed_path_ledger(); + + let error = fold_observer_interval( + &ledger, + &fixture.expected, + &root_identity, + &start, + &end, + &mut qualification, + &[ObserverEvent { + path: LedgerPath::parse("new-prefix").unwrap(), + flags: EvidenceFlags::PROVIDER_COMPLETE_PREFIX, + sequence: 101, + }], + ) + .unwrap_err(); + + assert!(matches!(error, Error::ChangeLedgerReconcileRequired { .. })); + assert_eq!(fixture.ledger_prefixes(), vec!["existing".to_string()]); + let scope: (String, i64, i64) = fixture + .db + .conn + .query_row( + "SELECT trust_state,durable_offset,folded_offset + FROM changed_path_scopes WHERE scope_id=?1", + [scope_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(scope, ("overflow".into(), 100, 100)); + let segment_folded: i64 = fixture + .db + .conn + .query_row( + "SELECT folded_end_offset FROM changed_path_observer_segments + WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(segment_folded, 100); + } + + #[test] + fn publication_candidate_cap_accepts_boundary_and_rolls_back_overflow() { + for (cap, succeeds) in [(2_i64, true), (1_i64, false)] { + let fixture = Fixture::new(); + fs::write(fixture.root().join("cap-one.txt"), b"one\n").unwrap(); + fs::write(fixture.root().join("cap-two.txt"), b"two\n").unwrap(); + fixture + .db + .conn + .execute( + "UPDATE changed_path_scopes SET max_candidate_rows=?1 WHERE scope_id=?2", + params![cap, fixture.expected.scope_id.to_text()], + ) + .unwrap(); + let attempt = fixture.begin_observed(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + + let result = attempt.publish(&fixture.db, &ledger, &fixture.policy); + let (trust, attempt_state, candidates): (String, String, i64) = fixture + .db + .conn + .query_row( + "SELECT s.trust_state,r.state, + (SELECT COUNT(*) FROM changed_path_entries e WHERE e.scope_id=s.scope_id) + FROM changed_path_scopes s JOIN changed_path_reconciliations r + ON r.scope_id=s.scope_id ORDER BY r.created_at DESC LIMIT 1", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + if succeeds { + assert!(result.is_ok()); + assert_eq!(trust, "trusted"); + assert_eq!(attempt_state, "published"); + assert_eq!(candidates, 2); + } else { + assert!(result.is_err()); + assert_eq!(trust, "overflow"); + assert_eq!(attempt_state, "failed"); + assert_eq!(candidates, 0); + } + } + } + + #[test] + fn workspace_root_replacement_cannot_publish_trust() { + let fixture = Fixture::new(); + fs::write(fixture.root().join("add.txt"), b"added\n").unwrap(); + let attempt = fixture.begin_observed(); + let root = fixture.root().to_path_buf(); + let displaced = root.with_extension("reconcile-displaced"); + fs::rename(&root, &displaced).unwrap(); + fs::create_dir(&root).unwrap(); + + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let result = attempt.publish(&fixture.db, &ledger, &fixture.policy); + + fs::remove_dir(&root).unwrap(); + fs::rename(&displaced, &root).unwrap(); + assert!(result.is_err()); + let trust: String = fixture + .db + .conn + .query_row( + "SELECT trust_state FROM changed_path_scopes WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + ) + .unwrap(); + assert_ne!(trust, "trusted"); + } + + #[test] + fn nested_directory_replacement_forces_a_fresh_scan_before_publication() { + let fixture = Fixture::new(); + let directory = fixture.root().join("nested"); + fs::create_dir(&directory).unwrap(); + fs::write(directory.join("original.txt"), b"original\n").unwrap(); + let displaced = fixture.root().join("nested-displaced"); + let mutation_directory = directory.clone(); + let mutation_displaced = displaced.clone(); + let observer = EventDuringFenceObserver { + event: ObserverEvent { + path: LedgerPath::parse("nested/replacement.txt").unwrap(), + flags: EvidenceFlags::CREATE, + sequence: 31, + }, + mutation: Mutex::new(Some(Box::new(move || { + fs::rename(&mutation_directory, &mutation_displaced).unwrap(); + fs::create_dir(&mutation_directory).unwrap(); + fs::write(mutation_directory.join("replacement.txt"), b"replacement\n").unwrap(); + }))), + }; + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let result = reconcile_full( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + "directory_replacement", + ); + fs::remove_dir_all(&directory).unwrap(); + fs::rename(&displaced, &directory).unwrap(); + + let report = result.unwrap(); + assert_eq!(report.retries, 1); + assert!(report.published); + let rows = fixture.ledger_rows(); + assert!(rows.contains(&("nested/replacement.txt".into(), EvidenceFlags::CREATE.0))); + assert!(!rows.iter().any(|(path, _)| path == "nested/original.txt")); + } + + #[test] + fn evidence_arriving_after_scan_is_folded_through_end_fence() { + let fixture = Fixture::new(); + let path = fixture.root().join("modify.txt"); + let observer = EventDuringFenceObserver { + event: ObserverEvent { + path: LedgerPath::parse("modify.txt").unwrap(), + flags: EvidenceFlags::CONTENT, + sequence: 31, + }, + mutation: Mutex::new(Some(Box::new(move || { + fs::write(path, b"changed during end fence\n").unwrap(); + }))), + }; + let ledger = ChangedPathLedger::new(&fixture.db.conn); + + let report = reconcile_full( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + "event_during_scan", + ) + .unwrap(); + + assert!(report.published); + assert_eq!(report.start_sequence, 30); + assert_eq!(report.end_sequence, 31); + assert_eq!( + fixture.ledger_rows(), + vec![("modify.txt".into(), EvidenceFlags::CONTENT.0)] + ); + } + + #[test] + fn evidence_later_than_end_fence_is_retained() { + let fixture = Fixture::new(); + fs::write(fixture.root().join("add.txt"), b"added\n").unwrap(); + let attempt = fixture.begin_observed(); + fixture + .db + .conn + .execute( + "INSERT INTO changed_path_entries( + scope_id,normalized_path,event_flags,source_mask, + first_sequence,last_sequence,provider_id,provider_sequence, + created_at,updated_at + ) VALUES(?1,'later.txt',?2,?3,11,11,'later-provider',11,?4,?4)", + params![ + fixture.expected.scope_id.to_text(), + EvidenceFlags::CONTENT.0, + super::super::EvidenceSource::Observer.mask(), + now_ts(), + ], + ) + .unwrap(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + + attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .unwrap(); + + assert_eq!( + fixture.ledger_rows(), + vec![ + ("add.txt".into(), EvidenceFlags::CREATE.0), + ("later.txt".into(), EvidenceFlags::CONTENT.0), + ] + ); + } + + #[test] + fn only_persisted_provider_complete_prefixes_are_qualified() { + let fixture = Fixture::new(); + let scope_id = fixture.expected.scope_id.to_text(); + fixture.persist_live_provider_prefix("src"); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + + let proof = persisted_proven_prefixes( + &ledger, + &fixture.expected, + &[LedgerPath::parse("src").unwrap()], + ) + .unwrap(); + assert_eq!(proof.prefixes, vec![LedgerPath::parse("src").unwrap()]); + assert!(persisted_proven_prefixes( + &ledger, + &fixture.expected, + &[LedgerPath::parse("user-prefix").unwrap()], + ) + .is_err()); + + fixture + .db + .conn + .execute( + "DELETE FROM changed_path_observer_owners WHERE scope_id=?1", + [&scope_id], + ) + .unwrap(); + assert!(persisted_proven_prefixes( + &ledger, + &fixture.expected, + &[LedgerPath::parse("src").unwrap()], + ) + .is_err()); + + fixture + .db + .conn + .execute( + "UPDATE changed_path_scopes + SET trust_state='stale_baseline', trust_reason='global_policy_change' + WHERE scope_id=?1", + [&scope_id], + ) + .unwrap(); + assert!(persisted_proven_prefixes( + &ledger, + &fixture.expected, + &[LedgerPath::parse("src").unwrap()], + ) + .is_err()); + + fixture + .db + .conn + .execute( + "UPDATE changed_path_scopes SET trust_state='overflow' WHERE scope_id=?1", + [&scope_id], + ) + .unwrap(); + assert!(persisted_proven_prefixes( + &ledger, + &fixture.expected, + &[LedgerPath::parse("src").unwrap()], + ) + .is_err()); + } + + #[test] + fn proven_prefix_report_never_promotes_global_trust() { + let fixture = Fixture::new(); + fixture.persist_live_provider_prefix("src"); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let proof = persisted_proven_prefixes( + &ledger, + &fixture.expected, + &[LedgerPath::parse("src").unwrap()], + ) + .unwrap(); + let observer = FakeQualifiedObserver::new(); + let mut attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::ProvenPrefixes(proof), + "prefix_refresh", + ) + .unwrap(); + attempt + .observe(&fixture.db, &ledger, &observer, &fixture.policy) + .unwrap(); + + let report = attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .unwrap(); + + assert!(!report.published); + assert!(report.refreshed); + assert_eq!(report.trust_state, "reconciling"); + assert_eq!(fixture.latest_attempt_state(), "published"); + } + + #[test] + fn proven_prefix_proof_is_bound_to_exact_owner_and_provider_cut() { + for mutation in [ + "UPDATE changed_path_observer_owners SET owner_token='replacement-owner'", + "UPDATE changed_path_prefixes SET provider_sequence=provider_sequence+1", + "UPDATE changed_path_scopes SET durable_offset=1, folded_offset=1", + "UPDATE changed_path_scopes SET continuity_generation=continuity_generation+1", + ] { + let fixture = Fixture::new(); + fixture.persist_live_provider_prefix("src"); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let proof = persisted_proven_prefixes( + &ledger, + &fixture.expected, + &[LedgerPath::parse("src").unwrap()], + ) + .unwrap(); + fixture.db.conn.execute(mutation, []).unwrap(); + let observer = FakeQualifiedObserver::new(); + + assert!(begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::ProvenPrefixes(proof), + "stale_prefix_proof", + ) + .is_err()); + } + } + + #[test] + fn stale_baseline_rejects_previously_sealed_prefix_proof_at_start() { + let fixture = Fixture::new(); + fixture.persist_live_provider_prefix("src"); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let proof = persisted_proven_prefixes( + &ledger, + &fixture.expected, + &[LedgerPath::parse("src").unwrap()], + ) + .unwrap(); + fixture + .db + .conn + .execute( + "UPDATE changed_path_scopes + SET trust_state='stale_baseline',trust_reason='policy changed' + WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + ) + .unwrap(); + let observer = FakeQualifiedObserver::new(); + + assert!(begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::ProvenPrefixes(proof), + "stale_prefix_start", + ) + .is_err()); + } + + #[test] + fn stale_baseline_after_ready_rejects_prefix_publication() { + let fixture = Fixture::new(); + fixture.persist_live_provider_prefix("src"); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let proof = persisted_proven_prefixes( + &ledger, + &fixture.expected, + &[LedgerPath::parse("src").unwrap()], + ) + .unwrap(); + let observer = FakeQualifiedObserver::new(); + let mut attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::ProvenPrefixes(proof), + "stale_prefix_publish", + ) + .unwrap(); + attempt + .observe(&fixture.db, &ledger, &observer, &fixture.policy) + .unwrap(); + fixture + .db + .conn + .execute( + "UPDATE changed_path_scopes + SET trust_state='stale_baseline',trust_reason='policy changed' + WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + ) + .unwrap(); + + assert!(attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .is_err()); + assert_eq!(fixture.latest_attempt_state(), "failed"); + } + + #[test] + fn proven_prefix_publish_replaces_only_covered_owned_rows() { + let fixture = Fixture::new(); + fs::create_dir_all(fixture.root().join("src")).unwrap(); + fs::write(fixture.root().join("src/current.txt"), b"current\n").unwrap(); + fixture.persist_live_provider_prefix("src"); + let scope = fixture.expected.scope_id.to_text(); + let provider = hex::encode(&fixture.expected.provider_identity); + for (path, source_mask, provider_id, provider_sequence) in [ + ( + "src/stale-reconcile.txt", + super::super::EvidenceSource::Reconciliation.mask(), + None, + None, + ), + ( + "src/stale-provider.txt", + super::super::EvidenceSource::Observer.mask(), + Some(provider.as_str()), + Some(2_i64), + ), + ( + "src/later-provider.txt", + super::super::EvidenceSource::Observer.mask(), + Some(provider.as_str()), + Some(99_i64), + ), + ( + "src/intent.txt", + super::super::EvidenceSource::Intent.mask(), + None, + None, + ), + ( + "outside.txt", + super::super::EvidenceSource::Reconciliation.mask(), + None, + None, + ), + ] { + fixture + .db + .conn + .execute( + "INSERT INTO changed_path_entries( + scope_id,normalized_path,event_flags,source_mask, + first_sequence,last_sequence,provider_id,provider_sequence, + created_at,updated_at + ) VALUES(?1,?2,?3,?4,1,1,?5,?6,?7,?7)", + params![ + scope, + path, + EvidenceFlags::CONTENT.0, + source_mask, + provider_id, + provider_sequence, + now_ts(), + ], + ) + .unwrap(); + } + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let proof = persisted_proven_prefixes( + &ledger, + &fixture.expected, + &[LedgerPath::parse("src").unwrap()], + ) + .unwrap(); + let observer = FakeQualifiedObserver::new(); + let mut attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::ProvenPrefixes(proof), + "prefix_replace", + ) + .unwrap(); + attempt + .observe(&fixture.db, &ledger, &observer, &fixture.policy) + .unwrap(); + + let report = attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .unwrap(); + let paths = fixture + .db + .conn + .prepare( + "SELECT normalized_path FROM changed_path_entries WHERE scope_id=?1 + ORDER BY normalized_path COLLATE BINARY", + ) + .unwrap() + .query_map([scope], |row| row.get::<_, String>(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + + assert!(report.refreshed); + assert!(!report.published); + assert!(paths.contains(&"src/current.txt".to_string())); + assert!(!paths.contains(&"src/stale-reconcile.txt".to_string())); + assert!(!paths.contains(&"src/stale-provider.txt".to_string())); + assert!(paths.contains(&"src/later-provider.txt".to_string())); + assert!(paths.contains(&"src/intent.txt".to_string())); + assert!(paths.contains(&"outside.txt".to_string())); + } + + #[test] + fn each_proven_prefix_uses_its_own_provider_sequence_cut() { + let fixture = Fixture::new(); + fixture.persist_live_provider_prefix("a"); + let scope = fixture.expected.scope_id.to_text(); + let provider = hex::encode(&fixture.expected.provider_identity); + fixture + .db + .conn + .execute( + "INSERT INTO changed_path_prefixes( + scope_id,normalized_prefix,completeness_reason,event_flags, + source_mask,first_sequence,last_sequence,provider_id, + provider_sequence,created_at,updated_at + ) VALUES(?1,'b','provider_complete',0,?2,1,8,?3,8,?4,?4)", + params![ + scope, + super::super::EvidenceSource::Observer.mask(), + provider, + now_ts(), + ], + ) + .unwrap(); + fixture + .db + .conn + .execute( + "INSERT INTO changed_path_entries( + scope_id,normalized_path,event_flags,source_mask, + first_sequence,last_sequence,provider_id,provider_sequence, + created_at,updated_at + ) VALUES(?1,'a/later.txt',?2,?3,5,5,?4,5,?5,?5)", + params![ + scope, + EvidenceFlags::CONTENT.0, + super::super::EvidenceSource::Observer.mask(), + provider, + now_ts(), + ], + ) + .unwrap(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let proof = persisted_proven_prefixes( + &ledger, + &fixture.expected, + &[ + LedgerPath::parse("a").unwrap(), + LedgerPath::parse("b").unwrap(), + ], + ) + .unwrap(); + let observer = FakeQualifiedObserver::new(); + let mut attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::ProvenPrefixes(proof), + "per_prefix_cut", + ) + .unwrap(); + attempt + .observe(&fixture.db, &ledger, &observer, &fixture.policy) + .unwrap(); + attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .unwrap(); + + let preserved: bool = fixture + .db + .conn + .query_row( + "SELECT EXISTS(SELECT 1 FROM changed_path_entries + WHERE scope_id=?1 AND normalized_path='a/later.txt')", + [scope], + |row| row.get(0), + ) + .unwrap(); + assert!(preserved); + } + + #[test] + fn mismatched_sealed_observer_proof_fails_closed() { + let fixture = Fixture::new(); + fs::write(fixture.root().join("add.txt"), b"added\n").unwrap(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let observer = FakeQualifiedObserver::with_corrupt_proof(); + let mut attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::Full, + "bad_proof", + ) + .unwrap(); + attempt + .observe(&fixture.db, &ledger, &observer, &fixture.policy) + .unwrap(); + + assert!(attempt + .publish(&fixture.db, &ledger, &fixture.policy) + .is_err()); + assert!(fixture.ledger_rows().is_empty()); + } + + #[cfg(unix)] + #[test] + fn symlinks_and_non_regular_entries_are_not_candidates() { + use std::os::unix::fs::symlink; + + let fixture = Fixture::new(); + symlink("modify.txt", fixture.root().join("linked.txt")).unwrap(); + fs::create_dir(fixture.root().join("empty-dir")).unwrap(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let observer = FakeQualifiedObserver::new(); + + let report = reconcile_full( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + "regular_files_only", + ) + .unwrap(); + + assert_eq!(report.observed_candidates, 0); + assert!(fixture.ledger_rows().is_empty()); + } + + #[test] + fn starting_a_new_attempt_abandons_crash_left_staging() { + let fixture = Fixture::new(); + let observer = FakeQualifiedObserver::new(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let first = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::Full, + "first", + ) + .unwrap(); + let first_id = first.attempt_id.clone(); + drop(first); + let second_observer = FakeQualifiedObserver::new(); + let _second = begin_reconciliation( + &fixture.db, + &ledger, + &second_observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::Full, + "restart", + ) + .unwrap(); + + let state: String = fixture + .db + .conn + .query_row( + "SELECT state FROM changed_path_reconciliations WHERE attempt_id=?1", + [first_id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(state, "abandoned"); + } + + #[test] + fn scan_end_fence_drain_callback_and_flush_errors_terminalize_attempts() { + for point in [ObserverFailurePoint::EndFence, ObserverFailurePoint::Drain] { + let fixture = Fixture::new(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let observer = FailingObserver { + point, + callback_path: None, + }; + let mut attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::Full, + "terminal_failure", + ) + .unwrap(); + assert!(attempt + .observe(&fixture.db, &ledger, &observer, &fixture.policy) + .is_err()); + assert_eq!(fixture.latest_attempt_state(), "failed"); + } + + #[cfg(unix)] + { + let fixture = Fixture::new(); + let blocked = fixture.root().join("blocked-directory"); + fs::create_dir(&blocked).unwrap(); + fs::write(blocked.join("file.txt"), b"blocked\n").unwrap(); + let mut blocked_permissions = fs::metadata(&blocked).unwrap().permissions(); + blocked_permissions.set_mode(0o000); + fs::set_permissions(&blocked, blocked_permissions).unwrap(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let observer = FakeQualifiedObserver::new(); + let mut attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::Full, + "scan_failure", + ) + .unwrap(); + let result = attempt.observe(&fixture.db, &ledger, &observer, &fixture.policy); + let mut restore_permissions = fs::metadata(&blocked).unwrap().permissions(); + restore_permissions.set_mode(0o700); + fs::set_permissions(&blocked, restore_permissions).unwrap(); + assert!(result.is_err()); + assert_eq!(fixture.latest_attempt_state(), "failed"); + } + + #[cfg(unix)] + { + use std::os::unix::fs::symlink; + let fixture = Fixture::new(); + let external = tempfile::tempdir().unwrap(); + fs::write(external.path().join("file.txt"), b"external\n").unwrap(); + symlink(external.path(), fixture.root().join("linked")).unwrap(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let observer = FailingObserver { + point: ObserverFailurePoint::Callback, + callback_path: Some(LedgerPath::parse("linked/file.txt").unwrap()), + }; + let mut attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::Full, + "callback_failure", + ) + .unwrap(); + assert!(attempt + .observe(&fixture.db, &ledger, &observer, &fixture.policy) + .is_err()); + assert_eq!(fixture.latest_attempt_state(), "failed"); + } + + { + let fixture = Fixture::new(); + fs::write(fixture.root().join("flush.txt"), b"flush\n").unwrap(); + fixture + .db + .conn + .execute_batch( + "CREATE TRIGGER fail_reconciliation_flush + BEFORE INSERT ON changed_path_reconciliation_rows + BEGIN SELECT RAISE(ABORT, 'injected flush failure'); END;", + ) + .unwrap(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let observer = FakeQualifiedObserver::new(); + let mut attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::Full, + "flush_failure", + ) + .unwrap(); + assert!(attempt + .observe(&fixture.db, &ledger, &observer, &fixture.policy) + .is_err()); + assert_eq!(fixture.latest_attempt_state(), "failed"); + } + } + + #[test] + fn observer_callback_only_spools_then_replays_after_primary_transaction() { + let fixture = Fixture::new(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let observer = PrimaryTransactionObserver { + conn: &fixture.db.conn, + path: fixture.root().join("modify.txt"), + }; + let mut attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::Full, + "callback_spool", + ) + .unwrap(); + + attempt + .observe(&fixture.db, &ledger, &observer, &fixture.policy) + .unwrap(); + + let staged_hash: String = fixture + .db + .conn + .query_row( + "SELECT content_hash FROM changed_path_reconciliation_rows + WHERE attempt_id=?1 AND normalized_path='modify.txt'", + [&attempt.attempt_id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + staged_hash, + hex::encode(Sha256::digest(b"after callback\n")) + ); + assert_eq!(fixture.latest_attempt_state(), "ready"); + } + + #[test] + fn directory_guards_cannot_collide_with_legitimate_user_paths() { + let fixture = Fixture::new(); + fs::create_dir(fixture.root().join("a")).unwrap(); + fs::create_dir(fixture.root().join("#directory-guard")).unwrap(); + fs::write( + fixture.root().join("#directory-guard/61"), + b"legitimate user file\n", + ) + .unwrap(); + let observer = FakeQualifiedObserver::new(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let mut attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::Full, + "guard_collision", + ) + .unwrap(); + attempt + .observe(&fixture.db, &ledger, &observer, &fixture.policy) + .unwrap(); + + let candidate_exists: bool = fixture + .db + .conn + .query_row( + "SELECT EXISTS(SELECT 1 FROM changed_path_reconciliation_rows + WHERE attempt_id=?1 AND normalized_path='#directory-guard/61' + AND before_identity LIKE 'flags:%')", + [&attempt.attempt_id], + |row| row.get(0), + ) + .unwrap(); + let guard_exists: bool = fixture + .db + .conn + .query_row( + "SELECT EXISTS(SELECT 1 FROM changed_path_reconciliation_guards + WHERE attempt_id=?1 AND relative_path=?2)", + params![attempt.attempt_id, b"a".as_slice()], + |row| row.get(0), + ) + .unwrap(); + assert!(candidate_exists); + assert!(guard_exists); + } + + #[test] + fn retryable_identity_race_restarts_and_reports_actual_retry() { + let fixture = Fixture::new(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let observer = RetryOnceObserver { + attempts: AtomicU64::new(0), + }; + + let report = reconcile_full( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + "retry_once", + ) + .unwrap(); + + assert_eq!(observer.attempts.load(Ordering::SeqCst), 2); + assert_eq!(report.retries, 1); + let failed: i64 = fixture + .db + .conn + .query_row( + "SELECT COUNT(*) FROM changed_path_reconciliations WHERE state='failed'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(failed, 1); + } + + #[test] + fn hundred_thousand_staged_rows_keep_batch_memory_bounded() { + let fixture = Fixture::new(); + let observer = FakeQualifiedObserver::new(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let attempt = begin_reconciliation( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + ReconcileMode::Full, + "scale", + ) + .unwrap(); + let mut writer = StagingWriter::new(&fixture.db.conn, &attempt.attempt_id); + for index in 0..100_000u64 { + writer + .push(StagedRow { + path: format!("scale/{index:06}.txt"), + row_kind: "entry", + file_kind: Some("Text".into()), + content_hash: Some(format!("{index:064x}")), + executable: Some(false), + size_bytes: Some(1), + flags: EvidenceFlags::CREATE, + identity: None, + source_sequence: None, + }) + .unwrap(); + } + writer.flush().unwrap(); + + let count: i64 = fixture + .db + .conn + .query_row( + "SELECT COUNT(*) FROM changed_path_reconciliation_rows WHERE attempt_id=?1", + [&attempt.attempt_id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, 100_000); + assert_eq!(writer.peak_batch_rows, STAGING_BATCH_ROWS as u64); + assert!(writer.pending.capacity() <= STAGING_BATCH_ROWS); + } + + #[test] + fn hundred_thousand_files_reconcile_end_to_end_with_bounded_streaming() { + let fixture = Fixture::new(); + for directory in 0..100_u64 { + let path = fixture.root().join(format!("scale/{directory:03}")); + fs::create_dir_all(&path).unwrap(); + for file in 0..1_000_u64 { + std::fs::File::create(path.join(format!("{file:04}.empty"))).unwrap(); + } + } + fixture + .db + .conn + .execute("DELETE FROM worktree_file_index", []) + .unwrap(); + let ledger = ChangedPathLedger::new(&fixture.db.conn); + let observer = FakeQualifiedObserver::new(); + + let report = reconcile_full( + &fixture.db, + &ledger, + &observer, + &fixture.expected, + &fixture.policy, + "hundred_thousand_end_to_end", + ) + .unwrap(); + + let published: i64 = fixture + .db + .conn + .query_row( + "SELECT COUNT(*) FROM changed_path_entries WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(observer.drains.load(Ordering::SeqCst), 1); + assert_eq!(report.observed_files, 100_004); + assert_eq!(report.observed_candidates, 100_000); + assert_eq!(report.candidate_rows, 100_000); + assert_eq!(published, 100_000); + assert!(report.hashed_bytes > 0); + assert!(report.peak_batch_rows <= STAGING_BATCH_ROWS as u64); + assert!(report.peak_buffer_bytes <= 2 * 64 * 1024 + 6); + } + + #[test] + fn realistic_filesystem_hash_and_baseline_gate_ignores_worktree_index_authority() { + let temp = tempfile::tempdir().unwrap(); + for directory in 0..32_u64 { + let path = temp.path().join(format!("tree/{directory:02}")); + fs::create_dir_all(&path).unwrap(); + for file in 0..16_u64 { + fs::write( + path.join(format!("{file:02}.txt")), + format!("baseline-{directory}-{file}\n"), + ) + .unwrap(); + } + } + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let branch = db.current_branch().unwrap(); + let head = db.resolve_branch_ref(&branch).unwrap(); + let scope = ScopeIdentity { + scope_id: ScopeId([81; 32]), + kind: ScopeKind::Workspace, + owner_id: "realistic-reconcile".into(), + }; + let fingerprint = [82; 32]; + let provider = ProviderIdentity { + identity: vec![83], + capabilities: ProviderCapabilities { + durable_cursor: true, + linearizable_fence: true, + rename_pairing: true, + overflow_scope: true, + filesystem_supported: true, + clean_proof_allowed: true, + power_loss_durability: false, + }, + }; + db.changed_path_ledger() + .begin_scope( + &scope, + &BaselineIdentity { + ref_name: head.name.clone(), + ref_generation: u64::try_from(head.generation).unwrap(), + change_id: head.change_id, + root_id: head.root_id.clone(), + }, + &PolicyIdentity { + fingerprint, + generation: 1, + }, + &FilesystemIdentity(vec![84]), + &provider, + ) + .unwrap(); + let expected = ExpectedScope { + scope_id: scope.scope_id, + epoch: 1, + ref_name: head.name, + ref_generation: u64::try_from(head.generation).unwrap(), + baseline_root: head.root_id, + policy_fingerprint: fingerprint, + policy_generation: 1, + filesystem_identity: vec![84], + provider_identity: provider.identity, + }; + let policy = CompiledPolicy::for_reconciliation_test( + RecordingPolicySnapshot { + workspace_root: db.workspace_root.clone(), + ignore_gitignored: true, + dependency_files: Vec::new(), + case_sensitive: true, + rule_sources: Vec::new(), + }, + fingerprint, + &expected, + ); + install_test_observer_continuity(&db.conn, &expected); + for index in 0..128_u64 { + let directory = index / 16; + let file = index % 16; + fs::write( + temp.path() + .join(format!("tree/{directory:02}/{file:02}.txt")), + format!("modified-{index}\n"), + ) + .unwrap(); + } + for index in 128..256_u64 { + let directory = index / 16; + let file = index % 16; + fs::remove_file( + temp.path() + .join(format!("tree/{directory:02}/{file:02}.txt")), + ) + .unwrap(); + } + for index in 0..128_u64 { + fs::write( + temp.path().join(format!("tree/new-{index:03}.txt")), + format!("new-{index}\n"), + ) + .unwrap(); + } + db.conn + .execute("DELETE FROM worktree_file_index", []) + .unwrap(); + let ledger = db.changed_path_ledger(); + let observer = FakeQualifiedObserver::new(); + + let report = reconcile_full( + &db, + &ledger, + &observer, + &expected, + &policy, + "realistic_gate", + ) + .unwrap(); + + assert_eq!(report.observed_files, 512); + assert_eq!(report.observed_candidates, 384); + assert!(report.hashed_bytes > 0); + assert!(report.peak_batch_rows <= STAGING_BATCH_ROWS as u64); + assert!(report.peak_buffer_bytes <= 2 * 64 * 1024 + 6); + } +} diff --git a/trail/src/db/change_ledger/recovery.rs b/trail/src/db/change_ledger/recovery.rs new file mode 100644 index 0000000..c11befa --- /dev/null +++ b/trail/src/db/change_ledger/recovery.rs @@ -0,0 +1,5831 @@ +use rusqlite::{params, Connection, OptionalExtension, Transaction, TransactionBehavior}; + +use super::intent::{ + authoritative_ref_matches_target, db_u64, durable_intent_barrier, load_intent, sql_u64, + stage_intent_evidence, validate_qualified_filesystem_proof, IntentId, IntentState, + PersistedIntent, +}; +use super::{ChangedPathLedger, ExpectedScope}; +use crate::db::util::now_ts; +use crate::error::{Error, Result}; +use crate::ObjectId; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum RecoveryDecision { + FinishPublication, + RetainCandidatesAndAcknowledge, + Abort, + FullReconciliation, +} + +pub(crate) fn recover_scope( + ledger: &ChangedPathLedger<'_>, + expected: &ExpectedScope, +) -> Result> { + let ids = pending_intent_ids(ledger.conn, &expected.scope_id.to_text())?; + let mut decisions = Vec::with_capacity(ids.len()); + for id in ids { + let tx = Transaction::new_unchecked(ledger.conn, TransactionBehavior::Immediate)?; + let intent = load_intent(&tx, &id)?.ok_or_else(|| { + Error::Corrupt(format!( + "pending changed-path intent `{}` disappeared", + id.0 + )) + })?; + if intent.state.is_terminal() { + tx.commit()?; + continue; + } + let exact_scope = intent_matches_expected_scope(&intent, expected) + && scope_cas_matches(&tx, expected)? + && current_change_matches(&tx, &intent)?; + let target_published = authoritative_ref_matches_target(&tx, &intent)?; + let qualified_proof = intent.verified_cut.as_ref().is_some_and(|proof| { + ledger.database_path().is_ok_and(|database_path| { + validate_qualified_filesystem_proof(&tx, database_path, expected, &intent, proof) + .is_ok() + }) + }); + let decision = if exact_scope + && target_published + && matches!( + intent.state, + IntentState::FilesystemApplied | IntentState::Published + ) + && qualified_proof + { + if intent.state == IntentState::FilesystemApplied { + stage_intent_evidence(&tx, &intent)?; + let published = tx.execute( + "UPDATE changed_path_intents SET lifecycle_state='published',updated_at=?1 + WHERE intent_id=?2 AND lifecycle_state='filesystem_applied'", + params![now_ts(), intent.id.0], + )?; + if published != 1 { + return Err(Error::Corrupt(format!( + "intent `{}` changed during recovery publication", + intent.id.0 + ))); + } + } + if finish_publication(&tx, &intent, expected)? { + RecoveryDecision::FinishPublication + } else { + RecoveryDecision::FullReconciliation + } + } else if exact_scope && intent.state == IntentState::Prepared { + retain_and_fail_closed(&tx, &intent, "prepared_intent_recovery_ambiguous")?; + RecoveryDecision::FullReconciliation + } else if exact_scope { + retain_and_fail_closed(&tx, &intent, "intent_publication_state_ambiguous")?; + RecoveryDecision::RetainCandidatesAndAcknowledge + } else { + retain_and_fail_closed(&tx, &intent, "intent_scope_identity_changed")?; + RecoveryDecision::FullReconciliation + }; + tx.commit()?; + decisions.push((id, decision)); + } + if !decisions.is_empty() { + durable_intent_barrier(ledger.conn)?; + crate::db::util::test_crash_point("changed_path_after_recovery_commit"); + } + Ok(decisions) +} + +impl ChangedPathLedger<'_> { + pub(crate) fn recover_scope( + &self, + expected: &ExpectedScope, + ) -> Result> { + recover_scope(self, expected) + } + + pub(crate) fn recover(&self) -> Result> { + let scopes = scopes_with_pending_intents(self.conn)?; + let mut decisions = Vec::new(); + for expected in scopes { + decisions.extend(recover_scope(self, &expected)?); + } + Ok(decisions) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct IntentGcRoot { + pub(crate) change_id: crate::ChangeId, + pub(crate) root_id: ObjectId, + pub(crate) operation_id: Option, +} + +pub(crate) fn ledger_gc_roots(conn: &Connection) -> Result> { + let mut roots = Vec::new(); + let mut statement = conn.prepare( + "SELECT target_change_id,target_root_id,target_operation_id FROM changed_path_intents + WHERE lifecycle_state IN ('prepared','filesystem_applied','published')", + )?; + let rows = statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + )) + })?; + for row in rows { + let (change, root, operation) = row?; + roots.push(IntentGcRoot { + change_id: crate::ChangeId(change), + root_id: ObjectId(root), + operation_id: operation.map(ObjectId), + }); + } + Ok(roots) +} + +pub(crate) fn mark_backup_scopes_untrusted(conn: &Connection) -> Result<()> { + let tx = Transaction::new_unchecked(conn, TransactionBehavior::Immediate)?; + tx.execute( + "UPDATE changed_path_scopes + SET trust_state='untrusted_gap',trust_reason='backup_without_observer_segments', + continuity_generation=continuity_generation+1,updated_at=?1 + WHERE retired_at IS NULL", + [now_ts()], + )?; + tx.execute( + "UPDATE changed_path_intents SET lifecycle_state='aborted', + failure_reason='backup_did_not_fence_observer_segments',updated_at=?1 + WHERE lifecycle_state IN ('prepared','filesystem_applied','published')", + [now_ts()], + )?; + tx.execute("DELETE FROM changed_path_observer_owners", [])?; + tx.execute("DELETE FROM changed_path_observer_segments", [])?; + tx.commit()?; + Ok(()) +} + +pub(crate) fn rotate_restored_scopes( + conn: &Connection, + filesystem_identity: &[u8], + scope_root: &str, + previous_epoch: u64, + previous_continuity_generation: u64, +) -> Result<()> { + let tx = Transaction::new_unchecked(conn, TransactionBehavior::Immediate)?; + let now = now_ts(); + let (source_epoch, source_continuity): (i64, i64) = tx.query_row( + "SELECT COALESCE(MAX(epoch),0),COALESCE(MAX(continuity_generation),0) + FROM changed_path_scopes WHERE retired_at IS NULL", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + let next_epoch = db_u64(source_epoch, "restored source epoch")? + .max(previous_epoch) + .checked_add(1) + .ok_or_else(|| Error::Corrupt("restored scope epoch overflow".into()))?; + let next_continuity = db_u64(source_continuity, "restored source continuity")? + .max(previous_continuity_generation) + .checked_add(1) + .ok_or_else(|| Error::Corrupt("restored scope continuity overflow".into()))?; + tx.execute( + "UPDATE changed_path_intents SET lifecycle_state='aborted', + failure_reason='restore_rotated_scope_identity',updated_at=?1 + WHERE lifecycle_state IN ('prepared','filesystem_applied','published')", + [now], + )?; + tx.execute("DELETE FROM changed_path_observer_owners", [])?; + tx.execute("DELETE FROM changed_path_observer_segments", [])?; + tx.execute( + "UPDATE changed_path_scopes SET epoch=?1,scope_root=?2, + filesystem_identity=?3,scope_root_identity=?3, + provider_id=NULL,provider_identity=NULL,provider_cursor=NULL,provider_fence=NULL, + observer_owner_token=NULL,observer_heartbeat_at=NULL, + durable_offset=0,folded_offset=0,trust_state='untrusted_gap', + trust_reason='restored_filesystem_identity_rotated', + continuity_generation=?4,updated_at=?5 + WHERE retired_at IS NULL", + params![ + sql_u64(next_epoch, "restored target epoch")?, + scope_root, + hex::encode(filesystem_identity), + sql_u64(next_continuity, "restored target continuity")?, + now + ], + )?; + tx.commit()?; + Ok(()) +} + +#[derive(Clone)] +pub(crate) struct SegmentDeletionToken { + scope_id: super::ScopeId, + epoch: u64, + segment_id: String, + directory: std::sync::Arc, + identity: PersistedSegmentDeletion, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PersistedSegmentDeletion { + original_leaf: String, + quarantine_leaf: String, + scope_directory_identity: (u64, u64), + quarantine_identity: (u64, u64), + segment_identity: (u64, u64), + file_length: u64, + file_hash: [u8; 32], + durable_end_offset: u64, + durable_hash: [u8; 32], + limits: super::PersistedLogLimits, + owner_token: [u8; 32], + first_sequence: u64, + last_sequence: Option, + previous_segment_id: Option, + previous_segment_hash: [u8; 32], + source_state: String, + state: String, +} + +impl std::fmt::Debug for SegmentDeletionToken { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SegmentDeletionToken") + .field("scope_id", &self.scope_id.to_text()) + .field("epoch", &self.epoch) + .field("segment_id", &self.segment_id) + .field("leaf", &self.identity.original_leaf) + .field("quarantine_leaf", &self.identity.quarantine_leaf) + .field("state", &self.identity.state) + .finish() + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct RetirementIdentity { + scope_id: super::ScopeId, + epoch: u64, + scope_kind: String, + owner_id: String, + scope_root: String, + ref_name: String, + ref_generation: u64, + change_id: String, + baseline_root_id: ObjectId, + policy_fingerprint: String, + policy_generation: u64, + filesystem_identity: String, + continuity_generation: u64, + provider_identity: Option, + retired_at: Option, +} + +#[cfg(debug_assertions)] +#[derive(Clone, Copy, Eq, PartialEq)] +enum DeletionSubstitutionPoint { + Parent, + BeforeQuarantineMove, + AfterDirectRenameBeforeVerify, +} + +#[cfg(debug_assertions)] +thread_local! { + static DELETION_SUBSTITUTION_HOOK: std::cell::RefCell< + Option<(DeletionSubstitutionPoint, Box)> + > = const { std::cell::RefCell::new(None) }; +} + +#[cfg(debug_assertions)] +fn install_deletion_substitution_hook( + point: DeletionSubstitutionPoint, + hook: impl FnOnce() + 'static, +) { + DELETION_SUBSTITUTION_HOOK.with(|slot| { + *slot.borrow_mut() = Some((point, Box::new(hook))); + }); +} + +#[cfg(debug_assertions)] +fn run_deletion_substitution_hook(point: DeletionSubstitutionPoint) { + DELETION_SUBSTITUTION_HOOK.with(|slot| { + let should_run = slot + .borrow() + .as_ref() + .is_some_and(|(installed, _)| *installed == point); + if should_run { + if let Some((_, hook)) = slot.borrow_mut().take() { + hook(); + } + } + }); +} + +#[cfg(debug_assertions)] +fn clear_deletion_substitution_hook() { + DELETION_SUBSTITUTION_HOOK.with(|slot| { + slot.borrow_mut().take(); + }); +} + +pub(crate) fn retire_scope( + conn: &Connection, + database_path: &std::path::Path, + expected: &ExpectedScope, +) -> Result> { + let identity = + load_retirement_identity(conn, &expected.scope_id.to_text())?.ok_or_else(|| { + Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "scope retirement lookup failed".into(), + command: "trail status".into(), + } + })?; + let expected_provider_identity = hex::encode(&expected.provider_identity); + if identity.epoch != expected.epoch + || identity.ref_name != expected.ref_name + || identity.ref_generation != expected.ref_generation + || identity.baseline_root_id != expected.baseline_root + || identity.policy_fingerprint != hex::encode(expected.policy_fingerprint) + || identity.policy_generation != expected.policy_generation + || identity.filesystem_identity != hex::encode(&expected.filesystem_identity) + || identity.provider_identity.as_deref() != Some(expected_provider_identity.as_str()) + { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "scope retirement expected identity changed".into(), + command: "trail status".into(), + }); + } + retire_scope_identity(conn, database_path, &identity) +} + +fn retire_scope_identity( + conn: &Connection, + database_path: &std::path::Path, + expected: &RetirementIdentity, +) -> Result> { + let retiring = if expected.retired_at.is_none() { + begin_scope_retirement(conn, expected)? + } else { + expected.clone() + }; + let quiesced_rows = if retiring.retired_at.is_none() { + ensure_segment_quarantine_allocations(conn, database_path, &retiring)? + } else { + Vec::new() + }; + let tx = Transaction::new_unchecked(conn, TransactionBehavior::Immediate)?; + let now = now_ts(); + let current_matches: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM changed_path_scopes + WHERE scope_id=?1 AND epoch=?2 AND scope_kind=?3 AND owner_id=?4 AND scope_root=?5 + AND ref_name=?6 AND ref_generation=?7 AND change_id=?8 AND baseline_root_id=?9 + AND policy_fingerprint=?10 AND policy_dependency_generation=?11 + AND filesystem_identity=?12 AND continuity_generation=?13 + AND provider_identity IS ?14 AND retired_at IS ?15)", + params![ + retiring.scope_id.to_text(), + sql_u64(retiring.epoch, "scope epoch")?, + retiring.scope_kind, + retiring.owner_id, + retiring.scope_root, + retiring.ref_name, + sql_u64(retiring.ref_generation, "ref generation")?, + retiring.change_id, + retiring.baseline_root_id.0, + retiring.policy_fingerprint, + sql_u64(retiring.policy_generation, "policy generation")?, + retiring.filesystem_identity, + sql_u64(retiring.continuity_generation, "continuity generation")?, + retiring.provider_identity, + retiring.retired_at, + ], + |row| row.get(0), + )?; + if !current_matches { + return Err(Error::ChangeLedgerReconcileRequired { + scope: retiring.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "scope retirement exact row changed".into(), + command: "trail status".into(), + }); + }; + if retiring.retired_at.is_none() { + prepare_segment_deletion_transactions( + &tx, + database_path, + retiring.scope_id, + retiring.epoch, + &quiesced_rows, + )?; + } else { + validate_segment_deletion_transaction_coverage(&tx, retiring.scope_id, retiring.epoch)?; + } + if retiring.retired_at.is_some() { + tx.commit()?; + durable_intent_barrier(conn)?; + return load_segment_deletion_tokens( + conn, + database_path, + retiring.scope_id, + retiring.epoch, + ); + } + let changed = tx.execute( + "UPDATE changed_path_scopes SET retired_at=?1,trust_state='untrusted_gap', + trust_reason='scope_retired', + observer_owner_token=NULL,observer_heartbeat_at=NULL,updated_at=?1 + WHERE scope_id=?2 AND epoch=?3 AND continuity_generation=?4 + AND provider_identity IS ?5 AND retired_at IS NULL + AND trust_state='untrusted_gap' AND trust_reason='scope_retiring'", + params![ + now, + retiring.scope_id.to_text(), + sql_u64(retiring.epoch, "scope epoch")?, + sql_u64(retiring.continuity_generation, "continuity generation")?, + retiring.provider_identity, + ], + )?; + if changed != 1 { + let observed: (i64, i64, Option, Option) = tx.query_row( + "SELECT epoch,continuity_generation,provider_identity,retired_at + FROM changed_path_scopes WHERE scope_id=?1", + [retiring.scope_id.to_text()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + )?; + return Err(Error::ChangeLedgerReconcileRequired { + scope: retiring.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: format!( + "scope retirement update CAS failed: expected epoch={} continuity={} provider={:?}; observed={observed:?}", + retiring.epoch, retiring.continuity_generation, retiring.provider_identity + ), + command: "trail status".into(), + }); + } + tx.execute( + "UPDATE changed_path_observer_segments SET state='retired',updated_at=?1 + WHERE scope_id=?2 AND epoch=?3 AND state='retiring'", + params![ + now, + retiring.scope_id.to_text(), + sql_u64(retiring.epoch, "scope epoch")? + ], + )?; + crate::db::util::test_crash_point("changed_path_deletion_before_retirement_commit"); + tx.commit()?; + crate::db::util::test_crash_point("changed_path_deletion_after_retirement_commit"); + durable_intent_barrier(conn)?; + crate::db::util::test_crash_point("changed_path_deletion_after_retirement_wal_barrier"); + load_segment_deletion_tokens(conn, database_path, retiring.scope_id, retiring.epoch) +} + +fn begin_scope_retirement( + conn: &Connection, + expected: &RetirementIdentity, +) -> Result { + let trust_reason: String = conn.query_row( + "SELECT trust_reason FROM changed_path_scopes WHERE scope_id=?1", + [expected.scope_id.to_text()], + |row| row.get(0), + )?; + if trust_reason == "scope_retiring" { + return load_retirement_identity(conn, &expected.scope_id.to_text())? + .ok_or_else(|| Error::Corrupt("retiring scope disappeared".into())); + } + let (_, rows) = load_retired_segment_rows(conn, expected.scope_id, expected.epoch)?; + if rows + .iter() + .any(|row| !matches!(row.state.as_str(), "open" | "sealed")) + { + return Err(Error::Corrupt( + "scope retirement started from an invalid observer segment state".into(), + )); + } + let tx = Transaction::new_unchecked(conn, TransactionBehavior::Immediate)?; + let now = now_ts(); + let current_matches: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM changed_path_scopes + WHERE scope_id=?1 AND epoch=?2 AND continuity_generation=?3 + AND provider_identity IS ?4 AND retired_at IS NULL)", + params![ + expected.scope_id.to_text(), + sql_u64(expected.epoch, "scope epoch")?, + sql_u64(expected.continuity_generation, "continuity generation")?, + expected.provider_identity, + ], + |row| row.get(0), + )?; + if !current_matches { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "scope changed before retirement revocation fence".into(), + command: "trail status".into(), + }); + } + let mut fence_nonce = [0_u8; 32]; + getrandom::getrandom(&mut fence_nonce).map_err(|error| { + Error::InvalidInput(format!("retirement fence nonce generation failed: {error}")) + })?; + let revoked = tx.execute( + "UPDATE changed_path_observer_owners + SET lease_state='revoked',fence_nonce=?1,updated_at=?2 + WHERE scope_id=?3 AND epoch=?4 AND lease_state='active'", + params![ + fence_nonce, + now, + expected.scope_id.to_text(), + sql_u64(expected.epoch, "scope epoch")?, + ], + )?; + let post_trigger_continuity = expected + .continuity_generation + .checked_add(u64::from(revoked != 0)) + .ok_or_else(|| Error::Corrupt("retirement continuity generation overflow".into()))?; + let changed = tx.execute( + "UPDATE changed_path_scopes + SET trust_state='untrusted_gap',trust_reason='scope_retiring', + continuity_generation=continuity_generation+?1, + observer_owner_token=NULL,observer_heartbeat_at=NULL,updated_at=?2 + WHERE scope_id=?3 AND epoch=?4 AND continuity_generation=?5 + AND provider_identity IS ?6 AND retired_at IS NULL", + params![ + i64::from(revoked == 0), + now, + expected.scope_id.to_text(), + sql_u64(expected.epoch, "scope epoch")?, + sql_u64(post_trigger_continuity, "continuity generation")?, + expected.provider_identity, + ], + )?; + if changed != 1 { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "scope retirement revocation fence CAS failed".into(), + command: "trail status".into(), + }); + } + tx.execute( + "UPDATE changed_path_observer_segments + SET retirement_source_state=state,state='retiring',updated_at=?1 + WHERE scope_id=?2 AND epoch=?3 AND state IN ('open','sealed')", + params![ + now, + expected.scope_id.to_text(), + sql_u64(expected.epoch, "scope epoch")?, + ], + )?; + tx.commit()?; + durable_intent_barrier(conn)?; + crate::db::util::test_crash_point("changed_path_deletion_after_retirement_fence_barrier"); + load_retirement_identity(conn, &expected.scope_id.to_text())? + .ok_or_else(|| Error::Corrupt("retiring scope disappeared after fence".into())) +} + +fn load_retirement_identity( + conn: &Connection, + scope_id: &str, +) -> Result> { + let row = conn + .query_row( + "SELECT scope_id,epoch,scope_kind,owner_id,scope_root,ref_name,ref_generation, + change_id,baseline_root_id,policy_fingerprint,policy_dependency_generation, + filesystem_identity,continuity_generation,provider_identity,retired_at + FROM changed_path_scopes WHERE scope_id=?1", + [scope_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, i64>(6)?, + row.get::<_, String>(7)?, + row.get::<_, String>(8)?, + row.get::<_, String>(9)?, + row.get::<_, i64>(10)?, + row.get::<_, String>(11)?, + row.get::<_, i64>(12)?, + row.get::<_, Option>(13)?, + row.get::<_, Option>(14)?, + )) + }, + ) + .optional()?; + let Some(row) = row else { + return Ok(None); + }; + let scope_bytes = hex::decode(&row.0).map_err(|error| Error::Corrupt(error.to_string()))?; + Ok(Some(RetirementIdentity { + scope_id: super::ScopeId( + scope_bytes + .try_into() + .map_err(|_| Error::Corrupt("invalid retirement scope id".into()))?, + ), + epoch: db_u64(row.1, "retirement scope epoch")?, + scope_kind: row.2, + owner_id: row.3, + scope_root: row.4, + ref_name: row.5, + ref_generation: db_u64(row.6, "retirement ref generation")?, + change_id: row.7, + baseline_root_id: ObjectId(row.8), + policy_fingerprint: row.9, + policy_generation: db_u64(row.10, "retirement policy generation")?, + filesystem_identity: row.11, + continuity_generation: db_u64(row.12, "retirement continuity generation")?, + provider_identity: row.13, + retired_at: row.14, + })) +} + +pub(crate) fn retire_deletion_scopes( + conn: &Connection, + database_path: &std::path::Path, + owner_ids: &[&str], + scope_roots: &[&str], + ref_names: &[&str], +) -> Result> { + let scope_ids = { + let mut statement = conn.prepare( + "SELECT scope_id,owner_id,scope_root,ref_name + FROM changed_path_scopes + WHERE scope_kind IN ('materialized_lane','workspace_view') + ORDER BY scope_id", + )?; + let rows = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + })? + .collect::, _>>()? + .into_iter() + .filter_map(|(scope_id, owner_id, scope_root, ref_name)| { + (owner_ids.contains(&owner_id.as_str()) + || scope_roots.contains(&scope_root.as_str()) + || ref_names.contains(&ref_name.as_str())) + .then_some(scope_id) + }) + .collect::>(); + rows + }; + let mut retired_paths = Vec::new(); + for scope_id in scope_ids { + let identity = load_retirement_identity(conn, &scope_id)?.ok_or_else(|| { + Error::ChangeLedgerReconcileRequired { + scope: scope_id.clone(), + state: "untrusted_gap".into(), + reason: "deletion scope disappeared before retirement".into(), + command: "trail status".into(), + } + })?; + retired_paths.extend(retire_scope_identity(conn, database_path, &identity)?); + } + Ok(retired_paths) +} + +fn validate_retired_segment_paths(paths: &[String]) -> Result<()> { + let mut unique = std::collections::HashSet::with_capacity(paths.len()); + for path in paths { + let parsed = std::path::Path::new(path); + let mut components = parsed.components(); + let confined = matches!( + (components.next(), components.next()), + (Some(std::path::Component::Normal(_)), None) + ); + if !confined + || path + .chars() + .any(|character| matches!(character, '/' | '\\' | '\0')) + || !path.ends_with(".cpl") + || path.len() > 128 + || !unique.insert(path) + { + return Err(Error::Corrupt(format!( + "observer segment retirement path is not confined: `{path}`" + ))); + } + } + Ok(()) +} + +#[derive(Debug)] +struct RetiredSegmentRow { + segment_id: String, + log_format_version: u64, + owner_token: [u8; 32], + provider_id: String, + first_sequence: u64, + last_sequence: Option, + durable_end_offset: u64, + folded_end_offset: u64, + previous_segment_id: Option, + previous_segment_hash: [u8; 32], + stored_segment_hash: Option<[u8; 32]>, + segment_path: String, + state: String, + source_state: String, + file_length: Option, + file_hash: Option<[u8; 32]>, + durable_hash: Option<[u8; 32]>, + source_identity: Option<(u64, u64)>, + quiescence_file: Option, +} + +#[derive(Clone, Debug)] +struct QuarantineAllocation { + attempt_nonce: String, + quarantine_leaf: String, + scope_directory_identity: (u64, u64), + source_identity: (u64, u64), + quarantine_identity: Option<(u64, u64)>, + state: String, +} + +fn allocation_quarantine_leaf( + scope_id: super::ScopeId, + epoch: u64, + segment_id: &str, + attempt_nonce: &str, +) -> String { + use sha2::{Digest, Sha256}; + + let mut hasher = Sha256::new(); + hasher.update(b"trail-segment-direct-quarantine-v1\0"); + hasher.update(scope_id.0); + hasher.update(epoch.to_le_bytes()); + hasher.update(segment_id.as_bytes()); + hasher.update(attempt_nonce.as_bytes()); + format!(".trail-delete-{}.cplq", hex::encode(hasher.finalize())) +} + +fn new_allocation_nonce() -> Result { + let mut nonce = [0_u8; 32]; + getrandom::getrandom(&mut nonce).map_err(|error| { + Error::InvalidInput(format!("allocation nonce generation failed: {error}")) + })?; + Ok(hex::encode(nonce)) +} + +fn insert_quarantine_allocation( + tx: &Transaction<'_>, + scope_id: super::ScopeId, + epoch: u64, + segment_id: &str, + scope_directory_identity: (u64, u64), + source_identity: (u64, u64), +) -> Result<()> { + let attempt_nonce = new_allocation_nonce()?; + let quarantine_leaf = allocation_quarantine_leaf(scope_id, epoch, segment_id, &attempt_nonce); + let now = now_ts(); + tx.execute( + "INSERT INTO changed_path_segment_quarantine_allocations( + attempt_nonce,scope_id,epoch,segment_id,quarantine_leaf, + scope_directory_device,scope_directory_inode,identity_policy, + source_segment_device,source_segment_inode,quarantine_device,quarantine_inode, + observed_conflict_device,observed_conflict_inode,retained_reason,state, + created_at,updated_at,allocated_at,bound_at,abandoned_at) + VALUES(?1,?2,?3,?4,?5,?6,?7,'direct_noreplace_same_directory_v1', + ?8,?9,NULL,NULL,NULL,NULL,NULL,'allocating',?10,?10,NULL,NULL,NULL)", + params![ + attempt_nonce, + scope_id.to_text(), + sql_u64(epoch, "scope epoch")?, + segment_id, + quarantine_leaf, + encode_fs_identity_part(scope_directory_identity.0), + encode_fs_identity_part(scope_directory_identity.1), + encode_fs_identity_part(source_identity.0), + encode_fs_identity_part(source_identity.1), + now, + ], + )?; + Ok(()) +} + +fn load_active_quarantine_allocation( + conn: &Connection, + scope_id: super::ScopeId, + epoch: u64, + segment_id: &str, +) -> Result { + let rows = conn + .prepare( + "SELECT attempt_nonce,quarantine_leaf, + scope_directory_device,scope_directory_inode, + source_segment_device,source_segment_inode, + quarantine_device,quarantine_inode,state,identity_policy + FROM changed_path_segment_quarantine_allocations + WHERE scope_id=?1 AND epoch=?2 AND segment_id=?3 + AND state IN ('allocating','allocated','bound')", + )? + .query_map( + params![ + scope_id.to_text(), + sql_u64(epoch, "scope epoch")?, + segment_id + ], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, String>(8)?, + row.get::<_, String>(9)?, + )) + }, + )? + .collect::, _>>()?; + if rows.len() != 1 { + return Err(Error::Corrupt(format!( + "segment requires exactly one active quarantine allocation: {}/{epoch}/{segment_id} observed={}", + scope_id.to_text(), + rows.len() + ))); + } + let row = &rows[0]; + if row.9 != "direct_noreplace_same_directory_v1" || row.6.is_some() != row.7.is_some() { + return Err(Error::Corrupt( + "quarantine allocation identity policy or identity pair is invalid".into(), + )); + } + Ok(QuarantineAllocation { + attempt_nonce: row.0.clone(), + quarantine_leaf: row.1.clone(), + scope_directory_identity: ( + decode_fs_identity_part(&row.2, "allocation scope directory device")?, + decode_fs_identity_part(&row.3, "allocation scope directory inode")?, + ), + source_identity: ( + decode_fs_identity_part(&row.4, "allocation source device")?, + decode_fs_identity_part(&row.5, "allocation source inode")?, + ), + quarantine_identity: row + .6 + .as_deref() + .zip(row.7.as_deref()) + .map(|(device, inode)| { + Ok::<(u64, u64), Error>(( + decode_fs_identity_part(device, "allocation quarantine device")?, + decode_fs_identity_part(inode, "allocation quarantine inode")?, + )) + }) + .transpose()?, + state: row.8.clone(), + }) +} + +fn load_retired_segment_rows( + conn: &Connection, + scope_id: super::ScopeId, + epoch: u64, +) -> Result<(super::PersistedLogLimits, Vec)> { + let limit_values: (i64, i64, i64) = conn.query_row( + "SELECT max_observer_log_bytes,max_segment_bytes,max_unfolded_tail_records + FROM changed_path_scopes WHERE scope_id=?1 AND epoch=?2", + params![scope_id.to_text(), sql_u64(epoch, "scope epoch")?], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + let limits = super::PersistedLogLimits { + max_log_bytes: db_u64(limit_values.0, "maximum observer log bytes")?, + max_segment_bytes: db_u64(limit_values.1, "maximum segment bytes")?, + max_unfolded_tail_records: usize::try_from(db_u64( + limit_values.2, + "maximum unfolded records", + )?) + .map_err(|_| Error::Corrupt("maximum unfolded records exceeds memory".into()))?, + }; + let rows = conn + .prepare( + "SELECT segment_id,owner_token,first_sequence,last_sequence,durable_end_offset, + previous_segment_id,previous_segment_hash,segment_hash,segment_path,state, + COALESCE(retirement_source_state,state),log_format_version,provider_id, + folded_end_offset,retirement_file_length,retirement_file_hash, + retirement_durable_hash,retirement_source_device,retirement_source_inode + FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=?2 ORDER BY first_sequence,segment_id", + )? + .query_map( + params![scope_id.to_text(), sql_u64(epoch, "scope epoch")?], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, String>(8)?, + row.get::<_, String>(9)?, + row.get::<_, String>(10)?, + row.get::<_, i64>(11)?, + row.get::<_, String>(12)?, + row.get::<_, i64>(13)?, + row.get::<_, Option>(14)?, + row.get::<_, Option>(15)?, + row.get::<_, Option>(16)?, + row.get::<_, Option>(17)?, + row.get::<_, Option>(18)?, + )) + }, + )? + .collect::, _>>()? + .into_iter() + .map(|row| { + Ok(RetiredSegmentRow { + segment_id: row.0, + log_format_version: db_u64(row.11, "retired segment log format")?, + owner_token: decode_hex_32(&row.1, "retired segment owner token")?, + provider_id: row.12, + first_sequence: db_u64(row.2, "retired segment first sequence")?, + last_sequence: row + .3 + .map(|value| db_u64(value, "retired segment last sequence")) + .transpose()?, + durable_end_offset: db_u64(row.4, "retired segment durable offset")?, + folded_end_offset: db_u64(row.13, "retired segment folded offset")?, + previous_segment_id: row.5, + previous_segment_hash: row + .6 + .as_deref() + .map(|value| decode_hex_32(value, "retired previous segment hash")) + .transpose()? + .unwrap_or([0; 32]), + stored_segment_hash: row + .7 + .as_deref() + .map(|value| decode_hex_32(value, "retired segment hash")) + .transpose()?, + segment_path: row.8, + state: row.9, + source_state: row.10, + file_length: row + .14 + .map(|value| db_u64(value, "retirement file length")) + .transpose()?, + file_hash: row + .15 + .as_deref() + .map(|value| decode_hex_32(value, "retirement file hash")) + .transpose()?, + durable_hash: row + .16 + .as_deref() + .map(|value| decode_hex_32(value, "retirement durable hash")) + .transpose()?, + source_identity: row + .17 + .as_deref() + .zip(row.18.as_deref()) + .map(|(device, inode)| { + Ok::<_, Error>(( + decode_fs_identity_part(device, "retirement source device")?, + decode_fs_identity_part(inode, "retirement source inode")?, + )) + }) + .transpose()?, + quiescence_file: None, + }) + }) + .collect::>>()?; + validate_retired_segment_paths( + &rows + .iter() + .map(|row| row.segment_path.clone()) + .collect::>(), + )?; + Ok((limits, rows)) +} + +fn scope_segment_directory_path( + database_path: &std::path::Path, + scope_id: super::ScopeId, +) -> Result { + let trail_root = database_path + .parent() + .and_then(std::path::Path::parent) + .ok_or_else(|| Error::Corrupt("retirement database has no Trail root".into()))?; + Ok(trail_root + .join("observer-segments") + .join(scope_id.to_text())) +} + +fn inspect_segments_before_allocation( + conn: &Connection, + database_path: &std::path::Path, + scope_id: super::ScopeId, + epoch: u64, +) -> Result<((u64, u64), Vec)> { + let (limits, mut rows) = load_retired_segment_rows(conn, scope_id, epoch)?; + if rows.is_empty() { + return Ok(((0, 0), rows)); + } + let directory = super::secure_fs::SecureDirectory::open_absolute( + &scope_segment_directory_path(database_path, scope_id)?, + )?; + let scope_directory_identity = directory.identity()?; + let mut expected_previous_segment_id: Option = None; + let mut expected_previous_segment_hash = [0; 32]; + let mut expected_first_sequence = 1_u64; + let mut saw_open_segment = false; + for row in &mut rows { + if saw_open_segment + || row.previous_segment_id != expected_previous_segment_id + || row.previous_segment_hash != expected_previous_segment_hash + || row.first_sequence != expected_first_sequence + { + return Err(Error::Corrupt(format!( + "retired observer segment metadata lineage is not exact at `{}`", + row.segment_id + ))); + } + let file = match directory.open_regular(&row.segment_path) { + Ok(file) => file, + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => { + let allocation = + load_active_quarantine_allocation(conn, scope_id, epoch, &row.segment_id)?; + let file = directory.open_regular(&allocation.quarantine_leaf)?; + let identity = super::secure_fs::file_identity(&file)?; + if identity != allocation.source_identity { + return Err(Error::InvalidInput( + "journaled quarantine does not match the authenticated source inode".into(), + )); + } + file + } + Err(error) => return Err(error), + }; + let source_identity = super::secure_fs::file_identity(&file)?; + match super::secure_fs::try_lock_observer_quiescence(&file) { + Ok(()) => {} + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::WouldBlock => { + return Err(Error::WorkspaceLocked(format!( + "observer segment writer has not acknowledged close: {}", + row.segment_id + ))); + } + Err(error) => return Err(error), + } + let authenticated = super::log::authenticate_segment_for_deletion( + &file, + &super::log::DeletionSegmentExpectation { + scope_id, + epoch, + segment_id: row.segment_id.clone(), + owner_token: row.owner_token, + first_sequence: row.first_sequence, + last_sequence: row.last_sequence, + durable_end_offset: row.durable_end_offset, + previous_segment_hash: row.previous_segment_hash, + stored_segment_hash: row.stored_segment_hash, + state: row.source_state.clone(), + limits, + }, + ) + .map_err(|error| Error::Corrupt(error.to_string()))?; + if row + .file_length + .is_some_and(|value| value != authenticated.file_length) + || row + .file_hash + .is_some_and(|value| value != authenticated.file_hash) + || row + .durable_hash + .is_some_and(|value| value != authenticated.durable_hash) + || row + .source_identity + .is_some_and(|value| value != source_identity) + { + return Err(Error::Corrupt( + "persisted retirement source metadata changed before allocation".into(), + )); + } + row.file_length = Some(authenticated.file_length); + row.file_hash = Some(authenticated.file_hash); + row.durable_hash = Some(authenticated.durable_hash); + row.source_identity = Some(source_identity); + row.quiescence_file = Some(file); + expected_previous_segment_id = Some(row.segment_id.clone()); + expected_previous_segment_hash = authenticated.durable_hash; + saw_open_segment = row.source_state == "open"; + expected_first_sequence = row + .last_sequence + .unwrap_or(row.first_sequence.saturating_sub(1)) + .checked_add(1) + .ok_or_else(|| Error::Corrupt("retired segment sequence overflow".into()))?; + } + Ok((scope_directory_identity, rows)) +} + +fn journal_missing_quarantine_allocations( + conn: &Connection, + expected: &RetirementIdentity, + scope_directory_identity: (u64, u64), + rows: &[RetiredSegmentRow], +) -> Result<()> { + if rows.is_empty() { + return Ok(()); + } + let tx = Transaction::new_unchecked(conn, TransactionBehavior::Immediate)?; + if load_retirement_identity(&tx, &expected.scope_id.to_text())?.as_ref() != Some(expected) { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "scope identity changed before quarantine allocation journal".into(), + command: "trail status".into(), + }); + } + for row in rows { + let source_identity = row.source_identity.ok_or_else(|| { + Error::Corrupt("retirement source identity was not authenticated".into()) + })?; + let file_length = row + .file_length + .ok_or_else(|| Error::Corrupt("retirement file length was not authenticated".into()))?; + let file_hash = row + .file_hash + .ok_or_else(|| Error::Corrupt("retirement file hash was not authenticated".into()))?; + let durable_hash = row.durable_hash.ok_or_else(|| { + Error::Corrupt("retirement durable hash was not authenticated".into()) + })?; + let source_published = tx.execute( + "UPDATE changed_path_observer_segments + SET retirement_file_length=?1,retirement_file_hash=?2, + retirement_durable_hash=?3,retirement_source_device=?4, + retirement_source_inode=?5,updated_at=?6 + WHERE scope_id=?7 AND epoch=?8 AND segment_id=?9 AND state='retiring' + AND owner_token=?10 AND provider_id=?11 AND log_format_version=?12 + AND first_sequence=?13 AND last_sequence IS ?14 + AND durable_end_offset=?15 AND folded_end_offset=?16 + AND segment_path=?17 AND retirement_source_state=?18", + params![ + sql_u64(file_length, "retirement file length")?, + hex::encode(file_hash), + hex::encode(durable_hash), + encode_fs_identity_part(source_identity.0), + encode_fs_identity_part(source_identity.1), + now_ts(), + expected.scope_id.to_text(), + sql_u64(expected.epoch, "scope epoch")?, + row.segment_id, + hex::encode(row.owner_token), + row.provider_id, + sql_u64(row.log_format_version, "segment log format")?, + sql_u64(row.first_sequence, "segment first sequence")?, + row.last_sequence + .map(|value| sql_u64(value, "segment last sequence")) + .transpose()?, + sql_u64(row.durable_end_offset, "segment durable offset")?, + sql_u64(row.folded_end_offset, "segment folded offset")?, + row.segment_path, + row.source_state, + ], + )?; + if source_published != 1 { + return Err(Error::Conflict(format!( + "retirement source metadata changed before publication: {}", + row.segment_id + ))); + } + let active: i64 = tx.query_row( + "SELECT COUNT(*) FROM changed_path_segment_quarantine_allocations + WHERE scope_id=?1 AND epoch=?2 AND segment_id=?3 + AND state IN ('allocating','allocated','bound')", + params![ + expected.scope_id.to_text(), + sql_u64(expected.epoch, "scope epoch")?, + row.segment_id + ], + |sql_row| sql_row.get(0), + )?; + match active { + 0 => insert_quarantine_allocation( + &tx, + expected.scope_id, + expected.epoch, + &row.segment_id, + scope_directory_identity, + source_identity, + )?, + 1 => {} + count => { + return Err(Error::Corrupt(format!( + "multiple active quarantine allocations exist for segment `{}`: {count}", + row.segment_id + ))); + } + } + } + tx.commit()?; + durable_intent_barrier(conn)?; + crate::db::util::test_crash_point("changed_path_deletion_after_allocation_journal_barrier"); + Ok(()) +} + +fn abandon_quarantine_allocation_and_replace( + conn: &Connection, + expected: &RetirementIdentity, + segment_id: &str, + allocation: &QuarantineAllocation, + observed_identity: Option<(u64, u64)>, + reason: &str, + replace: bool, +) -> Result<()> { + let tx = Transaction::new_unchecked(conn, TransactionBehavior::Immediate)?; + if load_retirement_identity(&tx, &expected.scope_id.to_text())?.as_ref() != Some(expected) { + return Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "scope identity changed while replacing quarantine allocation".into(), + command: "trail status".into(), + }); + } + let now = now_ts(); + let changed = tx.execute( + "UPDATE changed_path_segment_quarantine_allocations + SET state='abandoned',observed_conflict_device=?1,observed_conflict_inode=?2, + retained_reason=?3,updated_at=?4,abandoned_at=?4 + WHERE attempt_nonce=?5 AND state=?6", + params![ + observed_identity.map(|identity| encode_fs_identity_part(identity.0)), + observed_identity.map(|identity| encode_fs_identity_part(identity.1)), + reason, + now, + allocation.attempt_nonce, + allocation.state, + ], + )?; + if changed != 1 { + return Err(Error::Conflict( + "quarantine allocation changed while abandoning it".into(), + )); + } + if replace { + insert_quarantine_allocation( + &tx, + expected.scope_id, + expected.epoch, + segment_id, + allocation.scope_directory_identity, + allocation.source_identity, + )?; + } + tx.commit()?; + durable_intent_barrier(conn) +} + +fn mark_quarantine_allocation_allocated( + conn: &Connection, + allocation: &QuarantineAllocation, + identity: (u64, u64), +) -> Result<()> { + let tx = Transaction::new_unchecked(conn, TransactionBehavior::Immediate)?; + let now = now_ts(); + let changed = tx.execute( + "UPDATE changed_path_segment_quarantine_allocations + SET state='allocated',quarantine_device=?1, + quarantine_inode=?2,updated_at=?3,allocated_at=?3 + WHERE attempt_nonce=?4 AND state='allocating' + AND quarantine_device IS NULL AND quarantine_inode IS NULL", + params![ + encode_fs_identity_part(identity.0), + encode_fs_identity_part(identity.1), + now, + allocation.attempt_nonce, + ], + )?; + if changed != 1 { + return Err(Error::Conflict( + "quarantine allocation changed before identity publication".into(), + )); + } + tx.commit()?; + durable_intent_barrier(conn)?; + crate::db::util::test_crash_point("changed_path_deletion_after_allocation_identity_barrier"); + Ok(()) +} + +fn authenticate_direct_quarantine( + file: &std::fs::File, + row: &RetiredSegmentRow, + scope_id: super::ScopeId, + epoch: u64, + limits: super::PersistedLogLimits, +) -> Result<()> { + super::log::authenticate_segment_for_deletion( + file, + &super::log::DeletionSegmentExpectation { + scope_id, + epoch, + segment_id: row.segment_id.clone(), + owner_token: row.owner_token, + first_sequence: row.first_sequence, + last_sequence: row.last_sequence, + durable_end_offset: row.durable_end_offset, + previous_segment_hash: row.previous_segment_hash, + stored_segment_hash: row.stored_segment_hash, + state: row.source_state.clone(), + limits, + }, + ) + .map(|_| ()) + .map_err(|error| Error::Corrupt(error.to_string())) +} + +fn ensure_segment_quarantine_allocations( + conn: &Connection, + database_path: &std::path::Path, + expected: &RetirementIdentity, +) -> Result> { + let (scope_directory_identity, rows) = + inspect_segments_before_allocation(conn, database_path, expected.scope_id, expected.epoch)?; + if rows.is_empty() { + return Ok(rows); + } + journal_missing_quarantine_allocations(conn, expected, scope_directory_identity, &rows)?; + let directory = super::secure_fs::SecureDirectory::open_absolute( + &scope_segment_directory_path(database_path, expected.scope_id)?, + )?; + let (limits, _) = load_retired_segment_rows(conn, expected.scope_id, expected.epoch)?; + directory.verify_identity(scope_directory_identity)?; + for (index, row) in rows.iter().enumerate() { + let mut attempts = 0_usize; + loop { + attempts += 1; + if attempts > 64 { + return Err(Error::Conflict(format!( + "too many conflicting quarantine allocations for segment `{}`", + row.segment_id + ))); + } + let allocation = load_active_quarantine_allocation( + conn, + expected.scope_id, + expected.epoch, + &row.segment_id, + )?; + directory.verify_identity(allocation.scope_directory_identity)?; + match allocation.state.as_str() { + "allocated" => { + let opened = directory.open_regular(&allocation.quarantine_leaf)?; + let observed = super::secure_fs::file_identity(&opened)?; + if Some(observed) != allocation.quarantine_identity + || observed != allocation.source_identity + { + abandon_quarantine_allocation_and_replace( + conn, + expected, + &row.segment_id, + &allocation, + Some(observed), + "allocated_direct_quarantine_identity_mismatch", + false, + )?; + return Err(Error::InvalidInput( + "allocated direct quarantine identity changed".into(), + )); + } + authenticate_direct_quarantine( + &opened, + row, + expected.scope_id, + expected.epoch, + limits, + )?; + break; + } + "allocating" => match directory.open_regular(&allocation.quarantine_leaf) { + Ok(opened) => { + let observed = super::secure_fs::file_identity(&opened)?; + if observed != allocation.source_identity { + let source_still_present = + open_optional_regular(&directory, &row.segment_path)?.is_some(); + abandon_quarantine_allocation_and_replace( + conn, + expected, + &row.segment_id, + &allocation, + Some(observed), + "direct_quarantine_target_identity_mismatch", + source_still_present, + )?; + if source_still_present { + continue; + } + return Err(Error::InvalidInput( + "direct quarantine target does not match journaled source".into(), + )); + } + authenticate_direct_quarantine( + &opened, + row, + expected.scope_id, + expected.epoch, + limits, + )?; + crate::db::util::test_crash_point( + "changed_path_deletion_after_direct_quarantine_verify", + ); + opened.sync_all()?; + directory.sync()?; + crate::db::util::test_crash_point( + "changed_path_deletion_after_direct_quarantine_fsync", + ); + mark_quarantine_allocation_allocated(conn, &allocation, observed)?; + break; + } + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => { + let source = directory.open_regular(&row.segment_path)?; + if super::secure_fs::file_identity(&source)? != allocation.source_identity { + return Err(Error::InvalidInput( + "retirement source changed after allocation journal".into(), + )); + } + #[cfg(debug_assertions)] + run_deletion_substitution_hook( + DeletionSubstitutionPoint::BeforeQuarantineMove, + ); + match directory + .rename_leaf_noreplace(&row.segment_path, &allocation.quarantine_leaf) + { + Ok(()) => { + crate::db::util::test_crash_point( + "changed_path_deletion_after_direct_quarantine_rename", + ); + #[cfg(debug_assertions)] + run_deletion_substitution_hook( + DeletionSubstitutionPoint::AfterDirectRenameBeforeVerify, + ); + } + Err(Error::Io(error)) + if error.kind() == std::io::ErrorKind::AlreadyExists => + { + let observed = directory + .open_regular(&allocation.quarantine_leaf) + .ok() + .and_then(|opened| { + super::secure_fs::file_identity(&opened).ok() + }); + abandon_quarantine_allocation_and_replace( + conn, + expected, + &row.segment_id, + &allocation, + observed, + "direct_quarantine_target_preexisting", + true, + )?; + } + Err(error) => return Err(error), + } + } + Err(error) => return Err(error), + }, + state => { + return Err(Error::Corrupt(format!( + "unexpected active quarantine allocation state `{state}`" + ))); + } + } + } + if index + 1 < rows.len() { + crate::db::util::test_crash_point("changed_path_deletion_between_allocation_segments"); + } + } + crate::db::util::test_crash_point("changed_path_deletion_after_allocation_setup"); + Ok(rows) +} + +fn prepare_segment_deletion_transactions( + tx: &Transaction<'_>, + database_path: &std::path::Path, + scope_id: super::ScopeId, + epoch: u64, + rows: &[RetiredSegmentRow], +) -> Result<()> { + let (limits, persisted_rows) = load_retired_segment_rows(tx, scope_id, epoch)?; + if persisted_rows.len() != rows.len() { + return Err(Error::Corrupt( + "retirement segment set changed after writer quiescence".into(), + )); + } + if rows.is_empty() { + return Ok(()); + } + let trail_root = database_path + .parent() + .and_then(std::path::Path::parent) + .ok_or_else(|| Error::Corrupt("retirement database has no Trail root".into()))?; + let directory_path = trail_root + .join("observer-segments") + .join(scope_id.to_text()); + let directory = super::secure_fs::SecureDirectory::open_absolute(&directory_path)?; + let scope_directory_identity = directory.identity()?; + let (retirement_generation, retirement_owner_token, retirement_fence): (i64, String, Vec) = + tx.query_row( + "SELECT scope.continuity_generation,owner.owner_token,owner.fence_nonce + FROM changed_path_scopes scope + JOIN changed_path_observer_owners owner ON owner.scope_id=scope.scope_id + WHERE scope.scope_id=?1 AND scope.epoch=?2 + AND scope.trust_state='untrusted_gap' AND scope.trust_reason='scope_retiring' + AND owner.epoch=scope.epoch AND owner.lease_state='revoked'", + params![scope_id.to_text(), sql_u64(epoch, "scope epoch")?], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + if retirement_fence.len() != 32 { + return Err(Error::Corrupt( + "retirement owner fence nonce is not exact".into(), + )); + } + let mut expected_previous_segment_id: Option = None; + let mut expected_previous_segment_hash = [0; 32]; + let mut expected_first_sequence = 1_u64; + let mut saw_open_segment = false; + for row in rows { + if saw_open_segment + || row.previous_segment_id != expected_previous_segment_id + || row.previous_segment_hash != expected_previous_segment_hash + || row.first_sequence != expected_first_sequence + { + return Err(Error::Corrupt(format!( + "retired observer segment metadata lineage is not exact at `{}`", + row.segment_id + ))); + } + let existing: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM changed_path_segment_deletions + WHERE scope_id=?1 AND epoch=?2 AND segment_id=?3)", + params![ + scope_id.to_text(), + sql_u64(epoch, "scope epoch")?, + row.segment_id + ], + |sql_row| sql_row.get(0), + )?; + if existing { + return Err(Error::Corrupt(format!( + "segment deletion transaction already exists before retirement: {}", + row.segment_id + ))); + } + let allocation = load_active_quarantine_allocation(tx, scope_id, epoch, &row.segment_id)?; + if hex::encode(row.owner_token) != retirement_owner_token { + return Err(Error::Corrupt( + "retirement segment owner does not match revoked fence owner".into(), + )); + } + if allocation.state != "allocated" { + return Err(Error::Corrupt(format!( + "segment quarantine allocation is not ready for binding: {} state={}", + row.segment_id, allocation.state + ))); + } + if allocation.scope_directory_identity != scope_directory_identity { + return Err(Error::InvalidInput(format!( + "segment quarantine allocation parent identity changed: {}", + row.segment_id + ))); + } + let quarantine_identity = allocation + .quarantine_identity + .ok_or_else(|| Error::Corrupt("allocated quarantine identity is missing".into()))?; + let file = directory.open_regular(&allocation.quarantine_leaf)?; + let observed_identity = super::secure_fs::file_identity(&file)?; + if observed_identity != quarantine_identity + || observed_identity != allocation.source_identity + { + return Err(Error::InvalidInput( + "direct quarantine identity changed before retirement binding".into(), + )); + } + let segment_identity = allocation.source_identity; + let authenticated = super::log::authenticate_segment_for_deletion( + &file, + &super::log::DeletionSegmentExpectation { + scope_id, + epoch, + segment_id: row.segment_id.clone(), + owner_token: row.owner_token, + first_sequence: row.first_sequence, + last_sequence: row.last_sequence, + durable_end_offset: row.durable_end_offset, + previous_segment_hash: row.previous_segment_hash, + stored_segment_hash: row.stored_segment_hash, + state: row.source_state.clone(), + limits, + }, + ) + .map_err(|error| Error::Corrupt(error.to_string()))?; + expected_previous_segment_id = Some(row.segment_id.clone()); + expected_previous_segment_hash = authenticated.durable_hash; + saw_open_segment = row.source_state == "open"; + expected_first_sequence = row + .last_sequence + .unwrap_or(row.first_sequence.saturating_sub(1)) + .checked_add(1) + .ok_or_else(|| Error::Corrupt("retired segment sequence overflow".into()))?; + let now = now_ts(); + tx.execute( + "INSERT INTO changed_path_segment_deletions( + scope_id,epoch,segment_id,original_leaf,quarantine_leaf,allocation_nonce, + log_format_version,provider_id,folded_end_offset, + retirement_continuity_generation,retirement_fence_nonce, + scope_directory_device,scope_directory_inode,quarantine_device,quarantine_inode, + segment_device,segment_inode,file_length,file_hash,durable_end_offset, + durable_hash,max_observer_log_bytes,max_segment_bytes, + max_unfolded_tail_records,owner_token,first_sequence,last_sequence, + previous_segment_id,previous_segment_hash,source_state,state, + created_at,updated_at,completed_at) + VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14, + ?15,?16,?17,?18,?19,?20,?21,?22,?23,?24,?25,?26,?27, + ?28,?29,?30,'quiesced',?31,?31,?31)", + params![ + scope_id.to_text(), + sql_u64(epoch, "scope epoch")?, + row.segment_id, + row.segment_path, + allocation.quarantine_leaf, + allocation.attempt_nonce, + sql_u64(row.log_format_version, "segment log format")?, + row.provider_id, + sql_u64(row.folded_end_offset, "segment folded offset")?, + retirement_generation, + retirement_fence, + encode_fs_identity_part(scope_directory_identity.0), + encode_fs_identity_part(scope_directory_identity.1), + encode_fs_identity_part(quarantine_identity.0), + encode_fs_identity_part(quarantine_identity.1), + encode_fs_identity_part(segment_identity.0), + encode_fs_identity_part(segment_identity.1), + sql_u64(authenticated.file_length, "segment file length")?, + hex::encode(authenticated.file_hash), + sql_u64(row.durable_end_offset, "segment durable offset")?, + hex::encode(authenticated.durable_hash), + sql_u64(limits.max_log_bytes, "maximum observer log bytes")?, + sql_u64(limits.max_segment_bytes, "maximum segment bytes")?, + sql_u64( + u64::try_from(limits.max_unfolded_tail_records).map_err(|_| { + Error::InvalidInput("maximum unfolded records exceeds SQLite".into()) + })?, + "maximum unfolded records" + )?, + hex::encode(row.owner_token), + sql_u64(row.first_sequence, "segment first sequence")?, + row.last_sequence + .map(|value| sql_u64(value, "segment last sequence")) + .transpose()?, + row.previous_segment_id, + hex::encode(row.previous_segment_hash), + row.source_state, + now, + ], + )?; + let bound = tx.execute( + "UPDATE changed_path_segment_quarantine_allocations + SET state='bound',updated_at=?1,bound_at=?1 + WHERE attempt_nonce=?2 AND state='allocated'", + params![now, allocation.attempt_nonce], + )?; + if bound != 1 { + return Err(Error::Conflict(format!( + "segment quarantine allocation changed before binding: {}", + row.segment_id + ))); + } + } + directory.sync()?; + Ok(()) +} + +fn validate_segment_deletion_transaction_coverage( + conn: &Connection, + scope_id: super::ScopeId, + epoch: u64, +) -> Result<()> { + let (segments, deletions, missing, invalid_allocations): (i64, i64, i64, i64) = conn + .query_row( + "SELECT + (SELECT COUNT(*) FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=?2), + (SELECT COUNT(*) FROM changed_path_segment_deletions + WHERE scope_id=?1 AND epoch=?2), + (SELECT COUNT(*) FROM changed_path_observer_segments segment + WHERE segment.scope_id=?1 AND segment.epoch=?2 + AND NOT EXISTS( + SELECT 1 FROM changed_path_segment_deletions deletion + WHERE deletion.scope_id=segment.scope_id AND deletion.epoch=segment.epoch + AND deletion.segment_id=segment.segment_id)), + (SELECT COUNT(*) FROM changed_path_segment_deletions deletion + LEFT JOIN changed_path_segment_quarantine_allocations allocation + ON allocation.attempt_nonce=deletion.allocation_nonce + LEFT JOIN changed_path_observer_segments segment + ON segment.scope_id=deletion.scope_id AND segment.epoch=deletion.epoch + AND segment.segment_id=deletion.segment_id + LEFT JOIN changed_path_scopes scope ON scope.scope_id=deletion.scope_id + LEFT JOIN changed_path_observer_owners owner ON owner.scope_id=deletion.scope_id + WHERE deletion.scope_id=?1 AND deletion.epoch=?2 + AND (allocation.attempt_nonce IS NULL OR allocation.state<>'bound' + OR segment.segment_id IS NULL OR segment.state<>'retired' + OR scope.retired_at IS NULL OR scope.trust_reason<>'scope_retired' + OR owner.owner_token IS NULL OR owner.lease_state<>'revoked' + OR allocation.scope_id<>deletion.scope_id + OR allocation.epoch<>deletion.epoch + OR allocation.segment_id<>deletion.segment_id + OR allocation.quarantine_leaf<>deletion.quarantine_leaf + OR allocation.scope_directory_device<> + deletion.scope_directory_device + OR allocation.scope_directory_inode<> + deletion.scope_directory_inode + OR allocation.source_segment_device<>deletion.segment_device + OR allocation.source_segment_inode<>deletion.segment_inode + OR allocation.quarantine_device<>deletion.quarantine_device + OR allocation.quarantine_inode<>deletion.quarantine_inode + OR deletion.original_leaf<>segment.segment_path + OR deletion.log_format_version<>segment.log_format_version + OR deletion.provider_id<>segment.provider_id + OR deletion.provider_id<>owner.provider_id + OR deletion.provider_id IS NOT scope.provider_id + OR deletion.owner_token<>segment.owner_token + OR deletion.owner_token<>owner.owner_token + OR deletion.first_sequence<>segment.first_sequence + OR deletion.last_sequence IS NOT segment.last_sequence + OR deletion.durable_end_offset<>segment.durable_end_offset + OR deletion.folded_end_offset<>segment.folded_end_offset + OR deletion.previous_segment_id IS NOT segment.previous_segment_id + OR deletion.previous_segment_hash<> + COALESCE(segment.previous_segment_hash, + '0000000000000000000000000000000000000000000000000000000000000000') + OR deletion.source_state<>segment.retirement_source_state + OR deletion.file_length<>segment.retirement_file_length + OR deletion.file_hash<>segment.retirement_file_hash + OR deletion.durable_hash<>segment.retirement_durable_hash + OR deletion.segment_device<>segment.retirement_source_device + OR deletion.segment_inode<>segment.retirement_source_inode + OR deletion.retirement_continuity_generation<>scope.continuity_generation + OR deletion.retirement_fence_nonce<>owner.fence_nonce))", + params![scope_id.to_text(), sql_u64(epoch, "scope epoch")?], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + )?; + if segments != deletions || missing != 0 || invalid_allocations != 0 { + return Err(Error::Corrupt(format!( + "retired segment deletion transaction coverage mismatch: segments={segments} deletions={deletions} missing={missing} invalid_allocations={invalid_allocations}" + ))); + } + Ok(()) +} + +fn load_segment_deletion_tokens( + conn: &Connection, + database_path: &std::path::Path, + scope_id: super::ScopeId, + epoch: u64, +) -> Result> { + validate_segment_deletion_transaction_coverage(conn, scope_id, epoch)?; + let trail_root = database_path + .parent() + .and_then(std::path::Path::parent) + .ok_or_else(|| Error::Corrupt("retirement database has no Trail root".into()))?; + let rows = load_persisted_segment_deletions(conn, scope_id, epoch)?; + if rows.is_empty() { + return Ok(Vec::new()); + } + let directory = std::sync::Arc::new(super::secure_fs::SecureDirectory::open_absolute( + &trail_root + .join("observer-segments") + .join(scope_id.to_text()), + )?); + rows.into_iter() + .map(|(segment_id, identity)| { + directory.verify_identity(identity.scope_directory_identity)?; + let quarantined = directory.open_regular(&identity.quarantine_leaf)?; + if super::secure_fs::file_identity(&quarantined)? != identity.quarantine_identity { + return Err(Error::InvalidInput( + "direct quarantine identity changed while loading deletion token".into(), + )); + } + Ok(SegmentDeletionToken { + scope_id, + epoch, + segment_id, + directory: directory.clone(), + identity, + }) + }) + .collect() +} + +pub(crate) fn remove_retired_segments( + conn: &Connection, + tokens: &[SegmentDeletionToken], +) -> Result<()> { + for token in tokens { + let current = load_one_persisted_segment_deletion( + conn, + token.scope_id, + token.epoch, + &token.segment_id, + )?; + validate_deletion_token_identity(token, ¤t)?; + validate_retired_segment_paths(std::slice::from_ref(¤t.original_leaf))?; + token + .directory + .verify_identity(current.scope_directory_identity)?; + #[cfg(debug_assertions)] + run_deletion_substitution_hook(DeletionSubstitutionPoint::Parent); + let original = open_optional_regular(&token.directory, ¤t.original_leaf)?; + let quarantined = open_optional_regular(&token.directory, ¤t.quarantine_leaf)?; + if current.state != "quiesced" || original.is_some() || quarantined.is_none() { + return Err(Error::InvalidInput( + "quiesced direct segment retirement has missing or reappeared evidence".into(), + )); + } + let quarantined = quarantined.expect("checked direct quarantine presence"); + if super::secure_fs::file_identity(&quarantined)? != current.quarantine_identity { + return Err(Error::InvalidInput( + "quiesced direct quarantine inode changed".into(), + )); + } + authenticate_persisted_deletion_file(token, ¤t, &quarantined)?; + } + Ok(()) +} + +fn load_persisted_segment_deletions( + conn: &Connection, + scope_id: super::ScopeId, + epoch: u64, +) -> Result> { + conn.prepare( + "SELECT segment_id,original_leaf,quarantine_leaf, + scope_directory_device,scope_directory_inode,quarantine_device,quarantine_inode, + segment_device,segment_inode,file_length,file_hash,durable_end_offset, + durable_hash,max_observer_log_bytes,max_segment_bytes, + max_unfolded_tail_records,owner_token,first_sequence,last_sequence, + previous_segment_id,previous_segment_hash,source_state,state + FROM changed_path_segment_deletions + WHERE scope_id=?1 AND epoch=?2 ORDER BY segment_id", + )? + .query_map( + params![scope_id.to_text(), sql_u64(epoch, "scope epoch")?], + decode_persisted_segment_deletion, + )? + .collect::, _>>()? + .into_iter() + .map(decode_persisted_segment_deletion_values) + .collect() +} + +type PersistedSegmentDeletionValues = ( + String, + String, + String, + String, + String, + String, + String, + String, + String, + i64, + String, + i64, + String, + i64, + i64, + i64, + String, + i64, + Option, + Option, + String, + String, + String, +); + +fn decode_persisted_segment_deletion( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + row.get(6)?, + row.get(7)?, + row.get(8)?, + row.get(9)?, + row.get(10)?, + row.get(11)?, + row.get(12)?, + row.get(13)?, + row.get(14)?, + row.get(15)?, + row.get(16)?, + row.get(17)?, + row.get(18)?, + row.get(19)?, + row.get(20)?, + row.get(21)?, + row.get(22)?, + )) +} + +fn decode_persisted_segment_deletion_values( + row: PersistedSegmentDeletionValues, +) -> Result<(String, PersistedSegmentDeletion)> { + Ok(( + row.0, + PersistedSegmentDeletion { + original_leaf: row.1, + quarantine_leaf: row.2, + scope_directory_identity: ( + decode_fs_identity_part(&row.3, "scope directory device")?, + decode_fs_identity_part(&row.4, "scope directory inode")?, + ), + quarantine_identity: ( + decode_fs_identity_part(&row.5, "quarantine device")?, + decode_fs_identity_part(&row.6, "quarantine inode")?, + ), + segment_identity: ( + decode_fs_identity_part(&row.7, "segment device")?, + decode_fs_identity_part(&row.8, "segment inode")?, + ), + file_length: db_u64(row.9, "deletion file length")?, + file_hash: decode_hex_32(&row.10, "deletion file hash")?, + durable_end_offset: db_u64(row.11, "deletion durable offset")?, + durable_hash: decode_hex_32(&row.12, "deletion durable hash")?, + limits: super::PersistedLogLimits { + max_log_bytes: db_u64(row.13, "deletion maximum log bytes")?, + max_segment_bytes: db_u64(row.14, "deletion maximum segment bytes")?, + max_unfolded_tail_records: usize::try_from(db_u64( + row.15, + "deletion maximum unfolded records", + )?) + .map_err(|_| Error::Corrupt("deletion record cap exceeds memory".into()))?, + }, + owner_token: decode_hex_32(&row.16, "deletion owner token")?, + first_sequence: db_u64(row.17, "deletion first sequence")?, + last_sequence: row + .18 + .map(|value| db_u64(value, "deletion last sequence")) + .transpose()?, + previous_segment_id: row.19, + previous_segment_hash: decode_hex_32(&row.20, "deletion previous hash")?, + source_state: row.21, + state: row.22, + }, + )) +} + +fn load_one_persisted_segment_deletion( + conn: &Connection, + scope_id: super::ScopeId, + epoch: u64, + segment_id: &str, +) -> Result { + load_persisted_segment_deletions(conn, scope_id, epoch)? + .into_iter() + .find_map(|(observed_id, identity)| (observed_id == segment_id).then_some(identity)) + .ok_or_else(|| { + Error::Corrupt(format!( + "persisted segment deletion disappeared: {}/{epoch}/{segment_id}", + scope_id.to_text() + )) + }) +} + +fn validate_deletion_token_identity( + token: &SegmentDeletionToken, + current: &PersistedSegmentDeletion, +) -> Result<()> { + let mut expected = token.identity.clone(); + expected.state.clone_from(¤t.state); + if expected != *current { + return Err(Error::InvalidInput( + "persisted segment deletion identity changed after authority issuance".into(), + )); + } + Ok(()) +} + +fn open_optional_regular( + directory: &super::secure_fs::SecureDirectory, + leaf: &str, +) -> Result> { + match directory.open_regular(leaf) { + Ok(file) => Ok(Some(file)), + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } +} + +fn authenticate_persisted_deletion_file( + token: &SegmentDeletionToken, + identity: &PersistedSegmentDeletion, + file: &std::fs::File, +) -> Result<()> { + if super::secure_fs::file_identity(file)? != identity.segment_identity { + return Err(Error::InvalidInput( + "segment deletion inode does not match persisted authority".into(), + )); + } + let authenticated = super::log::authenticate_segment_for_deletion( + file, + &super::log::DeletionSegmentExpectation { + scope_id: token.scope_id, + epoch: token.epoch, + segment_id: token.segment_id.clone(), + owner_token: identity.owner_token, + first_sequence: identity.first_sequence, + last_sequence: identity.last_sequence, + durable_end_offset: identity.durable_end_offset, + previous_segment_hash: identity.previous_segment_hash, + stored_segment_hash: (identity.source_state == "sealed") + .then_some(identity.durable_hash), + state: identity.source_state.clone(), + limits: identity.limits, + }, + ) + .map_err(|error| Error::Corrupt(error.to_string()))?; + if authenticated.file_length != identity.file_length + || authenticated.file_hash != identity.file_hash + || authenticated.durable_hash != identity.durable_hash + { + return Err(Error::InvalidInput( + "segment deletion bytes do not match persisted authority".into(), + )); + } + Ok(()) +} + +fn update_segment_deletion_state( + conn: &Connection, + token: &SegmentDeletionToken, + expected_states: &[&str], + state: &str, + completed: bool, +) -> Result<()> { + let tx = Transaction::new_unchecked(conn, TransactionBehavior::Immediate)?; + let persisted = + load_one_persisted_segment_deletion(&tx, token.scope_id, token.epoch, &token.segment_id)?; + validate_deletion_token_identity(token, &persisted)?; + let current = persisted.state; + if current == state || (completed && current == "quiesced") { + tx.commit()?; + durable_intent_barrier(conn)?; + return Ok(()); + } + if !expected_states.contains(¤t.as_str()) { + return Err(Error::InvalidInput(format!( + "segment deletion state `{current}` cannot advance to `{state}`" + ))); + } + let now = now_ts(); + let changed = tx.execute( + "UPDATE changed_path_segment_deletions + SET state=?1,updated_at=?2,completed_at=?3 + WHERE scope_id=?4 AND epoch=?5 AND segment_id=?6 AND state=?7", + params![ + state, + now, + completed.then_some(now), + token.scope_id.to_text(), + sql_u64(token.epoch, "scope epoch")?, + token.segment_id, + current, + ], + )?; + if changed != 1 { + return Err(Error::Conflict( + "segment deletion state changed during durable transition".into(), + )); + } + tx.commit()?; + durable_intent_barrier(conn) +} + +fn encode_fs_identity_part(value: u64) -> String { + format!("{value:016x}") +} + +fn decode_fs_identity_part(value: &str, label: &str) -> Result { + u64::from_str_radix(value, 16).map_err(|_| Error::Corrupt(format!("invalid persisted {label}"))) +} + +fn decode_hex_32(value: &str, label: &str) -> Result<[u8; 32]> { + hex::decode(value) + .map_err(|_| Error::Corrupt(format!("invalid {label} encoding")))? + .try_into() + .map_err(|_| Error::Corrupt(format!("invalid {label} length"))) +} + +fn finish_publication( + tx: &Transaction<'_>, + intent: &PersistedIntent, + expected: &ExpectedScope, +) -> Result { + let changed = tx.execute( + "UPDATE changed_path_scopes SET change_id=?1,baseline_root_id=?2, + ref_generation=?3,updated_at=?4 + WHERE scope_id=?5 AND epoch=?6 AND ref_name=?7 AND ref_generation=?8 + AND change_id=?9 AND baseline_root_id=?10 AND trust_state='trusted'", + params![ + intent.target.change_id.0, + intent.target.root_id.0, + sql_u64( + intent.expected_ref_generation.saturating_add(1), + "target generation" + )?, + now_ts(), + intent.scope_id, + sql_u64(intent.expected_epoch, "scope epoch")?, + intent.expected_ref_name, + sql_u64(intent.expected_ref_generation, "ref generation")?, + intent.expected_change_id.0, + intent.expected_root_id.0 + ], + )?; + if changed != 1 { + retain_and_fail_closed(tx, intent, "intent_baseline_publication_cas_failed")?; + return Ok(false); + } + clear_intent_ownership(tx, intent)?; + let acknowledged = tx.execute( + "UPDATE changed_path_intents SET lifecycle_state='acknowledged',failure_reason=NULL, + updated_at=?1 WHERE intent_id=?2 AND lifecycle_state='published'", + params![now_ts(), intent.id.0], + )?; + if acknowledged != 1 { + return Err(Error::Corrupt(format!( + "published intent `{}` could not be acknowledged", + intent.id.0 + ))); + } + let _ = expected; + Ok(true) +} + +fn clear_intent_ownership(tx: &Transaction<'_>, intent: &PersistedIntent) -> Result<()> { + for table in ["changed_path_entries", "changed_path_prefixes"] { + let delete = format!("DELETE FROM {table} WHERE intent_id=?1 AND source_mask=2"); + tx.execute(&delete, [&intent.id.0])?; + let retain = format!( + "UPDATE {table} SET source_mask=source_mask & ~2,intent_id=NULL,updated_at=?1 + WHERE intent_id=?2 AND (source_mask & ~2) != 0" + ); + tx.execute(&retain, params![now_ts(), intent.id.0])?; + } + Ok(()) +} + +fn retain_and_fail_closed( + tx: &Transaction<'_>, + intent: &PersistedIntent, + reason: &str, +) -> Result<()> { + stage_intent_evidence(tx, intent)?; + tx.execute( + "UPDATE changed_path_intents SET lifecycle_state='aborted',failure_reason=?1, + updated_at=?2 WHERE intent_id=?3 AND lifecycle_state IN + ('prepared','filesystem_applied','published')", + params![reason, now_ts(), intent.id.0], + )?; + tx.execute( + "UPDATE changed_path_scopes SET + continuity_generation=continuity_generation+ + CASE WHEN trust_state IN ('trusted','reconciling') THEN 1 ELSE 0 END, + trust_state=CASE WHEN trust_state IN ('trusted','reconciling') + THEN 'untrusted_gap' ELSE trust_state END, + trust_reason=CASE WHEN trust_state IN ('trusted','reconciling') + THEN ?1 ELSE trust_reason END, + updated_at=?2 WHERE scope_id=?3", + params![reason, now_ts(), intent.scope_id], + )?; + Ok(()) +} + +fn pending_intent_ids(conn: &Connection, scope_id: &str) -> Result> { + conn.prepare( + "SELECT intent_id FROM changed_path_intents WHERE scope_id=?1 + AND lifecycle_state IN ('prepared','filesystem_applied','published') + ORDER BY created_at,intent_id", + )? + .query_map([scope_id], |row| row.get::<_, String>(0))? + .map(|row| row.map(IntentId).map_err(Error::from)) + .collect() +} + +fn intent_matches_expected_scope(intent: &PersistedIntent, expected: &ExpectedScope) -> bool { + intent.scope_id == expected.scope_id.to_text() + && intent.expected_epoch == expected.epoch + && intent.expected_ref_name == expected.ref_name + && intent.expected_ref_generation == expected.ref_generation + && intent.expected_root_id == expected.baseline_root + && intent + .verified_cut + .as_ref() + .is_none_or(|proof| proof.filesystem_identity == expected.filesystem_identity) +} + +fn current_change_matches(conn: &Connection, intent: &PersistedIntent) -> Result { + let change = conn + .query_row( + "SELECT change_id FROM changed_path_scopes WHERE scope_id=?1", + [&intent.scope_id], + |row| row.get::<_, String>(0), + ) + .optional()?; + Ok(change.as_deref() == Some(intent.expected_change_id.0.as_str())) +} + +fn scope_cas_matches(conn: &Connection, expected: &ExpectedScope) -> Result { + match super::intent::exact_scope_guard(conn, expected, false) { + Ok(()) => Ok(true), + Err(Error::ChangeLedgerReconcileRequired { .. }) => Ok(false), + Err(error) => Err(error), + } +} + +fn scopes_with_pending_intents(conn: &Connection) -> Result> { + let mut statement = conn.prepare( + "SELECT DISTINCT s.scope_id,s.epoch,s.ref_name,s.ref_generation,s.baseline_root_id, + s.policy_fingerprint,s.policy_dependency_generation,s.filesystem_identity, + COALESCE(s.provider_identity,'') + FROM changed_path_scopes s JOIN changed_path_intents i ON i.scope_id=s.scope_id + WHERE i.lifecycle_state IN ('prepared','filesystem_applied','published') + ORDER BY s.scope_id", + )?; + let rows = statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, i64>(6)?, + row.get::<_, String>(7)?, + row.get::<_, String>(8)?, + )) + })?; + rows.map(|row| { + let row = row?; + let scope_bytes = hex::decode(row.0).map_err(|error| Error::Corrupt(error.to_string()))?; + let scope_id: [u8; 32] = scope_bytes + .try_into() + .map_err(|_| Error::Corrupt("invalid scope id".into()))?; + let policy_bytes = hex::decode(row.5).map_err(|error| Error::Corrupt(error.to_string()))?; + let policy: [u8; 32] = policy_bytes + .try_into() + .map_err(|_| Error::Corrupt("invalid policy fingerprint".into()))?; + Ok(ExpectedScope { + scope_id: super::ScopeId(scope_id), + epoch: db_u64(row.1, "scope epoch")?, + ref_name: row.2, + ref_generation: db_u64(row.3, "ref generation")?, + baseline_root: ObjectId(row.4), + policy_fingerprint: policy, + policy_generation: db_u64(row.6, "policy generation")?, + filesystem_identity: hex::decode(row.7) + .map_err(|error| Error::Corrupt(error.to_string()))?, + provider_identity: hex::decode(row.8) + .map_err(|error| Error::Corrupt(error.to_string()))?, + }) + }) + .collect() +} + +#[cfg(debug_assertions)] +mod harness { + use rusqlite::{params, Connection}; + use std::fs::OpenOptions; + use std::process::{Command, Stdio}; + use std::time::{Duration, Instant}; + + use super::*; + use crate::db::change_ledger::intent::install_sidecar_ancestor_substitution_hook; + #[cfg(test)] + use crate::db::change_ledger::intent::{ + install_sidecar_post_validation_hook, install_sidecar_retry_hook, + }; + use crate::db::change_ledger::{ + mark_filesystem_applied, prepare_intent, publish_intent, BaselineIdentity, DirtyPrefix, + DurableCut, EvidenceCut, EvidenceFlags, EvidenceSource, FilesystemIdentity, IntentEvidence, + IntentProducer, IntentTarget, LedgerPath, PolicyIdentity, ProviderCapabilities, + ProviderIdentity, QualifiedFilesystemProof, ScopeId, ScopeIdentity, ScopeKind, + }; + use crate::db::{InitImportMode, Trail}; + use crate::model::{Actor, FileContentRef}; + use crate::{ChangeId, ObjectId}; + + struct Fixture { + _root: tempfile::TempDir, + db: Trail, + expected: ExpectedScope, + backup: std::path::PathBuf, + restore: std::path::PathBuf, + } + + impl Fixture { + fn new(tag: u8) -> Result { + let root = tempfile::tempdir()?; + let workspace = root.path().join("workspace"); + std::fs::create_dir_all(&workspace)?; + Trail::init(&workspace, "main", InitImportMode::Empty, false)?; + let db = Trail::open(&workspace)?; + let head = db.get_ref("refs/branches/main")?; + let scope = ScopeIdentity { + scope_id: ScopeId([tag; 32]), + kind: ScopeKind::Workspace, + owner_id: format!("recovery-harness-{tag}"), + }; + let baseline = BaselineIdentity { + ref_name: head.name.clone(), + ref_generation: head + .generation + .try_into() + .map_err(|_| Error::Corrupt("negative fixture generation".into()))?, + change_id: head.change_id.clone(), + root_id: head.root_id.clone(), + }; + let policy = PolicyIdentity { + fingerprint: [tag.wrapping_add(1); 32], + generation: 1, + }; + let filesystem = FilesystemIdentity(vec![tag, 0, 0xff]); + let provider = ProviderIdentity { + identity: vec![tag.wrapping_add(2), 0x80], + capabilities: ProviderCapabilities { + durable_cursor: true, + linearizable_fence: true, + rename_pairing: true, + overflow_scope: true, + filesystem_supported: true, + clean_proof_allowed: true, + power_loss_durability: true, + }, + }; + db.changed_path_ledger().begin_scope( + &scope, + &baseline, + &policy, + &filesystem, + &provider, + )?; + db.conn.execute( + "UPDATE changed_path_scopes SET trust_state='trusted',trust_reason='test_fence', + provider_cursor=?1 WHERE scope_id=?2", + params![b"cursor-1".as_slice(), scope.scope_id.to_text()], + )?; + let expected = ExpectedScope { + scope_id: scope.scope_id, + epoch: 1, + ref_name: baseline.ref_name, + ref_generation: baseline.ref_generation, + baseline_root: baseline.root_id, + policy_fingerprint: policy.fingerprint, + policy_generation: policy.generation, + filesystem_identity: filesystem.0, + provider_identity: provider.identity, + }; + let backup = root.path().join("backup"); + let restore = root.path().join("restore"); + Ok(Self { + _root: root, + db, + expected, + backup, + restore, + }) + } + + fn target(&self, suffix: &str, root: ObjectId) -> IntentTarget { + IntentTarget { + change_id: ChangeId(format!("change-recovery-{suffix}")), + root_id: root, + operation_id: None, + } + } + + fn evidence(path: &str) -> IntentEvidence { + IntentEvidence { + exact_paths: vec![LedgerPath::parse(path).unwrap()], + complete_prefixes: Vec::new(), + } + } + + fn qualified_proof(&self, sequence: u64, path: &str) -> Result { + let (proof, writer) = self.qualified_proof_with_writer(sequence, path)?; + drop(writer); + Ok(proof) + } + + fn qualified_proof_with_writer( + &self, + sequence: u64, + path: &str, + ) -> Result<(QualifiedFilesystemProof, super::super::SegmentWriter)> { + let provider_id: String = self.db.conn.query_row( + "SELECT provider_id FROM changed_path_scopes WHERE scope_id=?1", + [self.expected.scope_id.to_text()], + |row| row.get(0), + )?; + let owner_token_bytes = [self.expected.scope_id.0[0].wrapping_add(0x31); 32]; + let owner_token = hex::encode(owner_token_bytes); + let fence_nonce = b"qualified-fence".to_vec(); + let relative_directory = + format!("observer-segments/{}", self.expected.scope_id.to_text()); + let segment_directory = self.db.db_dir.join(&relative_directory); + let mut writer = super::super::SegmentWriter::acquire( + &self.db.db_dir.join(crate::db::DB_RELATIVE_PATH), + &segment_directory, + self.expected.scope_id, + self.expected.epoch, + owner_token_bytes, + &provider_id, + b"cursor-1".to_vec(), + Duration::from_secs(600), + )?; + let records = (1..=sequence) + .map(|record_sequence| { + Ok(super::super::ObserverRecord { + sequence: record_sequence, + source: EvidenceSource::Observer, + path: LedgerPath::parse(path)?, + flags: EvidenceFlags::CONTENT, + provider_cursor: format!("cursor-{}", record_sequence + 1).into_bytes(), + }) + }) + .collect::>>()?; + writer.append(&records)?; + let durable = writer.flush_durable()?; + writer.rotate()?; + let end_cursor = durable.provider_cursor.clone(); + self.insert_observer_event(path, sequence)?; + self.db.conn.execute( + "UPDATE changed_path_scopes SET provider_cursor=?1,durable_offset=?2, + folded_offset=?2,observer_owner_token=?3 WHERE scope_id=?4", + params![ + end_cursor, + sql_u64(durable.durable_end_offset, "fixture durable offset")?, + owner_token, + self.expected.scope_id.to_text() + ], + )?; + self.db.conn.execute( + "UPDATE changed_path_observer_owners + SET provider_identity=?1,fence_nonce=?2 WHERE scope_id=?3", + params![ + hex::encode(&self.expected.provider_identity), + fence_nonce, + self.expected.scope_id.to_text(), + ], + )?; + self.db.conn.execute( + "UPDATE changed_path_observer_segments SET folded_end_offset=durable_end_offset + WHERE scope_id=?1 AND segment_id=?2", + params![self.expected.scope_id.to_text(), durable.segment_id], + )?; + let segment = self.db.conn.query_row( + "SELECT segment_hash,segment_path,durable_end_offset,folded_end_offset + FROM changed_path_observer_segments + WHERE scope_id=?1 AND segment_id=?2 AND state='sealed'", + params![self.expected.scope_id.to_text(), durable.segment_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + )) + }, + )?; + let segment_hash: [u8; 32] = hex::decode(segment.0) + .map_err(|error| Error::Corrupt(error.to_string()))? + .try_into() + .map_err(|_| Error::Corrupt("invalid fixture segment hash".into()))?; + Ok(( + QualifiedFilesystemProof { + scope_id: self.expected.scope_id, + epoch: self.expected.epoch, + expected_root_id: self.expected.baseline_root.clone(), + scope_root_identity: self.expected.filesystem_identity.clone(), + filesystem_identity: self.expected.filesystem_identity.clone(), + provider_id, + provider_identity: self.expected.provider_identity.clone(), + observer_owner_token: owner_token, + owner_fence_nonce: Some(fence_nonce), + durable_segment_id: durable.segment_id.clone(), + durable_segment_hash: segment_hash, + segment_directory: relative_directory, + segment_path: segment.1, + start_cursor: Some(b"cursor-1".to_vec()), + publication_segment_id: durable.segment_id.clone(), + publication_cursor: end_cursor.clone(), + end_cursor, + start_sequence: 1, + end_cut: EvidenceCut { + source: EvidenceSource::Observer, + sequence, + durable_offset: durable.durable_end_offset, + folded_offset: durable.durable_end_offset, + }, + publication_cut: EvidenceCut { + source: EvidenceSource::Observer, + sequence, + durable_offset: durable.durable_end_offset, + folded_offset: durable.durable_end_offset, + }, + segment_durable_offset: db_u64(segment.2, "fixture segment durable")?, + segment_folded_offset: db_u64(segment.3, "fixture segment folded")?, + verified_paths: 1, + verified_prefixes: 0, + complete_root_interval: true, + complete_policy_interval: true, + persisted_evidence_through_end: true, + }, + writer, + )) + } + + fn advance_ref_to(&self, target: &IntentTarget) -> Result<()> { + self.db.conn.execute( + "UPDATE refs SET change_id=?1,root_id=?2,generation=generation+1,updated_at=?3 + WHERE name=?4 AND generation=?5", + params![ + target.change_id.0, + target.root_id.0, + now_ts(), + self.expected.ref_name, + sql_u64(self.expected.ref_generation, "fixture generation")? + ], + )?; + Ok(()) + } + + fn insert_observer_event(&self, path: &str, sequence: u64) -> Result<()> { + self.db.conn.execute( + "INSERT INTO changed_path_entries(scope_id,normalized_path,event_flags,source_mask, + first_sequence,last_sequence,provider_id,provider_sequence,intent_id,created_at,updated_at) + VALUES(?1,?2,?3,1,?4,?4,'observer',?4,NULL,?5,?5) + ON CONFLICT(scope_id,normalized_path) DO UPDATE SET + event_flags=changed_path_entries.event_flags|excluded.event_flags, + source_mask=changed_path_entries.source_mask|excluded.source_mask, + last_sequence=MAX(changed_path_entries.last_sequence,excluded.last_sequence), + provider_sequence=MAX(changed_path_entries.provider_sequence,excluded.provider_sequence), + updated_at=excluded.updated_at", + params![self.expected.scope_id.to_text(),path,EvidenceFlags::CONTENT.0, + sql_u64(sequence,"observer sequence")?,now_ts()], + )?; + Ok(()) + } + + fn insert_observer_prefix(&self, path: &str, sequence: u64) -> Result<()> { + self.db.conn.execute( + "INSERT INTO changed_path_prefixes( + scope_id,normalized_prefix,completeness_reason,event_flags,source_mask, + first_sequence,last_sequence,provider_id,provider_sequence,intent_id, + created_at,updated_at) + VALUES(?1,?2,'provider_complete',?3,1,?4,?4,'observer',?4,NULL,?5,?5) + ON CONFLICT(scope_id,normalized_prefix) DO UPDATE SET + event_flags=changed_path_prefixes.event_flags|excluded.event_flags, + source_mask=changed_path_prefixes.source_mask|excluded.source_mask, + first_sequence=MIN(changed_path_prefixes.first_sequence,excluded.first_sequence), + last_sequence=MAX(changed_path_prefixes.last_sequence,excluded.last_sequence), + provider_sequence=MAX(changed_path_prefixes.provider_sequence,excluded.provider_sequence), + updated_at=excluded.updated_at", + params![ + self.expected.scope_id.to_text(), + path, + EvidenceFlags::CONTENT.0, + sql_u64(sequence, "observer prefix sequence")?, + now_ts() + ], + )?; + Ok(()) + } + + fn proof_for_durable_segment( + &self, + durable: &DurableCut, + start_cursor: Vec, + start_sequence: u64, + verified_paths: u64, + verified_prefixes: u64, + ) -> Result { + let provider_id: String = self.db.conn.query_row( + "SELECT provider_id FROM changed_path_scopes WHERE scope_id=?1", + [self.expected.scope_id.to_text()], + |row| row.get(0), + )?; + let segment = self.db.conn.query_row( + "SELECT segment_hash,segment_path,durable_end_offset,folded_end_offset + FROM changed_path_observer_segments + WHERE scope_id=?1 AND segment_id=?2 AND state='sealed'", + params![self.expected.scope_id.to_text(), durable.segment_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + )) + }, + )?; + let segment_hash = hex::decode(segment.0) + .map_err(|error| Error::Corrupt(error.to_string()))? + .try_into() + .map_err(|_| Error::Corrupt("invalid fixture segment hash".into()))?; + Ok(QualifiedFilesystemProof { + scope_id: self.expected.scope_id, + epoch: self.expected.epoch, + expected_root_id: self.expected.baseline_root.clone(), + scope_root_identity: self.expected.filesystem_identity.clone(), + filesystem_identity: self.expected.filesystem_identity.clone(), + provider_id, + provider_identity: self.expected.provider_identity.clone(), + observer_owner_token: hex::encode( + [self.expected.scope_id.0[0].wrapping_add(0x31); 32], + ), + owner_fence_nonce: Some(b"qualified-fence".to_vec()), + durable_segment_id: durable.segment_id.clone(), + durable_segment_hash: segment_hash, + segment_directory: format!( + "observer-segments/{}", + self.expected.scope_id.to_text() + ), + segment_path: segment.1, + start_cursor: Some(start_cursor), + end_cursor: durable.provider_cursor.clone(), + publication_segment_id: durable.segment_id.clone(), + publication_cursor: durable.provider_cursor.clone(), + start_sequence, + end_cut: EvidenceCut { + source: EvidenceSource::Observer, + sequence: durable.last_sequence, + durable_offset: durable.durable_end_offset, + folded_offset: durable.durable_end_offset, + }, + publication_cut: EvidenceCut { + source: EvidenceSource::Observer, + sequence: durable.last_sequence, + durable_offset: durable.durable_end_offset, + folded_offset: durable.durable_end_offset, + }, + segment_durable_offset: db_u64(segment.2, "fixture segment durable")?, + segment_folded_offset: db_u64(segment.3, "fixture segment folded")?, + verified_paths, + verified_prefixes, + complete_root_interval: true, + complete_policy_interval: true, + persisted_evidence_through_end: true, + }) + } + } + + pub(super) fn acknowledgement_race() -> Result<()> { + let fixture = Fixture::new(0x61)?; + let target = fixture.target("ack", fixture.expected.baseline_root.clone()); + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &target, + &Fixture::evidence("src/lib.rs"), + )?; + mark_filesystem_applied( + &ledger, + &fixture.expected, + &intent, + &fixture.qualified_proof(7, "src/lib.rs")?, + )?; + fixture.insert_observer_event("src/lib.rs", 8)?; + fixture.advance_ref_to(&target)?; + publish_intent(&ledger, &fixture.expected, &intent)?; + let decisions = recover_scope(&ledger, &fixture.expected)?; + if decisions != vec![(intent.clone(), RecoveryDecision::FinishPublication)] { + return Err(Error::Corrupt(format!( + "unexpected recovery decision: {decisions:?}" + ))); + } + let row = fixture.db.conn.query_row( + "SELECT source_mask,provider_sequence,intent_id FROM changed_path_entries + WHERE scope_id=?1 AND normalized_path='src/lib.rs'", + [fixture.expected.scope_id.to_text()], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + )) + }, + )?; + if row != (EvidenceSource::Observer.mask(), Some(8), None) { + return Err(Error::Corrupt(format!( + "concurrent observer evidence was cleared: {row:?}" + ))); + } + Ok(()) + } + + pub(super) fn empty_rotation_anchor_authenticates_publication_cut() -> Result<()> { + let fixture = Fixture::new(0x80)?; + let target = fixture.target( + "empty-publication-anchor", + fixture.expected.baseline_root.clone(), + ); + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &target, + &Fixture::evidence("empty-publication-anchor.rs"), + )?; + let (mut proof, mut writer) = + fixture.qualified_proof_with_writer(1, "empty-publication-anchor.rs")?; + let publication = writer.flush_durable()?; + fixture.db.conn.execute( + "UPDATE changed_path_observer_segments SET folded_end_offset=durable_end_offset + WHERE scope_id=?1 AND segment_id=?2", + params![fixture.expected.scope_id.to_text(), publication.segment_id], + )?; + fixture.db.conn.execute( + "UPDATE changed_path_scopes SET provider_cursor=?1,durable_offset=?2,folded_offset=?2 + WHERE scope_id=?3", + params![ + publication.provider_cursor, + sql_u64( + publication.durable_end_offset, + "empty publication durable offset" + )?, + fixture.expected.scope_id.to_text(), + ], + )?; + proof.publication_segment_id = publication.segment_id; + proof.publication_cursor = publication.provider_cursor; + proof.publication_cut = EvidenceCut { + source: EvidenceSource::Observer, + sequence: publication.last_sequence, + durable_offset: publication.durable_end_offset, + folded_offset: publication.durable_end_offset, + }; + mark_filesystem_applied(&ledger, &fixture.expected, &intent, &proof)?; + drop(writer); + Ok(()) + } + + pub(super) fn authenticated_prefix_survives_later_observer_advance() -> Result<()> { + let fixture = Fixture::new(0x81)?; + let target = fixture.target("advanced-prefix", fixture.expected.baseline_root.clone()); + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &target, + &Fixture::evidence("advanced-prefix.rs"), + )?; + let (mut proof, mut writer) = + fixture.qualified_proof_with_writer(1, "advanced-prefix.rs")?; + + writer.append(&[super::super::ObserverRecord { + sequence: 2, + source: EvidenceSource::Observer, + path: LedgerPath::parse("advanced-prefix.rs")?, + flags: EvidenceFlags::CONTENT, + provider_cursor: b"cursor-3".to_vec(), + }])?; + let publication = writer.flush_durable()?; + fixture.db.conn.execute( + "UPDATE changed_path_observer_segments SET folded_end_offset=durable_end_offset + WHERE scope_id=?1 AND segment_id=?2", + params![fixture.expected.scope_id.to_text(), publication.segment_id], + )?; + fixture.db.conn.execute( + "UPDATE changed_path_scopes SET provider_cursor=?1,durable_offset=?2,folded_offset=?2 + WHERE scope_id=?3", + params![ + publication.provider_cursor, + sql_u64(publication.durable_end_offset, "publication durable offset")?, + fixture.expected.scope_id.to_text(), + ], + )?; + fixture.insert_observer_event("advanced-prefix.rs", 2)?; + proof.publication_cursor = publication.provider_cursor; + proof.publication_segment_id = publication.segment_id; + proof.publication_cut = EvidenceCut { + source: EvidenceSource::Observer, + sequence: 2, + durable_offset: publication.durable_end_offset, + folded_offset: publication.durable_end_offset, + }; + mark_filesystem_applied(&ledger, &fixture.expected, &intent, &proof)?; + + writer.append(&[super::super::ObserverRecord { + sequence: 3, + source: EvidenceSource::Observer, + path: LedgerPath::parse("advanced-prefix.rs")?, + flags: EvidenceFlags::CONTENT, + provider_cursor: b"cursor-4".to_vec(), + }])?; + let advanced = writer.flush_durable()?; + writer.rotate()?; + drop(writer); + fixture.db.conn.execute( + "UPDATE changed_path_observer_segments SET folded_end_offset=durable_end_offset + WHERE scope_id=?1 AND segment_id=?2", + params![fixture.expected.scope_id.to_text(), advanced.segment_id], + )?; + fixture.db.conn.execute( + "UPDATE changed_path_scopes SET provider_cursor=?1,durable_offset=?2,folded_offset=?2 + WHERE scope_id=?3", + params![ + advanced.provider_cursor, + sql_u64(advanced.durable_end_offset, "advanced durable offset")?, + fixture.expected.scope_id.to_text(), + ], + )?; + fixture.insert_observer_event("advanced-prefix.rs", 3)?; + durable_intent_barrier(&fixture.db.conn)?; + fixture.advance_ref_to(&target)?; + publish_intent(&ledger, &fixture.expected, &intent)?; + + let decisions = recover_scope(&ledger, &fixture.expected)?; + if decisions != vec![(intent.clone(), RecoveryDecision::FinishPublication)] { + return Err(Error::Corrupt(format!( + "authenticated prefix was not accepted after later observer advance: {decisions:?}" + ))); + } + let later_sequence: i64 = fixture.db.conn.query_row( + "SELECT provider_sequence FROM changed_path_entries + WHERE scope_id=?1 AND normalized_path='advanced-prefix.rs'", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + )?; + if later_sequence != 3 { + return Err(Error::Corrupt( + "later observer evidence was cleared with the intent prefix".into(), + )); + } + Ok(()) + } + + fn aggregate_bridge_without_interval_evidence_is_rejected(prefix: bool) -> Result<()> { + let fixture = Fixture::new(if prefix { 0x83 } else { 0x82 })?; + let bridge_path = if prefix { + "bridged-prefix" + } else { + "bridged-path.rs" + }; + let target = fixture.target("aggregate-bridge", fixture.expected.baseline_root.clone()); + let owner_token = [fixture.expected.scope_id.0[0].wrapping_add(0x31); 32]; + let provider_id: String = fixture.db.conn.query_row( + "SELECT provider_id FROM changed_path_scopes WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + )?; + let segment_directory = fixture + .db + .db_dir + .join("observer-segments") + .join(fixture.expected.scope_id.to_text()); + let mut writer = super::super::SegmentWriter::acquire( + &fixture.db.db_dir.join(crate::db::DB_RELATIVE_PATH), + &segment_directory, + fixture.expected.scope_id, + fixture.expected.epoch, + owner_token, + &provider_id, + b"cursor-1".to_vec(), + Duration::from_secs(600), + )?; + writer.append(&[super::super::ObserverRecord { + sequence: 1, + source: EvidenceSource::Observer, + path: LedgerPath::parse(bridge_path)?, + flags: EvidenceFlags::CONTENT, + provider_cursor: b"cursor-2".to_vec(), + }])?; + let before = writer.flush_durable()?; + writer.rotate()?; + if prefix { + fixture.insert_observer_prefix(bridge_path, 1)?; + } else { + fixture.insert_observer_event(bridge_path, 1)?; + } + fixture.db.conn.execute( + "UPDATE changed_path_observer_segments SET folded_end_offset=durable_end_offset + WHERE scope_id=?1 AND segment_id=?2", + params![fixture.expected.scope_id.to_text(), before.segment_id], + )?; + fixture.db.conn.execute( + "UPDATE changed_path_scopes SET provider_cursor=?1,durable_offset=?2,folded_offset=?2 + WHERE scope_id=?3", + params![ + before.provider_cursor, + sql_u64(before.durable_end_offset, "pre-intent durable offset")?, + fixture.expected.scope_id.to_text() + ], + )?; + + let evidence = if prefix { + IntentEvidence { + exact_paths: Vec::new(), + complete_prefixes: vec![DirtyPrefix { + path: LedgerPath::parse(bridge_path)?, + complete: true, + reason: "provider_complete".into(), + first_sequence: 2, + last_sequence: 2, + }], + } + } else { + Fixture::evidence(bridge_path) + }; + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &target, + &evidence, + )?; + + writer.append(&[super::super::ObserverRecord { + sequence: 2, + source: EvidenceSource::Observer, + path: LedgerPath::parse("unrelated-during-intent.rs")?, + flags: EvidenceFlags::CONTENT, + provider_cursor: b"cursor-3".to_vec(), + }])?; + let proof_cut = writer.flush_durable()?; + writer.rotate()?; + writer.append(&[super::super::ObserverRecord { + sequence: 3, + source: EvidenceSource::Observer, + path: LedgerPath::parse(bridge_path)?, + flags: EvidenceFlags::CONTENT, + provider_cursor: b"cursor-4".to_vec(), + }])?; + let after = writer.flush_durable()?; + writer.rotate()?; + drop(writer); + fixture.db.conn.execute( + "UPDATE changed_path_observer_segments SET folded_end_offset=durable_end_offset + WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + )?; + if prefix { + fixture.insert_observer_prefix(bridge_path, 3)?; + } else { + fixture.insert_observer_event(bridge_path, 3)?; + } + fixture.db.conn.execute( + "UPDATE changed_path_observer_owners + SET provider_identity=?1,fence_nonce=?2 WHERE scope_id=?3", + params![ + hex::encode(&fixture.expected.provider_identity), + b"qualified-fence".as_slice(), + fixture.expected.scope_id.to_text(), + ], + )?; + fixture.db.conn.execute( + "UPDATE changed_path_scopes SET provider_cursor=?1,durable_offset=?2,folded_offset=?2 + WHERE scope_id=?3", + params![ + after.provider_cursor, + sql_u64( + after.durable_end_offset.max(proof_cut.durable_end_offset), + "post-intent durable offset" + )?, + fixture.expected.scope_id.to_text(), + ], + )?; + let proof = fixture.proof_for_durable_segment( + &proof_cut, + b"cursor-2".to_vec(), + 2, + u64::from(!prefix), + u64::from(prefix), + )?; + + if mark_filesystem_applied(&ledger, &fixture.expected, &intent, &proof).is_ok() { + return Err(Error::Corrupt(format!( + "{} SQL aggregate bridged an authenticated interval with no matching record", + if prefix { "prefix" } else { "exact-path" } + ))); + } + Ok(()) + } + + pub(super) fn exact_path_aggregate_bridge_is_rejected() -> Result<()> { + aggregate_bridge_without_interval_evidence_is_rejected(false) + } + + pub(super) fn prefix_aggregate_bridge_is_rejected() -> Result<()> { + aggregate_bridge_without_interval_evidence_is_rejected(true) + } + + pub(super) fn authenticated_prefix_interval_preserves_later_suffix() -> Result<()> { + let fixture = Fixture::new(0x84)?; + let prefix = "authenticated-prefix"; + let target = fixture.target("prefix-interval", fixture.expected.baseline_root.clone()); + let evidence = IntentEvidence { + exact_paths: Vec::new(), + complete_prefixes: vec![DirtyPrefix { + path: LedgerPath::parse(prefix)?, + complete: true, + reason: "provider_complete".into(), + first_sequence: 1, + last_sequence: 1, + }], + }; + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &target, + &evidence, + )?; + let owner_token = [fixture.expected.scope_id.0[0].wrapping_add(0x31); 32]; + let provider_id: String = fixture.db.conn.query_row( + "SELECT provider_id FROM changed_path_scopes WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + )?; + let mut writer = super::super::SegmentWriter::acquire( + &fixture.db.db_dir.join(crate::db::DB_RELATIVE_PATH), + &fixture + .db + .db_dir + .join("observer-segments") + .join(fixture.expected.scope_id.to_text()), + fixture.expected.scope_id, + fixture.expected.epoch, + owner_token, + &provider_id, + b"cursor-1".to_vec(), + Duration::from_secs(600), + )?; + let complete_flags = + EvidenceFlags(EvidenceFlags::CONTENT.0 | EvidenceFlags::PROVIDER_COMPLETE_PREFIX.0); + writer.append(&[super::super::ObserverRecord { + sequence: 1, + source: EvidenceSource::Observer, + path: LedgerPath::parse(prefix)?, + flags: complete_flags, + provider_cursor: b"cursor-2".to_vec(), + }])?; + let proof_cut = writer.flush_durable()?; + writer.rotate()?; + fixture.insert_observer_prefix(prefix, 1)?; + fixture.db.conn.execute( + "UPDATE changed_path_observer_segments SET folded_end_offset=durable_end_offset + WHERE scope_id=?1 AND segment_id=?2", + params![fixture.expected.scope_id.to_text(), proof_cut.segment_id], + )?; + fixture.db.conn.execute( + "UPDATE changed_path_observer_owners + SET provider_identity=?1,fence_nonce=?2 WHERE scope_id=?3", + params![ + hex::encode(&fixture.expected.provider_identity), + b"qualified-fence".as_slice(), + fixture.expected.scope_id.to_text(), + ], + )?; + fixture.db.conn.execute( + "UPDATE changed_path_scopes SET provider_cursor=?1,durable_offset=?2,folded_offset=?2, + observer_owner_token=?3 WHERE scope_id=?4", + params![ + proof_cut.provider_cursor, + sql_u64(proof_cut.durable_end_offset, "prefix proof durable offset")?, + hex::encode(owner_token), + fixture.expected.scope_id.to_text(), + ], + )?; + let proof = fixture.proof_for_durable_segment(&proof_cut, b"cursor-1".to_vec(), 1, 0, 1)?; + mark_filesystem_applied(&ledger, &fixture.expected, &intent, &proof)?; + + writer.append(&[super::super::ObserverRecord { + sequence: 2, + source: EvidenceSource::Observer, + path: LedgerPath::parse(prefix)?, + flags: complete_flags, + provider_cursor: b"cursor-3".to_vec(), + }])?; + let suffix = writer.flush_durable()?; + writer.rotate()?; + drop(writer); + fixture.insert_observer_prefix(prefix, 2)?; + fixture.db.conn.execute( + "UPDATE changed_path_observer_segments SET folded_end_offset=durable_end_offset + WHERE scope_id=?1 AND segment_id=?2", + params![fixture.expected.scope_id.to_text(), suffix.segment_id], + )?; + fixture.db.conn.execute( + "UPDATE changed_path_scopes SET provider_cursor=?1,durable_offset=?2,folded_offset=?2 + WHERE scope_id=?3", + params![ + suffix.provider_cursor, + sql_u64( + suffix.durable_end_offset.max(proof_cut.durable_end_offset), + "prefix suffix durable offset" + )?, + fixture.expected.scope_id.to_text(), + ], + )?; + fixture.advance_ref_to(&target)?; + publish_intent(&ledger, &fixture.expected, &intent)?; + let decisions = recover_scope(&ledger, &fixture.expected)?; + if decisions != vec![(intent, RecoveryDecision::FinishPublication)] { + return Err(Error::Corrupt(format!( + "valid authenticated prefix interval did not publish: {decisions:?}" + ))); + } + let retained: (i64, i64, Option) = fixture.db.conn.query_row( + "SELECT source_mask,provider_sequence,intent_id FROM changed_path_prefixes + WHERE scope_id=?1 AND normalized_prefix=?2", + params![fixture.expected.scope_id.to_text(), prefix], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + if retained != (EvidenceSource::Observer.mask(), 2, None) { + return Err(Error::Corrupt(format!( + "later authenticated prefix suffix was not retained: {retained:?}" + ))); + } + Ok(()) + } + + #[cfg(unix)] + fn install_scope_directory_symlink_substitution(fixture: &Fixture) { + use std::os::unix::fs::symlink; + + let scope_directory = fixture + .db + .db_dir + .join("observer-segments") + .join(fixture.expected.scope_id.to_text()); + let retained_directory = fixture + .db + .db_dir + .join("observer-segments") + .join(format!("{}.retained", fixture.expected.scope_id.to_text())); + install_sidecar_ancestor_substitution_hook(move || { + std::fs::rename(&scope_directory, &retained_directory).unwrap(); + symlink(&retained_directory, &scope_directory).unwrap(); + }); + } + + #[cfg(unix)] + pub(super) fn ancestor_substitution_at_mark_is_rejected() -> Result<()> { + let fixture = Fixture::new(0x85)?; + let target = fixture.target( + "mark-directory-swap", + fixture.expected.baseline_root.clone(), + ); + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &target, + &Fixture::evidence("mark-directory-swap.rs"), + )?; + let proof = fixture.qualified_proof(1, "mark-directory-swap.rs")?; + install_scope_directory_symlink_substitution(&fixture); + + if mark_filesystem_applied(&ledger, &fixture.expected, &intent, &proof).is_ok() { + return Err(Error::Corrupt( + "ancestor directory substitution authorized filesystem-applied marking".into(), + )); + } + Ok(()) + } + + #[cfg(unix)] + pub(super) fn ancestor_substitution_at_recovery_is_rejected() -> Result<()> { + let fixture = Fixture::new(0x86)?; + let target = fixture.target( + "recovery-directory-swap", + fixture.expected.baseline_root.clone(), + ); + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &target, + &Fixture::evidence("recovery-directory-swap.rs"), + )?; + let proof = fixture.qualified_proof(1, "recovery-directory-swap.rs")?; + mark_filesystem_applied(&ledger, &fixture.expected, &intent, &proof)?; + fixture.advance_ref_to(&target)?; + install_scope_directory_symlink_substitution(&fixture); + + let decisions = recover_scope(&ledger, &fixture.expected)?; + if decisions == vec![(intent, RecoveryDecision::FinishPublication)] { + return Err(Error::Corrupt( + "ancestor directory substitution authorized publication recovery".into(), + )); + } + let trust: String = fixture.db.conn.query_row( + "SELECT trust_state FROM changed_path_scopes WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + )?; + if trust != "untrusted_gap" { + return Err(Error::Corrupt(format!( + "ancestor substitution did not fail closed: {trust}" + ))); + } + Ok(()) + } + + pub(super) fn gc_root_lifecycle() -> Result<()> { + let mut fixture = Fixture::new(0x62)?; + let baseline_ref = fixture.db.get_ref(&fixture.expected.ref_name)?; + std::fs::write( + fixture.db.workspace_root.join("intent-only.txt"), + b"reachable only through the intent graph\n", + )?; + fixture.db.record( + None, + Some("build intent-only graph".into()), + Actor::system(), + false, + )?; + let target_ref = fixture.db.get_ref(&fixture.expected.ref_name)?; + let files = fixture.db.load_root_files(&target_ref.root_id)?; + let mut dependent_objects = vec![ + target_ref.root_id.0.clone(), + target_ref.operation_id.0.clone(), + ]; + for entry in files.values() { + dependent_objects.push(match &entry.content { + FileContentRef::Text(id) + | FileContentRef::Opaque(id) + | FileContentRef::Binary(id) => id.0.clone(), + }); + } + fixture.db.conn.execute( + "UPDATE refs SET change_id=?1,root_id=?2,operation_id=?3,generation=?4,updated_at=?5 + WHERE name=?6", + params![ + baseline_ref.change_id.0, + baseline_ref.root_id.0, + baseline_ref.operation_id.0, + baseline_ref.generation, + now_ts(), + baseline_ref.name, + ], + )?; + let target = IntentTarget { + change_id: target_ref.change_id, + root_id: target_ref.root_id, + operation_id: Some(target_ref.operation_id), + }; + prepare_intent( + &fixture.db.changed_path_ledger(), + &fixture.expected, + IntentProducer::Checkout, + &target, + &Fixture::evidence("new.rs"), + )?; + fixture.db.gc(true)?; + fixture.db.gc(false)?; + for object_id in &dependent_objects { + let retained: bool = fixture.db.conn.query_row( + "SELECT EXISTS(SELECT 1 FROM objects WHERE object_id=?1)", + [object_id], + |row| row.get(0), + )?; + if !retained { + return Err(Error::Corrupt(format!( + "GC collected transitive intent dependency `{object_id}`" + ))); + } + } + fixture.db.gc(false)?; + let after_second: bool = fixture.db.conn.query_row( + "SELECT EXISTS(SELECT 1 FROM objects WHERE object_id=?1)", + [&target.root_id.0], + |row| row.get(0), + )?; + if after_second { + return Err(Error::Corrupt( + "terminal intent root remained pinned".into(), + )); + } + Ok(()) + } + + pub(super) fn crash_matrix() -> Result<()> { + let prepared = Fixture::new(0x63)?; + let target = prepared.target("prepared", prepared.expected.baseline_root.clone()); + let intent = prepare_intent( + &prepared.db.changed_path_ledger(), + &prepared.expected, + IntentProducer::LaneSync, + &target, + &Fixture::evidence("prepared.bin"), + )?; + prepared.advance_ref_to(&target)?; + let before: i64 = prepared.db.conn.query_row( + "SELECT continuity_generation FROM changed_path_scopes WHERE scope_id=?1", + [prepared.expected.scope_id.to_text()], + |row| row.get(0), + )?; + recover_scope(&prepared.db.changed_path_ledger(), &prepared.expected)?; + recover_scope(&prepared.db.changed_path_ledger(), &prepared.expected)?; + let state = prepared.db.conn.query_row( + "SELECT lifecycle_state,trust_state,continuity_generation,s.ref_generation, + s.change_id,i.expected_change_id FROM changed_path_intents i + JOIN changed_path_scopes s ON s.scope_id=i.scope_id WHERE i.intent_id=?1", + [&intent.0], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + )) + }, + )?; + if state.0 != "aborted" + || state.1 != "untrusted_gap" + || state.2 != before + 1 + || state.3 != sql_u64(prepared.expected.ref_generation, "fixture generation")? + || state.4 != state.5 + { + return Err(Error::Corrupt(format!( + "prepared crash recovery was not once-only: {state:?}" + ))); + } + + let retired = Fixture::new(0x67)?; + let retirement_before: i64 = retired.db.conn.query_row( + "SELECT continuity_generation FROM changed_path_scopes WHERE scope_id=?1", + [retired.expected.scope_id.to_text()], + |row| row.get(0), + )?; + let retired_leaf = create_real_retirement_segment(&retired, 0x67)?; + let retired_paths = + retire_scope(&retired.db.conn, &retired.db.sqlite_path, &retired.expected)?; + let retirement_state = retired.db.conn.query_row( + "SELECT s.continuity_generation,s.retired_at,o.lease_state,g.state + FROM changed_path_scopes s + JOIN changed_path_observer_owners o ON o.scope_id=s.scope_id + JOIN changed_path_observer_segments g ON g.scope_id=s.scope_id + WHERE s.scope_id=?1", + [retired.expected.scope_id.to_text()], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + }, + )?; + if retired_paths + .iter() + .map(|token| token.identity.original_leaf.as_str()) + .collect::>() + != vec![retired_leaf.as_str()] + || retirement_state.0 != retirement_before + 1 + || retirement_state.1.is_none() + || retirement_state.2 != "revoked" + || retirement_state.3 != "retired" + { + return Err(Error::Corrupt(format!( + "scope retirement was not ordered and once-only: {retirement_state:?} {retired_paths:?}" + ))); + } + + for (tag, publish_before_recovery) in [(0x64, false), (0x65, true)] { + let fixture = Fixture::new(tag)?; + let target = fixture.target("published", fixture.expected.baseline_root.clone()); + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Materialize, + &target, + &Fixture::evidence("target.dat"), + )?; + mark_filesystem_applied( + &ledger, + &fixture.expected, + &intent, + &fixture.qualified_proof(9, "target.dat")?, + )?; + fixture.advance_ref_to(&target)?; + if publish_before_recovery { + publish_intent(&ledger, &fixture.expected, &intent)?; + } + recover_scope(&ledger, &fixture.expected)?; + let state: String = fixture.db.conn.query_row( + "SELECT lifecycle_state FROM changed_path_intents WHERE intent_id=?1", + [&intent.0], + |row| row.get(0), + )?; + if state != "acknowledged" { + return Err(Error::Corrupt(format!( + "recoverable publication stayed {state}" + ))); + } + } + Ok(()) + } + + pub(super) fn retirement_path_and_reader_barrier() -> Result<()> { + let malicious = Fixture::new(0x6b)?; + malicious.db.conn.execute( + "INSERT INTO changed_path_observer_segments( + scope_id,epoch,segment_id,log_format_version,owner_token,provider_id, + first_sequence,last_sequence,durable_end_offset,folded_end_offset, + previous_segment_id,previous_segment_hash,segment_hash,segment_path,state, + created_at,sealed_at,updated_at) + VALUES(?1,1,'malicious',1,'owner','provider',1,1,1,1,NULL,NULL,?2, + '../outside.cpl','sealed',?3,?3,?3)", + params![ + malicious.expected.scope_id.to_text(), + hex::encode([1_u8; 32]), + now_ts() + ], + )?; + if retire_scope( + &malicious.db.conn, + &malicious.db.sqlite_path, + &malicious.expected, + ) + .is_ok() + { + return Err(Error::Corrupt( + "scope retirement returned an escaping observer segment path".into(), + )); + } + let malicious_retired: bool = malicious.db.conn.query_row( + "SELECT retired_at IS NOT NULL FROM changed_path_scopes WHERE scope_id=?1", + [malicious.expected.scope_id.to_text()], + |row| row.get(0), + )?; + if malicious_retired { + return Err(Error::Corrupt( + "invalid retirement path committed partial retirement".into(), + )); + } + + let reader_fixture = Fixture::new(0x6c)?; + let reader_leaf = create_real_retirement_segment(&reader_fixture, 0x6c)?; + let reader = Connection::open(reader_fixture.db.db_dir.join(crate::db::DB_RELATIVE_PATH))?; + reader.execute_batch("BEGIN DEFERRED")?; + let _: i64 = reader.query_row("SELECT COUNT(*) FROM changed_path_scopes", [], |row| { + row.get(0) + })?; + if retire_scope( + &reader_fixture.db.conn, + &reader_fixture.db.sqlite_path, + &reader_fixture.expected, + ) + .is_ok() + { + return Err(Error::Corrupt( + "retirement returned deletion authority while an older reader was active".into(), + )); + } + reader.execute_batch("COMMIT")?; + let paths = retire_scope( + &reader_fixture.db.conn, + &reader_fixture.db.sqlite_path, + &reader_fixture.expected, + )?; + if paths + .iter() + .map(|token| token.identity.original_leaf.as_str()) + .collect::>() + != vec![reader_leaf.as_str()] + { + return Err(Error::Corrupt(format!( + "retirement retry did not return confined paths: {paths:?}" + ))); + } + Ok(()) + } + + fn retired_segment_fixture( + tag: u8, + ) -> Result<( + Fixture, + Vec, + std::path::PathBuf, + String, + )> { + let fixture = Fixture::new(tag)?; + let leaf = create_real_retirement_segment(&fixture, tag)?; + let directory = fixture + .db + .db_dir + .join("observer-segments") + .join(fixture.expected.scope_id.to_text()); + let tokens = retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &fixture.expected)?; + Ok((fixture, tokens, directory, leaf)) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + fn direct_target_collision_is_retained(tag: u8) -> Result<()> { + let fixture = Fixture::new(tag)?; + create_real_retirement_segment(&fixture, tag)?; + let retirement = + load_retirement_identity(&fixture.db.conn, &fixture.expected.scope_id.to_text())? + .ok_or_else(|| Error::Corrupt("missing collision retirement identity".into()))?; + let retirement = begin_scope_retirement(&fixture.db.conn, &retirement)?; + let (scope_identity, rows) = inspect_segments_before_allocation( + &fixture.db.conn, + &fixture.db.sqlite_path, + fixture.expected.scope_id, + fixture.expected.epoch, + )?; + journal_missing_quarantine_allocations( + &fixture.db.conn, + &retirement, + scope_identity, + &rows, + )?; + drop(rows); + let quarantine_leaf: String = fixture.db.conn.query_row( + "SELECT quarantine_leaf FROM changed_path_segment_quarantine_allocations + WHERE scope_id=?1 AND state='allocating'", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + )?; + let directory = fixture + .db + .db_dir + .join("observer-segments") + .join(fixture.expected.scope_id.to_text()); + let quarantine = directory.join(&quarantine_leaf); + std::fs::write(&quarantine, b"foreign direct quarantine\n")?; + let tokens = retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &fixture.expected)?; + let (deletion_rows, audited): (i64, i64) = fixture.db.conn.query_row( + "SELECT + (SELECT COUNT(*) FROM changed_path_segment_deletions WHERE scope_id=?1), + (SELECT COUNT(*) FROM changed_path_segment_quarantine_allocations + WHERE scope_id=?1 AND quarantine_leaf=?2 + AND state='abandoned' + AND retained_reason='direct_quarantine_target_identity_mismatch')", + params![fixture.expected.scope_id.to_text(), quarantine_leaf], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + if std::fs::read(&quarantine)? != b"foreign direct quarantine\n" + || tokens.len() != 1 + || tokens[0].identity.quarantine_leaf == quarantine_leaf + || deletion_rows != 1 + || audited != 1 + { + return Err(Error::Corrupt( + "direct target collision was removed or adopted".into(), + )); + } + remove_retired_segments(&fixture.db.conn, &tokens)?; + Ok(()) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(super) fn orphan_quarantine_substitution_fails_closed() -> Result<()> { + direct_target_collision_is_retained(0x95) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(super) fn empty_orphan_quarantine_is_retained_and_rejected() -> Result<()> { + direct_target_collision_is_retained(0x96) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(super) fn no_orphan_allocates_fresh_quarantine_authority() -> Result<()> { + let fixture = Fixture::new(0x97)?; + create_real_retirement_segment(&fixture, 0x97)?; + let tokens = retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &fixture.expected)?; + if tokens.len() != 1 || tokens[0].identity.state != "quiesced" { + return Err(Error::Corrupt( + "normal retirement did not mint exactly one fresh prepared authority".into(), + )); + } + let quarantined = tokens[0] + .directory + .open_regular(&tokens[0].identity.quarantine_leaf)?; + if super::super::secure_fs::file_identity(&quarantined)? + != tokens[0].identity.quarantine_identity + { + return Err(Error::Corrupt("direct quarantine identity changed".into())); + } + remove_retired_segments(&fixture.db.conn, &tokens)?; + let retry = retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &fixture.expected)?; + remove_retired_segments(&fixture.db.conn, &retry)?; + Ok(()) + } + + fn create_real_retirement_segment(fixture: &Fixture, tag: u8) -> Result { + create_real_segment( + &fixture.db, + fixture.expected.scope_id, + fixture.expected.epoch, + tag, + ) + } + + fn create_real_segment(db: &Trail, scope_id: ScopeId, epoch: u64, tag: u8) -> Result { + let (leaf, writer) = create_retained_real_segment(db, scope_id, epoch, tag)?; + drop(writer); + Ok(leaf) + } + + fn create_retained_real_segment( + db: &Trail, + scope_id: ScopeId, + epoch: u64, + tag: u8, + ) -> Result<(String, super::super::SegmentWriter)> { + let directory = db.db_dir.join("observer-segments").join(scope_id.to_text()); + let provider_id: String = db.conn.query_row( + "SELECT provider_id FROM changed_path_scopes WHERE scope_id=?1", + [scope_id.to_text()], + |row| row.get(0), + )?; + let writer = super::super::SegmentWriter::acquire( + &db.sqlite_path, + &directory, + scope_id, + epoch, + [tag.wrapping_add(0x41); 32], + &provider_id, + b"retirement-test-cursor".to_vec(), + Duration::from_secs(600), + )?; + let leaf: String = db.conn.query_row( + "SELECT segment_path FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=?2", + params![scope_id.to_text(), sql_u64(epoch, "scope epoch")?], + |row| row.get(0), + )?; + Ok((leaf, writer)) + } + + pub(super) fn retirement_requires_retained_writer_quiescence() -> Result<()> { + let fixture = Fixture::new(0xa1)?; + let (leaf, mut writer) = create_retained_real_segment( + &fixture.db, + fixture.expected.scope_id, + fixture.expected.epoch, + 0xa1, + )?; + let original = fixture + .db + .db_dir + .join("observer-segments") + .join(fixture.expected.scope_id.to_text()) + .join(&leaf); + if retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &fixture.expected).is_ok() { + return Err(Error::Corrupt( + "retirement moved a segment while its writer FD remained retained".into(), + )); + } + if !original.exists() { + return Err(Error::Corrupt( + "retirement moved the segment before writer close acknowledgement".into(), + )); + } + let quiesced: i64 = fixture.db.conn.query_row( + "SELECT COUNT(*) FROM changed_path_segment_deletions WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + )?; + if quiesced != 0 { + return Err(Error::Corrupt( + "retirement published quiesced deletion before writer close acknowledgement".into(), + )); + } + let before_append = std::fs::metadata(&original)?.len(); + let late_record = super::super::ObserverRecord { + sequence: 1, + source: EvidenceSource::Observer, + path: LedgerPath::parse("late-after-retirement-fence.txt")?, + flags: EvidenceFlags::CONTENT, + provider_cursor: b"late-after-retirement-fence".to_vec(), + }; + if writer.append(&[late_record]).is_ok() + || std::fs::metadata(&original)?.len() != before_append + { + return Err(Error::Corrupt( + "revoked retained writer appended after the retirement fence".into(), + )); + } + drop(writer); + let retry = retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &fixture.expected)?; + if retry.len() != 1 || original.exists() { + return Err(Error::Corrupt( + "retirement did not converge after the retained writer closed".into(), + )); + } + Ok(()) + } + + pub(super) fn deletion_parent_substitution_uses_retained_authority() -> Result<()> { + let (fixture, tokens, directory, leaf) = retired_segment_fixture(0x87)?; + let retained = directory.with_extension("retained-directory"); + let replacement_file = directory.join(&leaf); + let original_file = retained.join(&leaf); + let hook_directory = directory.clone(); + let hook_retained = retained.clone(); + let hook_leaf = leaf.clone(); + install_deletion_substitution_hook(DeletionSubstitutionPoint::Parent, move || { + std::fs::rename(&hook_directory, &hook_retained).unwrap(); + std::fs::create_dir(&hook_directory).unwrap(); + std::fs::write( + hook_directory.join(&hook_leaf), + b"replacement must survive\n", + ) + .unwrap(); + }); + + remove_retired_segments(&fixture.db.conn, &tokens)?; + remove_retired_segments(&fixture.db.conn, &tokens)?; + if original_file.exists() || !replacement_file.exists() { + return Err(Error::Corrupt( + "retired deletion followed a substituted parent directory".into(), + )); + } + Ok(()) + } + + #[cfg(unix)] + pub(super) fn deletion_leaf_substitution_fails_closed() -> Result<()> { + use std::os::unix::fs::symlink; + + let (fixture, tokens, directory, segment_leaf) = retired_segment_fixture(0x88)?; + let leaf = directory.join(segment_leaf); + let outside = fixture._root.path().join("outside-segment.cpl"); + std::fs::write(&outside, b"outside must survive\n")?; + symlink(&outside, &leaf)?; + + if remove_retired_segments(&fixture.db.conn, &tokens).is_ok() { + return Err(Error::Corrupt( + "retired deletion accepted a substituted leaf".into(), + )); + } + if !outside.exists() || std::fs::symlink_metadata(&leaf).is_err() { + return Err(Error::Corrupt( + "retired deletion escaped its descriptor-relative leaf".into(), + )); + } + Ok(()) + } + + pub(super) fn deletion_name_substitution_after_verification_fails_closed() -> Result<()> { + let (fixture, tokens, directory, segment_leaf) = retired_segment_fixture(0x89)?; + let leaf = directory.join(segment_leaf); + let authorized = directory.join("post-verification-authorized.cpl"); + let quarantine = directory.join(&tokens[0].identity.quarantine_leaf); + std::fs::write(&leaf, b"replacement must survive\n")?; + + if remove_retired_segments(&fixture.db.conn, &tokens).is_ok() { + return Err(Error::Corrupt( + "retired deletion accepted a name substituted after verification".into(), + )); + } + if !leaf.exists() || authorized.exists() || !quarantine.exists() { + return Err(Error::Corrupt( + "retired deletion did not preserve the authorized inode and quarantine the substitution" + .into(), + )); + } + Ok(()) + } + + pub(super) fn deletion_substitution_after_quarantine_verification_fails_closed() -> Result<()> { + let (fixture, tokens, directory, _segment_leaf) = retired_segment_fixture(0x8c)?; + let quarantine = directory.join(&tokens[0].identity.quarantine_leaf); + let authorized = directory.join("post-quarantine-authorized.cpl"); + std::fs::rename(&quarantine, &authorized)?; + std::fs::write(&quarantine, b"replacement must survive\n")?; + + let rejection = remove_retired_segments(&fixture.db.conn, &tokens) + .err() + .ok_or_else(|| { + Error::Corrupt( + "retired deletion accepted substitution after quarantine verification".into(), + ) + })?; + if !quarantine.exists() || !authorized.exists() { + return Err(Error::Corrupt(format!( + "retired deletion did not preserve post-verification files: quarantine={} authorized={} directory={} rejection={rejection}", + quarantine.exists(), + authorized.exists(), + directory.display() + ))); + } + Ok(()) + } + + pub(super) fn deletion_retry_rejects_hostile_quarantine_replacement() -> Result<()> { + let (fixture, tokens, directory, _segment_leaf) = retired_segment_fixture(0x8d)?; + let quarantine = directory.join(&tokens[0].identity.quarantine_leaf); + let retained = directory.join("hostile-quarantine-retained.cpl"); + std::fs::rename(&quarantine, &retained)?; + std::fs::write(&quarantine, b"hostile replacement must survive\n")?; + + if retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &fixture.expected).is_ok() { + return Err(Error::Corrupt( + "retirement retry adopted a hostile quarantine replacement".into(), + )); + } + if !quarantine.exists() || !retained.exists() { + return Err(Error::Corrupt( + "retirement retry deleted a hostile or retained quarantine file".into(), + )); + } + Ok(()) + } + + pub(super) fn deletion_normal_retry_is_durably_idempotent() -> Result<()> { + let (fixture, tokens, directory, segment_leaf) = retired_segment_fixture(0x8e)?; + let original = directory.join(&segment_leaf); + let quarantine = directory.join(&tokens[0].identity.quarantine_leaf); + let expected_bytes = std::fs::read(&quarantine)?; + let expected_identity = tokens[0].identity.segment_identity; + remove_retired_segments(&fixture.db.conn, &tokens)?; + if original.exists() || !quarantine.exists() { + return Err(Error::Corrupt( + "normal retirement did not retain exactly one quarantined segment".into(), + )); + } + let retained = std::fs::File::open(&quarantine)?; + if super::super::secure_fs::file_identity(&retained)? != expected_identity + || std::fs::read(&quarantine)? != expected_bytes + { + return Err(Error::Corrupt( + "quiesced retirement did not retain the authenticated segment".into(), + )); + } + let retry = retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &fixture.expected)?; + remove_retired_segments(&fixture.db.conn, &retry)?; + remove_retired_segments(&fixture.db.conn, &retry)?; + let reopened = Connection::open(&fixture.db.sqlite_path)?; + let reopened_retry = retire_scope(&reopened, &fixture.db.sqlite_path, &fixture.expected)?; + remove_retired_segments(&reopened, &reopened_retry)?; + let quiesced: i64 = reopened.query_row( + "SELECT COUNT(*) FROM changed_path_segment_deletions + WHERE scope_id=?1 AND epoch=?2 AND state='quiesced' AND completed_at IS NOT NULL", + params![ + fixture.expected.scope_id.to_text(), + sql_u64(fixture.expected.epoch, "scope epoch")? + ], + |row| row.get(0), + )?; + if quiesced != 1 || !quarantine.exists() || original.exists() { + return Err(Error::Corrupt(format!( + "normal deletion retry did not retain one durable quiesced row and segment: rows={quiesced} quarantine={} original={}", + quarantine.exists(), + original.exists() + ))); + } + Ok(()) + } + + pub(super) fn deletion_quiesced_retry_rejects_missing_quarantine() -> Result<()> { + let (fixture, tokens, directory, segment_leaf) = retired_segment_fixture(0x8f)?; + let original = directory.join(segment_leaf); + let quarantine = directory.join(&tokens[0].identity.quarantine_leaf); + let retained = directory.join("missing-quarantine-retained.cpl"); + remove_retired_segments(&fixture.db.conn, &tokens)?; + std::fs::rename(&quarantine, &retained)?; + + if retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &fixture.expected).is_ok() { + return Err(Error::Corrupt( + "quiesced retirement retry accepted a missing quarantine entry".into(), + )); + } + if !retained.exists() || quarantine.exists() || original.exists() { + return Err(Error::Corrupt( + "missing-quarantine retry mutated retained filesystem evidence".into(), + )); + } + Ok(()) + } + + pub(super) fn deletion_quiesced_retry_rejects_reappeared_original() -> Result<()> { + let (fixture, tokens, directory, segment_leaf) = retired_segment_fixture(0x94)?; + let original = directory.join(segment_leaf); + let quarantine = directory.join(&tokens[0].identity.quarantine_leaf); + remove_retired_segments(&fixture.db.conn, &tokens)?; + std::fs::write(&original, b"reappeared original-name replacement\n")?; + + let retry = retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &fixture.expected)?; + if remove_retired_segments(&fixture.db.conn, &retry).is_ok() { + return Err(Error::Corrupt( + "quiesced retirement retry accepted a reappeared original name".into(), + )); + } + if !original.exists() || !quarantine.exists() { + return Err(Error::Corrupt( + "reappeared-original retry removed filesystem evidence".into(), + )); + } + Ok(()) + } + + pub(super) fn restored_nullable_provider_lane_deletion() -> Result<()> { + let root = tempfile::tempdir()?; + std::fs::write(root.path().join("lane.txt"), b"lane\n")?; + Trail::init(root.path(), "main", InitImportMode::WorkingTree, false)?; + let mut db = Trail::open(root.path())?; + let spawned = db.spawn_lane("restored-retire", Some("main"), true, None, None)?; + let branch = db.lane_branch("restored-retire")?; + let head = db.get_ref(&branch.ref_name)?; + let view = db.create_workspace_view( + &branch.lane_id, + &head.change_id, + &head.root_id, + "fuse-cow", + &root.path().join("restored-retire-view"), + )?; + let baseline = BaselineIdentity { + ref_name: head.name.clone(), + ref_generation: u64::try_from(head.generation) + .map_err(|_| Error::Corrupt("negative lane ref generation".into()))?, + change_id: head.change_id.clone(), + root_id: head.root_id.clone(), + }; + let policy = PolicyIdentity { + fingerprint: [0x8a; 32], + generation: 1, + }; + let filesystem = FilesystemIdentity(vec![0x8a]); + let provider = ProviderIdentity { + identity: vec![0x8b], + capabilities: ProviderCapabilities { + durable_cursor: true, + linearizable_fence: true, + rename_pairing: true, + overflow_scope: true, + filesystem_supported: true, + clean_proof_allowed: true, + power_loss_durability: true, + }, + }; + for identity in [ + ScopeIdentity { + scope_id: ScopeId([0x8a; 32]), + kind: ScopeKind::MaterializedLane, + owner_id: branch.lane_id.clone(), + }, + ScopeIdentity { + scope_id: ScopeId([0x8b; 32]), + kind: ScopeKind::WorkspaceView, + owner_id: view.view_id.clone(), + }, + ] { + db.changed_path_ledger().begin_scope( + &identity, + &baseline, + &policy, + &filesystem, + &provider, + )?; + } + db.conn.execute( + "UPDATE changed_path_scopes SET scope_root=?1 WHERE scope_id=?2", + params![spawned.workdir, ScopeId([0x8a; 32]).to_text()], + )?; + let backup = root.path().join("restored-retire-backup"); + db.create_backup(&backup, false)?; + let destination = root.path().join("restored-retire-destination"); + Trail::restore_backup(&destination, &backup, false)?; + let mut restored = Trail::open(&destination)?; + let nullable: i64 = restored.conn.query_row( + "SELECT COUNT(*) FROM changed_path_scopes + WHERE scope_kind IN ('materialized_lane','workspace_view') + AND provider_identity IS NULL AND retired_at IS NULL", + [], + |row| row.get(0), + )?; + if nullable != 2 { + return Err(Error::Corrupt(format!( + "restore did not produce two nullable-provider deletion scopes: {nullable}" + ))); + } + restored.remove_lane("restored-retire", true)?; + let retired: i64 = restored.conn.query_row( + "SELECT COUNT(*) FROM changed_path_scopes + WHERE scope_kind IN ('materialized_lane','workspace_view') AND retired_at IS NOT NULL", + [], + |row| row.get(0), + )?; + if retired != 2 { + return Err(Error::Corrupt(format!( + "restored nullable-provider lane/view scopes were not retired: {retired}" + ))); + } + Ok(()) + } + + #[cfg(unix)] + pub(super) fn non_utf_database_path_mark_recover_and_retire() -> Result<()> { + use std::os::unix::ffi::{OsStrExt, OsStringExt}; + + let fixture = Fixture::new(0x8c)?; + let Fixture { + _root, + db, + expected, + backup, + restore, + } = fixture; + let old_workspace = db.workspace_root.clone(); + drop(db); + let new_workspace = _root + .path() + .join(std::ffi::OsString::from_vec(b"workspace-\xff".to_vec())); + if let Err(error) = std::fs::rename(&old_workspace, &new_workspace) { + if cfg!(target_os = "macos") && error.raw_os_error() == Some(libc::EILSEQ) { + let conn = Connection::open_in_memory()?; + let database_path = new_workspace.join(".trail/index/trail.sqlite"); + let ledger = ChangedPathLedger::new_at(&conn, &database_path); + if ledger.database_path()?.as_os_str().as_bytes() + != database_path.as_os_str().as_bytes() + { + return Err(Error::Corrupt( + "macOS lossless database path plumbing changed raw bytes".into(), + )); + } + return Ok(()); + } + return Err(error.into()); + } + let db = Trail::open(&new_workspace)?; + let fixture = Fixture { + _root, + db, + expected, + backup, + restore, + }; + let target = fixture.target( + "non-utf-database-path", + fixture.expected.baseline_root.clone(), + ); + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Materialize, + &target, + &Fixture::evidence("non-utf.txt"), + )?; + let proof = fixture.qualified_proof(1, "non-utf.txt")?; + mark_filesystem_applied(&ledger, &fixture.expected, &intent, &proof)?; + fixture.advance_ref_to(&target)?; + recover_scope(&ledger, &fixture.expected)?; + + let mut retired_expected = fixture.expected.clone(); + retired_expected.ref_generation = retired_expected.ref_generation.saturating_add(1); + let tokens = retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &retired_expected)?; + remove_retired_segments(&fixture.db.conn, &tokens)?; + if tokens.is_empty() + || tokens.iter().any(|token| { + token + .directory + .open_regular(&token.identity.original_leaf) + .is_ok() + }) + { + return Err(Error::Corrupt( + "non-UTF database path did not retain proof/recovery/retirement authority".into(), + )); + } + Ok(()) + } + + pub(super) fn lane_deletion_retires_scope_first() -> Result<()> { + let root = tempfile::tempdir()?; + std::fs::write(root.path().join("lane.txt"), b"lane\n")?; + Trail::init(root.path(), "main", InitImportMode::WorkingTree, false)?; + let mut db = Trail::open(root.path())?; + let spawned = db.spawn_lane("retire-first", Some("main"), true, None, None)?; + let branch = db.lane_branch("retire-first")?; + let head = db.get_ref(&branch.ref_name)?; + let view = db.create_workspace_view( + &branch.lane_id, + &head.change_id, + &head.root_id, + "fuse-cow", + &root.path().join("retire-first-view"), + )?; + let scope = ScopeIdentity { + scope_id: ScopeId([0x6d; 32]), + kind: ScopeKind::MaterializedLane, + owner_id: branch.lane_id.clone(), + }; + let baseline = BaselineIdentity { + ref_name: head.name.clone(), + ref_generation: u64::try_from(head.generation) + .map_err(|_| Error::Corrupt("negative lane ref generation".into()))?, + change_id: head.change_id.clone(), + root_id: head.root_id.clone(), + }; + let policy = PolicyIdentity { + fingerprint: [0x6d; 32], + generation: 1, + }; + let filesystem = FilesystemIdentity(vec![0x6d]); + let provider = ProviderIdentity { + identity: vec![0x6e], + capabilities: ProviderCapabilities { + durable_cursor: true, + linearizable_fence: true, + rename_pairing: true, + overflow_scope: true, + filesystem_supported: true, + clean_proof_allowed: true, + power_loss_durability: true, + }, + }; + db.changed_path_ledger() + .begin_scope(&scope, &baseline, &policy, &filesystem, &provider)?; + let view_scope = ScopeIdentity { + scope_id: ScopeId([0x6e; 32]), + kind: ScopeKind::WorkspaceView, + owner_id: view.view_id.clone(), + }; + db.changed_path_ledger().begin_scope( + &view_scope, + &baseline, + &policy, + &filesystem, + &provider, + )?; + db.conn.execute( + "UPDATE changed_path_scopes SET scope_root=?1 WHERE scope_id=?2", + params![spawned.workdir, scope.scope_id.to_text()], + )?; + db.conn.execute( + "INSERT INTO changed_path_observer_segments( + scope_id,epoch,segment_id,log_format_version,owner_token,provider_id, + first_sequence,last_sequence,durable_end_offset,folded_end_offset, + previous_segment_id,previous_segment_hash,segment_hash,segment_path,state, + created_at,sealed_at,updated_at) + VALUES(?1,1,'lane-segment',1,'owner','provider',1,1,1,1,NULL,NULL,?2, + '../escape.cpl','sealed',?3,?3,?3)", + params![scope.scope_id.to_text(), hex::encode([3_u8; 32]), now_ts()], + )?; + db.conn.execute( + "INSERT INTO changed_path_observer_segments( + scope_id,epoch,segment_id,log_format_version,owner_token,provider_id, + first_sequence,last_sequence,durable_end_offset,folded_end_offset, + previous_segment_id,previous_segment_hash,segment_hash,segment_path,state, + created_at,sealed_at,updated_at) + VALUES(?1,1,'view-segment',1,'owner','provider',1,1,1,1,NULL,NULL,?2, + 'view-segment.cpl','sealed',?3,?3,?3)", + params![ + view_scope.scope_id.to_text(), + hex::encode([4_u8; 32]), + now_ts() + ], + )?; + if db.remove_lane("retire-first", true).is_ok() { + return Err(Error::Corrupt( + "lane deletion bypassed changed-path scope retirement".into(), + )); + } + let workdir = std::path::PathBuf::from( + spawned + .workdir + .ok_or_else(|| Error::Corrupt("materialized lane has no workdir".into()))?, + ); + if !workdir.exists() || db.try_get_ref(&branch.ref_name)?.is_none() { + return Err(Error::Corrupt( + "lane deletion mutated filesystem/ref before retirement completed".into(), + )); + } + db.conn.execute( + "DELETE FROM changed_path_observer_segments WHERE scope_id=?1", + [scope.scope_id.to_text()], + )?; + db.conn.execute( + "DELETE FROM changed_path_observer_segments WHERE scope_id=?1", + [view_scope.scope_id.to_text()], + )?; + let lane_leaf = create_real_segment(&db, scope.scope_id, 1, 0x6d)?; + let view_leaf = create_real_segment(&db, view_scope.scope_id, 1, 0x6e)?; + let segment_directory = db + .db_dir + .join("observer-segments") + .join(scope.scope_id.to_text()); + let segment_file = segment_directory.join(lane_leaf); + let view_segment_directory = db + .db_dir + .join("observer-segments") + .join(view_scope.scope_id.to_text()); + let view_segment_file = view_segment_directory.join(view_leaf); + let reader = Connection::open(db.db_dir.join(crate::db::DB_RELATIVE_PATH))?; + reader.execute_batch("BEGIN DEFERRED")?; + let _: i64 = reader.query_row("SELECT COUNT(*) FROM changed_path_scopes", [], |row| { + row.get(0) + })?; + if db.remove_lane("retire-first", true).is_ok() { + return Err(Error::Corrupt( + "lane deletion bypassed the pre-retirement reader barrier".into(), + )); + } + if db.remove_lane("retire-first", true).is_ok() { + return Err(Error::Corrupt( + "lane deletion retry bypassed the still-active reader barrier".into(), + )); + } + if !workdir.exists() || db.try_get_ref(&branch.ref_name)?.is_none() { + return Err(Error::Corrupt( + "lane deletion proceeded before the pre-retirement reader drained".into(), + )); + } + reader.execute_batch("COMMIT")?; + db.remove_lane("retire-first", true)?; + let retired: i64 = db.conn.query_row( + "SELECT COUNT(*) FROM changed_path_scopes + WHERE scope_id IN (?1,?2) AND retired_at IS NOT NULL", + params![scope.scope_id.to_text(), view_scope.scope_id.to_text()], + |row| row.get(0), + )?; + if retired != 2 || workdir.exists() || segment_file.exists() || view_segment_file.exists() { + return Err(Error::Corrupt( + "lane/view scopes or segments were not retired before workdir deletion".into(), + )); + } + Ok(()) + } + + #[test] + fn changed_path_subprocess_crash_helper() { + let Some(workspace) = std::env::var_os("TRAIL_TEST_CHANGED_PATH_CRASH_WORKSPACE") else { + return; + }; + run_real_crash_scenario(std::path::Path::new(&workspace)).unwrap(); + panic!("changed-path crash helper passed its requested crash point"); + } + + #[test] + fn deletion_subprocess_crash_helper() { + let Some(workspace) = std::env::var_os("TRAIL_TEST_DELETION_CRASH_WORKSPACE") else { + return; + }; + let lane = std::env::var("TRAIL_TEST_DELETION_CRASH_LANE").unwrap(); + let mut db = Trail::open(std::path::Path::new(&workspace)).unwrap(); + db.remove_lane(&lane, true).unwrap(); + panic!("deletion crash helper passed its requested crash point"); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn retirement_never_uses_separable_mkdir_open_authority() { + use std::os::unix::fs::PermissionsExt; + use std::sync::atomic::{AtomicBool, Ordering}; + + let fixture = Fixture::new(0xa4).unwrap(); + create_real_retirement_segment(&fixture, 0xa4).unwrap(); + let directory = fixture + .db + .db_dir + .join("observer-segments") + .join(fixture.expected.scope_id.to_text()); + let hook_ran = std::sync::Arc::new(AtomicBool::new(false)); + let hook_ran_capture = hook_ran.clone(); + let hook_directory = directory.clone(); + super::super::secure_fs::install_private_dir_create_open_hook(move || { + let allocated = std::fs::read_dir(&hook_directory) + .unwrap() + .filter_map(std::result::Result::ok) + .find(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(".trail-delete-") + && entry + .path() + .extension() + .is_some_and(|extension| extension == "d") + }) + .unwrap(); + let retained = allocated.path().with_extension("old-retained"); + std::fs::rename(allocated.path(), retained).unwrap(); + std::fs::create_dir(allocated.path()).unwrap(); + std::fs::set_permissions(allocated.path(), std::fs::Permissions::from_mode(0o700)) + .unwrap(); + hook_ran_capture.store(true, Ordering::SeqCst); + }); + + let result = retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &fixture.expected); + super::super::secure_fs::clear_private_dir_create_open_hook(); + + result.unwrap(); + assert!( + !hook_ran.load(Ordering::SeqCst), + "retirement crossed the separable mkdirat/openat substitution window" + ); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn direct_quarantine_rejects_source_substitution_before_atomic_rename() { + let fixture = Fixture::new(0xa5).unwrap(); + let source_leaf = create_real_retirement_segment(&fixture, 0xa5).unwrap(); + let directory = fixture + .db + .db_dir + .join("observer-segments") + .join(fixture.expected.scope_id.to_text()); + let source = directory.join(&source_leaf); + let retained = directory.join("authenticated-source-retained.cpl"); + let hook_source = source.clone(); + let hook_retained = retained.clone(); + install_deletion_substitution_hook( + DeletionSubstitutionPoint::BeforeQuarantineMove, + move || { + std::fs::rename(&hook_source, &hook_retained).unwrap(); + std::fs::write(&hook_source, b"hostile source replacement\n").unwrap(); + }, + ); + + let result = retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &fixture.expected); + clear_deletion_substitution_hook(); + assert!(result.is_err(), "substituted source was accepted"); + assert!(retained.is_file(), "authenticated source was not retained"); + let targets = std::fs::read_dir(&directory) + .unwrap() + .filter_map(std::result::Result::ok) + .filter(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(".trail-delete-") + }) + .collect::>(); + assert_eq!(targets.len(), 1); + assert_eq!( + std::fs::read(targets[0].path()).unwrap(), + b"hostile source replacement\n" + ); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn direct_quarantine_rejects_target_substitution_after_rename_before_verify() { + let fixture = Fixture::new(0xa6).unwrap(); + create_real_retirement_segment(&fixture, 0xa6).unwrap(); + let directory = fixture + .db + .db_dir + .join("observer-segments") + .join(fixture.expected.scope_id.to_text()); + let retained = directory.join("renamed-target-retained.cpl"); + let hook_directory = directory.clone(); + let hook_retained = retained.clone(); + install_deletion_substitution_hook( + DeletionSubstitutionPoint::AfterDirectRenameBeforeVerify, + move || { + let target = std::fs::read_dir(&hook_directory) + .unwrap() + .filter_map(std::result::Result::ok) + .find(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(".trail-delete-") + }) + .unwrap() + .path(); + std::fs::rename(&target, &hook_retained).unwrap(); + std::fs::write(&target, b"hostile target replacement\n").unwrap(); + }, + ); + + let result = retire_scope(&fixture.db.conn, &fixture.db.sqlite_path, &fixture.expected); + clear_deletion_substitution_hook(); + assert!(result.is_err(), "substituted direct target was accepted"); + assert!( + retained.is_file(), + "atomically moved source was not retained" + ); + let hostile = std::fs::read_dir(&directory) + .unwrap() + .filter_map(std::result::Result::ok) + .find(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(".trail-delete-") + }) + .unwrap() + .path(); + assert_eq!( + std::fs::read(hostile).unwrap(), + b"hostile target replacement\n" + ); + } + + #[test] + fn subprocess_kill_and_retry_covers_quarantine_deletion_boundaries() { + for (index, phase) in [ + "changed_path_deletion_after_retirement_fence_barrier", + "changed_path_deletion_after_allocation_journal_barrier", + "changed_path_deletion_after_direct_quarantine_rename", + "changed_path_deletion_after_direct_quarantine_verify", + "changed_path_deletion_after_direct_quarantine_fsync", + "changed_path_deletion_after_allocation_identity_barrier", + "changed_path_deletion_between_allocation_segments", + "changed_path_deletion_after_allocation_setup", + "changed_path_deletion_before_retirement_commit", + "changed_path_deletion_after_retirement_commit", + "changed_path_deletion_after_retirement_wal_barrier", + ] + .into_iter() + .enumerate() + { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("lane.txt"), b"lane\n").unwrap(); + Trail::init(root.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(root.path()).unwrap(); + let lane = format!("quarantine-crash-{index}"); + let spawned = db + .spawn_lane(&lane, Some("main"), true, None, None) + .unwrap(); + let branch = db.lane_branch(&lane).unwrap(); + let head = db.get_ref(&branch.ref_name).unwrap(); + let scope = ScopeIdentity { + scope_id: ScopeId([0x90_u8.wrapping_add(index as u8); 32]), + kind: ScopeKind::MaterializedLane, + owner_id: branch.lane_id.clone(), + }; + db.changed_path_ledger() + .begin_scope( + &scope, + &BaselineIdentity { + ref_name: head.name, + ref_generation: u64::try_from(head.generation).unwrap(), + change_id: head.change_id, + root_id: head.root_id, + }, + &PolicyIdentity { + fingerprint: [0x90; 32], + generation: 1, + }, + &FilesystemIdentity(vec![0x91]), + &ProviderIdentity { + identity: vec![0x92], + capabilities: ProviderCapabilities { + durable_cursor: true, + linearizable_fence: true, + rename_pairing: true, + overflow_scope: true, + filesystem_supported: true, + clean_proof_allowed: true, + power_loss_durability: true, + }, + }, + ) + .unwrap(); + db.conn + .execute( + "UPDATE changed_path_scopes SET scope_root=?1 WHERE scope_id=?2", + params![spawned.workdir, scope.scope_id.to_text()], + ) + .unwrap(); + let segment_directory = db + .db_dir + .join("observer-segments") + .join(scope.scope_id.to_text()); + let provider_id: String = db + .conn + .query_row( + "SELECT provider_id FROM changed_path_scopes WHERE scope_id=?1", + [scope.scope_id.to_text()], + |row| row.get(0), + ) + .unwrap(); + let mut writer = super::super::SegmentWriter::acquire( + &db.sqlite_path, + &segment_directory, + scope.scope_id, + 1, + [0x93_u8.wrapping_add(index as u8); 32], + &provider_id, + b"crash-retirement-cursor".to_vec(), + Duration::from_secs(600), + ) + .unwrap(); + writer + .append(&[super::super::ObserverRecord { + sequence: 1, + source: EvidenceSource::Observer, + path: LedgerPath::parse("allocation-crash-segment.txt").unwrap(), + flags: EvidenceFlags::CONTENT, + provider_cursor: b"crash-retirement-cursor-2".to_vec(), + }]) + .unwrap(); + writer.flush_durable().unwrap(); + writer.rotate().unwrap(); + let leaves = db + .conn + .prepare( + "SELECT segment_path FROM changed_path_observer_segments + WHERE scope_id=?1 AND epoch=1 ORDER BY first_sequence,segment_id", + ) + .unwrap() + .query_map([scope.scope_id.to_text()], |row| row.get::<_, String>(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + drop(writer); + assert_eq!(leaves.len(), 2); + let mut expected_segment_bytes = leaves + .iter() + .map(|leaf| std::fs::read(segment_directory.join(leaf)).unwrap()) + .collect::>(); + expected_segment_bytes.sort(); + let workdir = std::path::PathBuf::from(spawned.workdir.unwrap()); + let sqlite_path = db.sqlite_path.clone(); + drop(db); + + let ready = root.path().join(format!("{phase}.ready")); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "db::change_ledger::recovery::harness::deletion_subprocess_crash_helper", + "--nocapture", + ]) + .env("RUST_TEST_THREADS", "1") + .env("TRAIL_TEST_CRASH_AT", phase) + .env("TRAIL_TEST_CRASH_READY", &ready) + .env("TRAIL_TEST_DELETION_CRASH_WORKSPACE", root.path()) + .env("TRAIL_TEST_DELETION_CRASH_LANE", &lane) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + wait_for_crash_handshake(&mut child, &ready, phase); + child.kill().unwrap(); + let _ = child.wait().unwrap(); + + let mut injected_conflicts = Vec::new(); + if phase == "changed_path_deletion_after_allocation_journal_barrier" { + let crash_conn = Connection::open(&sqlite_path).unwrap(); + let allocation_leaves = crash_conn + .prepare( + "SELECT quarantine_leaf + FROM changed_path_segment_quarantine_allocations + WHERE scope_id=?1 AND state='allocating' ORDER BY segment_id", + ) + .unwrap() + .query_map([scope.scope_id.to_text()], |row| row.get::<_, String>(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + drop(crash_conn); + for leaf in allocation_leaves { + let path = segment_directory.join(leaf); + std::fs::write(&path, b"foreign direct crash target\n").unwrap(); + injected_conflicts.push(path); + } + } + + let mut reopened = Trail::open(root.path()).unwrap(); + reopened.remove_lane(&lane, true).unwrap(); + let retired: bool = reopened + .conn + .query_row( + "SELECT retired_at IS NOT NULL FROM changed_path_scopes WHERE scope_id=?1", + [scope.scope_id.to_text()], + |row| row.get(0), + ) + .unwrap(); + let retained_quarantines = reopened + .conn + .prepare( + "SELECT quarantine_leaf FROM changed_path_segment_deletions + WHERE scope_id=?1 AND epoch=1 ORDER BY segment_id", + ) + .unwrap() + .query_map([scope.scope_id.to_text()], |row| row.get::<_, String>(0)) + .unwrap() + .map(|leaf| segment_directory.join(leaf.unwrap())) + .collect::>(); + assert!(retired, "scope did not remain retired after {phase}"); + for leaf in &leaves { + assert!( + !segment_directory.join(leaf).exists(), + "original segment name remained after {phase}: {leaf}" + ); + } + assert_eq!( + retained_quarantines.len(), + 2, + "exactly two quarantined segments were not retained after {phase}" + ); + let mut retained_bytes = retained_quarantines + .iter() + .map(|path| std::fs::read(path).unwrap()) + .collect::>(); + retained_bytes.sort(); + assert_eq!(retained_bytes, expected_segment_bytes); + for conflict in &injected_conflicts { + assert!( + conflict.is_file(), + "foreign namespace was removed after {phase}" + ); + } + assert!(!workdir.exists(), "workdir remained after {phase}"); + let quiesced: i64 = reopened + .conn + .query_row( + "SELECT COUNT(*) FROM changed_path_segment_deletions + WHERE scope_id=?1 AND epoch=1 AND state='quiesced' + AND completed_at IS NOT NULL", + [scope.scope_id.to_text()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + quiesced, 2, + "retirement was not durably quiesced after {phase}" + ); + let (bound, abandoned): (i64, i64) = reopened + .conn + .query_row( + "SELECT + SUM(state='bound'),SUM(state='abandoned') + FROM changed_path_segment_quarantine_allocations WHERE scope_id=?1", + [scope.scope_id.to_text()], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(bound, 2, "allocations were not exactly bound after {phase}"); + if phase == "changed_path_deletion_after_allocation_journal_barrier" { + assert_eq!(abandoned, 2, "foreign allocations were not audited"); + } + } + } + + #[test] + fn subprocess_kill_and_reopen_covers_intent_durability_boundaries() { + for (index, (phase, tamper)) in [ + ("changed_path_after_object_graph", None), + ("changed_path_after_intent_prepare", None), + ("changed_path_after_filesystem_write", None), + ("changed_path_after_observer_durable", None), + ("changed_path_after_filesystem_applied", None), + ("changed_path_after_ref_publish", None), + ("changed_path_after_ref_publish", Some("missing")), + ("changed_path_after_ref_publish", Some("truncated")), + ("changed_path_after_ref_publish", Some("replaced")), + ("changed_path_after_ref_publish", Some("corrupt")), + ("changed_path_after_intent_publish", None), + ("changed_path_after_recovery_commit", None), + ] + .into_iter() + .enumerate() + { + let fixture = Fixture::new(0x70 + u8::try_from(index).unwrap()).unwrap(); + let label = tamper.map_or_else(|| phase.to_string(), |kind| format!("{phase}-{kind}")); + let ready = fixture.db.db_dir.join("tmp").join(format!("{label}.ready")); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "db::change_ledger::recovery::harness::changed_path_subprocess_crash_helper", + "--nocapture", + ]) + .env("RUST_TEST_THREADS", "1") + .env("TRAIL_TEST_CRASH_AT", phase) + .env("TRAIL_TEST_CRASH_READY", &ready) + .env( + "TRAIL_TEST_CHANGED_PATH_CRASH_WORKSPACE", + &fixture.db.workspace_root, + ) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + wait_for_crash_handshake(&mut child, &ready, phase); + child.kill().unwrap(); + let _ = child.wait().unwrap(); + + if let Some(tamper) = tamper { + let segment_path: String = fixture + .db + .conn + .query_row( + "SELECT segment_path FROM changed_path_observer_segments + WHERE scope_id=?1 AND state='sealed' ORDER BY first_sequence LIMIT 1", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + ) + .unwrap(); + let path = fixture + .db + .db_dir + .join("observer-segments") + .join(fixture.expected.scope_id.to_text()) + .join(segment_path); + match tamper { + "missing" => std::fs::remove_file(path).unwrap(), + "truncated" => { + let file = std::fs::OpenOptions::new().write(true).open(path).unwrap(); + let len = file.metadata().unwrap().len(); + file.set_len(len.saturating_sub(1)).unwrap(); + file.sync_all().unwrap(); + } + "replaced" => { + std::fs::remove_file(&path).unwrap(); + std::fs::write(path, b"replacement segment").unwrap(); + } + "corrupt" => { + use std::io::{Seek, Write}; + let mut file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + .unwrap(); + file.seek(std::io::SeekFrom::End(-1)).unwrap(); + file.write_all(&[0xff]).unwrap(); + file.sync_all().unwrap(); + } + _ => unreachable!(), + } + } + + let reopened = Trail::open(&fixture.db.workspace_root).unwrap(); + let ledger = reopened.changed_path_ledger(); + let _ = recover_scope(&ledger, &fixture.expected); + let fsck = reopened.fsck().unwrap(); + assert!( + fsck.errors.is_empty(), + "fsck failed after {phase}: {:?}", + fsck.errors + ); + let state: (String, i64, String) = reopened + .conn + .query_row( + "SELECT trust_state,ref_generation,baseline_root_id + FROM changed_path_scopes WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + if tamper.is_some() { + assert_eq!(state.0, "untrusted_gap", "tampered proof stayed clean"); + assert_eq!(state.2, fixture.expected.baseline_root.0); + } else if phase == "changed_path_after_object_graph" { + assert_eq!(state.0, "trusted", "object-only boundary changed trust"); + assert_eq!(state.2, fixture.expected.baseline_root.0); + } else if matches!( + phase, + "changed_path_after_intent_prepare" + | "changed_path_after_filesystem_write" + | "changed_path_after_observer_durable" + | "changed_path_after_filesystem_applied" + ) { + assert_eq!(state.0, "untrusted_gap", "false clean after {phase}"); + assert_eq!(state.2, fixture.expected.baseline_root.0); + } else { + assert_eq!( + state.0, "trusted", + "recoverable publication failed at {phase}" + ); + assert_eq!( + u64::try_from(state.1).unwrap(), + fixture.expected.ref_generation + 1 + ); + } + } + } + + #[test] + fn backup_restore_subprocess_crash_helper() { + let Some(mode) = std::env::var_os("TRAIL_TEST_BACKUP_RESTORE_CRASH_MODE") else { + return; + }; + let workspace = std::path::PathBuf::from( + std::env::var_os("TRAIL_TEST_BACKUP_RESTORE_WORKSPACE").unwrap(), + ); + let backup = + std::path::PathBuf::from(std::env::var_os("TRAIL_TEST_BACKUP_RESTORE_BACKUP").unwrap()); + if mode == "backup" { + Trail::open(&workspace) + .unwrap() + .create_backup(&backup, true) + .unwrap(); + } else { + Trail::restore_backup(&workspace, &backup, true).unwrap(); + } + panic!("backup/restore crash helper passed its requested crash point"); + } + + #[test] + fn subprocess_kill_preserves_old_or_new_backup_and_restore_tree() { + for phase in [ + "backup_restore_after_staging_sync", + "backup_restore_after_atomic_exchange", + ] { + let fixture = Fixture::new(0x79).unwrap(); + std::fs::write( + fixture.db.workspace_root.join(".trailignore"), + b"old-backup\n", + ) + .unwrap(); + fixture.db.create_backup(&fixture.backup, false).unwrap(); + std::fs::write( + fixture.db.workspace_root.join(".trailignore"), + b"new-backup\n", + ) + .unwrap(); + run_backup_restore_child( + "backup", + phase, + &fixture.db.workspace_root, + &fixture.backup, + &fixture + .db + .db_dir + .join("tmp") + .join(format!("backup-{phase}.ready")), + ); + let verification = Trail::verify_backup(&fixture.backup).unwrap(); + assert!( + verification.valid, + "backup invalid after {phase}: {:?}", + verification.errors + ); + let ignore = std::fs::read(fixture.backup.join(".trailignore")).unwrap(); + assert!(ignore == b"old-backup\n" || ignore == b"new-backup\n"); + } + + for phase in [ + "restore_after_ledger_rotation", + "restore_after_staged_workdir_rewrite", + "restore_after_staged_recovery", + "restore_after_staged_checkpoint", + "restore_after_staged_sync", + "backup_restore_after_staging_sync", + "restore_after_policy_staging", + "restore_after_policy_exchange_before_marker", + "restore_after_policy_publication", + "backup_restore_after_atomic_exchange", + "restore_after_trail_publication", + "restore_during_rollback", + "restore_during_finalization", + "restore_before_retained_cleanup", + "restore_after_retained_cleanup", + ] { + let mut fixture = Fixture::new(0x7b).unwrap(); + fixture + .db + .spawn_lane("restore-crash-lane", Some("main"), true, None, None) + .unwrap(); + std::fs::write( + fixture.db.workspace_root.join(".trailignore"), + b"new-policy-generation\n", + ) + .unwrap(); + fixture.db.create_backup(&fixture.backup, false).unwrap(); + let destination = fixture._root.path().join(format!("restore-{phase}")); + std::fs::write(destination.join("live.txt"), b"old live workspace\n").unwrap_or_else( + |_| { + std::fs::create_dir_all(&destination).unwrap(); + std::fs::write(destination.join("live.txt"), b"old live workspace\n").unwrap(); + }, + ); + Trail::init(&destination, "main", InitImportMode::WorkingTree, false).unwrap(); + std::fs::write(destination.join(".trail/live-marker"), b"old\n").unwrap(); + std::fs::write(destination.join(".trailignore"), b"old-policy-generation\n").unwrap(); + run_backup_restore_child( + "restore", + phase, + &destination, + &fixture.backup, + &destination.join(format!("{phase}.ready")), + ); + let old_is_intact = destination.join(".trail/live-marker").is_file(); + let new_is_intact = Trail::open(&destination) + .and_then(|db| db.fsck()) + .is_ok_and(|report| report.errors.is_empty()); + assert!( + old_is_intact || new_is_intact, + "neither old nor restored tree survived {phase}" + ); + let policy = std::fs::read(destination.join(".trailignore")).unwrap(); + assert_eq!( + policy, + if old_is_intact { + b"old-policy-generation\n".as_slice() + } else { + b"new-policy-generation\n".as_slice() + }, + "restore exposed a mixed DB/policy generation after {phase}" + ); + if !old_is_intact { + let restored = Trail::open(&destination).unwrap(); + let lane = restored.lane_details("restore-crash-lane").unwrap(); + let workdir = lane.branch.workdir.unwrap(); + let canonical_destination = destination.canonicalize().unwrap(); + assert!( + workdir.starts_with( + &canonical_destination + .join(".trail") + .to_string_lossy() + .to_string() + ), + "restored workdir was not rewritten after {phase}: {workdir}" + ); + assert!(std::path::Path::new(&workdir).is_dir()); + let identity: (String, String) = restored + .conn + .query_row( + "SELECT trust_state,filesystem_identity FROM changed_path_scopes + WHERE retired_at IS NULL LIMIT 1", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(identity.0, "untrusted_gap"); + assert!(!identity.1.is_empty()); + } + } + } + + #[test] + fn backup_create_supports_a_missing_output_parent() { + let fixture = Fixture::new(0x7d).unwrap(); + let backup = fixture._root.path().join("new-parent/backup"); + + fixture.db.create_backup(&backup, false).unwrap(); + + let verification = Trail::verify_backup(&backup).unwrap(); + assert!(verification.valid, "{:?}", verification.errors); + } + + #[cfg(all(unix, not(target_os = "macos")))] + #[test] + fn restore_supports_non_utf8_destination_with_fresh_identity() { + use std::os::unix::ffi::OsStringExt; + + let fixture = Fixture::new(0x7f).unwrap(); + fixture.db.create_backup(&fixture.backup, false).unwrap(); + let destination = fixture + ._root + .path() + .join(std::ffi::OsString::from_vec(b"restore-\xff".to_vec())); + Trail::restore_backup(&destination, &fixture.backup, false).unwrap(); + let restored = Trail::open(&destination).unwrap(); + let row: (String, String) = restored + .conn + .query_row( + "SELECT scope_root,filesystem_identity FROM changed_path_scopes", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert!(row.0.starts_with("os-bytes:")); + assert!(!row.1.is_empty()); + } + + fn run_backup_restore_child( + mode: &str, + phase: &str, + workspace: &std::path::Path, + backup: &std::path::Path, + ready: &std::path::Path, + ) { + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "db::change_ledger::recovery::harness::backup_restore_subprocess_crash_helper", + "--nocapture", + ]) + .env("RUST_TEST_THREADS", "1") + .env("TRAIL_TEST_CRASH_AT", phase) + .env("TRAIL_TEST_CRASH_READY", ready) + .env("TRAIL_TEST_BACKUP_RESTORE_CRASH_MODE", mode) + .env("TRAIL_TEST_BACKUP_RESTORE_WORKSPACE", workspace) + .env("TRAIL_TEST_BACKUP_RESTORE_BACKUP", backup) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + if phase == "restore_during_rollback" { + command.env("TRAIL_TEST_RESTORE_FORCE_ROLLBACK", "1"); + } + let mut child = command.spawn().unwrap(); + wait_for_crash_handshake(&mut child, ready, phase); + child.kill().unwrap(); + let _ = child.wait().unwrap(); + } + + fn wait_for_crash_handshake( + child: &mut std::process::Child, + ready: &std::path::Path, + phase: &str, + ) { + let deadline = Instant::now() + Duration::from_secs(15); + while Instant::now() < deadline { + if ready.is_file() { + return; + } + if let Some(status) = child.try_wait().unwrap() { + panic!("changed-path crash helper exited at {phase} before handshake: {status}"); + } + std::thread::sleep(Duration::from_millis(25)); + } + panic!("timed out waiting for changed-path crash helper at {phase}"); + } + + fn run_real_crash_scenario(workspace: &std::path::Path) -> Result<()> { + let mut db = Trail::open(workspace)?; + let scope_id: String = db.conn.query_row( + "SELECT scope_id FROM changed_path_scopes ORDER BY created_at LIMIT 1", + [], + |row| row.get(0), + )?; + let expected = load_expected_scope(&db.conn, &scope_id)?; + let baseline = db.get_ref(&expected.ref_name)?; + + let target_path = workspace.join("crash-target.txt"); + std::fs::write(&target_path, b"durable target graph and filesystem write\n")?; + db.record( + None, + Some("crash target graph".into()), + Actor::system(), + false, + )?; + let target_ref = db.get_ref(&expected.ref_name)?; + db.conn.execute( + "UPDATE refs SET change_id=?1,root_id=?2,operation_id=?3,generation=?4,updated_at=?5 + WHERE name=?6", + params![ + baseline.change_id.0, + baseline.root_id.0, + baseline.operation_id.0, + baseline.generation, + now_ts(), + baseline.name, + ], + )?; + crate::db::util::write_ref_file( + &db.db_dir, + &baseline.name, + &baseline.change_id, + &baseline.root_id, + &baseline.operation_id, + baseline.generation, + )?; + std::fs::remove_file(&target_path)?; + durable_intent_barrier(&db.conn)?; + crate::db::util::test_crash_point("changed_path_after_object_graph"); + + let target = IntentTarget { + change_id: target_ref.change_id.clone(), + root_id: target_ref.root_id.clone(), + operation_id: Some(target_ref.operation_id.clone()), + }; + let ledger = db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &expected, + IntentProducer::Checkout, + &target, + &Fixture::evidence("crash-target.txt"), + )?; + + std::fs::write(&target_path, b"durable target graph and filesystem write\n")?; + OpenOptions::new() + .read(true) + .open(&target_path)? + .sync_all()?; + crate::db::util::test_crash_point("changed_path_after_filesystem_write"); + + let provider_id: String = db.conn.query_row( + "SELECT provider_id FROM changed_path_scopes WHERE scope_id=?1", + [&scope_id], + |row| row.get(0), + )?; + let owner_token = [0x71_u8; 32]; + let segment_dir = db + .db_dir + .join("observer-segments") + .join(expected.scope_id.to_text()); + let mut writer = super::super::SegmentWriter::acquire( + &db.db_dir.join(crate::db::DB_RELATIVE_PATH), + &segment_dir, + expected.scope_id, + expected.epoch, + owner_token, + &provider_id, + b"cursor-1".to_vec(), + Duration::from_secs(600), + )?; + writer.append(&[super::super::ObserverRecord { + sequence: 1, + source: EvidenceSource::Observer, + path: LedgerPath::parse("crash-target.txt")?, + flags: EvidenceFlags::CONTENT, + provider_cursor: b"cursor-end".to_vec(), + }])?; + let durable = writer.flush_durable()?; + writer.rotate()?; + drop(writer); + db.conn.execute( + "UPDATE changed_path_observer_owners SET provider_identity=?1,fence_nonce=?2 + WHERE scope_id=?3", + params![ + hex::encode(&expected.provider_identity), + b"crash-fence".as_slice(), + scope_id, + ], + )?; + db.conn.execute( + "UPDATE changed_path_observer_segments SET folded_end_offset=durable_end_offset + WHERE scope_id=?1 AND segment_id=?2", + params![scope_id, durable.segment_id], + )?; + db.conn.execute( + "UPDATE changed_path_scopes SET provider_cursor=?1,durable_offset=?2,folded_offset=?2 + WHERE scope_id=?3", + params![ + durable.provider_cursor, + sql_u64(durable.durable_end_offset, "crash durable offset")?, + scope_id, + ], + )?; + db.conn.execute( + "INSERT INTO changed_path_entries(scope_id,normalized_path,event_flags,source_mask, + first_sequence,last_sequence,provider_id,provider_sequence,intent_id,created_at,updated_at) + VALUES(?1,'crash-target.txt',?2,1,1,1,?3,1,NULL,?4,?4)", + params![scope_id, EvidenceFlags::CONTENT.0, provider_id, now_ts()], + )?; + durable_intent_barrier(&db.conn)?; + crate::db::util::test_crash_point("changed_path_after_observer_durable"); + + let segment = db.conn.query_row( + "SELECT segment_id,segment_hash,segment_path,durable_end_offset,folded_end_offset + FROM changed_path_observer_segments + WHERE scope_id=?1 AND state='sealed'", + [&scope_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + )) + }, + )?; + let hash: [u8; 32] = hex::decode(segment.1) + .map_err(|error| Error::Corrupt(error.to_string()))? + .try_into() + .map_err(|_| Error::Corrupt("invalid crash segment hash".into()))?; + let proof = QualifiedFilesystemProof { + scope_id: expected.scope_id, + epoch: expected.epoch, + expected_root_id: expected.baseline_root.clone(), + scope_root_identity: expected.filesystem_identity.clone(), + filesystem_identity: expected.filesystem_identity.clone(), + provider_id, + provider_identity: expected.provider_identity.clone(), + observer_owner_token: hex::encode(owner_token), + owner_fence_nonce: Some(b"crash-fence".to_vec()), + durable_segment_id: segment.0.clone(), + durable_segment_hash: hash, + segment_directory: format!("observer-segments/{}", expected.scope_id.to_text()), + segment_path: segment.2, + start_cursor: Some(b"cursor-1".to_vec()), + end_cursor: b"cursor-end".to_vec(), + publication_segment_id: segment.0.clone(), + publication_cursor: b"cursor-end".to_vec(), + start_sequence: 1, + end_cut: EvidenceCut { + source: EvidenceSource::Observer, + sequence: 1, + durable_offset: durable.durable_end_offset, + folded_offset: durable.durable_end_offset, + }, + publication_cut: EvidenceCut { + source: EvidenceSource::Observer, + sequence: 1, + durable_offset: durable.durable_end_offset, + folded_offset: durable.durable_end_offset, + }, + segment_durable_offset: db_u64(segment.3, "crash segment durable")?, + segment_folded_offset: db_u64(segment.4, "crash segment folded")?, + verified_paths: 1, + verified_prefixes: 0, + complete_root_interval: true, + complete_policy_interval: true, + persisted_evidence_through_end: true, + }; + mark_filesystem_applied(&ledger, &expected, &intent, &proof)?; + + db.conn.execute( + "UPDATE refs SET change_id=?1,root_id=?2,operation_id=?3,generation=?4,updated_at=?5 + WHERE name=?6", + params![ + target_ref.change_id.0, + target_ref.root_id.0, + target_ref.operation_id.0, + target_ref.generation, + now_ts(), + target_ref.name, + ], + )?; + crate::db::util::write_ref_file( + &db.db_dir, + &target_ref.name, + &target_ref.change_id, + &target_ref.root_id, + &target_ref.operation_id, + target_ref.generation, + )?; + durable_intent_barrier(&db.conn)?; + crate::db::util::test_crash_point("changed_path_after_ref_publish"); + publish_intent(&ledger, &expected, &intent)?; + recover_scope(&ledger, &expected)?; + Ok(()) + } + + fn load_expected_scope(conn: &Connection, scope_id: &str) -> Result { + let row = conn.query_row( + "SELECT epoch,ref_name,ref_generation,baseline_root_id,policy_fingerprint, + policy_dependency_generation,filesystem_identity,provider_identity + FROM changed_path_scopes WHERE scope_id=?1", + [scope_id], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, String>(6)?, + row.get::<_, String>(7)?, + )) + }, + )?; + let scope: [u8; 32] = hex::decode(scope_id) + .map_err(|error| Error::Corrupt(error.to_string()))? + .try_into() + .map_err(|_| Error::Corrupt("invalid crash scope id".into()))?; + let policy: [u8; 32] = hex::decode(row.4) + .map_err(|error| Error::Corrupt(error.to_string()))? + .try_into() + .map_err(|_| Error::Corrupt("invalid crash policy".into()))?; + Ok(ExpectedScope { + scope_id: ScopeId(scope), + epoch: db_u64(row.0, "crash epoch")?, + ref_name: row.1, + ref_generation: db_u64(row.2, "crash ref generation")?, + baseline_root: ObjectId(row.3), + policy_fingerprint: policy, + policy_generation: db_u64(row.5, "crash policy generation")?, + filesystem_identity: hex::decode(row.6) + .map_err(|error| Error::Corrupt(error.to_string()))?, + provider_identity: hex::decode(row.7) + .map_err(|error| Error::Corrupt(error.to_string()))?, + }) + } + + pub(super) fn rejects_unqualified_or_stale_filesystem_proof() -> Result<()> { + let fixture = Fixture::new(0x68)?; + let target = fixture.target("stale-proof", fixture.expected.baseline_root.clone()); + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &target, + &Fixture::evidence("stale-proof.rs"), + )?; + mark_filesystem_applied( + &ledger, + &fixture.expected, + &intent, + &fixture.qualified_proof(11, "stale-proof.rs")?, + )?; + fixture.db.conn.execute( + "UPDATE changed_path_observer_segments SET segment_hash=?1 WHERE scope_id=?2", + params![ + hex::encode([0xff_u8; 32]), + fixture.expected.scope_id.to_text() + ], + )?; + fixture.advance_ref_to(&target)?; + + let decisions = recover_scope(&ledger, &fixture.expected)?; + if decisions == vec![(intent.clone(), RecoveryDecision::FinishPublication)] { + return Err(Error::Corrupt( + "an unqualified filesystem cut authorized baseline publication".into(), + )); + } + let state: String = fixture.db.conn.query_row( + "SELECT trust_state FROM changed_path_scopes WHERE scope_id=?1", + [fixture.expected.scope_id.to_text()], + |row| row.get(0), + )?; + if state != "untrusted_gap" { + return Err(Error::Corrupt(format!( + "unqualified proof did not fail closed: {state}" + ))); + } + Ok(()) + } + + pub(super) fn rejects_metadata_only_proof_without_sidecar() -> Result<()> { + let fixture = Fixture::new(0x80)?; + let target = fixture.target("missing-sidecar", fixture.expected.baseline_root.clone()); + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &target, + &Fixture::evidence("missing-sidecar.rs"), + )?; + let proof = fixture.qualified_proof(1, "missing-sidecar.rs")?; + std::fs::remove_file( + fixture + .db + .db_dir + .join(&proof.segment_directory) + .join(&proof.segment_path), + )?; + + if mark_filesystem_applied(&ledger, &fixture.expected, &intent, &proof).is_ok() { + return Err(Error::Corrupt( + "metadata-only intent proof was accepted without a segment sidecar".into(), + )); + } + Ok(()) + } + + #[cfg(test)] + fn append_unpublished_proof_suffix( + writer: &mut super::super::SegmentWriter, + sequence: u64, + path: &str, + ) { + writer + .append(&[super::super::ObserverRecord { + sequence, + source: EvidenceSource::Observer, + path: LedgerPath::parse(path).unwrap(), + flags: EvidenceFlags::CONTENT, + provider_cursor: format!("cursor-{}", sequence + 1).into_bytes(), + }]) + .unwrap(); + } + + #[cfg(test)] + #[test] + fn proof_validation_waits_for_authenticated_open_tail_publication() { + let fixture = Fixture::new(0x91).unwrap(); + let target = fixture.target("transient-tail", fixture.expected.baseline_root.clone()); + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &target, + &Fixture::evidence("transient-tail.rs"), + ) + .unwrap(); + let (proof, mut writer) = fixture + .qualified_proof_with_writer(1, "transient-tail.rs") + .unwrap(); + append_unpublished_proof_suffix(&mut writer, 2, "later-tail.rs"); + install_sidecar_retry_hook(move || { + writer.flush_durable().unwrap(); + }); + + mark_filesystem_applied(&ledger, &fixture.expected, &intent, &proof).unwrap(); + } + + #[cfg(test)] + #[test] + fn static_unpublished_open_tail_fails_after_idle_bound() { + let fixture = Fixture::new(0x92).unwrap(); + let target = fixture.target("static-tail", fixture.expected.baseline_root.clone()); + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &target, + &Fixture::evidence("static-tail.rs"), + ) + .unwrap(); + let (proof, mut writer) = fixture + .qualified_proof_with_writer(1, "static-tail.rs") + .unwrap(); + append_unpublished_proof_suffix(&mut writer, 2, "unpublished-tail.rs"); + let started = Instant::now(); + + let error = mark_filesystem_applied(&ledger, &fixture.expected, &intent, &proof) + .expect_err("static unpublished tail unexpectedly authorized the proof"); + + assert!(started.elapsed() >= crate::db::change_ledger::intent::SIDECAR_RECOVERY_IDLE_LIMIT); + assert!(started.elapsed() < Duration::from_secs(5)); + assert!(error + .to_string() + .contains("observer sidecar chain is incomplete or contains unpublished entries")); + drop(writer); + } + + #[cfg(test)] + #[test] + fn scope_owner_token_swap_during_sidecar_retry_fails_closed() { + let fixture = Fixture::new(0x93).unwrap(); + let target = fixture.target("owner-swap", fixture.expected.baseline_root.clone()); + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &target, + &Fixture::evidence("owner-swap.rs"), + ) + .unwrap(); + let (proof, mut writer) = fixture + .qualified_proof_with_writer(1, "owner-swap.rs") + .unwrap(); + append_unpublished_proof_suffix(&mut writer, 2, "owner-swap-tail.rs"); + let database_path = fixture.db.sqlite_path.clone(); + let scope_id = fixture.expected.scope_id.to_text(); + install_sidecar_retry_hook(move || { + let conn = Connection::open(database_path).unwrap(); + conn.execute( + "UPDATE changed_path_scopes SET observer_owner_token=?1 WHERE scope_id=?2", + params![hex::encode([0xee; 32]), scope_id], + ) + .unwrap(); + }); + let started = Instant::now(); + + let error = mark_filesystem_applied(&ledger, &fixture.expected, &intent, &proof) + .expect_err("scope owner-token swap unexpectedly authorized the proof"); + + assert!(started.elapsed() < Duration::from_secs(1)); + assert!(error + .to_string() + .contains("owner, fence, or sealed segment changed during sidecar verification")); + drop(writer); + } + + #[cfg(test)] + #[test] + fn scope_owner_token_swap_after_read_validation_loses_final_publication_cas() { + let fixture = Fixture::new(0x94).unwrap(); + let target = fixture.target( + "post-validation-owner-swap", + fixture.expected.baseline_root.clone(), + ); + let ledger = fixture.db.changed_path_ledger(); + let intent = prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &target, + &Fixture::evidence("post-validation-owner-swap.rs"), + ) + .unwrap(); + let proof = fixture + .qualified_proof(1, "post-validation-owner-swap.rs") + .unwrap(); + let database_path = fixture.db.sqlite_path.clone(); + let scope_id = fixture.expected.scope_id.to_text(); + install_sidecar_post_validation_hook(move || { + let conn = Connection::open(database_path).unwrap(); + conn.execute( + "UPDATE changed_path_scopes SET observer_owner_token=?1 WHERE scope_id=?2", + params![hex::encode([0xef; 32]), scope_id], + ) + .unwrap(); + }); + + let error = mark_filesystem_applied(&ledger, &fixture.expected, &intent, &proof) + .expect_err("post-validation owner-token swap unexpectedly won final intent CAS"); + + assert!(error + .to_string() + .contains("filesystem proof authority changed before intent publication")); + let state: String = fixture + .db + .conn + .query_row( + "SELECT lifecycle_state FROM changed_path_intents WHERE intent_id=?1", + [&intent.0], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(state, "prepared"); + } + + pub(super) fn ambiguous_recovery_blocks_next_intent() -> Result<()> { + let fixture = Fixture::new(0x69)?; + let first_target = + fixture.target("ambiguous-first", fixture.expected.baseline_root.clone()); + let ledger = fixture.db.changed_path_ledger(); + prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &first_target, + &Fixture::evidence("first.rs"), + )?; + recover_scope(&ledger, &fixture.expected)?; + + let second_target = + fixture.target("ambiguous-second", fixture.expected.baseline_root.clone()); + if prepare_intent( + &ledger, + &fixture.expected, + IntentProducer::Checkout, + &second_target, + &Fixture::evidence("second.rs"), + ) + .is_ok() + { + return Err(Error::Corrupt( + "a new intent was prepared while full reconciliation was required".into(), + )); + } + Ok(()) + } + + pub(super) fn backup_restore_rotation() -> Result<()> { + let fixture = Fixture::new(0x66)?; + fixture.db.create_backup(&fixture.backup, false)?; + let backup_conn = Connection::open(fixture.backup.join("index/trail.sqlite"))?; + let backup_state: String = + backup_conn.query_row("SELECT trust_state FROM changed_path_scopes", [], |row| { + row.get(0) + })?; + if backup_state != "untrusted_gap" { + return Err(Error::Corrupt("backup retained trusted scope".into())); + } + drop(backup_conn); + Trail::restore_backup(&fixture.restore, &fixture.backup, false)?; + let restored = Connection::open(fixture.restore.join(".trail/index/trail.sqlite"))?; + let row = restored.query_row( + "SELECT epoch,trust_state,provider_identity,provider_cursor,durable_offset, + folded_offset,filesystem_identity FROM changed_path_scopes", + [], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, String>(6)?, + )) + }, + )?; + if row.0 != 2 + || row.1 != "untrusted_gap" + || row.2.is_some() + || row.3.is_some() + || row.4 != 0 + || row.5 != 0 + || row.6 == hex::encode(&fixture.expected.filesystem_identity) + { + return Err(Error::Corrupt(format!( + "restore did not rotate ledger identity: {row:?}" + ))); + } + drop(restored); + Trail::restore_backup(&fixture.restore, &fixture.backup, true)?; + let restored_again = Connection::open(fixture.restore.join(".trail/index/trail.sqlite"))?; + let repeated: (i64, i64, String) = restored_again.query_row( + "SELECT epoch,continuity_generation,filesystem_identity + FROM changed_path_scopes", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + if repeated.0 <= row.0 || repeated.1 <= 1 || repeated.2 == row.6 { + return Err(Error::Corrupt(format!( + "repeated restore reused its ledger incarnation: first={row:?} repeated={repeated:?}" + ))); + } + Ok(()) + } + + pub(super) fn backup_overwrite_failure_preserves_previous() -> Result<()> { + let fixture = Fixture::new(0x6a)?; + fixture.db.create_backup(&fixture.backup, false)?; + let previous_manifest = std::fs::read(fixture.backup.join("manifest.json"))?; + std::fs::remove_file(fixture.db.db_dir.join(crate::db::CONFIG_FILE))?; + if fixture.db.create_backup(&fixture.backup, true).is_ok() { + return Err(Error::Corrupt( + "backup overwrite unexpectedly succeeded with a missing source config".into(), + )); + } + let retained = std::fs::read(fixture.backup.join("manifest.json"))?; + if retained != previous_manifest { + return Err(Error::Corrupt( + "failed backup overwrite did not retain the prior valid tree".into(), + )); + } + Ok(()) + } +} + +#[cfg(debug_assertions)] +pub(crate) fn run_acknowledgement_race() -> std::result::Result<(), String> { + harness::acknowledgement_race().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_gc_root_lifecycle() -> std::result::Result<(), String> { + harness::gc_root_lifecycle().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_crash_matrix() -> std::result::Result<(), String> { + harness::crash_matrix().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_backup_restore_rotation() -> std::result::Result<(), String> { + harness::backup_restore_rotation().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_qualified_proof_revalidation() -> std::result::Result<(), String> { + harness::rejects_unqualified_or_stale_filesystem_proof().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_ambiguous_recovery_gate() -> std::result::Result<(), String> { + harness::ambiguous_recovery_blocks_next_intent().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_backup_overwrite_rollback() -> std::result::Result<(), String> { + harness::backup_overwrite_failure_preserves_previous().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_deletion_post_verification_substitution_rejection( +) -> std::result::Result<(), String> { + harness::deletion_name_substitution_after_verification_fails_closed() + .map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_deletion_post_quarantine_verification_substitution_rejection( +) -> std::result::Result<(), String> { + harness::deletion_substitution_after_quarantine_verification_fails_closed() + .map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_deletion_retry_hostile_quarantine_replacement_rejection( +) -> std::result::Result<(), String> { + harness::deletion_retry_rejects_hostile_quarantine_replacement() + .map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_deletion_normal_retry_idempotence() -> std::result::Result<(), String> { + harness::deletion_normal_retry_is_durably_idempotent().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_retained_writer_quiescence() -> std::result::Result<(), String> { + harness::retirement_requires_retained_writer_quiescence().map_err(|error| error.to_string()) +} + +#[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] +pub(crate) fn run_orphan_quarantine_substitution_rejection() -> std::result::Result<(), String> { + harness::orphan_quarantine_substitution_fails_closed().map_err(|error| error.to_string()) +} + +#[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] +pub(crate) fn run_empty_orphan_quarantine_rejection() -> std::result::Result<(), String> { + harness::empty_orphan_quarantine_is_retained_and_rejected().map_err(|error| error.to_string()) +} + +#[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] +pub(crate) fn run_no_orphan_quarantine_allocation() -> std::result::Result<(), String> { + harness::no_orphan_allocates_fresh_quarantine_authority().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_deletion_quiesced_missing_quarantine_rejection() -> std::result::Result<(), String> +{ + harness::deletion_quiesced_retry_rejects_missing_quarantine().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_deletion_quiesced_reappeared_original_rejection( +) -> std::result::Result<(), String> { + harness::deletion_quiesced_retry_rejects_reappeared_original() + .map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_restored_nullable_provider_lane_deletion() -> std::result::Result<(), String> { + harness::restored_nullable_provider_lane_deletion().map_err(|error| error.to_string()) +} + +#[cfg(all(debug_assertions, unix))] +pub(crate) fn run_non_utf_database_path_mark_recover_and_retire() -> std::result::Result<(), String> +{ + harness::non_utf_database_path_mark_recover_and_retire().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_retirement_barrier() -> std::result::Result<(), String> { + harness::retirement_path_and_reader_barrier().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_lane_deletion_retirement() -> std::result::Result<(), String> { + harness::lane_deletion_retires_scope_first().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_missing_sidecar_rejection() -> std::result::Result<(), String> { + harness::rejects_metadata_only_proof_without_sidecar().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_advanced_prefix_recovery() -> std::result::Result<(), String> { + harness::empty_rotation_anchor_authenticates_publication_cut() + .and_then(|()| harness::authenticated_prefix_survives_later_observer_advance()) + .map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_exact_interval_bridge_rejection() -> std::result::Result<(), String> { + harness::exact_path_aggregate_bridge_is_rejected().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_prefix_interval_bridge_rejection() -> std::result::Result<(), String> { + harness::prefix_aggregate_bridge_is_rejected().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_valid_prefix_interval_recovery() -> std::result::Result<(), String> { + harness::authenticated_prefix_interval_preserves_later_suffix() + .map_err(|error| error.to_string()) +} + +#[cfg(all(debug_assertions, unix))] +pub(crate) fn run_mark_ancestor_substitution_rejection() -> std::result::Result<(), String> { + harness::ancestor_substitution_at_mark_is_rejected().map_err(|error| error.to_string()) +} + +#[cfg(all(debug_assertions, unix))] +pub(crate) fn run_recovery_ancestor_substitution_rejection() -> std::result::Result<(), String> { + harness::ancestor_substitution_at_recovery_is_rejected().map_err(|error| error.to_string()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_deletion_parent_substitution_rejection() -> std::result::Result<(), String> { + harness::deletion_parent_substitution_uses_retained_authority() + .map_err(|error| error.to_string()) +} + +#[cfg(all(debug_assertions, unix))] +pub(crate) fn run_deletion_leaf_substitution_rejection() -> std::result::Result<(), String> { + harness::deletion_leaf_substitution_fails_closed().map_err(|error| error.to_string()) +} diff --git a/trail/src/db/change_ledger/secure_fs.rs b/trail/src/db/change_ledger/secure_fs.rs new file mode 100644 index 0000000..098823b --- /dev/null +++ b/trail/src/db/change_ledger/secure_fs.rs @@ -0,0 +1,725 @@ +use std::fs::File; +use std::io::{Read, Write}; +use std::path::{Component, Path}; + +use crate::error::{Error, Result}; + +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] +thread_local! { + static VERIFIED_UNLINK_INNER_WINDOW_HOOK: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] +thread_local! { + static PRIVATE_DIR_CREATE_OPEN_HOOK: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] +pub(crate) fn install_private_dir_create_open_hook(hook: impl FnOnce() + 'static) { + PRIVATE_DIR_CREATE_OPEN_HOOK.with(|slot| *slot.borrow_mut() = Some(Box::new(hook))); +} + +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] +pub(crate) fn clear_private_dir_create_open_hook() { + PRIVATE_DIR_CREATE_OPEN_HOOK.with(|slot| slot.borrow_mut().take()); +} + +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] +fn run_private_dir_create_open_hook() { + PRIVATE_DIR_CREATE_OPEN_HOOK.with(|slot| { + if let Some(hook) = slot.borrow_mut().take() { + hook(); + } + }); +} + +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] +fn install_verified_unlink_inner_window_hook(hook: impl FnOnce() + 'static) { + VERIFIED_UNLINK_INNER_WINDOW_HOOK.with(|slot| { + *slot.borrow_mut() = Some(Box::new(hook)); + }); +} + +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] +fn run_verified_unlink_inner_window_hook() { + VERIFIED_UNLINK_INNER_WINDOW_HOOK.with(|slot| { + if let Some(hook) = slot.borrow_mut().take() { + hook(); + } + }); +} + +#[derive(Debug)] +pub(crate) struct SecureDirectory { + file: File, +} + +impl SecureDirectory { + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn open_absolute(path: &Path) -> Result { + use rustix::fs::{openat, Mode, OFlags, CWD}; + + if !path.is_absolute() { + return Err(Error::InvalidInput(format!( + "secure directory `{}` is not absolute", + path.display() + ))); + } + let mut file = File::from( + openat( + CWD, + Path::new("/"), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(|error| Error::Io(error.into()))?, + ); + for component in path + .strip_prefix(Path::new("/")) + .map_err(|error| Error::InvalidInput(error.to_string()))? + .components() + { + let Component::Normal(name) = component else { + return Err(Error::InvalidInput(format!( + "secure directory `{}` is not normalized", + path.display() + ))); + }; + file = File::from( + openat( + &file, + Path::new(name), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(|error| Error::Io(error.into()))?, + ); + } + Ok(Self { file }) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn open_absolute(path: &Path) -> Result { + Err(Error::InvalidInput(format!( + "secure descriptor-relative filesystem authority is unsupported for `{}`", + path.display() + ))) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn open_dir(&self, name: &str) -> Result { + use rustix::fs::{openat, Mode, OFlags}; + + validate_leaf(name)?; + let file = File::from( + openat( + &self.file, + Path::new(name), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(|error| Error::Io(error.into()))?, + ); + verify_entry_identity(&self.file, name, &file, true)?; + Ok(Self { file }) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn create_private_dir(&self, name: &str) -> Result { + use rustix::fs::{mkdirat, Mode}; + + validate_leaf(name)?; + mkdirat(&self.file, Path::new(name), Mode::from_raw_mode(0o700)) + .map_err(|error| Error::Io(error.into()))?; + #[cfg(all(test, any(target_os = "linux", target_os = "macos")))] + run_private_dir_create_open_hook(); + let directory = self.open_dir(name)?; + directory.verify_private()?; + Ok(directory) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn create_private_dir(&self, name: &str) -> Result { + let _ = (self, name); + Err(Error::InvalidInput( + "secure private directory creation is unsupported".into(), + )) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn open_private_dir(&self, name: &str) -> Result { + let directory = self.open_dir(name)?; + directory.verify_private()?; + Ok(directory) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn open_private_dir(&self, name: &str) -> Result { + let _ = (self, name); + Err(Error::InvalidInput( + "secure private directory open is unsupported".into(), + )) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn open_dir(&self, name: &str) -> Result { + let _ = (self, name); + Err(Error::InvalidInput( + "secure descriptor-relative directory traversal is unsupported".into(), + )) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn open_regular(&self, name: &str) -> Result { + use rustix::fs::{openat, Mode, OFlags}; + + validate_leaf(name)?; + let file = File::from( + openat( + &self.file, + Path::new(name), + OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(|error| Error::Io(error.into()))?, + ); + verify_entry_identity(&self.file, name, &file, false)?; + Ok(file) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn read_regular_optional_bounded( + &self, + name: &str, + max_bytes: u64, + ) -> Result>> { + let mut file = match self.open_regular(name) { + Ok(file) => file, + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(None); + } + Err(error) => return Err(error), + }; + if file.metadata()?.len() > max_bytes { + return Err(Error::InvalidInput(format!( + "secure marker `{name}` exceeds {max_bytes} bytes" + ))); + } + let mut bytes = Vec::new(); + std::io::Read::by_ref(&mut file) + .take(max_bytes.saturating_add(1)) + .read_to_end(&mut bytes)?; + if u64::try_from(bytes.len()).unwrap_or(u64::MAX) > max_bytes { + return Err(Error::InvalidInput(format!( + "secure marker `{name}` grew beyond {max_bytes} bytes" + ))); + } + self.verify_opened_regular(name, &file)?; + Ok(Some(bytes)) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn read_regular_optional_bounded( + &self, + name: &str, + max_bytes: u64, + ) -> Result>> { + let _ = (self, name, max_bytes); + Err(Error::InvalidInput( + "secure descriptor-relative marker read is unsupported".into(), + )) + } + + /// Atomically replace a regular leaf without following either the + /// temporary or destination pathname. The pinned directory descriptor is + /// the sole rename authority. + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn write_atomic_regular(&self, name: &str, bytes: &[u8]) -> Result<()> { + use rustix::fs::{openat, renameat, Mode, OFlags}; + + validate_leaf(name)?; + let mut random = [0_u8; 12]; + getrandom::getrandom(&mut random) + .map_err(|error| Error::Io(std::io::Error::other(error.to_string())))?; + let temporary = format!(".marker-{}.tmp", hex::encode(random)); + let mut file = File::from( + openat( + &self.file, + Path::new(&temporary), + OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::from_raw_mode(0o600), + ) + .map_err(|error| Error::Io(error.into()))?, + ); + let result = (|| { + file.write_all(bytes)?; + file.sync_all()?; + renameat( + &self.file, + Path::new(&temporary), + &self.file, + Path::new(name), + ) + .map_err(|error| Error::Io(error.into()))?; + self.sync() + })(); + if result.is_err() { + let _ = rustix::fs::unlinkat( + &self.file, + Path::new(&temporary), + rustix::fs::AtFlags::empty(), + ); + } + result + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn write_atomic_regular(&self, name: &str, bytes: &[u8]) -> Result<()> { + let _ = (self, name, bytes); + Err(Error::InvalidInput( + "secure descriptor-relative marker write is unsupported".into(), + )) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn remove_leaf(&self, name: &str) -> Result<()> { + validate_leaf(name)?; + match rustix::fs::unlinkat(&self.file, Path::new(name), rustix::fs::AtFlags::empty()) { + Ok(()) => self.sync(), + Err(error) if error == rustix::io::Errno::NOENT => Ok(()), + Err(error) => Err(Error::Io(error.into())), + } + } + + /// Remove one directory tree using only descriptor-relative traversal. + /// The caller must first authenticate this directory and the child inode + /// against persisted ownership authority. + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn remove_owned_tree_leaf( + &self, + name: &str, + expected_identity: (u64, u64), + ) -> Result<()> { + validate_leaf(name)?; + let child = self.open_dir(name)?; + child.verify_identity(expected_identity)?; + child.remove_owned_tree_contents()?; + // Reopen through the pinned parent immediately before rmdir so a + // pathname substitution cannot redirect deletion to another inode. + self.open_dir(name)?.verify_identity(expected_identity)?; + rustix::fs::unlinkat(&self.file, Path::new(name), rustix::fs::AtFlags::REMOVEDIR) + .map_err(|error| Error::Io(error.into()))?; + self.sync() + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + fn remove_owned_tree_contents(&self) -> Result<()> { + for name in self.entry_names()? { + let name = name.to_str().ok_or_else(|| { + Error::InvalidInput("owned materialization tree has a non-UTF8 leaf".into()) + })?; + match self.open_dir(name) { + Ok(directory) => { + let identity = directory.identity()?; + directory.remove_owned_tree_contents()?; + self.open_dir(name)?.verify_identity(identity)?; + rustix::fs::unlinkat( + &self.file, + Path::new(name), + rustix::fs::AtFlags::REMOVEDIR, + ) + .map_err(|error| Error::Io(error.into()))?; + } + Err(Error::Io(error)) if error.raw_os_error() == Some(libc::ENOTDIR) => { + let opened = self.open_regular(name)?; + self.verify_opened_regular(name, &opened)?; + rustix::fs::unlinkat(&self.file, Path::new(name), rustix::fs::AtFlags::empty()) + .map_err(|error| Error::Io(error.into()))?; + } + Err(error) => return Err(error), + } + } + self.sync() + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn remove_owned_tree_leaf( + &self, + name: &str, + expected_identity: (u64, u64), + ) -> Result<()> { + let _ = (self, name, expected_identity); + Err(Error::InvalidInput( + "secure owned-tree removal is unsupported".into(), + )) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn remove_leaf(&self, name: &str) -> Result<()> { + let _ = (self, name); + Err(Error::InvalidInput( + "secure descriptor-relative marker removal is unsupported".into(), + )) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn open_regular(&self, name: &str) -> Result { + let _ = (self, name); + Err(Error::InvalidInput( + "secure descriptor-relative file open is unsupported".into(), + )) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn entry_names(&self) -> Result> { + use std::os::unix::ffi::OsStrExt; + + let mut directory = + rustix::fs::Dir::read_from(&self.file).map_err(|error| Error::Io(error.into()))?; + let mut names = Vec::new(); + while let Some(entry) = directory.read() { + let entry = entry.map_err(|error| Error::Io(error.into()))?; + let bytes = entry.file_name().to_bytes(); + if matches!(bytes, b"." | b"..") { + continue; + } + names.push(std::ffi::OsStr::from_bytes(bytes).to_os_string()); + } + Ok(names) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn entry_names(&self) -> Result> { + let _ = self; + Err(Error::InvalidInput( + "secure descriptor-relative directory enumeration is unsupported".into(), + )) + } + + pub(crate) fn file(&self) -> &File { + &self.file + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn identity(&self) -> Result<(u64, u64)> { + descriptor_identity(&self.file) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn identity(&self) -> Result<(u64, u64)> { + let _ = self; + Err(Error::InvalidInput( + "secure directory identity is unsupported".into(), + )) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn verify_identity(&self, expected: (u64, u64)) -> Result<()> { + if self.identity()? != expected { + return Err(Error::InvalidInput( + "secure directory identity does not match persisted authority".into(), + )); + } + Ok(()) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn verify_identity(&self, expected: (u64, u64)) -> Result<()> { + let _ = (self, expected); + Err(Error::InvalidInput( + "secure directory identity validation is unsupported".into(), + )) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn verify_opened_regular(&self, name: &str, opened: &File) -> Result<()> { + validate_leaf(name)?; + verify_entry_identity(&self.file, name, opened, false) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn verify_same_opened_regular( + &self, + expected: &File, + observed: &File, + ) -> Result<()> { + use rustix::fs::{fstat, FileType}; + + let expected = fstat(expected).map_err(|error| Error::Io(error.into()))?; + let observed = fstat(observed).map_err(|error| Error::Io(error.into()))?; + if FileType::from_raw_mode(expected.st_mode) != FileType::RegularFile + || FileType::from_raw_mode(observed.st_mode) != FileType::RegularFile + || expected.st_dev != observed.st_dev + || expected.st_ino != observed.st_ino + { + return Err(Error::InvalidInput( + "secure filesystem quarantine inode does not match deletion authority".into(), + )); + } + Ok(()) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn verify_same_opened_regular( + &self, + expected: &File, + observed: &File, + ) -> Result<()> { + let _ = (self, expected, observed); + Err(Error::InvalidInput( + "secure descriptor-relative inode comparison is unsupported".into(), + )) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn rename_leaf_noreplace(&self, source: &str, destination: &str) -> Result<()> { + use rustix::fs::{renameat_with, RenameFlags}; + + validate_leaf(source)?; + validate_leaf(destination)?; + renameat_with( + &self.file, + Path::new(source), + &self.file, + Path::new(destination), + RenameFlags::NOREPLACE, + ) + .map_err(|error| Error::Io(error.into())) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn rename_leaf_to_noreplace( + &self, + source: &str, + destination_directory: &Self, + destination: &str, + ) -> Result<()> { + use rustix::fs::{renameat_with, RenameFlags}; + + validate_leaf(source)?; + validate_leaf(destination)?; + renameat_with( + &self.file, + Path::new(source), + &destination_directory.file, + Path::new(destination), + RenameFlags::NOREPLACE, + ) + .map_err(|error| Error::Io(error.into())) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn rename_leaf_to_noreplace( + &self, + source: &str, + destination_directory: &Self, + destination: &str, + ) -> Result<()> { + let _ = (self, source, destination_directory, destination); + Err(Error::InvalidInput( + "secure descriptor-relative cross-directory rename is unsupported".into(), + )) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn rename_leaf_noreplace(&self, source: &str, destination: &str) -> Result<()> { + let _ = (self, source, destination); + Err(Error::InvalidInput( + "secure descriptor-relative quarantine rename is unsupported".into(), + )) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn verify_opened_regular(&self, name: &str, opened: &File) -> Result<()> { + let _ = (self, name, opened); + Err(Error::InvalidInput( + "secure descriptor-relative validation is unsupported".into(), + )) + } + + #[cfg(all(test, any(target_os = "linux", target_os = "macos")))] + pub(crate) fn unlink_verified_regular(&self, name: &str, opened: &File) -> Result { + self.verify_opened_regular(name, opened)?; + run_verified_unlink_inner_window_hook(); + Err(Error::InvalidInput( + "verified pathname unlink has no inode-bound POSIX authority".into(), + )) + } + + pub(crate) fn sync(&self) -> Result<()> { + self.file.sync_all().map_err(Error::Io) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn restrict_private(&self) -> Result<()> { + rustix::fs::fchmod(&self.file, rustix::fs::Mode::from_raw_mode(0o700)) + .map_err(|error| Error::Io(error.into()))?; + self.verify_private() + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn restrict_private(&self) -> Result<()> { + let _ = self; + Err(Error::InvalidInput( + "secure private marker directory is unsupported".into(), + )) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + fn verify_private(&self) -> Result<()> { + use rustix::fs::{fstat, FileType}; + + let metadata = fstat(&self.file).map_err(|error| Error::Io(error.into()))?; + if FileType::from_raw_mode(metadata.st_mode) != FileType::Directory + || metadata.st_mode & 0o777 != 0o700 + || metadata.st_uid != rustix::process::geteuid().as_raw() + { + return Err(Error::InvalidInput( + "secure quarantine directory is not private".into(), + )); + } + Ok(()) + } +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub(crate) fn file_identity(file: &File) -> Result<(u64, u64)> { + use rustix::fs::{fstat, FileType}; + + let metadata = fstat(file).map_err(|error| Error::Io(error.into()))?; + if FileType::from_raw_mode(metadata.st_mode) != FileType::RegularFile { + return Err(Error::InvalidInput( + "secure filesystem deletion authority is not a regular file".into(), + )); + } + descriptor_identity_from_stat(metadata) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub(crate) fn lock_observer_writer(file: &File) -> Result<()> { + rustix::fs::flock(file, rustix::fs::FlockOperation::NonBlockingLockExclusive) + .map_err(|error| Error::Io(error.into())) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub(crate) fn lock_observer_writer(file: &File) -> Result<()> { + let _ = file; + Err(Error::InvalidInput( + "observer writer descriptor locking is unsupported".into(), + )) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub(crate) fn try_lock_observer_quiescence(file: &File) -> Result<()> { + rustix::fs::flock(file, rustix::fs::FlockOperation::NonBlockingLockExclusive) + .map_err(|error| Error::Io(error.into())) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub(crate) fn try_lock_observer_quiescence(file: &File) -> Result<()> { + let _ = file; + Err(Error::InvalidInput( + "observer quiescence descriptor locking is unsupported".into(), + )) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn descriptor_identity(file: &File) -> Result<(u64, u64)> { + let metadata = rustix::fs::fstat(file).map_err(|error| Error::Io(error.into()))?; + descriptor_identity_from_stat(metadata) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn descriptor_identity_from_stat(metadata: rustix::fs::Stat) -> Result<(u64, u64)> { + Ok(( + u64::try_from(metadata.st_dev) + .map_err(|_| Error::Corrupt("negative filesystem device identity".into()))?, + u64::try_from(metadata.st_ino) + .map_err(|_| Error::Corrupt("negative filesystem inode identity".into()))?, + )) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub(crate) fn file_identity(file: &File) -> Result<(u64, u64)> { + let _ = file; + Err(Error::InvalidInput( + "secure filesystem file identity is unsupported".into(), + )) +} + +fn validate_leaf(name: &str) -> Result<()> { + let mut components = Path::new(name).components(); + if !matches!( + (components.next(), components.next()), + (Some(Component::Normal(_)), None) + ) || name.contains(['/', '\0']) + { + return Err(Error::InvalidInput(format!( + "secure filesystem leaf is not confined: `{name}`" + ))); + } + Ok(()) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn verify_entry_identity(parent: &File, name: &str, opened: &File, directory: bool) -> Result<()> { + use rustix::fs::{fstat, statat, AtFlags, FileType}; + + let path_stat = statat(parent, Path::new(name), AtFlags::SYMLINK_NOFOLLOW) + .map_err(|error| Error::Io(error.into()))?; + let opened_stat = fstat(opened).map_err(|error| Error::Io(error.into()))?; + let expected_type = if directory { + FileType::Directory + } else { + FileType::RegularFile + }; + if FileType::from_raw_mode(path_stat.st_mode) != expected_type + || FileType::from_raw_mode(opened_stat.st_mode) != expected_type + || path_stat.st_dev != opened_stat.st_dev + || path_stat.st_ino != opened_stat.st_ino + { + return Err(Error::InvalidInput(format!( + "secure filesystem entry changed while opening `{name}`" + ))); + } + Ok(()) +} + +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] +mod tests { + use super::{install_verified_unlink_inner_window_hook, SecureDirectory}; + + #[test] + fn verified_unlink_never_deletes_an_inner_window_replacement() { + let root = tempfile::tempdir().unwrap(); + let root_path = root.path().canonicalize().unwrap(); + let original = root_path.join("segment.cplq"); + let retained = root_path.join("authorized.retained.cplq"); + std::fs::write(&original, b"authenticated original\n").unwrap(); + let directory = SecureDirectory::open_absolute(&root_path).unwrap(); + let opened = directory.open_regular("segment.cplq").unwrap(); + let hook_original = original.clone(); + let hook_retained = retained.clone(); + install_verified_unlink_inner_window_hook(move || { + std::fs::rename(&hook_original, &hook_retained).unwrap(); + std::fs::write(&hook_original, b"hostile replacement\n").unwrap(); + }); + + let result = directory.unlink_verified_regular("segment.cplq", &opened); + + assert!( + retained.exists(), + "the authenticated inode was not retained" + ); + assert!( + original.exists(), + "pathname unlink deleted the inner-window replacement" + ); + assert!( + result.is_err(), + "verified pathname unlink must fail closed when no inode-bound primitive exists" + ); + } +} diff --git a/trail/src/db/change_ledger/snapshot.rs b/trail/src/db/change_ledger/snapshot.rs new file mode 100644 index 0000000..2c57209 --- /dev/null +++ b/trail/src/db/change_ledger/snapshot.rs @@ -0,0 +1,1508 @@ +//! Fenced changed-path command snapshots. +//! +//! This module owns the authoritative command state machine so status, diff, +//! and record cannot accidentally assemble a partially-fenced result. + +use super::types::{trail_atomic_temp_target, trail_case_probe_token}; +use super::{ + BaselineIdentity, CandidateSnapshot, EvidenceAcknowledgementToken, EvidenceCut, ExpectedScope, + IntentEvidence, +}; +use crate::db::storage::file_kind_from_index; +use crate::db::{DiskFile, DiskManifest, OperationMetricsDelta}; +use crate::model::{FileDiffSummary, FileEntry}; +use crate::model::{Operation, RefRecord}; +use crate::{ObjectId, Result}; +use rusqlite::params; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::OnceLock; + +static COMMAND_AUTHORITY_OVERRIDE: AtomicBool = AtomicBool::new(false); + +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub(crate) const LEDGER_AUTHORITY_ENABLED: bool = true; +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub(crate) const LEDGER_AUTHORITY_ENABLED: bool = false; + +fn checked_activation_enabled() -> bool { + static CHECKED: OnceLock = OnceLock::new(); + *CHECKED.get_or_init(|| { + let platform = if cfg!(target_os = "linux") { + "linux" + } else if cfg!(target_os = "macos") { + "macos" + } else { + "unsupported" + }; + super::ActivationEvidence::from_checked_build().is_ok_and(|evidence| { + super::ledger_authority_enabled_for(platform, &evidence) && LEDGER_AUTHORITY_ENABLED + }) + }) +} + +/// Release builds activate only on qualified native platforms. Debug builds +/// remain explicitly opt-in so parallel tests cannot accidentally share +/// production authority through process-global state. +pub(crate) fn command_authority_enabled() -> bool { + if cfg!(debug_assertions) { + checked_activation_enabled() && COMMAND_AUTHORITY_OVERRIDE.load(Ordering::Acquire) + } else { + checked_activation_enabled() + } +} + +#[cfg(debug_assertions)] +pub(crate) fn set_command_authority_override(enabled: bool) { + COMMAND_AUTHORITY_OVERRIDE.store(enabled, Ordering::Release); +} + +#[derive(Clone, Debug)] +pub(crate) struct ObservedRecordCut { + pub(crate) expected: ExpectedScope, + pub(crate) c1: EvidenceCut, + pub(crate) c2: EvidenceCut, + pub(crate) acknowledgement_tokens: Vec, +} + +#[derive(Clone, Debug)] +pub(crate) struct FencedCandidateSnapshot { + pub(crate) candidates: CandidateSnapshot, + pub(crate) c2: EvidenceCut, +} + +#[derive(Clone, Debug)] +pub(crate) struct CandidateComparison { + pub(crate) selections: Vec, + pub(crate) baseline_files: BTreeMap, + pub(crate) disk_manifest: BTreeMap, + pub(crate) disk_files: Option>, + pub(crate) summaries: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CandidateMaterialization { + ManifestOnly, + RecordBytes, +} + +impl CandidateMaterialization { + fn retains_bytes(self) -> bool { + matches!(self, Self::RecordBytes) + } +} + +fn sparse_selection_intersects(selection: &[String], candidate: &str) -> bool { + selection.iter().any(|selected| { + candidate == selected + || candidate + .strip_prefix(selected) + .is_some_and(|rest| rest.starts_with('/')) + || selected + .strip_prefix(candidate) + .is_some_and(|rest| rest.starts_with('/')) + }) +} + +impl crate::Trail { + pub(crate) fn filter_controlled_internal_candidates( + &self, + policy: &super::CompiledPolicy, + snapshot: &mut CandidateSnapshot, + evidence: &IntentEvidence, + ) -> Result<()> { + let intended = evidence + .exact_paths + .iter() + .map(|path| path.as_str()) + .collect::>(); + let internal = snapshot + .acknowledgement_tokens + .iter() + .filter_map(|token| { + if trail_case_probe_token(token) + || trail_atomic_temp_target(token) + .as_deref() + .is_some_and(|target| intended.contains(target)) + { + Some(token.path.as_str().to_string()) + } else { + None + } + }) + .collect::>(); + if internal.is_empty() { + return Ok(()); + } + let baseline_internal = self.load_root_files_for_paths( + &snapshot.expected.baseline_root, + &internal.iter().cloned().collect::>(), + )?; + let pinned = self.open_pinned_worktree_root(policy)?; + if self.pinned_worktree_root_identity(&pinned) != snapshot.expected.filesystem_identity { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: snapshot.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "pinned workspace root changed before internal candidate filtering".into(), + command: "trail status".into(), + }); + } + let mut retained = Vec::with_capacity(snapshot.exact_paths.len()); + for path in std::mem::take(&mut snapshot.exact_paths) { + if internal.contains(path.as_str()) + && !baseline_internal.contains_key(path.as_str()) + && self + .read_pinned_candidate_path(&pinned, policy, path.as_str(), false)? + .is_none() + { + continue; + } + retained.push(path); + } + snapshot.exact_paths = retained; + if !self.verify_pinned_worktree_root(&pinned)? { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: snapshot.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "pinned workspace root changed during internal candidate filtering".into(), + command: "trail status".into(), + }); + } + Ok(()) + } + + pub(crate) fn compare_controlled_projection_target( + &self, + policy: &super::CompiledPolicy, + snapshot: &CandidateSnapshot, + target_root: &ObjectId, + materialization: CandidateMaterialization, + ) -> Result { + if snapshot.expected.policy_fingerprint != policy.fingerprint() { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: snapshot.expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "controlled projection policy changed before pinned verification".into(), + command: "trail status".into(), + }); + } + // The immutable target root was prebuilt before the intent. Candidate + // evidence and filesystem identity remain bound to the original scope; + // only the comparison baseline is substituted to prove the applied + // bytes equal that target before c2. + let mut target_snapshot = snapshot.clone(); + target_snapshot.expected.baseline_root = target_root.clone(); + self.compare_authoritative_candidates( + policy, + &target_snapshot, + target_root, + materialization, + ) + } + + pub(crate) fn compare_authoritative_candidates( + &self, + policy: &super::CompiledPolicy, + snapshot: &CandidateSnapshot, + root_id: &ObjectId, + materialization: CandidateMaterialization, + ) -> Result { + if snapshot.expected.baseline_root != *root_id + || snapshot.expected.policy_fingerprint != policy.fingerprint() + { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: snapshot.expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "candidate comparison baseline or compiled policy changed".into(), + command: "trail status".into(), + }); + } + let matcher = policy.recording_matcher()?; + let pinned = self.open_pinned_worktree_root(policy)?; + if self.pinned_worktree_root_identity(&pinned) != snapshot.expected.filesystem_identity { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: snapshot.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "pinned workspace root identity changed before candidate comparison".into(), + command: "trail status".into(), + }); + } + let transient_case_probes = snapshot + .acknowledgement_tokens + .iter() + .filter(|token| trail_case_probe_token(token)) + .map(|token| token.path.as_str().to_string()) + .collect::>(); + let baseline_case_probes = self.load_root_files_for_paths( + root_id, + &transient_case_probes.iter().cloned().collect::>(), + )?; + let mut exact = BTreeSet::new(); + for path in &snapshot.exact_paths { + if matcher.is_ignored(path.as_str(), false)? { + continue; + } + if transient_case_probes.contains(path.as_str()) + && !baseline_case_probes.contains_key(path.as_str()) + && self + .read_pinned_candidate_path(&pinned, policy, path.as_str(), false)? + .is_none() + { + continue; + } + exact.insert(path.as_str().to_string()); + } + let prefixes = snapshot + .prefixes + .iter() + .filter(|prefix| prefix.complete) + .map(|prefix| prefix.path.as_str().to_string()) + .filter_map(|prefix| match matcher.is_ignored(&prefix, false) { + Ok(false) => Some(Ok(prefix)), + Ok(true) => None, + Err(error) => Some(Err(error)), + }) + .collect::>>()?; + exact.retain(|path| { + !prefixes.iter().any(|prefix| { + path == prefix + || path + .strip_prefix(prefix) + .is_some_and(|rest| rest.starts_with('/')) + }) + }); + let mut selections = exact.iter().cloned().collect::>(); + selections.extend(prefixes.iter().cloned()); + selections.sort(); + selections.dedup(); + self.note_operation_metrics(OperationMetricsDelta { + authoritative_candidate_count: selections.len().try_into().unwrap_or(u64::MAX), + ..OperationMetricsDelta::default() + }); + let baseline_files = self.load_root_files_for_selections(root_id, &selections)?; + if selections.is_empty() { + if !self.verify_pinned_worktree_root(&pinned)? { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: snapshot.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "workspace root identity changed during candidate comparison".into(), + command: "trail status".into(), + }); + } + return Ok(CandidateComparison { + selections, + baseline_files, + disk_manifest: BTreeMap::new(), + disk_files: materialization.retains_bytes().then(Vec::new), + summaries: Vec::new(), + }); + } + let mut disk_manifest = BTreeMap::new(); + let mut disk_files = materialization.retains_bytes().then(Vec::new); + for path in exact { + if matcher.is_ignored(&path, false)? { + continue; + } + if let Some(file) = self.read_pinned_candidate_path( + &pinned, + policy, + &path, + materialization.retains_bytes(), + )? { + let bytes = file.bytes; + if let Some(disk_files) = disk_files.as_mut() { + disk_files.push(DiskFile { + path: path.clone(), + bytes: bytes.ok_or_else(|| { + crate::Error::Corrupt( + "record candidate read omitted captured contents".into(), + ) + })?, + executable: file.executable, + }); + } else if bytes.is_some() { + return Err(crate::Error::Corrupt( + "manifest-only candidate comparison retained file contents".into(), + )); + } + disk_manifest.insert( + path.clone(), + DiskManifest { + kind: file_kind_from_index(&file.file_kind)?, + executable: file.executable, + content_hash: file.content_hash, + }, + ); + } + } + self.visit_pinned_worktree_prefix_files( + &pinned, + &matcher, + &prefixes, + materialization.retains_bytes(), + |file| { + let bytes = file.bytes; + if let Some(disk_files) = disk_files.as_mut() { + disk_files.push(DiskFile { + path: file.path.clone(), + bytes: bytes.ok_or_else(|| { + crate::Error::Corrupt( + "record prefix read omitted captured contents".into(), + ) + })?, + executable: file.executable, + }); + } else if bytes.is_some() { + return Err(crate::Error::Corrupt( + "manifest-only prefix comparison retained file contents".into(), + )); + } + disk_manifest.insert( + file.path.clone(), + DiskManifest { + kind: file_kind_from_index(&file.file_kind)?, + executable: file.executable, + content_hash: file.content_hash, + }, + ); + Ok(()) + }, + )?; + if !self.verify_pinned_worktree_root(&pinned)? { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: snapshot.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "workspace root identity changed during candidate comparison".into(), + command: "trail status".into(), + }); + } + let summaries = self.diff_file_maps_to_manifest_for_paths( + &baseline_files, + &disk_manifest, + &selections, + )?; + Ok(CandidateComparison { + selections, + baseline_files, + disk_manifest, + disk_files, + summaries, + }) + } +} + +#[cfg(debug_assertions)] +pub(crate) fn run_command_flow() -> std::result::Result<(), String> { + run_command_flow_inner(false) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_command_long_lock_flow() -> std::result::Result<(), String> { + run_command_flow_inner(true) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_materialized_lane_snapshot_flow() -> std::result::Result<(), String> { + use crate::{InitImportMode, Trail}; + use sha2::{Digest, Sha256}; + use std::fs; + + let temp = tempfile::tempdir().map_err(|error| error.to_string())?; + fs::write(temp.path().join("tracked.txt"), b"baseline\n").map_err(|error| error.to_string())?; + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false) + .map_err(|error| error.to_string())?; + let mut db = Trail::open(temp.path()).map_err(|error| error.to_string())?; + let lane = db + .spawn_lane("ledger-lane", Some("main"), true, None, None) + .map_err(|error| error.to_string())?; + let workdir = std::path::PathBuf::from( + lane.workdir + .ok_or_else(|| "test lane was not materialized".to_string())?, + ); + + // Make the same path diverge in opposite directions. A comparison that + // accidentally pins Trail's primary workspace will report the workspace + // hash instead of the lane hash. + fs::write(temp.path().join("tracked.txt"), b"workspace-only\n") + .map_err(|error| error.to_string())?; + fs::write(workdir.join("tracked.txt"), b"lane-only\n").map_err(|error| error.to_string())?; + let (comparison, _) = db + .compare_materialized_lane_candidates("ledger-lane", CandidateMaterialization::ManifestOnly) + .map_err(|error| format!("lane authoritative comparison: {error}"))?; + let lane_hash = hex::encode(Sha256::digest(b"lane-only\n")); + let observed = comparison + .disk_manifest + .get("tracked.txt") + .map(|entry| entry.content_hash.as_str()); + if observed != Some(lane_hash.as_str()) { + return Err(format!( + "lane snapshot used the wrong pinned root: expected {lane_hash}, got {observed:?}" + )); + } + + // Missing marker is reconciliation input, never a clean shortcut. + fs::remove_file(workdir.join(".trail/workdir-manifest.json")) + .map_err(|error| error.to_string())?; + let (after_missing_marker, _) = db + .compare_materialized_lane_candidates("ledger-lane", CandidateMaterialization::ManifestOnly) + .map_err(|error| format!("missing-marker reconciliation: {error}"))?; + if after_missing_marker + .disk_manifest + .get("tracked.txt") + .map(|entry| entry.content_hash.as_str()) + != Some(lane_hash.as_str()) + { + return Err("missing-marker reconciliation lost the lane candidate".into()); + } + + // Forced sync implementations may replace the directory inode at the + // registered path. The old watcher/tail must be discarded and a new epoch + // reconciled against the replacement root before any clean decision. + let replaced = workdir.with_extension("replaced-root"); + fs::rename(&workdir, &replaced).map_err(|error| error.to_string())?; + fs::create_dir(&workdir).map_err(|error| error.to_string())?; + fs::create_dir(workdir.join(".trail")).map_err(|error| error.to_string())?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(workdir.join(".trail"), fs::Permissions::from_mode(0o700)) + .map_err(|error| error.to_string())?; + } + fs::write(workdir.join("tracked.txt"), b"lane-only\n").map_err(|error| error.to_string())?; + let (after_root_replacement, _) = db + .compare_materialized_lane_candidates("ledger-lane", CandidateMaterialization::ManifestOnly) + .map_err(|error| format!("replacement-root reconciliation: {error}"))?; + if after_root_replacement + .disk_manifest + .get("tracked.txt") + .map(|entry| entry.content_hash.as_str()) + != Some(lane_hash.as_str()) + { + return Err("replacement-root reconciliation used the retired watcher root".into()); + } + Ok(()) +} + +#[cfg(debug_assertions)] +pub(crate) fn run_materialized_candidate_lifecycle_flow() -> std::result::Result<(), String> { + use crate::db::OperationMetricsState; + use crate::{InitImportMode, PatchDocument, PatchEdit, Trail}; + use std::fs; + use std::sync::{Arc, Mutex, OnceLock}; + + static FLOW: OnceLock> = OnceLock::new(); + let _guard = FLOW + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + struct OverrideReset; + impl Drop for OverrideReset { + fn drop(&mut self) { + set_command_authority_override(false); + } + } + + let temp = tempfile::tempdir().map_err(|error| error.to_string())?; + for index in 0..100 { + let directory = temp.path().join(format!("pkg_{:05}", index / 100)); + fs::create_dir_all(&directory).map_err(|error| error.to_string())?; + fs::write( + directory.join(format!("module_{:03}.rs", index % 100)), + format!("baseline {index}\n"), + ) + .map_err(|error| error.to_string())?; + } + fs::write( + temp.path().join(".trail-case-probe-4242-a"), + "tracked probe-shaped file\n", + ) + .map_err(|error| error.to_string())?; + fs::write( + temp.path().join("pkg_00000/.module_000.rs.trail-tmp-4242"), + "tracked temp-shaped file\n", + ) + .map_err(|error| error.to_string())?; + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false) + .map_err(|error| error.to_string())?; + let mut db = Trail::open(temp.path()).map_err(|error| error.to_string())?; + let lane = db + .spawn_lane("candidate-lifecycle", Some("main"), true, None, None) + .map_err(|error| error.to_string())?; + let workdir = std::path::PathBuf::from( + lane.workdir + .ok_or_else(|| "candidate lifecycle lane was not materialized".to_string())?, + ); + db.compare_materialized_lane_candidates( + "candidate-lifecycle", + CandidateMaterialization::ManifestOnly, + ) + .map_err(|error| format!("warm materialized candidate runtime: {error}"))?; + set_command_authority_override(true); + let _reset = OverrideReset; + let metrics = Arc::new(OperationMetricsState::default()); + db.operation_metrics = Some(Arc::clone(&metrics)); + + let scope_id: String = db + .conn + .query_row( + "SELECT scope_id FROM changed_path_scopes + WHERE scope_kind='materialized_lane' AND owner_id=?1", + [&lane.lane_id], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + let assert_internal_residue = |db: &Trail, expected: i64, stage: &str| { + let count: i64 = db + .conn + .query_row( + "SELECT COUNT(*) FROM changed_path_entries + WHERE scope_id=?1 AND ( + normalized_path GLOB '.trail-case-probe-[0-9]*-a' + OR normalized_path GLOB '*.trail-tmp-[0-9]*' + )", + [&scope_id], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + if count != expected { + let rows: String = db + .conn + .query_row( + "SELECT COALESCE(GROUP_CONCAT( + normalized_path || ':' || first_sequence || '-' || last_sequence || ':' || event_flags, + ','), '') FROM changed_path_entries WHERE scope_id=?1 AND ( + normalized_path GLOB '.trail-case-probe-[0-9]*-a' + OR normalized_path GLOB '*.trail-tmp-[0-9]*' + )", + [&scope_id], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + return Err(format!( + "{stage}: expected {expected} internal candidate rows, found {count}: {rows}" + )); + } + Ok(()) + }; + let assert_candidates = + |metrics: &OperationMetricsState, operation: &str, expected: u64, stage: &str| { + let report = metrics.last_report(); + if report.operation != operation || report.authoritative_candidate_count != expected { + return Err(format!( + "{stage}: expected {} candidates for {}, got {} for {}", + expected, operation, report.authoritative_candidate_count, report.operation, + )); + } + Ok(()) + }; + + for count in [0usize, 1, 100] { + for index in 0..count { + let path = workdir.join(format!( + "pkg_{:05}/module_{:03}.rs", + index / 100, + index % 100 + )); + use std::io::Write as _; + writeln!( + fs::OpenOptions::new() + .append(true) + .open(path) + .map_err(|error| error.to_string())?, + "// record {count}:{index}" + ) + .map_err(|error| error.to_string())?; + } + db.record_lane_workdir( + "candidate-lifecycle", + Some(format!("candidate record {count}")), + ) + .map_err(|error| format!("record k={count}: {error}"))?; + assert_candidates( + &metrics, + "materialized_lane_record", + count as u64, + &format!("record k={count}"), + )?; + assert_internal_residue(&db, 0, &format!("record k={count}"))?; + } + + for count in [0usize, 1, 100] { + let edits = (0..count) + .map(|index| { + let relative = format!("pkg_{:05}/module_{:03}.rs", index / 100, index % 100); + let content = fs::read_to_string(workdir.join(&relative))?; + Ok(PatchEdit::Write { + path: relative, + content: format!("{content}// patch {count}:{index}\n"), + executable: false, + }) + }) + .collect::>>() + .map_err(|error| error.to_string())?; + db.apply_lane_patch( + "candidate-lifecycle", + PatchDocument { + base_change: None, + message: Some(format!("candidate patch {count}")), + session_id: None, + allow_ignored: false, + allow_stale: true, + edits, + }, + ) + .map_err(|error| format!("patch k={count}: {error}"))?; + assert_candidates( + &metrics, + "structured_patch", + count as u64, + &format!("patch k={count}"), + )?; + assert_internal_residue(&db, 0, &format!("patch k={count}"))?; + } + + fs::remove_file(workdir.join(".trail-case-probe-4242-a")).map_err(|error| error.to_string())?; + fs::rename( + workdir.join("pkg_00000/.module_000.rs.trail-tmp-4242"), + workdir.join("pkg_00000/module_000.rs"), + ) + .map_err(|error| error.to_string())?; + let adversarial = db.apply_lane_patch( + "candidate-lifecycle", + PatchDocument { + base_change: None, + message: Some("tracked control-shaped deletion must remain visible".into()), + session_id: None, + allow_ignored: false, + allow_stale: true, + edits: vec![PatchEdit::Write { + path: "pkg_00000/module_000.rs".into(), + content: "adversarial target\n".into(), + executable: false, + }], + }, + ); + if !matches!( + adversarial, + Err(crate::Error::ChangeLedgerReconcileRequired { .. }) + ) { + return Err(format!( + "tracked control-shaped deletions were not rejected: {adversarial:?}" + )); + } + let retained: i64 = db + .conn + .query_row( + "SELECT COUNT(*) FROM changed_path_entries WHERE scope_id=?1 + AND normalized_path IN( + '.trail-case-probe-4242-a', + 'pkg_00000/.module_000.rs.trail-tmp-4242' + )", + [&scope_id], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + if retained != 2 { + return Err(format!( + "tracked control-shaped evidence was consumed; retained={retained}" + )); + } + if db + .debug_persisted_materialized_case_insensitive(&workdir) + .map_err(|error| format!("read persisted case sensitivity: {error}"))? + .is_none() + { + return Err("exact workdir identity did not reuse persisted case sensitivity".into()); + } + let retired_workdir = workdir.with_extension("candidate-lifecycle-retired"); + fs::rename(&workdir, &retired_workdir).map_err(|error| error.to_string())?; + fs::create_dir(&workdir).map_err(|error| error.to_string())?; + if db + .debug_persisted_materialized_case_insensitive(&workdir) + .map_err(|error| format!("check replacement workdir identity: {error}"))? + .is_some() + { + return Err("replacement workdir reused stale persisted case sensitivity".into()); + } + Ok(()) +} + +#[cfg(debug_assertions)] +fn run_command_flow_inner(long_record_lock: bool) -> std::result::Result<(), String> { + use crate::model::Actor; + use crate::{InitImportMode, Trail}; + use sha2::{Digest, Sha256}; + use std::fs; + use std::sync::{Mutex, OnceLock}; + + static COMMAND_FLOW: OnceLock> = OnceLock::new(); + let _guard = COMMAND_FLOW + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + struct OverrideReset; + impl Drop for OverrideReset { + fn drop(&mut self) { + set_command_authority_override(false); + } + } + + let temp = tempfile::tempdir().map_err(|error| error.to_string())?; + Trail::init(temp.path(), "main", InitImportMode::Empty, false) + .map_err(|error| error.to_string())?; + fs::create_dir(temp.path().join("tree")).map_err(|error| error.to_string())?; + fs::write(temp.path().join("tree/nested.txt"), vec![b'x'; 256 * 1024]) + .map_err(|error| error.to_string())?; + fs::write(temp.path().join("tracked.txt"), b"baseline\n").map_err(|error| error.to_string())?; + let mut db = Trail::open(temp.path()).map_err(|error| error.to_string())?; + db.record(None, Some("baseline".into()), Actor::human(), false) + .map_err(|error| error.to_string())?; + super::prepare_workspace_daemon(&mut db, false).map_err(|error| error.to_string())?; + set_command_authority_override(true); + let _reset = OverrideReset; + + let (scope_id, provider_id): (String, String) = db + .conn + .query_row( + "SELECT scope_id,provider_id FROM changed_path_scopes + WHERE scope_kind='workspace'", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(|error| error.to_string())?; + db.conn + .execute( + "INSERT INTO changed_path_prefixes( + scope_id,normalized_prefix,completeness_reason,event_flags,source_mask, + first_sequence,last_sequence,provider_id,provider_sequence,created_at,updated_at + ) VALUES(?1,'tree','provider_complete',?2,?3,1,1,?4,1, + strftime('%s','now'),strftime('%s','now'))", + params![ + scope_id, + super::EvidenceFlags::PROVIDER_COMPLETE_PREFIX.0, + super::EvidenceSource::Observer.mask(), + provider_id, + ], + ) + .map_err(|error| error.to_string())?; + + let (metadata_only, _) = db + .with_workspace_authoritative_snapshot(|db, policy, candidates| { + db.compare_authoritative_candidates( + policy, + candidates, + &candidates.expected.baseline_root, + CandidateMaterialization::ManifestOnly, + ) + }) + .map_err(|error| format!("metadata-only complete-prefix comparison: {error}"))?; + if metadata_only.disk_files.is_some() + || !metadata_only.disk_manifest.contains_key("tree/nested.txt") + { + return Err("metadata-only complete-prefix comparison retained content bytes".into()); + } + let (record_bytes, _) = db + .with_workspace_authoritative_snapshot(|db, policy, candidates| { + db.compare_authoritative_candidates( + policy, + candidates, + &candidates.expected.baseline_root, + CandidateMaterialization::RecordBytes, + ) + }) + .map_err(|error| format!("record-byte complete-prefix comparison: {error}"))?; + let retained = record_bytes.disk_files.as_ref().ok_or_else(|| { + "record-byte complete-prefix comparison omitted its explicit materialization".to_string() + })?; + if retained.len() != 1 || retained[0].path != "tree/nested.txt" { + return Err("record-byte complete-prefix comparison retained the wrong path set".into()); + } + + let consume_attempts = std::cell::Cell::new(0_u8); + db.with_workspace_authoritative_snapshot(|_, _, candidates| { + let attempt = consume_attempts.get().saturating_add(1); + consume_attempts.set(attempt); + if attempt == 1 { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "command_test_consumer_retry".into(), + command: "trail status".into(), + }); + } + Ok(()) + }) + .map_err(|error| format!("consumer reconciliation retry: {error}"))?; + if consume_attempts.get() != 2 { + return Err("consumer reconciliation did not retry the entire fenced flow once".into()); + } + + let clean = db + .status_from_changed_path_ledger() + .map_err(|error| format!("initial authoritative status: {error}"))?; + if !clean.changed_paths.is_empty() { + return Err("empty authoritative candidate set was not clean".into()); + } + db.conn + .execute( + "UPDATE changed_path_scopes SET trust_state='untrusted_gap', + trust_reason='command_test_auto_reconcile', + continuity_generation=continuity_generation+1", + [], + ) + .map_err(|error| error.to_string())?; + let recovered = db + .status_from_changed_path_ledger() + .map_err(|error| format!("automatic reconciliation retry: {error}"))?; + if !recovered.changed_paths.is_empty() { + return Err("automatic reconciliation retry did not restore clean authority".into()); + } + fs::write(temp.path().join("tracked.txt"), b"changed\n").map_err(|error| error.to_string())?; + let dirty = db + .status_from_changed_path_ledger() + .map_err(|error| format!("dirty authoritative status: {error}"))?; + if dirty.changed_paths.len() != 1 || dirty.changed_paths[0].path != "tracked.txt" { + return Err(format!( + "authoritative candidate comparison missed tracked change: {:?}", + dirty + .changed_paths + .iter() + .map(|item| item.path.as_str()) + .collect::>() + )); + } + let changed_after_comparison = temp.path().join("tracked.txt"); + crate::db::install_observed_record_after_compare_hook(move || { + fs::write(changed_after_comparison, b"changed-after-comparison\n")?; + Ok(()) + }); + if long_record_lock { + let changed_with_lock = temp.path().join("tracked.txt"); + crate::db::install_observed_record_with_lock_hook(move || { + fs::write(changed_with_lock, b"changed-while-record-lock-held\n")?; + std::thread::sleep(std::time::Duration::from_millis(5_500)); + Ok(()) + }); + } + let recorded = db + .record(None, Some("observed".into()), Actor::human(), false) + .map_err(|error| format!("observed record: {error}"))?; + if recorded.operation.is_none() || recorded.changed_paths.len() != 1 { + return Err("observed record did not publish the candidate change".into()); + } + let recorded_files = db + .load_root_files_for_paths(&recorded.root_id, &["tracked.txt".to_string()]) + .map_err(|error| format!("load observed record root: {error}"))?; + let recorded_hash = recorded_files + .get("tracked.txt") + .map(|entry| entry.content_hash.as_str()); + let compared_hash = hex::encode(Sha256::digest(b"changed\n")); + if recorded_hash != Some(compared_hash.as_str()) { + return Err("observed record did not use the authorized pinned candidate bytes".into()); + } + let after_race = db + .status_from_changed_path_ledger() + .map_err(|error| format!("post-record authoritative status: {error}"))?; + if after_race.changed_paths.len() != 1 || after_race.changed_paths[0].path != "tracked.txt" { + return Err("post-comparison mutation was not retained as c2 evidence".into()); + } + let recorded_after_race = db + .record( + None, + Some("observed-after-race".into()), + Actor::human(), + false, + ) + .map_err(|error| format!("observed record after comparison race: {error}"))?; + if recorded_after_race.operation.is_none() || recorded_after_race.changed_paths.len() != 1 { + return Err("second observed record did not publish retained c2 evidence".into()); + } + let after = db + .status_from_changed_path_ledger() + .map_err(|error| format!("final post-record authoritative status: {error}"))?; + if !after.changed_paths.is_empty() { + return Err("atomic observed record did not acknowledge unchanged evidence".into()); + } + Ok(()) +} + +impl crate::Trail { + pub(crate) fn filter_controlled_lane_sparse_candidates( + &self, + lane: &str, + policy: &super::CompiledPolicy, + candidates: &mut CandidateSnapshot, + ) -> crate::Result>> { + let branch = self.lane_branch(lane)?; + let workdir = std::path::PathBuf::from(branch.workdir.as_deref().ok_or_else(|| { + crate::Error::InvalidInput(format!( + "lane `{}` does not have a materialized workdir", + branch.lane_id + )) + })?); + let declared_sparse = self + .lane_sparse_paths_from_metadata(&branch.lane_id)? + .is_some_and(|paths| !paths.is_empty()); + let selection = self.authenticated_lane_sparse_selection(&workdir)?; + if declared_sparse && selection.is_none() { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "sparse lane selection metadata is missing".into(), + command: format!("trail lane status {}", branch.lane_id), + }); + } + let Some(selected) = selection.as_ref() else { + return Ok(selection); + }; + let pinned = self.open_pinned_worktree_root(policy)?; + if self.pinned_worktree_root_identity(&pinned) != candidates.expected.filesystem_identity { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "sparse lane root identity changed before candidate filtering".into(), + command: format!("trail lane status {}", branch.lane_id), + }); + } + let mut exact_paths = Vec::new(); + for path in std::mem::take(&mut candidates.exact_paths) { + if sparse_selection_intersects(selected, path.as_str()) + || self.pinned_worktree_path_is_visible(&pinned, path.as_str())? + { + exact_paths.push(path); + } + } + candidates.exact_paths = exact_paths; + let mut prefixes = Vec::new(); + for prefix in std::mem::take(&mut candidates.prefixes) { + if sparse_selection_intersects(selected, prefix.path.as_str()) + || self.pinned_worktree_path_is_visible(&pinned, prefix.path.as_str())? + { + prefixes.push(prefix); + } + } + candidates.prefixes = prefixes; + if !self.verify_pinned_worktree_root(&pinned)? { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "sparse lane root identity changed during candidate filtering".into(), + command: format!("trail lane status {}", branch.lane_id), + }); + } + Ok(selection) + } + + pub(crate) fn compare_materialized_lane_candidates( + &self, + lane: &str, + materialization: CandidateMaterialization, + ) -> crate::Result<(CandidateComparison, FencedCandidateSnapshot)> { + let branch = self.lane_branch(lane)?; + let workdir = branch.workdir.as_deref().ok_or_else(|| { + crate::Error::InvalidInput(format!( + "lane `{}` does not have a materialized workdir", + branch.lane_id + )) + })?; + let workdir = std::path::PathBuf::from(workdir); + let declared_sparse = self + .lane_sparse_paths_from_metadata(&branch.lane_id)? + .is_some_and(|paths| !paths.is_empty()); + let selection_used = std::cell::RefCell::new(None::>>); + let result = + self.with_materialized_lane_authoritative_snapshot(lane, |db, policy, candidates| { + let sparse_selection = db.authenticated_lane_sparse_selection(&workdir)?; + if declared_sparse && sparse_selection.is_none() { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "sparse lane selection metadata is missing".into(), + command: format!("trail lane status {}", branch.lane_id), + }); + } + selection_used.replace(Some(sparse_selection.clone())); + let mut candidates = candidates.clone(); + if let Some(selection) = sparse_selection.as_ref() { + // Full reconciliation deliberately notices baseline files that + // are absent on disk. In a sparse lane, absence outside the + // authenticated materialized selection is not a deletion. Keep + // selected/intersecting paths plus any currently visible path + // (a newly created file outside the selection), and discard + // only baseline-only absences. + let pinned = db.open_pinned_worktree_root(policy)?; + if db.pinned_worktree_root_identity(&pinned) + != candidates.expected.filesystem_identity + { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "sparse lane root identity changed before candidate filtering" + .into(), + command: format!("trail lane status {}", branch.lane_id), + }); + } + let mut exact_paths = Vec::new(); + for path in std::mem::take(&mut candidates.exact_paths) { + if sparse_selection_intersects(selection, path.as_str()) + || db.pinned_worktree_path_is_visible(&pinned, path.as_str())? + { + exact_paths.push(path); + } + } + candidates.exact_paths = exact_paths; + let mut prefixes = Vec::new(); + for prefix in std::mem::take(&mut candidates.prefixes) { + if sparse_selection_intersects(selection, prefix.path.as_str()) + || db.pinned_worktree_path_is_visible(&pinned, prefix.path.as_str())? + { + prefixes.push(prefix); + } + } + candidates.prefixes = prefixes; + if !db.verify_pinned_worktree_root(&pinned)? { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "sparse lane root identity changed during candidate filtering" + .into(), + command: format!("trail lane status {}", branch.lane_id), + }); + } + candidates.acknowledgement_tokens.retain(|token| { + candidates + .exact_paths + .iter() + .any(|path| path == &token.path) + || candidates + .prefixes + .iter() + .any(|prefix| prefix.path == token.path) + }); + } + db.compare_authoritative_candidates( + policy, + &candidates, + &candidates.expected.baseline_root, + materialization, + ) + })?; + let current_selection = self.authenticated_lane_sparse_selection(&workdir)?; + if selection_used.into_inner() != Some(current_selection) { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: result.1.candidates.expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "sparse lane selection changed during authoritative snapshot".into(), + command: format!("trail lane status {}", branch.lane_id), + }); + } + Ok(result) + } + + pub(crate) fn with_workspace_authoritative_snapshot( + &self, + mut consume: F, + ) -> crate::Result<(T, FencedCandidateSnapshot)> + where + F: FnMut(&crate::Trail, &super::CompiledPolicy, &CandidateSnapshot) -> crate::Result, + { + let daemon_missing = self + .changed_path_daemon_registry + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .workspace + .is_none(); + if daemon_missing { + super::prepare_workspace_daemon(self, true)?; + } + let mut runtime = self + .changed_path_daemon_registry + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .workspace + .take() + .ok_or_else(|| { + crate::Error::DaemonUnavailable( + "changed-path observer runtime is unavailable".into(), + ) + })?; + let mut result = runtime.with_authoritative_snapshot(self, &mut consume); + if result + .as_ref() + .err() + .is_some_and(super::policy_runtime_restart_required) + { + drop(runtime); + super::prepare_workspace_daemon(self, true)?; + runtime = self + .changed_path_daemon_registry + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .workspace + .take() + .ok_or_else(|| { + crate::Error::DaemonUnavailable( + "restarted changed-path observer runtime is unavailable".into(), + ) + })?; + result = runtime.with_authoritative_snapshot(self, &mut consume); + } + self.changed_path_daemon_registry + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .workspace = Some(runtime); + result + } + + /// Consume an exact c1 -> compare -> c2 snapshot rooted at one qualified + /// materialized lane. Missing, stale, or structurally invalid marker state + /// is never interpreted as clean: it forces a full lane reconciliation and + /// a newly bound marker before the candidate set may be consumed. + pub(crate) fn with_materialized_lane_authoritative_snapshot( + &self, + lane: &str, + mut consume: F, + ) -> crate::Result<(T, FencedCandidateSnapshot)> + where + F: FnMut(&crate::Trail, &super::CompiledPolicy, &CandidateSnapshot) -> crate::Result, + { + let lane_id = self.lane_branch(lane)?.lane_id; + self.ensure_materialized_lane_snapshot_authority(&lane_id)?; + let branch = self.lane_branch(&lane_id)?; + let workdir = branch.workdir.as_deref().ok_or_else(|| { + crate::Error::InvalidInput(format!( + "lane `{}` does not have a materialized workdir", + branch.lane_id + )) + })?; + let sparse_selection_fingerprint = + self.materialized_lane_sparse_selection_fingerprint(std::path::Path::new(workdir))?; + let mut runtime = self + .changed_path_daemon_registry + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .materialized_lanes + .remove(&lane_id) + .ok_or_else(|| { + crate::Error::DaemonUnavailable(format!( + "changed-path observer runtime for materialized lane `{lane_id}` is unavailable" + )) + })?; + let mut result = runtime.with_authoritative_snapshot(self, &mut consume); + if result + .as_ref() + .err() + .is_some_and(super::policy_runtime_restart_required) + { + drop(runtime); + super::prepare_materialized_lane_daemon(self, &lane_id, true)?; + runtime = self + .changed_path_daemon_registry + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .materialized_lanes + .remove(&lane_id) + .ok_or_else(|| { + crate::Error::DaemonUnavailable(format!( + "restarted changed-path observer runtime for lane `{lane_id}` is unavailable" + )) + })?; + result = runtime.with_authoritative_snapshot(self, &mut consume); + } + self.changed_path_daemon_registry + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .materialized_lanes + .insert(lane_id.clone(), runtime); + // Keep the compact marker bound to c2. It remains only a restart hint; + // every clean claim still authenticates a fresh native fence. + // Publish the exact selection binding captured before c1. If the + // selection changes concurrently, validation sees a fingerprint + // mismatch and forces reconciliation instead of accepting false-clean + // candidates for a different sparse view. + let marker_result = + self.publish_lane_marker_for_sparse_selection(&lane_id, sparse_selection_fingerprint); + match (result, marker_result) { + (Ok(snapshot), Ok(())) => Ok(snapshot), + (Err(error), _) => Err(error), + (_, Err(error)) => Err(error), + } + } + + fn ensure_materialized_lane_snapshot_authority(&self, lane: &str) -> crate::Result<()> { + let branch = self.lane_branch(lane)?; + let workdir = branch.workdir.as_deref().ok_or_else(|| { + crate::Error::InvalidInput(format!( + "lane `{}` does not have a materialized workdir", + branch.lane_id + )) + })?; + let head = self.get_ref(&branch.ref_name)?; + let runtime_present = self + .changed_path_daemon_registry + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .materialized_lanes + .contains_key(&branch.lane_id); + let runtime_matches = runtime_present + && super::materialized_lane_daemon_matches_target(self, &branch.lane_id)?; + if runtime_present && !runtime_matches { + // A forced sync may atomically replace the root inode. Drop the old + // watcher before starting a new epoch against the replacement root; + // reusing its retained tail would falsely bridge two filesystems. + self.changed_path_daemon_registry + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .materialized_lanes + .remove(&branch.lane_id); + } + if !runtime_matches { + if runtime_present { + super::daemon::prepare_materialized_lane_daemon_verified_replacement( + self, + &branch.lane_id, + )?; + } else { + super::prepare_materialized_lane_daemon(self, &branch.lane_id, true)?; + } + } + let valid = self + .validated_materialized_lane_marker_v2(&branch, std::path::Path::new(workdir), &head)? + .is_some(); + if valid { + // Authenticate continuity and the current sidecar/owner chain. A + // persisted SQL `trusted` bit or marker alone is never clean + // authority. + if let Err(error) = super::materialized_lane_daemon_fence(self, &branch.lane_id) { + if !matches!(error, crate::Error::ChangeLedgerReconcileRequired { .. }) { + return Err(error); + } + if let Err(error) = super::materialized_lane_daemon_reconcile( + self, + &branch.lane_id, + "materialized_lane_authority_recovery", + ) { + if matches!( + error, + crate::Error::DaemonUnavailable(_) + | crate::Error::ChangeLedgerReconcileRequired { .. } + ) { + self.restart_verified_materialized_lane_runtime(&branch.lane_id)?; + } else { + return Err(error); + } + } + } + } else { + if let Err(error) = super::materialized_lane_daemon_reconcile( + self, + &branch.lane_id, + "materialized_lane_marker_reconciliation", + ) { + if matches!( + error, + crate::Error::DaemonUnavailable(_) + | crate::Error::ChangeLedgerReconcileRequired { .. } + ) { + self.restart_verified_materialized_lane_runtime(&branch.lane_id)?; + } else { + return Err(error); + } + } + // Reconciliation establishes the authoritative cut; publish its + // compact marker only after the runtime has been returned to the + // registry so the marker can bind the exact segment ID and cut. + self.publish_lane_marker_if_materialized(&branch.lane_id)?; + let branch = self.lane_branch(&branch.lane_id)?; + let head = self.get_ref(&branch.ref_name)?; + let workdir = branch.workdir.as_deref().ok_or_else(|| { + crate::Error::InvalidInput(format!( + "lane `{}` lost its materialized workdir during reconciliation", + branch.lane_id + )) + })?; + if self + .validated_materialized_lane_marker_v2( + &branch, + std::path::Path::new(workdir), + &head, + )? + .is_none() + { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: super::materialized_lane_scope_id( + &self.config.workspace.id.0, + &branch.lane_id, + ) + .to_text(), + state: "untrusted_gap".into(), + reason: + "materialized lane marker remained unqualified after full reconciliation" + .into(), + command: format!("trail lane status {}", branch.lane_id), + }); + } + } + Ok(()) + } + + fn restart_verified_materialized_lane_runtime(&self, lane: &str) -> crate::Result<()> { + self.changed_path_daemon_registry + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .materialized_lanes + .remove(lane); + super::daemon::prepare_materialized_lane_daemon_verified_replacement(self, lane)?; + Ok(()) + } + + /// Publish an observed workspace record as one SQLite transaction. The + /// immutable operation object is written first, but remains unreachable if + /// any index/ref/baseline/ack/lane CAS fails. The ref mirror and in-memory + /// daemon baseline are repaired only after commit. + pub(crate) fn commit_observed_record( + &mut self, + operation: &Operation, + expected_ref: &RefRecord, + observed: &ObservedRecordCut, + acknowledge_complete_prefixes: bool, + lane_id: Option<&str>, + ) -> Result { + if operation.before_root.as_ref() != Some(&observed.expected.baseline_root) + || operation.parents.as_slice() != [expected_ref.change_id.clone()] + || expected_ref.name != observed.expected.ref_name + || u64::try_from(expected_ref.generation).ok() != Some(observed.expected.ref_generation) + || observed.c1.sequence > observed.c2.sequence + { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: observed.expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "observed record does not match its fenced baseline and cuts".into(), + command: "trail status".into(), + }); + } + let (operation, operation_id) = self.store_operation_object_unindexed(operation)?; + self.conn.execute_batch("BEGIN IMMEDIATE;")?; + let transaction_result = (|| { + self.index_operation_in_transaction(&operation, &operation_id)?; + self.advance_ref_cas_in_transaction( + expected_ref, + &operation.change_id, + &operation.after_root, + &operation_id, + )?; + let touched = super::ChangedPathLedger::acknowledge_immutable_tokens_in_transaction( + &self.conn, + &observed.expected, + &observed.c2, + observed.c1.sequence, + &observed.acknowledgement_tokens, + acknowledge_complete_prefixes, + )?; + let changed = self.conn.execute( + "UPDATE changed_path_scopes SET ref_generation=?1,change_id=?2, + baseline_root_id=?3,updated_at=strftime('%s','now') + WHERE scope_id=?4 AND epoch=?5 AND ref_name=?6 AND ref_generation=?7 + AND baseline_root_id=?8 AND policy_fingerprint=?9 + AND policy_dependency_generation=?10 AND filesystem_identity=?11 + AND provider_identity=?12 AND trust_state='trusted' + AND durable_offset=?13 AND folded_offset=?13", + params![ + expected_ref.generation + 1, + operation.change_id.0, + operation.after_root.0, + observed.expected.scope_id.to_text(), + i64::try_from(observed.expected.epoch) + .map_err(|_| crate::Error::InvalidInput("scope epoch overflow".into()))?, + observed.expected.ref_name, + expected_ref.generation, + observed.expected.baseline_root.0, + hex::encode(observed.expected.policy_fingerprint), + i64::try_from(observed.expected.policy_generation).map_err(|_| { + crate::Error::InvalidInput("policy generation overflow".into()) + })?, + hex::encode(&observed.expected.filesystem_identity), + hex::encode(&observed.expected.provider_identity), + i64::try_from(observed.c2.durable_offset) + .map_err(|_| crate::Error::InvalidInput("observer cut overflow".into()))?, + ], + )?; + if changed != 1 { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: observed.expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "observed record lost the latest c2 scope CAS".into(), + command: "trail status".into(), + }); + } + if let Some(lane_id) = lane_id { + let changed = self.conn.execute( + "UPDATE lane_branches SET head_change=?1,head_root=?2,updated_at=strftime('%s','now') + WHERE lane_id=?3 AND ref_name=?4 AND head_change=?5 AND head_root=?6", + params![ + operation.change_id.0, + operation.after_root.0, + lane_id, + expected_ref.name, + expected_ref.change_id.0, + expected_ref.root_id.0, + ], + )?; + if changed != 1 { + return Err(crate::Error::StaleBranch(expected_ref.name.clone())); + } + } + self.note_operation_metrics(OperationMetricsDelta { + ledger_row_touch_count: touched, + ..OperationMetricsDelta::default() + }); + Ok(()) + })(); + match transaction_result { + Ok(()) => self.conn.execute_batch("COMMIT;")?, + Err(error) => { + let _ = self.conn.execute_batch("ROLLBACK;"); + return Err(error); + } + } + self.repair_ref_mirror( + expected_ref, + &operation.change_id, + &operation.after_root, + &operation_id, + ) + .map_err(|error| crate::Error::CommittedRepairRequired { + operation: operation_id.0.clone(), + repair: "ref mirror".into(), + reason: error.to_string(), + })?; + let target = BaselineIdentity { + ref_name: expected_ref.name.clone(), + ref_generation: u64::try_from(expected_ref.generation + 1) + .map_err(|_| crate::Error::InvalidInput("ref generation overflow".into()))?, + change_id: operation.change_id.clone(), + root_id: operation.after_root.clone(), + }; + // Materialized-lane runtime repair is wrapper-owned so a committed + // observed operation has exactly one post-commit transition. Workspace + // observed recording has no wrapper repair closure and remains local. + if lane_id.is_none() { + if let Some(runtime) = self + .changed_path_daemon_registry + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .workspace + .as_mut() + { + runtime + .accept_observed_baseline(&observed.expected, &target) + .map_err(|error| crate::Error::CommittedRepairRequired { + operation: operation_id.0.clone(), + repair: "workspace observer runtime".into(), + reason: error.to_string(), + })?; + } + } + Ok(operation_id) + } +} diff --git a/trail/src/db/change_ledger/store.rs b/trail/src/db/change_ledger/store.rs new file mode 100644 index 0000000..8aa814b --- /dev/null +++ b/trail/src/db/change_ledger/store.rs @@ -0,0 +1,2261 @@ +use rusqlite::{params, Connection, OptionalExtension}; + +use super::types::*; +use crate::db::util::now_ts; +use crate::error::{Error, Result}; + +pub(crate) struct ChangedPathLedger<'a> { + pub(super) conn: &'a Connection, + database_path: Option, +} + +#[derive(Debug)] +struct ScopeRow { + state: TrustState, + reason: String, + durable_offset: u64, + folded_offset: u64, + max_candidate_rows: u64, + max_prefix_rows: u64, +} + +impl<'a> ChangedPathLedger<'a> { + #[cfg(test)] + pub(crate) fn new(conn: &'a Connection) -> Self { + Self { + conn, + database_path: None, + } + } + + pub(crate) fn new_at(conn: &'a Connection, database_path: &std::path::Path) -> Self { + Self { + conn, + database_path: Some(database_path.to_path_buf()), + } + } + + pub(crate) fn database_path(&self) -> Result<&std::path::Path> { + self.database_path.as_deref().ok_or_else(|| { + Error::InvalidInput("changed-path database path authority is unavailable".into()) + }) + } + + pub(crate) fn begin_scope( + &self, + identity: &ScopeIdentity, + baseline: &BaselineIdentity, + policy: &PolicyIdentity, + filesystem: &FilesystemIdentity, + provider: &ProviderIdentity, + ) -> Result { + if identity.owner_id.is_empty() { + return Err(Error::InvalidInput( + "changed-path scope owner cannot be empty".into(), + )); + } + let now = now_ts(); + let scope_id = identity.scope_id.to_text(); + let filesystem_identity = hex::encode(&filesystem.0); + let provider_identity = hex::encode(&provider.identity); + let policy_fingerprint = hex::encode(policy.fingerprint); + self.conn.execute( + "INSERT INTO changed_path_scopes( + scope_id, schema_version, scope_kind, owner_id, + scope_root, scope_root_identity, filesystem_identity, + filesystem_kind, case_sensitive, ref_name, ref_generation, + change_id, baseline_root_id, policy_fingerprint, + policy_dependency_generation, trust_state, trust_reason, epoch, + provider_id, provider_identity, durable_cursor, + linearizable_fence, rename_pairing, overflow_scope, + filesystem_supported, clean_proof_allowed, + power_loss_durability, created_at, updated_at + ) VALUES( + ?1, 1, ?2, ?3, '', ?4, ?4, 'unknown', 1, ?5, ?6, ?7, ?8, + ?9, ?10, 'untrusted_gap', 'initial_reconciliation_required', 1, + ?11, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?19 + )", + params![ + scope_id, + identity.kind.as_str(), + identity.owner_id, + filesystem_identity, + baseline.ref_name, + sql_u64(baseline.ref_generation, "ref generation")?, + baseline.change_id.0, + baseline.root_id.0, + policy_fingerprint, + sql_u64(policy.generation, "policy generation")?, + provider_identity, + bool_sql(provider.capabilities.durable_cursor), + bool_sql(provider.capabilities.linearizable_fence), + bool_sql(provider.capabilities.rename_pairing), + bool_sql(provider.capabilities.overflow_scope), + bool_sql(provider.capabilities.filesystem_supported), + bool_sql(provider.capabilities.clean_proof_allowed), + bool_sql(provider.capabilities.power_loss_durability), + now, + ], + )?; + Ok(identity.scope_id) + } + + pub(crate) fn mark_prefix_dirty( + &self, + expected: &ExpectedScope, + prefix: &DirtyPrefix, + ) -> Result<()> { + if !prefix.complete { + return Err(Error::InvalidInput( + "incomplete dirty prefixes cannot be persisted as authoritative evidence".into(), + )); + } + if prefix.reason.is_empty() { + return Err(Error::InvalidInput( + "dirty prefix completeness reason cannot be empty".into(), + )); + } + if prefix.first_sequence > prefix.last_sequence { + return Err(Error::InvalidInput( + "dirty prefix first sequence exceeds last sequence".into(), + )); + } + self.recover_scope(expected)?; + + let mut tx = self.conn.unchecked_transaction()?; + let scope = evidence_write_guard(&tx, expected)?; + let mut savepoint = tx.savepoint()?; + let scope_id = expected.scope_id.to_text(); + let incoming = prefix.path.as_str(); + let mut statement = savepoint.prepare( + "SELECT normalized_prefix, completeness_reason, source_mask, + first_sequence, last_sequence + FROM changed_path_prefixes + WHERE scope_id = ?1 + AND ( + normalized_prefix = ?2 COLLATE BINARY + OR ( + normalized_prefix >= (?2 || '/') COLLATE BINARY + AND normalized_prefix < (?2 || '0') COLLATE BINARY + ) + OR ( + ?2 >= (normalized_prefix || '/') COLLATE BINARY + AND ?2 < (normalized_prefix || '0') COLLATE BINARY + ) + ) + ORDER BY normalized_prefix COLLATE BINARY", + )?; + let overlaps = statement + .query_map(params![scope_id, incoming], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + )) + })? + .collect::, _>>()?; + drop(statement); + + let ancestor = overlaps + .iter() + .filter(|(path, ..)| component_prefix(path, incoming)) + .min_by_key(|(path, ..)| path.len()); + let (merged_path, merged_reason) = ancestor + .map(|(path, reason, ..)| (path.clone(), reason.clone())) + .unwrap_or_else(|| (incoming.to_string(), prefix.reason.clone())); + let mut first_sequence = prefix.first_sequence; + let mut last_sequence = prefix.last_sequence; + let mut source_mask = EvidenceSource::Reconciliation.mask(); + for row in &overlaps { + first_sequence = first_sequence.min(db_u64(row.3, "prefix first sequence")?); + last_sequence = last_sequence.max(db_u64(row.4, "prefix last sequence")?); + source_mask |= row.2; + } + for (path, ..) in &overlaps { + savepoint.execute( + "DELETE FROM changed_path_prefixes + WHERE scope_id = ?1 AND normalized_prefix = ?2 COLLATE BINARY", + params![scope_id, path], + )?; + } + let now = now_ts(); + savepoint.execute( + "INSERT INTO changed_path_prefixes( + scope_id, normalized_prefix, completeness_reason, event_flags, + source_mask, first_sequence, last_sequence, provider_id, + provider_sequence, intent_id, created_at, updated_at + ) VALUES(?1, ?2, ?3, 0, ?4, ?5, ?6, 'reconciliation', NULL, NULL, ?7, ?7)", + params![ + scope_id, + merged_path, + merged_reason, + source_mask, + sql_u64(first_sequence, "prefix first sequence")?, + sql_u64(last_sequence, "prefix last sequence")?, + now, + ], + )?; + let overflowed = caps_exceeded(&savepoint, &scope, &scope_id)?; + if overflowed { + savepoint.rollback()?; + } + savepoint.commit()?; + if overflowed { + mark_scope_overflow(&tx, expected)?; + tx.commit()?; + return Err(reconcile_error( + expected, + TrustState::Overflow, + "persisted evidence cap exceeded", + )); + } + tx.commit()?; + Ok(()) + } + + pub(crate) fn mark_untrusted( + &self, + expected: &ExpectedScope, + state: TrustState, + reason: &str, + ) -> Result<()> { + if state == TrustState::Trusted { + return Err(Error::InvalidInput( + "mark_untrusted cannot promote a scope to trusted".into(), + )); + } + self.recover_scope(expected)?; + let encoded = EncodedExpected::new(expected)?; + let changed = self.conn.execute( + "UPDATE changed_path_scopes + SET trust_state = ?1, trust_reason = ?2, + continuity_generation = continuity_generation + 1, updated_at = ?3 + WHERE scope_id = ?4 AND epoch = ?5 AND ref_name = ?6 + AND ref_generation = ?7 AND baseline_root_id = ?8 + AND policy_fingerprint = ?9 + AND policy_dependency_generation = ?10 + AND filesystem_identity = ?11 AND provider_identity = ?12", + params![ + state.as_str(), + reason, + now_ts(), + encoded.scope_id, + encoded.epoch, + expected.ref_name, + encoded.ref_generation, + expected.baseline_root.0, + encoded.policy_fingerprint, + encoded.policy_generation, + encoded.filesystem_identity, + encoded.provider_identity, + ], + )?; + if changed == 0 { + return Err(stale_cas_error(self.conn, expected)); + } + Ok(()) + } + + pub(crate) fn snapshot_candidates( + &self, + expected: &ExpectedScope, + ) -> Result { + self.recover_scope(expected)?; + self.snapshot_candidates_in_transaction(expected, None, || {}) + } + + /// Read candidates while one exact live controlled intent is Prepared. + /// This narrowly avoids treating that caller-owned intent as crash + /// residue; every ordinary snapshot still runs full recovery first. + pub(crate) fn snapshot_candidates_for_controlled_intent( + &self, + expected: &ExpectedScope, + intent_id: &str, + start_cursor: Option<&[u8]>, + ) -> Result { + self.snapshot_candidates_in_transaction(expected, Some((intent_id, start_cursor)), || {}) + } + + #[cfg(test)] + fn snapshot_candidates_with_phase_hook( + &self, + expected: &ExpectedScope, + after_scope_read: F, + ) -> Result + where + F: FnOnce(), + { + self.snapshot_candidates_in_transaction(expected, None, after_scope_read) + } + + fn snapshot_candidates_in_transaction( + &self, + expected: &ExpectedScope, + controlled_intent: Option<(&str, Option<&[u8]>)>, + after_scope_read: F, + ) -> Result + where + F: FnOnce(), + { + let tx = self.conn.unchecked_transaction()?; + if let Some((intent_id, start_cursor)) = controlled_intent { + let exact: bool = tx.query_row( + "SELECT EXISTS( + SELECT 1 FROM changed_path_intents intent + JOIN changed_path_scopes scope ON scope.scope_id=intent.scope_id + WHERE intent.intent_id=?1 AND intent.scope_id=?2 + AND intent.expected_scope_epoch=?3 + AND intent.expected_ref_name=?4 + AND intent.expected_ref_generation=?5 + AND intent.expected_root_id=?6 + AND intent.expected_change_id=scope.change_id + AND scope.epoch=?3 AND scope.ref_name=?4 + AND scope.ref_generation=?5 AND scope.baseline_root_id=?6 + AND intent.lifecycle_state='prepared' AND intent.start_cursor IS ?7 + )", + params![ + intent_id, + expected.scope_id.to_text(), + i64::try_from(expected.epoch) + .map_err(|_| Error::InvalidInput("intent epoch overflow".into()))?, + expected.ref_name, + i64::try_from(expected.ref_generation).map_err(|_| Error::InvalidInput( + "intent ref generation overflow".into() + ))?, + expected.baseline_root.0, + start_cursor, + ], + |row| row.get(0), + )?; + if !exact { + return Err(Error::Conflict(format!( + "controlled intent `{intent_id}` changed before its fenced candidate read" + ))); + } + } + let scope = load_scope(&tx, expected)?; + if scope.state != TrustState::Trusted { + return Err(reconcile_error(expected, scope.state, &scope.reason)); + } + if scope.durable_offset != scope.folded_offset { + return Err(reconcile_error( + expected, + TrustState::UntrustedGap, + "durable observer evidence is not fully folded", + )); + } + after_scope_read(); + let scope_id = expected.scope_id.to_text(); + let exact_count = row_count(&tx, "changed_path_entries", &scope_id)?; + let prefix_count = row_count(&tx, "changed_path_prefixes", &scope_id)?; + if exact_count > scope.max_candidate_rows || prefix_count > scope.max_prefix_rows { + return Err(reconcile_error( + expected, + TrustState::Overflow, + "persisted evidence cap exceeded", + )); + } + + let exact_values = tx + .prepare( + "SELECT normalized_path,event_flags,source_mask,first_sequence,last_sequence, + provider_id,provider_sequence,intent_id + FROM changed_path_entries + WHERE scope_id = ?1 ORDER BY normalized_path COLLATE BINARY", + )? + .query_map([&scope_id], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + )) + })? + .collect::, _>>()?; + let exact_paths = exact_values + .iter() + .map(|row| LedgerPath::parse(&row.0)) + .collect::>>()?; + + let prefix_values = tx + .prepare( + "SELECT normalized_prefix,completeness_reason,event_flags,source_mask, + first_sequence,last_sequence,provider_id,provider_sequence,intent_id + FROM changed_path_prefixes + WHERE scope_id = ?1 ORDER BY normalized_prefix COLLATE BINARY", + )? + .query_map([&scope_id], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, Option>(8)?, + )) + })? + .collect::, _>>()?; + let prefixes = prefix_values + .iter() + .map( + |(path, reason, _flags, _source, first, last, _provider, _sequence, _intent)| { + Ok(DirtyPrefix { + path: LedgerPath::parse(path)?, + complete: true, + reason: reason.clone(), + first_sequence: db_u64(*first, "prefix first sequence")?, + last_sequence: db_u64(*last, "prefix last sequence")?, + }) + }, + ) + .collect::>>()?; + let mut acknowledgement_tokens = exact_values + .into_iter() + .map( + |( + path, + flags, + source_mask, + first, + last, + provider_id, + provider_sequence, + intent_id, + )| { + Ok(EvidenceAcknowledgementToken { + kind: EvidenceRowKind::Exact, + path: LedgerPath::parse(&path)?, + flags: EvidenceFlags(flags), + source_mask, + first_sequence: db_u64(first, "entry first sequence")?, + last_sequence: db_u64(last, "entry last sequence")?, + provider_id, + provider_sequence: provider_sequence + .map(|value| db_u64(value, "entry provider sequence")) + .transpose()?, + intent_id, + }) + }, + ) + .collect::>>()?; + acknowledgement_tokens.extend( + prefix_values + .into_iter() + .map( + |( + path, + _reason, + flags, + source_mask, + first, + last, + provider_id, + provider_sequence, + intent_id, + )| { + Ok(EvidenceAcknowledgementToken { + kind: EvidenceRowKind::CompletePrefix, + path: LedgerPath::parse(&path)?, + flags: EvidenceFlags(flags), + source_mask, + first_sequence: db_u64(first, "prefix first sequence")?, + last_sequence: db_u64(last, "prefix last sequence")?, + provider_id, + provider_sequence: provider_sequence + .map(|value| db_u64(value, "prefix provider sequence")) + .transpose()?, + intent_id, + }) + }, + ) + .collect::>>()?, + ); + let sequence = tx.query_row( + "SELECT COALESCE(MAX(provider_sequence), 0) + FROM ( + SELECT provider_sequence FROM changed_path_entries + WHERE scope_id = ?1 AND (source_mask & ?2) != 0 + AND provider_sequence IS NOT NULL + UNION ALL + SELECT provider_sequence FROM changed_path_prefixes + WHERE scope_id = ?1 AND (source_mask & ?2) != 0 + AND provider_sequence IS NOT NULL + )", + params![scope_id, EvidenceSource::Observer.mask()], + |row| row.get::<_, i64>(0), + )?; + let snapshot = CandidateSnapshot { + expected: expected.clone(), + cut: EvidenceCut { + source: EvidenceSource::Observer, + sequence: db_u64(sequence, "observer sequence")?, + durable_offset: scope.durable_offset, + folded_offset: scope.folded_offset, + }, + exact_paths, + prefixes, + acknowledgement_tokens, + trust: TrustState::Trusted, + }; + tx.commit()?; + Ok(snapshot) + } + + pub(crate) fn acknowledge( + &self, + expected: &ExpectedScope, + cut: &EvidenceCut, + owned: &OwnedEvidence, + ) -> Result<()> { + if cut.source != owned.source { + return Err(Error::InvalidInput( + "evidence cut and ownership source must match".into(), + )); + } + if owned.prefixes.iter().any(|prefix| !prefix.complete) { + return Err(Error::InvalidInput( + "incomplete prefixes cannot be acknowledged as authoritative".into(), + )); + } + self.recover_scope(expected)?; + let through = cut.sequence.min(owned.through_sequence); + let tx = self.conn.unchecked_transaction()?; + cas_guard(&tx, expected, true)?; + let scope = load_scope(&tx, expected)?; + validate_cut_boundaries(expected, &scope, cut)?; + let scope_id = expected.scope_id.to_text(); + for path in &owned.exact_paths { + tx.execute( + "DELETE FROM changed_path_entries + WHERE scope_id = ?1 AND normalized_path = ?2 COLLATE BINARY + AND source_mask = ?3 + AND ( + (?3 = ?5 AND provider_sequence IS NOT NULL + AND provider_sequence <= ?4) + OR (?3 != ?5 AND last_sequence <= ?4) + )", + params![ + scope_id, + path.as_str(), + owned.source.mask(), + sql_u64(through, "acknowledgement sequence")?, + EvidenceSource::Observer.mask(), + ], + )?; + } + for prefix in &owned.prefixes { + tx.execute( + "DELETE FROM changed_path_prefixes + WHERE scope_id = ?1 AND normalized_prefix = ?2 COLLATE BINARY + AND source_mask = ?3 + AND ( + (?3 = ?5 AND provider_sequence IS NOT NULL + AND provider_sequence <= ?4) + OR (?3 != ?5 AND last_sequence <= ?4) + )", + params![ + scope_id, + prefix.path.as_str(), + owned.source.mask(), + sql_u64(through, "acknowledgement sequence")?, + EvidenceSource::Observer.mask(), + ], + )?; + } + tx.commit()?; + Ok(()) + } + + pub(crate) fn acknowledge_immutable_tokens_in_transaction( + conn: &Connection, + expected: &ExpectedScope, + latest_cut: &EvidenceCut, + through_sequence: u64, + tokens: &[EvidenceAcknowledgementToken], + acknowledge_complete_prefixes: bool, + ) -> Result { + cas_guard(conn, expected, true)?; + let scope = load_scope(conn, expected)?; + validate_cut_boundaries(expected, &scope, latest_cut)?; + let mut deleted = 0u64; + for token in tokens { + // A mixed-source row or a row advanced beyond c1 cannot be proved + // fully represented by the observed record. + let source_is_acknowledgeable = if token.source_mask == EvidenceSource::Observer.mask() + { + token + .provider_sequence + .is_some_and(|sequence| sequence <= through_sequence) + } else if token.source_mask == EvidenceSource::Intent.mask() { + token.provider_sequence.is_none() && token.last_sequence <= through_sequence + } else { + false + }; + if !source_is_acknowledgeable + || (token.kind == EvidenceRowKind::CompletePrefix && !acknowledge_complete_prefixes) + { + continue; + } + let table = match token.kind { + EvidenceRowKind::Exact => "changed_path_entries", + EvidenceRowKind::CompletePrefix => "changed_path_prefixes", + }; + let path_column = match token.kind { + EvidenceRowKind::Exact => "normalized_path", + EvidenceRowKind::CompletePrefix => "normalized_prefix", + }; + let sql = format!( + "DELETE FROM {table} WHERE scope_id=?1 AND {path_column}=?2 COLLATE BINARY + AND event_flags=?3 AND source_mask=?4 AND first_sequence=?5 + AND last_sequence=?6 AND provider_id IS ?7 + AND provider_sequence IS ?8 AND intent_id IS ?9" + ); + deleted = deleted.saturating_add(conn.execute( + &sql, + params![ + expected.scope_id.to_text(), + token.path.as_str(), + token.flags.0, + token.source_mask, + sql_u64(token.first_sequence, "acknowledgement first sequence")?, + sql_u64(token.last_sequence, "acknowledgement last sequence")?, + token.provider_id, + token + .provider_sequence + .map(|value| sql_u64(value, "acknowledgement provider sequence")) + .transpose()?, + token.intent_id, + ], + )? as u64); + } + Ok(deleted) + } + + pub(crate) fn advance_baseline( + &self, + expected: &ExpectedScope, + target: &BaselineIdentity, + cut: &EvidenceCut, + ) -> Result<()> { + self.recover_scope(expected)?; + let tx = self.conn.unchecked_transaction()?; + cas_guard(&tx, expected, true)?; + let scope = load_scope(&tx, expected)?; + validate_cut_boundaries(expected, &scope, cut)?; + let encoded = EncodedExpected::new(expected)?; + let changed = tx.execute( + "UPDATE changed_path_scopes + SET ref_name = ?1, ref_generation = ?2, change_id = ?3, + baseline_root_id = ?4, updated_at = ?5 + WHERE scope_id = ?6 AND epoch = ?7 AND ref_name = ?8 + AND ref_generation = ?9 AND baseline_root_id = ?10 + AND policy_fingerprint = ?11 + AND policy_dependency_generation = ?12 + AND filesystem_identity = ?13 AND provider_identity = ?14 + AND trust_state = 'trusted'", + params![ + target.ref_name, + sql_u64(target.ref_generation, "target ref generation")?, + target.change_id.0, + target.root_id.0, + now_ts(), + encoded.scope_id, + encoded.epoch, + expected.ref_name, + encoded.ref_generation, + expected.baseline_root.0, + encoded.policy_fingerprint, + encoded.policy_generation, + encoded.filesystem_identity, + encoded.provider_identity, + ], + )?; + if changed == 0 { + return Err(stale_cas_error(&tx, expected)); + } + tx.commit()?; + Ok(()) + } + + pub(crate) fn upsert_exact( + &self, + expected: &ExpectedScope, + path: &LedgerPath, + flags: EvidenceFlags, + source: EvidenceSource, + sequence: u64, + ) -> Result<()> { + self.recover_scope(expected)?; + let mut tx = self.conn.unchecked_transaction()?; + let scope = evidence_write_guard(&tx, expected)?; + let mut savepoint = tx.savepoint()?; + let now = now_ts(); + let sequence = sql_u64(sequence, "source sequence")?; + let provider_sequence = (source == EvidenceSource::Observer).then_some(sequence); + savepoint.execute( + "INSERT INTO changed_path_entries( + scope_id, normalized_path, event_flags, source_mask, + first_sequence, last_sequence, provider_id, + provider_sequence, intent_id, created_at, updated_at + ) VALUES(?1, ?2, ?3, ?4, ?5, ?5, ?6, ?7, NULL, ?8, ?8) + ON CONFLICT(scope_id, normalized_path) DO UPDATE SET + event_flags = changed_path_entries.event_flags | excluded.event_flags, + source_mask = changed_path_entries.source_mask | excluded.source_mask, + first_sequence = MIN(changed_path_entries.first_sequence, excluded.first_sequence), + last_sequence = MAX(changed_path_entries.last_sequence, excluded.last_sequence), + provider_id = CASE + WHEN changed_path_entries.source_mask = excluded.source_mask + THEN excluded.provider_id ELSE NULL END, + provider_sequence = CASE + WHEN excluded.provider_sequence IS NULL + THEN changed_path_entries.provider_sequence + WHEN changed_path_entries.provider_sequence IS NULL + THEN excluded.provider_sequence + ELSE MAX(changed_path_entries.provider_sequence, excluded.provider_sequence) + END, + updated_at = excluded.updated_at", + params![ + expected.scope_id.to_text(), + path.as_str(), + flags.0, + source.mask(), + sequence, + source.as_str(), + provider_sequence, + now, + ], + )?; + let scope_id = expected.scope_id.to_text(); + let overflowed = caps_exceeded(&savepoint, &scope, &scope_id)?; + if overflowed { + savepoint.rollback()?; + } + savepoint.commit()?; + if overflowed { + mark_scope_overflow(&tx, expected)?; + tx.commit()?; + return Err(reconcile_error( + expected, + TrustState::Overflow, + "persisted evidence cap exceeded", + )); + } + tx.commit()?; + Ok(()) + } + + pub(crate) fn all_exact(&self, expected: &ExpectedScope) -> Result> { + load_scope(self.conn, expected)?; + let values = self + .conn + .prepare( + "SELECT normalized_path, event_flags, source_mask, + first_sequence, last_sequence + FROM changed_path_entries + WHERE scope_id = ?1 ORDER BY normalized_path COLLATE BINARY", + )? + .query_map([expected.scope_id.to_text()], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + )) + })? + .collect::, _>>()?; + values + .into_iter() + .map(|(path, flags, source_mask, first, last)| { + Ok(ExactEvidence { + path: LedgerPath::parse(&path)?, + flags: EvidenceFlags(flags), + source_mask, + first_sequence: db_u64(first, "entry first sequence")?, + last_sequence: db_u64(last, "entry last sequence")?, + }) + }) + .collect() + } +} + +struct EncodedExpected { + scope_id: String, + epoch: i64, + ref_generation: i64, + policy_fingerprint: String, + policy_generation: i64, + filesystem_identity: String, + provider_identity: String, +} + +impl EncodedExpected { + fn new(expected: &ExpectedScope) -> Result { + Ok(Self { + scope_id: expected.scope_id.to_text(), + epoch: sql_u64(expected.epoch, "scope epoch")?, + ref_generation: sql_u64(expected.ref_generation, "ref generation")?, + policy_fingerprint: hex::encode(expected.policy_fingerprint), + policy_generation: sql_u64(expected.policy_generation, "policy generation")?, + filesystem_identity: hex::encode(&expected.filesystem_identity), + provider_identity: hex::encode(&expected.provider_identity), + }) + } +} + +fn cas_guard(conn: &Connection, expected: &ExpectedScope, require_trusted: bool) -> Result<()> { + let encoded = EncodedExpected::new(expected)?; + let changed = conn.execute( + "UPDATE changed_path_scopes SET updated_at = updated_at + WHERE scope_id = ?1 AND epoch = ?2 AND ref_name = ?3 + AND ref_generation = ?4 AND baseline_root_id = ?5 + AND policy_fingerprint = ?6 AND policy_dependency_generation = ?7 + AND filesystem_identity = ?8 AND provider_identity = ?9 + AND (?10 = 0 OR trust_state = 'trusted')", + params![ + encoded.scope_id, + encoded.epoch, + expected.ref_name, + encoded.ref_generation, + expected.baseline_root.0, + encoded.policy_fingerprint, + encoded.policy_generation, + encoded.filesystem_identity, + encoded.provider_identity, + bool_sql(require_trusted), + ], + )?; + if changed == 0 { + return Err(stale_cas_error(conn, expected)); + } + Ok(()) +} + +fn load_scope(conn: &Connection, expected: &ExpectedScope) -> Result { + let encoded = EncodedExpected::new(expected)?; + let row = conn + .query_row( + "SELECT trust_state, trust_reason, durable_offset, folded_offset, + max_candidate_rows, max_prefix_rows + FROM changed_path_scopes + WHERE scope_id = ?1 AND epoch = ?2 AND ref_name = ?3 + AND ref_generation = ?4 AND baseline_root_id = ?5 + AND policy_fingerprint = ?6 AND policy_dependency_generation = ?7 + AND filesystem_identity = ?8 AND provider_identity = ?9", + params![ + encoded.scope_id, + encoded.epoch, + expected.ref_name, + encoded.ref_generation, + expected.baseline_root.0, + encoded.policy_fingerprint, + encoded.policy_generation, + encoded.filesystem_identity, + encoded.provider_identity, + ], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, i64>(5)?, + )) + }, + ) + .optional()?; + let Some((state, reason, durable_offset, folded_offset, max_candidate_rows, max_prefix_rows)) = + row + else { + return Err(stale_cas_error(conn, expected)); + }; + Ok(ScopeRow { + state: TrustState::parse(&state)?, + reason, + durable_offset: db_u64(durable_offset, "durable offset")?, + folded_offset: db_u64(folded_offset, "folded offset")?, + max_candidate_rows: db_u64(max_candidate_rows, "candidate row cap")?, + max_prefix_rows: db_u64(max_prefix_rows, "prefix row cap")?, + }) +} + +fn evidence_write_guard(conn: &Connection, expected: &ExpectedScope) -> Result { + cas_guard(conn, expected, false)?; + let scope = load_scope(conn, expected)?; + if matches!(scope.state, TrustState::Overflow | TrustState::Corrupt) { + return Err(reconcile_error(expected, scope.state, &scope.reason)); + } + Ok(scope) +} + +fn caps_exceeded(conn: &Connection, scope: &ScopeRow, scope_id: &str) -> Result { + let candidates = row_count(conn, "changed_path_entries", scope_id)?; + let prefixes = row_count(conn, "changed_path_prefixes", scope_id)?; + Ok(candidates > scope.max_candidate_rows || prefixes > scope.max_prefix_rows) +} + +fn mark_scope_overflow(conn: &Connection, expected: &ExpectedScope) -> Result<()> { + let encoded = EncodedExpected::new(expected)?; + let changed = conn.execute( + "UPDATE changed_path_scopes + SET trust_state = 'overflow', trust_reason = 'persisted evidence cap exceeded', + continuity_generation = continuity_generation + 1, updated_at = ?1 + WHERE scope_id = ?2 AND epoch = ?3 AND ref_name = ?4 + AND ref_generation = ?5 AND baseline_root_id = ?6 + AND policy_fingerprint = ?7 AND policy_dependency_generation = ?8 + AND filesystem_identity = ?9 AND provider_identity = ?10", + params![ + now_ts(), + encoded.scope_id, + encoded.epoch, + expected.ref_name, + encoded.ref_generation, + expected.baseline_root.0, + encoded.policy_fingerprint, + encoded.policy_generation, + encoded.filesystem_identity, + encoded.provider_identity, + ], + )?; + if changed == 0 { + return Err(stale_cas_error(conn, expected)); + } + Ok(()) +} + +fn validate_cut_boundaries( + expected: &ExpectedScope, + scope: &ScopeRow, + cut: &EvidenceCut, +) -> Result<()> { + if cut.durable_offset != scope.durable_offset || cut.folded_offset != scope.folded_offset { + return Err(reconcile_error( + expected, + TrustState::UntrustedGap, + "evidence cut does not match the persisted observer boundaries", + )); + } + Ok(()) +} + +fn row_count(conn: &Connection, table: &str, scope_id: &str) -> Result { + let sql = match table { + "changed_path_entries" => "SELECT COUNT(*) FROM changed_path_entries WHERE scope_id = ?1", + "changed_path_prefixes" => "SELECT COUNT(*) FROM changed_path_prefixes WHERE scope_id = ?1", + _ => return Err(Error::Corrupt("unknown changed-path row table".into())), + }; + let count = conn.query_row(sql, [scope_id], |row| row.get::<_, i64>(0))?; + db_u64(count, "evidence row count") +} + +fn component_prefix(prefix: &str, path: &str) -> bool { + prefix == path + || path + .strip_prefix(prefix) + .is_some_and(|remainder| remainder.starts_with('/')) +} + +fn stale_cas_error(conn: &Connection, expected: &ExpectedScope) -> Error { + let observed = conn + .query_row( + "SELECT trust_state, trust_reason FROM changed_path_scopes WHERE scope_id = ?1", + [expected.scope_id.to_text()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .optional() + .ok() + .flatten(); + let (state, reason) = observed.unwrap_or_else(|| { + ( + "missing".into(), + "scope is missing or its exact identity changed".into(), + ) + }); + reconcile_error( + expected, + TrustState::parse(&state).unwrap_or(TrustState::UntrustedGap), + &format!("stale scope CAS: {reason}"), + ) +} + +fn reconcile_error(expected: &ExpectedScope, state: TrustState, reason: &str) -> Error { + Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: state.as_str().into(), + reason: reason.into(), + command: "trail status".into(), + } +} + +fn bool_sql(value: bool) -> i64 { + i64::from(value) +} + +fn sql_u64(value: u64, label: &str) -> Result { + value + .try_into() + .map_err(|_| Error::InvalidInput(format!("{label} exceeds SQLite INTEGER range"))) +} + +fn db_u64(value: i64, label: &str) -> Result { + value + .try_into() + .map_err(|_| Error::Corrupt(format!("{label} is negative"))) +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + use rusqlite::{params, Connection}; + + use super::*; + use crate::db::{InitImportMode, Trail}; + use crate::error::Error; + use crate::{ChangeId, ObjectId}; + + #[cfg(unix)] + #[test] + fn ledger_retains_lossless_non_utf8_database_path_authority() { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + let conn = Connection::open_in_memory().unwrap(); + let path = std::path::Path::new(OsStr::from_bytes(b"/tmp/trail-\xff/index/trail.sqlite")); + let ledger = ChangedPathLedger::new_at(&conn, path); + + assert_eq!( + ledger.database_path().unwrap().as_os_str().as_bytes(), + path.as_os_str().as_bytes() + ); + } + + const FIXTURE_SCHEMA: &str = " + PRAGMA foreign_keys = ON; + CREATE TABLE changed_path_scopes ( + scope_id TEXT NOT NULL PRIMARY KEY, + schema_version INTEGER NOT NULL DEFAULT 1, + scope_kind TEXT NOT NULL, + owner_id TEXT NOT NULL, + scope_root TEXT NOT NULL, + scope_root_identity TEXT NOT NULL, + filesystem_identity TEXT NOT NULL, + filesystem_kind TEXT NOT NULL, + case_sensitive INTEGER NOT NULL, + ref_name TEXT NOT NULL, + ref_generation INTEGER NOT NULL, + change_id TEXT NOT NULL, + baseline_root_id TEXT NOT NULL, + policy_fingerprint TEXT NOT NULL, + policy_dependency_generation INTEGER NOT NULL, + trust_state TEXT NOT NULL, + trust_reason TEXT NOT NULL, + continuity_generation INTEGER NOT NULL DEFAULT 1, + epoch INTEGER NOT NULL, + provider_id TEXT, + provider_identity TEXT, + durable_cursor INTEGER NOT NULL DEFAULT 0, + linearizable_fence INTEGER NOT NULL DEFAULT 0, + rename_pairing INTEGER NOT NULL DEFAULT 0, + overflow_scope INTEGER NOT NULL DEFAULT 0, + filesystem_supported INTEGER NOT NULL DEFAULT 0, + clean_proof_allowed INTEGER NOT NULL DEFAULT 0, + power_loss_durability INTEGER NOT NULL DEFAULT 0, + durable_offset INTEGER NOT NULL DEFAULT 0, + folded_offset INTEGER NOT NULL DEFAULT 0, + max_candidate_rows INTEGER NOT NULL DEFAULT 250000, + max_prefix_rows INTEGER NOT NULL DEFAULT 16384, + max_observer_log_bytes INTEGER NOT NULL DEFAULT 268435456, + max_segment_bytes INTEGER NOT NULL DEFAULT 16777216, + max_unfolded_tail_records INTEGER NOT NULL DEFAULT 65536, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(scope_kind, owner_id) + ); + CREATE TABLE changed_path_intents ( + intent_id TEXT NOT NULL PRIMARY KEY, + schema_version INTEGER NOT NULL DEFAULT 1, + scope_id TEXT NOT NULL REFERENCES changed_path_scopes(scope_id) ON DELETE CASCADE, + producer TEXT NOT NULL, + expected_scope_epoch INTEGER NOT NULL, + expected_ref_name TEXT NOT NULL, + expected_ref_generation INTEGER NOT NULL, + expected_change_id TEXT NOT NULL, + expected_root_id TEXT NOT NULL, + target_change_id TEXT NOT NULL, + target_root_id TEXT NOT NULL, + target_operation_id TEXT, + start_cursor BLOB, + lifecycle_state TEXT NOT NULL DEFAULT 'prepared', + verified_cut BLOB, + failure_reason TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE changed_path_intent_paths ( + intent_id TEXT NOT NULL REFERENCES changed_path_intents(intent_id) ON DELETE CASCADE, + normalized_path TEXT NOT NULL, + event_flags INTEGER NOT NULL, + PRIMARY KEY(intent_id, normalized_path) + ); + CREATE TABLE changed_path_intent_prefixes ( + intent_id TEXT NOT NULL REFERENCES changed_path_intents(intent_id) ON DELETE CASCADE, + normalized_prefix TEXT NOT NULL, + completeness_reason TEXT NOT NULL, + event_flags INTEGER NOT NULL, + PRIMARY KEY(intent_id, normalized_prefix) + ); + CREATE TABLE changed_path_entries ( + scope_id TEXT NOT NULL REFERENCES changed_path_scopes(scope_id) ON DELETE CASCADE, + normalized_path TEXT COLLATE BINARY NOT NULL, + event_flags INTEGER NOT NULL, + source_mask INTEGER NOT NULL, + first_sequence INTEGER NOT NULL, + last_sequence INTEGER NOT NULL, + provider_id TEXT, + provider_sequence INTEGER, + intent_id TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY(scope_id, normalized_path) + ); + CREATE TABLE changed_path_prefixes ( + scope_id TEXT NOT NULL REFERENCES changed_path_scopes(scope_id) ON DELETE CASCADE, + normalized_prefix TEXT COLLATE BINARY NOT NULL, + completeness_reason TEXT NOT NULL, + event_flags INTEGER NOT NULL, + source_mask INTEGER NOT NULL, + first_sequence INTEGER NOT NULL, + last_sequence INTEGER NOT NULL, + provider_id TEXT, + provider_sequence INTEGER, + intent_id TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY(scope_id, normalized_prefix) + );"; + + fn scope_identity() -> ScopeIdentity { + ScopeIdentity { + scope_id: ScopeId([0xab; 32]), + kind: ScopeKind::Workspace, + owner_id: "workspace-main".into(), + } + } + + fn baseline() -> BaselineIdentity { + BaselineIdentity { + ref_name: "refs/branches/main".into(), + ref_generation: 7, + change_id: ChangeId("change-base".into()), + root_id: ObjectId("root-base".into()), + } + } + + fn policy() -> PolicyIdentity { + PolicyIdentity { + fingerprint: [0xcd; 32], + generation: 11, + } + } + + fn filesystem() -> FilesystemIdentity { + FilesystemIdentity(vec![0, 0xff, b'f', b's']) + } + + fn provider() -> ProviderIdentity { + ProviderIdentity { + identity: vec![0x80, 0, b'p'], + capabilities: ProviderCapabilities { + durable_cursor: true, + linearizable_fence: true, + rename_pairing: false, + overflow_scope: true, + filesystem_supported: true, + clean_proof_allowed: false, + power_loss_durability: true, + }, + } + } + + fn expected() -> ExpectedScope { + ExpectedScope { + scope_id: scope_identity().scope_id, + epoch: 1, + ref_name: baseline().ref_name, + ref_generation: baseline().ref_generation, + baseline_root: baseline().root_id, + policy_fingerprint: policy().fingerprint, + policy_generation: policy().generation, + filesystem_identity: filesystem().0, + provider_identity: provider().identity, + } + } + + fn fixture(max_candidates: u64, max_prefixes: u64) -> Connection { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(FIXTURE_SCHEMA).unwrap(); + ChangedPathLedger::new(&conn) + .begin_scope( + &scope_identity(), + &baseline(), + &policy(), + &filesystem(), + &provider(), + ) + .unwrap(); + conn.execute( + "UPDATE changed_path_scopes + SET max_candidate_rows = ?1, max_prefix_rows = ?2 + WHERE scope_id = ?3", + params![ + max_candidates, + max_prefixes, + scope_identity().scope_id.to_text() + ], + ) + .unwrap(); + conn + } + + fn set_trust(conn: &Connection, state: TrustState) { + conn.execute( + "UPDATE changed_path_scopes SET trust_state = ?1 WHERE scope_id = ?2", + params![state.as_str(), scope_identity().scope_id.to_text()], + ) + .unwrap(); + } + + fn prefix(path: &str, sequence: u64) -> DirtyPrefix { + DirtyPrefix { + path: LedgerPath::parse(path).unwrap(), + complete: true, + reason: format!("rescan-{path}"), + first_sequence: sequence, + last_sequence: sequence, + } + } + + #[test] + fn begin_scope_is_untrusted_and_encodes_binary_identities_canonically() { + let conn = fixture(10, 10); + let row = conn + .query_row( + "SELECT scope_id, trust_state, policy_fingerprint, + filesystem_identity, provider_identity, + durable_cursor, linearizable_fence, clean_proof_allowed + FROM changed_path_scopes", + [], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, i64>(6)?, + row.get::<_, i64>(7)?, + )) + }, + ) + .unwrap(); + assert_eq!(row.0, "ab".repeat(32)); + assert_eq!(row.1, "untrusted_gap"); + assert_eq!(row.2, "cd".repeat(32)); + assert_eq!(row.3, "00ff6673"); + assert_eq!(row.4, "800070"); + assert_eq!((row.5, row.6, row.7), (1, 1, 0)); + } + + #[test] + fn begin_scope_matches_the_fresh_v18_schema() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::Empty, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + ChangedPathLedger::new(&db.conn) + .begin_scope( + &scope_identity(), + &baseline(), + &policy(), + &filesystem(), + &provider(), + ) + .unwrap(); + assert_eq!( + db.conn + .query_row( + "SELECT trust_state FROM changed_path_scopes WHERE scope_id = ?1", + [scope_identity().scope_id.to_text()], + |row| row.get::<_, String>(0), + ) + .unwrap(), + "untrusted_gap" + ); + } + + #[test] + fn ledger_path_accepts_only_normalized_relative_slash_paths_and_preserves_case() { + assert_eq!( + LedgerPath::parse("Src/Crab.rs").unwrap().as_str(), + "Src/Crab.rs" + ); + for invalid in [ + "", + "/root", + ".", + "..", + "a/./b", + "a/../b", + "a//b", + "a/", + "a\\b", + "nul\0byte", + "e\u{301}.txt", + ] { + assert!(LedgerPath::parse(invalid).is_err(), "accepted {invalid:?}"); + } + assert!(LedgerPath::parse("é.txt").is_ok()); + } + + #[test] + fn only_trusted_scope_can_return_an_authoritative_snapshot() { + let conn = fixture(10, 10); + let ledger = ChangedPathLedger::new(&conn); + for state in [ + TrustState::Reconciling, + TrustState::Overflow, + TrustState::UntrustedGap, + TrustState::StaleBaseline, + TrustState::Corrupt, + ] { + set_trust(&conn, state); + assert!(matches!( + ledger.snapshot_candidates(&expected()), + Err(Error::ChangeLedgerReconcileRequired { .. }) + )); + } + set_trust(&conn, TrustState::Trusted); + assert_eq!( + ledger.snapshot_candidates(&expected()).unwrap().trust, + TrustState::Trusted + ); + } + + #[test] + fn snapshot_rejects_an_unfolded_durable_tail() { + let conn = fixture(10, 10); + conn.execute( + "UPDATE changed_path_scopes + SET trust_state = 'trusted', durable_offset = 8, folded_offset = 5", + [], + ) + .unwrap(); + + assert!(matches!( + ChangedPathLedger::new(&conn).snapshot_candidates(&expected()), + Err(Error::ChangeLedgerReconcileRequired { .. }) + )); + } + + #[test] + fn immutable_ack_token_retains_same_path_evidence_arriving_after_c1() { + let conn = fixture(10, 10); + set_trust(&conn, TrustState::Trusted); + let ledger = ChangedPathLedger::new(&conn); + let path = LedgerPath::parse("same.txt").unwrap(); + ledger + .upsert_exact( + &expected(), + &path, + EvidenceFlags::CONTENT, + EvidenceSource::Observer, + 1, + ) + .unwrap(); + let c1 = ledger.snapshot_candidates(&expected()).unwrap(); + ledger + .upsert_exact( + &expected(), + &path, + EvidenceFlags::MODE, + EvidenceSource::Observer, + 2, + ) + .unwrap(); + + ChangedPathLedger::acknowledge_immutable_tokens_in_transaction( + &conn, + &expected(), + &c1.cut, + c1.cut.sequence, + &c1.acknowledgement_tokens, + true, + ) + .unwrap(); + + let retained = ledger.all_exact(&expected()).unwrap(); + assert_eq!(retained.len(), 1); + assert_eq!(retained[0].last_sequence, 2); + assert_eq!( + retained[0].flags.0, + (EvidenceFlags::CONTENT | EvidenceFlags::MODE).0 + ); + } + + #[test] + fn partial_observed_record_retains_complete_prefix_and_mixed_source_rows() { + let conn = fixture(10, 10); + set_trust(&conn, TrustState::Trusted); + let scope = expected().scope_id.to_text(); + conn.execute( + "INSERT INTO changed_path_prefixes( + scope_id,normalized_prefix,completeness_reason,event_flags,source_mask, + first_sequence,last_sequence,provider_id,provider_sequence,intent_id, + created_at,updated_at + ) VALUES(?1,'src','provider_complete',?2,?3,1,1,'observer',1,NULL,1,1)", + params![ + scope, + EvidenceFlags::PROVIDER_COMPLETE_PREFIX.0, + EvidenceSource::Observer.mask() + ], + ) + .unwrap(); + let ledger = ChangedPathLedger::new(&conn); + ledger + .upsert_exact( + &expected(), + &LedgerPath::parse("mixed.txt").unwrap(), + EvidenceFlags::CONTENT, + EvidenceSource::Observer, + 1, + ) + .unwrap(); + let snapshot = ledger.snapshot_candidates(&expected()).unwrap(); + ledger + .upsert_exact( + &expected(), + &LedgerPath::parse("mixed.txt").unwrap(), + EvidenceFlags::CONTENT, + EvidenceSource::Intent, + 1, + ) + .unwrap(); + + ChangedPathLedger::acknowledge_immutable_tokens_in_transaction( + &conn, + &expected(), + &snapshot.cut, + snapshot.cut.sequence, + &snapshot.acknowledgement_tokens, + false, + ) + .unwrap(); + + assert_eq!( + conn.query_row( + "SELECT COUNT(*) FROM changed_path_prefixes WHERE scope_id=?1", + [expected().scope_id.to_text()], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1 + ); + assert_eq!(ledger.all_exact(&expected()).unwrap()[0].source_mask, 3); + } + + #[test] + fn snapshot_candidates_uses_one_read_snapshot_across_all_phases() { + let workspace = tempfile::tempdir().unwrap(); + let database = workspace.path().join("ledger.db"); + let reader = Connection::open(&database).unwrap(); + reader.execute_batch("PRAGMA journal_mode = WAL;").unwrap(); + reader.execute_batch(FIXTURE_SCHEMA).unwrap(); + ChangedPathLedger::new(&reader) + .begin_scope( + &scope_identity(), + &baseline(), + &policy(), + &filesystem(), + &provider(), + ) + .unwrap(); + let before = LedgerPath::parse("before").unwrap(); + ChangedPathLedger::new(&reader) + .upsert_exact( + &expected(), + &before, + EvidenceFlags::CONTENT, + EvidenceSource::Observer, + 1, + ) + .unwrap(); + set_trust(&reader, TrustState::Trusted); + let writer = Connection::open(&database).unwrap(); + + let snapshot = ChangedPathLedger::new(&reader) + .snapshot_candidates_with_phase_hook(&expected(), || { + ChangedPathLedger::new(&writer) + .upsert_exact( + &expected(), + &LedgerPath::parse("after").unwrap(), + EvidenceFlags::CONTENT, + EvidenceSource::Observer, + 2, + ) + .unwrap(); + }) + .unwrap(); + + assert_eq!(snapshot.exact_paths, vec![before]); + assert_eq!(snapshot.cut.sequence, 1); + assert_eq!( + writer + .query_row("SELECT COUNT(*) FROM changed_path_entries", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 2 + ); + } + + #[test] + fn every_scope_mutation_rejects_a_stale_exact_cas_tuple() { + let conn = fixture(10, 10); + set_trust(&conn, TrustState::Trusted); + let ledger = ChangedPathLedger::new(&conn); + let stale_values = { + let base = expected(); + let mut values = Vec::new(); + let mut stale = base.clone(); + stale.epoch += 1; + values.push(stale); + let mut stale = base.clone(); + stale.ref_name.push_str("-stale"); + values.push(stale); + let mut stale = base.clone(); + stale.ref_generation += 1; + values.push(stale); + let mut stale = base.clone(); + stale.baseline_root = ObjectId("root-stale".into()); + values.push(stale); + let mut stale = base.clone(); + stale.policy_fingerprint[0] ^= 1; + values.push(stale); + let mut stale = base.clone(); + stale.policy_generation += 1; + values.push(stale); + let mut stale = base.clone(); + stale.filesystem_identity.push(1); + values.push(stale); + let mut stale = base; + stale.provider_identity.push(1); + values.push(stale); + values + }; + + for stale in stale_values { + let assertions = [ + ledger.mark_untrusted(&stale, TrustState::StaleBaseline, "stale"), + ledger.mark_prefix_dirty(&stale, &prefix("src", 1)), + ledger.upsert_exact( + &stale, + &LedgerPath::parse("src/lib.rs").unwrap(), + EvidenceFlags::CONTENT, + EvidenceSource::Observer, + 1, + ), + ledger.acknowledge( + &stale, + &EvidenceCut { + source: EvidenceSource::Observer, + sequence: 1, + durable_offset: 1, + folded_offset: 1, + }, + &OwnedEvidence { + source: EvidenceSource::Observer, + through_sequence: 1, + exact_paths: Vec::new(), + prefixes: Vec::new(), + }, + ), + ledger.advance_baseline( + &stale, + &BaselineIdentity { + ref_name: "refs/branches/main".into(), + ref_generation: 8, + change_id: ChangeId("change-next".into()), + root_id: ObjectId("root-next".into()), + }, + &EvidenceCut { + source: EvidenceSource::Observer, + sequence: 1, + durable_offset: 1, + folded_offset: 1, + }, + ), + ]; + assert!(assertions + .into_iter() + .all(|result| matches!(result, Err(Error::ChangeLedgerReconcileRequired { .. })))); + } + } + + #[test] + fn mark_untrusted_rejects_trusted_as_an_input() { + let conn = fixture(10, 10); + let error = ChangedPathLedger::new(&conn) + .mark_untrusted(&expected(), TrustState::Trusted, "not allowed") + .unwrap_err(); + assert!(matches!(error, Error::InvalidInput(_))); + } + + #[test] + fn mark_untrusted_advances_continuity_generation_atomically() { + let conn = fixture(10, 10); + let ledger = ChangedPathLedger::new(&conn); + let before: i64 = conn + .query_row( + "SELECT continuity_generation FROM changed_path_scopes", + [], + |row| row.get(0), + ) + .unwrap(); + + ledger + .mark_untrusted(&expected(), TrustState::StaleBaseline, "policy invalidated") + .unwrap(); + + let after: (String, i64) = conn + .query_row( + "SELECT trust_state,continuity_generation FROM changed_path_scopes", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(after, ("stale_baseline".into(), before + 1)); + } + + #[test] + fn complete_prefixes_coalesce_on_component_boundaries() { + let conn = fixture(10, 10); + let ledger = ChangedPathLedger::new(&conn); + ledger + .mark_prefix_dirty(&expected(), &prefix("dir/sub", 9)) + .unwrap(); + ledger + .mark_prefix_dirty(&expected(), &prefix("directory", 4)) + .unwrap(); + ledger + .mark_prefix_dirty(&expected(), &prefix("dir", 2)) + .unwrap(); + + set_trust(&conn, TrustState::Trusted); + let snapshot = ledger.snapshot_candidates(&expected()).unwrap(); + assert_eq!(snapshot.prefixes.len(), 2); + assert_eq!(snapshot.prefixes[0].path.as_str(), "dir"); + assert!(snapshot.prefixes[0].complete); + assert_eq!(snapshot.prefixes[0].reason, "rescan-dir"); + assert_eq!( + ( + snapshot.prefixes[0].first_sequence, + snapshot.prefixes[0].last_sequence + ), + (2, 9) + ); + assert_eq!(snapshot.prefixes[1].path.as_str(), "directory"); + } + + #[test] + fn incomplete_prefixes_are_rejected_instead_of_persisted_as_authoritative() { + let conn = fixture(10, 10); + let mut incomplete = prefix("dir", 1); + incomplete.complete = false; + let error = ChangedPathLedger::new(&conn) + .mark_prefix_dirty(&expected(), &incomplete) + .unwrap_err(); + assert!(matches!(error, Error::InvalidInput(_))); + assert_eq!( + conn.query_row("SELECT COUNT(*) FROM changed_path_prefixes", [], |row| row + .get::<_, i64>( + 0 + )) + .unwrap(), + 0 + ); + } + + #[test] + fn non_observer_sequences_do_not_inflate_the_observer_cut() { + let conn = fixture(10, 10); + let ledger = ChangedPathLedger::new(&conn); + let mixed = LedgerPath::parse("mixed").unwrap(); + ledger + .upsert_exact( + &expected(), + &mixed, + EvidenceFlags::CONTENT, + EvidenceSource::Observer, + 5, + ) + .unwrap(); + ledger + .upsert_exact( + &expected(), + &mixed, + EvidenceFlags::MODE, + EvidenceSource::Intent, + 500, + ) + .unwrap(); + ledger + .upsert_exact( + &expected(), + &LedgerPath::parse("intent-only").unwrap(), + EvidenceFlags::CREATE, + EvidenceSource::Intent, + 900, + ) + .unwrap(); + + assert_eq!( + conn.query_row( + "SELECT provider_sequence FROM changed_path_entries + WHERE normalized_path = 'intent-only'", + [], + |row| row.get::<_, Option>(0), + ) + .unwrap(), + None + ); + set_trust(&conn, TrustState::Trusted); + assert_eq!( + ledger + .snapshot_candidates(&expected()) + .unwrap() + .cut + .sequence, + 5 + ); + } + + #[test] + fn observer_acknowledgement_retains_mixed_and_later_observer_evidence() { + let conn = fixture(10, 10); + let ledger = ChangedPathLedger::new(&conn); + let mixed = LedgerPath::parse("mixed").unwrap(); + let later = LedgerPath::parse("later").unwrap(); + ledger + .upsert_exact( + &expected(), + &mixed, + EvidenceFlags::CONTENT, + EvidenceSource::Observer, + 5, + ) + .unwrap(); + ledger + .upsert_exact( + &expected(), + &mixed, + EvidenceFlags::MODE, + EvidenceSource::Intent, + 500, + ) + .unwrap(); + ledger + .upsert_exact( + &expected(), + &later, + EvidenceFlags::CONTENT, + EvidenceSource::Observer, + 6, + ) + .unwrap(); + set_trust(&conn, TrustState::Trusted); + + ledger + .acknowledge( + &expected(), + &EvidenceCut { + source: EvidenceSource::Observer, + sequence: 5, + durable_offset: 0, + folded_offset: 0, + }, + &OwnedEvidence { + source: EvidenceSource::Observer, + through_sequence: 5, + exact_paths: vec![mixed, later], + prefixes: Vec::new(), + }, + ) + .unwrap(); + + assert_eq!( + ledger + .all_exact(&expected()) + .unwrap() + .into_iter() + .map(|evidence| evidence.path.0) + .collect::>(), + vec!["later".to_string(), "mixed".to_string()] + ); + } + + #[test] + fn acknowledgement_rejects_scope_boundary_mismatch_before_deleting_evidence() { + let conn = fixture(10, 10); + let path = LedgerPath::parse("kept").unwrap(); + let ledger = ChangedPathLedger::new(&conn); + ledger + .upsert_exact( + &expected(), + &path, + EvidenceFlags::CONTENT, + EvidenceSource::Observer, + 3, + ) + .unwrap(); + conn.execute( + "UPDATE changed_path_scopes + SET trust_state = 'trusted', durable_offset = 10, folded_offset = 10", + [], + ) + .unwrap(); + + assert!(matches!( + ledger.acknowledge( + &expected(), + &EvidenceCut { + source: EvidenceSource::Observer, + sequence: 3, + durable_offset: 9, + folded_offset: 9, + }, + &OwnedEvidence { + source: EvidenceSource::Observer, + through_sequence: 3, + exact_paths: vec![path], + prefixes: Vec::new(), + }, + ), + Err(Error::ChangeLedgerReconcileRequired { .. }) + )); + assert_eq!(ledger.all_exact(&expected()).unwrap().len(), 1); + } + + #[test] + fn baseline_advance_rejects_scope_boundary_mismatch() { + let conn = fixture(10, 10); + conn.execute( + "UPDATE changed_path_scopes + SET trust_state = 'trusted', durable_offset = 10, folded_offset = 10", + [], + ) + .unwrap(); + let error = ChangedPathLedger::new(&conn) + .advance_baseline( + &expected(), + &BaselineIdentity { + ref_name: "refs/branches/main".into(), + ref_generation: 8, + change_id: ChangeId("change-next".into()), + root_id: ObjectId("root-next".into()), + }, + &EvidenceCut { + source: EvidenceSource::Observer, + sequence: 3, + durable_offset: 9, + folded_offset: 9, + }, + ) + .unwrap_err(); + assert!(matches!(error, Error::ChangeLedgerReconcileRequired { .. })); + assert_eq!( + conn.query_row( + "SELECT ref_generation FROM changed_path_scopes", + [], + |row| { row.get::<_, i64>(0) } + ) + .unwrap(), + 7 + ); + } + + #[test] + fn acknowledgement_and_baseline_advance_leave_observer_offsets_unchanged() { + let conn = fixture(10, 10); + conn.execute( + "UPDATE changed_path_scopes + SET trust_state = 'trusted', durable_offset = 10, folded_offset = 10", + [], + ) + .unwrap(); + let ledger = ChangedPathLedger::new(&conn); + let cut = EvidenceCut { + source: EvidenceSource::Intent, + sequence: 500, + durable_offset: 10, + folded_offset: 10, + }; + ledger + .acknowledge( + &expected(), + &cut, + &OwnedEvidence { + source: EvidenceSource::Intent, + through_sequence: 500, + exact_paths: Vec::new(), + prefixes: Vec::new(), + }, + ) + .unwrap(); + ledger + .advance_baseline( + &expected(), + &BaselineIdentity { + ref_name: "refs/branches/main".into(), + ref_generation: 8, + change_id: ChangeId("change-next".into()), + root_id: ObjectId("root-next".into()), + }, + &cut, + ) + .unwrap(); + assert_eq!( + conn.query_row( + "SELECT durable_offset, folded_offset FROM changed_path_scopes", + [], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)), + ) + .unwrap(), + (10, 10) + ); + } + + #[test] + fn candidate_cap_rolls_back_the_over_cap_row_and_rejects_later_writes() { + let conn = fixture(1, 1); + let ledger = ChangedPathLedger::new(&conn); + ledger + .upsert_exact( + &expected(), + &LedgerPath::parse("a").unwrap(), + EvidenceFlags::CONTENT, + EvidenceSource::Observer, + 1, + ) + .unwrap(); + assert!(matches!( + ledger.upsert_exact( + &expected(), + &LedgerPath::parse("b").unwrap(), + EvidenceFlags::CONTENT, + EvidenceSource::Observer, + 2, + ), + Err(Error::ChangeLedgerReconcileRequired { .. }) + )); + assert_eq!( + conn.query_row("SELECT trust_state FROM changed_path_scopes", [], |row| row + .get::<_, String>(0)) + .unwrap(), + "overflow" + ); + assert_eq!( + conn.query_row("SELECT COUNT(*) FROM changed_path_entries", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 1 + ); + assert!(matches!( + ledger.upsert_exact( + &expected(), + &LedgerPath::parse("c").unwrap(), + EvidenceFlags::CONTENT, + EvidenceSource::Observer, + 3, + ), + Err(Error::ChangeLedgerReconcileRequired { .. }) + )); + assert_eq!( + conn.query_row("SELECT COUNT(*) FROM changed_path_entries", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 1 + ); + } + + #[test] + fn prefix_cap_rolls_back_the_over_cap_row_and_rejects_later_writes() { + let prefix_conn = fixture(10, 1); + let prefix_ledger = ChangedPathLedger::new(&prefix_conn); + prefix_ledger + .mark_prefix_dirty(&expected(), &prefix("a", 1)) + .unwrap(); + assert!(matches!( + prefix_ledger.mark_prefix_dirty(&expected(), &prefix("b", 2)), + Err(Error::ChangeLedgerReconcileRequired { .. }) + )); + assert_eq!( + prefix_conn + .query_row("SELECT trust_state FROM changed_path_scopes", [], |row| row + .get::<_, String>(0)) + .unwrap(), + "overflow" + ); + assert_eq!( + prefix_conn + .query_row("SELECT COUNT(*) FROM changed_path_prefixes", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 1 + ); + assert!(matches!( + prefix_ledger.mark_prefix_dirty(&expected(), &prefix("c", 3)), + Err(Error::ChangeLedgerReconcileRequired { .. }) + )); + assert_eq!( + prefix_conn + .query_row("SELECT COUNT(*) FROM changed_path_prefixes", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 1 + ); + } + + #[test] + fn prefix_cap_rollback_restores_rows_deleted_during_coalescing() { + let conn = fixture(10, 2); + let ledger = ChangedPathLedger::new(&conn); + ledger + .mark_prefix_dirty(&expected(), &prefix("dir/a", 1)) + .unwrap(); + ledger + .mark_prefix_dirty(&expected(), &prefix("dir/b", 2)) + .unwrap(); + let rows = || { + conn.prepare( + "SELECT normalized_prefix, completeness_reason, source_mask, + first_sequence, last_sequence, provider_id, + provider_sequence, created_at, updated_at + FROM changed_path_prefixes + ORDER BY normalized_prefix COLLATE BINARY", + ) + .unwrap() + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, i64>(7)?, + row.get::<_, i64>(8)?, + )) + }) + .unwrap() + .collect::, _>>() + .unwrap() + }; + let before = rows(); + conn.execute("UPDATE changed_path_scopes SET max_prefix_rows = 1", []) + .unwrap(); + + assert!(matches!( + ledger.mark_prefix_dirty(&expected(), &prefix("dir/a/sub", 3)), + Err(Error::ChangeLedgerReconcileRequired { .. }) + )); + assert_eq!(rows(), before); + } + + #[test] + fn overflow_and_corrupt_scopes_reject_all_evidence_writes() { + for state in [TrustState::Overflow, TrustState::Corrupt] { + let conn = fixture(10, 10); + set_trust(&conn, state); + let ledger = ChangedPathLedger::new(&conn); + assert!(matches!( + ledger.upsert_exact( + &expected(), + &LedgerPath::parse("blocked").unwrap(), + EvidenceFlags::CONTENT, + EvidenceSource::Observer, + 1, + ), + Err(Error::ChangeLedgerReconcileRequired { .. }) + )); + assert!(matches!( + ledger.mark_prefix_dirty(&expected(), &prefix("blocked", 1)), + Err(Error::ChangeLedgerReconcileRequired { .. }) + )); + assert_eq!( + conn.query_row( + "SELECT + (SELECT COUNT(*) FROM changed_path_entries), + (SELECT COUNT(*) FROM changed_path_prefixes)", + [], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)), + ) + .unwrap(), + (0, 0) + ); + } + } + + #[test] + fn trusted_baseline_advance_updates_the_identity_and_fences_the_old_expected_scope() { + let conn = fixture(10, 10); + conn.execute( + "UPDATE changed_path_scopes + SET trust_state = 'trusted', durable_offset = 34, folded_offset = 34", + [], + ) + .unwrap(); + let ledger = ChangedPathLedger::new(&conn); + let target = BaselineIdentity { + ref_name: "refs/branches/main".into(), + ref_generation: 8, + change_id: ChangeId("change-next".into()), + root_id: ObjectId("root-next".into()), + }; + ledger + .advance_baseline( + &expected(), + &target, + &EvidenceCut { + source: EvidenceSource::Observer, + sequence: 12, + durable_offset: 34, + folded_offset: 34, + }, + ) + .unwrap(); + assert_eq!( + conn.query_row( + "SELECT ref_generation, change_id, baseline_root_id, durable_offset, + folded_offset + FROM changed_path_scopes", + [], + |row| Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + )), + ) + .unwrap(), + (8, "change-next".into(), "root-next".into(), 34, 34) + ); + assert!(matches!( + ledger.snapshot_candidates(&expected()), + Err(Error::ChangeLedgerReconcileRequired { .. }) + )); + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(32))] + + #[test] + fn exact_event_coalescing_is_order_independent( + events in prop::collection::vec((0u8..4, 0u8..6, 0u64..100), 1..30) + ) { + fn apply(events: &[(u8, u8, u64)]) -> ExactEvidence { + let conn = fixture(100, 10); + let ledger = ChangedPathLedger::new(&conn); + let path = LedgerPath::parse("src/lib.rs").unwrap(); + for &(source, flag, sequence) in events { + ledger.upsert_exact( + &expected(), + &path, + EvidenceFlags::from_index(flag), + EvidenceSource::from_index(source), + sequence, + ).unwrap(); + } + ledger.all_exact(&expected()).unwrap().remove(0) + } + + let forward = apply(&events); + let expected_flags = events.iter().fold(EvidenceFlags::default(), |mut flags, event| { + flags |= EvidenceFlags::from_index(event.1); + flags + }); + let expected_sources = events.iter().fold(0, |mask, event| { + mask | EvidenceSource::from_index(event.0).mask() + }); + prop_assert_eq!(forward.path.as_str(), "src/lib.rs"); + prop_assert_eq!(forward.flags, expected_flags); + prop_assert_eq!(forward.source_mask, expected_sources); + prop_assert_eq!(forward.first_sequence, events.iter().map(|event| event.2).min().unwrap()); + prop_assert_eq!(forward.last_sequence, events.iter().map(|event| event.2).max().unwrap()); + let mut reversed = events.clone(); + reversed.reverse(); + prop_assert_eq!(forward, apply(&reversed)); + } + + #[test] + fn acknowledgement_never_clears_later_or_other_source_evidence( + events in prop::collection::vec((0u8..4, 0u64..40), 1..30), + cut_sequence in 0u64..40, + acknowledged_source in 0u8..4, + ) { + let conn = fixture(100, 10); + let ledger = ChangedPathLedger::new(&conn); + let path = LedgerPath::parse("src/lib.rs").unwrap(); + for &(source, sequence) in &events { + ledger.upsert_exact( + &expected(), + &path, + EvidenceFlags::CONTENT, + EvidenceSource::from_index(source), + sequence, + ).unwrap(); + } + set_trust(&conn, TrustState::Trusted); + let source = EvidenceSource::from_index(acknowledged_source); + ledger.acknowledge( + &expected(), + &EvidenceCut { + source, + sequence: cut_sequence, + durable_offset: 0, + folded_offset: 0, + }, + &OwnedEvidence { + source, + through_sequence: cut_sequence, + exact_paths: vec![path], + prefixes: Vec::new(), + }, + ).unwrap(); + + let remaining = ledger.all_exact(&expected()).unwrap(); + let has_later = events.iter().any(|&(event_source, sequence)| { + EvidenceSource::from_index(event_source) == source && sequence > cut_sequence + }); + let has_other = events.iter().any(|&(event_source, _)| { + EvidenceSource::from_index(event_source) != source + }); + prop_assert_eq!(remaining.is_empty(), !has_later && !has_other); + } + } +} diff --git a/trail/src/db/change_ledger/types.rs b/trail/src/db/change_ledger/types.rs new file mode 100644 index 0000000..42fa0a1 --- /dev/null +++ b/trail/src/db/change_ledger/types.rs @@ -0,0 +1,509 @@ +use std::ops::{BitOr, BitOrAssign}; + +use serde::{Deserialize, Serialize}; +use unicode_normalization::UnicodeNormalization; + +use crate::error::{Error, Result}; +use crate::{ChangeId, ObjectId}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum TrustState { + Trusted, + Reconciling, + Overflow, + UntrustedGap, + StaleBaseline, + Corrupt, +} + +impl TrustState { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Trusted => "trusted", + Self::Reconciling => "reconciling", + Self::Overflow => "overflow", + Self::UntrustedGap => "untrusted_gap", + Self::StaleBaseline => "stale_baseline", + Self::Corrupt => "corrupt", + } + } + + pub(crate) fn parse(value: &str) -> Result { + match value { + "trusted" => Ok(Self::Trusted), + "reconciling" => Ok(Self::Reconciling), + "overflow" => Ok(Self::Overflow), + "untrusted_gap" => Ok(Self::UntrustedGap), + "stale_baseline" => Ok(Self::StaleBaseline), + "corrupt" => Ok(Self::Corrupt), + other => Err(Error::Corrupt(format!( + "unknown changed-path trust state `{other}`" + ))), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct ProviderCapabilities { + pub(crate) durable_cursor: bool, + pub(crate) linearizable_fence: bool, + pub(crate) rename_pairing: bool, + pub(crate) overflow_scope: bool, + pub(crate) filesystem_supported: bool, + pub(crate) clean_proof_allowed: bool, + pub(crate) power_loss_durability: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub(crate) struct ScopeId(pub(crate) [u8; 32]); + +impl ScopeId { + pub(crate) fn to_text(self) -> String { + hex::encode(self.0) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ScopeKind { + Workspace, + MaterializedLane, + WorkspaceView, +} + +impl ScopeKind { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Workspace => "workspace", + Self::MaterializedLane => "materialized_lane", + Self::WorkspaceView => "workspace_view", + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +pub(crate) struct LedgerPath(pub(crate) String); + +impl LedgerPath { + pub(crate) fn parse(value: &str) -> Result { + let invalid = |reason: &str| Error::InvalidPath { + path: value.to_string(), + reason: reason.to_string(), + }; + if value.is_empty() { + return Err(invalid("ledger paths cannot be empty")); + } + if value.starts_with('/') + || (value.len() >= 3 + && value.as_bytes()[0].is_ascii_alphabetic() + && value.as_bytes()[1] == b':' + && value.as_bytes()[2] == b'/') + { + return Err(invalid("ledger paths must be relative")); + } + if value.contains('\\') { + return Err(invalid("ledger paths use `/` separators")); + } + if value.contains('\0') { + return Err(invalid("ledger paths cannot contain NUL")); + } + if !value.nfc().eq(value.chars()) { + return Err(invalid("ledger paths must be Unicode NFC normalized")); + } + if value + .split('/') + .any(|component| component.is_empty() || component == "." || component == "..") + { + return Err(invalid("ledger paths must contain normalized components")); + } + Ok(Self(value.to_string())) + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ScopeIdentity { + pub(crate) scope_id: ScopeId, + pub(crate) kind: ScopeKind, + pub(crate) owner_id: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct BaselineIdentity { + pub(crate) ref_name: String, + pub(crate) ref_generation: u64, + pub(crate) change_id: ChangeId, + pub(crate) root_id: ObjectId, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PolicyIdentity { + pub(crate) fingerprint: [u8; 32], + pub(crate) generation: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct FilesystemIdentity(pub(crate) Vec); + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ProviderIdentity { + pub(crate) identity: Vec, + pub(crate) capabilities: ProviderCapabilities, +} + +/// Exact structural binding for Git-sourced changed-path evidence. Git may +/// contribute advisory dirty paths when this record is not fully equivalent, +/// but only a completely qualified record may prove an empty candidate set. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct GitEvidenceQualification { + pub(crate) head_oid: String, + pub(crate) head_identity: Vec, + pub(crate) worktree_top_level: String, + pub(crate) worktree_identity: Vec, + pub(crate) index_identity: Vec, + pub(crate) split_index_identity: Option>, + pub(crate) shared_index_identity: Option>, + pub(crate) mapped_trail_root: ObjectId, + pub(crate) ledger_baseline_root: ObjectId, + pub(crate) filesystem_identity: Vec, + pub(crate) policy_fingerprint: [u8; 32], + pub(crate) filesystem_equivalent: bool, + pub(crate) worktree_equivalent: bool, + pub(crate) policy_equivalent: bool, + pub(crate) head_equivalent: bool, + pub(crate) index_equivalent: bool, + pub(crate) mode_equivalent: bool, + pub(crate) symlink_equivalent: bool, + pub(crate) sparse_equivalent: bool, + pub(crate) submodule_equivalent: bool, + pub(crate) ignore_equivalent: bool, + pub(crate) case_equivalent: bool, + pub(crate) fsmonitor_qualified: bool, + pub(crate) untracked_cache_qualified: bool, + pub(crate) clean_proof_allowed: bool, + pub(crate) advisory_reasons: Vec, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct GitStructuralMetrics { + pub(crate) subprocess_count: u64, + pub(crate) index_refresh_count: u64, + pub(crate) trace2_region_count: u64, + pub(crate) trace2_bytes: u64, + pub(crate) output_bytes: u64, + pub(crate) output_record_count: u64, + pub(crate) index_read_count: u64, + pub(crate) index_bytes: u64, + pub(crate) shared_index_read_count: u64, + pub(crate) shared_index_bytes: u64, + pub(crate) fsmonitor_qualification_count: u64, + pub(crate) untracked_cache_qualification_count: u64, + pub(crate) external_adapter_global_work: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct QualifiedGitCandidates { + pub(crate) qualification: GitEvidenceQualification, + pub(crate) exact_paths: Vec, + pub(crate) rename_pairs: Vec<(String, String)>, + pub(crate) metrics: GitStructuralMetrics, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum EvidenceSource { + Observer, + Intent, + Reconciliation, + GitAdvisory, +} + +impl EvidenceSource { + pub(crate) const fn mask(self) -> i64 { + match self { + Self::Observer => 1, + Self::Intent => 2, + Self::Reconciliation => 4, + Self::GitAdvisory => 8, + } + } + + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Observer => "observer", + Self::Intent => "intent", + Self::Reconciliation => "reconciliation", + Self::GitAdvisory => "git_advisory", + } + } + + #[cfg(test)] + pub(crate) const fn from_index(index: u8) -> Self { + match index % 4 { + 0 => Self::Observer, + 1 => Self::Intent, + 2 => Self::Reconciliation, + _ => Self::GitAdvisory, + } + } +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[serde(transparent)] +pub(crate) struct EvidenceFlags(pub(crate) i64); + +impl EvidenceFlags { + pub(crate) const CREATE: Self = Self(1 << 0); + pub(crate) const CONTENT: Self = Self(1 << 1); + pub(crate) const MODE: Self = Self(1 << 2); + pub(crate) const DELETE: Self = Self(1 << 3); + pub(crate) const RENAME_FROM: Self = Self(1 << 4); + pub(crate) const RENAME_TO: Self = Self(1 << 5); + pub(crate) const PROVIDER_COMPLETE_PREFIX: Self = Self(1 << 6); + /// A controlled projection path is covered by any authenticated native + /// mutation for that exact path. The pinned comparison after c1 proves the + /// final bytes/type/mode; requiring CONTENT would reject create, delete, + /// rename, and chmod-only producers. + pub(crate) const ANY_MUTATION: Self = Self( + Self::CREATE.0 + | Self::CONTENT.0 + | Self::MODE.0 + | Self::DELETE.0 + | Self::RENAME_FROM.0 + | Self::RENAME_TO.0, + ); + + #[cfg(test)] + pub(crate) const fn from_index(index: u8) -> Self { + match index % 6 { + 0 => Self::CREATE, + 1 => Self::CONTENT, + 2 => Self::MODE, + 3 => Self::DELETE, + 4 => Self::RENAME_FROM, + _ => Self::RENAME_TO, + } + } +} + +impl BitOr for EvidenceFlags { + type Output = Self; + + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl BitOrAssign for EvidenceFlags { + fn bitor_assign(&mut self, rhs: Self) { + self.0 |= rhs.0; + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(crate) struct EvidenceCut { + pub(crate) source: EvidenceSource, + pub(crate) sequence: u64, + pub(crate) durable_offset: u64, + pub(crate) folded_offset: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct DirtyPrefix { + pub(crate) path: LedgerPath, + pub(crate) complete: bool, + pub(crate) reason: String, + pub(crate) first_sequence: u64, + pub(crate) last_sequence: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct OwnedEvidence { + pub(crate) source: EvidenceSource, + pub(crate) through_sequence: u64, + pub(crate) exact_paths: Vec, + pub(crate) prefixes: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CandidateSnapshot { + pub(crate) expected: ExpectedScope, + pub(crate) cut: EvidenceCut, + pub(crate) exact_paths: Vec, + pub(crate) prefixes: Vec, + /// Immutable row identities captured in the same read transaction as the + /// candidate set. A checkpoint may acknowledge a row only when every + /// field still matches this token; merging newer or differently-owned + /// evidence therefore always leaves the row pending. + pub(crate) acknowledgement_tokens: Vec, + pub(crate) trust: TrustState, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct EvidenceAcknowledgementToken { + pub(crate) kind: EvidenceRowKind, + pub(crate) path: LedgerPath, + pub(crate) flags: EvidenceFlags, + pub(crate) source_mask: i64, + pub(crate) first_sequence: u64, + pub(crate) last_sequence: u64, + pub(crate) provider_id: Option, + pub(crate) provider_sequence: Option, + pub(crate) intent_id: Option, +} + +pub(crate) fn trail_case_probe_token(token: &EvidenceAcknowledgementToken) -> bool { + let lifecycle = EvidenceFlags::CREATE.0 | EvidenceFlags::DELETE.0; + let allowed = lifecycle | EvidenceFlags::MODE.0; + if token.kind != EvidenceRowKind::Exact + || token.source_mask != EvidenceSource::Observer.mask() + || token.provider_sequence.is_none() + || token.intent_id.is_some() + || token.first_sequence > token.last_sequence + // Native providers may report create/delete separately or coalesce the + // short-lived probe to either endpoint. Accept only lifecycle/mode + // flags and require at least one endpoint; the exact random name plus + // pinned absence and baseline checks provide the state binding. + || token.flags.0 & lifecycle == 0 + || token.flags.0 & !allowed != 0 + { + return false; + } + let path = token.path.as_str(); + let Some(nonce) = path + .strip_prefix(".trail-case-probe-") + .and_then(|path| path.strip_suffix("-a")) + else { + return false; + }; + !nonce.is_empty() && nonce.bytes().all(|byte| byte.is_ascii_digit()) +} + +pub(crate) fn trail_atomic_temp_target(token: &EvidenceAcknowledgementToken) -> Option { + let required_flags = + EvidenceFlags::CREATE.0 | EvidenceFlags::CONTENT.0 | EvidenceFlags::RENAME_FROM.0; + if token.kind != EvidenceRowKind::Exact + || token.source_mask != EvidenceSource::Observer.mask() + || token.provider_sequence.is_none() + || token.intent_id.is_some() + || token.first_sequence > token.last_sequence + // Native rename streams attach RENAME_FROM to the temporary source + // and RENAME_TO to the intended destination; requiring both on the + // temporary row can never match a real atomic replacement. CREATE + + // CONTENT + RENAME_FROM, the exact numeric Trail temp name, a matching + // intended target, and the later pinned absence check together bind + // this row to the controlled projection without hiding persistent + // user files that merely resemble Trail's namespace. + || token.flags.0 & required_flags != required_flags + { + return None; + } + let path = token.path.as_str(); + let (temporary, nonce) = path.rsplit_once(".trail-tmp-")?; + if nonce.is_empty() || !nonce.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let (parent, temporary_leaf) = temporary + .rsplit_once('/') + .map_or((None, temporary), |(parent, leaf)| (Some(parent), leaf)); + let target_leaf = temporary_leaf.strip_prefix('.')?; + if target_leaf.is_empty() { + return None; + } + Some(match parent { + Some(parent) => format!("{parent}/{target_leaf}"), + None => target_leaf.to_string(), + }) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum EvidenceRowKind { + Exact, + CompletePrefix, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ExpectedScope { + pub(crate) scope_id: ScopeId, + pub(crate) epoch: u64, + pub(crate) ref_name: String, + pub(crate) ref_generation: u64, + pub(crate) baseline_root: ObjectId, + pub(crate) policy_fingerprint: [u8; 32], + pub(crate) policy_generation: u64, + pub(crate) filesystem_identity: Vec, + pub(crate) provider_identity: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ExactEvidence { + pub(crate) path: LedgerPath, + pub(crate) flags: EvidenceFlags, + pub(crate) source_mask: i64, + pub(crate) first_sequence: u64, + pub(crate) last_sequence: u64, +} + +#[cfg(test)] +mod internal_control_tests { + use super::*; + + fn observer_token(path: &str, flags: i64) -> EvidenceAcknowledgementToken { + EvidenceAcknowledgementToken { + kind: EvidenceRowKind::Exact, + path: LedgerPath::parse(path).unwrap(), + flags: EvidenceFlags(flags), + source_mask: EvidenceSource::Observer.mask(), + first_sequence: 10, + last_sequence: 10, + provider_id: Some("native".into()), + provider_sequence: Some(10), + intent_id: None, + } + } + + #[test] + fn internal_control_tokens_require_exact_names_and_lifecycles() { + let case_flags = EvidenceFlags::CREATE.0 | EvidenceFlags::DELETE.0; + assert!(trail_case_probe_token(&observer_token( + ".trail-case-probe-123-a", + case_flags, + ))); + assert!(!trail_case_probe_token(&observer_token( + "src/.trail-case-probe-123-a", + case_flags, + ))); + assert!(!trail_case_probe_token(&observer_token( + ".trail-case-probe-user-a", + case_flags, + ))); + assert!(!trail_case_probe_token(&observer_token( + ".trail-case-probe-123-a", + EvidenceFlags::CONTENT.0, + ))); + + let rename_flags = EvidenceFlags::CREATE.0 + | EvidenceFlags::CONTENT.0 + | EvidenceFlags::MODE.0 + | EvidenceFlags::RENAME_FROM.0; + assert_eq!( + trail_atomic_temp_target(&observer_token("src/.lib.rs.trail-tmp-456", rename_flags,)), + Some("src/lib.rs".into()) + ); + assert_eq!( + trail_atomic_temp_target(&observer_token("src/.lib.rs.trail-tmp-user", rename_flags,)), + None + ); + assert_eq!( + trail_atomic_temp_target(&observer_token( + "src/.unrelated.rs.trail-tmp-456", + EvidenceFlags::CREATE.0, + )), + None + ); + } +} diff --git a/trail/src/db/core/audit.rs b/trail/src/db/core/audit.rs index 41f9d78..647b098 100644 --- a/trail/src/db/core/audit.rs +++ b/trail/src/db/core/audit.rs @@ -6,6 +6,12 @@ impl Trail { mut input: ExternalMutationAuditInput, ) -> Result { let _lock = self.acquire_write_lock()?; + #[cfg(debug_assertions)] + if let Some(milliseconds) = std::env::var_os("TRAIL_TEST_EXTERNAL_AUDIT_HOLD_MS") + .and_then(|value| value.to_string_lossy().parse::().ok()) + { + std::thread::sleep(std::time::Duration::from_millis(milliseconds)); + } if let Some(lane_handle) = input.lane_id.clone() { if let Some((lane_id, ref_name)) = self.external_mutation_lane_identity(&lane_handle)? { input.lane_id = Some(lane_id); diff --git a/trail/src/db/core/backup.rs b/trail/src/db/core/backup.rs index 80e001b..ce4a76c 100644 --- a/trail/src/db/core/backup.rs +++ b/trail/src/db/core/backup.rs @@ -1,5 +1,9 @@ use super::*; mod create; +mod publication; mod restore; +mod restore_transaction; mod verify; + +pub(crate) use restore_transaction::recover_restore_publication; diff --git a/trail/src/db/core/backup/create.rs b/trail/src/db/core/backup/create.rs index 47abc4b..0d35baf 100644 --- a/trail/src/db/core/backup/create.rs +++ b/trail/src/db/core/backup/create.rs @@ -1,4 +1,8 @@ use super::*; +use crate::db::change_ledger::mark_backup_scopes_untrusted; +use crate::db::core::backup::publication::{ + publish_staged_tree, remove_any, remove_retained_tree, sibling_stage, +}; impl Trail { pub fn create_backup( @@ -13,26 +17,40 @@ impl Trail { "backup output cannot be inside .trail".to_string(), )); } - if output.exists() { - if !overwrite { - return Err(Error::WorkspaceExists(output)); + self.changed_path_ledger().recover()?; + if output.exists() && !overwrite { + return Err(Error::WorkspaceExists(output)); + } + let parent = output + .parent() + .ok_or_else(|| Error::InvalidInput("backup output has no parent".into()))?; + fs::create_dir_all(parent)?; + let stage = sibling_stage(&output, "backup-stage")?; + let mut report = match self.create_backup_inner(&stage) { + Ok(report) => report, + Err(error) => { + let _ = remove_any(&stage); + return Err(error); } - if output.is_dir() { - fs::remove_dir_all(&output)?; - } else { - fs::remove_file(&output)?; + }; + let retained = match publish_staged_tree(&stage, &output) { + Ok(retained) => retained, + Err(error) => { + let _ = remove_any(&stage); + return Err(error); } - } - - let result = self.create_backup_inner(&output); - if result.is_err() { - let _ = fs::remove_dir_all(&output); - } - result + }; + remove_retained_tree(retained, parent)?; + report.path = output.to_string_lossy().to_string(); + report.manifest_path = backup_manifest_path(&output).to_string_lossy().to_string(); + report.sqlite_path = backup_sqlite_path(&output).to_string_lossy().to_string(); + Ok(report) } pub(crate) fn create_backup_inner(&self, output: &Path) -> Result { fs::create_dir_all(output.join("index"))?; + fs::write(output.join("index").join(SCHEMA_EXCLUSION_FILE), [])?; + fs::write(output.join("index").join(SCHEMA_VALIDATION_LEADER_FILE), [])?; fs::create_dir_all(output.join("refs/branches"))?; fs::create_dir_all(output.join("refs/lanes"))?; @@ -47,6 +65,20 @@ impl Trail { let sqlite_path_text = sqlite_path.to_string_lossy().to_string(); self.conn .execute("VACUUM main INTO ?1", params![sqlite_path_text])?; + let backup_conn = Connection::open(&sqlite_path)?; + mark_backup_scopes_untrusted(&backup_conn)?; + let checkpoint_busy: i64 = + backup_conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |row| row.get(0))?; + if checkpoint_busy != 0 { + return Err(Error::Conflict( + "backup SQLite checkpoint remained busy".into(), + )); + } + drop(backup_conn); + OpenOptions::new() + .read(true) + .open(&sqlite_path)? + .sync_all()?; let (sqlite_bytes, sqlite_sha256) = file_digest(&sqlite_path)?; let worktree_bytes = @@ -77,6 +109,10 @@ impl Trail { }; let manifest_path = backup_manifest_path(output); fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)?; + OpenOptions::new() + .read(true) + .open(&manifest_path)? + .sync_all()?; Ok(BackupCreateReport { path: output.to_string_lossy().to_string(), diff --git a/trail/src/db/core/backup/publication.rs b/trail/src/db/core/backup/publication.rs new file mode 100644 index 0000000..9dfb496 --- /dev/null +++ b/trail/src/db/core/backup/publication.rs @@ -0,0 +1,203 @@ +use super::*; + +pub(super) fn sibling_stage(target: &Path, label: &str) -> Result { + let parent = target + .parent() + .ok_or_else(|| Error::InvalidInput("publication target has no parent".into()))?; + let leaf = target + .file_name() + .ok_or_else(|| Error::InvalidInput("publication target has no file name".into()))? + .to_string_lossy(); + for _ in 0..32 { + let candidate = parent.join(format!(".{leaf}.{label}-{}", now_nanos())); + match fs::create_dir(&candidate) { + Ok(()) => return Ok(candidate), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error.into()), + } + } + Err(Error::Conflict( + "could not allocate sibling publication stage".into(), + )) +} + +pub(super) fn sync_tree_bottom_up(root: &Path) -> Result<()> { + let mut directories = Vec::new(); + for entry in walkdir::WalkDir::new(root).follow_links(false) { + let entry = entry.map_err(|error| Error::Io(error.into()))?; + let file_type = entry.file_type(); + if file_type.is_file() { + OpenOptions::new() + .read(true) + .open(entry.path())? + .sync_all()?; + } else if file_type.is_dir() { + directories.push(entry.path().to_path_buf()); + } + } + directories.sort_by_key(|path| std::cmp::Reverse(path.components().count())); + for directory in directories { + sync_directory_strict(&directory)?; + } + Ok(()) +} + +pub(super) fn publish_staged_tree(stage: &Path, target: &Path) -> Result> { + publish_staged_tree_with_exchange(stage, target, atomic_exchange) +} + +fn publish_staged_tree_with_exchange( + stage: &Path, + target: &Path, + exchange: impl FnOnce(&Path, &Path) -> Result, +) -> Result> { + let parent = target + .parent() + .ok_or_else(|| Error::InvalidInput("publication target has no parent".into()))?; + sync_tree_bottom_up(stage)?; + sync_directory_strict(parent)?; + test_crash_point("backup_restore_after_staging_sync"); + + if !target.exists() { + fs::rename(stage, target)?; + sync_directory_strict(parent)?; + test_crash_point("backup_restore_after_atomic_publish"); + return Ok(None); + } + + if exchange(stage, target)? { + sync_directory_strict(parent)?; + test_crash_point("backup_restore_after_atomic_exchange"); + return Ok(Some(stage.to_path_buf())); + } + + Err(Error::Conflict( + "atomic directory exchange is unsupported; live tree was not moved".into(), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_exchange_refuses_before_moving_the_live_tree() { + let root = tempfile::tempdir().unwrap(); + let target = root.path().join("live"); + let stage = root.path().join("stage"); + fs::create_dir(&target).unwrap(); + fs::create_dir(&stage).unwrap(); + fs::write(target.join("marker"), b"old").unwrap(); + fs::write(stage.join("marker"), b"new").unwrap(); + + let result = publish_staged_tree_with_exchange(&stage, &target, |_, _| Ok(false)); + + assert!(result.is_err()); + assert_eq!(fs::read(target.join("marker")).unwrap(), b"old"); + assert_eq!(fs::read(stage.join("marker")).unwrap(), b"new"); + } +} + +pub(super) fn remove_retained_tree(path: Option, parent: &Path) -> Result<()> { + if let Some(path) = path { + remove_any(&path)?; + sync_directory_strict(parent)?; + } + Ok(()) +} + +pub(super) fn remove_any(path: &Path) -> Result<()> { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + if metadata.is_dir() && !metadata.file_type().is_symlink() { + fs::remove_dir_all(path)?; + } else { + fs::remove_file(path)?; + } + Ok(()) +} + +pub(super) fn sync_directory_strict(path: &Path) -> Result<()> { + #[cfg(unix)] + { + OpenOptions::new().read(true).open(path)?.sync_all()?; + } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000; + OpenOptions::new() + .read(true) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS) + .open(path)? + .sync_all()?; + } + #[cfg(not(any(unix, windows)))] + { + let _ = path; + } + Ok(()) +} + +#[cfg(target_os = "macos")] +pub(super) fn atomic_exchange(left: &Path, right: &Path) -> Result { + use std::ffi::CString; + use std::os::unix::ffi::OsStrExt; + + let left = CString::new(left.as_os_str().as_bytes()) + .map_err(|_| Error::InvalidInput("publication path contains NUL".into()))?; + let right = CString::new(right.as_os_str().as_bytes()) + .map_err(|_| Error::InvalidInput("publication path contains NUL".into()))?; + let result = unsafe { libc::renamex_np(left.as_ptr(), right.as_ptr(), libc::RENAME_SWAP) }; + if result == 0 { + Ok(true) + } else { + let error = std::io::Error::last_os_error(); + if matches!(error.raw_os_error(), Some(code) if code == libc::ENOTSUP || code == libc::EINVAL || code == libc::EXDEV) + { + Ok(false) + } else { + Err(error.into()) + } + } +} + +#[cfg(target_os = "linux")] +pub(super) fn atomic_exchange(left: &Path, right: &Path) -> Result { + use std::ffi::CString; + use std::os::unix::ffi::OsStrExt; + + let left = CString::new(left.as_os_str().as_bytes()) + .map_err(|_| Error::InvalidInput("publication path contains NUL".into()))?; + let right = CString::new(right.as_os_str().as_bytes()) + .map_err(|_| Error::InvalidInput("publication path contains NUL".into()))?; + let result = unsafe { + libc::syscall( + libc::SYS_renameat2, + libc::AT_FDCWD, + left.as_ptr(), + libc::AT_FDCWD, + right.as_ptr(), + libc::RENAME_EXCHANGE, + ) + }; + if result == 0 { + Ok(true) + } else { + let error = std::io::Error::last_os_error(); + if matches!(error.raw_os_error(), Some(code) if code == libc::ENOSYS || code == libc::ENOTSUP || code == libc::EINVAL || code == libc::EXDEV) + { + Ok(false) + } else { + Err(error.into()) + } + } +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +pub(super) fn atomic_exchange(_left: &Path, _right: &Path) -> Result { + Ok(false) +} diff --git a/trail/src/db/core/backup/restore.rs b/trail/src/db/core/backup/restore.rs index f07df97..7b93e29 100644 --- a/trail/src/db/core/backup/restore.rs +++ b/trail/src/db/core/backup/restore.rs @@ -1,4 +1,9 @@ use super::*; +use crate::db::change_ledger::rotate_restored_scopes; +use crate::db::core::backup::publication::{remove_any, sibling_stage, sync_tree_bottom_up}; +use crate::db::core::backup::restore_transaction::{ + recover_restore_publication, RestorePublication, +}; impl Trail { pub fn restore_backup( @@ -7,7 +12,8 @@ impl Trail { force: bool, ) -> Result { fs::create_dir_all(workspace_root.as_ref())?; - let workspace_root = workspace_root.as_ref().canonicalize()?; + let workspace_root = canonicalize_lossless(workspace_root.as_ref())?; + recover_restore_publication(&workspace_root)?; let backup_path = absolute_path(backup_path.as_ref())?; let verification = Self::verify_backup(&backup_path)?; if !verification.valid { @@ -29,14 +35,20 @@ impl Trail { return Err(Error::WorkspaceExists(db_dir)); } } + let (previous_epoch, previous_continuity) = if replaced_existing { + previous_scope_rotation(&db_dir)? + } else { + (0, 0) + }; + let temp_dir = sibling_stage(&db_dir, "restore-stage")?; - let temp_dir = workspace_root.join(format!(".trail.restore-{}", now_nanos())); - if temp_dir.exists() { - fs::remove_dir_all(&temp_dir)?; - } - - let restore_result = (|| -> Result<()> { + let restore_result = (|| -> Result<(u64, crate::model::FsckReport)> { fs::create_dir_all(temp_dir.join("index"))?; + fs::write(temp_dir.join("index").join(SCHEMA_EXCLUSION_FILE), [])?; + fs::write( + temp_dir.join("index").join(SCHEMA_VALIDATION_LEADER_FILE), + [], + )?; fs::create_dir_all(temp_dir.join("refs/branches"))?; fs::create_dir_all(temp_dir.join("refs/lanes"))?; fs::copy(backup_path.join(CONFIG_FILE), temp_dir.join(CONFIG_FILE))?; @@ -46,44 +58,66 @@ impl Trail { temp_dir.join(DB_RELATIVE_PATH), )?; copy_dir_recursive(&backup_path.join("worktrees"), &temp_dir.join("worktrees"))?; - Ok(()) - })(); - if let Err(err) = restore_result { - let _ = fs::remove_dir_all(&temp_dir); - return Err(err); - } - - if replaced_existing { - fs::remove_dir_all(&db_dir)?; - } - if let Err(err) = fs::rename(&temp_dir, &db_dir) { - let _ = fs::remove_dir_all(&temp_dir); - return Err(Error::Io(err)); - } + let restored_conn = Connection::open(temp_dir.join(DB_RELATIVE_PATH))?; + let filesystem_identity = fresh_restored_filesystem_identity(&workspace_root)?; + let scope_root = restored_scope_root(&workspace_root); + rotate_restored_scopes( + &restored_conn, + &filesystem_identity, + &scope_root, + previous_epoch, + previous_continuity, + )?; + drop(restored_conn); + test_crash_point("restore_after_ledger_rotation"); - let backup_trailignore = backup_path.join(".trailignore"); - let workspace_trailignore = workspace_root.join(".trailignore"); - let restored_trailignore = - if backup_trailignore.is_file() && (force || !workspace_trailignore.exists()) { - fs::copy(&backup_trailignore, &workspace_trailignore)?; - true - } else { - if !workspace_trailignore.exists() { - write_default_trailignore(&workspace_root)?; + let mut db = Trail::open_without_recovering_derived_paths(&workspace_root, &temp_dir)?; + let rewritten_workdirs = { + let _lock = db.acquire_write_lock()?; + let rewritten = db.rewrite_restored_lane_workdir_paths()?; + db.drain_pending_path_index_derived_repairs_from_restore_stage(&db_dir)?; + rewritten + }; + test_crash_point("restore_after_staged_workdir_rewrite"); + db.recover_after_open()?; + test_crash_point("restore_after_staged_recovery"); + let fsck = db.fsck()?; + if !fsck.errors.is_empty() { + return Err(Error::Corrupt(format!( + "restored backup failed fsck: {}", + fsck.errors.join("; ") + ))); + } + let checkpoint_busy: i64 = + db.conn + .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |row| row.get(0))?; + if checkpoint_busy != 0 { + return Err(Error::Conflict( + "restored SQLite checkpoint remained busy".into(), + )); + } + drop(db); + test_crash_point("restore_after_staged_checkpoint"); + sync_tree_bottom_up(&temp_dir)?; + test_crash_point("restore_after_staged_sync"); + Ok((rewritten_workdirs, fsck)) + })(); + let (rewritten_workdirs, fsck) = match restore_result { + Ok(prepared) => prepared, + Err(err) => { + let _ = remove_any(&temp_dir); + return Err(err); + } + }; + let (publication, restored_trailignore) = + match RestorePublication::prepare(&workspace_root, &temp_dir, &backup_path, force) { + Ok(prepared) => prepared, + Err(error) => { + let _ = remove_any(&temp_dir); + return Err(error); } - false }; - - let mut db = Trail::open(&workspace_root)?; - let rewritten_workdirs = db.rewrite_restored_lane_workdir_paths()?; - let fsck = db.fsck()?; - if !fsck.errors.is_empty() { - return Err(Error::Corrupt(format!( - "restored backup failed fsck: {}", - fsck.errors.join("; ") - ))); - } - + publication.publish()?; Ok(BackupRestoreReport { workspace: workspace_root.to_string_lossy().to_string(), db_dir: db_dir.to_string_lossy().to_string(), @@ -99,3 +133,77 @@ impl Trail { }) } } + +fn restored_scope_root(workspace_root: &Path) -> String { + workspace_root + .to_str() + .map(str::to_owned) + .unwrap_or_else(|| { + format!( + "os-bytes:{}", + hex::encode(workspace_root.as_os_str().as_encoded_bytes()) + ) + }) +} + +fn previous_scope_rotation(db_dir: &Path) -> Result<(u64, u64)> { + let sqlite = db_dir.join(DB_RELATIVE_PATH); + if !sqlite.is_file() { + return Ok((0, 0)); + } + let flags = + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX; + let conn = Connection::open_with_flags(sqlite, flags)?; + let (epoch, continuity): (i64, i64) = conn.query_row( + "SELECT COALESCE(MAX(epoch),0),COALESCE(MAX(continuity_generation),0) + FROM changed_path_scopes", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + Ok(( + u64::try_from(epoch).map_err(|_| Error::Corrupt("negative scope epoch".into()))?, + u64::try_from(continuity) + .map_err(|_| Error::Corrupt("negative scope continuity".into()))?, + )) +} + +fn fresh_restored_filesystem_identity(workspace_root: &Path) -> Result> { + let mut nonce = [0_u8; 32]; + getrandom::getrandom(&mut nonce) + .map_err(|error| Error::Io(std::io::Error::other(error.to_string())))?; + let mut identity = b"trail-restored-filesystem-v2\0".to_vec(); + identity.extend_from_slice(&nonce); + identity.extend_from_slice(workspace_root.as_os_str().as_encoded_bytes()); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let metadata = fs::metadata(workspace_root)?; + identity.extend_from_slice(&metadata.dev().to_be_bytes()); + identity.extend_from_slice(&metadata.ino().to_be_bytes()); + } + #[cfg(windows)] + { + let platform = windows_file_identity(workspace_root)?; + identity.extend_from_slice(&platform.volume_serial_number.to_be_bytes()); + identity.extend_from_slice(&platform.file_index.to_be_bytes()); + } + Ok(identity) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::ffi::{OsStrExt, OsStringExt}; + + #[test] + fn restored_scope_root_encodes_non_utf8_bytes_losslessly() { + let path = PathBuf::from(std::ffi::OsString::from_vec(b"/tmp/trail-\xff".to_vec())); + let encoded = restored_scope_root(&path); + + assert_eq!(encoded, "os-bytes:2f746d702f747261696c2dff"); + assert_eq!( + hex::decode(encoded.strip_prefix("os-bytes:").unwrap()).unwrap(), + path.as_os_str().as_bytes() + ); + } +} diff --git a/trail/src/db/core/backup/restore_transaction.rs b/trail/src/db/core/backup/restore_transaction.rs new file mode 100644 index 0000000..ad1e6ea --- /dev/null +++ b/trail/src/db/core/backup/restore_transaction.rs @@ -0,0 +1,375 @@ +use super::*; +use crate::db::core::backup::publication::{ + atomic_exchange, publish_staged_tree, remove_any, sync_directory_strict, +}; +use std::fs::File; + +const RESTORE_MARKER: &str = ".trail-restore-transaction.json"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +enum RestorePhase { + Prepared, + PolicyPublished, + RollingBack, + Finalizing, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +struct RestoreMarker { + format_version: u32, + phase: RestorePhase, + db_stage_leaf: String, + policy_stage_leaf: String, + had_old_db: bool, + had_old_policy: bool, + new_db_sha256: String, + new_policy_sha256: String, + old_policy_sha256: Option, +} + +pub(super) struct RestorePublication { + workspace_root: PathBuf, + marker: RestoreMarker, +} + +impl RestorePublication { + pub(super) fn prepare( + workspace_root: &Path, + db_stage: &Path, + backup_path: &Path, + force: bool, + ) -> Result<(Self, bool)> { + let policy_target = workspace_root.join(".trailignore"); + let backup_policy = backup_path.join(".trailignore"); + let restored_policy = backup_policy.is_file() && (force || !policy_target.exists()); + let policy_stage = allocate_policy_stage(workspace_root)?; + if restored_policy { + fs::copy(&backup_policy, &policy_stage)?; + } else if policy_target.is_file() { + fs::copy(&policy_target, &policy_stage)?; + } else { + fs::write( + &policy_stage, + format!("{}\n", DEFAULT_CRABIGNORE_PATTERNS.join("\n")), + )?; + } + OpenOptions::new() + .read(true) + .open(&policy_stage)? + .sync_all()?; + sync_directory_strict(workspace_root)?; + + let db_stage_leaf = confined_leaf(db_stage, workspace_root, "restore DB stage")?; + let policy_stage_leaf = + confined_leaf(&policy_stage, workspace_root, "restore policy stage")?; + let marker = RestoreMarker { + format_version: 1, + phase: RestorePhase::Prepared, + db_stage_leaf, + policy_stage_leaf, + had_old_db: workspace_root.join(".trail").exists(), + had_old_policy: policy_target.exists(), + new_db_sha256: digest_file(&db_stage.join(DB_RELATIVE_PATH))?, + new_policy_sha256: digest_file(&policy_stage)?, + old_policy_sha256: policy_target + .is_file() + .then(|| digest_file(&policy_target)) + .transpose()?, + }; + write_marker(workspace_root, &marker)?; + test_crash_point("restore_after_policy_staging"); + Ok(( + Self { + workspace_root: workspace_root.to_path_buf(), + marker, + }, + restored_policy, + )) + } + + pub(super) fn publish(mut self) -> Result<()> { + let policy_stage = self.path(&self.marker.policy_stage_leaf); + let policy_target = self.workspace_root.join(".trailignore"); + if let Err(error) = publish_policy_entry(&policy_stage, &policy_target) { + let _ = self.rollback(); + return Err(error); + } + test_crash_point("restore_after_policy_exchange_before_marker"); + self.set_phase(RestorePhase::PolicyPublished)?; + test_crash_point("restore_after_policy_publication"); + + #[cfg(test)] + if std::env::var_os("TRAIL_TEST_RESTORE_FORCE_ROLLBACK").is_some() { + self.set_phase(RestorePhase::RollingBack)?; + test_crash_point("restore_during_rollback"); + self.rollback()?; + return Err(Error::Conflict("forced restore rollback".into())); + } + + let db_stage = self.path(&self.marker.db_stage_leaf); + let db_target = self.workspace_root.join(".trail"); + if let Err(error) = publish_staged_tree(&db_stage, &db_target) { + self.set_phase(RestorePhase::RollingBack)?; + test_crash_point("restore_during_rollback"); + self.rollback()?; + return Err(error); + } + self.set_phase(RestorePhase::Finalizing)?; + test_crash_point("restore_after_trail_publication"); + test_crash_point("restore_during_finalization"); + test_crash_point("restore_before_retained_cleanup"); + finish_new_pair(&self.workspace_root, &self.marker)?; + test_crash_point("restore_after_retained_cleanup"); + Ok(()) + } + + fn set_phase(&mut self, phase: RestorePhase) -> Result<()> { + self.marker.phase = phase; + write_marker(&self.workspace_root, &self.marker) + } + + fn rollback(&self) -> Result<()> { + restore_old_pair(&self.workspace_root, &self.marker) + } + + fn path(&self, leaf: &str) -> PathBuf { + self.workspace_root.join(leaf) + } +} + +pub(crate) fn recover_restore_publication(workspace_root: &Path) -> Result<()> { + let marker_path = workspace_root.join(RESTORE_MARKER); + let bytes = match fs::read(&marker_path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + let marker: RestoreMarker = serde_json::from_slice(&bytes)?; + validate_marker(&marker)?; + match marker.phase { + RestorePhase::Prepared | RestorePhase::RollingBack => { + restore_old_pair(workspace_root, &marker) + } + RestorePhase::PolicyPublished => { + if db_target_is_new(workspace_root, &marker)? { + finish_new_pair(workspace_root, &marker) + } else { + restore_old_pair(workspace_root, &marker) + } + } + RestorePhase::Finalizing => finish_new_pair(workspace_root, &marker), + } +} + +fn restore_old_pair(workspace_root: &Path, marker: &RestoreMarker) -> Result<()> { + restore_old_entry( + workspace_root, + ".trailignore", + &marker.policy_stage_leaf, + marker.had_old_policy, + marker.old_policy_sha256.as_deref(), + &marker.new_policy_sha256, + )?; + let db_target = workspace_root.join(".trail"); + let db_stage = workspace_root.join(&marker.db_stage_leaf); + if db_target_is_new(workspace_root, marker)? { + if marker.had_old_db { + exchange_required(&db_stage, &db_target, "restore DB rollback")?; + } else { + remove_any(&db_target)?; + } + } + remove_any(&db_stage)?; + remove_any(&workspace_root.join(&marker.policy_stage_leaf))?; + remove_marker(workspace_root) +} + +fn finish_new_pair(workspace_root: &Path, marker: &RestoreMarker) -> Result<()> { + if !db_target_is_new(workspace_root, marker)? { + return Err(Error::Corrupt( + "restore transaction cannot finalize without the staged DB generation".into(), + )); + } + let policy_target = workspace_root.join(".trailignore"); + if digest_optional(&policy_target)?.as_deref() != Some(marker.new_policy_sha256.as_str()) { + let policy_stage = workspace_root.join(&marker.policy_stage_leaf); + if digest_optional(&policy_stage)?.as_deref() != Some(marker.new_policy_sha256.as_str()) { + return Err(Error::Corrupt( + "restore transaction lost the staged policy generation".into(), + )); + } + publish_policy_entry(&policy_stage, &policy_target)?; + } + remove_any(&workspace_root.join(&marker.db_stage_leaf))?; + remove_any(&workspace_root.join(&marker.policy_stage_leaf))?; + remove_marker(workspace_root) +} + +fn restore_old_entry( + workspace_root: &Path, + target_leaf: &str, + stage_leaf: &str, + had_old: bool, + old_digest: Option<&str>, + new_digest: &str, +) -> Result<()> { + let target = workspace_root.join(target_leaf); + let stage = workspace_root.join(stage_leaf); + if had_old { + if digest_optional(&target)?.as_deref() == old_digest { + return Ok(()); + } + if digest_optional(&stage)?.as_deref() != old_digest { + return Err(Error::Corrupt(format!( + "restore transaction lost retained `{target_leaf}`" + ))); + } + if target.exists() { + exchange_required(&stage, &target, "restore policy rollback")?; + } else { + fs::rename(&stage, &target)?; + sync_directory_strict(workspace_root)?; + } + } else if digest_optional(&target)?.as_deref() == Some(new_digest) { + remove_any(&target)?; + sync_directory_strict(workspace_root)?; + } + Ok(()) +} + +fn publish_policy_entry(stage: &Path, target: &Path) -> Result<()> { + let parent = target + .parent() + .ok_or_else(|| Error::InvalidInput("restore policy has no parent".into()))?; + OpenOptions::new().read(true).open(stage)?.sync_all()?; + sync_directory_strict(parent)?; + if target.exists() { + exchange_required(stage, target, "restore policy publication")?; + } else { + fs::rename(stage, target)?; + sync_directory_strict(parent)?; + } + Ok(()) +} + +fn exchange_required(left: &Path, right: &Path, label: &str) -> Result<()> { + if !atomic_exchange(left, right)? { + return Err(Error::Conflict(format!( + "atomic exchange is unsupported for {label}; live entries were not moved" + ))); + } + let parent = right + .parent() + .ok_or_else(|| Error::InvalidInput(format!("{label} target has no parent")))?; + sync_directory_strict(parent) +} + +fn db_target_is_new(workspace_root: &Path, marker: &RestoreMarker) -> Result { + Ok( + digest_optional(&workspace_root.join(".trail").join(DB_RELATIVE_PATH))?.as_deref() + == Some(marker.new_db_sha256.as_str()), + ) +} + +fn write_marker(workspace_root: &Path, marker: &RestoreMarker) -> Result<()> { + validate_marker(marker)?; + let marker_path = workspace_root.join(RESTORE_MARKER); + let temporary = workspace_root.join(format!(".{RESTORE_MARKER}.tmp-{}", now_nanos())); + fs::write(&temporary, serde_json::to_vec(marker)?)?; + OpenOptions::new().read(true).open(&temporary)?.sync_all()?; + fs::rename(&temporary, &marker_path)?; + sync_directory_strict(workspace_root) +} + +fn remove_marker(workspace_root: &Path) -> Result<()> { + remove_any(&workspace_root.join(RESTORE_MARKER))?; + sync_directory_strict(workspace_root) +} + +fn validate_marker(marker: &RestoreMarker) -> Result<()> { + if marker.format_version != 1 { + return Err(Error::Corrupt( + "unsupported restore transaction marker".into(), + )); + } + validate_marker_leaf(&marker.db_stage_leaf)?; + validate_marker_leaf(&marker.policy_stage_leaf)?; + if marker.db_stage_leaf == marker.policy_stage_leaf + || marker.new_db_sha256.len() != 64 + || marker.new_policy_sha256.len() != 64 + || marker + .old_policy_sha256 + .as_ref() + .is_some_and(|hash| hash.len() != 64) + { + return Err(Error::Corrupt("invalid restore transaction marker".into())); + } + Ok(()) +} + +fn validate_marker_leaf(leaf: &str) -> Result<()> { + let mut components = Path::new(leaf).components(); + if !matches!( + (components.next(), components.next()), + (Some(Component::Normal(_)), None) + ) || leaf.contains(['/', '\0']) + { + return Err(Error::Corrupt(format!( + "restore transaction path is not confined: `{leaf}`" + ))); + } + Ok(()) +} + +fn confined_leaf(path: &Path, parent: &Path, label: &str) -> Result { + if path.parent() != Some(parent) { + return Err(Error::InvalidInput(format!("{label} is not a sibling"))); + } + let leaf = path + .file_name() + .and_then(|leaf| leaf.to_str()) + .ok_or_else(|| Error::InvalidInput(format!("{label} has no UTF-8 leaf")))? + .to_string(); + validate_marker_leaf(&leaf)?; + Ok(leaf) +} + +fn allocate_policy_stage(workspace_root: &Path) -> Result { + for _ in 0..32 { + let path = workspace_root.join(format!("..trailignore.restore-stage-{}", now_nanos())); + match OpenOptions::new().write(true).create_new(true).open(&path) { + Ok(file) => { + file.sync_all()?; + return Ok(path); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error.into()), + } + } + Err(Error::Conflict( + "could not allocate restore policy stage".into(), + )) +} + +fn digest_optional(path: &Path) -> Result> { + match digest_file(path) { + Ok(digest) => Ok(Some(digest)), + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } +} + +fn digest_file(path: &Path) -> Result { + let mut file = File::open(path)?; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(hex::encode(hasher.finalize())) +} diff --git a/trail/src/db/core/backup/verify.rs b/trail/src/db/core/backup/verify.rs index 8dc7253..39423fd 100644 --- a/trail/src/db/core/backup/verify.rs +++ b/trail/src/db/core/backup/verify.rs @@ -74,10 +74,15 @@ impl Trail { )); let verify_open = (|| -> Result { fs::create_dir_all(verify_dir.join("index"))?; + fs::write(verify_dir.join("index").join(SCHEMA_EXCLUSION_FILE), [])?; + fs::write( + verify_dir.join("index").join(SCHEMA_VALIDATION_LEADER_FILE), + [], + )?; fs::copy(path.join(CONFIG_FILE), verify_dir.join(CONFIG_FILE))?; fs::copy(path.join(HEAD_FILE), verify_dir.join(HEAD_FILE))?; fs::copy(&sqlite_path, verify_dir.join(DB_RELATIVE_PATH))?; - Trail::open_with_db_dir(&verify_dir, &verify_dir) + Trail::open_without_recovering_derived_paths(&verify_dir, &verify_dir) })(); match verify_open { Ok(db) => match db.fsck() { @@ -88,6 +93,23 @@ impl Trail { errors.extend(fsck.errors); workspace_id.get_or_insert_with(|| db.config.workspace.id.clone()); branch.get_or_insert(db.current_branch()?); + let trusted_scopes: i64 = db.conn.query_row( + "SELECT COUNT(*) FROM changed_path_scopes + WHERE retired_at IS NULL AND trust_state='trusted'", + [], + |row| row.get(0), + )?; + let live_observers: i64 = db.conn.query_row( + "SELECT (SELECT COUNT(*) FROM changed_path_observer_owners) + + (SELECT COUNT(*) FROM changed_path_observer_segments)", + [], + |row| row.get(0), + )?; + if trusted_scopes != 0 || live_observers != 0 { + errors.push( + "changed-path backup is not fenced or marked untrusted".to_string(), + ); + } } Err(err) => errors.push(format!("fsck failed: {err}")), }, diff --git a/trail/src/db/core/doctor.rs b/trail/src/db/core/doctor.rs index b933444..5a001cb 100644 --- a/trail/src/db/core/doctor.rs +++ b/trail/src/db/core/doctor.rs @@ -17,7 +17,7 @@ impl Trail { doctor_activity::push_pending_approvals_check(self, &mut checks); doctor_activity::push_active_leases_check(self, &mut checks); - doctor_activity::push_merge_queue_check(self, &mut checks); + doctor_activity::push_lane_merge_queue_check(self, &mut checks); doctor_activity::push_conflicts_check(self, &mut checks); doctor_activity::push_lanes_check(self, &mut checks); diff --git a/trail/src/db/core/doctor_activity.rs b/trail/src/db/core/doctor_activity.rs index e001541..4850af9 100644 --- a/trail/src/db/core/doctor_activity.rs +++ b/trail/src/db/core/doctor_activity.rs @@ -46,8 +46,8 @@ pub(super) fn push_active_leases_check(db: &Trail, checks: &mut Vec } } -pub(super) fn push_merge_queue_check(db: &Trail, checks: &mut Vec) { - match db.list_merge_queue() { +pub(super) fn push_lane_merge_queue_check(db: &Trail, checks: &mut Vec) { + match db.list_lane_merge_queue() { Ok(entries) => { let queued = entries .iter() @@ -71,14 +71,14 @@ pub(super) fn push_merge_queue_check(db: &Trail, checks: &mut Vec) "ok" }; let message = if status == "ok" { - "merge queue has no pending attention".to_string() + "lane merge queue has no pending attention".to_string() } else { format!( - "merge queue has {queued} queued, {running} running, {conflicted} conflicted, and {failed} failed item(s)" + "lane merge queue has {queued} queued, {running} running, {conflicted} conflicted, and {failed} failed item(s)" ) }; checks.push(doctor_check( - "merge_queue", + "lane_merge_queue", status, message, Some(serde_json::json!({ @@ -91,9 +91,9 @@ pub(super) fn push_merge_queue_check(db: &Trail, checks: &mut Vec) )); } Err(err) => checks.push(doctor_check( - "merge_queue", + "lane_merge_queue", "error", - format!("could not list merge queue: {err}"), + format!("could not list lane merge queue: {err}"), None, )), } diff --git a/trail/src/db/core/doctor_storage.rs b/trail/src/db/core/doctor_storage.rs index 6e4b347..9a1b82b 100644 --- a/trail/src/db/core/doctor_storage.rs +++ b/trail/src/db/core/doctor_storage.rs @@ -233,10 +233,13 @@ pub(super) fn push_workspace_views_check(db: &Trail, checks: &mut Vec Path::new("/dev/fuse").exists(), - "nfs-cow" if cfg!(target_os = "macos") => Path::new("/sbin/mount_nfs").is_file(), - "overlay-cow" if cfg!(target_os = "windows") => true, - _ => true, + "fuse" if cfg!(target_os = "linux") => Path::new("/dev/fuse").exists(), + "fuse" if cfg!(target_os = "macos") => cfg!(feature = "macfuse"), + "nfs" if cfg!(target_os = "macos") => Path::new("/sbin/mount_nfs").is_file(), + "dokan" if cfg!(target_os = "windows") => true, + "clone" | "virtual" => true, + "fuse" | "nfs" | "dokan" => false, + _ => false, }; if !backend_available { errors.push(format!( diff --git a/trail/src/db/core/init.rs b/trail/src/db/core/init.rs index 89cc38b..4429e6e 100644 --- a/trail/src/db/core/init.rs +++ b/trail/src/db/core/init.rs @@ -4,6 +4,58 @@ const AUTO_MINIMAL_FILES_THRESHOLD: usize = 10_000; const AUTO_MINIMAL_BYTES_THRESHOLD: u64 = 128 * 1024 * 1024; const DETAILED_INIT_CHANGES_FILE_THRESHOLD: usize = 10_000; +#[cfg(test)] +thread_local! { + static SCHEMA_HANDOFF_HOOK: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; + static SCHEMA_PRIMARY_OPEN_HOOK: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; + static SCHEMA_PROLLY_OPEN_HOOK: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(test)] +fn install_schema_primary_open_hook(hook: impl FnOnce(&Path) + 'static) { + SCHEMA_PRIMARY_OPEN_HOOK.with(|slot| *slot.borrow_mut() = Some(Box::new(hook))); +} + +#[cfg(test)] +fn run_schema_primary_open_hook(db_dir: &Path) { + SCHEMA_PRIMARY_OPEN_HOOK.with(|slot| { + if let Some(hook) = slot.borrow_mut().take() { + hook(db_dir); + } + }); +} + +#[cfg(test)] +fn install_schema_prolly_open_hook(hook: impl FnOnce(&Path) + 'static) { + SCHEMA_PROLLY_OPEN_HOOK.with(|slot| *slot.borrow_mut() = Some(Box::new(hook))); +} + +#[cfg(test)] +fn run_schema_prolly_open_hook(db_dir: &Path) { + SCHEMA_PROLLY_OPEN_HOOK.with(|slot| { + if let Some(hook) = slot.borrow_mut().take() { + hook(db_dir); + } + }); +} + +#[cfg(test)] +fn install_schema_handoff_hook(hook: impl FnOnce(&Path) + 'static) { + SCHEMA_HANDOFF_HOOK.with(|slot| *slot.borrow_mut() = Some(Box::new(hook))); +} + +#[cfg(test)] +fn run_schema_handoff_hook(db_dir: &Path) { + SCHEMA_HANDOFF_HOOK.with(|slot| { + if let Some(hook) = slot.borrow_mut().take() { + hook(db_dir); + } + }); +} + impl Trail { pub fn init( workspace_root: impl AsRef, @@ -50,7 +102,8 @@ impl Trail { text_policy: Option<&str>, prolly_backend: Option<&str>, ) -> Result { - let workspace_root = workspace_root.as_ref().canonicalize()?; + let workspace_root = canonicalize_lossless(workspace_root.as_ref())?; + super::backup::recover_restore_publication(&workspace_root)?; let db_dir = workspace_root.join(".trail"); if db_dir.exists() { if !force { @@ -60,6 +113,14 @@ impl Trail { } fs::create_dir_all(db_dir.join("index"))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&db_dir, fs::Permissions::from_mode(0o700))?; + fs::set_permissions(db_dir.join("index"), fs::Permissions::from_mode(0o700))?; + } + fs::write(db_dir.join("index").join(SCHEMA_EXCLUSION_FILE), [])?; + fs::write(db_dir.join("index").join(SCHEMA_VALIDATION_LEADER_FILE), [])?; fs::create_dir_all(db_dir.join("refs/branches"))?; fs::create_dir_all(db_dir.join("refs/lanes"))?; fs::create_dir_all(db_dir.join("worktrees"))?; @@ -95,8 +156,7 @@ impl Trail { fs::write(db_dir.join(HEAD_FILE), format!("{branch}\n"))?; write_default_trailignore(&workspace_root)?; - let mut db = Self::open_at(workspace_root, db_dir, config)?; - db.init_schema()?; + let mut db = Self::open_at(workspace_root, db_dir, config, SchemaOpenMode::FreshCreate)?; let actor = Actor::system(); let change_id = db.allocate_change_id(&actor.id, "init")?; @@ -209,12 +269,13 @@ impl Trail { } pub fn discover(start: impl AsRef) -> Result { - let mut current = start.as_ref().canonicalize()?; + let mut current = canonicalize_lossless(start.as_ref())?; loop { + super::backup::recover_restore_publication(¤t)?; let db_dir = current.join(".trail"); if db_dir.is_dir() { let config = read_config(&db_dir)?; - return Self::open_at(current, db_dir, config); + return Self::open_at(current, db_dir, config, SchemaOpenMode::Existing); } if !current.pop() { return Err(Error::WorkspaceNotFound(start.as_ref().to_path_buf())); @@ -223,44 +284,149 @@ impl Trail { } pub fn open(workspace_root: impl AsRef) -> Result { - let workspace_root = workspace_root.as_ref().canonicalize()?; + let workspace_root = canonicalize_lossless(workspace_root.as_ref())?; + super::backup::recover_restore_publication(&workspace_root)?; let db_dir = workspace_root.join(".trail"); if !db_dir.is_dir() { return Err(Error::WorkspaceNotFound(workspace_root)); } let config = read_config(&db_dir)?; - Self::open_at(workspace_root, db_dir, config) + Self::open_at(workspace_root, db_dir, config, SchemaOpenMode::Existing) } pub fn open_with_db_dir( workspace_root: impl AsRef, db_dir: impl AsRef, ) -> Result { - let workspace_root = workspace_root.as_ref().canonicalize()?; - let db_dir = db_dir.as_ref().canonicalize()?; + let workspace_root = canonicalize_lossless(workspace_root.as_ref())?; + super::backup::recover_restore_publication(&workspace_root)?; + let db_dir = canonicalize_lossless(db_dir.as_ref())?; if !db_dir.is_dir() { return Err(Error::WorkspaceNotFound(db_dir)); } let config = read_config(&db_dir)?; - Self::open_at(workspace_root, db_dir, config) + Self::open_at(workspace_root, db_dir, config, SchemaOpenMode::Existing) } pub(crate) fn open_at( workspace_root: PathBuf, db_dir: PathBuf, config: TrailConfig, + schema_mode: SchemaOpenMode, + ) -> Result { + let db = + Self::open_at_without_recovery(workspace_root, db_dir, config, schema_mode, false)?; + db.recover_after_open()?; + Ok(db) + } + + pub(crate) fn open_without_recovering_derived_paths( + workspace_root: impl AsRef, + db_dir: impl AsRef, + ) -> Result { + let workspace_root = canonicalize_lossless(workspace_root.as_ref())?; + let db_dir = canonicalize_lossless(db_dir.as_ref())?; + if !db_dir.is_dir() { + return Err(Error::WorkspaceNotFound(db_dir)); + } + let config = read_config(&db_dir)?; + Self::open_at_without_recovery( + workspace_root, + db_dir, + config, + SchemaOpenMode::Existing, + false, + ) + } + + pub(crate) fn open_without_recovering_derived_paths_under_write_lock( + workspace_root: impl AsRef, + db_dir: impl AsRef, + ) -> Result { + let workspace_root = canonicalize_lossless(workspace_root.as_ref())?; + let db_dir = canonicalize_lossless(db_dir.as_ref())?; + if !db_dir.is_dir() { + return Err(Error::WorkspaceNotFound(db_dir)); + } + let config = read_config(&db_dir)?; + // The caller owns the workspace write lock and already holds a Trail handle whose + // mutable handoff completed schema validation under that same exclusion. + Self::open_at_without_recovery( + workspace_root, + db_dir, + config, + SchemaOpenMode::Existing, + true, + ) + } + + fn open_at_without_recovery( + workspace_root: PathBuf, + db_dir: PathBuf, + config: TrailConfig, + schema_mode: SchemaOpenMode, + writer_exclusion_held: bool, ) -> Result { - fs::create_dir_all(db_dir.join("index"))?; let sqlite_path = db_dir.join(DB_RELATIVE_PATH); + let validated_schema = match schema_mode { + SchemaOpenMode::FreshCreate => None, + SchemaOpenMode::Existing if writer_exclusion_held => None, + SchemaOpenMode::Existing => { + Some(Self::with_write_lock_wait(Duration::from_secs(10), || { + preflight_existing_schema(&sqlite_path, &config.storage.prolly_backend) + })?) + } + }; + if schema_mode == SchemaOpenMode::FreshCreate { + fs::create_dir_all(db_dir.join("index"))?; + } + #[cfg(test)] + if schema_mode == SchemaOpenMode::Existing && !writer_exclusion_held { + run_schema_handoff_hook(&db_dir); + } + if let Some(validated) = &validated_schema { + validated.verify_unchanged()?; + } register_sqlite_vec_extension()?; - let store = open_prolly_store(&config, &sqlite_path)?; - let conn = Connection::open(&sqlite_path)?; + let operation_metrics = + operation_metrics_are_enabled().then(|| Arc::new(OperationMetricsState::default())); + #[cfg(test)] + if validated_schema.is_some() { + run_schema_primary_open_hook(&db_dir); + } + let conn = match schema_mode { + SchemaOpenMode::FreshCreate => Connection::open(&sqlite_path)?, + SchemaOpenMode::Existing => Connection::open_with_flags( + &sqlite_path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE + | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, + )?, + }; + if let Some(validated) = &validated_schema { + validated.verify_connection(&conn)?; + } + conn.set_db_config( + rusqlite::config::DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, + true, + )?; + #[cfg(test)] + if validated_schema.is_some() { + run_schema_prolly_open_hook(&db_dir); + } + let store = open_prolly_store( + &config, + &sqlite_path, + operation_metrics.clone(), + schema_mode, + validated_schema.as_ref(), + )?; apply_sqlite_pragmas(&conn)?; let prolly = Prolly::new(store.clone(), prolly_config()); let root_prolly = Prolly::new(store.clone(), root_map_prolly_config()); let db = Self { workspace_root, db_dir, + sqlite_path, conn, store, prolly, @@ -268,12 +434,56 @@ impl Trail { config, object_cache: Mutex::new(ObjectCache::default()), daemon_worktree_cache: None, + changed_path_daemon_registry: Mutex::new( + change_ledger::ChangedPathDaemonRegistry::default(), + ), + git_handoff_metrics: Cell::new(GitHandoffMetrics::default()), + case_fold_index_metrics: Cell::new(CaseFoldIndexMetrics::default()), + operation_metrics, }; - db.init_schema()?; - db.recover_workspace_views()?; + if schema_mode == SchemaOpenMode::FreshCreate { + db.create_schema_v18()?; + } Ok(db) } + pub(crate) fn recover_after_open(&self) -> Result<()> { + let lock_path = self.db_dir.join("lock"); + if lock_path.exists() + && fs::read_to_string(&lock_path) + .ok() + .and_then(|holder| { + holder.split_whitespace().find_map(|part| { + part.strip_prefix("pid=") + .and_then(|value| value.parse::().ok()) + }) + }) + .is_none() + { + // Legacy or manually managed writer locks have always allowed a read-only open. + // Defer recovery until a later open after that writer releases its lock. + return Ok(()); + } + match Self::with_write_lock_wait(Duration::from_secs(30), || { + let _lock = self.acquire_write_lock()?; + self.recover_after_open_under_write_lock() + }) { + Err(Error::WorkspaceLocked(_)) => Ok(()), + result => result, + } + } + + fn recover_after_open_under_write_lock(&self) -> Result<()> { + if self.has_pending_path_index_derived_repairs()? { + self.drain_pending_path_index_derived_repairs()?; + } + self.recover_materialization_stages()?; + self.recover_workspace_views()?; + self.recover_workspace_environment_sync_attempts()?; + self.recover_workspace_runtime_leases()?; + Ok(()) + } + pub fn workspace_root(&self) -> &Path { &self.workspace_root } @@ -282,6 +492,10 @@ impl Trail { &self.db_dir } + pub(crate) fn changed_path_ledger(&self) -> super::change_ledger::ChangedPathLedger<'_> { + super::change_ledger::ChangedPathLedger::new_at(&self.conn, &self.sqlite_path) + } + pub fn config(&self) -> &TrailConfig { &self.config } @@ -343,38 +557,17 @@ impl Trail { } pub(crate) fn acquire_write_lock(&self) -> Result { - let path = self.db_dir.join("lock"); - let mut delay = Duration::from_millis(2); - let mut file = loop { - match OpenOptions::new().write(true).create_new(true).open(&path) { - Ok(file) => break file, - Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { - let holder = - fs::read_to_string(&path).unwrap_or_else(|_| "unknown writer".to_string()); - if is_stale_lock_holder(&holder) - && fs::read_to_string(&path).unwrap_or_default() == holder - { - match fs::remove_file(&path) { - Ok(()) => continue, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, - Err(err) => return Err(Error::Io(err)), - } - } - let should_wait = WRITE_LOCK_WAIT_DEADLINE - .with(|deadline| deadline.get()) - .is_some_and(|deadline| Instant::now() < deadline); - if should_wait { - std::thread::sleep(delay); - delay = (delay * 2).min(Duration::from_millis(50)); - continue; - } - return Err(Error::WorkspaceLocked(holder.trim().to_string())); - } - Err(err) => return Err(Error::Io(err)), - } - }; - writeln!(file, "pid={} created_at={}", std::process::id(), now_ts())?; - Ok(WorkspaceLock { path }) + // Every in-process Trail mutation is an authorized handoff against + // native observer durability. Register before waiting so no new + // observer append/heartbeat can enter, then retain the exclusion for + // the complete workspace-lock lifetime. A writer in another process + // cannot publish this process-local authority and remains terminal to + // observer continuity. + let exclusion = begin_authorized_observer_write_exclusion(&self.db_dir); + wait_for_observer_workspace_lock(&self.db_dir); + let mut lock = acquire_workspace_lock(&self.db_dir)?; + lock.observer_write_exclusion = Some(exclusion); + Ok(lock) } } @@ -435,16 +628,1409 @@ fn worktree_path_scan_is_large(scan: &WorktreePathScan) -> bool { || scan.total_bytes > AUTO_MINIMAL_BYTES_THRESHOLD } -fn is_stale_lock_holder(holder: &str) -> bool { - let Some(pid) = lock_holder_pid(holder) else { - return false; +#[cfg(test)] +mod schema_handoff_tests { + use super::*; + use std::io::Write; + use std::process::{Child, Stdio}; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Barrier, }; - !process_is_alive(pid) -} -fn lock_holder_pid(holder: &str) -> Option { - holder.split_whitespace().find_map(|part| { - part.strip_prefix("pid=") - .and_then(|value| value.parse::().ok()) - }) + #[cfg(unix)] + fn assert_open_replacement_is_rejected_before_mutation( + install: impl FnOnce(Box), + ) { + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_path = root.path().join(".trail").join(DB_RELATIVE_PATH); + let original_bytes = fs::read(&db_path).unwrap(); + let replacement = db_path.with_extension("replacement.sqlite"); + fs::copy(&db_path, &replacement).unwrap(); + let replacement_bytes = fs::read(&replacement).unwrap(); + let retained = db_path.with_extension("validated.sqlite"); + let hook_db = db_path.clone(); + let hook_replacement = replacement.clone(); + let hook_retained = retained.clone(); + install(Box::new(move |_| { + fs::rename(&hook_db, &hook_retained).unwrap(); + fs::rename(&hook_replacement, &hook_db).unwrap(); + })); + let error = Trail::open(root.path()).err(); + assert!( + matches!(error, Some(Error::SchemaReinitializeRequired { .. })), + "replacement rejection returned {error:?}" + ); + assert_eq!(fs::read(retained).unwrap(), original_bytes); + assert_eq!(fs::read(db_path).unwrap(), replacement_bytes); + } + + #[cfg(unix)] + #[test] + fn primary_sqlite_open_binds_the_validated_inode_before_any_statement() { + assert_open_replacement_is_rejected_before_mutation(|hook| { + install_schema_primary_open_hook(hook) + }); + } + + #[cfg(unix)] + #[test] + fn prolly_sqlite_open_binds_the_validated_inode_before_any_pragma() { + assert_open_replacement_is_rejected_before_mutation(|hook| { + install_schema_prolly_open_hook(hook) + }); + } + + const CROSS_PROCESS_TEST: &str = + "db::core::init::schema_handoff_tests::cross_process_schema_validation_fanout"; + const SCHEMA_LEADER_LOCK_PROBE_TEST: &str = + "db::core::init::schema_handoff_tests::schema_leader_lock_probe"; + + fn wait_for_path(path: &Path, deadline: Duration) { + let started = Instant::now(); + while !path.exists() { + assert!( + started.elapsed() < deadline, + "timed out waiting for {}", + path.display() + ); + std::thread::sleep(Duration::from_millis(10)); + } + } + + fn validation_count(path: &Path) -> usize { + fs::read_to_string(path).unwrap_or_default().lines().count() + } + + fn spawn_schema_children( + root: &Path, + counter: &Path, + go: &Path, + count: usize, + mode: &str, + configure: impl Fn(&mut Command), + ) -> Vec { + let ready_dir = go.with_extension("ready"); + fs::create_dir(&ready_dir).unwrap(); + let children = (0..count) + .map(|index| { + let ready = ready_dir.join(format!("{index}.ready")); + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", CROSS_PROCESS_TEST, "--nocapture"]) + .env("RUST_TEST_THREADS", "1") + .env("TRAIL_TEST_SCHEMA_CHILD", mode) + .env("TRAIL_TEST_SCHEMA_WORKSPACE", root) + .env("TRAIL_TEST_SCHEMA_GO", go) + .env("TRAIL_TEST_SCHEMA_READY", &ready) + .env("TRAIL_TEST_SCHEMA_VALIDATION_COUNTER", counter) + // These waves prove exact-one fanout while the authenticated + // leader remains live for the server's bounded grace period. + // A process that exits earlier may conservatively trigger a + // second read-only validation. + .env("TRAIL_TEST_SCHEMA_CHILD_LINGER_MS", "400") + .stdout(Stdio::null()) + .stderr(Stdio::inherit()); + configure(&mut command); + command.spawn().unwrap() + }) + .collect::>(); + let started = Instant::now(); + while fs::read_dir(&ready_dir).unwrap().count() != count { + assert!(started.elapsed() < Duration::from_secs(10)); + std::thread::sleep(Duration::from_millis(5)); + } + children + } + + fn release_and_wait(mut children: Vec, go: &Path) { + fs::write(go, []).unwrap(); + let deadline = Instant::now() + Duration::from_secs(15); + for child in &mut children { + loop { + if let Some(status) = child.try_wait().unwrap() { + assert!(status.success(), "schema validation child {status}"); + break; + } + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("schema validation child timed out"); + } + std::thread::sleep(Duration::from_millis(10)); + } + } + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + fn test_schema_validation_server( + root: &Path, + ) -> (SchemaValidationServer, PathBuf, PathBuf, String, String) { + use std::os::unix::net::UnixListener; + + let lock_path = root.join("schema-validation.lock"); + let leader_exclusion = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .unwrap(); + rustix::fs::fcntl_lock( + &leader_exclusion, + rustix::fs::FlockOperation::NonBlockingLockExclusive, + ) + .unwrap(); + let socket_path = root.join("schema-validation.socket"); + let listener = UnixListener::bind(&socket_path).unwrap(); + listener.set_nonblocking(true).unwrap(); + let socket_cleanup = SchemaValidationRuntimeEntry::capture(&socket_path).unwrap(); + let socket_authority = + secure_schema_validation_socket(&socket_path, &socket_cleanup).unwrap(); + let announcement_path = root.join("schema-validation.announce"); + fs::write(&announcement_path, b"test announcement").unwrap(); + let announcement_cleanup = + SchemaValidationRuntimeEntry::capture(&announcement_path).unwrap(); + let nonce = "a".repeat(64); + let generation = "b".repeat(64); + ( + SchemaValidationServer { + listener, + nonce: nonce.clone(), + generation: generation.clone(), + backend: "sqlite".to_owned(), + outcome: CrossProcessSchemaValidationOutcome::Success("a".repeat(64)), + _leader_exclusion: leader_exclusion, + _socket_authority: socket_authority, + _socket_cleanup: socket_cleanup, + _announcement_cleanup: announcement_cleanup, + panic_on_serve: false, + start_delay: Duration::ZERO, + shutdown_delay: Duration::ZERO, + }, + socket_path, + lock_path, + nonce, + generation, + ) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + fn assert_schema_leader_lock_available_to_child(lock_path: &Path) { + let status = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", SCHEMA_LEADER_LOCK_PROBE_TEST, "--nocapture"]) + .env("TRAIL_TEST_SCHEMA_LEADER_LOCK_PROBE", lock_path) + .status() + .unwrap(); + assert!(status.success(), "schema leader lock remained held"); + } + + fn run_schema_child() -> bool { + fn linger_after_open() { + if let Some(delay) = std::env::var_os("TRAIL_TEST_SCHEMA_CHILD_LINGER_MS") { + std::thread::sleep(Duration::from_millis( + delay.to_string_lossy().parse::().unwrap(), + )); + } + } + + let Ok(mode) = std::env::var("TRAIL_TEST_SCHEMA_CHILD") else { + return false; + }; + let root = PathBuf::from(std::env::var_os("TRAIL_TEST_SCHEMA_WORKSPACE").unwrap()); + let go = PathBuf::from(std::env::var_os("TRAIL_TEST_SCHEMA_GO").unwrap()); + fs::write(std::env::var_os("TRAIL_TEST_SCHEMA_READY").unwrap(), []).unwrap(); + wait_for_path(&go, Duration::from_secs(10)); + let result = Trail::open(root); + match mode.as_str() { + "success" | "crash" => result.unwrap(), + "failure" => { + let error = match result { + Ok(_) => panic!("injected schema failure opened"), + Err(error) => error, + }; + assert!(matches!(error, Error::SchemaReinitializeRequired { .. })); + assert!(error.to_string().contains("cross-process injected failure")); + linger_after_open(); + return true; + } + "schema-failure" => { + assert!(matches!( + result, + Err(Error::SchemaReinitializeRequired { .. }) + )); + linger_after_open(); + return true; + } + other => panic!("unknown schema child mode {other}"), + }; + linger_after_open(); + true + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn schema_leader_lock_probe() { + let Some(lock_path) = std::env::var_os("TRAIL_TEST_SCHEMA_LEADER_LOCK_PROBE") else { + return; + }; + let lock = OpenOptions::new() + .read(true) + .write(true) + .open(lock_path) + .unwrap(); + rustix::fs::fcntl_lock(&lock, rustix::fs::FlockOperation::NonBlockingLockExclusive) + .unwrap(); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn cross_process_schema_validation_fanout() { + if run_schema_child() { + return; + } + + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let writer = Connection::open(root.path().join(".trail").join(DB_RELATIVE_PATH)).unwrap(); + writer + .execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;") + .unwrap(); + writer + .execute( + "UPDATE schema_meta SET value='success-wave' WHERE key='app.version'", + [], + ) + .unwrap(); + let counter = root.path().join("success-count"); + let go = root.path().join("success-go"); + let started_marker = root.path().join("success-started"); + let started = Instant::now(); + let children = + spawn_schema_children(root.path(), &counter, &go, 16, "success", |command| { + command + .env("TRAIL_TEST_SCHEMA_VALIDATION_DELAY_MS", "1000") + .env("TRAIL_TEST_SCHEMA_VALIDATION_STARTED", &started_marker); + }); + fs::write(&go, []).unwrap(); + wait_for_path(&started_marker, Duration::from_secs(5)); + let exclusion = + File::open(root.path().join(".trail/index").join(SCHEMA_EXCLUSION_FILE)).unwrap(); + assert_eq!( + rustix::fs::flock( + &exclusion, + rustix::fs::FlockOperation::NonBlockingLockExclusive, + ) + .unwrap_err(), + rustix::io::Errno::AGAIN, + "external schema writer was not excluded during validation" + ); + release_and_wait(children, &go); + assert_eq!(validation_count(&counter), 1); + assert!(started.elapsed() < Duration::from_secs(3)); + + writer + .execute( + "UPDATE schema_meta SET value='failure-wave' WHERE key='app.version'", + [], + ) + .unwrap(); + let failure_counter = root.path().join("failure-count"); + let failure_go = root.path().join("failure-go"); + let children = spawn_schema_children( + root.path(), + &failure_counter, + &failure_go, + 16, + "failure", + |command| { + command + .env( + "TRAIL_TEST_SCHEMA_VALIDATION_FAIL", + "cross-process injected failure", + ) + .env("TRAIL_TEST_SCHEMA_VALIDATION_DELAY_MS", "500"); + }, + ); + release_and_wait(children, &failure_go); + assert_eq!(validation_count(&failure_counter), 1); + + writer + .execute( + "UPDATE schema_meta SET value='crash-wave' WHERE key='app.version'", + [], + ) + .unwrap(); + let crash_counter = root.path().join("crash-count"); + let crash_go = root.path().join("crash-go"); + let crash_once = root.path().join("crash-once"); + let mut children = spawn_schema_children( + root.path(), + &crash_counter, + &crash_go, + 16, + "crash", + |command| { + command.env("TRAIL_TEST_SCHEMA_VALIDATION_CRASH_ONCE", &crash_once); + }, + ); + fs::write(&crash_go, []).unwrap(); + wait_for_path(&crash_once, Duration::from_secs(5)); + let leader_pid = fs::read_to_string(&crash_once) + .unwrap() + .trim() + .parse::() + .unwrap(); + let leader = children + .iter_mut() + .find(|child| child.id() == leader_pid) + .unwrap(); + leader.kill().unwrap(); + let _ = leader.wait().unwrap(); + children.retain(|child| child.id() != leader_pid); + release_and_wait(children, &crash_go); + assert_eq!( + validation_count(&crash_counter), + 2, + "crash validation leaders: {}", + fs::read_to_string(&crash_counter).unwrap_or_default() + ); + + writer + .execute( + "UPDATE schema_meta SET value='later-generation' WHERE key='app.version'", + [], + ) + .unwrap(); + let later_counter = root.path().join("later-count"); + let later_go = root.path().join("later-go"); + let children = spawn_schema_children( + root.path(), + &later_counter, + &later_go, + 16, + "success", + |command| { + command.env("TRAIL_TEST_SCHEMA_VALIDATION_DELAY_MS", "500"); + }, + ); + release_and_wait(children, &later_go); + assert_eq!(validation_count(&later_counter), 1); + + let db_path = + canonicalize_lossless(&root.path().join(".trail").join(DB_RELATIVE_PATH)).unwrap(); + let (runtime_dir, namespace) = schema_validation_runtime_namespace(&db_path) + .unwrap() + .unwrap(); + let prefix = format!("{}-", &namespace[..24]); + let leftovers = fs::read_dir(runtime_dir) + .unwrap() + .filter_map(|entry| entry.ok()) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| { + name.starts_with(&prefix) + && (name.ends_with(".socket") || name.ends_with(".announce")) + }) + .collect::>(); + assert!( + leftovers.is_empty(), + "schema validation runtime artifacts survived bounded cleanup: {leftovers:?}" + ); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn ipc_result_is_bound_to_exact_main_wal_shm_generation_and_backend() { + let mut generation = SchemaGeneration( + ["", "-wal", "-shm", "-journal"] + .into_iter() + .enumerate() + .map(|(index, suffix)| SchemaFileGeneration { + suffix, + present: true, + device: 10, + inode: 20 + index as u64, + length: 30 + index as u64, + modified_seconds: 40, + modified_nanoseconds: 50, + changed_seconds: 60, + changed_nanoseconds: 70, + }) + .collect(), + ); + let original_key = schema_generation_key(&generation); + let response = schema_validation_wire_result( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + &original_key, + "sqlite", + &CrossProcessSchemaValidationOutcome::Success("d".repeat(64)), + ); + for suffix in ["", "-wal", "-shm"] { + let index = generation + .0 + .iter() + .position(|file| file.suffix == suffix) + .unwrap(); + generation.0[index].inode += 1; + let changed_key = schema_generation_key(&generation); + assert!(parse_schema_validation_wire_result( + &response, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + &changed_key, + "sqlite", + ) + .is_none()); + generation.0[index].inode -= 1; + } + assert!(parse_schema_validation_wire_result( + &response, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + &original_key, + "slatedb", + ) + .is_none()); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_schema_socket_is_really_published_owner_only() { + use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; + + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_path = + canonicalize_lossless(&root.path().join(".trail").join(DB_RELATIVE_PATH)).unwrap(); + let mut generation = schema_generation(&db_path).unwrap(); + + let mut authenticated_wal_digest = None; + let (validation, foreground_leader) = coordinate_schema_snapshot_validation( + &db_path, + "sqlite", + &mut generation, + &mut authenticated_wal_digest, + ); + validation.unwrap(); + assert!( + foreground_leader.is_none(), + "Linux silently fell back to per-process validation instead of publishing IPC" + ); + assert!(authenticated_wal_digest.is_some()); + + let (runtime_dir, namespace) = schema_validation_runtime_namespace(&db_path) + .unwrap() + .unwrap(); + let prefix = format!("{}-", &namespace[..24]); + let announcement = fs::read_dir(&runtime_dir) + .unwrap() + .filter_map(|entry| entry.ok()) + .find(|entry| { + let name = entry.file_name(); + let name = name.to_string_lossy(); + name.starts_with(&prefix) && name.ends_with(".announce") + }) + .expect("Linux schema validation announcement was not published"); + let (_, socket_path) = read_schema_validation_announcement( + &announcement.path(), + std::process::id(), + &namespace, + ) + .expect("published Linux announcement was not authenticated"); + let metadata = fs::symlink_metadata(&socket_path).unwrap(); + assert!(metadata.file_type().is_socket()); + assert_eq!(metadata.uid(), rustix::process::getuid().as_raw()); + assert_eq!(metadata.permissions().mode() & 0o777, 0o600); + + assert!(stop_schema_validation_server(&db_path)); + } + + #[cfg(target_os = "linux")] + #[test] + fn linux_schema_socket_rejects_last_moment_path_substitution() { + use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; + use std::os::unix::net::UnixListener; + + let root = tempfile::tempdir().unwrap(); + let socket_path = root.path().join("schema.socket"); + let retained_path = root.path().join("retained.socket"); + let _listener = UnixListener::bind(&socket_path).unwrap(); + let expected = SchemaValidationRuntimeEntry::capture(&socket_path).unwrap(); + let hook_retained = retained_path.clone(); + install_schema_socket_authority_hook( + &socket_path, + Box::new(move |path| { + fs::rename(path, &hook_retained).unwrap(); + let replacement = UnixListener::bind(path).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o644)).unwrap(); + drop(replacement); + }), + ); + + let result = secure_schema_validation_socket(&socket_path, &expected); + assert!( + result.is_err(), + "descriptor-bound socket authority accepted a substituted pathname" + ); + let retained = fs::symlink_metadata(&retained_path).unwrap(); + let replacement = fs::symlink_metadata(&socket_path).unwrap(); + assert!(retained.file_type().is_socket()); + assert_eq!(retained.dev(), expected.device); + assert_eq!(retained.ino(), expected.inode); + assert_eq!(retained.permissions().mode() & 0o777, 0o600); + assert!(replacement.file_type().is_socket()); + assert_ne!(replacement.ino(), expected.inode); + assert_eq!(replacement.permissions().mode() & 0o777, 0o644); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn ipc_rejects_spoofed_announcement_and_nonleader_peer() { + use std::os::unix::fs::PermissionsExt; + use std::os::unix::net::UnixListener; + + let root = tempfile::tempdir().unwrap(); + let announcement = root.path().join("spoof.announce"); + fs::write( + &announcement, + format!( + "trail-schema-validation-announce-v1\n{}\nwrong-namespace\n{}\n{}\n", + std::process::id(), + "a".repeat(64), + hex::encode( + root.path() + .join("spoof.socket") + .as_os_str() + .as_encoded_bytes() + ), + ), + ) + .unwrap(); + fs::set_permissions(&announcement, fs::Permissions::from_mode(0o600)).unwrap(); + assert!(read_schema_validation_announcement( + &announcement, + std::process::id(), + "expected-namespace", + ) + .is_none()); + + let socket = root.path().join("peer.socket"); + let listener = UnixListener::bind(&socket).unwrap(); + let accept = std::thread::spawn(move || listener.accept().unwrap()); + assert!(request_schema_validation_result( + &socket, + &"b".repeat(64), + &"c".repeat(64), + "sqlite", + std::process::id().wrapping_add(1), + ) + .is_none()); + drop(accept.join().unwrap()); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn real_main_wal_and_shm_changes_each_start_one_fresh_multiprocess_validation() { + fn open_stable_writer(db_path: &Path, label: &str) -> Connection { + let writer = Connection::open(db_path).unwrap(); + writer + .execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;") + .unwrap(); + writer + .execute( + "UPDATE schema_meta SET value=?1 WHERE key='app.version'", + [label], + ) + .unwrap(); + writer + } + + for (index, suffix) in ["", "-wal", "-shm"].into_iter().enumerate() { + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_dir = root.path().join(".trail"); + let db_path = db_dir.join(DB_RELATIVE_PATH); + let writer = open_stable_writer(&db_path, &format!("before-{index}")); + let before = schema_generation(&db_path).unwrap(); + assert!(before + .0 + .iter() + .find(|file| file.suffix == suffix) + .is_some_and(|file| file.present)); + + let counter = root.path().join(format!("{index}-count")); + let initial_go = root.path().join(format!("{index}-initial-go")); + let children = spawn_schema_children( + root.path(), + &counter, + &initial_go, + 16, + "success", + |command| { + command.env("TRAIL_TEST_SCHEMA_VALIDATION_DELAY_MS", "100"); + }, + ); + release_and_wait(children, &initial_go); + assert_eq!(validation_count(&counter), 1); + drop(writer); + + let lock = acquire_workspace_lock(&db_dir).unwrap(); + if suffix.is_empty() { + let replacement = db_path.with_extension("validation-replacement"); + fs::copy(&db_path, &replacement).unwrap(); + fs::rename(&replacement, &db_path).unwrap(); + } else { + let mut sidecar = db_path.as_os_str().to_os_string(); + sidecar.push(suffix); + match fs::remove_file(PathBuf::from(sidecar)) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => panic!("failed to rotate {suffix}: {error}"), + } + } + let writer = open_stable_writer(&db_path, &format!("after-{index}")); + if suffix == "-shm" { + let mut shm = db_path.as_os_str().to_os_string(); + shm.push("-shm"); + assert!(PathBuf::from(shm).exists()); + } + let after = schema_generation(&db_path).unwrap(); + let before_file = before.0.iter().find(|file| file.suffix == suffix).unwrap(); + let after_file = after.0.iter().find(|file| file.suffix == suffix).unwrap(); + assert_ne!( + before_file, after_file, + "{suffix} generation did not change" + ); + drop(lock); + + let fresh_go = root.path().join(format!("{index}-fresh-go")); + let children = + spawn_schema_children(root.path(), &counter, &fresh_go, 16, "success", |command| { + command.env("TRAIL_TEST_SCHEMA_VALIDATION_DELAY_MS", "100"); + }); + release_and_wait(children, &fresh_go); + assert_eq!( + validation_count(&counter), + 2, + "{suffix} change did not start exactly one fresh validation" + ); + drop(writer); + } + + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_dir = root.path().join(".trail"); + let db_path = db_dir.join(DB_RELATIVE_PATH); + let writer = open_stable_writer(&db_path, "backend-before"); + let counter = root.path().join("backend-count"); + let sqlite_go = root.path().join("backend-sqlite-go"); + let children = + spawn_schema_children(root.path(), &counter, &sqlite_go, 16, "success", |_| {}); + release_and_wait(children, &sqlite_go); + assert_eq!(validation_count(&counter), 1); + drop(writer); + let lock = acquire_workspace_lock(&db_dir).unwrap(); + let config_path = db_dir.join(CONFIG_FILE); + let config = fs::read_to_string(&config_path).unwrap().replace( + "prolly_backend = \"sqlite\"", + "prolly_backend = \"slatedb\"", + ); + fs::write(&config_path, config).unwrap(); + drop(lock); + let slatedb_go = root.path().join("backend-slatedb-go"); + let children = spawn_schema_children( + root.path(), + &counter, + &slatedb_go, + 16, + "schema-failure", + |_| {}, + ); + release_and_wait(children, &slatedb_go); + assert_eq!( + validation_count(&counter), + 2, + "backend change reused the sqlite validation result" + ); + } + + #[test] + fn schema_snapshot_exclusion_is_retained_through_mutable_handoff() { + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let observed = Arc::new(AtomicBool::new(false)); + let observed_in_hook = observed.clone(); + install_schema_handoff_hook(move |db_dir| { + assert!(matches!( + acquire_workspace_lock(db_dir), + Err(Error::WorkspaceLocked(_)) + )); + observed_in_hook.store(true, Ordering::SeqCst); + }); + let reopened = Trail::open(root.path()).unwrap(); + drop(reopened); + assert!(observed.load(Ordering::SeqCst)); + } + + #[test] + fn same_process_observer_writers_serialize_without_weakening_external_lock_failure() { + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_dir = root.path().join(".trail"); + let database = db_dir.join(DB_RELATIVE_PATH); + + let first = acquire_workspace_lock_for_observer(&db_dir, &database).unwrap(); + let (acquired, received) = std::sync::mpsc::sync_channel(1); + let other_db_dir = db_dir.clone(); + let other_database = database.clone(); + let waiter = std::thread::spawn(move || { + let second = + acquire_workspace_lock_for_observer(&other_db_dir, &other_database).unwrap(); + acquired.send(()).unwrap(); + drop(second); + }); + assert!(received + .recv_timeout(std::time::Duration::from_millis(100)) + .is_err()); + drop(first); + received + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + waiter.join().unwrap(); + + let observer = acquire_workspace_lock_for_observer(&db_dir, &database).unwrap(); + let (mutated, mutation_received) = std::sync::mpsc::sync_channel(1); + let workspace = root.path().to_path_buf(); + let mutation = std::thread::spawn(move || { + let db = Trail::open(workspace).unwrap(); + let lock = db.acquire_write_lock().unwrap(); + mutated.send(()).unwrap(); + drop(lock); + }); + assert!(mutation_received + .recv_timeout(std::time::Duration::from_millis(100)) + .is_err()); + drop(observer); + mutation_received + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + mutation.join().unwrap(); + + let external = acquire_workspace_lock(&db_dir).unwrap(); + assert!(matches!( + acquire_workspace_lock_for_observer(&db_dir, &database), + Err(Error::WorkspaceLocked(_)) + )); + drop(external); + } + + #[test] + fn one_hundred_ordinary_opens_share_one_unchanged_generation_validation() { + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_path = + canonicalize_lossless(&root.path().join(".trail").join(DB_RELATIVE_PATH)).unwrap(); + let writer = Connection::open(&db_path).unwrap(); + writer + .execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;") + .unwrap(); + writer + .execute( + "UPDATE schema_meta SET value='singleflight' WHERE key='app.version'", + [], + ) + .unwrap(); + let barrier = Arc::new(Barrier::new(100)); + let started = Instant::now(); + let handles = (0..100) + .map(|_| { + let root = root.path().to_path_buf(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + Trail::open(root).map(drop) + }) + }) + .collect::>(); + for handle in handles { + handle.join().unwrap().unwrap(); + } + assert_eq!(schema_validation_count(&db_path), 1); + assert!( + started.elapsed() < Duration::from_secs(5), + "100 shared schema opens exceeded the five second budget" + ); + drop(writer); + } + + #[test] + fn live_wal_advance_during_schema_snapshot_retries_without_reinitialize_guidance() { + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_path = + canonicalize_lossless(&root.path().join(".trail").join(DB_RELATIVE_PATH)).unwrap(); + // Retain the writer across the generation recapture. Dropping the last + // writer here can checkpoint and remove the WAL/SHM files, which is a + // main-database rewrite rather than the in-place WAL churn this test + // is meant to model. + let writer = Arc::new(Mutex::new(Some(Connection::open(&db_path).unwrap()))); + writer + .lock() + .unwrap() + .as_ref() + .unwrap() + .execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;") + .unwrap(); + let hook_writer = writer.clone(); + install_schema_snapshot_generation_hook(&db_path, move |_path| { + hook_writer + .lock() + .unwrap() + .as_ref() + .unwrap() + .execute( + "UPDATE schema_meta SET value='live-wal-retry' WHERE key='app.version'", + [], + ) + .unwrap(); + }); + + Trail::open(root.path()).unwrap(); + assert_eq!(schema_validation_count(&db_path), 2); + drop(writer.lock().unwrap().take()); + } + + #[test] + fn low_level_snapshot_failure_retries_only_when_same_wal_advanced() { + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_path = + canonicalize_lossless(&root.path().join(".trail").join(DB_RELATIVE_PATH)).unwrap(); + let writer = Connection::open(&db_path).unwrap(); + writer + .execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;") + .unwrap(); + writer + .execute( + "UPDATE schema_meta SET value='snapshot-error-baseline' WHERE key='app.version'", + [], + ) + .unwrap(); + fail_next_schema_validation(&db_path); + let advancing_path = db_path.clone(); + let advancing = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(20)); + let connection = Connection::open(advancing_path).unwrap(); + connection + .execute( + "UPDATE schema_meta SET value='snapshot-error-retry' WHERE key='app.version'", + [], + ) + .unwrap(); + }); + + Trail::open(root.path()).unwrap(); + advancing.join().unwrap(); + assert_eq!(schema_validation_count(&db_path), 2); + drop(writer); + } + + #[cfg(unix)] + #[test] + fn live_wal_advance_during_digest_retries_without_reinitialize_guidance() { + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_path = + canonicalize_lossless(&root.path().join(".trail").join(DB_RELATIVE_PATH)).unwrap(); + let writer = Arc::new(Mutex::new(Some(Connection::open(&db_path).unwrap()))); + writer + .lock() + .unwrap() + .as_ref() + .unwrap() + .execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;") + .unwrap(); + writer + .lock() + .unwrap() + .as_ref() + .unwrap() + .execute( + "UPDATE schema_meta SET value='digest-baseline' WHERE key='app.version'", + [], + ) + .unwrap(); + let mut wal = db_path.as_os_str().to_os_string(); + wal.push("-wal"); + let wal = PathBuf::from(wal); + let hook_writer = writer.clone(); + install_schema_wal_digest_hook( + &wal, + Box::new(move |_path| { + hook_writer + .lock() + .unwrap() + .as_ref() + .unwrap() + .execute( + "UPDATE schema_meta SET value='live-digest-retry' WHERE key='app.version'", + [], + ) + .unwrap(); + }), + ); + + Trail::open(root.path()).unwrap(); + assert_eq!(schema_validation_count(&db_path), 2); + drop(writer.lock().unwrap().take()); + } + + #[test] + fn snapshot_retry_rejects_main_wal_and_shm_pathname_substitution() { + for suffix in ["", "-wal", "-shm"] { + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_path = + canonicalize_lossless(&root.path().join(".trail").join(DB_RELATIVE_PATH)).unwrap(); + let writer = Connection::open(&db_path).unwrap(); + writer + .execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;") + .unwrap(); + writer + .execute( + "UPDATE schema_meta SET value='snapshot-substitution' WHERE key='app.version'", + [], + ) + .unwrap(); + let suffix = suffix.to_string(); + install_schema_snapshot_generation_hook(&db_path, move |path| { + let mut selected = path.as_os_str().to_os_string(); + selected.push(&suffix); + let selected = PathBuf::from(selected); + assert!( + selected.exists(), + "missing schema file `{}`", + selected.display() + ); + let replacement = selected.with_extension(format!( + "{}snapshot-replacement", + selected + .extension() + .and_then(|value| value.to_str()) + .map(|value| format!("{value}.")) + .unwrap_or_default() + )); + fs::copy(&selected, &replacement).unwrap(); + fs::rename(&replacement, &selected).unwrap(); + }); + + let error = match Trail::open(root.path()) { + Ok(_) => panic!("schema pathname substitution unexpectedly opened"), + Err(error) => error, + }; + assert!(matches!(error, Error::SchemaReinitializeRequired { .. })); + assert!(error + .to_string() + .contains("generation changed during snapshot validation")); + drop(writer); + } + } + + #[test] + fn generation_replacement_or_sidecar_change_invalidates_mutable_handoff() { + for changed_suffix in ["", "-wal", "-shm"] { + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_path = root.path().join(".trail").join(DB_RELATIVE_PATH); + let writer = Connection::open(&db_path).unwrap(); + writer + .execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;") + .unwrap(); + writer + .execute( + "UPDATE schema_meta SET value='toctou' WHERE key='app.version'", + [], + ) + .unwrap(); + let mut changed_path = db_path.as_os_str().to_os_string(); + changed_path.push(changed_suffix); + let changed_path = PathBuf::from(changed_path); + assert!(changed_path.exists(), "missing sidecar {changed_suffix}"); + install_schema_handoff_hook(move |_| { + if changed_suffix.is_empty() { + let replacement = changed_path.with_extension("replacement"); + fs::copy(&changed_path, &replacement).unwrap(); + fs::rename(replacement, &changed_path).unwrap(); + } else { + OpenOptions::new() + .append(true) + .open(&changed_path) + .unwrap() + .write_all(b"generation-change") + .unwrap(); + } + }); + assert!(matches!( + Trail::open(root.path()), + Err(Error::SchemaReinitializeRequired { .. }) + )); + drop(writer); + } + } + + #[cfg(unix)] + #[test] + fn restored_mtime_wal_byte_change_invalidates_authenticated_handoff() { + use std::ffi::CString; + use std::io::{Read, Seek, SeekFrom, Write}; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::fs::MetadataExt; + + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_path = root.path().join(".trail").join(DB_RELATIVE_PATH); + let writer = Connection::open(&db_path).unwrap(); + writer + .execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;") + .unwrap(); + writer + .execute( + "UPDATE schema_meta SET value='digest-race' WHERE key='app.version'", + [], + ) + .unwrap(); + let mut wal = db_path.as_os_str().to_os_string(); + wal.push("-wal"); + let wal = PathBuf::from(wal); + let hook_wal = wal.clone(); + install_schema_handoff_hook(move |_| { + let metadata = fs::metadata(&hook_wal).unwrap(); + let mut file = OpenOptions::new() + .read(true) + .write(true) + .open(&hook_wal) + .unwrap(); + file.seek(SeekFrom::End(-1)).unwrap(); + let mut byte = [0_u8; 1]; + file.read_exact(&mut byte).unwrap(); + file.seek(SeekFrom::End(-1)).unwrap(); + file.write_all(&[byte[0] ^ 0xff]).unwrap(); + file.sync_all().unwrap(); + let path = CString::new(hook_wal.as_os_str().as_bytes()).unwrap(); + let times = [ + libc::timespec { + tv_sec: metadata.atime(), + tv_nsec: metadata.atime_nsec(), + }, + libc::timespec { + tv_sec: metadata.mtime(), + tv_nsec: metadata.mtime_nsec(), + }, + ]; + assert_eq!( + unsafe { libc::utimensat(libc::AT_FDCWD, path.as_ptr(), times.as_ptr(), 0) }, + 0 + ); + }); + + assert!(matches!( + Trail::open(root.path()), + Err(Error::SchemaReinitializeRequired { .. }) + )); + drop(writer); + } + + #[cfg(unix)] + #[test] + fn wal_digest_rejects_last_component_symlink_substitution() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_path = root.path().join(".trail").join(DB_RELATIVE_PATH); + let writer = Connection::open(&db_path).unwrap(); + writer + .execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;") + .unwrap(); + writer + .execute( + "UPDATE schema_meta SET value='digest-symlink' WHERE key='app.version'", + [], + ) + .unwrap(); + + let expected = schema_generation(&db_path).unwrap(); + let mut wal = db_path.as_os_str().to_os_string(); + wal.push("-wal"); + let wal = PathBuf::from(wal); + let retained = wal.with_extension("wal.digest-retained"); + let hook_retained = retained.clone(); + install_schema_wal_digest_hook( + &wal, + Box::new(move |path| { + fs::rename(path, &hook_retained).unwrap(); + symlink(&hook_retained, path).unwrap(); + }), + ); + + let error = schema_wal_digest(&db_path, &expected).unwrap_err(); + assert!(matches!(error, Error::SchemaReinitializeRequired { .. })); + assert!(wal.symlink_metadata().unwrap().file_type().is_symlink()); + + fs::remove_file(&wal).unwrap(); + fs::rename(&retained, &wal).unwrap(); + drop(writer); + } + + #[cfg(unix)] + #[test] + fn wal_digest_continuous_churn_is_bounded_and_fails_closed() { + use std::os::unix::fs::PermissionsExt; + + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_path = root.path().join(".trail").join(DB_RELATIVE_PATH); + let writer = Connection::open(&db_path).unwrap(); + writer + .execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;") + .unwrap(); + writer + .execute( + "UPDATE schema_meta SET value='digest-churn' WHERE key='app.version'", + [], + ) + .unwrap(); + + let expected = schema_generation(&db_path).unwrap(); + let mut wal = db_path.as_os_str().to_os_string(); + wal.push("-wal"); + let wal = PathBuf::from(wal); + let mut owner_only = false; + install_schema_wal_digest_attempt_hook( + &wal, + Box::new(move |path| { + owner_only = !owner_only; + let mode = if owner_only { 0o600 } else { 0o640 }; + fs::set_permissions(path, fs::Permissions::from_mode(mode)).unwrap(); + }), + ); + + let started = Instant::now(); + let error = + schema_wal_digest_allowing_wal_ctime_transition(&db_path, &expected).unwrap_err(); + clear_schema_wal_digest_attempt_hook(&wal); + assert!(matches!(error, Error::SchemaReinitializeRequired { .. })); + assert!(error.to_string().contains("stable descriptor snapshot")); + assert!( + started.elapsed() < Duration::from_secs(2), + "bounded WAL digest retry took {:?}", + started.elapsed() + ); + drop(writer); + } + + #[test] + fn schema_validation_leader_failure_propagates_and_next_open_retries() { + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_path = + canonicalize_lossless(&root.path().join(".trail").join(DB_RELATIVE_PATH)).unwrap(); + fail_next_schema_validation(&db_path); + delay_next_schema_validation_server_shutdown_for_test(&db_path, Duration::from_millis(300)); + let barrier = Arc::new(Barrier::new(16)); + let started = Instant::now(); + let handles = (0..16) + .map(|_| { + let root = root.path().to_path_buf(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + Trail::open(root) + }) + }) + .collect::>(); + for handle in handles { + assert!(matches!( + handle.join().unwrap(), + Err(Error::SchemaReinitializeRequired { .. }) + )); + } + assert!( + started.elapsed() < Duration::from_millis(250), + "invalid-schema peers paid the detached IPC fanout lifetime: {:?}", + started.elapsed() + ); + let retry_started = Instant::now(); + Trail::open(root.path()).unwrap(); + assert!( + retry_started.elapsed() < Duration::from_millis(250), + "retry waited for the failed result server to retire: {:?}", + retry_started.elapsed() + ); + assert_eq!(schema_validation_count(&db_path), 2); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn production_server_start_does_not_rendezvous_with_the_spawned_thread() { + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let db_path = + canonicalize_lossless(&root.path().join(".trail").join(DB_RELATIVE_PATH)).unwrap(); + delay_next_schema_validation_server_start_for_test(&db_path, Duration::from_millis(300)); + + let started = Instant::now(); + Trail::open(root.path()).unwrap(); + assert!( + started.elapsed() < Duration::from_millis(250), + "production start rendezvoused with the delayed server thread: {:?}", + started.elapsed() + ); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn generation_replacement_does_not_wait_for_the_old_server() { + let root = tempfile::tempdir().unwrap(); + Trail::init(root.path(), "main", InitImportMode::Empty, false).unwrap(); + let canonical_db_path = + canonicalize_lossless(&root.path().join(".trail").join(DB_RELATIVE_PATH)).unwrap(); + delay_next_schema_validation_server_shutdown_for_test( + &canonical_db_path, + Duration::from_millis(300), + ); + Trail::open(root.path()).unwrap(); + + let db_path = root.path().join(".trail").join(DB_RELATIVE_PATH); + let writer = Connection::open(&db_path).unwrap(); + writer + .execute_batch("PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0;") + .unwrap(); + writer + .execute( + "UPDATE schema_meta SET value='replacement-latency' WHERE key='app.version'", + [], + ) + .unwrap(); + + let started = Instant::now(); + Trail::open(root.path()).unwrap(); + assert!( + started.elapsed() < Duration::from_millis(250), + "replacement waited for the old result server: {:?}", + started.elapsed() + ); + drop(writer); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn completed_schema_servers_naturally_leave_a_bounded_registry() { + let roots = (0..24) + .map(|_| tempfile::tempdir().unwrap()) + .collect::>(); + let keys = roots + .iter() + .map(|root| root.path().join("workspace.sqlite")) + .collect::>(); + let resources = roots + .iter() + .zip(&keys) + .map(|(root, key)| { + let (server, socket, lock, _, _) = test_schema_validation_server(root.path()); + start_schema_validation_server(key, server); + (socket, root.path().join("schema-validation.announce"), lock) + }) + .collect::>(); + + let deadline = Instant::now() + Duration::from_secs(2); + loop { + let registered = SCHEMA_VALIDATION_SERVERS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if keys.iter().all(|key| !registered.contains_key(key)) { + break; + } + drop(registered); + assert!( + Instant::now() < deadline, + "completed detached servers remained registered" + ); + std::thread::sleep(Duration::from_millis(10)); + } + + for (socket, announcement, lock) in resources { + assert!(!socket.exists()); + assert!(!announcement.exists()); + assert_schema_leader_lock_available_to_child(&lock); + } + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn schema_validation_server_returns_immediately_and_serves_a_late_follower() { + let root = tempfile::tempdir().unwrap(); + let (server, socket_path, lock_path, nonce, generation) = + test_schema_validation_server(root.path()); + + let started = Instant::now(); + let shutdown = Arc::new(SchemaValidationServerShutdown::default()); + let server = spawn_schema_validation_server(server, shutdown.clone()).unwrap(); + assert!( + started.elapsed() < Duration::from_millis(5), + "starting the background IPC server delayed mutable handoff: {:?}", + started.elapsed() + ); + + std::thread::sleep(Duration::from_millis(20)); + assert!(matches!( + request_schema_validation_result( + &socket_path, + &nonce, + &generation, + "sqlite", + std::process::id(), + ), + Some(CrossProcessSchemaValidationOutcome::Success(digest)) + if digest == "a".repeat(64) + )); + let stopped = Instant::now(); + shutdown.stop(); + server.join().unwrap(); + assert!( + stopped.elapsed() < Duration::from_millis(50), + "schema server shutdown did not wake immediately: {:?}", + stopped.elapsed() + ); + assert!(!socket_path.exists()); + assert!(!root.path().join("schema-validation.announce").exists()); + assert_schema_leader_lock_available_to_child(&lock_path); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn schema_validation_server_panic_releases_ipc_and_leader_lock() { + let root = tempfile::tempdir().unwrap(); + let (mut server, socket_path, lock_path, _, _) = test_schema_validation_server(root.path()); + server.panic_on_serve = true; + let key = root.path().join("workspace.sqlite"); + + start_schema_validation_server(&key, server); + let deadline = Instant::now() + Duration::from_secs(2); + while SCHEMA_VALIDATION_SERVERS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .contains_key(&key) + { + assert!( + Instant::now() < deadline, + "panicked detached server remained registered" + ); + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!socket_path.exists()); + assert!(!root.path().join("schema-validation.announce").exists()); + assert_schema_leader_lock_available_to_child(&lock_path); + } } diff --git a/trail/src/db/core/readonly.rs b/trail/src/db/core/readonly.rs index 0a912fd..62905be 100644 --- a/trail/src/db/core/readonly.rs +++ b/trail/src/db/core/readonly.rs @@ -70,7 +70,6 @@ impl Trail { "refs", "daemon.json", "daemon.token", - "worktree-daemon-cache.json", ] { collect_read_only_file_fingerprints( &self.db_dir.join(rel), diff --git a/trail/src/db/core/status.rs b/trail/src/db/core/status.rs index 560bd0b..932dcef 100644 --- a/trail/src/db/core/status.rs +++ b/trail/src/db/core/status.rs @@ -1,22 +1,83 @@ use super::*; impl Trail { + pub(crate) fn status_from_changed_path_ledger(&self) -> Result { + let branch = self.current_branch()?; + let head = self.resolve_branch_ref(&branch)?; + let (comparison, _fenced) = + self.with_workspace_authoritative_command_snapshot(|db, policy, candidates, _git| { + db.compare_authoritative_candidates( + policy, + candidates, + &head.root_id, + crate::db::change_ledger::CandidateMaterialization::ManifestOnly, + ) + })?; + debug_assert!(comparison.disk_files.is_none()); + let changed_paths = comparison.summaries; + let worktree_state = worktree_state_from_changes(&changed_paths); + let suggestions = self.status_suggestions(&branch, &worktree_state, &changed_paths); + Ok(StatusReport { + branch, + head, + worktree_state, + changed_paths, + suggestions, + }) + } + pub fn status(&self, branch: Option<&str>) -> Result { + let metrics = self.operation_metrics.clone(); + let result_metrics = metrics.clone(); + profile_operation_metrics(metrics.as_ref(), OperationMetricsKind::Status, || { + let authoritative_current_branch = if crate::db::command_authority_enabled() { + match branch { + None => true, + Some(branch) => self.current_branch()? == branch, + } + } else { + false + }; + let result = if authoritative_current_branch { + self.note_operation_metrics(OperationMetricsDelta { + selected_worktree_index_sqlite_not_applicable_count: 1, + ..OperationMetricsDelta::default() + }); + self.status_from_changed_path_ledger() + } else { + self.status_profiled(branch) + }; + if let (Some(metrics), Ok(report)) = (&result_metrics, &result) { + metrics.add(OperationMetricsDelta { + final_path_count: saturating_u64_from_usize(report.changed_paths.len()), + ..OperationMetricsDelta::default() + }); + } + result + }) + } + + fn status_profiled(&self, branch: Option<&str>) -> Result { let current_branch = self.current_branch()?; let branch = branch.map(str::to_string).unwrap_or(current_branch.clone()); let head = self.resolve_branch_ref(&branch)?; + let daemon_snapshot = if branch == current_branch { + self.daemon_worktree_snapshot() + } else { + None + }; if branch == current_branch { - if let Some(report) = self.status_from_daemon_cache(&branch, &head)? { + if let Some(report) = + self.status_from_daemon_snapshot(&branch, &head, daemon_snapshot.as_ref())? + { return Ok(report); } } - let snapshot_generation = self - .daemon_worktree_snapshot() - .map(|snapshot| match snapshot { - DaemonWorktreeSnapshot::Clean { generation, .. } - | DaemonWorktreeSnapshot::Dirty { generation, .. } - | DaemonWorktreeSnapshot::Overflow { generation } => generation, - }); + let snapshot_generation = daemon_snapshot.as_ref().map(|snapshot| match snapshot { + DaemonWorktreeSnapshot::Clean { generation, .. } + | DaemonWorktreeSnapshot::Dirty { generation, .. } + | DaemonWorktreeSnapshot::Overflow { generation } => *generation, + }); let changed_paths = self.status_changed_paths_uncached(¤t_branch, &branch, &head.root_id)?; if branch == current_branch { @@ -34,6 +95,25 @@ impl Trail { } pub(crate) fn status_read_only(&self, branch: Option<&str>) -> Result { + let metrics = self.operation_metrics.clone(); + let result_metrics = metrics.clone(); + profile_operation_metrics( + metrics.as_ref(), + OperationMetricsKind::StatusReadOnly, + || { + let result = self.status_read_only_profiled(branch); + if let (Some(metrics), Ok(report)) = (&result_metrics, &result) { + metrics.add(OperationMetricsDelta { + final_path_count: saturating_u64_from_usize(report.changed_paths.len()), + ..OperationMetricsDelta::default() + }); + } + result + }, + ) + } + + fn status_read_only_profiled(&self, branch: Option<&str>) -> Result { let current_branch = self.current_branch()?; let branch = branch.map(str::to_string).unwrap_or(current_branch.clone()); let head = self.resolve_branch_ref(&branch)?; @@ -93,20 +173,19 @@ impl Trail { root_id: &ObjectId, ) -> Result> { if branch == current_branch { - if let Some(paths) = self.scan_git_dirty_tracked_paths()? { + let policy = self.workspace_ignore_policy_snapshot(); + if let Some(paths) = self.scan_git_dirty_tracked_paths_with_policy(&policy)? { if paths.is_empty() { return Ok(Vec::new()); } return Ok(self - .selected_worktree_snapshot_for_root(root_id, &paths)? + .selected_worktree_snapshot_for_root_with_policy(root_id, &paths, &policy)? .summaries); } } let refresh = self.refresh_worktree_index_streaming_report()?; - if !refresh.changed - && self - .worktree_index_baseline_root()? - .is_some_and(|baseline| baseline == root_id.clone()) + let baseline = self.worktree_index_baseline_root()?; + if !refresh.changed && self.clean_baseline_matches_visible_root(baseline.as_ref(), root_id) { return Ok(Vec::new()); } @@ -124,12 +203,15 @@ impl Trail { root_id: &ObjectId, ) -> Result> { if branch == current_branch { - if let Some(paths) = self.scan_git_dirty_tracked_paths()? { + let policy = self.workspace_ignore_policy_snapshot(); + if let Some(paths) = self.scan_git_dirty_tracked_paths_with_policy(&policy)? { if paths.is_empty() { return Ok(Vec::new()); } return Ok(self - .selected_worktree_snapshot_for_root_read_only(root_id, &paths)? + .selected_worktree_snapshot_for_root_read_only_with_policy( + root_id, &paths, &policy, + )? .summaries); } } @@ -138,30 +220,42 @@ impl Trail { self.diff_root_to_disk_manifest(root_id, &disk_manifest) } - fn status_from_daemon_cache( + fn status_from_daemon_snapshot( &self, branch: &str, head: &RefRecord, + snapshot: Option<&DaemonWorktreeSnapshot>, ) -> Result> { - let Some(snapshot) = self.daemon_worktree_snapshot() else { + let Some(snapshot) = snapshot else { return Ok(None); }; match snapshot { DaemonWorktreeSnapshot::Clean { generation: _, root_id: Some(root_id), - } if root_id == head.root_id => Ok(Some(clean_status_report(branch, head))), + } => { + if self.clean_baseline_matches_visible_root(Some(&root_id), &head.root_id) { + Ok(Some(clean_status_report(branch, head))) + } else { + Ok(None) + } + } DaemonWorktreeSnapshot::Dirty { generation, paths } => { if paths.len() > self.daemon_dirty_path_limit() { return Ok(None); } - let snapshot = self.selected_worktree_snapshot_for_root(&head.root_id, &paths)?; + let policy = self.workspace_ignore_policy_snapshot(); + let snapshot = self.selected_worktree_snapshot_for_root_with_policy( + &head.root_id, + &paths, + &policy, + )?; let changed_paths = snapshot.summaries; self.reconcile_daemon_status_paths( &head.root_id, &paths, &changed_paths, - generation, + *generation, ); let worktree_state = worktree_state_from_changes(&changed_paths); let suggestions = self.status_suggestions(branch, &worktree_state, &changed_paths); diff --git a/trail/src/db/core/workspace/ignore.rs b/trail/src/db/core/workspace/ignore.rs index 3f4fb12..2b87672 100644 --- a/trail/src/db/core/workspace/ignore.rs +++ b/trail/src/db/core/workspace/ignore.rs @@ -1,6 +1,117 @@ use super::*; +impl WorkspaceIgnorePolicySnapshot { + pub(crate) fn check(&self, path: &str) -> Result { + let path = normalize_relative_path(path)?; + if is_default_ignored(&path) { + return Ok(IgnoreCheckReport { + path, + ignored: true, + source: Some("hardcoded".to_string()), + }); + } + + let abs = safe_join(&self.workspace_root, &path)?; + if let Some(metrics) = &self.metrics { + metrics.add(OperationMetricsDelta { + filesystem_stat_count: 1, + ..OperationMetricsDelta::default() + }); + } + let is_dir = match fs::symlink_metadata(abs) { + Ok(metadata) => metadata.is_dir(), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => false, + Err(err) => return Err(Error::Io(err)), + }; + self.check_normalized_with_is_dir(path, is_dir) + } + + pub(crate) fn check_with_is_dir(&self, path: &str, is_dir: bool) -> Result { + let path = normalize_relative_path(path)?; + if is_default_ignored(&path) { + return Ok(IgnoreCheckReport { + path, + ignored: true, + source: Some("hardcoded".to_string()), + }); + } + self.check_normalized_with_is_dir(path, is_dir) + } + + fn check_normalized_with_is_dir( + &self, + path: String, + is_dir: bool, + ) -> Result { + let matcher = self + .matcher + .get_or_init(|| self.compile_matcher()) + .as_ref() + .map_err(|message| Error::InvalidInput(message.clone()))?; + let ignored = matcher + .matched_path_or_any_parents(path_from_rel(&path), is_dir) + .is_ignore(); + Ok(IgnoreCheckReport { + path, + ignored, + source: ignored.then(|| "workspace".to_string()), + }) + } + + fn compile_matcher(&self) -> std::result::Result<::ignore::gitignore::Gitignore, String> { + let mut builder = ::ignore::gitignore::GitignoreBuilder::new(&self.workspace_root); + let trailignore = self.workspace_root.join(".trailignore"); + let gitignore = self.workspace_root.join(".gitignore"); + // A single metadata probe per dependency preserves `Path::exists` + // semantics while exposing exactly the files and bytes read by this + // operation-local matcher. + let trailignore_metadata = fs::metadata(&trailignore).ok(); + let gitignore_metadata = fs::metadata(&gitignore).ok(); + let trailignore_exists = trailignore_metadata.is_some(); + let gitignore_exists = gitignore_metadata.is_some(); + let dependency_bytes = trailignore_metadata + .as_ref() + .map(fs::Metadata::len) + .unwrap_or(0) + .saturating_add( + gitignore_metadata + .as_ref() + .map(fs::Metadata::len) + .unwrap_or(0), + ); + if let Some(metrics) = &self.metrics { + metrics.add(OperationMetricsDelta { + policy_build_count: 1, + policy_dependency_file_count: u64::from(trailignore_exists) + .saturating_add(u64::from(gitignore_exists)), + policy_dependency_bytes: dependency_bytes, + filesystem_stat_count: 2, + ..OperationMetricsDelta::default() + }); + } + if trailignore_exists { + if let Some(err) = builder.add(trailignore) { + return Err(err.to_string()); + } + } + if gitignore_exists { + if let Some(err) = builder.add(gitignore) { + return Err(err.to_string()); + } + } + builder.build().map_err(|err| err.to_string()) + } +} + impl Trail { + pub(crate) fn workspace_ignore_policy_snapshot(&self) -> WorkspaceIgnorePolicySnapshot { + WorkspaceIgnorePolicySnapshot { + workspace_root: self.workspace_root.clone(), + metrics: self.operation_metrics.clone(), + matcher: OnceLock::new(), + } + } + pub fn ignore_list(&self) -> Result { let path = self.workspace_root.join(".trailignore"); let patterns = read_ignore_patterns(&path)?; @@ -63,39 +174,641 @@ impl Trail { } pub fn ignore_check(&self, path: &str) -> Result { - let path = normalize_relative_path(path)?; - if is_default_ignored(&path) { - return Ok(IgnoreCheckReport { - path, - ignored: true, - source: Some("hardcoded".to_string()), - }); - } - let abs = self.workspace_root.join(path_from_rel(&path)); - let is_dir = abs.is_dir(); - let mut builder = ::ignore::gitignore::GitignoreBuilder::new(&self.workspace_root); - let trailignore = self.workspace_root.join(".trailignore"); - if trailignore.exists() { - if let Some(err) = builder.add(trailignore) { - return Err(Error::InvalidInput(err.to_string())); - } + self.workspace_ignore_policy_snapshot().check(path) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn write_scale_files(root: &Path, count: usize) { + let src = root.join("src"); + fs::create_dir_all(&src).unwrap(); + for index in 0..count { + fs::write( + src.join(format!("file_{index:04}.txt")), + format!("baseline {index}\n"), + ) + .unwrap(); } - let gitignore = self.workspace_root.join(".gitignore"); - if gitignore.exists() { - if let Some(err) = builder.add(gitignore) { - return Err(Error::InvalidInput(err.to_string())); - } + } + + fn git(root: &Path, args: &[&str]) { + let output = Command::new("git") + .arg("-C") + .arg(root) + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + fn init_git_scale_workspace(file_count: usize) -> tempfile::TempDir { + let temp = tempfile::tempdir().unwrap(); + git(temp.path(), &["init"]); + git(temp.path(), &["config", "user.email", "trail@example.test"]); + git(temp.path(), &["config", "user.name", "Trail Test"]); + fs::write(temp.path().join(".trailignore"), "").unwrap(); + write_scale_files(temp.path(), file_count); + git(temp.path(), &["add", ".trailignore", "src"]); + git(temp.path(), &["commit", "-m", "baseline"]); + Trail::init(temp.path(), "main", InitImportMode::GitTracked, false).unwrap(); + temp + } + + fn init_git_policy_visibility_workspace( + policy_path: &str, + policy_pattern: &str, + ) -> tempfile::TempDir { + let temp = tempfile::tempdir().unwrap(); + git(temp.path(), &["init"]); + git(temp.path(), &["config", "user.email", "trail@example.test"]); + git(temp.path(), &["config", "user.name", "Trail Test"]); + fs::create_dir_all(temp.path().join("nested")).unwrap(); + if policy_path != ".trailignore" { + fs::write(temp.path().join(".trailignore"), "").unwrap(); } - let matcher = builder - .build() - .map_err(|err| Error::InvalidInput(err.to_string()))?; - let ignored = matcher - .matched_path_or_any_parents(path_from_rel(&path), is_dir) - .is_ignore(); - Ok(IgnoreCheckReport { - path, - ignored, - source: ignored.then(|| "workspace".to_string()), + fs::write(temp.path().join(policy_path), policy_pattern).unwrap(); + fs::write(temp.path().join("nested/hidden.txt"), "hidden baseline\n").unwrap(); + git( + temp.path(), + &[ + "add", + "-f", + ".trailignore", + policy_path, + "nested/hidden.txt", + ], + ); + git(temp.path(), &["commit", "-m", "policy baseline"]); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + fs::write(temp.path().join(".git/info/exclude"), ".trail/\n").unwrap(); + temp + } + + fn assert_selected_locality(report: &OperationMetricsReport) { + assert_eq!(report.full_filesystem_walk_count, 0); + assert_eq!(report.full_root_range_count, 0); + assert_eq!(report.selected_worktree_index_sqlite_full_scan_count, 0); + } + + fn assert_operation_report( + report: &OperationMetricsReport, + operation: &str, + git_global_work_count: u64, + policy_build_count: u64, + ) { + assert_eq!(report.operation, operation); + assert_eq!(report.outcome, OperationMetricsOutcome::Success); + assert_eq!(report.git_global_work_count, git_global_work_count); + assert_eq!(report.git_subprocess_count, git_global_work_count); + assert_eq!(report.policy_build_count, policy_build_count); + } + + fn assert_selected_sqlite_accounting(report: &OperationMetricsReport) { + assert!(report.selected_worktree_index_sqlite_accounting_complete); + assert!(report.selected_worktree_index_sqlite_envelope_count > 0); + assert_eq!(report.selected_worktree_index_sqlite_full_scan_count, 0); + } + + fn assert_no_daemon_persistence(report: &OperationMetricsReport) { + assert_eq!(report.daemon_cumulative_rewrite_count, 0); + assert_eq!(report.daemon_cumulative_rewrite_bytes, 0); + assert_eq!(report.daemon_cumulative_rewrite_count_total, 0); + assert_eq!(report.daemon_cumulative_rewrite_bytes_total, 0); + } + + fn install_daemon_cache( + db: &mut Trail, + dirty_paths: &[&str], + overflow: bool, + baseline_root_id: Option, + ) { + db.daemon_worktree_cache = Some(DaemonWorktreeCache { + state: Arc::new(Mutex::new(DaemonWorktreeCacheState { + dirty_paths: dirty_paths.iter().map(|path| path.to_string()).collect(), + overflow, + initialized: true, + baseline_root_id, + generation: 1, + policy_invalidation_index: None, + })), + persist: None, + watcher: None, + }); + } + + #[test] + fn workspace_ignore_policy_snapshot_reuses_matcher_and_invalidates_between_operations() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + fs::write(temp.path().join(".trailignore"), "old.txt\n").unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let metrics = db.operation_metrics.as_ref().unwrap(); + + profile_operation_metrics(Some(metrics), OperationMetricsKind::Status, || { + let policy = db.workspace_ignore_policy_snapshot(); + assert!(policy.check("old.txt")?.ignored); + assert!(!policy.check("new.txt")?.ignored); + fs::write(temp.path().join(".trailignore"), "new.txt\n")?; + assert!(policy.check("old.txt")?.ignored); + assert!(!policy.check("new.txt")?.ignored); + Ok::<(), Error>(()) }) + .unwrap(); + let first = operation_metrics_report(Some(metrics)).unwrap(); + assert_eq!(first.policy_build_count, 1); + + profile_operation_metrics(Some(metrics), OperationMetricsKind::Status, || { + let policy = db.workspace_ignore_policy_snapshot(); + assert!(!policy.check("old.txt")?.ignored); + assert!(policy.check("new.txt")?.ignored); + Ok::<(), Error>(()) + }) + .unwrap(); + let second = operation_metrics_report(Some(metrics)).unwrap(); + assert_eq!(second.policy_build_count, 1); + } + + #[test] + fn workspace_ignore_policy_snapshot_does_not_compile_for_hardcoded_ignore() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let metrics = db.operation_metrics.as_ref().unwrap(); + + profile_operation_metrics(Some(metrics), OperationMetricsKind::Status, || { + let policy = db.workspace_ignore_policy_snapshot(); + assert!(policy.check(".trail/index/trail.sqlite")?.ignored); + Ok::<(), Error>(()) + }) + .unwrap(); + let report = operation_metrics_report(Some(metrics)).unwrap(); + assert_eq!(report.policy_build_count, 0); + assert_eq!(report.policy_dependency_file_count, 0); + assert_eq!(report.filesystem_stat_count, 0); + } + + #[test] + fn workspace_ignore_policy_snapshot_caches_exact_error_and_retry_rebuilds() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + let trailignore = temp.path().join(".trailignore"); + fs::write(&trailignore, "invalid\\\n").unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let mut expected_builder = ::ignore::gitignore::GitignoreBuilder::new(&db.workspace_root); + let expected_message = expected_builder + .add(db.workspace_root.join(".trailignore")) + .expect("fixture must be rejected by the existing ignore parser") + .to_string(); + let metrics = db.operation_metrics.as_ref().unwrap(); + + let error = profile_operation_metrics( + Some(metrics), + OperationMetricsKind::Status, + || -> Result<()> { + let policy = db.workspace_ignore_policy_snapshot(); + for _ in 0..2 { + match policy.check("candidate.txt").unwrap_err() { + Error::InvalidInput(message) => { + assert_eq!(message, expected_message); + } + other => panic!("unexpected policy error: {other}"), + } + } + Err(Error::InvalidInput(expected_message.clone())) + }, + ) + .unwrap_err(); + assert!(matches!(error, Error::InvalidInput(message) if message == expected_message)); + let failed = operation_metrics_report(Some(metrics)).unwrap(); + assert_eq!(failed.outcome, OperationMetricsOutcome::Error); + assert_eq!(failed.policy_build_count, 1); + + fs::write(&trailignore, "candidate.txt\n").unwrap(); + profile_operation_metrics(Some(metrics), OperationMetricsKind::Status, || { + let policy = db.workspace_ignore_policy_snapshot(); + assert!(policy.check("candidate.txt")?.ignored); + Ok::<(), Error>(()) + }) + .unwrap(); + let retry = operation_metrics_report(Some(metrics)).unwrap(); + assert_eq!(retry.outcome, OperationMetricsOutcome::Success); + assert_eq!(retry.policy_build_count, 1); + } + + #[test] + fn git_selected_status_1k_reuses_one_policy_snapshot() { + let temp = init_git_scale_workspace(1_000); + fs::write(temp.path().join("src/file_0001.txt"), "changed one\n").unwrap(); + fs::write(temp.path().join("src/file_0002.txt"), "changed two\n").unwrap(); + let db = Trail::open(temp.path()).unwrap(); + + let status = db.status(None).unwrap(); + assert_eq!(status.changed_paths.len(), 2); + let report = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_selected_locality(&report); + assert_selected_sqlite_accounting(&report); + assert_eq!(report.policy_build_count, 1); + assert_eq!(report.git_subprocess_count, 1); + assert_eq!(report.git_global_work_count, 1); + } + + #[test] + fn explicit_selected_record_1k_reuses_one_policy_snapshot() { + let temp = tempfile::tempdir().unwrap(); + write_scale_files(temp.path(), 1_000); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + fs::write(temp.path().join("src/file_0001.txt"), "changed one\n").unwrap(); + fs::write(temp.path().join("src/file_0002.txt"), "changed two\n").unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + + let recorded = db + .record_with_options( + Some("main"), + Some("selected policy reuse".to_string()), + Actor::human(), + RecordOptions { + paths: vec!["src".to_string(), "src/file_0001.txt".to_string()], + ..RecordOptions::default() + }, + ) + .unwrap(); + assert_eq!(recorded.changed_paths.len(), 2); + let report = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_selected_locality(&report); + assert_eq!(report.policy_build_count, 2); + assert_eq!(report.git_global_work_count, 0); + assert_eq!(report.input_path_count, 2); + assert_eq!(report.canonical_path_count, 1); + assert_eq!(report.filesystem_read_count, 1_000); + } + + #[test] + fn git_policy_file_change_is_labeled_as_global_fallback() { + let temp = init_git_scale_workspace(8); + fs::write(temp.path().join(".trailignore"), "generated/\n").unwrap(); + let db = Trail::open(temp.path()).unwrap(); + + let _status = db.status(None).unwrap(); + let report = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_eq!(report.git_global_work_count, 1); + assert_eq!(report.full_filesystem_walk_count, 1); + assert_eq!(report.policy_build_count, 1); + } + + #[test] + fn git_root_and_nested_policy_changes_force_full_fallback_before_filtering() { + for (policy_path, pattern) in [ + ("nested/.trailignore", "hidden.txt\n"), + ("nested/.gitignore", "hidden.txt\n"), + (".trailignore", "nested/hidden.txt\n"), + (".gitignore", "nested/hidden.txt\n"), + ] { + let temp = init_git_policy_visibility_workspace(policy_path, pattern); + fs::write(temp.path().join(policy_path), "").unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let policy = db.workspace_ignore_policy_snapshot(); + + assert!( + db.scan_git_dirty_tracked_paths_with_policy(&policy) + .unwrap() + .is_none(), + "policy candidate {policy_path} must force global fallback" + ); + + let status = db.status(None).unwrap(); + assert!( + status + .changed_paths + .iter() + .any(|change| change.path == "nested/hidden.txt"), + "policy candidate {policy_path} produced {:?}", + status.changed_paths + ); + let report = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_eq!(report.git_global_work_count, 1, "{policy_path}"); + assert_eq!(report.full_filesystem_walk_count, 1, "{policy_path}"); + assert_eq!(report.input_path_count, 1, "{policy_path}"); + assert_eq!(report.canonical_path_count, 0, "{policy_path}"); + assert_eq!(report.policy_build_count, 1, "{policy_path}"); + } + } + + #[test] + fn git_regular_rename_candidates_preserve_both_endpoints() { + let temp = init_git_scale_workspace(8); + git( + temp.path(), + &["mv", "src/file_0001.txt", "src/renamed_0001.txt"], + ); + let db = Trail::open(temp.path()).unwrap(); + let policy = db.workspace_ignore_policy_snapshot(); + + let paths = db + .scan_git_dirty_tracked_paths_with_policy(&policy) + .unwrap() + .expect("regular rename remains eligible for selected handling"); + + assert_eq!( + paths, + vec![ + "src/file_0001.txt".to_string(), + "src/renamed_0001.txt".to_string() + ] + ); + } + + #[test] + fn git_candidate_metrics_keep_exact_endpoints_but_canonicalize_parent_overlap() { + let temp = tempfile::tempdir().unwrap(); + git(temp.path(), &["init"]); + git(temp.path(), &["config", "user.email", "trail@example.test"]); + git(temp.path(), &["config", "user.name", "Trail Test"]); + fs::write(temp.path().join(".trailignore"), "").unwrap(); + fs::write(temp.path().join("node"), "baseline node\n").unwrap(); + git(temp.path(), &["add", ".trailignore", "node"]); + git(temp.path(), &["commit", "-m", "overlap baseline"]); + Trail::init(temp.path(), "main", InitImportMode::GitTracked, false).unwrap(); + fs::write(temp.path().join(".git/info/exclude"), ".trail/\n").unwrap(); + fs::remove_file(temp.path().join("node")).unwrap(); + fs::create_dir(temp.path().join("node")).unwrap(); + fs::write(temp.path().join("node/child.txt"), "replacement child\n").unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let metrics = db.operation_metrics.as_ref().unwrap(); + + let paths = profile_operation_metrics( + Some(metrics), + OperationMetricsKind::Status, + || -> Result> { + let policy = db.workspace_ignore_policy_snapshot(); + Ok(db + .scan_git_dirty_tracked_paths_with_policy(&policy)? + .expect("valid Git overlap candidates")) + }, + ) + .unwrap(); + + assert_eq!( + paths, + vec!["node".to_string(), "node/child.txt".to_string()] + ); + let report = operation_metrics_report(Some(metrics)).unwrap(); + assert_eq!(report.input_path_count, 2); + assert_eq!(report.canonical_path_count, 1); + } + + #[test] + fn git_candidate_metrics_report_raw_input_before_policy_error() { + let temp = init_git_scale_workspace(1); + fs::write(temp.path().join(".git/info/exclude"), ".trail/\n").unwrap(); + fs::write(temp.path().join(".trailignore"), "invalid\\\n").unwrap(); + git(temp.path(), &["add", ".trailignore"]); + git(temp.path(), &["commit", "-m", "invalid policy"]); + fs::write(temp.path().join("src/file_0000.txt"), "dirty candidate\n").unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let metrics = db.operation_metrics.as_ref().unwrap(); + + let result = profile_operation_metrics( + Some(metrics), + OperationMetricsKind::Status, + || -> Result<()> { + let policy = db.workspace_ignore_policy_snapshot(); + db.scan_git_dirty_tracked_paths_with_policy(&policy)?; + Ok(()) + }, + ); + + assert!(matches!(result, Err(Error::InvalidInput(_)))); + let report = operation_metrics_report(Some(metrics)).unwrap(); + assert_eq!(report.outcome, OperationMetricsOutcome::Error); + assert_eq!(report.input_path_count, 1); + assert_eq!(report.canonical_path_count, 0); + } + + #[test] + fn selected_directory_keeps_nested_walkbuilder_ignore_semantics() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir_all(temp.path().join("src/nested")).unwrap(); + fs::write(temp.path().join("src/keep.txt"), "base\n").unwrap(); + fs::write(temp.path().join("src/nested/.trailignore"), "ignored.log\n").unwrap(); + fs::write(temp.path().join("src/nested/ignored.log"), "ignored base\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + fs::write(temp.path().join("src/keep.txt"), "changed\n").unwrap(); + fs::write( + temp.path().join("src/nested/ignored.log"), + "ignored changed\n", + ) + .unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + + let recorded = db + .record_with_options( + Some("main"), + Some("nested ignore compatibility".to_string()), + Actor::human(), + RecordOptions { + paths: vec!["src".to_string()], + ..RecordOptions::default() + }, + ) + .unwrap(); + assert_eq!( + recorded.changed_paths.len(), + 1, + "unexpected nested-ignore changes: {:?}", + recorded.changed_paths + ); + assert_eq!(recorded.changed_paths[0].path, "src/keep.txt"); + let report = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_operation_report(&report, "record", 0, 2); + assert_eq!(report.bounded_filesystem_walk_count, 1); + } + + #[test] + fn daemon_selected_status_diff_and_record_1k_structural_matrix() { + let temp = tempfile::tempdir().unwrap(); + write_scale_files(temp.path(), 996); + fs::create_dir_all(temp.path().join("overlap/nested")).unwrap(); + fs::write(temp.path().join("overlap/nested/a.txt"), "overlap base\n").unwrap(); + fs::create_dir_all(temp.path().join("delete/sub")).unwrap(); + fs::write(temp.path().join("delete/sub/b.txt"), "delete base\n").unwrap(); + fs::write(temp.path().join("clean_a.txt"), "clean a\n").unwrap(); + fs::write(temp.path().join("clean_b.txt"), "clean b\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let head = db.resolve_branch_ref("main").unwrap(); + + install_daemon_cache(&mut db, &[], false, Some(head.root_id.clone())); + assert!(db.status(None).unwrap().changed_paths.is_empty()); + let clean_status = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_selected_locality(&clean_status); + assert_operation_report(&clean_status, "status", 0, 0); + assert_no_daemon_persistence(&clean_status); + + assert!(db.diff_dirty(false, false).unwrap().files.is_empty()); + let clean_diff = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_selected_locality(&clean_diff); + assert_operation_report(&clean_diff, "diff", 0, 0); + assert_no_daemon_persistence(&clean_diff); + + assert!(db + .record( + Some("main"), + Some("daemon clean no-op".to_string()), + Actor::human(), + false, + ) + .unwrap() + .operation + .is_none()); + let clean_record = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_selected_locality(&clean_record); + assert_operation_report(&clean_record, "record", 0, 0); + assert_no_daemon_persistence(&clean_record); + + fs::write( + temp.path().join("overlap/nested/a.txt"), + "overlap changed\n", + ) + .unwrap(); + install_daemon_cache(&mut db, &["overlap", "overlap/nested/a.txt"], false, None); + let dirty_status = db.status(None).unwrap(); + assert_eq!(dirty_status.changed_paths.len(), 1); + let status_metrics = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_selected_locality(&status_metrics); + assert_selected_sqlite_accounting(&status_metrics); + assert_operation_report(&status_metrics, "status", 0, 2); + assert_eq!(status_metrics.bounded_filesystem_walk_count, 1); + assert_eq!(status_metrics.bounded_root_range_count, 1); + assert_eq!(status_metrics.filesystem_read_count, 1); + assert_eq!(status_metrics.input_path_count, 2); + assert_eq!(status_metrics.canonical_path_count, 1); + assert_no_daemon_persistence(&status_metrics); + + fs::write(temp.path().join("overlap/nested/a.txt"), "overlap base\n").unwrap(); + fs::remove_dir_all(temp.path().join("delete")).unwrap(); + install_daemon_cache(&mut db, &["delete", "delete/sub"], false, None); + let deleted = db.diff_dirty(false, false).unwrap(); + assert_eq!(deleted.files.len(), 1); + assert_eq!(deleted.files[0].path, "delete/sub/b.txt"); + assert_eq!(deleted.files[0].kind, FileChangeKind::Deleted); + let diff_metrics = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_selected_locality(&diff_metrics); + assert_selected_sqlite_accounting(&diff_metrics); + assert_operation_report(&diff_metrics, "diff", 0, 1); + assert_eq!(diff_metrics.bounded_root_range_count, 1); + assert_no_daemon_persistence(&diff_metrics); + + fs::write(temp.path().join("src/file_0001.txt"), "daemon record\n").unwrap(); + install_daemon_cache(&mut db, &["src/file_0001.txt"], false, None); + let recorded = db + .record( + Some("main"), + Some("daemon selected record".to_string()), + Actor::human(), + false, + ) + .unwrap(); + assert!(recorded.operation.is_some()); + assert_eq!(recorded.changed_paths.len(), 1); + let record_metrics = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_selected_locality(&record_metrics); + assert_selected_sqlite_accounting(&record_metrics); + assert_operation_report(&record_metrics, "record", 0, 1); + assert_no_daemon_persistence(&record_metrics); + + fs::write(temp.path().join(".trailignore"), "invalid\\\n").unwrap(); + fs::write(temp.path().join("src/file_0002.txt"), "daemon retry\n").unwrap(); + install_daemon_cache(&mut db, &["src/file_0002.txt"], false, None); + assert!(matches!(db.status(None), Err(Error::InvalidInput(_)))); + let failed = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_eq!(failed.operation, "status"); + assert_eq!(failed.outcome, OperationMetricsOutcome::Error); + assert_eq!(failed.policy_build_count, 1); + assert_selected_locality(&failed); + + fs::write(temp.path().join(".trailignore"), "").unwrap(); + install_daemon_cache(&mut db, &["src/file_0002.txt"], false, None); + assert_eq!(db.status(None).unwrap().changed_paths.len(), 1); + let retry = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_selected_locality(&retry); + assert_selected_sqlite_accounting(&retry); + assert_operation_report(&retry, "status", 0, 1); + + fs::write(temp.path().join(".trailignore"), "generated/\n").unwrap(); + install_daemon_cache(&mut db, &[], true, None); + let _fallback = db.status(None).unwrap(); + let fallback = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_eq!(fallback.operation, "status"); + assert_eq!(fallback.outcome, OperationMetricsOutcome::Success); + assert_eq!(fallback.full_filesystem_walk_count, 1); + assert_eq!(fallback.git_global_work_count, 1); + assert_eq!(fallback.policy_build_count, 1); + } + + #[test] + fn git_selected_diff_record_and_noop_1k_structural_matrix() { + let temp = init_git_scale_workspace(1_000); + let mut db = Trail::open(temp.path()).unwrap(); + + assert!(db.status(None).unwrap().changed_paths.is_empty()); + let clean = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_selected_locality(&clean); + assert_operation_report(&clean, "status", 1, 0); + + git( + temp.path(), + &["mv", "src/file_0001.txt", "src/renamed_0001.txt"], + ); + let renamed = db.diff_dirty(false, false).unwrap(); + assert_eq!(renamed.files.len(), 1); + assert_eq!(renamed.files[0].path, "src/renamed_0001.txt"); + assert_eq!( + renamed.files[0].old_path.as_deref(), + Some("src/file_0001.txt") + ); + assert_eq!(renamed.files[0].kind, FileChangeKind::Renamed); + let diff = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_selected_locality(&diff); + assert_selected_sqlite_accounting(&diff); + assert_operation_report(&diff, "diff", 1, 1); + assert_eq!(diff.root_point_key_count, 2); + + git(temp.path(), &["reset", "--hard", "HEAD"]); + fs::write(temp.path().join("src/file_0002.txt"), "git record\n").unwrap(); + let recorded = db + .record( + Some("main"), + Some("Git selected record".to_string()), + Actor::human(), + false, + ) + .unwrap(); + assert!(recorded.operation.is_some()); + assert_eq!(recorded.changed_paths.len(), 1); + let record = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_selected_locality(&record); + assert_selected_sqlite_accounting(&record); + assert_operation_report(&record, "record", 1, 1); + + let noop = db + .record( + Some("main"), + Some("Git candidate no-op".to_string()), + Actor::human(), + false, + ) + .unwrap(); + assert!(noop.operation.is_none()); + let noop_metrics = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_selected_locality(&noop_metrics); + assert_selected_sqlite_accounting(&noop_metrics); + assert_operation_report(&noop_metrics, "record", 1, 1); } } diff --git a/trail/src/db/lane/control.rs b/trail/src/db/lane/control.rs index 924e40e..cd28c42 100644 --- a/trail/src/db/lane/control.rs +++ b/trail/src/db/lane/control.rs @@ -1,6 +1,8 @@ use super::*; mod acp_sessions; +mod agent_capture; +mod agent_evidence; mod approvals; mod conversations; mod events; diff --git a/trail/src/db/lane/control/acp_sessions.rs b/trail/src/db/lane/control/acp_sessions.rs index 5951332..b7479ce 100644 --- a/trail/src/db/lane/control/acp_sessions.rs +++ b/trail/src/db/lane/control/acp_sessions.rs @@ -8,6 +8,7 @@ impl Trail { lane: &str, trail_session_id: &str, cwd: &str, + path_mappings: &[AcpPathMapping], provider: Option<&str>, model: Option<&str>, upstream_command_json: Option<&str>, @@ -26,6 +27,7 @@ impl Trail { )); } let status = validate_acp_session_status(status)?; + let path_mappings_json = serde_json::to_string(path_mappings)?; let branch = self.lane_branch(lane)?; let session = self.lane_session(trail_session_id)?; if session.lane_id != branch.lane_id { @@ -42,13 +44,14 @@ impl Trail { .unwrap_or(now); self.conn.execute( "INSERT INTO lane_acp_sessions \ - (acp_session_id, upstream_session_id, lane_id, trail_session_id, cwd, provider, model, upstream_command_json, status, created_at, updated_at) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) \ + (acp_session_id, upstream_session_id, lane_id, trail_session_id, cwd, path_mappings_json, provider, model, upstream_command_json, status, created_at, updated_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12) \ ON CONFLICT(acp_session_id) DO UPDATE SET \ upstream_session_id = excluded.upstream_session_id, \ lane_id = excluded.lane_id, \ trail_session_id = excluded.trail_session_id, \ cwd = excluded.cwd, \ + path_mappings_json = excluded.path_mappings_json, \ provider = excluded.provider, \ model = excluded.model, \ upstream_command_json = excluded.upstream_command_json, \ @@ -60,6 +63,7 @@ impl Trail { branch.lane_id, trail_session_id, cwd, + path_mappings_json, provider, model, upstream_command_json, @@ -100,6 +104,71 @@ impl Trail { self.lane_acp_session(acp_session_id) } + pub fn update_lane_acp_session_configuration( + &mut self, + acp_session_id: &str, + mode_id: Option<&str>, + config: Option<(&str, &serde_json::Value)>, + ) -> Result { + let _lock = self.acquire_write_lock()?; + let acp_session_id = validate_external_id("ACP session id", acp_session_id)?; + let existing = self.lane_acp_session(acp_session_id)?; + let mode_id = mode_id + .map(|value| validate_external_id("ACP mode id", value)) + .transpose()? + .or(existing.current_mode_id.as_deref()); + let mut config_options = existing + .config_options + .as_object() + .cloned() + .unwrap_or_default(); + if let Some((config_id, value)) = config { + let config_id = validate_external_id("ACP config id", config_id)?; + config_options.insert( + config_id.to_string(), + crate::db::redact_sensitive_json(value.clone()), + ); + } + self.conn.execute( + "UPDATE lane_acp_sessions + SET current_mode_id = ?1, config_options_json = ?2, updated_at = ?3 + WHERE acp_session_id = ?4", + params![ + mode_id, + serde_json::to_string(&config_options)?, + now_ts(), + acp_session_id + ], + )?; + self.lane_acp_session(acp_session_id) + } + + pub fn replace_lane_acp_session_configuration_options( + &mut self, + acp_session_id: &str, + config_options: &serde_json::Map, + ) -> Result { + let _lock = self.acquire_write_lock()?; + let acp_session_id = validate_external_id("ACP session id", acp_session_id)?; + self.lane_acp_session(acp_session_id)?; + for config_id in config_options.keys() { + validate_external_id("ACP config id", config_id)?; + } + let config_options = + crate::db::redact_sensitive_json(serde_json::Value::Object(config_options.clone())); + self.conn.execute( + "UPDATE lane_acp_sessions + SET config_options_json = ?1, updated_at = ?2 + WHERE acp_session_id = ?3", + params![ + serde_json::to_string(&config_options)?, + now_ts(), + acp_session_id + ], + )?; + self.lane_acp_session(acp_session_id) + } + pub fn lane_acp_session(&self, acp_session_id: &str) -> Result { self.try_lane_acp_session(acp_session_id)? .ok_or_else(|| Error::InvalidInput(format!("ACP session `{acp_session_id}` not found"))) @@ -109,14 +178,14 @@ impl Trail { let sessions = if let Some(lane) = lane { let branch = self.lane_branch(lane)?; let mut stmt = self.conn.prepare( - "SELECT acp_session_id, upstream_session_id, lane_id, trail_session_id, cwd, provider, model, upstream_command_json, status, created_at, updated_at \ + "SELECT acp_session_id, upstream_session_id, lane_id, trail_session_id, cwd, path_mappings_json, provider, model, upstream_command_json, status, created_at, updated_at, current_mode_id, config_options_json \ FROM lane_acp_sessions WHERE lane_id = ?1 ORDER BY updated_at DESC, acp_session_id DESC", )?; let rows = stmt.query_map(params![branch.lane_id], lane_acp_session_row)?; rows.collect::, _>>()? } else { let mut stmt = self.conn.prepare( - "SELECT acp_session_id, upstream_session_id, lane_id, trail_session_id, cwd, provider, model, upstream_command_json, status, created_at, updated_at \ + "SELECT acp_session_id, upstream_session_id, lane_id, trail_session_id, cwd, path_mappings_json, provider, model, upstream_command_json, status, created_at, updated_at, current_mode_id, config_options_json \ FROM lane_acp_sessions ORDER BY updated_at DESC, acp_session_id DESC", )?; let rows = stmt.query_map([], lane_acp_session_row)?; @@ -129,7 +198,7 @@ impl Trail { let acp_session_id = validate_external_id("ACP session id", acp_session_id)?; self.conn .query_row( - "SELECT acp_session_id, upstream_session_id, lane_id, trail_session_id, cwd, provider, model, upstream_command_json, status, created_at, updated_at \ + "SELECT acp_session_id, upstream_session_id, lane_id, trail_session_id, cwd, path_mappings_json, provider, model, upstream_command_json, status, created_at, updated_at, current_mode_id, config_options_json \ FROM lane_acp_sessions WHERE acp_session_id = ?1", params![acp_session_id], lane_acp_session_row, @@ -140,18 +209,29 @@ impl Trail { } pub(super) fn lane_acp_session_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let path_mappings_json: String = row.get(5)?; + let path_mappings = serde_json::from_str(&path_mappings_json).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure(5, rusqlite::types::Type::Text, Box::new(error)) + })?; + let config_options_json: String = row.get(13)?; + let config_options = serde_json::from_str(&config_options_json).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure(13, rusqlite::types::Type::Text, Box::new(error)) + })?; Ok(LaneAcpSession { acp_session_id: row.get(0)?, upstream_session_id: row.get(1)?, lane_id: row.get(2)?, trail_session_id: row.get(3)?, cwd: row.get(4)?, - provider: row.get(5)?, - model: row.get(6)?, - upstream_command_json: row.get(7)?, - status: row.get(8)?, - created_at: row.get(9)?, - updated_at: row.get(10)?, + path_mappings, + provider: row.get(6)?, + model: row.get(7)?, + upstream_command_json: row.get(8)?, + status: row.get(9)?, + current_mode_id: row.get(12)?, + config_options, + created_at: row.get(10)?, + updated_at: row.get(11)?, }) } @@ -175,10 +255,67 @@ fn validate_acp_session_status(value: &str) -> Result<&'static str> { "loaded" => Ok("loaded"), "resumed" => Ok("resumed"), "closed" => Ok("closed"), + "deleted" => Ok("deleted"), "failed" => Ok("failed"), "cancelled" => Ok("cancelled"), other => Err(Error::InvalidInput(format!( - "ACP session status must be starting, active, loaded, resumed, closed, failed, or cancelled, got `{other}`" + "ACP session status must be starting, active, loaded, resumed, closed, deleted, failed, or cancelled, got `{other}`" ))), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn path_mappings_round_trip_through_create_list_and_update() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "mapping fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("mapping", None, false, None, None).unwrap(); + let session = db + .start_lane_session("mapping", Some("mapping".to_string()), None) + .unwrap() + .session; + let mappings = vec![ + AcpPathMapping { + original: "/workspace/packages/app".to_string(), + effective: "/lane/packages/app".to_string(), + isolated: true, + }, + AcpPathMapping { + original: "/external".to_string(), + effective: "/external".to_string(), + isolated: false, + }, + ]; + + let created = db + .upsert_lane_acp_session( + "acp-mapping", + Some("upstream-mapping"), + "mapping", + &session.session_id, + "/lane/packages/app", + &mappings, + Some("test"), + None, + None, + "active", + ) + .unwrap(); + assert_eq!(created.path_mappings, mappings); + assert_eq!( + db.list_lane_acp_sessions(Some("mapping")).unwrap().sessions[0].path_mappings, + mappings + ); + assert_eq!( + db.update_lane_acp_session_status("acp-mapping", "closed") + .unwrap() + .path_mappings, + mappings + ); + } +} diff --git a/trail/src/db/lane/control/agent_capture.rs b/trail/src/db/lane/control/agent_capture.rs new file mode 100644 index 0000000..c658a9c --- /dev/null +++ b/trail/src/db/lane/control/agent_capture.rs @@ -0,0 +1,4409 @@ +use super::*; +use crate::agent_hooks::{ + parse_agent_hook_payload, AgentHookInstallPlan, AgentHookInstallScope, + AgentHookInstallationRecord, AgentHookParseContext, AgentProviderRegistry, +}; + +const AGENT_HOOK_RECEIPT_OBJECT_KIND: &str = "AgentHookReceipt"; +const AGENT_HOOK_RECEIPT_OBJECT_VERSION: u16 = 1; + +#[derive(Clone, Debug, Serialize, serde::Deserialize)] +struct AgentHookReceiptObject { + version: u16, + provider: String, + native_event: String, + payload_digest: String, + redaction_profile: String, + payload: serde_json::Value, +} + +#[derive(Clone, Debug, Serialize, serde::Deserialize)] +struct LaneArtifactObject { + version: u16, + artifact_kind: String, + format: String, + content_digest: String, + content: Vec, +} + +impl Trail { + /// Persist ownership only after the corresponding filesystem mutation succeeds. + pub fn record_agent_hook_installation( + &mut self, + plan: &AgentHookInstallPlan, + lane: Option<&str>, + ) -> Result { + let _lock = self.acquire_write_lock()?; + let lane_id = lane + .map(|lane| self.lane_branch(lane).map(|branch| branch.lane_id)) + .transpose()?; + let now = now_millis(); + let inventory = serde_json::to_string(&plan.ownership_inventory)?; + self.conn.execute( + "INSERT INTO agent_hook_installations + (installation_id, workspace_id, provider, scope, config_path, lane_id, + manifest_digest, manifest_signature_json, ownership_inventory_json, + config_before_digest, config_after_digest, adapter_version, + provider_version_range, detected_provider_version, capability_status, + status, installed_at, verified_at, last_receipt_at, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, NULL, ?8, ?9, ?10, ?11, + ?12, NULL, 'configured', 'installed', ?13, ?13, NULL, NULL) + ON CONFLICT(installation_id) DO UPDATE SET + provider = excluded.provider, + scope = excluded.scope, + config_path = excluded.config_path, + lane_id = excluded.lane_id, + manifest_digest = excluded.manifest_digest, + ownership_inventory_json = excluded.ownership_inventory_json, + config_before_digest = excluded.config_before_digest, + config_after_digest = excluded.config_after_digest, + adapter_version = excluded.adapter_version, + provider_version_range = excluded.provider_version_range, + capability_status = excluded.capability_status, + status = 'installed', + installed_at = excluded.installed_at, + verified_at = excluded.verified_at", + params![ + plan.installation_id, + self.config.workspace.id.0, + plan.provider, + plan.scope.as_str(), + plan.config_path.to_string_lossy(), + lane_id, + plan.manifest_digest, + inventory, + plan.before_digest, + plan.after_digest, + plan.adapter_version, + plan.provider_version_range, + now, + ], + )?; + self.agent_hook_installation(&plan.installation_id) + } + + pub fn agent_hook_installation( + &self, + installation_id: &str, + ) -> Result { + self.conn + .query_row( + AGENT_HOOK_INSTALLATION_BY_ID_SELECT, + params![installation_id, self.config.workspace.id.0], + map_agent_hook_installation, + ) + .optional()? + .ok_or_else(|| Error::ObjectNotFound { + kind: "agent hook installation", + id: installation_id.to_string(), + }) + } + + pub fn list_agent_hook_installations( + &self, + provider: Option<&str>, + ) -> Result> { + let sql = if provider.is_some() { + AGENT_HOOK_INSTALLATION_BY_PROVIDER_SELECT + } else { + AGENT_HOOK_INSTALLATION_LIST_SELECT + }; + let mut statement = self.conn.prepare(sql)?; + let mut records = Vec::new(); + if let Some(provider) = provider { + let rows = statement.query_map( + params![self.config.workspace.id.0, provider], + map_agent_hook_installation, + )?; + for row in rows { + records.push(row?); + } + } else { + let rows = statement.query_map( + params![self.config.workspace.id.0], + map_agent_hook_installation, + )?; + for row in rows { + records.push(row?); + } + } + Ok(records) + } + + pub fn mark_agent_hook_installation_removed( + &mut self, + installation_id: &str, + ) -> Result { + let _lock = self.acquire_write_lock()?; + let changed = self.conn.execute( + "UPDATE agent_hook_installations + SET status = 'removed', verified_at = ?2 + WHERE installation_id = ?1 AND workspace_id = ?3", + params![installation_id, now_millis(), self.config.workspace.id.0], + )?; + if changed == 0 { + return Err(Error::ObjectNotFound { + kind: "agent hook installation", + id: installation_id.to_string(), + }); + } + self.agent_hook_installation(installation_id) + } + + pub fn begin_agent_capture_run( + &mut self, + input: AgentCaptureRunInput, + ) -> Result { + let _lock = self.acquire_write_lock()?; + validate_agent_provider(&input.owner_agent)?; + if let Some(executor) = input.executor_agent.as_deref() { + validate_agent_provider(executor)?; + } + validate_agent_capture_id("owner session id", &input.owner_session_id, 256)?; + if let Some(work_item) = input.work_item_id.as_deref() { + validate_agent_capture_id("work item id", work_item, 512)?; + } + validate_agent_capture_lease_ms(input.lease_ms)?; + if let Some(metadata) = input.metadata_json.as_deref() { + serde_json::from_str::(metadata).map_err(|error| { + Error::InvalidInput(format!("invalid capture run metadata JSON: {error}")) + })?; + } + + let workdir = PathBuf::from(input.workdir.trim()); + if !workdir.is_absolute() { + return Err(Error::InvalidInput( + "agent capture run workdir must be absolute".to_string(), + )); + } + let canonical_workdir = workdir.canonicalize().map_err(|error| { + Error::InvalidInput(format!( + "cannot canonicalize agent capture run workdir `{}`: {error}", + workdir.display() + )) + })?; + if !canonical_workdir.starts_with(&self.workspace_root) { + return Err(Error::InvalidInput(format!( + "agent capture run workdir `{}` is outside workspace `{}`", + canonical_workdir.display(), + self.workspace_root.display() + ))); + } + let lane_id = input + .lane + .as_deref() + .map(|lane| self.lane_branch(lane).map(|branch| branch.lane_id)) + .transpose()?; + let now = now_millis(); + let expires_at = now.saturating_add(i64::try_from(input.lease_ms).unwrap_or(i64::MAX)); + let capture_run_id = format!( + "capture_run_{}", + crate::ids::short_hash( + format!( + "{}:{}:{}:{}:{}", + self.config.workspace.id.0, + input.owner_agent, + input.owner_session_id, + canonical_workdir.display(), + now_nanos() + ) + .as_bytes(), + 24, + ) + ); + self.conn.execute( + "INSERT INTO agent_capture_runs + (capture_run_id, workspace_id, lane_id, workdir, canonical_workdir, + owner_agent, owner_session_id, executor_agent, work_item_id, status, + created_at, updated_at, expires_at, ended_at, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 'active', ?10, ?10, + ?11, NULL, ?12)", + params![ + capture_run_id, + self.config.workspace.id.0, + lane_id, + workdir.to_string_lossy(), + canonical_workdir.to_string_lossy(), + input.owner_agent, + input.owner_session_id, + input.executor_agent, + input.work_item_id, + now, + expires_at, + input.metadata_json, + ], + )?; + self.agent_capture_run(&capture_run_id) + } + + pub fn agent_capture_run(&self, capture_run_id: &str) -> Result { + validate_agent_capture_id("capture run id", capture_run_id, 256)?; + self.conn + .query_row( + AGENT_CAPTURE_RUN_SELECT_BY_ID, + params![capture_run_id], + agent_capture_run_row, + ) + .optional()? + .ok_or_else(|| { + Error::InvalidInput(format!("agent capture run `{capture_run_id}` not found")) + }) + } + + pub fn list_agent_capture_runs( + &self, + active_only: bool, + limit: usize, + ) -> Result> { + self.list_agent_capture_runs_page(active_only, 0, limit) + } + + pub fn list_agent_capture_runs_page( + &self, + active_only: bool, + offset: usize, + limit: usize, + ) -> Result> { + let mut statement = self.conn.prepare( + "SELECT capture_run_id, workspace_id, lane_id, workdir, canonical_workdir, + owner_agent, owner_session_id, executor_agent, work_item_id, status, + created_at, updated_at, expires_at, ended_at, metadata_json + FROM agent_capture_runs + WHERE workspace_id = ?1 AND (?2 = 0 OR status = 'active') + ORDER BY updated_at DESC, capture_run_id DESC LIMIT ?3 OFFSET ?4", + )?; + let runs = statement + .query_map( + params![ + self.config.workspace.id.0, + if active_only { 1 } else { 0 }, + i64::try_from(limit.clamp(1, 1_000)).unwrap_or(1_000), + i64::try_from(offset.min(1_000_000)).unwrap_or(1_000_000) + ], + agent_capture_run_row, + )? + .collect::, _>>()?; + Ok(runs) + } + + pub fn renew_agent_capture_run( + &mut self, + capture_run_id: &str, + owner_agent: &str, + owner_session_id: &str, + lease_ms: u64, + ) -> Result { + let _lock = self.acquire_write_lock()?; + validate_agent_capture_id("capture run id", capture_run_id, 256)?; + validate_agent_provider(owner_agent)?; + validate_agent_capture_id("owner session id", owner_session_id, 256)?; + validate_agent_capture_lease_ms(lease_ms)?; + let now = now_millis(); + let expires_at = now.saturating_add(i64::try_from(lease_ms).unwrap_or(i64::MAX)); + let updated = self.conn.execute( + "UPDATE agent_capture_runs SET updated_at = ?1, expires_at = ?2 + WHERE capture_run_id = ?3 AND owner_agent = ?4 AND owner_session_id = ?5 + AND status = 'active' AND expires_at > ?1", + params![ + now, + expires_at, + capture_run_id, + owner_agent, + owner_session_id + ], + )?; + if updated != 1 { + return Err(Error::InvalidInput(format!( + "agent capture run `{capture_run_id}` is expired, ended, or owned by another session" + ))); + } + self.agent_capture_run(capture_run_id) + } + + pub fn end_agent_capture_run( + &mut self, + capture_run_id: &str, + owner_agent: &str, + owner_session_id: &str, + ) -> Result { + let _lock = self.acquire_write_lock()?; + validate_agent_capture_id("capture run id", capture_run_id, 256)?; + validate_agent_provider(owner_agent)?; + validate_agent_capture_id("owner session id", owner_session_id, 256)?; + let now = now_millis(); + let updated = self.conn.execute( + "UPDATE agent_capture_runs + SET status = 'ended', updated_at = ?1, ended_at = ?1 + WHERE capture_run_id = ?2 AND owner_agent = ?3 AND owner_session_id = ?4 + AND status = 'active'", + params![now, capture_run_id, owner_agent, owner_session_id], + )?; + if updated != 1 { + let existing = self.agent_capture_run(capture_run_id)?; + if existing.status == "ended" + && existing.owner_agent == owner_agent + && existing.owner_session_id == owner_session_id + { + return Ok(existing); + } + return Err(Error::InvalidInput(format!( + "agent capture run `{capture_run_id}` is not active for this owner" + ))); + } + self.agent_capture_run(capture_run_id) + } + + /// Expire abandoned managed runs and close their still-open captured work as interrupted. + pub fn reconcile_expired_agent_capture_runs(&mut self) -> Result { + let now = now_millis(); + let mut expired_run_ids = { + let mut statement = self.conn.prepare( + "SELECT capture_run_id FROM agent_capture_runs + WHERE workspace_id = ?1 AND status = 'active' AND expires_at <= ?2 + ORDER BY capture_run_id", + )?; + let rows = statement + .query_map(params![self.config.workspace.id.0, now], |row| row.get(0))? + .collect::, _>>()?; + rows + }; + if !expired_run_ids.is_empty() { + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "UPDATE agent_capture_runs SET status = 'expired', updated_at = ?2, ended_at = ?2 + WHERE workspace_id = ?1 AND status = 'active' AND expires_at <= ?2", + params![self.config.workspace.id.0, now], + )?; + } + + let mappings = { + let mut statement = self.conn.prepare( + "SELECT s.mapping_id + FROM lane_agent_sessions s + JOIN agent_capture_runs r ON r.capture_run_id = s.capture_run_id + WHERE s.workspace_id = ?1 AND r.status = 'expired' + AND s.status IN ('active', 'finalizing') + ORDER BY s.mapping_id", + )?; + let rows = statement + .query_map(params![self.config.workspace.id.0], |row| row.get(0))? + .collect::, _>>()?; + rows + }; + let mut interrupted_mapping_ids = Vec::new(); + let mut interrupted_turn_ids = Vec::new(); + for mapping_id in mappings { + let mapping = self.lane_agent_session(&mapping_id)?; + let lane = self.lane_name_by_id(&mapping.lane_id)?; + if let Some(turn_id) = self.open_turn_for_agent_session(&mapping.trail_session_id)? { + if self.lane_details(&lane)?.branch.workdir.is_some() { + self.record_lane_workdir_for_turn( + &lane, + &turn_id, + Some("recovered interrupted agent turn".to_string()), + )?; + } + self.end_lane_turn(&turn_id, "interrupted")?; + self.create_turn_evidence_manifest(&turn_id)?; + interrupted_turn_ids.push(turn_id); + } + let session = self.lane_session(&mapping.trail_session_id)?; + if session.status == "active" { + self.end_lane_session(&mapping.trail_session_id, "interrupted")?; + } + let _lock = self.acquire_write_lock()?; + let changed = self.conn.execute( + "UPDATE lane_agent_sessions + SET status = 'interrupted', pending_turn_outcome = NULL, + session_close_requested = 1, finalization_owner = NULL, + finalization_lease_expires_at = NULL, updated_at = ?3 + WHERE mapping_id = ?1 AND status = ?2", + params![ + mapping_id, + agent_capture_phase_name(mapping.status), + now_millis() + ], + )?; + if changed == 1 { + interrupted_mapping_ids.push(mapping_id); + } + } + expired_run_ids.sort(); + interrupted_mapping_ids.sort(); + interrupted_turn_ids.sort(); + Ok(AgentCaptureRecoveryReport { + expired_run_ids, + interrupted_mapping_ids, + interrupted_turn_ids, + }) + } + + /// Resolve a governing run without guessing between equally specific candidates. + pub fn match_agent_capture_run( + &self, + cwd: impl AsRef, + agent: &str, + ) -> Result> { + validate_agent_provider(agent)?; + let canonical_cwd = cwd.as_ref().canonicalize().map_err(|error| { + Error::InvalidInput(format!( + "cannot canonicalize capture cwd `{}`: {error}", + cwd.as_ref().display() + )) + })?; + let now = now_millis(); + let mut stmt = self.conn.prepare( + "SELECT capture_run_id, workspace_id, lane_id, workdir, canonical_workdir, + owner_agent, owner_session_id, executor_agent, work_item_id, status, + created_at, updated_at, expires_at, ended_at, metadata_json + FROM agent_capture_runs + WHERE workspace_id = ?1 AND status = 'active' AND expires_at > ?2 + AND (owner_agent = ?3 OR executor_agent = ?3) + ORDER BY canonical_workdir ASC, created_at ASC", + )?; + let runs = stmt + .query_map( + params![self.config.workspace.id.0, now, agent], + agent_capture_run_row, + )? + .collect::, _>>()?; + let mut candidates = runs + .into_iter() + .filter_map(|run| { + let path = PathBuf::from(&run.canonical_workdir); + canonical_cwd + .starts_with(&path) + .then_some((path.components().count(), run)) + }) + .collect::>(); + candidates.sort_by(|(left_depth, left), (right_depth, right)| { + right_depth + .cmp(left_depth) + .then_with(|| left.capture_run_id.cmp(&right.capture_run_id)) + }); + let Some((best_depth, best)) = candidates.first() else { + return Ok(None); + }; + if candidates + .get(1) + .is_some_and(|(second_depth, _)| second_depth == best_depth) + { + return Err(Error::InvalidInput(format!( + "multiple agent capture runs equally match `{}` for `{agent}`", + canonical_cwd.display() + ))); + } + Ok(Some(best.clone())) + } + + /// Create the stable provider-native to Trail session mapping exactly once. + pub fn ensure_lane_agent_session( + &mut self, + input: LaneAgentSessionInput, + ) -> Result { + let _lock = self.acquire_write_lock()?; + validate_agent_provider(&input.provider)?; + validate_agent_capture_id("native session id", &input.native_session_id, 1024)?; + if let Some(parent) = input.parent_native_session_id.as_deref() { + validate_agent_capture_id("parent native session id", parent, 1024)?; + } + validate_ref_segment(&input.lane)?; + validate_agent_capture_id("Trail session id", &input.trail_session_id, 256)?; + let branch = self.lane_branch(&input.lane)?; + let session = self.lane_session(&input.trail_session_id)?; + if session.lane_id != branch.lane_id { + return Err(Error::InvalidInput(format!( + "session `{}` does not belong to lane `{}`", + input.trail_session_id, input.lane + ))); + } + if let Some(run_id) = input.capture_run_id.as_deref() { + validate_agent_capture_id("capture run id", run_id, 256)?; + let run: Option<(String, Option, String, i64)> = self + .conn + .query_row( + "SELECT workspace_id, lane_id, status, expires_at + FROM agent_capture_runs WHERE capture_run_id = ?1", + params![run_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .optional()?; + let Some((run_workspace, run_lane, run_status, expires_at)) = run else { + return Err(Error::InvalidInput(format!( + "agent capture run `{run_id}` not found" + ))); + }; + if run_workspace != self.config.workspace.id.0 + || run_lane + .as_deref() + .is_some_and(|lane| lane != branch.lane_id) + || run_status != "active" + || expires_at <= now_millis() + { + return Err(Error::InvalidInput(format!( + "agent capture run `{run_id}` is not active for lane `{}`", + input.lane + ))); + } + } + + if let Some(existing) = + self.try_lane_agent_session(&input.provider, &input.native_session_id)? + { + if existing.trail_session_id != input.trail_session_id + || existing.lane_id != branch.lane_id + { + return Err(Error::InvalidInput(format!( + "native session `{}` is already mapped to a different Trail session", + input.native_session_id + ))); + } + // Managed runs stamp only mappings created while the run governs capture. + // Existing direct sessions are deliberately never restamped here. + return Ok(existing); + } + + let workspace_id = self.config.workspace.id.0.clone(); + let mapping_id = format!( + "agent_session_{}", + crate::ids::short_hash( + format!( + "{}:{}:{}", + workspace_id, input.provider, input.native_session_id + ) + .as_bytes(), + 24, + ) + ); + let now = now_millis(); + self.conn.execute( + "INSERT INTO lane_agent_sessions + (mapping_id, workspace_id, provider, native_session_id, + parent_native_session_id, trail_session_id, lane_id, capture_run_id, + primary_transport, transcript_identity, transcript_offset, resume_json, + last_attestation_id, status, pending_turn_outcome, + session_close_requested, capture_epoch, finalization_owner, + finalization_lease_expires_at, next_receive_sequence, created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, NULL, NULL, NULL, + 'idle', NULL, 0, 1, NULL, NULL, 1, ?11, ?11)", + params![ + mapping_id, + workspace_id, + input.provider, + input.native_session_id, + input.parent_native_session_id, + input.trail_session_id, + branch.lane_id, + input.capture_run_id, + agent_capture_transport_name(input.primary_transport), + input.transcript_identity, + now, + ], + )?; + self.lane_agent_session(&mapping_id) + } + + pub fn lane_agent_session(&self, mapping_id: &str) -> Result { + validate_agent_capture_id("agent session mapping id", mapping_id, 256)?; + self.conn + .query_row( + LANE_AGENT_SESSION_SELECT_BY_ID, + params![mapping_id], + lane_agent_session_row, + ) + .optional()? + .ok_or_else(|| { + Error::InvalidInput(format!("agent session mapping `{mapping_id}` not found")) + }) + } + + pub fn try_lane_agent_session( + &self, + provider: &str, + native_session_id: &str, + ) -> Result> { + validate_agent_provider(provider)?; + validate_agent_capture_id("native session id", native_session_id, 1024)?; + self.conn + .query_row( + "SELECT mapping_id, workspace_id, provider, native_session_id, + parent_native_session_id, trail_session_id, lane_id, capture_run_id, + primary_transport, transcript_identity, transcript_offset, resume_json, + last_attestation_id, status, pending_turn_outcome, + session_close_requested, capture_epoch, finalization_owner, + finalization_lease_expires_at, next_receive_sequence, created_at, updated_at + FROM lane_agent_sessions + WHERE workspace_id = ?1 AND provider = ?2 AND native_session_id = ?3", + params![self.config.workspace.id.0, provider, native_session_id], + lane_agent_session_row, + ) + .optional() + .map_err(Error::from) + } + + /// Resolve a provider-native session id for user-facing selectors. Native ids are + /// normally globally unique, but fail closed if two providers claim the same id. + pub fn try_lane_agent_session_by_native_id( + &self, + native_session_id: &str, + ) -> Result> { + validate_agent_capture_id("native session id", native_session_id, 1024)?; + let mut statement = self.conn.prepare( + "SELECT mapping_id, workspace_id, provider, native_session_id, + parent_native_session_id, trail_session_id, lane_id, capture_run_id, + primary_transport, transcript_identity, transcript_offset, resume_json, + last_attestation_id, status, pending_turn_outcome, + session_close_requested, capture_epoch, finalization_owner, + finalization_lease_expires_at, next_receive_sequence, created_at, updated_at + FROM lane_agent_sessions + WHERE workspace_id = ?1 AND native_session_id = ?2 + ORDER BY updated_at DESC, mapping_id DESC + LIMIT 2", + )?; + let mappings = statement + .query_map( + params![self.config.workspace.id.0, native_session_id], + lane_agent_session_row, + )? + .collect::, _>>()?; + match mappings.as_slice() { + [] => Ok(None), + [mapping] => Ok(Some(mapping.clone())), + _ => Err(Error::Conflict(format!( + "native session id `{native_session_id}` is ambiguous across providers" + ))), + } + } + + pub fn list_lane_agent_sessions( + &self, + provider: Option<&str>, + status: Option, + limit: usize, + ) -> Result> { + self.list_lane_agent_sessions_page(provider, status, 0, limit) + } + + pub fn list_lane_agent_sessions_page( + &self, + provider: Option<&str>, + status: Option, + offset: usize, + limit: usize, + ) -> Result> { + if let Some(provider) = provider { + validate_agent_provider(provider)?; + } + let status_name = status.map(agent_capture_phase_name); + let mut statement = self.conn.prepare( + "SELECT mapping_id, workspace_id, provider, native_session_id, + parent_native_session_id, trail_session_id, lane_id, capture_run_id, + primary_transport, transcript_identity, transcript_offset, resume_json, + last_attestation_id, status, pending_turn_outcome, + session_close_requested, capture_epoch, finalization_owner, + finalization_lease_expires_at, next_receive_sequence, created_at, updated_at + FROM lane_agent_sessions + WHERE workspace_id = ?1 AND (?2 IS NULL OR provider = ?2) + AND (?3 IS NULL OR status = ?3) + ORDER BY updated_at DESC, mapping_id DESC LIMIT ?4 OFFSET ?5", + )?; + let rows = statement + .query_map( + params![ + self.config.workspace.id.0, + provider, + status_name, + i64::try_from(limit.clamp(1, 1_000)).unwrap_or(1_000), + i64::try_from(offset.min(1_000_000)).unwrap_or(1_000_000) + ], + lane_agent_session_row, + )? + .collect::, _>>() + .map_err(Error::from)?; + Ok(rows) + } + + /// Acquire or renew the one durable finalizer lease for a native session. + pub fn acquire_agent_finalization_lease( + &mut self, + mapping_id: &str, + owner: &str, + ttl_ms: u64, + outcome: AgentTurnOutcome, + request_session_close: bool, + ) -> Result { + let _lock = self.acquire_write_lock()?; + validate_agent_capture_id("agent session mapping id", mapping_id, 256)?; + validate_agent_capture_id("finalization owner", owner, 256)?; + if !(1_000..=300_000).contains(&ttl_ms) { + return Err(Error::InvalidInput( + "agent finalization lease must be between 1000 and 300000 milliseconds".to_string(), + )); + } + let now = now_millis(); + let expires_at = now.saturating_add(i64::try_from(ttl_ms).unwrap_or(i64::MAX)); + let updated = self.conn.execute( + "UPDATE lane_agent_sessions + SET status = 'finalizing', pending_turn_outcome = ?1, + session_close_requested = CASE + WHEN session_close_requested = 1 OR ?2 = 1 THEN 1 ELSE 0 END, + finalization_owner = ?3, finalization_lease_expires_at = ?4, + updated_at = ?5 + WHERE mapping_id = ?6 + AND status IN ('idle', 'active', 'finalizing') + AND (status != 'finalizing' OR finalization_owner = ?3 + OR finalization_lease_expires_at IS NULL + OR finalization_lease_expires_at <= ?5)", + params![ + agent_turn_outcome_name(outcome), + request_session_close, + owner, + expires_at, + now, + mapping_id, + ], + )?; + let mapping = self.lane_agent_session(mapping_id)?; + Ok(AgentFinalizationLeaseReport { + expires_at: mapping.finalization_lease_expires_at.unwrap_or(expires_at), + mapping, + acquired: updated == 1, + owner: owner.to_string(), + }) + } + + /// Persist a bounded, redacted native receipt before semantic processing. + pub fn persist_agent_hook_receipt( + &mut self, + input: AgentHookReceiptInput, + ) -> Result { + let _lock = self.acquire_write_lock()?; + validate_agent_receipt_input(&input)?; + + let workspace_id = self.config.workspace.id.0.clone(); + if let Some(installation_id) = input.installation_id.as_deref() { + let installation: Option<(String, String, String)> = self + .conn + .query_row( + "SELECT workspace_id, provider, status FROM agent_hook_installations + WHERE installation_id = ?1", + params![installation_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional()?; + let Some((installation_workspace, installation_provider, installation_status)) = + installation + else { + return Err(Error::InvalidInput(format!( + "agent hook installation `{installation_id}` not found" + ))); + }; + if installation_workspace != workspace_id || installation_provider != input.provider { + return Err(Error::InvalidInput(format!( + "agent hook installation `{installation_id}` does not authorize provider `{}` in this workspace", + input.provider + ))); + } + if installation_status != "installed" { + return Err(Error::InvalidInput(format!( + "agent hook installation `{installation_id}` is `{installation_status}` and cannot ingest receipts" + ))); + } + } + + let redacted_payload = redact_sensitive_json(input.payload.clone()); + let payload_bytes = serde_json::to_vec(&redacted_payload)?; + let configured_limit = + usize::try_from(self.config.lane.max_event_payload_bytes).unwrap_or(usize::MAX); + let payload_limit = if configured_limit == 0 { + AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES + } else { + configured_limit.min(AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES) + }; + if payload_bytes.len() > payload_limit { + return Err(Error::InvalidInput(format!( + "agent hook receipt payload is {} bytes; maximum is {payload_limit}", + payload_bytes.len() + ))); + } + let payload_digest = format!("sha256:{}", sha256_hex(&payload_bytes)); + + if let Some(existing) = self.try_agent_hook_receipt_by_dedupe_key( + &workspace_id, + &input.provider, + &input.dedupe_key, + )? { + if existing.payload_digest != payload_digest { + return Err(Error::InvalidInput(format!( + "agent hook dedupe key `{}` was reused with different payload content", + input.dedupe_key + ))); + } + return Ok(AgentHookReceiptIngestReport { + receipt: existing, + duplicate: true, + }); + } + + let stored = AgentHookReceiptObject { + version: AGENT_HOOK_RECEIPT_OBJECT_VERSION, + provider: input.provider.clone(), + native_event: input.native_event.clone(), + payload_digest: payload_digest.clone(), + redaction_profile: "trail-sensitive-json/v1".to_string(), + payload: redacted_payload, + }; + let raw_object_id = self.put_object( + AGENT_HOOK_RECEIPT_OBJECT_KIND, + AGENT_HOOK_RECEIPT_OBJECT_VERSION, + &stored, + )?; + let receipt_id = format!( + "receipt_{}", + crate::ids::short_hash( + format!( + "{}:{}:{}:{}", + workspace_id, input.provider, input.dedupe_key, payload_digest + ) + .as_bytes(), + 24, + ) + ); + let now = now_millis(); + self.conn.execute( + "INSERT INTO agent_hook_receipts + (receipt_id, workspace_id, installation_id, mapping_id, provider, + native_event, native_session_id, native_turn_id, transport, dedupe_key, + payload_digest, raw_object_id, raw_artifact_id, receive_sequence, + connection_id, direction, connection_sequence, status, + attempt_count, next_attempt_at, diagnostic, occurred_at, received_at, + processed_at, updated_at) + VALUES (?1, ?2, ?3, NULL, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, NULL, + NULL, ?12, ?13, ?14, 'received', 0, NULL, NULL, ?15, ?16, NULL, ?16)", + params![ + receipt_id, + workspace_id, + input.installation_id, + input.provider, + input.native_event, + input.native_session_id, + input.native_turn_id, + agent_capture_transport_name(input.transport), + input.dedupe_key, + payload_digest, + raw_object_id.0, + input.connection_id, + input.direction, + input + .connection_sequence + .map(i64::try_from) + .transpose() + .map_err(|_| { + Error::InvalidInput( + "ACP connection sequence exceeds SQLite integer range".to_string(), + ) + })?, + input.occurred_at, + now, + ], + )?; + Ok(AgentHookReceiptIngestReport { + receipt: self.agent_hook_receipt(&receipt_id)?, + duplicate: false, + }) + } + + pub fn agent_hook_receipt(&self, receipt_id: &str) -> Result { + validate_agent_capture_id("receipt id", receipt_id, 256)?; + self.conn + .query_row( + "SELECT receipt_id, workspace_id, installation_id, mapping_id, provider, + native_event, native_session_id, native_turn_id, transport, dedupe_key, + payload_digest, raw_object_id, raw_artifact_id, receive_sequence, + connection_id, direction, connection_sequence, status, + attempt_count, next_attempt_at, diagnostic, occurred_at, received_at, + processed_at, updated_at + FROM agent_hook_receipts WHERE receipt_id = ?1", + params![receipt_id], + agent_hook_receipt_row, + ) + .optional()? + .ok_or_else(|| { + Error::InvalidInput(format!("agent hook receipt `{receipt_id}` not found")) + }) + } + + pub(crate) fn mark_agent_hook_receipt_processed( + &mut self, + receipt_id: &str, + ) -> Result { + let _lock = self.acquire_write_lock()?; + let now = now_millis(); + self.conn.execute( + "UPDATE agent_hook_receipts + SET status = 'processed', processed_at = COALESCE(processed_at, ?1), updated_at = ?1, + diagnostic = NULL, next_attempt_at = NULL + WHERE receipt_id = ?2", + params![now, receipt_id], + )?; + self.agent_hook_receipt(receipt_id) + } + + pub(crate) fn pending_acp_capture_payloads(&self) -> Result> { + let mut stmt = self.conn.prepare( + "SELECT raw_object_id + FROM agent_hook_receipts + WHERE workspace_id = ?1 AND provider = 'trail-acp' + AND status IN ('received', 'retry') + ORDER BY received_at ASC, connection_id ASC, connection_sequence ASC, + direction ASC, receipt_id ASC", + )?; + let object_ids = stmt + .query_map(params![self.config.workspace.id.0], |row| { + Ok(ObjectId(row.get(0)?)) + })? + .collect::, _>>()?; + drop(stmt); + + object_ids + .into_iter() + .map(|object_id| { + let object: AgentHookReceiptObject = + self.get_object(AGENT_HOOK_RECEIPT_OBJECT_KIND, &object_id)?; + Ok(object.payload) + }) + .collect() + } + + pub fn list_agent_hook_receipts( + &self, + provider: Option<&str>, + status: Option<&str>, + limit: usize, + ) -> Result> { + self.list_agent_hook_receipts_page(provider, status, 0, limit) + } + + pub fn list_agent_hook_receipts_page( + &self, + provider: Option<&str>, + status: Option<&str>, + offset: usize, + limit: usize, + ) -> Result> { + if let Some(provider) = provider { + validate_agent_provider(provider)?; + } + if let Some(status) = status { + validate_agent_receipt_status(status)?; + } + let limit = limit.clamp(1, 1_000) as i64; + let mut stmt = self.conn.prepare( + "SELECT receipt_id, workspace_id, installation_id, mapping_id, provider, + native_event, native_session_id, native_turn_id, transport, dedupe_key, + payload_digest, raw_object_id, raw_artifact_id, receive_sequence, + connection_id, direction, connection_sequence, status, + attempt_count, next_attempt_at, diagnostic, occurred_at, received_at, + processed_at, updated_at + FROM agent_hook_receipts + WHERE (?1 IS NULL OR provider = ?1) AND (?2 IS NULL OR status = ?2) + ORDER BY received_at DESC, receipt_id DESC LIMIT ?3 OFFSET ?4", + )?; + let receipts = stmt + .query_map( + params![ + provider, + status, + limit, + i64::try_from(offset.min(1_000_000)).unwrap_or(1_000_000) + ], + agent_hook_receipt_row, + )? + .collect::, _>>() + .map_err(Error::from)?; + Ok(receipts) + } + + pub fn recover_stale_agent_hook_receipts(&mut self, stale_after_ms: u64) -> Result { + if !(1_000..=86_400_000).contains(&stale_after_ms) { + return Err(Error::InvalidInput( + "stale receipt timeout must be between 1000 and 86400000 milliseconds".to_string(), + )); + } + let now = now_millis(); + let cutoff = now.saturating_sub(i64::try_from(stale_after_ms).unwrap_or(i64::MAX)); + let _lock = self.acquire_write_lock()?; + self.conn + .execute( + "UPDATE agent_hook_receipts + SET status = 'retry', next_attempt_at = ?1, + diagnostic = 'processing lease expired; receipt recovered for replay', + updated_at = ?1 + WHERE workspace_id = ?2 AND status = 'processing' AND updated_at <= ?3", + params![now, self.config.workspace.id.0, cutoff], + ) + .map_err(Error::from) + } + + pub fn retry_agent_hook_receipt(&mut self, receipt_id: &str) -> Result { + let existing = self.agent_hook_receipt(receipt_id)?; + if existing.status == "received" { + return Ok(existing); + } + if !matches!(existing.status.as_str(), "retry" | "quarantined") { + return Err(Error::Conflict(format!( + "agent hook receipt `{receipt_id}` cannot be retried from `{}`", + existing.status + ))); + } + let _lock = self.acquire_write_lock()?; + let changed = self.conn.execute( + "UPDATE agent_hook_receipts + SET status = 'received', next_attempt_at = NULL, diagnostic = NULL, + updated_at = ?2 + WHERE receipt_id = ?1 AND status IN ('retry', 'quarantined')", + params![receipt_id, now_millis()], + )?; + if changed != 1 { + return Err(Error::Conflict(format!( + "agent hook receipt `{receipt_id}` changed while requesting retry" + ))); + } + self.agent_hook_receipt(receipt_id) + } + + pub fn discard_agent_hook_receipt(&mut self, receipt_id: &str) -> Result { + let existing = self.agent_hook_receipt(receipt_id)?; + if existing.status == "discarded" { + return Ok(existing); + } + if !matches!( + existing.status.as_str(), + "received" | "retry" | "quarantined" + ) { + return Err(Error::Conflict(format!( + "agent hook receipt `{receipt_id}` cannot be discarded from `{}`", + existing.status + ))); + } + let _lock = self.acquire_write_lock()?; + let changed = self.conn.execute( + "UPDATE agent_hook_receipts + SET status = 'discarded', next_attempt_at = NULL, + diagnostic = 'discarded explicitly by operator', updated_at = ?2 + WHERE receipt_id = ?1 AND status IN ('received', 'retry', 'quarantined')", + params![receipt_id, now_millis()], + )?; + if changed != 1 { + return Err(Error::Conflict(format!( + "agent hook receipt `{receipt_id}` changed while discarding" + ))); + } + self.agent_hook_receipt(receipt_id) + } + + pub fn replay_pending_agent_hook_receipts( + &mut self, + limit: usize, + ) -> Result { + let recovered_stale_receipts = self.recover_stale_agent_hook_receipts(60_000)?; + let now = now_millis(); + let mut statement = self.conn.prepare( + "SELECT receipt_id FROM agent_hook_receipts + WHERE workspace_id = ?1 + AND (status = 'received' OR + (status = 'retry' AND (next_attempt_at IS NULL OR next_attempt_at <= ?2))) + ORDER BY received_at, receipt_id LIMIT ?3", + )?; + let receipt_ids = statement + .query_map( + params![ + self.config.workspace.id.0, + now, + i64::try_from(limit.clamp(1, 1_000)).unwrap_or(1_000) + ], + |row| row.get::<_, String>(0), + )? + .collect::, _>>()?; + drop(statement); + let mut replayed = Vec::new(); + let mut failures = Vec::new(); + for receipt_id in receipt_ids { + match self.replay_agent_hook_receipt(&receipt_id) { + Ok(report) => replayed.push(report), + Err(error) => failures.push(AgentHookReplayFailure { + receipt_id, + code: error.code().to_string(), + message: redact_agent_capture_diagnostic(&error.to_string()), + }), + } + } + Ok(AgentHookReplayBatchReport { + recovered_stale_receipts, + replayed, + failures, + }) + } + + pub fn record_lane_artifact(&mut self, input: LaneArtifactInput) -> Result { + const MAX_ARTIFACT_BYTES: usize = 64 * 1024 * 1024; + if input.content.len() > MAX_ARTIFACT_BYTES { + return Err(Error::InvalidInput(format!( + "agent artifact is {} bytes; maximum is {MAX_ARTIFACT_BYTES}", + input.content.len() + ))); + } + validate_agent_provider(&input.provider)?; + for (name, value) in [ + ("artifact kind", input.artifact_kind.as_str()), + ("artifact format", input.format.as_str()), + ("artifact trust", input.trust.as_str()), + ] { + validate_agent_capture_id(name, value, 128)?; + } + if let Some(metadata) = input.metadata_json.as_deref() { + serde_json::from_str::(metadata).map_err(|error| { + Error::InvalidInput(format!("invalid agent artifact metadata JSON: {error}")) + })?; + } + if input + .start_offset + .zip(input.end_offset) + .is_some_and(|(start, end)| start > end) + { + return Err(Error::InvalidInput( + "agent artifact start_offset exceeds end_offset".to_string(), + )); + } + let branch = self.lane_branch(&input.lane)?; + let session = self.lane_session(&input.session_id)?; + if session.lane_id != branch.lane_id { + return Err(Error::InvalidInput(format!( + "session `{}` does not belong to lane `{}`", + input.session_id, input.lane + ))); + } + if let Some(turn_id) = input.turn_id.as_deref() { + let turn = self.lane_turn(turn_id)?; + if turn.session_id.as_deref() != Some(input.session_id.as_str()) { + return Err(Error::InvalidInput(format!( + "turn `{turn_id}` does not belong to session `{}`", + input.session_id + ))); + } + } + if let Some(supersedes) = input.supersedes_artifact_id.as_deref() { + self.lane_artifact(supersedes)?; + } + let content_digest = format!("sha256:{}", sha256_hex(&input.content)); + let object = LaneArtifactObject { + version: 1, + artifact_kind: input.artifact_kind.clone(), + format: input.format.clone(), + content_digest: content_digest.clone(), + content: input.content.clone(), + }; + let object_id = self.put_object("LaneArtifact", 1, &object)?; + let now = now_millis(); + let artifact_id = format!( + "artifact_{}", + crate::ids::short_hash( + format!( + "{}:{}:{}:{}:{}:{}", + self.config.workspace.id.0, + input.session_id, + input.turn_id.as_deref().unwrap_or("session"), + input.artifact_kind, + content_digest, + input.start_offset.unwrap_or(0) + ) + .as_bytes(), + 24, + ) + ); + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "INSERT INTO lane_artifacts + (artifact_id, workspace_id, lane_id, session_id, turn_id, provider, + artifact_kind, format, source, source_locator_redacted, content_object_id, + content_digest, size_bytes, start_offset, end_offset, redaction_profile, + retention_status, trust, supersedes_artifact_id, created_at, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, + ?14, ?15, ?16, 'active', ?17, ?18, ?19, ?20) + ON CONFLICT(artifact_id) DO NOTHING", + params![ + artifact_id, + self.config.workspace.id.0, + branch.lane_id, + input.session_id, + input.turn_id, + input.provider, + input.artifact_kind, + input.format, + agent_evidence_source_name(input.source), + input + .source_locator_redacted + .as_deref() + .map(redact_sensitive_text), + object_id.0, + content_digest, + i64::try_from(input.content.len()).unwrap_or(i64::MAX), + input + .start_offset + .and_then(|value| i64::try_from(value).ok()), + input.end_offset.and_then(|value| i64::try_from(value).ok()), + input.redaction_profile, + input.trust, + input.supersedes_artifact_id, + now, + input.metadata_json, + ], + )?; + self.lane_artifact(&artifact_id) + } + + pub fn lane_artifact(&self, artifact_id: &str) -> Result { + self.conn + .query_row( + LANE_ARTIFACT_BY_ID_SELECT, + params![artifact_id], + map_lane_artifact, + ) + .optional()? + .ok_or_else(|| Error::ObjectNotFound { + kind: "lane artifact", + id: artifact_id.to_string(), + }) + } + + pub fn lane_artifact_content(&self, artifact_id: &str) -> Result> { + let artifact = self.lane_artifact(artifact_id)?; + let object_id = + artifact + .content_object_id + .as_ref() + .ok_or_else(|| Error::ObjectNotFound { + kind: "lane artifact content", + id: artifact_id.to_string(), + })?; + let object: LaneArtifactObject = self.get_object("LaneArtifact", object_id)?; + if object.content_digest != artifact.content_digest + || object.artifact_kind != artifact.artifact_kind + || object.format != artifact.format + || format!("sha256:{}", sha256_hex(&object.content)) != artifact.content_digest + { + return Err(Error::Corrupt(format!( + "lane artifact `{artifact_id}` content does not match its immutable metadata" + ))); + } + Ok(object.content) + } + + /// Remove attachment access while preserving immutable digest and evidence identity. + pub fn redact_lane_artifact( + &mut self, + artifact_id: &str, + reason: &str, + ) -> Result { + validate_agent_capture_id("artifact redaction reason", reason, 512)?; + let existing = self.lane_artifact(artifact_id)?; + if existing.retention_status == "redacted" { + return Ok(existing); + } + { + let _lock = self.acquire_write_lock()?; + let changed = self.conn.execute( + "UPDATE lane_artifacts + SET content_object_id = NULL, retention_status = 'redacted' + WHERE artifact_id = ?1 AND retention_status = 'active'", + params![artifact_id], + )?; + if changed != 1 { + return Err(Error::Conflict(format!( + "agent artifact `{artifact_id}` changed while redacting attachment access" + ))); + } + } + let lane = self.lane_name_by_id(&existing.lane_id)?; + self.add_lane_session_event( + &lane, + &existing.session_id, + "artifact.redacted", + Some(serde_json::json!({ + "artifact_id": artifact_id, + "content_digest": existing.content_digest, + "reason": redact_agent_capture_diagnostic(reason), + })), + )?; + self.lane_artifact(artifact_id) + } + + pub fn list_lane_artifacts( + &self, + session_id: &str, + turn_id: Option<&str>, + limit: usize, + ) -> Result> { + self.list_lane_artifacts_page(session_id, turn_id, 0, limit) + } + + pub fn list_lane_artifacts_page( + &self, + session_id: &str, + turn_id: Option<&str>, + offset: usize, + limit: usize, + ) -> Result> { + self.lane_session(session_id)?; + let mut statement = self.conn.prepare( + "SELECT artifact_id, workspace_id, lane_id, session_id, turn_id, provider, + artifact_kind, format, source, source_locator_redacted, content_object_id, + content_digest, size_bytes, start_offset, end_offset, redaction_profile, + retention_status, trust, supersedes_artifact_id, created_at, metadata_json + FROM lane_artifacts + WHERE session_id = ?1 AND (?2 IS NULL OR turn_id = ?2) + ORDER BY created_at DESC, artifact_id DESC LIMIT ?3 OFFSET ?4", + )?; + let artifacts = statement + .query_map( + params![ + session_id, + turn_id, + i64::try_from(limit.clamp(1, 1_000)).unwrap_or(1_000), + i64::try_from(offset.min(1_000_000)).unwrap_or(1_000_000) + ], + map_lane_artifact, + )? + .collect::, _>>() + .map_err(Error::from)?; + Ok(artifacts) + } + + /// Replay one durable receipt through the provider parser and shared lifecycle machine. + pub fn replay_agent_hook_receipt(&mut self, receipt_id: &str) -> Result { + let receipt = self.agent_hook_receipt(receipt_id)?; + if receipt.status == "processed" { + return self.refresh_processed_agent_hook_receipt_artifacts(receipt); + } + { + let _lock = self.acquire_write_lock()?; + let changed = self.conn.execute( + "UPDATE agent_hook_receipts + SET status = 'processing', attempt_count = attempt_count + 1, + diagnostic = NULL, updated_at = ?2 + WHERE receipt_id = ?1 AND status IN ('received', 'retry')", + params![receipt_id, now_millis()], + )?; + if changed == 0 { + return Err(Error::Conflict(format!( + "agent hook receipt `{receipt_id}` is not replayable from status `{}`", + receipt.status + ))); + } + } + + match self.replay_agent_hook_receipt_inner(receipt_id) { + Ok(report) => Ok(report), + Err(error) => { + let status = if matches!(error, Error::InvalidInput(_) | Error::Conflict(_)) { + "quarantined" + } else { + "retry" + }; + let diagnostic = redact_agent_capture_diagnostic(&error.to_string()); + let attempt_count = self.agent_hook_receipt(receipt_id)?.attempt_count.max(1); + let exponent = attempt_count.saturating_sub(1).min(8); + let backoff_ms = 1_000_i64.saturating_mul(1_i64 << exponent).min(300_000); + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "UPDATE agent_hook_receipts + SET status = ?2, diagnostic = ?3, + next_attempt_at = CASE WHEN ?2 = 'retry' THEN ?4 ELSE NULL END, + updated_at = ?5 + WHERE receipt_id = ?1", + params![ + receipt_id, + status, + diagnostic, + now_millis().saturating_add(backoff_ms), + now_millis(), + ], + )?; + Err(error) + } + } + } + + fn refresh_processed_agent_hook_receipt_artifacts( + &mut self, + receipt: AgentHookReceipt, + ) -> Result { + let Some(mapping_id) = receipt.mapping_id.clone() else { + return Ok(AgentHookReplayReport { + receipt, + mapping: None, + normalized_events: Vec::new(), + actions: Vec::new(), + diagnostics: Vec::new(), + replayed: false, + }); + }; + let mapping = self.lane_agent_session(&mapping_id)?; + let object: AgentHookReceiptObject = + self.get_object(AGENT_HOOK_RECEIPT_OBJECT_KIND, &receipt.raw_object_id)?; + let registry = AgentProviderRegistry::built_in()?; + let events = parse_agent_hook_payload( + ®istry, + &receipt.provider, + &receipt.native_event, + &object.payload, + AgentHookParseContext { + receipt_id: receipt.receipt_id.clone(), + workspace_id: receipt.workspace_id.clone(), + lane_id: Some(mapping.lane_id.clone()), + capture_run_id: mapping.capture_run_id.clone(), + provider_version: None, + raw_digest: receipt.payload_digest.clone(), + received_at: receipt.received_at, + transport: receipt.transport, + }, + )?; + let mut actions = Vec::new(); + for event in &events { + if event.event_type.kind().terminal_outcome().is_some() + || event.event_type.kind() == AgentLifecycleEventKind::SessionEnded + { + self.import_agent_transcript_snapshot(&mapping, event)?; + actions.push(AgentCaptureAction::ImportTranscript); + } + } + Ok(AgentHookReplayReport { + receipt, + mapping: Some(self.lane_agent_session(&mapping_id)?), + normalized_events: events, + actions, + diagnostics: Vec::new(), + replayed: false, + }) + } + + fn replay_agent_hook_receipt_inner( + &mut self, + receipt_id: &str, + ) -> Result { + let receipt = self.agent_hook_receipt(receipt_id)?; + let object: AgentHookReceiptObject = + self.get_object(AGENT_HOOK_RECEIPT_OBJECT_KIND, &receipt.raw_object_id)?; + if object.payload_digest != receipt.payload_digest || object.provider != receipt.provider { + return Err(Error::Corrupt(format!( + "agent hook receipt `{receipt_id}` object does not match its journal row" + ))); + } + let mapping = self.ensure_mapping_for_agent_hook_receipt(&receipt, &object.payload)?; + let receipt = self.assign_agent_hook_receipt_mapping(receipt_id, &mapping.mapping_id)?; + let registry = AgentProviderRegistry::built_in()?; + let events = parse_agent_hook_payload( + ®istry, + &receipt.provider, + &receipt.native_event, + &object.payload, + AgentHookParseContext { + receipt_id: receipt.receipt_id.clone(), + workspace_id: receipt.workspace_id.clone(), + lane_id: Some(mapping.lane_id.clone()), + capture_run_id: mapping.capture_run_id.clone(), + provider_version: None, + raw_digest: receipt.payload_digest.clone(), + received_at: receipt.received_at, + transport: receipt.transport, + }, + )?; + let lane_name = self.lane_name_by_id(&mapping.lane_id)?; + let mut actions = Vec::new(); + let mut diagnostics = Vec::new(); + let mut current = self.lane_agent_session(&mapping.mapping_id)?; + for event in &events { + if current.primary_transport == AgentCaptureTransport::Hybrid + && self.hybrid_acp_owns_session(¤t.trail_session_id)? + { + let mut hybrid_actions = vec![AgentCaptureAction::AppendEvidence]; + if event.event_type.kind().terminal_outcome().is_some() + || event.event_type.kind() == AgentLifecycleEventKind::SessionEnded + { + hybrid_actions.push(AgentCaptureAction::ImportTranscript); + } + self.execute_agent_capture_actions(&lane_name, ¤t, event, &hybrid_actions)?; + actions.extend(hybrid_actions); + continue; + } + let open_turn = self.open_turn_for_agent_session(¤t.trail_session_id)?; + let already_recorded = self.agent_lifecycle_event_already_recorded( + ¤t.trail_session_id, + &event.event_id, + )?; + if already_recorded + && event.event_type.kind().terminal_outcome().is_some() + && open_turn.is_none() + && matches!( + current.status, + AgentCapturePhase::Idle + | AgentCapturePhase::Ended + | AgentCapturePhase::Interrupted + ) + { + continue; + } + let transition = AgentCaptureCoordinator::decide( + current.status, + event.event_type.kind(), + AgentTransitionContext { + has_session: true, + has_open_turn: open_turn.is_some(), + duplicate: false, + has_new_evidence: true, + session_close_requested: current.session_close_requested, + pending_turn_outcome: current.pending_turn_outcome, + }, + ); + diagnostics.extend(transition.diagnostics.clone()); + let replay_actions = transition + .actions + .iter() + .filter(|action| { + !(already_recorded && matches!(action, AgentCaptureAction::AppendEvidence)) + }) + .cloned() + .collect::>(); + self.execute_agent_capture_actions(&lane_name, ¤t, event, &replay_actions)?; + actions.extend(transition.actions.clone()); + let persisted = self.lane_agent_session(¤t.mapping_id)?; + self.update_agent_mapping_phase( + ¤t.mapping_id, + persisted.status, + persisted.finalization_owner.as_deref(), + transition.new_phase, + transition + .actions + .iter() + .find_map(|action| match action { + AgentCaptureAction::RequestTurnFinalization { outcome } => Some(*outcome), + _ => None, + }) + .or(current.pending_turn_outcome), + transition + .actions + .iter() + .any(|action| matches!(action, AgentCaptureAction::RequestSessionClose)) + || current.session_close_requested, + )?; + current = self.lane_agent_session(¤t.mapping_id)?; + + if transition.new_phase == AgentCapturePhase::Finalizing { + let completed = AgentCaptureCoordinator::decide( + AgentCapturePhase::Finalizing, + AgentLifecycleEventKind::FinalizationCompleted, + AgentTransitionContext { + has_session: true, + has_open_turn: self + .open_turn_for_agent_session(¤t.trail_session_id)? + .is_some(), + duplicate: false, + has_new_evidence: false, + session_close_requested: current.session_close_requested, + pending_turn_outcome: current.pending_turn_outcome, + }, + ); + self.execute_agent_capture_actions( + &lane_name, + ¤t, + event, + &completed.actions, + )?; + actions.extend(completed.actions.clone()); + diagnostics.extend(completed.diagnostics); + let persisted = self.lane_agent_session(¤t.mapping_id)?; + self.update_agent_mapping_phase( + ¤t.mapping_id, + persisted.status, + persisted.finalization_owner.as_deref(), + completed.new_phase, + None, + current.session_close_requested, + )?; + current = self.lane_agent_session(¤t.mapping_id)?; + } + } + let now = now_millis(); + { + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "UPDATE agent_hook_receipts + SET status = 'processed', processed_at = ?2, next_attempt_at = NULL, + diagnostic = NULL, updated_at = ?2 + WHERE receipt_id = ?1 AND status = 'processing'", + params![receipt_id, now], + )?; + if let Some(installation_id) = receipt.installation_id.as_deref() { + self.conn.execute( + "UPDATE agent_hook_installations + SET last_receipt_at = ?2, verified_at = ?2 + WHERE installation_id = ?1", + params![installation_id, now], + )?; + } + } + Ok(AgentHookReplayReport { + receipt: self.agent_hook_receipt(receipt_id)?, + mapping: Some(current), + normalized_events: events, + actions, + diagnostics, + replayed: true, + }) + } + + fn ensure_mapping_for_agent_hook_receipt( + &mut self, + receipt: &AgentHookReceipt, + payload: &serde_json::Value, + ) -> Result { + let native_session_id = receipt.native_session_id.as_deref().ok_or_else(|| { + Error::InvalidInput(format!( + "agent hook receipt `{}` has no native session id", + receipt.receipt_id + )) + })?; + if let Some(mapping) = self.try_lane_agent_session(&receipt.provider, native_session_id)? { + return Ok(mapping); + } + let capture_run = + payload_string_value(payload, &["cwd", "workspaceRoot", "workspace_root"]) + .map(|cwd| self.match_agent_capture_run(cwd, &receipt.provider)) + .transpose()? + .flatten(); + if let Some((lane_id, trail_session_id)) = + self.match_acp_session_for_native_id(&receipt.provider, native_session_id)? + { + let lane = self.lane_name_by_id(&lane_id)?; + return self.ensure_lane_agent_session(LaneAgentSessionInput { + provider: receipt.provider.clone(), + native_session_id: native_session_id.to_string(), + parent_native_session_id: payload_string_value( + payload, + &["parent_session_id", "parentSessionId"], + ) + .map(str::to_string), + lane, + trail_session_id, + capture_run_id: capture_run.as_ref().map(|run| run.capture_run_id.clone()), + primary_transport: AgentCaptureTransport::Hybrid, + transcript_identity: payload_string_value( + payload, + &["transcript_path", "transcriptPath"], + ) + .map(str::to_string), + }); + } + let installation_lane_id = receipt + .installation_id + .as_deref() + .map(|installation_id| { + self.conn.query_row( + "SELECT lane_id FROM agent_hook_installations WHERE installation_id = ?1", + params![installation_id], + |row| row.get::<_, Option>(0), + ) + }) + .transpose()? + .flatten(); + let lane_id = installation_lane_id.or_else(|| { + capture_run + .as_ref() + .and_then(|capture_run| capture_run.lane_id.clone()) + }); + let lane = if let Some(lane_id) = lane_id.as_deref() { + self.lane_name_by_id(lane_id)? + } else { + let lane = format!("agent-{}", receipt.provider); + if self.lane_branch(&lane).is_err() { + self.spawn_lane(&lane, None, false, Some(receipt.provider.clone()), None)?; + } + lane + }; + if let Some(capture_run) = capture_run.as_ref() { + let managed_branch = self.lane_branch(&lane)?; + if capture_run.lane_id.as_deref() == Some(managed_branch.lane_id.as_str()) { + let managed_session = self.lane_session(&capture_run.owner_session_id)?; + if managed_session.lane_id == managed_branch.lane_id { + return self.ensure_lane_agent_session(LaneAgentSessionInput { + provider: receipt.provider.clone(), + native_session_id: native_session_id.to_string(), + parent_native_session_id: payload_string_value( + payload, + &["parent_session_id", "parentSessionId"], + ) + .map(str::to_string), + lane, + trail_session_id: managed_session.session_id, + capture_run_id: Some(capture_run.capture_run_id.clone()), + primary_transport: AgentCaptureTransport::Hybrid, + transcript_identity: payload_string_value( + payload, + &["transcript_path", "transcriptPath"], + ) + .map(str::to_string), + }); + } + } + } + let trail_session_id = format!( + "session_hook_{}", + crate::ids::short_hash( + format!( + "{}:{}:{}", + receipt.workspace_id, receipt.provider, native_session_id + ) + .as_bytes(), + 16, + ) + ); + let session = self + .start_lane_session( + &lane, + Some(format!("{} native session", receipt.provider)), + Some(trail_session_id), + )? + .session; + self.ensure_lane_agent_session(LaneAgentSessionInput { + provider: receipt.provider.clone(), + native_session_id: native_session_id.to_string(), + parent_native_session_id: payload_string_value( + payload, + &["parent_session_id", "parentSessionId"], + ) + .map(str::to_string), + lane, + trail_session_id: session.session_id, + capture_run_id: capture_run.map(|capture_run| capture_run.capture_run_id), + primary_transport: receipt.transport, + transcript_identity: payload_string_value( + payload, + &["transcript_path", "transcriptPath"], + ) + .map(str::to_string), + }) + } + + /// Snapshot the real workspace used by a native provider into its virtual lane. + /// Native hooks run in the user's checkout, not Trail's materialized lane workdir, + /// so `record_lane_workdir_for_turn` cannot observe their writes. The baseline is + /// captured before a turn begins; the terminal checkpoint is attached to that turn. + fn record_native_agent_workspace_checkpoint( + &mut self, + lane: &str, + session_id: &str, + turn_id: Option<&str>, + message: Option, + ) -> Result { + self.reset_case_fold_index_metrics(); + validate_ref_segment(lane)?; + let branch = self.lane_branch(lane)?; + if let Some(turn_id) = turn_id { + let turn = self.lane_turn(turn_id)?; + if turn.lane_id != branch.lane_id || turn.session_id.as_deref() != Some(session_id) { + return Err(Error::InvalidInput(format!( + "turn `{turn_id}` does not belong to native session `{session_id}`" + ))); + } + if turn.ended_at.is_some() { + return Err(Error::InvalidInput(format!( + "turn `{turn_id}` is already ended" + ))); + } + } + + let head = self.get_ref(&branch.ref_name)?; + // Refreshing a native workspace index walks the repository-shaped + // checkout rather than an explicitly bounded sparse selection. + self.note_full_filesystem_path_scan(); + self.refresh_worktree_index_streaming_report()?; + let summaries = self.diff_root_to_worktree_index(&head.root_id)?; + if summaries.is_empty() { + self.set_worktree_index_baseline(&head.root_id)?; + return Ok(LaneRecordReport { + lane_id: branch.lane_id, + operation: None, + root_id: head.root_id, + changed_paths: Vec::new(), + path_index: self.case_fold_index_metrics_report(), + upper_recovery_walks: 0, + generated_dirty_paths: 0, + }); + } + + self.ensure_lane_record_policy(&branch, &summaries)?; + let paths = summaries + .iter() + .map(|summary| summary.path.clone()) + .collect::>(); + let previous_files = self.load_root_files_for_paths(&head.root_id, &paths)?; + let disk_files = self.scan_visible_files_for_paths(&paths)?; + let actor = Actor::lane(lane); + let change_id = self.allocate_change_id(&actor.id, "native_agent_checkpoint")?; + let built = self.build_root_for_selected_record_incremental( + &head.root_id, + &previous_files, + &disk_files, + &paths, + false, + &change_id, + )?; + let diff = self.diff_file_maps(&previous_files, &built.files)?; + if diff.changes.is_empty() { + self.set_worktree_index_baseline(&head.root_id)?; + return Ok(LaneRecordReport { + lane_id: branch.lane_id, + operation: None, + root_id: head.root_id, + changed_paths: Vec::new(), + path_index: self.case_fold_index_metrics_report(), + upper_recovery_walks: 0, + generated_dirty_paths: 0, + }); + } + self.ensure_lane_record_file_size_policy(&built.files, &diff.summaries)?; + + let operation = Operation { + version: OP_OBJECT_VERSION, + change_id: change_id.clone(), + kind: OperationKind::LaneRecord, + parents: vec![head.change_id.clone()], + before_root: Some(head.root_id.clone()), + after_root: built.root_id.clone(), + branch: branch.ref_name.clone(), + actor, + session_id: Some(session_id.to_string()), + message: message.as_deref().map(redact_sensitive_text), + changes: diff.changes, + created_at: now_ts(), + }; + let operation_id = self.store_operation(&operation)?; + self.advance_ref_cas(&head, &change_id, &built.root_id, &operation_id)?; + self.conn.execute( + "UPDATE lane_branches SET head_change = ?1, head_root = ?2, updated_at = ?3 WHERE lane_id = ?4", + params![change_id.0, built.root_id.0, now_ts(), branch.lane_id], + )?; + self.set_worktree_index_baseline(&built.root_id)?; + + let payload = serde_json::json!({ + "workspace": self.workspace_root, + "root_id": built.root_id.0.clone(), + "changed_paths": diff.summaries.iter().map(|item| item.path.clone()).collect::>(), + "checkpoint_kind": if turn_id.is_some() { "turn" } else { "baseline" } + }); + if let Some(turn_id) = turn_id { + self.add_lane_turn_event( + turn_id, + "workspace.checkpoint", + Some(payload), + Some(&change_id.0), + None, + )?; + self.update_lane_turn_progress( + turn_id, + "workspace_checkpoint_recorded", + Some(&change_id), + )?; + } else { + self.add_lane_session_event(lane, session_id, "workspace.baseline", Some(payload))?; + } + Ok(LaneRecordReport { + lane_id: branch.lane_id, + operation: Some(change_id), + root_id: built.root_id, + changed_paths: diff.summaries, + path_index: self.case_fold_index_metrics_report(), + upper_recovery_walks: 0, + generated_dirty_paths: 0, + }) + } + + fn execute_agent_capture_actions( + &mut self, + lane: &str, + mapping: &LaneAgentSession, + event: &AgentLifecycleEvent, + actions: &[AgentCaptureAction], + ) -> Result<()> { + for action in actions { + match action { + AgentCaptureAction::EnsureSession + | AgentCaptureAction::CreateCaptureEpoch + | AgentCaptureAction::WarnDuplicate + | AgentCaptureAction::RecoverInterruptedTurn + | AgentCaptureAction::DeferUntilFinalized => {} + AgentCaptureAction::CaptureBaseline => { + if self.lane_details(lane)?.branch.workdir.is_some() { + self.record_lane_workdir( + lane, + Some(format!("agent {} turn baseline", event.provider)), + )?; + } else { + self.record_native_agent_workspace_checkpoint( + lane, + &mapping.trail_session_id, + None, + Some(format!("agent {} turn baseline", event.provider)), + )?; + } + } + AgentCaptureAction::BeginTurn { synthetic } => { + if self + .open_turn_for_agent_session(&mapping.trail_session_id)? + .is_none() + { + let details = self.lane_details(lane)?; + let envelope = TurnEnvelope::new_agent_turn(TurnEnvelopeInput { + kind: "agent_hook_turn".to_string(), + protocol: "native-hooks".to_string(), + host: payload_string_value( + &event.payload, + &["host", "surface", "client"], + ) + .map(str::to_string), + agent: Some(event.provider.clone()), + adapter: Some(format!("trail/{}@1", event.provider)), + provider: Some(event.provider.clone()), + model: payload_string_value(&event.payload, &["model"]) + .map(str::to_string), + session: TurnEnvelopeSession { + trail_session_id: Some(mapping.trail_session_id.clone()), + upstream_session_id: event.native.session_id.clone(), + ..TurnEnvelopeSession::default() + }, + prompt: TurnEnvelopePrompt { + summary: payload_string_value( + &event.payload, + &["prompt", "message"], + ) + .map(|prompt| prompt.chars().take(160).collect()), + ..TurnEnvelopePrompt::default() + }, + workspace: TurnEnvelopeWorkspace { + lane: Some(lane.to_string()), + cwd: payload_string_value( + &event.payload, + &["cwd", "workspaceRoot", "workspace_root"], + ) + .map(str::to_string), + effective_cwd: details.branch.workdir.clone(), + workdir_mode: Some("native-provider".to_string()), + base_change: Some(details.branch.base_change.clone()), + before_change: Some(details.branch.head_change.clone()), + }, + }); + self.begin_lane_session_turn( + lane, + &mapping.trail_session_id, + Some(envelope.to_metadata_value()), + )?; + if *synthetic { + if let Some(turn_id) = + self.open_turn_for_agent_session(&mapping.trail_session_id)? + { + self.add_lane_turn_event( + &turn_id, + "diagnostic", + Some(serde_json::json!({ + "code": "synthetic_turn_start", + "native_turn_id": event.native.turn_id, + "capture_run_id": event.capture_run_id, + })), + None, + None, + )?; + } + } + } + } + AgentCaptureAction::AppendEvidence => { + let payload = serde_json::json!({ + "schema": event.schema, + "version": event.version, + "event_id": event.event_id, + "native": event.native, + "correlation": event.correlation, + "evidence": event.evidence, + "payload": event.payload, + }); + if let Some(turn_id) = + self.open_turn_for_agent_session(&mapping.trail_session_id)? + { + self.add_lane_turn_event( + &turn_id, + event.event_type.as_str(), + Some(payload), + None, + None, + )?; + self.capture_agent_event_message(&turn_id, event)?; + } else { + self.add_lane_session_event( + lane, + &mapping.trail_session_id, + event.event_type.as_str(), + Some(payload), + )?; + } + } + AgentCaptureAction::StartRootSpan => { + self.ensure_agent_trace_span( + mapping, + event, + "agent", + "Native agent turn", + None, + false, + )?; + } + AgentCaptureAction::StartToolSpan { synthetic } => { + self.ensure_agent_trace_span( + mapping, + event, + "tool", + payload_string_value(&event.payload, &["tool_name", "toolName", "tool"]) + .unwrap_or("Native tool call"), + event.native.tool_id.as_deref(), + *synthetic, + )?; + } + AgentCaptureAction::EndToolSpan => { + self.end_agent_trace_span( + mapping, + event, + "tool", + event.native.tool_id.as_deref(), + )?; + } + AgentCaptureAction::StartSubagentSpan { synthetic } => { + self.ensure_agent_trace_span( + mapping, + event, + "subagent", + "Native subagent", + event.native.subagent_id.as_deref(), + *synthetic, + )?; + } + AgentCaptureAction::EndSubagentSpan => { + self.end_agent_trace_span( + mapping, + event, + "subagent", + event.native.subagent_id.as_deref(), + )?; + } + AgentCaptureAction::StartCompactionSpan => { + self.ensure_agent_trace_span( + mapping, + event, + "compaction", + "Native context compaction", + None, + false, + )?; + } + AgentCaptureAction::EndCompactionSpan => { + self.end_agent_trace_span(mapping, event, "compaction", None)?; + } + AgentCaptureAction::RequestTurnFinalization { outcome } => { + let lease = self.acquire_agent_finalization_lease( + &mapping.mapping_id, + &format!("receipt:{}", event.evidence.receipt_id), + 30_000, + *outcome, + false, + )?; + if !lease.acquired { + return Err(Error::WorkspaceLocked(format!( + "agent session `{}` finalization is owned by `{}` until {}", + mapping.mapping_id, + lease + .mapping + .finalization_owner + .as_deref() + .unwrap_or("unknown"), + lease.expires_at + ))); + } + } + AgentCaptureAction::ReconcileWorkdir => { + if let Some(turn_id) = + self.open_turn_for_agent_session(&mapping.trail_session_id)? + { + if self.lane_details(lane)?.branch.workdir.is_some() { + self.record_lane_workdir_for_turn( + lane, + &turn_id, + Some(format!("agent {} turn", event.provider)), + )?; + } else { + self.record_native_agent_workspace_checkpoint( + lane, + &mapping.trail_session_id, + Some(&turn_id), + Some(format!("agent {} turn checkpoint", event.provider)), + )?; + } + } + } + AgentCaptureAction::ImportTranscript => { + if let Err(error) = self.import_agent_transcript_snapshot(mapping, event) { + self.record_agent_capture_diagnostic( + lane, + mapping, + "TRANSCRIPT_IMPORT_DEFERRED", + &error.to_string(), + )?; + } + } + AgentCaptureAction::CloseTurn { outcome } => { + if let Some(turn_id) = + self.open_turn_for_agent_session(&mapping.trail_session_id)? + { + self.close_open_agent_trace_spans( + &turn_id, + agent_turn_outcome_name(*outcome), + )?; + self.end_lane_turn(&turn_id, agent_turn_outcome_name(*outcome))?; + self.create_turn_evidence_manifest(&turn_id)?; + self.classify_session_activity(&mapping.trail_session_id, 10_000)?; + } + } + AgentCaptureAction::FinalizeAttestation => { + if self + .lane_session_turns(&mapping.trail_session_id)? + .iter() + .any(|turn| turn.ended_at.is_some()) + { + self.create_session_attestation( + &mapping.trail_session_id, + "native-hooks-on-finalization", + Some(serde_json::json!({ + "provider": event.provider, + "receipt_id": event.evidence.receipt_id, + })), + )?; + } + } + AgentCaptureAction::RequestSessionClose => { + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "UPDATE lane_agent_sessions SET session_close_requested = 1, + updated_at = ?2 WHERE mapping_id = ?1", + params![mapping.mapping_id, now_millis()], + )?; + } + AgentCaptureAction::CloseSession { interrupted } => { + let session = self.lane_session(&mapping.trail_session_id)?; + if session.status == "active" { + self.end_lane_session( + &mapping.trail_session_id, + if *interrupted { + "interrupted" + } else { + "completed" + }, + )?; + } + } + } + } + Ok(()) + } + + fn ensure_agent_trace_span( + &mut self, + mapping: &LaneAgentSession, + event: &AgentLifecycleEvent, + span_type: &str, + name: &str, + native_id: Option<&str>, + synthetic: bool, + ) -> Result<()> { + let Some(turn_id) = self.open_turn_for_agent_session(&mapping.trail_session_id)? else { + return Ok(()); + }; + let spans = self.list_lane_trace_spans( + None, + Some(&mapping.trail_session_id), + Some(&turn_id), + None, + 1_000, + )?; + if spans.iter().any(|span| { + span.attributes.as_ref().is_some_and(|attributes| { + attributes + .get("lifecycle_event_id") + .and_then(serde_json::Value::as_str) + == Some(event.event_id.as_str()) + }) + }) { + return Ok(()); + } + let parent_span_id = if span_type == "agent" { + None + } else { + spans + .iter() + .find(|span| span.span_type == "agent" && span.ended_at.is_none()) + .map(|span| span.span_id.as_str()) + }; + self.start_lane_trace_span( + &turn_id, + span_type, + name, + parent_span_id, + None, + Some(serde_json::json!({ + "transport": "native-hooks", + "provider": event.provider, + "mapping_id": mapping.mapping_id, + "lifecycle_event_id": event.event_id, + "native_id": native_id, + "synthetic": synthetic, + })), + )?; + Ok(()) + } + + fn end_agent_trace_span( + &mut self, + mapping: &LaneAgentSession, + event: &AgentLifecycleEvent, + span_type: &str, + native_id: Option<&str>, + ) -> Result<()> { + let Some(turn_id) = self.open_turn_for_agent_session(&mapping.trail_session_id)? else { + return Ok(()); + }; + let spans = self.list_lane_trace_spans( + None, + Some(&mapping.trail_session_id), + Some(&turn_id), + None, + 1_000, + )?; + let span = spans.iter().find(|span| { + span.span_type == span_type + && span.ended_at.is_none() + && (native_id.is_none() + || span.attributes.as_ref().is_some_and(|attributes| { + attributes + .get("native_id") + .and_then(serde_json::Value::as_str) + == native_id + })) + }); + let Some(span) = span else { + return Ok(()); + }; + let status = match event.event_type.kind() { + AgentLifecycleEventKind::ToolFailed | AgentLifecycleEventKind::SubagentFailed => { + "failed" + } + _ => "completed", + }; + self.end_lane_trace_span( + &span.span_id, + status, + Some(serde_json::json!({ + "lifecycle_event_id": event.event_id, + "native_id": native_id, + })), + )?; + Ok(()) + } + + fn close_open_agent_trace_spans(&mut self, turn_id: &str, status: &str) -> Result<()> { + let mut spans = self.list_lane_trace_spans(None, None, Some(turn_id), None, 1_000)?; + spans.retain(|span| span.ended_at.is_none()); + spans.sort_by(|left, right| { + left.parent_span_id + .is_none() + .cmp(&right.parent_span_id.is_none()) + .then_with(|| right.started_at.cmp(&left.started_at)) + }); + for span in spans { + self.end_lane_trace_span( + &span.span_id, + status, + Some(serde_json::json!({"reason":"turn_finalization"})), + )?; + } + Ok(()) + } + + fn import_agent_transcript_snapshot( + &mut self, + mapping: &LaneAgentSession, + event: &AgentLifecycleEvent, + ) -> Result> { + const MAX_TRANSCRIPT_BYTES: u64 = 64 * 1024 * 1024; + + let canonical_export_locator = payload_string_value( + &event.payload, + &[ + "canonical_export_path", + "canonicalExportPath", + "export_path", + "exportPath", + ], + ) + .map(str::to_string); + let transcript_locator = payload_string_value( + &event.payload, + &[ + "transcript_path", + "transcriptPath", + "session_file", + "sessionFile", + ], + ) + .map(str::to_string) + .or_else(|| mapping.transcript_identity.clone()); + let (locator, artifact_kind, evidence_source, trust) = + if let Some(locator) = canonical_export_locator { + ( + locator, + "export", + AgentEvidenceSource::CanonicalExport, + "provider-canonical-export", + ) + } else if let Some(locator) = transcript_locator { + ( + locator, + "transcript", + AgentEvidenceSource::NativeTranscript, + "provider-native", + ) + } else { + return self.record_reconstructed_agent_transcript(mapping, event); + }; + if locator.is_empty() || locator.chars().any(char::is_control) { + return Err(Error::InvalidPath { + path: "agent transcript locator".to_string(), + reason: "locator is empty or contains control characters".to_string(), + }); + } + let raw = PathBuf::from(&locator); + let candidate = if raw.is_absolute() { + raw + } else { + let cwd = + payload_string_value(&event.payload, &["cwd", "workspaceRoot", "workspace_root"]) + .map(PathBuf::from) + .unwrap_or_else(|| self.workspace_root.clone()); + cwd.join(raw) + }; + reject_agent_transcript_symlinks(&candidate)?; + let canonical = candidate + .canonicalize() + .map_err(|error| Error::InvalidPath { + path: "agent transcript locator".to_string(), + reason: format!("cannot resolve provider transcript: {error}"), + })?; + if !self.agent_transcript_path_allowed(&event.provider, &canonical)? { + return Err(Error::InvalidPath { + path: "agent transcript locator".to_string(), + reason: "provider transcript is outside the workspace and approved provider roots" + .to_string(), + }); + } + let metadata = std::fs::metadata(&canonical)?; + if !metadata.is_file() { + return Err(Error::InvalidPath { + path: "agent transcript locator".to_string(), + reason: "provider transcript is not a regular file".to_string(), + }); + } + if metadata.len() > MAX_TRANSCRIPT_BYTES { + return Err(Error::InvalidInput(format!( + "provider transcript is {} bytes; maximum is {MAX_TRANSCRIPT_BYTES}", + metadata.len() + ))); + } + let content = std::fs::read(&canonical)?; + if content.len() as u64 > MAX_TRANSCRIPT_BYTES { + return Err(Error::InvalidInput(format!( + "provider transcript grew beyond {MAX_TRANSCRIPT_BYTES} bytes while reading" + ))); + } + let prior_offset = mapping.transcript_offset.unwrap_or(0); + let previous_offset = prior_offset.min(content.len() as u64); + let end_offset = content.len() as u64; + let content_digest = format!("sha256:{}", sha256_hex(&content)); + let previous_digest: Option = self + .conn + .query_row( + "SELECT content_digest FROM lane_artifacts + WHERE session_id = ?1 AND source = ?2 + ORDER BY created_at DESC, artifact_id DESC LIMIT 1", + params![ + mapping.trail_session_id, + agent_evidence_source_name(evidence_source) + ], + |row| row.get(0), + ) + .optional()?; + let truncated = prior_offset > end_offset; + let rewritten = previous_digest + .as_deref() + .is_some_and(|previous| previous != content_digest); + let turn_id = self + .open_turn_for_agent_session(&mapping.trail_session_id)? + .or(self.agent_lifecycle_event_turn_id(&mapping.trail_session_id, &event.event_id)?); + let lane = self.lane_name_by_id(&mapping.lane_id)?; + let locator_digest = format!( + "sha256:{}", + sha256_hex(canonical.to_string_lossy().as_bytes()) + ); + let artifact = self.record_lane_artifact(LaneArtifactInput { + lane, + session_id: mapping.trail_session_id.clone(), + turn_id, + provider: event.provider.clone(), + artifact_kind: artifact_kind.to_string(), + format: agent_transcript_format(&canonical).to_string(), + source: evidence_source, + source_locator_redacted: canonical + .file_name() + .and_then(|name| name.to_str()) + .map(|name| format!("provider://sessions/{name}")), + content, + start_offset: Some(0), + end_offset: Some(end_offset), + redaction_profile: Some("trail-sensitive-json/v1".to_string()), + trust: trust.to_string(), + supersedes_artifact_id: None, + metadata_json: Some( + serde_json::json!({ + "snapshot": "full", + "artifact_source": agent_evidence_source_name(evidence_source), + "delta_start_offset": previous_offset, + "delta_end_offset": end_offset, + "truncated": truncated, + "rewritten": rewritten, + "locator_digest": locator_digest, + "receipt_id": event.evidence.receipt_id, + }) + .to_string(), + ), + })?; + let _lock = self.acquire_write_lock()?; + let changed = self.conn.execute( + "UPDATE lane_agent_sessions + SET transcript_identity = ?2, transcript_offset = ?3, updated_at = ?4 + WHERE mapping_id = ?1 + AND transcript_offset IS ?5", + params![ + mapping.mapping_id, + canonical.to_string_lossy(), + i64::try_from(end_offset).unwrap_or(i64::MAX), + now_millis(), + mapping + .transcript_offset + .and_then(|offset| i64::try_from(offset).ok()), + ], + )?; + if changed != 1 { + return Err(Error::Conflict(format!( + "agent transcript offset advanced concurrently for `{}`", + mapping.mapping_id + ))); + } + Ok(Some(artifact)) + } + + fn record_reconstructed_agent_transcript( + &mut self, + mapping: &LaneAgentSession, + event: &AgentLifecycleEvent, + ) -> Result> { + let turn_id = self + .open_turn_for_agent_session(&mapping.trail_session_id)? + .or(self.agent_lifecycle_event_turn_id(&mapping.trail_session_id, &event.event_id)?); + let Some(turn_id) = turn_id else { + return Ok(None); + }; + let details = self.show_lane_turn(&turn_id)?; + let content = serde_json::to_vec(&serde_json::json!({ + "schema": "trail.reconstructed_transcript", + "version": 1, + "provider": event.provider, + "native_session_id": event.native.session_id, + "native_turn_id": event.native.turn_id, + "messages": details.messages, + "events": details.events, + }))?; + let lane = self.lane_name_by_id(&mapping.lane_id)?; + self.record_lane_artifact(LaneArtifactInput { + lane, + session_id: mapping.trail_session_id.clone(), + turn_id: Some(turn_id), + provider: event.provider.clone(), + artifact_kind: "transcript".to_string(), + format: "application/json".to_string(), + source: AgentEvidenceSource::Reconstructed, + source_locator_redacted: None, + content, + start_offset: None, + end_offset: None, + redaction_profile: Some("trail-sensitive-json/v1".to_string()), + trust: "reconstructed-from-receipts".to_string(), + supersedes_artifact_id: None, + metadata_json: Some( + serde_json::json!({ + "receipt_id": event.evidence.receipt_id, + "fidelity": "reconstructed", + }) + .to_string(), + ), + }) + .map(Some) + } + + fn agent_transcript_path_allowed(&self, provider: &str, path: &Path) -> Result { + if path.starts_with(&self.workspace_root) { + return Ok(true); + } + let Some(home) = std::env::var_os("HOME").map(PathBuf::from) else { + return Ok(false); + }; + let roots: &[&str] = match provider { + "codex" => &[".codex/sessions"], + "claude-code" => &[".claude/projects"], + "pi" => &[".pi/agent/sessions"], + "opencode" => &[".local/share/opencode", ".opencode"], + "cursor" => &[".cursor/projects", ".cursor/sessions"], + "gemini" => &[".gemini/tmp", ".gemini/sessions"], + "copilot" => &[".copilot/session-state"], + "grok" => &[".grok/sessions"], + _ => &[], + }; + for root in roots { + let candidate = home.join(root); + if candidate.exists() && path.starts_with(candidate.canonicalize()?) { + return Ok(true); + } + } + Ok(false) + } + + fn record_agent_capture_diagnostic( + &mut self, + lane: &str, + mapping: &LaneAgentSession, + code: &str, + message: &str, + ) -> Result<()> { + let payload = Some(serde_json::json!({ + "code": code, + "message": redact_agent_capture_diagnostic(message), + "mapping_id": mapping.mapping_id, + })); + if let Some(turn_id) = self.open_turn_for_agent_session(&mapping.trail_session_id)? { + self.add_lane_turn_event(&turn_id, "diagnostic", payload, None, None)?; + } else { + self.add_lane_session_event(lane, &mapping.trail_session_id, "diagnostic", payload)?; + } + Ok(()) + } + + fn capture_agent_event_message( + &mut self, + turn_id: &str, + event: &AgentLifecycleEvent, + ) -> Result<()> { + let (role, keys): (&str, &[&str]) = match event.event_type.kind() { + AgentLifecycleEventKind::MessageUser => ("user", &["prompt", "message", "text"]), + AgentLifecycleEventKind::MessageAssistantCompleted => ( + "assistant", + &[ + "last_assistant_message", + "lastAssistantMessage", + "assistant_response", + "assistantResponse", + "message", + "text", + ], + ), + _ => return Ok(()), + }; + if let Some(text) = payload_string_value(&event.payload, keys) { + if !text.is_empty() { + let duplicate = self + .show_lane_turn(turn_id)? + .messages + .iter() + .any(|message| message.role == role && message.body == text); + if !duplicate { + self.add_lane_turn_message(turn_id, role, text)?; + } + } + } + Ok(()) + } + + fn match_acp_session_for_native_id( + &self, + provider: &str, + native_session_id: &str, + ) -> Result> { + let mut statement = self.conn.prepare( + "SELECT lane_id, trail_session_id FROM lane_acp_sessions + WHERE (acp_session_id = ?1 OR upstream_session_id = ?1) + AND (provider IS NULL OR provider = ?2) + ORDER BY updated_at DESC LIMIT 2", + )?; + let rows = statement + .query_map(params![native_session_id, provider], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()?; + if rows.len() > 1 && rows[0] != rows[1] { + return Err(Error::Conflict(format!( + "native session `{native_session_id}` ambiguously matches multiple ACP sessions" + ))); + } + Ok(rows.into_iter().next()) + } + + fn hybrid_acp_owns_session(&self, trail_session_id: &str) -> Result { + self.conn + .query_row( + "SELECT EXISTS( + SELECT 1 FROM lane_acp_sessions + WHERE trail_session_id = ?1 + AND status IN ('starting', 'active', 'loaded', 'resumed') + )", + params![trail_session_id], + |row| row.get(0), + ) + .map_err(Error::from) + } + + fn update_agent_mapping_phase( + &mut self, + mapping_id: &str, + expected_phase: AgentCapturePhase, + expected_finalization_owner: Option<&str>, + phase: AgentCapturePhase, + pending_outcome: Option, + session_close_requested: bool, + ) -> Result<()> { + let _lock = self.acquire_write_lock()?; + let now = now_millis(); + let changed = self.conn.execute( + "UPDATE lane_agent_sessions + SET status = ?2, pending_turn_outcome = ?3, + session_close_requested = ?4, + finalization_owner = CASE WHEN ?2 = 'finalizing' THEN finalization_owner ELSE NULL END, + finalization_lease_expires_at = CASE WHEN ?2 = 'finalizing' THEN finalization_lease_expires_at ELSE NULL END, + updated_at = ?5 + WHERE mapping_id = ?1 AND status = ?6 + AND (?6 != 'finalizing' OR + (finalization_owner = ?7 AND finalization_lease_expires_at > ?5))", + params![ + mapping_id, + agent_capture_phase_name(phase), + pending_outcome.map(agent_turn_outcome_name), + if session_close_requested { 1 } else { 0 }, + now, + agent_capture_phase_name(expected_phase), + expected_finalization_owner, + ], + )?; + if changed != 1 { + return Err(Error::WorkspaceLocked(format!( + "agent session `{mapping_id}` changed phase or lost its finalization lease" + ))); + } + Ok(()) + } + + fn lane_name_by_id(&self, lane_id: &str) -> Result { + self.conn + .query_row( + "SELECT name FROM lanes WHERE lane_id = ?1", + params![lane_id], + |row| row.get(0), + ) + .optional()? + .ok_or_else(|| Error::InvalidInput(format!("lane id `{lane_id}` not found"))) + } + + fn open_turn_for_agent_session(&self, session_id: &str) -> Result> { + self.conn + .query_row( + "SELECT turn_id FROM lane_turns + WHERE session_id = ?1 AND ended_at IS NULL + ORDER BY started_at DESC, rowid DESC LIMIT 1", + params![session_id], + |row| row.get(0), + ) + .optional() + .map_err(Error::from) + } + + fn agent_lifecycle_event_already_recorded( + &self, + session_id: &str, + event_id: &str, + ) -> Result { + self.conn + .query_row( + "SELECT EXISTS( + SELECT 1 FROM lane_events + WHERE session_id = ?1 AND json_extract(payload_json, '$.event_id') = ?2 + )", + params![session_id, event_id], + |row| row.get::<_, bool>(0), + ) + .map_err(Error::from) + } + + fn agent_lifecycle_event_turn_id( + &self, + session_id: &str, + event_id: &str, + ) -> Result> { + self.conn + .query_row( + "SELECT turn_id FROM lane_events + WHERE session_id = ?1 AND json_extract(payload_json, '$.event_id') = ?2 + AND turn_id IS NOT NULL + ORDER BY created_at DESC, rowid DESC LIMIT 1", + params![session_id, event_id], + |row| row.get(0), + ) + .optional() + .map_err(Error::from) + } + + /// Attach a durable receipt to a resolved native session and allocate its logical order. + pub fn assign_agent_hook_receipt_mapping( + &mut self, + receipt_id: &str, + mapping_id: &str, + ) -> Result { + let _lock = self.acquire_write_lock()?; + validate_agent_capture_id("receipt id", receipt_id, 256)?; + validate_agent_capture_id("agent session mapping id", mapping_id, 256)?; + let receipt = self.agent_hook_receipt(receipt_id)?; + if let Some(existing_mapping) = receipt.mapping_id.as_deref() { + if existing_mapping == mapping_id { + return Ok(receipt); + } + return Err(Error::InvalidInput(format!( + "agent hook receipt `{receipt_id}` already belongs to mapping `{existing_mapping}`" + ))); + } + + let mapping: Option<(String, String, i64)> = self + .conn + .query_row( + "SELECT workspace_id, provider, next_receive_sequence + FROM lane_agent_sessions WHERE mapping_id = ?1", + params![mapping_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional()?; + let Some((workspace_id, provider, sequence)) = mapping else { + return Err(Error::InvalidInput(format!( + "agent session mapping `{mapping_id}` not found" + ))); + }; + if workspace_id != receipt.workspace_id || provider != receipt.provider { + return Err(Error::InvalidInput(format!( + "agent hook receipt `{receipt_id}` and mapping `{mapping_id}` have different workspace or provider identities" + ))); + } + + self.conn + .execute_batch("SAVEPOINT assign_agent_receipt_mapping")?; + let update = (|| -> Result<()> { + let mapping_updated = self.conn.execute( + "UPDATE lane_agent_sessions + SET next_receive_sequence = next_receive_sequence + 1, updated_at = ?1 + WHERE mapping_id = ?2 AND next_receive_sequence = ?3", + params![now_millis(), mapping_id, sequence], + )?; + if mapping_updated != 1 { + return Err(Error::WorkspaceLocked(format!( + "agent session mapping `{mapping_id}` receive sequence changed concurrently" + ))); + } + self.conn.execute( + "UPDATE agent_hook_receipts + SET mapping_id = ?1, receive_sequence = ?2, updated_at = ?3 + WHERE receipt_id = ?4 AND mapping_id IS NULL", + params![mapping_id, sequence, now_millis(), receipt_id], + )?; + Ok(()) + })(); + match update { + Ok(()) => self + .conn + .execute_batch("RELEASE SAVEPOINT assign_agent_receipt_mapping")?, + Err(error) => { + let _ = self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT assign_agent_receipt_mapping; + RELEASE SAVEPOINT assign_agent_receipt_mapping", + ); + return Err(error); + } + } + self.agent_hook_receipt(receipt_id) + } + + fn try_agent_hook_receipt_by_dedupe_key( + &self, + workspace_id: &str, + provider: &str, + dedupe_key: &str, + ) -> Result> { + self.conn + .query_row( + "SELECT receipt_id, workspace_id, installation_id, mapping_id, provider, + native_event, native_session_id, native_turn_id, transport, dedupe_key, + payload_digest, raw_object_id, raw_artifact_id, receive_sequence, + connection_id, direction, connection_sequence, status, + attempt_count, next_attempt_at, diagnostic, occurred_at, received_at, + processed_at, updated_at + FROM agent_hook_receipts + WHERE workspace_id = ?1 AND provider = ?2 AND dedupe_key = ?3", + params![workspace_id, provider, dedupe_key], + agent_hook_receipt_row, + ) + .optional() + .map_err(Error::from) + } +} + +const AGENT_HOOK_INSTALLATION_BY_ID_SELECT: &str = + "SELECT installation_id, workspace_id, provider, scope, config_path, + lane_id, manifest_digest, ownership_inventory_json, + config_before_digest, config_after_digest, adapter_version, + provider_version_range, detected_provider_version, capability_status, + status, installed_at, verified_at, last_receipt_at + FROM agent_hook_installations + WHERE installation_id = ?1 AND workspace_id = ?2"; + +const LANE_ARTIFACT_BY_ID_SELECT: &str = + "SELECT artifact_id, workspace_id, lane_id, session_id, turn_id, provider, + artifact_kind, format, source, source_locator_redacted, content_object_id, + content_digest, size_bytes, start_offset, end_offset, redaction_profile, + retention_status, trust, supersedes_artifact_id, created_at, metadata_json + FROM lane_artifacts WHERE artifact_id = ?1"; + +fn map_lane_artifact(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let source: String = row.get(8)?; + let source = parse_agent_evidence_source(&source).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure(8, rusqlite::types::Type::Text, Box::new(error)) + })?; + Ok(LaneArtifact { + artifact_id: row.get(0)?, + workspace_id: row.get(1)?, + lane_id: row.get(2)?, + session_id: row.get(3)?, + turn_id: row.get(4)?, + provider: row.get(5)?, + artifact_kind: row.get(6)?, + format: row.get(7)?, + source, + source_locator_redacted: row.get(9)?, + content_object_id: row.get::<_, Option>(10)?.map(ObjectId), + content_digest: row.get(11)?, + size_bytes: u64::try_from(row.get::<_, i64>(12)?).unwrap_or(0), + start_offset: row + .get::<_, Option>(13)? + .and_then(|value| u64::try_from(value).ok()), + end_offset: row + .get::<_, Option>(14)? + .and_then(|value| u64::try_from(value).ok()), + redaction_profile: row.get(15)?, + retention_status: row.get(16)?, + trust: row.get(17)?, + supersedes_artifact_id: row.get(18)?, + created_at: row.get(19)?, + metadata_json: row.get(20)?, + }) +} + +const AGENT_HOOK_INSTALLATION_LIST_SELECT: &str = + "SELECT installation_id, workspace_id, provider, scope, config_path, + lane_id, manifest_digest, ownership_inventory_json, + config_before_digest, config_after_digest, adapter_version, + provider_version_range, detected_provider_version, capability_status, + status, installed_at, verified_at, last_receipt_at + FROM agent_hook_installations + WHERE workspace_id = ?1 + ORDER BY provider, scope, config_path"; + +const AGENT_HOOK_INSTALLATION_BY_PROVIDER_SELECT: &str = + "SELECT installation_id, workspace_id, provider, scope, config_path, + lane_id, manifest_digest, ownership_inventory_json, + config_before_digest, config_after_digest, adapter_version, + provider_version_range, detected_provider_version, capability_status, + status, installed_at, verified_at, last_receipt_at + FROM agent_hook_installations + WHERE workspace_id = ?1 AND provider = ?2 + ORDER BY scope, config_path"; + +fn map_agent_hook_installation( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + let scope: String = row.get(3)?; + let scope = match scope.as_str() { + "project" => AgentHookInstallScope::Project, + "user" => AgentHookInstallScope::User, + other => { + return Err(rusqlite::Error::FromSqlConversionFailure( + 3, + rusqlite::types::Type::Text, + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("unknown agent hook installation scope `{other}`"), + ) + .into(), + )); + } + }; + let inventory: String = row.get(7)?; + let ownership_inventory = serde_json::from_str(&inventory).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure(7, rusqlite::types::Type::Text, Box::new(error)) + })?; + Ok(AgentHookInstallationRecord { + installation_id: row.get(0)?, + workspace_id: row.get(1)?, + provider: row.get(2)?, + scope, + config_path: PathBuf::from(row.get::<_, String>(4)?), + lane_id: row.get(5)?, + manifest_digest: row.get(6)?, + ownership_inventory, + config_before_digest: row.get(8)?, + config_after_digest: row.get(9)?, + adapter_version: row.get(10)?, + provider_version_range: row.get(11)?, + detected_provider_version: row.get(12)?, + capability_status: row.get(13)?, + status: row.get(14)?, + installed_at: row.get(15)?, + verified_at: row.get(16)?, + last_receipt_at: row.get(17)?, + }) +} + +const AGENT_CAPTURE_RUN_SELECT_BY_ID: &str = + "SELECT capture_run_id, workspace_id, lane_id, workdir, canonical_workdir, + owner_agent, owner_session_id, executor_agent, work_item_id, status, + created_at, updated_at, expires_at, ended_at, metadata_json + FROM agent_capture_runs WHERE capture_run_id = ?1"; + +fn agent_capture_run_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(AgentCaptureRun { + capture_run_id: row.get(0)?, + workspace_id: row.get(1)?, + lane_id: row.get(2)?, + workdir: row.get(3)?, + canonical_workdir: row.get(4)?, + owner_agent: row.get(5)?, + owner_session_id: row.get(6)?, + executor_agent: row.get(7)?, + work_item_id: row.get(8)?, + status: row.get(9)?, + created_at: row.get(10)?, + updated_at: row.get(11)?, + expires_at: row.get(12)?, + ended_at: row.get(13)?, + metadata_json: row.get(14)?, + }) +} + +const LANE_AGENT_SESSION_SELECT_BY_ID: &str = + "SELECT mapping_id, workspace_id, provider, native_session_id, + parent_native_session_id, trail_session_id, lane_id, capture_run_id, + primary_transport, transcript_identity, transcript_offset, resume_json, + last_attestation_id, status, pending_turn_outcome, + session_close_requested, capture_epoch, finalization_owner, + finalization_lease_expires_at, next_receive_sequence, created_at, updated_at + FROM lane_agent_sessions WHERE mapping_id = ?1"; + +fn lane_agent_session_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let primary_transport = parse_agent_capture_transport(row.get::<_, String>(8)?.as_str()) + .map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 8, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?; + let status = + parse_agent_capture_phase(row.get::<_, String>(13)?.as_str()).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 13, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?; + let pending_turn_outcome = row + .get::<_, Option>(14)? + .map(|value| parse_agent_turn_outcome(&value)) + .transpose() + .map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 14, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?; + Ok(LaneAgentSession { + mapping_id: row.get(0)?, + workspace_id: row.get(1)?, + provider: row.get(2)?, + native_session_id: row.get(3)?, + parent_native_session_id: row.get(4)?, + trail_session_id: row.get(5)?, + lane_id: row.get(6)?, + capture_run_id: row.get(7)?, + primary_transport, + transcript_identity: row.get(9)?, + transcript_offset: row + .get::<_, Option>(10)? + .map(u64::try_from) + .transpose() + .map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 10, + rusqlite::types::Type::Integer, + Box::new(error), + ) + })?, + resume_json: row.get(11)?, + last_attestation_id: row.get(12)?, + status, + pending_turn_outcome, + session_close_requested: row.get(15)?, + capture_epoch: u64::try_from(row.get::<_, i64>(16)?).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 16, + rusqlite::types::Type::Integer, + Box::new(error), + ) + })?, + finalization_owner: row.get(17)?, + finalization_lease_expires_at: row.get(18)?, + next_receive_sequence: u64::try_from(row.get::<_, i64>(19)?).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 19, + rusqlite::types::Type::Integer, + Box::new(error), + ) + })?, + created_at: row.get(20)?, + updated_at: row.get(21)?, + }) +} + +fn agent_hook_receipt_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let transport = + parse_agent_capture_transport(row.get::<_, String>(8)?.as_str()).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 8, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?; + let receive_sequence = row + .get::<_, Option>(13)? + .map(u64::try_from) + .transpose() + .map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 13, + rusqlite::types::Type::Integer, + Box::new(error), + ) + })?; + let connection_sequence = row + .get::<_, Option>(16)? + .map(u64::try_from) + .transpose() + .map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 16, + rusqlite::types::Type::Integer, + Box::new(error), + ) + })?; + let attempt_count = u32::try_from(row.get::<_, i64>(18)?).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 18, + rusqlite::types::Type::Integer, + Box::new(error), + ) + })?; + Ok(AgentHookReceipt { + receipt_id: row.get(0)?, + workspace_id: row.get(1)?, + installation_id: row.get(2)?, + mapping_id: row.get(3)?, + provider: row.get(4)?, + native_event: row.get(5)?, + native_session_id: row.get(6)?, + native_turn_id: row.get(7)?, + transport, + dedupe_key: row.get(9)?, + payload_digest: row.get(10)?, + raw_object_id: ObjectId(row.get(11)?), + raw_artifact_id: row.get(12)?, + receive_sequence, + connection_id: row.get(14)?, + direction: row.get(15)?, + connection_sequence, + status: row.get(17)?, + attempt_count, + next_attempt_at: row.get(19)?, + diagnostic: row.get(20)?, + occurred_at: row.get(21)?, + received_at: row.get(22)?, + processed_at: row.get(23)?, + updated_at: row.get(24)?, + }) +} + +fn validate_agent_receipt_input(input: &AgentHookReceiptInput) -> Result<()> { + validate_agent_provider(&input.provider)?; + validate_agent_capture_id("native event", &input.native_event, 256)?; + validate_agent_capture_id("dedupe key", &input.dedupe_key, 1024)?; + if let Some(value) = input.installation_id.as_deref() { + validate_agent_capture_id("installation id", value, 256)?; + } + if let Some(value) = input.native_session_id.as_deref() { + validate_agent_capture_id("native session id", value, 1024)?; + } + if let Some(value) = input.native_turn_id.as_deref() { + validate_agent_capture_id("native turn id", value, 1024)?; + } + match ( + input.connection_id.as_deref(), + input.direction.as_deref(), + input.connection_sequence, + ) { + (None, None, None) => {} + (Some(connection_id), Some(direction), Some(sequence)) => { + validate_agent_capture_id("connection id", connection_id, 256)?; + if !matches!(direction, "client_to_agent" | "agent_to_client") { + return Err(Error::InvalidInput(format!( + "ACP receipt direction must be client_to_agent or agent_to_client, got `{direction}`" + ))); + } + i64::try_from(sequence).map_err(|_| { + Error::InvalidInput( + "ACP connection sequence exceeds SQLite integer range".to_string(), + ) + })?; + } + _ => { + return Err(Error::InvalidInput( + "ACP receipt connection identity requires id, direction, and sequence together" + .to_string(), + )); + } + } + if input.occurred_at.is_some_and(|value| value < 0) { + return Err(Error::InvalidInput( + "agent hook occurred_at must be non-negative milliseconds".to_string(), + )); + } + Ok(()) +} + +fn validate_agent_provider(value: &str) -> Result<()> { + validate_agent_capture_id("provider", value, 64)?; + if !value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + { + return Err(Error::InvalidInput( + "agent provider must use lowercase ASCII letters, digits, and hyphens".to_string(), + )); + } + Ok(()) +} + +fn validate_agent_capture_id(label: &str, value: &str, maximum: usize) -> Result<()> { + if value.is_empty() || value.len() > maximum || value.chars().any(char::is_control) { + return Err(Error::InvalidInput(format!( + "agent {label} must contain 1 to {maximum} non-control bytes" + ))); + } + Ok(()) +} + +fn validate_agent_receipt_status(status: &str) -> Result<()> { + match status { + "received" | "processing" | "processed" | "retry" | "quarantined" | "discarded" => Ok(()), + _ => Err(Error::InvalidInput(format!( + "unknown agent hook receipt status `{status}`" + ))), + } +} + +fn validate_agent_capture_lease_ms(lease_ms: u64) -> Result<()> { + if !(1_000..=86_400_000).contains(&lease_ms) { + return Err(Error::InvalidInput( + "agent capture run lease must be between 1000 and 86400000 milliseconds".to_string(), + )); + } + Ok(()) +} + +fn agent_capture_transport_name(transport: AgentCaptureTransport) -> &'static str { + match transport { + AgentCaptureTransport::Acp => "acp", + AgentCaptureTransport::NativeHooks => "native-hooks", + AgentCaptureTransport::Terminal => "terminal", + AgentCaptureTransport::Hybrid => "hybrid", + } +} + +fn agent_evidence_source_name(source: AgentEvidenceSource) -> &'static str { + match source { + AgentEvidenceSource::Acp => "acp", + AgentEvidenceSource::NativeHook => "native_hook", + AgentEvidenceSource::NativeTranscript => "native_transcript", + AgentEvidenceSource::CanonicalExport => "canonical_export", + AgentEvidenceSource::WorkdirObserved => "workdir_observed", + AgentEvidenceSource::Reconstructed => "reconstructed", + } +} + +fn parse_agent_evidence_source(value: &str) -> Result { + match value { + "acp" => Ok(AgentEvidenceSource::Acp), + "native_hook" => Ok(AgentEvidenceSource::NativeHook), + "native_transcript" => Ok(AgentEvidenceSource::NativeTranscript), + "canonical_export" => Ok(AgentEvidenceSource::CanonicalExport), + "workdir_observed" => Ok(AgentEvidenceSource::WorkdirObserved), + "reconstructed" => Ok(AgentEvidenceSource::Reconstructed), + _ => Err(Error::Corrupt(format!( + "unknown agent evidence source `{value}`" + ))), + } +} + +fn reject_agent_transcript_symlinks(path: &Path) -> Result<()> { + if std::fs::symlink_metadata(path) + .map(|metadata| metadata.file_type().is_symlink()) + .unwrap_or(false) + { + return Err(Error::InvalidPath { + path: "agent transcript locator".to_string(), + reason: "provider transcript locator is a symbolic link".to_string(), + }); + } + Ok(()) +} + +fn agent_transcript_format(path: &Path) -> &'static str { + match path + .extension() + .and_then(|extension| extension.to_str()) + .map(str::to_ascii_lowercase) + .as_deref() + { + Some("jsonl" | "ndjson") => "application/x-ndjson", + Some("json") => "application/json", + Some("md" | "markdown") => "text/markdown", + Some("txt") => "text/plain", + _ => "application/octet-stream", + } +} + +fn parse_agent_capture_transport(value: &str) -> std::result::Result { + match value { + "acp" => Ok(AgentCaptureTransport::Acp), + "native-hooks" => Ok(AgentCaptureTransport::NativeHooks), + "terminal" => Ok(AgentCaptureTransport::Terminal), + "hybrid" => Ok(AgentCaptureTransport::Hybrid), + _ => Err(Error::Corrupt(format!( + "unknown agent capture transport `{value}`" + ))), + } +} + +fn payload_string_value<'a>(payload: &'a serde_json::Value, keys: &[&str]) -> Option<&'a str> { + keys.iter().find_map(|key| { + payload + .get(*key) + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) + }) +} + +fn redact_agent_capture_diagnostic(value: &str) -> String { + redact_sensitive_text(value).chars().take(2_048).collect() +} + +fn agent_capture_phase_name(value: AgentCapturePhase) -> &'static str { + match value { + AgentCapturePhase::Idle => "idle", + AgentCapturePhase::Active => "active", + AgentCapturePhase::Finalizing => "finalizing", + AgentCapturePhase::Ended => "ended", + AgentCapturePhase::Interrupted => "interrupted", + } +} + +fn parse_agent_capture_phase(value: &str) -> std::result::Result { + match value { + "idle" => Ok(AgentCapturePhase::Idle), + "active" => Ok(AgentCapturePhase::Active), + "finalizing" => Ok(AgentCapturePhase::Finalizing), + "ended" => Ok(AgentCapturePhase::Ended), + "interrupted" => Ok(AgentCapturePhase::Interrupted), + _ => Err(Error::Corrupt(format!( + "unknown agent capture phase `{value}`" + ))), + } +} + +fn agent_turn_outcome_name(value: AgentTurnOutcome) -> &'static str { + match value { + AgentTurnOutcome::Completed => "completed", + AgentTurnOutcome::Failed => "failed", + AgentTurnOutcome::Cancelled => "cancelled", + AgentTurnOutcome::Interrupted => "interrupted", + } +} + +fn parse_agent_turn_outcome(value: &str) -> std::result::Result { + match value { + "completed" => Ok(AgentTurnOutcome::Completed), + "failed" => Ok(AgentTurnOutcome::Failed), + "cancelled" => Ok(AgentTurnOutcome::Cancelled), + "interrupted" => Ok(AgentTurnOutcome::Interrupted), + _ => Err(Error::Corrupt(format!( + "unknown agent turn outcome `{value}`" + ))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn receipt_ingress_is_redacted_durable_and_idempotent() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let input = AgentHookReceiptInput { + installation_id: None, + provider: "codex".to_string(), + native_event: "AfterTool".to_string(), + native_session_id: Some("native-1".to_string()), + native_turn_id: Some("turn-1".to_string()), + transport: AgentCaptureTransport::NativeHooks, + connection_id: None, + direction: None, + connection_sequence: None, + dedupe_key: "native-1:AfterTool:1".to_string(), + payload: serde_json::json!({ + "tool": "shell", + "api_key": "must-not-survive", + "output": "authorization: Bearer must-not-survive" + }), + occurred_at: Some(1), + }; + let first = db.persist_agent_hook_receipt(input.clone()).unwrap(); + assert!(!first.duplicate); + let second = db.persist_agent_hook_receipt(input).unwrap(); + assert!(second.duplicate); + assert_eq!(first.receipt.receipt_id, second.receipt.receipt_id); + + let stored: AgentHookReceiptObject = db + .get_object(AGENT_HOOK_RECEIPT_OBJECT_KIND, &first.receipt.raw_object_id) + .unwrap(); + let rendered = serde_json::to_string(&stored.payload).unwrap(); + assert!(!rendered.contains("must-not-survive")); + assert!(rendered.contains("REDACTED")); + + drop(db); + let reopened = Trail::open(temp.path()).unwrap(); + assert_eq!( + reopened + .agent_hook_receipt(&first.receipt.receipt_id) + .unwrap(), + first.receipt + ); + } + + #[test] + fn dedupe_key_collision_with_different_payload_fails_closed() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let mut input = AgentHookReceiptInput { + installation_id: None, + provider: "claude-code".to_string(), + native_event: "Stop".to_string(), + native_session_id: Some("native-1".to_string()), + native_turn_id: None, + transport: AgentCaptureTransport::NativeHooks, + connection_id: None, + direction: None, + connection_sequence: None, + dedupe_key: "native-1:Stop:1".to_string(), + payload: serde_json::json!({"result": 1}), + occurred_at: None, + }; + db.persist_agent_hook_receipt(input.clone()).unwrap(); + input.payload = serde_json::json!({"result": 2}); + assert!(db.persist_agent_hook_receipt(input).is_err()); + } + + #[test] + fn receipt_ingress_rejects_payloads_over_the_hard_limit() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let result = db.persist_agent_hook_receipt(AgentHookReceiptInput { + installation_id: None, + provider: "codex".to_string(), + native_event: "AfterTool".to_string(), + native_session_id: Some("oversized-session".to_string()), + native_turn_id: Some("oversized-turn".to_string()), + transport: AgentCaptureTransport::NativeHooks, + connection_id: None, + direction: None, + connection_sequence: None, + dedupe_key: "oversized:one".to_string(), + payload: serde_json::json!({ + "output": "x".repeat(AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES + 1) + }), + occurred_at: None, + }); + assert!(result.is_err()); + assert!(db + .list_agent_hook_receipts(Some("codex"), None, 10) + .unwrap() + .is_empty()); + } + + #[test] + fn durable_receipts_replay_into_one_native_session_turn_and_message() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + let transcript_path = temp.path().join("claude-session.jsonl"); + let transcript = b"{\"type\":\"user\",\"message\":\"make the change\"}\n{\"type\":\"assistant\",\"message\":\"done\"}\n"; + std::fs::write(&transcript_path, transcript).unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let persist = |db: &mut Trail, event: &str, dedupe: &str, payload: serde_json::Value| { + db.persist_agent_hook_receipt(AgentHookReceiptInput { + installation_id: None, + provider: "claude-code".to_string(), + native_event: event.to_string(), + native_session_id: Some("native-replay-1".to_string()), + native_turn_id: payload + .get("turn_id") + .and_then(serde_json::Value::as_str) + .map(str::to_string), + transport: AgentCaptureTransport::NativeHooks, + connection_id: None, + direction: None, + connection_sequence: None, + dedupe_key: dedupe.to_string(), + payload, + occurred_at: None, + }) + .unwrap() + .receipt + .receipt_id + }; + let start = persist( + &mut db, + "SessionStart", + "replay:start", + serde_json::json!({ + "session_id":"native-replay-1", + "source":"startup", + "transcript_path": transcript_path, + }), + ); + let prompt = persist( + &mut db, + "UserPromptSubmit", + "replay:prompt", + serde_json::json!({ + "session_id":"native-replay-1", + "turn_id":"native-turn-1", + "prompt":"make the change" + }), + ); + let tool_start = persist( + &mut db, + "PreToolUse", + "replay:tool:start", + serde_json::json!({ + "session_id":"native-replay-1", + "turn_id":"native-turn-1", + "tool_use_id":"tool-1", + "tool_name":"shell" + }), + ); + let tool_end = persist( + &mut db, + "PostToolUse", + "replay:tool:end", + serde_json::json!({ + "session_id":"native-replay-1", + "turn_id":"native-turn-1", + "tool_use_id":"tool-1", + "result":"ok" + }), + ); + let stop = persist( + &mut db, + "Stop", + "replay:stop", + serde_json::json!({ + "session_id":"native-replay-1", + "turn_id":"native-turn-1", + "last_assistant_message":"done" + }), + ); + let late_subagent_stop = persist( + &mut db, + "SubagentStop", + "replay:subagent:late-stop", + serde_json::json!({ + "session_id":"native-replay-1", + "turn_id":"native-turn-1", + "agent_id":"subagent-1", + "status":"completed" + }), + ); + db.replay_agent_hook_receipt(&start).unwrap(); + let prompt_report = db.replay_agent_hook_receipt(&prompt).unwrap(); + assert_eq!(prompt_report.normalized_events.len(), 2); + db.replay_agent_hook_receipt(&tool_start).unwrap(); + db.replay_agent_hook_receipt(&tool_end).unwrap(); + let stop_report = db.replay_agent_hook_receipt(&stop).unwrap(); + assert_eq!(stop_report.receipt.status, "processed"); + let mapping = stop_report.mapping.unwrap(); + assert_eq!(mapping.status, AgentCapturePhase::Idle); + let late_report = db.replay_agent_hook_receipt(&late_subagent_stop).unwrap(); + assert_eq!(late_report.mapping.unwrap().status, AgentCapturePhase::Idle); + let turns = db.lane_session_turns(&mapping.trail_session_id).unwrap(); + assert_eq!(turns.len(), 1); + assert_eq!(turns[0].status, "completed"); + let details = db.show_lane_turn(&turns[0].turn_id).unwrap(); + assert_eq!(details.messages.len(), 1); + assert_eq!(details.messages[0].body, "make the change"); + let spans = db + .list_lane_trace_spans( + None, + Some(&mapping.trail_session_id), + Some(&turns[0].turn_id), + None, + 10, + ) + .unwrap(); + assert_eq!(spans.len(), 2); + assert!(spans.iter().all(|span| span.ended_at.is_some())); + assert!(spans.iter().any(|span| span.span_type == "agent")); + assert!(spans.iter().any(|span| span.span_type == "tool")); + let artifacts = db + .list_lane_artifacts(&mapping.trail_session_id, Some(&turns[0].turn_id), 10) + .unwrap(); + assert_eq!(artifacts.len(), 1); + assert_eq!(artifacts[0].artifact_kind, "transcript"); + assert_eq!(artifacts[0].start_offset, Some(0)); + assert_eq!(artifacts[0].end_offset, Some(transcript.len() as u64)); + assert_eq!( + db.lane_artifact_content(&artifacts[0].artifact_id).unwrap(), + transcript + ); + let mapping_after_import = db.lane_agent_session(&mapping.mapping_id).unwrap(); + assert_eq!( + mapping_after_import.transcript_offset, + Some(transcript.len() as u64) + ); + assert_eq!( + db.turn_evidence_manifest(&turns[0].turn_id) + .unwrap() + .turn_id, + turns[0].turn_id + ); + + let end = persist( + &mut db, + "SessionEnd", + "replay:end", + serde_json::json!({"session_id":"native-replay-1","reason":"exit"}), + ); + let end_report = db.replay_agent_hook_receipt(&end).unwrap(); + assert_eq!(end_report.mapping.unwrap().status, AgentCapturePhase::Ended); + let attestations = db + .list_session_attestations(&mapping.trail_session_id) + .unwrap(); + assert_eq!(attestations.len(), 1); + assert!( + db.verify_session_attestation(&attestations[0].attestation_id) + .unwrap() + .valid + ); + } + + #[test] + fn native_replay_checkpoints_real_workspace_changes_and_resolves_native_id() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "before\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let persist = |db: &mut Trail, event: &str, suffix: &str, payload: serde_json::Value| { + db.persist_agent_hook_receipt(AgentHookReceiptInput { + installation_id: None, + provider: "claude-code".to_string(), + native_event: event.to_string(), + native_session_id: Some("native-workspace-change".to_string()), + native_turn_id: None, + transport: AgentCaptureTransport::NativeHooks, + connection_id: None, + direction: None, + connection_sequence: None, + dedupe_key: format!("native-workspace-change:{suffix}"), + payload, + occurred_at: None, + }) + .unwrap() + .receipt + .receipt_id + }; + let started = persist( + &mut db, + "SessionStart", + "start", + serde_json::json!({"session_id":"native-workspace-change","cwd":temp.path().to_string_lossy()}), + ); + let prompt = persist( + &mut db, + "UserPromptSubmit", + "prompt", + serde_json::json!({"session_id":"native-workspace-change","prompt":"change README","cwd":temp.path().to_string_lossy()}), + ); + db.replay_agent_hook_receipt(&started).unwrap(); + let mapping = db + .replay_agent_hook_receipt(&prompt) + .unwrap() + .mapping + .unwrap(); + assert_eq!( + db.try_lane_agent_session_by_native_id("native-workspace-change") + .unwrap() + .unwrap() + .mapping_id, + mapping.mapping_id + ); + + crate::db::prepare_workspace_daemon(&mut db, false).unwrap(); + std::fs::write(temp.path().join("README.md"), "after\n").unwrap(); + crate::db::workspace_daemon_fence(&mut db, None, None).unwrap(); + let workspace_scope_before_lane: (String, i64, String, i64) = db + .conn + .query_row( + "SELECT baseline_root_id,ref_generation,ref_name, + (SELECT COUNT(*) FROM changed_path_entries e + WHERE e.scope_id=s.scope_id) + FROM changed_path_scopes s WHERE scope_kind='workspace'", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .unwrap(); + assert!(workspace_scope_before_lane.3 > 0); + let stop = persist( + &mut db, + "Stop", + "stop", + serde_json::json!({"session_id":"native-workspace-change","cwd":temp.path().to_string_lossy()}), + ); + let mapping = db + .replay_agent_hook_receipt(&stop) + .unwrap() + .mapping + .unwrap(); + let lane = db.lane_name_by_id(&mapping.lane_id).unwrap(); + let branch = db.lane_branch(&lane).unwrap(); + let diff = db + .diff_refs(&branch.base_change.0, &branch.head_change.0, false) + .unwrap(); + assert_eq!(diff.files.len(), 1); + assert_eq!(diff.files[0].path, "README.md"); + let turns = db.lane_session_turns(&mapping.trail_session_id).unwrap(); + assert_eq!(turns.len(), 1); + assert_ne!(Some(turns[0].before_change.clone()), turns[0].after_change); + let view = db.agent_task_view("native-workspace-change").unwrap(); + assert_eq!( + view.task.session_id.as_deref(), + Some(mapping.trail_session_id.as_str()) + ); + assert_eq!(view.task.latest_checkpoint, Some(branch.head_change)); + assert_eq!(view.transcript.unwrap().turns.len(), 1); + let workspace_scope_after_lane: (String, i64, String, i64) = db + .conn + .query_row( + "SELECT baseline_root_id,ref_generation,ref_name, + (SELECT COUNT(*) FROM changed_path_entries e + WHERE e.scope_id=s.scope_id) + FROM changed_path_scopes s WHERE scope_kind='workspace'", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .unwrap(); + assert_eq!(workspace_scope_after_lane, workspace_scope_before_lane); + } + + #[test] + fn artifact_content_is_bounded_addressed_and_scoped_to_the_exact_turn() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane( + "artifact", + None, + false, + Some("claude-code".to_string()), + None, + ) + .unwrap(); + let session = db + .start_lane_session("artifact", Some("artifact session".to_string()), None) + .unwrap() + .session; + let turn = db + .begin_lane_session_turn("artifact", &session.session_id, None) + .unwrap() + .turn; + let input = LaneArtifactInput { + lane: "artifact".to_string(), + session_id: session.session_id.clone(), + turn_id: Some(turn.turn_id.clone()), + provider: "claude-code".to_string(), + artifact_kind: "native_transcript".to_string(), + format: "jsonl".to_string(), + source: AgentEvidenceSource::NativeTranscript, + source_locator_redacted: Some("token=secret".to_string()), + content: b"{\"role\":\"user\"}\n".to_vec(), + start_offset: Some(0), + end_offset: Some(16), + redaction_profile: Some("trail-default-v1".to_string()), + trust: "provider-native".to_string(), + supersedes_artifact_id: None, + metadata_json: Some("{\"fixture\":true}".to_string()), + }; + let artifact = db.record_lane_artifact(input.clone()).unwrap(); + assert!(artifact.content_digest.starts_with("sha256:")); + assert_eq!( + artifact.source_locator_redacted.as_deref(), + Some("token=[REDACTED]") + ); + assert_eq!( + db.record_lane_artifact(input).unwrap().artifact_id, + artifact.artifact_id + ); + assert_eq!( + db.list_lane_artifacts(&session.session_id, Some(&turn.turn_id), 10) + .unwrap(), + vec![artifact] + ); + } + + #[test] + fn native_mapping_allocates_monotonic_receipt_sequences_and_one_finalizer() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("hooks", None, false, Some("codex".to_string()), None) + .unwrap(); + let session = db + .start_lane_session( + "hooks", + Some("native hooks".to_string()), + Some("session_hooks".to_string()), + ) + .unwrap() + .session; + let mapping = db + .ensure_lane_agent_session(LaneAgentSessionInput { + provider: "codex".to_string(), + native_session_id: "native-hooks-1".to_string(), + parent_native_session_id: None, + lane: "hooks".to_string(), + trail_session_id: session.session_id, + capture_run_id: None, + primary_transport: AgentCaptureTransport::NativeHooks, + transcript_identity: None, + }) + .unwrap(); + + let make_receipt = |dedupe_key: &str| AgentHookReceiptInput { + installation_id: None, + provider: "codex".to_string(), + native_event: "event".to_string(), + native_session_id: Some("native-hooks-1".to_string()), + native_turn_id: None, + transport: AgentCaptureTransport::NativeHooks, + connection_id: None, + direction: None, + connection_sequence: None, + dedupe_key: dedupe_key.to_string(), + payload: serde_json::json!({"key": dedupe_key}), + occurred_at: None, + }; + let first = db + .persist_agent_hook_receipt(make_receipt("event:1")) + .unwrap() + .receipt; + let second = db + .persist_agent_hook_receipt(make_receipt("event:2")) + .unwrap() + .receipt; + assert_eq!( + db.assign_agent_hook_receipt_mapping(&first.receipt_id, &mapping.mapping_id) + .unwrap() + .receive_sequence, + Some(1) + ); + assert_eq!( + db.assign_agent_hook_receipt_mapping(&second.receipt_id, &mapping.mapping_id) + .unwrap() + .receive_sequence, + Some(2) + ); + + let first_lease = db + .acquire_agent_finalization_lease( + &mapping.mapping_id, + "worker-1", + 30_000, + AgentTurnOutcome::Failed, + true, + ) + .unwrap(); + assert!(first_lease.acquired); + assert_eq!(first_lease.mapping.status, AgentCapturePhase::Finalizing); + assert_eq!( + first_lease.mapping.pending_turn_outcome, + Some(AgentTurnOutcome::Failed) + ); + let competing = db + .acquire_agent_finalization_lease( + &mapping.mapping_id, + "worker-2", + 30_000, + AgentTurnOutcome::Completed, + false, + ) + .unwrap(); + assert!(!competing.acquired); + assert_eq!( + competing.mapping.finalization_owner.as_deref(), + Some("worker-1") + ); + } + + #[test] + fn managed_runs_choose_longest_workdir_and_reject_ambiguity() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + std::fs::create_dir_all(temp.path().join("apps/web")).unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + + let outer = db + .begin_agent_capture_run(AgentCaptureRunInput { + lane: None, + workdir: temp.path().to_string_lossy().to_string(), + owner_agent: "codex".to_string(), + owner_session_id: "outer".to_string(), + executor_agent: None, + work_item_id: None, + lease_ms: 60_000, + metadata_json: None, + }) + .unwrap(); + let inner = db + .begin_agent_capture_run(AgentCaptureRunInput { + lane: None, + workdir: temp.path().join("apps").to_string_lossy().to_string(), + owner_agent: "codex".to_string(), + owner_session_id: "inner".to_string(), + executor_agent: None, + work_item_id: Some("web".to_string()), + lease_ms: 60_000, + metadata_json: Some("{\"source\":\"test\"}".to_string()), + }) + .unwrap(); + assert_eq!( + db.match_agent_capture_run(temp.path().join("apps/web"), "codex") + .unwrap() + .unwrap() + .capture_run_id, + inner.capture_run_id + ); + db.renew_agent_capture_run(&outer.capture_run_id, "codex", "outer", 120_000) + .unwrap(); + + db.begin_agent_capture_run(AgentCaptureRunInput { + lane: None, + workdir: temp.path().join("apps").to_string_lossy().to_string(), + owner_agent: "codex".to_string(), + owner_session_id: "ambiguous".to_string(), + executor_agent: None, + work_item_id: None, + lease_ms: 60_000, + metadata_json: None, + }) + .unwrap(); + assert!(db + .match_agent_capture_run(temp.path().join("apps/web"), "codex") + .is_err()); + let ended = db + .end_agent_capture_run(&outer.capture_run_id, "codex", "outer") + .unwrap(); + assert_eq!(ended.status, "ended"); + } + + #[test] + fn stale_receipts_recover_with_backoff_and_explicit_operator_resolution() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let receipt = db + .persist_agent_hook_receipt(AgentHookReceiptInput { + installation_id: None, + provider: "codex".to_string(), + native_event: "SessionStart".to_string(), + native_session_id: Some("stale-session".to_string()), + native_turn_id: None, + transport: AgentCaptureTransport::NativeHooks, + connection_id: None, + direction: None, + connection_sequence: None, + dedupe_key: "stale:1".to_string(), + payload: serde_json::json!({"session_id":"stale-session"}), + occurred_at: None, + }) + .unwrap() + .receipt; + db.conn + .execute( + "UPDATE agent_hook_receipts SET status = 'processing', updated_at = 0 + WHERE receipt_id = ?1", + params![receipt.receipt_id], + ) + .unwrap(); + assert_eq!(db.recover_stale_agent_hook_receipts(1_000).unwrap(), 1); + assert_eq!( + db.agent_hook_receipt(&receipt.receipt_id).unwrap().status, + "retry" + ); + assert_eq!( + db.retry_agent_hook_receipt(&receipt.receipt_id) + .unwrap() + .status, + "received" + ); + assert_eq!( + db.discard_agent_hook_receipt(&receipt.receipt_id) + .unwrap() + .status, + "discarded" + ); + assert!(db.retry_agent_hook_receipt(&receipt.receipt_id).is_err()); + } + + #[test] + fn one_hundred_concurrent_duplicate_receipts_create_one_journal_row() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let workspace = std::sync::Arc::new(temp.path().to_path_buf()); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(100)); + let started = std::time::Instant::now(); + let handles = (0..100) + .map(|_| { + let workspace = std::sync::Arc::clone(&workspace); + let barrier = std::sync::Arc::clone(&barrier); + std::thread::spawn(move || { + let mut db = Trail::open(workspace.as_path()).unwrap(); + barrier.wait(); + Trail::with_write_lock_wait(Duration::from_secs(10), || { + db.persist_agent_hook_receipt(AgentHookReceiptInput { + installation_id: None, + provider: "codex".to_string(), + native_event: "SessionStart".to_string(), + native_session_id: Some("concurrent-session".to_string()), + native_turn_id: None, + transport: AgentCaptureTransport::NativeHooks, + connection_id: None, + direction: None, + connection_sequence: None, + dedupe_key: "concurrent:one".to_string(), + payload: serde_json::json!({"session_id":"concurrent-session"}), + occurred_at: None, + }) + }) + .unwrap() + .duplicate + }) + }) + .collect::>(); + let duplicate_count = handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .filter(|duplicate| *duplicate) + .count(); + assert_eq!(duplicate_count, 99); + assert!( + started.elapsed() < Duration::from_secs(15), + "100 concurrent duplicate receipts exceeded the 15 second CI budget" + ); + let db = Trail::open(workspace.as_path()).unwrap(); + assert_eq!( + db.list_agent_hook_receipts(Some("codex"), None, 100) + .unwrap() + .len(), + 1 + ); + assert!( + std::fs::metadata(workspace.join(".trail/index/trail.sqlite")) + .unwrap() + .len() + < 32 * 1024 * 1024, + "deduplicated ingress exceeded the 32 MiB database-growth budget" + ); + } + + #[test] + fn expired_managed_run_interrupts_open_turn_and_session() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("expired", None, false, Some("codex".to_string()), None) + .unwrap(); + let session = db + .start_lane_session("expired", Some("expired run".to_string()), None) + .unwrap() + .session; + let turn = db + .begin_lane_session_turn("expired", &session.session_id, None) + .unwrap() + .turn; + let run = db + .begin_agent_capture_run(AgentCaptureRunInput { + lane: Some("expired".to_string()), + workdir: temp.path().to_string_lossy().to_string(), + owner_agent: "codex".to_string(), + owner_session_id: "owner-expired".to_string(), + executor_agent: None, + work_item_id: None, + lease_ms: 60_000, + metadata_json: None, + }) + .unwrap(); + let mapping = db + .ensure_lane_agent_session(LaneAgentSessionInput { + provider: "codex".to_string(), + native_session_id: "native-expired".to_string(), + parent_native_session_id: None, + lane: "expired".to_string(), + trail_session_id: session.session_id.clone(), + capture_run_id: Some(run.capture_run_id.clone()), + primary_transport: AgentCaptureTransport::NativeHooks, + transcript_identity: None, + }) + .unwrap(); + db.conn + .execute( + "UPDATE lane_agent_sessions SET status = 'active' WHERE mapping_id = ?1", + params![mapping.mapping_id], + ) + .unwrap(); + db.conn + .execute( + "UPDATE agent_capture_runs SET expires_at = 0 WHERE capture_run_id = ?1", + params![run.capture_run_id], + ) + .unwrap(); + + let report = db.reconcile_expired_agent_capture_runs().unwrap(); + assert_eq!(report.expired_run_ids, vec![run.capture_run_id]); + assert_eq!( + report.interrupted_mapping_ids, + vec![mapping.mapping_id.clone()] + ); + assert_eq!(report.interrupted_turn_ids, vec![turn.turn_id.clone()]); + assert_eq!(db.lane_turn(&turn.turn_id).unwrap().status, "interrupted"); + assert_eq!( + db.lane_session(&session.session_id).unwrap().status, + "interrupted" + ); + assert_eq!( + db.lane_agent_session(&mapping.mapping_id).unwrap().status, + AgentCapturePhase::Interrupted + ); + } + + #[test] + fn hybrid_native_receipts_enrich_acp_owned_turn_without_duplication() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("hybrid", None, false, Some("codex".to_string()), None) + .unwrap(); + let session = db + .start_lane_session("hybrid", Some("hybrid".to_string()), None) + .unwrap() + .session; + db.upsert_lane_acp_session( + "acp-hybrid-session", + Some("native-hybrid-session"), + "hybrid", + &session.session_id, + &temp.path().to_string_lossy(), + &[], + Some("codex"), + None, + None, + "active", + ) + .unwrap(); + let turn = db + .begin_lane_session_turn("hybrid", &session.session_id, None) + .unwrap() + .turn; + db.add_lane_turn_message(&turn.turn_id, "user", "same prompt") + .unwrap(); + let prompt = db + .persist_agent_hook_receipt(AgentHookReceiptInput { + installation_id: None, + provider: "codex".to_string(), + native_event: "UserPromptSubmit".to_string(), + native_session_id: Some("native-hybrid-session".to_string()), + native_turn_id: Some("native-turn".to_string()), + transport: AgentCaptureTransport::NativeHooks, + connection_id: None, + direction: None, + connection_sequence: None, + dedupe_key: "hybrid:prompt".to_string(), + payload: serde_json::json!({ + "session_id":"native-hybrid-session", + "turn_id":"native-turn", + "prompt":"same prompt" + }), + occurred_at: None, + }) + .unwrap() + .receipt; + let report = db.replay_agent_hook_receipt(&prompt.receipt_id).unwrap(); + let mapping = report.mapping.unwrap(); + assert_eq!(mapping.primary_transport, AgentCaptureTransport::Hybrid); + assert_eq!(mapping.trail_session_id, session.session_id); + assert_eq!(mapping.status, AgentCapturePhase::Idle); + let details = db.show_lane_turn(&turn.turn_id).unwrap(); + assert_eq!(details.messages.len(), 1); + assert_eq!(details.messages[0].body, "same prompt"); + assert!(details + .events + .iter() + .any(|event| event.event_type == "turn.started")); + assert!(db.lane_turn(&turn.turn_id).unwrap().ended_at.is_none()); + } + + #[test] + fn canonical_export_precedes_transcript_and_missing_native_artifact_is_reconstructed() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + let transcript_path = temp.path().join("native.jsonl"); + let export_path = temp.path().join("canonical.json"); + std::fs::write(&transcript_path, b"native transcript").unwrap(); + std::fs::write(&export_path, b"{\"canonical\":true}").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let persist = |db: &mut Trail, + session_id: &str, + event: &str, + suffix: &str, + payload: serde_json::Value| { + db.persist_agent_hook_receipt(AgentHookReceiptInput { + installation_id: None, + provider: "opencode".to_string(), + native_event: event.to_string(), + native_session_id: Some(session_id.to_string()), + native_turn_id: None, + transport: AgentCaptureTransport::NativeHooks, + connection_id: None, + direction: None, + connection_sequence: None, + dedupe_key: format!("{session_id}:{suffix}"), + payload, + occurred_at: None, + }) + .unwrap() + .receipt + .receipt_id + }; + + let session_id = "opencode-canonical"; + let started = persist( + &mut db, + session_id, + "session.created", + "start", + serde_json::json!({ + "sessionID": session_id, + "transcript_path": transcript_path, + }), + ); + let prompt = persist( + &mut db, + session_id, + "chat.message", + "prompt", + serde_json::json!({"sessionID":session_id,"message":"do it"}), + ); + let ended = persist( + &mut db, + session_id, + "session.idle", + "end", + serde_json::json!({ + "sessionID":session_id, + "canonical_export_path": export_path, + }), + ); + db.replay_agent_hook_receipt(&started).unwrap(); + db.replay_agent_hook_receipt(&prompt).unwrap(); + let mapping = db + .replay_agent_hook_receipt(&ended) + .unwrap() + .mapping + .unwrap(); + let artifacts = db + .list_lane_artifacts(&mapping.trail_session_id, None, 10) + .unwrap(); + assert_eq!(artifacts.len(), 1); + assert_eq!(artifacts[0].artifact_kind, "export"); + assert_eq!(artifacts[0].source, AgentEvidenceSource::CanonicalExport); + assert_eq!( + db.lane_artifact_content(&artifacts[0].artifact_id).unwrap(), + b"{\"canonical\":true}" + ); + std::fs::write(&export_path, b"{\"canonical\":false}").unwrap(); + let refreshed = db.replay_agent_hook_receipt(&ended).unwrap(); + assert!(!refreshed.replayed); + assert_eq!( + refreshed.actions, + vec![AgentCaptureAction::ImportTranscript] + ); + let refreshed_artifacts = db + .list_lane_artifacts(&mapping.trail_session_id, None, 10) + .unwrap(); + assert_eq!(refreshed_artifacts.len(), 2); + assert!(refreshed_artifacts + .iter() + .all(|artifact| artifact.turn_id.is_some())); + assert_eq!( + db.lane_artifact_content(&refreshed_artifacts[0].artifact_id) + .unwrap(), + b"{\"canonical\":false}" + ); + std::fs::write(&export_path, b"{}").unwrap(); + db.replay_agent_hook_receipt(&ended).unwrap(); + let truncated = db + .list_lane_artifacts(&mapping.trail_session_id, None, 10) + .unwrap(); + assert_eq!(truncated.len(), 3); + let metadata: serde_json::Value = + serde_json::from_str(truncated[0].metadata_json.as_deref().unwrap()).unwrap(); + assert_eq!(metadata["truncated"], true); + assert_eq!(truncated[0].end_offset, Some(2)); + + let reconstructed_session = "opencode-reconstructed"; + let prompt = persist( + &mut db, + reconstructed_session, + "chat.message", + "prompt", + serde_json::json!({"sessionID":reconstructed_session,"message":"do it"}), + ); + let ended = persist( + &mut db, + reconstructed_session, + "session.idle", + "end", + serde_json::json!({"sessionID":reconstructed_session}), + ); + db.replay_agent_hook_receipt(&prompt).unwrap(); + let mapping = db + .replay_agent_hook_receipt(&ended) + .unwrap() + .mapping + .unwrap(); + let artifacts = db + .list_lane_artifacts(&mapping.trail_session_id, None, 10) + .unwrap(); + assert_eq!(artifacts.len(), 1); + assert_eq!(artifacts[0].artifact_kind, "transcript"); + assert_eq!(artifacts[0].source, AgentEvidenceSource::Reconstructed); + assert!( + String::from_utf8(db.lane_artifact_content(&artifacts[0].artifact_id).unwrap()) + .unwrap() + .contains("trail.reconstructed_transcript") + ); + } + + #[cfg(unix)] + #[test] + fn transcript_import_rejects_leaf_symlinks_and_paths_outside_approved_roots() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let target = temp.path().join("transcript.jsonl"); + let link = temp.path().join("transcript-link.jsonl"); + std::fs::write(&target, "{}\n").unwrap(); + symlink(&target, &link).unwrap(); + + assert!(reject_agent_transcript_symlinks(&link).is_err()); + assert!(!db + .agent_transcript_path_allowed("opencode", std::path::Path::new("/etc/hosts")) + .unwrap()); + } +} diff --git a/trail/src/db/lane/control/agent_evidence.rs b/trail/src/db/lane/control/agent_evidence.rs new file mode 100644 index 0000000..8217f56 --- /dev/null +++ b/trail/src/db/lane/control/agent_evidence.rs @@ -0,0 +1,2091 @@ +use super::*; + +use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; +use sha2::{Digest, Sha256}; + +const TURN_EVIDENCE_OBJECT_KIND: &str = "TurnEvidenceManifest"; +const SESSION_ATTESTATION_OBJECT_KIND: &str = "SessionAttestation"; + +impl Trail { + /// Freeze the exact evidence already attached to one completed turn. + pub fn create_turn_evidence_manifest(&mut self, turn_id: &str) -> Result { + if let Some(existing) = self.try_turn_evidence_manifest(turn_id)? { + return Ok(existing); + } + let turn = self.lane_turn(turn_id)?; + if turn.ended_at.is_none() { + return Err(Error::InvalidInput(format!( + "turn `{turn_id}` must be completed before its evidence manifest is frozen" + ))); + } + let session_id = turn.session_id.clone().ok_or_else(|| { + Error::InvalidInput(format!("turn `{turn_id}` has no owning session")) + })?; + let events = self.lane_turn_events(turn_id)?; + let messages = self.lane_turn_messages(turn_id)?; + let artifacts = self.list_lane_artifacts(&session_id, Some(turn_id), 1_000)?; + let mut coverage = TurnEvidenceCoverage { + event_ids: events + .iter() + .map(|event| { + event + .payload + .as_ref() + .and_then(|payload| payload.get("event_id")) + .and_then(serde_json::Value::as_str) + .unwrap_or(&event.event_id) + .to_string() + }) + .collect(), + message_ids: messages + .iter() + .map(|message| message.id.0.clone()) + .collect(), + artifact_ids: artifacts + .iter() + .map(|artifact| artifact.artifact_id.clone()) + .collect(), + ..TurnEvidenceCoverage::default() + }; + for event in &events { + if let Some(payload) = event.payload.as_ref() { + collect_json_strings(payload, "receipt_id", &mut coverage.receipt_ids); + collect_json_strings(payload, "span_id", &mut coverage.tool_span_ids); + collect_json_strings(payload, "approval_id", &mut coverage.approval_ids); + } + } + let after_change = turn + .after_change + .clone() + .unwrap_or_else(|| turn.before_change.clone()); + if after_change != turn.before_change { + coverage.change_ids.push(after_change.clone()); + } + sort_dedup_coverage(&mut coverage); + let statement = TurnEvidenceStatement { + schema: TURN_EVIDENCE_MANIFEST_SCHEMA.to_string(), + version: TURN_EVIDENCE_MANIFEST_VERSION, + workspace_id: self.config.workspace.id.0.clone(), + lane_id: turn.lane_id.clone(), + session_id: session_id.clone(), + turn_id: turn_id.to_string(), + before_change: turn.before_change, + after_change, + turn_status: turn.status, + coverage, + }; + let statement_bytes = canonical_json_bytes(&statement)?; + let digest = digest_bytes(&statement_bytes); + let manifest_id = format!("evidence_{}", crate::ids::short_hash(digest.as_bytes(), 24)); + let object_id = self.put_object( + TURN_EVIDENCE_OBJECT_KIND, + TURN_EVIDENCE_MANIFEST_VERSION, + &statement, + )?; + let created_at = now_millis(); + { + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "INSERT INTO lane_turn_evidence_manifests + (manifest_id, lane_id, session_id, turn_id, schema_version, + object_id, digest, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + ON CONFLICT(turn_id) DO NOTHING", + params![ + manifest_id, + statement.lane_id, + session_id, + turn_id, + TURN_EVIDENCE_MANIFEST_VERSION, + object_id.0, + digest, + created_at, + ], + )?; + } + let stored = self.turn_evidence_manifest(turn_id)?; + if stored.digest != digest { + return Err(Error::Conflict(format!( + "turn `{turn_id}` already has a different immutable evidence manifest" + ))); + } + Ok(stored) + } + + pub fn turn_evidence_manifest(&self, turn_id: &str) -> Result { + self.try_turn_evidence_manifest(turn_id)? + .ok_or_else(|| Error::ObjectNotFound { + kind: "turn evidence manifest", + id: turn_id.to_string(), + }) + } + + pub fn list_turn_evidence_manifests( + &self, + session_id: &str, + limit: usize, + ) -> Result> { + self.lane_session(session_id)?; + let mut statement = self.conn.prepare( + "SELECT manifest_id, lane_id, session_id, turn_id, schema_version, + object_id, digest, created_at + FROM lane_turn_evidence_manifests + WHERE session_id = ?1 + ORDER BY created_at, turn_id LIMIT ?2", + )?; + let rows = statement + .query_map( + params![ + session_id, + i64::try_from(limit.clamp(1, 1_000)).unwrap_or(1_000) + ], + evidence_manifest_row, + )? + .collect::, _>>()?; + rows.into_iter() + .map(|row| self.materialize_turn_evidence_manifest(row)) + .collect() + } + + fn try_turn_evidence_manifest(&self, turn_id: &str) -> Result> { + let row = self + .conn + .query_row( + "SELECT manifest_id, lane_id, session_id, turn_id, schema_version, + object_id, digest, created_at + FROM lane_turn_evidence_manifests WHERE turn_id = ?1", + params![turn_id], + evidence_manifest_row, + ) + .optional()?; + row.map(|row| self.materialize_turn_evidence_manifest(row)) + .transpose() + } + + fn materialize_turn_evidence_manifest( + &self, + row: EvidenceManifestRow, + ) -> Result { + let statement: TurnEvidenceStatement = + self.get_object(TURN_EVIDENCE_OBJECT_KIND, &row.object_id)?; + if statement.schema != TURN_EVIDENCE_MANIFEST_SCHEMA + || statement.version != row.schema_version + || statement.turn_id != row.turn_id + || statement.session_id != row.session_id + || statement.lane_id != row.lane_id + { + return Err(Error::Corrupt(format!( + "turn evidence manifest `{}` row/object identity mismatch", + row.manifest_id + ))); + } + let actual_digest = digest_bytes(&canonical_json_bytes(&statement)?); + if actual_digest != row.digest { + return Err(Error::Corrupt(format!( + "turn evidence manifest `{}` digest mismatch", + row.manifest_id + ))); + } + Ok(TurnEvidenceManifest { + manifest_id: row.manifest_id, + lane_id: row.lane_id, + session_id: row.session_id, + turn_id: row.turn_id, + schema_version: row.schema_version, + object_id: row.object_id, + digest: row.digest, + created_at: row.created_at, + statement, + }) + } + + pub fn create_provenance_node(&mut self, input: ProvenanceNodeInput) -> Result { + validate_evidence_text("provenance node kind", &input.node_kind, 128)?; + validate_evidence_text("provenance summary", &input.summary, 4_096)?; + validate_evidence_text( + "provenance source confidence", + &input.source_confidence, + 128, + )?; + let session = self.lane_session(&input.session_id)?; + validate_optional_turn_for_session(self, input.turn_id.as_deref(), &input.session_id)?; + if let Some(artifact_id) = input.artifact_id.as_deref() { + let artifact = self.lane_artifact(artifact_id)?; + if artifact.session_id != input.session_id { + return Err(Error::InvalidInput(format!( + "artifact `{artifact_id}` does not belong to session `{}`", + input.session_id + ))); + } + } + if let Some(message_id) = input.message_id.as_deref() { + let message = self.message(message_id)?; + if message.session_id.as_deref() != Some(input.session_id.as_str()) { + return Err(Error::InvalidInput(format!( + "message `{message_id}` does not belong to session `{}`", + input.session_id + ))); + } + } + let attributes_json = input + .attributes + .as_ref() + .map(canonical_json_string) + .transpose()?; + let identity = canonical_json_bytes(&serde_json::json!({ + "session_id": input.session_id, + "turn_id": input.turn_id, + "node_kind": input.node_kind, + "event_id": input.event_id, + "span_id": input.span_id, + "message_id": input.message_id, + "change_id": input.change_id, + "artifact_id": input.artifact_id, + "source_confidence": input.source_confidence, + "classifier_version": input.classifier_version, + "attributes": input.attributes, + }))?; + let node_id = format!("provenance_node_{}", crate::ids::short_hash(&identity, 24)); + let now = now_millis(); + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "INSERT INTO lane_provenance_nodes + (provenance_node_id, lane_id, session_id, turn_id, node_kind, summary, + event_id, span_id, message_id, change_id, artifact_id, source_confidence, + classifier_version, created_at, attributes_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15) + ON CONFLICT(provenance_node_id) DO NOTHING", + params![ + node_id, + session.lane_id, + input.session_id, + input.turn_id, + input.node_kind, + redact_sensitive_text(&input.summary), + input.event_id, + input.span_id, + input.message_id, + input.change_id, + input.artifact_id, + input.source_confidence, + input.classifier_version, + now, + attributes_json, + ], + )?; + self.provenance_node(&node_id) + } + + pub fn provenance_node(&self, node_id: &str) -> Result { + self.conn + .query_row( + PROVENANCE_NODE_SELECT_BY_ID, + params![node_id], + provenance_node_row, + ) + .optional()? + .ok_or_else(|| Error::ObjectNotFound { + kind: "provenance node", + id: node_id.to_string(), + }) + } + + pub fn create_provenance_edge(&mut self, input: ProvenanceEdgeInput) -> Result { + validate_evidence_text("provenance relation", &input.relation, 128)?; + validate_evidence_text( + "provenance source confidence", + &input.source_confidence, + 128, + )?; + let from = self.provenance_node(&input.from_node_id)?; + let to = self.provenance_node(&input.to_node_id)?; + if from.session_id != to.session_id || from.lane_id != to.lane_id { + return Err(Error::InvalidInput( + "provenance edges cannot cross session or lane boundaries".to_string(), + )); + } + if let Some(receipt_id) = input.receipt_id.as_deref() { + let receipt = self.agent_hook_receipt(receipt_id)?; + if receipt.mapping_id.is_none() { + return Err(Error::InvalidInput(format!( + "receipt `{receipt_id}` is not mapped to a captured session" + ))); + } + } + let attributes_json = input + .attributes + .as_ref() + .map(canonical_json_string) + .transpose()?; + let identity = format!( + "{}:{}:{}:{}", + input.from_node_id, + input.to_node_id, + input.relation, + input.receipt_id.as_deref().unwrap_or("") + ); + let edge_id = format!( + "provenance_edge_{}", + crate::ids::short_hash(identity.as_bytes(), 24) + ); + let now = now_millis(); + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "INSERT INTO lane_provenance_edges + (provenance_edge_id, lane_id, session_id, from_node_id, to_node_id, + relation, source_confidence, receipt_id, created_at, attributes_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) + ON CONFLICT(from_node_id, to_node_id, relation, COALESCE(receipt_id, '')) + DO NOTHING", + params![ + edge_id, + from.lane_id, + from.session_id, + input.from_node_id, + input.to_node_id, + input.relation, + input.source_confidence, + input.receipt_id, + now, + attributes_json, + ], + )?; + self.provenance_edge(&edge_id).or_else(|_| { + self.conn + .query_row( + "SELECT provenance_edge_id, lane_id, session_id, from_node_id, + to_node_id, relation, source_confidence, receipt_id, + created_at, attributes_json + FROM lane_provenance_edges + WHERE from_node_id = ?1 AND to_node_id = ?2 AND relation = ?3 + AND COALESCE(receipt_id, '') = COALESCE(?4, '')", + params![ + from.provenance_node_id, + to.provenance_node_id, + input.relation, + input.receipt_id + ], + provenance_edge_row, + ) + .map_err(Error::from) + }) + } + + pub fn provenance_edge(&self, edge_id: &str) -> Result { + self.conn + .query_row( + "SELECT provenance_edge_id, lane_id, session_id, from_node_id, + to_node_id, relation, source_confidence, receipt_id, + created_at, attributes_json + FROM lane_provenance_edges WHERE provenance_edge_id = ?1", + params![edge_id], + provenance_edge_row, + ) + .optional()? + .ok_or_else(|| Error::ObjectNotFound { + kind: "provenance edge", + id: edge_id.to_string(), + }) + } + + pub fn list_session_provenance( + &self, + session_id: &str, + limit: usize, + ) -> Result<(Vec, Vec)> { + self.list_session_provenance_page(session_id, 0, limit) + } + + pub fn list_session_provenance_page( + &self, + session_id: &str, + offset: usize, + limit: usize, + ) -> Result<(Vec, Vec)> { + self.lane_session(session_id)?; + let limit = i64::try_from(limit.clamp(1, 10_000)).unwrap_or(10_000); + let mut node_statement = self.conn.prepare( + "SELECT provenance_node_id, lane_id, session_id, turn_id, node_kind, + summary, event_id, span_id, message_id, change_id, artifact_id, + source_confidence, classifier_version, created_at, attributes_json + FROM lane_provenance_nodes WHERE session_id = ?1 + ORDER BY created_at, provenance_node_id LIMIT ?2 OFFSET ?3", + )?; + let nodes = node_statement + .query_map( + params![ + session_id, + limit, + i64::try_from(offset.min(1_000_000)).unwrap_or(1_000_000) + ], + provenance_node_row, + )? + .collect::, _>>()?; + let mut edge_statement = self.conn.prepare( + "SELECT provenance_edge_id, lane_id, session_id, from_node_id, + to_node_id, relation, source_confidence, receipt_id, + created_at, attributes_json + FROM lane_provenance_edges WHERE session_id = ?1 + ORDER BY created_at, provenance_edge_id LIMIT ?2 OFFSET ?3", + )?; + let edges = edge_statement + .query_map( + params![ + session_id, + limit, + i64::try_from(offset.min(1_000_000)).unwrap_or(1_000_000) + ], + provenance_edge_row, + )? + .collect::, _>>()?; + Ok((nodes, edges)) + } + + /// Materialize deterministic, explicitly derived activity labels from factual events. + pub fn classify_session_activity( + &mut self, + session_id: &str, + limit: usize, + ) -> Result { + const CLASSIFIER: &str = "trail-activity-rules/v1"; + let events = self.list_lane_events(None, Some(session_id), None, None, limit)?; + let mut nodes = Vec::new(); + let mut edges = Vec::new(); + for event in events.into_iter().rev() { + let Some((activity_kind, summary)) = classify_event_activity(&event.event_type) else { + continue; + }; + let source = self.create_provenance_node(ProvenanceNodeInput { + session_id: session_id.to_string(), + turn_id: event.turn_id.clone(), + node_kind: "source_event".to_string(), + summary: format!("Observed `{}` event", event.event_type), + event_id: Some(event.event_id.clone()), + span_id: None, + message_id: event + .message_id + .as_ref() + .map(|message_id| message_id.0.clone()), + change_id: event.change_id.as_ref().map(ToString::to_string), + artifact_id: None, + source_confidence: "factual".to_string(), + classifier_version: None, + attributes: Some(serde_json::json!({"event_type": event.event_type})), + })?; + let derived = self.create_provenance_node(ProvenanceNodeInput { + session_id: session_id.to_string(), + turn_id: event.turn_id, + node_kind: activity_kind.to_string(), + summary: summary.to_string(), + event_id: None, + span_id: None, + message_id: None, + change_id: None, + artifact_id: None, + source_confidence: "deterministic-derived".to_string(), + classifier_version: Some(CLASSIFIER.to_string()), + attributes: Some(serde_json::json!({ + "rule": event.event_type, + "claims_hidden_reasoning": false, + })), + })?; + let edge = self.create_provenance_edge(ProvenanceEdgeInput { + from_node_id: derived.provenance_node_id.clone(), + to_node_id: source.provenance_node_id.clone(), + relation: "derived_from".to_string(), + source_confidence: "deterministic-derived".to_string(), + receipt_id: None, + attributes: Some(serde_json::json!({"classifier_version": CLASSIFIER})), + })?; + nodes.push(source); + nodes.push(derived); + edges.push(edge); + } + nodes.sort_by(|left, right| left.provenance_node_id.cmp(&right.provenance_node_id)); + nodes.dedup_by(|left, right| left.provenance_node_id == right.provenance_node_id); + edges.sort_by(|left, right| left.provenance_edge_id.cmp(&right.provenance_edge_id)); + edges.dedup_by(|left, right| left.provenance_edge_id == right.provenance_edge_id); + Ok(ActivityClassificationReport { + session_id: session_id.to_string(), + classifier_version: CLASSIFIER.to_string(), + nodes, + edges, + }) + } + + pub fn create_session_attestation( + &mut self, + session_id: &str, + capture_policy: &str, + metadata: Option, + ) -> Result { + validate_evidence_text("attestation capture policy", capture_policy, 128)?; + let session = self.lane_session(session_id)?; + let turns = self.lane_session_turns(session_id)?; + for turn in &turns { + if turn.ended_at.is_some() { + self.create_turn_evidence_manifest(&turn.turn_id)?; + } + } + let previous = self.latest_session_attestation(session_id)?; + let mut covered = std::collections::BTreeSet::new(); + { + let mut statement = self.conn.prepare( + "SELECT turn_id FROM lane_session_attestation_turns + WHERE attestation_id IN ( + SELECT attestation_id FROM lane_session_attestations WHERE session_id = ?1 + )", + )?; + for turn_id in + statement.query_map(params![session_id], |row| row.get::<_, String>(0))? + { + covered.insert(turn_id?); + } + } + let manifests = self + .list_turn_evidence_manifests(session_id, 1_000)? + .into_iter() + .filter(|manifest| !covered.contains(&manifest.turn_id)) + .collect::>(); + if manifests.is_empty() { + return previous.ok_or_else(|| { + Error::InvalidInput(format!( + "session `{session_id}` has no completed turns to attest" + )) + }); + } + let capture_run_id = self + .conn + .query_row( + "SELECT capture_run_id FROM lane_agent_sessions + WHERE trail_session_id = ?1 ORDER BY created_at LIMIT 1", + params![session_id], + |row| row.get::<_, Option>(0), + ) + .optional()? + .flatten(); + let attested_turns = manifests + .iter() + .map(|manifest| SessionAttestationTurn { + turn_id: manifest.turn_id.clone(), + change_id: (manifest.statement.after_change != manifest.statement.before_change) + .then(|| manifest.statement.after_change.0.clone()), + evidence_manifest_id: manifest.manifest_id.clone(), + evidence_digest: manifest.digest.clone(), + }) + .collect::>(); + let statement = SessionAttestationStatement { + schema: SESSION_ATTESTATION_SCHEMA.to_string(), + version: SESSION_ATTESTATION_VERSION, + workspace_id: self.config.workspace.id.0.clone(), + lane_id: session.lane_id.clone(), + session_id: session_id.to_string(), + capture_run_id: capture_run_id.clone(), + previous_attestation_id: previous + .as_ref() + .map(|attestation| attestation.attestation_id.clone()), + turns: attested_turns.clone(), + capture_policy: capture_policy.to_string(), + }; + let statement_bytes = canonical_json_bytes(&statement)?; + let statement_digest = digest_bytes(&statement_bytes); + let attestation_id = format!( + "attestation_{}", + crate::ids::short_hash(statement_digest.as_bytes(), 24) + ); + let object_id = self.put_object( + SESSION_ATTESTATION_OBJECT_KIND, + SESSION_ATTESTATION_VERSION, + &statement, + )?; + let metadata_json = metadata.as_ref().map(canonical_json_string).transpose()?; + let now = now_millis(); + let _lock = self.acquire_write_lock()?; + self.conn + .execute_batch("SAVEPOINT create_session_attestation")?; + let result = (|| -> Result<()> { + self.conn.execute( + "INSERT INTO lane_session_attestations + (attestation_id, lane_id, session_id, capture_run_id, + previous_attestation_id, statement_object_id, statement_digest, + signature_json, status, created_at, superseded_by, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, NULL, 'unsigned', ?8, NULL, ?9) + ON CONFLICT(attestation_id) DO NOTHING", + params![ + attestation_id, + session.lane_id, + session_id, + capture_run_id, + statement.previous_attestation_id, + object_id.0, + statement_digest, + now, + metadata_json, + ], + )?; + for turn in &attested_turns { + self.conn.execute( + "INSERT INTO lane_session_attestation_turns + (attestation_id, turn_id, change_id, evidence_manifest_id) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(attestation_id, turn_id) DO NOTHING", + params![ + attestation_id, + turn.turn_id, + turn.change_id, + turn.evidence_manifest_id, + ], + )?; + } + self.conn.execute( + "UPDATE lane_agent_sessions SET last_attestation_id = ?2, updated_at = ?3 + WHERE trail_session_id = ?1", + params![session_id, attestation_id, now], + )?; + Ok(()) + })(); + match result { + Ok(()) => self + .conn + .execute_batch("RELEASE SAVEPOINT create_session_attestation")?, + Err(error) => { + self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT create_session_attestation; + RELEASE SAVEPOINT create_session_attestation", + )?; + return Err(error); + } + } + self.session_attestation(&attestation_id) + } + + pub fn session_attestation(&self, attestation_id: &str) -> Result { + let row = self + .conn + .query_row( + SESSION_ATTESTATION_SELECT_BY_ID, + params![attestation_id], + session_attestation_row, + ) + .optional()? + .ok_or_else(|| Error::ObjectNotFound { + kind: "session attestation", + id: attestation_id.to_string(), + })?; + self.materialize_session_attestation(row) + } + + pub fn list_session_attestations(&self, session_id: &str) -> Result> { + self.list_session_attestations_page(session_id, 0, 1_000) + } + + pub fn list_session_attestations_page( + &self, + session_id: &str, + offset: usize, + limit: usize, + ) -> Result> { + self.lane_session(session_id)?; + let mut statement = self.conn.prepare( + "SELECT attestation_id, lane_id, session_id, capture_run_id, + previous_attestation_id, statement_object_id, statement_digest, + signature_json, status, created_at, superseded_by, metadata_json + FROM lane_session_attestations WHERE session_id = ?1 + ORDER BY created_at, attestation_id LIMIT ?2 OFFSET ?3", + )?; + let rows = statement + .query_map( + params![ + session_id, + i64::try_from(limit.clamp(1, 1_000)).unwrap_or(1_000), + i64::try_from(offset.min(1_000_000)).unwrap_or(1_000_000) + ], + session_attestation_row, + )? + .collect::, _>>()?; + rows.into_iter() + .map(|row| self.materialize_session_attestation(row)) + .collect() + } + + pub fn verify_session_attestation( + &self, + attestation_id: &str, + ) -> Result { + let attestation = self.session_attestation(attestation_id)?; + let statement: SessionAttestationStatement = self.get_object( + SESSION_ATTESTATION_OBJECT_KIND, + &attestation.statement_object_id, + )?; + let statement_digest_valid = + digest_bytes(&canonical_json_bytes(&statement)?) == attestation.statement_digest; + let mut diagnostics = Vec::new(); + if !statement_digest_valid { + diagnostics.push("attestation statement digest mismatch".to_string()); + } + let mut evidence_digests_valid = true; + for turn in &statement.turns { + match self.turn_evidence_manifest(&turn.turn_id) { + Ok(manifest) + if manifest.manifest_id == turn.evidence_manifest_id + && manifest.digest == turn.evidence_digest => {} + _ => { + evidence_digests_valid = false; + diagnostics.push(format!( + "turn `{}` evidence manifest no longer verifies", + turn.turn_id + )); + } + } + } + let chain_valid = self.verify_attestation_chain(&attestation, 1_000)?; + if !chain_valid { + diagnostics.push("attestation predecessor chain is invalid".to_string()); + } + let (signature_status, signature_valid) = + self.verify_attestation_signature(&attestation, &mut diagnostics)?; + let valid = + statement_digest_valid && evidence_digests_valid && chain_valid && signature_valid; + Ok(AttestationVerificationReport { + attestation_id: attestation_id.to_string(), + statement_digest_valid, + evidence_digests_valid, + chain_valid, + signature_status, + valid, + diagnostics, + }) + } + + pub fn sign_session_attestation( + &mut self, + attestation_id: &str, + secret_key: &[u8; 32], + ) -> Result { + let attestation = self.session_attestation(attestation_id)?; + let signing_key = SigningKey::from_bytes(secret_key); + let verifying_key = signing_key.verifying_key(); + let public_key_hex = hex::encode(verifying_key.as_bytes()); + let key_id = attestation_key_id(verifying_key.as_bytes()); + if self.attestation_key_revocation(&key_id)?.is_some() { + return Err(Error::Conflict(format!( + "attestation signing key `{key_id}` is revoked" + ))); + } + let signature = signing_key.sign(attestation.statement_digest.as_bytes()); + let envelope = AttestationSignature { + algorithm: "ed25519".to_string(), + key_id, + public_key_hex, + signature_hex: hex::encode(signature.to_bytes()), + }; + let signature_json = serde_json::to_string(&envelope)?; + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "UPDATE lane_session_attestations + SET signature_json = ?2, status = 'signed' + WHERE attestation_id = ?1", + params![attestation_id, signature_json], + )?; + self.session_attestation(attestation_id) + } + + pub fn revoke_attestation_key( + &mut self, + public_key: &[u8; 32], + reason: &str, + metadata: Option, + ) -> Result { + validate_evidence_text("attestation key revocation reason", reason, 4_096)?; + VerifyingKey::from_bytes(public_key) + .map_err(|error| Error::InvalidInput(format!("invalid Ed25519 public key: {error}")))?; + let key_id = attestation_key_id(public_key); + let public_key_hex = hex::encode(public_key); + let metadata_json = metadata.as_ref().map(canonical_json_string).transpose()?; + let now = now_millis(); + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "INSERT INTO agent_attestation_key_revocations + (key_id, public_key_hex, reason, revoked_at, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(key_id) DO UPDATE SET + reason = excluded.reason, + revoked_at = excluded.revoked_at, + metadata_json = excluded.metadata_json", + params![key_id, public_key_hex, reason, now, metadata_json], + )?; + self.attestation_key_revocation(&key_id)? + .ok_or_else(|| Error::Corrupt(format!("revocation `{key_id}` was not persisted"))) + } + + pub fn attestation_key_revocation( + &self, + key_id: &str, + ) -> Result> { + self.conn + .query_row( + "SELECT key_id, public_key_hex, reason, revoked_at, metadata_json + FROM agent_attestation_key_revocations WHERE key_id = ?1", + params![key_id], + attestation_key_revocation_row, + ) + .optional() + .map_err(Error::from) + } + + fn verify_attestation_signature( + &self, + attestation: &SessionAttestation, + diagnostics: &mut Vec, + ) -> Result<(String, bool)> { + let Some(signature_value) = attestation.signature.clone() else { + return Ok(("unsigned".to_string(), true)); + }; + let envelope: AttestationSignature = match serde_json::from_value(signature_value) { + Ok(envelope) => envelope, + Err(error) => { + diagnostics.push(format!("invalid attestation signature envelope: {error}")); + return Ok(("invalid-envelope".to_string(), false)); + } + }; + if envelope.algorithm != "ed25519" { + diagnostics.push(format!( + "unsupported attestation signature algorithm `{}`", + envelope.algorithm + )); + return Ok(("unsupported-algorithm".to_string(), false)); + } + let public_key = decode_fixed_hex::<32>(&envelope.public_key_hex, "public key")?; + let signature_bytes = decode_fixed_hex::<64>(&envelope.signature_hex, "signature")?; + if attestation_key_id(&public_key) != envelope.key_id { + diagnostics.push("attestation key id does not match public key".to_string()); + return Ok(("invalid-key-id".to_string(), false)); + } + if self.attestation_key_revocation(&envelope.key_id)?.is_some() { + diagnostics.push(format!( + "attestation key `{}` has been revoked", + envelope.key_id + )); + return Ok(("revoked".to_string(), false)); + } + let verifying_key = VerifyingKey::from_bytes(&public_key).map_err(|error| { + Error::InvalidInput(format!("invalid attestation public key: {error}")) + })?; + let signature = Signature::from_bytes(&signature_bytes); + match verifying_key.verify(attestation.statement_digest.as_bytes(), &signature) { + Ok(()) => Ok(("valid".to_string(), true)), + Err(error) => { + diagnostics.push(format!( + "attestation signature verification failed: {error}" + )); + Ok(("invalid".to_string(), false)) + } + } + } + + fn latest_session_attestation(&self, session_id: &str) -> Result> { + let row = self + .conn + .query_row( + "SELECT attestation_id, lane_id, session_id, capture_run_id, + previous_attestation_id, statement_object_id, statement_digest, + signature_json, status, created_at, superseded_by, metadata_json + FROM lane_session_attestations WHERE session_id = ?1 + ORDER BY created_at DESC, attestation_id DESC LIMIT 1", + params![session_id], + session_attestation_row, + ) + .optional()?; + row.map(|row| self.materialize_session_attestation(row)) + .transpose() + } + + fn materialize_session_attestation( + &self, + row: SessionAttestationRow, + ) -> Result { + let statement: SessionAttestationStatement = + self.get_object(SESSION_ATTESTATION_OBJECT_KIND, &row.statement_object_id)?; + if statement.schema != SESSION_ATTESTATION_SCHEMA + || statement.version != SESSION_ATTESTATION_VERSION + || statement.session_id != row.session_id + || statement.lane_id != row.lane_id + || digest_bytes(&canonical_json_bytes(&statement)?) != row.statement_digest + { + return Err(Error::Corrupt(format!( + "session attestation `{}` statement is invalid", + row.attestation_id + ))); + } + Ok(SessionAttestation { + attestation_id: row.attestation_id, + lane_id: row.lane_id, + session_id: row.session_id, + capture_run_id: row.capture_run_id, + previous_attestation_id: row.previous_attestation_id, + statement_object_id: row.statement_object_id, + statement_digest: row.statement_digest, + signature: row.signature, + status: row.status, + created_at: row.created_at, + superseded_by: row.superseded_by, + metadata: row.metadata, + turns: statement.turns, + }) + } + + fn verify_attestation_chain( + &self, + attestation: &SessionAttestation, + max_depth: usize, + ) -> Result { + let mut current = attestation.clone(); + let mut seen = std::collections::BTreeSet::new(); + for _ in 0..max_depth { + if !seen.insert(current.attestation_id.clone()) { + return Ok(false); + } + let Some(previous_id) = current.previous_attestation_id.as_deref() else { + return Ok(true); + }; + let previous = self.session_attestation(previous_id)?; + if previous.session_id != attestation.session_id + || previous.created_at > current.created_at + { + return Ok(false); + } + current = previous; + } + Ok(false) + } + + pub fn propose_learning(&mut self, input: LearningInput) -> Result { + validate_evidence_text("learning scope", &input.scope, 128)?; + validate_evidence_text("learning body", &input.body, 32 * 1024)?; + if input + .confidence + .is_some_and(|confidence| !(0.0..=1.0).contains(&confidence)) + { + return Err(Error::InvalidInput( + "learning confidence must be between 0 and 1".to_string(), + )); + } + let session = self.lane_session(&input.session_id)?; + validate_optional_turn_for_session(self, input.turn_id.as_deref(), &input.session_id)?; + if let Some(artifact_id) = input.source_artifact_id.as_deref() { + let artifact = self.lane_artifact(artifact_id)?; + if artifact.session_id != input.session_id { + return Err(Error::InvalidInput(format!( + "learning artifact `{artifact_id}` does not belong to session `{}`", + input.session_id + ))); + } + } + let body = redact_sensitive_text(&input.body); + let anchor_json = input + .anchor + .as_ref() + .map(canonical_json_string) + .transpose()?; + let metadata_json = input + .metadata + .as_ref() + .map(canonical_json_string) + .transpose()?; + let identity = canonical_json_bytes(&serde_json::json!({ + "session_id": input.session_id, + "turn_id": input.turn_id, + "scope": input.scope, + "body": body, + "source_artifact_id": input.source_artifact_id, + "anchor": input.anchor, + }))?; + let learning_id = format!("learning_{}", crate::ids::short_hash(&identity, 24)); + let now = now_millis(); + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "INSERT INTO lane_learnings + (learning_id, lane_id, session_id, turn_id, scope, body, status, + confidence, source_artifact_id, anchor_json, created_at, reviewed_at, + reviewer, expires_at, superseded_by, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'proposed', ?7, ?8, ?9, ?10, + NULL, NULL, ?11, NULL, ?12) + ON CONFLICT(learning_id) DO NOTHING", + params![ + learning_id, + session.lane_id, + input.session_id, + input.turn_id, + input.scope, + body, + input.confidence, + input.source_artifact_id, + anchor_json, + now, + input.expires_at, + metadata_json, + ], + )?; + self.learning(&learning_id) + } + + pub fn review_learning( + &mut self, + learning_id: &str, + accept: bool, + reviewer: &str, + ) -> Result { + validate_evidence_text("learning reviewer", reviewer, 256)?; + let existing = self.learning(learning_id)?; + if !matches!( + existing.status.as_str(), + "proposed" | "accepted" | "rejected" + ) { + return Err(Error::Conflict(format!( + "learning `{learning_id}` cannot be reviewed from status `{}`", + existing.status + ))); + } + let status = if accept { "accepted" } else { "rejected" }; + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "UPDATE lane_learnings SET status = ?2, reviewed_at = ?3, reviewer = ?4 + WHERE learning_id = ?1", + params![learning_id, status, now_millis(), reviewer], + )?; + self.learning(learning_id) + } + + pub fn learning(&self, learning_id: &str) -> Result { + self.conn + .query_row(LEARNING_SELECT_BY_ID, params![learning_id], learning_row) + .optional()? + .ok_or_else(|| Error::ObjectNotFound { + kind: "learning", + id: learning_id.to_string(), + }) + } + + pub fn list_learnings( + &self, + session_id: Option<&str>, + status: Option<&str>, + limit: usize, + ) -> Result> { + self.list_learnings_page(session_id, status, 0, limit) + } + + pub fn list_learnings_page( + &self, + session_id: Option<&str>, + status: Option<&str>, + offset: usize, + limit: usize, + ) -> Result> { + if let Some(session_id) = session_id { + self.lane_session(session_id)?; + } + if let Some(status) = status { + if !matches!( + status, + "proposed" | "accepted" | "rejected" | "expired" | "superseded" + ) { + return Err(Error::InvalidInput(format!( + "unknown learning status `{status}`" + ))); + } + } + let mut statement = self.conn.prepare( + "SELECT learning_id, lane_id, session_id, turn_id, scope, body, status, + confidence, source_artifact_id, anchor_json, created_at, reviewed_at, + reviewer, expires_at, superseded_by, metadata_json + FROM lane_learnings + WHERE (?1 IS NULL OR session_id = ?1) AND (?2 IS NULL OR status = ?2) + ORDER BY created_at DESC, learning_id DESC LIMIT ?3 OFFSET ?4", + )?; + let learnings = statement + .query_map( + params![ + session_id, + status, + i64::try_from(limit.clamp(1, 1_000)).unwrap_or(1_000), + i64::try_from(offset.min(1_000_000)).unwrap_or(1_000_000) + ], + learning_row, + )? + .collect::, _>>()?; + Ok(learnings) + } + + pub fn supersede_learning( + &mut self, + learning_id: &str, + superseded_by: &str, + reviewer: &str, + ) -> Result { + validate_evidence_text("learning reviewer", reviewer, 256)?; + let existing = self.learning(learning_id)?; + let replacement = self.learning(superseded_by)?; + if existing.lane_id != replacement.lane_id || existing.scope != replacement.scope { + return Err(Error::InvalidInput( + "a learning may only be superseded by one in the same lane and scope".to_string(), + )); + } + if learning_id == superseded_by { + return Err(Error::InvalidInput( + "a learning cannot supersede itself".to_string(), + )); + } + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "UPDATE lane_learnings + SET status = 'superseded', superseded_by = ?2, reviewed_at = ?3, reviewer = ?4 + WHERE learning_id = ?1", + params![learning_id, superseded_by, now_millis(), reviewer], + )?; + self.learning(learning_id) + } + + pub fn expire_learnings(&mut self, at_millis: i64) -> Result { + if at_millis < 0 { + return Err(Error::InvalidInput( + "learning expiry timestamp must be non-negative".to_string(), + )); + } + let _lock = self.acquire_write_lock()?; + self.conn + .execute( + "UPDATE lane_learnings SET status = 'expired', reviewed_at = ?1 + WHERE expires_at IS NOT NULL AND expires_at <= ?1 + AND status IN ('proposed', 'accepted')", + params![at_millis], + ) + .map_err(Error::from) + } + + /// Resolve explicitly requested, reviewed learning context under a strict byte bound. + pub fn accepted_learning_context( + &self, + lane: &str, + scopes: &[String], + max_bytes: usize, + ) -> Result> { + if scopes.is_empty() || max_bytes == 0 || max_bytes > 1024 * 1024 { + return Err(Error::InvalidInput( + "accepted learning context requires scopes and a 1..=1048576 byte bound" + .to_string(), + )); + } + let branch = self.lane_branch(lane)?; + let now = now_millis(); + let mut statement = self.conn.prepare( + "SELECT learning_id, lane_id, session_id, turn_id, scope, body, status, + confidence, source_artifact_id, anchor_json, created_at, reviewed_at, + reviewer, expires_at, superseded_by, metadata_json + FROM lane_learnings + WHERE lane_id = ?1 AND status = 'accepted' + AND (expires_at IS NULL OR expires_at > ?2) + ORDER BY confidence DESC, created_at DESC, learning_id", + )?; + let candidates = statement + .query_map(params![branch.lane_id, now], learning_row)? + .collect::, _>>()?; + let allowed = scopes + .iter() + .map(String::as_str) + .collect::>(); + let mut total = 0usize; + let mut selected = Vec::new(); + for learning in candidates { + if !allowed.contains(learning.scope.as_str()) { + continue; + } + let size = learning.body.len(); + if total.saturating_add(size) > max_bytes { + continue; + } + total += size; + selected.push(learning); + } + Ok(selected) + } + + pub fn link_git_commit_to_agent(&mut self, input: GitAgentLinkInput) -> Result { + if !matches!(input.git_commit.len(), 40 | 64) + || !input + .git_commit + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + return Err(Error::InvalidInput( + "git agent link commit must be a full 40- or 64-character hexadecimal object id" + .to_string(), + )); + } + validate_evidence_text("git link confidence", &input.confidence, 128)?; + validate_evidence_text("git link source", &input.source, 128)?; + let session = self.lane_session(&input.session_id)?; + validate_optional_turn_for_session(self, input.turn_id.as_deref(), &input.session_id)?; + for change in [ + input.from_change.as_deref(), + input.through_change.as_deref(), + ] + .into_iter() + .flatten() + { + self.operation(&ChangeId(change.to_string()))?; + } + if input.from_change.is_none() && input.through_change.is_none() { + return Err(Error::InvalidInput( + "git agent link requires an exact from_change or through_change identity" + .to_string(), + )); + } + let metadata_json = input + .metadata + .as_ref() + .map(canonical_json_string) + .transpose()?; + let identity = format!( + "{}:{}:{}:{}", + input.git_commit.to_ascii_lowercase(), + input.session_id, + input.turn_id.as_deref().unwrap_or(""), + input.source + ); + let link_id = format!( + "git_agent_link_{}", + crate::ids::short_hash(identity.as_bytes(), 24) + ); + let _lock = self.acquire_write_lock()?; + self.conn.execute( + "INSERT INTO git_agent_links + (git_agent_link_id, git_commit, lane_id, session_id, turn_id, + from_change, through_change, confidence, source, created_at, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + ON CONFLICT(git_commit, session_id, COALESCE(turn_id, ''), source) DO NOTHING", + params![ + link_id, + input.git_commit.to_ascii_lowercase(), + session.lane_id, + input.session_id, + input.turn_id, + input.from_change, + input.through_change, + input.confidence, + input.source, + now_millis(), + metadata_json, + ], + )?; + self.git_agent_link(&link_id).or_else(|_| { + self.conn + .query_row( + "SELECT git_agent_link_id, git_commit, lane_id, session_id, turn_id, + from_change, through_change, confidence, source, created_at, metadata_json + FROM git_agent_links WHERE git_commit = ?1 AND session_id = ?2 + AND COALESCE(turn_id, '') = COALESCE(?3, '') AND source = ?4", + params![ + input.git_commit.to_ascii_lowercase(), + input.session_id, + input.turn_id, + input.source + ], + git_agent_link_row, + ) + .map_err(Error::from) + }) + } + + pub fn git_agent_link(&self, link_id: &str) -> Result { + self.conn + .query_row( + GIT_AGENT_LINK_SELECT_BY_ID, + params![link_id], + git_agent_link_row, + ) + .optional()? + .ok_or_else(|| Error::ObjectNotFound { + kind: "git agent link", + id: link_id.to_string(), + }) + } + + pub fn list_git_agent_links(&self, session_id: &str) -> Result> { + self.list_git_agent_links_page(session_id, 0, 1_000) + } + + pub fn list_git_agent_links_page( + &self, + session_id: &str, + offset: usize, + limit: usize, + ) -> Result> { + self.lane_session(session_id)?; + let mut statement = self.conn.prepare( + "SELECT git_agent_link_id, git_commit, lane_id, session_id, turn_id, + from_change, through_change, confidence, source, created_at, metadata_json + FROM git_agent_links WHERE session_id = ?1 + ORDER BY created_at, git_agent_link_id LIMIT ?2 OFFSET ?3", + )?; + let links = statement + .query_map( + params![ + session_id, + i64::try_from(limit.clamp(1, 1_000)).unwrap_or(1_000), + i64::try_from(offset.min(1_000_000)).unwrap_or(1_000_000) + ], + git_agent_link_row, + )? + .collect::, _>>()?; + Ok(links) + } + + pub fn export_agent_trace( + &self, + session_id: &str, + include_attachments: bool, + ) -> Result { + let session = self.lane_session(session_id)?; + ensure_export_bound( + &self.conn, + "lane_turn_evidence_manifests", + session_id, + 1_000, + )?; + ensure_export_bound(&self.conn, "lane_artifacts", session_id, 1_000)?; + ensure_export_bound(&self.conn, "lane_provenance_nodes", session_id, 10_000)?; + ensure_export_bound(&self.conn, "lane_provenance_edges", session_id, 10_000)?; + ensure_export_bound(&self.conn, "lane_session_attestations", session_id, 1_000)?; + ensure_export_bound(&self.conn, "lane_learnings", session_id, 1_000)?; + ensure_export_bound(&self.conn, "git_agent_links", session_id, 1_000)?; + + let mut evidence_manifests = self.list_turn_evidence_manifests(session_id, 1_000)?; + evidence_manifests.sort_by(|left, right| left.turn_id.cmp(&right.turn_id)); + let mut artifacts = self + .list_lane_artifacts(session_id, None, 1_000)? + .into_iter() + .map(|artifact| { + let content = include_attachments + .then(|| self.lane_artifact_content(&artifact.artifact_id)) + .transpose()?; + Ok(PortableAgentArtifact { + artifact_id: artifact.artifact_id, + provider: artifact.provider, + artifact_kind: artifact.artifact_kind, + format: artifact.format, + source: artifact.source, + content_digest: artifact.content_digest, + size_bytes: artifact.size_bytes, + start_offset: artifact.start_offset, + end_offset: artifact.end_offset, + trust: artifact.trust, + retention_status: artifact.retention_status, + content, + }) + }) + .collect::>>()?; + artifacts.sort_by(|left, right| left.artifact_id.cmp(&right.artifact_id)); + let (mut provenance_nodes, mut provenance_edges) = + self.list_session_provenance(session_id, 10_000)?; + provenance_nodes + .sort_by(|left, right| left.provenance_node_id.cmp(&right.provenance_node_id)); + provenance_edges + .sort_by(|left, right| left.provenance_edge_id.cmp(&right.provenance_edge_id)); + let mut attestations = self.list_session_attestations(session_id)?; + attestations.sort_by(|left, right| left.attestation_id.cmp(&right.attestation_id)); + let mut learnings = self.list_learnings(Some(session_id), None, 1_000)?; + learnings.sort_by(|left, right| left.learning_id.cmp(&right.learning_id)); + let mut git_links = self.list_git_agent_links(session_id)?; + git_links.sort_by(|left, right| left.git_agent_link_id.cmp(&right.git_agent_link_id)); + let trace = PortableAgentTrace { + schema: AGENT_TRACE_SCHEMA.to_string(), + version: AGENT_TRACE_VERSION, + source_workspace_id: self.config.workspace.id.0.clone(), + lane_id: session.lane_id, + session_id: session_id.to_string(), + session_status: session.status, + evidence_manifests, + artifacts, + provenance_nodes, + provenance_edges, + attestations, + learnings, + git_links, + }; + let verification = trace.verify(); + if !verification.valid { + return Err(Error::Corrupt(format!( + "generated portable trace failed verification: {}", + verification.diagnostics.join("; ") + ))); + } + Ok(trace) + } +} + +struct EvidenceManifestRow { + manifest_id: String, + lane_id: String, + session_id: String, + turn_id: String, + schema_version: u16, + object_id: ObjectId, + digest: String, + created_at: i64, +} + +fn classify_event_activity(event_type: &str) -> Option<(&'static str, &'static str)> { + if event_type.starts_with("tool.") || event_type.starts_with("tool_") { + Some(("tool_activity", "Agent used a tool")) + } else if event_type.starts_with("workspace.") || event_type.contains("checkpoint") { + Some(( + "workspace_activity", + "Agent changed or checkpointed workspace state", + )) + } else if event_type.starts_with("plan.") || event_type.contains("plan_update") { + Some(("plan_activity", "Agent updated its explicit plan")) + } else if event_type.starts_with("approval.") || event_type.contains("permission") { + Some(( + "approval_activity", + "Agent participated in an approval decision", + )) + } else if event_type.starts_with("message.") || event_type.contains("message_flushed") { + Some(( + "communication_activity", + "Agent exchanged a captured message", + )) + } else { + None + } +} + +const PROVENANCE_NODE_SELECT_BY_ID: &str = + "SELECT provenance_node_id, lane_id, session_id, turn_id, node_kind, + summary, event_id, span_id, message_id, change_id, artifact_id, + source_confidence, classifier_version, created_at, attributes_json + FROM lane_provenance_nodes WHERE provenance_node_id = ?1"; + +fn provenance_node_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(ProvenanceNode { + provenance_node_id: row.get(0)?, + lane_id: row.get(1)?, + session_id: row.get(2)?, + turn_id: row.get(3)?, + node_kind: row.get(4)?, + summary: row.get(5)?, + event_id: row.get(6)?, + span_id: row.get(7)?, + message_id: row.get(8)?, + change_id: row.get(9)?, + artifact_id: row.get(10)?, + source_confidence: row.get(11)?, + classifier_version: row.get(12)?, + created_at: row.get(13)?, + attributes: parse_optional_json_column(row, 14)?, + }) +} + +fn provenance_edge_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(ProvenanceEdge { + provenance_edge_id: row.get(0)?, + lane_id: row.get(1)?, + session_id: row.get(2)?, + from_node_id: row.get(3)?, + to_node_id: row.get(4)?, + relation: row.get(5)?, + source_confidence: row.get(6)?, + receipt_id: row.get(7)?, + created_at: row.get(8)?, + attributes: parse_optional_json_column(row, 9)?, + }) +} + +const SESSION_ATTESTATION_SELECT_BY_ID: &str = + "SELECT attestation_id, lane_id, session_id, capture_run_id, + previous_attestation_id, statement_object_id, statement_digest, + signature_json, status, created_at, superseded_by, metadata_json + FROM lane_session_attestations WHERE attestation_id = ?1"; + +struct SessionAttestationRow { + attestation_id: String, + lane_id: String, + session_id: String, + capture_run_id: Option, + previous_attestation_id: Option, + statement_object_id: ObjectId, + statement_digest: String, + signature: Option, + status: String, + created_at: i64, + superseded_by: Option, + metadata: Option, +} + +fn session_attestation_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(SessionAttestationRow { + attestation_id: row.get(0)?, + lane_id: row.get(1)?, + session_id: row.get(2)?, + capture_run_id: row.get(3)?, + previous_attestation_id: row.get(4)?, + statement_object_id: ObjectId(row.get(5)?), + statement_digest: row.get(6)?, + signature: parse_optional_json_column(row, 7)?, + status: row.get(8)?, + created_at: row.get(9)?, + superseded_by: row.get(10)?, + metadata: parse_optional_json_column(row, 11)?, + }) +} + +const LEARNING_SELECT_BY_ID: &str = + "SELECT learning_id, lane_id, session_id, turn_id, scope, body, status, + confidence, source_artifact_id, anchor_json, created_at, reviewed_at, + reviewer, expires_at, superseded_by, metadata_json + FROM lane_learnings WHERE learning_id = ?1"; + +fn learning_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(Learning { + learning_id: row.get(0)?, + lane_id: row.get(1)?, + session_id: row.get(2)?, + turn_id: row.get(3)?, + scope: row.get(4)?, + body: row.get(5)?, + status: row.get(6)?, + confidence: row.get(7)?, + source_artifact_id: row.get(8)?, + anchor: parse_optional_json_column(row, 9)?, + created_at: row.get(10)?, + reviewed_at: row.get(11)?, + reviewer: row.get(12)?, + expires_at: row.get(13)?, + superseded_by: row.get(14)?, + metadata: parse_optional_json_column(row, 15)?, + }) +} + +const GIT_AGENT_LINK_SELECT_BY_ID: &str = + "SELECT git_agent_link_id, git_commit, lane_id, session_id, turn_id, + from_change, through_change, confidence, source, created_at, metadata_json + FROM git_agent_links WHERE git_agent_link_id = ?1"; + +fn git_agent_link_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(GitAgentLink { + git_agent_link_id: row.get(0)?, + git_commit: row.get(1)?, + lane_id: row.get(2)?, + session_id: row.get(3)?, + turn_id: row.get(4)?, + from_change: row.get(5)?, + through_change: row.get(6)?, + confidence: row.get(7)?, + source: row.get(8)?, + created_at: row.get(9)?, + metadata: parse_optional_json_column(row, 10)?, + }) +} + +fn attestation_key_revocation_row( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + Ok(AttestationKeyRevocation { + key_id: row.get(0)?, + public_key_hex: row.get(1)?, + reason: row.get(2)?, + revoked_at: row.get(3)?, + metadata: parse_optional_json_column(row, 4)?, + }) +} + +fn parse_optional_json_column( + row: &rusqlite::Row<'_>, + index: usize, +) -> rusqlite::Result> { + row.get::<_, Option>(index)? + .map(|value| { + serde_json::from_str(&value).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + index, + rusqlite::types::Type::Text, + Box::new(error), + ) + }) + }) + .transpose() +} + +fn validate_optional_turn_for_session( + db: &Trail, + turn_id: Option<&str>, + session_id: &str, +) -> Result<()> { + if let Some(turn_id) = turn_id { + let turn = db.lane_turn(turn_id)?; + if turn.session_id.as_deref() != Some(session_id) { + return Err(Error::InvalidInput(format!( + "turn `{turn_id}` does not belong to session `{session_id}`" + ))); + } + } + Ok(()) +} + +fn validate_evidence_text(name: &str, value: &str, max_bytes: usize) -> Result<()> { + if value.trim().is_empty() || value.len() > max_bytes || value.chars().any(char::is_control) { + return Err(Error::InvalidInput(format!( + "{name} must contain 1..={max_bytes} non-control bytes" + ))); + } + Ok(()) +} + +fn canonical_json_string(value: &serde_json::Value) -> Result { + serde_json::to_string(value).map_err(Error::from) +} + +fn attestation_key_id(public_key: &[u8; 32]) -> String { + let digest = Sha256::digest(public_key); + format!("ed25519_{}", hex::encode(&digest[..16])) +} + +fn decode_fixed_hex(value: &str, name: &str) -> Result<[u8; N]> { + let bytes = hex::decode(value) + .map_err(|error| Error::InvalidInput(format!("invalid {name} hex: {error}")))?; + bytes.try_into().map_err(|bytes: Vec| { + Error::InvalidInput(format!( + "invalid {name} length {}; expected {N} bytes", + bytes.len() + )) + }) +} + +fn ensure_export_bound( + conn: &rusqlite::Connection, + table: &str, + session_id: &str, + maximum: i64, +) -> Result<()> { + let count: i64 = conn.query_row( + &format!("SELECT COUNT(*) FROM {table} WHERE session_id = ?1"), + params![session_id], + |row| row.get(0), + )?; + if count > maximum { + return Err(Error::InvalidInput(format!( + "session `{session_id}` has {count} rows in {table}; bounded export maximum is {maximum}" + ))); + } + Ok(()) +} + +fn evidence_manifest_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + let raw_version: i64 = row.get(4)?; + let schema_version = u16::try_from(raw_version).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 4, + rusqlite::types::Type::Integer, + Box::new(error), + ) + })?; + Ok(EvidenceManifestRow { + manifest_id: row.get(0)?, + lane_id: row.get(1)?, + session_id: row.get(2)?, + turn_id: row.get(3)?, + schema_version, + object_id: ObjectId(row.get(5)?), + digest: row.get(6)?, + created_at: row.get(7)?, + }) +} + +fn collect_json_strings(value: &serde_json::Value, key: &str, output: &mut Vec) { + match value { + serde_json::Value::Object(object) => { + if let Some(value) = object.get(key).and_then(serde_json::Value::as_str) { + output.push(value.to_string()); + } + for value in object.values() { + collect_json_strings(value, key, output); + } + } + serde_json::Value::Array(values) => { + for value in values { + collect_json_strings(value, key, output); + } + } + _ => {} + } +} + +fn sort_dedup_coverage(coverage: &mut TurnEvidenceCoverage) { + coverage.receipt_ids.sort(); + coverage.receipt_ids.dedup(); + coverage.event_ids.sort(); + coverage.event_ids.dedup(); + coverage.message_ids.sort(); + coverage.message_ids.dedup(); + coverage.artifact_ids.sort(); + coverage.artifact_ids.dedup(); + coverage + .change_ids + .sort_by(|left, right| left.0.cmp(&right.0)); + coverage.change_ids.dedup(); + coverage.tool_span_ids.sort(); + coverage.tool_span_ids.dedup(); + coverage.approval_ids.sort(); + coverage.approval_ids.dedup(); +} + +fn canonical_json_bytes(value: &T) -> Result> { + serde_json::to_vec(value).map_err(Error::from) +} + +fn digest_bytes(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + format!("sha256:{}", hex::encode(digest)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn completed_turn_evidence_manifest_is_exact_content_addressed_and_idempotent() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("evidence", None, false, Some("codex".to_string()), None) + .unwrap(); + let session = db + .start_lane_session("evidence", Some("evidence".to_string()), None) + .unwrap() + .session; + let turn = db + .begin_lane_session_turn("evidence", &session.session_id, None) + .unwrap() + .turn; + let message = db + .add_lane_turn_message(&turn.turn_id, "user", "make it exact") + .unwrap(); + db.add_lane_turn_event( + &turn.turn_id, + "tool.completed", + Some(serde_json::json!({ + "event_id":"normalized-event-1", + "evidence":{"receipt_id":"receipt-1"}, + "span_id":"span-1" + })), + None, + Some(&message.message_id.0), + ) + .unwrap(); + let artifact = db + .record_lane_artifact(LaneArtifactInput { + lane: "evidence".to_string(), + session_id: session.session_id.clone(), + turn_id: Some(turn.turn_id.clone()), + provider: "codex".to_string(), + artifact_kind: "native_transcript".to_string(), + format: "jsonl".to_string(), + source: AgentEvidenceSource::NativeTranscript, + source_locator_redacted: None, + content: b"transcript".to_vec(), + start_offset: Some(0), + end_offset: Some(10), + redaction_profile: None, + trust: "provider-native".to_string(), + supersedes_artifact_id: None, + metadata_json: None, + }) + .unwrap(); + db.end_lane_turn(&turn.turn_id, "completed").unwrap(); + + let first = db.create_turn_evidence_manifest(&turn.turn_id).unwrap(); + let second = db.create_turn_evidence_manifest(&turn.turn_id).unwrap(); + assert_eq!(first, second); + assert_eq!(first.statement.coverage.receipt_ids, vec!["receipt-1"]); + assert!(first + .statement + .coverage + .event_ids + .contains(&"normalized-event-1".to_string())); + assert_eq!( + first.statement.coverage.message_ids, + vec![message.message_id.0.clone()] + ); + assert_eq!( + first.statement.coverage.artifact_ids, + vec![artifact.artifact_id.clone()] + ); + assert_eq!(first.statement.coverage.tool_span_ids, vec!["span-1"]); + assert!(first.digest.starts_with("sha256:")); + + let prompt_node_input = ProvenanceNodeInput { + session_id: session.session_id.clone(), + turn_id: Some(turn.turn_id.clone()), + node_kind: "user_message".to_string(), + summary: "user requested exact evidence".to_string(), + event_id: None, + span_id: None, + message_id: Some(message.message_id.0.clone()), + change_id: None, + artifact_id: None, + source_confidence: "native-structured".to_string(), + classifier_version: None, + attributes: Some(serde_json::json!({"role":"user"})), + }; + let prompt_node = db + .create_provenance_node(prompt_node_input.clone()) + .unwrap(); + assert_eq!( + db.create_provenance_node(prompt_node_input) + .unwrap() + .provenance_node_id, + prompt_node.provenance_node_id + ); + let tool_node = db + .create_provenance_node(ProvenanceNodeInput { + session_id: session.session_id.clone(), + turn_id: Some(turn.turn_id.clone()), + node_kind: "tool_result".to_string(), + summary: "tool completed".to_string(), + event_id: Some("normalized-event-1".to_string()), + span_id: Some("span-1".to_string()), + message_id: None, + change_id: None, + artifact_id: Some(artifact.artifact_id), + source_confidence: "native-structured".to_string(), + classifier_version: None, + attributes: None, + }) + .unwrap(); + let edge_input = ProvenanceEdgeInput { + from_node_id: prompt_node.provenance_node_id.clone(), + to_node_id: tool_node.provenance_node_id.clone(), + relation: "caused".to_string(), + source_confidence: "observed-order".to_string(), + receipt_id: None, + attributes: None, + }; + let edge = db.create_provenance_edge(edge_input.clone()).unwrap(); + assert_eq!( + db.create_provenance_edge(edge_input) + .unwrap() + .provenance_edge_id, + edge.provenance_edge_id + ); + let (nodes, edges) = db + .list_session_provenance(&session.session_id, 100) + .unwrap(); + assert_eq!(nodes.len(), 2); + assert_eq!(edges, vec![edge]); + + let classification = db + .classify_session_activity(&session.session_id, 100) + .unwrap(); + assert_eq!(classification.classifier_version, "trail-activity-rules/v1"); + assert_eq!(classification.nodes.len(), 2); + assert_eq!(classification.edges.len(), 1); + assert!(classification + .nodes + .iter() + .any(|node| node.node_kind == "tool_activity" + && node.source_confidence == "deterministic-derived")); + assert_eq!( + db.classify_session_activity(&session.session_id, 100) + .unwrap() + .edges[0] + .provenance_edge_id, + classification.edges[0].provenance_edge_id + ); + + let first_attestation = db + .create_session_attestation( + &session.session_id, + "on-end", + Some(serde_json::json!({"test":true})), + ) + .unwrap(); + assert_eq!(first_attestation.turns.len(), 1); + assert_eq!( + db.create_session_attestation(&session.session_id, "on-end", None) + .unwrap() + .attestation_id, + first_attestation.attestation_id + ); + assert!( + db.verify_session_attestation(&first_attestation.attestation_id) + .unwrap() + .valid + ); + + let second_turn = db + .begin_lane_session_turn("evidence", &session.session_id, None) + .unwrap() + .turn; + db.add_lane_turn_message(&second_turn.turn_id, "user", "second turn") + .unwrap(); + db.end_lane_turn(&second_turn.turn_id, "completed").unwrap(); + let second_attestation = db + .create_session_attestation(&session.session_id, "on-end", None) + .unwrap(); + assert_eq!( + second_attestation.previous_attestation_id.as_deref(), + Some(first_attestation.attestation_id.as_str()) + ); + assert_eq!(second_attestation.turns.len(), 1); + assert_eq!( + second_attestation.turns[0].turn_id, + second_turn.turn_id.clone() + ); + assert!( + db.verify_session_attestation(&second_attestation.attestation_id) + .unwrap() + .valid + ); + + let learning = db + .propose_learning(LearningInput { + session_id: session.session_id.clone(), + turn_id: Some(second_turn.turn_id.clone()), + scope: "workspace".to_string(), + body: "api_token=secret".to_string(), + confidence: Some(0.8), + source_artifact_id: None, + anchor: Some(serde_json::json!({"turn":second_turn.turn_id})), + expires_at: None, + metadata: None, + }) + .unwrap(); + assert_eq!(learning.body, "api_token=[REDACTED]"); + let accepted = db + .review_learning(&learning.learning_id, true, "reviewer@example") + .unwrap(); + assert_eq!(accepted.status, "accepted"); + assert_eq!( + db.list_learnings(Some(&session.session_id), Some("accepted"), 10) + .unwrap(), + vec![accepted.clone()] + ); + assert_eq!( + db.accepted_learning_context("evidence", &["workspace".to_string()], 1_024) + .unwrap(), + vec![accepted.clone()] + ); + let replacement = db + .propose_learning(LearningInput { + body: "use the verified replacement".to_string(), + confidence: Some(0.9), + anchor: Some(serde_json::json!({"supersedes":learning.learning_id})), + ..LearningInput { + session_id: session.session_id.clone(), + turn_id: Some(second_turn.turn_id.clone()), + scope: "workspace".to_string(), + body: String::new(), + confidence: None, + source_artifact_id: None, + anchor: None, + expires_at: None, + metadata: None, + } + }) + .unwrap(); + let replacement = db + .review_learning(&replacement.learning_id, true, "reviewer@example") + .unwrap(); + assert_eq!( + db.supersede_learning( + &learning.learning_id, + &replacement.learning_id, + "reviewer@example" + ) + .unwrap() + .status, + "superseded" + ); + + let link_input = GitAgentLinkInput { + session_id: session.session_id.clone(), + turn_id: Some(second_turn.turn_id.clone()), + git_commit: "a".repeat(40), + from_change: Some(second_turn.before_change.0.clone()), + through_change: Some(second_turn.before_change.0.clone()), + confidence: "exact-change".to_string(), + source: "trail-export".to_string(), + metadata: None, + }; + let link = db.link_git_commit_to_agent(link_input.clone()).unwrap(); + assert_eq!( + db.link_git_commit_to_agent(link_input) + .unwrap() + .git_agent_link_id, + link.git_agent_link_id + ); + assert_eq!( + db.list_git_agent_links(&session.session_id).unwrap(), + vec![link] + ); + + let secret = [7_u8; 32]; + let signed = db + .sign_session_attestation(&second_attestation.attestation_id, &secret) + .unwrap(); + assert_eq!(signed.status, "signed"); + let signed_verification = db + .verify_session_attestation(&signed.attestation_id) + .unwrap(); + assert!(signed_verification.valid); + assert_eq!(signed_verification.signature_status, "valid"); + let public = SigningKey::from_bytes(&secret).verifying_key().to_bytes(); + let revocation = db + .revoke_attestation_key(&public, "test rotation", None) + .unwrap(); + assert!(revocation.key_id.starts_with("ed25519_")); + let revoked_verification = db + .verify_session_attestation(&signed.attestation_id) + .unwrap(); + assert!(!revoked_verification.valid); + assert_eq!(revoked_verification.signature_status, "revoked"); + + let trace = db.export_agent_trace(&session.session_id, true).unwrap(); + assert!(trace.verify().valid); + let bytes = trace.to_canonical_json().unwrap(); + let imported = PortableAgentTrace::from_json(&bytes).unwrap(); + assert_eq!(imported.to_canonical_json().unwrap(), bytes); + let mut tampered = imported; + tampered.artifacts[0].content.as_mut().unwrap()[0] ^= 1; + assert!(!tampered.verify().artifact_digests_valid); + + let manifest_digest = first.digest.clone(); + let artifact_id = trace.artifacts[0].artifact_id.clone(); + let redacted = db + .redact_lane_artifact(&artifact_id, "retention policy") + .unwrap(); + assert_eq!(redacted.retention_status, "redacted"); + assert!(redacted.content_object_id.is_none()); + assert!(db.lane_artifact_content(&artifact_id).is_err()); + assert_eq!( + db.turn_evidence_manifest(&turn.turn_id).unwrap().digest, + manifest_digest + ); + } + + #[test] + fn open_turn_cannot_freeze_an_evidence_manifest() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("open-evidence", None, false, None, None) + .unwrap(); + let turn = db + .begin_lane_turn("open-evidence", None, None, None) + .unwrap() + .turn; + assert!(db.create_turn_evidence_manifest(&turn.turn_id).is_err()); + } +} diff --git a/trail/src/db/lane/control/sessions.rs b/trail/src/db/lane/control/sessions.rs index ab18ddf..1d023fc 100644 --- a/trail/src/db/lane/control/sessions.rs +++ b/trail/src/db/lane/control/sessions.rs @@ -2,6 +2,21 @@ use super::acp_sessions::lane_acp_session_row; use super::*; impl Trail { + pub fn update_lane_session_title( + &mut self, + session_id: &str, + title: Option<&str>, + ) -> Result { + let _lock = self.acquire_write_lock()?; + validate_session_id(session_id)?; + self.lane_session(session_id)?; + self.conn.execute( + "UPDATE lane_sessions SET title = ?1 WHERE session_id = ?2", + params![title, session_id], + )?; + self.lane_session(session_id) + } + pub fn start_lane_session( &mut self, lane: &str, @@ -168,11 +183,8 @@ impl Trail { for turn in details.turns { let turn_details = self.show_lane_turn(&turn.turn_id)?; let turn_envelope = turn_details.turn_envelope; - let checkpoint = if turn_envelope - .as_ref() - .is_some_and(|envelope| envelope.outcome.no_changes) - { - None + let checkpoint = if let Some(envelope) = turn_envelope.as_ref() { + envelope.outcome.checkpoint.clone() } else { turn_details.turn.after_change.clone().or_else(|| { turn_details @@ -284,7 +296,7 @@ impl Trail { fn acp_session_for_session(&self, session_id: &str) -> Result> { let mut stmt = self.conn.prepare( - "SELECT acp_session_id, upstream_session_id, lane_id, trail_session_id, cwd, provider, model, upstream_command_json, status, created_at, updated_at \ + "SELECT acp_session_id, upstream_session_id, lane_id, trail_session_id, cwd, path_mappings_json, provider, model, upstream_command_json, status, created_at, updated_at, current_mode_id, config_options_json \ FROM lane_acp_sessions WHERE trail_session_id = ?1 ORDER BY updated_at DESC, acp_session_id DESC LIMIT 1", )?; stmt.query_row(params![session_id], lane_acp_session_row) diff --git a/trail/src/db/lane/control/turn_setup.rs b/trail/src/db/lane/control/turn_setup.rs index bfa455a..6367b2e 100644 --- a/trail/src/db/lane/control/turn_setup.rs +++ b/trail/src/db/lane/control/turn_setup.rs @@ -1,3 +1,4 @@ +use super::super::workdir::MaterializationPolicy; use super::*; impl Trail { @@ -20,6 +21,7 @@ impl Trail { from: Option<&str>, base_change: Option<&str>, ) -> Result { + // TRAIL_FS_PRODUCER: turn_lane_spawn Materialize controlled let source_selector = match base_change.or(from) { Some(selector) => selector.to_string(), None => self.current_branch()?, @@ -30,59 +32,220 @@ impl Trail { if self.try_get_ref(&ref_name)?.is_some() { return Err(Error::InvalidInput(format!("lane `{lane}` already exists"))); } + let mut materialization = None; let workdir = if self.default_lane_materialize_for_ref(Some(&source_selector))? { - let dir = self.materialize_lane_workdir(lane, &source.root_id, None)?; + let dir = self.resolve_lane_workdir_path(lane, None)?; + let outcome = self.materialize_lane_root_staged( + &source.root_id, + &dir, + false, + MaterializationPolicy::Auto, + )?; + materialization = Some(outcome); Some(dir.to_string_lossy().to_string()) } else { None }; - self.set_ref( - &ref_name, - &source.change_id, - &source.root_id, - &source.operation_id, - )?; + let metadata_json = materialization + .as_ref() + .map(|outcome| { + serde_json::to_string(&serde_json::json!({ + "requested_workdir_mode": "auto", + "workdir_mode": outcome.resolved_mode.as_str(), + "workdir_backend": outcome.backend.as_str(), + "materialization": outcome.report, + "sparse_paths": [], + "include_neighbors": false, + "transparent_cow_available": false + })) + }) + .transpose()?; let now = now_ts(); - self.conn.execute( - "INSERT INTO lanes (lane_id, name, kind, provider, model, created_at, metadata_json) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", - params![ - lane_id, - lane, - "coding-lane", - Option::::None, - Option::::None, - now, - Option::::None - ], + self.conn.execute_batch("BEGIN IMMEDIATE;")?; + let association = (|| -> Result<()> { + self.insert_new_ref_database_only( + &ref_name, + &source.change_id, + &source.root_id, + &source.operation_id, + )?; + super::super::lifecycle::fail_lane_association_if_requested("turn_after_ref")?; + self.conn.execute( + "INSERT INTO lanes (lane_id, name, kind, provider, model, created_at, metadata_json) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + lane_id, + lane, + "coding-lane", + Option::::None, + Option::::None, + now, + metadata_json + ], + )?; + super::super::lifecycle::fail_lane_association_if_requested("turn_after_lane")?; + self.conn.execute( + "INSERT INTO lane_branches \ + (lane_id, ref_name, base_change, head_change, base_root, head_root, session_id, workdir, status, created_at, updated_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, NULL, ?7, 'active', ?8, ?8)", + params![ + lane_id, + ref_name, + source.change_id.0, + source.change_id.0, + source.root_id.0, + source.root_id.0, + workdir.clone(), + now + ], + )?; + super::super::lifecycle::fail_lane_association_if_requested("turn_after_branch")?; + Ok(()) + })(); + match association { + Ok(()) => self.conn.execute_batch("COMMIT;")?, + Err(error) => { + let _ = self.conn.execute_batch("ROLLBACK;"); + if let Some(outcome) = materialization.as_ref() { + self.abort_materialization_operation(&outcome.materialization_operation_id)?; + } + return Err(error); + } + } + let committed_operation = materialization + .as_ref() + .map(|outcome| outcome.materialization_operation_id.clone()) + .unwrap_or_else(|| source.operation_id.0.clone()); + super::super::lifecycle::committed_lane_step( + &committed_operation, + "turn lane ref mirror", + (|| { + super::super::lifecycle::fail_lane_association_if_requested("turn_ref_repair")?; + self.repair_new_ref_mirror( + &ref_name, + &source.change_id, + &source.root_id, + &source.operation_id, + ) + })(), )?; - self.conn.execute( - "INSERT INTO lane_branches \ - (lane_id, ref_name, base_change, head_change, base_root, head_root, session_id, workdir, status, created_at, updated_at) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, NULL, ?7, 'active', ?8, ?8)", - params![ - lane_id, - ref_name, - source.change_id.0, - source.change_id.0, - source.root_id.0, - source.root_id.0, - workdir.clone(), - now - ], + if let Some(outcome) = materialization.as_ref() { + super::super::lifecycle::committed_lane_step( + &committed_operation, + "turn lane materialization journal completion", + (|| { + super::super::lifecycle::fail_lane_association_if_requested( + "turn_journal_completion", + )?; + self.complete_materialization_operation(&outcome.materialization_operation_id) + })(), + )?; + } + super::super::lifecycle::committed_lane_step( + &committed_operation, + "turn lane post-association reconciliation", + super::super::lifecycle::fail_lane_association_if_requested("turn_after_commit"), )?; - self.insert_lane_event( - &format!("lane_{}", crate::ids::short_hash(lane.as_bytes(), 8)), - "lane_spawned", - Some(&source.change_id), - None, - &serde_json::json!({ - "ref_name": lane_ref(lane), - "base_root": source.root_id.0.clone(), - "workdir": workdir.clone(), - "source": "api" - }), + if workdir.is_some() && crate::db::change_ledger::command_authority_enabled() { + let expected = + crate::db::change_ledger::prepare_materialized_lane_controlled_projection( + self, &lane_id, + ) + .map_err(|error| Error::CommittedRepairRequired { + operation: materialization + .as_ref() + .map(|outcome| outcome.materialization_operation_id.clone()) + .unwrap_or_else(|| source.operation_id.0.clone()), + repair: "turn lane ledger reconciliation".into(), + reason: error.to_string(), + })?; + let evidence = crate::db::change_ledger::IntentEvidence { + exact_paths: Vec::new(), + complete_prefixes: Vec::new(), + }; + crate::db::change_ledger::run_projection_alignment( + self, + &expected, + crate::db::change_ledger::IntentProducer::Materialize, + &evidence, + crate::db::change_ledger::ProjectionAlignmentMode::Aligned, + |db, intent| { + crate::db::change_ledger::with_materialized_lane_controlled_interval( + db, + &lane_id, + intent, + &evidence, + |_| Ok(()), + |db, policy, candidates| { + let comparison = db.compare_controlled_projection_target( + policy, + candidates, + &source.root_id, + crate::db::change_ledger::CandidateMaterialization::ManifestOnly, + )?; + if comparison.summaries.is_empty() { + Ok(()) + } else { + Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: + "turn lane materialization did not match its target root" + .into(), + command: format!("trail lane status {lane_id}"), + }) + } + }, + ) + }, + |db| db.publish_lane_marker_if_materialized(&lane_id), + ) + .map_err(|error| Error::CommittedRepairRequired { + operation: materialization + .as_ref() + .map(|outcome| outcome.materialization_operation_id.clone()) + .unwrap_or_else(|| source.operation_id.0.clone()), + repair: "turn lane ledger alignment".into(), + reason: error.to_string(), + })?; + } else if workdir.is_some() { + super::super::lifecycle::committed_lane_step( + &committed_operation, + "turn lane marker publication", + (|| { + super::super::lifecycle::fail_lane_association_if_requested("turn_marker")?; + self.publish_lane_marker_if_materialized(lane) + })(), + )?; + } + super::super::lifecycle::committed_lane_step( + &committed_operation, + "turn lane event publication", + (|| { + super::super::lifecycle::fail_lane_association_if_requested("turn_event")?; + self.insert_lane_event( + &format!("lane_{}", crate::ids::short_hash(lane.as_bytes(), 8)), + "lane_spawned", + Some(&source.change_id), + None, + &serde_json::json!({ + "ref_name": lane_ref(lane), + "base_root": source.root_id.0.clone(), + "workdir": workdir.clone(), + "requested_workdir_mode": materialization.as_ref().map(|_| "auto"), + "workdir_mode": materialization.as_ref().map(|outcome| outcome.resolved_mode.as_str()), + "workdir_backend": materialization.as_ref().map(|outcome| outcome.backend.as_str()), + "materialization": materialization.as_ref().map(|outcome| &outcome.report), + "source": "api" + }), + ) + })(), )?; - self.lane_branch(lane) + let branch = self.lane_branch(lane); + super::super::lifecycle::committed_lane_step( + &committed_operation, + "turn lane branch readback", + branch, + ) } } diff --git a/trail/src/db/lane/gates/runner.rs b/trail/src/db/lane/gates/runner.rs index 1905a73..51767d0 100644 --- a/trail/src/db/lane/gates/runner.rs +++ b/trail/src/db/lane/gates/runner.rs @@ -100,7 +100,7 @@ impl Trail { let layer_ids = if let Some(view) = &view { self.workspace_layer_bindings_for_source_upper(Path::new(&view.source_upper))? .into_iter() - .map(|binding| binding.layer_id) + .filter_map(|binding| binding.layer_id) .collect::>() } else { Vec::new() @@ -172,8 +172,8 @@ impl Trail { ) }; - let overlay_mount = if workdir_mode == LaneWorkdirMode::OverlayCow { - Some(self.mount_overlay_cow_workdir_for_lane(lane)?) + let fuse_mount = if workdir_mode == LaneWorkdirMode::FuseCow { + Some(self.mount_fuse_cow_workdir_for_lane(lane)?) } else { None }; @@ -182,6 +182,12 @@ impl Trail { } else { None }; + #[cfg(target_os = "windows")] + let dokan_mount = if workdir_mode == LaneWorkdirMode::DokanCow { + Some(self.mount_dokan_cow_workdir_for_lane(lane)?) + } else { + None + }; let environment = view .as_ref() .map(|view| self.workspace_command_environment(view, &source_root)) @@ -194,7 +200,9 @@ impl Trail { &environment, )?; drop(nfs_mount); - drop(overlay_mount); + drop(fuse_mount); + #[cfg(target_os = "windows")] + drop(dokan_mount); let threshold_met = score .zip(threshold) .map(|(score, threshold)| score >= threshold); diff --git a/trail/src/db/lane/identity.rs b/trail/src/db/lane/identity.rs index b3366b1..fc9a52e 100644 --- a/trail/src/db/lane/identity.rs +++ b/trail/src/db/lane/identity.rs @@ -26,16 +26,70 @@ impl Trail { rows.collect::, _>>()? }; - let mut rewritten = 0; - for (lane_id, name) in rows { - let workdir = self.default_lane_workdir_path(&name)?; - self.conn.execute( - "UPDATE lane_branches SET workdir = ?1, updated_at = ?2 WHERE lane_id = ?3", - params![workdir.to_string_lossy(), now_ts(), lane_id], + self.conn.execute_batch("BEGIN IMMEDIATE;")?; + let rewrite = (|| -> Result { + let mut rewritten = 0; + for (lane_id, name) in rows { + let workdir = self.default_lane_workdir_path(&name)?; + self.conn.execute( + "UPDATE lane_branches SET workdir = ?1, updated_at = ?2 WHERE lane_id = ?3", + params![workdir.to_string_lossy(), now_ts(), lane_id], + )?; + rewritten += 1; + } + + // Backups do not contain `.trail/views`. Invalidate every view and + // its derived environment state before normal open recovery can + // inspect source-workspace absolute paths from the copied SQLite + // store. A lane can create a fresh view in the restored workspace. + self.conn.execute_batch( + "DELETE FROM environment_secret_access_audit + WHERE generation_id IN (SELECT generation_id FROM environment_generations); + DELETE FROM environment_generation_runtime_secrets + WHERE generation_id IN (SELECT generation_id FROM environment_generations); + DELETE FROM environment_generation_runtime_resources + WHERE generation_id IN (SELECT generation_id FROM environment_generations); + DELETE FROM environment_generation_external_artifacts + WHERE generation_id IN (SELECT generation_id FROM environment_generations); + DELETE FROM environment_generation_caches + WHERE generation_id IN (SELECT generation_id FROM environment_generations); + DELETE FROM environment_generation_edges + WHERE generation_id IN (SELECT generation_id FROM environment_generations); + DELETE FROM environment_generation_outputs + WHERE generation_id IN (SELECT generation_id FROM environment_generations); + DELETE FROM environment_generation_components + WHERE generation_id IN (SELECT generation_id FROM environment_generations); + DELETE FROM environment_view_generations; + DELETE FROM environment_generations; + DELETE FROM environment_sync_attempts; + DELETE FROM environment_component_runtime_secrets; + DELETE FROM environment_component_runtime_resources; + DELETE FROM environment_component_external_artifacts; + DELETE FROM environment_component_caches; + DELETE FROM environment_component_dependencies; + DELETE FROM environment_component_output_bindings; + DELETE FROM environment_component_bindings; + DELETE FROM environment_component_states; + DELETE FROM workspace_environment_states; + DELETE FROM workspace_view_layers; + DELETE FROM workspace_git_shadows; + DELETE FROM workspace_views;", )?; - rewritten += 1; + Ok(rewritten) + })(); + match rewrite { + Ok(rewritten) => { + if let Err(err) = self.conn.execute_batch("COMMIT;") { + let _ = self.conn.execute_batch("ROLLBACK;"); + return Err(Error::from(err)); + } + Ok(rewritten) + } + Err(err) => { + let _ = self.conn.execute_batch("ROLLBACK;"); + Err(err) + } } - Ok(rewritten) } pub fn lane_details(&self, lane: &str) -> Result { @@ -78,8 +132,8 @@ impl Trail { .as_ref() .map(|_| worktree_state_from_changes(&workdir_changed_paths)); let queued_merges: i64 = self.conn.query_row( - "SELECT COUNT(*) FROM merge_queue WHERE source_ref = ?1 AND status IN ('queued', 'running')", - params![details.branch.ref_name], + "SELECT COUNT(*) FROM lane_merge_queue WHERE lane_id = ?1 AND status IN ('queued', 'running')", + params![details.branch.lane_id], |row| row.get(0), )?; Ok(LaneStatusReport { diff --git a/trail/src/db/lane/leases.rs b/trail/src/db/lane/leases.rs index 2fe9f3b..f9e8faf 100644 --- a/trail/src/db/lane/leases.rs +++ b/trail/src/db/lane/leases.rs @@ -8,7 +8,6 @@ impl Trail { mode: &str, ttl_secs: u64, ) -> Result { - let _lock = self.acquire_write_lock()?; validate_ref_segment(lane)?; let mode = parse_lease_mode(mode)?; if ttl_secs == 0 { @@ -17,6 +16,15 @@ impl Trail { )); } let branch = self.lane_branch(lane)?; + if crate::db::change_ledger::command_authority_enabled() + && self.lane_uses_native_materialized_ledger(&branch)? + { + crate::db::change_ledger::materialized_lane_daemon_expected_scope( + self, + &branch.lane_id, + )?; + } + let _lock = self.acquire_write_lock()?; let path = path.map(normalize_relative_path).transpose()?; let file_id = if let Some(path) = &path { let ref_record = self.get_ref(&branch.ref_name)?; @@ -219,7 +227,7 @@ impl Trail { } fn claim_sparse_hydration( - &self, + &mut self, lane: &str, branch: &LaneBranch, path: &str, diff --git a/trail/src/db/lane/lifecycle.rs b/trail/src/db/lane/lifecycle.rs index e89c698..8134328 100644 --- a/trail/src/db/lane/lifecycle.rs +++ b/trail/src/db/lane/lifecycle.rs @@ -1,5 +1,60 @@ +use super::workdir::{MaterializationOutcome, MaterializationPolicy}; use super::*; +#[cfg(debug_assertions)] +std::thread_local! { + static FAIL_SPARSE_SELECTION_WRITE: std::cell::Cell = const { std::cell::Cell::new(false) }; + static FAIL_LANE_ASSOCIATION_BOUNDARY: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; +} + +#[cfg(debug_assertions)] +pub(crate) fn set_sparse_selection_write_failure_for_current_thread(enabled: bool) { + FAIL_SPARSE_SELECTION_WRITE.with(|fail| fail.set(enabled)); +} + +#[cfg(debug_assertions)] +fn fail_sparse_selection_write_if_requested() -> Result<()> { + if FAIL_SPARSE_SELECTION_WRITE.with(std::cell::Cell::get) { + return Err(Error::Io(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected sparse-selection publication failure", + ))); + } + Ok(()) +} + +#[cfg(test)] +pub(crate) fn set_lane_association_failure_for_current_thread(boundary: Option<&'static str>) { + FAIL_LANE_ASSOCIATION_BOUNDARY.with(|selected| *selected.borrow_mut() = boundary); +} + +#[cfg(debug_assertions)] +pub(crate) fn fail_lane_association_if_requested(boundary: &'static str) -> Result<()> { + if FAIL_LANE_ASSOCIATION_BOUNDARY.with(|selected| *selected.borrow() == Some(boundary)) { + return Err(Error::InvalidInput(format!( + "injected lane association failure at {boundary}" + ))); + } + Ok(()) +} + +#[cfg(not(debug_assertions))] +pub(crate) fn fail_lane_association_if_requested(_boundary: &'static str) -> Result<()> { + Ok(()) +} + +pub(crate) fn committed_lane_step( + operation: &str, + repair: &str, + result: Result, +) -> Result { + result.map_err(|error| Error::CommittedRepairRequired { + operation: operation.to_string(), + repair: repair.to_string(), + reason: error.to_string(), + }) +} + const LARGE_LANE_MATERIALIZE_FILE_THRESHOLD: u64 = 10_000; impl Trail { @@ -29,7 +84,7 @@ impl Trail { sparse_paths: &[String], ) -> Result { let mode = if let Some("auto") = requested_mode { - self.resolve_automatic_lane_workdir_mode(from)? + LaneWorkdirMode::Auto } else if let Some(requested_mode) = requested_mode { parse_lane_workdir_mode(requested_mode)? } else if no_materialize || materialize == Some(false) { @@ -37,9 +92,9 @@ impl Trail { } else if !sparse_paths.is_empty() { LaneWorkdirMode::Sparse } else if custom_workdir || materialize == Some(true) { - LaneWorkdirMode::FullCow + LaneWorkdirMode::Auto } else if self.default_lane_materialize_for_ref(from)? { - LaneWorkdirMode::FullCow + LaneWorkdirMode::Auto } else { LaneWorkdirMode::Virtual }; @@ -63,41 +118,6 @@ impl Trail { Ok(mode) } - fn resolve_automatic_lane_workdir_mode(&self, from: Option<&str>) -> Result { - #[cfg(target_os = "windows")] - { - let _ = from; - Ok(LaneWorkdirMode::OverlayCow) - } - #[cfg(not(target_os = "windows"))] - { - #[cfg(target_os = "macos")] - { - if Path::new("/sbin/mount_nfs").is_file() { - return Ok(LaneWorkdirMode::NfsCow); - } - } - #[cfg(target_os = "linux")] - { - if Path::new("/dev/fuse").exists() { - return Ok(LaneWorkdirMode::OverlayCow); - } - } - let source = match from { - Some(refish) => self.resolve_refish(refish)?, - None => self.resolve_branch_ref(&self.current_branch()?)?, - }; - let root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, &source.root_id)?; - if root.file_count > LARGE_LANE_MATERIALIZE_FILE_THRESHOLD { - return Err(Error::InvalidInput(format!( - "automatic layered workdir selection found no available mount backend for a {}-path root; install/enable FUSE, use nfs-cow on macOS, Dokan on Windows, or explicitly accept full-cow", - root.file_count - ))); - } - Ok(LaneWorkdirMode::FullCow) - } - } - pub fn spawn_lane( &mut self, name: &str, @@ -156,7 +176,7 @@ impl Trail { ) -> Result { let workdir_mode = if materialize { if sparse_paths.is_empty() { - LaneWorkdirMode::FullCow + LaneWorkdirMode::Auto } else { LaneWorkdirMode::Sparse } @@ -186,7 +206,64 @@ impl Trail { sparse_paths: &[String], include_neighbors: bool, ) -> Result { - let _lock = self.acquire_write_lock()?; + self.spawn_lane_with_workdir_mode_paths_and_neighbors_inner( + name, + from, + workdir_mode, + provider, + model, + workdir, + sparse_paths, + include_neighbors, + false, + ) + } + + #[doc(hidden)] + pub fn spawn_lane_with_deferred_initial_ledger( + &mut self, + name: &str, + from: Option<&str>, + workdir_mode: LaneWorkdirMode, + provider: Option, + model: Option, + workdir: Option, + sparse_paths: &[String], + include_neighbors: bool, + ) -> Result { + self.spawn_lane_with_workdir_mode_paths_and_neighbors_inner( + name, + from, + workdir_mode, + provider, + model, + workdir, + sparse_paths, + include_neighbors, + true, + ) + } + + #[allow(clippy::too_many_arguments)] + fn spawn_lane_with_workdir_mode_paths_and_neighbors_inner( + &mut self, + name: &str, + from: Option<&str>, + workdir_mode: LaneWorkdirMode, + provider: Option, + model: Option, + workdir: Option, + sparse_paths: &[String], + include_neighbors: bool, + defer_initial_ledger: bool, + ) -> Result { + // TRAIL_FS_PRODUCER: lane_spawn_materialize Materialize controlled + let ledger_authority = crate::db::change_ledger::command_authority_enabled(); + let _lock = if ledger_authority { + None + } else { + Some(self.acquire_write_lock()?) + }; validate_ref_segment(name)?; validate_lane_workdir_mode_request(&workdir_mode, workdir.is_some(), sparse_paths)?; let sparse_paths = normalize_record_paths(sparse_paths)?; @@ -204,27 +281,68 @@ impl Trail { } else { None }; - let overlay_available = workdir_mode.is_transparent_cow(); + let transparent_cow_available = workdir_mode.is_transparent_cow(); let mut sparse_policy_paths = None; + let mut resolved_workdir_mode = workdir_mode.clone(); + let mut workdir_backend = workdir_mode + .default_backend() + .unwrap_or(WorkdirBackend::Clone); + let mut materialization_report = None; + let mut materialization_operation_id = None; let materialized_workdir = if let Some(dir) = &workdir_path { match &workdir_mode { - LaneWorkdirMode::OverlayCow => { - self.prepare_overlay_cow_lane_workdir(name, dir, workdir.is_some())?; + LaneWorkdirMode::FuseCow => { + self.prepare_fuse_cow_lane_workdir(name, dir, workdir.is_some())?; + } + LaneWorkdirMode::DokanCow => { + #[cfg(target_os = "windows")] + self.prepare_dokan_cow_lane_workdir(name, dir, workdir.is_some())?; + #[cfg(not(target_os = "windows"))] + return Err(Error::InvalidInput( + "dokan-cow workdirs are currently supported only on Windows".to_string(), + )); } LaneWorkdirMode::NfsCow => { self.prepare_nfs_cow_lane_workdir(name, dir, workdir.is_some())?; } - LaneWorkdirMode::Sparse | LaneWorkdirMode::FullCow => { - self.materialize_lane_workdir_at_paths_with_neighbors( + LaneWorkdirMode::Sparse => { + let (report, operation_id) = self + .materialize_lane_workdir_at_paths_with_neighbors( + &source.root_id, + dir, + workdir.is_some(), + &sparse_paths, + include_neighbors, + )?; + materialization_operation_id = operation_id; + if let Some(report) = report { + workdir_backend = report.backend(); + materialization_report = Some(report); + } + if !sparse_paths.is_empty() { + sparse_policy_paths = self.sparse_workdir_paths(dir)?; + } + } + LaneWorkdirMode::NativeCow + | LaneWorkdirMode::PortableCopy + | LaneWorkdirMode::Auto => { + let policy = match workdir_mode { + LaneWorkdirMode::NativeCow => MaterializationPolicy::StrictNative, + LaneWorkdirMode::PortableCopy => MaterializationPolicy::Portable, + LaneWorkdirMode::Auto => MaterializationPolicy::Auto, + _ => unreachable!(), + }; + let outcome = self.materialize_lane_root_staged( &source.root_id, dir, workdir.is_some(), - &sparse_paths, - include_neighbors, + policy, )?; - if !sparse_paths.is_empty() { - sparse_policy_paths = self.sparse_workdir_paths(dir)?; - } + resolved_workdir_mode = outcome.resolved_mode; + workdir_backend = outcome.backend; + materialization_operation_id = + Some(outcome.materialization_operation_id.clone()); + materialization_report = Some(outcome.report); } LaneWorkdirMode::Virtual => {} } @@ -233,86 +351,378 @@ impl Trail { None }; let sparse_paths_for_report = sparse_policy_paths.clone().unwrap_or_default(); + let requested_workdir_mode = workdir_mode.clone(); let metadata_json = serde_json::to_string(&serde_json::json!({ - "workdir_mode": workdir_mode.as_str(), - "cow_backend": workdir_mode.cow_backend(), + "requested_workdir_mode": requested_workdir_mode.as_str(), + "workdir_mode": resolved_workdir_mode.as_str(), + "workdir_backend": workdir_backend.as_str(), + "materialization": materialization_report, "sparse_paths": sparse_paths_for_report, "include_neighbors": include_neighbors, - "overlay_available": overlay_available + "transparent_cow_available": transparent_cow_available }))?; - self.set_ref( - &ref_name, - &source.change_id, - &source.root_id, - &source.operation_id, - )?; let now = now_ts(); - self.conn.execute( - "INSERT INTO lanes (lane_id, name, kind, provider, model, created_at, metadata_json) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", - params![ - lane_id, - name, - "coding-lane", - provider, - model, - now, - metadata_json - ], + self.conn.execute_batch("BEGIN IMMEDIATE;")?; + let association = (|| -> Result<()> { + self.insert_new_ref_database_only( + &ref_name, + &source.change_id, + &source.root_id, + &source.operation_id, + )?; + fail_lane_association_if_requested("spawn_after_ref")?; + self.conn.execute( + "INSERT INTO lanes (lane_id, name, kind, provider, model, created_at, metadata_json) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![lane_id, name, "coding-lane", provider, model, now, metadata_json], + )?; + fail_lane_association_if_requested("spawn_after_lane")?; + self.conn.execute( + "INSERT INTO lane_branches \ + (lane_id, ref_name, base_change, head_change, base_root, head_root, session_id, workdir, status, created_at, updated_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 'active', ?9, ?9)", + params![ + lane_id, + ref_name, + source.change_id.0, + source.change_id.0, + source.root_id.0, + source.root_id.0, + Option::::None, + materialized_workdir, + now + ], + )?; + fail_lane_association_if_requested("spawn_after_branch")?; + Ok(()) + })(); + match association { + Ok(()) => self.conn.execute_batch("COMMIT;")?, + Err(error) => { + let _ = self.conn.execute_batch("ROLLBACK;"); + if let Some(operation_id) = materialization_operation_id.as_deref() { + self.abort_materialization_operation(operation_id)?; + } + return Err(error); + } + } + let committed_operation = materialization_operation_id + .clone() + .unwrap_or_else(|| source.operation_id.0.clone()); + committed_lane_step( + &committed_operation, + "new lane ref mirror", + (|| { + fail_lane_association_if_requested("spawn_ref_repair")?; + self.repair_new_ref_mirror( + &ref_name, + &source.change_id, + &source.root_id, + &source.operation_id, + ) + })(), )?; - self.conn.execute( - "INSERT INTO lane_branches \ - (lane_id, ref_name, base_change, head_change, base_root, head_root, session_id, workdir, status, created_at, updated_at) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 'active', ?9, ?9)", - params![ + if let Some(operation_id) = materialization_operation_id.as_deref() { + committed_lane_step( + &committed_operation, + "new lane materialization journal completion", + (|| { + fail_lane_association_if_requested("spawn_journal_completion")?; + self.complete_materialization_operation(operation_id) + })(), + )?; + } + committed_lane_step( + &committed_operation, + "initial lane post-association reconciliation", + fail_lane_association_if_requested("spawn_after_commit"), + )?; + if defer_initial_ledger + && ledger_authority + && materialized_workdir.is_some() + && !workdir_mode.is_transparent_cow() + { + return Ok(LaneSpawnReport { lane_id, ref_name, - source.change_id.0, - source.change_id.0, - source.root_id.0, - source.root_id.0, - Option::::None, - materialized_workdir, - now - ], - )?; - if workdir_mode.is_transparent_cow() { - let mountpoint = materialized_workdir.as_deref().ok_or_else(|| { - Error::Corrupt("transparent COW lane has no mountpoint".to_string()) + base_change: source.change_id, + workdir: materialized_workdir, + requested_workdir_mode, + workdir_mode: resolved_workdir_mode, + workdir_backend: Some(workdir_backend), + materialization: materialization_report, + sparse_paths: sparse_policy_paths.unwrap_or_default(), + transparent_cow_available, + }); + } + if ledger_authority && materialized_workdir.is_some() && !workdir_mode.is_transparent_cow() + { + let expected = + crate::db::change_ledger::prepare_materialized_lane_controlled_projection( + self, &lane_id, + ) + .map_err(|error| Error::CommittedRepairRequired { + operation: materialization_operation_id + .clone() + .unwrap_or_else(|| source.operation_id.0.clone()), + repair: "initial materialized lane ledger reconciliation".into(), + reason: error.to_string(), + })?; + let evidence = crate::db::change_ledger::IntentEvidence { + exact_paths: Vec::new(), + complete_prefixes: Vec::new(), + }; + crate::db::change_ledger::run_projection_alignment( + self, + &expected, + crate::db::change_ledger::IntentProducer::Materialize, + &evidence, + crate::db::change_ledger::ProjectionAlignmentMode::Aligned, + |db, intent| { + crate::db::change_ledger::with_materialized_lane_controlled_interval( + db, + &lane_id, + intent, + &evidence, + |_| Ok(()), + |db, policy, candidates| { + let comparison = db.compare_controlled_projection_target( + policy, + candidates, + &source.root_id, + crate::db::change_ledger::CandidateMaterialization::ManifestOnly, + )?; + if comparison.summaries.is_empty() { + Ok(()) + } else { + Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: format!( + "initial lane materialization did not match its target root: {:?}", + comparison.summaries + ), + command: format!("trail lane status {lane_id}"), + }) + } + }, + ) + }, + |db| db.publish_lane_marker_if_materialized(&lane_id), + ) + .map_err(|error| Error::CommittedRepairRequired { + operation: materialization_operation_id + .clone() + .unwrap_or_else(|| source.operation_id.0.clone()), + repair: "initial materialized lane ledger alignment".into(), + reason: error.to_string(), })?; - self.create_workspace_view( - &lane_id, - &source.change_id, - &source.root_id, - platform_workspace_backend(&workdir_mode), - Path::new(mountpoint), + } else if materialized_workdir.is_some() { + committed_lane_step( + &committed_operation, + "initial lane marker publication", + (|| { + fail_lane_association_if_requested("spawn_marker")?; + self.publish_lane_marker_if_materialized(name) + })(), )?; } - self.insert_lane_event( - &lane_id, - "lane_spawned", - Some(&source.change_id), - None, - &serde_json::json!({ - "ref_name": ref_name.clone(), - "base_root": source.root_id.0.clone(), - "workdir": materialized_workdir.clone(), - "workdir_mode": workdir_mode.as_str(), - "cow_backend": workdir_mode.cow_backend(), - "sparse_paths": sparse_policy_paths.clone().unwrap_or_default(), - "include_neighbors": include_neighbors, - "overlay_available": overlay_available - }), + if workdir_mode.is_transparent_cow() { + committed_lane_step( + &committed_operation, + "initial lane workspace view publication", + (|| { + fail_lane_association_if_requested("spawn_workspace_view")?; + let mountpoint = materialized_workdir.as_deref().ok_or_else(|| { + Error::Corrupt("transparent COW lane has no mountpoint".to_string()) + })?; + self.create_workspace_view( + &lane_id, + &source.change_id, + &source.root_id, + platform_workspace_backend(&workdir_mode), + Path::new(mountpoint), + ) + })(), + )?; + } + committed_lane_step( + &committed_operation, + "initial lane event publication", + (|| { + fail_lane_association_if_requested("spawn_event")?; + self.insert_lane_event( + &lane_id, + "lane_spawned", + Some(&source.change_id), + None, + &serde_json::json!({ + "ref_name": ref_name.clone(), + "base_root": source.root_id.0.clone(), + "workdir": materialized_workdir.clone(), + "requested_workdir_mode": requested_workdir_mode.as_str(), + "workdir_mode": resolved_workdir_mode.as_str(), + "workdir_backend": workdir_backend.as_str(), + "materialization": materialization_report, + "sparse_paths": sparse_policy_paths.clone().unwrap_or_default(), + "include_neighbors": include_neighbors, + "transparent_cow_available": transparent_cow_available + }), + ) + })(), )?; Ok(LaneSpawnReport { lane_id, ref_name, base_change: source.change_id, workdir: materialized_workdir, - cow_backend: workdir_mode.cow_backend().map(str::to_string), + requested_workdir_mode, + workdir_mode: resolved_workdir_mode, + workdir_backend: Some(workdir_backend), + materialization: materialization_report, sparse_paths: sparse_policy_paths.unwrap_or_default(), - overlay_available, + transparent_cow_available, + }) + } + + #[doc(hidden)] + pub fn resume_deferred_initial_lane_ledger(&mut self, lane: &str) -> Result { + let details = self.lane_details(lane)?; + let metadata = details.record.metadata_json.as_deref().ok_or_else(|| { + Error::Corrupt(format!( + "lane `{}` is missing its spawn metadata", + details.record.name + )) + })?; + let metadata: serde_json::Value = serde_json::from_str(metadata)?; + let metadata_field = |name: &str| { + metadata + .get(name) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| Error::Corrupt(format!("lane spawn metadata is missing `{name}`"))) + }; + let requested_workdir_mode = + LaneWorkdirMode::parse(metadata_field("requested_workdir_mode")?) + .ok_or_else(|| Error::Corrupt("invalid requested lane workdir mode".into()))?; + let workdir_mode = LaneWorkdirMode::parse(metadata_field("workdir_mode")?) + .ok_or_else(|| Error::Corrupt("invalid resolved lane workdir mode".into()))?; + let workdir_backend = serde_json::from_value::( + metadata + .get("workdir_backend") + .cloned() + .ok_or_else(|| Error::Corrupt("lane spawn metadata is missing backend".into()))?, + )?; + let materialization = metadata + .get("materialization") + .filter(|value| !value.is_null()) + .cloned() + .map(serde_json::from_value::) + .transpose()?; + let sparse_paths = metadata + .get("sparse_paths") + .cloned() + .map(serde_json::from_value::>) + .transpose()? + .unwrap_or_default(); + let include_neighbors = metadata + .get("include_neighbors") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + let transparent_cow_available = metadata + .get("transparent_cow_available") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + + if details.branch.workdir.is_some() && !workdir_mode.is_transparent_cow() { + let expected = + crate::db::change_ledger::prepare_materialized_lane_controlled_projection( + self, + &details.branch.lane_id, + )?; + let evidence = crate::db::change_ledger::IntentEvidence { + exact_paths: Vec::new(), + complete_prefixes: Vec::new(), + }; + crate::db::change_ledger::run_projection_alignment( + self, + &expected, + crate::db::change_ledger::IntentProducer::Materialize, + &evidence, + crate::db::change_ledger::ProjectionAlignmentMode::Aligned, + |db, intent| { + crate::db::change_ledger::with_materialized_lane_controlled_interval( + db, + &details.branch.lane_id, + intent, + &evidence, + |_| Ok(()), + |db, policy, candidates| { + let comparison = db.compare_controlled_projection_target( + policy, + candidates, + &details.branch.head_root, + crate::db::change_ledger::CandidateMaterialization::ManifestOnly, + )?; + if comparison.summaries.is_empty() { + Ok(()) + } else { + Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: format!( + "initial lane materialization did not match its target root: {:?}", + comparison.summaries + ), + command: format!( + "trail lane status {}", + details.branch.lane_id + ), + }) + } + }, + ) + }, + |db| db.publish_lane_marker_if_materialized(&details.branch.lane_id), + )?; + } + + let event_exists = self.conn.query_row( + "SELECT EXISTS(SELECT 1 FROM lane_events WHERE lane_id=?1 AND event_type='lane_spawned')", + [&details.branch.lane_id], + |row| row.get::<_, bool>(0), + )?; + if !event_exists { + self.insert_lane_event( + &details.branch.lane_id, + "lane_spawned", + Some(&details.branch.base_change), + None, + &serde_json::json!({ + "ref_name": details.branch.ref_name, + "base_root": details.branch.base_root.0, + "workdir": details.branch.workdir, + "requested_workdir_mode": requested_workdir_mode.as_str(), + "workdir_mode": workdir_mode.as_str(), + "workdir_backend": workdir_backend.as_str(), + "materialization": materialization, + "sparse_paths": sparse_paths, + "include_neighbors": include_neighbors, + "transparent_cow_available": transparent_cow_available, + }), + )?; + } + + Ok(LaneSpawnReport { + lane_id: details.branch.lane_id, + ref_name: details.branch.ref_name, + base_change: details.branch.base_change, + workdir: details.branch.workdir, + requested_workdir_mode, workdir_mode, + workdir_backend: Some(workdir_backend), + materialization, + sparse_paths, + transparent_cow_available, }) } @@ -321,7 +731,13 @@ impl Trail { lane: &str, workdir: Option, ) -> Result { - let _lock = self.acquire_write_lock()?; + // TRAIL_FS_PRODUCER: lane_ensure_materialized Materialize controlled + let ledger_authority = crate::db::change_ledger::command_authority_enabled(); + let _lock = if ledger_authority { + None + } else { + Some(self.acquire_write_lock()?) + }; validate_ref_segment(lane)?; let branch = self.lane_branch(lane)?; if let Some(existing) = branch.workdir.clone() { @@ -337,81 +753,168 @@ impl Trail { } let record = self.lane_record(&branch.lane_id)?; let workdir_mode = self.lane_workdir_mode_for(&record, &branch)?; + let requested_workdir_mode = self.lane_requested_workdir_mode_for(&record, &branch)?; + let workdir_backend = self.lane_workdir_backend_for(&record)?; + let materialization = self.lane_materialization_report_for(&record)?; let sparse_paths = self.lane_report_sparse_paths(&branch)?; - let overlay_available = workdir_mode.is_transparent_cow(); + let transparent_cow_available = workdir_mode.is_transparent_cow(); return Ok(LaneWorkdirReport { lane_id: branch.lane_id, workdir: Some(existing), - cow_backend: workdir_mode.cow_backend().map(str::to_string), + requested_workdir_mode, + workdir_backend, + materialization, sparse_paths, - overlay_available, + transparent_cow_available, workdir_mode, }); } let head = self.get_ref(&branch.ref_name)?; - let dir = self.materialize_lane_workdir(lane, &head.root_id, workdir.as_deref())?; + let dir = self.resolve_lane_workdir_path(lane, workdir.as_deref())?; + let outcome = self.materialize_lane_root_staged( + &head.root_id, + &dir, + workdir.is_some(), + MaterializationPolicy::Auto, + )?; let workdir = dir.to_string_lossy().to_string(); - self.conn.execute( - "UPDATE lane_branches SET workdir = ?1, updated_at = ?2 WHERE lane_id = ?3", - params![workdir, now_ts(), branch.lane_id], + self.conn.execute_batch("BEGIN IMMEDIATE;")?; + let association = (|| -> Result<()> { + self.update_lane_materialization_metadata( + &branch.lane_id, + &LaneWorkdirMode::Auto, + &outcome, + )?; + fail_lane_association_if_requested("ensure_after_lane_metadata")?; + let changed = self.conn.execute( + "UPDATE lane_branches SET workdir = ?1, updated_at = ?2 + WHERE lane_id = ?3 AND workdir IS NULL AND head_root=?4", + params![workdir, now_ts(), branch.lane_id, head.root_id.0], + )?; + if changed != 1 { + return Err(Error::StaleBranch(branch.ref_name.clone())); + } + fail_lane_association_if_requested("ensure_after_branch")?; + Ok(()) + })(); + match association { + Ok(()) => self.conn.execute_batch("COMMIT;")?, + Err(error) => { + let _ = self.conn.execute_batch("ROLLBACK;"); + self.abort_materialization_operation(&outcome.materialization_operation_id)?; + return Err(error); + } + } + let committed_operation = outcome.materialization_operation_id.clone(); + committed_lane_step( + &committed_operation, + "ensured lane materialization journal completion", + (|| { + fail_lane_association_if_requested("ensure_journal_completion")?; + self.complete_materialization_operation(&committed_operation) + })(), + )?; + committed_lane_step( + &committed_operation, + "ensured lane post-association reconciliation", + fail_lane_association_if_requested("ensure_after_commit"), )?; - self.insert_lane_event( - &branch.lane_id, - "workdir_materialized", - Some(&head.change_id), - None, - &serde_json::json!({ - "workdir": workdir, - "root_id": head.root_id.0 - }), + if ledger_authority { + let expected = + crate::db::change_ledger::prepare_materialized_lane_controlled_projection( + self, + &branch.lane_id, + ) + .map_err(|error| Error::CommittedRepairRequired { + operation: outcome.materialization_operation_id.clone(), + repair: "ensured materialized lane ledger reconciliation".into(), + reason: error.to_string(), + })?; + let evidence = crate::db::change_ledger::IntentEvidence { + exact_paths: Vec::new(), + complete_prefixes: Vec::new(), + }; + crate::db::change_ledger::run_projection_alignment( + self, + &expected, + crate::db::change_ledger::IntentProducer::Materialize, + &evidence, + crate::db::change_ledger::ProjectionAlignmentMode::Aligned, + |db, intent| { + crate::db::change_ledger::with_materialized_lane_controlled_interval( + db, + &branch.lane_id, + intent, + &evidence, + |_| Ok(()), + |db, policy, candidates| { + let comparison = db.compare_controlled_projection_target( + policy, + candidates, + &head.root_id, + crate::db::change_ledger::CandidateMaterialization::ManifestOnly, + )?; + if comparison.summaries.is_empty() { + Ok(()) + } else { + Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: + "ensured lane materialization did not match its target root" + .into(), + command: format!("trail lane status {}", branch.lane_id), + }) + } + }, + ) + }, + |db| db.publish_lane_marker_if_materialized(&branch.lane_id), + ) + .map_err(|error| Error::CommittedRepairRequired { + operation: outcome.materialization_operation_id.clone(), + repair: "ensured materialized lane ledger alignment".into(), + reason: error.to_string(), + })?; + } + committed_lane_step( + &committed_operation, + "ensured lane event publication", + (|| { + fail_lane_association_if_requested("ensure_event")?; + self.insert_lane_event( + &branch.lane_id, + "workdir_materialized", + Some(&head.change_id), + None, + &serde_json::json!({ + "workdir": workdir, + "root_id": head.root_id.0 + }), + ) + })(), + )?; + committed_lane_step( + &committed_operation, + "ensured lane marker publication", + (|| { + fail_lane_association_if_requested("ensure_marker")?; + self.publish_lane_marker_if_materialized(lane) + })(), )?; Ok(LaneWorkdirReport { lane_id: branch.lane_id, workdir: Some(dir.to_string_lossy().to_string()), - workdir_mode: LaneWorkdirMode::FullCow, - cow_backend: (LaneWorkdirMode::FullCow).cow_backend().map(str::to_string), + requested_workdir_mode: LaneWorkdirMode::Auto, + workdir_mode: outcome.resolved_mode, + workdir_backend: Some(outcome.backend), + materialization: Some(outcome.report), sparse_paths: Vec::new(), - overlay_available: false, + transparent_cow_available: false, }) } - pub(crate) fn materialize_lane_workdir( - &self, - name: &str, - root_id: &ObjectId, - custom_workdir: Option<&Path>, - ) -> Result { - let dir = self.resolve_lane_workdir_path(name, custom_workdir)?; - self.materialize_lane_workdir_at(root_id, &dir, custom_workdir.is_some())?; - Ok(dir) - } - - pub(crate) fn materialize_lane_workdir_at( - &self, - root_id: &ObjectId, - dir: &Path, - custom_workdir: bool, - ) -> Result<()> { - self.materialize_lane_workdir_at_paths(root_id, dir, custom_workdir, &[]) - } - - pub(crate) fn materialize_lane_workdir_at_paths( - &self, - root_id: &ObjectId, - dir: &Path, - custom_workdir: bool, - sparse_paths: &[String], - ) -> Result<()> { - self.materialize_lane_workdir_at_paths_with_neighbors( - root_id, - dir, - custom_workdir, - sparse_paths, - false, - ) - } - pub(crate) fn materialize_lane_workdir_at_paths_with_neighbors( &self, root_id: &ObjectId, @@ -419,84 +922,27 @@ impl Trail { custom_workdir: bool, sparse_paths: &[String], include_neighbors: bool, - ) -> Result<()> { - prepare_lane_workdir(dir, custom_workdir)?; + ) -> Result<(Option, Option)> { if sparse_paths.is_empty() { - let root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, root_id)?; - if root.file_count > LARGE_LANE_MATERIALIZE_FILE_THRESHOLD { - return self.materialize_lane_workdir_at_streaming(root_id, dir); - } + let outcome = self.materialize_lane_root_staged( + root_id, + dir, + custom_workdir, + MaterializationPolicy::Auto, + )?; + return Ok((None, Some(outcome.materialization_operation_id))); } - let empty = BTreeMap::new(); - let files = if sparse_paths.is_empty() { - self.load_root_files(root_id)? - } else if include_neighbors { + let files = if include_neighbors { self.load_root_files_for_selections_with_neighbors(root_id, sparse_paths)? } else { self.load_root_files_for_selections(root_id, sparse_paths)? }; - let mut materialized = None; - let cloned_from_workspace = if sparse_paths.is_empty() { - let source_stamps = - match self.workspace_file_stamps_if_clean_index_matches(root_id, &files)? { - Some(stamps) => Some(stamps), - None => self.workspace_file_stamps_if_entries_match(&files)?, - }; - if let Some(source_stamps) = source_stamps { - if let Some(report) = materialize_from_workspace_cow_report( - &self.workspace_root, - dir, - &files, - &source_stamps, - false, - )? { - materialized = Some(report); - true - } else { - false - } - } else { - false - } - } else { - false - }; - if !cloned_from_workspace { - materialized = Some(if sparse_paths.is_empty() { - self.materialize_files_best_effort_at_report(dir, &empty, &files)? - } else { - self.materialize_new_files_best_effort_at_with_workspace_cow_report(dir, &files)? - }); - } - if sparse_paths.is_empty() { - remove_sparse_workdir_manifest(dir)?; - } else { - self.write_sparse_workdir_manifest(dir, files.keys())?; - } - if let Some(report) = materialized { - self.write_clean_workdir_manifest_from_stamps( - dir, - root_id, - &files, - files.keys(), - report.stamps, - )?; - } else { - self.write_clean_workdir_manifest(dir, root_id, &files, files.keys())?; - } - Ok(()) - } - - fn materialize_lane_workdir_at_streaming(&self, root_id: &ObjectId, dir: &Path) -> Result<()> { - let report = self.materialize_root_files_at_streaming(root_id, dir, false)?; - remove_sparse_workdir_manifest(dir)?; - self.write_clean_workdir_manifest_from_disk_manifest_and_stamps( - dir, - root_id, - &report.disk_manifest, - report.disk_manifest.keys(), - report.materialized.stamps, - ) + let outcome = + self.materialize_sparse_lane_root_staged(root_id, dir, custom_workdir, &files)?; + Ok(( + Some(outcome.report), + Some(outcome.materialization_operation_id), + )) } pub(crate) fn sparse_workdir_paths(&self, dir: &Path) -> Result>> { @@ -560,12 +1006,106 @@ impl Trail { } } if branch.workdir.is_some() { - Ok(LaneWorkdirMode::FullCow) + Ok(LaneWorkdirMode::NativeCow) } else { Ok(LaneWorkdirMode::Virtual) } } + pub(crate) fn lane_requested_workdir_mode_for( + &self, + record: &LaneRecord, + branch: &LaneBranch, + ) -> Result { + if let Some(metadata_json) = &record.metadata_json { + let value: serde_json::Value = serde_json::from_str(metadata_json)?; + if let Some(mode) = value + .get("requested_workdir_mode") + .and_then(serde_json::Value::as_str) + { + return parse_lane_workdir_mode(mode); + } + } + self.lane_workdir_mode_for(record, branch) + } + + pub(crate) fn lane_workdir_backend_for( + &self, + record: &LaneRecord, + ) -> Result> { + let Some(metadata_json) = &record.metadata_json else { + return Ok(None); + }; + let value: serde_json::Value = serde_json::from_str(metadata_json)?; + let Some(backend) = value.get("workdir_backend") else { + return Ok(None); + }; + serde_json::from_value(backend.clone()) + .map(Some) + .map_err(Error::Json) + } + + pub(crate) fn lane_materialization_report_for( + &self, + record: &LaneRecord, + ) -> Result> { + let Some(metadata_json) = &record.metadata_json else { + return Ok(None); + }; + let value: serde_json::Value = serde_json::from_str(metadata_json)?; + let Some(report) = value.get("materialization") else { + return Ok(None); + }; + if report.is_null() { + return Ok(None); + } + serde_json::from_value(report.clone()) + .map(Some) + .map_err(Error::Json) + } + + pub(crate) fn update_lane_materialization_metadata( + &self, + lane_id: &str, + requested_mode: &LaneWorkdirMode, + outcome: &MaterializationOutcome, + ) -> Result<()> { + let existing = self + .conn + .query_row( + "SELECT metadata_json FROM lanes WHERE lane_id = ?1", + params![lane_id], + |row| row.get::<_, Option>(0), + )? + .unwrap_or_else(|| "{}".to_string()); + let mut value: serde_json::Value = serde_json::from_str(&existing)?; + let object = value.as_object_mut().ok_or_else(|| { + Error::Corrupt(format!("lane `{lane_id}` metadata is not a JSON object")) + })?; + object.insert( + "requested_workdir_mode".to_string(), + serde_json::json!(requested_mode.as_str()), + ); + object.insert( + "workdir_mode".to_string(), + serde_json::json!(outcome.resolved_mode.as_str()), + ); + object.insert( + "workdir_backend".to_string(), + serde_json::json!(outcome.backend.as_str()), + ); + object.remove("cow_backend"); + object.insert( + "materialization".to_string(), + serde_json::to_value(&outcome.report)?, + ); + self.conn.execute( + "UPDATE lanes SET metadata_json = ?1 WHERE lane_id = ?2", + params![serde_json::to_string(&value)?, lane_id], + )?; + Ok(()) + } + pub(crate) fn lane_report_sparse_paths(&self, branch: &LaneBranch) -> Result> { if let Some(workdir) = &branch.workdir { if let Some(paths) = self.lane_sparse_workdir_paths(branch, Path::new(workdir))? { @@ -635,7 +1175,9 @@ impl Trail { "version": 1, "materialized_paths": normalized.into_iter().collect::>() }); - write_file_atomic(&manifest, &serde_json::to_vec(&body)?, false)?; + #[cfg(debug_assertions)] + fail_sparse_selection_write_if_requested()?; + write_file_atomic(&manifest, &serde_json::to_vec(&body)?, true)?; Ok(()) } @@ -738,9 +1280,24 @@ impl Trail { } fn parse_lane_workdir_mode(value: &str) -> Result { + match value { + "overlay-cow" | "overlay_cow" => { + return Err(Error::InvalidInput( + "unsupported lane workdir mode `overlay-cow`; this build uses the hard-cutover modes `fuse-cow` and `dokan-cow`; remove and recreate the lane with the platform-appropriate mode" + .to_string(), + )); + } + "full-cow" | "full_cow" => { + return Err(Error::InvalidInput( + "unsupported lane workdir mode `full-cow`; this mode was renamed to `native-cow` to describe filesystem-native clone/reflink materialization; remove and recreate the lane with `native-cow`" + .to_string(), + )); + } + _ => {} + } LaneWorkdirMode::parse(value).ok_or_else(|| { Error::InvalidInput(format!( - "unknown lane workdir mode `{value}`; expected auto, virtual, sparse, full-cow, overlay-cow, or nfs-cow" + "unknown lane workdir mode `{value}`; expected auto, virtual, sparse, native-cow, portable-copy, fuse-cow, nfs-cow, or dokan-cow" )) }) } @@ -748,9 +1305,12 @@ fn parse_lane_workdir_mode(value: &str) -> Result { fn platform_workspace_backend(mode: &LaneWorkdirMode) -> &'static str { match mode { LaneWorkdirMode::NfsCow => "nfs", - LaneWorkdirMode::OverlayCow if cfg!(target_os = "windows") => "dokan", - LaneWorkdirMode::OverlayCow => "fuse", - LaneWorkdirMode::Sparse | LaneWorkdirMode::FullCow => "clone", + LaneWorkdirMode::FuseCow => "fuse", + LaneWorkdirMode::DokanCow => "dokan", + LaneWorkdirMode::Auto + | LaneWorkdirMode::Sparse + | LaneWorkdirMode::NativeCow + | LaneWorkdirMode::PortableCopy => "clone", LaneWorkdirMode::Virtual => "virtual", } } @@ -761,6 +1321,14 @@ fn validate_lane_workdir_mode_request( sparse_paths: &[String], ) -> Result<()> { match mode { + LaneWorkdirMode::Auto | LaneWorkdirMode::PortableCopy => { + if !sparse_paths.is_empty() { + return Err(Error::InvalidInput(format!( + "{} lane workdir mode cannot be combined with sparse paths", + mode.as_str() + ))); + } + } LaneWorkdirMode::Virtual => { if custom_workdir { return Err(Error::InvalidInput( @@ -780,20 +1348,35 @@ fn validate_lane_workdir_mode_request( )); } } - LaneWorkdirMode::FullCow => { + LaneWorkdirMode::NativeCow => { if !sparse_paths.is_empty() { return Err(Error::InvalidInput( - "full-cow lane workdir mode cannot be combined with sparse paths".to_string(), + "native-cow lane workdir mode cannot be combined with sparse paths".to_string(), )); } } - LaneWorkdirMode::OverlayCow => { + LaneWorkdirMode::FuseCow => { if !sparse_paths.is_empty() { return Err(Error::InvalidInput( - "overlay-cow lane workdir mode cannot be combined with sparse paths" - .to_string(), + "fuse-cow lane workdir mode cannot be combined with sparse paths".to_string(), )); } + #[cfg(not(any(target_os = "linux", all(target_os = "macos", feature = "macfuse"))))] + return Err(Error::InvalidInput( + "fuse-cow workdirs require Linux FUSE or a macOS build with --features macfuse" + .to_string(), + )); + } + LaneWorkdirMode::DokanCow => { + if !sparse_paths.is_empty() { + return Err(Error::InvalidInput( + "dokan-cow lane workdir mode cannot be combined with sparse paths".to_string(), + )); + } + #[cfg(not(target_os = "windows"))] + return Err(Error::InvalidInput( + "dokan-cow workdirs are currently supported only on Windows".to_string(), + )); } LaneWorkdirMode::NfsCow => { if !sparse_paths.is_empty() { @@ -826,13 +1409,362 @@ pub(crate) fn selected_file_entries( } fn sparse_workdir_manifest_path(dir: &Path) -> PathBuf { - dir.join(".trail").join("sparse-workdir.json") + dir.join(".trail").join("sparse-selection.json") } -fn remove_sparse_workdir_manifest(dir: &Path) -> Result<()> { - let manifest = sparse_workdir_manifest_path(dir); - if manifest.exists() { - fs::remove_file(manifest)?; +#[cfg(test)] +mod hard_cutover_tests { + use super::*; + + static AUTHORITY_TEST: std::sync::OnceLock> = std::sync::OnceLock::new(); + + struct AuthorityReset; + + impl Drop for AuthorityReset { + fn drop(&mut self) { + crate::db::set_command_authority_override(false); + } + } + + fn initialized_trail() -> (tempfile::TempDir, Trail) { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + (workspace, db) + } + + fn assert_lane_association_absent(db: &Trail, name: &str) { + assert!(db.try_get_ref(&lane_ref(name)).unwrap().is_none()); + let lane_count: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM lanes WHERE name=?1", [name], |row| { + row.get(0) + }) + .unwrap(); + let branch_count: i64 = db + .conn + .query_row( + "SELECT COUNT(*) FROM lane_branches WHERE ref_name=?1", + [lane_ref(name)], + |row| row.get(0), + ) + .unwrap(); + assert_eq!((lane_count, branch_count), (0, 0)); + } + + fn assert_lane_association_present(db: &Trail, name: &str) { + assert!(db.try_get_ref(&lane_ref(name)).unwrap().is_some()); + assert!(db.lane_branch(name).is_ok()); + } + + fn materialization_journal_count(db: &Trail) -> usize { + let journal = db.db_dir.join("materialization-operations"); + if !journal.is_dir() { + return 0; + } + fs::read_dir(journal) + .unwrap() + .filter_map(std::result::Result::ok) + .filter(|entry| entry.path().extension().and_then(|ext| ext.to_str()) == Some("json")) + .count() + } + + #[cfg(unix)] + #[test] + fn controlled_lane_prepare_is_marker_free_but_ordinary_prepare_repairs_marker() { + use std::os::unix::fs::MetadataExt; + + let (_workspace, mut db) = initialized_trail(); + let spawned = db + .spawn_lane("marker-free-prepare", Some("main"), true, None, None) + .unwrap(); + let workdir = PathBuf::from(spawned.workdir.unwrap()); + let marker = workdir.join(".trail/workdir-manifest.json"); + fs::remove_file(&marker).unwrap(); + + crate::db::change_ledger::prepare_materialized_lane_controlled_projection( + &mut db, + "marker-free-prepare", + ) + .unwrap(); + assert!( + !marker.exists(), + "new controlled daemon preparation wrote its watched marker" + ); + + crate::db::change_ledger::prepare_materialized_lane_daemon( + &db, + "marker-free-prepare", + true, + ) + .unwrap(); + let ordinary_marker_inode = fs::metadata(&marker).unwrap().ino(); + + crate::db::change_ledger::prepare_materialized_lane_controlled_projection( + &mut db, + "marker-free-prepare", + ) + .unwrap(); + assert_eq!( + fs::metadata(&marker).unwrap().ino(), + ordinary_marker_inode, + "existing controlled daemon preparation rewrote its watched marker" + ); + } + + #[test] + fn repeated_authoritative_materialized_spawn_and_record_setup_has_no_transient_repair() { + let _guard = AUTHORITY_TEST + .get_or_init(|| std::sync::Mutex::new(())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let _reset = AuthorityReset; + + for index in 0..4 { + crate::db::set_command_authority_override(false); + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "base\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + crate::db::set_command_authority_override(true); + let lane = format!("repeated-authority-{index}"); + let spawned = db + .spawn_lane(&lane, Some("main"), true, None, None) + .unwrap_or_else(|error| panic!("materialized spawn {index} failed: {error}")); + let workdir = PathBuf::from(spawned.workdir.unwrap()); + fs::write( + workdir.join("README.md"), + format!("recorded lane contents {index}\n"), + ) + .unwrap(); + db.record_lane_workdir(&lane, Some(format!("record setup {index}"))) + .unwrap_or_else(|error| panic!("materialized record {index} failed: {error}")); + } + } + + #[test] + fn removed_cow_mode_reports_the_recreate_lifecycle() { + let overlay_error = parse_lane_workdir_mode("overlay-cow").unwrap_err(); + let overlay_message = overlay_error.to_string(); + assert!(overlay_message.contains("hard-cutover modes `fuse-cow` and `dokan-cow`")); + assert!(overlay_message.contains("remove and recreate the lane")); + + let native_error = parse_lane_workdir_mode("full-cow").unwrap_err(); + let native_message = native_error.to_string(); + assert!(native_message.contains("renamed to `native-cow`")); + assert!(native_message.contains("remove and recreate the lane")); + } + + #[test] + fn lane_spawn_sql_association_rolls_back_at_every_boundary() { + for boundary in ["spawn_after_ref", "spawn_after_lane", "spawn_after_branch"] { + let (_workspace, mut db) = initialized_trail(); + set_lane_association_failure_for_current_thread(Some(boundary)); + let result = db.spawn_lane("atomic-spawn", Some("main"), false, None, None); + set_lane_association_failure_for_current_thread(None); + assert!(result.is_err(), "boundary {boundary} did not fail"); + assert_lane_association_absent(&db, "atomic-spawn"); + } + } + + #[test] + fn sparse_lane_spawn_rolls_back_publication_and_journal_at_every_sql_boundary() { + for boundary in ["spawn_after_ref", "spawn_after_lane", "spawn_after_branch"] { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root contents").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let destination = workspace.path().join(format!("sparse-{boundary}")); + set_lane_association_failure_for_current_thread(Some(boundary)); + let result = db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "atomic-sparse", + Some("main"), + LaneWorkdirMode::Sparse, + None, + None, + Some(destination.clone()), + &["README.md".to_string()], + false, + ); + set_lane_association_failure_for_current_thread(None); + assert!(result.is_err(), "boundary {boundary} did not fail"); + assert_lane_association_absent(&db, "atomic-sparse"); + assert!(!destination.exists()); + let journal_dir = db.db_dir.join("materialization-operations"); + assert!( + !journal_dir.exists() + || fs::read_dir(&journal_dir) + .unwrap() + .filter_map(std::result::Result::ok) + .all( + |entry| entry.path().extension().and_then(|ext| ext.to_str()) + != Some("json") + ) + ); + drop(db); + Trail::open(workspace.path()).unwrap(); + assert!(!destination.exists()); + } + } + + #[test] + fn turn_lane_spawn_sql_association_rolls_back_at_every_boundary() { + for boundary in ["turn_after_ref", "turn_after_lane", "turn_after_branch"] { + let (_workspace, mut db) = initialized_trail(); + set_lane_association_failure_for_current_thread(Some(boundary)); + let result = db.lane_branch_for_turn("atomic-turn", Some("main"), None); + set_lane_association_failure_for_current_thread(None); + assert!(result.is_err(), "boundary {boundary} did not fail"); + assert_lane_association_absent(&db, "atomic-turn"); + } + } + + #[test] + fn lane_ensure_sql_association_rolls_back_at_every_boundary() { + for boundary in ["ensure_after_lane_metadata", "ensure_after_branch"] { + let (workspace, mut db) = initialized_trail(); + db.spawn_lane("atomic-ensure", Some("main"), false, None, None) + .unwrap(); + let before = db.lane_record("atomic-ensure").unwrap().metadata_json; + let destination = workspace.path().join(format!("ensure-{boundary}")); + set_lane_association_failure_for_current_thread(Some(boundary)); + let result = + db.ensure_lane_workdir_materialized("atomic-ensure", Some(destination.clone())); + set_lane_association_failure_for_current_thread(None); + assert!(result.is_err(), "boundary {boundary} did not fail"); + let branch = db.lane_branch("atomic-ensure").unwrap(); + assert!(branch.workdir.is_none()); + assert_eq!( + db.lane_record("atomic-ensure").unwrap().metadata_json, + before + ); + assert!(!destination.exists()); + assert_eq!(materialization_journal_count(&db), 0); + drop(db); + Trail::open(workspace.path()).unwrap(); + assert!(!destination.exists()); + } + } + + #[test] + fn materialized_turn_spawn_rolls_back_owned_publication_at_every_boundary() { + for boundary in ["turn_after_ref", "turn_after_lane", "turn_after_branch"] { + let (_workspace, mut db) = initialized_trail(); + db.config_set("lane.default_materialize", "true").unwrap(); + let destination = db + .default_lane_workdir_path("atomic-materialized-turn") + .unwrap(); + set_lane_association_failure_for_current_thread(Some(boundary)); + let result = db.lane_branch_for_turn("atomic-materialized-turn", Some("main"), None); + set_lane_association_failure_for_current_thread(None); + assert!(result.is_err(), "boundary {boundary} did not fail"); + assert_lane_association_absent(&db, "atomic-materialized-turn"); + assert_eq!(materialization_journal_count(&db), 0); + assert!(!destination.exists()); + } + } + + #[test] + fn post_commit_lane_failures_are_distinct_from_rolled_back_publication() { + let (_workspace, mut db) = initialized_trail(); + set_lane_association_failure_for_current_thread(Some("spawn_after_commit")); + let spawn = db.spawn_lane("committed-spawn", Some("main"), false, None, None); + set_lane_association_failure_for_current_thread(None); + assert!(matches!(spawn, Err(Error::CommittedRepairRequired { .. }))); + assert_lane_association_present(&db, "committed-spawn"); + + set_lane_association_failure_for_current_thread(Some("turn_after_commit")); + let turn = db.lane_branch_for_turn("committed-turn", Some("main"), None); + set_lane_association_failure_for_current_thread(None); + assert!(matches!(turn, Err(Error::CommittedRepairRequired { .. }))); + assert_lane_association_present(&db, "committed-turn"); + + db.spawn_lane("committed-ensure", Some("main"), false, None, None) + .unwrap(); + set_lane_association_failure_for_current_thread(Some("ensure_after_commit")); + let ensure = db.ensure_lane_workdir_materialized("committed-ensure", None); + set_lane_association_failure_for_current_thread(None); + assert!(matches!(ensure, Err(Error::CommittedRepairRequired { .. }))); + assert!(db + .lane_branch("committed-ensure") + .unwrap() + .workdir + .is_some()); + } + + #[test] + fn all_post_commit_lane_steps_preserve_committed_repair_semantics() { + for boundary in ["spawn_ref_repair", "spawn_event"] { + let (_workspace, mut db) = initialized_trail(); + set_lane_association_failure_for_current_thread(Some(boundary)); + let result = db.spawn_lane("committed-spawn", Some("main"), false, None, None); + set_lane_association_failure_for_current_thread(None); + assert!( + matches!(result, Err(Error::CommittedRepairRequired { .. })), + "boundary {boundary} returned {result:?}" + ); + assert_lane_association_present(&db, "committed-spawn"); + } + + for boundary in ["spawn_journal_completion", "spawn_marker"] { + let (_workspace, mut db) = initialized_trail(); + set_lane_association_failure_for_current_thread(Some(boundary)); + let result = db.spawn_lane("committed-spawn", Some("main"), true, None, None); + set_lane_association_failure_for_current_thread(None); + assert!(matches!(result, Err(Error::CommittedRepairRequired { .. }))); + assert_lane_association_present(&db, "committed-spawn"); + } + + for boundary in ["ensure_journal_completion", "ensure_event", "ensure_marker"] { + let (_workspace, mut db) = initialized_trail(); + db.spawn_lane("committed-ensure", Some("main"), false, None, None) + .unwrap(); + set_lane_association_failure_for_current_thread(Some(boundary)); + let result = db.ensure_lane_workdir_materialized("committed-ensure", None); + set_lane_association_failure_for_current_thread(None); + assert!(matches!(result, Err(Error::CommittedRepairRequired { .. }))); + assert!(db + .lane_branch("committed-ensure") + .unwrap() + .workdir + .is_some()); + } + + for boundary in ["turn_ref_repair", "turn_event"] { + let (_workspace, mut db) = initialized_trail(); + set_lane_association_failure_for_current_thread(Some(boundary)); + let result = db.lane_branch_for_turn("committed-turn", Some("main"), None); + set_lane_association_failure_for_current_thread(None); + assert!( + matches!(result, Err(Error::CommittedRepairRequired { .. })), + "boundary {boundary} returned {result:?}" + ); + assert_lane_association_present(&db, "committed-turn"); + } + + for boundary in ["turn_journal_completion", "turn_marker"] { + let (_workspace, mut db) = initialized_trail(); + db.config_set("lane.default_materialize", "true").unwrap(); + set_lane_association_failure_for_current_thread(Some(boundary)); + let result = db.lane_branch_for_turn("committed-turn", Some("main"), None); + set_lane_association_failure_for_current_thread(None); + assert!(matches!(result, Err(Error::CommittedRepairRequired { .. }))); + assert_lane_association_present(&db, "committed-turn"); + } + + for repair in [ + "journal completion", + "marker publication", + "workspace view publication", + "event publication", + "ref mirror repair", + ] { + let result: Result<()> = committed_lane_step( + "operation_test", + repair, + Err(Error::InvalidInput("injected post-commit failure".into())), + ); + assert!(matches!(result, Err(Error::CommittedRepairRequired { .. }))); + } } - Ok(()) } diff --git a/trail/src/db/lane/mod.rs b/trail/src/db/lane/mod.rs index 8120cc7..db47fb3 100644 --- a/trail/src/db/lane/mod.rs +++ b/trail/src/db/lane/mod.rs @@ -14,11 +14,29 @@ mod readiness; mod rewind; mod turns; mod workdir; +mod workspace_cargo; +mod workspace_cmake; +mod workspace_environment; mod workspace_git; +mod workspace_go; mod workspace_layer; mod workspace_node; +mod workspace_oci; +mod workspace_plugin; +mod workspace_python; +mod workspace_recipe; +mod workspace_runtime; mod workspace_view; +#[cfg(debug_assertions)] +pub(crate) use lifecycle::set_sparse_selection_write_failure_for_current_thread; pub(crate) use workdir::ViewMutationBarrier; -pub(crate) use workspace_layer::WorkspaceLayerBinding; +#[cfg(debug_assertions)] +pub(crate) use workdir::{ + install_lane_record_after_c2_write_for_current_thread, run_changed_path_view_flow, + set_lane_record_postcommit_failure_for_current_thread, +}; +pub(crate) use workspace_layer::{ + EnvironmentLayerActivation, EnvironmentLayerOutputActivation, WorkspaceLayerBinding, +}; pub(crate) use workspace_view::WorkspaceMountLease; diff --git a/trail/src/db/lane/patch_edits.rs b/trail/src/db/lane/patch_edits.rs index 5e3b7e8..900848a 100644 --- a/trail/src/db/lane/patch_edits.rs +++ b/trail/src/db/lane/patch_edits.rs @@ -41,7 +41,12 @@ impl Trail { lines_by_path.insert(path.clone(), self.load_text_lines(text_id)?); } let lines = lines_by_path.get_mut(&path).expect("path was loaded"); - let Some(line_idx) = lines.iter().position(|line| line.line_id_key() == *line_id) + let parsed_line_id = parse_line_id_key(line_id) + .map_err(|error| Error::PatchRejected(error.to_string()))?; + let storage_line_id = line_id_key_value(&parsed_line_id); + let Some(line_idx) = lines + .iter() + .position(|line| line.line_id_key() == storage_line_id) else { return Err(Error::PatchRejected(format!( "replace_line line_id `{line_id}` not found in `{path}`" @@ -213,7 +218,13 @@ impl Trail { ))); }; let mut lines = self.load_text_lines(text_id)?; - let Some(line_idx) = lines.iter().position(|line| line.line_id_key() == line_id) else { + let parsed_line_id = + parse_line_id_key(&line_id).map_err(|error| Error::PatchRejected(error.to_string()))?; + let storage_line_id = line_id_key_value(&parsed_line_id); + let Some(line_idx) = lines + .iter() + .position(|line| line.line_id_key() == storage_line_id) + else { return Err(Error::PatchRejected(format!( "replace_line line_id `{line_id}` not found in `{path}`" ))); @@ -272,7 +283,7 @@ mod tests { let head = db.get_ref("refs/branches/main").unwrap(); let files = db.load_root_files(&head.root_id).unwrap(); let why = db.why("README.md:1", Some("main")).unwrap(); - let line_id = format!("{}:{}", why.line_id.origin_change.0, why.line_id.local_seq); + let line_id = why.line_id.alias(); let edit = PatchEdit::ReplaceLine { path: "README.md".to_string(), line_id, @@ -296,7 +307,7 @@ mod tests { .apply_patch_edit_to_files( edit, &mut files, - &ChangeId("ch_internal_replace_line_guard".to_string()), + &ChangeId("change_internal_replace_line_guard".to_string()), &mut file_seq, &mut line_seq, &mut manual_line_changes, diff --git a/trail/src/db/lane/patch_policy.rs b/trail/src/db/lane/patch_policy.rs index 14a70e2..4e3eb2f 100644 --- a/trail/src/db/lane/patch_policy.rs +++ b/trail/src/db/lane/patch_policy.rs @@ -16,17 +16,59 @@ impl Trail { &self, head_root_id: &ObjectId, previous_touched: &BTreeMap, - target_touched: &BTreeMap, - ) -> Result<()> { - let mut paths = self - .load_root_paths(head_root_id)? - .into_iter() - .collect::>(); - for path in previous_touched.keys() { - paths.remove(path); + edits: &[PatchEdit], + ) -> Result { + let mut target_paths = previous_touched.keys().cloned().collect::>(); + for edit in edits { + match edit { + PatchEdit::Write { path, .. } | PatchEdit::WriteBytes { path, .. } => { + target_paths.insert(normalize_relative_path(path)?); + } + PatchEdit::ReplaceLine { path, .. } => { + let path = normalize_relative_path(path)?; + if !target_paths.contains(&path) { + return Err(Error::PatchRejected(format!( + "replace_line path `{path}` is absent" + ))); + } + } + PatchEdit::Delete { path } => { + let path = normalize_relative_path(path)?; + if !target_paths.remove(&path) { + return Err(Error::PatchRejected(format!( + "delete path `{path}` is absent" + ))); + } + } + PatchEdit::Rename { from, to } => { + let from = normalize_relative_path(from)?; + let to = normalize_relative_path(to)?; + if target_paths.contains(&to) { + return Err(Error::PatchRejected(format!( + "rename destination `{to}` already exists" + ))); + } + if !target_paths.remove(&from) { + return Err(Error::PatchRejected(format!( + "rename source `{from}` is absent" + ))); + } + target_paths.insert(to); + } + } } - paths.extend(target_touched.keys().cloned()); - validate_no_case_fold_collisions(paths.iter()) + let removals = previous_touched + .keys() + .filter(|path| !target_paths.contains(*path)) + .cloned() + .collect::>(); + let additions = target_paths + .iter() + .filter(|path| !previous_touched.contains_key(*path)) + .cloned() + .collect::>(); + let previous_root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, head_root_id)?; + self.validate_and_update_case_fold_index(&previous_root, &removals, &additions) } pub(crate) fn ensure_lane_record_policy( @@ -43,21 +85,22 @@ impl Trail { head_root_id: &ObjectId, summaries: &[FileDiffSummary], ) -> Result<()> { - let mut paths = self - .load_root_paths(head_root_id)? - .into_iter() - .collect::>(); + let mut removals = Vec::new(); + let mut additions = Vec::new(); for summary in summaries { let path = normalize_relative_path(&summary.path)?; if let Some(old_path) = &summary.old_path { - paths.remove(&normalize_relative_path(old_path)?); + removals.push(normalize_relative_path(old_path)?); } - paths.remove(&path); - if summary.kind != FileChangeKind::Deleted { - paths.insert(path); + match summary.kind { + FileChangeKind::Added => additions.push(path), + FileChangeKind::Deleted => removals.push(path), + FileChangeKind::Renamed => additions.push(path), + FileChangeKind::Modified | FileChangeKind::TypeChanged => {} } } - validate_no_case_fold_collisions(paths.iter()) + let previous_root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, head_root_id)?; + self.validate_case_fold_index_update_read_only(&previous_root, &removals, &additions) } pub(crate) fn preview_lane_record_policy( @@ -220,7 +263,6 @@ impl Trail { if paths.is_empty() { return Ok(()); } - validate_no_case_fold_collisions(paths.iter())?; self.ensure_changed_path_limit(paths.len())?; self.ensure_sparse_policy_paths_allowed(branch, &paths)?; self.ensure_claim_policy_paths_allowed(branch, &paths) @@ -240,7 +282,6 @@ impl Trail { if paths.is_empty() { return Ok(preview); } - validate_no_case_fold_collisions(paths.iter())?; let max_changed_paths = self.config.lane.max_changed_paths; if max_changed_paths > 0 && paths.len() as u64 > max_changed_paths { diff --git a/trail/src/db/lane/patching.rs b/trail/src/db/lane/patching.rs index df131c9..6e6d3c8 100644 --- a/trail/src/db/lane/patching.rs +++ b/trail/src/db/lane/patching.rs @@ -1,13 +1,108 @@ use super::*; +#[cfg(debug_assertions)] +std::thread_local! { + static FAIL_PATCH_POST_COMMIT: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +#[cfg(test)] +fn set_patch_post_commit_failure_for_current_thread(enabled: bool) { + FAIL_PATCH_POST_COMMIT.with(|fail| fail.set(enabled)); +} + +#[cfg(debug_assertions)] +fn fail_patch_post_commit_if_requested() -> Result<()> { + if FAIL_PATCH_POST_COMMIT.with(std::cell::Cell::get) { + return Err(Error::Io(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected lane patch post-commit failure", + ))); + } + Ok(()) +} + +#[cfg(not(debug_assertions))] +fn fail_patch_post_commit_if_requested() -> Result<()> { + Ok(()) +} + +fn lane_patch_committed_repair(operation: &ObjectId, error: Error) -> Error { + match error { + Error::CommittedRepairRequired { .. } => error, + error => Error::CommittedRepairRequired { + operation: operation.0.clone(), + repair: "lane patch post-commit metadata and marker".into(), + reason: error.to_string(), + }, + } +} + +fn retry_patch_after_full_reconciliation(error: &Error) -> bool { + matches!( + error, + Error::ChangeLedgerReconcileRequired { reason, .. } + if reason == "controlled interval sidecar chain requires reconciliation" + ) +} + impl Trail { pub fn apply_lane_patch( &mut self, lane: &str, patch: PatchDocument, ) -> Result { - let _lock = self.acquire_write_lock()?; - self.apply_lane_patch_locked(lane, patch, None) + let metrics = self.operation_metrics.clone(); + let result_metrics = metrics.clone(); + profile_operation_metrics( + metrics.as_ref(), + OperationMetricsKind::StructuredPatch, + || { + self.reset_case_fold_index_metrics(); + if crate::db::change_ledger::command_authority_enabled() { + let branch = self.lane_branch(lane)?; + if self.lane_uses_native_materialized_ledger(&branch)? { + crate::db::change_ledger::materialized_lane_daemon_expected_scope( + self, lane, + )?; + } + } + let _lock = if crate::db::change_ledger::command_authority_enabled() { + None + } else { + Some(self.acquire_write_lock()?) + }; + // A native observer may have durably advanced its sidecar while + // SQLite publication was momentarily unavailable. The + // controlled projection is target-based (not an incremental + // edit of workdir bytes), so after a full reconciliation it is + // safe to retry the exact patch once. Keeping this here makes + // CLI, HTTP, and MCP callers share the same recovery behavior. + let result = self.apply_lane_patch_with_reconciliation_retry(lane, patch, None); + if let (Some(metrics), Ok(report)) = (&result_metrics, &result) { + metrics.add(OperationMetricsDelta { + final_path_count: saturating_u64_from_usize(report.changed_paths.len()), + ..OperationMetricsDelta::default() + }); + } + result + }, + ) + } + + pub(crate) fn apply_lane_patch_with_reconciliation_retry( + &mut self, + lane: &str, + patch: PatchDocument, + api_turn: Option<&LaneTurn>, + ) -> Result { + let retry_patch = patch.clone(); + match self.apply_lane_patch_locked(lane, patch, api_turn) { + Err(error) if retry_patch_after_full_reconciliation(&error) => { + crate::db::change_ledger::materialized_lane_daemon_full_reconcile(self, lane)?; + self.apply_lane_patch_locked(lane, retry_patch, api_turn) + } + result => result, + } } pub(crate) fn apply_lane_patch_locked( @@ -16,10 +111,14 @@ impl Trail { patch: PatchDocument, api_turn: Option<&LaneTurn>, ) -> Result { + // TRAIL_FS_PRODUCER: structured_patch_projection StructuredPatchProjection controlled + self.reset_case_fold_index_metrics(); validate_ref_segment(lane)?; let lane_row = self.lane_branch(lane)?; let ref_name = lane_row.ref_name.clone(); let head = self.get_ref(&ref_name)?; + let controlled_projection = crate::db::change_ledger::command_authority_enabled() + && self.lane_uses_native_materialized_ledger(&lane_row)?; if let Some(turn) = api_turn { if turn.lane_id != lane_row.lane_id { return Err(Error::InvalidInput(format!( @@ -49,6 +148,7 @@ impl Trail { } } } + let empty_patch = patch.edits.is_empty(); for edit in &patch.edits { self.ensure_patch_edit_allowed(edit, patch.allow_ignored)?; } @@ -72,6 +172,11 @@ impl Trail { self.ensure_lane_patch_policy(&lane_row, &patch, &touched_paths)?; let previous_touched = self.load_root_files_for_paths(&head.root_id, &touched_paths)?; self.preflight_replace_line_batch(&patch.edits, &previous_touched)?; + let case_fold_tree = self.ensure_patch_final_root_paths_safe( + &head.root_id, + &previous_touched, + &patch.edits, + )?; let actor = Actor::lane(lane); let change_id = self.allocate_change_id(&actor.id, "lane_patch")?; let mut target_touched = previous_touched.clone(); @@ -90,13 +195,12 @@ impl Trail { )?; } - self.ensure_patch_final_root_paths_safe(&head.root_id, &previous_touched, &target_touched)?; - - let built = self.build_root_from_touched_file_entries_incremental( + let built = self.build_root_from_touched_file_entries_incremental_with_case_fold_tree( &head.root_id, &previous_touched, &target_touched, &change_id, + case_fold_tree, )?; let mut diff = self.diff_file_maps(&previous_touched, &target_touched)?; for (path, file_id, line) in manual_line_changes { @@ -114,7 +218,7 @@ impl Trail { } } } - if diff.changes.is_empty() { + if diff.changes.is_empty() && !empty_patch { return Err(Error::PatchRejected( "patch produced no changes".to_string(), )); @@ -153,27 +257,125 @@ impl Trail { changes: diff.changes, created_at: now_ts(), }; - let operation_id = self.store_operation(&operation)?; - self.advance_ref_cas(&head, &change_id, &built.root_id, &operation_id)?; - let message_id = if let Some(message) = patch_message { - Some(self.store_message( - "lane", - &message, - Some(&lane_row.lane_id), - patch_session_id.as_deref(), - Some(&change_id), - operation.created_at, - )?) + // All reads that may recover/reconcile public authority, plus patch + // policy and cleanliness preflight, must finish before Prepared. The + // controlled apply closure below contains only the bounded durable + // filesystem projection. + let controlled_expected = if controlled_projection { + Some( + crate::db::change_ledger::prepare_materialized_lane_controlled_projection( + self, + &lane_row.lane_id, + )?, + ) } else { None }; - self.insert_lane_event_with_context( + let operation_id = if let Some(expected) = controlled_expected { + let evidence = crate::db::change_ledger::IntentEvidence { + exact_paths: touched_paths + .iter() + .map(|path| crate::db::change_ledger::LedgerPath::parse(path)) + .collect::>>()?, + complete_prefixes: Vec::new(), + }; + crate::db::change_ledger::run_ref_advancing_projection( + self, + &expected, + &head, + &lane_row.lane_id, + crate::db::change_ledger::IntentProducer::StructuredPatchProjection, + &operation, + &evidence, + crate::db::change_ledger::RefAdvancingProjectionMode::ControlledIntent, + |db, intent| { + crate::db::change_ledger::with_materialized_lane_controlled_interval( + db, + &lane_row.lane_id, + intent, + &evidence, + |db| { + db.invalidate_lane_marker_if_materialized(&lane_row)?; + db.apply_lane_patch_workdir_projection( + &lane_row, + &head, + &built.root_id, + &changed_summaries, + &previous_touched, + &target_touched, + false, + ) + }, + |db, policy, candidates| { + let comparison = db.compare_controlled_projection_target( + policy, + candidates, + &built.root_id, + crate::db::change_ledger::CandidateMaterialization::ManifestOnly, + )?; + if comparison.summaries.is_empty() { + Ok(()) + } else { + Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "structured patch pinned verification did not match its target root".into(), + command: format!("trail lane status {}", lane_row.lane_id), + }) + } + }, + ) + }, + |db, publication| { + crate::db::change_ledger::accept_materialized_lane_daemon_baseline( + db, + &lane_row.lane_id, + &expected, + &publication.baseline, + ) + }, + )? + .operation_id + } else { + self.invalidate_lane_marker_if_materialized(&lane_row)?; + self.apply_lane_patch_workdir_projection( + &lane_row, + &head, + &built.root_id, + &changed_summaries, + &previous_touched, + &target_touched, + true, + )?; + crate::db::change_ledger::commit_lane_operation_atomic( + self, + &head, + &lane_row.lane_id, + &operation, + None, + )? + }; + let post_commit = (|| -> Result<()> { + fail_patch_post_commit_if_requested()?; + let message_id = if let Some(message) = patch_message { + Some(self.store_message( + "lane", + &message, + Some(&lane_row.lane_id), + patch_session_id.as_deref(), + Some(&change_id), + operation.created_at, + )?) + } else { + None + }; + self.insert_lane_event_with_context( &lane_row.lane_id, patch_session_id.as_deref(), Some(&turn_id), "patch_applied", - Some(&change_id), - message_id.as_ref(), + Some(&change_id), + message_id.as_ref(), &serde_json::json!({ "ref_name": ref_name.clone(), "root_id": built.root_id.0.clone(), @@ -183,74 +385,34 @@ impl Trail { "changed_paths": changed_summaries.iter().map(|item| item.path.clone()).collect::>() }), )?; - self.conn.execute( - "UPDATE lane_branches SET head_change = ?1, head_root = ?2, session_id = COALESCE(?3, session_id), updated_at = ?4 \ - WHERE lane_id = ?5", - params![ - change_id.0, - built.root_id.0, - patch_session_id, - now_ts(), - lane_row.lane_id - ], - )?; - if let Some(workdir) = &lane_row.workdir { - let workdir_path = Path::new(workdir); - let sparse_paths = self.lane_sparse_workdir_paths(&lane_row, workdir_path)?; - let (previous_files, target_files) = if let Some(sparse_paths) = sparse_paths { - let mut selected_paths = sparse_paths; - for summary in &changed_summaries { - selected_paths.push(summary.path.clone()); - if let Some(old_path) = &summary.old_path { - selected_paths.push(old_path.clone()); - } - } - selected_paths.sort(); - selected_paths.dedup(); - let previous_files = - self.load_root_files_for_paths(&head.root_id, &selected_paths)?; - let target_files = - self.load_root_files_for_paths(&built.root_id, &selected_paths)?; - self.write_sparse_workdir_manifest(workdir_path, target_files.keys())?; - (previous_files, target_files) - } else if self.refresh_clean_materialized_workdir_for_lane_patch( - workdir_path, - &head.root_id, - &built.root_id, - &previous_touched, - &target_touched, - )? { - (BTreeMap::new(), BTreeMap::new()) - } else { - ( - self.load_root_files(&head.root_id)?, - self.load_root_files(&built.root_id)?, - ) - }; - if !previous_files.is_empty() || !target_files.is_empty() { - self.materialize_files_best_effort_at( - workdir_path, - &previous_files, - &target_files, - )?; - self.write_clean_workdir_manifest( - workdir_path, - &built.root_id, - &target_files, - target_files.keys(), + if patch_session_id.is_some() { + self.conn.execute( + "UPDATE lane_branches SET session_id=COALESCE(?1,session_id),updated_at=?2 + WHERE lane_id=?3 AND head_change=?4 AND head_root=?5", + params![ + patch_session_id, + now_ts(), + lane_row.lane_id, + change_id.0, + built.root_id.0, + ], )?; } - } - if api_turn.is_some() { - self.update_lane_turn_progress(&turn_id, "patch_applied", Some(&change_id))?; - } else { - self.finish_lane_turn(&turn_id, "patch_applied", Some(&change_id))?; - } + if api_turn.is_some() { + self.update_lane_turn_progress(&turn_id, "patch_applied", Some(&change_id))?; + } else { + self.finish_lane_turn(&turn_id, "patch_applied", Some(&change_id))?; + } + self.publish_lane_marker_if_materialized(lane)?; + Ok(()) + })(); + post_commit.map_err(|error| lane_patch_committed_repair(&operation_id, error))?; Ok(LanePatchReport { lane_id: lane_row.lane_id, operation: change_id, root_id: built.root_id, changed_paths: changed_summaries, + path_index: self.case_fold_index_metrics_report(), }) } @@ -262,13 +424,7 @@ impl Trail { previous_touched: &BTreeMap, target_touched: &BTreeMap, ) -> Result { - if !matches!( - self.cached_workdir_manifest_status(workdir_path, previous_root_id)?, - CachedWorkdirManifestStatus::Clean - ) { - return Ok(false); - } - if !self.clean_workdir_manifest_allows_file_subset_update( + if !self.clean_workdir_manifest_allows_touched_path_update( workdir_path, previous_root_id, previous_touched, @@ -286,4 +442,230 @@ impl Trail { target_touched, ) } + + fn apply_lane_patch_workdir_projection( + &self, + lane_row: &LaneBranch, + head: &RefRecord, + next_root_id: &ObjectId, + changed_summaries: &[FileDiffSummary], + previous_touched: &BTreeMap, + target_touched: &BTreeMap, + allow_legacy_manifest_shortcut: bool, + ) -> Result<()> { + let Some(workdir) = &lane_row.workdir else { + return Ok(()); + }; + let workdir_path = Path::new(workdir); + let sparse_paths = self.lane_sparse_workdir_paths(lane_row, workdir_path)?; + let (previous_files, target_files) = if let Some(sparse_paths) = sparse_paths { + let mut selected_paths = sparse_paths; + for summary in changed_summaries { + selected_paths.push(summary.path.clone()); + if let Some(old_path) = &summary.old_path { + selected_paths.push(old_path.clone()); + } + } + selected_paths.sort(); + selected_paths.dedup(); + let previous_files = self.load_root_files_for_paths(&head.root_id, &selected_paths)?; + let target_files = self.load_root_files_for_paths(next_root_id, &selected_paths)?; + self.write_sparse_workdir_manifest(workdir_path, target_files.keys())?; + (previous_files, target_files) + } else if allow_legacy_manifest_shortcut + && self.refresh_clean_materialized_workdir_for_lane_patch( + workdir_path, + &head.root_id, + next_root_id, + previous_touched, + target_touched, + )? + { + (BTreeMap::new(), BTreeMap::new()) + } else if !allow_legacy_manifest_shortcut { + // Controlled projection already holds exact changed-path evidence + // and verifies the projected bytes against the immutable target + // root before publication. Project only the touched maps: loading + // and materializing both complete roots here would turn a k-path + // patch (including genuine k=0) back into O(repository paths). + self.materialize_files_at(workdir_path, previous_touched, target_touched)?; + return Ok(()); + } else { + ( + self.load_root_files(&head.root_id)?, + self.load_root_files(next_root_id)?, + ) + }; + if !previous_files.is_empty() || !target_files.is_empty() { + if crate::db::change_ledger::command_authority_enabled() { + self.materialize_files_at(workdir_path, &previous_files, &target_files)?; + } else { + self.materialize_files_best_effort_at( + workdir_path, + &previous_files, + &target_files, + )?; + } + self.write_clean_workdir_manifest( + workdir_path, + next_root_id, + &target_files, + target_files.keys(), + )?; + } + Ok(()) + } + + pub(crate) fn lane_uses_native_materialized_ledger(&self, branch: &LaneBranch) -> Result { + let Some(workdir) = branch.workdir.as_deref() else { + return Ok(false); + }; + if !Path::new(workdir).is_dir() { + return Ok(false); + } + let mode = self.lane_workdir_mode_for(&self.lane_record(&branch.lane_id)?, branch)?; + Ok(!mode.is_transparent_cow()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn controlled_materialized_projection_never_loads_complete_roots() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "base\n").unwrap(); + for index in 0..128 { + fs::write( + workspace.path().join(format!("untouched-{index:03}.txt")), + format!("untouched {index}\n"), + ) + .unwrap(); + } + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane("bounded-patch", Some("main"), true, None, None) + .unwrap(); + let branch = db.lane_branch("bounded-patch").unwrap(); + let head = db.get_ref(&branch.ref_name).unwrap(); + let workdir = PathBuf::from(branch.workdir.as_deref().unwrap()); + let previous = db + .load_root_files_for_paths(&head.root_id, &["README.md".to_string()]) + .unwrap(); + + let metrics = Arc::new(OperationMetricsState::default()); + db.operation_metrics = Some(Arc::clone(&metrics)); + db.apply_lane_patch_workdir_projection( + &branch, + &head, + &head.root_id, + &[], + &BTreeMap::new(), + &BTreeMap::new(), + false, + ) + .unwrap(); + assert!(workdir.join("README.md").is_file()); + assert!(workdir.join("untouched-127.txt").is_file()); + + db.apply_lane_patch_workdir_projection( + &branch, + &head, + &head.root_id, + &[], + &previous, + &BTreeMap::new(), + false, + ) + .unwrap(); + assert!(!workdir.join("README.md").exists()); + assert!(workdir.join("untouched-127.txt").is_file()); + let counters = metrics.snapshot(); + assert_eq!(counters.full_root_range_count, 0); + assert_eq!(counters.full_filesystem_walk_count, 0); + } + + #[test] + fn empty_patch_records_a_successful_noop_operation() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "base\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let metrics = Arc::new(OperationMetricsState::default()); + db.operation_metrics = Some(Arc::clone(&metrics)); + db.spawn_lane("empty-patch", Some("main"), false, None, None) + .unwrap(); + let before = db.get_ref("refs/lanes/empty-patch").unwrap(); + let before_files = db.load_root_files(&before.root_id).unwrap(); + let report = db + .apply_lane_patch( + "empty-patch", + PatchDocument { + base_change: Some(before.change_id.0.clone()), + message: Some("empty patch checkpoint".into()), + session_id: None, + allow_ignored: false, + allow_stale: false, + edits: Vec::new(), + }, + ) + .unwrap(); + let after = db.get_ref("refs/lanes/empty-patch").unwrap(); + let after_files = db.load_root_files(&after.root_id).unwrap(); + + assert!(report.changed_paths.is_empty()); + assert_eq!(before_files, after_files); + assert_eq!(report.root_id, after.root_id); + assert_ne!(after.change_id, before.change_id); + assert_eq!(report.operation, after.change_id); + let operation_metrics = metrics.last_report(); + assert_eq!(operation_metrics.operation, "structured_patch"); + assert_eq!(operation_metrics.outcome, OperationMetricsOutcome::Success); + assert_eq!(operation_metrics.final_path_count, 0); + } + + #[test] + fn post_commit_patch_failure_reports_repair_required_and_keeps_committed_head() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "base\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane("patch-repair", Some("main"), false, None, None) + .unwrap(); + let before = db.get_ref("refs/lanes/patch-repair").unwrap(); + let patch = PatchDocument { + base_change: Some(before.change_id.0.clone()), + message: Some("committed patch".into()), + session_id: None, + allow_ignored: false, + allow_stale: false, + edits: vec![PatchEdit::Write { + path: "README.md".into(), + content: "patched\n".into(), + executable: false, + }], + }; + + set_patch_post_commit_failure_for_current_thread(true); + let result = db.apply_lane_patch("patch-repair", patch); + set_patch_post_commit_failure_for_current_thread(false); + + let error = result.unwrap_err(); + let after = db.get_ref("refs/lanes/patch-repair").unwrap(); + assert_ne!(after.change_id, before.change_id); + assert_ne!(after.root_id, before.root_id); + match error { + Error::CommittedRepairRequired { + operation, + repair, + reason, + } => { + assert_eq!(operation, after.operation_id.0); + assert!(repair.contains("lane patch")); + assert!(reason.contains("injected lane patch post-commit failure")); + } + other => panic!("expected committed repair error, got {other:?}"), + } + } } diff --git a/trail/src/db/lane/readiness.rs b/trail/src/db/lane/readiness.rs index e889b89..b0a2b0e 100644 --- a/trail/src/db/lane/readiness.rs +++ b/trail/src/db/lane/readiness.rs @@ -123,6 +123,50 @@ impl Trail { )), } } + if let Some(generation) = self.active_environment_generation(lane)? { + for component in generation.components { + for resource in component.runtime_resources { + for secret in &resource.secret_statuses { + if secret.reference.required && secret.status != "available" { + blockers.push(readiness_issue( + "environment_secret_unavailable", + format!( + "required secret reference `{}` for runtime resource `{}` is unavailable", + secret.reference.name, resource.declaration.name + ), + Some(serde_json::json!({ + "generation_id": generation.generation_id, + "component_id": component.component_id, + "resource": resource.declaration.name, + "secret": secret.reference.name, + "provider": secret.reference.provider, + "status": secret.status, + "reason": secret.reason, + })), + )); + } + } + if resource.status != "running" || resource.health_status != "healthy" { + blockers.push(readiness_issue( + "environment_runtime_unhealthy", + format!( + "runtime resource `{}` for component `{}` is not healthy", + resource.declaration.name, component.component_id + ), + Some(serde_json::json!({ + "generation_id": generation.generation_id, + "component_id": component.component_id, + "resource": resource.declaration.name, + "allocation_id": resource.allocation_id, + "status": resource.status, + "health_status": resource.health_status, + "reason": resource.reason, + })), + )); + } + } + } + } for layer in self.workspace_view_layer_reports(&view.view_id)? { if layer.state != "ready" || self.verify_workspace_layer(&layer.layer_id).is_err() { blockers.push(readiness_issue( diff --git a/trail/src/db/lane/rewind.rs b/trail/src/db/lane/rewind.rs index 13e8e45..32ddeed 100644 --- a/trail/src/db/lane/rewind.rs +++ b/trail/src/db/lane/rewind.rs @@ -1,5 +1,42 @@ use super::*; +#[cfg(debug_assertions)] +std::thread_local! { + static FAIL_REWIND_POST_COMMIT: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +#[cfg(test)] +fn set_rewind_post_commit_failure_for_current_thread(enabled: bool) { + FAIL_REWIND_POST_COMMIT.with(|fail| fail.set(enabled)); +} + +#[cfg(debug_assertions)] +fn fail_rewind_post_commit_if_requested() -> Result<()> { + if FAIL_REWIND_POST_COMMIT.with(std::cell::Cell::get) { + return Err(Error::Io(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected lane rewind post-commit failure", + ))); + } + Ok(()) +} + +#[cfg(not(debug_assertions))] +fn fail_rewind_post_commit_if_requested() -> Result<()> { + Ok(()) +} + +fn lane_rewind_committed_repair(operation: &str, error: Error) -> Error { + match error { + Error::CommittedRepairRequired { .. } => error, + error => Error::CommittedRepairRequired { + operation: operation.to_string(), + repair: "lane rewind post-commit metadata and workdir".into(), + reason: error.to_string(), + }, + } +} + impl Trail { pub fn rewind_lane( &mut self, @@ -8,6 +45,7 @@ impl Trail { record_current: bool, sync_workdir: bool, ) -> Result { + // TRAIL_FS_PRODUCER: lane_rewind_projection RestoreProjection controlled validate_ref_segment(lane)?; if target.trim().is_empty() { return Err(Error::InvalidInput( @@ -31,8 +69,30 @@ impl Trail { } } + let controlled_rewind = + if crate::db::change_ledger::command_authority_enabled() && sync_workdir { + let branch = self.lane_branch(lane)?; + let materialized = branch + .workdir + .as_deref() + .is_some_and(|workdir| Path::new(workdir).is_dir()); + let native = materialized + && !self + .lane_workdir_mode_for(&self.lane_record(&branch.lane_id)?, &branch)? + .is_transparent_cow(); + if native { + crate::db::change_ledger::materialized_lane_daemon_expected_scope(self, lane)?; + } + native + } else { + false + }; let mut report = { - let _lock = self.acquire_write_lock()?; + let _lock = if crate::db::change_ledger::command_authority_enabled() { + None + } else { + Some(self.acquire_write_lock()?) + }; let branch = self.lane_branch(lane)?; let head = self.get_ref(&branch.ref_name)?; if sync_workdir { @@ -79,33 +139,135 @@ impl Trail { changes: diff.changes, created_at: now_ts(), }; - let operation_id = self.store_operation(&operation)?; - self.advance_ref_cas(&head, &change_id, &target_ref.root_id, &operation_id)?; - self.conn.execute( - "UPDATE lane_branches SET head_change = ?1, head_root = ?2, status = 'active', updated_at = ?3 WHERE lane_id = ?4", - params![change_id.0, target_ref.root_id.0, now_ts(), branch.lane_id], - )?; - self.insert_lane_event_with_context( - &branch.lane_id, - branch.session_id.as_deref(), - None, - "lane_rewound", - Some(&change_id), - None, - &serde_json::json!({ - "target": target, - "target_change": target_ref.change_id.0, - "target_root": target_ref.root_id.0, - "previous_change": head.change_id.0, - "previous_root": head.root_id.0, - "record_current": record_current, - "recorded_current": recorded_current.as_ref().map(|id| id.0.clone()), - "preserved_branch": preserved_branch.clone(), - "preserved_ref": preserved_ref.clone(), - "sync_workdir": sync_workdir, - "changed_paths": diff.summaries.iter().map(|item| item.path.clone()).collect::>() - }), - )?; + let operation_id = if controlled_rewind { + let expected = + crate::db::change_ledger::prepare_materialized_lane_controlled_projection( + self, + &branch.lane_id, + )?; + let mut evidence_paths = diff + .summaries + .iter() + .flat_map(|summary| { + std::iter::once(summary.path.as_str()).chain(summary.old_path.as_deref()) + }) + .map(crate::db::change_ledger::LedgerPath::parse) + .collect::>>()?; + evidence_paths.sort(); + evidence_paths.dedup(); + let evidence = crate::db::change_ledger::IntentEvidence { + exact_paths: evidence_paths, + complete_prefixes: Vec::new(), + }; + crate::db::change_ledger::run_ref_advancing_projection( + self, + &expected, + &head, + &branch.lane_id, + crate::db::change_ledger::IntentProducer::RestoreProjection, + &operation, + &evidence, + crate::db::change_ledger::RefAdvancingProjectionMode::ControlledIntent, + |db, intent| { + crate::db::change_ledger::with_materialized_lane_controlled_interval( + db, + &branch.lane_id, + intent, + &evidence, + |db| { + db.invalidate_lane_marker_if_materialized(&branch)?; + db.apply_rewind_workdir_projection( + &branch, + &head.root_id, + &target_ref.root_id, + ) + }, + |db, policy, candidates| { + let comparison = db.compare_controlled_projection_target( + policy, + candidates, + &target_ref.root_id, + crate::db::change_ledger::CandidateMaterialization::ManifestOnly, + )?; + if comparison.summaries.is_empty() { + Ok(()) + } else { + Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "rewind pinned verification did not match its target root".into(), + command: format!("trail lane status {}", branch.lane_id), + }) + } + }, + ) + }, + |db, publication| { + crate::db::change_ledger::accept_materialized_lane_daemon_baseline( + db, + &branch.lane_id, + &expected, + &publication.baseline, + ) + }, + )? + .operation_id + } else { + self.invalidate_lane_marker_if_materialized(&branch)?; + crate::db::change_ledger::commit_lane_operation_atomic( + self, + &head, + &branch.lane_id, + &operation, + None, + )? + }; + let post_commit = (|| -> Result<()> { + fail_rewind_post_commit_if_requested()?; + if crate::db::change_ledger::command_authority_enabled() && !sync_workdir { + self.conn.execute( + "UPDATE changed_path_scopes + SET trust_state='stale_baseline',trust_reason='rewind_without_workdir_alignment',updated_at=?1 + WHERE scope_id=?2 AND retired_at IS NULL", + params![ + now_ts(), + crate::db::change_ledger::materialized_lane_scope_id( + &self.config.workspace.id.0, + &branch.lane_id, + ) + .to_text() + ], + )?; + } + self.conn.execute( + "UPDATE lane_branches SET status='active',updated_at=?1 + WHERE lane_id=?2 AND head_change=?3 AND head_root=?4", + params![now_ts(), branch.lane_id, change_id.0, target_ref.root_id.0], + )?; + self.insert_lane_event_with_context( + &branch.lane_id, + branch.session_id.as_deref(), + None, + "lane_rewound", + Some(&change_id), + None, + &serde_json::json!({ + "target": target, + "target_change": target_ref.change_id.0, + "target_root": target_ref.root_id.0, + "previous_change": head.change_id.0, + "previous_root": head.root_id.0, + "record_current": record_current, + "recorded_current": recorded_current.as_ref().map(|id| id.0.clone()), + "preserved_branch": preserved_branch.clone(), + "preserved_ref": preserved_ref.clone(), + "sync_workdir": sync_workdir, + "changed_paths": diff.summaries.iter().map(|item| item.path.clone()).collect::>() + }), + )?; + Ok(()) + })(); + post_commit.map_err(|error| lane_rewind_committed_repair(&operation_id.0, error))?; LaneRewindReport { lane_id: branch.lane_id, @@ -121,13 +283,15 @@ impl Trail { recorded_current, preserved_branch, preserved_ref, - workdir: branch.workdir, - workdir_synced: false, + workdir: branch.workdir.clone(), + workdir_synced: controlled_rewind && branch.workdir.is_some(), } }; - if sync_workdir && report.workdir.is_some() { - let sync = self.sync_lane_workdir(lane, true)?; + if sync_workdir && report.workdir.is_some() && !controlled_rewind { + let sync = self + .sync_lane_workdir(lane, true) + .map_err(|error| lane_rewind_committed_repair(&report.operation.0, error))?; report.workdir = Some(sync.workdir); report.workdir_synced = true; } @@ -159,14 +323,98 @@ impl Trail { "could not allocate rewind preservation branch for `{lane}`" ))) } + + fn apply_rewind_workdir_projection( + &self, + branch: &LaneBranch, + previous_root: &ObjectId, + target_root: &ObjectId, + ) -> Result<()> { + let Some(workdir) = branch.workdir.as_deref() else { + return Ok(()); + }; + let workdir = Path::new(workdir); + let sparse = self.lane_sparse_workdir_paths(branch, workdir)?; + let (previous_files, target_files) = if let Some(paths) = sparse { + ( + self.load_root_files_for_selections(previous_root, &paths)?, + self.load_root_files_for_selections(target_root, &paths)?, + ) + } else { + ( + self.load_root_files(previous_root)?, + self.load_root_files(target_root)?, + ) + }; + if crate::db::change_ledger::command_authority_enabled() { + self.materialize_files_at(workdir, &previous_files, &target_files)?; + } else { + self.materialize_files_best_effort_at(workdir, &previous_files, &target_files)?; + } + if self.lane_sparse_workdir_paths(branch, workdir)?.is_some() { + self.write_sparse_workdir_manifest(workdir, target_files.keys())?; + } + self.write_clean_workdir_manifest(workdir, target_root, &target_files, target_files.keys()) + } } fn short_rewind_change_id(change_id: &ChangeId) -> String { - change_id - .0 - .strip_prefix("ch_") + crate::ids::change_id_hash(&change_id.0) .unwrap_or(&change_id.0) .chars() .take(12) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn post_commit_rewind_failure_reports_repair_required_and_keeps_committed_head() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "base\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let spawned = db + .spawn_lane("rewind-repair", Some("main"), false, None, None) + .unwrap(); + let base_change = spawned.base_change; + let base_root = db.get_ref("refs/lanes/rewind-repair").unwrap().root_id; + let patch = PatchDocument { + base_change: Some(base_change.0.clone()), + message: Some("advance before rewind".into()), + session_id: None, + allow_ignored: false, + allow_stale: false, + edits: vec![PatchEdit::Write { + path: "README.md".into(), + content: "advanced\n".into(), + executable: false, + }], + }; + db.apply_lane_patch("rewind-repair", patch).unwrap(); + let before = db.get_ref("refs/lanes/rewind-repair").unwrap(); + + set_rewind_post_commit_failure_for_current_thread(true); + let result = db.rewind_lane("rewind-repair", &base_change.0, false, false); + set_rewind_post_commit_failure_for_current_thread(false); + + let error = result.unwrap_err(); + let after = db.get_ref("refs/lanes/rewind-repair").unwrap(); + assert_ne!(after.change_id, before.change_id); + assert_eq!(after.root_id, base_root); + match error { + Error::CommittedRepairRequired { + operation, + repair, + reason, + } => { + assert_eq!(operation, after.operation_id.0); + assert!(repair.contains("lane rewind")); + assert!(reason.contains("injected lane rewind post-commit failure")); + } + other => panic!("expected committed repair error, got {other:?}"), + } + } +} diff --git a/trail/src/db/lane/turns.rs b/trail/src/db/lane/turns.rs index 615017a..c580c71 100644 --- a/trail/src/db/lane/turns.rs +++ b/trail/src/db/lane/turns.rs @@ -6,6 +6,7 @@ impl Trail { turn_id: &str, patch: PatchDocument, ) -> Result { + self.reset_case_fold_index_metrics(); let _lock = self.acquire_write_lock()?; let turn = self.lane_turn(turn_id)?; if turn.ended_at.is_some() { @@ -13,7 +14,7 @@ impl Trail { "turn `{turn_id}` is already ended" ))); } - self.apply_lane_patch_locked(&turn.lane_id, patch, Some(&turn)) + self.apply_lane_patch_with_reconciliation_retry(&turn.lane_id, patch, Some(&turn)) } pub fn end_lane_turn(&mut self, turn_id: &str, status: &str) -> Result { diff --git a/trail/src/db/lane/workdir.rs b/trail/src/db/lane/workdir.rs index 69bba60..fa7a018 100644 --- a/trail/src/db/lane/workdir.rs +++ b/trail/src/db/lane/workdir.rs @@ -1,9 +1,13 @@ use super::*; +#[cfg(target_os = "windows")] +mod dokan; +mod fuse; mod lifecycle; mod manifest; +mod marker; +mod materialize; mod nfs_overlay; -mod overlay; mod record; mod sync; mod view_barrier; @@ -13,6 +17,12 @@ mod view_core; mod view_journal; mod view_layout; +pub(crate) use materialize::*; +#[cfg(debug_assertions)] +pub(crate) use record::{ + install_lane_record_after_c2_write_for_current_thread, + set_lane_record_postcommit_failure_for_current_thread, +}; pub(crate) use view_barrier::*; #[cfg(test)] pub(crate) use view_conformance::*; diff --git a/trail/src/db/lane/workdir/overlay/dokan_overlay.rs b/trail/src/db/lane/workdir/dokan.rs similarity index 87% rename from trail/src/db/lane/workdir/overlay/dokan_overlay.rs rename to trail/src/db/lane/workdir/dokan.rs index 0cf03de..09cd440 100644 --- a/trail/src/db/lane/workdir/overlay/dokan_overlay.rs +++ b/trail/src/db/lane/workdir/dokan.rs @@ -7,7 +7,7 @@ use std::sync::{mpsc, Mutex, MutexGuard, Once}; use std::thread::{self, JoinHandle}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use dokan::{ +use ::dokan::{ init, unmount, CreateFileInfo, DiskSpaceInfo, FileInfo, FileSystemHandler, FileSystemMounter, FillDataError, FillDataResult, FindData, MountFlags, MountOptions, OperationInfo, OperationResult, VolumeInfo, IO_SECURITY_CONTEXT, @@ -25,21 +25,22 @@ static DOKAN_INIT: Once = Once::new(); const OVERLAY_META_DIR: &str = ".trail"; -pub(crate) struct OverlayCowMount { +pub(crate) struct DokanCowMount { mountpoint: PathBuf, mount_name: U16CString, worker: Option>, + #[allow(dead_code)] lease: WorkspaceMountLease, } -impl OverlayCowMount { +impl DokanCowMount { #[allow(dead_code)] pub(crate) fn mountpoint(&self) -> &Path { &self.mountpoint } } -impl Drop for OverlayCowMount { +impl Drop for DokanCowMount { fn drop(&mut self) { let _ = unmount(&self.mount_name); if let Some(worker) = self.worker.take() { @@ -48,7 +49,7 @@ impl Drop for OverlayCowMount { } } -pub(crate) fn prepare_overlay_cow_workdir( +pub(crate) fn prepare_dokan_cow_workdir( db: &Trail, lane: &str, dir: &Path, @@ -62,31 +63,53 @@ pub(crate) fn prepare_overlay_cow_workdir( Ok(upperdir) } -pub(crate) fn mount_overlay_cow_for_lane(db: &Trail, lane: &str) -> Result { +pub(crate) fn mount_dokan_cow_for_lane(db: &Trail, lane: &str) -> Result { + mount_dokan_cow_for_lane_with_view(db, lane, None) +} + +pub(crate) fn mount_dokan_cow_for_lane_with_ephemeral_bindings( + db: &Trail, + lane: &str, + source_upper: PathBuf, + source_root: ObjectId, + bindings: Vec, +) -> Result { + mount_dokan_cow_for_lane_with_view(db, lane, Some((source_upper, source_root, bindings))) +} + +fn mount_dokan_cow_for_lane_with_view( + db: &Trail, + lane: &str, + ephemeral: Option<(PathBuf, ObjectId, Vec)>, +) -> Result { validate_ref_segment(lane)?; let branch = db.lane_branch(lane)?; let record = db.lane_record(&branch.lane_id)?; let mode = db.lane_workdir_mode_for(&record, &branch)?; - if mode != LaneWorkdirMode::OverlayCow { + if mode != LaneWorkdirMode::DokanCow { return Err(Error::InvalidInput(format!( - "lane `{lane}` uses workdir mode `{}`; expected overlay-cow", + "lane `{lane}` uses workdir mode `{}`; expected dokan-cow", mode.as_str() ))); } let Some(workdir) = branch.workdir.clone() else { return Err(Error::InvalidInput(format!( - "overlay-cow lane `{lane}` has no mountpoint" + "dokan-cow lane `{lane}` has no mountpoint" ))); }; let mountpoint = PathBuf::from(workdir); prepare_overlay_mountpoint(&mountpoint, false)?; - let upperdir = overlay_upperdir(db, lane)?; + let mut lease = db.acquire_workspace_mount_lease(lane, "dokan")?; + let head = db.get_ref(&branch.ref_name)?; + let (upperdir, source_root, bindings) = match ephemeral { + Some((upperdir, source_root, bindings)) => (upperdir, source_root, Some(bindings)), + None => (overlay_upperdir(db, lane)?, head.root_id, None), + }; fs::create_dir_all(upperdir.join(OVERLAY_META_DIR))?; - let head = db.get_ref(&branch.ref_name)?; let mount_name = U16CString::from_str(mountpoint.to_string_lossy().as_ref()).map_err(|_| { Error::InvalidInput(format!( - "overlay-cow mountpoint `{}` is not a valid Windows mount string", + "dokan-cow mountpoint `{}` is not a valid Windows mount string", mountpoint.display() )) })?; @@ -95,9 +118,9 @@ pub(crate) fn mount_overlay_cow_for_lane(db: &Trail, lane: &str) -> Result Result Result Result> { +pub(crate) fn dokan_candidate_paths(db: &Trail, lane: &str) -> Result { let upper = overlay_upperdir(db, lane)?; let branch = db.lane_branch(lane)?; let head = db.get_ref(&branch.ref_name)?; - Ok( - recover_view_checkpoint_candidates_for_root(db, &upper, &head.root_id)? - .paths - .into_iter() - .collect(), - ) + recover_view_checkpoint_candidates_for_root(db, &upper, &head.root_id) +} + +impl Trail { + pub(crate) fn prepare_dokan_cow_lane_workdir( + &self, + lane: &str, + dir: &Path, + custom_workdir: bool, + ) -> Result { + prepare_dokan_cow_workdir(self, lane, dir, custom_workdir) + } + + pub fn mount_dokan_cow_workdir_for_lane(&self, lane: &str) -> Result { + mount_dokan_cow_for_lane(self, lane) + } + + pub(crate) fn mount_dokan_cow_workdir_for_lane_with_ephemeral_bindings( + &self, + lane: &str, + source_upper: PathBuf, + source_root: ObjectId, + bindings: Vec, + ) -> Result { + mount_dokan_cow_for_lane_with_ephemeral_bindings( + self, + lane, + source_upper, + source_root, + bindings, + ) + } + + pub(crate) fn dokan_cow_candidate_paths_for_lane( + &self, + lane: &str, + ) -> Result { + dokan_candidate_paths(self, lane) + } + + pub(crate) fn maybe_mount_dokan_cow_workdir_for_lane( + &self, + lane: &str, + ) -> Result> { + validate_ref_segment(lane)?; + let branch = self.lane_branch(lane)?; + let record = self.lane_record(&branch.lane_id)?; + if self.lane_workdir_mode_for(&record, &branch)? == LaneWorkdirMode::DokanCow { + Ok(Some(mount_dokan_cow_for_lane(self, lane)?)) + } else { + Ok(None) + } + } } fn overlay_mount_error(mountpoint: &Path, err: &str) -> Error { Error::InvalidInput(format!( - "failed to mount overlay-cow workdir at `{}` with Dokan: {err}. Install Dokan 2.x and ensure the Dokan driver service is running.", + "failed to mount dokan-cow workdir at `{}` with Dokan: {err}. Install Dokan 2.x and ensure the Dokan driver service is running.", mountpoint.display() )) } @@ -170,18 +240,18 @@ fn prepare_overlay_mountpoint(path: &Path, custom_workdir: bool) -> Result<()> { if metadata.file_type().is_symlink() { return Err(Error::InvalidPath { path: path.to_string_lossy().to_string(), - reason: "overlay-cow mountpoint cannot be a symlink".to_string(), + reason: "dokan-cow mountpoint cannot be a symlink".to_string(), }); } if !metadata.is_dir() { return Err(Error::InvalidPath { path: path.to_string_lossy().to_string(), - reason: "overlay-cow mountpoint must be a directory".to_string(), + reason: "dokan-cow mountpoint must be a directory".to_string(), }); } if fs::read_dir(path)?.next().transpose()?.is_some() && custom_workdir { return Err(Error::InvalidInput(format!( - "custom overlay-cow workdir `{}` must be empty", + "custom dokan-cow workdir `{}` must be empty", path.display() ))); } @@ -230,13 +300,17 @@ impl DokanOverlayFs { db_dir: PathBuf, upperdir: PathBuf, root_id: ObjectId, + ephemeral_bindings: Option>, ) -> Result { + let db = Trail::open_with_db_dir(workspace_root, db_dir)?; + let core = match ephemeral_bindings { + Some(bindings) => { + ViewCore::new_lazy_with_ephemeral_bindings(db, upperdir, root_id, bindings)? + } + None => ViewCore::new_lazy(db, upperdir, root_id)?, + }; Ok(Self { - core: Mutex::new(ViewCore::new_lazy( - Trail::open_with_db_dir(workspace_root, db_dir)?, - upperdir, - root_id, - )?), + core: Mutex::new(core), }) } @@ -727,7 +801,7 @@ mod mounted_conformance { db.spawn_lane_with_workdir_mode_paths_and_neighbors( "dokan-conformance", Some("main"), - LaneWorkdirMode::OverlayCow, + LaneWorkdirMode::DokanCow, None, None, None, @@ -736,7 +810,7 @@ mod mounted_conformance { ) .unwrap(); let mount = db - .mount_overlay_cow_workdir_for_lane("dokan-conformance") + .mount_dokan_cow_workdir_for_lane("dokan-conformance") .unwrap(); let workdir = PathBuf::from( db.lane_workdir("dokan-conformance") @@ -774,7 +848,7 @@ mod mounted_conformance { .spawn_lane_with_workdir_mode_paths_and_neighbors( "foreground-dokan", Some("main"), - LaneWorkdirMode::OverlayCow, + LaneWorkdirMode::DokanCow, None, None, None, @@ -828,7 +902,7 @@ mod mounted_conformance { .spawn_lane_with_workdir_mode_paths_and_neighbors( "daemon-dokan", Some("main"), - LaneWorkdirMode::OverlayCow, + LaneWorkdirMode::DokanCow, None, None, None, diff --git a/trail/src/db/lane/workdir/overlay.rs b/trail/src/db/lane/workdir/fuse.rs similarity index 87% rename from trail/src/db/lane/workdir/overlay.rs rename to trail/src/db/lane/workdir/fuse.rs index d380a85..e34de3c 100644 --- a/trail/src/db/lane/workdir/overlay.rs +++ b/trail/src/db/lane/workdir/fuse.rs @@ -22,7 +22,7 @@ mod fuse_overlay { const TTL: Duration = Duration::from_secs(1); const OVERLAY_META_DIR: &str = ".trail"; - pub(crate) struct OverlayCowMount { + pub(crate) struct FuseCowMount { #[allow(dead_code)] session: fuser::BackgroundSession, #[allow(dead_code)] @@ -31,18 +31,18 @@ mod fuse_overlay { lease: WorkspaceMountLease, } - impl OverlayCowMount { + impl FuseCowMount { #[allow(dead_code)] pub(crate) fn mountpoint(&self) -> &Path { &self.mountpoint } } - impl Drop for OverlayCowMount { + impl Drop for FuseCowMount { fn drop(&mut self) {} } - pub(crate) fn prepare_overlay_cow_workdir( + pub(crate) fn prepare_fuse_cow_workdir( db: &Trail, lane: &str, dir: &Path, @@ -56,72 +56,89 @@ mod fuse_overlay { Ok(upperdir) } - pub(crate) fn mount_overlay_cow_for_lane(db: &Trail, lane: &str) -> Result { + pub(crate) fn mount_fuse_cow_for_lane(db: &Trail, lane: &str) -> Result { + mount_fuse_cow_for_lane_with_view(db, lane, None) + } + + pub(crate) fn mount_fuse_cow_for_lane_with_ephemeral_bindings( + db: &Trail, + lane: &str, + source_upper: PathBuf, + source_root: ObjectId, + bindings: Vec, + ) -> Result { + mount_fuse_cow_for_lane_with_view(db, lane, Some((source_upper, source_root, bindings))) + } + + fn mount_fuse_cow_for_lane_with_view( + db: &Trail, + lane: &str, + ephemeral: Option<(PathBuf, ObjectId, Vec)>, + ) -> Result { validate_ref_segment(lane)?; let branch = db.lane_branch(lane)?; let record = db.lane_record(&branch.lane_id)?; let mode = db.lane_workdir_mode_for(&record, &branch)?; - if mode != LaneWorkdirMode::OverlayCow { + if mode != LaneWorkdirMode::FuseCow { return Err(Error::InvalidInput(format!( - "lane `{lane}` uses workdir mode `{}`; expected overlay-cow", + "lane `{lane}` uses workdir mode `{}`; expected fuse-cow", mode.as_str() ))); } let Some(workdir) = branch.workdir.clone() else { return Err(Error::InvalidInput(format!( - "overlay-cow lane `{lane}` has no mountpoint" + "fuse-cow lane `{lane}` has no mountpoint" ))); }; let mountpoint = PathBuf::from(workdir); prepare_overlay_mountpoint(&mountpoint, false)?; - let upperdir = overlay_upperdir(db, lane)?; + ensure_platform_fuse_ready()?; + let mut lease = db.acquire_workspace_mount_lease(lane, "fuse")?; + let head = db.get_ref(&branch.ref_name)?; + let (upperdir, source_root, bindings) = match ephemeral { + Some((upperdir, source_root, bindings)) => (upperdir, source_root, Some(bindings)), + None => (overlay_upperdir(db, lane)?, head.root_id, None), + }; fs::create_dir_all(&upperdir)?; fs::create_dir_all(upperdir.join(OVERLAY_META_DIR))?; - let head = db.get_ref(&branch.ref_name)?; let fs = SharedOverlayFs::new( db.workspace_root.clone(), db.db_dir.clone(), upperdir, - head.root_id, + source_root, + bindings, )?; #[cfg(target_os = "linux")] - let mut options = vec![MountOption::FSName(format!("trail-overlay-cow-{lane}"))]; + let mut options = vec![MountOption::FSName(format!("trail-fuse-cow-{lane}"))]; #[cfg(target_os = "macos")] - let options = vec![MountOption::FSName(format!("trail-overlay-cow-{lane}"))]; + let options = vec![MountOption::FSName(format!("trail-fuse-cow-{lane}"))]; #[cfg(target_os = "linux")] { - options.push(MountOption::Subtype("trail-overlay-cow".to_string())); + options.push(MountOption::Subtype("trail-fuse-cow".to_string())); options.push(MountOption::RW); options.push(MountOption::NoAtime); } - ensure_platform_fuse_ready()?; - let mut lease = db.acquire_workspace_mount_lease(lane, "fuse")?; let session = fuser::spawn_mount2(fs, &mountpoint, &options) .map_err(|err| overlay_mount_error(&mountpoint, err))?; lease.mark_mounted()?; - Ok(OverlayCowMount { + Ok(FuseCowMount { session, mountpoint, lease, }) } - pub(crate) fn overlay_candidate_paths(db: &Trail, lane: &str) -> Result> { + pub(crate) fn fuse_candidate_paths(db: &Trail, lane: &str) -> Result { let upper = overlay_upperdir(db, lane)?; let branch = db.lane_branch(lane)?; let head = db.get_ref(&branch.ref_name)?; - Ok( - recover_view_checkpoint_candidates_for_root(db, &upper, &head.root_id)? - .paths - .into_iter() - .collect(), - ) + recover_view_checkpoint_candidates_for_root(db, &upper, &head.root_id) } fn overlay_mount_error(mountpoint: &Path, err: std::io::Error) -> Error { Error::InvalidInput(format!( - "failed to mount overlay-cow workdir at `{}`: {err}. On macOS install macFUSE; on Linux ensure /dev/fuse is available and your user can mount FUSE filesystems.", + "failed to mount fuse-cow workdir at `{}`: {err}. On macOS install macFUSE; on Linux ensure /dev/fuse is available and your user can mount FUSE filesystems.", mountpoint.display() )) } @@ -132,7 +149,7 @@ mod fuse_overlay { return Ok(()); } Err(Error::InvalidInput( - "overlay-cow workdirs require `/dev/fuse`; enable FUSE for this Linux environment" + "fuse-cow workdirs require `/dev/fuse`; enable FUSE for this Linux environment" .to_string(), )) } @@ -145,7 +162,7 @@ mod fuse_overlay { let loader = Path::new("/Library/Filesystems/macfuse.fs/Contents/Resources/load_macfuse"); if !loader.exists() { return Err(Error::InvalidInput( - "overlay-cow workdirs require macFUSE; install macFUSE and approve its system extension".to_string(), + "fuse-cow workdirs require macFUSE; install macFUSE and approve its system extension".to_string(), )); } run_macos_fuse_loader(loader, Duration::from_secs(5))?; @@ -222,18 +239,18 @@ mod fuse_overlay { if metadata.file_type().is_symlink() { return Err(Error::InvalidPath { path: path.to_string_lossy().to_string(), - reason: "overlay-cow mountpoint cannot be a symlink".to_string(), + reason: "fuse-cow mountpoint cannot be a symlink".to_string(), }); } if !metadata.is_dir() { return Err(Error::InvalidPath { path: path.to_string_lossy().to_string(), - reason: "overlay-cow mountpoint must be a directory".to_string(), + reason: "fuse-cow mountpoint must be a directory".to_string(), }); } if fs::read_dir(path)?.next().transpose()?.is_some() && custom_workdir { return Err(Error::InvalidInput(format!( - "custom overlay-cow workdir `{}` must be empty", + "custom fuse-cow workdir `{}` must be empty", path.display() ))); } @@ -256,13 +273,17 @@ mod fuse_overlay { db_dir: PathBuf, upperdir: PathBuf, root_id: ObjectId, + ephemeral_bindings: Option>, ) -> Result { + let db = Trail::open_with_db_dir(workspace_root, db_dir)?; + let core = match ephemeral_bindings { + Some(bindings) => { + ViewCore::new_lazy_with_ephemeral_bindings(db, upperdir, root_id, bindings)? + } + None => ViewCore::new_lazy(db, upperdir, root_id)?, + }; Ok(Self { - core: ViewCore::new_lazy( - Trail::open_with_db_dir(workspace_root, db_dir)?, - upperdir, - root_id, - )?, + core, handles: HashMap::new(), directory_handles: HashMap::new(), next_fh: 1, @@ -760,7 +781,7 @@ mod fuse_overlay { db.spawn_lane_with_workdir_mode_paths_and_neighbors( "fuse-conformance", Some("main"), - LaneWorkdirMode::OverlayCow, + LaneWorkdirMode::FuseCow, None, None, None, @@ -769,7 +790,7 @@ mod fuse_overlay { ) .unwrap(); let mount = db - .mount_overlay_cow_workdir_for_lane("fuse-conformance") + .mount_fuse_cow_workdir_for_lane("fuse-conformance") .unwrap(); let workdir = PathBuf::from( db.lane_workdir("fuse-conformance") @@ -847,7 +868,7 @@ mod fuse_overlay { db.spawn_lane_with_workdir_mode_paths_and_neighbors( lane, Some("main"), - LaneWorkdirMode::OverlayCow, + LaneWorkdirMode::FuseCow, None, None, None, @@ -880,40 +901,10 @@ mod fuse_overlay { let target_a = PathBuf::from(&view_a.generated_upper).join("target"); assert!(tree_has_name_fragment(&target_a, "libshared_dep")); - let cargo_version = std::process::Command::new("cargo") - .arg("--version") - .output() - .unwrap(); - let key = WorkspaceLayerKeyV1 { - kind: "compiler-results".to_string(), - adapter: "cargo-target-seed".to_string(), - adapter_version: 1, - inputs: BTreeMap::from([ - ("source_root".to_string(), first.source_root.0.clone()), - ("command".to_string(), "cargo build --offline".to_string()), - ]), - tool_versions: BTreeMap::from([( - "cargo".to_string(), - String::from_utf8_lossy(&cargo_version.stdout) - .trim() - .to_string(), - )]), - platform: std::env::consts::OS.to_string(), - architecture: std::env::consts::ARCH.to_string(), - portability_scope: "source-root-toolchain-platform".to_string(), - strategy: "immutable-target-seed".to_string(), - }; let layer = db - .publish_workspace_layer_from_directory(&key, &target_a) + .sync_workspace_environment("rust-seed-b", "cargo", None) .unwrap(); - db.attach_workspace_layer( - "rust-seed-b", - &layer.layer_id, - "target", - "cargo-target-seed", - &layer.cache_key, - ) - .unwrap(); + assert_eq!(layer.adapter, "cargo-target-seed"); let second = db .exec_lane_workspace( @@ -1010,7 +1001,7 @@ mod fuse_overlay { db.spawn_lane_with_workdir_mode_paths_and_neighbors( lane, Some("main"), - LaneWorkdirMode::OverlayCow, + LaneWorkdirMode::FuseCow, None, None, None, @@ -1048,12 +1039,8 @@ mod fuse_overlay { assert_eq!(binding_count, 2); let mount_started = std::time::Instant::now(); - let mount_a = db - .mount_overlay_cow_workdir_for_lane("node-fuse-a") - .unwrap(); - let mount_b = db - .mount_overlay_cow_workdir_for_lane("node-fuse-b") - .unwrap(); + let mount_a = db.mount_fuse_cow_workdir_for_lane("node-fuse-a").unwrap(); + let mount_b = db.mount_fuse_cow_workdir_for_lane("node-fuse-b").unwrap(); let mount_ms = mount_started.elapsed().as_millis(); let workdir_a = PathBuf::from(db.lane_workdir("node-fuse-a").unwrap().workdir.unwrap()); let workdir_b = PathBuf::from(db.lane_workdir("node-fuse-b").unwrap().workdir.unwrap()); @@ -1205,120 +1192,110 @@ mod fuse_overlay { #[cfg(any(target_os = "linux", all(target_os = "macos", feature = "macfuse")))] pub(crate) use fuse_overlay::*; -#[cfg(target_os = "windows")] -mod dokan_overlay; - -#[cfg(target_os = "windows")] -pub(crate) use dokan_overlay::*; - -#[cfg(not(any( - target_os = "linux", - all(target_os = "macos", feature = "macfuse"), - target_os = "windows" -)))] -pub(crate) struct OverlayCowMount { +#[cfg(not(any(target_os = "linux", all(target_os = "macos", feature = "macfuse"))))] +pub(crate) struct FuseCowMount { #[allow(dead_code)] mountpoint: PathBuf, } -#[cfg(not(any( - target_os = "linux", - all(target_os = "macos", feature = "macfuse"), - target_os = "windows" -)))] -impl OverlayCowMount { +#[cfg(not(any(target_os = "linux", all(target_os = "macos", feature = "macfuse"))))] +impl FuseCowMount { #[allow(dead_code)] pub(crate) fn mountpoint(&self) -> &Path { &self.mountpoint } } -#[cfg(not(any( - target_os = "linux", - all(target_os = "macos", feature = "macfuse"), - target_os = "windows" -)))] -impl Drop for OverlayCowMount { +#[cfg(not(any(target_os = "linux", all(target_os = "macos", feature = "macfuse"))))] +impl Drop for FuseCowMount { fn drop(&mut self) {} } -#[cfg(not(any( - target_os = "linux", - all(target_os = "macos", feature = "macfuse"), - target_os = "windows" -)))] -pub(crate) fn prepare_overlay_cow_workdir( +#[cfg(not(any(target_os = "linux", all(target_os = "macos", feature = "macfuse"))))] +pub(crate) fn prepare_fuse_cow_workdir( _db: &Trail, _lane: &str, _dir: &Path, _custom_workdir: bool, ) -> Result { Err(Error::InvalidInput( - "overlay-cow workdirs require Linux FUSE, a macOS build with --features macfuse, or Windows Dokan" - .to_string(), + "fuse-cow workdirs require Linux FUSE or a macOS build with --features macfuse".to_string(), )) } -#[cfg(not(any( - target_os = "linux", - all(target_os = "macos", feature = "macfuse"), - target_os = "windows" -)))] -pub(crate) fn mount_overlay_cow_for_lane(_db: &Trail, lane: &str) -> Result { +#[cfg(not(any(target_os = "linux", all(target_os = "macos", feature = "macfuse"))))] +pub(crate) fn mount_fuse_cow_for_lane(_db: &Trail, lane: &str) -> Result { Err(Error::InvalidInput(format!( - "overlay-cow lane `{lane}` cannot be mounted on this platform" + "fuse-cow lane `{lane}` cannot be mounted on this platform" ))) } -#[cfg(not(any( - target_os = "linux", - all(target_os = "macos", feature = "macfuse"), - target_os = "windows" -)))] -pub(crate) fn overlay_candidate_paths(_db: &Trail, lane: &str) -> Result> { +#[cfg(not(any(target_os = "linux", all(target_os = "macos", feature = "macfuse"))))] +pub(crate) fn mount_fuse_cow_for_lane_with_ephemeral_bindings( + _db: &Trail, + lane: &str, + _source_upper: PathBuf, + _source_root: ObjectId, + _bindings: Vec, +) -> Result { Err(Error::InvalidInput(format!( - "overlay-cow lane `{lane}` cannot be inspected on this platform" + "fuse-cow lane `{lane}` cannot be mounted on this platform" ))) } -impl Trail { - pub(crate) fn overlay_clean_workdir_manifest_path_for_lane( - &self, - lane: &str, - ) -> Result { - validate_ref_segment(lane)?; - Ok(self - .workspace_view_paths_for_lane_name(lane) - .meta_dir - .join("workdir-manifest.json")) - } +#[cfg(not(any(target_os = "linux", all(target_os = "macos", feature = "macfuse"))))] +pub(crate) fn fuse_candidate_paths(_db: &Trail, lane: &str) -> Result { + Err(Error::InvalidInput(format!( + "fuse-cow lane `{lane}` cannot be inspected on this platform" + ))) +} - pub(crate) fn prepare_overlay_cow_lane_workdir( +impl Trail { + pub(crate) fn prepare_fuse_cow_lane_workdir( &self, lane: &str, dir: &Path, custom_workdir: bool, ) -> Result { - prepare_overlay_cow_workdir(self, lane, dir, custom_workdir) + prepare_fuse_cow_workdir(self, lane, dir, custom_workdir) } - pub fn mount_overlay_cow_workdir_for_lane(&self, lane: &str) -> Result { - mount_overlay_cow_for_lane(self, lane) + pub fn mount_fuse_cow_workdir_for_lane(&self, lane: &str) -> Result> { + mount_fuse_cow_for_lane(self, lane) } - pub(crate) fn overlay_cow_candidate_paths_for_lane(&self, lane: &str) -> Result> { - overlay_candidate_paths(self, lane) + pub(crate) fn mount_fuse_cow_workdir_for_lane_with_ephemeral_bindings( + &self, + lane: &str, + source_upper: PathBuf, + source_root: ObjectId, + bindings: Vec, + ) -> Result> { + mount_fuse_cow_for_lane_with_ephemeral_bindings( + self, + lane, + source_upper, + source_root, + bindings, + ) + } + + pub(crate) fn fuse_cow_candidate_paths_for_lane( + &self, + lane: &str, + ) -> Result { + fuse_candidate_paths(self, lane) } - pub(crate) fn maybe_mount_overlay_cow_workdir_for_lane( + pub(crate) fn maybe_mount_fuse_cow_workdir_for_lane( &self, lane: &str, - ) -> Result> { + ) -> Result> { validate_ref_segment(lane)?; let branch = self.lane_branch(lane)?; let record = self.lane_record(&branch.lane_id)?; - if self.lane_workdir_mode_for(&record, &branch)? == LaneWorkdirMode::OverlayCow { - Ok(Some(mount_overlay_cow_for_lane(self, lane)?)) + if self.lane_workdir_mode_for(&record, &branch)? == LaneWorkdirMode::FuseCow { + Ok(Some(mount_fuse_cow_for_lane(self, lane)?)) } else { Ok(None) } diff --git a/trail/src/db/lane/workdir/lifecycle.rs b/trail/src/db/lane/workdir/lifecycle.rs index 6d6f4b9..1405729 100644 --- a/trail/src/db/lane/workdir/lifecycle.rs +++ b/trail/src/db/lane/workdir/lifecycle.rs @@ -1,4 +1,5 @@ use super::*; +use crate::db::change_ledger::{remove_retired_segments, retire_deletion_scopes}; impl Trail { pub fn lane_timeline(&self, lane: &str, limit: usize) -> Result> { @@ -50,6 +51,19 @@ impl Trail { "lane `{lane}` has unmerged changes; pass --force to remove" ))); } + let mut owners = vec![branch.lane_id.as_str(), lane]; + if let Some(view) = &preserved_view { + owners.push(view.view_id.as_str()); + } + let roots = branch.workdir.as_deref().into_iter().collect::>(); + let retired_segments = retire_deletion_scopes( + &self.conn, + &self.sqlite_path, + &owners, + &roots, + &[branch.ref_name.as_str()], + )?; + remove_retired_segments(&self.conn, &retired_segments)?; remove_ref_file(&self.db_dir, &branch.ref_name)?; self.conn .execute("DELETE FROM refs WHERE name = ?1", params![branch.ref_name])?; @@ -59,7 +73,7 @@ impl Trail { fs::remove_dir_all(&path)?; } } - for backend in ["overlay-cow", "nfs-cow"] { + for backend in ["fuse-cow", "nfs-cow", "dokan-cow"] { let state = self.db_dir.join(backend).join(lane); if state.exists() { fs::remove_dir_all(state)?; diff --git a/trail/src/db/lane/workdir/manifest.rs b/trail/src/db/lane/workdir/manifest.rs index 1b43d47..0bf7c22 100644 --- a/trail/src/db/lane/workdir/manifest.rs +++ b/trail/src/db/lane/workdir/manifest.rs @@ -17,6 +17,63 @@ struct CleanWorkdirManifestEntry { } impl Trail { + pub(crate) fn repair_clean_workdir_manifest_root_mirror( + &self, + path: &Path, + old_root_id: &ObjectId, + new_root_id: &ObjectId, + ) -> bool { + let bytes = match fs::read(path) { + Ok(bytes) => bytes, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return true, + Err(_) => return false, + }; + let mut manifest: CleanWorkdirManifest = match serde_json::from_slice(&bytes) { + Ok(manifest) => manifest, + Err(_) => return remove_clean_workdir_manifest_path(path).is_ok(), + }; + if manifest.version != CLEAN_WORKDIR_MANIFEST_VERSION { + return remove_clean_workdir_manifest_path(path).is_ok(); + } + if manifest.root_id == new_root_id.0 { + return true; + } + if manifest.root_id != old_root_id.0 { + return remove_clean_workdir_manifest_path(path).is_ok(); + } + manifest.root_id = new_root_id.0.clone(); + if self + .write_clean_workdir_manifest_entries_to_path(path, new_root_id, manifest.files) + .is_ok() + { + return true; + } + remove_clean_workdir_manifest_path(path).is_ok() + } + + pub(crate) fn preflight_clean_workdir_manifest_root_retarget( + &self, + dir: &Path, + manifest_path: Option<&Path>, + old_root_id: &ObjectId, + ) -> Result { + let path = manifest_path + .map(Path::to_path_buf) + .unwrap_or_else(|| clean_workdir_manifest_path(dir)); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(err) => return Err(Error::Io(err)), + }; + let manifest: CleanWorkdirManifest = serde_json::from_slice(&bytes).map_err(|err| { + Error::Corrupt(format!( + "clean workdir manifest `{}` cannot be retargeted: {err}", + path.display() + )) + })?; + Ok(manifest.version == CLEAN_WORKDIR_MANIFEST_VERSION && manifest.root_id == old_root_id.0) + } + pub(crate) fn cached_workdir_manifest_status( &self, dir: &Path, @@ -35,6 +92,13 @@ impl Trail { manifest_path: &Path, root_id: &ObjectId, ) -> Result { + if crate::db::change_ledger::command_authority_enabled() + && self.registered_materialized_lane_for_root(dir)?.is_some() + { + // Registered lane authority is the qualified v2 marker plus its + // ledger snapshot. A legacy N-entry manifest is never consulted. + return Ok(CachedWorkdirManifestStatus::Missing); + } let Some(manifest) = self.read_clean_workdir_manifest_from_path(manifest_path)? else { return Ok(CachedWorkdirManifestStatus::Missing); }; @@ -105,6 +169,45 @@ impl Trail { }) } + /// Verify a freshly written materialization manifest without changing the + /// workdir. The caller must pin case sensitivity before its durability + /// barrier; probing it here would create and unlink a file after the + /// barrier, and the ordinary cached-manifest path may retire an invalid + /// manifest while classifying it. + pub(crate) fn clean_workdir_manifest_matches_read_only( + &self, + dir: &Path, + root_id: &ObjectId, + case_insensitive: bool, + ) -> Result { + let manifest_path = clean_workdir_manifest_path(dir); + let bytes = match fs::read(&manifest_path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(Error::Io(error)), + }; + let manifest = match serde_json::from_slice::(&bytes) { + Ok(manifest) => manifest, + Err(_) => return Ok(false), + }; + if manifest.version != CLEAN_WORKDIR_MANIFEST_VERSION || manifest.root_id != root_id.0 { + return Ok(false); + } + let manifest_paths = manifest.files.keys().cloned().collect::>(); + let stamps = self.scan_workdir_file_stamps_with_pinned_paths_case_sensitivity( + dir, + &manifest_paths, + case_insensitive, + )?; + Ok(stamps.len() == manifest.files.len() + && stamps.iter().all(|(path, stamp)| { + manifest + .files + .get(path) + .is_some_and(|cached| cached.stamp == *stamp) + })) + } + pub(crate) fn write_clean_workdir_manifest<'a, I>( &self, dir: &Path, @@ -300,30 +403,6 @@ impl Trail { self.write_clean_workdir_manifest_entries_to_path(manifest_path, root_id, entries) } - pub(crate) fn write_clean_workdir_manifest_from_disk_manifest_and_stamps<'a, I>( - &self, - dir: &Path, - root_id: &ObjectId, - disk_manifest: &BTreeMap, - expected_paths: I, - stamps: BTreeMap, - ) -> Result<()> - where - I: IntoIterator, - { - let expected = expected_paths - .into_iter() - .map(|path| normalize_relative_path(path)) - .collect::>>()?; - self.write_clean_workdir_manifest_from_disk_manifest_stamps_for_paths( - dir, - root_id, - disk_manifest, - expected, - stamps, - ) - } - fn write_clean_workdir_manifest_from_disk_manifest_stamps_for_paths( &self, dir: &Path, @@ -369,6 +448,22 @@ impl Trail { previous: &BTreeMap, target: &BTreeMap, ) -> Result { + if crate::db::change_ledger::command_authority_enabled() + && self.registered_materialized_lane_for_root(dir)?.is_some() + { + let target_paths = target.keys().cloned().collect::>(); + let disk = self.scan_files_under_for_paths(dir, &target_paths)?; + let disk_manifest = self.disk_manifest(&disk); + if !self + .diff_file_maps_to_manifest_for_paths(target, &disk_manifest, &target_paths)? + .is_empty() + { + self.invalidate_materialized_lane_marker(dir)?; + return Ok(false); + } + self.invalidate_materialized_lane_marker(dir)?; + return Ok(true); + } let Some(mut manifest) = self.read_clean_workdir_manifest(dir)? else { return Ok(false); }; @@ -413,13 +508,23 @@ impl Trail { Ok(true) } - pub(crate) fn clean_workdir_manifest_allows_file_subset_update( + pub(crate) fn clean_workdir_manifest_allows_touched_path_update( &self, dir: &Path, previous_root_id: &ObjectId, previous: &BTreeMap, target: &BTreeMap, ) -> Result { + if crate::db::change_ledger::command_authority_enabled() + && self.registered_materialized_lane_for_root(dir)?.is_some() + { + let previous_paths = previous.keys().cloned().collect::>(); + let disk = self.scan_files_under_for_paths(dir, &previous_paths)?; + let disk_manifest = self.disk_manifest(&disk); + return Ok(self + .diff_file_maps_to_manifest_for_paths(previous, &disk_manifest, &previous_paths)? + .is_empty()); + } let Some(manifest) = self.read_clean_workdir_manifest(dir)? else { return Ok(false); }; @@ -431,15 +536,59 @@ impl Trail { return Ok(false); } - let mut paths = manifest.files.keys().cloned().collect::>(); - for path in previous.keys() { - if !target.contains_key(path) { - paths.remove(path); + let case_insensitive = is_case_insensitive_filesystem(dir)?; + let candidate_paths = previous + .keys() + .chain(target.keys()) + .cloned() + .collect::>() + .into_iter() + .collect::>(); + let observed = + observed_exact_paths_for_candidates(dir, &candidate_paths, case_insensitive)?; + let observed_by_folded = index_observed_paths_by_folded(&observed); + + for (path, entry) in previous { + let Some(cached) = manifest.files.get(path) else { + return Ok(false); + }; + if cached.kind != entry.kind || cached.content_hash != entry.content_hash { + return Ok(false); + } + if observed.get(path) != Some(&ObservedPathKind::RegularFile) { + return Ok(false); + } + let metadata = match fs::symlink_metadata(safe_join(dir, path)?) { + Ok(metadata) => metadata, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(err) => return Err(Error::Io(err)), + }; + if metadata.file_type().is_symlink() + || !metadata.is_file() + || cached.stamp != WorkdirFileStamp::from_metadata(&metadata) + { + return Ok(false); } } - paths.extend(target.keys().cloned()); - if is_case_insensitive_filesystem(dir)? { - validate_no_case_fold_collisions(paths.iter())?; + + let removed_paths = previous + .keys() + .filter(|path| !target.contains_key(*path)) + .cloned() + .collect::>(); + for path in target.keys().filter(|path| !previous.contains_key(*path)) { + let folded = case_insensitive_path_key(path); + let observed_aliases = observed_by_folded.get(&folded); + let aliases_are_removed_previous = observed_aliases.is_none_or(|aliases| { + aliases + .iter() + .all(|observed_path| removed_paths.contains(*observed_path)) + }); + if observed_aliases.is_some_and(|aliases| !aliases.is_empty()) + && !aliases_are_removed_previous + { + return Ok(false); + } } Ok(true) } @@ -460,6 +609,15 @@ impl Trail { root_id: &ObjectId, entries: BTreeMap, ) -> Result<()> { + if let Some(workdir) = workdir_root_for_manifest_path(path) { + if crate::db::change_ledger::command_authority_enabled() + && self + .registered_materialized_lane_for_root(workdir)? + .is_some() + { + return self.invalidate_materialized_lane_marker(workdir); + } + } let parent = path.parent().ok_or_else(|| Error::InvalidPath { path: path.to_string_lossy().to_string(), reason: "clean workdir manifest has no parent".to_string(), @@ -470,7 +628,7 @@ impl Trail { root_id: root_id.0.clone(), files: entries, }; - write_file_atomic(&path, &serde_json::to_vec(&manifest)?, false)?; + write_file_atomic(&path, &serde_json::to_vec(&manifest)?, true)?; Ok(()) } @@ -480,6 +638,16 @@ impl Trail { root_id: &ObjectId, target: &BTreeMap, ) -> Result { + if crate::db::change_ledger::command_authority_enabled() + && self.registered_materialized_lane_for_root(dir)?.is_some() + { + let paths = target.keys().cloned().collect::>(); + let disk = self.scan_files_under_for_paths(dir, &paths)?; + let disk_manifest = self.disk_manifest(&disk); + return Ok(self + .diff_file_maps_to_manifest_for_paths(target, &disk_manifest, &paths)? + .is_empty()); + } let Some(manifest) = self.read_clean_workdir_manifest(dir)? else { return Ok(false); }; @@ -523,6 +691,15 @@ impl Trail { &self, path: &Path, ) -> Result> { + if let Some(workdir) = workdir_root_for_manifest_path(path) { + if crate::db::change_ledger::command_authority_enabled() + && self + .registered_materialized_lane_for_root(workdir)? + .is_some() + { + return Ok(None); + } + } let bytes = match fs::read(path) { Ok(bytes) => bytes, Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), @@ -581,27 +758,221 @@ impl Trail { root: &Path, pinned_paths: &[String], ) -> Result> { + let root = root.canonicalize()?; + let case_insensitive = match self.persisted_materialized_case_insensitive(&root)? { + Some(case_insensitive) => case_insensitive, + None => is_case_insensitive_filesystem(&root)?, + }; + self.scan_workdir_file_stamps_with_pinned_paths_case_sensitivity( + &root, + pinned_paths, + case_insensitive, + ) + } + + fn persisted_materialized_case_insensitive(&self, root: &Path) -> Result> { + if !crate::db::change_ledger::command_authority_enabled() { + return Ok(None); + } + let normalized = normalize_workdir_path(&root.to_path_buf())?; + let persisted = self + .conn + .query_row( + "SELECT scope.case_sensitive,scope.filesystem_identity + FROM lane_branches branch + JOIN changed_path_scopes scope + ON scope.scope_kind='materialized_lane' AND scope.owner_id=branch.lane_id + WHERE branch.workdir=?1 AND branch.status<>'removed' + AND scope.trust_state='trusted' AND scope.retired_at IS NULL", + [normalized.to_string_lossy().as_ref()], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)), + ) + .optional()?; + let Some((case_sensitive, persisted_identity)) = persisted else { + return Ok(None); + }; + let persisted_identity = hex::decode(persisted_identity) + .map_err(|_| Error::Corrupt("invalid materialized filesystem identity".into()))?; + if self.pinned_worktree_identity_for_path(root)? != persisted_identity { + return Ok(None); + } + match case_sensitive { + 0 => Ok(Some(true)), + 1 => Ok(Some(false)), + _ => Err(Error::Corrupt( + "invalid materialized case-sensitivity metadata".into(), + )), + } + } + + #[cfg(debug_assertions)] + pub(crate) fn debug_persisted_materialized_case_insensitive( + &self, + root: &Path, + ) -> Result> { + self.persisted_materialized_case_insensitive(root) + } + + fn scan_workdir_file_stamps_with_pinned_paths_case_sensitivity( + &self, + root: &Path, + pinned_paths: &[String], + case_insensitive: bool, + ) -> Result> { + self.scan_workdir_file_stamps_with_pinned_paths_case_sensitivity_and_insertion_count( + root, + pinned_paths, + case_insensitive, + ) + .map(|(files, _)| files) + } + + fn scan_workdir_file_stamps_with_pinned_paths_case_sensitivity_and_insertion_count( + &self, + root: &Path, + pinned_paths: &[String], + case_insensitive: bool, + ) -> Result<(BTreeMap, usize)> { let root = root.canonicalize()?; let mut files = self.scan_workdir_file_stamps(&root)?; + let mut exact_paths = files.keys().cloned().collect::>(); + let observed = observed_exact_paths_for_candidates(&root, pinned_paths, case_insensitive)?; + let actual_paths = observed.keys().cloned().collect::>(); + let folded_paths = actual_paths + .iter() + .map(|path| case_insensitive_path_key(path)) + .collect::>(); + let mut observed_insertions = 0; + for (path, kind) in &observed { + if exact_paths.contains(path) || *kind != ObservedPathKind::RegularFile { + continue; + } + observed_insertions += 1; + let Some(stamp) = open_observed_exact_regular_file_stamp(&root, path)? else { + continue; + }; + exact_paths.insert(path.clone()); + files.insert(path.clone(), stamp); + } for path in pinned_paths { let path = normalize_relative_path(path)?; + if !pinned_path_needs_probe( + case_insensitive, + &exact_paths, + &actual_paths, + &folded_paths, + &path, + ) { + continue; + } let abs = safe_join(&root, &path)?; let metadata = match fs::symlink_metadata(&abs) { Ok(metadata) => metadata, Err(err) if err.kind() == std::io::ErrorKind::NotFound => { files.remove(&path); + exact_paths.remove(&path); continue; } Err(err) => return Err(Error::Io(err)), }; if metadata.file_type().is_symlink() || !metadata.is_file() { files.remove(&path); + exact_paths.remove(&path); continue; } + exact_paths.insert(path.clone()); files.insert(path, WorkdirFileStamp::from_metadata(&metadata)); } - Ok(files) + Ok((files, observed_insertions)) + } + + fn registered_materialized_lane_for_root(&self, root: &Path) -> Result> { + let normalized = normalize_workdir_path(&root.to_path_buf())?; + self.conn + .query_row( + "SELECT lane.name FROM lanes lane JOIN lane_branches branch USING(lane_id) + WHERE branch.workdir=?1 AND branch.status<>'removed'", + [normalized.to_string_lossy().as_ref()], + |row| row.get(0), + ) + .optional() + .map_err(Error::from) + } +} + +fn workdir_root_for_manifest_path(path: &Path) -> Option<&Path> { + (path.file_name().and_then(|name| name.to_str()) == Some("workdir-manifest.json") + && path.parent()?.file_name().and_then(|name| name.to_str()) == Some(".trail")) + .then(|| path.parent().and_then(Path::parent)) + .flatten() +} + +fn open_observed_exact_regular_file_stamp( + root: &Path, + path: &str, +) -> Result> { + let abs = safe_join(root, path)?; + #[cfg(not(unix))] + { + let metadata = match fs::symlink_metadata(&abs) { + Ok(metadata) => metadata, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(Error::Io(err)), + }; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Ok(None); + } + } + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC); + } + let file = match options.open(&abs) { + Ok(file) => file, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => return Err(Error::Io(err)), + }; + let metadata = file.metadata()?; + if !metadata.is_file() { + return Ok(None); } + #[cfg(not(unix))] + { + let final_metadata = fs::symlink_metadata(&abs)?; + if final_metadata.file_type().is_symlink() || !final_metadata.is_file() { + return Ok(None); + } + } + Ok(Some(WorkdirFileStamp::from_metadata(&metadata))) +} + +fn index_observed_paths_by_folded( + observed: &BTreeMap, +) -> BTreeMap> { + let mut indexed = BTreeMap::>::new(); + for path in observed.keys() { + indexed + .entry(case_insensitive_path_key(path)) + .or_default() + .push(path); + } + indexed +} + +fn pinned_path_needs_probe( + case_insensitive: bool, + exact_paths: &BTreeSet, + actual_paths: &BTreeSet, + folded_paths: &BTreeSet, + path: &str, +) -> bool { + !case_insensitive + || exact_paths.contains(path) + || actual_paths.contains(path) + || !folded_paths.contains(&case_insensitive_path_key(path)) } fn clean_workdir_manifest_path(dir: &Path) -> PathBuf { @@ -768,4 +1139,223 @@ mod tests { assert!(!clean_workdir_manifest_path(workdir.path()).exists()); } + + #[test] + fn pinned_path_probe_skips_different_spelling_already_seen_by_directory_scan() { + let visible_paths = ["readme.md".to_string()] + .into_iter() + .collect::>(); + let actual_paths = visible_paths.clone(); + let folded_paths = actual_paths + .iter() + .map(|path| case_insensitive_path_key(path)) + .collect::>(); + + assert!(!pinned_path_needs_probe( + true, + &visible_paths, + &actual_paths, + &folded_paths, + "README.md" + )); + assert!(pinned_path_needs_probe( + true, + &visible_paths, + &actual_paths, + &folded_paths, + "readme.md" + )); + assert!(pinned_path_needs_probe( + true, + &visible_paths, + &actual_paths, + &folded_paths, + "other.md" + )); + assert!(pinned_path_needs_probe( + false, + &visible_paths, + &actual_paths, + &folded_paths, + "README.md" + )); + + let ignored_visible = BTreeSet::new(); + assert!(!pinned_path_needs_probe( + true, + &ignored_visible, + &actual_paths, + &folded_paths, + "README.md" + )); + assert!(pinned_path_needs_probe( + true, + &ignored_visible, + &actual_paths, + &folded_paths, + "readme.md" + )); + } + + #[test] + fn ignored_actual_spelling_prevents_fabricated_pinned_alias() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("readme.md"), "ignored\n").unwrap(); + fs::write(workspace.path().join(".trailignore"), "readme.md\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::Empty, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let visible = db.scan_workdir_file_stamps(workspace.path()).unwrap(); + assert!(!visible.contains_key("readme.md")); + + let observed = + observed_exact_paths_for_candidates(workspace.path(), &["README.md".to_string()], true) + .unwrap(); + let actual_paths = observed.keys().cloned().collect::>(); + let folded_paths = actual_paths + .iter() + .map(|path| case_insensitive_path_key(path)) + .collect::>(); + assert!(actual_paths.contains("readme.md")); + assert!(!pinned_path_needs_probe( + true, + &BTreeSet::new(), + &actual_paths, + &folded_paths, + "README.md", + )); + + let (stamps, observed_insertions) = db + .scan_workdir_file_stamps_with_pinned_paths_case_sensitivity_and_insertion_count( + workspace.path(), + &["README.md".to_string()], + true, + ) + .unwrap(); + assert_eq!(observed_insertions, 1); + assert!(stamps.contains_key("readme.md")); + assert!(!stamps.contains_key("README.md")); + } + + #[test] + fn visible_exact_pinned_file_needs_no_observed_insertion_open() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("visible.md"), "visible\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::Empty, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + + let (stamps, observed_insertions) = db + .scan_workdir_file_stamps_with_pinned_paths_case_sensitivity_and_insertion_count( + workspace.path(), + &["visible.md".to_string()], + true, + ) + .unwrap(); + + assert_eq!(observed_insertions, 0); + assert!(stamps.contains_key("visible.md")); + } + + #[test] + fn observed_fold_index_supports_ten_thousand_constant_domain_lookups() { + let observed = (0..10_000) + .map(|index| { + ( + format!("Dir-{index:05}/File.txt"), + ObservedPathKind::RegularFile, + ) + }) + .collect::>(); + + let folded = index_observed_paths_by_folded(&observed); + + assert_eq!(folded.len(), 10_000); + for path in observed.keys() { + let aliases = folded + .get(&case_insensitive_path_key(path)) + .expect("folded path is indexed once"); + assert_eq!(aliases.as_slice(), &[path.as_str()]); + } + } + + #[test] + fn touched_manifest_guard_does_not_probe_unrelated_large_manifest_paths() { + let (_workspace, workdir, db, head, files) = + workdir_manifest_from_materialization_stamps_fixture(); + let metadata = fs::symlink_metadata(workdir.path().join("a.txt")).unwrap(); + let template = CleanWorkdirManifestEntry { + stamp: WorkdirFileStamp::from_metadata(&metadata), + kind: files["a.txt"].kind.clone(), + content_hash: files["a.txt"].content_hash.clone(), + }; + let mut entries = BTreeMap::new(); + entries.insert("a.txt".to_string(), template.clone()); + for idx in 0..10_000 { + entries.insert(format!("missing/{idx:05}.txt"), template.clone()); + } + db.write_clean_workdir_manifest_entries(workdir.path(), &head.root_id, entries) + .unwrap(); + let previous = [("a.txt".to_string(), files["a.txt"].clone())] + .into_iter() + .collect::>(); + let target = previous.clone(); + + assert!(db + .clean_workdir_manifest_allows_touched_path_update( + workdir.path(), + &head.root_id, + &previous, + &target, + ) + .unwrap()); + } + + #[test] + fn touched_manifest_guard_allows_case_only_rename_from_removed_path() { + let (_workspace, workdir, db, head, files) = + workdir_manifest_from_materialization_stamps_fixture(); + db.write_clean_workdir_manifest(workdir.path(), &head.root_id, &files, files.keys()) + .unwrap(); + let previous = [("a.txt".to_string(), files["a.txt"].clone())] + .into_iter() + .collect::>(); + let target = [("A.txt".to_string(), files["a.txt"].clone())] + .into_iter() + .collect::>(); + + assert!(db + .clean_workdir_manifest_allows_touched_path_update( + workdir.path(), + &head.root_id, + &previous, + &target, + ) + .unwrap()); + } + + #[test] + fn touched_manifest_guard_rejects_external_new_target() { + let (_workspace, workdir, db, head, files) = + workdir_manifest_from_materialization_stamps_fixture(); + db.write_clean_workdir_manifest(workdir.path(), &head.root_id, &files, files.keys()) + .unwrap(); + fs::write(workdir.path().join("external.txt"), "external\n").unwrap(); + let previous = [("a.txt".to_string(), files["a.txt"].clone())] + .into_iter() + .collect::>(); + let target = [ + ("a.txt".to_string(), files["a.txt"].clone()), + ("external.txt".to_string(), files["a.txt"].clone()), + ] + .into_iter() + .collect::>(); + + assert!(!db + .clean_workdir_manifest_allows_touched_path_update( + workdir.path(), + &head.root_id, + &previous, + &target, + ) + .unwrap()); + } } diff --git a/trail/src/db/lane/workdir/marker.rs b/trail/src/db/lane/workdir/marker.rs new file mode 100644 index 0000000..65c216b --- /dev/null +++ b/trail/src/db/lane/workdir/marker.rs @@ -0,0 +1,432 @@ +use super::*; + +use crate::db::change_ledger::secure_fs::SecureDirectory; +use crate::db::change_ledger::{EvidenceCut, EvidenceSource, ScopeId}; +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; + +const MARKER_FILE: &str = "workdir-manifest.json"; +const MAX_MARKER_BYTES: u64 = 16 * 1024; +const SPARSE_SELECTION_FILE: &str = "sparse-selection.json"; +const MAX_SPARSE_SELECTION_BYTES: u64 = 1024 * 1024; + +pub(crate) const MATERIALIZED_LANE_MARKER_VERSION: u16 = 2; + +/// Compact authority marker. It deliberately contains no path manifest: the +/// changed-path ledger is the only scalable candidate authority. +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +pub(crate) struct MaterializedLaneMarkerV2 { + pub(crate) version: u16, + pub(crate) scope_id: ScopeId, + pub(crate) filesystem_identity: Vec, + pub(crate) ref_name: String, + pub(crate) ref_generation: u64, + pub(crate) root_id: ObjectId, + pub(crate) policy_fingerprint: [u8; 32], + pub(crate) epoch: u64, + pub(crate) provider_cut: EvidenceCut, + pub(crate) provider_segment_id: String, + pub(crate) sparse_selection_fingerprint: [u8; 32], +} + +impl Trail { + pub(crate) fn materialized_lane_sparse_selection_fingerprint( + &self, + workdir: &Path, + ) -> Result<[u8; 32]> { + actual_sparse_selection_fingerprint(workdir) + } + + pub(crate) fn authenticated_lane_sparse_selection( + &self, + workdir: &Path, + ) -> Result>> { + let bytes = secure_marker_directory(workdir, false)? + .map(|metadata| { + metadata.read_regular_optional_bounded( + SPARSE_SELECTION_FILE, + MAX_SPARSE_SELECTION_BYTES, + ) + }) + .transpose()? + .flatten(); + let Some(bytes) = bytes else { + return Ok(None); + }; + Ok(Some(parse_sparse_selection_paths(&bytes)?)) + } + + pub(crate) fn capture_materialized_lane_marker( + &self, + workdir: &Path, + ) -> Result>> { + let Some(metadata) = secure_marker_directory(workdir, false)? else { + return Ok(None); + }; + metadata.read_regular_optional_bounded(MARKER_FILE, MAX_MARKER_BYTES) + } + + pub(crate) fn restore_materialized_lane_marker( + &self, + workdir: &Path, + bytes: Option<&[u8]>, + ) -> Result<()> { + match bytes { + Some(bytes) => secure_marker_directory(workdir, true)? + .ok_or_else(|| Error::Corrupt("lane metadata directory disappeared".into()))? + .write_atomic_regular(MARKER_FILE, bytes), + None => self.invalidate_materialized_lane_marker(workdir), + } + } + + pub(crate) fn publish_lane_marker_if_materialized(&self, lane: &str) -> Result<()> { + // Until command authority is platform-qualified, the existing V1 + // clean-workdir manifest remains the only cache. Never replace it with + // an epoch-zero V2 marker that could look like ledger authority. + let branch = self.lane_branch(lane)?; + let Some(workdir) = branch.workdir.as_deref() else { + return Ok(()); + }; + let workdir = Path::new(workdir); + if !workdir.is_dir() { + return Ok(()); + } + let head = self.get_ref(&branch.ref_name)?; + let sparse_selection_fingerprint = actual_sparse_selection_fingerprint(workdir)?; + self.publish_materialized_lane_marker_v2( + &branch, + workdir, + &head, + sparse_selection_fingerprint, + ) + } + + pub(crate) fn publish_lane_marker_for_sparse_selection( + &self, + lane: &str, + sparse_selection_fingerprint: [u8; 32], + ) -> Result<()> { + let branch = self.lane_branch(lane)?; + let Some(workdir) = branch.workdir.as_deref() else { + return Ok(()); + }; + let workdir = Path::new(workdir); + if !workdir.is_dir() { + return Ok(()); + } + let head = self.get_ref(&branch.ref_name)?; + self.publish_materialized_lane_marker_v2( + &branch, + workdir, + &head, + sparse_selection_fingerprint, + ) + } + + pub(crate) fn invalidate_lane_marker_if_materialized(&self, branch: &LaneBranch) -> Result<()> { + if let Some(workdir) = branch.workdir.as_deref() { + let workdir = Path::new(workdir); + if workdir.is_dir() { + self.invalidate_materialized_lane_marker(workdir)?; + } + } + Ok(()) + } + + pub(crate) fn publish_materialized_lane_marker_v2( + &self, + branch: &LaneBranch, + workdir: &Path, + head: &RefRecord, + sparse_selection_fingerprint: [u8; 32], + ) -> Result<()> { + if branch.ref_name != head.name + || branch.head_change != head.change_id + || branch.head_root != head.root_id + || branch.workdir.as_deref() != Some(workdir.to_string_lossy().as_ref()) + { + return Err(Error::StaleBranch(branch.ref_name.clone())); + } + let scope_id = crate::db::change_ledger::materialized_lane_scope_id( + &self.config.workspace.id.0, + &branch.lane_id, + ); + let runtime_cut = + crate::db::change_ledger::materialized_lane_daemon_marker_cut(self, &branch.lane_id)?; + let scope = self + .conn + .query_row( + "SELECT scope.epoch,scope.durable_offset,scope.folded_offset, + scope.filesystem_identity,scope.policy_fingerprint + FROM changed_path_scopes scope + WHERE scope.scope_id=?1 AND scope.scope_kind='materialized_lane' AND scope.owner_id=?2 + AND ref_name=?3 AND ref_generation=?4 AND baseline_root_id=?5 + AND retired_at IS NULL", + params![ + scope_id.to_text(), + branch.lane_id, + head.name, + head.generation, + head.root_id.0, + ], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + )) + }, + ) + .optional()?; + // Before a qualified lane daemon owns the scope the marker remains an + // explicit reconciliation marker, never clean authority. + let (epoch, provider_cut, provider_segment_id, filesystem_identity, policy_fingerprint) = + match (scope, runtime_cut) { + ( + Some((epoch, durable, folded, filesystem_identity, policy_fingerprint)), + Some((runtime_cut, segment_id)), + ) => ( + u64::try_from(epoch) + .map_err(|_| Error::Corrupt("negative lane epoch".into()))?, + if runtime_cut.durable_offset + == u64::try_from(durable) + .map_err(|_| Error::Corrupt("negative lane durable cut".into()))? + && runtime_cut.folded_offset + == u64::try_from(folded) + .map_err(|_| Error::Corrupt("negative lane folded cut".into()))? + { + runtime_cut + } else { + return Err(Error::ChangeLedgerReconcileRequired { + scope: scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "lane marker runtime cut does not match persisted scope".into(), + command: format!("trail lane status {}", branch.lane_id), + }); + }, + segment_id, + hex::decode(filesystem_identity) + .map_err(|_| Error::Corrupt("invalid lane filesystem identity".into()))?, + decode_marker_fingerprint(&policy_fingerprint)?, + ), + // No live qualified runtime means there is no V2 authority to + // publish. Preserve any legacy cache in authority-off mode; a + // later exact runtime reconciliation will replace it atomically. + _ => return Ok(()), + }; + let marker = MaterializedLaneMarkerV2 { + version: MATERIALIZED_LANE_MARKER_VERSION, + scope_id, + filesystem_identity, + ref_name: head.name.clone(), + ref_generation: u64::try_from(head.generation) + .map_err(|_| Error::Corrupt("negative lane ref generation".into()))?, + root_id: head.root_id.clone(), + policy_fingerprint, + epoch, + provider_cut, + provider_segment_id, + sparse_selection_fingerprint, + }; + let metadata = secure_marker_directory(workdir, true)?.ok_or_else(|| { + Error::Corrupt("materialized lane metadata directory disappeared".into()) + })?; + metadata.write_atomic_regular(MARKER_FILE, &serde_json::to_vec(&marker)?) + } + + pub(crate) fn invalidate_materialized_lane_marker(&self, workdir: &Path) -> Result<()> { + if let Some(metadata) = secure_marker_directory(workdir, false)? { + metadata.remove_leaf(MARKER_FILE)?; + } + Ok(()) + } + + /// Return a marker only when its full binding still matches a trusted, + /// qualified materialized-lane scope. Every other case is reconciliation, + /// including absent/v1/future markers and owner/provider loss. + pub(crate) fn validated_materialized_lane_marker_v2( + &self, + branch: &LaneBranch, + workdir: &Path, + head: &RefRecord, + ) -> Result> { + let Some(metadata) = secure_marker_directory(workdir, false)? else { + return Ok(None); + }; + let Some(bytes) = metadata.read_regular_optional_bounded(MARKER_FILE, MAX_MARKER_BYTES)? + else { + return Ok(None); + }; + let marker = match serde_json::from_slice::(&bytes) { + Ok(marker) if marker.version == MATERIALIZED_LANE_MARKER_VERSION => marker, + _ => { + metadata.remove_leaf(MARKER_FILE)?; + return Ok(None); + } + }; + let generation = u64::try_from(head.generation) + .map_err(|_| Error::Corrupt("negative lane ref generation".into()))?; + let expected_scope = crate::db::change_ledger::materialized_lane_scope_id( + &self.config.workspace.id.0, + &branch.lane_id, + ); + let structural = marker.scope_id == expected_scope + && marker.filesystem_identity == materialized_lane_root_identity(workdir)? + && marker.ref_name == head.name + && marker.ref_generation == generation + && marker.root_id == head.root_id + && marker.sparse_selection_fingerprint == actual_sparse_selection_fingerprint(workdir)? + && marker.epoch > 0 + && marker.provider_cut.source == EvidenceSource::Observer + && marker.provider_cut.durable_offset == marker.provider_cut.folded_offset + && !marker.provider_segment_id.is_empty(); + if !structural { + metadata.remove_leaf(MARKER_FILE)?; + return Ok(None); + } + let qualified: bool = self.conn.query_row( + "SELECT EXISTS( + SELECT 1 FROM changed_path_scopes scope + JOIN changed_path_observer_owners owner + ON owner.scope_id=scope.scope_id AND owner.epoch=scope.epoch + JOIN changed_path_observer_segments segment + ON segment.scope_id=scope.scope_id AND segment.epoch=scope.epoch + AND segment.segment_id=?10 + WHERE scope.scope_id=?1 AND scope.scope_kind='materialized_lane' AND scope.owner_id=?2 + AND scope.epoch=?3 AND scope.ref_name=?4 AND scope.ref_generation=?5 + AND scope.baseline_root_id=?6 AND scope.policy_fingerprint=?7 + AND scope.filesystem_identity=?8 AND scope.trust_state='trusted' + AND scope.clean_proof_allowed=1 AND scope.linearizable_fence=1 + AND scope.filesystem_supported=1 AND scope.power_loss_durability=1 + AND scope.durable_offset=?9 AND scope.folded_offset=?9 + AND owner.lease_state='active' AND owner.error_state IS NULL + AND owner.error_at IS NULL AND owner.expires_at>?12 + AND owner.fence_nonce IS NOT NULL + AND owner.provider_id=scope.provider_id + AND owner.provider_identity=scope.provider_identity + AND segment.owner_token=owner.owner_token + AND segment.provider_id=owner.provider_id + AND segment.first_sequence<=?11 AND segment.last_sequence>=?11 + AND segment.durable_end_offset>=?9 AND segment.folded_end_offset=?9 + AND segment.state IN ('open','sealed') + AND scope.retired_at IS NULL)", + params![ + marker.scope_id.to_text(), + branch.lane_id, + i64::try_from(marker.epoch).map_err(|_| Error::InvalidInput("lane epoch overflow".into()))?, + marker.ref_name, + i64::try_from(marker.ref_generation).map_err(|_| Error::InvalidInput("lane ref generation overflow".into()))?, + marker.root_id.0, + hex::encode(marker.policy_fingerprint), + hex::encode(&marker.filesystem_identity), + i64::try_from(marker.provider_cut.durable_offset).map_err(|_| Error::InvalidInput("lane provider cut overflow".into()))?, + marker.provider_segment_id, + i64::try_from(marker.provider_cut.sequence).map_err(|_| Error::InvalidInput("lane provider sequence overflow".into()))?, + now_ts(), + ], + |row| row.get(0), + )?; + if !qualified { + return Ok(None); + } + Ok(Some(marker)) + } +} + +fn materialized_lane_root_identity(workdir: &Path) -> Result> { + #[cfg(unix)] + { + let file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(workdir)?; + let metadata = file.metadata()?; + return Ok(format!( + "root-v1:dev={};ino={};mode={};uid={};gid={}", + metadata.dev(), + metadata.ino(), + metadata.mode(), + metadata.uid(), + metadata.gid() + ) + .into_bytes()); + } + #[cfg(not(unix))] + { + let canonical = workdir.canonicalize()?; + Ok(format!("root-v1:path={}", canonical.display()).into_bytes()) + } +} + +fn actual_sparse_selection_fingerprint(workdir: &Path) -> Result<[u8; 32]> { + let mut digest = Sha256::new(); + digest.update(b"trail-sparse-selection-v2\0"); + let bytes = secure_marker_directory(workdir, false)? + .map(|metadata| { + metadata + .read_regular_optional_bounded(SPARSE_SELECTION_FILE, MAX_SPARSE_SELECTION_BYTES) + }) + .transpose()? + .flatten(); + match bytes { + Some(bytes) => { + for path in parse_sparse_selection_paths(&bytes)? { + digest.update(path.as_bytes()); + digest.update([0]); + } + } + None => digest.update(b"full"), + } + Ok(digest.finalize().into()) +} + +fn parse_sparse_selection_paths(bytes: &[u8]) -> Result> { + let value: serde_json::Value = serde_json::from_slice(bytes)?; + let raw_paths = value.get("materialized_paths").ok_or_else(|| { + Error::Corrupt( + "invalid sparse selection: required `materialized_paths` field is missing".into(), + ) + })?; + let raw_paths = raw_paths.as_array().ok_or_else(|| { + Error::Corrupt("invalid sparse selection: `materialized_paths` must be an array".into()) + })?; + let mut paths = raw_paths + .iter() + .enumerate() + .map(|(index, value)| { + let path = value.as_str().ok_or_else(|| { + Error::Corrupt(format!( + "invalid sparse selection: `materialized_paths[{index}]` must be a string" + )) + })?; + normalize_relative_path(path) + }) + .collect::>>()?; + paths.sort(); + paths.dedup(); + Ok(paths) +} + +fn decode_marker_fingerprint(encoded: &str) -> Result<[u8; 32]> { + hex::decode(encoded) + .ok() + .and_then(|bytes| bytes.try_into().ok()) + .ok_or_else(|| Error::Corrupt("invalid materialized lane policy fingerprint".into())) +} + +fn secure_marker_directory(workdir: &Path, create: bool) -> Result> { + let root = SecureDirectory::open_absolute(workdir)?; + match root.open_dir(".trail") { + Ok(directory) => { + directory.restrict_private()?; + Ok(Some(directory)) + } + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound && create => { + root.create_private_dir(".trail").map(Some) + } + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } +} diff --git a/trail/src/db/lane/workdir/materialize.rs b/trail/src/db/lane/workdir/materialize.rs new file mode 100644 index 0000000..2de944a --- /dev/null +++ b/trail/src/db/lane/workdir/materialize.rs @@ -0,0 +1,1347 @@ +use super::*; +use crate::db::change_ledger::secure_fs::SecureDirectory; +use rayon::prelude::*; + +const MAX_MATERIALIZATION_OPERATION_BYTES: u64 = 64 * 1024; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum MaterializationPolicy { + StrictNative, + Portable, + Auto, +} + +#[derive(Clone, Debug)] +pub(crate) struct MaterializationOutcome { + pub(crate) resolved_mode: LaneWorkdirMode, + pub(crate) backend: WorkdirBackend, + pub(crate) report: MaterializationReport, + pub(crate) materialization_operation_id: String, +} + +#[derive(Debug)] +enum NativeAttemptError { + Unavailable(MaterializationFallbackReason), + Hard(Error), +} + +struct NativeSource { + root: PathBuf, + stamps: BTreeMap, +} + +#[derive(Clone, Copy, Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum MaterializationOperationState { + Preparing, + Materializing, + Verified, + Published, +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +struct MaterializationOperationRecord { + version: u16, + operation_id: String, + parent: String, + parent_device: u64, + parent_inode: u64, + destination_leaf: String, + stage_leaf: String, + owned_device: u64, + owned_inode: u64, + root_id: String, + state: MaterializationOperationState, + owner_pid: u32, + owner_start_token: String, +} + +struct RegisteredMaterializationStage { + record_path: PathBuf, + stage_path: PathBuf, + parent_authority: SecureDirectory, + stage_authority: SecureDirectory, + record: MaterializationOperationRecord, +} + +impl RegisteredMaterializationStage { + fn path(&self) -> &Path { + &self.stage_path + } + + fn set_state(&mut self, state: MaterializationOperationState) -> Result<()> { + self.record.state = state; + write_materialization_record(&self.record_path, &self.record) + } + + fn publish(&mut self) -> Result<()> { + let expected_parent = (self.record.parent_device, self.record.parent_inode); + let expected_stage = (self.record.owned_device, self.record.owned_inode); + // Retain the originally authenticated descriptor for publication, and + // also require the path binding to still name that same parent. + SecureDirectory::open_absolute(Path::new(&self.record.parent))? + .verify_identity(expected_parent)?; + self.parent_authority.verify_identity(expected_parent)?; + self.stage_authority.verify_identity(expected_stage)?; + self.parent_authority + .open_dir(&self.record.stage_leaf)? + .verify_identity(expected_stage)?; + self.parent_authority + .rename_leaf_noreplace(&self.record.stage_leaf, &self.record.destination_leaf)?; + self.parent_authority.sync()?; + self.parent_authority + .open_dir(&self.record.destination_leaf)? + .verify_identity(expected_stage)?; + Ok(()) + } + + fn abort(self) { + if remove_owned_materialization_tree(&self.record, true).is_ok() { + let _ = remove_materialization_record(&self.record_path); + } + } +} + +impl From for NativeAttemptError { + fn from(error: Error) -> Self { + Self::Hard(error) + } +} + +impl From for NativeAttemptError { + fn from(error: std::io::Error) -> Self { + Self::Hard(Error::Io(error)) + } +} + +impl Trail { + pub(crate) fn materialize_lane_root_staged( + &self, + root_id: &ObjectId, + destination: &Path, + custom_workdir: bool, + policy: MaterializationPolicy, + ) -> Result { + prepare_staged_destination(destination, custom_workdir)?; + match policy { + MaterializationPolicy::StrictNative => self + .materialize_strict_native_attempt(root_id, destination) + .map_err(native_attempt_error), + MaterializationPolicy::Portable => { + self.materialize_portable_attempt(root_id, destination, None) + } + MaterializationPolicy::Auto => { + match self.materialize_strict_native_attempt(root_id, destination) { + Ok(outcome) => Ok(outcome), + Err(NativeAttemptError::Unavailable(reason)) => { + self.materialize_portable_attempt(root_id, destination, Some(reason)) + } + Err(NativeAttemptError::Hard(error)) => Err(error), + } + } + } + } + + pub(crate) fn materialize_sparse_lane_root_staged( + &self, + root_id: &ObjectId, + destination: &Path, + custom_workdir: bool, + files: &BTreeMap, + ) -> Result { + prepare_staged_destination(destination, custom_workdir)?; + let mut operation = self.create_materialization_stage(destination, root_id)?; + let operation_id = operation.record.operation_id.clone(); + let stage = operation.path().to_path_buf(); + let result = (|| { + let empty = BTreeMap::new(); + let mut stamps = BTreeMap::new(); + let mut report = MaterializationReport::default(); + for (path, entry) in files { + let mut cloned = false; + match materialize_workspace_file_cow_status_if_matching_with_durability( + &self.workspace_root, + &stage, + path, + entry, + true, + )? { + WorkspaceCowMaterializeStatus::Cloned(stamp) => { + stamps.insert(path.clone(), stamp); + report.cloned_files += 1; + report.cloned_bytes += entry.size_bytes; + cloned = true; + } + WorkspaceCowMaterializeStatus::Skipped + | WorkspaceCowMaterializeStatus::Unavailable(_) => {} + } + if !cloned { + let one = BTreeMap::from([(path.clone(), entry.clone())]); + let materialized = self.materialize_files_at_report(&stage, &empty, &one)?; + stamps.extend(materialized.stamps); + report.copied_files += 1; + report.copied_bytes += entry.size_bytes; + } + } + + self.write_sparse_workdir_manifest(&stage, files.keys())?; + self.write_clean_workdir_manifest_from_stamps( + &stage, + root_id, + files, + files.keys(), + stamps, + )?; + ensure_staged_manifest_is_clean(self, &stage, root_id)?; + let backend = report.backend(); + operation.set_state(MaterializationOperationState::Verified)?; + operation.publish()?; + ensure_staged_manifest_is_clean(self, destination, root_id)?; + operation.set_state(MaterializationOperationState::Published)?; + Ok(MaterializationOutcome { + resolved_mode: LaneWorkdirMode::Sparse, + backend, + report, + materialization_operation_id: operation_id, + }) + })(); + if result.is_err() { + operation.abort(); + } + result + } + + fn materialize_strict_native_attempt( + &self, + root_id: &ObjectId, + destination: &Path, + ) -> std::result::Result { + let files = self.load_root_files(root_id)?; + let source = self + .resolve_native_materialization_source(root_id, &files)? + .ok_or(NativeAttemptError::Unavailable( + MaterializationFallbackReason::NativeSourceUnavailable, + ))?; + + let mut operation = self.create_materialization_stage(destination, root_id)?; + let operation_id = operation.record.operation_id.clone(); + let stage = operation.path().to_path_buf(); + let result = (|| { + verify_same_native_filesystem(&source.root, &stage)?; + probe_native_clone(&stage)?; + // Pin this while the stage is still allowed to change. Case + // sensitivity detection creates and unlinks a probe file and must + // never run during either post-barrier verification. + let case_insensitive = is_case_insensitive_filesystem(&stage)?; + + let mut stamps = BTreeMap::new(); + let mut report = MaterializationReport::default(); + let results = files + .par_iter() + .map(|(path, entry)| { + let source_stamp = + source + .stamps + .get(path) + .ok_or(NativeAttemptError::Unavailable( + MaterializationFallbackReason::NativeSourceUnavailable, + ))?; + let status = materialize_workspace_file_cow_status_if_stamp_matches( + &source.root, + &stage, + path, + entry, + *source_stamp, + false, + )?; + Ok((path.clone(), entry.size_bytes, status)) + }) + .collect::>>(); + for result in results { + let (path, size_bytes, status) = result?; + match status { + WorkspaceCowMaterializeStatus::Cloned(stamp) => { + stamps.insert(path, stamp); + report.cloned_files += 1; + report.cloned_bytes += size_bytes; + } + WorkspaceCowMaterializeStatus::Skipped => { + return Err(NativeAttemptError::Unavailable( + MaterializationFallbackReason::NativeSourceUnavailable, + )); + } + WorkspaceCowMaterializeStatus::Unavailable(reason) => { + return Err(NativeAttemptError::Unavailable(fallback_reason_for_clone( + reason, + ))); + } + } + } + + self.write_clean_workdir_manifest_from_stamps( + &stage, + root_id, + &files, + files.keys(), + stamps, + )?; + // The stage is private and cannot be published until this batch + // durability barrier succeeds. Flush every source and cloned + // inode with ordinary fsync, every created directory ancestor + // bottom-up, and finally one full volume/filesystem barrier. The + // manifest is deliberately written first so `.trail`, its file, + // and the stage-parent binding are included in the same cut. + sync_native_materialization_stage(&source.root, &stage, files.keys())?; + ensure_staged_manifest_is_clean_read_only(self, &stage, root_id, case_insensitive)?; + operation.set_state(MaterializationOperationState::Verified)?; + operation.publish()?; + ensure_staged_manifest_is_clean_read_only( + self, + destination, + root_id, + case_insensitive, + )?; + operation.set_state(MaterializationOperationState::Published)?; + Ok(MaterializationOutcome { + resolved_mode: LaneWorkdirMode::NativeCow, + backend: WorkdirBackend::Clone, + report, + materialization_operation_id: operation_id, + }) + })(); + if result.is_err() { + operation.abort(); + } + result + } + + fn materialize_portable_attempt( + &self, + root_id: &ObjectId, + destination: &Path, + fallback_reason: Option, + ) -> Result { + let files = self.load_root_files(root_id)?; + let source = self.resolve_native_materialization_source(root_id, &files)?; + let mut operation = self.create_materialization_stage(destination, root_id)?; + let operation_id = operation.record.operation_id.clone(); + let stage = operation.path().to_path_buf(); + let result = (|| { + let mut stamps = BTreeMap::new(); + let mut report = MaterializationReport { + fallback_reason, + ..MaterializationReport::default() + }; + let empty = BTreeMap::new(); + for (path, entry) in &files { + let mut cloned = false; + let clone_status = if let Some(native_source) = source.as_ref() { + if let Some(source_stamp) = native_source.stamps.get(path) { + Some(materialize_workspace_file_cow_status_if_stamp_matches( + &native_source.root, + &stage, + path, + entry, + *source_stamp, + true, + )?) + } else { + None + } + } else { + Some( + materialize_workspace_file_cow_status_if_matching_with_durability( + &self.workspace_root, + &stage, + path, + entry, + true, + )?, + ) + }; + if let Some(status) = clone_status { + match status { + WorkspaceCowMaterializeStatus::Cloned(stamp) => { + stamps.insert(path.clone(), stamp); + report.cloned_files += 1; + report.cloned_bytes += entry.size_bytes; + cloned = true; + } + WorkspaceCowMaterializeStatus::Skipped + | WorkspaceCowMaterializeStatus::Unavailable(_) => {} + } + } + if !cloned { + let one = BTreeMap::from([(path.clone(), entry.clone())]); + let materialized = self.materialize_files_at_report(&stage, &empty, &one)?; + stamps.extend(materialized.stamps); + report.copied_files += 1; + report.copied_bytes += entry.size_bytes; + } + } + + self.write_clean_workdir_manifest_from_stamps( + &stage, + root_id, + &files, + files.keys(), + stamps, + )?; + ensure_staged_manifest_is_clean(self, &stage, root_id)?; + let backend = report.backend(); + operation.set_state(MaterializationOperationState::Verified)?; + operation.publish()?; + ensure_staged_manifest_is_clean(self, destination, root_id)?; + operation.set_state(MaterializationOperationState::Published)?; + Ok(MaterializationOutcome { + resolved_mode: LaneWorkdirMode::PortableCopy, + backend, + report, + materialization_operation_id: operation_id, + }) + })(); + if result.is_err() { + operation.abort(); + } + result + } + + fn resolve_native_materialization_source( + &self, + root_id: &ObjectId, + files: &BTreeMap, + ) -> Result> { + // Strict source selection must not trust a possibly stale daemon/index + // snapshot. Hash the current workspace entries before choosing it. + let workspace_stamps = self.workspace_file_stamps_if_entries_match(files)?; + if let Some(stamps) = workspace_stamps { + return Ok(Some(NativeSource { + root: self.workspace_root.clone(), + stamps, + })); + } + + let mut statement = self.conn.prepare( + "SELECT workdir FROM lane_branches \ + WHERE head_root = ?1 AND workdir IS NOT NULL ORDER BY updated_at DESC", + )?; + let candidates = statement + .query_map(params![root_id.0], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + for candidate in candidates { + let root = PathBuf::from(candidate); + if !root.is_dir() + || !matches!( + self.cached_workdir_manifest_status(&root, root_id)?, + CachedWorkdirManifestStatus::Clean + ) + { + continue; + } + let mut stamps = BTreeMap::new(); + let mut complete = true; + for path in files.keys() { + let source = safe_join(&root, path)?; + let metadata = match fs::symlink_metadata(&source) { + Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => { + metadata + } + Ok(_) => { + complete = false; + break; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + complete = false; + break; + } + Err(error) => return Err(Error::Io(error)), + }; + stamps.insert(path.clone(), WorktreeFileStamp::from_metadata(&metadata)); + } + if complete { + return Ok(Some(NativeSource { root, stamps })); + } + } + Ok(None) + } + + fn create_materialization_stage( + &self, + destination: &Path, + root_id: &ObjectId, + ) -> Result { + let parent = destination + .parent() + .ok_or_else(|| Error::InvalidPath { + path: destination.to_string_lossy().to_string(), + reason: "lane workdir has no parent".to_string(), + })? + .canonicalize()?; + let leaf = destination + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| Error::InvalidPath { + path: destination.to_string_lossy().to_string(), + reason: "lane workdir leaf is not UTF-8".into(), + })?; + let (parent_device, parent_inode) = SecureDirectory::open_absolute(&parent)?.identity()?; + let journal_dir = self.db_dir.join("materialization-operations"); + create_dir_all_durable(&journal_dir)?; + let journal = SecureDirectory::open_absolute(&journal_dir.canonicalize()?)?; + for _ in 0..32 { + let parent_authority = SecureDirectory::open_absolute(&parent)?; + parent_authority.verify_identity((parent_device, parent_inode))?; + let operation_id = format!("materialize-{}", now_nanos()); + let stage_leaf = format!(".{leaf}.trail-{operation_id}"); + let stage = parent.join(&stage_leaf); + let record_path = journal_dir.join(format!("{operation_id}.json")); + let record = MaterializationOperationRecord { + version: 2, + operation_id, + parent: parent.to_string_lossy().to_string(), + parent_device, + parent_inode, + destination_leaf: leaf.to_string(), + stage_leaf, + owned_device: 0, + owned_inode: 0, + root_id: root_id.0.clone(), + state: MaterializationOperationState::Preparing, + owner_pid: std::process::id(), + owner_start_token: current_process_start_token(), + }; + let record_leaf = record_path + .file_name() + .and_then(|leaf| leaf.to_str()) + .ok_or_else(|| Error::Corrupt("invalid materialization record leaf".into()))?; + if journal + .read_regular_optional_bounded(record_leaf, MAX_MATERIALIZATION_OPERATION_BYTES)? + .is_some() + { + continue; + } + journal.write_atomic_regular(record_leaf, &serde_json::to_vec(&record)?)?; + match parent_authority.create_private_dir(&record.stage_leaf) { + Ok(stage_authority) => { + let mut registered = RegisteredMaterializationStage { + record_path, + stage_path: stage, + parent_authority, + stage_authority, + record, + }; + let initialized = (|| { + registered.stage_authority.sync()?; + registered.parent_authority.sync()?; + let owned = registered.stage_authority.identity()?; + registered.record.owned_device = owned.0; + registered.record.owned_inode = owned.1; + registered.set_state(MaterializationOperationState::Materializing) + })(); + match initialized { + Ok(()) => return Ok(registered), + Err(error) => { + registered.abort(); + return Err(error); + } + } + } + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::AlreadyExists => { + let _ = journal.remove_leaf(record_leaf); + } + Err(error) => { + let _ = journal.remove_leaf(record_leaf); + return Err(error); + } + } + } + Err(Error::InvalidInput( + "could not create a unique workdir materialization stage".to_string(), + )) + } + + pub(crate) fn recover_materialization_stages(&self) -> Result<()> { + let journal_dir = self.db_dir.join("materialization-operations"); + if !journal_dir.is_dir() { + return Ok(()); + } + let journal = SecureDirectory::open_absolute(&journal_dir)?; + for entry in journal.entry_names()? { + let Some(entry) = entry.to_str() else { + return Err(Error::Corrupt( + "materialization journal has a non-UTF8 entry".into(), + )); + }; + if !entry.ends_with(".json") { + continue; + } + let bytes = journal + .read_regular_optional_bounded(entry, MAX_MATERIALIZATION_OPERATION_BYTES)? + .ok_or_else(|| Error::Corrupt("materialization journal entry vanished".into()))?; + let record: MaterializationOperationRecord = serde_json::from_slice(&bytes)?; + if record.version != 2 + || entry.strip_suffix(".json") != Some(record.operation_id.as_str()) + { + return Err(Error::Corrupt(format!( + "invalid materialization operation record `{entry}`" + ))); + } + if process_matches_start_token(record.owner_pid, &record.owner_start_token) { + continue; + } + let associated = self.materialization_record_has_lane_association(&record)?; + if !associated { + remove_owned_materialization_tree(&record, true)?; + } + journal.remove_leaf(entry)?; + } + Ok(()) + } + + pub(crate) fn complete_materialization_operation(&self, operation_id: &str) -> Result<()> { + let journal_dir = self.db_dir.join("materialization-operations"); + let journal = SecureDirectory::open_absolute(&journal_dir)?; + let name = format!("{operation_id}.json"); + let bytes = journal + .read_regular_optional_bounded(&name, MAX_MATERIALIZATION_OPERATION_BYTES)? + .ok_or_else(|| Error::Corrupt("materialization operation record disappeared".into()))?; + let record: MaterializationOperationRecord = serde_json::from_slice(&bytes)?; + let associated = self.materialization_record_has_lane_association(&record)?; + if !associated { + return Err(Error::Corrupt( + "materialized destination was not atomically associated with a lane row".into(), + )); + } + journal.remove_leaf(&name) + } + + pub(crate) fn abort_materialization_operation(&self, operation_id: &str) -> Result<()> { + let journal_dir = self.db_dir.join("materialization-operations"); + let journal = SecureDirectory::open_absolute(&journal_dir)?; + let name = format!("{operation_id}.json"); + let bytes = journal + .read_regular_optional_bounded(&name, MAX_MATERIALIZATION_OPERATION_BYTES)? + .ok_or_else(|| Error::Corrupt("materialization operation record disappeared".into()))?; + let record: MaterializationOperationRecord = serde_json::from_slice(&bytes)?; + if record.version != 2 || record.operation_id != operation_id { + return Err(Error::Corrupt(format!( + "invalid materialization operation record `{name}`" + ))); + } + if self.materialization_record_has_lane_association(&record)? { + return Err(Error::Corrupt( + "refusing to abort materialization associated with a lane row".into(), + )); + } + remove_owned_materialization_tree(&record, true)?; + journal.remove_leaf(&name) + } + + fn materialization_record_has_lane_association( + &self, + record: &MaterializationOperationRecord, + ) -> Result { + let mut statement = self.conn.prepare( + "SELECT workdir FROM lane_branches + WHERE workdir IS NOT NULL AND status<>'removed'", + )?; + let candidates = statement + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + let parent = SecureDirectory::open_absolute(Path::new(&record.parent))?; + parent.verify_identity((record.parent_device, record.parent_inode))?; + let expected_destination = Path::new(&record.parent).join(&record.destination_leaf); + for candidate in candidates { + let candidate = normalize_workdir_path(&PathBuf::from(candidate))?; + let candidate = match candidate.canonicalize() { + Ok(candidate) => candidate, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(Error::Io(error)), + }; + if candidate != expected_destination { + continue; + } + let Some(leaf) = candidate.file_name().and_then(|leaf| leaf.to_str()) else { + continue; + }; + match parent.open_dir(leaf) { + Ok(directory) + if directory.identity()? == (record.owned_device, record.owned_inode) => + { + return Ok(true); + } + Ok(_) => {} + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(false) + } +} + +fn write_materialization_record( + record_path: &Path, + record: &MaterializationOperationRecord, +) -> Result<()> { + let parent = record_path + .parent() + .ok_or_else(|| Error::Corrupt("materialization operation record has no parent".into()))?; + let leaf = record_path + .file_name() + .and_then(|leaf| leaf.to_str()) + .ok_or_else(|| Error::Corrupt("invalid materialization operation record leaf".into()))?; + SecureDirectory::open_absolute(&parent.canonicalize()?)? + .write_atomic_regular(leaf, &serde_json::to_vec(record)?) +} + +fn remove_materialization_record(record_path: &Path) -> Result<()> { + let parent = record_path + .parent() + .ok_or_else(|| Error::Corrupt("materialization operation record has no parent".into()))?; + let leaf = record_path + .file_name() + .and_then(|leaf| leaf.to_str()) + .ok_or_else(|| Error::Corrupt("invalid materialization operation record leaf".into()))?; + SecureDirectory::open_absolute(&parent.canonicalize()?)?.remove_leaf(leaf) +} + +fn remove_owned_materialization_tree( + record: &MaterializationOperationRecord, + include_published_destination: bool, +) -> Result<()> { + if record.owned_device == 0 || record.owned_inode == 0 { + let parent = SecureDirectory::open_absolute(Path::new(&record.parent))?; + parent.verify_identity((record.parent_device, record.parent_inode))?; + return match parent.open_dir(&record.stage_leaf) { + Ok(stage) if stage.entry_names()?.is_empty() => { + let identity = stage.identity()?; + parent.remove_owned_tree_leaf(&record.stage_leaf, identity) + } + Ok(_) => Err(Error::Corrupt(format!( + "unbound materialization stage `{}` contains data without persisted inode authority", + record.stage_leaf + ))), + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + }; + } + let parent = SecureDirectory::open_absolute(Path::new(&record.parent))?; + parent.verify_identity((record.parent_device, record.parent_inode))?; + let leaf = match parent.open_dir(&record.stage_leaf) { + Ok(stage) => { + stage.verify_identity((record.owned_device, record.owned_inode))?; + record.stage_leaf.as_str() + } + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => { + if !include_published_destination { + return Ok(()); + } + let destination = parent.open_dir(&record.destination_leaf)?; + destination.verify_identity((record.owned_device, record.owned_inode))?; + record.destination_leaf.as_str() + } + Err(error) => return Err(error), + }; + parent.remove_owned_tree_leaf(leaf, (record.owned_device, record.owned_inode)) +} + +fn native_attempt_error(error: NativeAttemptError) -> Error { + match error { + NativeAttemptError::Unavailable(MaterializationFallbackReason::CloneUnsupported) => { + Error::CloneUnsupported + } + NativeAttemptError::Unavailable(MaterializationFallbackReason::CrossDevice) => { + Error::CloneCrossDevice + } + NativeAttemptError::Unavailable(MaterializationFallbackReason::NativeSourceUnavailable) => { + Error::NativeCowSourceUnavailable + } + NativeAttemptError::Hard(error) => error, + } +} + +fn fallback_reason_for_clone(reason: NativeCloneUnavailable) -> MaterializationFallbackReason { + match reason { + NativeCloneUnavailable::Unsupported => MaterializationFallbackReason::CloneUnsupported, + NativeCloneUnavailable::CrossDevice => MaterializationFallbackReason::CrossDevice, + } +} + +fn prepare_staged_destination(destination: &Path, custom_workdir: bool) -> Result<()> { + let parent = destination.parent().ok_or_else(|| Error::InvalidPath { + path: destination.to_string_lossy().to_string(), + reason: "lane workdir has no parent".to_string(), + })?; + create_dir_all_durable(parent)?; + match fs::symlink_metadata(destination) { + Ok(metadata) => { + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(Error::InvalidPath { + path: destination.to_string_lossy().to_string(), + reason: "lane workdir must be an absent or empty directory".to_string(), + }); + } + if fs::read_dir(destination)?.next().is_some() { + let qualifier = if custom_workdir { "custom " } else { "" }; + return Err(Error::InvalidInput(format!( + "{qualifier}lane workdir `{}` must be empty or absent", + destination.display() + ))); + } + fs::remove_dir(destination)?; + sync_directory_strict(parent)?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(Error::Io(error)), + } + Ok(()) +} + +#[cfg(unix)] +fn verify_same_native_filesystem( + source: &Path, + stage: &Path, +) -> std::result::Result<(), NativeAttemptError> { + use std::os::unix::fs::MetadataExt; + + if fs::metadata(source)?.dev() != fs::metadata(stage)?.dev() { + return Err(NativeAttemptError::Unavailable( + MaterializationFallbackReason::CrossDevice, + )); + } + Ok(()) +} + +#[cfg(not(unix))] +fn verify_same_native_filesystem( + _source: &Path, + _stage: &Path, +) -> std::result::Result<(), NativeAttemptError> { + Ok(()) +} + +fn probe_native_clone(stage: &Path) -> std::result::Result<(), NativeAttemptError> { + let source = stage.join(".trail-native-cow-probe-source"); + let destination = stage.join(".trail-native-cow-probe-destination"); + fs::write(&source, b"trail-native-cow-probe")?; + let outcome = clone_file_native(&source, &destination)?; + let _ = fs::remove_file(&source); + let _ = fs::remove_file(&destination); + match outcome { + NativeCloneOutcome::Cloned => Ok(()), + NativeCloneOutcome::Unavailable(reason) => Err(NativeAttemptError::Unavailable( + fallback_reason_for_clone(reason), + )), + } +} + +#[derive(Debug, PartialEq, Eq)] +struct NativeMaterializationDurabilityInventory { + source_files: BTreeSet, + destination_files: BTreeSet, + directories_bottom_up: Vec, +} + +fn native_materialization_durability_inventory<'a, I>( + source: &Path, + stage: &Path, + paths: I, +) -> Result +where + I: IntoIterator, +{ + let mut source_files = BTreeSet::new(); + let mut destination_files = BTreeSet::new(); + let mut directories = BTreeSet::new(); + for path in paths { + let normalized = normalize_relative_path(path)?; + let relative = path_from_rel(&normalized); + source_files.insert(source.join(&relative)); + destination_files.insert(stage.join(&relative)); + let mut parent = relative.parent(); + while let Some(relative_parent) = parent { + if relative_parent.as_os_str().is_empty() { + break; + } + directories.insert(stage.join(relative_parent)); + parent = relative_parent.parent(); + } + } + let manifest = stage.join(".trail/workdir-manifest.json"); + destination_files.insert(manifest); + directories.insert(stage.join(".trail")); + directories.insert(stage.to_path_buf()); + if let Some(parent) = stage.parent() { + directories.insert(parent.to_path_buf()); + } + let mut directories_bottom_up = directories.into_iter().collect::>(); + directories_bottom_up.sort_by_key(|path| std::cmp::Reverse(path.components().count())); + Ok(NativeMaterializationDurabilityInventory { + source_files, + destination_files, + directories_bottom_up, + }) +} + +fn sync_native_materialization_stage<'a, I>(source: &Path, stage: &Path, paths: I) -> Result<()> +where + I: IntoIterator, +{ + let inventory = native_materialization_durability_inventory(source, stage, paths)?; + let mut files = inventory.source_files.into_iter().collect::>(); + files.extend(inventory.destination_files); + files.par_iter().try_for_each(|path| -> Result<()> { + let file = OpenOptions::new().read(true).open(path)?; + // Rust's File::sync_all maps to F_FULLFSYNC on Apple platforms. + // Use POSIX fsync for each inode, then one F_FULLFSYNC below. + rustix::fs::fsync(&file).map_err(|error| Error::Io(error.into())) + })?; + for directory in inventory.directories_bottom_up { + let directory = OpenOptions::new().read(true).open(directory)?; + rustix::fs::fsync(&directory).map_err(|error| Error::Io(error.into()))?; + } + let stage_authority = OpenOptions::new().read(true).open(stage)?; + #[cfg(target_os = "linux")] + { + rustix::fs::syncfs(&stage_authority).map_err(|error| Error::Io(error.into()))?; + } + #[cfg(target_os = "macos")] + { + rustix::fs::fcntl_fullfsync(&stage_authority).map_err(|error| Error::Io(error.into()))?; + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + stage_authority.sync_all()?; + } + Ok(()) +} + +fn ensure_staged_manifest_is_clean(trail: &Trail, stage: &Path, root_id: &ObjectId) -> Result<()> { + if !matches!( + trail.cached_workdir_manifest_status(stage, root_id)?, + CachedWorkdirManifestStatus::Clean + ) { + return Err(Error::Corrupt(format!( + "staged lane workdir `{}` did not verify against root `{}`", + stage.display(), + root_id.0 + ))); + } + Ok(()) +} + +fn ensure_staged_manifest_is_clean_read_only( + trail: &Trail, + stage: &Path, + root_id: &ObjectId, + case_insensitive: bool, +) -> Result<()> { + if !trail.clean_workdir_manifest_matches_read_only(stage, root_id, case_insensitive)? { + return Err(Error::Corrupt(format!( + "staged lane workdir `{}` did not verify read-only against root `{}`", + stage.display(), + root_id.0 + ))); + } + Ok(()) +} + +#[cfg(test)] +fn publish_materialization_stage(stage: &Path, destination: &Path) -> Result<()> { + let stage_parent = stage.parent().ok_or_else(|| Error::InvalidPath { + path: stage.to_string_lossy().to_string(), + reason: "materialization stage has no parent".into(), + })?; + let destination_parent = destination.parent().ok_or_else(|| Error::InvalidPath { + path: destination.to_string_lossy().to_string(), + reason: "lane workdir destination has no parent".into(), + })?; + let stage_parent = stage_parent.canonicalize()?; + let destination_parent = destination_parent.canonicalize()?; + if stage_parent != destination_parent { + return Err(Error::InvalidInput( + "materialization stage and destination must share a parent".into(), + )); + } + let stage_leaf = stage + .file_name() + .and_then(|leaf| leaf.to_str()) + .ok_or_else(|| Error::InvalidInput("materialization stage leaf is invalid".into()))?; + let destination_leaf = destination + .file_name() + .and_then(|leaf| leaf.to_str()) + .ok_or_else(|| Error::InvalidInput("lane workdir destination leaf is invalid".into()))?; + let parent = SecureDirectory::open_absolute(&stage_parent)?; + let stage_identity = parent.open_dir(stage_leaf)?.identity()?; + parent.rename_leaf_noreplace(stage_leaf, destination_leaf)?; + parent.sync()?; + parent + .open_dir(destination_leaf)? + .verify_identity(stage_identity)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, PartialEq, Eq)] + struct ReadOnlyVerificationEntry { + is_dir: bool, + len: u64, + modified: Option, + contents: Option>, + } + + fn read_only_verification_snapshot( + root: &Path, + ) -> BTreeMap { + fn visit( + root: &Path, + path: &Path, + snapshot: &mut BTreeMap, + ) { + let metadata = fs::symlink_metadata(path).unwrap(); + let relative = path.strip_prefix(root).unwrap().to_path_buf(); + snapshot.insert( + relative, + ReadOnlyVerificationEntry { + is_dir: metadata.is_dir(), + len: metadata.len(), + modified: metadata.modified().ok(), + contents: metadata.is_file().then(|| fs::read(path).unwrap()), + }, + ); + if metadata.is_dir() { + let mut children = fs::read_dir(path) + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect::>(); + children.sort(); + for child in children { + visit(root, &child, snapshot); + } + } + } + + let mut snapshot = BTreeMap::new(); + visit(root, root, &mut snapshot); + snapshot + } + + #[test] + fn batch_durability_inventory_covers_nested_inodes_manifest_and_all_ancestors() { + let source = Path::new("/source"); + let stage = Path::new("/parent/stage"); + let paths = ["nested/deeper/file.txt".to_string(), "top.txt".to_string()]; + + let inventory = + native_materialization_durability_inventory(source, stage, paths.iter()).unwrap(); + + assert_eq!( + inventory.source_files, + BTreeSet::from([ + source.join("nested/deeper/file.txt"), + source.join("top.txt") + ]) + ); + assert_eq!( + inventory.destination_files, + BTreeSet::from([ + stage.join("nested/deeper/file.txt"), + stage.join("top.txt"), + stage.join(".trail/workdir-manifest.json") + ]) + ); + let directories = inventory + .directories_bottom_up + .iter() + .cloned() + .collect::>(); + assert_eq!( + directories, + BTreeSet::from([ + stage.join("nested/deeper"), + stage.join("nested"), + stage.join(".trail"), + stage.to_path_buf(), + stage.parent().unwrap().to_path_buf(), + ]) + ); + assert!(inventory + .directories_bottom_up + .windows(2) + .all(|pair| pair[0].components().count() >= pair[1].components().count())); + } + + #[test] + fn post_barrier_verifications_do_not_mutate_stage_or_published_destination() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root contents").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let root_id = db.resolve_branch_ref("main").unwrap().root_id; + let files = db.load_root_files(&root_id).unwrap(); + + let holder = tempfile::tempdir().unwrap(); + let stage = holder.path().join("stage"); + let destination = holder.path().join("destination"); + fs::create_dir(&stage).unwrap(); + fs::write(stage.join("README.md"), "root contents").unwrap(); + + // This is the only case-sensitivity probe. It deliberately happens + // before the manifest/barrier cut represented by the first snapshot. + let case_insensitive = is_case_insensitive_filesystem(&stage).unwrap(); + db.write_clean_workdir_manifest(&stage, &root_id, &files, files.keys()) + .unwrap(); + + let before_stage_verification = read_only_verification_snapshot(&stage); + ensure_staged_manifest_is_clean_read_only(&db, &stage, &root_id, case_insensitive).unwrap(); + assert_eq!( + read_only_verification_snapshot(&stage), + before_stage_verification + ); + + fs::rename(&stage, &destination).unwrap(); + let before_destination_verification = read_only_verification_snapshot(&destination); + ensure_staged_manifest_is_clean_read_only(&db, &destination, &root_id, case_insensitive) + .unwrap(); + assert_eq!( + read_only_verification_snapshot(&destination), + before_destination_verification + ); + } + + #[test] + fn publication_never_overwrites_a_concurrent_destination() { + let temp = tempfile::tempdir().unwrap(); + let stage = temp.path().join("stage"); + let destination = temp.path().join("workdir"); + fs::create_dir(&stage).unwrap(); + fs::write(stage.join("new.txt"), "new").unwrap(); + fs::create_dir(&destination).unwrap(); + fs::write(destination.join("rival.txt"), "rival").unwrap(); + + assert!(publish_materialization_stage(&stage, &destination).is_err()); + assert_eq!( + fs::read_to_string(destination.join("rival.txt")).unwrap(), + "rival" + ); + assert_eq!(fs::read_to_string(stage.join("new.txt")).unwrap(), "new"); + } + + #[test] + fn staged_destination_rejects_unowned_nonempty_directory() { + let temp = tempfile::tempdir().unwrap(); + let destination = temp.path().join("workdir"); + fs::create_dir(&destination).unwrap(); + fs::write(destination.join("owned-by-user.txt"), "keep").unwrap(); + + assert!(prepare_staged_destination(&destination, true).is_err()); + assert_eq!( + fs::read_to_string(destination.join("owned-by-user.txt")).unwrap(), + "keep" + ); + } + + #[test] + fn publication_rejects_parent_path_substitution() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let root_id = db.resolve_branch_ref("main").unwrap().root_id; + let holder = tempfile::tempdir().unwrap(); + let parent = holder.path().join("parent"); + let displaced = holder.path().join("displaced"); + fs::create_dir(&parent).unwrap(); + let destination = parent.join("workdir"); + let mut operation = db + .create_materialization_stage(&destination, &root_id) + .unwrap(); + let stage_leaf = operation.record.stage_leaf.clone(); + + fs::rename(&parent, &displaced).unwrap(); + fs::create_dir(&parent).unwrap(); + assert!(operation.publish().is_err()); + assert!(!destination.exists()); + assert!(displaced.join(&stage_leaf).is_dir()); + + fs::remove_dir(&parent).unwrap(); + fs::rename(&displaced, &parent).unwrap(); + operation.abort(); + assert!(!parent.join(stage_leaf).exists()); + } + + #[test] + fn startup_recovery_removes_crashed_sparse_publication_before_association() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root contents").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let root_id = db.resolve_branch_ref("main").unwrap().root_id; + let files = db + .load_root_files_for_selections(&root_id, &["README.md".to_string()]) + .unwrap(); + let holder = tempfile::tempdir().unwrap(); + let destination = holder.path().join("sparse-workdir"); + let outcome = db + .materialize_sparse_lane_root_staged(&root_id, &destination, true, &files) + .unwrap(); + let record_path = db + .db_dir + .join("materialization-operations") + .join(format!("{}.json", outcome.materialization_operation_id)); + let mut record: MaterializationOperationRecord = + serde_json::from_slice(&fs::read(&record_path).unwrap()).unwrap(); + record.owner_pid = u32::MAX; + record.owner_start_token = "dead:sparse-spawn".to_string(); + write_materialization_record(&record_path, &record).unwrap(); + assert!(destination.join("README.md").is_file()); + drop(db); + + Trail::open(workspace.path()).unwrap(); + + assert!(!destination.exists()); + assert!(!record_path.exists()); + } + + #[test] + fn startup_recovery_removes_only_registered_incomplete_stage() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let root_id = db.resolve_branch_ref("main").unwrap().root_id; + let parent = tempfile::tempdir().unwrap(); + let destination = parent.path().join("workdir"); + let registered = db + .create_materialization_stage(&destination, &root_id) + .unwrap(); + let mut registered = registered; + let stage = registered.path().to_path_buf(); + let record = registered.record_path.clone(); + fs::write(stage.join("partial.txt"), "partial").unwrap(); + let unregistered = parent.path().join(".workdir.trail-unregistered"); + fs::create_dir(&unregistered).unwrap(); + registered.record.owner_pid = u32::MAX; + registered.record.owner_start_token = "dead:test-owner".to_string(); + registered + .set_state(MaterializationOperationState::Materializing) + .unwrap(); + drop(registered); + drop(db); + + Trail::open(workspace.path()).unwrap(); + + assert!(!stage.exists()); + assert!(!record.exists()); + assert!(unregistered.is_dir()); + } + + #[test] + fn startup_recovery_removes_published_destination_without_lane_association() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let root_id = db.resolve_branch_ref("main").unwrap().root_id; + let parent = tempfile::tempdir().unwrap(); + let destination = parent.path().join("workdir"); + let mut registered = db + .create_materialization_stage(&destination, &root_id) + .unwrap(); + let stage = registered.path().to_path_buf(); + let record = registered.record_path.clone(); + fs::write(stage.join("complete.txt"), "complete").unwrap(); + fs::rename(&stage, &destination).unwrap(); + registered.record.owner_pid = u32::MAX; + registered.record.owner_start_token = "dead:test-owner".to_string(); + registered + .set_state(MaterializationOperationState::Published) + .unwrap(); + drop(registered); + drop(db); + + Trail::open(workspace.path()).unwrap(); + + assert!(!destination.exists()); + assert!(!record.exists()); + } + + #[test] + fn startup_recovery_keeps_associated_destination_after_lane_head_advances() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let head = db.resolve_branch_ref("main").unwrap(); + let parent = tempfile::tempdir().unwrap(); + let destination = parent.path().join("workdir"); + let mut registered = db + .create_materialization_stage(&destination, &head.root_id) + .unwrap(); + let stage = registered.path().to_path_buf(); + let record = registered.record_path.clone(); + fs::write(stage.join("complete.txt"), "complete").unwrap(); + fs::rename(&stage, &destination).unwrap(); + registered.record.owner_pid = u32::MAX; + registered.record.owner_start_token = "dead:test-owner".to_string(); + registered + .set_state(MaterializationOperationState::Published) + .unwrap(); + let now = now_ts(); + db.conn + .execute( + "INSERT INTO lanes + (lane_id,name,kind,provider,model,created_at,metadata_json) + VALUES ('lane_recovery','recovery','coding-lane',NULL,NULL,?1,NULL)", + [now], + ) + .unwrap(); + db.conn + .execute( + "INSERT INTO lane_branches + (lane_id,ref_name,base_change,head_change,base_root,head_root,session_id, + workdir,status,created_at,updated_at) + VALUES ('lane_recovery',?1,?2,?2,?3,?3,NULL,?4,'active',?5,?5)", + params![ + head.name, + head.change_id.0, + head.root_id.0, + destination.to_string_lossy(), + now, + ], + ) + .unwrap(); + db.conn + .execute( + "UPDATE lane_branches SET head_root='root_advanced_after_materialization' + WHERE lane_id='lane_recovery'", + [], + ) + .unwrap(); + drop(registered); + drop(db); + + Trail::open(workspace.path()).unwrap(); + + assert_eq!( + fs::read_to_string(destination.join("complete.txt")).unwrap(), + "complete" + ); + assert!(!record.exists()); + } + + #[test] + fn startup_recovery_ignores_atomic_journal_temporary_files() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let journal_dir = db.db_dir.join("materialization-operations"); + fs::create_dir_all(&journal_dir).unwrap(); + let temporary = journal_dir.join(".materialize-1.json.trail-tmp-2"); + fs::write(&temporary, b"partial").unwrap(); + drop(db); + + Trail::open(workspace.path()).unwrap(); + + assert!(temporary.exists()); + } +} diff --git a/trail/src/db/lane/workdir/nfs_overlay.rs b/trail/src/db/lane/workdir/nfs_overlay.rs index fe146e6..05bd5cf 100644 --- a/trail/src/db/lane/workdir/nfs_overlay.rs +++ b/trail/src/db/lane/workdir/nfs_overlay.rs @@ -23,7 +23,8 @@ mod macos { const ROOT_INO: u64 = 1; const OVERLAY_META_DIR: &str = ".trail"; - const MOUNT_STATE_FILE: &str = "mount.json"; + const NFS_MOUNT_STATE_FILE: &str = "nfs-mount.json"; + const LEGACY_NFS_MOUNT_STATE_FILE: &str = "mount.json"; type CowCore = ViewCore; type NodeKind = ViewNodeKind; @@ -270,6 +271,52 @@ mod macos { } } + struct PendingNfsMount { + mountpoint: PathBuf, + state_path: PathBuf, + shutdown: Option>, + worker: Option>, + mounted: bool, + committed: bool, + } + + impl PendingNfsMount { + fn commit( + mut self, + ) -> ( + PathBuf, + PathBuf, + tokio::sync::oneshot::Sender<()>, + JoinHandle<()>, + ) { + self.committed = true; + ( + std::mem::take(&mut self.mountpoint), + std::mem::take(&mut self.state_path), + self.shutdown.take().expect("pending NFS shutdown sender"), + self.worker.take().expect("pending NFS worker"), + ) + } + } + + impl Drop for PendingNfsMount { + fn drop(&mut self) { + if self.committed { + return; + } + if self.mounted { + let _ = unmount(&self.mountpoint); + } + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + let _ = fs::remove_file(&self.state_path); + } + } + pub(crate) fn prepare_nfs_cow_workdir( db: &Trail, lane: &str, @@ -284,32 +331,40 @@ mod macos { Ok(upper) } - pub(crate) fn nfs_clean_manifest_path(db: &Trail, lane: &str) -> Result { - Ok(db - .workspace_view_paths_for_lane_name(lane) - .meta_dir - .join("workdir-manifest.json")) - } - - pub(crate) fn nfs_candidate_paths(db: &Trail, lane: &str) -> Result> { + pub(crate) fn nfs_candidate_paths(db: &Trail, lane: &str) -> Result { let upper = nfs_upperdir(db, lane)?; let branch = db.lane_branch(lane)?; let head = db.get_ref(&branch.ref_name)?; - Ok( - recover_view_checkpoint_candidates_for_root(db, &upper, &head.root_id)? - .paths - .into_iter() - .filter(|path| { - !Path::new(path) - .file_name() - .and_then(OsStr::to_str) - .is_some_and(is_macos_junk) - }) - .collect(), - ) + let mut candidates = + recover_view_checkpoint_candidates_for_root(db, &upper, &head.root_id)?; + candidates.paths.retain(|path| { + !Path::new(path) + .file_name() + .and_then(OsStr::to_str) + .is_some_and(is_macos_junk) + }); + Ok(candidates) } pub(crate) fn mount_nfs_cow_for_lane(db: &Trail, lane: &str) -> Result { + mount_nfs_cow_for_lane_with_view(db, lane, None) + } + + pub(crate) fn mount_nfs_cow_for_lane_with_ephemeral_bindings( + db: &Trail, + lane: &str, + source_upper: PathBuf, + source_root: ObjectId, + bindings: Vec, + ) -> Result { + mount_nfs_cow_for_lane_with_view(db, lane, Some((source_upper, source_root, bindings))) + } + + fn mount_nfs_cow_for_lane_with_view( + db: &Trail, + lane: &str, + ephemeral: Option<(PathBuf, ObjectId, Vec)>, + ) -> Result { validate_ref_segment(lane)?; let branch = db.lane_branch(lane)?; let record = db.lane_record(&branch.lane_id)?; @@ -326,20 +381,45 @@ mod macos { "nfs-cow requires /sbin/mount_nfs on macOS".to_string(), )); } - let mut lease = db.acquire_workspace_mount_lease(lane, "nfs")?; - fs::create_dir_all(&mountpoint)?; - let upper = nfs_upperdir(db, lane)?; - fs::create_dir_all(upper.join(OVERLAY_META_DIR))?; let state_path = db .workspace_view_paths_for_lane_name(lane) .meta_dir - .join(MOUNT_STATE_FILE); + .join(NFS_MOUNT_STATE_FILE); + let legacy_state_path = state_path.with_file_name(LEGACY_NFS_MOUNT_STATE_FILE); + if fs::read(&legacy_state_path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .is_some_and(|value| { + value + .get("pid") + .and_then(serde_json::Value::as_i64) + .is_some() + }) + { + recover_stale_mount(&mountpoint, &legacy_state_path)?; + } + // A dead loopback server can leave the mountpoint in an unresponsive + // NFS state. Recover it before *any* path operation on the mountpoint; + // even `create_dir_all` or metadata can otherwise block in the kernel. + recover_stale_mount(&mountpoint, &state_path)?; + let mut lease = db.acquire_workspace_mount_lease(lane, "nfs")?; + fs::create_dir_all(&mountpoint)?; let head = db.get_ref(&branch.ref_name)?; - let core = CowCore::new_lazy( - Trail::open_with_db_dir(db.workspace_root.clone(), db.db_dir.clone())?, - upper.clone(), - head.root_id, - )?; + let (upper, source_root, bindings) = match ephemeral { + Some((upper, source_root, bindings)) => (upper, source_root, Some(bindings)), + None => (nfs_upperdir(db, lane)?, head.root_id, None), + }; + fs::create_dir_all(upper.join(OVERLAY_META_DIR))?; + let core_db = Trail::open_with_db_dir(db.workspace_root.clone(), db.db_dir.clone())?; + let core = match bindings { + Some(bindings) => CowCore::new_lazy_with_ephemeral_bindings( + core_db, + upper.clone(), + source_root, + bindings, + )?, + None => CowCore::new_lazy(core_db, upper.clone(), source_root)?, + }; let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -349,7 +429,6 @@ mod macos { .block_on(NFSTcpListener::bind("127.0.0.1:0", adapter)) .map_err(Error::Io)?; let port = listener.get_listen_port(); - recover_stale_mount(&mountpoint, &state_path)?; let state_file = OpenOptions::new() .write(true) .create_new(true) @@ -359,9 +438,15 @@ mod macos { "nfs-cow lane `{lane}` is already being mounted: {err}" )) })?; + let process_start_token = current_process_start_token(); serde_json::to_writer( &state_file, - &serde_json::json!({"pid": std::process::id(), "port": port, "mountpoint": mountpoint}), + &serde_json::json!({ + "pid": std::process::id(), + "process_start_token": process_start_token, + "port": port, + "mountpoint": mountpoint, + }), )?; state_file.sync_all()?; let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); @@ -370,32 +455,43 @@ mod macos { tokio::select! { _ = listener.handle_forever() => {}, _ = shutdown_rx => {} } }) }); + let mut pending_mount = PendingNfsMount { + mountpoint: mountpoint.clone(), + state_path: state_path.clone(), + shutdown: Some(shutdown_tx), + worker: Some(worker), + mounted: false, + committed: false, + }; // Directory mutations must be visible to the checkpoint scan that runs // immediately after the agent exits. Attribute caching can otherwise - // retain a pre-create READDIR result and omit brand-new paths. - let opts = format!("locallocks,vers=3,tcp,rsize=1048576,wsize=1048576,noac,actimeo=0,nonegnamecache,nobrowse,port={port},mountport={port}"); + // retain a pre-create READDIR result and omit brand-new paths. macOS + // also defaults to `nosync`, where a write syscall may return before + // this userspace server has received the WRITE RPC. A lane can then be + // unmounted and remounted with only the preceding truncate visible. + // `sync` makes successful writes an actual durability boundary; the + // server itself fsyncs every WRITE before reporting FILE_SYNC. + let opts = format!( + "locallocks,vers=3,tcp,sync,rsize=1048576,wsize=1048576,noac,actimeo=0,nonegnamecache,nobrowse,port={port},mountport={port}" + ); let status = Command::new("/sbin/mount_nfs") .args(["-o", &opts, "127.0.0.1:/", &mountpoint.to_string_lossy()]) .status()?; if !status.success() { - let _ = shutdown_tx.send(()); - let _ = worker.join(); - let _ = fs::remove_file(&state_path); return Err(Error::InvalidInput(format!( "mount_nfs failed for `{}` with {status}", mountpoint.display() ))); } if !is_nfs_mount(&mountpoint) { - let _ = shutdown_tx.send(()); - let _ = worker.join(); - let _ = fs::remove_file(&state_path); return Err(Error::InvalidInput(format!( "mount_nfs returned success, but `{}` is not an active NFS mount", mountpoint.display() ))); } + pending_mount.mounted = true; lease.mark_mounted()?; + let (mountpoint, state_path, shutdown_tx, worker) = pending_mount.commit(); Ok(NfsCowMount { mountpoint, state_path, @@ -411,25 +507,56 @@ mod macos { } fn recover_stale_mount(mountpoint: &Path, state: &Path) -> Result<()> { + let mut known_dead_owner = false; if let Ok(bytes) = fs::read(state) { if let Ok(value) = serde_json::from_slice::(&bytes) { if let Some(pid) = value.get("pid").and_then(serde_json::Value::as_i64) { - if unsafe { libc::kill(pid as i32, 0) } == 0 { + let token = value + .get("process_start_token") + .and_then(serde_json::Value::as_str); + if token.is_some_and(|token| process_matches_start_token(pid as u32, token)) { return Err(Error::InvalidInput(format!( "nfs-cow mount `{}` is already active in process {pid}", mountpoint.display() ))); } + known_dead_owner = true; } } } - if is_nfs_mount(mountpoint) { + if known_dead_owner { + force_unmount_stale_without_probe(mountpoint)?; + } else if is_nfs_mount(mountpoint) { unmount(mountpoint)?; } let _ = fs::remove_file(state); Ok(()) } + fn force_unmount_stale_without_probe(path: &Path) -> Result<()> { + let cpath = CString::new(path.to_string_lossy().as_bytes()) + .map_err(|_| Error::InvalidInput("invalid NFS mount path".to_string()))?; + if unsafe { libc::unmount(cpath.as_ptr(), libc::MNT_FORCE) } == 0 { + return Ok(()); + } + let error = std::io::Error::last_os_error(); + if error + .raw_os_error() + .is_some_and(|code| code == libc::EINVAL || code == libc::ENOENT) + { + return Ok(()); + } + let status = Command::new("/sbin/umount").arg("-f").arg(path).status()?; + if status.success() { + Ok(()) + } else { + Err(Error::InvalidInput(format!( + "failed to force-unmount stale NFS mount `{}` after owner death: {error}", + path.display() + ))) + } + } + fn unmount(path: &Path) -> Result<()> { if !is_nfs_mount(path) { return Ok(()); @@ -532,6 +659,9 @@ mod macos { mod tests { use super::*; + static COMMAND_AUTHORITY_TEST: std::sync::OnceLock> = + std::sync::OnceLock::new(); + #[test] fn cow_core_copies_up_writes_and_persists_whiteouts() { let temp = tempfile::tempdir().unwrap(); @@ -649,7 +779,7 @@ mod macos { let temp = tempfile::tempdir().unwrap(); let mountpoint = temp.path().join("mount"); fs::create_dir_all(&mountpoint).unwrap(); - let state = temp.path().join(MOUNT_STATE_FILE); + let state = temp.path().join(NFS_MOUNT_STATE_FILE); fs::write(&state, br#"{"pid":2147483647,"port":1}"#).unwrap(); recover_stale_mount(&mountpoint, &state).unwrap(); @@ -657,6 +787,68 @@ mod macos { assert!(!state.exists()); } + #[test] + fn stale_mount_state_uses_process_start_identity_not_pid_alone() { + let temp = tempfile::tempdir().unwrap(); + let mountpoint = temp.path().join("mount"); + fs::create_dir_all(&mountpoint).unwrap(); + let state = temp.path().join(NFS_MOUNT_STATE_FILE); + fs::write( + &state, + serde_json::to_vec(&serde_json::json!({ + "pid": std::process::id(), + "process_start_token": "reused-pid", + "port": 1, + })) + .unwrap(), + ) + .unwrap(); + + recover_stale_mount(&mountpoint, &state).unwrap(); + assert!(!state.exists()); + + fs::write( + &state, + serde_json::to_vec(&serde_json::json!({ + "pid": std::process::id(), + "process_start_token": current_process_start_token(), + "port": 1, + })) + .unwrap(), + ) + .unwrap(); + assert!(recover_stale_mount(&mountpoint, &state).is_err()); + assert!(state.exists()); + } + + #[test] + fn pending_nfs_mount_drop_cleans_state_and_worker_before_publication() { + let temp = tempfile::tempdir().unwrap(); + let mountpoint = temp.path().join("mount"); + fs::create_dir_all(&mountpoint).unwrap(); + let state_path = temp.path().join(NFS_MOUNT_STATE_FILE); + fs::write(&state_path, b"pending").unwrap(); + let (shutdown, stopped) = tokio::sync::oneshot::channel(); + let worker = thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let _ = runtime.block_on(stopped); + }); + + drop(PendingNfsMount { + mountpoint, + state_path: state_path.clone(), + shutdown: Some(shutdown), + worker: Some(worker), + mounted: false, + committed: false, + }); + + assert!(!state_path.exists()); + } + #[test] fn nfs_adapter_validates_names_paginates_and_rejects_stale_inodes() { use nfsserve::nfs::nfsstring; @@ -889,6 +1081,119 @@ mod macos { assert_eq!(checkpoint.source_paths, vec!["daemon.txt"]); } + #[test] + fn command_authority_checkpoints_unmounted_nfs_view_from_qualified_journal() { + if std::env::var_os("TRAIL_RUN_NFS_COW_TESTS").is_none() { + return; + } + let _serial = COMMAND_AUTHORITY_TEST + .get_or_init(|| std::sync::Mutex::new(())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + struct AuthorityReset; + impl Drop for AuthorityReset { + fn drop(&mut self) { + crate::db::change_ledger::set_command_authority_override(false); + } + } + + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "baseline\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let spawned = db + .spawn_lane_with_workdir_mode_paths_and_neighbors( + "authority-nfs", + Some("main"), + LaneWorkdirMode::NfsCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let lane_id = spawned.lane_id; + let workdir = PathBuf::from(spawned.workdir.unwrap()); + assert!(workdir.read_dir().unwrap().next().is_none()); + + crate::db::change_ledger::set_command_authority_override(true); + let _reset = AuthorityReset; + let empty = db + .checkpoint_lane_workspace("authority-nfs", Some("empty".into())) + .unwrap(); + assert!(empty.source_paths.is_empty()); + + let mounted = db.start_lane_workspace_mount("authority-nfs").unwrap(); + assert!(mounted.healthy); + fs::write(workdir.join("changed.txt"), "changed\n").unwrap(); + db.request_lane_workspace_unmount("authority-nfs").unwrap(); + assert!(workdir.read_dir().unwrap().next().is_none()); + let changed = db + .checkpoint_lane_workspace("authority-nfs", Some("changed".into())) + .unwrap(); + assert_eq!(changed.source_paths, vec!["changed.txt"]); + + let observer_scopes: i64 = db + .conn + .query_row( + "SELECT COUNT(*) FROM changed_path_scopes + WHERE scope_kind='materialized_lane' AND owner_id=?1", + [&lane_id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(observer_scopes, 0); + } + + #[test] + fn command_authority_nfs_checkpoint_rejects_missing_journal_pair() { + let _serial = COMMAND_AUTHORITY_TEST + .get_or_init(|| std::sync::Mutex::new(())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + struct AuthorityReset; + impl Drop for AuthorityReset { + fn drop(&mut self) { + crate::db::change_ledger::set_command_authority_override(false); + } + } + + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "baseline\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "authority-nfs-corrupt", + Some("main"), + LaneWorkdirMode::NfsCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let view = db + .lane_workspace_view("authority-nfs-corrupt") + .unwrap() + .unwrap(); + let layout = ViewUpperLayout::from_source_upper(PathBuf::from(view.source_upper)); + fs::remove_file(layout.journal_path()).unwrap(); + fs::remove_file(layout.whiteout_journal_path()).unwrap(); + + crate::db::change_ledger::set_command_authority_override(true); + let _reset = AuthorityReset; + let error = db + .checkpoint_lane_workspace("authority-nfs-corrupt", None) + .unwrap_err(); + assert!(matches!( + error, + Error::ChangeLedgerReconcileRequired { state, .. } + if state == "unqualified_view_journal" + )); + } + #[test] fn real_nfs_mount_records_new_modified_and_renamed_files() { if std::env::var_os("TRAIL_RUN_NFS_COW_TESTS").is_none() { @@ -1056,7 +1361,7 @@ mod macos { } #[test] - fn nfs_real_node_layer_is_shared_and_writable_installs_are_isolated() { + fn nfs_real_node_layer_bulk_replacement_is_isolated() { if std::env::var_os("TRAIL_RUN_NFS_COW_TESTS").is_none() { return; } @@ -1172,66 +1477,58 @@ mod macos { ); assert_eq!(sha256_hex(&fs::read(&layer_file).unwrap()), immutable_hash); - let clean = Command::new("rm") - .args(["-rf", "node_modules"]) - .current_dir(&workdir_a) - .output() - .unwrap(); - assert!(clean.status.success()); - assert!(!workdir_a.join("node_modules").exists()); - assert!(workdir_b.join("node_modules/lodash/lodash.js").is_file()); - assert_eq!(sha256_hex(&fs::read(&layer_file).unwrap()), immutable_hash); - - let install = Command::new("npm") - .args(["ci", "--ignore-scripts", "--no-audit", "--no-fund"]) - .env("npm_config_cache", temp.path().join("npm-test-cache")) - .current_dir(&workdir_a) - .output() - .unwrap(); + drop(mount_a); + let replace_started = std::time::Instant::now(); + let replaced = db.sync_node_dependencies("node-nfs-a", None).unwrap(); + let replace_ms = replace_started.elapsed().as_millis(); + assert_eq!(replaced.layer_id, first.layer_id); assert!( - install.status.success(), - "npm ci through NFS failed: {}", - String::from_utf8_lossy(&install.stderr) + replace_ms < 30_000, + "bulk dependency replacement took {replace_ms} ms" + ); + let remount_a = db.mount_nfs_cow_workdir_for_lane("node-nfs-a").unwrap(); + assert_eq!( + sha256_hex(&fs::read(workdir_a.join("node_modules/lodash/lodash.js")).unwrap()), + immutable_hash + ); + assert_eq!( + sha256_hex(&fs::read(workdir_b.join("node_modules/lodash/lodash.js")).unwrap()), + immutable_hash ); - let reinstalled_prettier = Command::new("node_modules/.bin/prettier") + assert_eq!(sha256_hex(&fs::read(&layer_file).unwrap()), immutable_hash); + let restored_prettier = Command::new("node_modules/.bin/prettier") .arg("--version") .current_dir(&workdir_a) .output() .unwrap(); assert!( - reinstalled_prettier.status.success(), - "npm-created bin symlink did not execute through NFS: {}", - String::from_utf8_lossy(&reinstalled_prettier.stderr) + restored_prettier.status.success(), + "bulk-replaced npm bin symlink did not execute through NFS: {}", + String::from_utf8_lossy(&restored_prettier.stderr) ); - for path in [ - workdir_a.join("node_modules/lodash/lodash.js"), - workdir_b.join("node_modules/lodash/lodash.js"), - layer_file.clone(), - ] { - assert_eq!(sha256_hex(&fs::read(path).unwrap()), immutable_hash); - } let lane_head_before = db.lane_details("node-nfs-a").unwrap().branch.head_change; let checkpoint = db.checkpoint_lane_workspace("node-nfs-a", None).unwrap(); assert!(checkpoint.source_paths.is_empty()); - assert!(checkpoint.generated_dirty_paths > 500); + assert_eq!(checkpoint.generated_dirty_paths, 0); assert_eq!( db.lane_details("node-nfs-a").unwrap().branch.head_change, lane_head_before ); let view_a = db.lane_workspace_view("node-nfs-a").unwrap().unwrap(); let view_b = db.lane_workspace_view("node-nfs-b").unwrap().unwrap(); - assert!(Path::new(&view_a.generated_upper) - .join("node_modules/lodash/lodash.js") - .is_file()); + assert!(!Path::new(&view_a.generated_upper) + .join("node_modules") + .exists()); assert!(!Path::new(&view_b.generated_upper) .join("node_modules/lodash/lodash.js") .exists()); db.verify_workspace_layer(&first.layer_id).unwrap(); eprintln!( - "macos-nfs-node-layer layer_entries={} mount_two_ms={} shared_layer={} generated_a_bytes={} generated_b_bytes={}", + "macos-nfs-node-layer layer_entries={} mount_two_ms={} bulk_replace_ms={} shared_layer={} generated_a_bytes={} generated_b_bytes={}", first.entry_count, mount_ms, + replace_ms, first.layer_id, db.lane_workspace_space("node-nfs-a") .unwrap() @@ -1241,7 +1538,354 @@ mod macos { .generated_upper_bytes, ); drop(mount_b); + drop(remount_a); + } + + struct NfsFrameworkFixture<'a> { + name: &'static str, + package_json: &'static str, + files: &'a [(&'static str, &'static str)], + package_probe: &'static str, + bin: &'static str, + build_args: &'static [&'static str], + build_mode: &'static str, + build_timeout_secs: u64, + require_build_success: bool, + build_output: &'static str, + min_layer_entries: u64, + } + + #[test] + fn nfs_large_nextjs_and_vite_layers_build_and_bulk_replace() { + if std::env::var_os("TRAIL_RUN_NFS_FRAMEWORK_BENCH").is_none() { + return; + } + for tool in ["node", "npm"] { + assert!( + Command::new(tool) + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()), + "{tool} is required for the NFS framework benchmark" + ); + } + + let next_files = [ + ( + "app/layout.jsx", + "export const metadata = { title: 'Trail Next benchmark' };\nexport default function Layout({ children }) { return {children}; }\n", + ), + ( + "app/page.jsx", + "export default function Page() { return
Trail Next benchmark
; }\n", + ), + ( + "next.config.mjs", + "export default { turbopack: { root: process.cwd() }, outputFileTracingRoot: process.cwd() };\n", + ), + ]; + let vite_files = [ + ( + "index.html", + "
\n", + ), + ( + "src/main.jsx", + "import React from 'react';\nimport { createRoot } from 'react-dom/client';\ncreateRoot(document.getElementById('root')).render(
Trail Vite benchmark
);\n", + ), + ( + "vite.config.mjs", + "import { defineConfig } from 'vite';\nimport react from '@vitejs/plugin-react';\nexport default defineConfig({ plugins: [react()] });\n", + ), + ]; + let fixtures = [ + NfsFrameworkFixture { + name: "next", + package_json: r#"{"name":"trail-next-nfs-bench","version":"1.0.0","private":true,"scripts":{"build":"next build"},"dependencies":{"next":"16.2.10","react":"19.2.7","react-dom":"19.2.7"}}"#, + files: &next_files, + package_probe: "next/package.json", + bin: "next", + build_args: &["build"], + build_mode: "turbopack", + build_timeout_secs: 120, + require_build_success: false, + build_output: ".next/BUILD_ID", + min_layer_entries: 2_000, + }, + NfsFrameworkFixture { + name: "vite", + package_json: r#"{"name":"trail-vite-nfs-bench","version":"1.0.0","private":true,"scripts":{"build":"vite build"},"dependencies":{"react":"19.2.7","react-dom":"19.2.7"},"devDependencies":{"@vitejs/plugin-react":"6.0.3","vite":"8.1.4"}}"#, + files: &vite_files, + package_probe: "vite/package.json", + bin: "vite", + build_args: &["build"], + build_mode: "rolldown", + build_timeout_secs: 120, + require_build_success: true, + build_output: "dist/index.html", + min_layer_entries: 300, + }, + ]; + + let filter = std::env::var("TRAIL_NFS_FRAMEWORK_FILTER").ok(); + let mut ran = 0_u32; + for fixture in fixtures { + if filter.as_deref().is_some_and(|name| name != fixture.name) { + continue; + } + run_nfs_framework_benchmark(fixture); + ran += 1; + } + assert!(ran > 0, "framework benchmark filter matched no fixture"); + } + + fn run_nfs_framework_benchmark(fixture: NfsFrameworkFixture<'_>) { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("package.json"), fixture.package_json).unwrap(); + fs::write( + temp.path().join(".gitignore"), + "node_modules/\n.next/\ndist/\ntarget/\n", + ) + .unwrap(); + for (path, contents) in fixture.files { + let path = temp.path().join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, contents).unwrap(); + } + let lock_started = std::time::Instant::now(); + let lock = Command::new("npm") + .args([ + "install", + "--package-lock-only", + "--ignore-scripts", + "--no-audit", + "--no-fund", + ]) + .current_dir(temp.path()) + .output() + .unwrap(); + let lock_ms = lock_started.elapsed().as_millis(); + assert!( + lock.status.success(), + "{} lock generation failed: {}", + fixture.name, + String::from_utf8_lossy(&lock.stderr) + ); + + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let lane_a = format!("{}-nfs-a", fixture.name); + let lane_b = format!("{}-nfs-b", fixture.name); + for lane in [&lane_a, &lane_b] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + LaneWorkdirMode::NfsCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + } + + let cold_started = std::time::Instant::now(); + let first = db.sync_node_dependencies(&lane_a, None).unwrap(); + let cold_sync_ms = cold_started.elapsed().as_millis(); + let hit_started = std::time::Instant::now(); + let second = db.sync_node_dependencies(&lane_b, None).unwrap(); + let cache_hit_ms = hit_started.elapsed().as_millis(); + assert_eq!(first.layer_id, second.layer_id); + assert_eq!(first.cache_key, second.cache_key); + assert!( + first.entry_count >= fixture.min_layer_entries, + "{} layer had only {} entries", + fixture.name, + first.entry_count + ); + assert!( + first.logical_bytes >= 10 * 1024 * 1024, + "{} layer had only {} logical bytes", + fixture.name, + first.logical_bytes + ); + assert!(cache_hit_ms < 30_000); + + let layer_probe = Path::new(&first.storage_path).join(fixture.package_probe); + let immutable_hash = sha256_hex(&fs::read(&layer_probe).unwrap()); + let layer_bin = Path::new(&first.storage_path) + .join(".bin") + .join(fixture.bin); + assert!(fs::symlink_metadata(&layer_bin) + .unwrap() + .file_type() + .is_symlink()); + db.verify_workspace_layer(&first.layer_id).unwrap(); + + let mount_started = std::time::Instant::now(); + let mount_a = db.mount_nfs_cow_workdir_for_lane(&lane_a).unwrap(); + let mount_b = db.mount_nfs_cow_workdir_for_lane(&lane_b).unwrap(); + let mount_two_ms = mount_started.elapsed().as_millis(); + let workdir_a = PathBuf::from(db.lane_workdir(&lane_a).unwrap().workdir.unwrap()); + let workdir_b = PathBuf::from(db.lane_workdir(&lane_b).unwrap().workdir.unwrap()); + + let bin_started = std::time::Instant::now(); + let version = Command::new(format!("node_modules/.bin/{}", fixture.bin)) + .arg("--version") + .current_dir(&workdir_a) + .output() + .unwrap(); + let bin_probe_ms = bin_started.elapsed().as_millis(); + assert!( + version.status.success(), + "{} bin failed through NFS: {}", + fixture.name, + String::from_utf8_lossy(&version.stderr) + ); + + let mut build_command = vec![format!("node_modules/.bin/{}", fixture.bin)]; + build_command.extend(fixture.build_args.iter().map(|arg| (*arg).to_string())); + let build = run_command_with_timeout_env( + &build_command, + &workdir_a, + Duration::from_secs(fixture.build_timeout_secs), + &[ + ("CI".to_string(), "1".to_string()), + ("NEXT_TELEMETRY_DISABLED".to_string(), "1".to_string()), + ], + ) + .unwrap(); + let build_ms = build.duration_ms; + if fixture.require_build_success { + assert!( + build.success, + "{} build failed through NFS\nstdout:\n{}\nstderr:\n{}", + fixture.name, + String::from_utf8_lossy(&build.stdout), + String::from_utf8_lossy(&build.stderr) + ); + } else { + assert!( + build.success || build.timed_out, + "{} build failed before its benchmark budget\nstdout:\n{}\nstderr:\n{}", + fixture.name, + String::from_utf8_lossy(&build.stdout), + String::from_utf8_lossy(&build.stderr) + ); + } + if build.success { + assert!(workdir_a.join(fixture.build_output).is_file()); + } + assert!(!workdir_b.join(fixture.build_output).exists()); + + let probe_a = workdir_a.join("node_modules").join(fixture.package_probe); + let probe_b = workdir_b.join("node_modules").join(fixture.package_probe); + fs::write(&probe_a, "lane-a-private\n").unwrap(); + assert_eq!(fs::read_to_string(&probe_a).unwrap(), "lane-a-private\n"); + assert_eq!(sha256_hex(&fs::read(&probe_b).unwrap()), immutable_hash); + assert_eq!(sha256_hex(&fs::read(&layer_probe).unwrap()), immutable_hash); + drop(mount_a); + let view_a = db.lane_workspace_view(&lane_a).unwrap().unwrap(); + let private_root = Path::new(&view_a.generated_upper).join("node_modules"); + let private_started = std::time::Instant::now(); + let private_entries = + materialize_private_dependency_tree(Path::new(&first.storage_path), &private_root); + let private_materialize_ms = private_started.elapsed().as_millis(); + assert_eq!(private_entries, first.entry_count); + assert!(private_root.join(fixture.package_probe).is_file()); + + let replace_started = std::time::Instant::now(); + let replaced = db.sync_node_dependencies(&lane_a, None).unwrap(); + let bulk_replace_ms = replace_started.elapsed().as_millis(); + assert_eq!(replaced.layer_id, first.layer_id); + assert!( + bulk_replace_ms < 60_000, + "{} bulk replacement took {} ms", + fixture.name, + bulk_replace_ms + ); + assert!(!private_root.exists()); + + let remount_a = db.mount_nfs_cow_workdir_for_lane(&lane_a).unwrap(); + assert_eq!(sha256_hex(&fs::read(&probe_a).unwrap()), immutable_hash); + assert_eq!(sha256_hex(&fs::read(&probe_b).unwrap()), immutable_hash); + if build.success { + assert!(workdir_a.join(fixture.build_output).is_file()); + } + let restored_bin = Command::new(format!("node_modules/.bin/{}", fixture.bin)) + .arg("--version") + .current_dir(&workdir_a) + .output() + .unwrap(); + assert!(restored_bin.status.success()); + + let checkpoint = db.checkpoint_lane_workspace(&lane_a, None).unwrap(); + assert!(checkpoint.source_paths.is_empty()); + if build.success { + assert!(checkpoint.generated_dirty_paths > 0); + } + let space_a = db.lane_workspace_space(&lane_a).unwrap(); + let space_b = db.lane_workspace_space(&lane_b).unwrap(); + assert_eq!(space_b.generated_upper_bytes, 0); + eprintln!( + "macos-nfs-framework name={} build_mode={} layer_entries={} logical_bytes={} physical_bytes={} lock_ms={} cold_sync_ms={} cache_hit_ms={} mount_two_ms={} bin_probe_ms={} build_ms={} build_success={} build_timed_out={} private_materialize_ms={} private_entries={} bulk_replace_ms={} generated_a_bytes={} generated_b_bytes={}", + fixture.name, + fixture.build_mode, + first.entry_count, + first.logical_bytes, + first.physical_bytes.unwrap_or(0), + lock_ms, + cold_sync_ms, + cache_hit_ms, + mount_two_ms, + bin_probe_ms, + build_ms, + build.success, + build.timed_out, + private_materialize_ms, + private_entries, + bulk_replace_ms, + space_a.generated_upper_bytes, + space_b.generated_upper_bytes, + ); + drop(mount_b); + drop(remount_a); + } + + fn materialize_private_dependency_tree(source: &Path, destination: &Path) -> u64 { + if destination.exists() { + fs::remove_dir_all(destination).unwrap(); + } + fs::create_dir_all(destination).unwrap(); + let mut entries = 0_u64; + for entry in walkdir::WalkDir::new(source).follow_links(false) { + let entry = entry.unwrap(); + if entry.path() == source { + continue; + } + entries += 1; + let relative = entry.path().strip_prefix(source).unwrap(); + let target = destination.join(relative); + if entry.file_type().is_dir() { + fs::create_dir_all(&target).unwrap(); + } else if entry.file_type().is_symlink() { + fs::create_dir_all(target.parent().unwrap()).unwrap(); + std::os::unix::fs::symlink(fs::read_link(entry.path()).unwrap(), target) + .unwrap(); + } else { + fs::create_dir_all(target.parent().unwrap()).unwrap(); + clone_or_copy_projected_file(entry.path(), &target).unwrap(); + let metadata = fs::metadata(entry.path()).unwrap(); + fs::set_permissions( + &target, + fs::Permissions::from_mode(metadata.permissions().mode() | 0o200), + ) + .unwrap(); + } + } + entries } #[test] @@ -1330,37 +1974,10 @@ mod macos { let target_a = PathBuf::from(&view_a.generated_upper).join("target"); assert!(tree_has_name_fragment(&target_a, "libshared_dep")); - let cargo_version = Command::new("cargo").arg("--version").output().unwrap(); - let key = WorkspaceLayerKeyV1 { - kind: "compiler-results".to_string(), - adapter: "cargo-target-seed".to_string(), - adapter_version: 1, - inputs: BTreeMap::from([ - ("source_root".to_string(), first.source_root.0.clone()), - ("command".to_string(), "cargo build --offline".to_string()), - ]), - tool_versions: BTreeMap::from([( - "cargo".to_string(), - String::from_utf8_lossy(&cargo_version.stdout) - .trim() - .to_string(), - )]), - platform: std::env::consts::OS.to_string(), - architecture: std::env::consts::ARCH.to_string(), - portability_scope: "source-root-toolchain-platform".to_string(), - strategy: "immutable-target-seed".to_string(), - }; let layer = db - .publish_workspace_layer_from_directory(&key, &target_a) + .sync_workspace_environment("rust-nfs-b", "cargo", None) .unwrap(); - db.attach_workspace_layer( - "rust-nfs-b", - &layer.layer_id, - "target", - "cargo-target-seed", - &layer.cache_key, - ) - .unwrap(); + assert_eq!(layer.adapter, "cargo-target-seed"); let second = db .exec_lane_workspace( @@ -1576,14 +2193,20 @@ pub(crate) fn mount_nfs_cow_for_lane(_db: &Trail, _lane: &str) -> Result Result { +pub(crate) fn mount_nfs_cow_for_lane_with_ephemeral_bindings( + _db: &Trail, + _lane: &str, + _source_upper: PathBuf, + _source_root: ObjectId, + _bindings: Vec, +) -> Result { Err(Error::InvalidInput( "nfs-cow workdirs are currently supported only on macOS".to_string(), )) } #[cfg(not(target_os = "macos"))] -pub(crate) fn nfs_candidate_paths(_db: &Trail, _lane: &str) -> Result> { +pub(crate) fn nfs_candidate_paths(_db: &Trail, _lane: &str) -> Result { Err(Error::InvalidInput( "nfs-cow workdirs are currently supported only on macOS".to_string(), )) @@ -1599,15 +2222,30 @@ impl Trail { prepare_nfs_cow_workdir(self, lane, dir, custom) } - pub fn mount_nfs_cow_workdir_for_lane(&self, lane: &str) -> Result { + pub fn mount_nfs_cow_workdir_for_lane(&self, lane: &str) -> Result> { mount_nfs_cow_for_lane(self, lane) } - pub(crate) fn nfs_clean_workdir_manifest_path_for_lane(&self, lane: &str) -> Result { - nfs_clean_manifest_path(self, lane) + pub(crate) fn mount_nfs_cow_workdir_for_lane_with_ephemeral_bindings( + &self, + lane: &str, + source_upper: PathBuf, + source_root: ObjectId, + bindings: Vec, + ) -> Result> { + mount_nfs_cow_for_lane_with_ephemeral_bindings( + self, + lane, + source_upper, + source_root, + bindings, + ) } - pub(crate) fn nfs_cow_candidate_paths_for_lane(&self, lane: &str) -> Result> { + pub(crate) fn nfs_cow_candidate_paths_for_lane( + &self, + lane: &str, + ) -> Result { nfs_candidate_paths(self, lane) } diff --git a/trail/src/db/lane/workdir/record.rs b/trail/src/db/lane/workdir/record.rs index 8bee636..67f14b4 100644 --- a/trail/src/db/lane/workdir/record.rs +++ b/trail/src/db/lane/workdir/record.rs @@ -1,5 +1,98 @@ use super::*; +#[cfg(debug_assertions)] +std::thread_local! { + static LANE_RECORD_AFTER_C2_WRITE: std::cell::RefCell)>> = + const { std::cell::RefCell::new(None) }; + static FAIL_LANE_RECORD_POSTCOMMIT_BOUNDARY: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(debug_assertions)] +pub(crate) fn install_lane_record_after_c2_write_for_current_thread(path: PathBuf, bytes: Vec) { + LANE_RECORD_AFTER_C2_WRITE.with(|pending| *pending.borrow_mut() = Some((path, bytes))); +} + +#[cfg(debug_assertions)] +pub(crate) fn set_lane_record_postcommit_failure_for_current_thread( + boundary: Option<&'static str>, +) { + FAIL_LANE_RECORD_POSTCOMMIT_BOUNDARY.with(|selected| *selected.borrow_mut() = boundary); +} + +#[cfg(debug_assertions)] +fn run_lane_record_after_c2_write() -> Result<()> { + let pending = LANE_RECORD_AFTER_C2_WRITE.with(|pending| pending.borrow_mut().take()); + if let Some((path, bytes)) = pending { + fs::write(path, bytes)?; + } + Ok(()) +} + +#[cfg(not(debug_assertions))] +fn run_lane_record_after_c2_write() -> Result<()> { + Ok(()) +} + +#[cfg(debug_assertions)] +fn fail_lane_record_postcommit_if_requested(boundary: &'static str) -> Result<()> { + if FAIL_LANE_RECORD_POSTCOMMIT_BOUNDARY.with(|selected| *selected.borrow() == Some(boundary)) { + return Err(Error::InvalidInput(format!( + "injected lane record postcommit failure at {boundary}" + ))); + } + Ok(()) +} + +#[cfg(not(debug_assertions))] +fn fail_lane_record_postcommit_if_requested(_boundary: &'static str) -> Result<()> { + Ok(()) +} + +fn lane_record_committed_repair_error( + operation: &ObjectId, + repair: &'static str, + error: Error, +) -> Error { + match error { + Error::CommittedRepairRequired { .. } => error, + error => Error::CommittedRepairRequired { + operation: operation.0.clone(), + repair: repair.into(), + reason: error.to_string(), + }, + } +} + +enum RecordChangedPathsCaseFold { + AlreadySafe, + NeedsValidation, + LegacyUnavailable, + Collision { path: String, previous: String }, +} + +struct RecordChangedPaths { + summaries: Vec, + case_fold: RecordChangedPathsCaseFold, +} + +fn record_changed_paths_case_fold( + state: &RecordCaseFoldResolutionState, +) -> RecordChangedPathsCaseFold { + match state { + RecordCaseFoldResolutionState::Indexed { .. } => RecordChangedPathsCaseFold::AlreadySafe, + RecordCaseFoldResolutionState::LegacyUnavailable => { + RecordChangedPathsCaseFold::LegacyUnavailable + } + RecordCaseFoldResolutionState::Collision { path, previous } => { + RecordChangedPathsCaseFold::Collision { + path: path.clone(), + previous: previous.clone(), + } + } + } +} + impl Trail { pub fn preview_lane_workdir_record(&self, lane: &str) -> Result { validate_ref_segment(lane)?; @@ -14,17 +107,33 @@ impl Trail { return Err(Error::WorkspaceNotFound(workdir_path)); } let head = self.get_ref(&branch.ref_name)?; - let changed_paths = - self.lane_workdir_record_changed_paths(&branch, &head, &workdir_path)?; + let changed = + self.lane_workdir_record_changed_paths_with_case_fold(&branch, &head, &workdir_path)?; + let changed_paths = changed.summaries; let (ignored_paths, risky_paths) = self.preview_lane_workdir_path_warnings(&workdir_path)?; let oversized_files = self.lane_record_oversized_files_on_disk(&workdir_path, &changed_paths)?; let mut policy = self.preview_lane_record_policy(&branch, &changed_paths)?; - if policy.error.is_none() { - if let Err(err) = self - .ensure_record_final_root_paths_safe_from_summaries(&head.root_id, &changed_paths) - { + if policy.error.is_none() && !changed_paths.is_empty() { + let case_fold_result: Result<()> = match changed.case_fold { + RecordChangedPathsCaseFold::AlreadySafe => Ok(()), + RecordChangedPathsCaseFold::NeedsValidation => self + .ensure_record_final_root_paths_safe_from_summaries( + &head.root_id, + &changed_paths, + ), + RecordChangedPathsCaseFold::LegacyUnavailable => Err(Error::PathIndexRequired( + "legacy root has no case-fold index; run `trail index rebuild`".to_string(), + )), + RecordChangedPathsCaseFold::Collision { path, previous } => { + Err(Error::InvalidPath { + path, + reason: format!("case-insensitive path collision with `{previous}`"), + }) + } + }; + if let Err(err) = case_fold_result { policy.allowed = false; policy.error = Some(err.to_string()); } @@ -53,8 +162,28 @@ impl Trail { lane: &str, message: Option, ) -> Result { - let _lock = self.acquire_write_lock()?; - self.record_lane_workdir_locked(lane, message, None) + let metrics = self.operation_metrics.clone(); + let result_metrics = metrics.clone(); + profile_operation_metrics( + metrics.as_ref(), + OperationMetricsKind::MaterializedLaneRecord, + || { + self.reset_case_fold_index_metrics(); + let _lock = if crate::db::change_ledger::command_authority_enabled() { + None + } else { + Some(self.acquire_write_lock()?) + }; + let result = self.record_lane_workdir_locked(lane, message, None); + if let (Some(metrics), Ok(report)) = (&result_metrics, &result) { + metrics.add(OperationMetricsDelta { + final_path_count: saturating_u64_from_usize(report.changed_paths.len()), + ..OperationMetricsDelta::default() + }); + } + result + }, + ) } pub fn record_lane_workdir_for_turn( @@ -63,8 +192,28 @@ impl Trail { turn_id: &str, message: Option, ) -> Result { - let _lock = self.acquire_write_lock()?; - self.record_lane_workdir_locked(lane, message, Some(turn_id)) + let metrics = self.operation_metrics.clone(); + let result_metrics = metrics.clone(); + profile_operation_metrics( + metrics.as_ref(), + OperationMetricsKind::MaterializedLaneRecord, + || { + self.reset_case_fold_index_metrics(); + let _lock = if crate::db::change_ledger::command_authority_enabled() { + None + } else { + Some(self.acquire_write_lock()?) + }; + let result = self.record_lane_workdir_locked(lane, message, Some(turn_id)); + if let (Some(metrics), Ok(report)) = (&result_metrics, &result) { + metrics.add(OperationMetricsDelta { + final_path_count: saturating_u64_from_usize(report.changed_paths.len()), + ..OperationMetricsDelta::default() + }); + } + result + }, + ) } fn record_lane_workdir_locked( @@ -73,6 +222,8 @@ impl Trail { message: Option, existing_turn_id: Option<&str>, ) -> Result { + // TRAIL_FS_PRODUCER: observed_lane_checkpoint ObservedCheckpoint controlled + self.reset_case_fold_index_metrics(); validate_ref_segment(lane)?; let branch = self.lane_branch(lane)?; ensure_lane_record_message_has_no_secrets(message.as_deref())?; @@ -107,55 +258,141 @@ impl Trail { let is_sparse = sparse_paths.is_some(); let workdir_mode = self.lane_workdir_mode_for(&self.lane_record(&branch.lane_id)?, &branch)?; - let _mutation_barrier = if workdir_mode.is_transparent_cow() { - let view = self.lane_workspace_view(lane)?.ok_or_else(|| { - Error::Corrupt(format!( - "layered lane `{lane}` has no persisted workspace view" - )) - })?; - Some(ViewMutationBarrier::exclusive(Path::new(&view.meta_dir))?) + // Layered COW lanes are authorized by their persistent view mutation + // journal and exclusive view barrier. Their mountpoint is intentionally + // empty after unmount, so starting a filesystem observer there would + // both inspect the wrong authority and fail closed on absent `.trail` + // storage. Ordinary materialized lanes retain the native observer + // handoff before candidate comparison. + if crate::db::change_ledger::command_authority_enabled() + && !workdir_mode.is_transparent_cow() + { + crate::db::change_ledger::materialized_lane_daemon_expected_scope(self, lane)?; + } + // Authority-on mounted views still take the workspace writer lock + // before the view barrier. Materialized observer-backed lanes retain + // their explicit daemon handoff instead of deadlocking that observer + // behind a lock held by its caller. + let _view_workspace_lock = if workdir_mode.is_transparent_cow() + && crate::db::change_ledger::command_authority_enabled() + { + Some(self.acquire_write_lock()?) } else { None }; - let source_upper = if workdir_mode.is_transparent_cow() { - Some(self.workspace_view_paths_for_lane(lane)?.source_upper) + let workspace_view = if workdir_mode.is_transparent_cow() { + Some(self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::Corrupt(format!( + "layered lane `{lane}` has no persisted workspace view" + )) + })?) } else { None }; + let mut mutation_barrier = workspace_view + .as_ref() + .map(|view| ViewMutationBarrier::exclusive(Path::new(&view.meta_dir))) + .transpose()?; + let source_upper = workspace_view + .as_ref() + .map(|view| PathBuf::from(&view.source_upper)); let record_scan_root = source_upper.as_deref().unwrap_or(&workdir_path); - let overlay_manifest_path = self.lane_overlay_clean_manifest_path(&branch)?; - let cached_status = if matches!( - &workdir_mode, - LaneWorkdirMode::OverlayCow | LaneWorkdirMode::NfsCow - ) { + let layered_manifest_path = self.lane_layered_clean_manifest_path(&branch)?; + let mut observed_record_cut = None; + let mut observed_record_files = None; + let mut upper_recovery_walks = 0_u64; + let mut generated_dirty_paths = 0_u64; + let cached_status = if crate::db::change_ledger::command_authority_enabled() + && !workdir_mode.is_transparent_cow() + { + let (mut comparison, fenced) = self.compare_materialized_lane_candidates( + &branch.lane_id, + crate::db::change_ledger::CandidateMaterialization::RecordBytes, + )?; + observed_record_cut = Some(crate::db::change_ledger::ObservedRecordCut { + expected: fenced.candidates.expected.clone(), + c1: fenced.candidates.cut.clone(), + c2: fenced.c2.clone(), + acknowledgement_tokens: fenced.candidates.acknowledgement_tokens.clone(), + }); + // The comparison owns the exact descriptor-relative bytes read + // between c1 and c2. A later same-path write must remain pending + // evidence, never replace the bytes authorized for this record. + observed_record_files = comparison.disk_files.take(); + if observed_record_files.is_none() { + return Err(Error::Corrupt( + "lane record candidate comparison omitted pinned contents".into(), + )); + } + run_lane_record_after_c2_write()?; + if comparison.summaries.is_empty() { + CachedWorkdirManifestStatus::Clean + } else { + CachedWorkdirManifestStatus::Dirty { + disk_manifest: comparison.disk_manifest, + candidate_paths: Some(comparison.selections), + } + } + } else if workdir_mode.is_transparent_cow() { // Derive the delta from the persistent upper layer instead of an // FUSE/NFS READDIR result or a scan of the composed repository. - let candidate_paths = if workdir_mode == LaneWorkdirMode::NfsCow { - self.nfs_cow_candidate_paths_for_lane(lane)? - } else { - self.overlay_cow_candidate_paths_for_lane(lane)? - }; + let candidates = self.transparent_cow_candidate_paths_for_lane(lane, &workdir_mode)?; + upper_recovery_walks = candidates.upper_recovery_walks; + generated_dirty_paths = candidates + .generated_paths + .len() + .try_into() + .unwrap_or(u64::MAX); + let candidate_paths = candidates.paths.into_iter().collect::>(); + self.note_operation_metrics(OperationMetricsDelta { + authoritative_candidate_count: candidate_paths.len().try_into().unwrap_or(u64::MAX), + ..OperationMetricsDelta::default() + }); let disk_files = self.scan_files_under_for_paths(record_scan_root, &candidate_paths)?; + let disk_manifest = self.disk_manifest(&disk_files); + // The exclusive view mutation barrier pins the qualified journal + // and its source upper for this checkpoint. Reuse these selected + // bytes when building the root instead of reading every candidate + // a second time under the same barrier. + observed_record_files = Some(disk_files); CachedWorkdirManifestStatus::Dirty { - disk_manifest: self.disk_manifest(&disk_files), + disk_manifest, candidate_paths: Some(candidate_paths), } } else { + // A full materialized workdir is repository-shaped: validating its + // manifest walks every materialized path (or falls back to a full + // scan when the manifest is missing). Sparse workdirs remain + // bounded by their explicit selection and are not counted here. + if !is_sparse { + self.note_full_filesystem_path_scan(); + } self.lane_cached_workdir_manifest_status( &workdir_path, - overlay_manifest_path.as_deref(), + layered_manifest_path.as_deref(), &head.root_id, )? }; if matches!(cached_status, CachedWorkdirManifestStatus::Clean) { if workdir_mode.is_transparent_cow() { - self.complete_workspace_checkpoint(lane, &head.root_id, None)?; + self.complete_workspace_checkpoint( + lane, + &head.root_id, + None, + mutation_barrier.as_mut().ok_or_else(|| { + Error::Corrupt("layered checkpoint lost its mutation barrier".into()) + })?, + )?; } + self.publish_lane_marker_if_materialized(lane)?; return Ok(LaneRecordReport { lane_id: branch.lane_id, operation: None, root_id: head.root_id, changed_paths: Vec::new(), + path_index: self.case_fold_index_metrics_report(), + upper_recovery_walks, + generated_dirty_paths, }); } if let Some(session_id) = &branch.session_id { @@ -170,7 +407,7 @@ impl Trail { disk_manifest, candidate_paths, } => { - let (summaries, previous_files, use_disk_manifest_for_clean) = + let (summaries, previous_files, use_disk_manifest_for_clean, record_preflight) = if let Some(mut selected_paths) = sparse_paths.clone() { selected_paths.extend(disk_manifest.keys().cloned()); selected_paths.sort(); @@ -181,17 +418,28 @@ impl Trail { &previous_files, &disk_manifest, &selected_paths, - ); - (summaries, previous_files, false) + )?; + (summaries, previous_files, false, None) } else if let Some(candidate_paths) = candidate_paths { + let resolution = self.resolve_record_case_fold_candidates_read_only( + &head.root_id, + &candidate_paths, + &disk_manifest, + )?; + let candidate_paths = resolution.selected_paths.clone(); let previous_files = self.load_root_files_for_paths(&head.root_id, &candidate_paths)?; let summaries = self.diff_file_maps_to_manifest_for_paths( &previous_files, &disk_manifest, &candidate_paths, - ); - (summaries, previous_files, true) + )?; + ( + summaries, + previous_files, + true, + Some(self.finalize_record_case_fold_resolution(resolution)?), + ) } else { let summaries = self.diff_root_to_disk_manifest(&head.root_id, &disk_manifest)?; @@ -201,7 +449,7 @@ impl Trail { .collect::>(); let previous_files = self.load_root_files_for_paths(&head.root_id, &selected_paths)?; - (summaries, previous_files, true) + (summaries, previous_files, true, None) }; let materialized_paths = disk_manifest.keys().cloned().collect::>(); if summaries.is_empty() { @@ -214,7 +462,7 @@ impl Trail { if use_disk_manifest_for_clean { self.write_lane_clean_workdir_manifest_from_disk_manifest( &workdir_path, - overlay_manifest_path.as_deref(), + layered_manifest_path.as_deref(), &head.root_id, &disk_manifest, materialized_paths.iter(), @@ -222,35 +470,93 @@ impl Trail { } else { self.write_lane_clean_workdir_manifest( &workdir_path, - overlay_manifest_path.as_deref(), + layered_manifest_path.as_deref(), &head.root_id, &previous_files, materialized_paths.iter(), )?; } if workdir_mode.is_transparent_cow() { - self.complete_workspace_checkpoint(lane, &head.root_id, None)?; + self.complete_workspace_checkpoint( + lane, + &head.root_id, + None, + mutation_barrier.as_mut().ok_or_else(|| { + Error::Corrupt( + "layered checkpoint lost its mutation barrier".into(), + ) + })?, + )?; } + self.publish_lane_marker_if_materialized(lane)?; return Ok(LaneRecordReport { lane_id: branch.lane_id, operation: None, root_id: head.root_id, changed_paths: Vec::new(), + path_index: self.case_fold_index_metrics_report(), + upper_recovery_walks, + generated_dirty_paths, }); } let selected_paths = summaries .iter() .map(|summary| summary.path.clone()) .collect::>(); - let disk_files = - self.scan_files_under_for_paths(record_scan_root, &selected_paths)?; - let built = self.build_root_for_selected_disk_files_incremental( - &head.root_id, - &previous_files, - &disk_files, - &selected_paths, - &change_id, - )?; + let selected_disk_paths = record_preflight + .as_ref() + .map(|preflight| { + preflight + .expected_observed_present_paths + .iter() + .cloned() + .collect::>() + }) + .unwrap_or_else(|| { + selected_paths + .iter() + .filter(|path| disk_manifest.contains_key(*path)) + .cloned() + .collect::>() + }); + let scanned_disk_files; + let disk_files = if let Some(captured) = observed_record_files.as_ref() { + captured.as_slice() + } else { + // Legacy and transparent-COW record paths do not use the + // authoritative c1/c2 snapshot and retain their existing + // selected filesystem scan. + scanned_disk_files = + self.scan_files_under_for_paths(record_scan_root, &selected_disk_paths)?; + scanned_disk_files.as_slice() + }; + let built = if observed_record_files.is_some() { + self.build_root_for_selected_disk_files_incremental( + &head.root_id, + &previous_files, + disk_files, + &selected_paths, + &change_id, + )? + } else if let Some(record_preflight) = record_preflight { + self.build_root_for_selected_disk_files_incremental_with_record_preflight( + record_scan_root, + &head.root_id, + &previous_files, + &disk_files, + &selected_paths, + &change_id, + record_preflight, + )? + } else { + self.build_root_for_selected_disk_files_incremental( + &head.root_id, + &previous_files, + &disk_files, + &selected_paths, + &change_id, + )? + }; let clean_disk_manifest = use_disk_manifest_for_clean.then_some(disk_manifest); ( built, @@ -287,16 +593,20 @@ impl Trail { if summaries.is_empty() { self.write_lane_clean_workdir_manifest_from_disk_manifest( &workdir_path, - overlay_manifest_path.as_deref(), + layered_manifest_path.as_deref(), &head.root_id, &disk_manifest, materialized_paths.iter(), )?; + self.publish_lane_marker_if_materialized(lane)?; return Ok(LaneRecordReport { lane_id: branch.lane_id, operation: None, root_id: head.root_id, changed_paths: Vec::new(), + path_index: self.case_fold_index_metrics_report(), + upper_recovery_walks, + generated_dirty_paths, }); } let selected_paths = summaries @@ -329,7 +639,7 @@ impl Trail { if let Some(disk_manifest) = &clean_disk_manifest { self.write_lane_clean_workdir_manifest_from_disk_manifest( &workdir_path, - overlay_manifest_path.as_deref(), + layered_manifest_path.as_deref(), &head.root_id, disk_manifest, materialized_paths.iter(), @@ -337,20 +647,31 @@ impl Trail { } else { self.write_lane_clean_workdir_manifest( &workdir_path, - overlay_manifest_path.as_deref(), + layered_manifest_path.as_deref(), &head.root_id, &built.files, materialized_paths.iter(), )?; } if workdir_mode.is_transparent_cow() { - self.complete_workspace_checkpoint(lane, &head.root_id, None)?; + self.complete_workspace_checkpoint( + lane, + &head.root_id, + None, + mutation_barrier.as_mut().ok_or_else(|| { + Error::Corrupt("layered checkpoint lost its mutation barrier".into()) + })?, + )?; } + self.publish_lane_marker_if_materialized(lane)?; return Ok(LaneRecordReport { lane_id: branch.lane_id, operation: None, root_id: head.root_id, changed_paths: Vec::new(), + path_index: self.case_fold_index_metrics_report(), + upper_recovery_walks, + generated_dirty_paths, }); } self.ensure_lane_record_policy(&branch, &diff.summaries)?; @@ -387,99 +708,253 @@ impl Trail { changes: diff.changes, created_at: now_ts(), }; - let operation_id = self.store_operation(&operation)?; - self.advance_ref_cas(&head, &change_id, &built.root_id, &operation_id)?; - test_crash_point("checkpoint_after_ref_advance"); - let message_id = if let Some(message) = message { - Some(self.store_message( - "lane", - &message, - Some(&branch.lane_id), - branch.session_id.as_deref(), - Some(&change_id), - operation.created_at, - )?) + let workspace_checkpoint = workspace_view + .as_ref() + .map(|view| -> Result<_> { + let cut = ViewMutationJournal::open(Path::new(&view.source_upper))?.cut(); + if !cut.recovery_qualified { + return Err(Error::ChangeLedgerReconcileRequired { + scope: view.view_id.clone(), + state: "unqualified_view_journal".into(), + reason: "workspace checkpoint cannot qualify either changed-path or whiteout evidence".into(), + command: "trail ledger reconcile".into(), + }); + } + Ok(crate::db::change_ledger::WorkspaceViewCheckpointPublication { + view_id: view.view_id.clone(), + expected_generation: view.generation, + journal_sequence: cut.sequence, + next_generation: cut.generation.saturating_add(1), + journal_qualified: cut.qualified, + }) + }) + .transpose()?; + self.invalidate_lane_marker_if_materialized(&branch)?; + let committed_operation_id = if let Some(observed) = observed_record_cut.as_ref() { + let evidence = crate::db::change_ledger::IntentEvidence { + exact_paths: Vec::new(), + complete_prefixes: Vec::new(), + }; + crate::db::change_ledger::run_ref_advancing_projection( + self, + &observed.expected, + &head, + &branch.lane_id, + crate::db::change_ledger::IntentProducer::ObservedCheckpoint, + &operation, + &evidence, + crate::db::change_ledger::RefAdvancingProjectionMode::ObservedCut { + cut: observed, + acknowledge_complete_prefixes: true, + }, + |_, _| { + Err(Error::InvalidInput( + "observed checkpoint cannot enter a controlled intent interval".into(), + )) + }, + |db, publication| { + crate::db::change_ledger::accept_materialized_lane_daemon_baseline( + db, + &branch.lane_id, + &observed.expected, + &publication.baseline, + ) + }, + )? + .operation_id } else { - None + crate::db::change_ledger::commit_lane_operation_atomic( + self, + &head, + &branch.lane_id, + &operation, + workspace_checkpoint.as_ref(), + )? }; - self.conn.execute( - "UPDATE lane_branches SET head_change = ?1, head_root = ?2, updated_at = ?3 WHERE lane_id = ?4", - params![change_id.0, built.root_id.0, now_ts(), branch.lane_id], - )?; + test_crash_point("checkpoint_after_ref_advance"); + let message_id = (|| { + fail_lane_record_postcommit_if_requested("message")?; + if let Some(message) = message { + Ok(Some(self.store_message( + "lane", + &message, + Some(&branch.lane_id), + branch.session_id.as_deref(), + Some(&change_id), + operation.created_at, + )?)) + } else { + Ok(None) + } + })() + .map_err(|error| { + lane_record_committed_repair_error( + &committed_operation_id, + "lane record message", + error, + ) + })?; test_crash_point("checkpoint_after_lane_head_update"); - if is_sparse { - self.write_sparse_workdir_manifest(&workdir_path, materialized_paths.iter())?; - } - if let Some(disk_manifest) = &clean_disk_manifest { - self.write_lane_clean_workdir_manifest_from_disk_manifest( - &workdir_path, - overlay_manifest_path.as_deref(), - &built.root_id, - disk_manifest, - materialized_paths.iter(), - )?; - } else { - self.write_lane_clean_workdir_manifest( - &workdir_path, - overlay_manifest_path.as_deref(), - &built.root_id, - &built.files, - materialized_paths.iter(), - )?; - } - self.insert_lane_event_with_context( - &branch.lane_id, - branch.session_id.as_deref(), - Some(&turn_id), - "workdir_recorded", - Some(&change_id), - message_id.as_ref(), - &serde_json::json!({ - "workdir": workdir, - "root_id": built.root_id.0.clone(), - "session_id": branch.session_id.clone(), - "changed_paths": diff.summaries.iter().map(|item| item.path.clone()).collect::>() - }), - )?; - if existing_turn.is_some() { - self.update_lane_turn_progress(&turn_id, "workdir_recorded", Some(&change_id))?; - } else { - self.finish_lane_turn(&turn_id, "completed", Some(&change_id))?; - } - if workdir_mode.is_transparent_cow() { - self.complete_workspace_checkpoint(lane, &built.root_id, Some(&change_id))?; - } + (|| { + fail_lane_record_postcommit_if_requested("manifest")?; + if is_sparse { + self.write_sparse_workdir_manifest(&workdir_path, materialized_paths.iter())?; + } + if let Some(disk_manifest) = &clean_disk_manifest { + self.write_lane_clean_workdir_manifest_from_disk_manifest( + &workdir_path, + layered_manifest_path.as_deref(), + &built.root_id, + disk_manifest, + materialized_paths.iter(), + )?; + } else { + self.write_lane_clean_workdir_manifest( + &workdir_path, + layered_manifest_path.as_deref(), + &built.root_id, + &built.files, + materialized_paths.iter(), + )?; + } + Ok(()) + })() + .map_err(|error| { + lane_record_committed_repair_error( + &committed_operation_id, + "lane record manifest", + error, + ) + })?; + (|| { + fail_lane_record_postcommit_if_requested("event")?; + self.insert_lane_event_with_context( + &branch.lane_id, + branch.session_id.as_deref(), + Some(&turn_id), + "workdir_recorded", + Some(&change_id), + message_id.as_ref(), + &serde_json::json!({ + "workdir": workdir, + "root_id": built.root_id.0.clone(), + "session_id": branch.session_id.clone(), + "changed_paths": diff.summaries.iter().map(|item| item.path.clone()).collect::>() + }), + ) + })() + .map_err(|error| { + lane_record_committed_repair_error( + &committed_operation_id, + "lane record event", + error, + ) + })?; + (|| { + fail_lane_record_postcommit_if_requested("turn")?; + if existing_turn.is_some() { + self.update_lane_turn_progress(&turn_id, "workdir_recorded", Some(&change_id)) + } else { + self.finish_lane_turn(&turn_id, "completed", Some(&change_id)) + } + })() + .map_err(|error| { + lane_record_committed_repair_error(&committed_operation_id, "lane record turn", error) + })?; + (|| { + fail_lane_record_postcommit_if_requested("checkpoint")?; + if let Some(checkpoint) = &workspace_checkpoint { + self.repair_workspace_checkpoint_mirror( + lane, + &built.root_id, + Some(&change_id), + checkpoint.journal_sequence, + checkpoint.next_generation, + checkpoint.journal_qualified, + mutation_barrier.as_mut().ok_or_else(|| { + Error::Corrupt("layered checkpoint repair lost its mutation barrier".into()) + })?, + )?; + } + Ok(()) + })() + .map_err(|error| { + lane_record_committed_repair_error( + &committed_operation_id, + "lane record workspace checkpoint", + error, + ) + })?; + (|| { + fail_lane_record_postcommit_if_requested("marker")?; + self.publish_lane_marker_if_materialized(lane) + })() + .map_err(|error| { + lane_record_committed_repair_error(&committed_operation_id, "lane record marker", error) + })?; Ok(LaneRecordReport { lane_id: branch.lane_id, operation: Some(change_id), root_id: built.root_id, changed_paths: diff.summaries, + path_index: self.case_fold_index_metrics_report(), + upper_recovery_walks, + generated_dirty_paths, }) } - pub(crate) fn lane_overlay_clean_manifest_path( + pub(crate) fn lane_layered_clean_manifest_path( &self, branch: &LaneBranch, ) -> Result> { let record = self.lane_record(&branch.lane_id)?; - match self.lane_workdir_mode_for(&record, branch)? { - LaneWorkdirMode::OverlayCow => Ok(Some( - self.overlay_clean_workdir_manifest_path_for_lane(&record.name)?, - )), - LaneWorkdirMode::NfsCow => Ok(Some( - self.nfs_clean_workdir_manifest_path_for_lane(&record.name)?, - )), - _ => Ok(None), + if self + .lane_workdir_mode_for(&record, branch)? + .is_transparent_cow() + { + return Ok(Some( + self.workspace_view_paths_for_lane_name(&record.name) + .meta_dir + .join("workdir-manifest.json"), + )); + } + Ok(None) + } + + fn transparent_cow_candidate_paths_for_lane( + &self, + lane: &str, + mode: &LaneWorkdirMode, + ) -> Result { + match mode { + LaneWorkdirMode::FuseCow => self.fuse_cow_candidate_paths_for_lane(lane), + LaneWorkdirMode::NfsCow => self.nfs_cow_candidate_paths_for_lane(lane), + LaneWorkdirMode::DokanCow => { + #[cfg(target_os = "windows")] + { + self.dokan_cow_candidate_paths_for_lane(lane) + } + #[cfg(not(target_os = "windows"))] + { + Err(Error::InvalidInput( + "dokan-cow workdirs are currently supported only on Windows".to_string(), + )) + } + } + _ => Err(Error::InvalidInput(format!( + "lane `{lane}` uses non-layered workdir mode `{}`", + mode.as_str() + ))), } } pub(crate) fn lane_cached_workdir_manifest_status( &self, workdir_path: &Path, - overlay_manifest_path: Option<&Path>, + layered_manifest_path: Option<&Path>, root_id: &ObjectId, ) -> Result { - if let Some(manifest_path) = overlay_manifest_path { + if let Some(manifest_path) = layered_manifest_path { self.cached_workdir_manifest_status_from_path(workdir_path, manifest_path, root_id) } else { self.cached_workdir_manifest_status(workdir_path, root_id) @@ -489,7 +964,7 @@ impl Trail { fn write_lane_clean_workdir_manifest<'a, I>( &self, workdir_path: &Path, - overlay_manifest_path: Option<&Path>, + layered_manifest_path: Option<&Path>, root_id: &ObjectId, files: &BTreeMap, expected_paths: I, @@ -497,7 +972,7 @@ impl Trail { where I: IntoIterator, { - if let Some(manifest_path) = overlay_manifest_path { + if let Some(manifest_path) = layered_manifest_path { self.write_clean_workdir_manifest_to_path( workdir_path, manifest_path, @@ -513,7 +988,7 @@ impl Trail { fn write_lane_clean_workdir_manifest_from_disk_manifest<'a, I>( &self, workdir_path: &Path, - overlay_manifest_path: Option<&Path>, + layered_manifest_path: Option<&Path>, root_id: &ObjectId, disk_manifest: &BTreeMap, expected_paths: I, @@ -521,7 +996,7 @@ impl Trail { where I: IntoIterator, { - if let Some(manifest_path) = overlay_manifest_path { + if let Some(manifest_path) = layered_manifest_path { self.write_clean_workdir_manifest_from_disk_manifest_to_path( workdir_path, manifest_path, @@ -545,34 +1020,72 @@ impl Trail { head: &RefRecord, workdir_path: &Path, ) -> Result> { + Ok(self + .lane_workdir_record_changed_paths_with_case_fold(branch, head, workdir_path)? + .summaries) + } + + fn lane_workdir_record_changed_paths_with_case_fold( + &self, + branch: &LaneBranch, + head: &RefRecord, + workdir_path: &Path, + ) -> Result { let lane_record = self.lane_record(&branch.lane_id)?; let workdir_mode = self.lane_workdir_mode_for(&lane_record, branch)?; if workdir_mode.is_transparent_cow() { - let candidate_paths = if workdir_mode == LaneWorkdirMode::NfsCow { - self.nfs_cow_candidate_paths_for_lane(&lane_record.name)? - } else { - self.overlay_cow_candidate_paths_for_lane(&lane_record.name)? - }; + let candidate_paths = self + .transparent_cow_candidate_paths_for_lane(&lane_record.name, &workdir_mode)? + .paths + .into_iter() + .collect::>(); let source_upper = self .workspace_view_paths_for_lane(&lane_record.name)? .source_upper; let disk_files = self.scan_files_under_for_paths(&source_upper, &candidate_paths)?; let disk_manifest = self.disk_manifest(&disk_files); - let previous_files = self.load_root_files_for_paths(&head.root_id, &candidate_paths)?; - return Ok(self.diff_file_maps_to_manifest_for_paths( - &previous_files, - &disk_manifest, + let resolution = self.resolve_record_case_fold_candidates_read_only( + &head.root_id, &candidate_paths, - )); + &disk_manifest, + )?; + let case_fold = record_changed_paths_case_fold(&resolution.state); + let candidate_paths = resolution.selected_paths; + let previous_files = self.load_root_files_for_paths(&head.root_id, &candidate_paths)?; + return Ok(RecordChangedPaths { + summaries: self.diff_file_maps_to_manifest_for_paths( + &previous_files, + &disk_manifest, + &candidate_paths, + )?, + case_fold, + }); + } + if crate::db::change_ledger::command_authority_enabled() { + let (comparison, _) = self.compare_materialized_lane_candidates( + &branch.lane_id, + crate::db::change_ledger::CandidateMaterialization::ManifestOnly, + )?; + return Ok(RecordChangedPaths { + case_fold: if comparison.summaries.is_empty() { + RecordChangedPathsCaseFold::AlreadySafe + } else { + RecordChangedPathsCaseFold::NeedsValidation + }, + summaries: comparison.summaries, + }); } let sparse_paths = self.lane_sparse_workdir_paths(branch, workdir_path)?; - let overlay_manifest_path = self.lane_overlay_clean_manifest_path(branch)?; + let layered_manifest_path = self.lane_layered_clean_manifest_path(branch)?; match self.lane_cached_workdir_manifest_status( workdir_path, - overlay_manifest_path.as_deref(), + layered_manifest_path.as_deref(), &head.root_id, )? { - CachedWorkdirManifestStatus::Clean => Ok(Vec::new()), + CachedWorkdirManifestStatus::Clean => Ok(RecordChangedPaths { + summaries: Vec::new(), + case_fold: RecordChangedPathsCaseFold::AlreadySafe, + }), CachedWorkdirManifestStatus::Dirty { disk_manifest, candidate_paths, @@ -583,21 +1096,38 @@ impl Trail { selected_paths.dedup(); let previous_files = self.load_root_files_for_selections(&head.root_id, &selected_paths)?; - Ok(self.diff_file_maps_to_manifest_for_paths( - &previous_files, - &disk_manifest, - &selected_paths, - )) + Ok(RecordChangedPaths { + summaries: self.diff_file_maps_to_manifest_for_paths( + &previous_files, + &disk_manifest, + &selected_paths, + )?, + case_fold: RecordChangedPathsCaseFold::NeedsValidation, + }) } else if let Some(candidate_paths) = candidate_paths { + let resolution = self.resolve_record_case_fold_candidates_read_only( + &head.root_id, + &candidate_paths, + &disk_manifest, + )?; + let case_fold = record_changed_paths_case_fold(&resolution.state); + let candidate_paths = resolution.selected_paths; let previous_files = self.load_root_files_for_paths(&head.root_id, &candidate_paths)?; - Ok(self.diff_file_maps_to_manifest_for_paths( - &previous_files, - &disk_manifest, - &candidate_paths, - )) + Ok(RecordChangedPaths { + summaries: self.diff_file_maps_to_manifest_for_paths( + &previous_files, + &disk_manifest, + &candidate_paths, + )?, + case_fold, + }) } else { - self.diff_root_to_disk_manifest(&head.root_id, &disk_manifest) + Ok(RecordChangedPaths { + summaries: self + .diff_root_to_disk_manifest(&head.root_id, &disk_manifest)?, + case_fold: RecordChangedPathsCaseFold::NeedsValidation, + }) } } CachedWorkdirManifestStatus::Missing => { @@ -609,13 +1139,20 @@ impl Trail { selected_paths.dedup(); let previous_files = self.load_root_files_for_selections(&head.root_id, &selected_paths)?; - Ok(self.diff_file_maps_to_manifest_for_paths( - &previous_files, - &disk_manifest, - &selected_paths, - )) + Ok(RecordChangedPaths { + summaries: self.diff_file_maps_to_manifest_for_paths( + &previous_files, + &disk_manifest, + &selected_paths, + )?, + case_fold: RecordChangedPathsCaseFold::NeedsValidation, + }) } else { - self.diff_root_to_disk_manifest(&head.root_id, &disk_manifest) + Ok(RecordChangedPaths { + summaries: self + .diff_root_to_disk_manifest(&head.root_id, &disk_manifest)?, + case_fold: RecordChangedPathsCaseFold::NeedsValidation, + }) } } } @@ -677,7 +1214,7 @@ impl Trail { Ok(oversized.into_values().collect()) } - fn ensure_lane_record_file_size_policy( + pub(crate) fn ensure_lane_record_file_size_policy( &self, files: &BTreeMap, summaries: &[FileDiffSummary], diff --git a/trail/src/db/lane/workdir/sync.rs b/trail/src/db/lane/workdir/sync.rs index 98880c4..7ae86d8 100644 --- a/trail/src/db/lane/workdir/sync.rs +++ b/trail/src/db/lane/workdir/sync.rs @@ -6,13 +6,18 @@ impl Trail { let branch = self.lane_branch(lane)?; let record = self.lane_record(&branch.lane_id)?; let workdir_mode = self.lane_workdir_mode_for(&record, &branch)?; + let requested_workdir_mode = self.lane_requested_workdir_mode_for(&record, &branch)?; + let workdir_backend = self.lane_workdir_backend_for(&record)?; + let materialization = self.lane_materialization_report_for(&record)?; let sparse_paths = self.lane_report_sparse_paths(&branch)?; Ok(LaneWorkdirReport { lane_id: branch.lane_id, workdir: branch.workdir, - cow_backend: workdir_mode.cow_backend().map(str::to_string), + requested_workdir_mode, + workdir_backend, + materialization, sparse_paths, - overlay_available: workdir_mode.is_transparent_cow(), + transparent_cow_available: workdir_mode.is_transparent_cow(), workdir_mode, }) } @@ -43,7 +48,14 @@ impl Trail { Some(hydrate) => hydrate, None => branch_has_sparse_workdir(self, &branch)?, }; - let _lock = if hydrate { + let ledger_authority = crate::db::change_ledger::command_authority_enabled(); + if hydrate && ledger_authority { + crate::db::change_ledger::materialized_lane_daemon_expected_scope( + self, + &branch.lane_id, + )?; + } + let _lock = if hydrate && !ledger_authority { Some(self.acquire_write_lock()?) } else { None @@ -105,7 +117,7 @@ impl Trail { paths: &[String], include_neighbors: bool, ) -> Result { - let _lock = self.acquire_write_lock()?; + // TRAIL_FS_PRODUCER: lane_sync LaneSync controlled validate_ref_segment(lane)?; let selected_paths = normalize_record_paths(paths)?; let path_scoped = !selected_paths.is_empty(); @@ -118,6 +130,8 @@ impl Trail { let workdir_path = PathBuf::from(&workdir); let workdir_mode = self.lane_workdir_mode_for(&self.lane_record(&branch.lane_id)?, &branch)?; + let requested_workdir_mode = + self.lane_requested_workdir_mode_for(&self.lane_record(&branch.lane_id)?, &branch)?; if workdir_mode == LaneWorkdirMode::NfsCow { return self.sync_nfs_cow_lane_workdir( lane, @@ -203,58 +217,219 @@ impl Trail { "lane `{lane}` workdir has unrecorded changes; run `trail lane record {lane}` or pass `--force` to sync: {preview}{suffix}" ))); } + let ledger_authority = crate::db::change_ledger::command_authority_enabled(); + let marker_before = if ledger_authority && workdir_path_is_dir { + self.capture_materialized_lane_marker(&workdir_path)? + } else { + None + }; + let expected = if ledger_authority && workdir_path_is_dir { + Some( + crate::db::change_ledger::prepare_materialized_lane_controlled_projection( + self, + &branch.lane_id, + )?, + ) + } else { + None + }; let rescue_workdir = if force && workdir_path_is_non_dir { + // TRAIL_FS_PRODUCER: lane_sync_rescue exempt_rescue_output exempt Some(self.rescue_replaced_lane_workdir_path(lane, &workdir, &workdir_path)?) } else if force && workdir_exists && !changed_paths.is_empty() { Some(self.rescue_dirty_lane_workdir(lane, &workdir, &workdir_path, &changed_paths)?) } else { None }; - if path_scoped { - if workdir_path_is_non_dir { - remove_existing_lane_workdir_path(&workdir_path)?; + // A controlled interval requires a stable directory inode before the + // native observer is bound. A forced non-directory replacement was + // already copied to the explicit rescue output above. + if workdir_path_is_non_dir { + remove_existing_lane_workdir_path(&workdir_path)?; + } + fs::create_dir_all(&workdir_path)?; + let mut evidence_paths = changed_paths + .iter() + .flat_map(|summary| { + std::iter::once(summary.path.as_str()).chain(summary.old_path.as_deref()) + }) + .map(crate::db::change_ledger::LedgerPath::parse) + .collect::>>()?; + if force { + evidence_paths.extend( + target_files + .keys() + .map(|path| crate::db::change_ledger::LedgerPath::parse(path)) + .collect::>>()?, + ); + } + evidence_paths.sort(); + evidence_paths.dedup(); + let evidence = crate::db::change_ledger::IntentEvidence { + exact_paths: evidence_paths, + complete_prefixes: Vec::new(), + }; + let mut materialization_outcome = None; + let projected_target_files = if path_scoped { + target_files.clone() + } else if let Some(paths) = &sparse_paths { + self.selected_file_entries(&target_files, paths) + } else { + target_files.clone() + }; + if ledger_authority { + let expected = expected.ok_or_else(|| Error::ChangeLedgerReconcileRequired { + scope: crate::db::change_ledger::materialized_lane_scope_id( + &self.config.workspace.id.0, + &branch.lane_id, + ) + .to_text(), + state: "reconciling".into(), + reason: "materialized lane root must be initialized before controlled sync".into(), + command: format!("trail lane workdir ensure {}", branch.lane_id), + })?; + let projection_result = crate::db::change_ledger::run_projection_alignment( + self, + &expected, + crate::db::change_ledger::IntentProducer::LaneSync, + &evidence, + crate::db::change_ledger::ProjectionAlignmentMode::Aligned, + |db, intent| { + crate::db::change_ledger::with_materialized_lane_controlled_interval( + db, + &branch.lane_id, + intent, + &evidence, + |db| { + if path_scoped { + db.materialize_sparse_lane_workdir_paths( + &workdir_path, + &head.root_id, + sparse_paths.clone().unwrap_or_default(), + &projected_target_files, + force, + )?; + } else if force || !workdir_exists { + db.invalidate_lane_marker_if_materialized(&branch)?; + materialization_outcome = db.materialize_full_lane_workdir_staged( + &workdir_path, + &head.root_id, + &projected_target_files, + sparse_paths.is_some(), + &requested_workdir_mode, + )?; + } else { + db.invalidate_lane_marker_if_materialized(&branch)?; + if !changed_paths.is_empty() { + db.materialize_files_at( + &workdir_path, + &projected_target_files, + &projected_target_files, + )?; + } + if sparse_paths.is_some() { + db.write_sparse_workdir_manifest( + &workdir_path, + projected_target_files.keys(), + )?; + } + db.write_clean_workdir_manifest( + &workdir_path, + &head.root_id, + &projected_target_files, + projected_target_files.keys(), + )?; + } + Ok(()) + }, + |db, policy, candidates| { + let comparison = db.compare_authoritative_candidates( + policy, + candidates, + &head.root_id, + crate::db::change_ledger::CandidateMaterialization::ManifestOnly, + )?; + if comparison.summaries.is_empty() { + Ok(()) + } else { + Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "lane sync pinned verification did not match its target root" + .into(), + command: format!("trail lane status {}", branch.lane_id), + }) + } + }, + ) + }, + |db| db.publish_lane_marker_if_materialized(&branch.lane_id), + ); + if let Err(error) = projection_result { + self.restore_materialized_lane_marker(&workdir_path, marker_before.as_deref())?; + return Err(error); } - fs::create_dir_all(&workdir_path)?; - self.materialize_sparse_lane_workdir_paths( - &workdir_path, - &head.root_id, - sparse_paths.unwrap_or_default(), - &target_files, - force, - )?; } else { - let target_files = if let Some(paths) = &sparse_paths { - self.selected_file_entries(&target_files, paths) - } else { - target_files - }; - if force || !workdir_exists { - self.materialize_full_lane_workdir_staged( + if path_scoped { + self.materialize_sparse_lane_workdir_paths( &workdir_path, &head.root_id, - &target_files, + sparse_paths.clone().unwrap_or_default(), + &projected_target_files, + force, + )?; + } else if force || !workdir_exists { + self.invalidate_lane_marker_if_materialized(&branch)?; + materialization_outcome = self.materialize_full_lane_workdir_staged( + &workdir_path, + &head.root_id, + &projected_target_files, sparse_paths.is_some(), + &requested_workdir_mode, )?; } else { - fs::create_dir_all(&workdir_path)?; + self.invalidate_lane_marker_if_materialized(&branch)?; if !changed_paths.is_empty() { self.materialize_files_best_effort_at( &workdir_path, - &target_files, - &target_files, + &projected_target_files, + &projected_target_files, )?; } if sparse_paths.is_some() { - self.write_sparse_workdir_manifest(&workdir_path, target_files.keys())?; + self.write_sparse_workdir_manifest( + &workdir_path, + projected_target_files.keys(), + )?; } self.write_clean_workdir_manifest( &workdir_path, &head.root_id, - &target_files, - target_files.keys(), + &projected_target_files, + projected_target_files.keys(), )?; } + self.publish_lane_marker_if_materialized(&branch.lane_id)?; } + if let Some(outcome) = &materialization_outcome { + self.update_lane_materialization_metadata( + &branch.lane_id, + &requested_workdir_mode, + outcome, + )?; + } + let resolved_workdir_mode = materialization_outcome + .as_ref() + .map(|outcome| outcome.resolved_mode.clone()) + .unwrap_or_else(|| workdir_mode.clone()); + let workdir_backend = match materialization_outcome.as_ref() { + Some(outcome) => Some(outcome.backend), + None => self.lane_workdir_backend_for(&self.lane_record(&branch.lane_id)?)?, + }; + let materialization = match materialization_outcome.as_ref() { + Some(outcome) => Some(outcome.report.clone()), + None => self.lane_materialization_report_for(&self.lane_record(&branch.lane_id)?)?, + }; self.insert_lane_event( &branch.lane_id, "workdir_synced", @@ -266,6 +441,10 @@ impl Trail { "rescue_workdir": rescue_workdir.clone(), "paths": selected_paths, "include_neighbors": include_neighbors, + "requested_workdir_mode": requested_workdir_mode.as_str(), + "workdir_mode": materialization_outcome.as_ref().map(|outcome| outcome.resolved_mode.as_str()), + "workdir_backend": materialization_outcome.as_ref().map(|outcome| outcome.backend.as_str()), + "materialization": materialization_outcome.as_ref().map(|outcome| &outcome.report), "changed_paths": changed_paths.iter().map(|item| item.path.clone()).collect::>() }), )?; @@ -274,6 +453,10 @@ impl Trail { workdir, head_change: head.change_id, root_id: head.root_id, + requested_workdir_mode, + workdir_mode: resolved_workdir_mode, + workdir_backend, + materialization, forced: force, rescue_workdir, changed_paths, @@ -331,6 +514,10 @@ impl Trail { workdir, head_change: head.change_id, root_id: head.root_id, + requested_workdir_mode: LaneWorkdirMode::NfsCow, + workdir_mode: LaneWorkdirMode::NfsCow, + workdir_backend: Some(WorkdirBackend::Nfs), + materialization: None, forced: force, rescue_workdir, changed_paths, @@ -343,16 +530,74 @@ impl Trail { root_id: &ObjectId, target_files: &BTreeMap, sparse: bool, - ) -> Result<()> { + requested_mode: &LaneWorkdirMode, + ) -> Result> { + // A registered root may already be watched by the lane observer. Keep + // its directory inode stable; replacing the root would silently + // detach inotify/FSEvents authority from subsequent writes. + if workdir_path.is_dir() { + let disk = self.scan_files_under(workdir_path)?; + let mut deletion_parents = BTreeSet::new(); + for path in disk + .iter() + .map(|file| file.path.as_str()) + .filter(|path| !target_files.contains_key(*path)) + { + let absolute = safe_join(workdir_path, path)?; + match fs::remove_file(absolute) { + Ok(()) => { + if let Some(parent) = Path::new(path).parent() { + deletion_parents.insert(workdir_path.join(parent)); + } else { + deletion_parents.insert(workdir_path.to_path_buf()); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(Error::Io(error)), + } + } + for parent in deletion_parents { + File::open(parent)?.sync_all()?; + } + self.materialize_files_at(workdir_path, target_files, target_files)?; + File::open(workdir_path)?.sync_all()?; + if sparse { + self.write_sparse_workdir_manifest(workdir_path, target_files.keys())?; + } + self.write_clean_workdir_manifest( + workdir_path, + root_id, + target_files, + target_files.keys(), + )?; + return Ok(None); + } let parent = workdir_path.parent().ok_or_else(|| Error::InvalidPath { path: workdir_path.to_string_lossy().to_string(), reason: "workdir path has no parent".to_string(), })?; fs::create_dir_all(parent)?; + if !sparse + && matches!( + requested_mode, + LaneWorkdirMode::Auto | LaneWorkdirMode::NativeCow | LaneWorkdirMode::PortableCopy + ) + { + let replacement = create_unique_lane_workdir_replacement_target(parent, workdir_path)?; + let policy = materialization_policy_for_mode(requested_mode); + let outcome = + self.materialize_lane_root_staged(root_id, &replacement, false, policy)?; + if let Err(error) = replace_lane_workdir_with_stage(workdir_path, &replacement) { + let _ = fs::remove_dir_all(&replacement); + return Err(error); + } + self.complete_materialization_operation(&outcome.materialization_operation_id)?; + return Ok(Some(outcome)); + } let stage_dir = create_unique_lane_workdir_sync_stage_dir(parent, workdir_path)?; let result = (|| -> Result<()> { let empty = BTreeMap::new(); - self.materialize_files_best_effort_at(&stage_dir, &empty, target_files)?; + self.materialize_files_at(&stage_dir, &empty, target_files)?; if sparse { self.write_sparse_workdir_manifest(&stage_dir, target_files.keys())?; } @@ -376,7 +621,7 @@ impl Trail { if result.is_err() { let _ = fs::remove_dir_all(&stage_dir); } - result + result.map(|()| None) } fn rescue_dirty_lane_workdir( @@ -499,13 +744,14 @@ impl Trail { } pub(crate) fn hydrate_sparse_lane_workdir_paths_unlocked( - &self, + &mut self, lane: &str, branch: &LaneBranch, paths: &[String], force: bool, include_neighbors: bool, ) -> Result> { + // TRAIL_FS_PRODUCER: sparse_hydration LaneSync controlled let selected_paths = normalize_record_paths(paths)?; if selected_paths.is_empty() { return Ok(Vec::new()); @@ -549,13 +795,78 @@ impl Trail { ))); } - self.materialize_sparse_lane_workdir_paths( - &workdir_path, - &head.root_id, - sparse_paths, - &target_files, - force, + if !crate::db::change_ledger::command_authority_enabled() { + self.materialize_sparse_lane_workdir_paths( + &workdir_path, + &head.root_id, + sparse_paths, + &target_files, + force, + )?; + self.publish_lane_marker_if_materialized(lane)?; + return Ok(target_files.keys().cloned().collect()); + } + + let marker_before = self.capture_materialized_lane_marker(&workdir_path)?; + let expected = crate::db::change_ledger::prepare_materialized_lane_controlled_projection( + self, + &branch.lane_id, )?; + let evidence = crate::db::change_ledger::IntentEvidence { + exact_paths: target_files + .keys() + .map(|path| crate::db::change_ledger::LedgerPath::parse(path)) + .collect::>>()?, + complete_prefixes: Vec::new(), + }; + let projection_result = crate::db::change_ledger::run_projection_alignment( + self, + &expected, + crate::db::change_ledger::IntentProducer::LaneSync, + &evidence, + crate::db::change_ledger::ProjectionAlignmentMode::Aligned, + |db, intent| { + crate::db::change_ledger::with_materialized_lane_controlled_interval( + db, + &branch.lane_id, + intent, + &evidence, + |db| { + db.materialize_sparse_lane_workdir_paths( + &workdir_path, + &head.root_id, + sparse_paths, + &target_files, + force, + ) + }, + |db, policy, candidates| { + let comparison = db.compare_authoritative_candidates( + policy, + candidates, + &head.root_id, + crate::db::change_ledger::CandidateMaterialization::ManifestOnly, + )?; + if comparison.summaries.is_empty() { + Ok(()) + } else { + Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "sparse hydration pinned verification did not match its target root" + .into(), + command: format!("trail lane status {}", branch.lane_id), + }) + } + }, + ) + }, + |db| db.publish_lane_marker_if_materialized(&branch.lane_id), + ); + if let Err(error) = projection_result { + self.restore_materialized_lane_marker(&workdir_path, marker_before.as_deref())?; + return Err(error); + } Ok(target_files.keys().cloned().collect()) } @@ -570,10 +881,19 @@ impl Trail { let write_files = self.sparse_hydration_write_files(workdir_path, target_files, force)?; let transaction = SparseHydrationTransaction::begin(workdir_path, write_files.keys())?; let result = (|| -> Result<()> { - self.materialize_new_files_best_effort_at_with_workspace_cow( - workdir_path, - &write_files, - )?; + // The marker is one of the transaction snapshots, so a failure in + // bytes or sparse-selection publication restores all three. + if crate::db::change_ledger::command_authority_enabled() { + self.invalidate_materialized_lane_marker(workdir_path)?; + } + if crate::db::change_ledger::command_authority_enabled() { + self.materialize_files_at(workdir_path, &BTreeMap::new(), &write_files)?; + } else { + self.materialize_new_files_best_effort_at_with_workspace_cow( + workdir_path, + &write_files, + )?; + } let mut materialized_paths = sparse_paths; materialized_paths.extend(target_files.keys().cloned()); @@ -639,7 +959,7 @@ impl Trail { let disk_files = self.scan_files_under_for_paths(workdir_path, &target_paths)?; let disk_manifest = self.disk_manifest(&disk_files); let diffs = - self.diff_file_maps_to_manifest_for_paths(target_files, &disk_manifest, &target_paths); + self.diff_file_maps_to_manifest_for_paths(target_files, &disk_manifest, &target_paths)?; if let Some(diff) = diffs.first() { let detail = sparse_hydration_diff_detail( diff, @@ -722,13 +1042,7 @@ impl Trail { let head_files = self.selected_file_entries(target_files, &candidate_paths); let disk_manifest = self.disk_manifest(&disk_files); - Ok( - self.diff_file_maps_to_manifest_for_paths( - &head_files, - &disk_manifest, - &candidate_paths, - ), - ) + self.diff_file_maps_to_manifest_for_paths(&head_files, &disk_manifest, &candidate_paths) } } @@ -772,7 +1086,11 @@ fn create_unique_lane_workdir_rescue_dir(rescue_root: &Path, lane: &str) -> Resu for _ in 0..16 { let candidate = rescue_root.join(format!("{lane}-{}", now_nanos())); match fs::create_dir(&candidate) { - Ok(()) => return Ok(candidate), + Ok(()) => { + sync_directory_strict(&candidate)?; + sync_directory_strict(rescue_root)?; + return Ok(candidate); + } Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => continue, Err(err) => return Err(Error::Io(err)), } @@ -793,7 +1111,11 @@ fn create_unique_lane_workdir_sync_stage_dir( for _ in 0..16 { let candidate = parent.join(format!(".{leaf}.trail-sync-{}", now_nanos())); match fs::create_dir(&candidate) { - Ok(()) => return Ok(candidate), + Ok(()) => { + sync_directory_strict(&candidate)?; + sync_directory_strict(parent)?; + return Ok(candidate); + } Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => continue, Err(err) => return Err(Error::Io(err)), } @@ -803,6 +1125,36 @@ fn create_unique_lane_workdir_sync_stage_dir( )) } +fn create_unique_lane_workdir_replacement_target( + parent: &Path, + workdir_path: &Path, +) -> Result { + let leaf = workdir_path + .file_name() + .map(|name| name.to_string_lossy()) + .unwrap_or_else(|| "workdir".into()); + for _ in 0..16 { + let candidate = parent.join(format!(".{leaf}.trail-next-{}", now_nanos())); + match fs::symlink_metadata(&candidate) { + Ok(_) => continue, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(candidate), + Err(error) => return Err(Error::Io(error)), + } + } + Err(Error::InvalidInput( + "could not reserve a unique lane workdir replacement target".to_string(), + )) +} + +fn materialization_policy_for_mode(mode: &LaneWorkdirMode) -> MaterializationPolicy { + match mode { + LaneWorkdirMode::NativeCow => MaterializationPolicy::StrictNative, + LaneWorkdirMode::PortableCopy => MaterializationPolicy::Portable, + LaneWorkdirMode::Auto => MaterializationPolicy::Auto, + _ => MaterializationPolicy::Portable, + } +} + fn replace_lane_workdir_with_stage(workdir_path: &Path, stage_dir: &Path) -> Result<()> { let backup_path = move_existing_lane_workdir_to_backup(workdir_path)?; let replace_result = fs::rename(stage_dir, workdir_path).map_err(Error::Io); @@ -922,7 +1274,7 @@ impl SparseHydrationTransaction { let backup_dir_for_cleanup = backup_dir.clone(); let result = (|| -> Result { let mut paths = BTreeSet::new(); - paths.insert(".trail/sparse-workdir.json".to_string()); + paths.insert(".trail/sparse-selection.json".to_string()); paths.insert(".trail/workdir-manifest.json".to_string()); paths.extend(write_paths.into_iter().cloned()); diff --git a/trail/src/db/lane/workdir/view_barrier.rs b/trail/src/db/lane/workdir/view_barrier.rs index e729546..9948ea4 100644 --- a/trail/src/db/lane/workdir/view_barrier.rs +++ b/trail/src/db/lane/workdir/view_barrier.rs @@ -1,6 +1,6 @@ use std::fs::{self, File, OpenOptions}; use std::io::{self, Read, Seek, SeekFrom, Write}; -use std::path::Path; +use std::path::{Path, PathBuf}; /// Cross-process reader/writer barrier for one workspace view. /// @@ -10,8 +10,13 @@ use std::path::Path; /// Operating-system locks are released automatically if a mount or checkpoint /// process exits abruptly. pub(crate) struct ViewMutationBarrier { + authority: File, file: File, + meta_dir: PathBuf, + path: PathBuf, + exclusive: bool, checkpoint_sequence: u64, + checkpoint_generation: u64, } impl ViewMutationBarrier { @@ -25,16 +30,30 @@ impl ViewMutationBarrier { fn acquire(meta_dir: &Path, exclusive: bool) -> io::Result { fs::create_dir_all(meta_dir)?; - let mut file = OpenOptions::new() - .read(true) - .write(true) - .create(true) - .open(meta_dir.join("checkpoint-barrier.lock"))?; - lock_file(&file, exclusive)?; - let checkpoint_sequence = read_checkpoint_sequence(&mut file)?; + if !fs::symlink_metadata(meta_dir)?.file_type().is_dir() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "workspace view metadata path is not a real directory", + )); + } + let path = meta_dir.join("checkpoint-barrier.lock"); + let authority = open_barrier_authority(meta_dir)?; + lock_file(&authority, exclusive)?; + validate_authority_identity(meta_dir, &authority)?; + let mut file = open_barrier_child(&authority, &path)?; + validate_barrier_child_identity(&authority, &path, &file)?; + validate_or_initialize_barrier_identity(meta_dir, &authority, &file)?; + let (checkpoint_sequence, checkpoint_generation) = read_checkpoint_cut(&mut file)?; + validate_authority_identity(meta_dir, &authority)?; + validate_barrier_child_identity(&authority, &path, &file)?; Ok(Self { + authority, file, + meta_dir: meta_dir.to_path_buf(), + path, + exclusive, checkpoint_sequence, + checkpoint_generation, }) } @@ -42,35 +61,337 @@ impl ViewMutationBarrier { self.checkpoint_sequence } + pub(crate) fn checkpoint_generation(&self) -> u64 { + self.checkpoint_generation + } + + pub(crate) fn validate(&self) -> io::Result<()> { + validate_authority_identity(&self.meta_dir, &self.authority)?; + validate_barrier_child_identity(&self.authority, &self.path, &self.file)?; + validate_or_initialize_barrier_identity(&self.meta_dir, &self.authority, &self.file) + } + /// Update the epoch observed by future shared-lock holders. /// /// The caller must already hold the view's exclusive barrier lock. The /// file is updated in place so the inode carrying the OS lock is never /// replaced. - pub(crate) fn record_checkpoint_sequence(meta_dir: &Path, sequence: u64) -> io::Result<()> { - fs::create_dir_all(meta_dir)?; - let mut file = OpenOptions::new() - .read(true) - .write(true) - .create(true) - .open(meta_dir.join("checkpoint-barrier.lock"))?; - file.seek(SeekFrom::Start(0))?; - file.set_len(0)?; - write!(file, "{sequence}\n")?; - file.sync_data() + pub(crate) fn record_checkpoint_cut( + &mut self, + sequence: u64, + generation: u64, + ) -> io::Result<()> { + if !self.exclusive { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "checkpoint cut requires the held exclusive view barrier", + )); + } + self.validate()?; + self.file.seek(SeekFrom::Start(0))?; + self.file.set_len(0)?; + write!(self.file, "{sequence} {generation}\n")?; + self.file.sync_data()?; + self.validate()?; + self.checkpoint_sequence = sequence; + self.checkpoint_generation = generation; + Ok(()) } } -fn read_checkpoint_sequence(file: &mut File) -> io::Result { +#[cfg(unix)] +fn open_barrier_authority(meta_dir: &Path) -> io::Result { + use std::os::unix::fs::OpenOptionsExt; + + let mut options = OpenOptions::new(); + options + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC); + let authority = options.open(meta_dir)?; + let metadata = authority.metadata()?; + if !metadata.is_dir() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "workspace view barrier authority is not a directory", + )); + } + validate_authority_identity(meta_dir, &authority)?; + Ok(authority) +} + +#[cfg(windows)] +fn open_barrier_authority(meta_dir: &Path) -> io::Result { + open_barrier_no_follow(&meta_dir.join("checkpoint-barrier.authority.lock")) +} + +#[cfg(unix)] +fn validate_authority_identity(path: &Path, authority: &File) -> io::Result<()> { + use std::os::unix::fs::MetadataExt; + + let published = fs::symlink_metadata(path)?; + let held = authority.metadata()?; + if !published.file_type().is_dir() + || held.dev() != published.dev() + || held.ino() != published.ino() + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "workspace view barrier authority directory was replaced", + )); + } + Ok(()) +} + +#[cfg(windows)] +fn validate_authority_identity(_path: &Path, _authority: &File) -> io::Result<()> { + Ok(()) +} + +#[cfg(unix)] +fn open_regular_at( + authority: &File, + name: &str, + read: bool, + write: bool, + create: bool, + exclusive: bool, +) -> io::Result { + use std::ffi::CString; + use std::os::fd::{AsRawFd, FromRawFd}; + + let name = CString::new(name) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid barrier child name"))?; + let mut flags = libc::O_NOFOLLOW | libc::O_CLOEXEC; + flags |= match (read, write) { + (true, true) => libc::O_RDWR, + (false, true) => libc::O_WRONLY, + _ => libc::O_RDONLY, + }; + if create { + flags |= libc::O_CREAT; + } + if exclusive { + flags |= libc::O_EXCL; + } + let fd = unsafe { libc::openat(authority.as_raw_fd(), name.as_ptr(), flags, 0o600) }; + if fd < 0 { + return Err(io::Error::last_os_error()); + } + let file = unsafe { File::from_raw_fd(fd) }; + let metadata = file.metadata()?; + use std::os::unix::fs::MetadataExt; + if !metadata.is_file() || metadata.nlink() != 1 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "workspace view barrier child is unsafe", + )); + } + Ok(file) +} + +#[cfg(unix)] +fn open_barrier_child(authority: &File, _path: &Path) -> io::Result { + open_regular_at( + authority, + "checkpoint-barrier.lock", + true, + true, + true, + false, + ) +} + +#[cfg(windows)] +fn open_barrier_child(_authority: &File, path: &Path) -> io::Result { + open_barrier_no_follow(path) +} + +#[cfg(unix)] +fn validate_barrier_child_identity(authority: &File, _path: &Path, held: &File) -> io::Result<()> { + use std::os::unix::fs::MetadataExt; + + let published = open_regular_at( + authority, + "checkpoint-barrier.lock", + true, + false, + false, + false, + )?; + let published = published.metadata()?; + let held = held.metadata()?; + if published.dev() != held.dev() || published.ino() != held.ino() || held.nlink() != 1 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "workspace view barrier pathname changed under its pinned authority", + )); + } + Ok(()) +} + +#[cfg(windows)] +fn validate_barrier_child_identity(_authority: &File, path: &Path, held: &File) -> io::Result<()> { + validate_barrier_identity(path, held) +} + +#[cfg(unix)] +fn validate_or_initialize_barrier_identity( + _meta_dir: &Path, + authority: &File, + file: &File, +) -> io::Result<()> { + use std::os::unix::fs::MetadataExt; + + let metadata = file.metadata()?; + let expected = format!("{} {}\n", metadata.dev(), metadata.ino()); + let init_lock = open_regular_at( + authority, + "checkpoint-barrier.identity-init.lock", + true, + true, + true, + false, + )?; + lock_file(&init_lock, true)?; + let result = match open_regular_at( + authority, + "checkpoint-barrier.identity", + false, + true, + true, + true, + ) { + Ok(mut identity) => { + identity.write_all(expected.as_bytes())?; + identity.sync_all()?; + authority.sync_all()?; + Ok(()) + } + Err(err) if err.kind() == io::ErrorKind::AlreadyExists => { + let mut identity = open_regular_at( + authority, + "checkpoint-barrier.identity", + true, + false, + false, + false, + )?; + let identity_metadata = identity.metadata()?; + if !identity_metadata.is_file() || identity_metadata.nlink() != 1 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "workspace view barrier identity is unsafe", + )); + } + let mut observed = String::new(); + identity.read_to_string(&mut observed)?; + if observed != expected { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "workspace view barrier identity does not match its authority", + )); + } + Ok(()) + } + Err(err) => Err(err), + }; + let _ = unlock_file(&init_lock); + result +} + +#[cfg(windows)] +fn validate_or_initialize_barrier_identity( + _meta_dir: &Path, + _authority: &File, + _file: &File, +) -> io::Result<()> { + Ok(()) +} + +#[cfg(windows)] +fn open_barrier_no_follow(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true).write(true).create(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC); + options.mode(0o600); + } + let file = options.open(path)?; + let metadata = file.metadata()?; + if !metadata.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "workspace view barrier is not a regular file", + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if metadata.nlink() != 1 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "workspace view barrier has an unsafe hard-link count", + )); + } + } + Ok(file) +} + +#[cfg(windows)] +fn validate_barrier_identity(path: &Path, file: &File) -> io::Result<()> { + let path_metadata = fs::symlink_metadata(path)?; + if !path_metadata.file_type().is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "workspace view barrier pathname was replaced", + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let held = file.metadata()?; + if held.dev() != path_metadata.dev() || held.ino() != path_metadata.ino() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "workspace view barrier pathname changed while acquiring its lock", + )); + } + } + Ok(()) +} + +fn read_checkpoint_cut(file: &mut File) -> io::Result<(u64, u64)> { file.seek(SeekFrom::Start(0))?; let mut value = String::new(); file.read_to_string(&mut value)?; - Ok(value.trim().parse::().unwrap_or(0)) + if value.trim().is_empty() { + return Ok((0, 0)); + } + let mut fields = value.split_whitespace(); + let sequence = fields + .next() + .and_then(|field| field.parse::().ok()) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "invalid checkpoint sequence"))?; + let generation = fields + .next() + .and_then(|field| field.parse::().ok()) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "invalid checkpoint generation") + })?; + if fields.next().is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "checkpoint cut has trailing fields", + )); + } + Ok((sequence, generation)) } impl Drop for ViewMutationBarrier { fn drop(&mut self) { - let _ = unlock_file(&self.file); + let _ = unlock_file(&self.authority); } } @@ -190,4 +511,71 @@ mod tests { mutation.join().unwrap(); checkpoint.join().unwrap(); } + + #[cfg(unix)] + #[test] + fn checkpoint_cut_rejects_path_aba_and_never_follows_replacement_symlink() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let mut barrier = ViewMutationBarrier::exclusive(temp.path()).unwrap(); + let path = temp.path().join("checkpoint-barrier.lock"); + let held = temp.path().join("checkpoint-barrier.held"); + let victim = temp.path().join("victim.txt"); + fs::write(&victim, b"preserve me").unwrap(); + fs::rename(&path, &held).unwrap(); + symlink(&victim, &path).unwrap(); + + assert!(barrier.record_checkpoint_cut(7, 3).is_err()); + assert_eq!(fs::read(victim).unwrap(), b"preserve me"); + drop(barrier); + assert!(ViewMutationBarrier::shared(temp.path()).is_err()); + } + + #[cfg(unix)] + #[test] + fn checkpoint_revalidates_pinned_meta_directory_after_replacement() { + let temp = tempfile::tempdir().unwrap(); + let meta_dir = temp.path().join("meta"); + let held_dir = temp.path().join("meta-held"); + let mut barrier = ViewMutationBarrier::exclusive(&meta_dir).unwrap(); + fs::rename(&meta_dir, &held_dir).unwrap(); + fs::create_dir(&meta_dir).unwrap(); + + assert!(barrier.record_checkpoint_cut(9, 4).is_err()); + assert_eq!( + fs::read(held_dir.join("checkpoint-barrier.lock")).unwrap(), + b"" + ); + assert!(!meta_dir.join("checkpoint-barrier.lock").exists()); + } + + #[cfg(unix)] + #[test] + fn replacement_exclusive_cannot_split_authority_from_old_shared_holder() { + let temp = tempfile::tempdir().unwrap(); + let shared = ViewMutationBarrier::shared(temp.path()).unwrap(); + let path = temp.path().join("checkpoint-barrier.lock"); + let held = temp.path().join("checkpoint-barrier.held"); + fs::rename(&path, &held).unwrap(); + fs::write(&path, b"0 0\n").unwrap(); + + let meta_dir = temp.path().to_path_buf(); + let (attempted_tx, attempted_rx) = mpsc::channel(); + let (finished_tx, finished_rx) = mpsc::channel(); + let checkpoint = thread::spawn(move || { + attempted_tx.send(()).unwrap(); + finished_tx + .send(ViewMutationBarrier::exclusive(&meta_dir).is_ok()) + .unwrap(); + }); + attempted_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + assert!(finished_rx + .recv_timeout(Duration::from_millis(150)) + .is_err()); + + drop(shared); + assert!(!finished_rx.recv_timeout(Duration::from_secs(2)).unwrap()); + checkpoint.join().unwrap(); + } } diff --git a/trail/src/db/lane/workdir/view_core.rs b/trail/src/db/lane/workdir/view_core.rs index df083ad..ca05f93 100644 --- a/trail/src/db/lane/workdir/view_core.rs +++ b/trail/src/db/lane/workdir/view_core.rs @@ -29,6 +29,65 @@ const ENOTEMPTY: i32 = 39; pub(crate) const VIEW_ROOT_INO: u64 = 1; +#[cfg(test)] +std::thread_local! { + static FAIL_RENAME_AFTER_INTENT: std::cell::Cell = const { std::cell::Cell::new(false) }; + static FAIL_RENAME_BEFORE_DURABILITY_FENCE: std::cell::Cell = const { std::cell::Cell::new(false) }; + static FAIL_NAMESPACE_BEFORE_DURABILITY_FENCE: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +#[cfg(test)] +fn fail_rename_after_intent_for_current_thread() { + FAIL_RENAME_AFTER_INTENT.with(|fail| fail.set(true)); +} + +#[cfg(test)] +fn fail_rename_after_intent_if_requested() -> std::result::Result<(), i32> { + if FAIL_RENAME_AFTER_INTENT.with(|fail| fail.replace(false)) { + Err(EIO) + } else { + Ok(()) + } +} + +#[cfg(test)] +fn fail_rename_before_durability_fence_for_current_thread() { + FAIL_RENAME_BEFORE_DURABILITY_FENCE.with(|fail| fail.set(true)); +} + +#[cfg(test)] +fn fail_rename_before_durability_fence_if_requested() -> std::result::Result<(), i32> { + if FAIL_RENAME_BEFORE_DURABILITY_FENCE.with(|fail| fail.replace(false)) { + Err(EIO) + } else { + Ok(()) + } +} + +#[cfg(not(test))] +fn fail_rename_before_durability_fence_if_requested() -> std::result::Result<(), i32> { + Ok(()) +} + +#[cfg(test)] +fn fail_namespace_before_durability_fence_for_current_thread() { + FAIL_NAMESPACE_BEFORE_DURABILITY_FENCE.with(|fail| fail.set(true)); +} + +#[cfg(test)] +fn fail_namespace_before_durability_fence_if_requested() -> std::result::Result<(), i32> { + if FAIL_NAMESPACE_BEFORE_DURABILITY_FENCE.with(|fail| fail.replace(false)) { + Err(EIO) + } else { + Ok(()) + } +} + +#[cfg(not(test))] +fn fail_namespace_before_durability_fence_if_requested() -> std::result::Result<(), i32> { + Ok(()) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum ViewNodeKind { File, @@ -61,6 +120,44 @@ pub(crate) struct ViewCore { dir_epoch: SystemTime, next_ino: u64, journal: ViewMutationJournal, + generation_lease: ViewGenerationLease, +} + +const LAYER_MOUNT_RESET_INTENT_VERSION: u16 = 2; + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +struct LayerMountResetIntent { + version: u16, + mount_path: String, + upper_class: ViewPathClass, + replacement_layer_id: String, + #[serde(default)] + binding_removed: bool, + removed_whiteouts: Vec, +} + +/// A durable, filesystem-side half of workspace-layer activation. +/// +/// The reset intent is written before the private upper is moved aside. If the +/// process dies, `ViewCore::new_with_lower` compares the intended replacement +/// with SQLite: a committed binding discards the backup, while an uncommitted +/// binding restores it. Dropping this value intentionally leaves the durable +/// intent for crash recovery. +#[must_use = "a prepared layer reset must be committed or rolled back"] +pub(crate) struct PreparedLayerMountReset { + layout: ViewUpperLayout, + mount_path: String, + upper_class: ViewPathClass, + intent_path: PathBuf, + backup_path: PathBuf, + removed_whiteouts: Vec, +} + +struct RecoveredLayerMountReset { + mount_path: String, + intent_path: PathBuf, + backup_path: PathBuf, + removed_whiteouts: Vec, } enum ViewLower { @@ -83,12 +180,55 @@ impl ViewCore { Self::new_with_lower(db, upperdir, ViewLower::Root(root_id)) } + /// Construct a temporary mounted view with an explicit desired binding + /// set. This is used while initializing path-sensitive private outputs at + /// the lane's stable mountpoint before their generation is activated. + /// + /// Reset recovery is deliberately skipped: the supplied upper layout is + /// ephemeral and cannot contain a durable activation intent. The real + /// lane layout continues to use `new_lazy`, which always performs normal + /// SQLite-correlated recovery. + pub(crate) fn new_lazy_with_ephemeral_bindings( + db: Trail, + upperdir: PathBuf, + root_id: ObjectId, + layers: Vec, + ) -> Result { + Self::new_with_lower_and_bindings(db, upperdir, ViewLower::Root(root_id), Some(layers)) + } + fn new_with_lower(db: Trail, upperdir: PathBuf, lower: ViewLower) -> Result { + Self::new_with_lower_and_bindings(db, upperdir, lower, None) + } + + fn new_with_lower_and_bindings( + db: Trail, + upperdir: PathBuf, + lower: ViewLower, + ephemeral_layers: Option>, + ) -> Result { let dir_epoch = SystemTime::now(); let layout = ViewUpperLayout::from_source_upper(upperdir); layout.ensure()?; - let journal = ViewMutationJournal::open(&layout.source_upper)?; - let layers = db.workspace_layer_bindings_for_source_upper(&layout.source_upper)?; + ViewMutationJournal::initialize_storage(&layout.source_upper)?; + let _barrier = ViewMutationBarrier::shared(&layout.meta_dir)?; + let mut journal = ViewMutationJournal::open(&layout.source_upper)?; + journal.observe_checkpoint( + _barrier.checkpoint_sequence(), + _barrier.checkpoint_generation(), + )?; + let generation_lease = + ViewGenerationLease::acquire(&layout.source_upper, journal.generation())?; + let ephemeral = ephemeral_layers.is_some(); + let layers = match ephemeral_layers { + Some(layers) => layers, + None => db.workspace_layer_bindings_for_source_upper(&layout.source_upper)?, + }; + let recovered_resets = if ephemeral { + Vec::new() + } else { + recover_layer_mount_resets(&layout, &layers)? + }; let mut core = Self { db, layout, @@ -101,8 +241,35 @@ impl ViewCore { dir_epoch, next_ino: VIEW_ROOT_INO + 1, journal, + generation_lease, }; - core.whiteouts = core.load_whiteouts()?; + core.whiteouts = core.journal.whiteouts().clone(); + if !recovered_resets.is_empty() { + for recovered in &recovered_resets { + if !recovered.removed_whiteouts.is_empty() { + let class = core.path_class(&recovered.mount_path); + let sequence = core.journal.append_classified_with_whiteouts( + ViewMutationKind::Metadata, + recovered.mount_path.clone(), + class, + None, + None, + recovered + .removed_whiteouts + .iter() + .cloned() + .map(ViewWhiteoutChange::Insert) + .collect(), + )?; + core.journal.commit_whiteouts(sequence)?; + } + core.whiteouts + .extend(recovered.removed_whiteouts.iter().cloned()); + } + for recovered in recovered_resets { + finish_layer_mount_reset_recovery(&recovered)?; + } + } Ok(core) } @@ -139,7 +306,28 @@ impl ViewCore { } pub(crate) fn upper_path(&self, path: &str) -> std::result::Result { - self.upper_path_in_class(classify_view_path(path), path) + self.upper_path_in_class(self.path_class(path), path) + } + + fn path_class(&self, path: &str) -> ViewPathClass { + let conventional = classify_view_path(path); + if matches!( + conventional, + ViewPathClass::Internal | ViewPathClass::Secret + ) { + return conventional; + } + let binding = self.layers.iter().find(|binding| { + path == binding.mount_path + || path + .strip_prefix(&binding.mount_path) + .is_some_and(|rest| rest.starts_with('/')) + }); + match binding.map(|binding| binding.kind.as_str()) { + Some("dependency") => ViewPathClass::Dependency, + Some("compiler-results" | "generated" | "build") => ViewPathClass::Generated, + _ => conventional, + } } fn upper_path_in_class( @@ -172,7 +360,7 @@ impl ViewCore { if metadata.file_type().is_symlink() && !(allow_leaf_symlink && index + 1 == components.len()) => { - return Err(EPERM) + return Err(EPERM); } Ok(_) => {} Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} @@ -183,13 +371,13 @@ impl ViewCore { } fn upper_metadata(&self, path: &str) -> Option { - self.upper_path_in_class_with_leaf(classify_view_path(path), path, true) + self.upper_path_in_class_with_leaf(self.path_class(path), path, true) .ok() .and_then(|path| fs::symlink_metadata(path).ok()) } fn upper_path_with_leaf_symlink(&self, path: &str) -> std::result::Result { - self.upper_path_in_class_with_leaf(classify_view_path(path), path, true) + self.upper_path_in_class_with_leaf(self.path_class(path), path, true) } pub(crate) fn is_whiteouted(&self, path: &str) -> bool { @@ -211,6 +399,9 @@ impl ViewCore { fn layer_path(&self, path: &str) -> std::result::Result, i32> { for binding in &self.layers { + let Some(storage_path) = &binding.storage_path else { + continue; + }; let suffix = if path == binding.mount_path { Some("") } else { @@ -221,9 +412,9 @@ impl ViewCore { continue; }; let candidate = if suffix.is_empty() { - binding.storage_path.clone() + storage_path.clone() } else { - safe_join(&binding.storage_path, suffix).map_err(|_| EINVAL)? + safe_join(storage_path, suffix).map_err(|_| EINVAL)? }; let metadata = match fs::symlink_metadata(&candidate) { Ok(metadata) => metadata, @@ -232,7 +423,7 @@ impl ViewCore { }; if metadata.file_type().is_symlink() { let canonical = candidate.canonicalize().map_err(io_errno)?; - let root = binding.storage_path.canonicalize().map_err(io_errno)?; + let root = storage_path.canonicalize().map_err(io_errno)?; if !canonical.starts_with(root) { return Err(EPERM); } @@ -349,7 +540,7 @@ impl ViewCore { if path.is_empty() { return Ok(Some(ViewNodeKind::Directory)); } - if classify_view_path(path) == ViewPathClass::Internal || self.is_whiteouted(path) { + if self.path_class(path) == ViewPathClass::Internal || self.is_whiteouted(path) { return Ok(None); } if let Some(metadata) = self.upper_metadata(path) { @@ -705,18 +896,35 @@ impl ViewCore { fn begin_mutation(&mut self) -> std::result::Result { let barrier = ViewMutationBarrier::shared(&self.layout.meta_dir).map_err(|_| EIO)?; self.journal - .observe_checkpoint(barrier.checkpoint_sequence()); + .observe_checkpoint( + barrier.checkpoint_sequence(), + barrier.checkpoint_generation(), + ) + .map_err(|_| EIO)?; + self.generation_lease + .advance(&self.layout.source_upper, self.journal.generation()) + .map_err(|_| EIO)?; + if !self.journal.is_qualified() { + return Err(EIO); + } Ok(barrier) } + fn begin_qualified_view_mutation(&mut self) -> std::result::Result { + self.begin_mutation() + } + #[allow(dead_code)] pub(crate) fn ensure_upper_file( &mut self, path: &str, truncate: bool, ) -> std::result::Result { - let _barrier = self.begin_mutation()?; - self.ensure_upper_file_under_barrier(path, truncate) + // TRAIL_FS_PRODUCER: mounted_cow_copy_up CowPublication controlled + let barrier = self.begin_qualified_view_mutation()?; + let file = self.ensure_upper_file_under_barrier(path, truncate)?; + barrier.validate().map_err(|_| EIO)?; + Ok(file) } fn ensure_upper_file_under_barrier( @@ -727,9 +935,28 @@ impl ViewCore { if self.node_kind(path)? == Some(ViewNodeKind::Directory) { return Err(EISDIR); } - self.ensure_upper_parent(path)?; let upper = self.upper_path(path)?; - if !upper.exists() { + let created_upper = !upper.exists(); + let class = self.path_class(path); + let whiteout_changes = if self.whiteouts.contains(path) { + vec![ViewWhiteoutChange::Remove(path.to_string())] + } else { + Vec::new() + }; + let has_whiteout_changes = !whiteout_changes.is_empty(); + let intent_sequence = self + .journal + .append_classified_with_whiteouts( + ViewMutationKind::Write, + path.to_string(), + class, + None, + None, + whiteout_changes, + ) + .map_err(|_| EIO)?; + self.ensure_upper_parent(path)?; + if created_upper { let visible_size = if truncate { 0 } else if let Some(layer_path) = self.layer_path(path)? { @@ -767,11 +994,6 @@ impl ViewCore { File::create(&upper).map_err(io_errno)?; } } - self.whiteouts.remove(path); - self.save_whiteouts().map_err(|_| EIO)?; - self.journal - .append(ViewMutationKind::Write, path, None) - .map_err(|_| EIO)?; let file = OpenOptions::new() .read(true) .write(true) @@ -781,6 +1003,15 @@ impl ViewCore { if truncate { file.sync_data().map_err(io_errno)?; } + if created_upper || has_whiteout_changes { + self.sync_namespace_publication(path)?; + } + if has_whiteout_changes { + self.journal + .commit_whiteouts(intent_sequence) + .map_err(|_| EIO)?; + } + self.whiteouts.remove(path); Ok(file) } @@ -790,7 +1021,8 @@ impl ViewCore { offset: u64, data: &[u8], ) -> std::result::Result { - let _barrier = self.begin_mutation()?; + // TRAIL_FS_PRODUCER: mounted_cow_write CowPublication controlled + let barrier = self.begin_qualified_view_mutation()?; let path = self.path_for_ino(ino)?; let visible_size = self.attr(&path)?.size; let proposed_size = visible_size.max(offset.saturating_add(data.len() as u64)); @@ -799,6 +1031,7 @@ impl ViewCore { write_all_file_at(&file, data, offset).map_err(io_errno)?; file.sync_data().map_err(io_errno)?; test_crash_point("checkpoint_after_source_sync"); + barrier.validate().map_err(|_| EIO)?; self.attr(&path) } @@ -809,12 +1042,25 @@ impl ViewCore { mode: u32, exclusive: bool, ) -> std::result::Result { - let _barrier = self.begin_mutation()?; + // TRAIL_FS_PRODUCER: mounted_cow_create CowPublication controlled + let barrier = self.begin_qualified_view_mutation()?; let path = self.child_path(parent, name)?; if exclusive && self.node_kind(&path)?.is_some() { return Err(EEXIST); } self.enforce_mutation_quota(&path, Some(0), self.upper_metadata(&path).is_none())?; + let class = self.path_class(&path); + let intent_sequence = self + .journal + .append_classified_with_whiteouts( + ViewMutationKind::Create, + path.clone(), + class, + None, + None, + vec![ViewWhiteoutChange::Remove(path.clone())], + ) + .map_err(|_| EIO)?; self.ensure_upper_parent(&path)?; let file = OpenOptions::new() .read(true) @@ -826,12 +1072,13 @@ impl ViewCore { .map_err(io_errno)?; set_file_mode(&self.upper_path(&path)?, mode & 0o777).map_err(io_errno)?; file.sync_data().map_err(io_errno)?; - self.whiteouts.remove(&path); - self.save_whiteouts().map_err(|_| EIO)?; + self.sync_namespace_publication(&path)?; self.journal - .append(ViewMutationKind::Create, &path, None) + .commit_whiteouts(intent_sequence) .map_err(|_| EIO)?; + self.whiteouts.remove(&path); self.touch_parent(&path); + barrier.validate().map_err(|_| EIO)?; self.attr(&path) } @@ -841,21 +1088,35 @@ impl ViewCore { name: &str, mode: u32, ) -> std::result::Result { - let _barrier = self.begin_mutation()?; + // TRAIL_FS_PRODUCER: mounted_cow_mkdir CowPublication controlled + let barrier = self.begin_qualified_view_mutation()?; let path = self.child_path(parent, name)?; if self.node_kind(&path)?.is_some() { return Err(EEXIST); } self.enforce_mutation_quota(&path, None, false)?; + let class = self.path_class(&path); + let intent_sequence = self + .journal + .append_classified_with_whiteouts( + ViewMutationKind::Mkdir, + path.clone(), + class, + None, + None, + vec![ViewWhiteoutChange::Remove(path.clone())], + ) + .map_err(|_| EIO)?; fs::create_dir_all(self.upper_path(&path)?).map_err(io_errno)?; set_file_mode(&self.upper_path(&path)?, mode & 0o777).map_err(io_errno)?; - self.whiteouts.remove(&path); - self.save_whiteouts().map_err(|_| EIO)?; + self.sync_namespace_publication(&path)?; self.journal - .append(ViewMutationKind::Mkdir, &path, None) + .commit_whiteouts(intent_sequence) .map_err(|_| EIO)?; + self.whiteouts.remove(&path); self.touch_parent(&path); self.touch_dir(path.clone()); + barrier.validate().map_err(|_| EIO)?; self.attr(&path) } @@ -866,21 +1127,35 @@ impl ViewCore { name: &str, target: &Path, ) -> std::result::Result { - let _barrier = self.begin_mutation()?; + // TRAIL_FS_PRODUCER: mounted_cow_symlink CowPublication controlled + let barrier = self.begin_qualified_view_mutation()?; let path = self.child_path(parent, name)?; if self.node_kind(&path)?.is_some() { return Err(EEXIST); } validate_view_symlink_target(&path, target)?; self.enforce_mutation_quota(&path, Some(target.to_string_lossy().len() as u64), true)?; + let class = self.path_class(&path); + let intent_sequence = self + .journal + .append_classified_with_whiteouts( + ViewMutationKind::Create, + path.clone(), + class, + None, + None, + vec![ViewWhiteoutChange::Remove(path.clone())], + ) + .map_err(|_| EIO)?; self.ensure_upper_parent(&path)?; std::os::unix::fs::symlink(target, self.upper_path(&path)?).map_err(io_errno)?; - self.whiteouts.remove(&path); - self.save_whiteouts().map_err(|_| EIO)?; + self.sync_namespace_publication(&path)?; self.journal - .append(ViewMutationKind::Create, &path, None) + .commit_whiteouts(intent_sequence) .map_err(|_| EIO)?; + self.whiteouts.remove(&path); self.touch_parent(&path); + barrier.validate().map_err(|_| EIO)?; self.attr(&path) } @@ -890,8 +1165,16 @@ impl ViewCore { size: Option, mode: Option, ) -> std::result::Result { - let _barrier = self.begin_mutation()?; let path = self.path_for_ino(ino)?; + if size.is_none() && mode.is_none() { + return self.attr(&path); + } + // TRAIL_FS_PRODUCER: mounted_cow_setattr CowPublication controlled + let barrier = self.begin_qualified_view_mutation()?; + let class = self.path_class(&path); + self.journal + .append_classified(ViewMutationKind::Metadata, path.clone(), class, None, None) + .map_err(|_| EIO)?; if let Some(size) = size { self.enforce_mutation_quota(&path, Some(size), false)?; let file = self.ensure_upper_file_under_barrier(&path, false)?; @@ -908,14 +1191,13 @@ impl ViewCore { set_file_mode(&self.upper_path(&path)?, mode & 0o777).map_err(io_errno)?; } } - self.journal - .append(ViewMutationKind::Metadata, &path, None) - .map_err(|_| EIO)?; + barrier.validate().map_err(|_| EIO)?; self.attr(&path) } pub(crate) fn remove(&mut self, parent: u64, name: &str) -> std::result::Result<(), i32> { - let _barrier = self.begin_mutation()?; + // TRAIL_FS_PRODUCER: mounted_cow_remove CowPublication controlled + let barrier = self.begin_qualified_view_mutation()?; let path = self.child_path(parent, name)?; self.enforce_mutation_quota(&path, None, false)?; let kind = self.node_kind(&path)?.ok_or(ENOENT)?; @@ -923,6 +1205,27 @@ impl ViewCore { if kind == ViewNodeKind::Directory && !self.children(ino)?.is_empty() { return Err(ENOTEMPTY); } + let hides_lower = self.layer_path(&path)?.is_some() + || self.layer_directory_exists(&path)? + || self.lower_file(&path)?.is_some() + || self.lower_directory_exists(&path)?; + let whiteout_changes = if hides_lower { + vec![ViewWhiteoutChange::Insert(path.clone())] + } else { + Vec::new() + }; + let class = self.path_class(&path); + let intent_sequence = self + .journal + .append_classified_with_whiteouts( + ViewMutationKind::Delete, + path.clone(), + class, + None, + None, + whiteout_changes, + ) + .map_err(|_| EIO)?; if let Some(metadata) = self.upper_metadata(&path) { if metadata.is_dir() { fs::remove_dir(self.upper_path(&path)?).map_err(io_errno)?; @@ -930,21 +1233,256 @@ impl ViewCore { fs::remove_file(self.upper_path_with_leaf_symlink(&path)?).map_err(io_errno)?; } } - if self.layer_path(&path)?.is_some() - || self.layer_directory_exists(&path)? - || self.lower_file(&path)?.is_some() - || self.lower_directory_exists(&path)? - { + self.sync_namespace_publication(&path)?; + if hides_lower { + self.journal + .commit_whiteouts(intent_sequence) + .map_err(|_| EIO)?; + } + if hides_lower { self.whiteouts.insert(path.clone()); } - self.save_whiteouts().map_err(|_| EIO)?; - self.journal - .append(ViewMutationKind::Delete, &path, None) - .map_err(|_| EIO)?; + self.touch_parent(&path); + barrier.validate().map_err(|_| EIO)?; + Ok(()) + } + + /// Prepare an adapter-owned mount replacement without irreversibly + /// deleting private generated state. The returned transaction is paired + /// with the SQLite layer-binding savepoint by the caller. + pub(crate) fn prepare_declared_layer_mount_path( + &mut self, + path: &str, + layer_kind: &str, + replacement_layer_id: &str, + ) -> Result { + let path = normalize_relative_path(path)?; + if matches!( + classify_view_path(&path), + ViewPathClass::Internal | ViewPathClass::Secret + ) { + return Err(Error::InvalidInput(format!( + "workspace layer replacement path `{path}` is protected internal or secret state" + ))); + } + let class = match layer_kind { + "dependency" => ViewPathClass::Dependency, + "compiler-results" | "generated" | "build" => ViewPathClass::Generated, + other => { + return Err(Error::InvalidInput(format!( + "workspace layer kind `{other}` cannot own a writable mount path" + ))); + } + }; + self.prepare_validated_layer_mount_path(&path, class, replacement_layer_id, false) + } + + /// Prepare a writable-private output replacement. Its durable binding + /// identity is component/output/key-derived rather than a workspace layer + /// ID, but crash recovery is otherwise identical to layer replacement. + pub(crate) fn prepare_declared_private_mount_path( + &mut self, + path: &str, + layer_kind: &str, + binding_identity: &str, + ) -> Result { + self.prepare_declared_layer_mount_path(path, layer_kind, binding_identity) + } + + /// Preserve a compatible writable-private output and make an empty first + /// binding visible without manufacturing an immutable lower layer. + pub(crate) fn ensure_declared_private_mount_path( + &mut self, + path: &str, + layer_kind: &str, + ) -> Result<()> { + let path = normalize_relative_path(path)?; + if matches!( + classify_view_path(&path), + ViewPathClass::Internal | ViewPathClass::Secret + ) { + return Err(Error::InvalidInput(format!( + "writable-private path `{path}` is protected internal or secret state" + ))); + } + let class = match layer_kind { + "dependency" => ViewPathClass::Dependency, + "compiler-results" | "generated" | "build" => ViewPathClass::Generated, + other => { + return Err(Error::InvalidInput(format!( + "environment kind `{other}` cannot own a writable-private path" + ))); + } + }; + let upper = self + .upper_path_in_class_with_leaf(class, &path, true) + .map_err(|errno| Error::Io(std::io::Error::from_raw_os_error(errno)))?; + let _barrier = self + .begin_mutation() + .map_err(|errno| Error::Io(std::io::Error::from_raw_os_error(errno)))?; + let intent_sequence = self.journal.append_classified_with_whiteouts( + ViewMutationKind::Mkdir, + path.clone(), + class, + None, + None, + vec![ViewWhiteoutChange::RemoveTree(path.clone())], + )?; + match fs::symlink_metadata(&upper) { + Ok(metadata) if metadata.is_dir() => Ok(()), + Ok(_) => Err(Error::InvalidInput(format!( + "writable-private mount `{path}` is not a directory" + ))), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + fs::create_dir_all(&upper)?; + Ok(()) + } + Err(err) => Err(Error::Io(err)), + }?; + self.sync_namespace_publication(&path) + .map_err(|errno| Error::Io(std::io::Error::from_raw_os_error(errno)))?; + self.journal.commit_whiteouts(intent_sequence)?; + let prefix = format!("{path}/"); + self.whiteouts + .retain(|whiteout| whiteout != &path && !whiteout.starts_with(&prefix)); self.touch_parent(&path); Ok(()) } + /// Prepare removal of an adapter-owned mount with the same durable + /// rollback semantics as replacement. Recovery commits the reset only when + /// SQLite no longer contains a binding at this path. + pub(crate) fn prepare_declared_layer_unmount_path( + &mut self, + path: &str, + layer_kind: &str, + ) -> Result { + let path = normalize_relative_path(path)?; + if matches!( + classify_view_path(&path), + ViewPathClass::Internal | ViewPathClass::Secret + ) { + return Err(Error::InvalidInput(format!( + "workspace layer removal path `{path}` is protected internal or secret state" + ))); + } + let class = match layer_kind { + "dependency" => ViewPathClass::Dependency, + "compiler-results" | "generated" | "build" => ViewPathClass::Generated, + other => { + return Err(Error::InvalidInput(format!( + "workspace layer kind `{other}` cannot own a writable mount path" + ))); + } + }; + self.prepare_validated_layer_mount_path(&path, class, "binding-removed", true) + } + + #[cfg(test)] + pub(crate) fn prepare_layer_mount_path( + &mut self, + path: &str, + replacement_layer_id: &str, + ) -> Result { + let path = normalize_relative_path(path)?; + let class = classify_view_path(&path); + if !matches!(class, ViewPathClass::Dependency | ViewPathClass::Generated) { + return Err(Error::InvalidInput(format!( + "workspace layer replacement path `{path}` is not dependency or generated state" + ))); + } + self.prepare_validated_layer_mount_path(&path, class, replacement_layer_id, false) + } + + fn prepare_validated_layer_mount_path( + &mut self, + path: &str, + class: ViewPathClass, + replacement_layer_id: &str, + binding_removed: bool, + ) -> Result { + if replacement_layer_id.trim().is_empty() { + return Err(Error::InvalidInput( + "replacement workspace layer id cannot be empty".to_string(), + )); + } + let source_path = self + .upper_path_in_class_with_leaf(ViewPathClass::Source, path, true) + .map_err(|errno| Error::Io(std::io::Error::from_raw_os_error(errno)))?; + if fs::symlink_metadata(&source_path).is_ok() { + return Err(Error::InvalidInput(format!( + "adapter mount `{path}` overlaps pre-existing source-upper state; preserve or remove that source state before synchronizing the environment" + ))); + } + let _barrier = self + .begin_mutation() + .map_err(|errno| Error::Io(std::io::Error::from_raw_os_error(errno)))?; + let upper = self + .upper_path_in_class_with_leaf(class, path, true) + .map_err(|errno| Error::Io(std::io::Error::from_raw_os_error(errno)))?; + let prefix = format!("{path}/"); + let removed_whiteouts = self + .whiteouts + .iter() + .filter(|whiteout| *whiteout == path || whiteout.starts_with(&prefix)) + .cloned() + .collect::>(); + let (intent_path, backup_path) = create_layer_mount_reset_paths(&self.layout, path)?; + let intent = LayerMountResetIntent { + version: LAYER_MOUNT_RESET_INTENT_VERSION, + mount_path: path.to_string(), + upper_class: class, + replacement_layer_id: replacement_layer_id.to_string(), + binding_removed, + removed_whiteouts: removed_whiteouts.clone(), + }; + write_file_atomic(&intent_path, &serde_json::to_vec_pretty(&intent)?, true)?; + let intent_sequence = self.journal.append_classified_with_whiteouts( + ViewMutationKind::Metadata, + path.to_string(), + class, + None, + None, + vec![ViewWhiteoutChange::RemoveTree(path.to_string())], + )?; + + let reset = (|| -> Result<()> { + match fs::symlink_metadata(&upper) { + Ok(_) => { + fs::rename(&upper, &backup_path)?; + if let Some(parent) = upper.parent() { + sync_directory(parent); + } + if let Some(parent) = backup_path.parent() { + sync_directory(parent); + } + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(Error::Io(err)), + } + self.whiteouts + .retain(|whiteout| whiteout != path && !whiteout.starts_with(&prefix)); + Ok(()) + })(); + let prepared = PreparedLayerMountReset { + layout: self.layout.clone(), + mount_path: path.to_string(), + upper_class: class, + intent_path, + backup_path, + removed_whiteouts, + }; + if let Err(err) = reset { + let _ = prepared.rollback(self); + return Err(err); + } + self.sync_namespace_publication(path) + .map_err(|errno| Error::Io(std::io::Error::from_raw_os_error(errno)))?; + self.journal.commit_whiteouts(intent_sequence)?; + self.touch_parent(path); + Ok(prepared) + } + pub(crate) fn rename( &mut self, from_parent: u64, @@ -952,7 +1490,8 @@ impl ViewCore { to_parent: u64, to_name: &str, ) -> std::result::Result<(), i32> { - let _barrier = self.begin_mutation()?; + // TRAIL_FS_PRODUCER: mounted_cow_rename CowPublication controlled + let barrier = self.begin_qualified_view_mutation()?; let old = self.child_path(from_parent, from_name)?; let new = self.child_path(to_parent, to_name)?; if old == new { @@ -963,7 +1502,6 @@ impl ViewCore { if kind == ViewNodeKind::Directory && new.starts_with(&format!("{old}/")) { return Err(EINVAL); } - self.ensure_upper_parent(&new)?; if let Some(target_kind) = self.node_kind(&new)? { if kind == ViewNodeKind::File && target_kind == ViewNodeKind::Directory { return Err(EISDIR); @@ -978,6 +1516,25 @@ impl ViewCore { } } } + let old_class = self.path_class(&old); + let new_class = self.path_class(&new); + let intent_sequence = self + .journal + .append_classified_with_whiteouts( + ViewMutationKind::Rename, + old.clone(), + old_class, + Some(new.clone()), + Some(new_class), + vec![ + ViewWhiteoutChange::Insert(old.clone()), + ViewWhiteoutChange::RemoveTree(new.clone()), + ], + ) + .map_err(|_| EIO)?; + #[cfg(test)] + fail_rename_after_intent_if_requested()?; + self.ensure_upper_parent(&new)?; if self.upper_metadata(&new).is_some() { let metadata = self.upper_metadata(&new).unwrap(); if metadata.is_dir() { @@ -1003,15 +1560,17 @@ impl ViewCore { validate_view_symlink_target(&new, &target)?; create_view_symlink(&target, &self.upper_path(&new)?).map_err(io_errno)?; } - self.whiteouts.insert(old.clone()); - self.whiteouts.remove(&new); - self.save_whiteouts().map_err(|_| EIO)?; + fail_rename_before_durability_fence_if_requested()?; + self.sync_rename_publication(&old, &new)?; self.journal - .append(ViewMutationKind::Rename, &old, Some(&new)) + .commit_whiteouts(intent_sequence) .map_err(|_| EIO)?; + self.whiteouts.insert(old.clone()); + let new_prefix = format!("{new}/"); + self.whiteouts + .retain(|path| path != &new && !path.starts_with(&new_prefix)); self.touch_parent(&old); self.touch_parent(&new); - let new_prefix = format!("{new}/"); let replaced_inodes = self .ino_by_path .keys() @@ -1037,13 +1596,13 @@ impl ViewCore { self.ino_by_path.insert(moved.clone(), ino); self.path_by_ino.insert(ino, moved); } + barrier.validate().map_err(|_| EIO)?; Ok(()) } /// Returns the checkpoint candidate set without walking the composed view. /// The journal is the fast path; upper and whiteout scans make recovery /// correct after an interrupted append. - #[cfg(test)] pub(crate) fn checkpoint_candidates(&self) -> Result { match &self.lower { #[cfg(test)] @@ -1064,7 +1623,9 @@ impl ViewCore { let target_path = self.upper_path(target)?; clone_or_copy_projected_file(&layer_path, &target_path).map_err(|_| EIO)?; let metadata = fs::metadata(layer_path).map_err(io_errno)?; - return set_file_mode(&target_path, copy_up_mode(&metadata)).map_err(io_errno); + set_file_mode(&target_path, copy_up_mode(&metadata)).map_err(io_errno)?; + self.sync_copied_file(&target_path)?; + return Ok(()); } let entry = self.lower_file(source)?.ok_or(ENOENT)?; self.ensure_upper_parent(target)?; @@ -1079,7 +1640,96 @@ impl ViewCore { entry.mode & 0o777 }, ) - .map_err(io_errno) + .map_err(io_errno)?; + self.sync_copied_file(&target_path) + } + + fn sync_copied_file(&self, path: &Path) -> std::result::Result<(), i32> { + OpenOptions::new() + .read(true) + .open(path) + .and_then(|file| file.sync_all()) + .map_err(io_errno)?; + if let Some(parent) = path.parent() { + sync_directory_strict(parent).map_err(|_| EIO)?; + } + Ok(()) + } + + fn sync_namespace_publication(&self, path: &str) -> std::result::Result<(), i32> { + fail_namespace_before_durability_fence_if_requested()?; + let upper = self.upper_path_with_leaf_symlink(path)?; + match fs::symlink_metadata(&upper) { + Ok(metadata) if metadata.is_file() => { + OpenOptions::new() + .read(true) + .open(&upper) + .and_then(|file| file.sync_all()) + .map_err(io_errno)?; + } + Ok(metadata) if metadata.is_dir() => { + sync_directory_strict(&upper).map_err(|_| EIO)?; + } + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(io_errno(err)), + } + + let root = self.layout.upper_for_class(self.path_class(path)); + let mut cursor = upper.parent(); + while let Some(directory) = cursor { + if directory.is_dir() { + sync_directory_strict(directory).map_err(|_| EIO)?; + } + if directory == root { + break; + } + cursor = directory.parent(); + } + Ok(()) + } + + fn sync_rename_publication(&self, old: &str, new: &str) -> std::result::Result<(), i32> { + let mut directories = BTreeSet::new(); + for root in [ + &self.layout.source_upper, + &self.layout.generated_upper, + &self.layout.scratch_upper, + ] { + let source = safe_join(root, old).map_err(|_| EINVAL)?; + if let Some(parent) = source.parent() { + insert_directory_ancestry(&mut directories, parent, root); + } + let destination = safe_join(root, new).map_err(|_| EINVAL)?; + if let Some(parent) = destination.parent() { + insert_directory_ancestry(&mut directories, parent, root); + } + let metadata = match fs::symlink_metadata(&destination) { + Ok(metadata) => metadata, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, + Err(err) => return Err(io_errno(err)), + }; + if metadata.is_dir() { + for entry in walkdir::WalkDir::new(&destination).follow_links(false) { + let entry = entry.map_err(|_| EIO)?; + if entry.file_type().is_dir() { + directories.insert(entry.path().to_path_buf()); + } + } + } else if metadata.is_file() { + OpenOptions::new() + .read(true) + .open(&destination) + .and_then(|file| file.sync_all()) + .map_err(io_errno)?; + } + } + for directory in directories { + if directory.is_dir() { + sync_directory_strict(&directory).map_err(|_| EIO)?; + } + } + Ok(()) } fn merge_lower_subtree_into_upper(&self, root: &str) -> std::result::Result<(), i32> { @@ -1174,58 +1824,338 @@ impl ViewCore { } Ok(()) } +} - fn load_whiteouts(&self) -> Result> { - let mut whiteouts = BTreeSet::new(); - let classes = [ - ViewPathClass::Source, - ViewPathClass::Dependency, - ViewPathClass::Generated, - ViewPathClass::Scratch, - ViewPathClass::Secret, - ]; - for path in classes - .into_iter() - .map(|class| self.layout.whiteouts_path(class)) - .chain(std::iter::once(self.layout.legacy_whiteouts_path())) - { - match fs::read(path) { - Ok(bytes) => { - for path in serde_json::from_slice::>(&bytes)? { - whiteouts.insert(normalize_relative_path(&path)?); - } - } - Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} - Err(err) => return Err(Error::Io(err)), +fn insert_directory_ancestry(directories: &mut BTreeSet, start: &Path, root: &Path) { + let mut cursor = Some(start); + while let Some(directory) = cursor { + if !directory.starts_with(root) { + break; + } + directories.insert(directory.to_path_buf()); + if directory == root { + break; + } + cursor = directory.parent(); + } +} + +#[cfg(all(debug_assertions, unix))] +pub(crate) fn run_changed_path_view_flow() -> std::result::Result<(), String> { + let temp = tempfile::tempdir().map_err(|err| err.to_string())?; + fs::write(temp.path().join("README.md"), b"baseline\n").map_err(|err| err.to_string())?; + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false) + .map_err(|err| err.to_string())?; + let db = Trail::open(temp.path()).map_err(|err| err.to_string())?; + let root = db + .resolve_branch_ref("main") + .map_err(|err| err.to_string())? + .root_id; + + let upper = temp.path().join("view/source-upper"); + let mut view = ViewCore::new_lazy( + Trail::open(temp.path()).map_err(|err| err.to_string())?, + upper.clone(), + root.clone(), + ) + .map_err(|err| err.to_string())?; + let readme = view + .lookup(VIEW_ROOT_INO, "README.md") + .map_err(|code| format!("lookup failed with errno {code}"))?; + fail_next_view_journal_sync_for_current_thread(); + if view.write(readme, 0, b"new").is_ok() { + return Err("injected journal sync failure unexpectedly exposed a write".to_string()); + } + if upper.join("README.md").exists() { + return Err("copy-up became visible before its intent was durable".to_string()); + } + if fs::read(temp.path().join("README.md")).map_err(|err| err.to_string())? != b"baseline\n" { + return Err("journal sync failure changed the lower file".to_string()); + } + + view.write(readme, 0, b"changed") + .map_err(|code| format!("write failed with errno {code}"))?; + let qualified = view + .checkpoint_candidates() + .map_err(|err| err.to_string())?; + if !qualified.qualified || qualified.upper_recovery_walks != 0 { + return Err("qualified view did not retain the zero-walk fast path".to_string()); + } + OpenOptions::new() + .append(true) + .open(ViewUpperLayout::from_source_upper(upper.clone()).journal_path()) + .and_then(|mut file| file.write_all(b"corrupt-complete-record\n")) + .map_err(|err| err.to_string())?; + let recovered = view + .checkpoint_candidates() + .map_err(|err| err.to_string())?; + if recovered.qualified + || recovered.upper_recovery_walks == 0 + || !recovered.paths.contains("README.md") + { + return Err("untrusted view did not reconcile its upper tree".to_string()); + } + + let whiteout_upper = temp.path().join("whiteout-view/source-upper"); + let mut whiteout_view = ViewCore::new_lazy( + Trail::open(temp.path()).map_err(|err| err.to_string())?, + whiteout_upper.clone(), + root.clone(), + ) + .map_err(|err| err.to_string())?; + whiteout_view + .remove(VIEW_ROOT_INO, "README.md") + .map_err(|code| format!("remove failed with errno {code}"))?; + drop(whiteout_view); + let reopened = ViewCore::new_lazy( + Trail::open(temp.path()).map_err(|err| err.to_string())?, + whiteout_upper, + root, + ) + .map_err(|err| err.to_string())?; + if !reopened.is_whiteouted("README.md") { + return Err("incremental whiteout was not replayed".to_string()); + } + let whiteout_candidates = reopened + .checkpoint_candidates() + .map_err(|err| err.to_string())?; + if !whiteout_candidates.qualified + || whiteout_candidates.upper_recovery_walks != 0 + || !whiteout_candidates.paths.contains("README.md") + { + return Err("qualified incremental whiteout lost checkpoint authority".to_string()); + } + Ok(()) +} + +impl PreparedLayerMountReset { + pub(crate) fn install_private_directory(&self, source: &Path) -> Result<()> { + if !source.is_dir() { + return Err(Error::InvalidInput(format!( + "writable-private seed `{}` is not a directory", + source.display() + ))); + } + let upper = safe_join( + self.layout.upper_for_class(self.upper_class), + &self.mount_path, + )?; + if fs::symlink_metadata(&upper).is_ok() { + return Err(Error::Corrupt(format!( + "writable-private destination `{}` was recreated during activation", + upper.display() + ))); + } + copy_dir_recursive(source, &upper)?; + if let Some(parent) = upper.parent() { + sync_directory(parent); + } + Ok(()) + } + + /// SQLite already committed the replacement. Cleanup is best-effort; a + /// retained intent is safe because the next view open observes the new + /// binding and completes cleanup idempotently. + pub(crate) fn commit(self) { + let _ = remove_layer_mount_reset_path(&self.backup_path); + if !self.backup_path.exists() { + let _ = fs::remove_file(&self.intent_path); + if let Some(parent) = self.intent_path.parent() { + sync_directory(parent); } } - Ok(whiteouts) } - fn save_whiteouts(&self) -> Result<()> { - fs::create_dir_all(&self.layout.meta_dir)?; - for class in [ - ViewPathClass::Source, - ViewPathClass::Dependency, - ViewPathClass::Generated, - ViewPathClass::Scratch, - ViewPathClass::Secret, - ] { - let paths = self - .whiteouts - .iter() - .filter(|path| classify_view_path(path) == class) - .collect::>(); - write_file_atomic( - &self.layout.whiteouts_path(class), - &serde_json::to_vec(&paths)?, - false, + pub(crate) fn rollback(self, core: &mut ViewCore) -> Result<()> { + let whiteout_sequence = if !self.removed_whiteouts.is_empty() { + let class = core.path_class(&self.mount_path); + let sequence = core.journal.append_classified_with_whiteouts( + ViewMutationKind::Metadata, + self.mount_path.clone(), + class, + None, + None, + self.removed_whiteouts + .iter() + .cloned() + .map(ViewWhiteoutChange::Insert) + .collect(), )?; + Some(sequence) + } else { + None + }; + let upper = safe_join( + self.layout.upper_for_class(self.upper_class), + &self.mount_path, + )?; + remove_layer_mount_reset_path(&upper)?; + if self.backup_path.exists() { + if let Some(parent) = upper.parent() { + fs::create_dir_all(parent)?; + } + fs::rename(&self.backup_path, &upper)?; + if let Some(parent) = upper.parent() { + sync_directory(parent); + } + } + if let Some(sequence) = whiteout_sequence { + core.journal.commit_whiteouts(sequence)?; + } + core.whiteouts.extend(self.removed_whiteouts); + remove_layer_mount_reset_path(&self.intent_path)?; + if let Some(parent) = self.intent_path.parent() { + sync_directory(parent); } Ok(()) } } +fn layer_mount_reset_intents_dir(layout: &ViewUpperLayout) -> PathBuf { + layout.meta_dir.join("layer-reset-intents") +} + +fn layer_mount_reset_backups_dir(layout: &ViewUpperLayout) -> PathBuf { + layout.meta_dir.join("layer-reset-backups") +} + +fn create_layer_mount_reset_paths( + layout: &ViewUpperLayout, + mount_path: &str, +) -> Result<(PathBuf, PathBuf)> { + let intents = layer_mount_reset_intents_dir(layout); + let backups = layer_mount_reset_backups_dir(layout); + fs::create_dir_all(&intents)?; + fs::create_dir_all(&backups)?; + for attempt in 0..32_u8 { + let id = sha256_hex( + format!( + "{}:{}:{}:{}", + std::process::id(), + now_nanos(), + mount_path, + attempt + ) + .as_bytes(), + ); + let intent = intents.join(format!("{id}.json")); + let backup = backups.join(&id); + if !intent.exists() && !backup.exists() { + return Ok((intent, backup)); + } + } + Err(Error::InvalidInput( + "could not allocate a unique workspace-layer reset intent".to_string(), + )) +} + +fn recover_layer_mount_resets( + layout: &ViewUpperLayout, + layers: &[WorkspaceLayerBinding], +) -> Result> { + let intents_dir = layer_mount_reset_intents_dir(layout); + let entries = match fs::read_dir(&intents_dir) { + Ok(entries) => entries, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(err) => return Err(Error::Io(err)), + }; + let mut intent_paths = entries + .collect::, _>>()? + .into_iter() + .map(|entry| entry.path()) + .filter(|path| path.extension().and_then(|value| value.to_str()) == Some("json")) + .collect::>(); + intent_paths.sort(); + + let mut recovered = Vec::new(); + for intent_path in intent_paths { + let id = intent_path + .file_stem() + .and_then(|value| value.to_str()) + .filter(|value| value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit())) + .ok_or_else(|| { + Error::Corrupt(format!( + "invalid workspace-layer reset intent name `{}`", + intent_path.display() + )) + })?; + let intent: LayerMountResetIntent = serde_json::from_slice(&fs::read(&intent_path)?)?; + if intent.version != 1 && intent.version != LAYER_MOUNT_RESET_INTENT_VERSION { + return Err(Error::Corrupt(format!( + "unsupported workspace-layer reset intent version {}", + intent.version + ))); + } + let mount_path = normalize_relative_path(&intent.mount_path)?; + if !matches!( + intent.upper_class, + ViewPathClass::Dependency | ViewPathClass::Generated + ) { + return Err(Error::Corrupt(format!( + "workspace-layer reset for `{mount_path}` targets protected upper class `{}`", + intent.upper_class.as_str() + ))); + } + let backup_path = layer_mount_reset_backups_dir(layout).join(id); + let binding_committed = if intent.binding_removed { + layers + .iter() + .all(|binding| binding.mount_path != mount_path) + } else { + layers.iter().any(|binding| { + binding.mount_path == mount_path + && binding.binding_identity == intent.replacement_layer_id + }) + }; + if binding_committed { + remove_layer_mount_reset_path(&backup_path)?; + remove_layer_mount_reset_path(&intent_path)?; + sync_directory(&intents_dir); + continue; + } + + let upper = safe_join(layout.upper_for_class(intent.upper_class), &mount_path)?; + remove_layer_mount_reset_path(&upper)?; + if backup_path.exists() { + if let Some(parent) = upper.parent() { + fs::create_dir_all(parent)?; + } + fs::rename(&backup_path, &upper)?; + if let Some(parent) = upper.parent() { + sync_directory(parent); + } + } + recovered.push(RecoveredLayerMountReset { + mount_path, + intent_path, + backup_path, + removed_whiteouts: intent.removed_whiteouts, + }); + } + Ok(recovered) +} + +fn finish_layer_mount_reset_recovery(recovered: &RecoveredLayerMountReset) -> Result<()> { + remove_layer_mount_reset_path(&recovered.backup_path)?; + remove_layer_mount_reset_path(&recovered.intent_path)?; + if let Some(parent) = recovered.intent_path.parent() { + sync_directory(parent); + } + Ok(()) +} + +fn remove_layer_mount_reset_path(path: &Path) -> Result<()> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => { + fs::remove_dir_all(path)?; + } + Ok(_) => fs::remove_file(path)?, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(Error::Io(err)), + } + Ok(()) +} + #[derive(Clone, Copy, Debug, Default)] struct ViewUpperUsage { logical_bytes: u64, @@ -1461,10 +2391,124 @@ mod tests { assert!(!eof); } + #[test] + fn interrupted_layer_reset_restores_private_upper_and_whiteouts_when_binding_did_not_commit() { + let (_temp, db, root, upper) = fixture(); + let layout = ViewUpperLayout::from_source_upper(upper.clone()); + fs::create_dir_all(layout.generated_upper.join("node_modules/pkg")).unwrap(); + fs::write( + layout.generated_upper.join("node_modules/pkg/private.js"), + "private\n", + ) + .unwrap(); + let mut view = lazy_core(&db, &root, upper.clone()); + view.whiteouts + .insert("node_modules/pkg/removed.js".to_string()); + + let reset = view + .prepare_declared_layer_mount_path("node_modules", "dependency", "layer-not-committed") + .unwrap(); + assert!(!layout.generated_upper.join("node_modules").exists()); + let seed = _temp.path().join("replacement-seed"); + fs::create_dir_all(seed.join("pkg")).unwrap(); + fs::write(seed.join("pkg/replacement.js"), "replacement\n").unwrap(); + reset.install_private_directory(&seed).unwrap(); + assert!(layout + .generated_upper + .join("node_modules/pkg/replacement.js") + .is_file()); + drop(reset); // simulate process loss: recovery owns the durable intent + drop(view); + + let reopened = lazy_core(&db, &root, upper); + assert_eq!( + fs::read(layout.generated_upper.join("node_modules/pkg/private.js")).unwrap(), + b"private\n" + ); + assert!(reopened.is_whiteouted("node_modules/pkg/removed.js")); + assert!(fs::read_dir(layer_mount_reset_intents_dir(&layout)) + .unwrap() + .next() + .is_none()); + } + + #[test] + fn interrupted_private_seed_is_kept_only_after_matching_binding_commit() { + let (temp, db, root, upper) = fixture(); + let layout = ViewUpperLayout::from_source_upper(upper.clone()); + fs::create_dir_all(layout.generated_upper.join("build")).unwrap(); + fs::write(layout.generated_upper.join("build/old.txt"), "old\n").unwrap(); + let seed = temp.path().join("private-seed"); + fs::create_dir_all(&seed).unwrap(); + fs::write(seed.join("new.txt"), "new\n").unwrap(); + + let mut view = lazy_core(&db, &root, upper); + let reset = view + .prepare_declared_private_mount_path("build", "generated", "private_committed_identity") + .unwrap(); + reset.install_private_directory(&seed).unwrap(); + drop(reset); + drop(view); + + let bindings = vec![WorkspaceLayerBinding { + binding_identity: "private_committed_identity".to_string(), + layer_id: None, + mount_path: "build".to_string(), + storage_path: None, + kind: "generated".to_string(), + priority: 100, + }]; + assert!(recover_layer_mount_resets(&layout, &bindings) + .unwrap() + .is_empty()); + assert_eq!( + fs::read(layout.generated_upper.join("build/new.txt")).unwrap(), + b"new\n" + ); + assert!(!layout.generated_upper.join("build/old.txt").exists()); + assert!(fs::read_dir(layer_mount_reset_intents_dir(&layout)) + .unwrap() + .next() + .is_none()); + } + + #[test] + fn interrupted_layer_unmount_discards_private_upper_when_binding_removal_committed() { + let (_temp, db, root, upper) = fixture(); + let layout = ViewUpperLayout::from_source_upper(upper.clone()); + fs::create_dir_all(layout.generated_upper.join("generated/old")).unwrap(); + fs::write( + layout.generated_upper.join("generated/old/private.txt"), + "private\n", + ) + .unwrap(); + let mut view = lazy_core(&db, &root, upper.clone()); + view.whiteouts + .insert("generated/old/removed.txt".to_string()); + + let reset = view + .prepare_declared_layer_unmount_path("generated/old", "generated") + .unwrap(); + assert!(!layout.generated_upper.join("generated/old").exists()); + drop(reset); // SQLite contains no binding, so recovery commits removal. + drop(view); + + let reopened = lazy_core(&db, &root, upper); + assert!(!layout.generated_upper.join("generated/old").exists()); + assert!(!reopened.is_whiteouted("generated/old/removed.txt")); + assert!(fs::read_dir(layer_mount_reset_intents_dir(&layout)) + .unwrap() + .next() + .is_none()); + } + #[test] fn view_core_conformance_copy_up_write_delete_and_reopen() { let (temp, db, root, upper) = fixture(); let mut view = core(&db, &root, upper.clone()); + assert!(view + .prepare_layer_mount_path("README.md", "layer-test") + .is_err()); let readme = view.lookup(VIEW_ROOT_INO, "README.md").unwrap(); assert!(!upper.join("README.md").exists()); view.write(readme, 0, b"changed\n").unwrap(); @@ -1483,6 +2527,293 @@ mod tests { assert!(reopened.is_whiteouted("README.md")); } + #[test] + fn crash_after_rename_intent_does_not_publish_whiteout_state() { + let (_temp, db, root, upper) = fixture(); + let mut view = core(&db, &root, upper.clone()); + view.journal + .append_classified_with_whiteouts( + ViewMutationKind::Rename, + "README.md".into(), + ViewPathClass::Source, + Some("renamed.md".into()), + Some(ViewPathClass::Source), + vec![ + ViewWhiteoutChange::Insert("README.md".into()), + ViewWhiteoutChange::RemoveTree("renamed.md".into()), + ], + ) + .unwrap(); + drop(view); // crash before the filesystem rename and commit record + + let mut reopened = core(&db, &root, upper); + assert!(reopened.lookup(VIEW_ROOT_INO, "README.md").is_ok()); + assert!(reopened.lookup(VIEW_ROOT_INO, "renamed.md").is_err()); + assert!(!reopened.is_whiteouted("README.md")); + } + + #[test] + fn rename_io_failure_after_intent_does_not_publish_whiteout_state() { + let (_temp, db, root, upper) = fixture(); + let mut view = core(&db, &root, upper.clone()); + fail_rename_after_intent_for_current_thread(); + assert!(matches!( + view.rename(VIEW_ROOT_INO, "README.md", VIEW_ROOT_INO, "renamed.md"), + Err(code) if code == EIO + )); + drop(view); + + let mut reopened = core(&db, &root, upper); + assert!(reopened.lookup(VIEW_ROOT_INO, "README.md").is_ok()); + assert!(reopened.lookup(VIEW_ROOT_INO, "renamed.md").is_err()); + assert!(!reopened.is_whiteouted("README.md")); + } + + #[test] + fn rename_durability_fence_precedes_semantic_commit() { + let (_temp, db, root, upper) = fixture(); + let mut view = core(&db, &root, upper.clone()); + let readme = view.lookup(VIEW_ROOT_INO, "README.md").unwrap(); + view.write(readme, 0, b"changed\n").unwrap(); + fail_rename_before_durability_fence_for_current_thread(); + + assert!(matches!( + view.rename(VIEW_ROOT_INO, "README.md", VIEW_ROOT_INO, "renamed.md"), + Err(code) if code == EIO + )); + assert!(upper.join("renamed.md").is_file()); + drop(view); + + let mut reopened = core(&db, &root, upper); + assert!(reopened.lookup(VIEW_ROOT_INO, "README.md").is_ok()); + assert!(reopened.lookup(VIEW_ROOT_INO, "renamed.md").is_ok()); + assert!(!reopened.is_whiteouted("README.md")); + let candidates = reopened.checkpoint_candidates().unwrap(); + assert!(candidates.paths.contains("README.md")); + assert!(candidates.paths.contains("renamed.md")); + } + + #[test] + fn namespace_durability_fence_precedes_remove_whiteout_commit() { + let (_temp, db, root, upper) = fixture(); + let mut view = core(&db, &root, upper.clone()); + fail_namespace_before_durability_fence_for_current_thread(); + + assert!(matches!( + view.remove(VIEW_ROOT_INO, "README.md"), + Err(code) if code == EIO + )); + drop(view); + + let mut reopened = core(&db, &root, upper); + assert!(reopened.lookup(VIEW_ROOT_INO, "README.md").is_ok()); + assert!(!reopened.is_whiteouted("README.md")); + assert!(reopened + .checkpoint_candidates() + .unwrap() + .paths + .contains("README.md")); + } + + #[test] + fn first_copy_up_requires_namespace_durability_before_write() { + let (_temp, db, root, upper) = fixture(); + let mut view = core(&db, &root, upper.clone()); + let readme = view.lookup(VIEW_ROOT_INO, "README.md").unwrap(); + fail_namespace_before_durability_fence_for_current_thread(); + + assert!(matches!( + view.write(readme, 0, b"changed\n"), + Err(code) if code == EIO + )); + assert_eq!(fs::read(upper.join("README.md")).unwrap(), b"baseline\n"); + drop(view); + + let mut reopened = core(&db, &root, upper); + let reopened_readme = reopened.lookup(VIEW_ROOT_INO, "README.md").unwrap(); + assert_eq!( + reopened.read(reopened_readme, 0, 32).unwrap().0, + b"baseline\n" + ); + assert!(reopened + .checkpoint_candidates() + .unwrap() + .paths + .contains("README.md")); + } + + #[test] + fn semantic_upper_mutation_is_not_visible_before_intent_is_durable() { + let (temp, db, root, upper) = fixture(); + let mut view = core(&db, &root, upper.clone()); + let readme = view.lookup(VIEW_ROOT_INO, "README.md").unwrap(); + fail_next_view_journal_sync_for_current_thread(); + + assert!(matches!(view.write(readme, 0, b"new"), Err(code) if code == EIO)); + assert!(!upper.join("README.md").exists()); + assert_eq!( + fs::read(temp.path().join("README.md")).unwrap(), + b"baseline\n" + ); + assert_eq!(view.read(readme, 0, 32).unwrap().0, b"baseline\n"); + } + + #[test] + fn live_handle_reloads_after_noop_checkpoint_rotates_generation() { + let (_temp, db, root, upper) = fixture(); + let mut view = core(&db, &root, upper.clone()); + let layout = ViewUpperLayout::from_source_upper(upper.clone()); + { + let mut checkpoint = ViewMutationBarrier::exclusive(&layout.meta_dir).unwrap(); + ViewMutationJournal::rotate_after_checkpoint(&upper, 0, 1).unwrap(); + checkpoint.record_checkpoint_cut(0, 1).unwrap(); + } + + let readme = view.lookup(VIEW_ROOT_INO, "README.md").unwrap(); + view.write(readme, 0, b"changed\n").unwrap(); + + let candidates = view.checkpoint_candidates().unwrap(); + assert!(candidates.qualified); + assert_eq!(candidates.journal_sequence, 1); + assert!(candidates.paths.contains("README.md")); + let active = ViewMutationJournal::open(&upper).unwrap(); + assert_eq!(active.generation(), 1); + assert!(active.dirty_source_paths().contains("README.md")); + } + + #[test] + fn untrusted_view_scans_upper_instead_of_claiming_zero_recovery_walks() { + let (_temp, db, root, upper) = fixture(); + let mut view = core(&db, &root, upper.clone()); + let readme = view.lookup(VIEW_ROOT_INO, "README.md").unwrap(); + view.write(readme, 0, b"changed").unwrap(); + let journal_path = ViewUpperLayout::from_source_upper(upper.clone()).journal_path(); + OpenOptions::new() + .append(true) + .open(journal_path) + .unwrap() + .write_all(b"corrupt-complete-record\n") + .unwrap(); + + let candidates = view.checkpoint_candidates().unwrap(); + assert!(!candidates.qualified); + assert!(candidates.upper_recovery_walks > 0); + assert!(candidates.paths.contains("README.md")); + } + + #[derive(Clone, Copy)] + enum JournalDamage { + Missing, + Corrupt, + Gapped, + ValidPrefix, + } + + fn damage_mutation_journal(upper: &Path, damage: JournalDamage) { + let path = ViewUpperLayout::from_source_upper(upper.to_path_buf()).journal_path(); + match damage { + JournalDamage::Missing => fs::remove_file(path).unwrap(), + JournalDamage::Corrupt => OpenOptions::new() + .append(true) + .open(path) + .unwrap() + .write_all(b"corrupt-complete-record\n") + .unwrap(), + JournalDamage::Gapped => fs::write( + path, + br#"{"sequence":99,"generation":0,"class":"source","kind":"delete","path":"README.md"} +"#, + ) + .unwrap(), + JournalDamage::ValidPrefix => fs::write(path, b"").unwrap(), + } + } + + #[test] + fn damaged_path_journal_never_loses_deleted_lower_path() { + for damage in [ + JournalDamage::Missing, + JournalDamage::Corrupt, + JournalDamage::Gapped, + JournalDamage::ValidPrefix, + ] { + let (_temp, db, root, upper) = fixture(); + let mut view = core(&db, &root, upper.clone()); + view.remove(VIEW_ROOT_INO, "README.md").unwrap(); + damage_mutation_journal(&upper, damage); + + let candidates = view.checkpoint_candidates().unwrap(); + assert!(!candidates.qualified); + assert!(candidates.upper_recovery_walks > 0); + assert!(candidates.paths.contains("README.md")); + } + } + + #[test] + fn damaged_path_journal_recovers_both_rename_sides() { + for damage in [ + JournalDamage::Missing, + JournalDamage::Corrupt, + JournalDamage::Gapped, + JournalDamage::ValidPrefix, + ] { + let (_temp, db, root, upper) = fixture(); + let mut view = core(&db, &root, upper.clone()); + view.rename(VIEW_ROOT_INO, "README.md", VIEW_ROOT_INO, "RENAMED.md") + .unwrap(); + damage_mutation_journal(&upper, damage); + + let candidates = view.checkpoint_candidates().unwrap(); + assert!(!candidates.qualified); + assert!(candidates.paths.contains("README.md")); + assert!(candidates.paths.contains("RENAMED.md")); + } + } + + #[test] + fn valid_prefix_truncation_recovers_written_upper_path() { + let (_temp, db, root, upper) = fixture(); + let mut view = core(&db, &root, upper.clone()); + let readme = view.lookup(VIEW_ROOT_INO, "README.md").unwrap(); + view.write(readme, 0, b"changed\n").unwrap(); + damage_mutation_journal(&upper, JournalDamage::ValidPrefix); + + let candidates = view.checkpoint_candidates().unwrap(); + assert!(!candidates.qualified); + assert!(candidates.upper_recovery_walks > 0); + assert!(candidates.paths.contains("README.md")); + } + + #[test] + fn recovery_fails_closed_when_both_independent_journals_are_lost() { + let (_temp, db, root, upper) = fixture(); + let mut view = core(&db, &root, upper.clone()); + view.remove(VIEW_ROOT_INO, "README.md").unwrap(); + let layout = ViewUpperLayout::from_source_upper(upper.clone()); + fs::remove_file(layout.journal_path()).unwrap(); + fs::remove_file(layout.whiteout_journal_path()).unwrap(); + + assert!(matches!( + view.checkpoint_candidates(), + Err(Error::ChangeLedgerReconcileRequired { .. }) + )); + } + + #[test] + fn qualified_view_replays_incremental_whiteouts_without_upper_walk() { + let (_temp, db, root, upper) = fixture(); + let mut view = core(&db, &root, upper.clone()); + view.remove(VIEW_ROOT_INO, "README.md").unwrap(); + drop(view); + + let reopened = core(&db, &root, upper); + assert!(reopened.is_whiteouted("README.md")); + let candidates = reopened.checkpoint_candidates().unwrap(); + assert!(candidates.qualified); + assert_eq!(candidates.upper_recovery_walks, 0); + assert!(candidates.paths.contains("README.md")); + } + #[test] fn view_core_conformance_rename_directory_preserves_mixed_contents_and_mode() { let (_temp, db, root, upper) = fixture(); @@ -1507,6 +2838,20 @@ mod tests { assert!(view.lookup(VIEW_ROOT_INO, "moved").is_ok()); } + #[test] + fn view_core_noop_setattr_does_not_create_checkpoint_candidate() { + let (_temp, db, root, upper) = fixture(); + let mut view = lazy_core(&db, &root, upper.clone()); + let readme = view.lookup(VIEW_ROOT_INO, "README.md").unwrap(); + + let attr = view.setattr(readme, None, None).unwrap(); + + assert_eq!(attr.ino, readme); + assert!(!upper.join("README.md").exists()); + let candidates = view.checkpoint_candidates().unwrap(); + assert!(!candidates.paths.contains("README.md")); + } + #[test] fn directory_rename_moves_files_from_every_classified_upper() { let (temp, db, root, _legacy_upper) = fixture(); diff --git a/trail/src/db/lane/workdir/view_journal.rs b/trail/src/db/lane/workdir/view_journal.rs index b3e0e36..10a0cce 100644 --- a/trail/src/db/lane/workdir/view_journal.rs +++ b/trail/src/db/lane/workdir/view_journal.rs @@ -1,8 +1,60 @@ use super::*; use serde::{Deserialize, Serialize}; -use std::collections::BTreeSet; -use std::fs::{self, OpenOptions}; -use std::io::Write; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; + +#[cfg(debug_assertions)] +std::thread_local! { + static FAIL_NEXT_VIEW_JOURNAL_SYNC: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +#[cfg(test)] +std::thread_local! { + static REPLACE_NEXT_VIEW_JOURNAL_AFTER_OPEN: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +#[cfg(debug_assertions)] +pub(crate) fn fail_next_view_journal_sync_for_current_thread() { + FAIL_NEXT_VIEW_JOURNAL_SYNC.with(|fail| fail.set(true)); +} + +#[cfg(debug_assertions)] +fn fail_view_journal_sync_if_requested() -> Result<()> { + if FAIL_NEXT_VIEW_JOURNAL_SYNC.with(|fail| fail.replace(false)) { + return Err(Error::Io(std::io::Error::new( + std::io::ErrorKind::Other, + "injected workspace view journal sync failure", + ))); + } + Ok(()) +} + +#[cfg(not(debug_assertions))] +fn fail_view_journal_sync_if_requested() -> Result<()> { + Ok(()) +} + +#[cfg(test)] +fn replace_next_view_journal_after_open_for_current_thread() { + REPLACE_NEXT_VIEW_JOURNAL_AFTER_OPEN.with(|replace| replace.set(true)); +} + +#[cfg(test)] +fn replace_view_journal_after_open_if_requested(path: &Path) -> std::io::Result<()> { + if REPLACE_NEXT_VIEW_JOURNAL_AFTER_OPEN.with(|replace| replace.replace(false)) { + let held = path.with_extension("aba-held"); + fs::rename(path, &held)?; + OpenOptions::new().create_new(true).write(true).open(path)?; + } + Ok(()) +} + +#[cfg(not(test))] +fn replace_view_journal_after_open_if_requested(_path: &Path) -> std::io::Result<()> { + Ok(()) +} #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] @@ -15,10 +67,18 @@ pub(crate) enum ViewMutationKind { Rename, } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case", tag = "operation", content = "path")] +pub(crate) enum ViewWhiteoutChange { + Insert(String), + Remove(String), + RemoveTree(String), +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub(crate) struct ViewMutationRecord { pub(crate) sequence: u64, - #[serde(default)] + pub(crate) generation: u64, pub(crate) class: ViewPathClass, pub(crate) kind: ViewMutationKind, pub(crate) path: String, @@ -26,25 +86,144 @@ pub(crate) struct ViewMutationRecord { pub(crate) destination: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) destination_class: Option, + pub(crate) whiteouts: Vec, + phase: ViewJournalPhase, + previous_hash: String, + record_hash: String, } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +struct ViewWhiteoutRecord { + sequence: u64, + generation: u64, + changes: Vec, + phase: ViewJournalPhase, + previous_hash: String, + record_hash: String, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum ViewJournalPhase { + Committed, + Intent, + Commit, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +struct ViewJournalState { + version: u16, + active_generation: u64, + base_sequence: u64, + mutation_base_hash: String, + whiteout_base_hash: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +struct ViewJournalTailAnchor { + version: u16, + generation: u64, + mutation_sequence: u64, + mutation_hash: String, + whiteout_sequence: u64, + whiteout_hash: String, +} + +const VIEW_JOURNAL_STATE_VERSION: u16 = 2; +const VIEW_JOURNAL_TAIL_VERSION: u16 = 1; +const VIEW_JOURNAL_GENESIS_HASH: &str = + "0000000000000000000000000000000000000000000000000000000000000000"; + /// Append-only dirty-path index for a workspace view. /// -/// The upper tree and whiteouts remain authoritative. A missing or truncated -/// journal tail is recoverable by scanning the upper; a complete corrupt line -/// is rejected because it cannot be attributed to an interrupted append. +/// A fully replayable journal is authoritative. Missing, truncated, corrupt, +/// or gapped input is explicitly unqualified so checkpointing reconciles the +/// upper tree instead of treating an incomplete path set as clean. pub(crate) struct ViewMutationJournal { + upperdir: PathBuf, path: PathBuf, + whiteout_path: PathBuf, next_sequence: u64, + base_sequence: u64, clean_sequence: u64, + generation: u64, dirty_paths: BTreeSet, dirty_source_paths: BTreeSet, + dirty_generated_paths: BTreeSet, + whiteouts: BTreeSet, + recovery_whiteouts: BTreeSet, + mutation_hash: String, + whiteout_hash: String, + whiteout_sequence: u64, + pending_mutations: BTreeMap, + pending_whiteouts: BTreeMap, + qualified: bool, + whiteouts_qualified: bool, +} + +pub(crate) type ViewIntentWriter = ViewMutationJournal; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ViewJournalCut { + pub(crate) sequence: u64, + pub(crate) generation: u64, + pub(crate) qualified: bool, + pub(crate) recovery_qualified: bool, +} + +#[derive(Debug, Serialize, Deserialize)] +struct ViewGenerationLeaseRecord { + generation: u64, + pid: u32, + process_start_token: String, +} + +pub(crate) struct ViewGenerationLease { + path: PathBuf, + generation: u64, +} + +impl ViewGenerationLease { + pub(crate) fn acquire(upperdir: &Path, generation: u64) -> Result { + static NEXT_LEASE: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + let layout = ViewUpperLayout::from_source_upper(upperdir.to_path_buf()); + let dir = layout.meta_dir.join("active-handles"); + fs::create_dir_all(&dir)?; + let id = NEXT_LEASE.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = dir.join(format!("{}-{id}.json", std::process::id())); + let record = ViewGenerationLeaseRecord { + generation, + pid: std::process::id(), + process_start_token: crate::db::util::current_process_start_token(), + }; + write_file_atomic(&path, &serde_json::to_vec(&record)?, false)?; + Ok(Self { path, generation }) + } + + pub(crate) fn advance(&mut self, upperdir: &Path, generation: u64) -> Result<()> { + if self.generation == generation { + return Ok(()); + } + let replacement = Self::acquire(upperdir, generation)?; + let old = std::mem::replace(self, replacement); + drop(old); + Ok(()) + } +} + +impl Drop for ViewGenerationLease { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } } #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct ViewCheckpointCandidates { pub(crate) journal_sequence: u64, pub(crate) paths: BTreeSet, + pub(crate) generated_paths: BTreeSet, + pub(crate) qualified: bool, + pub(crate) upper_recovery_walks: u64, } #[cfg(test)] @@ -54,27 +233,13 @@ pub(crate) fn recover_view_checkpoint_candidates( ) -> Result { let journal = ViewMutationJournal::open(upperdir)?; let mut paths = journal.dirty_source_paths().clone(); - for entry in walkdir::WalkDir::new(upperdir) { - let entry = entry.map_err(|err| Error::InvalidInput(err.to_string()))?; - if !entry.file_type().is_file() { - continue; - } - let path = normalize_relative_path( - &entry - .path() - .strip_prefix(upperdir) - .map_err(|err| Error::InvalidInput(err.to_string()))? - .to_string_lossy(), - )?; - if path == ".trail" || path.starts_with(".trail/") { - continue; - } - if classify_view_path(&path).checkpoints() { - paths.insert(path); - } - } - let layout = ViewUpperLayout::from_source_upper(upperdir.to_path_buf()); - let whiteouts = load_source_whiteouts(&layout)?; + let upper_recovery_walks = if journal.is_qualified() { + 0 + } else { + scan_source_upper(upperdir, &mut paths)? + }; + ensure_checkpoint_recovery_qualified(upperdir, &journal)?; + let whiteouts = journal.checkpoint_whiteouts().clone(); for whiteout in whiteouts { let whiteout = normalize_relative_path(&whiteout)?; if lower_files.contains_key(&whiteout) { @@ -91,6 +256,9 @@ pub(crate) fn recover_view_checkpoint_candidates( Ok(ViewCheckpointCandidates { journal_sequence: journal.last_sequence(), paths, + generated_paths: journal.dirty_generated_paths().clone(), + qualified: journal.is_qualified(), + upper_recovery_walks, }) } @@ -101,27 +269,13 @@ pub(crate) fn recover_view_checkpoint_candidates_for_root( ) -> Result { let journal = ViewMutationJournal::open(upperdir)?; let mut paths = journal.dirty_source_paths().clone(); - for entry in walkdir::WalkDir::new(upperdir) { - let entry = entry.map_err(|err| Error::InvalidInput(err.to_string()))?; - if !entry.file_type().is_file() { - continue; - } - let path = normalize_relative_path( - &entry - .path() - .strip_prefix(upperdir) - .map_err(|err| Error::InvalidInput(err.to_string()))? - .to_string_lossy(), - )?; - if path == ".trail" || path.starts_with(".trail/") { - continue; - } - if classify_view_path(&path).checkpoints() { - paths.insert(path); - } - } - let layout = ViewUpperLayout::from_source_upper(upperdir.to_path_buf()); - let whiteouts = load_source_whiteouts(&layout)?; + let upper_recovery_walks = if journal.is_qualified() { + 0 + } else { + scan_source_upper(upperdir, &mut paths)? + }; + ensure_checkpoint_recovery_qualified(upperdir, &journal)?; + let whiteouts = journal.checkpoint_whiteouts().clone(); for whiteout in whiteouts { let whiteout = normalize_relative_path(&whiteout)?; paths.extend( @@ -132,70 +286,333 @@ pub(crate) fn recover_view_checkpoint_candidates_for_root( Ok(ViewCheckpointCandidates { journal_sequence: journal.last_sequence(), paths, + generated_paths: journal.dirty_generated_paths().clone(), + qualified: journal.is_qualified(), + upper_recovery_walks, + }) +} + +fn ensure_checkpoint_recovery_qualified( + upperdir: &Path, + journal: &ViewMutationJournal, +) -> Result<()> { + if journal.recovery_is_qualified() { + return Ok(()); + } + Err(Error::ChangeLedgerReconcileRequired { + scope: upperdir.display().to_string(), + state: "unqualified_view_journal".into(), + reason: "both the changed-path journal and independent whiteout journal are missing, corrupt, or gapped; refusing a potentially false-clean checkpoint".into(), + command: "trail ledger reconcile".into(), }) } impl ViewMutationJournal { + /// Establish the durable empty journal pair for a newly-created view. + /// The state file is written last and is the qualification anchor: if it + /// survives while either stream is missing, recovery knows evidence was + /// lost and fails closed rather than mistaking the view for pristine. + pub(crate) fn initialize_storage(upperdir: &Path) -> Result<()> { + let layout = ViewUpperLayout::from_source_upper(upperdir.to_path_buf()); + layout.ensure()?; + let state_path = layout.journal_state_path(); + if state_path.exists() { + return Ok(()); + } + if layout.journal_path().exists() || layout.whiteout_journal_path().exists() { + return Err(Error::Corrupt(format!( + "workspace view journal state is missing under `{}`; reinitialize this workspace", + layout.meta_dir.display() + ))); + } + for path in [layout.journal_path(), layout.whiteout_journal_path()] { + let file = OpenOptions::new().create_new(true).write(true).open(path)?; + file.sync_all()?; + } + sync_directory_strict(&layout.meta_dir)?; + write_tail_anchor( + &layout, + &ViewJournalTailAnchor { + version: VIEW_JOURNAL_TAIL_VERSION, + generation: 0, + mutation_sequence: 0, + mutation_hash: VIEW_JOURNAL_GENESIS_HASH.to_string(), + whiteout_sequence: 0, + whiteout_hash: VIEW_JOURNAL_GENESIS_HASH.to_string(), + }, + )?; + write_file_atomic( + &state_path, + &serde_json::to_vec(&ViewJournalState { + version: VIEW_JOURNAL_STATE_VERSION, + active_generation: 0, + base_sequence: 0, + mutation_base_hash: VIEW_JOURNAL_GENESIS_HASH.to_string(), + whiteout_base_hash: VIEW_JOURNAL_GENESIS_HASH.to_string(), + })?, + true, + )?; + sync_directory_strict(&layout.meta_dir) + } + pub(crate) fn open(upperdir: &Path) -> Result { - let path = ViewUpperLayout::from_source_upper(upperdir.to_path_buf()).journal_path(); + let layout = ViewUpperLayout::from_source_upper(upperdir.to_path_buf()); + let state = read_journal_state(&layout); + let state_qualified = state.is_ok(); + let tail_anchor = read_tail_anchor(&layout); + let tail_qualified = tail_anchor.is_ok(); + let state = state.unwrap_or(ViewJournalState { + version: VIEW_JOURNAL_STATE_VERSION, + active_generation: 0, + base_sequence: 0, + mutation_base_hash: VIEW_JOURNAL_GENESIS_HASH.to_string(), + whiteout_base_hash: VIEW_JOURNAL_GENESIS_HASH.to_string(), + }); + let path = mutation_journal_path_for_generation(&layout, state.active_generation); + let whiteout_path = whiteout_journal_path_for_generation(&layout, state.active_generation); let mut journal = Self { + upperdir: upperdir.to_path_buf(), path, - next_sequence: 1, - clean_sequence: clean_checkpoint_sequence(upperdir)?, + whiteout_path, + next_sequence: state.base_sequence.saturating_add(1), + base_sequence: state.base_sequence, + clean_sequence: state.base_sequence, + generation: state.active_generation, dirty_paths: BTreeSet::new(), dirty_source_paths: BTreeSet::new(), + dirty_generated_paths: BTreeSet::new(), + whiteouts: BTreeSet::new(), + recovery_whiteouts: BTreeSet::new(), + mutation_hash: state.mutation_base_hash, + whiteout_hash: state.whiteout_base_hash, + whiteout_sequence: state.base_sequence, + pending_mutations: BTreeMap::new(), + pending_whiteouts: BTreeMap::new(), + qualified: state_qualified && tail_qualified, + whiteouts_qualified: state_qualified && tail_qualified, }; journal.replay()?; + journal.replay_whiteouts()?; + match tail_anchor { + Ok(anchor) + if anchor.version == VIEW_JOURNAL_TAIL_VERSION + && anchor.generation == journal.generation => + { + if anchor.mutation_sequence != journal.last_sequence() + || anchor.mutation_hash != journal.mutation_hash + { + journal.qualified = false; + } + if anchor.whiteout_sequence != journal.whiteout_sequence + || anchor.whiteout_hash != journal.whiteout_hash + { + journal.whiteouts_qualified = false; + } + } + _ => { + journal.qualified = false; + journal.whiteouts_qualified = false; + } + } Ok(journal) } + #[cfg(test)] pub(crate) fn append( &mut self, kind: ViewMutationKind, path: &str, destination: Option<&str>, ) -> Result { - let path = normalize_relative_path(path)?; - let destination = destination.map(normalize_relative_path).transpose()?; - let class = classify_view_path(&path); - let destination_class = destination.as_deref().map(classify_view_path); + let class = classify_view_path(path); + let destination_class = destination.map(classify_view_path); + self.append_classified( + kind, + path.to_string(), + class, + destination.map(str::to_string), + destination_class, + ) + } + + pub(crate) fn append_classified( + &mut self, + kind: ViewMutationKind, + path: String, + class: ViewPathClass, + destination: Option, + destination_class: Option, + ) -> Result { + self.append_classified_with_whiteouts( + kind, + path, + class, + destination, + destination_class, + Vec::new(), + ) + } + + pub(crate) fn append_classified_with_whiteouts( + &mut self, + kind: ViewMutationKind, + path: String, + class: ViewPathClass, + destination: Option, + destination_class: Option, + whiteouts: Vec, + ) -> Result { + let path = normalize_relative_path(&path)?; + let destination = destination + .as_deref() + .map(normalize_relative_path) + .transpose()?; if matches!(kind, ViewMutationKind::Write | ViewMutationKind::Metadata) && self.dirty_paths.contains(&path) && destination.is_none() + && whiteouts.is_empty() { return Ok(self.next_sequence.saturating_sub(1)); } - let record = ViewMutationRecord { + let phase = if whiteouts.is_empty() { + ViewJournalPhase::Committed + } else { + ViewJournalPhase::Intent + }; + let mut record = ViewMutationRecord { sequence: self.next_sequence, + generation: self.generation, class, kind, path, destination, destination_class, + whiteouts: whiteouts + .into_iter() + .map(normalize_whiteout_change) + .collect::>>()?, + phase, + previous_hash: self.mutation_hash.clone(), + record_hash: String::new(), }; - let mut encoded = serde_json::to_vec(&record)?; - encoded.push(b'\n'); - if let Some(parent) = self.path.parent() { - fs::create_dir_all(parent)?; + record.record_hash = mutation_record_hash(&record)?; + fail_view_journal_sync_if_requested()?; + append_authenticated_record(&self.path, &record)?; + self.mutation_hash = record.record_hash.clone(); + self.apply_dirty(&record); + if phase == ViewJournalPhase::Committed { + self.apply_whiteouts(&record); + } else { + self.pending_mutations + .insert(record.sequence, record.clone()); } - let mut file = OpenOptions::new() - .create(true) - .append(true) - .open(&self.path)?; - file.write_all(&encoded)?; - file.sync_data()?; - self.apply(&record); self.next_sequence += 1; + self.persist_tail_anchor()?; + + let mut whiteout_record = ViewWhiteoutRecord { + sequence: record.sequence, + generation: record.generation, + changes: record.whiteouts.clone(), + phase, + previous_hash: self.whiteout_hash.clone(), + record_hash: String::new(), + }; + whiteout_record.record_hash = whiteout_record_hash(&whiteout_record)?; + if let Err(error) = append_authenticated_record(&self.whiteout_path, &whiteout_record) { + self.whiteouts_qualified = false; + return Err(error); + } + self.whiteout_hash = whiteout_record.record_hash.clone(); + self.whiteout_sequence = whiteout_record.sequence; + if phase == ViewJournalPhase::Committed { + apply_whiteout_changes(&mut self.recovery_whiteouts, &record.whiteouts); + } else { + self.pending_whiteouts + .insert(record.sequence, whiteout_record); + } + self.persist_tail_anchor()?; Ok(record.sequence) } - pub(crate) fn observe_checkpoint(&mut self, sequence: u64) { - if sequence <= self.clean_sequence { - return; + /// Publish the semantic whiteout half of a durable filesystem mutation. + /// Intent records remain useful dirty-path evidence, but replay never + /// applies their whiteouts until this matching commit is durable. + pub(crate) fn commit_whiteouts(&mut self, sequence: u64) -> Result<()> { + let intent = self + .pending_mutations + .get(&sequence) + .cloned() + .ok_or_else(|| { + Error::Corrupt(format!( + "workspace view mutation {sequence} has no pending whiteout intent" + )) + })?; + let whiteout_intent = self + .pending_whiteouts + .get(&sequence) + .cloned() + .ok_or_else(|| { + Error::Corrupt(format!( + "workspace view mutation {sequence} has no independent whiteout intent" + )) + })?; + + let mut commit = intent.clone(); + commit.phase = ViewJournalPhase::Commit; + commit.previous_hash = self.mutation_hash.clone(); + commit.record_hash.clear(); + commit.record_hash = mutation_record_hash(&commit)?; + append_authenticated_record(&self.path, &commit)?; + self.mutation_hash = commit.record_hash.clone(); + self.persist_tail_anchor()?; + self.apply_whiteouts(&intent); + self.pending_mutations.remove(&sequence); + + let mut whiteout_commit = whiteout_intent.clone(); + whiteout_commit.phase = ViewJournalPhase::Commit; + whiteout_commit.previous_hash = self.whiteout_hash.clone(); + whiteout_commit.record_hash.clear(); + whiteout_commit.record_hash = whiteout_record_hash(&whiteout_commit)?; + if let Err(error) = append_authenticated_record(&self.whiteout_path, &whiteout_commit) { + self.whiteouts_qualified = false; + return Err(error); + } + self.whiteout_hash = whiteout_commit.record_hash; + self.whiteout_sequence = whiteout_commit.sequence; + self.persist_tail_anchor()?; + apply_whiteout_changes(&mut self.recovery_whiteouts, &whiteout_intent.changes); + self.pending_whiteouts.remove(&sequence); + Ok(()) + } + + fn persist_tail_anchor(&self) -> Result<()> { + let layout = ViewUpperLayout::from_source_upper(self.upperdir.clone()); + write_tail_anchor( + &layout, + &ViewJournalTailAnchor { + version: VIEW_JOURNAL_TAIL_VERSION, + generation: self.generation, + mutation_sequence: self.last_sequence(), + mutation_hash: self.mutation_hash.clone(), + whiteout_sequence: self.whiteout_sequence, + whiteout_hash: self.whiteout_hash.clone(), + }, + ) + } + + pub(crate) fn observe_checkpoint(&mut self, sequence: u64, generation: u64) -> Result<()> { + if sequence == self.clean_sequence && generation == self.generation { + return Ok(()); + } + let reloaded = Self::open(&self.upperdir)?; + if reloaded.base_sequence != sequence || reloaded.generation != generation { + return Err(Error::Corrupt(format!( + "workspace checkpoint cut ({sequence}, {generation}) does not match journal state ({}, {})", + reloaded.base_sequence, reloaded.generation, + ))); } - self.clean_sequence = sequence; - self.dirty_paths.clear(); - self.dirty_source_paths.clear(); + *self = reloaded; + Ok(()) } #[cfg(test)] @@ -207,101 +624,647 @@ impl ViewMutationJournal { &self.dirty_source_paths } + pub(crate) fn dirty_generated_paths(&self) -> &BTreeSet { + &self.dirty_generated_paths + } + + pub(crate) fn whiteouts(&self) -> &BTreeSet { + &self.whiteouts + } + + fn checkpoint_whiteouts(&self) -> &BTreeSet { + if self.qualified { + &self.whiteouts + } else { + &self.recovery_whiteouts + } + } + + pub(crate) fn recovery_is_qualified(&self) -> bool { + self.qualified || self.whiteouts_qualified + } + pub(crate) fn last_sequence(&self) -> u64 { self.next_sequence.saturating_sub(1) } + pub(crate) fn cut(&self) -> ViewJournalCut { + ViewJournalCut { + sequence: self.last_sequence(), + generation: self.generation, + qualified: self.qualified, + recovery_qualified: self.recovery_is_qualified(), + } + } + + pub(crate) fn is_qualified(&self) -> bool { + self.qualified + } + + pub(crate) fn generation(&self) -> u64 { + self.generation + } + + pub(crate) fn rotate_after_checkpoint( + upperdir: &Path, + sequence: u64, + next_generation: u64, + ) -> Result<()> { + let layout = ViewUpperLayout::from_source_upper(upperdir.to_path_buf()); + let state = read_journal_state(&layout)?; + if state.base_sequence == sequence && state.active_generation == next_generation { + return Ok(()); + } + let journal = Self::open(upperdir)?; + if journal.last_sequence() != sequence || next_generation <= state.active_generation { + return Err(Error::Corrupt(format!( + "cannot rotate workspace journal generation {} at sequence {}", + state.active_generation, sequence + ))); + } + for path in [ + mutation_journal_path_for_generation(&layout, next_generation), + whiteout_journal_path_for_generation(&layout, next_generation), + ] { + match OpenOptions::new().create_new(true).write(true).open(&path) { + Ok(file) => file.sync_all()?, + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { + let metadata = fs::symlink_metadata(&path)?; + if !metadata.file_type().is_file() || metadata.len() != 0 { + return Err(Error::Corrupt(format!( + "workspace journal generation path `{}` is unsafe or non-empty", + path.display() + ))); + } + } + Err(err) => return Err(Error::Io(err)), + } + } + sync_directory_strict(&layout.meta_dir)?; + write_file_atomic( + &layout.journal_state_path(), + &serde_json::to_vec(&ViewJournalState { + version: VIEW_JOURNAL_STATE_VERSION, + active_generation: next_generation, + base_sequence: sequence, + mutation_base_hash: journal.mutation_hash.clone(), + whiteout_base_hash: journal.whiteout_hash.clone(), + })?, + true, + )?; + write_tail_anchor( + &layout, + &ViewJournalTailAnchor { + version: VIEW_JOURNAL_TAIL_VERSION, + generation: next_generation, + mutation_sequence: sequence, + mutation_hash: journal.mutation_hash.clone(), + whiteout_sequence: sequence, + whiteout_hash: journal.whiteout_hash.clone(), + }, + )?; + sync_directory_strict(&layout.meta_dir)?; + compact_inactive_generations(&layout, next_generation) + } + fn replay(&mut self) -> Result<()> { - let bytes = match fs::read(&self.path) { + let bytes = match read_regular_no_follow(&self.path) { Ok(bytes) => bytes, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + self.qualified = false; + return Ok(()); + } Err(err) => return Err(Error::Io(err)), }; - let mut previous = 0; + let mut previous = self.base_sequence; + let mut previous_hash = self.mutation_hash.clone(); for line in bytes.split_inclusive(|byte| *byte == b'\n') { if line.last() != Some(&b'\n') { + self.qualified = false; break; } let payload = &line[..line.len() - 1]; if payload.is_empty() { continue; } - let mut record: ViewMutationRecord = - serde_json::from_slice(payload).map_err(|err| { - Error::Corrupt(format!( - "workspace view mutation journal `{}` has an invalid complete record: {err}", - self.path.display() - )) - })?; - record.path = normalize_relative_path(&record.path)?; - record.destination = record + let mut record: ViewMutationRecord = match serde_json::from_slice(payload) { + Ok(record) => record, + Err(_) => { + self.qualified = false; + break; + } + }; + if record.generation != self.generation + || record.previous_hash != previous_hash + || !mutation_record_hash(&record).is_ok_and(|hash| hash == record.record_hash) + { + self.qualified = false; + break; + } + record.path = match normalize_relative_path(&record.path) { + Ok(path) => path, + Err(_) => { + self.qualified = false; + break; + } + }; + let destination = record .destination .as_deref() .map(normalize_relative_path) - .transpose()?; - if record.sequence <= previous { - return Err(Error::Corrupt(format!( - "workspace view mutation journal `{}` is not strictly ordered at sequence {}", - self.path.display(), - record.sequence - ))); + .transpose(); + record.destination = match destination { + Ok(destination) => destination, + Err(_) => { + self.qualified = false; + break; + } + }; + for change in &mut record.whiteouts { + *change = match normalize_whiteout_change(change.clone()) { + Ok(change) => change, + Err(_) => { + self.qualified = false; + break; + } + }; + } + if !self.qualified { + break; } - previous = record.sequence; - if record.sequence > self.clean_sequence { - self.apply(&record); + match record.phase { + ViewJournalPhase::Committed => { + if !record.whiteouts.is_empty() || record.sequence != previous.saturating_add(1) + { + self.qualified = false; + break; + } + previous = record.sequence; + self.apply(&record); + } + ViewJournalPhase::Intent => { + if record.whiteouts.is_empty() || record.sequence != previous.saturating_add(1) + { + self.qualified = false; + break; + } + previous = record.sequence; + self.apply_dirty(&record); + self.pending_mutations + .insert(record.sequence, record.clone()); + } + ViewJournalPhase::Commit => { + let Some(intent) = self.pending_mutations.remove(&record.sequence) else { + self.qualified = false; + break; + }; + if !same_mutation_payload(&intent, &record) { + self.qualified = false; + break; + } + self.apply_whiteouts(&intent); + } } + previous_hash = record.record_hash; } self.next_sequence = previous.saturating_add(1).max(1); + self.mutation_hash = previous_hash; + Ok(()) + } + + fn replay_whiteouts(&mut self) -> Result<()> { + let bytes = match read_regular_no_follow(&self.whiteout_path) { + Ok(bytes) => bytes, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + self.whiteouts_qualified = false; + return Ok(()); + } + Err(err) => return Err(Error::Io(err)), + }; + let mut previous = self.base_sequence; + let mut previous_hash = self.whiteout_hash.clone(); + for line in bytes.split_inclusive(|byte| *byte == b'\n') { + if line.last() != Some(&b'\n') { + self.whiteouts_qualified = false; + break; + } + let payload = &line[..line.len() - 1]; + if payload.is_empty() { + continue; + } + let mut record: ViewWhiteoutRecord = match serde_json::from_slice(payload) { + Ok(record) => record, + Err(_) => { + self.whiteouts_qualified = false; + break; + } + }; + if record.generation != self.generation + || record.previous_hash != previous_hash + || !whiteout_record_hash(&record).is_ok_and(|hash| hash == record.record_hash) + { + self.whiteouts_qualified = false; + break; + } + for change in &mut record.changes { + *change = match normalize_whiteout_change(change.clone()) { + Ok(change) => change, + Err(_) => { + self.whiteouts_qualified = false; + break; + } + }; + } + if !self.whiteouts_qualified { + break; + } + match record.phase { + ViewJournalPhase::Committed => { + if !record.changes.is_empty() || record.sequence != previous.saturating_add(1) { + self.whiteouts_qualified = false; + break; + } + previous = record.sequence; + } + ViewJournalPhase::Intent => { + if record.changes.is_empty() || record.sequence != previous.saturating_add(1) { + self.whiteouts_qualified = false; + break; + } + previous = record.sequence; + self.pending_whiteouts + .insert(record.sequence, record.clone()); + } + ViewJournalPhase::Commit => { + let Some(intent) = self.pending_whiteouts.remove(&record.sequence) else { + self.whiteouts_qualified = false; + break; + }; + if !same_whiteout_payload(&intent, &record) { + self.whiteouts_qualified = false; + break; + } + apply_whiteout_changes(&mut self.recovery_whiteouts, &intent.changes); + } + } + previous_hash = record.record_hash; + } + self.whiteout_hash = previous_hash; + self.whiteout_sequence = previous; + if !self.pending_whiteouts.is_empty() { + // An intent without a commit is deliberately not independent + // whiteout authority. The mutation stream still supplies safe + // dirty-path evidence and can qualify recovery on its own. + self.whiteouts_qualified = false; + } + if self.whiteouts_qualified && previous != self.last_sequence() { + // Each mutation has a matching independent whiteout record. The + // longer complete stream is evidence that the shorter stream lost + // a valid suffix, even when that suffix ended on a record boundary. + if previous > self.last_sequence() { + self.qualified = false; + } else { + self.whiteouts_qualified = false; + } + } Ok(()) } fn apply(&mut self, record: &ViewMutationRecord) { + self.apply_whiteouts(record); + self.apply_dirty(record); + } + + fn apply_dirty(&mut self, record: &ViewMutationRecord) { self.dirty_paths.insert(record.path.clone()); if record.class.checkpoints() { self.dirty_source_paths.insert(record.path.clone()); + } else if matches!( + record.class, + ViewPathClass::Dependency | ViewPathClass::Generated + ) { + self.dirty_generated_paths.insert(record.path.clone()); } if let Some(destination) = &record.destination { self.dirty_paths.insert(destination.clone()); - if record + let destination_class = record .destination_class - .unwrap_or_else(|| classify_view_path(destination)) - .checkpoints() - { + .unwrap_or_else(|| classify_view_path(destination)); + if destination_class.checkpoints() { self.dirty_source_paths.insert(destination.clone()); + } else if matches!( + destination_class, + ViewPathClass::Dependency | ViewPathClass::Generated + ) { + self.dirty_generated_paths.insert(destination.clone()); + } + } + } + + fn apply_whiteouts(&mut self, record: &ViewMutationRecord) { + for change in &record.whiteouts { + match change { + ViewWhiteoutChange::Insert(path) => { + self.whiteouts.insert(path.clone()); + } + ViewWhiteoutChange::Remove(path) => { + self.whiteouts.remove(path); + } + ViewWhiteoutChange::RemoveTree(path) => { + let prefix = format!("{path}/"); + self.whiteouts + .retain(|item| item != path && !item.starts_with(&prefix)); + } } } } } -fn clean_checkpoint_sequence(upperdir: &Path) -> Result { - let layout = ViewUpperLayout::from_source_upper(upperdir.to_path_buf()); - let path = layout.meta_dir.join("clean-checkpoint.json"); - let bytes = match fs::read(&path) { - Ok(bytes) => bytes, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(0), - Err(err) => return Err(Error::Io(err)), - }; - let marker: serde_json::Value = serde_json::from_slice(&bytes)?; - marker["journal_sequence"].as_u64().ok_or_else(|| { - Error::Corrupt(format!( - "workspace checkpoint marker `{}` has no journal sequence", - path.display() - )) +fn mutation_record_hash(record: &ViewMutationRecord) -> Result { + let mut authenticated = record.clone(); + authenticated.record_hash.clear(); + Ok(hex::encode(Sha256::digest(serde_json::to_vec( + &authenticated, + )?))) +} + +fn whiteout_record_hash(record: &ViewWhiteoutRecord) -> Result { + let mut authenticated = record.clone(); + authenticated.record_hash.clear(); + Ok(hex::encode(Sha256::digest(serde_json::to_vec( + &authenticated, + )?))) +} + +fn same_mutation_payload(intent: &ViewMutationRecord, commit: &ViewMutationRecord) -> bool { + intent.sequence == commit.sequence + && intent.generation == commit.generation + && intent.class == commit.class + && intent.kind == commit.kind + && intent.path == commit.path + && intent.destination == commit.destination + && intent.destination_class == commit.destination_class + && intent.whiteouts == commit.whiteouts + && intent.phase == ViewJournalPhase::Intent + && commit.phase == ViewJournalPhase::Commit +} + +fn same_whiteout_payload(intent: &ViewWhiteoutRecord, commit: &ViewWhiteoutRecord) -> bool { + intent.sequence == commit.sequence + && intent.generation == commit.generation + && intent.changes == commit.changes + && intent.phase == ViewJournalPhase::Intent + && commit.phase == ViewJournalPhase::Commit +} + +fn append_authenticated_record(path: &Path, record: &T) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + if !fs::symlink_metadata(parent)?.file_type().is_dir() { + return Err(Error::Corrupt(format!( + "workspace journal parent `{}` is not a real directory", + parent.display() + ))); + } + } + let mut encoded = serde_json::to_vec(record)?; + encoded.push(b'\n'); + let mut file = open_regular_no_follow(path, true, true)?; + replace_view_journal_after_open_if_requested(path)?; + file.write_all(&encoded)?; + file.sync_all()?; + validate_regular_file_identity(path, &file)?; + if let Some(parent) = path.parent() { + sync_directory_strict(parent)?; + } + validate_regular_file_identity(path, &file)?; + Ok(()) +} + +fn read_regular_no_follow(path: &Path) -> std::io::Result> { + let mut file = open_regular_no_follow(path, false, false)?; + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + Ok(bytes) +} + +fn open_regular_no_follow(path: &Path, append: bool, create: bool) -> std::io::Result { + let mut options = OpenOptions::new(); + options + .read(!append) + .write(append) + .append(append) + .create(create); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC); + options.mode(0o600); + } + let file = options.open(path)?; + let metadata = file.metadata()?; + if !metadata.is_file() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "workspace journal is not a regular file", + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if metadata.nlink() != 1 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "workspace journal has an unsafe hard-link count", + )); + } + } + Ok(file) +} + +fn validate_regular_file_identity(path: &Path, file: &File) -> std::io::Result<()> { + let path_metadata = fs::symlink_metadata(path)?; + let held = file.metadata()?; + if !path_metadata.file_type().is_file() || !held.is_file() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "workspace journal pathname was replaced", + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if held.nlink() != 1 + || path_metadata.nlink() != 1 + || held.dev() != path_metadata.dev() + || held.ino() != path_metadata.ino() + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "workspace journal pathname changed while appending", + )); + } + } + Ok(()) +} + +fn journal_tail_anchor_path(layout: &ViewUpperLayout) -> PathBuf { + layout.meta_dir.join("journal-tail.json") +} + +fn read_tail_anchor(layout: &ViewUpperLayout) -> Result { + let anchor: ViewJournalTailAnchor = + serde_json::from_slice(&read_regular_no_follow(&journal_tail_anchor_path(layout))?)?; + if anchor.version != VIEW_JOURNAL_TAIL_VERSION { + return Err(Error::Corrupt(format!( + "unsupported workspace view journal tail version {}", + anchor.version + ))); + } + Ok(anchor) +} + +fn write_tail_anchor(layout: &ViewUpperLayout, anchor: &ViewJournalTailAnchor) -> Result<()> { + write_file_atomic( + &journal_tail_anchor_path(layout), + &serde_json::to_vec(anchor)?, + true, + )?; + sync_directory_strict(&layout.meta_dir) +} + +fn read_journal_state(layout: &ViewUpperLayout) -> Result { + let state: ViewJournalState = + serde_json::from_slice(&read_regular_no_follow(&layout.journal_state_path())?)?; + if state.version != VIEW_JOURNAL_STATE_VERSION { + return Err(Error::Corrupt(format!( + "unsupported workspace view journal state version {}", + state.version + ))); + } + Ok(state) +} + +fn mutation_journal_path_for_generation(layout: &ViewUpperLayout, generation: u64) -> PathBuf { + if generation == 0 { + layout.journal_path() + } else { + layout + .meta_dir + .join(format!("mutation-journal.g{generation}.jsonl")) + } +} + +fn whiteout_journal_path_for_generation(layout: &ViewUpperLayout, generation: u64) -> PathBuf { + if generation == 0 { + layout.whiteout_journal_path() + } else { + layout + .meta_dir + .join(format!("whiteout-journal.g{generation}.jsonl")) + } +} + +fn compact_inactive_generations(layout: &ViewUpperLayout, active_generation: u64) -> Result<()> { + let leases_dir = layout.meta_dir.join("active-handles"); + let mut retained = BTreeSet::new(); + if let Ok(entries) = fs::read_dir(&leases_dir) { + for entry in entries { + let entry = entry?; + let record = fs::read(entry.path()) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + match record { + Some(record) + if crate::db::util::process_matches_start_token( + record.pid, + &record.process_start_token, + ) => + { + retained.insert(record.generation); + } + _ => { + let _ = fs::remove_file(entry.path()); + } + } + } + } + // Retain one prior generation for post-crash inspection. Everything + // older is bounded unless an active adapter still owns that generation. + retained.insert(active_generation); + retained.insert(active_generation.saturating_sub(1)); + for generation in 0..active_generation.saturating_sub(1) { + if retained.contains(&generation) { + continue; + } + for path in [ + mutation_journal_path_for_generation(layout, generation), + whiteout_journal_path_for_generation(layout, generation), + ] { + match fs::remove_file(path) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(Error::Io(err)), + } + } + } + sync_directory_strict(&layout.meta_dir) +} + +fn apply_whiteout_changes(whiteouts: &mut BTreeSet, changes: &[ViewWhiteoutChange]) { + for change in changes { + match change { + ViewWhiteoutChange::Insert(path) => { + whiteouts.insert(path.clone()); + } + ViewWhiteoutChange::Remove(path) => { + whiteouts.remove(path); + } + ViewWhiteoutChange::RemoveTree(path) => { + let prefix = format!("{path}/"); + whiteouts.retain(|item| item != path && !item.starts_with(&prefix)); + } + } + } +} + +fn normalize_whiteout_change(change: ViewWhiteoutChange) -> Result { + Ok(match change { + ViewWhiteoutChange::Insert(path) => { + ViewWhiteoutChange::Insert(normalize_relative_path(&path)?) + } + ViewWhiteoutChange::Remove(path) => { + ViewWhiteoutChange::Remove(normalize_relative_path(&path)?) + } + ViewWhiteoutChange::RemoveTree(path) => { + ViewWhiteoutChange::RemoveTree(normalize_relative_path(&path)?) + } }) } -fn load_source_whiteouts(layout: &ViewUpperLayout) -> Result> { - for path in [ - layout.whiteouts_path(ViewPathClass::Source), - layout.legacy_whiteouts_path(), - ] { - match fs::read(path) { - Ok(bytes) => return serde_json::from_slice(&bytes).map_err(Error::from), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} - Err(err) => return Err(Error::Io(err)), +fn scan_source_upper(upperdir: &Path, paths: &mut BTreeSet) -> Result { + let mut walks = 0_u64; + for entry in walkdir::WalkDir::new(upperdir) { + let entry = entry.map_err(|err| Error::InvalidInput(err.to_string()))?; + walks = walks.saturating_add(1); + if !entry.file_type().is_file() { + continue; + } + let path = normalize_relative_path( + &entry + .path() + .strip_prefix(upperdir) + .map_err(|err| Error::InvalidInput(err.to_string()))? + .to_string_lossy(), + )?; + if path == ".trail" || path.starts_with(".trail/") { + continue; + } + if classify_view_path(&path).checkpoints() { + paths.insert(path); } } - Ok(Vec::new()) + Ok(walks) } #[cfg(test)] @@ -312,6 +1275,7 @@ mod tests { fn view_mutation_journal_replays_paths_and_ignores_truncated_tail() { let temp = tempfile::tempdir().unwrap(); let upper = temp.path().join("upper"); + ViewMutationJournal::initialize_storage(&upper).unwrap(); let mut journal = ViewMutationJournal::open(&upper).unwrap(); journal .append(ViewMutationKind::Write, "README.md", None) @@ -329,6 +1293,7 @@ mod tests { let replayed = ViewMutationJournal::open(&upper).unwrap(); assert_eq!(replayed.last_sequence(), 2); + assert!(!replayed.is_qualified()); assert_eq!( replayed.dirty_paths(), &BTreeSet::from([ @@ -340,16 +1305,245 @@ mod tests { } #[test] - fn view_mutation_journal_rejects_corrupt_complete_record() { + fn view_mutation_journal_marks_corrupt_complete_record_unqualified() { let temp = tempfile::tempdir().unwrap(); let upper = temp.path().join("upper"); + ViewMutationJournal::initialize_storage(&upper).unwrap(); let path = ViewUpperLayout::from_source_upper(upper.clone()).journal_path(); fs::create_dir_all(path.parent().unwrap()).unwrap(); fs::write(&path, b"not-json\n").unwrap(); - assert!(matches!( - ViewMutationJournal::open(&upper), - Err(Error::Corrupt(_)) - )); + let journal = ViewMutationJournal::open(&upper).unwrap(); + assert!(!journal.is_qualified()); + assert_eq!(journal.last_sequence(), 0); + } + + #[test] + fn authenticated_record_rejects_valid_json_path_and_generation_tampering() { + let temp = tempfile::tempdir().unwrap(); + let upper = temp.path().join("upper"); + ViewMutationJournal::initialize_storage(&upper).unwrap(); + let mut journal = ViewMutationJournal::open(&upper).unwrap(); + journal + .append(ViewMutationKind::Write, "changed.txt", None) + .unwrap(); + let path = ViewUpperLayout::from_source_upper(upper.clone()).journal_path(); + let mut record: serde_json::Value = serde_json::from_slice( + fs::read(&path) + .unwrap() + .split(|byte| *byte == b'\n') + .next() + .unwrap(), + ) + .unwrap(); + record["path"] = serde_json::Value::String("innocent.txt".into()); + record["generation"] = serde_json::Value::from(99_u64); + let mut bytes = serde_json::to_vec(&record).unwrap(); + bytes.push(b'\n'); + fs::write(path, bytes).unwrap(); + + let replayed = ViewMutationJournal::open(&upper).unwrap(); + assert!(!replayed.is_qualified()); + assert!(!replayed.dirty_paths().contains("innocent.txt")); + } + + #[test] + fn tail_anchor_rejects_correlated_valid_boundary_truncation() { + let temp = tempfile::tempdir().unwrap(); + let upper = temp.path().join("upper"); + ViewMutationJournal::initialize_storage(&upper).unwrap(); + let mut journal = ViewMutationJournal::open(&upper).unwrap(); + journal + .append(ViewMutationKind::Write, "changed.txt", None) + .unwrap(); + drop(journal); + + let layout = ViewUpperLayout::from_source_upper(upper.clone()); + fs::write(layout.journal_path(), b"").unwrap(); + fs::write(layout.whiteout_journal_path(), b"").unwrap(); + + let replayed = ViewMutationJournal::open(&upper).unwrap(); + assert!(!replayed.is_qualified()); + assert!(!replayed.recovery_is_qualified()); + assert!(recover_view_checkpoint_candidates(&upper, &BTreeMap::new()).is_err()); + } + + #[cfg(unix)] + #[test] + fn journal_append_rejects_post_open_pathname_aba() { + let temp = tempfile::tempdir().unwrap(); + let upper = temp.path().join("upper"); + ViewMutationJournal::initialize_storage(&upper).unwrap(); + let mut journal = ViewMutationJournal::open(&upper).unwrap(); + replace_next_view_journal_after_open_for_current_thread(); + + assert!(journal + .append(ViewMutationKind::Write, "changed.txt", None) + .is_err()); + + let layout = ViewUpperLayout::from_source_upper(upper.clone()); + assert_eq!(fs::read(layout.journal_path()).unwrap(), b""); + let anchor = read_tail_anchor(&layout).unwrap(); + assert_eq!(anchor.mutation_sequence, 0); + assert_eq!(anchor.whiteout_sequence, 0); + } + + #[test] + fn uncommitted_whiteout_intent_is_dirty_evidence_but_not_view_state() { + let temp = tempfile::tempdir().unwrap(); + let upper = temp.path().join("upper"); + ViewMutationJournal::initialize_storage(&upper).unwrap(); + let mut journal = ViewMutationJournal::open(&upper).unwrap(); + journal + .append_classified_with_whiteouts( + ViewMutationKind::Rename, + "old.txt".into(), + ViewPathClass::Source, + Some("new.txt".into()), + Some(ViewPathClass::Source), + vec![ + ViewWhiteoutChange::Insert("old.txt".into()), + ViewWhiteoutChange::RemoveTree("new.txt".into()), + ], + ) + .unwrap(); + drop(journal); // crash after intent sync, before the filesystem rename + + let replayed = ViewMutationJournal::open(&upper).unwrap(); + assert!(replayed.is_qualified()); + assert!(replayed.dirty_paths().contains("old.txt")); + assert!(replayed.dirty_paths().contains("new.txt")); + assert!(!replayed.whiteouts().contains("old.txt")); + } + + #[test] + fn authenticated_journal_separates_generated_dirty_paths_without_upper_walk() { + let temp = tempfile::tempdir().unwrap(); + let upper = temp.path().join("upper"); + ViewMutationJournal::initialize_storage(&upper).unwrap(); + let mut journal = ViewMutationJournal::open(&upper).unwrap(); + journal + .append_classified( + ViewMutationKind::Write, + "README.md".into(), + ViewPathClass::Source, + None, + None, + ) + .unwrap(); + journal + .append_classified( + ViewMutationKind::Write, + "target/debug/app".into(), + ViewPathClass::Generated, + None, + None, + ) + .unwrap(); + journal + .append_classified( + ViewMutationKind::Write, + "node_modules/pkg/index.js".into(), + ViewPathClass::Dependency, + None, + None, + ) + .unwrap(); + drop(journal); + + let candidates = recover_view_checkpoint_candidates(&upper, &BTreeMap::new()).unwrap(); + assert_eq!(candidates.paths, BTreeSet::from(["README.md".into()])); + assert_eq!( + candidates.generated_paths, + BTreeSet::from([ + "node_modules/pkg/index.js".into(), + "target/debug/app".into() + ]) + ); + assert_eq!(candidates.upper_recovery_walks, 0); + } + + #[cfg(unix)] + #[test] + fn journal_append_refuses_symlink_leaf_without_touching_target() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let upper = temp.path().join("upper"); + ViewMutationJournal::initialize_storage(&upper).unwrap(); + let mut journal = ViewMutationJournal::open(&upper).unwrap(); + let path = ViewUpperLayout::from_source_upper(upper).journal_path(); + let victim = temp.path().join("victim.txt"); + fs::write(&victim, b"preserve me").unwrap(); + fs::remove_file(&path).unwrap(); + symlink(&victim, &path).unwrap(); + + assert!(journal + .append(ViewMutationKind::Write, "changed.txt", None) + .is_err()); + assert_eq!(fs::read(victim).unwrap(), b"preserve me"); + } + + #[test] + fn journal_generations_rotate_with_global_sequences_and_bounded_growth() { + let temp = tempfile::tempdir().unwrap(); + let upper = temp.path().join("upper"); + ViewMutationJournal::initialize_storage(&upper).unwrap(); + + for generation in 1..=8_u64 { + let mut journal = ViewMutationJournal::open(&upper).unwrap(); + assert_eq!(journal.generation(), generation - 1); + let sequence = journal + .append(ViewMutationKind::Write, &format!("file-{generation}"), None) + .unwrap(); + assert_eq!(sequence, generation); + ViewMutationJournal::rotate_after_checkpoint(&upper, sequence, generation).unwrap(); + } + + let layout = ViewUpperLayout::from_source_upper(upper.clone()); + let journal_files = fs::read_dir(&layout.meta_dir) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_name().to_string_lossy().contains("journal.g")) + .count(); + assert!( + journal_files <= 4, + "inactive generations were not compacted" + ); + let reopened = ViewMutationJournal::open(&upper).unwrap(); + assert_eq!(reopened.generation(), 8); + assert_eq!(reopened.last_sequence(), 8); + assert!(reopened.is_qualified()); + } + + #[test] + fn active_generation_lease_retains_old_journals_until_handle_closes() { + let temp = tempfile::tempdir().unwrap(); + let upper = temp.path().join("upper"); + ViewMutationJournal::initialize_storage(&upper).unwrap(); + let layout = ViewUpperLayout::from_source_upper(upper.clone()); + let lease = ViewGenerationLease::acquire(&upper, 0).unwrap(); + + for generation in 1..=3_u64 { + let mut journal = ViewMutationJournal::open(&upper).unwrap(); + let sequence = journal + .append( + ViewMutationKind::Write, + &format!("leased-{generation}"), + None, + ) + .unwrap(); + ViewMutationJournal::rotate_after_checkpoint(&upper, sequence, generation).unwrap(); + } + assert!(mutation_journal_path_for_generation(&layout, 0).exists()); + + drop(lease); + let mut journal = ViewMutationJournal::open(&upper).unwrap(); + let sequence = journal + .append(ViewMutationKind::Write, "after-close", None) + .unwrap(); + ViewMutationJournal::rotate_after_checkpoint(&upper, sequence, 4).unwrap(); + assert!(!mutation_journal_path_for_generation(&layout, 0).exists()); + assert!(!whiteout_journal_path_for_generation(&layout, 0).exists()); } } diff --git a/trail/src/db/lane/workdir/view_layout.rs b/trail/src/db/lane/workdir/view_layout.rs index ec48e06..dab4af6 100644 --- a/trail/src/db/lane/workdir/view_layout.rs +++ b/trail/src/db/lane/workdir/view_layout.rs @@ -126,19 +126,16 @@ impl ViewUpperLayout { } } - pub(crate) fn whiteouts_path(&self, class: ViewPathClass) -> PathBuf { - self.meta_dir - .join(format!("{}-whiteouts.json", class.as_str())) + pub(crate) fn journal_path(&self) -> PathBuf { + self.meta_dir.join("mutation-journal.jsonl") } - pub(crate) fn legacy_whiteouts_path(&self) -> PathBuf { - self.source_upper - .join(".trail") - .join("overlay-whiteouts.json") + pub(crate) fn whiteout_journal_path(&self) -> PathBuf { + self.meta_dir.join("whiteout-journal.jsonl") } - pub(crate) fn journal_path(&self) -> PathBuf { - self.meta_dir.join("mutation-journal.jsonl") + pub(crate) fn journal_state_path(&self) -> PathBuf { + self.meta_dir.join("journal-state.json") } } diff --git a/trail/src/db/lane/workspace_cargo.rs b/trail/src/db/lane/workspace_cargo.rs new file mode 100644 index 0000000..f83bd74 --- /dev/null +++ b/trail/src/db/lane/workspace_cargo.rs @@ -0,0 +1,411 @@ +use super::workspace_environment::{ + resolve_workspace_tool_executable, WorkspaceEnvironmentAdapter, + WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentCacheAccess, + WorkspaceEnvironmentCacheProtocol, WorkspaceEnvironmentCommand, WorkspaceEnvironmentOutput, + WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, WorkspaceEnvironmentSandboxPolicy, +}; +use super::*; + +pub(crate) struct CargoTargetSeedAdapter; + +pub(crate) static CARGO_TARGET_SEED_ADAPTER: CargoTargetSeedAdapter = CargoTargetSeedAdapter; + +static CARGO_TARGET_SEED_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = + WorkspaceEnvironmentAdapterMetadata { + canonical_identity: "trail/cargo-target-seed@1", + namespace: "trail", + name: "cargo-target-seed", + contract_major: 1, + implementation_version: env!("CARGO_PKG_VERSION"), + distribution_digest: "builtin:cargo-target-seed-plan-v1", + selectors: &["trail/cargo-target-seed@1", "cargo-target-seed", "cargo"], + kind: "compiler-results", + layer_adapter_name: "cargo-target-seed", + discovery_markers: &["Cargo.toml"], + supported_operating_systems: &["linux", "macos", "windows"], + supported_architectures: &["aarch64", "x86_64"], + stability: "experimental", + description: + "Locked Cargo target seed keyed by the complete source root and Rust toolchain identity", + }; + +impl WorkspaceEnvironmentAdapter for CargoTargetSeedAdapter { + fn metadata(&self) -> &'static WorkspaceEnvironmentAdapterMetadata { + &CARGO_TARGET_SEED_ADAPTER_METADATA + } + + fn component_id(&self, component_root: &str) -> Result { + let root = normalize_component_root(component_root)?; + Ok(if root.is_empty() { + "cargo-target-seed".to_string() + } else { + format!("cargo-target-seed:{root}") + }) + } + + fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result { + let root = normalize_component_root(component_root)?; + Ok(db + .root_file_entry(source_root, &join_repo_path(&root, "Cargo.toml"))? + .is_some() + && db + .root_file_entry(source_root, &join_repo_path(&root, "Cargo.lock"))? + .is_some()) + } + + fn plan( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result { + let component_root = normalize_component_root(component_root)?; + let manifest_path = join_repo_path(&component_root, "Cargo.toml"); + let lock_path = join_repo_path(&component_root, "Cargo.lock"); + let manifest = db + .root_file_entry(source_root, &manifest_path)? + .ok_or_else(|| { + Error::InvalidInput(format!( + "Cargo component `{}` has no Cargo.toml", + display_component_root(&component_root) + )) + })?; + let lock = db.root_file_entry(source_root, &lock_path)?.ok_or_else(|| { + Error::InvalidInput(format!( + "Cargo component `{}` has no Cargo.lock; generate and record a lockfile before synchronizing a target seed", + display_component_root(&component_root) + )) + })?; + let cargo_version = command_identity("cargo", &["--version"])?; + let rustc_identity = command_identity("rustc", &["-vV"])?; + let cargo_tool = resolve_workspace_tool_executable("cargo")?; + let rustc_tool = resolve_workspace_tool_executable("rustc")?; + let host_target = rustc_identity + .lines() + .find_map(|line| line.strip_prefix("host: ")) + .unwrap_or("unknown") + .to_string(); + let has_sccache = command_is_available("sccache"); + let mut tool_versions = BTreeMap::from([ + ("cargo".to_string(), cargo_version.clone()), + ("rustc-vV".to_string(), rustc_identity.clone()), + ("target".to_string(), host_target.clone()), + ("cargo-executable".to_string(), cargo_tool.identity.clone()), + ("rustc-executable".to_string(), rustc_tool.identity), + ]); + let cargo_cache = db.declare_workspace_environment_cache( + self.identity(), + "cargo-home", + WorkspaceEnvironmentCacheProtocol::LockedIndex, + WorkspaceEnvironmentCacheAccess::ToolConcurrent, + BTreeMap::from([ + ("cargo".to_string(), cargo_version), + ("cargo_executable".to_string(), cargo_tool.identity.clone()), + ("platform".to_string(), std::env::consts::OS.to_string()), + ( + "architecture".to_string(), + std::env::consts::ARCH.to_string(), + ), + ]), + )?; + let mut caches = vec![cargo_cache.clone()]; + let mut cache_names = vec![cargo_cache.name.clone()]; + let mut environment = BTreeMap::from([ + ( + "CARGO_HOME".to_string(), + cargo_cache.storage_path.to_string_lossy().into_owned(), + ), + ("CARGO_NET_OFFLINE".to_string(), "true".to_string()), + ( + "CARGO_INCREMENTAL".to_string(), + if has_sccache { "0" } else { "1" }.to_string(), + ), + ]); + let rustup_home = std::env::var_os("RUSTUP_HOME") + .map(PathBuf::from) + .or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".rustup"))); + if let Some(rustup_home) = rustup_home.filter(|path| path.is_dir()) { + environment.insert( + "RUSTUP_HOME".to_string(), + rustup_home.to_string_lossy().into_owned(), + ); + } + let rustup_toolchain = std::env::var("RUSTUP_TOOLCHAIN").ok(); + if let Some(toolchain) = &rustup_toolchain { + environment.insert("RUSTUP_TOOLCHAIN".to_string(), toolchain.clone()); + } + if has_sccache { + let sccache_tool = resolve_workspace_tool_executable("sccache")?; + let sccache_version = command_identity("sccache", &["--version"])?; + let sccache_cache = db.declare_workspace_environment_cache( + self.identity(), + "sccache", + WorkspaceEnvironmentCacheProtocol::CompilerCache, + WorkspaceEnvironmentCacheAccess::ToolConcurrent, + BTreeMap::from([ + ("sccache".to_string(), sccache_version.clone()), + ("rustc".to_string(), rustc_identity), + ("target".to_string(), host_target), + ("platform".to_string(), std::env::consts::OS.to_string()), + ( + "architecture".to_string(), + std::env::consts::ARCH.to_string(), + ), + ]), + )?; + environment.insert( + "RUSTC_WRAPPER".to_string(), + sccache_tool.path.to_string_lossy().into_owned(), + ); + environment.insert( + "SCCACHE_DIR".to_string(), + sccache_cache.storage_path.to_string_lossy().into_owned(), + ); + tool_versions.insert("sccache".to_string(), sccache_version); + tool_versions.insert("sccache-executable".to_string(), sccache_tool.identity); + cache_names.push(sccache_cache.name.clone()); + caches.push(sccache_cache); + } + let mut remove_environment = vec![ + "CARGO_TARGET_DIR".to_string(), + "CARGO_ENCODED_RUSTFLAGS".to_string(), + "RUSTFLAGS".to_string(), + "RUSTDOCFLAGS".to_string(), + "RUSTC_WORKSPACE_WRAPPER".to_string(), + ]; + if !has_sccache { + remove_environment.push("RUSTC_WRAPPER".to_string()); + } + let mut fetch_environment = environment.clone(); + fetch_environment.insert("CARGO_NET_OFFLINE".to_string(), "false".to_string()); + let working_directory = if component_root.is_empty() { + "project".to_string() + } else { + format!("project/{component_root}") + }; + let output_path = format!("{working_directory}/target"); + let mount_path = if component_root.is_empty() { + "target".to_string() + } else { + format!("{component_root}/target") + }; + Ok(WorkspaceEnvironmentPlan { + component_id: self.component_id(&component_root)?, + adapter_identity: self.identity().to_string(), + adapter_version: 1, + implementation_version: env!("CARGO_PKG_VERSION").to_string(), + distribution_digest: "builtin:cargo-target-seed-plan-v1".to_string(), + kind: "compiler-results".to_string(), + dependencies: Vec::new(), + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: "compiler-results".to_string(), + adapter: self.layer_adapter_name().to_string(), + adapter_version: 1, + inputs: BTreeMap::from([ + ("source_root".to_string(), source_root.0.clone()), + (manifest_path, manifest.content_hash), + (lock_path, lock.content_hash), + ( + "output_contract".to_string(), + format!("immutable-seed-private:{mount_path}"), + ), + ( + "adapter_implementation".to_string(), + env!("CARGO_PKG_VERSION").to_string(), + ), + ( + "adapter_distribution_digest".to_string(), + "builtin:cargo-target-seed-plan-v1".to_string(), + ), + ( + "rustup_toolchain".to_string(), + rustup_toolchain.unwrap_or_default(), + ), + ]), + tool_versions, + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: "source-root-toolchain-target-platform".to_string(), + strategy: format!( + "cargo-build-locked-offline-target-seed-v1:{}", + if has_sccache { + "sccache" + } else { + "incremental" + } + ), + }, + inputs: Vec::new(), + source_projection: Some((source_root.clone(), "project".to_string())), + pre_commands: vec![WorkspaceEnvironmentCommand { + program: "cargo".to_string(), + resolved_program: cargo_tool.path.clone(), + executable_identity: cargo_tool.identity.clone(), + args: vec!["fetch".to_string(), "--locked".to_string()], + working_directory: working_directory.clone(), + environment: fetch_environment, + remove_environment: remove_environment.clone(), + cache_names: cache_names.clone(), + }], + command: Some(WorkspaceEnvironmentCommand { + program: "cargo".to_string(), + resolved_program: cargo_tool.path, + executable_identity: cargo_tool.identity, + args: vec![ + "build".to_string(), + "--locked".to_string(), + "--offline".to_string(), + "--target-dir".to_string(), + "target".to_string(), + ], + working_directory, + environment, + remove_environment, + cache_names, + }), + mounted_commands: Vec::new(), + caches, + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin, + outputs: vec![WorkspaceEnvironmentOutput { + name: "target-seed".to_string(), + output_path, + mount_path, + policy: WorkspaceEnvironmentOutputPolicy::ImmutableSeedPrivate, + create_if_missing: false, + }], + stale_reason: + "source root, Cargo lockfile, Rust toolchain, target, or build policy changed" + .to_string(), + }) + } +} + +fn normalize_component_root(component_root: &str) -> Result { + if component_root.trim_matches('/').is_empty() { + Ok(String::new()) + } else { + normalize_relative_path(component_root) + } +} + +fn display_component_root(component_root: &str) -> &str { + if component_root.is_empty() { + "." + } else { + component_root + } +} + +fn join_repo_path(root: &str, name: &str) -> String { + if root.is_empty() { + name.to_string() + } else { + format!("{root}/{name}") + } +} + +fn command_identity(program: &str, args: &[&str]) -> Result { + let output = Command::new(program).args(args).output().map_err(|err| { + Error::InvalidInput(format!("required tool `{program}` is unavailable: {err}")) + })?; + if !output.status.success() { + return Err(Error::InvalidInput(format!( + "`{program} {}` failed with {}", + args.join(" "), + output.status + ))); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +fn command_is_available(program: &str) -> bool { + Command::new(program) + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cargo_adapter_builds_once_and_reuses_one_immutable_target_seed() { + if command_identity("cargo", &["--version"]).is_err() + || command_identity("rustc", &["-vV"]).is_err() + { + return; + } + let workspace = tempfile::tempdir().unwrap(); + fs::create_dir_all(workspace.path().join("src")).unwrap(); + fs::write( + workspace.path().join("Cargo.toml"), + "[package]\nname = \"trail-cargo-adapter-test\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + ) + .unwrap(); + fs::write( + workspace.path().join("src/lib.rs"), + "pub fn answer() -> u64 { 42 }\n", + ) + .unwrap(); + let lock = Command::new("cargo") + .args(["generate-lockfile", "--offline"]) + .current_dir(workspace.path()) + .output() + .unwrap(); + assert!( + lock.status.success(), + "cargo generate-lockfile failed: {}", + String::from_utf8_lossy(&lock.stderr) + ); + + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + for lane in ["cargo-one", "cargo-two"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + mode.clone(), + None, + None, + None, + &[], + false, + ) + .unwrap(); + } + let first = db + .sync_workspace_environment("cargo-one", "auto", None) + .unwrap(); + let second = db + .sync_workspace_environment("cargo-two", "trail/cargo-target-seed@1", None) + .unwrap(); + assert_eq!(first.layer_id, second.layer_id); + assert_eq!(first.cache_key, second.cache_key); + assert!(Path::new(&first.storage_path).join("debug").is_dir()); + + let status = db.environment_component_status("cargo-two").unwrap(); + assert_eq!(status.len(), 1); + assert_eq!(status[0].component.component_id, "cargo-target-seed"); + assert_eq!(status[0].adapter.name, "cargo-target-seed"); + assert_eq!( + status[0].adapter.implementation_version, + env!("CARGO_PKG_VERSION") + ); + assert_eq!( + status[0].adapter.distribution_digest.as_deref(), + Some("builtin:cargo-target-seed-plan-v1") + ); + assert_eq!(status[0].status, "ready"); + } +} diff --git a/trail/src/db/lane/workspace_cmake.rs b/trail/src/db/lane/workspace_cmake.rs new file mode 100644 index 0000000..8a1601e --- /dev/null +++ b/trail/src/db/lane/workspace_cmake.rs @@ -0,0 +1,429 @@ +use super::workspace_environment::{ + resolve_workspace_tool_executable, WorkspaceEnvironmentAdapter, + WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentOutput, + WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, WorkspaceEnvironmentSandboxPolicy, +}; +use super::*; + +pub(crate) struct CmakeBuildTreeAdapter; + +pub(crate) static CMAKE_BUILD_TREE_ADAPTER: CmakeBuildTreeAdapter = CmakeBuildTreeAdapter; + +static CMAKE_BUILD_TREE_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = + WorkspaceEnvironmentAdapterMetadata { + canonical_identity: "trail/cmake-build@1", + namespace: "trail", + name: "cmake-build", + contract_major: 1, + implementation_version: env!("CARGO_PKG_VERSION"), + distribution_digest: "builtin:cmake-build-plan-v1", + selectors: &["trail/cmake-build@1", "cmake-build", "cmake"], + kind: "build", + layer_adapter_name: "cmake-build", + discovery_markers: &["CMakeLists.txt"], + supported_operating_systems: &["linux", "macos", "windows"], + supported_architectures: &["aarch64", "x86_64"], + stability: "experimental", + description: "Lane-private CMake build tree with configure deferred to the mounted lane", + }; + +impl WorkspaceEnvironmentAdapter for CmakeBuildTreeAdapter { + fn metadata(&self) -> &'static WorkspaceEnvironmentAdapterMetadata { + &CMAKE_BUILD_TREE_ADAPTER_METADATA + } + + fn component_id(&self, component_root: &str) -> Result { + let root = normalize_component_root(component_root)?; + Ok(if root.is_empty() { + "cmake-build".to_string() + } else { + format!("cmake-build:{root}") + }) + } + + fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result { + let root = normalize_component_root(component_root)?; + Ok(db + .root_file_entry(source_root, &join_repo_path(&root, "CMakeLists.txt"))? + .is_some()) + } + + fn plan( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result { + let component_root = normalize_component_root(component_root)?; + let manifest_path = join_repo_path(&component_root, "CMakeLists.txt"); + if db.root_file_entry(source_root, &manifest_path)?.is_none() { + return Err(Error::InvalidInput(format!( + "CMake component `{}` has no CMakeLists.txt", + display_component_root(&component_root) + ))); + } + let cmake = resolve_workspace_tool_executable("cmake")?; + let implementation_version = env!("CARGO_PKG_VERSION").to_string(); + let distribution_digest = "builtin:cmake-build-plan-v1".to_string(); + let mount_path = join_repo_path(&component_root, "build"); + let component_id = self.component_id(&component_root)?; + let inputs = BTreeMap::from([ + ("component_id".to_string(), component_id.clone()), + ("component_root".to_string(), component_root.clone()), + ("manifest".to_string(), manifest_path), + ( + "adapter_implementation".to_string(), + implementation_version.clone(), + ), + ( + "adapter_distribution_digest".to_string(), + distribution_digest.clone(), + ), + ( + "output_contract".to_string(), + format!("writable-private:{mount_path}"), + ), + ( + "configure_phase".to_string(), + "deferred-to-mounted-lane".to_string(), + ), + ]); + Ok(WorkspaceEnvironmentPlan { + component_id, + adapter_identity: self.identity().to_string(), + adapter_version: 1, + implementation_version, + distribution_digest, + kind: "build".to_string(), + dependencies: Vec::new(), + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: "build".to_string(), + adapter: self.layer_adapter_name().to_string(), + adapter_version: 1, + inputs, + tool_versions: BTreeMap::from([("cmake-executable".to_string(), cmake.identity)]), + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: "lane-private-host-tool".to_string(), + strategy: "cmake-build-tree-private-v1".to_string(), + }, + inputs: Vec::new(), + source_projection: None, + pre_commands: Vec::new(), + command: None, + mounted_commands: Vec::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin, + outputs: vec![WorkspaceEnvironmentOutput { + name: "build-tree".to_string(), + // No command consumes this staging path. The host creates an + // empty private directory directly in the final lane upper. + output_path: "private/build".to_string(), + mount_path, + policy: WorkspaceEnvironmentOutputPolicy::WritablePrivate, + create_if_missing: true, + }], + stale_reason: + "CMake executable, host platform, architecture, component root, or adapter policy changed" + .to_string(), + }) + } +} + +fn normalize_component_root(component_root: &str) -> Result { + if component_root.trim_matches('/').is_empty() { + Ok(String::new()) + } else { + normalize_relative_path(component_root) + } +} + +fn join_repo_path(root: &str, name: &str) -> String { + if root.is_empty() { + name.to_string() + } else { + format!("{root}/{name}") + } +} + +fn display_component_root(component_root: &str) -> &str { + if component_root.is_empty() { + "." + } else { + component_root + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::OsStr; + + #[test] + fn cmake_discovery_is_pinned_and_side_effect_free() { + let workspace = tempfile::tempdir().unwrap(); + fs::write( + workspace.path().join("CMakeLists.txt"), + "cmake_minimum_required(VERSION 3.20)\nproject(example)\n", + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let root = db.resolve_branch_ref("main").unwrap().root_id; + assert!(CMAKE_BUILD_TREE_ADAPTER.detect(&db, &root, "").unwrap()); + assert_eq!( + CMAKE_BUILD_TREE_ADAPTER.component_id("").unwrap(), + "cmake-build" + ); + assert_eq!( + CMAKE_BUILD_TREE_ADAPTER.component_id("native/lib").unwrap(), + "cmake-build:native/lib" + ); + } + + #[test] + fn cmake_sync_provisions_private_state_without_publishing_a_layer() { + if resolve_workspace_tool_executable("cmake").is_err() { + return; + } + let workspace = tempfile::tempdir().unwrap(); + fs::write( + workspace.path().join("CMakeLists.txt"), + "cmake_minimum_required(VERSION 3.20)\nproject(example)\n", + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "cmake", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let plan = db + .plan_workspace_environment("cmake", "trail/cmake-build@1", None) + .unwrap(); + assert!(plan.commands.is_empty()); + assert_eq!(plan.outputs[0].policy, "writable_private"); + assert!(plan.tools.contains_key("cmake-executable")); + let report = db + .sync_workspace_environment_component("cmake", "trail/cmake-build@1", None, None) + .unwrap(); + assert!(report.layers.is_empty()); + assert_eq!( + report.generation.components[0].outputs[0].policy, + "writable_private" + ); + assert!(report.generation.components[0].outputs[0] + .layer_id + .is_none()); + } + + #[cfg(unix)] + #[test] + fn real_cmake_configure_build_and_clean_stay_lane_private() { + #[cfg(target_os = "linux")] + if std::env::var_os("TRAIL_RUN_FUSE_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + #[cfg(target_os = "macos")] + if std::env::var_os("TRAIL_RUN_NFS_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + if resolve_workspace_tool_executable("cmake").is_err() + || resolve_workspace_tool_executable("make").is_err() + || resolve_workspace_tool_executable("cc").is_err() + { + return; + } + let workspace = tempfile::tempdir().unwrap(); + fs::write( + workspace.path().join("CMakeLists.txt"), + "cmake_minimum_required(VERSION 3.20)\nproject(trail_lane C)\nadd_executable(hello main.c)\n", + ) + .unwrap(); + fs::write( + workspace.path().join("main.c"), + "#include \nint main(void) { puts(\"hello\"); return 0; }\n", + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + for lane in ["cmake-a", "cmake-b"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let report = db + .sync_workspace_environment_component(lane, "trail/cmake-build@1", None, None) + .unwrap(); + assert!(report.layers.is_empty()); + } + + for lane in ["cmake-a", "cmake-b"] { + #[cfg(target_os = "macos")] + let mounted = db.mount_nfs_cow_workdir_for_lane(lane).unwrap(); + #[cfg(target_os = "linux")] + let mounted = db.mount_fuse_cow_workdir_for_lane(lane).unwrap(); + let workdir = PathBuf::from(db.lane_workdir(lane).unwrap().workdir.unwrap()); + let configured = Command::new("cmake") + .args(["-S", ".", "-B", "build", "-G", "Unix Makefiles"]) + .current_dir(&workdir) + .status() + .unwrap(); + assert!(configured.success()); + let built = Command::new("cmake") + .args(["--build", "build", "--parallel", "2"]) + .current_dir(&workdir) + .status() + .unwrap(); + assert!(built.success()); + assert!(workdir.join("build/hello").is_file()); + let cache = fs::read_to_string(workdir.join("build/CMakeCache.txt")).unwrap(); + assert!(cache.contains(workdir.to_string_lossy().as_ref())); + drop(mounted); + } + + #[cfg(target_os = "macos")] + let mounted = db.mount_nfs_cow_workdir_for_lane("cmake-a").unwrap(); + #[cfg(target_os = "linux")] + let mounted = db.mount_fuse_cow_workdir_for_lane("cmake-a").unwrap(); + let workdir_a = PathBuf::from(db.lane_workdir("cmake-a").unwrap().workdir.unwrap()); + let cleaned = Command::new("cmake") + .args(["--build", "build", "--target", "clean"]) + .current_dir(&workdir_a) + .status() + .unwrap(); + assert!(cleaned.success()); + assert!(!workdir_a.join("build/hello").exists()); + drop(mounted); + + #[cfg(target_os = "macos")] + let mounted = db.mount_nfs_cow_workdir_for_lane("cmake-b").unwrap(); + #[cfg(target_os = "linux")] + let mounted = db.mount_fuse_cow_workdir_for_lane("cmake-b").unwrap(); + let workdir_b = PathBuf::from(db.lane_workdir("cmake-b").unwrap().workdir.unwrap()); + assert!(workdir_b.join("build/hello").is_file()); + drop(mounted); + assert!(db.list_workspace_layers().unwrap().is_empty()); + } + + #[cfg(windows)] + #[test] + fn real_windows_cmake_build_and_clean_stay_lane_private() { + if std::env::var_os("TRAIL_RUN_DOKAN_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + if resolve_workspace_tool_executable("cmake").is_err() { + return; + } + let workspace = tempfile::tempdir().unwrap(); + fs::write( + workspace.path().join("CMakeLists.txt"), + "cmake_minimum_required(VERSION 3.20)\nproject(trail_lane C)\nadd_executable(hello main.c)\n", + ) + .unwrap(); + fs::write( + workspace.path().join("main.c"), + "#include \nint main(void) { puts(\"hello\"); return 0; }\n", + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + for lane in ["cmake-a", "cmake-b"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + LaneWorkdirMode::FuseCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let report = db + .sync_workspace_environment_component(lane, "trail/cmake-build@1", None, None) + .unwrap(); + assert!(report.layers.is_empty()); + } + let executable = |workdir: &Path| { + [ + workdir.join("build/Debug/hello.exe"), + workdir.join("build/hello.exe"), + ] + .into_iter() + .find(|path| path.is_file()) + .expect("CMake did not produce hello.exe") + }; + for lane in ["cmake-a", "cmake-b"] { + let mounted = db.mount_fuse_cow_workdir_for_lane(lane).unwrap(); + let workdir = PathBuf::from(db.lane_workdir(lane).unwrap().workdir.unwrap()); + assert!(Command::new("cmake") + .args(["-S", ".", "-B", "build"]) + .current_dir(&workdir) + .status() + .unwrap() + .success()); + assert!(Command::new("cmake") + .args(["--build", "build", "--config", "Debug", "--parallel", "2"]) + .current_dir(&workdir) + .status() + .unwrap() + .success()); + assert!(executable(&workdir).is_file()); + let cache = fs::read_to_string(workdir.join("build/CMakeCache.txt")) + .unwrap() + .replace('\\', "/"); + assert!(cache.contains(&workdir.to_string_lossy().replace('\\', "/"))); + drop(mounted); + } + let mounted = db.mount_fuse_cow_workdir_for_lane("cmake-a").unwrap(); + let workdir_a = PathBuf::from(db.lane_workdir("cmake-a").unwrap().workdir.unwrap()); + assert!(Command::new("cmake") + .args(["--build", "build", "--target", "clean", "--config", "Debug"]) + .current_dir(&workdir_a) + .status() + .unwrap() + .success()); + assert!(![ + workdir_a.join("build/Debug/hello.exe"), + workdir_a.join("build/hello.exe") + ] + .iter() + .any(|path| path.exists())); + drop(mounted); + let mounted = db.mount_fuse_cow_workdir_for_lane("cmake-b").unwrap(); + let workdir_b = PathBuf::from(db.lane_workdir("cmake-b").unwrap().workdir.unwrap()); + assert!(executable(&workdir_b).is_file()); + drop(mounted); + assert!(db.list_workspace_layers().unwrap().is_empty()); + } +} diff --git a/trail/src/db/lane/workspace_environment.rs b/trail/src/db/lane/workspace_environment.rs new file mode 100644 index 0000000..05b2f1c --- /dev/null +++ b/trail/src/db/lane/workspace_environment.rs @@ -0,0 +1,6801 @@ +use super::workdir::{ViewPathClass, ViewUpperLayout}; +use super::workspace_layer::make_tree_writable; +use super::*; +use std::ffi::OsString; +use std::process::Stdio; +use std::thread; + +/// One repository file that the host projects into an adapter-owned staging +/// directory. Adapters describe the mapping; they never receive writable +/// access to the lane source view. +#[derive(Clone, Debug)] +pub(crate) struct WorkspaceEnvironmentInput { + pub(crate) source_path: String, + pub(crate) staging_path: String, + pub(crate) entry: FileEntry, +} + +/// A command plan is deliberately argv-based. Trail owns the working +/// directory, environment injection, staging tree, and publication boundary. +#[derive(Clone, Debug)] +pub(crate) struct WorkspaceEnvironmentCommand { + pub(crate) program: String, + pub(crate) resolved_program: PathBuf, + pub(crate) executable_identity: String, + pub(crate) args: Vec, + pub(crate) working_directory: String, + pub(crate) environment: BTreeMap, + pub(crate) remove_environment: Vec, + /// Names of host-owned cache declarations used by this action. Commands + /// never acquire arbitrary writable host paths by supplying directories. + pub(crate) cache_names: Vec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum WorkspaceEnvironmentCacheProtocol { + ContentStore, + CompilerCache, + LockedIndex, +} + +impl WorkspaceEnvironmentCacheProtocol { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::ContentStore => "content_store", + Self::CompilerCache => "compiler_cache", + Self::LockedIndex => "locked_index", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum WorkspaceEnvironmentCacheAccess { + /// The ecosystem tool is certified to coordinate concurrent writers and + /// recover or evict partial entries without changing correctness. + ToolConcurrent, + /// Trail serializes all commands using the namespace. + HostExclusive, +} + +impl WorkspaceEnvironmentCacheAccess { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::ToolConcurrent => "tool_concurrent", + Self::HostExclusive => "host_exclusive", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct WorkspaceEnvironmentCache { + pub(crate) name: String, + pub(crate) namespace_id: String, + pub(crate) storage_path: PathBuf, + pub(crate) protocol: WorkspaceEnvironmentCacheProtocol, + pub(crate) access: WorkspaceEnvironmentCacheAccess, + pub(crate) compatibility: BTreeMap, +} + +/// Immutable identity owned by an external provider rather than Trail's +/// filesystem layer store. Trail records and composes the identity but never +/// deletes provider-owned content. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct WorkspaceEnvironmentExternalArtifact { + pub(crate) name: String, + pub(crate) artifact_type: String, + pub(crate) provider: String, + pub(crate) reference: String, + pub(crate) digest: String, + pub(crate) platform: String, + pub(crate) cleanup_owner: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct WorkspaceEnvironmentRuntimeResource { + pub(crate) name: String, + pub(crate) runtime_type: String, + pub(crate) provider: String, + pub(crate) artifact_name: String, + pub(crate) container_port: u16, + pub(crate) protocol: String, + pub(crate) health_type: String, + pub(crate) health_timeout_ms: u64, + pub(crate) restart_policy: String, + pub(crate) cleanup_owner: String, + pub(crate) volume_target: Option, + pub(crate) secrets: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct WorkspaceEnvironmentSecretReference { + pub(crate) name: String, + pub(crate) provider: String, + pub(crate) reference: String, + pub(crate) version: Option, + pub(crate) purpose: String, + pub(crate) injection: String, + pub(crate) target: String, + pub(crate) environment: Option, + pub(crate) required: bool, +} + +/// One independently mounted directory produced by a component action. +/// +/// A component has one canonical key. Immutable-seeded outputs share one +/// content-addressed lower layer, while writable-private outputs live only in +/// the owning lane's generated upper. Every output binding still activates as +/// part of one environment generation. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum WorkspaceEnvironmentOutputPolicy { + ImmutableSeedPrivate, + WritablePrivate, +} + +impl WorkspaceEnvironmentOutputPolicy { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::ImmutableSeedPrivate => "immutable_seed_private", + Self::WritablePrivate => "writable_private", + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct WorkspaceEnvironmentOutput { + pub(crate) name: String, + pub(crate) output_path: String, + pub(crate) mount_path: String, + pub(crate) policy: WorkspaceEnvironmentOutputPolicy, + pub(crate) create_if_missing: bool, +} + +/// Host-owned semantics for one directed component relationship. +/// +/// Every edge participates in graph validation and deterministic ordering. Only +/// identity-bearing edges enter the target component's canonical artifact key; +/// runtime and binding-order edges are instead retained in generation +/// provenance so changing them does not manufacture a different artifact. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum WorkspaceEnvironmentEdgeType { + BuildRequires, + RuntimeRequires, + BindsAfter, + InvalidatesWith, +} + +impl WorkspaceEnvironmentEdgeType { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::BuildRequires => "build_requires", + Self::RuntimeRequires => "runtime_requires", + Self::BindsAfter => "binds_after", + Self::InvalidatesWith => "invalidates_with", + } + } + + pub(crate) fn parse(value: &str) -> Result { + match value { + "build_requires" | "ordering_invalidation" => Ok(Self::BuildRequires), + "runtime_requires" => Ok(Self::RuntimeRequires), + "binds_after" => Ok(Self::BindsAfter), + "invalidates_with" => Ok(Self::InvalidatesWith), + other => Err(Error::InvalidInput(format!( + "unknown environment edge type `{other}`; expected build_requires, runtime_requires, binds_after, or invalidates_with" + ))), + } + } + + fn identity_input_name(self, component_id: &str) -> Option { + match self { + // Preserve the schema-v8 key shape for legacy `depends_on` edges. + Self::BuildRequires => Some(format!("dependency:{component_id}")), + Self::InvalidatesWith => Some(format!("dependency:invalidates_with:{component_id}")), + Self::RuntimeRequires | Self::BindsAfter => None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct WorkspaceEnvironmentDependency { + pub(crate) component_id: String, + pub(crate) edge_type: WorkspaceEnvironmentEdgeType, +} + +impl WorkspaceEnvironmentDependency { + pub(crate) fn build_requires(component_id: impl Into) -> Self { + Self { + component_id: component_id.into(), + edge_type: WorkspaceEnvironmentEdgeType::BuildRequires, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ResolvedWorkspaceEnvironmentDependency { + pub(crate) component_id: String, + pub(crate) component_key: String, + pub(crate) edge_type: WorkspaceEnvironmentEdgeType, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum WorkspaceEnvironmentSandboxPolicy { + /// Existing first-party adapters whose ecosystem-specific execution policy + /// is maintained with Trail. These remain open-world until migrated onto + /// individually certified capability profiles. + TrustedBuiltin, + /// Repository-authored argv command. The host must execute it inside a + /// supported kernel sandbox or fail closed before launching the tool. + RestrictedRecipe, + /// An installed protocol-v2 plugin staging action with host-owned cache + /// namespaces. Cache access is projected into the deny-by-default native + /// sandbox and is never available to planning or mounted actions. + RestrictedPluginStaging, + /// An installed protocol-v2 plugin requested mounted initialization. The + /// host authorizes the typed action only after normalizing the authenticated + /// package plan, then applies the same native deny-by-default sandbox while + /// the command runs in an ephemeral candidate lane view. + RestrictedPluginMounted, +} + +/// Normalized plan emitted by every built-in environment adapter. +/// +/// The layer key and output contract are data so the host can validate and +/// execute the plan without giving adapters publication or database access. +#[derive(Clone, Debug)] +pub(crate) struct WorkspaceEnvironmentPlan { + pub(crate) component_id: String, + pub(crate) adapter_identity: String, + pub(crate) adapter_version: u32, + pub(crate) implementation_version: String, + pub(crate) distribution_digest: String, + pub(crate) kind: String, + /// Stable logical component relationships. Trail validates and resolves + /// these edges; adapters never execute or attach dependencies themselves. + pub(crate) dependencies: Vec, + /// Populated only by host graph finalization. This retains the exact + /// upstream keys for both identity-bearing and generation-only edges. + pub(crate) resolved_dependencies: Vec, + pub(crate) layer_key: WorkspaceLayerKeyV1, + pub(crate) inputs: Vec, + /// Optional complete pinned source projection for adapters such as Cargo + /// whose build graph can include arbitrary workspace source files. The + /// host streams this root in bounded chunks during explicit sync. + pub(crate) source_projection: Option<(ObjectId, String)>, + pub(crate) pre_commands: Vec, + /// Optional host-executed initialization. Writable-private adapters may + /// provision an empty persistent directory and defer path-sensitive tools + /// (for example CMake configure) to execution inside the mounted lane. + pub(crate) command: Option, + /// Commands that must observe the lane's stable mountpoint. Trail runs + /// them against an ephemeral candidate view and persists only declared + /// writable-private outputs before normal atomic activation. + pub(crate) mounted_commands: Vec, + /// Performance-only host cache namespaces. Eviction may make execution + /// slower but must never change component correctness or readiness. + pub(crate) caches: Vec, + /// Provider-owned immutable identities such as pinned OCI manifests. + /// These have no manufactured filesystem output and survive cache GC as + /// generation provenance only. + pub(crate) external_artifacts: Vec, + /// Per-lane runtime resources derived from immutable external artifacts. + /// The declaration is identity-bearing; provider allocation IDs and host + /// ports are assigned only after the generation commits. + pub(crate) runtime_resources: Vec, + pub(crate) sandbox_policy: WorkspaceEnvironmentSandboxPolicy, + pub(crate) outputs: Vec, + pub(crate) stale_reason: String, +} + +struct PreparedPrivateEnvironmentOutputs { + _staging: tempfile::TempDir, + paths: Vec, +} + +enum PreparedEnvironmentArtifacts { + Immutable(WorkspaceLayerReport), + WritablePrivate(Option), + MetadataOnly, +} + +/// Static, side-effect-free facts used to register and discover an adapter. +/// +/// Keeping these facts outside `detect` means the host can build a catalog and +/// scan only relevant manifest names without invoking ecosystem tooling. +#[derive(Clone, Debug)] +pub(crate) struct WorkspaceEnvironmentAdapterMetadata { + pub(crate) canonical_identity: &'static str, + pub(crate) namespace: &'static str, + pub(crate) name: &'static str, + pub(crate) contract_major: u32, + pub(crate) implementation_version: &'static str, + pub(crate) distribution_digest: &'static str, + pub(crate) selectors: &'static [&'static str], + pub(crate) kind: &'static str, + pub(crate) layer_adapter_name: &'static str, + pub(crate) discovery_markers: &'static [&'static str], + pub(crate) supported_operating_systems: &'static [&'static str], + pub(crate) supported_architectures: &'static [&'static str], + pub(crate) stability: &'static str, + pub(crate) description: &'static str, +} + +/// Ecosystem code is restricted to discovery and deterministic planning. +/// Trail remains responsible for executing, validating, publishing, binding, +/// persisting, and reporting the resulting environment. +pub(crate) trait WorkspaceEnvironmentAdapter: Sync { + fn metadata(&self) -> &'static WorkspaceEnvironmentAdapterMetadata; + + fn identity(&self) -> &'static str { + self.metadata().canonical_identity + } + + fn kind(&self) -> &'static str { + self.metadata().kind + } + + /// Stable legacy/storage name used by WorkspaceLayerKeyV1. Keeping this + /// separate lets public component reports use the canonical contract + /// identity without invalidating already-published layer keys. + fn layer_adapter_name(&self) -> &'static str { + self.metadata().layer_adapter_name + } + + fn accepts_selector(&self, selector: &str) -> bool { + self.metadata().selectors.contains(&selector) + } + + fn component_id(&self, component_root: &str) -> Result; + + fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result; + + fn plan( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result; +} + +fn builtin_environment_adapters() -> [&'static dyn WorkspaceEnvironmentAdapter; 6] { + [ + &super::workspace_node::NODE_WORKSPACE_ADAPTER, + &super::workspace_cargo::CARGO_TARGET_SEED_ADAPTER, + &super::workspace_cmake::CMAKE_BUILD_TREE_ADAPTER, + &super::workspace_go::GO_VENDOR_ADAPTER, + &super::workspace_python::PYTHON_VENV_ADAPTER, + &super::workspace_oci::OCI_IMAGE_ADAPTER, + ] +} + +pub(super) fn registered_environment_adapter_metadata( +) -> Vec<&'static WorkspaceEnvironmentAdapterMetadata> { + let mut metadata = builtin_environment_adapters() + .into_iter() + .map(WorkspaceEnvironmentAdapter::metadata) + .collect::>(); + metadata.push(&super::workspace_recipe::COMMAND_RECIPE_ADAPTER_METADATA); + metadata +} + +fn builtin_environment_adapter_for_selector( + selector: &str, +) -> Option<&'static dyn WorkspaceEnvironmentAdapter> { + builtin_environment_adapters() + .into_iter() + .find(|adapter| adapter.accepts_selector(selector)) +} + +impl Trail { + /// Return the adapters compiled into this Trail host. + /// + /// This is intentionally side-effect free: listing adapters never probes + /// tools, reads repository files, or executes adapter code beyond static + /// metadata access. + pub fn workspace_environment_adapters(&self) -> Result { + let mut adapters = registered_environment_adapter_metadata() + .into_iter() + .map(|metadata| EnvironmentAdapterCatalogEntryReport { + identity: EnvironmentAdapterIdentityReport { + namespace: metadata.namespace.to_string(), + name: metadata.name.to_string(), + contract_major: metadata.contract_major, + implementation_version: metadata.implementation_version.to_string(), + distribution_digest: Some(metadata.distribution_digest.to_string()), + }, + canonical_identity: metadata.canonical_identity.to_string(), + selectors: metadata + .selectors + .iter() + .map(|selector| (*selector).to_string()) + .collect(), + kind: metadata.kind.to_string(), + layer_adapter_name: metadata.layer_adapter_name.to_string(), + discovery_markers: metadata + .discovery_markers + .iter() + .map(|marker| (*marker).to_string()) + .collect(), + protocols: Vec::new(), + supported_operating_systems: metadata + .supported_operating_systems + .iter() + .map(|value| (*value).to_string()) + .collect(), + supported_architectures: metadata + .supported_architectures + .iter() + .map(|value| (*value).to_string()) + .collect(), + source: if metadata.canonical_identity + == super::workspace_recipe::COMMAND_RECIPE_ADAPTER_METADATA.canonical_identity + { + "recipe" + } else { + "builtin" + } + .to_string(), + publisher: Some("trail".to_string()), + publisher_key_id: None, + trust: "builtin".to_string(), + certification_tier: format!("builtin-{}", metadata.stability), + stability: metadata.stability.to_string(), + description: metadata.description.to_string(), + }) + .collect::>(); + let mut selectors = adapters + .iter() + .flat_map(|adapter| { + adapter + .selectors + .iter() + .map(move |selector| (selector.clone(), adapter.canonical_identity.clone())) + }) + .collect::>(); + for plugin in self.installed_environment_plugins()? { + let metadata = plugin.manifest.adapter; + let (namespace, name, contract_major) = + super::workspace_plugin::validate_plugin_identity(&metadata.canonical_identity)?; + for selector in &metadata.selectors { + if let Some(other) = + selectors.insert(selector.clone(), metadata.canonical_identity.clone()) + { + return Err(Error::Corrupt(format!( + "adapter selector `{selector}` is claimed by both `{other}` and `{}`", + metadata.canonical_identity + ))); + } + } + adapters.push(EnvironmentAdapterCatalogEntryReport { + identity: EnvironmentAdapterIdentityReport { + namespace, + name, + contract_major, + implementation_version: metadata.implementation_version, + distribution_digest: Some(plugin.distribution_digest), + }, + canonical_identity: metadata.canonical_identity, + selectors: metadata.selectors, + kind: metadata.kind, + layer_adapter_name: metadata.layer_adapter_name, + discovery_markers: metadata.discovery_markers, + protocols: metadata.protocols, + supported_operating_systems: metadata.supported_operating_systems, + supported_architectures: metadata.supported_architectures, + source: "plugin".to_string(), + publisher: plugin.publisher, + publisher_key_id: plugin.publisher_key_id, + trust: plugin.trust, + certification_tier: plugin.certification_tier, + stability: metadata.stability, + description: metadata.description, + }); + } + adapters.sort_by(|left, right| left.canonical_identity.cmp(&right.canonical_identity)); + Ok(EnvironmentAdapterCatalogReport { + contract_major: 1, + adapters, + }) + } + + /// Recover synchronization records whose owning process no longer exists. + /// + /// Component rows retain their predecessor `attached_key` while building, + /// so recovery can distinguish a stale-but-usable predecessor from a first + /// build that produced no usable environment. + pub(crate) fn recover_workspace_environment_sync_attempts(&self) -> Result<()> { + let running = { + let mut stmt = self.conn.prepare( + "SELECT attempt_id, view_id, owner_pid, owner_start_token + FROM environment_sync_attempts WHERE status = 'running' + ORDER BY started_at, attempt_id", + )?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, String>(3)?, + )) + })?; + rows.collect::, _>>()? + }; + for (attempt_id, view_id, owner_pid, owner_start_token) in running { + if u32::try_from(owner_pid) + .ok() + .is_some_and(|pid| process_matches_start_token(pid, &owner_start_token)) + { + continue; + } + let building_count = self.conn.query_row( + "SELECT COUNT(*) FROM environment_component_states + WHERE view_id = ?1 AND status = 'building'", + params![view_id], + |row| row.get::<_, i64>(0), + )?; + let recovered_status = if building_count == 0 { + "succeeded" + } else { + "abandoned" + }; + let reason = if building_count == 0 { + "synchronization owner exited after component activation; recovered as complete" + } else { + "synchronization owner exited before activation; the predecessor environment remains authoritative" + }; + self.conn + .execute_batch("SAVEPOINT trail_environment_attempt_recovery")?; + let recovery = (|| -> Result<()> { + if building_count > 0 { + self.conn.execute( + "UPDATE environment_component_states + SET status = CASE WHEN attached_key IS NULL THEN 'failed' ELSE 'stale' END, + reason = ?1, + updated_at = ?2 + WHERE view_id = ?3 AND status = 'building'", + params![reason, now_ts(), view_id], + )?; + self.conn.execute( + "UPDATE workspace_environment_states + SET status = CASE WHEN attached_key IS NULL THEN 'failed' ELSE 'stale' END, + reason = ?1, + updated_at = ?2 + WHERE view_id = ?3 AND status = 'building'", + params![reason, now_ts(), view_id], + )?; + } + self.conn.execute( + "UPDATE environment_sync_attempts + SET status = ?1, reason = ?2, updated_at = ?3, finished_at = ?3 + WHERE attempt_id = ?4 AND status = 'running'", + params![recovered_status, reason, now_ts(), attempt_id], + )?; + Ok(()) + })(); + match recovery { + Ok(()) => self + .conn + .execute_batch("RELEASE SAVEPOINT trail_environment_attempt_recovery")?, + Err(err) => { + let _ = self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT trail_environment_attempt_recovery; + RELEASE SAVEPOINT trail_environment_attempt_recovery", + ); + return Err(err); + } + } + self.cleanup_mounted_environment_candidates(&attempt_id); + } + Ok(()) + } + + fn cleanup_mounted_environment_candidates(&self, attempt_id: &str) { + let staging = self.db_dir.join("cache/staging"); + let prefix = format!("mounted-environment-{attempt_id}-"); + let Ok(entries) = fs::read_dir(staging) else { + return; + }; + for entry in entries.filter_map(std::result::Result::ok) { + if !entry.file_name().to_string_lossy().starts_with(&prefix) { + continue; + } + let path = entry.path(); + make_tree_writable(&path); + let _ = fs::remove_dir_all(path); + } + } + + fn begin_workspace_environment_sync_attempt( + &self, + view_id: &str, + source_root: &ObjectId, + mode: &str, + ) -> Result { + self.recover_workspace_environment_sync_attempts()?; + if let Some((attempt_id, owner_pid)) = self + .conn + .query_row( + "SELECT attempt_id, owner_pid FROM environment_sync_attempts + WHERE view_id = ?1 AND status = 'running'", + params![view_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)), + ) + .optional()? + { + return Err(Error::InvalidInput(format!( + "workspace view `{view_id}` is already synchronizing in attempt `{attempt_id}` owned by process {owner_pid}" + ))); + } + let owner_start_token = current_process_start_token(); + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let digest = sha256_hex( + format!( + "{view_id}:{}:{}:{owner_start_token}:{nonce}:{mode}", + source_root.0, + std::process::id() + ) + .as_bytes(), + ); + let attempt_id = format!("envsync_{}", &digest[..24]); + self.conn.execute( + "INSERT INTO environment_sync_attempts + (attempt_id, view_id, source_root, mode, owner_pid, owner_start_token, status, reason, started_at, updated_at, finished_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'running', NULL, ?7, ?7, NULL)", + params![ + attempt_id, + view_id, + source_root.0, + mode, + i64::from(std::process::id()), + owner_start_token, + now_ts() + ], + )?; + Ok(attempt_id) + } + + fn finish_workspace_environment_sync_attempt( + &self, + attempt_id: &str, + status: &str, + reason: Option<&str>, + ) -> Result<()> { + if !matches!(status, "succeeded" | "failed") { + return Err(Error::InvalidInput(format!( + "invalid environment synchronization attempt status `{status}`" + ))); + } + let updated = self.conn.execute( + "UPDATE environment_sync_attempts + SET status = ?1, reason = ?2, updated_at = ?3, finished_at = ?3 + WHERE attempt_id = ?4 AND status = 'running'", + params![status, reason, now_ts(), attempt_id], + )?; + if updated != 1 { + return Err(Error::Corrupt(format!( + "environment synchronization attempt `{attempt_id}` was not running" + ))); + } + Ok(()) + } + + /// Discover built-in environment components without invoking package + /// managers, compilers, network providers, or repository code. + pub fn discover_workspace_environment( + &self, + lane: &str, + component_root: Option<&str>, + ) -> Result { + let branch = self.lane_branch(lane)?; + let head = self.get_ref(&branch.ref_name)?; + let mut roots = BTreeSet::new(); + let plugins = self.installed_environment_plugins()?; + if !plugins.is_empty() { + // This also rejects selectors that collide with built-ins before + // any external code is invoked. + let _ = self.workspace_environment_adapters()?; + } + let mut discovery_markers = builtin_environment_adapters() + .into_iter() + .flat_map(|adapter| adapter.metadata().discovery_markers.iter().copied()) + .map(str::to_string) + .collect::>(); + discovery_markers.extend( + plugins + .iter() + .flat_map(|plugin| plugin.manifest.adapter.discovery_markers.iter().cloned()), + ); + if let Some(component_root) = component_root { + roots.insert(if component_root.trim_matches('/').is_empty() { + String::new() + } else { + normalize_relative_path(component_root)? + }); + } else { + self.for_each_root_file_chunk(&head.root_id, 1024, |chunk| { + for (path, _) in chunk { + let file_name = path.rsplit('/').next().unwrap_or(path.as_str()); + if !discovery_markers.contains(file_name) { + continue; + } + roots.insert( + path.rsplit_once('/') + .map(|(parent, _)| parent.to_string()) + .unwrap_or_default(), + ); + } + Ok(()) + })?; + } + + let mut components = Vec::new(); + let mut conflicts = Vec::new(); + for root in roots { + for adapter in builtin_environment_adapters() { + if adapter.detect(self, &head.root_id, &root)? { + components.push(EnvironmentDiscoveredComponentReport { + component_id: adapter.component_id(&root)?, + component_root: root.clone(), + kind: adapter.kind().to_string(), + adapter_identity: adapter.identity().to_string(), + }); + } + } + for plugin in &plugins { + if !super::workspace_plugin::environment_plugin_supports_current_host(plugin) { + continue; + } + if let Some(component) = + self.discover_environment_plugin_component(plugin, &head.root_id, &root)? + { + components.push(component); + } + } + } + components.extend(self.command_recipe_discovery(&head.root_id, component_root)?); + components.sort_by(|left, right| { + ( + &left.component_root, + &left.component_id, + &left.adapter_identity, + ) + .cmp(&( + &right.component_root, + &right.component_id, + &right.adapter_identity, + )) + }); + for duplicate in components.windows(2) { + if duplicate[0].component_id == duplicate[1].component_id { + conflicts.push(EnvironmentDiscoveryConflictReport { + component_root: duplicate[0].component_root.clone(), + adapter_identities: vec![ + duplicate[0].adapter_identity.clone(), + duplicate[1].adapter_identity.clone(), + ], + reason: format!( + "multiple adapters proposed logical component `{}`", + duplicate[0].component_id + ), + }); + } + } + Ok(EnvironmentDiscoveryReport { + source_root: head.root_id, + components, + conflicts, + }) + } + + /// Return the complete desired ordering/invalidation graph without + /// executing adapter commands or mutating lane state. + pub fn workspace_environment_graph( + &self, + lane: &str, + discovery_root: Option<&str>, + ) -> Result { + let discovery = self.discover_workspace_environment(lane, discovery_root)?; + if !discovery.conflicts.is_empty() { + return Err(Error::InvalidInput(format!( + "environment discovery found {} unresolved component identity conflict(s); inspect `trail env discover {lane}`", + discovery.conflicts.len() + ))); + } + let roots = discovery + .components + .iter() + .map(|component| { + ( + component.component_id.clone(), + component.component_root.clone(), + ) + }) + .collect::>(); + let finalized = + self.plan_discovered_environment_graph(&discovery.source_root, &discovery.components)?; + let keys = finalized + .iter() + .map(|(plan, component_key)| (plan.component_id.clone(), component_key.clone())) + .collect::>(); + let mut nodes = Vec::with_capacity(finalized.len()); + let mut edges = Vec::new(); + for (topological_index, (plan, component_key)) in finalized.into_iter().enumerate() { + let external_artifacts = environment_external_artifact_activations(&plan); + let runtime_resources = environment_runtime_resource_activations(&plan); + for dependency in &plan.dependencies { + edges.push(EnvironmentGraphEdgeReport { + source_component_id: dependency.component_id.clone(), + source_component_key: keys.get(&dependency.component_id).cloned().ok_or_else( + || { + Error::Corrupt(format!( + "finalized environment graph lost key for dependency `{}`", + dependency.component_id + )) + }, + )?, + target_component_id: plan.component_id.clone(), + edge_type: dependency.edge_type.as_str().to_string(), + }); + } + nodes.push(EnvironmentGraphNodeReport { + topological_index: topological_index as u64, + component_root: roots.get(&plan.component_id).cloned().ok_or_else(|| { + Error::Corrupt(format!( + "finalized environment graph lost root for component `{}`", + plan.component_id + )) + })?, + component_id: plan.component_id, + kind: plan.kind, + adapter_identity: plan.adapter_identity, + component_key, + dependencies: plan + .dependencies + .into_iter() + .map(|dependency| dependency.component_id) + .collect(), + caches: plan.caches.iter().map(environment_cache_report).collect(), + external_artifacts, + runtime_resources, + outputs: plan + .outputs + .into_iter() + .map(|output| EnvironmentPlanOutputReport { + name: output.name, + output_path: output.output_path, + mount_path: output.mount_path, + policy: output.policy.as_str().to_string(), + }) + .collect(), + }); + } + Ok(EnvironmentGraphReport { + source_root: discovery.source_root, + total_nodes: nodes.len() as u64, + total_edges: edges.len() as u64, + offset: 0, + next_offset: None, + nodes, + edges, + }) + } + + pub fn workspace_environment_graph_page( + &self, + lane: &str, + discovery_root: Option<&str>, + offset: u64, + limit: u64, + ) -> Result { + if limit == 0 || limit > 1_000 { + return Err(Error::InvalidInput( + "environment graph limit must be between 1 and 1000".to_string(), + )); + } + let graph = self.workspace_environment_graph(lane, discovery_root)?; + let start = usize::try_from(offset) + .unwrap_or(usize::MAX) + .min(graph.nodes.len()); + let end = start.saturating_add(limit as usize).min(graph.nodes.len()); + let target_ids = graph.nodes[start..end] + .iter() + .map(|node| node.component_id.clone()) + .collect::>(); + let next_offset = (end < graph.nodes.len()).then_some(end as u64); + Ok(EnvironmentGraphReport { + source_root: graph.source_root, + total_nodes: graph.total_nodes, + total_edges: graph.total_edges, + offset, + next_offset, + nodes: graph.nodes[start..end].to_vec(), + edges: graph + .edges + .into_iter() + .filter(|edge| target_ids.contains(&edge.target_component_id)) + .collect(), + }) + } + + /// Resolve and normalize one component without executing its commands or + /// mutating environment state. + pub fn plan_workspace_environment( + &self, + lane: &str, + adapter_selector: &str, + component_root: Option<&str>, + ) -> Result { + self.plan_workspace_environment_component(lane, adapter_selector, component_root, None) + } + + pub fn plan_workspace_environment_component( + &self, + lane: &str, + adapter_selector: &str, + component_root: Option<&str>, + component_id: Option<&str>, + ) -> Result { + let (source_root, plan) = self.resolve_workspace_environment_plan( + lane, + adapter_selector, + component_root.unwrap_or(""), + component_id, + )?; + let plan = self.finalize_workspace_environment_plan_preview(lane, &source_root, plan)?; + let component_key = self.workspace_layer_cache_key(&plan.layer_key)?; + let output_path = plan + .outputs + .first() + .map(|output| output.output_path.clone()) + .unwrap_or_default(); + let mount_path = plan + .outputs + .first() + .map(|output| output.mount_path.clone()) + .unwrap_or_default(); + let commands = plan + .pre_commands + .iter() + .chain(plan.command.iter()) + .map(|command| EnvironmentPlanCommandReport { + phase: "staging".to_string(), + program: command.program.clone(), + resolved_program: command.resolved_program.to_string_lossy().into_owned(), + executable_identity: command.executable_identity.clone(), + args: command.args.clone(), + working_directory: command.working_directory.clone(), + environment_names: command.environment.keys().cloned().collect(), + }) + .chain( + plan.mounted_commands + .iter() + .map(|command| EnvironmentPlanCommandReport { + phase: "mounted_initialization".to_string(), + program: command.program.clone(), + resolved_program: command.resolved_program.to_string_lossy().into_owned(), + executable_identity: command.executable_identity.clone(), + args: command.args.clone(), + working_directory: command.working_directory.clone(), + environment_names: command.environment.keys().cloned().collect(), + }), + ) + .collect::>(); + let process = commands + .iter() + .map(|command| command.resolved_program.clone()) + .collect::>(); + let has_runtime_secrets = plan + .runtime_resources + .iter() + .any(|resource| !resource.secrets.is_empty()); + let capabilities = if !plan.external_artifacts.is_empty() + || !plan.runtime_resources.is_empty() + { + EnvironmentCapabilityReport { + filesystem_read: Vec::new(), + filesystem_write: Vec::new(), + process: Vec::new(), + network: "none".to_string(), + shell: "none".to_string(), + scripts: "none".to_string(), + secrets: if has_runtime_secrets { + "opaque-references-only; host-runtime-file-handle-resolution".to_string() + } else { + "none".to_string() + }, + sandbox: "not-applicable-metadata-only".to_string(), + } + } else { + match plan.sandbox_policy { + WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginStaging + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted => { + EnvironmentCapabilityReport { + filesystem_read: plan + .inputs + .iter() + .map(|input| input.source_path.clone()) + .collect(), + filesystem_write: plan + .outputs + .iter() + .map(|output| { + if plan.sandbox_policy + == WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted + { + output.mount_path.clone() + } else { + output.output_path.clone() + } + }) + .chain( + plan.caches + .iter() + .map(|cache| cache.storage_path.to_string_lossy().into_owned()), + ) + .collect(), + process, + network: "deny".to_string(), + shell: "deny".to_string(), + scripts: "deny".to_string(), + secrets: "deny".to_string(), + sandbox: restricted_recipe_sandbox_name().to_string(), + } + } + WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin => EnvironmentCapabilityReport { + filesystem_read: if plan.source_projection.is_some() { + vec!["pinned-source-root/**".to_string()] + } else { + plan.inputs + .iter() + .map(|input| input.source_path.clone()) + .collect() + }, + filesystem_write: plan + .outputs + .iter() + .map(|output| output.output_path.clone()) + .chain( + plan.caches + .iter() + .map(|cache| cache.storage_path.to_string_lossy().into_owned()), + ) + .collect(), + process, + network: "adapter-managed-open-world".to_string(), + shell: "argv-direct; child-process policy adapter-managed".to_string(), + scripts: "adapter-managed".to_string(), + secrets: "none-declared".to_string(), + sandbox: "trusted-builtin".to_string(), + }, + } + }; + let tools = plan.layer_key.tool_versions.clone(); + let external_artifacts = environment_external_artifact_activations(&plan); + let runtime_resources = environment_runtime_resource_activations(&plan); + Ok(EnvironmentPlanReport { + source_root, + component_id: plan.component_id, + adapter_identity: plan.adapter_identity, + kind: plan.kind, + component_key, + dependencies: plan + .dependencies + .iter() + .map(|dependency| dependency.component_id.clone()) + .collect(), + dependency_edges: plan + .resolved_dependencies + .iter() + .map(|dependency| EnvironmentGenerationDependencyReport { + component_id: dependency.component_id.clone(), + component_key: dependency.component_key.clone(), + edge_type: dependency.edge_type.as_str().to_string(), + }) + .collect(), + caches: plan.caches.iter().map(environment_cache_report).collect(), + external_artifacts, + runtime_resources, + inputs: plan + .inputs + .into_iter() + .map(|input| EnvironmentPlanInputReport { + source_path: input.source_path, + staging_path: input.staging_path, + content_hash: input.entry.content_hash, + size_bytes: input.entry.size_bytes, + }) + .collect(), + tools, + commands, + outputs: plan + .outputs + .iter() + .map(|output| EnvironmentPlanOutputReport { + name: output.name.clone(), + output_path: output.output_path.clone(), + mount_path: output.mount_path.clone(), + policy: output.policy.as_str().to_string(), + }) + .collect(), + output_path, + mount_path, + portability_scope: plan.layer_key.portability_scope, + capabilities, + }) + } + + fn resolve_workspace_environment_plan( + &self, + lane: &str, + adapter_selector: &str, + component_root: &str, + component_id: Option<&str>, + ) -> Result<(ObjectId, WorkspaceEnvironmentPlan)> { + let branch = self.lane_branch(lane)?; + let head = self.get_ref(&branch.ref_name)?; + let command_metadata = &super::workspace_recipe::COMMAND_RECIPE_ADAPTER_METADATA; + let (plan, builtin_adapter, plugin_adapter, expected_component_id) = if adapter_selector + == "auto" + { + let discovery = self.discover_workspace_environment(lane, Some(component_root))?; + if !discovery.conflicts.is_empty() { + return Err(Error::InvalidInput(format!( + "environment discovery found {} unresolved component conflict(s) at `{}`", + discovery.conflicts.len(), + if component_root.is_empty() { + "." + } else { + component_root + } + ))); + } + let candidates = discovery + .components + .iter() + .filter(|component| component_id.is_none_or(|id| component.component_id == id)) + .collect::>(); + match candidates.as_slice() { + [component] + if component.adapter_identity == command_metadata.canonical_identity => + { + ( + self.command_recipe_plan(&head.root_id, &component.component_id)?, + None, + None, + component.component_id.clone(), + ) + } + [component] => { + if let Some(adapter) = + builtin_environment_adapter_for_selector(&component.adapter_identity) + { + ( + adapter.plan(self, &head.root_id, &component.component_root)?, + Some(adapter), + None, + component.component_id.clone(), + ) + } else { + let plugin = self + .environment_plugin_for_selector(&component.adapter_identity)? + .ok_or_else(|| { + Error::Corrupt(format!( + "discovered adapter `{}` is no longer installed", + component.adapter_identity + )) + })?; + let plan = self.plan_environment_plugin_component( + &plugin, + &head.root_id, + &component.component_root, + &component.component_id, + )?; + (plan, None, Some(plugin), component.component_id.clone()) + } + } + [] => { + return Err(Error::InvalidInput(format!( + "no workspace environment adapter detected at `{}`; specify --adapter explicitly", + if component_root.is_empty() { + "." + } else { + component_root + } + ))); + } + components => { + return Err(Error::InvalidInput(format!( + "multiple workspace environment adapters or components detected at `{}`: {}; specify --adapter explicitly", + if component_root.is_empty() { + "." + } else { + component_root + }, + components + .iter() + .map(|component| component.adapter_identity.as_str()) + .collect::>() + .join(", ") + ))); + } + } + } else if command_metadata.selectors.contains(&adapter_selector) { + let plan = if let Some(component_id) = component_id { + self.command_recipe_plan(&head.root_id, component_id)? + } else { + self.command_recipe_plan_for_root(&head.root_id, component_root)? + }; + let expected = plan.component_id.clone(); + (plan, None, None, expected) + } else { + if let Some(adapter) = builtin_environment_adapter_for_selector(adapter_selector) { + let expected = adapter.component_id(component_root)?; + if component_id.is_some_and(|component_id| component_id != expected) { + return Err(Error::InvalidInput(format!( + "adapter `{}` proposes component `{expected}`, not requested component `{}`", + adapter.identity(), + component_id.unwrap_or_default() + ))); + } + ( + adapter.plan(self, &head.root_id, component_root)?, + Some(adapter), + None, + expected, + ) + } else if let Some(plugin) = self.environment_plugin_for_selector(adapter_selector)? { + let discovered = self + .discover_environment_plugin_component(&plugin, &head.root_id, component_root)? + .ok_or_else(|| { + Error::InvalidInput(format!( + "adapter `{}` did not detect a component at `{}`", + plugin.manifest.adapter.canonical_identity, + if component_root.is_empty() { + "." + } else { + component_root + } + )) + })?; + if component_id.is_some_and(|component_id| component_id != discovered.component_id) + { + return Err(Error::InvalidInput(format!( + "adapter `{}` proposes component `{}`, not requested component `{}`", + plugin.manifest.adapter.canonical_identity, + discovered.component_id, + component_id.unwrap_or_default() + ))); + } + let expected = discovered.component_id; + let plan = self.plan_environment_plugin_component( + &plugin, + &head.root_id, + component_root, + &expected, + )?; + (plan, None, Some(plugin), expected) + } else { + let available = self + .workspace_environment_adapters()? + .adapters + .into_iter() + .map(|adapter| adapter.canonical_identity) + .collect::>() + .join(", "); + return Err(Error::InvalidInput(format!( + "unknown workspace environment adapter `{adapter_selector}`; available adapters: {available}" + ))); + } + }; + if let Some(adapter) = builtin_adapter { + self.validate_workspace_environment_plan(adapter, component_root, &plan)?; + } else if let Some(plugin) = &plugin_adapter { + self.validate_environment_plugin_plan(plugin, &expected_component_id, &plan)?; + } else { + self.validate_command_recipe_plan(&expected_component_id, &plan)?; + } + self.validate_environment_mounts_do_not_shadow_source(&head.root_id, &[&plan])?; + Ok((head.root_id, plan)) + } + + /// Finalize only the dependency closure needed by a read-only plan. This + /// makes first-use planning useful without requiring attached state and + /// avoids invoking unrelated adapters elsewhere in a large monorepo. + fn finalize_workspace_environment_plan_preview( + &self, + lane: &str, + source_root: &ObjectId, + selected: WorkspaceEnvironmentPlan, + ) -> Result { + if selected.dependencies.is_empty() { + return Ok(selected); + } + let selected_id = selected.component_id.clone(); + let discovery = self.discover_workspace_environment(lane, None)?; + let mut discovered = BTreeMap::new(); + for component in &discovery.components { + if discovered + .insert(component.component_id.clone(), component) + .is_some() + { + return Err(Error::InvalidInput(format!( + "environment dependency closure has ambiguous component `{}`", + component.component_id + ))); + } + } + let mut closure = BTreeMap::from([(selected_id.clone(), selected)]); + let mut pending = closure[&selected_id] + .dependencies + .iter() + .map(|dependency| dependency.component_id.clone()) + .collect::>(); + while let Some(component_id) = pending.pop_first() { + if closure.contains_key(&component_id) { + continue; + } + let component = discovered.get(&component_id).ok_or_else(|| { + Error::InvalidInput(format!( + "environment component `{selected_id}` requires missing component `{component_id}`" + )) + })?; + let plan = self.plan_discovered_environment_component(source_root, component)?; + pending.extend( + plan.dependencies + .iter() + .map(|dependency| dependency.component_id.clone()), + ); + closure.insert(component_id, plan); + } + let plans = closure.into_values().collect::>(); + self.validate_environment_plan_mount_collisions(&plans)?; + self.validate_environment_mounts_do_not_shadow_source( + source_root, + &plans.iter().collect::>(), + )?; + self.finalize_workspace_environment_plan_graph(plans)? + .into_iter() + .find_map(|(plan, _)| (plan.component_id == selected_id).then_some(plan)) + .ok_or_else(|| { + Error::Corrupt(format!( + "environment dependency closure lost selected component `{selected_id}`" + )) + }) + } + + /// Resolve one component against the dependency keys that are actually + /// active in this lane. A single-component sync must never silently build + /// against a missing or stale predecessor; callers can use `sync-all` to + /// construct and activate the complete graph atomically. + fn finalize_single_workspace_environment_plan( + &self, + lane: &str, + mut plan: WorkspaceEnvironmentPlan, + ) -> Result { + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` does not have a layered workspace view" + )) + })?; + normalize_workspace_environment_dependencies(&plan.component_id, &mut plan.dependencies)?; + plan.resolved_dependencies.clear(); + for dependency in &plan.dependencies { + let state = self + .conn + .query_row( + "SELECT attached_key, status FROM environment_component_states + WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, &dependency.component_id], + |row| Ok((row.get::<_, Option>(0)?, row.get::<_, String>(1)?)), + ) + .optional()?; + let Some((Some(component_key), status)) = state else { + return Err(Error::InvalidInput(format!( + "environment component `{}` requires `{}`, which is not attached; run `trail env sync-all {lane}`", + plan.component_id, dependency.component_id + ))); + }; + if status != "ready" { + return Err(Error::InvalidInput(format!( + "environment component `{}` requires `{}`, which is `{status}`; run `trail env sync-all {lane}`", + plan.component_id, dependency.component_id + ))); + } + if let Some(input_name) = dependency + .edge_type + .identity_input_name(&dependency.component_id) + { + plan.layer_key + .inputs + .insert(input_name, component_key.clone()); + } + plan.resolved_dependencies + .push(ResolvedWorkspaceEnvironmentDependency { + component_id: dependency.component_id.clone(), + component_key, + edge_type: dependency.edge_type, + }); + } + Ok(plan) + } + + /// Validate and finalize a complete component DAG. The result is stable + /// topological order; every downstream canonical key includes the exact + /// finalized key of each direct dependency. + fn finalize_workspace_environment_plan_graph( + &self, + plans: Vec, + ) -> Result> { + let mut by_id = BTreeMap::new(); + for mut plan in plans { + normalize_workspace_environment_dependencies( + &plan.component_id, + &mut plan.dependencies, + )?; + plan.resolved_dependencies.clear(); + if by_id.insert(plan.component_id.clone(), plan).is_some() { + return Err(Error::InvalidInput( + "environment graph contains a duplicate component identity".to_string(), + )); + } + } + for plan in by_id.values() { + for dependency in &plan.dependencies { + if !by_id.contains_key(&dependency.component_id) { + return Err(Error::InvalidInput(format!( + "environment component `{}` requires missing component `{}`", + plan.component_id, dependency.component_id + ))); + } + } + } + + let mut indegree = by_id + .iter() + .map(|(component_id, plan)| (component_id.clone(), plan.dependencies.len())) + .collect::>(); + let mut dependents = BTreeMap::>::new(); + for plan in by_id.values() { + for dependency in &plan.dependencies { + dependents + .entry(dependency.component_id.clone()) + .or_default() + .insert(plan.component_id.clone()); + } + } + let mut ready = indegree + .iter() + .filter_map(|(component_id, count)| (*count == 0).then_some(component_id.clone())) + .collect::>(); + let mut finalized_keys = BTreeMap::::new(); + let mut ordered = Vec::with_capacity(by_id.len()); + while let Some(component_id) = ready.pop_first() { + let mut plan = by_id.remove(&component_id).ok_or_else(|| { + Error::Corrupt(format!( + "environment graph lost ready component `{component_id}`" + )) + })?; + for dependency in &plan.dependencies { + let dependency_key = + finalized_keys + .get(&dependency.component_id) + .ok_or_else(|| { + Error::Corrupt(format!( + "environment graph ordered `{component_id}` before `{}`", + dependency.component_id + )) + })?; + if let Some(input_name) = dependency + .edge_type + .identity_input_name(&dependency.component_id) + { + plan.layer_key + .inputs + .insert(input_name, dependency_key.clone()); + } + plan.resolved_dependencies + .push(ResolvedWorkspaceEnvironmentDependency { + component_id: dependency.component_id.clone(), + component_key: dependency_key.clone(), + edge_type: dependency.edge_type, + }); + } + let key = self.workspace_layer_cache_key(&plan.layer_key)?; + finalized_keys.insert(component_id.clone(), key.clone()); + ordered.push((plan, key)); + if let Some(children) = dependents.get(&component_id) { + for child in children { + let count = indegree.get_mut(child).ok_or_else(|| { + Error::Corrupt(format!("environment graph lost indegree for `{child}`")) + })?; + *count = count.checked_sub(1).ok_or_else(|| { + Error::Corrupt(format!( + "environment graph indegree underflow for `{child}`" + )) + })?; + if *count == 0 { + ready.insert(child.clone()); + } + } + } + } + if !by_id.is_empty() { + let cycle = environment_dependency_cycle(&by_id)?; + return Err(Error::InvalidInput(format!( + "environment component dependency cycle: {}", + cycle.join(" -> ") + ))); + } + Ok(ordered) + } + + /// Synchronize one adapter component through the host-owned staging and + /// immutable-layer lifecycle. The legacy Node wrapper delegates here. + pub fn sync_workspace_environment( + &self, + lane: &str, + adapter_selector: &str, + component_root: Option<&str>, + ) -> Result { + let (_, plan) = self.resolve_workspace_environment_plan( + lane, + adapter_selector, + component_root.unwrap_or(""), + None, + )?; + if plan.outputs.is_empty() { + return Err(Error::InvalidInput(format!( + "adapter `{adapter_selector}` has no filesystem layer output; call the generation-oriented environment sync API instead" + ))); + } + if plan + .outputs + .iter() + .any(|output| output.policy != WorkspaceEnvironmentOutputPolicy::ImmutableSeedPrivate) + { + return Err(Error::InvalidInput(format!( + "adapter `{adapter_selector}` uses writable-private outputs; call the generation-oriented environment sync API instead of the legacy layer API" + ))); + } + let mut report = self.sync_workspace_environment_component( + lane, + adapter_selector, + component_root, + None, + )?; + if report.layers.len() != 1 { + return Err(Error::Corrupt(format!( + "legacy immutable-layer synchronization expected one layer but generation `{}` contains {}", + report.generation.generation_id, + report.layers.len() + ))); + } + Ok(report.layers.remove(0)) + } + + pub fn sync_workspace_environment_component( + &self, + lane: &str, + adapter_selector: &str, + component_root: Option<&str>, + component_id: Option<&str>, + ) -> Result { + let component_root = component_root.unwrap_or(""); + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` does not have a layered workspace view" + )) + })?; + if let (Some(pid), Some(token)) = (view.owner_pid, view.owner_start_token.as_deref()) { + if process_matches_start_token(pid, token) { + return Err(Error::InvalidInput(format!( + "lane `{lane}` has an active workspace writer in process {pid}; run `trail lane unmount {lane}` before synchronizing its environment" + ))); + } + } + + let (source_root, plan) = self.resolve_workspace_environment_plan( + lane, + adapter_selector, + component_root, + component_id, + )?; + let plan = self.finalize_single_workspace_environment_plan(lane, plan)?; + let cache_key = self.workspace_layer_cache_key(&plan.layer_key)?; + let predecessor_key = self + .workspace_environment_rows(lane)? + .into_iter() + .find(|state| state.adapter == plan.component_id) + .and_then(|state| state.attached_key); + let attempt_id = + self.begin_workspace_environment_sync_attempt(&view.view_id, &source_root, "single")?; + let result = (|| -> Result { + self.set_workspace_environment_state( + lane, + &plan, + &cache_key, + predecessor_key.as_deref(), + "building", + None, + )?; + let prepared = + self.prepare_workspace_environment_artifacts(&view.view_id, &plan, &cache_key); + let mut prepared = match prepared { + Ok(prepared) => prepared, + Err(err) => { + self.set_workspace_environment_state( + lane, + &plan, + &cache_key, + predecessor_key.as_deref(), + "failed", + Some(&err.to_string()), + )?; + return Err(err); + } + }; + if let Err(err) = self.initialize_mounted_workspace_environment_plans( + lane, + &attempt_id, + &source_root, + &[(&plan, cache_key.as_str())], + std::slice::from_mut(&mut prepared), + &[], + ) { + self.set_workspace_environment_state( + lane, + &plan, + &cache_key, + predecessor_key.as_deref(), + "failed", + Some(&err.to_string()), + )?; + return Err(err); + } + let (layer_id, private_paths) = match &prepared { + PreparedEnvironmentArtifacts::Immutable(layer) => { + (Some(layer.layer_id.as_str()), None) + } + PreparedEnvironmentArtifacts::WritablePrivate(Some(outputs)) => { + (None, Some(outputs.paths.as_slice())) + } + PreparedEnvironmentArtifacts::WritablePrivate(None) => (None, None), + PreparedEnvironmentArtifacts::MetadataOnly => (None, None), + }; + let activation = EnvironmentLayerActivation { + layer_id: layer_id.map(str::to_string), + outputs: environment_output_activations( + &plan, + layer_id, + &cache_key, + private_paths, + )?, + component_id: plan.component_id.clone(), + adapter_identity: plan.adapter_identity.clone(), + adapter_version: plan.adapter_version, + implementation_version: plan.implementation_version.clone(), + distribution_digest: plan.distribution_digest.clone(), + kind: plan.kind.clone(), + dependencies: environment_dependency_activations(&plan)?, + caches: environment_cache_activations(&plan), + external_artifacts: environment_external_artifact_activations(&plan), + runtime_resources: environment_runtime_resource_activations(&plan), + expected_key: cache_key.clone(), + canonical_key: plan.layer_key.clone(), + }; + self.ensure_workspace_environment_source_root(lane, &source_root)?; + if let Err(err) = + self.replace_declared_workspace_layers_at_source(lane, &[activation], &source_root) + { + self.set_workspace_environment_state( + lane, + &plan, + &cache_key, + predecessor_key.as_deref(), + "failed", + Some(&err.to_string()), + )?; + return Err(err); + } + let generation = self.active_environment_generation(lane)?.ok_or_else(|| { + Error::Corrupt("environment activation committed without a generation".to_string()) + })?; + let layers = match prepared { + PreparedEnvironmentArtifacts::Immutable(layer) => vec![layer], + PreparedEnvironmentArtifacts::WritablePrivate(_) => Vec::new(), + PreparedEnvironmentArtifacts::MetadataOnly => Vec::new(), + }; + Ok(EnvironmentSyncReport { generation, layers }) + })(); + match result { + Ok(report) => { + self.finish_workspace_environment_sync_attempt(&attempt_id, "succeeded", None)?; + Ok(report) + } + Err(err) => { + let reason = err.to_string(); + if let Err(finish_err) = self.finish_workspace_environment_sync_attempt( + &attempt_id, + "failed", + Some(&reason), + ) { + return Err(Error::Corrupt(format!( + "environment synchronization failed: {reason}; additionally failed to finalize attempt `{attempt_id}`: {finish_err}" + ))); + } + Err(err) + } + } + } + + /// User-facing synchronization semantics: activate the immutable + /// generation first, then reconcile every declared private runtime + /// resource and return only after its health contract is satisfied. + /// Provider failure leaves the generation attached with explicit failed + /// runtime state so readiness remains fail-closed and retry is idempotent. + pub fn sync_workspace_environment_component_with_runtime( + &self, + lane: &str, + adapter_selector: &str, + component_root: Option<&str>, + component_id: Option<&str>, + ) -> Result { + let mut report = self.sync_workspace_environment_component( + lane, + adapter_selector, + component_root, + component_id, + )?; + if report + .generation + .components + .iter() + .any(|component| !component.runtime_resources.is_empty()) + { + report.generation = self.reconcile_workspace_environment_runtime(lane)?; + report.generation = self.cleanup_retired_workspace_environment_runtime(lane)?; + } + Ok(report) + } + + /// Build every discovered component first, then activate all bindings and + /// one complete generation atomically. Published but unbound layers remain + /// harmless cache entries if any build or activation fails. + pub fn sync_all_workspace_environments( + &self, + lane: &str, + discovery_root: Option<&str>, + ) -> Result { + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` does not have a layered workspace view" + )) + })?; + if let (Some(pid), Some(token)) = (view.owner_pid, view.owner_start_token.as_deref()) { + if process_matches_start_token(pid, token) { + return Err(Error::InvalidInput(format!( + "lane `{lane}` has an active workspace writer in process {pid}; run `trail lane unmount {lane}` before synchronizing its environment" + ))); + } + } + let discovery = self.discover_workspace_environment(lane, discovery_root)?; + if !discovery.conflicts.is_empty() { + return Err(Error::InvalidInput(format!( + "environment discovery found {} unresolved component identity conflict(s)", + discovery.conflicts.len() + ))); + } + let existing = self + .workspace_environment_rows(lane)? + .into_iter() + .map(|state| (state.adapter, state.attached_key)) + .collect::>(); + if discovery.components.is_empty() && (discovery_root.is_some() || existing.is_empty()) { + return Err(Error::InvalidInput(format!( + "no workspace environment components were discovered for lane `{lane}`" + ))); + } + let raw_plans = self.plan_discovered_environment_components( + &discovery.source_root, + &discovery.components, + )?; + self.validate_environment_plan_mount_collisions(&raw_plans)?; + let planned = self + .finalize_workspace_environment_plan_graph(raw_plans)? + .into_iter() + .map(|(plan, expected_key)| { + let predecessor = existing.get(&plan.component_id).cloned().flatten(); + (plan, expected_key, predecessor) + }) + .collect::>(); + let desired_component_ids = planned + .iter() + .map(|(plan, _, _)| plan.component_id.as_str()) + .collect::>(); + let removed_components = if discovery_root.is_none() { + existing + .keys() + .filter(|component_id| !desired_component_ids.contains(component_id.as_str())) + .cloned() + .collect::>() + } else { + Vec::new() + }; + self.validate_environment_mounts_do_not_shadow_source( + &discovery.source_root, + &planned.iter().map(|(plan, _, _)| plan).collect::>(), + )?; + + let attempt_id = self.begin_workspace_environment_sync_attempt( + &view.view_id, + &discovery.source_root, + "batch", + )?; + let result = (|| -> Result { + for (plan, expected_key, predecessor) in &planned { + self.set_workspace_environment_state( + lane, + plan, + expected_key, + predecessor.as_deref(), + "building", + None, + )?; + } + let mut prepared = Vec::with_capacity(planned.len()); + for (plan, expected_key, _) in &planned { + match self.prepare_workspace_environment_artifacts( + &view.view_id, + plan, + expected_key, + ) { + Ok(artifacts) => prepared.push(artifacts), + Err(err) => { + let reason = format!( + "atomic environment synchronization aborted before activation: {err}" + ); + for (plan, expected_key, predecessor) in &planned { + let _ = self.set_workspace_environment_state( + lane, + plan, + expected_key, + predecessor.as_deref(), + "failed", + Some(&reason), + ); + } + return Err(err); + } + } + } + let planned_refs = planned + .iter() + .map(|(plan, expected_key, _)| (plan, expected_key.as_str())) + .collect::>(); + if let Err(err) = self.initialize_mounted_workspace_environment_plans( + lane, + &attempt_id, + &discovery.source_root, + &planned_refs, + &mut prepared, + &removed_components, + ) { + let reason = format!( + "atomic environment synchronization aborted during mounted initialization: {err}" + ); + for (plan, expected_key, predecessor) in &planned { + let _ = self.set_workspace_environment_state( + lane, + plan, + expected_key, + predecessor.as_deref(), + "failed", + Some(&reason), + ); + } + return Err(err); + } + let activations = planned + .iter() + .zip(&prepared) + .map(|((plan, expected_key, _), artifacts)| { + let (layer_id, private_paths) = match artifacts { + PreparedEnvironmentArtifacts::Immutable(layer) => { + (Some(layer.layer_id.as_str()), None) + } + PreparedEnvironmentArtifacts::WritablePrivate(Some(outputs)) => { + (None, Some(outputs.paths.as_slice())) + } + PreparedEnvironmentArtifacts::WritablePrivate(None) => (None, None), + PreparedEnvironmentArtifacts::MetadataOnly => (None, None), + }; + Ok(EnvironmentLayerActivation { + layer_id: layer_id.map(str::to_string), + outputs: environment_output_activations( + plan, + layer_id, + expected_key, + private_paths, + )?, + component_id: plan.component_id.clone(), + adapter_identity: plan.adapter_identity.clone(), + adapter_version: plan.adapter_version, + implementation_version: plan.implementation_version.clone(), + distribution_digest: plan.distribution_digest.clone(), + kind: plan.kind.clone(), + dependencies: environment_dependency_activations(plan)?, + caches: environment_cache_activations(plan), + external_artifacts: environment_external_artifact_activations(plan), + runtime_resources: environment_runtime_resource_activations(plan), + expected_key: expected_key.clone(), + canonical_key: plan.layer_key.clone(), + }) + }) + .collect::>>()?; + self.ensure_workspace_environment_source_root(lane, &discovery.source_root)?; + if let Err(err) = self.replace_declared_workspace_layers_with_removals_at_source( + lane, + &activations, + &removed_components, + &discovery.source_root, + ) { + let reason = format!("atomic environment activation failed: {err}"); + for (plan, expected_key, predecessor) in &planned { + let _ = self.set_workspace_environment_state( + lane, + plan, + expected_key, + predecessor.as_deref(), + "failed", + Some(&reason), + ); + } + return Err(err); + } + let generation = self.active_environment_generation(lane)?.ok_or_else(|| { + Error::Corrupt("environment activation committed without a generation".to_string()) + })?; + let layers = prepared + .into_iter() + .filter_map(|artifacts| match artifacts { + PreparedEnvironmentArtifacts::Immutable(layer) => Some(layer), + PreparedEnvironmentArtifacts::WritablePrivate(_) => None, + PreparedEnvironmentArtifacts::MetadataOnly => None, + }) + .collect(); + Ok(EnvironmentSyncReport { generation, layers }) + })(); + match result { + Ok(report) => { + self.finish_workspace_environment_sync_attempt(&attempt_id, "succeeded", None)?; + Ok(report) + } + Err(err) => { + let reason = err.to_string(); + if let Err(finish_err) = self.finish_workspace_environment_sync_attempt( + &attempt_id, + "failed", + Some(&reason), + ) { + return Err(Error::Corrupt(format!( + "atomic environment synchronization failed: {reason}; additionally failed to finalize attempt `{attempt_id}`: {finish_err}" + ))); + } + Err(err) + } + } + } + + pub fn sync_all_workspace_environments_with_runtime( + &self, + lane: &str, + discovery_root: Option<&str>, + ) -> Result { + let mut report = self.sync_all_workspace_environments(lane, discovery_root)?; + if report + .generation + .components + .iter() + .any(|component| !component.runtime_resources.is_empty()) + { + report.generation = self.reconcile_workspace_environment_runtime(lane)?; + report.generation = self.cleanup_retired_workspace_environment_runtime(lane)?; + } + Ok(report) + } + + fn validate_workspace_environment_plan( + &self, + adapter: &dyn WorkspaceEnvironmentAdapter, + component_root: &str, + plan: &WorkspaceEnvironmentPlan, + ) -> Result<()> { + let expected_component = adapter.component_id(component_root)?; + if plan.component_id != expected_component { + return Err(Error::Corrupt(format!( + "adapter `{}` planned component `{}` but host expected `{expected_component}`", + adapter.identity(), + plan.component_id + ))); + } + if plan.adapter_identity != adapter.identity() + || plan.layer_key.adapter != adapter.layer_adapter_name() + || plan.layer_key.adapter_version != plan.adapter_version + { + return Err(Error::Corrupt(format!( + "component `{}` returned inconsistent adapter identity or version", + plan.component_id + ))); + } + self.validate_workspace_environment_plan_common(plan) + } + + fn validate_command_recipe_plan( + &self, + expected_component_id: &str, + plan: &WorkspaceEnvironmentPlan, + ) -> Result<()> { + let mut tool_identities = BTreeMap::new(); + self.validate_command_recipe_plan_with_tool_cache( + expected_component_id, + plan, + &mut tool_identities, + ) + } + + fn validate_command_recipe_plan_with_tool_cache( + &self, + expected_component_id: &str, + plan: &WorkspaceEnvironmentPlan, + tool_identities: &mut BTreeMap, + ) -> Result<()> { + let metadata = &super::workspace_recipe::COMMAND_RECIPE_ADAPTER_METADATA; + if plan.component_id != expected_component_id + || plan.adapter_identity != metadata.canonical_identity + || plan.layer_key.adapter != metadata.layer_adapter_name + || plan.layer_key.adapter_version != metadata.contract_major + || plan.sandbox_policy != WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + || !plan.pre_commands.is_empty() + || !plan.mounted_commands.is_empty() + || plan.command.is_none() + { + return Err(Error::Corrupt(format!( + "command recipe `{expected_component_id}` returned an inconsistent identity, policy, or action plan" + ))); + } + self.validate_workspace_environment_plan_common_with_tool_cache(plan, tool_identities) + } + + fn validate_environment_plugin_plan( + &self, + plugin: &super::workspace_plugin::InstalledEnvironmentPlugin, + expected_component_id: &str, + plan: &WorkspaceEnvironmentPlan, + ) -> Result<()> { + let metadata = &plugin.manifest.adapter; + let protocol = super::workspace_plugin::selected_environment_plugin_protocol(plugin)?; + let action_policy_is_valid = match protocol { + trail_environment_adapter_sdk::PROTOCOL_V1 => { + plan.sandbox_policy == WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + && plan.command.is_some() + && plan.mounted_commands.is_empty() + } + trail_environment_adapter_sdk::PROTOCOL_V2 if plan.mounted_commands.is_empty() => { + plan.sandbox_policy + == if plan.caches.is_empty() { + WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + } else { + WorkspaceEnvironmentSandboxPolicy::RestrictedPluginStaging + } + && plan.command.is_some() + } + trail_environment_adapter_sdk::PROTOCOL_V2 => { + plan.sandbox_policy == WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted + } + _ => false, + }; + if plan.component_id != expected_component_id + || plan.adapter_identity != metadata.canonical_identity + || plan.adapter_version != 1 + || plan.implementation_version != metadata.implementation_version + || plan.distribution_digest != plugin.distribution_digest + || plan.kind != metadata.kind + || plan.layer_key.adapter != metadata.layer_adapter_name + || plan.layer_key.adapter_version != 1 + || plan.layer_key.inputs.get("adapter_executable_digest") + != Some(&plugin.executable_digest) + || plan.layer_key.inputs.get("protocol") != Some(&protocol.to_string()) + || !action_policy_is_valid + || !plan.pre_commands.is_empty() + { + return Err(Error::Corrupt(format!( + "plugin component `{expected_component_id}` returned an inconsistent identity, provenance, or policy plan" + ))); + } + self.validate_workspace_environment_plan_common(plan) + } + + fn validate_workspace_environment_plan_common( + &self, + plan: &WorkspaceEnvironmentPlan, + ) -> Result<()> { + let mut tool_identities = BTreeMap::new(); + self.validate_workspace_environment_plan_common_with_tool_cache(plan, &mut tool_identities) + } + + fn validate_workspace_environment_plan_common_with_tool_cache( + &self, + plan: &WorkspaceEnvironmentPlan, + tool_identities: &mut BTreeMap, + ) -> Result<()> { + validate_environment_component_identity(&plan.component_id)?; + let mut dependencies = BTreeSet::new(); + for dependency in &plan.dependencies { + validate_environment_component_identity(&dependency.component_id)?; + if dependency.component_id == plan.component_id { + return Err(Error::InvalidInput(format!( + "environment component `{}` cannot depend on itself", + plan.component_id + ))); + } + if !dependencies.insert(&dependency.component_id) { + return Err(Error::InvalidInput(format!( + "environment component `{}` repeats dependency `{}`", + plan.component_id, dependency.component_id + ))); + } + } + if !plan.resolved_dependencies.is_empty() { + return Err(Error::Corrupt(format!( + "environment component `{}` supplied host-owned resolved dependency state", + plan.component_id + ))); + } + if plan + .layer_key + .inputs + .keys() + .any(|name| name.starts_with("dependency:")) + { + return Err(Error::InvalidInput(format!( + "environment component `{}` used the host-reserved `dependency:` key namespace", + plan.component_id + ))); + } + if plan.implementation_version.trim().is_empty() + || plan.distribution_digest.trim().is_empty() + || plan.layer_key.inputs.get("adapter_implementation") + != Some(&plan.implementation_version) + || plan.layer_key.inputs.get("adapter_distribution_digest") + != Some(&plan.distribution_digest) + { + return Err(Error::Corrupt(format!( + "component `{}` omitted adapter implementation provenance from its layer key", + plan.component_id + ))); + } + if plan.kind != plan.layer_key.kind { + return Err(Error::Corrupt(format!( + "component `{}` returned inconsistent layer kind", + plan.component_id + ))); + } + if !matches!( + plan.kind.as_str(), + "dependency" | "compiler-results" | "generated" | "build" | "external" + ) { + return Err(Error::InvalidInput(format!( + "component `{}` declared unsupported environment kind `{}`", + plan.component_id, plan.kind + ))); + } + let Some((_, _, contract_major)) = parse_canonical_adapter_identity(&plan.adapter_identity) + else { + return Err(Error::InvalidInput(format!( + "component `{}` declared malformed adapter identity `{}`", + plan.component_id, plan.adapter_identity + ))); + }; + if contract_major != plan.adapter_version { + return Err(Error::InvalidInput(format!( + "component `{}` adapter identity major does not match adapter_version", + plan.component_id + ))); + } + if plan.outputs.len() > 32 { + return Err(Error::InvalidInput(format!( + "component `{}` declares more than 32 outputs", + plan.component_id + ))); + } + let metadata_only = plan.outputs.is_empty(); + if metadata_only { + if plan.kind != "external" + || (plan.external_artifacts.is_empty() && plan.runtime_resources.is_empty()) + || plan.command.is_some() + || !plan.pre_commands.is_empty() + || !plan.mounted_commands.is_empty() + || !plan.caches.is_empty() + || plan.source_projection.is_some() + { + return Err(Error::InvalidInput(format!( + "component `{}` may omit filesystem outputs only as an action-free external/runtime component", + plan.component_id + ))); + } + } else if !plan.external_artifacts.is_empty() || !plan.runtime_resources.is_empty() { + return Err(Error::InvalidInput(format!( + "component `{}` mixes external/runtime resources with filesystem outputs; split independently owned resources into separate components", + plan.component_id + ))); + } + if plan.external_artifacts.len() > 32 { + return Err(Error::InvalidInput(format!( + "component `{}` declares more than 32 external artifacts", + plan.component_id + ))); + } + let mut external_names = BTreeSet::new(); + for artifact in &plan.external_artifacts { + validate_workspace_environment_external_artifact(artifact)?; + if !external_names.insert(&artifact.name) { + return Err(Error::InvalidInput(format!( + "component `{}` repeats external artifact `{}`", + plan.component_id, artifact.name + ))); + } + } + if !plan.external_artifacts.is_empty() { + let identity = workspace_external_artifacts_identity(&plan.external_artifacts)?; + if plan.layer_key.inputs.get("external_artifact_contract") != Some(&identity) { + return Err(Error::Corrupt(format!( + "component `{}` omitted its external artifact contract from the canonical key", + plan.component_id + ))); + } + } + if plan.runtime_resources.len() > 32 { + return Err(Error::InvalidInput(format!( + "component `{}` declares more than 32 runtime resources", + plan.component_id + ))); + } + let mut runtime_names = BTreeSet::new(); + for resource in &plan.runtime_resources { + validate_workspace_environment_runtime_resource(resource)?; + if !runtime_names.insert(&resource.name) { + return Err(Error::InvalidInput(format!( + "component `{}` repeats runtime resource `{}`", + plan.component_id, resource.name + ))); + } + if !external_names.contains(&resource.artifact_name) { + return Err(Error::InvalidInput(format!( + "component `{}` runtime resource `{}` references missing external artifact `{}`", + plan.component_id, resource.name, resource.artifact_name + ))); + } + } + if !plan.runtime_resources.is_empty() { + let identity = workspace_runtime_resources_identity(&plan.runtime_resources)?; + if plan.layer_key.inputs.get("runtime_resource_contract") != Some(&identity) { + return Err(Error::Corrupt(format!( + "component `{}` omitted its runtime resource contract from the canonical key", + plan.component_id + ))); + } + } + let mut output_names = BTreeSet::new(); + let mut output_policies = BTreeSet::new(); + let mut output_paths = Vec::<(&str, &str)>::new(); + let mut mount_paths = Vec::<(&str, &str)>::new(); + for output in &plan.outputs { + output_policies.insert(output.policy); + if output.name.is_empty() || !output_names.insert(output.name.clone()) { + return Err(Error::InvalidInput(format!( + "component `{}` has an empty or duplicate output name `{}`", + plan.component_id, output.name + ))); + } + normalize_relative_path(&output.mount_path)?; + normalize_relative_path(&output.output_path)?; + for (other_name, other_path) in &output_paths { + if environment_mounts_overlap(&output.output_path, other_path) { + return Err(Error::InvalidInput(format!( + "component `{}` output `{}` path overlaps output `{other_name}`", + plan.component_id, output.name + ))); + } + } + for (other_name, other_path) in &mount_paths { + if environment_mounts_overlap(&output.mount_path, other_path) { + return Err(Error::InvalidInput(format!( + "component `{}` output `{}` mount overlaps output `{other_name}`", + plan.component_id, output.name + ))); + } + } + output_paths.push((&output.name, &output.output_path)); + mount_paths.push((&output.name, &output.mount_path)); + } + if !metadata_only && output_policies.len() != 1 { + return Err(Error::InvalidInput(format!( + "component `{}` mixes immutable-seeded and writable-private outputs; split it into independently keyed components until heterogeneous action publication is available", + plan.component_id + ))); + } + if !plan.caches.is_empty() { + match plan.sandbox_policy { + WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin => {} + WorkspaceEnvironmentSandboxPolicy::RestrictedPluginStaging + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted + if plan.caches.iter().all(|cache| { + cache.access == WorkspaceEnvironmentCacheAccess::HostExclusive + }) => {} + _ => { + return Err(Error::InvalidInput(format!( + "component `{}` requests host cache namespaces without a certified cache access contract", + plan.component_id + ))); + } + } + } + let mut cache_names = BTreeSet::new(); + for cache in &plan.caches { + if !cache_names.insert(cache.name.clone()) { + return Err(Error::InvalidInput(format!( + "component `{}` repeats cache namespace `{}`", + plan.component_id, cache.name + ))); + } + let expected = self.declare_workspace_environment_cache( + &plan.adapter_identity, + &cache.name, + cache.protocol, + cache.access, + cache.compatibility.clone(), + )?; + if expected.namespace_id != cache.namespace_id + || expected.storage_path != cache.storage_path + { + return Err(Error::Corrupt(format!( + "component `{}` cache `{}` does not match its host-owned namespace identity", + plan.component_id, cache.name + ))); + } + } + if plan.mounted_commands.is_empty() { + if plan.layer_key.inputs.contains_key("mounted_action") { + return Err(Error::Corrupt(format!( + "component `{}` declares a mounted action identity without a mounted action", + plan.component_id + ))); + } + } else { + if !matches!( + plan.sandbox_policy, + WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted + ) || output_policies + != BTreeSet::from([WorkspaceEnvironmentOutputPolicy::WritablePrivate]) + { + return Err(Error::InvalidInput(format!( + "component `{}` may use mounted initialization only as a trusted built-in or authorized protocol-v2 plugin with writable-private outputs", + plan.component_id + ))); + } + let action_identity = workspace_mounted_commands_identity(&plan.mounted_commands)?; + if plan.layer_key.inputs.get("mounted_action") != Some(&action_identity) { + return Err(Error::Corrupt(format!( + "component `{}` omitted its mounted initialization action from the canonical key", + plan.component_id + ))); + } + } + for command in plan.pre_commands.iter().chain(plan.command.iter()) { + normalize_relative_path(&command.working_directory)?; + if command.program.trim().is_empty() { + return Err(Error::InvalidInput(format!( + "component `{}` has an empty build executable", + plan.component_id + ))); + } + let current_identity = + if let Some(identity) = tool_identities.get(&command.resolved_program) { + identity.clone() + } else { + let identity = workspace_tool_identity_for_path(&command.resolved_program)?; + tool_identities.insert(command.resolved_program.clone(), identity.clone()); + identity + }; + if !command.resolved_program.is_absolute() + || current_identity != command.executable_identity + { + return Err(Error::InvalidInput(format!( + "component `{}` executable identity for `{}` changed after planning", + plan.component_id, command.program + ))); + } + for name in command.environment.keys() { + if name.is_empty() + || !name + .chars() + .all(|ch| ch == '_' || ch.is_ascii_alphanumeric()) + { + return Err(Error::InvalidInput(format!( + "component `{}` has invalid environment key `{name}`", + plan.component_id + ))); + } + } + let mut command_caches = BTreeSet::new(); + for cache_name in &command.cache_names { + if !cache_names.contains(cache_name) || !command_caches.insert(cache_name) { + return Err(Error::InvalidInput(format!( + "component `{}` command references missing or duplicate cache `{cache_name}`", + plan.component_id + ))); + } + } + } + for command in &plan.mounted_commands { + if !command.working_directory.is_empty() { + normalize_relative_path(&command.working_directory)?; + } + if command.program.trim().is_empty() || !command.cache_names.is_empty() { + return Err(Error::InvalidInput(format!( + "component `{}` has an invalid mounted initialization command", + plan.component_id + ))); + } + let current_identity = + if let Some(identity) = tool_identities.get(&command.resolved_program) { + identity.clone() + } else { + let identity = workspace_tool_identity_for_path(&command.resolved_program)?; + tool_identities.insert(command.resolved_program.clone(), identity.clone()); + identity + }; + if !command.resolved_program.is_absolute() + || current_identity != command.executable_identity + { + return Err(Error::InvalidInput(format!( + "component `{}` mounted executable identity for `{}` changed after planning", + plan.component_id, command.program + ))); + } + for name in command.environment.keys() { + if name.is_empty() + || !name + .chars() + .all(|ch| ch == '_' || ch.is_ascii_alphanumeric()) + { + return Err(Error::InvalidInput(format!( + "component `{}` has invalid mounted environment key `{name}`", + plan.component_id + ))); + } + } + } + let mut staging_paths = BTreeSet::new(); + for input in &plan.inputs { + normalize_relative_path(&input.source_path)?; + let staging = normalize_relative_path(&input.staging_path)?; + if !staging_paths.insert(staging.clone()) { + return Err(Error::InvalidInput(format!( + "component `{}` maps more than one input to `{staging}`", + plan.component_id + ))); + } + } + if let Some((_, staging_root)) = &plan.source_projection { + normalize_relative_path(staging_root)?; + } + Ok(()) + } + + fn validate_environment_mounts_do_not_shadow_source( + &self, + source_root: &ObjectId, + plans: &[&WorkspaceEnvironmentPlan], + ) -> Result<()> { + let mut mounts = BTreeMap::::new(); + for plan in plans { + for output in &plan.outputs { + let mount_path = normalize_relative_path(&output.mount_path)?; + let folded = case_insensitive_path_key(&mount_path); + let first = folded.split('/').next().unwrap_or_default(); + if matches!(first, ".trail" | ".git") { + return Err(Error::InvalidInput(format!( + "environment component `{}` output `{}` cannot mount inside reserved path `{}`", + plan.component_id, output.name, output.mount_path + ))); + } + mounts.insert( + folded, + (plan.component_id.clone(), output.name.clone(), mount_path), + ); + } + } + self.for_each_root_file_chunk(source_root, 1024, |chunk| { + for (path, _) in chunk { + let folded_path = case_insensitive_path_key(&path); + if let Some((_, (component_id, output_name, mount))) = + environment_path_ancestor(&mounts, &folded_path) + { + return Err(Error::InvalidInput(format!( + "environment component `{component_id}` output `{output_name}` mount `{mount}` would shadow pinned source file `{path}`" + ))); + } + } + Ok(()) + }) + } + + fn prepare_workspace_environment_artifacts( + &self, + view_id: &str, + plan: &WorkspaceEnvironmentPlan, + component_key: &str, + ) -> Result { + let Some(policy) = plan.outputs.first().map(|output| output.policy) else { + if !plan.external_artifacts.is_empty() { + return Ok(PreparedEnvironmentArtifacts::MetadataOnly); + } + return Err(Error::Corrupt(format!( + "component `{}` reached execution without outputs or external artifacts", + plan.component_id + ))); + }; + match policy { + WorkspaceEnvironmentOutputPolicy::ImmutableSeedPrivate => self + .build_workspace_layer_singleflight(&plan.layer_key, |build_dir| { + self.execute_workspace_environment_plan(plan, build_dir) + }) + .map(PreparedEnvironmentArtifacts::Immutable), + WorkspaceEnvironmentOutputPolicy::WritablePrivate => { + if self.writable_private_outputs_are_compatible(view_id, plan, component_key)? { + return Ok(PreparedEnvironmentArtifacts::WritablePrivate(None)); + } + let outputs = self.execute_writable_private_environment_plan(plan)?; + Ok(PreparedEnvironmentArtifacts::WritablePrivate(Some(outputs))) + } + } + } + + /// Run path-sensitive initializers at the lane's final mountpoint without + /// exposing the current generation to partial output. + /// + /// The candidate view uses the pinned source root and desired immutable + /// bindings, but every writable class has a temporary upper. Existing + /// private outputs are presented as read-only lower directories. After all + /// commands succeed, only newly prepared writable-private outputs are + /// copied back into host staging; normal generation activation remains the + /// sole mutation of the real lane upper and SQLite bindings. + fn initialize_mounted_workspace_environment_plans( + &self, + lane: &str, + attempt_id: &str, + source_root: &ObjectId, + planned: &[(&WorkspaceEnvironmentPlan, &str)], + prepared: &mut [PreparedEnvironmentArtifacts], + removed_components: &[String], + ) -> Result<()> { + if planned.len() != prepared.len() { + return Err(Error::Corrupt( + "mounted environment initialization lost plan/artifact alignment".to_string(), + )); + } + let needs_initialization = + planned + .iter() + .zip(prepared.iter()) + .any(|((plan, _), artifacts)| { + !plan.mounted_commands.is_empty() + && matches!( + artifacts, + PreparedEnvironmentArtifacts::WritablePrivate(Some(_)) + ) + }); + if !needs_initialization { + return Ok(()); + } + + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` does not have a layered workspace view" + )) + })?; + let branch = self.lane_branch(lane)?; + let record = self.lane_record(&branch.lane_id)?; + let mode = self.lane_workdir_mode_for(&record, &branch)?; + if !mode.is_transparent_cow() { + return Err(Error::InvalidInput(format!( + "lane `{lane}` uses `{}` and cannot run mounted environment initialization", + mode.as_str() + ))); + } + + let candidate_root = self.db_dir.join("cache/staging"); + fs::create_dir_all(&candidate_root)?; + let candidate = tempfile::Builder::new() + .prefix(&format!("mounted-environment-{attempt_id}-")) + .tempdir_in(&candidate_root)?; + let candidate_source_upper = candidate.path().join("view/source-upper"); + let candidate_layout = ViewUpperLayout::from_source_upper(candidate_source_upper.clone()); + candidate_layout.ensure()?; + let real_layout = ViewUpperLayout::from_source_upper(PathBuf::from(&view.source_upper)); + + let replaced_components = planned + .iter() + .map(|(plan, _)| plan.component_id.clone()) + .chain(removed_components.iter().cloned()) + .collect::>(); + let desired_mounts = planned + .iter() + .flat_map(|(plan, _)| { + plan.outputs + .iter() + .map(|output| (plan.component_id.as_str(), output.mount_path.as_str())) + }) + .collect::>(); + let mut current_output_stmt = self.conn.prepare( + "SELECT component_id, mount_path + FROM environment_component_output_bindings + WHERE view_id = ?1", + )?; + let current_outputs = current_output_stmt + .query_map(params![&view.view_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()?; + for (owner, existing_mount) in ¤t_outputs { + if replaced_components.contains(owner) { + continue; + } + for (component, desired_mount) in &desired_mounts { + if environment_mounts_overlap(existing_mount, desired_mount) { + return Err(Error::InvalidInput(format!( + "environment component `{component}` mounted initializer would overlap active component `{owner}` at `{existing_mount}`" + ))); + } + } + } + let retired_mounts = + current_outputs + .iter() + .filter_map(|(component, mount)| { + replaced_components + .contains(component) + .then_some(mount.clone()) + }) + .chain(planned.iter().flat_map(|(plan, _)| { + plan.outputs.iter().map(|output| output.mount_path.clone()) + })) + .collect::>(); + + let mut bindings = + self.workspace_layer_bindings_for_source_upper(&real_layout.source_upper)?; + for binding in &mut bindings { + if binding.layer_id.is_none() { + let class = environment_upper_class(&binding.kind)?; + binding.storage_path = Some(safe_join( + real_layout.upper_for_class(class), + &binding.mount_path, + )?); + } + } + bindings.retain(|binding| !retired_mounts.contains(&binding.mount_path)); + + let mut candidate_outputs = Vec::<(PathBuf, PathBuf, String, String)>::new(); + let mut persisted_mounts = Vec::<(ViewPathClass, String)>::new(); + for ((plan, component_key), artifacts) in planned.iter().zip(prepared.iter()) { + let (layer_id, private_paths) = match artifacts { + PreparedEnvironmentArtifacts::Immutable(layer) => { + (Some(layer.layer_id.as_str()), None) + } + PreparedEnvironmentArtifacts::WritablePrivate(Some(outputs)) => { + (None, Some(outputs.paths.as_slice())) + } + PreparedEnvironmentArtifacts::WritablePrivate(None) => (None, None), + PreparedEnvironmentArtifacts::MetadataOnly => (None, None), + }; + let outputs = + environment_output_activations(plan, layer_id, component_key, private_paths)?; + for (output_index, output) in outputs.iter().enumerate() { + let class = environment_upper_class(&plan.kind)?; + let storage_path = match artifacts { + PreparedEnvironmentArtifacts::Immutable(layer) => { + let root = Path::new(&layer.storage_path); + Some(if output.layer_subpath.is_empty() { + root.to_path_buf() + } else { + safe_join(root, &output.layer_subpath)? + }) + } + PreparedEnvironmentArtifacts::WritablePrivate(Some(private)) => { + let seed = private.paths.get(output_index).ok_or_else(|| { + Error::Corrupt(format!( + "component `{}` lost writable-private seed `{}`", + plan.component_id, output.name + )) + })?; + let destination = + safe_join(candidate_layout.upper_for_class(class), &output.mount_path)?; + copy_dir_recursive(seed, &destination)?; + candidate_outputs.push(( + seed.clone(), + destination, + plan.component_id.clone(), + output.name.clone(), + )); + persisted_mounts.push((class, output.mount_path.clone())); + None + } + PreparedEnvironmentArtifacts::WritablePrivate(None) => Some(safe_join( + real_layout.upper_for_class(class), + &output.mount_path, + )?), + PreparedEnvironmentArtifacts::MetadataOnly => None, + }; + bindings.push(WorkspaceLayerBinding { + binding_identity: output.binding_identity.clone(), + layer_id: layer_id.map(str::to_string), + mount_path: output.mount_path.clone(), + storage_path, + kind: plan.kind.clone(), + priority: 100, + }); + } + } + bindings.sort_by(|left, right| { + right + .mount_path + .len() + .cmp(&left.mount_path.len()) + .then_with(|| right.priority.cmp(&left.priority)) + .then_with(|| left.mount_path.cmp(&right.mount_path)) + }); + + let run = || -> Result<()> { + let mut command_index = 0usize; + for ((plan, _), artifacts) in planned.iter().zip(prepared.iter()) { + if !matches!( + artifacts, + PreparedEnvironmentArtifacts::WritablePrivate(Some(_)) + ) { + continue; + } + for command in &plan.mounted_commands { + let process_root = candidate + .path() + .join("process") + .join(format!("{command_index:04}")); + self.run_mounted_workspace_environment_command( + lane, + plan, + command, + Path::new(&view.mountpoint), + &process_root, + )?; + command_index += 1; + } + } + Ok(()) + }; + + let run_result = match mode { + LaneWorkdirMode::FuseCow => { + let mount = self.mount_fuse_cow_workdir_for_lane_with_ephemeral_bindings( + lane, + candidate_source_upper.clone(), + source_root.clone(), + bindings, + )?; + let result = run(); + drop(mount); + result + } + LaneWorkdirMode::NfsCow => { + let mount = self.mount_nfs_cow_workdir_for_lane_with_ephemeral_bindings( + lane, + candidate_source_upper, + source_root.clone(), + bindings, + )?; + let result = run(); + drop(mount); + result + } + LaneWorkdirMode::DokanCow => { + #[cfg(target_os = "windows")] + { + let mount = self.mount_dokan_cow_workdir_for_lane_with_ephemeral_bindings( + lane, + candidate_source_upper, + source_root.clone(), + bindings, + )?; + let result = run(); + drop(mount); + result + } + #[cfg(not(target_os = "windows"))] + return Err(Error::InvalidInput( + "dokan-cow workdirs are currently supported only on Windows".to_string(), + )); + } + _ => unreachable!(), + }; + run_result?; + test_crash_point("environment_after_mounted_initialization"); + + validate_mounted_environment_candidate_writes(&candidate_layout, &persisted_mounts)?; + for (seed, output, component_id, output_name) in candidate_outputs { + if !output.is_dir() { + return Err(Error::InvalidInput(format!( + "mounted initialization for component `{component_id}` did not produce output `{output_name}`" + ))); + } + make_tree_writable(&seed); + if seed.exists() { + fs::remove_dir_all(&seed)?; + } + copy_dir_recursive(&output, &seed)?; + } + Ok(()) + } + + fn ensure_workspace_environment_source_root( + &self, + lane: &str, + expected: &ObjectId, + ) -> Result<()> { + let branch = self.lane_branch(lane)?; + let current = self.get_ref(&branch.ref_name)?.root_id; + if ¤t != expected { + return Err(Error::InvalidInput(format!( + "lane `{lane}` advanced from pinned source root `{expected}` to `{current}` during environment synchronization; retry against the new lane head" + ))); + } + Ok(()) + } + + fn run_mounted_workspace_environment_command( + &self, + lane: &str, + plan: &WorkspaceEnvironmentPlan, + command_plan: &WorkspaceEnvironmentCommand, + mountpoint: &Path, + process_root: &Path, + ) -> Result<()> { + let working_directory = if command_plan.working_directory.is_empty() { + mountpoint.to_path_buf() + } else { + safe_join(mountpoint, &command_plan.working_directory)? + }; + if !working_directory.is_dir() { + return Err(Error::InvalidInput(format!( + "mounted initialization working directory `{}` for component `{}` does not exist", + command_plan.working_directory, plan.component_id + ))); + } + let isolated_home = process_root.join("home"); + let isolated_tmp = process_root.join("tmp"); + fs::create_dir_all(&isolated_home)?; + fs::create_dir_all(&isolated_tmp)?; + let current_identity = workspace_tool_identity_for_path(&command_plan.resolved_program)?; + if current_identity != command_plan.executable_identity { + return Err(Error::InvalidInput(format!( + "mounted initialization executable `{}` changed after the component key was computed", + command_plan.program + ))); + } + let (launcher, launcher_args) = match plan.sandbox_policy { + WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin => ( + command_plan.resolved_program.clone(), + command_plan.args.iter().map(OsString::from).collect(), + ), + WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted => { + // Reuse the native recipe sandbox with the candidate mount as + // its read root and the declared final mount targets as its + // only writable paths. The adapter never learns candidate + // upper paths or receives mount authority. + let mut mounted_plan = plan.clone(); + for output in &mut mounted_plan.outputs { + output.output_path.clone_from(&output.mount_path); + } + for input in &mut mounted_plan.inputs { + input.staging_path.clone_from(&input.source_path); + } + let executable_directory = isolated_home.join("tool"); + fs::create_dir_all(&executable_directory)?; + let executable_name = + command_plan.resolved_program.file_name().ok_or_else(|| { + Error::Corrupt(format!( + "mounted plugin executable `{}` has no file name", + command_plan.resolved_program.display() + )) + })?; + let staged_executable = executable_directory.join(executable_name); + fs::copy(&command_plan.resolved_program, &staged_executable)?; + fs::set_permissions( + &staged_executable, + fs::metadata(&command_plan.resolved_program)?.permissions(), + )?; + let expected_digest = command_plan + .executable_identity + .rsplit_once(":sha256:") + .map(|(_, digest)| digest) + .ok_or_else(|| { + Error::Corrupt(format!( + "mounted plugin executable identity for `{}` has no digest", + command_plan.program + )) + })?; + let staged_digest = sha256_hex(&fs::read(&staged_executable)?); + if staged_digest != expected_digest { + return Err(Error::Corrupt(format!( + "staged mounted plugin executable `{}` failed identity verification", + command_plan.program + ))); + } + let mut sandboxed_command = command_plan.clone(); + sandboxed_command.resolved_program = staged_executable; + self.restricted_recipe_launcher( + &mounted_plan, + &sandboxed_command, + mountpoint, + &isolated_home, + &isolated_tmp, + )? + } + WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginStaging => { + return Err(Error::Corrupt(format!( + "repository command component `{}` unexpectedly requested mounted initialization", + plan.component_id + ))); + } + }; + let mut command = Command::new(launcher); + command + .args(launcher_args) + .current_dir(&working_directory) + .env_clear() + .envs(&command_plan.environment) + .env("HOME", &isolated_home) + .env("TMPDIR", &isolated_tmp) + .env("TMP", &isolated_tmp) + .env("TEMP", &isolated_tmp); + if plan.sandbox_policy == WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin { + command + .env("TRAIL_WORKSPACE", self.workspace_root()) + .env("TRAIL_LANE", lane) + .env("TRAIL_ENVIRONMENT_COMPONENT", &plan.component_id) + .env("TRAIL_ENVIRONMENT_INITIALIZATION", "1"); + } + if let Some(path) = std::env::var_os("PATH") { + command.env("PATH", path); + } + #[cfg(windows)] + for name in ["SystemRoot", "ComSpec", "PATHEXT"] { + if let Some(value) = std::env::var_os(name) { + command.env(name, value); + } + } + for name in &command_plan.remove_environment { + command.env_remove(name); + } + let status = + if plan.sandbox_policy == WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted { + run_supervised_mounted_plugin_process(&mut command).map_err(|err| { + Error::InvalidInput(format!( + "failed to supervise mounted initializer `{}` for component `{}`: {err}", + command_plan.program, plan.component_id + )) + })? + } else { + command.status().map_err(|err| { + Error::InvalidInput(format!( + "failed to launch mounted initializer `{}` for component `{}`: {err}", + command_plan.program, plan.component_id + )) + })? + }; + if !status.success() { + return Err(Error::InvalidInput(format!( + "mounted initialization for component `{}` failed with {status}; the previous environment generation remains active", + plan.component_id + ))); + } + Ok(()) + } + + fn writable_private_outputs_are_compatible( + &self, + view_id: &str, + plan: &WorkspaceEnvironmentPlan, + component_key: &str, + ) -> Result { + let mut stmt = self.conn.prepare( + "SELECT output_name, mount_path, policy, binding_identity + FROM environment_component_output_bindings + WHERE view_id = ?1 AND component_id = ?2 + ORDER BY output_name", + )?; + let actual = stmt + .query_map(params![view_id, &plan.component_id], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + })? + .collect::, _>>()?; + let mut expected = plan + .outputs + .iter() + .map(|output| { + ( + output.name.clone(), + output.mount_path.clone(), + output.policy.as_str().to_string(), + writable_private_binding_identity( + &plan.component_id, + &output.name, + component_key, + ), + ) + }) + .collect::>(); + expected.sort(); + if actual != expected { + return Ok(false); + } + let source_upper = self.conn.query_row( + "SELECT source_upper FROM workspace_views WHERE view_id = ?1", + params![view_id], + |row| row.get::<_, String>(0), + )?; + let layout = + super::workdir::ViewUpperLayout::from_source_upper(PathBuf::from(source_upper)); + let class = match plan.kind.as_str() { + "dependency" => super::workdir::ViewPathClass::Dependency, + "compiler-results" | "generated" | "build" => super::workdir::ViewPathClass::Generated, + _ => return Ok(false), + }; + for output in &plan.outputs { + let upper = safe_join(layout.upper_for_class(class), &output.mount_path)?; + if !upper.is_dir() { + return Ok(false); + } + } + Ok(true) + } + + fn execute_writable_private_environment_plan( + &self, + plan: &WorkspaceEnvironmentPlan, + ) -> Result { + if matches!( + plan.sandbox_policy, + WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginStaging + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted + ) { + ensure_restricted_recipe_sandbox_available()?; + } + #[cfg(target_os = "windows")] + let staging_parent = std::env::temp_dir(); + #[cfg(not(target_os = "windows"))] + let staging_parent = PathBuf::from("/tmp"); + if !staging_parent.is_dir() { + return Err(Error::InvalidInput( + "writable-private environment staging requires an available host temporary directory" + .to_string(), + )); + } + let staging = tempfile::Builder::new() + .prefix("trail-private-environment-") + .tempdir_in(staging_parent)?; + let root = fs::canonicalize(staging.path())?; + let restricted = matches!( + plan.sandbox_policy, + WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginStaging + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted + ); + let paths = + self.execute_workspace_environment_plan_in_directory(plan, &root, restricted)?; + if restricted { + let mut entries = 0usize; + for output in &paths { + self.validate_restricted_recipe_output(plan, output, &mut entries)?; + } + } + Ok(PreparedPrivateEnvironmentOutputs { + _staging: staging, + paths, + }) + } + + fn execute_workspace_environment_plan( + &self, + plan: &WorkspaceEnvironmentPlan, + build_dir: &Path, + ) -> Result { + if matches!( + plan.sandbox_policy, + WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginStaging + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted + ) { + ensure_restricted_recipe_sandbox_available()?; + #[cfg(target_os = "windows")] + let sandbox_parent = std::env::temp_dir(); + #[cfg(not(target_os = "windows"))] + let sandbox_parent = Path::new("/tmp"); + if !sandbox_parent.is_dir() { + return Err(Error::InvalidInput( + "restricted command recipes require an available host temporary directory" + .to_string(), + )); + } + let sandbox = tempfile::Builder::new() + .prefix("trail-environment-") + .tempdir_in(&sandbox_parent)?; + let sandbox_root = fs::canonicalize(sandbox.path())?; + let outputs = + self.execute_workspace_environment_plan_in_directory(plan, &sandbox_root, true)?; + let mut output_entries = 0usize; + for output in &outputs { + self.validate_restricted_recipe_output(plan, output, &mut output_entries)?; + } + if let [output] = outputs.as_slice() { + let destination = build_dir.join("recipe-output"); + if destination.exists() { + return Err(Error::Corrupt(format!( + "restricted recipe publication destination `{}` already exists", + destination.display() + ))); + } + copy_dir_recursive(output, &destination)?; + return Ok(destination); + } + return package_workspace_environment_outputs(build_dir, "recipe-outputs", &outputs); + } + let outputs = + self.execute_workspace_environment_plan_in_directory(plan, build_dir, false)?; + package_workspace_environment_outputs(build_dir, "published-outputs", &outputs) + } + + fn validate_restricted_recipe_output( + &self, + plan: &WorkspaceEnvironmentPlan, + output: &Path, + entries: &mut usize, + ) -> Result<()> { + for entry in walkdir::WalkDir::new(output).follow_links(false) { + let entry = entry.map_err(|err| { + Error::InvalidInput(format!( + "cannot inspect output for command component `{}`: {err}", + plan.component_id + )) + })?; + *entries += 1; + if *entries > 1_000_000 { + return Err(Error::InvalidInput(format!( + "command component `{}` output exceeds one million filesystem entries", + plan.component_id + ))); + } + let metadata = fs::symlink_metadata(entry.path())?; + let file_type = metadata.file_type(); + if file_type.is_symlink() { + return Err(Error::InvalidInput(format!( + "command component `{}` output contains unsupported symlink `{}`", + plan.component_id, + entry.path().display() + ))); + } + if !metadata.is_dir() && !metadata.is_file() { + return Err(Error::InvalidInput(format!( + "command component `{}` output contains unsupported filesystem entry `{}`", + plan.component_id, + entry.path().display() + ))); + } + #[cfg(unix)] + if metadata.is_file() { + use std::os::unix::fs::MetadataExt; + if metadata.nlink() != 1 { + return Err(Error::InvalidInput(format!( + "command component `{}` output contains hard-linked file `{}`", + plan.component_id, + entry.path().display() + ))); + } + if metadata.mode() & 0o6000 != 0 { + return Err(Error::InvalidInput(format!( + "command component `{}` output contains set-id file `{}`", + plan.component_id, + entry.path().display() + ))); + } + } + #[cfg(windows)] + { + use std::os::windows::fs::MetadataExt; + + if metadata.file_attributes() & winapi::um::winnt::FILE_ATTRIBUTE_REPARSE_POINT != 0 + { + return Err(Error::InvalidInput(format!( + "command component `{}` output contains unsupported reparse point `{}`", + plan.component_id, + entry.path().display() + ))); + } + if metadata.is_file() && windows_file_identity(entry.path())?.number_of_links != 1 { + return Err(Error::InvalidInput(format!( + "command component `{}` output contains hard-linked or unverifiable file `{}`", + plan.component_id, + entry.path().display() + ))); + } + } + } + Ok(()) + } + + fn validate_environment_plan_mount_collisions( + &self, + plans: &[WorkspaceEnvironmentPlan], + ) -> Result<()> { + let mut mounts = BTreeMap::::new(); + for plan in plans { + for output in &plan.outputs { + let folded = case_insensitive_path_key(&output.mount_path); + if let Some((_, (component_id, output_name, mount_path))) = + environment_path_overlap_in_map(&mounts, &folded) + { + return Err(Error::InvalidInput(format!( + "environment component `{}` output `{}` mount `{}` overlaps component `{component_id}` output `{output_name}` mount `{mount_path}`", + plan.component_id, output.name, output.mount_path + ))); + } + mounts.insert( + folded, + (&plan.component_id, &output.name, &output.mount_path), + ); + } + } + Ok(()) + } + + fn execute_workspace_environment_plan_in_directory( + &self, + plan: &WorkspaceEnvironmentPlan, + build_dir: &Path, + create_output_before_command: bool, + ) -> Result> { + if let Some((source_root, staging_root)) = &plan.source_projection { + self.for_each_root_file_chunk(source_root, 1024, |chunk| { + for (path, entry) in chunk { + let staging_path = format!("{staging_root}/{path}"); + self.materialize_workspace_environment_input(build_dir, &staging_path, &entry)?; + } + Ok(()) + })?; + } + for input in &plan.inputs { + self.materialize_workspace_environment_input( + build_dir, + &input.staging_path, + &input.entry, + )?; + } + let mut outputs = Vec::with_capacity(plan.outputs.len()); + for declaration in &plan.outputs { + let output = safe_join(build_dir, &declaration.output_path)?; + if create_output_before_command && declaration.create_if_missing && !output.exists() { + fs::create_dir_all(&output)?; + } + outputs.push(output); + } + for command in &plan.pre_commands { + self.run_workspace_environment_command(plan, command, build_dir)?; + } + if let Some(command) = &plan.command { + self.run_workspace_environment_command(plan, command, build_dir)?; + } + for (declaration, output) in plan.outputs.iter().zip(&outputs) { + if declaration.create_if_missing && !output.exists() { + fs::create_dir_all(output)?; + } + if !output.is_dir() { + return Err(Error::InvalidInput(format!( + "environment build for component `{}` did not produce declared output `{}` at `{}`", + plan.component_id, declaration.name, declaration.output_path + ))); + } + } + Ok(outputs) + } + + fn acquire_workspace_environment_cache_uses( + &self, + plan: &WorkspaceEnvironmentPlan, + command: &WorkspaceEnvironmentCommand, + ) -> Result> { + let by_name = plan + .caches + .iter() + .map(|cache| (cache.name.as_str(), cache)) + .collect::>(); + let mut names = command.cache_names.clone(); + names.sort(); + names.dedup(); + let mut guards = Vec::with_capacity(names.len()); + for name in names { + let cache = by_name.get(name.as_str()).ok_or_else(|| { + Error::Corrupt(format!( + "component `{}` command lost cache declaration `{name}`", + plan.component_id + )) + })?; + if let Ok(metadata) = fs::symlink_metadata(&cache.storage_path) { + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(Error::InvalidInput(format!( + "environment cache namespace `{}` is not a real directory", + cache.namespace_id + ))); + } + } + fs::create_dir_all(&cache.storage_path)?; + let namespace_parent = fs::canonicalize(self.db_dir.join("cache/namespaces"))?; + let canonical_storage = fs::canonicalize(&cache.storage_path)?; + if canonical_storage != namespace_parent.join(&cache.namespace_id) { + return Err(Error::InvalidInput(format!( + "environment cache namespace `{}` escapes host cache storage", + cache.namespace_id + ))); + } + let compatibility_json = serde_json::to_vec(&cache.compatibility)?; + self.conn.execute( + "INSERT OR IGNORE INTO environment_cache_namespaces + (namespace_id, adapter_identity, cache_name, protocol, access, authority, scope, compatibility_json, storage_path, last_used_at, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, 'performance_only', 'workspace', ?6, ?7, ?8, ?8)", + params![ + &cache.namespace_id, + &plan.adapter_identity, + &cache.name, + cache.protocol.as_str(), + cache.access.as_str(), + &compatibility_json, + cache.storage_path.to_string_lossy(), + now_ts() + ], + )?; + let changed = self.conn.execute( + "UPDATE environment_cache_namespaces SET last_used_at = ?1 + WHERE namespace_id = ?2 AND adapter_identity = ?3 AND cache_name = ?4 + AND protocol = ?5 AND access = ?6 AND authority = 'performance_only' + AND scope = 'workspace' AND compatibility_json = ?7 AND storage_path = ?8", + params![ + now_ts(), + &cache.namespace_id, + &plan.adapter_identity, + &cache.name, + cache.protocol.as_str(), + cache.access.as_str(), + &compatibility_json, + cache.storage_path.to_string_lossy() + ], + )?; + if changed != 1 { + return Err(Error::Corrupt(format!( + "environment cache namespace `{}` has conflicting provenance", + cache.namespace_id + ))); + } + + let token = format!("{}:{}", std::process::id(), current_process_start_token()); + let lease_dir = self + .db_dir + .join("cache/namespace-leases") + .join(&cache.namespace_id); + fs::create_dir_all(&lease_dir)?; + let maintenance_dir = self.db_dir.join("cache/namespace-maintenance"); + fs::create_dir_all(&maintenance_dir)?; + let maintenance_path = maintenance_dir.join(format!("{}.lock", cache.namespace_id)); + let lease_deadline = Instant::now() + Duration::from_secs(300); + let lease_path = loop { + if maintenance_path.exists() { + if environment_cache_owner_file_is_stale(&maintenance_path)? { + let _ = fs::remove_file(&maintenance_path); + continue; + } + if Instant::now() >= lease_deadline { + return Err(Error::InvalidInput(format!( + "timed out waiting for environment cache maintenance `{}`", + cache.name + ))); + } + thread::sleep(Duration::from_millis(50)); + continue; + } + let lease_path = lease_dir.join(format!( + "lease_{}", + crate::ids::short_hash( + format!("{token}:{}:{:?}", now_nanos(), thread::current().id()).as_bytes(), + 24 + ) + )); + let mut lease = OpenOptions::new() + .write(true) + .create_new(true) + .open(&lease_path)?; + lease.write_all(token.as_bytes())?; + lease.sync_all()?; + if maintenance_path.exists() { + let _ = fs::remove_file(&lease_path); + continue; + } + break lease_path; + }; + + let mut use_guard = WorkspaceEnvironmentCacheUseGuard { + lease_path, + exclusive: None, + }; + let exclusive = if cache.access == WorkspaceEnvironmentCacheAccess::HostExclusive { + let lock_dir = self.db_dir.join("cache/namespace-locks"); + fs::create_dir_all(&lock_dir)?; + let lock_path = lock_dir.join(format!("{}.lock", cache.namespace_id)); + let deadline = Instant::now() + Duration::from_secs(300); + loop { + match OpenOptions::new() + .write(true) + .create_new(true) + .open(&lock_path) + { + Ok(mut lock) => { + lock.write_all(token.as_bytes())?; + lock.sync_all()?; + break Some((lock_path, token.clone())); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + if environment_cache_owner_file_is_stale(&lock_path)? { + let stale = lock_path.with_extension(format!( + "stale.{}", + crate::ids::short_hash( + format!("{}:{}", now_nanos(), token).as_bytes(), + 16 + ) + )); + let _ = fs::rename(&lock_path, stale); + continue; + } + if Instant::now() >= deadline { + return Err(Error::InvalidInput(format!( + "timed out waiting for exclusive environment cache `{}`", + cache.name + ))); + } + thread::sleep(Duration::from_millis(50)); + } + Err(error) => return Err(Error::Io(error)), + } + } + } else { + None + }; + use_guard.exclusive = exclusive; + guards.push(use_guard); + } + Ok(guards) + } + + fn run_workspace_environment_command( + &self, + plan: &WorkspaceEnvironmentPlan, + command_plan: &WorkspaceEnvironmentCommand, + build_dir: &Path, + ) -> Result<()> { + let _cache_uses = self.acquire_workspace_environment_cache_uses(plan, command_plan)?; + let working_directory = safe_join(build_dir, &command_plan.working_directory)?; + let isolated_home = build_dir.join(".trail-home"); + let isolated_tmp = build_dir.join(".trail-tmp"); + fs::create_dir_all(&working_directory)?; + fs::create_dir_all(&isolated_home)?; + fs::create_dir_all(&isolated_tmp)?; + let current_identity = workspace_tool_identity_for_path(&command_plan.resolved_program)?; + if current_identity != command_plan.executable_identity { + return Err(Error::InvalidInput(format!( + "environment build executable `{}` changed after the component key was computed", + command_plan.program + ))); + } + let (launcher, launcher_args) = match plan.sandbox_policy { + WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin => ( + command_plan.resolved_program.clone(), + command_plan.args.iter().map(OsString::from).collect(), + ), + WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + | WorkspaceEnvironmentSandboxPolicy::RestrictedPluginStaging => self + .restricted_recipe_launcher( + plan, + command_plan, + build_dir, + &isolated_home, + &isolated_tmp, + )?, + WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted => self + .restricted_recipe_launcher( + plan, + command_plan, + build_dir, + &isolated_home, + &isolated_tmp, + )?, + }; + let mut command = Command::new(launcher); + command + .args(launcher_args) + .current_dir(&working_directory) + .env_clear() + .envs(&command_plan.environment) + .env("HOME", &isolated_home) + .env("TMPDIR", &isolated_tmp) + .env("TMP", &isolated_tmp) + .env("TEMP", &isolated_tmp); + if let Some(path) = std::env::var_os("PATH") { + command.env("PATH", path); + } + #[cfg(windows)] + for name in ["SystemRoot", "ComSpec", "PATHEXT"] { + if let Some(value) = std::env::var_os(name) { + command.env(name, value); + } + } + for name in &command_plan.remove_environment { + command.env_remove(name); + } + let status = command.status().map_err(|err| { + Error::InvalidInput(format!( + "failed to launch `{}` for component `{}`: {err}", + command_plan.program, plan.component_id + )) + })?; + if !status.success() { + return Err(Error::InvalidInput(format!( + "environment build for component `{}` failed with {status}", + plan.component_id + ))); + } + Ok(()) + } + + fn restricted_recipe_launcher( + &self, + plan: &WorkspaceEnvironmentPlan, + command_plan: &WorkspaceEnvironmentCommand, + build_dir: &Path, + isolated_home: &Path, + isolated_tmp: &Path, + ) -> Result<(PathBuf, Vec)> { + let cache_paths = command_plan + .cache_names + .iter() + .map(|name| { + plan.caches + .iter() + .find(|cache| &cache.name == name) + .map(|cache| cache.storage_path.clone()) + .ok_or_else(|| { + Error::Corrupt(format!( + "component `{}` lost cache declaration `{name}` before sandbox launch", + plan.component_id + )) + }) + }) + .collect::>>()?; + #[cfg(target_os = "macos")] + { + let launcher = PathBuf::from("/usr/bin/sandbox-exec"); + if !launcher.is_file() { + return Err(Error::InvalidInput( + "restricted command recipes require `/usr/bin/sandbox-exec` on macOS" + .to_string(), + )); + } + let build_dir = fs::canonicalize(build_dir)?; + let outputs = plan + .outputs + .iter() + .map(|output| -> Result { + let path = safe_join(&build_dir, &output.output_path)?; + Ok(fs::canonicalize(path)?) + }) + .collect::>>()?; + let caches = cache_paths + .iter() + .map(fs::canonicalize) + .collect::>>()?; + let readable_inputs = plan + .inputs + .iter() + .map(|input| -> Result { + Ok(fs::canonicalize(safe_join( + &build_dir, + &input.staging_path, + )?)?) + }) + .collect::>>()?; + let working_directory = if command_plan.working_directory.is_empty() { + build_dir.clone() + } else { + fs::canonicalize(safe_join(&build_dir, &command_plan.working_directory)?)? + }; + let isolated_home = fs::canonicalize(isolated_home)?; + let isolated_tmp = fs::canonicalize(isolated_tmp)?; + let executable = command_plan.resolved_program.clone(); + let canonical_executable = fs::canonicalize(&command_plan.resolved_program)?; + let executable_rules = if canonical_executable == executable { + format!("(literal \"{}\")", sandbox_profile_escape(&executable)) + } else { + format!( + "(literal \"{}\") (literal \"{}\")", + sandbox_profile_escape(&executable), + sandbox_profile_escape(&canonical_executable) + ) + }; + let output_rules = outputs + .iter() + .chain(&caches) + .map(|output| format!("(subpath \"{}\")", sandbox_profile_escape(output))) + .collect::>() + .join(" "); + let input_rules = readable_inputs + .iter() + .map(|input| format!("(literal \"{}\")", sandbox_profile_escape(input))) + .collect::>() + .join(" "); + let profile = format!( + "(version 1)\n\ + (deny default)\n\ + (import \"system.sb\")\n\ + (deny mach-lookup)\n\ + (deny mach-register)\n\ + (deny process-fork)\n\ + (deny process-exec)\n\ + (allow process-exec {})\n\ + (deny file-write*)\n\ + (allow file-write* {} (subpath \"{}\") (subpath \"{}\"))\n\ + (deny file-read* (subpath \"/Users\") (subpath \"/Volumes\") (subpath \"/private/etc\") (subpath \"/private/var\"))\n\ + (allow file-read-metadata (subpath \"{}\"))\n\ + (allow file-read* (literal \"{}\") {} {} (subpath \"{}\") (subpath \"{}\") (literal \"{}\") (literal \"{}\") (subpath \"/bin\") (subpath \"/usr\") (subpath \"/System\") (subpath \"/Library\") (subpath \"/opt/homebrew\") (subpath \"/nix/store\"))\n\ + (deny network*)", + executable_rules, + output_rules, + sandbox_profile_escape(&isolated_home), + sandbox_profile_escape(&isolated_tmp), + sandbox_profile_escape(&build_dir), + sandbox_profile_escape(&working_directory), + input_rules, + output_rules, + sandbox_profile_escape(&isolated_home), + sandbox_profile_escape(&isolated_tmp), + sandbox_profile_escape(&executable), + sandbox_profile_escape(&canonical_executable), + ); + let mut args = vec![ + OsString::from("-p"), + OsString::from(profile), + executable.as_os_str().to_owned(), + ]; + args.extend(command_plan.args.iter().map(OsString::from)); + Ok((launcher, args)) + } + #[cfg(all(target_os = "linux", not(test)))] + { + let launcher = std::env::current_exe()?; + let build_dir = fs::canonicalize(build_dir)?; + let outputs = plan + .outputs + .iter() + .map(|output| -> Result { + let path = safe_join(&build_dir, &output.output_path)?; + Ok(fs::canonicalize(path)?) + }) + .collect::>>()?; + let caches = cache_paths + .iter() + .map(fs::canonicalize) + .collect::>>()?; + let readable_inputs = plan + .inputs + .iter() + .map(|input| -> Result { + Ok(fs::canonicalize(safe_join( + &build_dir, + &input.staging_path, + )?)?) + }) + .collect::>>()?; + let isolated_home = fs::canonicalize(isolated_home)?; + let isolated_tmp = fs::canonicalize(isolated_tmp)?; + let executable = fs::canonicalize(&command_plan.resolved_program)?; + let mut args = vec![ + OsString::from("__environment-sandbox"), + OsString::from("--root"), + build_dir.into_os_string(), + ]; + for input in readable_inputs { + args.push(OsString::from("--read")); + args.push(input.into_os_string()); + } + for output in outputs { + args.push(OsString::from("--output")); + args.push(output.into_os_string()); + } + for cache in caches { + args.push(OsString::from("--cache")); + args.push(cache.into_os_string()); + } + args.extend([ + OsString::from("--home"), + isolated_home.into_os_string(), + OsString::from("--tmp"), + isolated_tmp.into_os_string(), + OsString::from("--program"), + executable.into_os_string(), + OsString::from("--"), + ]); + args.extend(command_plan.args.iter().map(OsString::from)); + Ok((launcher, args)) + } + #[cfg(all(target_os = "windows", not(test)))] + { + let launcher = std::env::current_exe()?; + let build_dir = fs::canonicalize(build_dir)?; + let outputs = plan + .outputs + .iter() + .map(|output| -> Result { + let path = safe_join(&build_dir, &output.output_path)?; + Ok(fs::canonicalize(path)?) + }) + .collect::>>()?; + let caches = cache_paths + .iter() + .map(fs::canonicalize) + .collect::>>()?; + let readable_inputs = plan + .inputs + .iter() + .map(|input| -> Result { + Ok(fs::canonicalize(safe_join( + &build_dir, + &input.staging_path, + )?)?) + }) + .collect::>>()?; + let isolated_home = fs::canonicalize(isolated_home)?; + let isolated_tmp = fs::canonicalize(isolated_tmp)?; + let executable = fs::canonicalize(&command_plan.resolved_program)?; + let mut args = vec![ + OsString::from("__environment-sandbox"), + OsString::from("--root"), + build_dir.into_os_string(), + ]; + for input in readable_inputs { + args.push(OsString::from("--read")); + args.push(input.into_os_string()); + } + for output in outputs { + args.push(OsString::from("--output")); + args.push(output.into_os_string()); + } + for cache in caches { + args.push(OsString::from("--cache")); + args.push(cache.into_os_string()); + } + args.extend([ + OsString::from("--home"), + isolated_home.into_os_string(), + OsString::from("--tmp"), + isolated_tmp.into_os_string(), + OsString::from("--program"), + executable.into_os_string(), + OsString::from("--"), + ]); + args.extend(command_plan.args.iter().map(OsString::from)); + Ok((launcher, args)) + } + #[cfg(any( + all(target_os = "linux", test), + all(target_os = "windows", test), + not(any(target_os = "macos", target_os = "linux", target_os = "windows")) + ))] + { + let _ = ( + plan, + command_plan, + build_dir, + isolated_home, + isolated_tmp, + cache_paths, + ); + Err(Error::InvalidInput(format!( + "restricted command recipe sandboxing is unavailable on {}; Trail refuses to run the repository command without kernel enforcement", + std::env::consts::OS + ))) + } + } + + fn materialize_workspace_environment_input( + &self, + build_dir: &Path, + staging_path: &str, + entry: &FileEntry, + ) -> Result<()> { + let destination = safe_join(build_dir, staging_path)?; + if let Some(parent) = destination.parent() { + fs::create_dir_all(parent)?; + } + let projection = self.project_entry_file(entry)?; + fs::copy(projection, &destination)?; + let mut permissions = fs::metadata(&destination)?.permissions(); + permissions.set_readonly(false); + #[cfg(unix)] + permissions.set_mode(if entry.executable { + 0o755 + } else { + entry.mode & 0o666 + }); + fs::set_permissions(&destination, permissions)?; + // Mounted immutable source entries deliberately expose a stable epoch + // mtime. Match it in staging so build systems such as Cargo do not + // reject a target seed solely because host projection happened later. + OpenOptions::new() + .write(true) + .open(&destination)? + .set_modified(SystemTime::UNIX_EPOCH)?; + Ok(()) + } + + pub fn workspace_environment_status( + &self, + lane: &str, + ) -> Result> { + self.workspace_environment_rows(lane) + } + + /// Component-oriented status surface. Unlike the legacy dependency report, + /// this keeps the logical component and versioned adapter identities + /// separate so multiple components can use the same adapter safely. + pub fn environment_component_status( + &self, + lane: &str, + ) -> Result> { + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` does not have a layered workspace view" + )) + })?; + let mut stmt = self.conn.prepare( + "SELECT view_id, component_id, adapter_identity, adapter_version, implementation_version, distribution_digest, kind, expected_key, attached_key, status, reason, updated_at FROM environment_component_states WHERE view_id = ?1 ORDER BY component_id", + )?; + let reports = stmt + .query_map(params![view.view_id], |row| { + let component_id = row.get::<_, String>(1)?; + let adapter_identity = row.get::<_, String>(2)?; + let adapter_version = row.get::<_, u32>(3)?; + let (namespace, name, contract_major) = + split_adapter_identity(&adapter_identity, adapter_version); + Ok(EnvironmentComponentStateReport { + view_id: row.get(0)?, + component: EnvironmentComponentIdentityReport { + component_id, + kind: row.get(6)?, + }, + adapter: EnvironmentAdapterIdentityReport { + namespace, + name, + contract_major, + implementation_version: row.get(4)?, + distribution_digest: row.get(5)?, + }, + expected_key: row.get(7)?, + attached_key: row.get(8)?, + status: row.get(9)?, + reason: row.get(10)?, + updated_at: row.get(11)?, + }) + })? + .collect::, _>>()?; + Ok(reports) + } + + pub fn active_environment_generation( + &self, + lane: &str, + ) -> Result> { + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` does not have a layered workspace view" + )) + })?; + let generation = self + .conn + .query_row( + "SELECT g.generation_id, g.view_id, g.generation_sequence, g.source_root, + g.specification_digest, g.predecessor_generation_id, g.state, + g.created_at, g.activated_at, g.retired_at + FROM environment_view_generations a + JOIN environment_generations g ON g.generation_id = a.generation_id + WHERE a.view_id = ?1", + params![&view.view_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, u64>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, String>(6)?, + row.get::<_, i64>(7)?, + row.get::<_, Option>(8)?, + row.get::<_, Option>(9)?, + )) + }, + ) + .optional()?; + let Some(( + generation_id, + view_id, + generation_sequence, + source_root, + specification_digest, + predecessor_generation_id, + state, + created_at, + activated_at, + retired_at, + )) = generation + else { + return Ok(None); + }; + let mut stmt = self.conn.prepare( + "SELECT component_id, adapter_identity, kind, component_key, layer_id, mount_path + FROM environment_generation_components + WHERE generation_id = ?1 ORDER BY component_id", + )?; + let mut components = stmt + .query_map(params![&generation_id], |row| { + Ok(EnvironmentGenerationComponentReport { + component_id: row.get(0)?, + adapter_identity: row.get(1)?, + kind: row.get(2)?, + component_key: row.get(3)?, + layer_id: row.get(4)?, + mount_path: row.get(5)?, + dependencies: Vec::new(), + outputs: Vec::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + }) + })? + .collect::, _>>()?; + for component in &mut components { + let mut dependency_stmt = self.conn.prepare( + "SELECT dependency_component_id, dependency_component_key, edge_type + FROM environment_generation_edges + WHERE generation_id = ?1 AND component_id = ?2 + ORDER BY dependency_component_id", + )?; + component.dependencies = dependency_stmt + .query_map(params![&generation_id, &component.component_id], |row| { + Ok(EnvironmentGenerationDependencyReport { + component_id: row.get(0)?, + component_key: row.get(1)?, + edge_type: row.get(2)?, + }) + })? + .collect::, _>>()?; + let mut output_stmt = self.conn.prepare( + "SELECT output_name, policy, storage_identity, layer_id, mount_path, layer_subpath + FROM environment_generation_outputs + WHERE generation_id = ?1 AND component_id = ?2 + ORDER BY output_name", + )?; + component.outputs = output_stmt + .query_map(params![&generation_id, &component.component_id], |row| { + Ok(EnvironmentGenerationOutputReport { + name: row.get(0)?, + policy: row.get(1)?, + storage_identity: row.get(2)?, + layer_id: row.get(3)?, + mount_path: row.get(4)?, + layer_subpath: row.get(5)?, + }) + })? + .collect::, _>>()?; + let mut cache_stmt = self.conn.prepare( + "SELECT cache_name, namespace_id, protocol, access, compatibility_json + FROM environment_generation_caches + WHERE generation_id = ?1 AND component_id = ?2 + ORDER BY cache_name", + )?; + component.caches = cache_stmt + .query_map(params![&generation_id, &component.component_id], |row| { + let compatibility = row.get::<_, Vec>(4)?; + let compatibility = + serde_json::from_slice(&compatibility).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 4, + rusqlite::types::Type::Blob, + Box::new(error), + ) + })?; + Ok(EnvironmentCacheReport { + name: row.get(0)?, + namespace_id: row.get(1)?, + protocol: row.get(2)?, + access: row.get(3)?, + authority: "performance_only".to_string(), + scope: "workspace".to_string(), + compatibility, + }) + })? + .collect::, _>>()?; + let mut external_stmt = self.conn.prepare( + "SELECT artifact_name, artifact_type, provider, reference, digest, platform, cleanup_owner + FROM environment_generation_external_artifacts + WHERE generation_id = ?1 AND component_id = ?2 + ORDER BY artifact_name", + )?; + component.external_artifacts = external_stmt + .query_map(params![&generation_id, &component.component_id], |row| { + Ok(EnvironmentExternalArtifactReport { + name: row.get(0)?, + artifact_type: row.get(1)?, + provider: row.get(2)?, + reference: row.get(3)?, + digest: row.get(4)?, + platform: row.get(5)?, + cleanup_owner: row.get(6)?, + }) + })? + .collect::, _>>()?; + let mut runtime_stmt = self.conn.prepare( + "SELECT resource_name, runtime_type, provider, artifact_name, + image_reference, image_digest, image_platform, container_port, + protocol, health_type, health_timeout_ms, restart_policy, + cleanup_owner, volume_target, allocation_id, provider_resource_id, + container_name, network_name, volume_name, host_port, status, + health_status, reason, created_at, updated_at, started_at, stopped_at + FROM environment_generation_runtime_resources + WHERE generation_id = ?1 AND component_id = ?2 + ORDER BY resource_name", + )?; + component.runtime_resources = runtime_stmt + .query_map(params![&generation_id, &component.component_id], |row| { + Ok(EnvironmentRuntimeResourceReport { + declaration: EnvironmentRuntimeDeclarationReport { + name: row.get(0)?, + runtime_type: row.get(1)?, + provider: row.get(2)?, + artifact_name: row.get(3)?, + container_port: row.get(7)?, + protocol: row.get(8)?, + health_type: row.get(9)?, + health_timeout_ms: row.get(10)?, + restart_policy: row.get(11)?, + cleanup_owner: row.get(12)?, + volume_target: row.get(13)?, + secrets: Vec::new(), + }, + image_reference: row.get(4)?, + image_digest: row.get(5)?, + image_platform: row.get(6)?, + allocation_id: row.get(14)?, + provider_resource_id: row.get(15)?, + container_name: row.get(16)?, + network_name: row.get(17)?, + volume_name: row.get(18)?, + host_port: row.get(19)?, + status: row.get(20)?, + health_status: row.get(21)?, + reason: row.get(22)?, + created_at: row.get(23)?, + updated_at: row.get(24)?, + started_at: row.get(25)?, + stopped_at: row.get(26)?, + secret_statuses: Vec::new(), + }) + })? + .collect::, _>>()?; + for resource in &mut component.runtime_resources { + let mut secret_stmt = self.conn.prepare( + "SELECT secret_name, provider, reference, version, purpose, injection, + target, environment, required, status, reason, resolved_at, updated_at + FROM environment_generation_runtime_secrets + WHERE generation_id = ?1 AND component_id = ?2 AND resource_name = ?3 + ORDER BY secret_name", + )?; + let statuses = secret_stmt + .query_map( + params![ + &generation_id, + &component.component_id, + &resource.declaration.name + ], + |row| { + let reference = EnvironmentSecretReferenceReport { + name: row.get(0)?, + provider: row.get(1)?, + reference: row.get(2)?, + version: row.get(3)?, + purpose: row.get(4)?, + injection: row.get(5)?, + target: row.get(6)?, + environment: row.get(7)?, + required: row.get(8)?, + }; + Ok(EnvironmentSecretStatusReport { + reference, + status: row.get(9)?, + reason: row.get(10)?, + resolved_at: row.get(11)?, + updated_at: row.get(12)?, + }) + }, + )? + .collect::, _>>()?; + resource.declaration.secrets = statuses + .iter() + .map(|status| status.reference.clone()) + .collect(); + resource.secret_statuses = statuses; + } + } + Ok(Some(EnvironmentGenerationReport { + generation_id, + view_id, + generation_sequence, + source_root: ObjectId(source_root), + specification_digest, + predecessor_generation_id, + state, + components, + created_at, + activated_at, + retired_at, + })) + } + + pub(crate) fn workspace_environment_rows( + &self, + lane: &str, + ) -> Result> { + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` does not have a layered workspace view" + )) + })?; + let mut stmt = self.conn.prepare( + "SELECT view_id, adapter, expected_key, attached_key, status, reason, updated_at FROM workspace_environment_states WHERE view_id = ?1 ORDER BY adapter", + )?; + let reports = stmt + .query_map(params![view.view_id], |row| { + Ok(WorkspaceEnvironmentReport { + view_id: row.get(0)?, + adapter: row.get(1)?, + expected_key: row.get(2)?, + attached_key: row.get(3)?, + status: row.get(4)?, + reason: row.get(5)?, + updated_at: row.get(6)?, + }) + })? + .collect::, _>>()?; + Ok(reports) + } + + pub fn explain_workspace_environment_staleness( + &self, + lane: &str, + component_id: &str, + ) -> Result { + self.explain_workspace_environment_staleness_page(lane, component_id, 0, 256) + } + + pub fn explain_workspace_environment_staleness_page( + &self, + lane: &str, + component_id: &str, + offset: u64, + limit: u64, + ) -> Result { + if limit == 0 || limit > 1_000 { + return Err(Error::InvalidInput( + "environment explanation limit must be between 1 and 1000".to_string(), + )); + } + let state = self + .environment_component_status(lane)? + .into_iter() + .find(|state| state.component.component_id == component_id) + .ok_or_else(|| { + Error::InvalidInput(format!( + "environment component `{component_id}` is not recorded for lane `{lane}`" + )) + })?; + let branch = self.lane_branch(lane)?; + let head = self.get_ref(&branch.ref_name)?; + let discovery = self.discover_workspace_environment(lane, None)?; + if !discovery.conflicts.is_empty() { + return Err(Error::InvalidInput(format!( + "environment discovery found {} conflict(s) while explaining staleness", + discovery.conflicts.len() + ))); + } + let Some(component) = discovery + .components + .iter() + .find(|component| component.component_id == component_id) + else { + return paginate_stale_explanation( + component_id, + "stale", + state.expected_key, + state.attached_key, + true, + vec![EnvironmentStaleChangeReport { + dimension: "component".to_string(), + name: component_id.to_string(), + change: "removed_or_adapter_unavailable".to_string(), + }], + offset, + limit, + ); + }; + let finalized = + self.plan_discovered_environment_graph(&head.root_id, &discovery.components)?; + let (plan, expected_key) = finalized + .into_iter() + .find(|(plan, _)| plan.component_id == component.component_id) + .ok_or_else(|| { + Error::Corrupt(format!( + "environment graph lost discovered component `{component_id}`" + )) + })?; + let (complete, changes) = match state.attached_key.as_deref() { + Some(attached_key) if attached_key == expected_key => (true, Vec::new()), + Some(attached_key) => match self.workspace_layer_key_by_cache_key(attached_key)? { + Some(previous) => (true, diff_workspace_layer_keys(&previous, &plan.layer_key)), + None => ( + false, + vec![EnvironmentStaleChangeReport { + dimension: "provenance".to_string(), + name: "canonical_layer_key".to_string(), + change: "unavailable_for_legacy_or_missing_layer".to_string(), + }], + ), + }, + None => ( + true, + vec![EnvironmentStaleChangeReport { + dimension: "attachment".to_string(), + name: component_id.to_string(), + change: "not_attached".to_string(), + }], + ), + }; + let status = if state.attached_key.as_deref() == Some(expected_key.as_str()) { + "ready".to_string() + } else { + "stale".to_string() + }; + paginate_stale_explanation( + component_id, + &status, + expected_key, + state.attached_key, + complete, + changes, + offset, + limit, + ) + } + + pub fn refresh_workspace_environment_staleness(&self, lane: &str) -> Result<()> { + let branch = self.lane_branch(lane)?; + let head = self.get_ref(&branch.ref_name)?; + let discovery = self.discover_workspace_environment(lane, None)?; + if !discovery.conflicts.is_empty() { + return Err(Error::InvalidInput(format!( + "environment discovery found {} conflict(s) while refreshing staleness", + discovery.conflicts.len() + ))); + } + let discovered = discovery + .components + .iter() + .map(|component| (component.component_id.clone(), component)) + .collect::>(); + let finalized = + self.plan_discovered_environment_graph(&head.root_id, &discovery.components)?; + let mut finalized = finalized + .into_iter() + .map(|(plan, key)| (plan.component_id.clone(), (plan, key))) + .collect::>(); + let mut planned_states = Vec::new(); + for state in self.environment_component_status(lane)? { + let component_id = state.component.component_id; + if !discovered.contains_key(&component_id) { + self.mark_environment_component_stale_without_plan( + lane, + &component_id, + "component is no longer discovered or its adapter is not installed", + )?; + continue; + } + let (plan, expected_key) = finalized.remove(&component_id).ok_or_else(|| { + Error::Corrupt(format!( + "environment graph lost discovered component `{component_id}`" + )) + })?; + planned_states.push((state.attached_key, plan, expected_key)); + } + for (attached_key, plan, expected_key) in planned_states { + let ready = attached_key.as_deref() == Some(expected_key.as_str()); + let exact_reason = if ready { + None + } else if let Some(attached_key) = attached_key.as_deref() { + self.workspace_layer_key_by_cache_key(attached_key)? + .map(|previous| { + format_stale_changes(&diff_workspace_layer_keys(&previous, &plan.layer_key)) + }) + .filter(|reason| !reason.is_empty()) + } else { + Some("component has no attached environment artifact".to_string()) + }; + let reason = (!ready).then(|| { + exact_reason + .as_deref() + .unwrap_or(plan.stale_reason.as_str()) + }); + self.set_workspace_environment_state( + lane, + &plan, + &expected_key, + attached_key.as_deref(), + if ready { "ready" } else { "stale" }, + reason, + )?; + } + Ok(()) + } + + fn plan_discovered_environment_component( + &self, + source_root: &ObjectId, + component: &EnvironmentDiscoveredComponentReport, + ) -> Result { + if component.adapter_identity + == super::workspace_recipe::COMMAND_RECIPE_ADAPTER_METADATA.canonical_identity + { + let plan = self.command_recipe_plan(source_root, &component.component_id)?; + self.validate_command_recipe_plan(&component.component_id, &plan)?; + return Ok(plan); + } + if let Some(adapter) = builtin_environment_adapter_for_selector(&component.adapter_identity) + { + let plan = adapter.plan(self, source_root, &component.component_root)?; + self.validate_workspace_environment_plan(adapter, &component.component_root, &plan)?; + return Ok(plan); + } + let plugin = self + .environment_plugin_for_selector(&component.adapter_identity)? + .ok_or_else(|| { + Error::InvalidInput(format!( + "adapter `{}` is no longer installed", + component.adapter_identity + )) + })?; + let plan = self.plan_environment_plugin_component( + &plugin, + source_root, + &component.component_root, + &component.component_id, + )?; + self.validate_environment_plugin_plan(&plugin, &component.component_id, &plan)?; + Ok(plan) + } + + fn plan_discovered_environment_components( + &self, + source_root: &ObjectId, + components: &[EnvironmentDiscoveredComponentReport], + ) -> Result> { + let recipe_identity = + super::workspace_recipe::COMMAND_RECIPE_ADAPTER_METADATA.canonical_identity; + let recipe_ids = components + .iter() + .filter(|component| component.adapter_identity == recipe_identity) + .map(|component| component.component_id.clone()) + .collect::>(); + let mut recipe_plans = if recipe_ids.is_empty() { + BTreeMap::new() + } else { + self.command_recipe_plans(source_root, &recipe_ids)? + }; + let mut tool_identities = BTreeMap::new(); + let mut plans = Vec::with_capacity(components.len()); + for component in components { + let plan = if component.adapter_identity == recipe_identity { + let plan = recipe_plans + .remove(&component.component_id) + .ok_or_else(|| { + Error::Corrupt(format!( + "batch recipe planner lost component `{}`", + component.component_id + )) + })?; + self.validate_command_recipe_plan_with_tool_cache( + &component.component_id, + &plan, + &mut tool_identities, + )?; + plan + } else { + self.plan_discovered_environment_component(source_root, component)? + }; + plans.push(plan); + } + Ok(plans) + } + + pub(super) fn plan_discovered_environment_graph( + &self, + source_root: &ObjectId, + components: &[EnvironmentDiscoveredComponentReport], + ) -> Result> { + let raw_plans = self.plan_discovered_environment_components(source_root, components)?; + self.validate_environment_plan_mount_collisions(&raw_plans)?; + self.validate_environment_mounts_do_not_shadow_source( + source_root, + &raw_plans.iter().collect::>(), + )?; + self.finalize_workspace_environment_plan_graph(raw_plans) + } + + fn mark_environment_component_stale_without_plan( + &self, + lane: &str, + component_id: &str, + reason: &str, + ) -> Result<()> { + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` does not have a layered workspace view" + )) + })?; + let changed = self.conn.execute( + "UPDATE environment_component_states SET status = 'stale', reason = ?1, updated_at = ?2 WHERE view_id = ?3 AND component_id = ?4", + params![reason, now_ts(), &view.view_id, component_id], + )?; + if changed != 1 { + return Err(Error::Corrupt(format!( + "environment component `{component_id}` disappeared during stale refresh" + ))); + } + self.conn.execute( + "UPDATE workspace_environment_states SET status = 'stale', reason = ?1, updated_at = ?2 WHERE view_id = ?3 AND adapter = ?4", + params![reason, now_ts(), &view.view_id, component_id], + )?; + Ok(()) + } + + pub(crate) fn set_workspace_environment_state( + &self, + lane: &str, + plan: &WorkspaceEnvironmentPlan, + expected_key: &str, + attached_key: Option<&str>, + status: &str, + reason: Option<&str>, + ) -> Result<()> { + if !matches!(status, "building" | "ready" | "stale" | "failed") { + return Err(Error::InvalidInput(format!( + "invalid workspace environment status `{status}`" + ))); + } + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` does not have a layered workspace view" + )) + })?; + // Keep the legacy projection and normalized component state in one + // SQLite savepoint. Readers can therefore never observe one identity + // model advancing without the other. + self.conn + .execute_batch("SAVEPOINT trail_environment_state")?; + let writes = (|| -> Result<()> { + self.conn.execute( + "INSERT OR REPLACE INTO workspace_environment_states (view_id, adapter, expected_key, attached_key, status, reason, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + view.view_id, + plan.component_id, + expected_key, + attached_key, + status, + reason, + now_ts() + ], + )?; + self.conn.execute( + "INSERT OR REPLACE INTO environment_component_states (view_id, component_id, adapter_identity, adapter_version, implementation_version, distribution_digest, kind, expected_key, attached_key, status, reason, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", + params![ + view.view_id, + plan.component_id, + plan.adapter_identity, + plan.adapter_version, + plan.implementation_version, + plan.distribution_digest, + plan.kind, + expected_key, + attached_key, + status, + reason, + now_ts() + ], + )?; + Ok(()) + })(); + match writes { + Ok(()) => { + self.conn + .execute_batch("RELEASE SAVEPOINT trail_environment_state")?; + Ok(()) + } + Err(err) => { + let _ = self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT trail_environment_state; RELEASE SAVEPOINT trail_environment_state", + ); + Err(err) + } + } + } +} + +fn run_supervised_mounted_plugin_process( + command: &mut Command, +) -> Result { + let mut child = command.spawn()?; + let child_pid = child.id(); + let child_start_token = match process_start_token(child_pid) { + Some(token) => token, + None => { + if let Some(status) = child.try_wait()? { + return Ok(status); + } + let _ = child.kill(); + let _ = child.wait(); + return Err(Error::InvalidInput(format!( + "cannot authenticate mounted plugin process {child_pid} for parent-death supervision" + ))); + } + }; + let mut watchdog = match Command::new(std::env::current_exe()?) + .arg("__process-watchdog") + .arg(std::process::id().to_string()) + .arg(child_pid.to_string()) + .arg(&child_start_token) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + { + Ok(watchdog) => watchdog, + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(Error::Io(error)); + } + }; + loop { + if let Some(status) = child.try_wait()? { + let watchdog_status = watchdog.wait()?; + if !watchdog_status.success() { + return Err(Error::InvalidInput(format!( + "mounted plugin process watchdog failed with {watchdog_status}" + ))); + } + return Ok(status); + } + if let Some(watchdog_status) = watchdog.try_wait()? { + let _ = child.kill(); + let _ = child.wait(); + return Err(Error::InvalidInput(format!( + "mounted plugin process watchdog exited early with {watchdog_status}" + ))); + } + thread::sleep(Duration::from_millis(20)); + } +} + +pub(crate) fn workspace_mounted_commands_identity( + commands: &[WorkspaceEnvironmentCommand], +) -> Result { + let commands = commands + .iter() + .map(|command| { + let mut remove_environment = command.remove_environment.clone(); + remove_environment.sort(); + serde_json::json!({ + "program": command.program, + "resolved_program": command.resolved_program.to_string_lossy(), + "executable_identity": command.executable_identity, + "args": command.args, + "working_directory": command.working_directory, + "environment": command.environment, + "remove_environment": remove_environment, + }) + }) + .collect::>(); + Ok(sha256_hex(&serde_json::to_vec(&commands)?)) +} + +fn environment_upper_class(kind: &str) -> Result { + match kind { + "dependency" => Ok(ViewPathClass::Dependency), + "compiler-results" | "generated" | "build" => Ok(ViewPathClass::Generated), + other => Err(Error::InvalidInput(format!( + "environment kind `{other}` cannot own a mounted writable output" + ))), + } +} + +fn validate_mounted_environment_candidate_writes( + layout: &ViewUpperLayout, + persisted_mounts: &[(ViewPathClass, String)], +) -> Result<()> { + let generated_mounts = persisted_mounts + .iter() + .filter_map(|(class, mount)| { + matches!(class, ViewPathClass::Dependency | ViewPathClass::Generated) + .then_some(mount.as_str()) + }) + .collect::>(); + for (label, root, allowed, ignore_internal) in [ + ( + "source", + layout.source_upper.as_path(), + Vec::<&str>::new(), + true, + ), + ( + "generated", + layout.generated_upper.as_path(), + generated_mounts, + false, + ), + ( + "scratch", + layout.scratch_upper.as_path(), + Vec::<&str>::new(), + false, + ), + ] { + for entry in walkdir::WalkDir::new(root).follow_links(false) { + let entry = entry.map_err(|err| { + Error::InvalidInput(format!( + "cannot inspect mounted initialization {label} upper: {err}" + )) + })?; + if entry.path() == root { + continue; + } + let relative = entry + .path() + .strip_prefix(root) + .map_err(|_| Error::Corrupt("candidate upper escaped its root".to_string()))? + .to_string_lossy() + .replace('\\', "/"); + if ignore_internal && (relative == ".trail" || relative.starts_with(".trail/")) { + continue; + } + if allowed + .iter() + .any(|mount| environment_mounts_overlap(&relative, mount)) + { + continue; + } + return Err(Error::InvalidInput(format!( + "mounted environment initialization wrote undeclared {label} path `{relative}`; only newly prepared writable-private outputs may persist" + ))); + } + } + Ok(()) +} + +fn split_adapter_identity(identity: &str, fallback_major: u32) -> (String, String, u32) { + let Some((namespace, remainder)) = identity.split_once('/') else { + return ("legacy".to_string(), identity.to_string(), fallback_major); + }; + let Some((name, major)) = remainder.rsplit_once('@') else { + return (namespace.to_string(), remainder.to_string(), fallback_major); + }; + ( + namespace.to_string(), + name.to_string(), + major.parse().unwrap_or(fallback_major), + ) +} + +fn parse_canonical_adapter_identity(identity: &str) -> Option<(String, String, u32)> { + let (namespace, remainder) = identity.split_once('/')?; + let (name, major) = remainder.rsplit_once('@')?; + if namespace.is_empty() || name.is_empty() || namespace.contains('@') || name.contains('/') { + return None; + } + Some((namespace.to_string(), name.to_string(), major.parse().ok()?)) +} + +fn environment_mounts_overlap(left: &str, right: &str) -> bool { + left == right + || left + .strip_prefix(right) + .is_some_and(|rest| rest.starts_with('/')) + || right + .strip_prefix(left) + .is_some_and(|rest| rest.starts_with('/')) +} + +fn environment_path_ancestor<'a, T>( + paths: &'a BTreeMap, + path: &str, +) -> Option<(&'a str, &'a T)> { + if let Some((stored, value)) = paths.get_key_value(path) { + return Some((stored, value)); + } + let mut prefix = String::new(); + let mut segments = path.split('/').peekable(); + while let Some(segment) = segments.next() { + if !prefix.is_empty() { + prefix.push('/'); + } + prefix.push_str(segment); + if segments.peek().is_some() { + if let Some((stored, value)) = paths.get_key_value(&prefix) { + return Some((stored, value)); + } + } + } + None +} + +fn environment_path_overlap_in_map<'a, T>( + paths: &'a BTreeMap, + path: &str, +) -> Option<(&'a str, &'a T)> { + environment_path_ancestor(paths, path).or_else(|| { + paths + .range(path.to_string()..) + .next() + .filter(|(stored, _)| stored.starts_with(&format!("{path}/"))) + .map(|(stored, value)| (stored.as_str(), value)) + }) +} + +fn diff_workspace_layer_keys( + previous: &WorkspaceLayerKeyV1, + current: &WorkspaceLayerKeyV1, +) -> Vec { + let mut changes = Vec::new(); + diff_layer_key_map("input", &previous.inputs, ¤t.inputs, &mut changes); + diff_layer_key_map( + "tool", + &previous.tool_versions, + ¤t.tool_versions, + &mut changes, + ); + for (name, previous, current) in [ + ("kind", previous.kind.as_str(), current.kind.as_str()), + ( + "adapter", + previous.adapter.as_str(), + current.adapter.as_str(), + ), + ( + "platform", + previous.platform.as_str(), + current.platform.as_str(), + ), + ( + "architecture", + previous.architecture.as_str(), + current.architecture.as_str(), + ), + ( + "portability_scope", + previous.portability_scope.as_str(), + current.portability_scope.as_str(), + ), + ( + "strategy", + previous.strategy.as_str(), + current.strategy.as_str(), + ), + ] { + if previous != current { + changes.push(EnvironmentStaleChangeReport { + dimension: "policy".to_string(), + name: name.to_string(), + change: "modified".to_string(), + }); + } + } + if previous.adapter_version != current.adapter_version { + changes.push(EnvironmentStaleChangeReport { + dimension: "policy".to_string(), + name: "adapter_version".to_string(), + change: "modified".to_string(), + }); + } + if changes.is_empty() && previous != current { + changes.push(EnvironmentStaleChangeReport { + dimension: "canonical_key".to_string(), + name: "serialized_value".to_string(), + change: "modified".to_string(), + }); + } + changes +} + +fn diff_layer_key_map( + dimension: &str, + previous: &BTreeMap, + current: &BTreeMap, + changes: &mut Vec, +) { + for name in previous + .keys() + .chain(current.keys()) + .collect::>() + { + let change = match (previous.get(name), current.get(name)) { + (None, Some(_)) => Some("added"), + (Some(_), None) => Some("removed"), + (Some(left), Some(right)) if left != right => Some("modified"), + _ => None, + }; + if let Some(change) = change { + changes.push(EnvironmentStaleChangeReport { + dimension: dimension.to_string(), + name: if dimension == "input" { + name.strip_prefix("input:").unwrap_or(name).to_string() + } else { + name.clone() + }, + change: change.to_string(), + }); + } + } +} + +fn format_stale_changes(changes: &[EnvironmentStaleChangeReport]) -> String { + const MAX_NAMES: usize = 12; + if changes.is_empty() { + return String::new(); + } + let mut rendered = changes + .iter() + .take(MAX_NAMES) + .map(|change| format!("{}:{} {}", change.dimension, change.name, change.change)) + .collect::>(); + if changes.len() > MAX_NAMES { + rendered.push(format!("and {} more", changes.len() - MAX_NAMES)); + } + format!("environment identity changed: {}", rendered.join(", ")) +} + +#[allow(clippy::too_many_arguments)] +fn paginate_stale_explanation( + component_id: &str, + status: &str, + expected_key: String, + attached_key: Option, + provenance_complete: bool, + changes: Vec, + offset: u64, + limit: u64, +) -> Result { + let total_changes = changes.len() as u64; + let start = usize::try_from(offset) + .unwrap_or(usize::MAX) + .min(changes.len()); + let limit = usize::try_from(limit).unwrap_or(usize::MAX); + let end = start.saturating_add(limit).min(changes.len()); + let next_offset = (end < changes.len()).then_some(end as u64); + Ok(EnvironmentStaleExplanationReport { + component_id: component_id.to_string(), + status: status.to_string(), + expected_key, + attached_key, + complete: provenance_complete && start == 0 && next_offset.is_none(), + provenance_complete, + total_changes, + offset, + next_offset, + changes: changes[start..end].to_vec(), + }) +} + +fn environment_output_activations( + plan: &WorkspaceEnvironmentPlan, + layer_id: Option<&str>, + component_key: &str, + private_paths: Option<&[PathBuf]>, +) -> Result> { + if private_paths.is_some_and(|paths| paths.len() != plan.outputs.len()) { + return Err(Error::Corrupt(format!( + "component `{}` produced a different number of private outputs than it declared", + plan.component_id + ))); + } + let package_outputs = layer_id.is_some() && plan.outputs.len() > 1; + plan.outputs + .iter() + .enumerate() + .map(|(index, output)| { + let binding_identity = match output.policy { + WorkspaceEnvironmentOutputPolicy::ImmutableSeedPrivate => layer_id + .ok_or_else(|| { + Error::Corrupt(format!( + "component `{}` immutable output `{}` has no prepared layer", + plan.component_id, output.name + )) + })? + .to_string(), + WorkspaceEnvironmentOutputPolicy::WritablePrivate => { + if layer_id.is_some() { + return Err(Error::Corrupt(format!( + "component `{}` writable-private output `{}` unexpectedly has a layer", + plan.component_id, output.name + ))); + } + writable_private_binding_identity( + &plan.component_id, + &output.name, + component_key, + ) + } + }; + Ok(EnvironmentLayerOutputActivation { + name: output.name.clone(), + mount_path: output.mount_path.clone(), + policy: output.policy.as_str().to_string(), + binding_identity, + private_seed: private_paths.map(|paths| paths[index].clone()), + layer_subpath: if package_outputs { + format!("outputs/{index:04}") + } else { + String::new() + }, + }) + }) + .collect() +} + +fn environment_dependency_activations( + plan: &WorkspaceEnvironmentPlan, +) -> Result> { + if plan.dependencies.len() != plan.resolved_dependencies.len() { + return Err(Error::Corrupt(format!( + "finalized environment component `{}` lost resolved dependency provenance", + plan.component_id + ))); + } + plan.resolved_dependencies + .iter() + .map(|dependency| { + if let Some(input_name) = dependency + .edge_type + .identity_input_name(&dependency.component_id) + { + if plan.layer_key.inputs.get(&input_name) != Some(&dependency.component_key) { + return Err(Error::Corrupt(format!( + "finalized environment component `{}` omitted identity edge `{}`", + plan.component_id, input_name + ))); + } + } + Ok(( + dependency.component_id.clone(), + dependency.component_key.clone(), + dependency.edge_type.as_str().to_string(), + )) + }) + .collect() +} + +fn environment_cache_report(cache: &WorkspaceEnvironmentCache) -> EnvironmentCacheReport { + EnvironmentCacheReport { + name: cache.name.clone(), + namespace_id: cache.namespace_id.clone(), + protocol: cache.protocol.as_str().to_string(), + access: cache.access.as_str().to_string(), + authority: "performance_only".to_string(), + scope: "workspace".to_string(), + compatibility: cache.compatibility.clone(), + } +} + +fn environment_cache_activations(plan: &WorkspaceEnvironmentPlan) -> Vec { + plan.caches.iter().map(environment_cache_report).collect() +} + +fn environment_external_artifact_report( + artifact: &WorkspaceEnvironmentExternalArtifact, +) -> EnvironmentExternalArtifactReport { + EnvironmentExternalArtifactReport { + name: artifact.name.clone(), + artifact_type: artifact.artifact_type.clone(), + provider: artifact.provider.clone(), + reference: artifact.reference.clone(), + digest: artifact.digest.clone(), + platform: artifact.platform.clone(), + cleanup_owner: artifact.cleanup_owner.clone(), + } +} + +fn environment_external_artifact_activations( + plan: &WorkspaceEnvironmentPlan, +) -> Vec { + let mut artifacts = plan + .external_artifacts + .iter() + .map(environment_external_artifact_report) + .collect::>(); + artifacts.sort_by(|left, right| left.name.cmp(&right.name)); + artifacts +} + +pub(super) fn workspace_external_artifacts_identity( + artifacts: &[WorkspaceEnvironmentExternalArtifact], +) -> Result { + let mut reports = artifacts + .iter() + .map(environment_external_artifact_report) + .collect::>(); + reports.sort_by(|left, right| left.name.cmp(&right.name)); + Ok(serde_json::to_string(&reports)?) +} + +fn environment_runtime_resource_report( + resource: &WorkspaceEnvironmentRuntimeResource, +) -> EnvironmentRuntimeDeclarationReport { + EnvironmentRuntimeDeclarationReport { + name: resource.name.clone(), + runtime_type: resource.runtime_type.clone(), + provider: resource.provider.clone(), + artifact_name: resource.artifact_name.clone(), + container_port: resource.container_port, + protocol: resource.protocol.clone(), + health_type: resource.health_type.clone(), + health_timeout_ms: resource.health_timeout_ms, + restart_policy: resource.restart_policy.clone(), + cleanup_owner: resource.cleanup_owner.clone(), + volume_target: resource.volume_target.clone(), + secrets: resource + .secrets + .iter() + .map(|secret| EnvironmentSecretReferenceReport { + name: secret.name.clone(), + provider: secret.provider.clone(), + reference: secret.reference.clone(), + version: secret.version.clone(), + purpose: secret.purpose.clone(), + injection: secret.injection.clone(), + target: secret.target.clone(), + environment: secret.environment.clone(), + required: secret.required, + }) + .collect(), + } +} + +fn environment_runtime_resource_activations( + plan: &WorkspaceEnvironmentPlan, +) -> Vec { + let mut resources = plan + .runtime_resources + .iter() + .map(environment_runtime_resource_report) + .collect::>(); + resources.sort_by(|left, right| left.name.cmp(&right.name)); + resources +} + +pub(super) fn workspace_runtime_resources_identity( + resources: &[WorkspaceEnvironmentRuntimeResource], +) -> Result { + let mut reports = resources + .iter() + .map(environment_runtime_resource_report) + .collect::>(); + reports.sort_by(|left, right| left.name.cmp(&right.name)); + Ok(serde_json::to_string(&reports)?) +} + +fn validate_workspace_environment_runtime_resource( + resource: &WorkspaceEnvironmentRuntimeResource, +) -> Result<()> { + validate_environment_runtime_declaration_report(&environment_runtime_resource_report(resource)) +} + +pub(super) fn validate_environment_runtime_declaration_report( + resource: &EnvironmentRuntimeDeclarationReport, +) -> Result<()> { + if resource.name.is_empty() + || resource.name.len() > 128 + || !resource.name.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') + }) + || resource.runtime_type != "container" + || resource.provider != "oci" + || resource.artifact_name.is_empty() + || resource.container_port == 0 + || resource.protocol != "tcp" + || resource.health_type != "tcp" + || !(1_000..=300_000).contains(&resource.health_timeout_ms) + || !matches!( + resource.restart_policy.as_str(), + "never" | "on_failure" | "always" + ) + || resource.cleanup_owner != "trail" + { + return Err(Error::InvalidInput(format!( + "invalid runtime resource declaration `{}`", + resource.name + ))); + } + if let Some(target) = resource.volume_target.as_deref() { + if target.len() > 4096 + || !target.starts_with('/') + || target.contains('\\') + || target.chars().any(char::is_control) + || target + .split('/') + .skip(1) + .any(|segment| segment.is_empty() || segment == "." || segment == "..") + || ["/proc", "/sys", "/dev", "/run", "/etc"] + .iter() + .any(|reserved| target == *reserved || target.starts_with(&format!("{reserved}/"))) + { + return Err(Error::InvalidInput(format!( + "runtime resource `{}` has invalid or reserved volume target `{target}`", + resource.name + ))); + } + } + if resource.secrets.len() > 16 { + return Err(Error::InvalidInput(format!( + "runtime resource `{}` declares more than 16 secret references", + resource.name + ))); + } + let mut secret_names = BTreeSet::new(); + let mut secret_targets = BTreeSet::new(); + for secret in &resource.secrets { + if secret.name.is_empty() + || secret.name.len() > 128 + || !secret.name.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') + }) + || !secret_names.insert(secret.name.clone()) + || !matches!(secret.provider.as_str(), "file" | "environment_file") + || secret.reference.is_empty() + || secret.reference.len() > 4096 + || secret.reference.chars().any(char::is_control) + || secret.purpose.is_empty() + || secret.purpose.len() > 256 + || secret.purpose.chars().any(char::is_control) + || secret.injection != "file" + || !secret.target.starts_with("/run/secrets/") + || secret.target.len() > 4096 + || secret.target.contains('\\') + || secret.target.chars().any(char::is_control) + || secret + .target + .split('/') + .skip(1) + .any(|segment| segment.is_empty() || segment == "." || segment == "..") + || !secret_targets.insert(secret.target.clone()) + || secret.environment.as_deref().is_some_and(|name| { + name.is_empty() + || name.len() > 128 + || name.as_bytes()[0].is_ascii_digit() + || !name.bytes().all(|byte| { + byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_' + }) + }) + || secret.version.as_deref().is_some_and(|version| { + version.is_empty() || version.len() > 256 || version.chars().any(char::is_control) + }) + || secret.provider == "file" && !secret.reference.starts_with('/') + || secret.provider == "environment_file" + && !secret.reference.chars().all(|character| { + character.is_ascii_uppercase() || character.is_ascii_digit() || character == '_' + }) + { + return Err(Error::InvalidInput(format!( + "runtime resource `{}` has invalid secret reference `{}`", + resource.name, secret.name + ))); + } + } + Ok(()) +} + +fn validate_workspace_environment_external_artifact( + artifact: &WorkspaceEnvironmentExternalArtifact, +) -> Result<()> { + validate_environment_external_artifact_report(&environment_external_artifact_report(artifact)) +} + +pub(super) fn validate_environment_external_artifact_report( + artifact: &EnvironmentExternalArtifactReport, +) -> Result<()> { + if artifact.name.is_empty() + || artifact.name.len() > 128 + || !artifact.name.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '-' | '_') + }) + || artifact.artifact_type != "oci_image" + || artifact.provider != "oci" + || artifact.cleanup_owner != "external" + { + return Err(Error::InvalidInput(format!( + "invalid external artifact declaration `{}`", + artifact.name + ))); + } + let digest = artifact.digest.strip_prefix("sha256:").ok_or_else(|| { + Error::InvalidInput(format!( + "external artifact `{}` requires a sha256 digest", + artifact.name + )) + })?; + if digest.len() != 64 + || !digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(Error::InvalidInput(format!( + "external artifact `{}` has a malformed sha256 digest", + artifact.name + ))); + } + let (repository, reference_digest) = artifact.reference.rsplit_once('@').ok_or_else(|| { + Error::InvalidInput(format!( + "external OCI artifact `{}` must use a digest-pinned reference", + artifact.name + )) + })?; + if repository.is_empty() + || repository.len() > 2048 + || repository.contains('@') + || repository.starts_with('/') + || repository.ends_with('/') + || !repository + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"._-/:".contains(&byte)) + || reference_digest != artifact.digest + { + return Err(Error::InvalidInput(format!( + "external OCI artifact `{}` has an invalid or mismatched reference", + artifact.name + ))); + } + let Some((operating_system, architecture)) = artifact.platform.split_once('/') else { + return Err(Error::InvalidInput(format!( + "external OCI artifact `{}` requires an os/architecture platform", + artifact.name + ))); + }; + if !matches!(operating_system, "linux" | "windows") + || !matches!(architecture, "amd64" | "arm64") + { + return Err(Error::InvalidInput(format!( + "external OCI artifact `{}` uses unsupported platform `{}`", + artifact.name, artifact.platform + ))); + } + Ok(()) +} + +struct WorkspaceEnvironmentCacheUseGuard { + lease_path: PathBuf, + exclusive: Option<(PathBuf, String)>, +} + +impl Drop for WorkspaceEnvironmentCacheUseGuard { + fn drop(&mut self) { + let _ = fs::remove_file(&self.lease_path); + if let Some((path, token)) = &self.exclusive { + if fs::read_to_string(path) + .ok() + .is_some_and(|owner| owner == *token) + { + let _ = fs::remove_file(path); + } + } + } +} + +fn environment_cache_owner_file_is_stale(path: &Path) -> Result { + let value = match fs::read_to_string(path) { + Ok(value) => value, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(true), + Err(error) => return Err(Error::Io(error)), + }; + let Some((pid, start_token)) = value.split_once(':') else { + return environment_cache_malformed_owner_is_stale(path); + }; + let Ok(pid) = pid.parse::() else { + return environment_cache_malformed_owner_is_stale(path); + }; + Ok(!process_matches_start_token(pid, start_token)) +} + +fn environment_cache_malformed_owner_is_stale(path: &Path) -> Result { + let modified = fs::metadata(path)?.modified()?; + Ok(SystemTime::now() + .duration_since(modified) + .unwrap_or_default() + >= Duration::from_secs(5)) +} + +pub(super) fn environment_cache_namespace_has_live_leases( + db_dir: &Path, + namespace_id: &str, + cleanup_stale: bool, +) -> Result { + let lease_dir = db_dir.join("cache/namespace-leases").join(namespace_id); + let entries = match fs::read_dir(&lease_dir) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(Error::Io(error)), + }; + let mut live = false; + for entry in entries { + let path = entry?.path(); + if environment_cache_owner_file_is_stale(&path)? { + if cleanup_stale { + let _ = fs::remove_file(path); + } + } else { + live = true; + } + } + if !live && cleanup_stale { + let _ = fs::remove_dir(&lease_dir); + } + Ok(live) +} + +fn normalize_workspace_environment_dependencies( + component_id: &str, + dependencies: &mut Vec, +) -> Result<()> { + dependencies.sort_by(|left, right| left.component_id.cmp(&right.component_id)); + for pair in dependencies.windows(2) { + if pair[0].component_id == pair[1].component_id { + return Err(Error::InvalidInput(format!( + "environment component `{component_id}` declares multiple edge types for dependency `{}`", + pair[0].component_id + ))); + } + } + Ok(()) +} + +fn writable_private_binding_identity( + component_id: &str, + output_name: &str, + component_key: &str, +) -> String { + format!( + "private_{}", + &sha256_hex(format!("{component_id}\0{output_name}\0{component_key}").as_bytes())[..32] + ) +} + +fn package_workspace_environment_outputs( + build_dir: &Path, + package_name: &str, + outputs: &[PathBuf], +) -> Result { + match outputs { + [] => Err(Error::Corrupt( + "environment execution produced no normalized outputs".to_string(), + )), + [output] => Ok(output.clone()), + outputs => { + let destination = build_dir.join(format!(".trail-{package_name}")); + if destination.exists() { + return Err(Error::Corrupt(format!( + "environment output package destination `{}` already exists", + destination.display() + ))); + } + for (index, output) in outputs.iter().enumerate() { + copy_dir_recursive(output, &destination.join(format!("outputs/{index:04}")))?; + } + Ok(destination) + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct ResolvedWorkspaceTool { + pub(crate) path: PathBuf, + pub(crate) identity: String, +} + +pub(crate) fn resolve_workspace_tool_executable(program: &str) -> Result { + let path = std::env::var_os("PATH").ok_or_else(|| { + Error::InvalidInput(format!("PATH is unavailable while resolving `{program}`")) + })?; + #[cfg(not(windows))] + let names = vec![program.to_string()]; + #[cfg(windows)] + let names = if Path::new(program).extension().is_none() { + vec![ + program.to_string(), + format!("{program}.exe"), + format!("{program}.cmd"), + ] + } else { + vec![program.to_string()] + }; + for directory in std::env::split_paths(&path) { + for name in &names { + let candidate = directory.join(name); + if !candidate.is_file() { + continue; + } + // Preserve the selected executable path when invoking it. Tools + // such as rustup dispatch from the `cargo`/`rustc` shim name; using + // the canonical rustup target directly changes argv[0] semantics. + // Identity still hashes the canonical file behind the shim. + let executable_path = if candidate.is_absolute() { + candidate + } else { + std::env::current_dir()?.join(candidate) + }; + let identity = workspace_tool_identity_for_path(&executable_path)?; + return Ok(ResolvedWorkspaceTool { + path: executable_path, + identity, + }); + } + } + Err(Error::InvalidInput(format!( + "required tool `{program}` was not found on PATH" + ))) +} + +impl Trail { + pub(crate) fn declare_workspace_environment_cache( + &self, + adapter_identity: &str, + name: &str, + protocol: WorkspaceEnvironmentCacheProtocol, + access: WorkspaceEnvironmentCacheAccess, + compatibility: BTreeMap, + ) -> Result { + if name.is_empty() + || name.len() > 128 + || !name.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '-' | '_') + }) + { + return Err(Error::InvalidInput(format!( + "invalid environment cache name `{name}`" + ))); + } + if compatibility.is_empty() || compatibility.len() > 32 { + return Err(Error::InvalidInput(format!( + "environment cache `{name}` requires 1-32 compatibility dimensions" + ))); + } + for (dimension, value) in &compatibility { + if dimension.is_empty() + || dimension.len() > 128 + || value.is_empty() + || value.len() > 4096 + || dimension.contains('\0') + || value.contains('\0') + { + return Err(Error::InvalidInput(format!( + "environment cache `{name}` has an invalid compatibility dimension" + ))); + } + } + if contains_sensitive_json(&serde_json::to_value(&compatibility)?) { + return Err(Error::InvalidInput(format!( + "environment cache `{name}` compatibility contains secret-like data; use only non-secret version or tool identities" + ))); + } + let contract = serde_json::json!({ + "schema": "trail.environment-cache/v1", + "adapter_identity": adapter_identity, + "name": name, + "protocol": protocol.as_str(), + "access": access.as_str(), + "authority": "performance_only", + "scope": "workspace", + "compatibility": &compatibility, + }); + let namespace_id = format!("cache_{}", sha256_hex(&serde_json::to_vec(&contract)?)); + let storage_path = self.db_dir.join("cache/namespaces").join(&namespace_id); + Ok(WorkspaceEnvironmentCache { + name: name.to_string(), + namespace_id, + storage_path, + protocol, + access, + compatibility, + }) + } +} + +/// Return one deterministic full cycle from the Kahn remainder without +/// recursion. Every remaining node has at least one dependency in the +/// remainder, so repeatedly following the lexicographically first edge must +/// eventually revisit a node. +fn environment_dependency_cycle( + remaining: &BTreeMap, +) -> Result> { + let mut current = remaining + .keys() + .next() + .cloned() + .ok_or_else(|| Error::Corrupt("empty dependency-cycle remainder".to_string()))?; + let mut path = Vec::::new(); + let mut positions = BTreeMap::::new(); + loop { + if let Some(position) = positions.get(¤t).copied() { + let mut cycle = path[position..].to_vec(); + cycle.push(current); + return Ok(cycle); + } + positions.insert(current.clone(), path.len()); + path.push(current.clone()); + let plan = remaining.get(¤t).ok_or_else(|| { + Error::Corrupt(format!( + "dependency-cycle traversal lost component `{current}`" + )) + })?; + current = plan + .dependencies + .iter() + .filter(|dependency| remaining.contains_key(&dependency.component_id)) + .map(|dependency| &dependency.component_id) + .min() + .cloned() + .ok_or_else(|| { + Error::Corrupt(format!( + "dependency-cycle remainder component `{current}` has no remaining dependency" + )) + })?; + } +} + +pub(super) fn validate_environment_component_identity(component_id: &str) -> Result<()> { + if component_id.is_empty() + || component_id.len() > 256 + || component_id.starts_with('/') + || component_id.ends_with('/') + || !component_id.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '_' | '-' | ':' | '/') + }) + || component_id + .split('/') + .any(|segment| segment.is_empty() || segment == "." || segment == "..") + { + return Err(Error::InvalidInput(format!( + "invalid environment component identity `{component_id}`" + ))); + } + Ok(()) +} + +fn workspace_tool_identity_for_path(path: &Path) -> Result { + let canonical = fs::canonicalize(path)?; + if !canonical.is_file() { + return Err(Error::InvalidInput(format!( + "workspace tool `{}` is not a regular file", + canonical.display() + ))); + } + Ok(format!( + "{}:sha256:{}", + canonical.to_string_lossy(), + sha256_hex(&fs::read(&canonical)?) + )) +} + +fn restricted_recipe_sandbox_name() -> &'static str { + #[cfg(target_os = "macos")] + { + "macos-sandbox-exec" + } + #[cfg(target_os = "linux")] + { + "linux-landlock-seccomp" + } + #[cfg(target_os = "windows")] + { + "windows-appcontainer-job" + } + #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] + { + "unavailable-fail-closed" + } +} + +fn ensure_restricted_recipe_sandbox_available() -> Result<()> { + #[cfg(target_os = "macos")] + { + if Path::new("/usr/bin/sandbox-exec").is_file() { + Ok(()) + } else { + Err(Error::InvalidInput( + "restricted command recipes require `/usr/bin/sandbox-exec` on macOS".to_string(), + )) + } + } + #[cfg(all(target_os = "linux", not(test)))] + { + Ok(()) + } + #[cfg(all(target_os = "windows", not(test)))] + { + Ok(()) + } + #[cfg(any( + all(target_os = "linux", test), + all(target_os = "windows", test), + not(any(target_os = "macos", target_os = "linux", target_os = "windows")) + ))] + { + Err(Error::InvalidInput(format!( + "restricted command recipe sandboxing is unavailable on {}; Trail refuses to run the repository command without native enforcement", + std::env::consts::OS + ))) + } +} + +#[cfg(target_os = "macos")] +pub(super) fn sandbox_profile_escape(path: &Path) -> String { + path.to_string_lossy() + .replace('\\', "\\\\") + .replace('"', "\\\"") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cache_test_plan( + db: &Trail, + access: WorkspaceEnvironmentCacheAccess, + ) -> (WorkspaceEnvironmentPlan, WorkspaceEnvironmentCommand) { + let cache = db + .declare_workspace_environment_cache( + "trail/test@1", + "test-cache", + WorkspaceEnvironmentCacheProtocol::LockedIndex, + access, + BTreeMap::from([ + ("tool".to_string(), "test-v1".to_string()), + ("platform".to_string(), std::env::consts::OS.to_string()), + ]), + ) + .unwrap(); + let command = WorkspaceEnvironmentCommand { + program: "test".to_string(), + resolved_program: std::env::current_exe().unwrap(), + executable_identity: "test".to_string(), + args: Vec::new(), + working_directory: "project".to_string(), + environment: BTreeMap::new(), + remove_environment: Vec::new(), + cache_names: vec![cache.name.clone()], + }; + let plan = WorkspaceEnvironmentPlan { + component_id: "cache-test".to_string(), + adapter_identity: "trail/test@1".to_string(), + adapter_version: 1, + implementation_version: "test".to_string(), + distribution_digest: "builtin:test".to_string(), + kind: "generated".to_string(), + dependencies: Vec::new(), + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: "generated".to_string(), + adapter: "test".to_string(), + adapter_version: 1, + inputs: BTreeMap::new(), + tool_versions: BTreeMap::new(), + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: "host".to_string(), + strategy: "cache-test".to_string(), + }, + inputs: Vec::new(), + source_projection: None, + pre_commands: Vec::new(), + command: Some(command.clone()), + mounted_commands: Vec::new(), + caches: vec![cache], + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin, + outputs: vec![WorkspaceEnvironmentOutput { + name: "output".to_string(), + output_path: "project/output".to_string(), + mount_path: "output".to_string(), + policy: WorkspaceEnvironmentOutputPolicy::WritablePrivate, + create_if_missing: true, + }], + stale_reason: "test".to_string(), + }; + (plan, command) + } + + #[test] + fn cache_namespace_identity_is_deterministic_and_compatibility_scoped() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let compatibility = BTreeMap::from([ + ("tool".to_string(), "v1".to_string()), + ("platform".to_string(), "test".to_string()), + ]); + let first = db + .declare_workspace_environment_cache( + "trail/test@1", + "packages", + WorkspaceEnvironmentCacheProtocol::ContentStore, + WorkspaceEnvironmentCacheAccess::ToolConcurrent, + compatibility.clone(), + ) + .unwrap(); + let repeated = db + .declare_workspace_environment_cache( + "trail/test@1", + "packages", + WorkspaceEnvironmentCacheProtocol::ContentStore, + WorkspaceEnvironmentCacheAccess::ToolConcurrent, + compatibility, + ) + .unwrap(); + assert_eq!(first, repeated); + let changed = db + .declare_workspace_environment_cache( + "trail/test@1", + "packages", + WorkspaceEnvironmentCacheProtocol::ContentStore, + WorkspaceEnvironmentCacheAccess::ToolConcurrent, + BTreeMap::from([ + ("tool".to_string(), "v2".to_string()), + ("platform".to_string(), "test".to_string()), + ]), + ) + .unwrap(); + assert_ne!(first.namespace_id, changed.namespace_id); + assert_eq!( + first.storage_path, + db.db_dir.join("cache/namespaces").join(&first.namespace_id) + ); + assert!(!first.storage_path.exists()); + assert!(db + .declare_workspace_environment_cache( + "trail/test@1", + "packages", + WorkspaceEnvironmentCacheProtocol::ContentStore, + WorkspaceEnvironmentCacheAccess::ToolConcurrent, + BTreeMap::from([("registry_token".to_string(), "do-not-persist".to_string())]), + ) + .is_err()); + } + + #[test] + fn host_exclusive_cache_serializes_users_and_gc_respects_live_leases() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let (plan, command) = cache_test_plan(&db, WorkspaceEnvironmentCacheAccess::HostExclusive); + let first = db + .acquire_workspace_environment_cache_uses(&plan, &command) + .unwrap(); + fs::write(plan.caches[0].storage_path.join("entry"), "cached\n").unwrap(); + + let (sender, receiver) = std::sync::mpsc::channel(); + let workspace_path = workspace.path().to_path_buf(); + let concurrent_plan = plan.clone(); + let concurrent_command = command.clone(); + let worker = thread::spawn(move || { + let concurrent = Trail::open(workspace_path).unwrap(); + let guard = concurrent + .acquire_workspace_environment_cache_uses(&concurrent_plan, &concurrent_command) + .unwrap(); + sender.send(()).unwrap(); + drop(guard); + }); + assert!(receiver.recv_timeout(Duration::from_millis(150)).is_err()); + let live_gc = db.workspace_cache_gc(false, Some(0)).unwrap(); + assert!(!live_gc + .deleted + .iter() + .any(|entry| entry.id == plan.caches[0].namespace_id)); + assert!(plan.caches[0].storage_path.is_dir()); + drop(first); + receiver.recv_timeout(Duration::from_secs(5)).unwrap(); + worker.join().unwrap(); + + let preview = db.workspace_cache_gc(true, Some(0)).unwrap(); + assert!(preview + .candidates + .iter() + .any(|entry| entry.kind == "environment_cache" + && entry.id == plan.caches[0].namespace_id)); + assert!(plan.caches[0].storage_path.is_dir()); + let collected = db.workspace_cache_gc(false, Some(0)).unwrap(); + assert!(collected + .deleted + .iter() + .any(|entry| entry.kind == "environment_cache" + && entry.id == plan.caches[0].namespace_id)); + assert!(!plan.caches[0].storage_path.exists()); + } + + #[test] + fn ordered_environment_path_lookup_finds_ancestors_and_descendants() { + let descendants = BTreeMap::from([("generated/nested".to_string(), "child")]); + assert_eq!( + environment_path_overlap_in_map(&descendants, "generated"), + Some(("generated/nested", &"child")) + ); + let ancestors = BTreeMap::from([("generated".to_string(), "parent")]); + assert_eq!( + environment_path_overlap_in_map(&ancestors, "generated/nested"), + Some(("generated", &"parent")) + ); + assert!(environment_path_overlap_in_map(&ancestors, "generated-sibling").is_none()); + } + + #[cfg(any(target_os = "linux", target_os = "macos", windows))] + #[test] + fn failed_mounted_initializer_preserves_previous_generation_and_real_uppers() { + #[cfg(target_os = "linux")] + if std::env::var_os("TRAIL_RUN_FUSE_COW_TESTS").as_deref() + != Some(std::ffi::OsStr::new("1")) + { + return; + } + #[cfg(target_os = "macos")] + if std::env::var_os("TRAIL_RUN_NFS_COW_TESTS").as_deref() != Some(std::ffi::OsStr::new("1")) + { + return; + } + #[cfg(windows)] + if std::env::var_os("TRAIL_RUN_DOKAN_COW_TESTS").as_deref() + != Some(std::ffi::OsStr::new("1")) + { + return; + } + #[cfg(windows)] + let tool = resolve_workspace_tool_executable("python") + .or_else(|_| resolve_workspace_tool_executable("python3")); + #[cfg(not(windows))] + let tool = resolve_workspace_tool_executable("python3") + .or_else(|_| resolve_workspace_tool_executable("python")); + let Ok(tool) = tool else { + return; + }; + + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "mounted-failure", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + + let component_id = "mounted-failure".to_string(); + let mount_path = ".candidate-output".to_string(); + let baseline_key = WorkspaceLayerKeyV1 { + kind: "generated".to_string(), + adapter: "test-mounted".to_string(), + adapter_version: 1, + inputs: BTreeMap::from([("revision".to_string(), "baseline".to_string())]), + tool_versions: BTreeMap::new(), + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: "host".to_string(), + strategy: "mounted-failure-baseline".to_string(), + }; + let baseline_cache_key = db.workspace_layer_cache_key(&baseline_key).unwrap(); + let baseline_seed = tempfile::tempdir().unwrap(); + fs::write(baseline_seed.path().join("keep.txt"), "previous\n").unwrap(); + db.replace_declared_workspace_layers( + "mounted-failure", + &[EnvironmentLayerActivation { + layer_id: None, + outputs: vec![EnvironmentLayerOutputActivation { + name: "output".to_string(), + mount_path: mount_path.clone(), + policy: "writable_private".to_string(), + binding_identity: writable_private_binding_identity( + &component_id, + "output", + &baseline_cache_key, + ), + private_seed: Some(baseline_seed.path().to_path_buf()), + layer_subpath: String::new(), + }], + component_id: component_id.clone(), + adapter_identity: "test/mounted@1".to_string(), + adapter_version: 1, + implementation_version: "test".to_string(), + distribution_digest: "builtin:test-mounted-baseline".to_string(), + kind: "generated".to_string(), + dependencies: Vec::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + expected_key: baseline_cache_key.clone(), + canonical_key: baseline_key, + }], + ) + .unwrap(); + let baseline_generation = db + .active_environment_generation("mounted-failure") + .unwrap() + .unwrap() + .generation_id; + + let mounted_command = WorkspaceEnvironmentCommand { + program: "python".to_string(), + resolved_program: tool.path, + executable_identity: tool.identity.clone(), + args: vec![ + "-c".to_string(), + "from pathlib import Path; out=Path('.candidate-output'); out.mkdir(exist_ok=True); (out/'partial.txt').write_text('partial'); Path('source-leak.txt').write_text('leak'); raise SystemExit(23)".to_string(), + ], + working_directory: String::new(), + environment: BTreeMap::new(), + remove_environment: Vec::new(), + cache_names: Vec::new(), + }; + let action_identity = + workspace_mounted_commands_identity(std::slice::from_ref(&mounted_command)).unwrap(); + let plan = WorkspaceEnvironmentPlan { + component_id: component_id.clone(), + adapter_identity: "test/mounted@1".to_string(), + adapter_version: 1, + implementation_version: "test".to_string(), + distribution_digest: "builtin:test-mounted-failure".to_string(), + kind: "generated".to_string(), + dependencies: Vec::new(), + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: "generated".to_string(), + adapter: "test-mounted".to_string(), + adapter_version: 1, + inputs: BTreeMap::from([ + ("revision".to_string(), "failure".to_string()), + ("mounted_action".to_string(), action_identity), + ]), + tool_versions: BTreeMap::from([("python".to_string(), tool.identity)]), + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: "host".to_string(), + strategy: "mounted-failure-candidate".to_string(), + }, + inputs: Vec::new(), + source_projection: None, + pre_commands: Vec::new(), + command: None, + mounted_commands: vec![mounted_command], + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin, + outputs: vec![WorkspaceEnvironmentOutput { + name: "output".to_string(), + output_path: "private/output".to_string(), + mount_path: mount_path.clone(), + policy: WorkspaceEnvironmentOutputPolicy::WritablePrivate, + create_if_missing: true, + }], + stale_reason: "test mounted initializer changed".to_string(), + }; + let candidate_key = db.workspace_layer_cache_key(&plan.layer_key).unwrap(); + let source_root = db + .get_ref(&db.lane_branch("mounted-failure").unwrap().ref_name) + .unwrap() + .root_id; + let private = db.execute_writable_private_environment_plan(&plan).unwrap(); + let mut artifacts = [PreparedEnvironmentArtifacts::WritablePrivate(Some(private))]; + let error = db + .initialize_mounted_workspace_environment_plans( + "mounted-failure", + "test-mounted-failure", + &source_root, + &[(&plan, candidate_key.as_str())], + &mut artifacts, + &[], + ) + .unwrap_err(); + assert!(error.to_string().contains("failed with")); + + let mut leaking_plan = plan.clone(); + leaking_plan.mounted_commands[0].args[1] = "from pathlib import Path; out=Path('.candidate-output'); out.mkdir(exist_ok=True); (out/'partial.txt').write_text('partial'); Path('source-leak.txt').write_text('leak')".to_string(); + leaking_plan + .layer_key + .inputs + .insert("revision".to_string(), "undeclared-write".to_string()); + leaking_plan.layer_key.inputs.insert( + "mounted_action".to_string(), + workspace_mounted_commands_identity(&leaking_plan.mounted_commands).unwrap(), + ); + let leaking_key = db + .workspace_layer_cache_key(&leaking_plan.layer_key) + .unwrap(); + let private = db + .execute_writable_private_environment_plan(&leaking_plan) + .unwrap(); + let mut artifacts = [PreparedEnvironmentArtifacts::WritablePrivate(Some(private))]; + let error = db + .initialize_mounted_workspace_environment_plans( + "mounted-failure", + "test-mounted-leak", + &source_root, + &[(&leaking_plan, leaking_key.as_str())], + &mut artifacts, + &[], + ) + .unwrap_err(); + assert!(error.to_string().contains("undeclared source path")); + + let active_generation = db + .active_environment_generation("mounted-failure") + .unwrap() + .unwrap(); + assert_eq!(active_generation.generation_id, baseline_generation); + let paths = db.workspace_view_paths_for_lane("mounted-failure").unwrap(); + assert_eq!( + fs::read_to_string(paths.generated_upper.join(&mount_path).join("keep.txt")).unwrap(), + "previous\n" + ); + assert!(!paths + .generated_upper + .join(&mount_path) + .join("partial.txt") + .exists()); + assert!(!paths.source_upper.join("source-leak.txt").exists()); + } + + #[test] + fn dead_environment_sync_attempt_recovers_predecessors_and_first_builds() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "recover-env", + Some("main"), + mode, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let view = db.lane_workspace_view("recover-env").unwrap().unwrap(); + for (component, attached) in [("with-predecessor", Some("old-key")), ("first", None)] { + db.conn + .execute( + "INSERT INTO environment_component_states + (view_id, component_id, adapter_identity, adapter_version, implementation_version, distribution_digest, kind, expected_key, attached_key, status, reason, updated_at) + VALUES (?1, ?2, 'trail/test@1', 1, 'test', 'builtin:test', 'dependency', 'new-key', ?3, 'building', NULL, ?4)", + params![view.view_id, component, attached, now_ts()], + ) + .unwrap(); + db.conn + .execute( + "INSERT INTO workspace_environment_states + (view_id, adapter, expected_key, attached_key, status, reason, updated_at) + VALUES (?1, ?2, 'new-key', ?3, 'building', NULL, ?4)", + params![view.view_id, component, attached, now_ts()], + ) + .unwrap(); + } + db.conn + .execute( + "INSERT INTO environment_sync_attempts + (attempt_id, view_id, source_root, mode, owner_pid, owner_start_token, status, reason, started_at, updated_at, finished_at) + VALUES ('dead-build', ?1, ?2, 'batch', -1, 'dead', 'running', NULL, ?3, ?3, NULL)", + params![view.view_id, view.base_root.0, now_ts()], + ) + .unwrap(); + drop(db); + + let db = Trail::open(workspace.path()).unwrap(); + let rows = { + let mut stmt = db + .conn + .prepare( + "SELECT component_id, status, attached_key, reason + FROM environment_component_states WHERE view_id = ?1 ORDER BY component_id", + ) + .unwrap(); + let mapped = stmt + .query_map(params![view.view_id], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + )) + }) + .unwrap(); + mapped.collect::, _>>().unwrap() + }; + assert_eq!(rows[0].0, "first"); + assert_eq!(rows[0].1, "failed"); + assert_eq!(rows[0].2, None); + assert_eq!(rows[1].0, "with-predecessor"); + assert_eq!(rows[1].1, "stale"); + assert_eq!(rows[1].2.as_deref(), Some("old-key")); + assert!(rows + .iter() + .all(|row| row.3.as_deref().unwrap().contains("predecessor"))); + assert_eq!( + db.conn + .query_row( + "SELECT status FROM environment_sync_attempts WHERE attempt_id = 'dead-build'", + [], + |row| row.get::<_, String>(0), + ) + .unwrap(), + "abandoned" + ); + + db.conn + .execute( + "INSERT INTO environment_sync_attempts + (attempt_id, view_id, source_root, mode, owner_pid, owner_start_token, status, reason, started_at, updated_at, finished_at) + VALUES ('dead-after-activation', ?1, ?2, 'single', -1, 'dead', 'running', NULL, ?3, ?3, NULL)", + params![view.view_id, view.base_root.0, now_ts()], + ) + .unwrap(); + drop(db); + let db = Trail::open(workspace.path()).unwrap(); + assert_eq!( + db.conn + .query_row( + "SELECT status FROM environment_sync_attempts WHERE attempt_id = 'dead-after-activation'", + [], + |row| row.get::<_, String>(0), + ) + .unwrap(), + "succeeded" + ); + } + + #[test] + fn adapter_identity_parser_preserves_contract_major_and_legacy_names() { + assert_eq!( + split_adapter_identity("trail/node@1", 9), + ("trail".to_string(), "node".to_string(), 1) + ); + assert_eq!( + split_adapter_identity("legacy/custom", 0), + ("legacy".to_string(), "custom".to_string(), 0) + ); + } + + #[test] + fn canonical_layer_key_diff_reports_edges_without_exposing_values() { + let previous = WorkspaceLayerKeyV1 { + kind: "dependency".to_string(), + adapter: "example".to_string(), + adapter_version: 1, + inputs: BTreeMap::from([ + ("removed.lock".to_string(), "old-secret-value".to_string()), + ("shared.lock".to_string(), "old-hash".to_string()), + ]), + tool_versions: BTreeMap::from([("tool".to_string(), "1".to_string())]), + platform: "linux".to_string(), + architecture: "x86_64".to_string(), + portability_scope: "platform".to_string(), + strategy: "old".to_string(), + }; + let mut current = previous.clone(); + current.inputs.remove("removed.lock"); + current + .inputs + .insert("added.lock".to_string(), "new-secret-value".to_string()); + current + .inputs + .insert("shared.lock".to_string(), "new-hash".to_string()); + current + .tool_versions + .insert("tool".to_string(), "2".to_string()); + current.strategy = "new".to_string(); + + let changes = diff_workspace_layer_keys(&previous, ¤t); + assert!(changes.iter().any(|change| { + change.dimension == "input" && change.name == "added.lock" && change.change == "added" + })); + assert!(changes.iter().any(|change| { + change.dimension == "input" + && change.name == "removed.lock" + && change.change == "removed" + })); + assert!(changes.iter().any(|change| { + change.dimension == "input" + && change.name == "shared.lock" + && change.change == "modified" + })); + assert!(changes.iter().any(|change| { + change.dimension == "tool" && change.name == "tool" && change.change == "modified" + })); + assert!(changes.iter().any(|change| { + change.dimension == "policy" && change.name == "strategy" && change.change == "modified" + })); + let rendered = format_stale_changes(&changes); + assert!(!rendered.contains("secret")); + assert!(!rendered.contains("old-hash")); + assert!(!rendered.contains("new-hash")); + let first = paginate_stale_explanation( + "test", + "stale", + "expected".to_string(), + Some("attached".to_string()), + true, + changes.clone(), + 0, + 2, + ) + .unwrap(); + assert!(!first.complete); + assert!(first.provenance_complete); + assert_eq!(first.total_changes, changes.len() as u64); + assert_eq!(first.changes.len(), 2); + assert_eq!(first.next_offset, Some(2)); + let remainder = paginate_stale_explanation( + "test", + "stale", + "expected".to_string(), + Some("attached".to_string()), + true, + changes.clone(), + 2, + 1_000, + ) + .unwrap(); + assert_eq!(remainder.changes.len(), changes.len() - 2); + assert!(remainder.next_offset.is_none()); + } + + #[test] + fn dependency_graph_finalization_scales_without_recursive_traversal() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let count = 10_000usize; + let plans = (0..count) + .rev() + .map(|index| { + let component_id = format!("component-{index:05}"); + WorkspaceEnvironmentPlan { + component_id: component_id.clone(), + adapter_identity: "trail/test@1".to_string(), + adapter_version: 1, + implementation_version: "test".to_string(), + distribution_digest: "builtin:test".to_string(), + kind: "generated".to_string(), + dependencies: (index > 0) + .then(|| format!("component-{:05}", index - 1)) + .into_iter() + .map(WorkspaceEnvironmentDependency::build_requires) + .collect(), + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: "generated".to_string(), + adapter: "test".to_string(), + adapter_version: 1, + inputs: BTreeMap::from([("component".to_string(), component_id.clone())]), + tool_versions: BTreeMap::new(), + platform: "test".to_string(), + architecture: "test".to_string(), + portability_scope: "test".to_string(), + strategy: "dependency-scale-test".to_string(), + }, + inputs: Vec::new(), + source_projection: None, + pre_commands: Vec::new(), + command: None, + mounted_commands: Vec::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin, + outputs: vec![WorkspaceEnvironmentOutput { + name: "output".to_string(), + output_path: format!("staging/{component_id}"), + mount_path: format!("generated/{component_id}"), + policy: WorkspaceEnvironmentOutputPolicy::WritablePrivate, + create_if_missing: true, + }], + stale_reason: "dependency changed".to_string(), + } + }) + .collect(); + let finalized = db.finalize_workspace_environment_plan_graph(plans).unwrap(); + assert_eq!(finalized.len(), count); + assert_eq!(finalized[0].0.component_id, "component-00000"); + assert_eq!(finalized[count - 1].0.component_id, "component-09999"); + assert_eq!( + finalized[count - 1].0.layer_key.inputs["dependency:component-09998"], + finalized[count - 2].1 + ); + } + + #[test] + fn typed_dependency_edges_separate_identity_from_generation_ordering() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let plan = |component_id: &str, + revision: &str, + dependencies: Vec| { + WorkspaceEnvironmentPlan { + component_id: component_id.to_string(), + adapter_identity: "trail/test@1".to_string(), + adapter_version: 1, + implementation_version: "test".to_string(), + distribution_digest: "builtin:test".to_string(), + kind: "generated".to_string(), + dependencies, + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: "generated".to_string(), + adapter: "test".to_string(), + adapter_version: 1, + inputs: BTreeMap::from([("revision".to_string(), revision.to_string())]), + tool_versions: BTreeMap::new(), + platform: "test".to_string(), + architecture: "test".to_string(), + portability_scope: "test".to_string(), + strategy: "typed-edge-test".to_string(), + }, + inputs: Vec::new(), + source_projection: None, + pre_commands: Vec::new(), + command: None, + mounted_commands: Vec::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin, + outputs: vec![WorkspaceEnvironmentOutput { + name: "output".to_string(), + output_path: format!("staging/{component_id}"), + mount_path: format!("generated/{component_id}"), + policy: WorkspaceEnvironmentOutputPolicy::WritablePrivate, + create_if_missing: true, + }], + stale_reason: "typed edge changed".to_string(), + } + }; + let dependencies = vec![ + WorkspaceEnvironmentDependency { + component_id: "build".to_string(), + edge_type: WorkspaceEnvironmentEdgeType::BuildRequires, + }, + WorkspaceEnvironmentDependency { + component_id: "runtime".to_string(), + edge_type: WorkspaceEnvironmentEdgeType::RuntimeRequires, + }, + WorkspaceEnvironmentDependency { + component_id: "binding".to_string(), + edge_type: WorkspaceEnvironmentEdgeType::BindsAfter, + }, + WorkspaceEnvironmentDependency { + component_id: "configuration".to_string(), + edge_type: WorkspaceEnvironmentEdgeType::InvalidatesWith, + }, + ]; + let finalize = |revisions: [&str; 4]| { + db.finalize_workspace_environment_plan_graph(vec![ + plan("build", revisions[0], Vec::new()), + plan("runtime", revisions[1], Vec::new()), + plan("binding", revisions[2], Vec::new()), + plan("configuration", revisions[3], Vec::new()), + plan("application", "1", dependencies.clone()), + ]) + .unwrap() + .into_iter() + .find(|(plan, _)| plan.component_id == "application") + .unwrap() + }; + + let baseline = finalize(["1", "1", "1", "1"]); + assert!(baseline.0.layer_key.inputs.contains_key("dependency:build")); + assert!(baseline + .0 + .layer_key + .inputs + .contains_key("dependency:invalidates_with:configuration")); + assert!(!baseline + .0 + .layer_key + .inputs + .keys() + .any(|key| key.contains("runtime") || key.contains("binding"))); + assert_ne!(finalize(["2", "1", "1", "1"]).1, baseline.1); + assert_ne!(finalize(["1", "1", "1", "2"]).1, baseline.1); + assert_eq!(finalize(["1", "2", "1", "1"]).1, baseline.1); + assert_eq!(finalize(["1", "1", "2", "1"]).1, baseline.1); + assert_eq!(baseline.0.resolved_dependencies.len(), 4); + } + + #[cfg(unix)] + #[test] + fn executor_rejects_a_tool_binary_changed_after_component_planning() { + use std::os::unix::fs::PermissionsExt; + + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let build = tempfile::tempdir().unwrap(); + let tool = build.path().join("tool"); + fs::write(&tool, "#!/bin/sh\nexit 0\n").unwrap(); + fs::set_permissions(&tool, fs::Permissions::from_mode(0o755)).unwrap(); + let identity = workspace_tool_identity_for_path(&tool).unwrap(); + let command = WorkspaceEnvironmentCommand { + program: "tool".to_string(), + resolved_program: tool.clone(), + executable_identity: identity, + args: Vec::new(), + working_directory: "project".to_string(), + environment: BTreeMap::new(), + remove_environment: Vec::new(), + cache_names: Vec::new(), + }; + let plan = WorkspaceEnvironmentPlan { + component_id: "test".to_string(), + adapter_identity: "trail/test@1".to_string(), + adapter_version: 1, + implementation_version: "test".to_string(), + distribution_digest: "builtin:test".to_string(), + kind: "dependency".to_string(), + dependencies: Vec::new(), + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: "dependency".to_string(), + adapter: "test".to_string(), + adapter_version: 1, + inputs: BTreeMap::new(), + tool_versions: BTreeMap::new(), + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: "host".to_string(), + strategy: "test".to_string(), + }, + inputs: Vec::new(), + source_projection: None, + pre_commands: Vec::new(), + command: Some(command.clone()), + mounted_commands: Vec::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin, + outputs: vec![WorkspaceEnvironmentOutput { + name: "primary".to_string(), + output_path: "project/output".to_string(), + mount_path: "output".to_string(), + policy: WorkspaceEnvironmentOutputPolicy::ImmutableSeedPrivate, + create_if_missing: false, + }], + stale_reason: "test".to_string(), + }; + fs::write(&tool, "#!/bin/sh\nexit 1\n").unwrap(); + let error = db + .run_workspace_environment_command(&plan, &command, build.path()) + .unwrap_err(); + assert!(error.to_string().contains("changed after")); + } + + #[test] + fn automatic_detection_rejects_ambiguous_polyglot_roots() { + let workspace = tempfile::tempdir().unwrap(); + fs::create_dir_all(workspace.path().join("src")).unwrap(); + fs::write( + workspace.path().join("package.json"), + r#"{"name":"polyglot","version":"1.0.0"}"#, + ) + .unwrap(); + fs::write( + workspace.path().join("package-lock.json"), + r#"{"name":"polyglot","version":"1.0.0","lockfileVersion":3,"packages":{}}"#, + ) + .unwrap(); + fs::write( + workspace.path().join("Cargo.toml"), + "[package]\nname = \"polyglot\"\nversion = \"0.1.0\"\nedition = \"2021\"\n", + ) + .unwrap(); + fs::write(workspace.path().join("src/lib.rs"), "pub fn value() {}\n").unwrap(); + let cargo_lock_ready = Command::new("cargo") + .args(["generate-lockfile", "--offline"]) + .current_dir(workspace.path()) + .output() + .is_ok_and(|output| output.status.success()); + if !cargo_lock_ready { + fs::write(workspace.path().join("Cargo.lock"), "version = 4\n").unwrap(); + } + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "polyglot", + Some("main"), + mode, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let discovery = db.discover_workspace_environment("polyglot", None).unwrap(); + assert_eq!( + discovery + .components + .iter() + .map(|component| component.adapter_identity.as_str()) + .collect::>(), + vec!["trail/cargo-target-seed@1", "trail/node@1"] + ); + assert!(discovery.conflicts.is_empty()); + let error = db + .sync_workspace_environment("polyglot", "auto", None) + .unwrap_err(); + let message = error.to_string(); + assert!(message.contains("multiple workspace environment adapters")); + assert!(message.contains("trail/node@1")); + assert!(message.contains("trail/cargo-target-seed@1")); + if cargo_lock_ready + && Command::new("cargo").arg("--version").output().is_ok() + && Command::new("rustc").arg("--version").output().is_ok() + && Command::new("npm").arg("--version").output().is_ok() + && Command::new("node").arg("--version").output().is_ok() + { + let synchronized = db + .sync_all_workspace_environments("polyglot", None) + .unwrap(); + assert_eq!(synchronized.layers.len(), 2); + assert_eq!(synchronized.generation.generation_sequence, 1); + assert_eq!( + synchronized + .generation + .components + .iter() + .map(|component| component.component_id.as_str()) + .collect::>(), + vec!["cargo-target-seed", "node"] + ); + } + } + + #[test] + fn environment_mount_validation_rejects_tracked_source_shadowing_and_reserved_paths() { + let workspace = tempfile::tempdir().unwrap(); + fs::create_dir_all(workspace.path().join("generated")).unwrap(); + fs::write( + workspace.path().join("generated/owned-by-source.txt"), + "source\n", + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let source_root = db.get_ref("refs/branches/main").unwrap().root_id; + let mut plan = WorkspaceEnvironmentPlan { + component_id: "generated-test".to_string(), + adapter_identity: "trail/test@1".to_string(), + adapter_version: 1, + implementation_version: "test".to_string(), + distribution_digest: "builtin:test".to_string(), + kind: "generated".to_string(), + dependencies: Vec::new(), + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: "generated".to_string(), + adapter: "test".to_string(), + adapter_version: 1, + inputs: BTreeMap::new(), + tool_versions: BTreeMap::new(), + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: "host".to_string(), + strategy: "test".to_string(), + }, + inputs: Vec::new(), + source_projection: None, + pre_commands: Vec::new(), + command: Some(WorkspaceEnvironmentCommand { + program: "test".to_string(), + resolved_program: std::env::current_exe().unwrap(), + executable_identity: "test".to_string(), + args: Vec::new(), + working_directory: "project".to_string(), + environment: BTreeMap::new(), + remove_environment: Vec::new(), + cache_names: Vec::new(), + }), + mounted_commands: Vec::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin, + outputs: vec![WorkspaceEnvironmentOutput { + name: "primary".to_string(), + output_path: "project/generated".to_string(), + mount_path: "GENERATED".to_string(), + policy: WorkspaceEnvironmentOutputPolicy::ImmutableSeedPrivate, + create_if_missing: true, + }], + stale_reason: "test".to_string(), + }; + + let error = db + .validate_environment_mounts_do_not_shadow_source(&source_root, &[&plan]) + .unwrap_err(); + assert!(error + .to_string() + .contains("would shadow pinned source file")); + + plan.outputs[0].mount_path = ".trail/generated".to_string(); + let error = db + .validate_environment_mounts_do_not_shadow_source(&source_root, &[&plan]) + .unwrap_err(); + assert!(error + .to_string() + .contains("cannot mount inside reserved path")); + } + + #[cfg(windows)] + #[test] + fn windows_recipe_publication_rejects_hard_link_aliases() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let staging = tempfile::tempdir().unwrap(); + let output = staging.path().join("output"); + fs::create_dir(&output).unwrap(); + let outside = staging.path().join("outside.txt"); + fs::write(&outside, "host canary\n").unwrap(); + fs::hard_link(&outside, output.join("alias.txt")).unwrap(); + + let command = WorkspaceEnvironmentCommand { + program: "test".to_string(), + resolved_program: std::env::current_exe().unwrap(), + executable_identity: "test".to_string(), + args: Vec::new(), + working_directory: "project".to_string(), + environment: BTreeMap::new(), + remove_environment: Vec::new(), + cache_names: Vec::new(), + }; + let plan = WorkspaceEnvironmentPlan { + component_id: "windows-output-validation".to_string(), + adapter_identity: "trail/test@1".to_string(), + adapter_version: 1, + implementation_version: "test".to_string(), + distribution_digest: "builtin:test".to_string(), + kind: "generated".to_string(), + dependencies: Vec::new(), + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: "generated".to_string(), + adapter: "test".to_string(), + adapter_version: 1, + inputs: BTreeMap::new(), + tool_versions: BTreeMap::new(), + platform: "windows".to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: "host".to_string(), + strategy: "test".to_string(), + }, + inputs: Vec::new(), + source_projection: None, + pre_commands: Vec::new(), + command, + mounted_commands: Vec::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe, + outputs: vec![WorkspaceEnvironmentOutput { + name: "primary".to_string(), + output_path: "project/output".to_string(), + mount_path: "output".to_string(), + policy: WorkspaceEnvironmentOutputPolicy::ImmutableSeedPrivate, + create_if_missing: true, + }], + stale_reason: "test".to_string(), + }; + let mut entries = 0; + let error = db + .validate_restricted_recipe_output(&plan, &output, &mut entries) + .unwrap_err(); + assert!(error.to_string().contains("hard-linked or unverifiable")); + } +} diff --git a/trail/src/db/lane/workspace_git.rs b/trail/src/db/lane/workspace_git.rs index e408d3e..6d473b8 100644 --- a/trail/src/db/lane/workspace_git.rs +++ b/trail/src/db/lane/workspace_git.rs @@ -1,6 +1,15 @@ use super::*; impl Trail { + /// A lane shadow is deliberately a compatibility/status projection. Its + /// private index and HEAD are never a clean proof for the real workspace, + /// even when the shadow still points at its pinned commit. + pub(crate) const fn workspace_git_shadow_clean_proof_allowed( + _shadow: &WorkspaceGitShadowReport, + ) -> bool { + false + } + pub(crate) fn ensure_workspace_git_shadow( &self, view: &LaneWorkspaceViewReport, @@ -125,6 +134,7 @@ impl Trail { } else { "diverged" }; + debug_assert!(!Self::workspace_git_shadow_clean_proof_allowed(shadow)); self.conn.execute( "UPDATE workspace_git_shadows SET current_head = ?1, status = ?2, updated_at = ?3 WHERE view_id = ?4", params![current_head, status, now_ts(), shadow.view_id], @@ -211,8 +221,10 @@ mod tests { let mut db = Trail::open(workspace.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "git-shadow", @@ -232,6 +244,7 @@ mod tests { .ensure_workspace_git_shadow(&view, &head.root_id) .unwrap() .unwrap(); + assert!(!Trail::workspace_git_shadow_clean_proof_allowed(&shadow)); let real_head = Command::new("git") .arg("-C") .arg(workspace.path()) diff --git a/trail/src/db/lane/workspace_go.rs b/trail/src/db/lane/workspace_go.rs new file mode 100644 index 0000000..afc3c62 --- /dev/null +++ b/trail/src/db/lane/workspace_go.rs @@ -0,0 +1,395 @@ +use super::workspace_environment::{ + resolve_workspace_tool_executable, WorkspaceEnvironmentAdapter, + WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentCacheAccess, + WorkspaceEnvironmentCacheProtocol, WorkspaceEnvironmentCommand, WorkspaceEnvironmentOutput, + WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, WorkspaceEnvironmentSandboxPolicy, +}; +use super::*; + +pub(crate) struct GoVendorAdapter; + +pub(crate) static GO_VENDOR_ADAPTER: GoVendorAdapter = GoVendorAdapter; + +static GO_VENDOR_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = + WorkspaceEnvironmentAdapterMetadata { + canonical_identity: "trail/go-vendor@1", + namespace: "trail", + name: "go-vendor", + contract_major: 1, + implementation_version: env!("CARGO_PKG_VERSION"), + distribution_digest: "builtin:go-vendor-plan-v1", + selectors: &["trail/go-vendor@1", "go-vendor", "go"], + kind: "dependency", + layer_adapter_name: "go-vendor", + discovery_markers: &["go.mod"], + supported_operating_systems: &["linux", "macos", "windows"], + supported_architectures: &["aarch64", "x86_64"], + stability: "experimental", + description: "Single-module Go vendor tree with shared module and compiler caches", + }; + +impl WorkspaceEnvironmentAdapter for GoVendorAdapter { + fn metadata(&self) -> &'static WorkspaceEnvironmentAdapterMetadata { + &GO_VENDOR_ADAPTER_METADATA + } + + fn component_id(&self, component_root: &str) -> Result { + let root = normalize_component_root(component_root)?; + Ok(if root.is_empty() { + "go-vendor".to_string() + } else { + format!("go-vendor:{root}") + }) + } + + fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result { + let root = normalize_component_root(component_root)?; + Ok(db + .root_file_entry(source_root, &join_repo_path(&root, "go.mod"))? + .is_some()) + } + + fn plan( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result { + let component_root = normalize_component_root(component_root)?; + let go_mod_path = join_repo_path(&component_root, "go.mod"); + let go_mod = db + .root_file_entry(source_root, &go_mod_path)? + .ok_or_else(|| { + Error::InvalidInput(format!( + "Go component `{}` has no go.mod", + display_component_root(&component_root) + )) + })?; + if db + .root_file_entry(source_root, &join_repo_path(&component_root, "go.work"))? + .is_some() + { + return Err(Error::InvalidInput( + "Go workspaces require a multi-module graph adapter; trail/go-vendor@1 supports one module root" + .to_string(), + )); + } + let go_sum_path = join_repo_path(&component_root, "go.sum"); + let go_sum = db.root_file_entry(source_root, &go_sum_path)?; + let go_version = command_identity("go", &["version"])?; + let go_tool = resolve_workspace_tool_executable("go")?; + let implementation_version = env!("CARGO_PKG_VERSION").to_string(); + let distribution_digest = "builtin:go-vendor-plan-v1".to_string(); + let cache_compatibility = BTreeMap::from([ + ("go".to_string(), go_version.clone()), + ("go_executable".to_string(), go_tool.identity.clone()), + ("platform".to_string(), std::env::consts::OS.to_string()), + ( + "architecture".to_string(), + std::env::consts::ARCH.to_string(), + ), + ]); + let module_cache = db.declare_workspace_environment_cache( + self.identity(), + "module-store", + WorkspaceEnvironmentCacheProtocol::ContentStore, + WorkspaceEnvironmentCacheAccess::ToolConcurrent, + cache_compatibility.clone(), + )?; + let build_cache = db.declare_workspace_environment_cache( + self.identity(), + "build-cache", + WorkspaceEnvironmentCacheProtocol::ContentStore, + WorkspaceEnvironmentCacheAccess::ToolConcurrent, + cache_compatibility, + )?; + let working_directory = if component_root.is_empty() { + "project".to_string() + } else { + format!("project/{component_root}") + }; + let mount_path = if component_root.is_empty() { + "vendor".to_string() + } else { + format!("{component_root}/vendor") + }; + let mut inputs = BTreeMap::from([ + ("source_root".to_string(), source_root.0.clone()), + (go_mod_path, go_mod.content_hash), + ( + "adapter_implementation".to_string(), + implementation_version.clone(), + ), + ( + "adapter_distribution_digest".to_string(), + distribution_digest.clone(), + ), + ( + "output_contract".to_string(), + format!("immutable-seed-private:{mount_path}"), + ), + ]); + inputs.insert( + go_sum_path, + go_sum + .map(|entry| entry.content_hash) + .unwrap_or_else(|| "missing".to_string()), + ); + let environment = BTreeMap::from([ + ("GOWORK".to_string(), "off".to_string()), + ( + "GOMODCACHE".to_string(), + module_cache.storage_path.to_string_lossy().into_owned(), + ), + ( + "GOCACHE".to_string(), + build_cache.storage_path.to_string_lossy().into_owned(), + ), + ]); + Ok(WorkspaceEnvironmentPlan { + component_id: self.component_id(&component_root)?, + adapter_identity: self.identity().to_string(), + adapter_version: 1, + implementation_version, + distribution_digest, + kind: "dependency".to_string(), + dependencies: Vec::new(), + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: "dependency".to_string(), + adapter: self.layer_adapter_name().to_string(), + adapter_version: 1, + inputs, + tool_versions: BTreeMap::from([ + ("go".to_string(), go_version), + ("go-executable".to_string(), go_tool.identity.clone()), + ]), + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: "source-root-go-toolchain-platform".to_string(), + strategy: "go-mod-vendor-v1".to_string(), + }, + inputs: Vec::new(), + source_projection: Some((source_root.clone(), "project".to_string())), + pre_commands: Vec::new(), + command: Some(WorkspaceEnvironmentCommand { + program: "go".to_string(), + resolved_program: go_tool.path, + executable_identity: go_tool.identity, + args: vec!["mod".to_string(), "vendor".to_string()], + working_directory: working_directory.clone(), + environment, + remove_environment: Vec::new(), + cache_names: vec![module_cache.name.clone(), build_cache.name.clone()], + }), + mounted_commands: Vec::new(), + caches: vec![module_cache, build_cache], + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin, + outputs: vec![WorkspaceEnvironmentOutput { + name: "vendor".to_string(), + output_path: format!("{working_directory}/vendor"), + mount_path, + policy: WorkspaceEnvironmentOutputPolicy::ImmutableSeedPrivate, + create_if_missing: true, + }], + stale_reason: + "source root, Go module graph, Go toolchain, platform, or adapter policy changed" + .to_string(), + }) + } +} + +fn normalize_component_root(component_root: &str) -> Result { + if component_root.trim_matches('/').is_empty() { + Ok(String::new()) + } else { + normalize_relative_path(component_root) + } +} + +fn join_repo_path(root: &str, name: &str) -> String { + if root.is_empty() { + name.to_string() + } else { + format!("{root}/{name}") + } +} + +fn display_component_root(component_root: &str) -> &str { + if component_root.is_empty() { + "." + } else { + component_root + } +} + +fn command_identity(program: &str, args: &[&str]) -> Result { + let output = Command::new(program).args(args).output().map_err(|err| { + Error::InvalidInput(format!("required tool `{program}` is unavailable: {err}")) + })?; + if !output.status.success() { + return Err(Error::InvalidInput(format!( + "`{program} {}` failed with {}", + args.join(" "), + output.status + ))); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn go_adapter_vendors_once_and_reuses_the_immutable_tree_across_lanes() { + if command_identity("go", &["version"]).is_err() { + return; + } + let workspace = tempfile::tempdir().unwrap(); + fs::create_dir_all(workspace.path().join("shared")).unwrap(); + fs::create_dir_all(workspace.path().join("tools")).unwrap(); + fs::write( + workspace.path().join("go.mod"), + "module example.test/app\n\ngo 1.22\n\nrequire example.test/shared v0.0.0\nreplace example.test/shared => ./shared\n", + ) + .unwrap(); + fs::write(workspace.path().join("go.sum"), "").unwrap(); + fs::write( + workspace.path().join("main.go"), + "package main\nimport _ \"example.test/shared\"\nfunc main() {}\n", + ) + .unwrap(); + fs::write( + workspace.path().join("shared/go.mod"), + "module example.test/shared\n\ngo 1.22\n", + ) + .unwrap(); + fs::write( + workspace.path().join("shared/shared.go"), + "package shared\nconst Value = 42\n", + ) + .unwrap(); + fs::write( + workspace.path().join("tools/go.mod"), + "module example.test/tools\n\ngo 1.22\n", + ) + .unwrap(); + fs::write( + workspace.path().join("tools/main.go"), + "package main\nfunc main() {}\n", + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + for lane in ["go-one", "go-two", "go-all"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + mode.clone(), + None, + None, + None, + &[], + false, + ) + .unwrap(); + } + let first = db + .sync_workspace_environment("go-one", "auto", None) + .unwrap(); + let second = db + .sync_workspace_environment("go-two", "trail/go-vendor@1", None) + .unwrap(); + assert_eq!(first.layer_id, second.layer_id); + assert!(Path::new(&first.storage_path) + .join("example.test/shared/shared.go") + .is_file()); + let status = db.environment_component_status("go-two").unwrap(); + assert_eq!(status[0].component.component_id, "go-vendor"); + assert_eq!(status[0].adapter.name, "go-vendor"); + assert_eq!(status[0].status, "ready"); + + let all = db.sync_all_workspace_environments("go-all", None).unwrap(); + assert_eq!(all.generation.generation_sequence, 1); + assert_eq!(all.layers.len(), 3); + assert_eq!( + all.generation + .components + .iter() + .map(|component| component.component_id.as_str()) + .collect::>(), + vec!["go-vendor", "go-vendor:shared", "go-vendor:tools"] + ); + + db.conn + .execute_batch( + "CREATE TRIGGER fail_generation_activation + BEFORE INSERT ON environment_generations + BEGIN + SELECT RAISE(ABORT, 'injected generation activation failure'); + END;", + ) + .unwrap(); + let activation_error = db + .sync_workspace_environment("go-one", "trail/go-vendor@1", Some("tools")) + .unwrap_err(); + assert!(activation_error.to_string().contains("injected generation")); + let unchanged = db.active_environment_generation("go-one").unwrap().unwrap(); + assert_eq!(unchanged.generation_sequence, 1); + assert_eq!(unchanged.components.len(), 1); + let view = db.lane_workspace_view("go-one").unwrap().unwrap(); + let tools_binding = db + .conn + .query_row( + "SELECT COUNT(*) FROM environment_component_bindings WHERE view_id = ?1 AND component_id = 'go-vendor:tools'", + params![view.view_id], + |row| row.get::<_, i64>(0), + ) + .unwrap(); + assert_eq!(tools_binding, 0); + db.conn + .execute_batch("DROP TRIGGER fail_generation_activation") + .unwrap(); + db.sync_workspace_environment("go-one", "trail/go-vendor@1", Some("tools")) + .unwrap(); + let generation = db.active_environment_generation("go-one").unwrap().unwrap(); + assert_eq!(generation.generation_sequence, 2); + assert_eq!(generation.components.len(), 2); + assert_eq!( + generation + .components + .iter() + .map(|component| component.component_id.as_str()) + .collect::>(), + vec!["go-vendor", "go-vendor:tools"] + ); + let predecessor = generation.predecessor_generation_id.unwrap(); + let command_environment = db.lane_workspace_environment("go-one").unwrap(); + assert_eq!( + command_environment + .iter() + .find(|(name, _)| name == "TRAIL_ENVIRONMENT_GENERATION") + .map(|(_, value)| value.as_str()), + Some(generation.generation_id.as_str()) + ); + let predecessor_state = db + .conn + .query_row( + "SELECT state FROM environment_generations WHERE generation_id = ?1", + params![predecessor], + |row| row.get::<_, String>(0), + ) + .unwrap(); + assert_eq!(predecessor_state, "retired"); + } +} diff --git a/trail/src/db/lane/workspace_layer.rs b/trail/src/db/lane/workspace_layer.rs index d70b2bb..01f7a6c 100644 --- a/trail/src/db/lane/workspace_layer.rs +++ b/trail/src/db/lane/workspace_layer.rs @@ -1,25 +1,69 @@ +use super::workdir::{PreparedLayerMountReset, ViewCore}; use super::*; use serde::{Deserialize, Serialize}; use std::thread; const LAYER_BUILD_LEASE_SECS: i64 = 300; +const WORKSPACE_LAYER_VERIFICATION_STAMP_VERSION: u16 = 1; +const WORKSPACE_LAYER_SIDECAR_MAX_BYTES: u64 = 64 * 1024; #[derive(Clone, Debug)] pub(crate) struct WorkspaceLayerBinding { + /// Durable identity used by filesystem-side activation recovery. For an + /// immutable binding this is the layer ID; writable-private bindings use a + /// component/output/key-derived identity and intentionally have no layer. + pub(crate) binding_identity: String, #[allow(dead_code)] - pub(crate) layer_id: String, + pub(crate) layer_id: Option, pub(crate) mount_path: String, - pub(crate) storage_path: PathBuf, + pub(crate) storage_path: Option, + pub(crate) kind: String, #[allow(dead_code)] pub(crate) priority: i64, } +#[derive(Clone, Debug)] +pub(crate) struct EnvironmentLayerActivation { + pub(crate) layer_id: Option, + pub(crate) outputs: Vec, + pub(crate) component_id: String, + pub(crate) adapter_identity: String, + pub(crate) adapter_version: u32, + pub(crate) implementation_version: String, + pub(crate) distribution_digest: String, + pub(crate) kind: String, + /// Direct dependencies, exact upstream generation keys, and typed edge + /// semantics. Only identity-bearing edge keys also occur in the canonical + /// component identity. + pub(crate) dependencies: Vec<(String, String, String)>, + pub(crate) caches: Vec, + pub(crate) external_artifacts: Vec, + pub(crate) runtime_resources: Vec, + pub(crate) expected_key: String, + pub(crate) canonical_key: WorkspaceLayerKeyV1, +} + +#[derive(Clone, Debug)] +pub(crate) struct EnvironmentLayerOutputActivation { + pub(crate) name: String, + pub(crate) mount_path: String, + pub(crate) policy: String, + pub(crate) binding_identity: String, + /// Staged initial content for a replaced writable-private output. None + /// means preserve the compatible lane-private directory in place. + pub(crate) private_seed: Option, + /// Empty for the historical single-output layer layout. + pub(crate) layer_subpath: String, +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] struct WorkspaceLayerManifest { version: u16, layer_id: String, kind: String, cache_key: String, + #[serde(default)] + layer_key: Option, adapter: String, adapter_version: u32, logical_bytes: u64, @@ -50,6 +94,32 @@ struct WorkspaceLayerPublishMarker { entry_count: u64, } +/// Durable evidence that a complete verification was performed for this exact +/// immutable directory identity. Routine attachment can validate this bounded +/// record instead of recursively reopening every file in a large dependency tree. +/// Explicit `cache verify` and readiness checks still perform a full scan. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +struct WorkspaceLayerVerificationStamp { + version: u16, + layer_id: String, + manifest_object_id: String, + root_identity: WorkspaceLayerRootIdentity, + logical_bytes: u64, + entry_count: u64, + verified_at: i64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +struct WorkspaceLayerRootIdentity { + platform: String, + values: Vec, +} + +#[cfg(test)] +std::thread_local! { + static WORKSPACE_LAYER_FULL_SCAN_COUNT: Cell = const { Cell::new(0) }; +} + impl Trail { pub fn workspace_layer_cache_key(&self, key: &WorkspaceLayerKeyV1) -> Result { validate_layer_key(key)?; @@ -67,7 +137,7 @@ impl Trail { let cache_key = self.workspace_layer_cache_key(key)?; if let Some(layer) = self.workspace_layer_by_cache_key(&cache_key)? { if layer.state == "ready" { - return self.verify_workspace_layer(&layer.layer_id); + return self.verify_workspace_layer_for_attach(&layer.layer_id); } } let lock_path = self @@ -94,7 +164,7 @@ impl Trail { Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { if let Some(layer) = self.workspace_layer_by_cache_key(&cache_key)? { if layer.state == "ready" { - return self.verify_workspace_layer(&layer.layer_id); + return self.verify_workspace_layer_for_attach(&layer.layer_id); } } if build_lock_is_stale(&lock_path)? { @@ -124,7 +194,7 @@ impl Trail { if let Some(layer) = self.workspace_layer_by_cache_key(&cache_key)? { if layer.state == "ready" { drop(guard); - return self.verify_workspace_layer(&layer.layer_id); + return self.verify_workspace_layer_for_attach(&layer.layer_id); } } let build_dir = self.db_dir.join("cache/staging").join(format!( @@ -169,7 +239,7 @@ impl Trail { let final_path = self.db_dir.join("cache/layers").join(&layer_id); if let Some(report) = self.workspace_layer_by_cache_key(&cache_key)? { if report.state == "ready" { - self.verify_workspace_layer(&report.layer_id)?; + self.verify_workspace_layer_for_attach(&report.layer_id)?; return Ok(report); } if final_path.is_dir() { @@ -239,6 +309,7 @@ impl Trail { layer_id: layer_id.clone(), kind: key.kind.clone(), cache_key: cache_key.clone(), + layer_key: Some(key.clone()), adapter: key.adapter.clone(), adapter_version: key.adapter_version, logical_bytes, @@ -297,10 +368,16 @@ impl Trail { ], )?; test_crash_point("layer_after_ready_state"); - self.workspace_layer_by_cache_key(&cache_key)? + let report = self + .workspace_layer_by_cache_key(&cache_key)? .ok_or_else(|| { Error::Corrupt("published workspace layer row disappeared".to_string()) - }) + })?; + // The layer is already durable and ready. A sidecar failure must + // not invalidate correct shared content; the next attach safely + // falls back to a full verification and retries the stamp. + let _ = write_workspace_layer_verification_stamp(&report, &manifest_id.0); + Ok(report) })(); if let Err(err) = &publish { self.conn.execute( @@ -346,7 +423,7 @@ impl Trail { final_path: &Path, ) -> Result { let marker_path = workspace_layer_marker_path(final_path); - let marker: WorkspaceLayerPublishMarker = serde_json::from_slice(&fs::read(&marker_path)?)?; + let marker: WorkspaceLayerPublishMarker = read_workspace_layer_sidecar(&marker_path)?; if marker.layer_id != layer_id || marker.cache_key != cache_key { return Err(Error::Corrupt(format!( "workspace layer publish marker `{}` has the wrong identity", @@ -358,7 +435,13 @@ impl Trail { &ObjectId(marker.manifest_object_id.clone()), )?; let actual = scan_layer_entries(final_path, false)?; - if manifest.entries != actual || manifest.cache_key != cache_key { + let manifest_key_matches = manifest + .layer_key + .as_ref() + .map(|layer_key| self.workspace_layer_cache_key(layer_key)) + .transpose()? + .is_none_or(|canonical| canonical == cache_key); + if manifest.entries != actual || manifest.cache_key != cache_key || !manifest_key_matches { return Err(Error::Corrupt(format!( "workspace layer `{layer_id}` cannot recover because its published tree is corrupt" ))); @@ -382,8 +465,13 @@ impl Trail { now_ts(), ], )?; - self.workspace_layer_by_cache_key(cache_key)? - .ok_or_else(|| Error::Corrupt("recovered workspace layer row disappeared".to_string())) + let report = self + .workspace_layer_by_cache_key(cache_key)? + .ok_or_else(|| { + Error::Corrupt("recovered workspace layer row disappeared".to_string()) + })?; + let _ = write_workspace_layer_verification_stamp(&report, &marker.manifest_object_id); + Ok(report) } pub fn workspace_layer_by_cache_key( @@ -400,6 +488,49 @@ impl Trail { .map_err(Error::from) } + pub(crate) fn workspace_layer_key_by_cache_key( + &self, + cache_key: &str, + ) -> Result> { + let layer_id = self + .conn + .query_row( + "SELECT layer_id FROM workspace_layers WHERE cache_key = ?1", + params![cache_key], + |row| row.get::<_, String>(0), + ) + .optional()?; + if let Some(layer_id) = layer_id { + let (_, _, manifest) = self.workspace_layer_verification_record(&layer_id)?; + if manifest.layer_key.is_some() { + return Ok(manifest.layer_key); + } + } + let bytes = self + .conn + .query_row( + "SELECT canonical_key_json FROM environment_component_key_provenance + WHERE component_key = ?1", + params![cache_key], + |row| row.get::<_, Vec>(0), + ) + .optional()?; + let Some(bytes) = bytes else { + return Ok(None); + }; + let key: WorkspaceLayerKeyV1 = serde_json::from_slice(&bytes).map_err(|error| { + Error::Corrupt(format!( + "environment key provenance `{cache_key}` is malformed: {error}" + )) + })?; + if self.workspace_layer_cache_key(&key)? != cache_key { + return Err(Error::Corrupt(format!( + "environment key provenance `{cache_key}` does not match its content identity" + ))); + } + Ok(Some(key)) + } + pub fn list_workspace_layers(&self) -> Result> { let mut stmt = self.conn.prepare( "SELECT layer_id, kind, cache_key, adapter, state, storage_path, logical_bytes, physical_bytes, entry_count, portability_scope FROM workspace_layers ORDER BY last_used_at DESC, layer_id ASC", @@ -414,12 +545,17 @@ impl Trail { pub(crate) fn workspace_reclaimable_cache_bytes(&self) -> Result { let layer_bytes = self.conn.query_row( "SELECT COALESCE(SUM(COALESCE(l.physical_bytes, 0)), 0) FROM workspace_layers l \ - WHERE l.state != 'building' AND NOT EXISTS (SELECT 1 FROM workspace_view_layers b WHERE b.layer_id = l.layer_id)", + WHERE l.state != 'building' + AND NOT EXISTS (SELECT 1 FROM workspace_view_layers b WHERE b.layer_id = l.layer_id) + AND NOT EXISTS (SELECT 1 FROM environment_generation_components g WHERE g.layer_id = l.layer_id)", [], |row| row.get::<_, i64>(0), )?; let blobs = cache_tree_usage(&self.db_dir.join("cache/blobs"))?; - Ok((layer_bytes.max(0) as u64).saturating_add(blobs)) + let environment_caches = cache_tree_usage(&self.db_dir.join("cache/namespaces"))?; + Ok((layer_bytes.max(0) as u64) + .saturating_add(blobs) + .saturating_add(environment_caches)) } pub fn workspace_cache_gc( @@ -436,7 +572,8 @@ impl Trail { { let mut stmt = self.conn.prepare( "SELECT l.layer_id, l.storage_path, COALESCE(l.physical_bytes, 0), l.last_used_at, l.state, \ - EXISTS(SELECT 1 FROM workspace_view_layers b WHERE b.layer_id = l.layer_id) \ + (EXISTS(SELECT 1 FROM workspace_view_layers b WHERE b.layer_id = l.layer_id) + OR EXISTS(SELECT 1 FROM environment_generation_components g WHERE g.layer_id = l.layer_id)) \ FROM workspace_layers l ORDER BY l.last_used_at ASC, l.layer_id ASC", )?; let rows = stmt.query_map([], |row| { @@ -472,6 +609,53 @@ impl Trail { }); } } + { + let mut stmt = self.conn.prepare( + "SELECT namespace_id, storage_path, last_used_at + FROM environment_cache_namespaces + ORDER BY last_used_at ASC, namespace_id ASC", + )?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + })?; + for row in rows { + let (namespace_id, storage_path, last_used_at) = row?; + if super::workspace_environment::environment_cache_namespace_has_live_leases( + &self.db_dir, + &namespace_id, + !dry_run, + )? { + continue; + } + let expected = cache_root.join("namespaces").join(&namespace_id); + let path = PathBuf::from(&storage_path); + if path != expected { + return Err(Error::Corrupt(format!( + "environment cache namespace `{namespace_id}` has invalid storage path `{storage_path}`" + ))); + } + found.push(CacheGcCandidate { + entry: WorkspaceCacheGcEntry { + kind: "environment_cache".to_string(), + id: namespace_id, + path: storage_path, + physical_bytes: cache_tree_usage(&path)?, + pinned: false, + reason: if last_used_at <= cutoff { + "performance_cache_retention_expired".to_string() + } else { + "performance_cache_lru".to_string() + }, + }, + last_used_at, + retention_expired: last_used_at <= cutoff, + }); + } + } let blob_root = cache_root.join("blobs"); if blob_root.exists() { for entry in walkdir::WalkDir::new(&blob_root).follow_links(false) { @@ -554,7 +738,8 @@ impl Trail { let path = PathBuf::from(&candidate.path); if candidate.kind == "layer" { let pinned = self.conn.query_row( - "SELECT EXISTS(SELECT 1 FROM workspace_view_layers WHERE layer_id = ?1)", + "SELECT (EXISTS(SELECT 1 FROM workspace_view_layers WHERE layer_id = ?1) + OR EXISTS(SELECT 1 FROM environment_generation_components WHERE layer_id = ?1))", params![candidate.id], |row| row.get::<_, i64>(0), )? != 0; @@ -572,7 +757,9 @@ impl Trail { if path.exists() { let previous_state = state.as_deref().unwrap_or("ready"); self.conn.execute( - "UPDATE workspace_layers SET state = 'deleting' WHERE layer_id = ?1 AND NOT EXISTS (SELECT 1 FROM workspace_view_layers WHERE layer_id = ?1)", + "UPDATE workspace_layers SET state = 'deleting' WHERE layer_id = ?1 + AND NOT EXISTS (SELECT 1 FROM workspace_view_layers WHERE layer_id = ?1) + AND NOT EXISTS (SELECT 1 FROM environment_generation_components WHERE layer_id = ?1)", params![candidate.id], )?; make_layer_root_writable(&path)?; @@ -600,7 +787,9 @@ impl Trail { ))); } if let Err(err) = self.conn.execute( - "DELETE FROM workspace_layers WHERE layer_id = ?1 AND NOT EXISTS (SELECT 1 FROM workspace_view_layers WHERE layer_id = ?1)", + "DELETE FROM workspace_layers WHERE layer_id = ?1 + AND NOT EXISTS (SELECT 1 FROM workspace_view_layers WHERE layer_id = ?1) + AND NOT EXISTS (SELECT 1 FROM environment_generation_components WHERE layer_id = ?1)", params![candidate.id], ) { let _ = fs::rename(&trash_path, &path); @@ -615,12 +804,80 @@ impl Trail { })?; } else { self.conn.execute( - "DELETE FROM workspace_layers WHERE layer_id = ?1 AND NOT EXISTS (SELECT 1 FROM workspace_view_layers WHERE layer_id = ?1)", + "DELETE FROM workspace_layers WHERE layer_id = ?1 + AND NOT EXISTS (SELECT 1 FROM workspace_view_layers WHERE layer_id = ?1) + AND NOT EXISTS (SELECT 1 FROM environment_generation_components WHERE layer_id = ?1)", params![candidate.id], )?; remove_workspace_layer_trash_entries(&trash, &candidate.id)?; } let _ = fs::remove_file(workspace_layer_marker_path(&path)); + let _ = fs::remove_file(workspace_layer_verification_stamp_path(&path)); + } else if candidate.kind == "environment_cache" { + let Some(_maintenance) = + acquire_environment_cache_maintenance(&self.db_dir, &candidate.id)? + else { + continue; + }; + if super::workspace_environment::environment_cache_namespace_has_live_leases( + &self.db_dir, + &candidate.id, + true, + )? { + continue; + } + let expected = cache_root.join("namespaces").join(&candidate.id); + if path != expected { + return Err(Error::Corrupt(format!( + "environment cache namespace `{}` escaped cache storage", + candidate.id + ))); + } + let trash_path = trash.join(format!( + "environment-cache.{}.{}", + candidate.id, + crate::ids::short_hash( + format!("{}:{}", candidate.id, now_nanos()).as_bytes(), + 12 + ) + )); + if path.exists() { + if !path.is_dir() { + return Err(Error::Corrupt(format!( + "environment cache namespace `{}` is not a directory", + candidate.id + ))); + } + fs::rename(&path, &trash_path).map_err(|error| { + Error::InvalidInput(format!( + "failed to atomically quarantine environment cache `{}`: {error}", + path.display() + )) + })?; + } + if let Err(error) = self.conn.execute( + "DELETE FROM environment_cache_namespaces WHERE namespace_id = ?1", + params![candidate.id], + ) { + if trash_path.exists() { + let _ = fs::rename(&trash_path, &path); + } + return Err(Error::from(error)); + } + if trash_path.exists() { + make_tree_writable(&trash_path); + fs::remove_dir_all(&trash_path).map_err(|error| { + Error::InvalidInput(format!( + "failed to remove quarantined environment cache `{}`: {error}", + trash_path.display() + )) + })?; + } + let _ = fs::remove_dir_all( + self.db_dir + .join("cache/namespace-leases") + .join(&candidate.id), + ); } else { if !path.is_file() { continue; @@ -665,7 +922,7 @@ impl Trail { view_id: &str, ) -> Result> { let mut stmt = self.conn.prepare( - "SELECT l.layer_id, l.kind, l.cache_key, l.adapter, l.state, l.storage_path, l.logical_bytes, l.physical_bytes, l.entry_count, l.portability_scope \ + "SELECT DISTINCT l.layer_id, l.kind, l.cache_key, l.adapter, l.state, l.storage_path, l.logical_bytes, l.physical_bytes, l.entry_count, l.portability_scope \ FROM workspace_view_layers b JOIN workspace_layers l ON l.layer_id = b.layer_id \ WHERE b.view_id = ?1 ORDER BY b.priority DESC, b.mount_path ASC", )?; @@ -675,7 +932,10 @@ impl Trail { Ok(rows) } - pub fn verify_workspace_layer(&self, layer_id: &str) -> Result { + fn workspace_layer_verification_record( + &self, + layer_id: &str, + ) -> Result<(WorkspaceLayerReport, String, WorkspaceLayerManifest)> { let row = self.conn.query_row( "SELECT layer_id, kind, cache_key, adapter, state, storage_path, logical_bytes, physical_bytes, entry_count, portability_scope, manifest_object_id FROM workspace_layers WHERE layer_id = ?1", params![layer_id], @@ -694,15 +954,97 @@ impl Trail { let manifest_id = manifest_id.ok_or_else(|| { Error::Corrupt(format!("workspace layer `{layer_id}` has no manifest")) })?; - let manifest: WorkspaceLayerManifest = - self.get_object(WORKSPACE_LAYER_MANIFEST_KIND, &ObjectId(manifest_id))?; + let manifest: WorkspaceLayerManifest = self.get_object( + WORKSPACE_LAYER_MANIFEST_KIND, + &ObjectId(manifest_id.clone()), + )?; + let manifest_key_matches = manifest + .layer_key + .as_ref() + .map(|key| self.workspace_layer_cache_key(key)) + .transpose() + .map_err(|error| { + Error::Corrupt(format!( + "workspace layer `{layer_id}` contains an invalid canonical key: {error}" + )) + })? + .is_none_or(|cache_key| cache_key == report.cache_key); + if manifest.version != WORKSPACE_LAYER_MANIFEST_VERSION + || !manifest_key_matches + || manifest.layer_id != layer_id + || manifest.kind != report.kind + || manifest.cache_key != report.cache_key + || manifest.adapter != report.adapter + || manifest.logical_bytes != report.logical_bytes + || manifest.entries.len() as u64 != report.entry_count + || manifest.portability_scope != report.portability_scope + { + report.state = "corrupt".to_string(); + return Err(Error::Corrupt(format!( + "workspace layer `{layer_id}` metadata does not match its immutable manifest" + ))); + } + Ok((report, manifest_id, manifest)) + } + + /// Bounded verification used by routine cache reuse and attachment. A + /// missing or stale stamp safely falls back to one complete verification, + /// which refreshes the durable stamp for later attaches. + pub(crate) fn verify_workspace_layer_for_attach( + &self, + layer_id: &str, + ) -> Result { + let (report, manifest_id, manifest) = self.workspace_layer_verification_record(layer_id)?; + if report.state != "ready" { + return Err(Error::Corrupt(format!( + "workspace layer `{layer_id}` is `{}` and cannot be attached", + report.state + ))); + } + let storage_path = Path::new(&report.storage_path); + let root_identity = workspace_layer_root_identity(storage_path)?; + let stamp_path = workspace_layer_verification_stamp_path(storage_path); + let marker_path = workspace_layer_marker_path(storage_path); + let stamp = read_workspace_layer_sidecar::(&stamp_path); + let marker = read_workspace_layer_sidecar::(&marker_path); + let stamp_matches = stamp.is_ok_and(|stamp| { + stamp.version == WORKSPACE_LAYER_VERIFICATION_STAMP_VERSION + && stamp.layer_id == layer_id + && stamp.manifest_object_id == manifest_id + && stamp.root_identity == root_identity + && stamp.logical_bytes == report.logical_bytes + && stamp.entry_count == report.entry_count + }); + let marker_matches = marker.is_ok_and(|marker| { + marker.layer_id == layer_id + && marker.cache_key == report.cache_key + && marker.manifest_object_id == manifest_id + && marker.logical_bytes == report.logical_bytes + && marker.entry_count == report.entry_count + && report + .physical_bytes + .is_none_or(|bytes| bytes == marker.physical_bytes) + }); + if stamp_matches && marker_matches && manifest.entries.len() as u64 == report.entry_count { + return Ok(report); + } + self.verify_workspace_layer(layer_id) + } + + /// Perform an explicit complete verification of every layer entry and + /// refresh the bounded attach-tier stamp only after all hashes match. + pub fn verify_workspace_layer(&self, layer_id: &str) -> Result { + let (mut report, manifest_id, manifest) = + self.workspace_layer_verification_record(layer_id)?; let actual = scan_layer_entries(Path::new(&report.storage_path), false)?; - if manifest.layer_id != layer_id || manifest.entries != actual { + if manifest.entries != actual { report.state = "corrupt".to_string(); return Err(Error::Corrupt(format!( "workspace layer `{layer_id}` does not match its immutable manifest" ))); } + let _ = write_workspace_layer_verification_stamp(&report, &manifest_id); + let _ = write_workspace_layer_publish_marker_from_report(&report, &manifest_id); Ok(report) } @@ -714,8 +1056,161 @@ impl Trail { adapter: &str, expected_key: &str, ) -> Result { + self.bind_workspace_layer( + lane, + layer_id, + mount_path, + adapter, + expected_key, + false, + None, + ) + } + + #[cfg(test)] + pub(crate) fn replace_workspace_layer( + &self, + lane: &str, + layer_id: &str, + mount_path: &str, + adapter: &str, + expected_key: &str, + ) -> Result { + self.bind_workspace_layer( + lane, + layer_id, + mount_path, + adapter, + expected_key, + true, + None, + ) + } + + #[cfg(test)] + #[allow(clippy::too_many_arguments)] + pub(crate) fn replace_declared_workspace_layer( + &self, + lane: &str, + layer_id: &str, + mount_path: &str, + component_id: &str, + adapter_identity: &str, + adapter_version: u32, + implementation_version: &str, + distribution_digest: &str, + kind: &str, + expected_key: &str, + ) -> Result { + self.replace_declared_workspace_layers( + lane, + &[EnvironmentLayerActivation { + layer_id: Some(layer_id.to_string()), + outputs: vec![EnvironmentLayerOutputActivation { + name: "primary".to_string(), + mount_path: mount_path.to_string(), + policy: "immutable_seed_private".to_string(), + binding_identity: layer_id.to_string(), + private_seed: None, + layer_subpath: String::new(), + }], + component_id: component_id.to_string(), + adapter_identity: adapter_identity.to_string(), + adapter_version, + implementation_version: implementation_version.to_string(), + distribution_digest: distribution_digest.to_string(), + kind: kind.to_string(), + dependencies: Vec::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + expected_key: expected_key.to_string(), + canonical_key: self + .workspace_layer_key_by_cache_key(expected_key)? + .ok_or_else(|| { + Error::Corrupt(format!( + "workspace layer `{layer_id}` has no canonical key provenance" + )) + })?, + }], + ) + } + + #[cfg(test)] + pub(crate) fn replace_declared_workspace_layers( + &self, + lane: &str, + requested: &[EnvironmentLayerActivation], + ) -> Result { + self.replace_declared_workspace_layers_with_removals(lane, requested, &[]) + } + + pub(crate) fn replace_declared_workspace_layers_at_source( + &self, + lane: &str, + requested: &[EnvironmentLayerActivation], + source_root: &ObjectId, + ) -> Result { + self.replace_declared_workspace_layers_with_removals_internal( + lane, + requested, + &[], + Some(source_root), + ) + } + + #[cfg(test)] + pub(crate) fn replace_declared_workspace_layers_with_removals( + &self, + lane: &str, + requested: &[EnvironmentLayerActivation], + removed_components: &[String], + ) -> Result { + self.replace_declared_workspace_layers_with_removals_internal( + lane, + requested, + removed_components, + None, + ) + } + + pub(crate) fn replace_declared_workspace_layers_with_removals_at_source( + &self, + lane: &str, + requested: &[EnvironmentLayerActivation], + removed_components: &[String], + source_root: &ObjectId, + ) -> Result { + self.replace_declared_workspace_layers_with_removals_internal( + lane, + requested, + removed_components, + Some(source_root), + ) + } + + fn replace_declared_workspace_layers_with_removals_internal( + &self, + lane: &str, + requested: &[EnvironmentLayerActivation], + removed_components: &[String], + source_root: Option<&ObjectId>, + ) -> Result { + if requested.is_empty() && removed_components.is_empty() { + return Err(Error::InvalidInput( + "environment activation requires at least one replacement or removal".to_string(), + )); + } let _lock = self.acquire_write_lock()?; - let layer = self.verify_workspace_layer(layer_id)?; + if let Some(expected) = source_root { + let branch = self.lane_branch(lane)?; + let current = self.get_ref(&branch.ref_name)?.root_id; + if ¤t != expected { + return Err(Error::InvalidInput(format!( + "lane `{lane}` advanced from pinned source root `{expected}` to `{current}` before environment activation; retry against the new lane head" + ))); + } + } let view = self.lane_workspace_view(lane)?.ok_or_else(|| { Error::InvalidInput(format!( "lane `{lane}` does not have a layered workspace view" @@ -724,92 +1219,1726 @@ impl Trail { if let (Some(pid), Some(token)) = (view.owner_pid, view.owner_start_token.as_deref()) { if process_matches_start_token(pid, token) { return Err(Error::InvalidInput(format!( - "workspace view `{}` has an active writer; unmount before changing layer bindings", + "workspace view `{}` for lane `{lane}` has an active writer; run `trail lane unmount {lane}` before changing environment bindings", view.view_id ))); } } - let mount_path = normalize_relative_path(mount_path)?; - self.conn.execute( - "INSERT OR REPLACE INTO workspace_view_layers (view_id, layer_id, mount_path, priority, read_only) VALUES (?1, ?2, ?3, 100, 1)", - params![view.view_id, layer.layer_id, mount_path], - )?; - self.conn.execute( - "UPDATE workspace_layers SET last_used_at = ?1 WHERE layer_id = ?2", - params![now_ts(), layer.layer_id], - )?; - self.conn.execute( - "INSERT OR REPLACE INTO workspace_environment_states (view_id, adapter, expected_key, attached_key, status, reason, updated_at) VALUES (?1, ?2, ?3, ?4, 'ready', NULL, ?5)", - params![view.view_id, adapter, expected_key, layer.cache_key, now_ts()], - )?; - self.conn.execute( - "UPDATE workspace_views SET generation = generation + 1, updated_at = ?1 WHERE view_id = ?2", - params![now_ts(), view.view_id], - )?; - self.lane_workspace_view(lane)?.ok_or_else(|| { - Error::Corrupt("workspace view disappeared after layer attach".to_string()) - }) - } - - pub(crate) fn workspace_layer_bindings_for_source_upper( - &self, - source_upper: &Path, - ) -> Result> { - let mut stmt = self.conn.prepare( - "SELECT l.layer_id, b.mount_path, l.storage_path, b.priority \ - FROM workspace_views v JOIN workspace_view_layers b ON b.view_id = v.view_id \ - JOIN workspace_layers l ON l.layer_id = b.layer_id \ - WHERE v.source_upper = ?1 AND b.read_only = 1 AND l.state = 'ready' \ - ORDER BY length(b.mount_path) DESC, b.priority DESC", - )?; - let rows = stmt - .query_map(params![source_upper.to_string_lossy()], |row| { - Ok(WorkspaceLayerBinding { - layer_id: row.get(0)?, - mount_path: row.get(1)?, - storage_path: PathBuf::from(row.get::<_, String>(2)?), - priority: row.get(3)?, - }) - }) - .map_err(Error::from)? - .collect::, _>>()?; - Ok(rows) - } -} - -struct CacheGcCandidate { - entry: WorkspaceCacheGcEntry, - last_used_at: i64, - retention_expired: bool, -} - -fn cache_tree_usage(path: &Path) -> Result { - if !path.exists() { - return Ok(0); - } - let mut bytes = 0_u64; - for entry in walkdir::WalkDir::new(path).follow_links(false) { - let entry = entry.map_err(|err| Error::InvalidInput(err.to_string()))?; - if entry.file_type().is_file() { - let metadata = entry.metadata().map_err(|err| Error::Io(err.into()))?; - bytes = bytes.saturating_add(cache_file_physical_bytes(&metadata)); - } - } - Ok(bytes) -} -fn cache_tree_logical_bytes(path: &Path) -> Result { - if !path.exists() { - return Ok(0); - } - let mut bytes = 0_u64; - for entry in walkdir::WalkDir::new(path).follow_links(false) { - let entry = entry.map_err(|err| Error::InvalidInput(err.to_string()))?; - if entry.file_type().is_file() { - let metadata = entry.metadata().map_err(|err| Error::Io(err.into()))?; - bytes = bytes.saturating_add(metadata.len()); + let mut requested_keys = BTreeMap::new(); + for activation in requested { + super::workspace_environment::validate_environment_component_identity( + &activation.component_id, + )?; + if requested_keys + .insert( + activation.component_id.clone(), + activation.expected_key.clone(), + ) + .is_some() + { + return Err(Error::InvalidInput(format!( + "environment activation contains component `{}` more than once", + activation.component_id + ))); + } } - } + let component_ids = requested_keys.keys().cloned().collect::>(); + let mut removed_ids = BTreeSet::new(); + for component_id in removed_components { + super::workspace_environment::validate_environment_component_identity(component_id)?; + if component_ids.contains(component_id) || !removed_ids.insert(component_id.clone()) { + return Err(Error::InvalidInput(format!( + "environment activation both replaces or repeats removed component `{component_id}`" + ))); + } + } + let replaced_component_ids = component_ids + .union(&removed_ids) + .cloned() + .collect::>(); + let dependent_components = + self.environment_dependency_descendants(&view.view_id, &replaced_component_ids)?; + let mut removed_bindings = Vec::with_capacity(removed_ids.len()); + for component_id in &removed_ids { + let mut stmt = self.conn.prepare( + "SELECT mount_path, kind, policy, binding_identity + FROM environment_component_output_bindings + WHERE view_id = ?1 AND component_id = ?2 + ORDER BY output_name", + )?; + let bindings = stmt + .query_map(params![&view.view_id, component_id], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + })? + .collect::, _>>()?; + removed_bindings.push((component_id.clone(), bindings)); + } + let mut resolved = Vec::with_capacity(requested.len()); + let mut mount_paths = Vec::<(String, String)>::new(); + for activation in requested { + if self.workspace_layer_cache_key(&activation.canonical_key)? != activation.expected_key + { + return Err(Error::Corrupt(format!( + "environment component `{}` canonical key does not match its expected key", + activation.component_id + ))); + } + let mut dependency_ids = BTreeSet::new(); + for (dependency, component_key, edge_type) in &activation.dependencies { + super::workspace_environment::validate_environment_component_identity(dependency)?; + if dependency == &activation.component_id + || !dependency_ids.insert(dependency.clone()) + { + return Err(Error::InvalidInput(format!( + "environment component `{}` has an invalid or duplicate dependency `{dependency}`", + activation.component_id + ))); + } + let identity_input = match edge_type.as_str() { + "build_requires" => Some(format!("dependency:{dependency}")), + "invalidates_with" => Some(format!("dependency:invalidates_with:{dependency}")), + "runtime_requires" | "binds_after" => None, + other => { + return Err(Error::InvalidInput(format!( + "environment component `{}` has unknown dependency edge type `{other}`", + activation.component_id + ))); + } + }; + if identity_input.as_ref().is_some_and(|input| { + activation.canonical_key.inputs.get(input) != Some(component_key) + }) { + return Err(Error::Corrupt(format!( + "environment component `{}` dependency `{dependency}` does not match its canonical key", + activation.component_id + ))); + } + if let Some(requested_key) = requested_keys.get(dependency) { + if requested_key != component_key { + return Err(Error::InvalidInput(format!( + "environment component `{}` requires `{dependency}` at `{component_key}`, but this activation provides `{requested_key}`", + activation.component_id + ))); + } + continue; + } + let attached = self + .conn + .query_row( + "SELECT attached_key, status FROM environment_component_states + WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, dependency], + |row| Ok((row.get::<_, Option>(0)?, row.get::<_, String>(1)?)), + ) + .optional()?; + if !matches!(attached, Some((Some(ref key), ref status)) if key == component_key && status == "ready") + { + return Err(Error::InvalidInput(format!( + "environment component `{}` requires `{dependency}` at `{component_key}`, but that dependency is not ready in the lane or this activation", + activation.component_id + ))); + } + } + let mut cache_names = BTreeSet::new(); + for cache in &activation.caches { + if !cache_names.insert(cache.name.clone()) + || !cache.namespace_id.starts_with("cache_") + || cache.namespace_id.len() != "cache_".len() + 64 + || !cache.namespace_id["cache_".len()..] + .chars() + .all(|character| character.is_ascii_hexdigit()) + || !matches!( + cache.protocol.as_str(), + "content_store" | "compiler_cache" | "locked_index" + ) + || !matches!(cache.access.as_str(), "tool_concurrent" | "host_exclusive") + || cache.authority != "performance_only" + || cache.scope != "workspace" + || cache.compatibility.is_empty() + { + return Err(Error::InvalidInput(format!( + "environment component `{}` has an invalid cache declaration `{}`", + activation.component_id, cache.name + ))); + } + } + let mut external_artifact_names = BTreeSet::new(); + for artifact in &activation.external_artifacts { + super::workspace_environment::validate_environment_external_artifact_report( + artifact, + )?; + if !external_artifact_names.insert(&artifact.name) { + return Err(Error::InvalidInput(format!( + "environment component `{}` repeats external artifact `{}`", + activation.component_id, artifact.name + ))); + } + } + let mut runtime_resource_names = BTreeSet::new(); + for resource in &activation.runtime_resources { + super::workspace_environment::validate_environment_runtime_declaration_report( + resource, + )?; + if !runtime_resource_names.insert(&resource.name) { + return Err(Error::InvalidInput(format!( + "environment component `{}` repeats runtime resource `{}`", + activation.component_id, resource.name + ))); + } + if !external_artifact_names.contains(&resource.artifact_name) { + return Err(Error::InvalidInput(format!( + "environment component `{}` runtime resource `{}` references missing external artifact `{}`", + activation.component_id, resource.name, resource.artifact_name + ))); + } + } + let layer = activation + .layer_id + .as_deref() + .map(|layer_id| self.verify_workspace_layer_for_attach(layer_id)) + .transpose()?; + if let Some(layer) = &layer { + if layer.kind != activation.kind { + return Err(Error::Corrupt(format!( + "component `{}` declares kind `{}` but layer `{}` has kind `{}`", + activation.component_id, activation.kind, layer.layer_id, layer.kind + ))); + } + } + let mut previous_stmt = self.conn.prepare( + "SELECT mount_path, kind, policy, binding_identity + FROM environment_component_output_bindings + WHERE view_id = ?1 AND component_id = ?2 ORDER BY output_name", + )?; + let previous_bindings = previous_stmt + .query_map(params![&view.view_id, &activation.component_id], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + })? + .collect::, _>>()?; + if activation.outputs.is_empty() + && activation.external_artifacts.is_empty() + && activation.runtime_resources.is_empty() + { + return Err(Error::InvalidInput(format!( + "environment component `{}` activation has neither outputs nor external/runtime resources", + activation.component_id + ))); + } + if (!activation.external_artifacts.is_empty() + || !activation.runtime_resources.is_empty()) + && (layer.is_some() || !activation.outputs.is_empty()) + { + return Err(Error::InvalidInput(format!( + "environment component `{}` mixes external/runtime resources with filesystem layer outputs", + activation.component_id + ))); + } + let mut names = BTreeSet::new(); + let mut outputs = Vec::with_capacity(activation.outputs.len()); + for output in &activation.outputs { + if !names.insert(output.name.clone()) { + return Err(Error::InvalidInput(format!( + "environment component `{}` activation repeats output `{}`", + activation.component_id, output.name + ))); + } + let mount_path = normalize_relative_path(&output.mount_path)?; + let layer_subpath = if output.layer_subpath.is_empty() { + String::new() + } else { + normalize_relative_path(&output.layer_subpath)? + }; + if output.binding_identity.trim().is_empty() { + return Err(Error::InvalidInput(format!( + "environment component `{}` output `{}` has an empty binding identity", + activation.component_id, output.name + ))); + } + match output.policy.as_str() { + "immutable_seed_private" => { + let layer = layer.as_ref().ok_or_else(|| { + Error::Corrupt(format!( + "environment component `{}` immutable output `{}` has no layer", + activation.component_id, output.name + )) + })?; + if output.private_seed.is_some() + || output.binding_identity != layer.layer_id + { + return Err(Error::Corrupt(format!( + "environment component `{}` immutable output `{}` has inconsistent storage identity", + activation.component_id, output.name + ))); + } + let source = if layer_subpath.is_empty() { + PathBuf::from(&layer.storage_path) + } else { + safe_join(Path::new(&layer.storage_path), &layer_subpath)? + }; + if !source.is_dir() { + return Err(Error::Corrupt(format!( + "environment component `{}` output `{}` refers to missing layer directory `{layer_subpath}`", + activation.component_id, output.name + ))); + } + } + "writable_private" => { + if layer.is_some() || !layer_subpath.is_empty() { + return Err(Error::Corrupt(format!( + "environment component `{}` writable-private output `{}` must not reference an immutable layer", + activation.component_id, output.name + ))); + } + if output + .private_seed + .as_ref() + .is_some_and(|path| !path.is_dir()) + { + return Err(Error::InvalidInput(format!( + "environment component `{}` writable-private seed for `{}` is not a directory", + activation.component_id, output.name + ))); + } + } + other => { + return Err(Error::InvalidInput(format!( + "environment component `{}` output `{}` has unsupported policy `{other}`", + activation.component_id, output.name + ))); + } + } + for (owner, existing) in &mount_paths { + if mount_paths_overlap(&mount_path, existing) { + return Err(Error::InvalidInput(format!( + "environment component `{}` mount `{mount_path}` overlaps batch component `{owner}` mount `{existing}`", + activation.component_id + ))); + } + } + mount_paths.push((activation.component_id.clone(), mount_path.clone())); + outputs.push(EnvironmentLayerOutputActivation { + name: output.name.clone(), + mount_path, + layer_subpath, + policy: output.policy.clone(), + binding_identity: output.binding_identity.clone(), + private_seed: output.private_seed.clone(), + }); + } + resolved.push((activation.clone(), layer, outputs, previous_bindings)); + } + self.validate_environment_batch_mount_ownership( + &view.view_id, + &replaced_component_ids, + &mount_paths, + )?; + + let mut core = ViewCore::new_lazy( + // The outer environment mutation owns the workspace writer lock + // and its Trail handle already completed open-time recovery. + Trail::open_without_recovering_derived_paths_under_write_lock( + self.workspace_root(), + self.db_dir(), + )?, + PathBuf::from(&view.source_upper), + view.base_root.clone(), + )?; + let mut resets: Vec = Vec::new(); + for (_, bindings) in &removed_bindings { + for (mount_path, kind, _, _) in bindings { + match core.prepare_declared_layer_unmount_path(mount_path, kind) { + Ok(reset) => resets.push(reset), + Err(err) => { + for reset in resets.into_iter().rev() { + let _ = reset.rollback(&mut core); + } + return Err(err); + } + } + } + } + for (component, layer, outputs, previous_bindings) in &resolved { + for output in outputs { + if output.policy == "writable_private" + && output.private_seed.is_none() + && previous_bindings + .iter() + .any(|(mount, _, policy, binding_identity)| { + mount == &output.mount_path + && policy == &output.policy + && binding_identity == &output.binding_identity + }) + { + if let Err(err) = + core.ensure_declared_private_mount_path(&output.mount_path, &component.kind) + { + for reset in resets.into_iter().rev() { + let _ = reset.rollback(&mut core); + } + return Err(err); + } + continue; + } + let prepared = if output.policy == "writable_private" { + core.prepare_declared_private_mount_path( + &output.mount_path, + &component.kind, + &output.binding_identity, + ) + } else { + let layer = layer.as_ref().ok_or_else(|| { + Error::Corrupt(format!( + "immutable output `{}` lost its prepared layer", + output.name + )) + })?; + core.prepare_declared_layer_mount_path( + &output.mount_path, + &layer.kind, + &layer.layer_id, + ) + }; + match prepared { + Ok(reset) => { + let install = if let Some(seed) = &output.private_seed { + reset.install_private_directory(seed) + } else if output.policy == "writable_private" { + core.ensure_declared_private_mount_path( + &output.mount_path, + &component.kind, + ) + } else { + Ok(()) + }; + if let Err(err) = install { + let _ = reset.rollback(&mut core); + for reset in resets.into_iter().rev() { + let _ = reset.rollback(&mut core); + } + return Err(err); + } + resets.push(reset); + } + Err(err) => { + for reset in resets.into_iter().rev() { + let _ = reset.rollback(&mut core); + } + return Err(err); + } + } + } + let replacement_mounts = outputs + .iter() + .map(|output| output.mount_path.as_str()) + .collect::>(); + for (previous_mount, previous_kind, _, _) in previous_bindings { + if replacement_mounts.contains(previous_mount.as_str()) { + continue; + } + match core.prepare_declared_layer_unmount_path(previous_mount, previous_kind) { + Ok(reset) => resets.push(reset), + Err(err) => { + for reset in resets.into_iter().rev() { + let _ = reset.rollback(&mut core); + } + return Err(err); + } + } + } + } + test_crash_point("environment_after_upper_resets"); + + self.conn + .execute_batch("SAVEPOINT trail_environment_activation")?; + let activation = (|| -> Result<()> { + for (component_id, bindings) in &removed_bindings { + for (mount_path, _, _, _) in bindings { + self.conn.execute( + "DELETE FROM workspace_view_layers WHERE view_id = ?1 AND mount_path = ?2", + params![&view.view_id, mount_path], + )?; + } + self.conn.execute( + "DELETE FROM environment_component_output_bindings WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, component_id], + )?; + self.conn.execute( + "DELETE FROM environment_component_bindings WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, component_id], + )?; + self.conn.execute( + "DELETE FROM environment_component_dependencies WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, component_id], + )?; + self.conn.execute( + "DELETE FROM environment_component_caches WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, component_id], + )?; + self.conn.execute( + "DELETE FROM environment_component_external_artifacts WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, component_id], + )?; + self.conn.execute( + "DELETE FROM environment_component_runtime_secrets WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, component_id], + )?; + self.conn.execute( + "DELETE FROM environment_component_runtime_resources WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, component_id], + )?; + self.conn.execute( + "DELETE FROM environment_component_states WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, component_id], + )?; + self.conn.execute( + "DELETE FROM workspace_environment_states WHERE view_id = ?1 AND adapter = ?2", + params![&view.view_id, component_id], + )?; + } + for (component, layer, outputs, previous_bindings) in &resolved { + self.conn.execute( + "INSERT OR IGNORE INTO environment_component_key_provenance + (component_key, canonical_key_json, created_at) VALUES (?1, ?2, ?3)", + params![ + &component.expected_key, + serde_json::to_vec(&component.canonical_key)?, + now_ts() + ], + )?; + let stored_key = self.conn.query_row( + "SELECT canonical_key_json FROM environment_component_key_provenance + WHERE component_key = ?1", + params![&component.expected_key], + |row| row.get::<_, Vec>(0), + )?; + let stored_key: WorkspaceLayerKeyV1 = + serde_json::from_slice(&stored_key).map_err(|error| { + Error::Corrupt(format!( + "environment key provenance `{}` is malformed: {error}", + component.expected_key + )) + })?; + if stored_key != component.canonical_key { + return Err(Error::Corrupt(format!( + "environment key provenance `{}` does not match its content identity", + component.expected_key + ))); + } + for (previous_mount, _, _, _) in previous_bindings { + self.conn.execute( + "DELETE FROM workspace_view_layers WHERE view_id = ?1 AND mount_path = ?2", + params![&view.view_id, previous_mount], + )?; + } + self.conn.execute( + "DELETE FROM environment_component_output_bindings WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, &component.component_id], + )?; + self.conn.execute( + "DELETE FROM environment_component_bindings WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, &component.component_id], + )?; + self.conn.execute( + "DELETE FROM environment_component_dependencies WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, &component.component_id], + )?; + self.conn.execute( + "DELETE FROM environment_component_caches WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, &component.component_id], + )?; + self.conn.execute( + "DELETE FROM environment_component_external_artifacts WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, &component.component_id], + )?; + self.conn.execute( + "DELETE FROM environment_component_runtime_secrets WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, &component.component_id], + )?; + self.conn.execute( + "DELETE FROM environment_component_runtime_resources WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, &component.component_id], + )?; + for (dependency, dependency_key, edge_type) in &component.dependencies { + self.conn.execute( + "INSERT INTO environment_component_dependencies + (view_id, component_id, dependency_component_id, dependency_component_key, edge_type, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + &view.view_id, + &component.component_id, + dependency, + dependency_key, + edge_type, + now_ts() + ], + )?; + } + for cache in &component.caches { + let compatibility_json = serde_json::to_vec(&cache.compatibility)?; + let storage_path = self + .db_dir + .join("cache/namespaces") + .join(&cache.namespace_id) + .to_string_lossy() + .into_owned(); + self.conn.execute( + "INSERT OR IGNORE INTO environment_cache_namespaces + (namespace_id, adapter_identity, cache_name, protocol, access, authority, scope, compatibility_json, storage_path, last_used_at, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, 'performance_only', 'workspace', ?6, ?7, ?8, ?8)", + params![ + &cache.namespace_id, + &component.adapter_identity, + &cache.name, + &cache.protocol, + &cache.access, + &compatibility_json, + &storage_path, + now_ts() + ], + )?; + let stored = self.conn.query_row( + "SELECT adapter_identity, cache_name, protocol, access, authority, scope, compatibility_json, storage_path + FROM environment_cache_namespaces WHERE namespace_id = ?1", + params![&cache.namespace_id], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, Vec>(6)?, + row.get::<_, String>(7)?, + )) + }, + )?; + if stored + != ( + component.adapter_identity.clone(), + cache.name.clone(), + cache.protocol.clone(), + cache.access.clone(), + "performance_only".to_string(), + "workspace".to_string(), + compatibility_json.clone(), + storage_path, + ) + { + return Err(Error::Corrupt(format!( + "environment cache namespace `{}` has conflicting provenance", + cache.namespace_id + ))); + } + self.conn.execute( + "INSERT INTO environment_component_caches + (view_id, component_id, cache_name, namespace_id, protocol, access, compatibility_json, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![ + &view.view_id, + &component.component_id, + &cache.name, + &cache.namespace_id, + &cache.protocol, + &cache.access, + &compatibility_json, + now_ts() + ], + )?; + } + for artifact in &component.external_artifacts { + self.conn.execute( + "INSERT INTO environment_component_external_artifacts + (view_id, component_id, artifact_name, artifact_type, provider, reference, digest, platform, cleanup_owner, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + params![ + &view.view_id, + &component.component_id, + &artifact.name, + &artifact.artifact_type, + &artifact.provider, + &artifact.reference, + &artifact.digest, + &artifact.platform, + &artifact.cleanup_owner, + now_ts() + ], + )?; + } + for resource in &component.runtime_resources { + self.conn.execute( + "INSERT INTO environment_component_runtime_resources + (view_id, component_id, resource_name, runtime_type, provider, artifact_name, + container_port, protocol, health_type, health_timeout_ms, restart_policy, + cleanup_owner, volume_target, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", + params![ + &view.view_id, + &component.component_id, + &resource.name, + &resource.runtime_type, + &resource.provider, + &resource.artifact_name, + resource.container_port, + &resource.protocol, + &resource.health_type, + resource.health_timeout_ms, + &resource.restart_policy, + &resource.cleanup_owner, + &resource.volume_target, + now_ts() + ], + )?; + for secret in &resource.secrets { + self.conn.execute( + "INSERT INTO environment_component_runtime_secrets + (view_id, component_id, resource_name, secret_name, provider, + reference, version, purpose, injection, target, environment, + required, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", + params![ + &view.view_id, + &component.component_id, + &resource.name, + &secret.name, + &secret.provider, + &secret.reference, + &secret.version, + &secret.purpose, + &secret.injection, + &secret.target, + &secret.environment, + secret.required, + now_ts() + ], + )?; + } + } + for output in outputs { + if output.policy == "immutable_seed_private" { + let layer = layer.as_ref().ok_or_else(|| { + Error::Corrupt(format!( + "immutable output `{}` lost its layer during activation", + output.name + )) + })?; + self.conn.execute( + "INSERT OR REPLACE INTO workspace_view_layers (view_id, layer_id, mount_path, priority, read_only, source_path) VALUES (?1, ?2, ?3, 100, 1, ?4)", + params![ + &view.view_id, + &layer.layer_id, + &output.mount_path, + &output.layer_subpath + ], + )?; + } + self.conn.execute( + "INSERT INTO environment_component_output_bindings + (view_id, component_id, output_name, mount_path, layer_subpath, policy, binding_identity, kind, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![ + &view.view_id, + &component.component_id, + &output.name, + &output.mount_path, + &output.layer_subpath, + &output.policy, + &output.binding_identity, + &component.kind, + now_ts() + ], + )?; + } + if let Some(layer) = layer { + self.conn.execute( + "UPDATE workspace_layers SET last_used_at = ?1 WHERE layer_id = ?2", + params![now_ts(), &layer.layer_id], + )?; + } + self.conn.execute( + "INSERT OR REPLACE INTO workspace_environment_states (view_id, adapter, expected_key, attached_key, status, reason, updated_at) VALUES (?1, ?2, ?3, ?4, 'ready', NULL, ?5)", + params![ + &view.view_id, + &component.component_id, + &component.expected_key, + &component.expected_key, + now_ts() + ], + )?; + self.conn.execute( + "INSERT OR REPLACE INTO environment_component_states (view_id, component_id, adapter_identity, adapter_version, implementation_version, distribution_digest, kind, expected_key, attached_key, status, reason, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 'ready', NULL, ?10)", + params![ + &view.view_id, + &component.component_id, + &component.adapter_identity, + component.adapter_version, + &component.implementation_version, + &component.distribution_digest, + &component.kind, + &component.expected_key, + &component.expected_key, + now_ts() + ], + )?; + if let Some(primary_output) = outputs.first() { + self.conn.execute( + "INSERT OR REPLACE INTO environment_component_bindings (view_id, component_id, mount_path, kind, updated_at) VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + &view.view_id, + &component.component_id, + &primary_output.mount_path, + &component.kind, + now_ts() + ], + )?; + } + } + for dependent in dependent_components + .iter() + .filter(|component| !replaced_component_ids.contains(*component)) + { + let reason = format!( + "an upstream environment dependency changed; run `trail env sync-all {lane}`" + ); + self.conn.execute( + "UPDATE environment_component_states + SET status = 'stale', reason = ?1, updated_at = ?2 + WHERE view_id = ?3 AND component_id = ?4 AND attached_key IS NOT NULL", + params![&reason, now_ts(), &view.view_id, dependent], + )?; + self.conn.execute( + "UPDATE workspace_environment_states + SET status = 'stale', reason = ?1, updated_at = ?2 + WHERE view_id = ?3 AND adapter = ?4 AND attached_key IS NOT NULL", + params![&reason, now_ts(), &view.view_id, dependent], + )?; + } + // Runtime/order-only edges select an upstream instance for this + // generation but do not define the consumer artifact. Advance + // those exact keys without rebuilding or marking the consumer + // stale when an upstream component is replaced. + for (component_id, component_key) in &requested_keys { + self.conn.execute( + "UPDATE environment_component_dependencies + SET dependency_component_key = ?1, updated_at = ?2 + WHERE view_id = ?3 AND dependency_component_id = ?4 + AND edge_type IN ('runtime_requires', 'binds_after')", + params![component_key, now_ts(), &view.view_id, component_id], + )?; + } + self.record_environment_generation(lane, &view.view_id)?; + self.conn.execute( + "UPDATE workspace_views SET generation = generation + 1, updated_at = ?1 WHERE view_id = ?2", + params![now_ts(), &view.view_id], + )?; + Ok(()) + })(); + let activation = activation.and_then(|()| { + self.conn + .execute_batch("RELEASE SAVEPOINT trail_environment_activation") + .map_err(Error::from) + }); + match activation { + Ok(()) => { + test_crash_point("environment_after_generation_commit"); + for reset in resets { + reset.commit(); + } + } + Err(err) => { + let _ = self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT trail_environment_activation; RELEASE SAVEPOINT trail_environment_activation", + ); + let mut rollback_failure = None; + for reset in resets.into_iter().rev() { + if let Err(reset_err) = reset.rollback(&mut core) { + rollback_failure = Some(reset_err); + break; + } + } + if let Some(reset_err) = rollback_failure { + return Err(Error::Corrupt(format!( + "environment activation failed ({err}); restoring private generated state also failed ({reset_err}); durable recovery intents were retained" + ))); + } + return Err(err); + } + } + self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::Corrupt("workspace view disappeared after environment activation".to_string()) + }) + } + + fn bind_workspace_layer( + &self, + lane: &str, + layer_id: &str, + mount_path: &str, + component_id: &str, + expected_key: &str, + replace_private_upper: bool, + normalized_environment: Option<(&str, u32, &str, &str, &str)>, + ) -> Result { + let _lock = self.acquire_write_lock()?; + let layer = self.verify_workspace_layer_for_attach(layer_id)?; + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` does not have a layered workspace view" + )) + })?; + if let (Some(pid), Some(token)) = (view.owner_pid, view.owner_start_token.as_deref()) { + if process_matches_start_token(pid, token) { + return Err(Error::InvalidInput(format!( + "workspace view `{}` for lane `{lane}` has an active writer; run `trail lane unmount {lane}` before changing layer bindings", + view.view_id, + ))); + } + } + let mount_path = normalize_relative_path(mount_path)?; + if normalized_environment.is_some() { + self.validate_environment_mount_ownership(&view.view_id, component_id, &mount_path)?; + } + let mut prepared_reset = if replace_private_upper { + let mut core = ViewCore::new_lazy( + // The outer layer mutation owns the workspace writer lock and + // its Trail handle already completed open-time recovery. + Trail::open_without_recovering_derived_paths_under_write_lock( + self.workspace_root(), + self.db_dir(), + )?, + PathBuf::from(&view.source_upper), + view.base_root.clone(), + )?; + let reset = + core.prepare_declared_layer_mount_path(&mount_path, &layer.kind, &layer.layer_id)?; + test_crash_point("layer_after_upper_reset"); + Some((core, reset)) + } else { + None + }; + + self.conn + .execute_batch("SAVEPOINT trail_layer_activation")?; + let activation = (|| -> Result<()> { + if normalized_environment.is_some() { + let previous_mount = self + .conn + .query_row( + "SELECT mount_path FROM environment_component_bindings WHERE view_id = ?1 AND component_id = ?2", + params![&view.view_id, component_id], + |row| row.get::<_, String>(0), + ) + .optional()?; + if previous_mount + .as_deref() + .is_some_and(|path| path != mount_path) + { + self.conn.execute( + "DELETE FROM workspace_view_layers WHERE view_id = ?1 AND mount_path = ?2", + params![&view.view_id, previous_mount], + )?; + } + } + self.conn.execute( + "INSERT OR REPLACE INTO workspace_view_layers (view_id, layer_id, mount_path, priority, read_only) VALUES (?1, ?2, ?3, 100, 1)", + params![&view.view_id, &layer.layer_id, &mount_path], + )?; + self.conn.execute( + "UPDATE workspace_layers SET last_used_at = ?1 WHERE layer_id = ?2", + params![now_ts(), &layer.layer_id], + )?; + self.conn.execute( + "INSERT OR REPLACE INTO workspace_environment_states (view_id, adapter, expected_key, attached_key, status, reason, updated_at) VALUES (?1, ?2, ?3, ?4, 'ready', NULL, ?5)", + params![&view.view_id, component_id, expected_key, &layer.cache_key, now_ts()], + )?; + if let Some(( + adapter_identity, + adapter_version, + implementation_version, + distribution_digest, + kind, + )) = normalized_environment + { + self.conn.execute( + "INSERT OR REPLACE INTO environment_component_states (view_id, component_id, adapter_identity, adapter_version, implementation_version, distribution_digest, kind, expected_key, attached_key, status, reason, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 'ready', NULL, ?10)", + params![ + &view.view_id, + component_id, + adapter_identity, + adapter_version, + implementation_version, + distribution_digest, + kind, + expected_key, + &layer.cache_key, + now_ts() + ], + )?; + self.conn.execute( + "INSERT OR REPLACE INTO environment_component_bindings (view_id, component_id, mount_path, kind, updated_at) VALUES (?1, ?2, ?3, ?4, ?5)", + params![&view.view_id, component_id, &mount_path, kind, now_ts()], + )?; + self.record_environment_generation(lane, &view.view_id)?; + } + self.conn.execute( + "UPDATE workspace_views SET generation = generation + 1, updated_at = ?1 WHERE view_id = ?2", + params![now_ts(), &view.view_id], + )?; + Ok(()) + })(); + let activation = activation.and_then(|()| { + self.conn + .execute_batch("RELEASE SAVEPOINT trail_layer_activation") + .map_err(Error::from) + }); + match activation { + Ok(()) => { + test_crash_point("layer_after_binding_commit"); + if let Some((_core, reset)) = prepared_reset.take() { + reset.commit(); + } + } + Err(err) => { + let _ = self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT trail_layer_activation; RELEASE SAVEPOINT trail_layer_activation", + ); + if let Some((mut core, reset)) = prepared_reset.take() { + if let Err(rollback_err) = reset.rollback(&mut core) { + return Err(Error::Corrupt(format!( + "workspace layer activation failed ({err}); restoring private generated state also failed ({rollback_err}); durable recovery intent was retained" + ))); + } + } + return Err(err); + } + } + self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::Corrupt("workspace view disappeared after layer attach".to_string()) + }) + } + + fn environment_dependency_descendants( + &self, + view_id: &str, + roots: &BTreeSet, + ) -> Result> { + let mut stmt = self.conn.prepare( + "SELECT component_id, dependency_component_id + FROM environment_component_dependencies + WHERE view_id = ?1 + AND edge_type IN ('build_requires', 'invalidates_with') + ORDER BY dependency_component_id, component_id", + )?; + let edges = stmt + .query_map(params![view_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()?; + let mut adjacency = BTreeMap::>::new(); + for (component, dependency) in edges { + adjacency.entry(dependency).or_default().push(component); + } + let mut queue = roots.iter().cloned().collect::>(); + let mut descendants = BTreeSet::new(); + while let Some(component) = queue.pop_front() { + if let Some(children) = adjacency.get(&component) { + for child in children { + if descendants.insert(child.clone()) { + queue.push_back(child.clone()); + } + } + } + } + Ok(descendants) + } + + fn validate_environment_mount_ownership( + &self, + view_id: &str, + component_id: &str, + mount_path: &str, + ) -> Result<()> { + let mut stmt = self.conn.prepare( + "SELECT component_id, mount_path FROM environment_component_bindings WHERE view_id = ?1", + )?; + let bindings = stmt + .query_map(params![view_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()?; + for (owner, existing) in bindings { + if owner != component_id && mount_paths_overlap(mount_path, &existing) { + return Err(Error::InvalidInput(format!( + "environment component `{component_id}` mount `{mount_path}` overlaps `{existing}` owned by component `{owner}`" + ))); + } + } + let mut stmt = self + .conn + .prepare("SELECT mount_path FROM workspace_view_layers WHERE view_id = ?1")?; + let existing_mounts = stmt + .query_map(params![view_id], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + for existing in existing_mounts { + let owned_by_component = self.conn.query_row( + "SELECT EXISTS(SELECT 1 FROM environment_component_bindings WHERE view_id = ?1 AND component_id = ?2 AND mount_path = ?3)", + params![view_id, component_id, &existing], + |row| row.get::<_, bool>(0), + )?; + if !owned_by_component + && existing != mount_path + && mount_paths_overlap(mount_path, &existing) + { + return Err(Error::InvalidInput(format!( + "environment component `{component_id}` mount `{mount_path}` overlaps existing layer mount `{existing}`" + ))); + } + } + Ok(()) + } + + fn validate_environment_batch_mount_ownership( + &self, + view_id: &str, + replaced_components: &BTreeSet, + requested_mounts: &[(String, String)], + ) -> Result<()> { + let mut stmt = self.conn.prepare( + "SELECT component_id, mount_path FROM environment_component_output_bindings WHERE view_id = ?1", + )?; + let bindings = stmt + .query_map(params![view_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()?; + for (owner, existing) in &bindings { + if replaced_components.contains(owner) { + continue; + } + for (component, requested) in requested_mounts { + if mount_paths_overlap(requested, existing) { + return Err(Error::InvalidInput(format!( + "environment component `{component}` mount `{requested}` overlaps `{existing}` owned by component `{owner}`" + ))); + } + } + } + + let known_mounts = bindings + .iter() + .map(|(_, mount)| mount.as_str()) + .collect::>(); + let mut stmt = self + .conn + .prepare("SELECT mount_path FROM workspace_view_layers WHERE view_id = ?1")?; + let layer_mounts = stmt + .query_map(params![view_id], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + for existing in layer_mounts { + if known_mounts.contains(existing.as_str()) { + continue; + } + for (component, requested) in requested_mounts { + if existing != *requested && mount_paths_overlap(requested, &existing) { + return Err(Error::InvalidInput(format!( + "environment component `{component}` mount `{requested}` overlaps existing layer mount `{existing}`" + ))); + } + } + } + Ok(()) + } + + fn record_environment_generation(&self, lane: &str, view_id: &str) -> Result { + let branch = self.lane_branch(lane)?; + let head = self.get_ref(&branch.ref_name)?; + let mut stmt = self.conn.prepare( + "SELECT s.component_id, s.adapter_identity, s.kind, + COALESCE(s.attached_key, s.expected_key), l.layer_id, b.mount_path + FROM environment_component_states s + LEFT JOIN environment_component_bindings b + ON b.view_id = s.view_id AND b.component_id = s.component_id + LEFT JOIN workspace_view_layers l + ON l.view_id = b.view_id AND l.mount_path = b.mount_path + WHERE s.view_id = ?1 + AND s.attached_key IS NOT NULL + ORDER BY s.component_id", + )?; + let mut components = stmt + .query_map(params![view_id], |row| { + Ok(EnvironmentGenerationComponentReport { + component_id: row.get(0)?, + adapter_identity: row.get(1)?, + kind: row.get(2)?, + component_key: row.get(3)?, + layer_id: row.get(4)?, + mount_path: row.get(5)?, + dependencies: Vec::new(), + outputs: Vec::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + }) + })? + .collect::, _>>()?; + for component in &mut components { + let dependencies = { + let mut dependency_stmt = self.conn.prepare( + "SELECT dependency_component_id, dependency_component_key, edge_type + FROM environment_component_dependencies + WHERE view_id = ?1 AND component_id = ?2 + ORDER BY dependency_component_id", + )?; + let dependencies = dependency_stmt + .query_map(params![view_id, &component.component_id], |row| { + Ok(EnvironmentGenerationDependencyReport { + component_id: row.get(0)?, + component_key: row.get(1)?, + edge_type: row.get(2)?, + }) + })? + .collect::, _>>()?; + dependencies + }; + component.dependencies = dependencies; + let mut output_stmt = self.conn.prepare( + "SELECT b.output_name, b.policy, b.binding_identity, l.layer_id, + b.mount_path, b.layer_subpath + FROM environment_component_output_bindings b + LEFT JOIN workspace_view_layers l + ON l.view_id = b.view_id AND l.mount_path = b.mount_path + WHERE b.view_id = ?1 AND b.component_id = ?2 + ORDER BY b.output_name", + )?; + component.outputs = output_stmt + .query_map(params![view_id, &component.component_id], |row| { + Ok(EnvironmentGenerationOutputReport { + name: row.get(0)?, + policy: row.get(1)?, + storage_identity: row.get(2)?, + layer_id: row.get(3)?, + mount_path: row.get(4)?, + layer_subpath: row.get(5)?, + }) + })? + .collect::, _>>()?; + let mut cache_stmt = self.conn.prepare( + "SELECT cache_name, namespace_id, protocol, access, compatibility_json + FROM environment_component_caches + WHERE view_id = ?1 AND component_id = ?2 + ORDER BY cache_name", + )?; + component.caches = cache_stmt + .query_map(params![view_id, &component.component_id], |row| { + let compatibility = row.get::<_, Vec>(4)?; + let compatibility = + serde_json::from_slice(&compatibility).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 4, + rusqlite::types::Type::Blob, + Box::new(error), + ) + })?; + Ok(EnvironmentCacheReport { + name: row.get(0)?, + namespace_id: row.get(1)?, + protocol: row.get(2)?, + access: row.get(3)?, + authority: "performance_only".to_string(), + scope: "workspace".to_string(), + compatibility, + }) + })? + .collect::, _>>()?; + let mut external_stmt = self.conn.prepare( + "SELECT artifact_name, artifact_type, provider, reference, digest, platform, cleanup_owner + FROM environment_component_external_artifacts + WHERE view_id = ?1 AND component_id = ?2 + ORDER BY artifact_name", + )?; + component.external_artifacts = external_stmt + .query_map(params![view_id, &component.component_id], |row| { + Ok(EnvironmentExternalArtifactReport { + name: row.get(0)?, + artifact_type: row.get(1)?, + provider: row.get(2)?, + reference: row.get(3)?, + digest: row.get(4)?, + platform: row.get(5)?, + cleanup_owner: row.get(6)?, + }) + })? + .collect::, _>>()?; + } + let specification_digest = sha256_hex(&serde_json::to_vec(&components)?); + let sequence = self.conn.query_row( + "SELECT COALESCE(MAX(generation_sequence), 0) + 1 FROM environment_generations WHERE view_id = ?1", + params![view_id], + |row| row.get::<_, u64>(0), + )?; + let predecessor = self + .conn + .query_row( + "SELECT generation_id FROM environment_view_generations WHERE view_id = ?1", + params![view_id], + |row| row.get::<_, String>(0), + ) + .optional()?; + let generation_id = format!( + "envgen_{}", + &sha256_hex( + format!( + "{view_id}:{sequence}:{}:{specification_digest}", + head.root_id.0 + ) + .as_bytes() + )[..32] + ); + let now = now_ts(); + self.conn.execute( + "INSERT INTO environment_generations + (generation_id, view_id, generation_sequence, source_root, specification_digest, predecessor_generation_id, state, created_at, activated_at, retired_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'active', ?7, ?7, NULL)", + params![ + &generation_id, + view_id, + sequence, + head.root_id.0, + &specification_digest, + predecessor.as_deref(), + now + ], + )?; + for component in &components { + self.conn.execute( + "INSERT INTO environment_generation_components + (generation_id, component_id, adapter_identity, kind, component_key, layer_id, mount_path) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + &generation_id, + &component.component_id, + &component.adapter_identity, + &component.kind, + &component.component_key, + &component.layer_id, + &component.mount_path + ], + )?; + for output in &component.outputs { + self.conn.execute( + "INSERT INTO environment_generation_outputs + (generation_id, component_id, output_name, policy, storage_identity, layer_id, mount_path, layer_subpath) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![ + &generation_id, + &component.component_id, + &output.name, + &output.policy, + &output.storage_identity, + &output.layer_id, + &output.mount_path, + &output.layer_subpath + ], + )?; + } + for dependency in &component.dependencies { + self.conn.execute( + "INSERT INTO environment_generation_edges + (generation_id, component_id, dependency_component_id, dependency_component_key, edge_type) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![ + &generation_id, + &component.component_id, + &dependency.component_id, + &dependency.component_key, + &dependency.edge_type + ], + )?; + } + for cache in &component.caches { + self.conn.execute( + "INSERT INTO environment_generation_caches + (generation_id, component_id, cache_name, namespace_id, protocol, access, compatibility_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + &generation_id, + &component.component_id, + &cache.name, + &cache.namespace_id, + &cache.protocol, + &cache.access, + serde_json::to_vec(&cache.compatibility)? + ], + )?; + } + for artifact in &component.external_artifacts { + self.conn.execute( + "INSERT INTO environment_generation_external_artifacts + (generation_id, component_id, artifact_name, artifact_type, provider, reference, digest, platform, cleanup_owner) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![ + &generation_id, + &component.component_id, + &artifact.name, + &artifact.artifact_type, + &artifact.provider, + &artifact.reference, + &artifact.digest, + &artifact.platform, + &artifact.cleanup_owner + ], + )?; + } + let mut runtime_stmt = self.conn.prepare( + "SELECT r.resource_name, r.runtime_type, r.provider, r.artifact_name, + a.reference, a.digest, a.platform, r.container_port, r.protocol, + r.health_type, r.health_timeout_ms, r.restart_policy, + r.cleanup_owner, r.volume_target + FROM environment_component_runtime_resources r + JOIN environment_component_external_artifacts a + ON a.view_id = r.view_id AND a.component_id = r.component_id + AND a.artifact_name = r.artifact_name + WHERE r.view_id = ?1 AND r.component_id = ?2 + ORDER BY r.resource_name", + )?; + let runtime_resources = runtime_stmt + .query_map(params![view_id, &component.component_id], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, String>(6)?, + row.get::<_, u16>(7)?, + row.get::<_, String>(8)?, + row.get::<_, String>(9)?, + row.get::<_, u64>(10)?, + row.get::<_, String>(11)?, + row.get::<_, String>(12)?, + row.get::<_, Option>(13)?, + )) + })? + .collect::, _>>()?; + for ( + resource_name, + runtime_type, + provider, + artifact_name, + image_reference, + image_digest, + image_platform, + container_port, + protocol, + health_type, + health_timeout_ms, + restart_policy, + cleanup_owner, + volume_target, + ) in runtime_resources + { + let names = environment_runtime_allocation_names( + &self.config.workspace.id.0, + view_id, + &generation_id, + &component.component_id, + &resource_name, + volume_target.is_some(), + ); + self.conn.execute( + "INSERT INTO environment_generation_runtime_resources + (generation_id, component_id, resource_name, runtime_type, provider, + artifact_name, image_reference, image_digest, image_platform, + container_port, protocol, health_type, health_timeout_ms, restart_policy, + cleanup_owner, volume_target, allocation_id, provider_resource_id, + container_name, network_name, volume_name, host_port, status, + health_status, reason, cleanup_token, owner_pid, owner_start_token, + created_at, updated_at, started_at, stopped_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, + ?14, ?15, ?16, ?17, NULL, ?18, ?19, ?20, NULL, 'pending', + 'pending', NULL, ?21, NULL, NULL, ?22, ?22, NULL, NULL)", + params![ + &generation_id, + &component.component_id, + &resource_name, + &runtime_type, + &provider, + &artifact_name, + &image_reference, + &image_digest, + &image_platform, + container_port, + &protocol, + &health_type, + health_timeout_ms, + &restart_policy, + &cleanup_owner, + &volume_target, + &names.allocation_id, + &names.container_name, + &names.network_name, + &names.volume_name, + &names.cleanup_token, + now + ], + )?; + let mut secret_stmt = self.conn.prepare( + "SELECT secret_name, provider, reference, version, purpose, injection, + target, environment, required + FROM environment_component_runtime_secrets + WHERE view_id = ?1 AND component_id = ?2 AND resource_name = ?3 + ORDER BY secret_name", + )?; + let secrets = secret_stmt + .query_map( + params![view_id, &component.component_id, &resource_name], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, String>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, bool>(8)?, + )) + }, + )? + .collect::, _>>()?; + for ( + secret_name, + secret_provider, + reference, + version, + purpose, + injection, + target, + environment, + required, + ) in secrets + { + self.conn.execute( + "INSERT INTO environment_generation_runtime_secrets + (generation_id, component_id, resource_name, secret_name, provider, + reference, version, purpose, injection, target, environment, required, + status, reason, resolved_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, + 'pending', NULL, NULL, ?13)", + params![ + &generation_id, + &component.component_id, + &resource_name, + &secret_name, + &secret_provider, + &reference, + &version, + &purpose, + &injection, + &target, + &environment, + required, + now + ], + )?; + } + } + } + if let Some(predecessor) = predecessor { + self.conn.execute( + "UPDATE environment_generations SET state = 'retired', retired_at = ?1 WHERE generation_id = ?2 AND state = 'active'", + params![now, predecessor], + )?; + } + self.conn.execute( + "INSERT OR REPLACE INTO environment_view_generations (view_id, generation_id, updated_at) VALUES (?1, ?2, ?3)", + params![view_id, &generation_id, now], + )?; + Ok(generation_id) + } + + pub(crate) fn workspace_layer_bindings_for_source_upper( + &self, + source_upper: &Path, + ) -> Result> { + let mut stmt = self.conn.prepare( + "SELECT l.layer_id, b.mount_path, l.storage_path, b.source_path, l.kind, b.priority \ + FROM workspace_views v JOIN workspace_view_layers b ON b.view_id = v.view_id \ + JOIN workspace_layers l ON l.layer_id = b.layer_id \ + WHERE v.source_upper = ?1 AND b.read_only = 1 AND l.state = 'ready' \ + ORDER BY length(b.mount_path) DESC, b.priority DESC", + )?; + let rows = stmt + .query_map(params![source_upper.to_string_lossy()], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, i64>(5)?, + )) + }) + .map_err(Error::from)? + .collect::, _>>()?; + let mut bindings = rows + .into_iter() + .map( + |(layer_id, mount_path, storage_path, source_path, kind, priority)| { + let storage_path = if source_path.is_empty() { + PathBuf::from(storage_path) + } else { + safe_join(Path::new(&storage_path), &source_path)? + }; + Ok(WorkspaceLayerBinding { + binding_identity: layer_id.clone(), + layer_id: Some(layer_id), + mount_path, + storage_path: Some(storage_path), + kind, + priority, + }) + }, + ) + .collect::>>()?; + let mut stmt = self.conn.prepare( + "SELECT o.binding_identity, o.mount_path, o.kind + FROM workspace_views v + JOIN environment_component_output_bindings o ON o.view_id = v.view_id + WHERE v.source_upper = ?1 AND o.policy = 'writable_private' + ORDER BY length(o.mount_path) DESC, o.mount_path", + )?; + let private = stmt + .query_map(params![source_upper.to_string_lossy()], |row| { + Ok(WorkspaceLayerBinding { + binding_identity: row.get(0)?, + layer_id: None, + mount_path: row.get(1)?, + storage_path: None, + kind: row.get(2)?, + priority: 100, + }) + })? + .collect::, _>>()?; + bindings.extend(private); + bindings.sort_by(|left, right| { + right + .mount_path + .len() + .cmp(&left.mount_path.len()) + .then_with(|| right.priority.cmp(&left.priority)) + .then_with(|| left.mount_path.cmp(&right.mount_path)) + }); + Ok(bindings) + } +} + +struct EnvironmentRuntimeAllocationNames { + allocation_id: String, + container_name: String, + network_name: String, + volume_name: Option, + cleanup_token: String, +} + +fn environment_runtime_allocation_names( + workspace_id: &str, + view_id: &str, + generation_id: &str, + component_id: &str, + resource_name: &str, + has_volume: bool, +) -> EnvironmentRuntimeAllocationNames { + let identity = sha256_hex( + format!("{workspace_id}\0{view_id}\0{generation_id}\0{component_id}\0{resource_name}") + .as_bytes(), + ); + let network_identity = + sha256_hex(format!("{workspace_id}\0{view_id}\0{generation_id}").as_bytes()); + let volume_identity = sha256_hex( + format!("{workspace_id}\0{view_id}\0{component_id}\0{resource_name}\0volume").as_bytes(), + ); + let mut slug = resource_name + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() { + character.to_ascii_lowercase() + } else { + '-' + } + }) + .collect::(); + slug.truncate(20); + let slug = slug.trim_matches('-'); + let slug = if slug.is_empty() { "service" } else { slug }; + EnvironmentRuntimeAllocationNames { + allocation_id: format!("runtime_{}", &identity[..32]), + container_name: format!("trail-{slug}-{}", &identity[..16]), + network_name: format!("trail-net-{}", &network_identity[..20]), + // A private data volume belongs to the logical lane service rather + // than one immutable generation. New container/image generations can + // therefore roll forward without silently discarding database state. + volume_name: has_volume.then(|| format!("trail-vol-{}", &volume_identity[..20])), + cleanup_token: format!("cleanup_{}", &identity[32..]), + } +} + +struct CacheGcCandidate { + entry: WorkspaceCacheGcEntry, + last_used_at: i64, + retention_expired: bool, +} + +fn mount_paths_overlap(left: &str, right: &str) -> bool { + left == right + || left + .strip_prefix(right) + .is_some_and(|rest| rest.starts_with('/')) + || right + .strip_prefix(left) + .is_some_and(|rest| rest.starts_with('/')) +} + +fn cache_tree_usage(path: &Path) -> Result { + if !path.exists() { + return Ok(0); + } + let mut bytes = 0_u64; + for entry in walkdir::WalkDir::new(path).follow_links(false) { + let entry = entry.map_err(|err| Error::InvalidInput(err.to_string()))?; + if entry.file_type().is_file() { + let metadata = entry.metadata().map_err(|err| Error::Io(err.into()))?; + bytes = bytes.saturating_add(cache_file_physical_bytes(&metadata)); + } + } + Ok(bytes) +} + +fn cache_tree_logical_bytes(path: &Path) -> Result { + if !path.exists() { + return Ok(0); + } + let mut bytes = 0_u64; + for entry in walkdir::WalkDir::new(path).follow_links(false) { + let entry = entry.map_err(|err| Error::InvalidInput(err.to_string()))?; + if entry.file_type().is_file() { + let metadata = entry.metadata().map_err(|err| Error::Io(err.into()))?; + bytes = bytes.saturating_add(metadata.len()); + } + } Ok(bytes) } @@ -865,15 +2994,64 @@ fn build_lock_is_stale(path: &Path) -> Result { Err(err) => return Err(Error::Io(err)), }; let Some((pid, token)) = value.split_once(':') else { - return Ok(true); + // create_new makes ownership exclusive before the builder can write + // and sync its token. A contender may observe that short empty-file + // window; do not steal a freshly created lock as malformed. + return malformed_build_lock_is_stale(path); }; let Ok(pid) = pid.parse::() else { - return Ok(true); + return malformed_build_lock_is_stale(path); }; Ok(!process_matches_start_token(pid, token)) } -fn make_tree_writable(path: &Path) { +fn acquire_environment_cache_maintenance( + db_dir: &Path, + namespace_id: &str, +) -> Result> { + let lock_dir = db_dir.join("cache/namespace-maintenance"); + fs::create_dir_all(&lock_dir)?; + let path = lock_dir.join(format!("{namespace_id}.lock")); + let token = format!("{}:{}", std::process::id(), current_process_start_token()); + loop { + match OpenOptions::new().write(true).create_new(true).open(&path) { + Ok(mut file) => { + file.write_all(token.as_bytes())?; + file.sync_all()?; + return Ok(Some(CacheBuildKeyGuard { path, token })); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + if !build_lock_is_stale(&path)? { + return Ok(None); + } + let stale = path.with_extension(format!( + "stale.{}", + crate::ids::short_hash( + format!("{}:{}", namespace_id, now_nanos()).as_bytes(), + 16 + ) + )); + match fs::rename(&path, stale) { + Ok(()) => continue, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(Error::Io(error)), + } + } + Err(error) => return Err(Error::Io(error)), + } + } +} + +fn malformed_build_lock_is_stale(path: &Path) -> Result { + const MALFORMED_LOCK_GRACE: Duration = Duration::from_secs(5); + let modified = fs::metadata(path)?.modified()?; + Ok(SystemTime::now() + .duration_since(modified) + .unwrap_or_default() + >= MALFORMED_LOCK_GRACE) +} + +pub(crate) fn make_tree_writable(path: &Path) { if !path.exists() { return; } @@ -937,6 +3115,148 @@ fn workspace_layer_marker_path(layer_path: &Path) -> PathBuf { layer_path.with_file_name(format!(".{name}.publish.json")) } +fn workspace_layer_verification_stamp_path(layer_path: &Path) -> PathBuf { + let name = layer_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("layer"); + layer_path.with_file_name(format!(".{name}.verified.json")) +} + +fn read_workspace_layer_sidecar(path: &Path) -> Result { + let metadata = fs::symlink_metadata(path)?; + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(Error::Corrupt(format!( + "workspace layer sidecar `{}` is not a regular file", + path.display() + ))); + } + if metadata.len() > WORKSPACE_LAYER_SIDECAR_MAX_BYTES { + return Err(Error::Corrupt(format!( + "workspace layer sidecar `{}` exceeds {WORKSPACE_LAYER_SIDECAR_MAX_BYTES} bytes", + path.display() + ))); + } + Ok(serde_json::from_slice(&fs::read(path)?)?) +} + +fn write_workspace_layer_verification_stamp( + report: &WorkspaceLayerReport, + manifest_object_id: &str, +) -> Result<()> { + let storage_path = Path::new(&report.storage_path); + let stamp = WorkspaceLayerVerificationStamp { + version: WORKSPACE_LAYER_VERIFICATION_STAMP_VERSION, + layer_id: report.layer_id.clone(), + manifest_object_id: manifest_object_id.to_string(), + root_identity: workspace_layer_root_identity(storage_path)?, + logical_bytes: report.logical_bytes, + entry_count: report.entry_count, + verified_at: now_ts(), + }; + write_file_atomic( + &workspace_layer_verification_stamp_path(storage_path), + &serde_json::to_vec_pretty(&stamp)?, + true, + ) +} + +fn write_workspace_layer_publish_marker_from_report( + report: &WorkspaceLayerReport, + manifest_object_id: &str, +) -> Result<()> { + let storage_path = Path::new(&report.storage_path); + let marker = WorkspaceLayerPublishMarker { + layer_id: report.layer_id.clone(), + cache_key: report.cache_key.clone(), + manifest_object_id: manifest_object_id.to_string(), + logical_bytes: report.logical_bytes, + physical_bytes: match report.physical_bytes { + Some(bytes) => bytes, + None => layer_physical_bytes(storage_path)?, + }, + entry_count: report.entry_count, + }; + write_file_atomic( + &workspace_layer_marker_path(storage_path), + &serde_json::to_vec_pretty(&marker)?, + true, + ) +} + +#[cfg(unix)] +fn workspace_layer_root_identity(path: &Path) -> Result { + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(Error::Corrupt(format!( + "workspace layer directory `{}` is missing or not a real directory", + path.display() + ))); + } + Ok(WorkspaceLayerRootIdentity { + platform: std::env::consts::OS.to_string(), + values: vec![ + metadata.dev(), + metadata.ino(), + metadata.mode() as u64, + metadata.nlink(), + metadata.uid() as u64, + metadata.gid() as u64, + metadata.mtime() as u64, + metadata.mtime_nsec() as u64, + metadata.ctime() as u64, + metadata.ctime_nsec() as u64, + ], + }) +} + +#[cfg(windows)] +fn workspace_layer_root_identity(path: &Path) -> Result { + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(Error::Corrupt(format!( + "workspace layer directory `{}` is missing or not a real directory", + path.display() + ))); + } + let identity = windows_file_identity(path)?; + Ok(WorkspaceLayerRootIdentity { + platform: std::env::consts::OS.to_string(), + values: vec![ + identity.attributes as u64, + identity.volume_serial_number as u64, + identity.file_index, + identity.number_of_links as u64, + identity.creation_time, + identity.last_write_time, + ], + }) +} + +#[cfg(not(any(unix, windows)))] +fn workspace_layer_root_identity(path: &Path) -> Result { + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(Error::Corrupt(format!( + "workspace layer directory `{}` is missing or not a real directory", + path.display() + ))); + } + let modified = metadata + .modified()? + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default(); + Ok(WorkspaceLayerRootIdentity { + platform: std::env::consts::OS.to_string(), + values: vec![ + metadata.len(), + modified.as_secs(), + modified.subsec_nanos() as u64, + metadata.permissions().readonly() as u64, + ], + }) +} + fn validate_layer_key(key: &WorkspaceLayerKeyV1) -> Result<()> { if key.kind.trim().is_empty() || key.adapter.trim().is_empty() @@ -1061,6 +3381,8 @@ fn scan_layer_entries( root: &Path, make_read_only: bool, ) -> Result> { + #[cfg(test)] + WORKSPACE_LAYER_FULL_SCAN_COUNT.with(|count| count.set(count.get().saturating_add(1))); if !root.is_dir() { return Err(Error::Corrupt(format!( "workspace layer directory `{}` is missing", @@ -1297,39 +3619,386 @@ fn layer_physical_bytes(root: &Path) -> Result { bytes = bytes.saturating_add(layer_file_physical_bytes(&metadata)); } } - Ok(bytes) -} - -#[cfg(unix)] -fn layer_file_physical_bytes(metadata: &fs::Metadata) -> u64 { - use std::os::unix::fs::MetadataExt; - metadata.blocks().saturating_mul(512) -} + Ok(bytes) +} + +#[cfg(unix)] +fn layer_file_physical_bytes(metadata: &fs::Metadata) -> u64 { + use std::os::unix::fs::MetadataExt; + metadata.blocks().saturating_mul(512) +} + +#[cfg(not(unix))] +fn layer_file_physical_bytes(metadata: &fs::Metadata) -> u64 { + metadata.len() +} + +#[cfg(test)] +mod tests { + use super::super::workdir::{ViewCore, VIEW_ROOT_INO}; + use super::*; + use std::process::Stdio; + use std::sync::atomic::{AtomicUsize, Ordering}; + + fn key() -> WorkspaceLayerKeyV1 { + WorkspaceLayerKeyV1 { + kind: "dependency".to_string(), + adapter: "node".to_string(), + adapter_version: 1, + inputs: BTreeMap::from([("package-lock.json".to_string(), "abc".to_string())]), + tool_versions: BTreeMap::from([("node".to_string(), "22.0.0".to_string())]), + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: "platform".to_string(), + strategy: "npm-ci-ignore-scripts".to_string(), + } + } + + #[test] + fn legacy_layer_manifests_without_canonical_keys_remain_readable() { + let manifest = WorkspaceLayerManifest { + version: WORKSPACE_LAYER_MANIFEST_VERSION, + layer_id: "layer_legacy".to_string(), + kind: "dependency".to_string(), + cache_key: "legacy-key".to_string(), + layer_key: Some(key()), + adapter: "node".to_string(), + adapter_version: 1, + logical_bytes: 0, + entries: BTreeMap::new(), + platform: "legacy".to_string(), + architecture: "legacy".to_string(), + portability_scope: "legacy".to_string(), + producer_version: "legacy".to_string(), + created_at: 1, + }; + let mut value = serde_json::to_value(manifest).unwrap(); + value.as_object_mut().unwrap().remove("layer_key"); + let decoded: WorkspaceLayerManifest = serde_json::from_value(value).unwrap(); + assert!(decoded.layer_key.is_none()); + } + + #[test] + fn dependency_activation_is_atomic_marks_descendants_stale_and_preserves_edge_history() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "dependencies", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + + let make_activation = |component: &str, + revision: &str, + dependencies: Vec<(String, String)>, + mount_path: &str, + seed: &Path| + -> EnvironmentLayerActivation { + let mut inputs = BTreeMap::from([("revision".to_string(), revision.to_string())]); + for (dependency, component_key) in &dependencies { + inputs.insert(format!("dependency:{dependency}"), component_key.clone()); + } + let canonical_key = WorkspaceLayerKeyV1 { + kind: "generated".to_string(), + adapter: "test".to_string(), + adapter_version: 1, + inputs, + tool_versions: BTreeMap::new(), + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: "host".to_string(), + strategy: "private-dependency-test".to_string(), + }; + let expected_key = db.workspace_layer_cache_key(&canonical_key).unwrap(); + EnvironmentLayerActivation { + layer_id: None, + outputs: vec![EnvironmentLayerOutputActivation { + name: "output".to_string(), + mount_path: mount_path.to_string(), + policy: "writable_private".to_string(), + binding_identity: format!("private-{component}-{revision}"), + private_seed: Some(seed.to_path_buf()), + layer_subpath: String::new(), + }], + component_id: component.to_string(), + adapter_identity: "trail/test@1".to_string(), + adapter_version: 1, + implementation_version: "test".to_string(), + distribution_digest: "builtin:test".to_string(), + kind: "generated".to_string(), + dependencies: dependencies + .into_iter() + .map(|(component_id, component_key)| { + (component_id, component_key, "build_requires".to_string()) + }) + .collect(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + expected_key, + canonical_key, + } + }; + + let seed_a1 = tempfile::tempdir().unwrap(); + let seed_b1 = tempfile::tempdir().unwrap(); + fs::write(seed_a1.path().join("value"), "a1\n").unwrap(); + fs::write(seed_b1.path().join("value"), "b1\n").unwrap(); + let a1 = make_activation("a", "1", Vec::new(), ".generated/a", seed_a1.path()); + let b1 = make_activation( + "b", + "1", + vec![("a".to_string(), a1.expected_key.clone())], + ".generated/b", + seed_b1.path(), + ); + let wrong_source = ObjectId("object_not_the_lane_head".to_string()); + let error = db + .replace_declared_workspace_layers_at_source( + "dependencies", + &[a1.clone(), b1.clone()], + &wrong_source, + ) + .unwrap_err(); + assert!(error + .to_string() + .contains("advanced from pinned source root")); + assert!(db + .active_environment_generation("dependencies") + .unwrap() + .is_none()); + db.replace_declared_workspace_layers("dependencies", &[a1.clone(), b1.clone()]) + .unwrap(); + let predecessor = db + .active_environment_generation("dependencies") + .unwrap() + .unwrap(); + assert_eq!( + predecessor.components[1].dependencies[0].component_key, + a1.expected_key + ); + + let mut invalid_b = b1.clone(); + invalid_b.dependencies[0].1 = "wrong-key".to_string(); + assert!(db + .replace_declared_workspace_layers("dependencies", &[invalid_b]) + .is_err()); + assert_eq!( + db.active_environment_generation("dependencies") + .unwrap() + .unwrap() + .generation_id, + predecessor.generation_id + ); + + let seed_a2 = tempfile::tempdir().unwrap(); + fs::write(seed_a2.path().join("value"), "a2\n").unwrap(); + let a2 = make_activation("a", "2", Vec::new(), ".generated/a", seed_a2.path()); + db.replace_declared_workspace_layers("dependencies", &[a2.clone()]) + .unwrap(); + let current = db + .active_environment_generation("dependencies") + .unwrap() + .unwrap(); + let current_b = current + .components + .iter() + .find(|component| component.component_id == "b") + .unwrap(); + assert_eq!(current_b.dependencies[0].component_key, a1.expected_key); + assert_ne!(current_b.dependencies[0].component_key, a2.expected_key); + assert_eq!( + db.environment_component_status("dependencies") + .unwrap() + .into_iter() + .find(|state| state.component.component_id == "b") + .unwrap() + .status, + "stale" + ); + + db.replace_declared_workspace_layers_with_removals("dependencies", &[], &["b".to_string()]) + .unwrap(); + let retired = db + .active_environment_generation("dependencies") + .unwrap() + .unwrap(); + assert_eq!( + retired + .components + .iter() + .map(|component| component.component_id.as_str()) + .collect::>(), + ["a"] + ); + assert!(db + .environment_component_status("dependencies") + .unwrap() + .into_iter() + .all(|state| state.component.component_id != "b")); + db.replace_declared_workspace_layers_with_removals("dependencies", &[], &["a".to_string()]) + .unwrap(); + assert!(db + .active_environment_generation("dependencies") + .unwrap() + .unwrap() + .components + .is_empty()); + } + + #[test] + fn runtime_dependency_replacement_advances_generation_without_rebuilding_consumer() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "runtime-edges", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let make_activation = |component: &str, + revision: &str, + dependencies: Vec<(String, String, String)>, + mount_path: &str, + seed: &Path| { + let canonical_key = WorkspaceLayerKeyV1 { + kind: "generated".to_string(), + adapter: "test".to_string(), + adapter_version: 1, + inputs: BTreeMap::from([("revision".to_string(), revision.to_string())]), + tool_versions: BTreeMap::new(), + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: "host".to_string(), + strategy: "runtime-edge-test".to_string(), + }; + let expected_key = db.workspace_layer_cache_key(&canonical_key).unwrap(); + EnvironmentLayerActivation { + layer_id: None, + outputs: vec![EnvironmentLayerOutputActivation { + name: "output".to_string(), + mount_path: mount_path.to_string(), + policy: "writable_private".to_string(), + binding_identity: format!("private-{component}-{revision}"), + private_seed: Some(seed.to_path_buf()), + layer_subpath: String::new(), + }], + component_id: component.to_string(), + adapter_identity: "trail/test@1".to_string(), + adapter_version: 1, + implementation_version: "test".to_string(), + distribution_digest: "builtin:test".to_string(), + kind: "generated".to_string(), + dependencies, + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + expected_key, + canonical_key, + } + }; + + let provider_seed_1 = tempfile::tempdir().unwrap(); + let provider_seed_2 = tempfile::tempdir().unwrap(); + let consumer_seed = tempfile::tempdir().unwrap(); + fs::write(provider_seed_1.path().join("value"), "provider-1\n").unwrap(); + fs::write(provider_seed_2.path().join("value"), "provider-2\n").unwrap(); + fs::write(consumer_seed.path().join("value"), "consumer\n").unwrap(); + let provider_1 = make_activation( + "provider", + "1", + Vec::new(), + ".generated/provider", + provider_seed_1.path(), + ); + let consumer = make_activation( + "consumer", + "1", + vec![( + "provider".to_string(), + provider_1.expected_key.clone(), + "runtime_requires".to_string(), + )], + ".generated/consumer", + consumer_seed.path(), + ); + db.replace_declared_workspace_layers( + "runtime-edges", + &[provider_1.clone(), consumer.clone()], + ) + .unwrap(); + + let provider_2 = make_activation( + "provider", + "2", + Vec::new(), + ".generated/provider", + provider_seed_2.path(), + ); + db.replace_declared_workspace_layers("runtime-edges", &[provider_2.clone()]) + .unwrap(); + let generation = db + .active_environment_generation("runtime-edges") + .unwrap() + .unwrap(); + let current_consumer = generation + .components + .iter() + .find(|component| component.component_id == "consumer") + .unwrap(); + assert_eq!(current_consumer.component_key, consumer.expected_key); + assert_eq!( + current_consumer.dependencies[0].edge_type, + "runtime_requires" + ); + assert_eq!( + current_consumer.dependencies[0].component_key, + provider_2.expected_key + ); + assert_eq!( + db.environment_component_status("runtime-edges") + .unwrap() + .into_iter() + .find(|state| state.component.component_id == "consumer") + .unwrap() + .status, + "ready" + ); + } -#[cfg(not(unix))] -fn layer_file_physical_bytes(metadata: &fs::Metadata) -> u64 { - metadata.len() -} - -#[cfg(test)] -mod tests { - use super::super::workdir::{ViewCore, VIEW_ROOT_INO}; - use super::*; - use std::process::Stdio; - use std::sync::atomic::{AtomicUsize, Ordering}; + fn reset_full_scan_count() { + WORKSPACE_LAYER_FULL_SCAN_COUNT.with(|count| count.set(0)); + } - fn key() -> WorkspaceLayerKeyV1 { - WorkspaceLayerKeyV1 { - kind: "dependency".to_string(), - adapter: "node".to_string(), - adapter_version: 1, - inputs: BTreeMap::from([("package-lock.json".to_string(), "abc".to_string())]), - tool_versions: BTreeMap::from([("node".to_string(), "22.0.0".to_string())]), - platform: std::env::consts::OS.to_string(), - architecture: std::env::consts::ARCH.to_string(), - portability_scope: "platform".to_string(), - strategy: "npm-ci-ignore-scripts".to_string(), - } + fn full_scan_count() -> u64 { + WORKSPACE_LAYER_FULL_SCAN_COUNT.with(Cell::get) } #[test] @@ -1360,8 +4029,10 @@ mod tests { let mut db = Trail::open(workspace.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "crash-source", @@ -1462,6 +4133,92 @@ mod tests { db.verify_workspace_layer(&first.layer_id).unwrap(); } + #[test] + fn warm_attach_uses_durable_verification_stamp_and_legacy_fallback_scans_once() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let built = tempfile::tempdir().unwrap(); + fs::create_dir_all(built.path().join("pkg")).unwrap(); + for index in 0..128 { + fs::write( + built.path().join(format!("pkg/{index:03}.js")), + format!("module.exports = {index};\n"), + ) + .unwrap(); + } + let layer = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + let storage = Path::new(&layer.storage_path); + let stamp = workspace_layer_verification_stamp_path(storage); + assert!(stamp.is_file()); + + reset_full_scan_count(); + let reused = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + assert_eq!(reused.layer_id, layer.layer_id); + db.verify_workspace_layer_for_attach(&layer.layer_id) + .unwrap(); + db.verify_workspace_layer_for_attach(&layer.layer_id) + .unwrap(); + assert_eq!(full_scan_count(), 0); + + fs::remove_file(&stamp).unwrap(); + reset_full_scan_count(); + db.verify_workspace_layer_for_attach(&layer.layer_id) + .unwrap(); + assert_eq!(full_scan_count(), 1); + assert!(stamp.is_file()); + + reset_full_scan_count(); + db.verify_workspace_layer_for_attach(&layer.layer_id) + .unwrap(); + assert_eq!(full_scan_count(), 0); + + fs::write( + &stamp, + vec![b'x'; WORKSPACE_LAYER_SIDECAR_MAX_BYTES as usize + 1], + ) + .unwrap(); + reset_full_scan_count(); + db.verify_workspace_layer_for_attach(&layer.layer_id) + .unwrap(); + assert_eq!(full_scan_count(), 1); + assert!(fs::metadata(&stamp).unwrap().len() < WORKSPACE_LAYER_SIDECAR_MAX_BYTES); + } + + #[test] + fn attach_stamp_invalidates_when_layer_root_identity_changes() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let built = tempfile::tempdir().unwrap(); + fs::write(built.path().join("artifact"), "immutable\n").unwrap(); + let layer = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + let storage = Path::new(&layer.storage_path); + make_layer_root_writable(storage).unwrap(); + fs::write(storage.join("injected"), "corrupt\n").unwrap(); + set_layer_read_only( + storage, + true, + layer_mode(&fs::symlink_metadata(storage).unwrap()), + ) + .unwrap(); + + reset_full_scan_count(); + let error = db + .verify_workspace_layer_for_attach(&layer.layer_id) + .unwrap_err(); + assert!(error.to_string().contains("immutable manifest")); + assert_eq!(full_scan_count(), 1); + } + #[test] fn layer_publish_rejects_escaping_symlink_and_secret_key_names() { let workspace = tempfile::tempdir().unwrap(); @@ -1492,8 +4249,10 @@ mod tests { let mut db = Trail::open(workspace.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; for lane in ["node-a", "node-b"] { db.spawn_lane_with_workdir_mode_paths_and_neighbors( @@ -1587,6 +4346,323 @@ mod tests { fs::read(Path::new(&layer.storage_path).join("pkg/index.js")).unwrap(), b"original\n" ); + + let private = views[0] + .1 + .create(package_a, "private.js", 0o644, true) + .unwrap(); + views[0].1.write(private.ino, 0, b"private\n").unwrap(); + assert!(views[0] + .0 + .generated_upper + .join("node_modules/pkg/private.js") + .is_file()); + drop(views); + + db.replace_workspace_layer( + "node-a", + &layer.layer_id, + "node_modules", + "node", + &layer.cache_key, + ) + .unwrap(); + let branch = db.lane_branch("node-a").unwrap(); + let head = db.get_ref(&branch.ref_name).unwrap(); + let paths = db.workspace_view_paths_for_lane("node-a").unwrap(); + let mut replaced = ViewCore::new_lazy( + Trail::open(workspace.path()).unwrap(), + paths.source_upper, + head.root_id, + ) + .unwrap(); + let modules = replaced.lookup(VIEW_ROOT_INO, "node_modules").unwrap(); + let package = replaced.lookup(modules, "pkg").unwrap(); + let restored = replaced.lookup(package, "index.js").unwrap(); + assert_eq!(replaced.read(restored, 0, 64).unwrap().0, b"original\n"); + assert!(replaced.lookup(package, "private.js").is_err()); + assert!(!paths.generated_upper.join("node_modules").exists()); + assert!(replaced.checkpoint_candidates().unwrap().paths.is_empty()); + } + + #[test] + fn retained_path_index_mirror_intent_does_not_self_lock_layer_replacement() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "retained-repair", + Some("main"), + mode, + None, + None, + None, + &[], + false, + ) + .unwrap(); + + let built = tempfile::tempdir().unwrap(); + fs::create_dir_all(built.path().join("pkg")).unwrap(); + fs::write(built.path().join("pkg/index.js"), "immutable\n").unwrap(); + let layer = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + db.attach_workspace_layer( + "retained-repair", + &layer.layer_id, + "node_modules", + "node", + &layer.cache_key, + ) + .unwrap(); + + let branch = db.lane_branch("retained-repair").unwrap(); + let head = db.get_ref(&branch.ref_name).unwrap(); + let manifest = db + .workspace_view_paths_for_lane("retained-repair") + .unwrap() + .meta_dir + .join("workdir-manifest.json"); + if manifest.exists() { + fs::remove_file(&manifest).unwrap(); + } + fs::create_dir(&manifest).unwrap(); + db.conn + .execute( + "INSERT INTO pending_path_index_derived_repairs \ + (ref_name, repair_kind, old_root, new_root, new_change, created_at) \ + VALUES (?1, 'lane_manifest', ?2, ?2, ?3, ?4)", + params![branch.ref_name, head.root_id.0, head.change_id.0, now_ts()], + ) + .unwrap(); + + db.replace_workspace_layer( + "retained-repair", + &layer.layer_id, + "node_modules", + "node", + &layer.cache_key, + ) + .unwrap(); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM pending_path_index_derived_repairs", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1 + ); + + fs::remove_dir(&manifest).unwrap(); + db.rebuild_indexes().unwrap(); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM pending_path_index_derived_repairs", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + } + + #[test] + fn declared_dependency_binding_classifies_nonstandard_mount_as_generated_upper() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "python-a", + Some("main"), + mode, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let built = tempfile::tempdir().unwrap(); + fs::create_dir_all(built.path().join("lib/python/site-packages/pkg")).unwrap(); + fs::write( + built + .path() + .join("lib/python/site-packages/pkg/__init__.py"), + "VALUE = 1\n", + ) + .unwrap(); + let layer = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + let paths = db.workspace_view_paths_for_lane("python-a").unwrap(); + fs::create_dir_all(paths.source_upper.join(".venv")).unwrap(); + fs::write(paths.source_upper.join(".venv/preexisting.txt"), "source\n").unwrap(); + let overlap = db + .replace_declared_workspace_layer( + "python-a", + &layer.layer_id, + ".venv", + "python-env", + "trail/python-test@1", + 1, + "test", + "builtin:python-test", + "dependency", + &layer.cache_key, + ) + .unwrap_err(); + assert!(overlap.to_string().contains("pre-existing source-upper")); + assert!(paths.source_upper.join(".venv/preexisting.txt").is_file()); + fs::remove_dir_all(paths.source_upper.join(".venv")).unwrap(); + db.attach_workspace_layer( + "python-a", + &layer.layer_id, + ".venv", + "python-env", + &layer.cache_key, + ) + .unwrap(); + + let branch = db.lane_branch("python-a").unwrap(); + let head = db.get_ref(&branch.ref_name).unwrap(); + let mut view = ViewCore::new_lazy( + Trail::open(workspace.path()).unwrap(), + paths.source_upper.clone(), + head.root_id, + ) + .unwrap(); + let venv = view.lookup(VIEW_ROOT_INO, ".venv").unwrap(); + let marker = view.create(venv, "private.txt", 0o644, true).unwrap(); + view.write(marker.ino, 0, b"private\n").unwrap(); + assert!(paths.generated_upper.join(".venv/private.txt").is_file()); + assert!(!paths.source_upper.join(".venv/private.txt").exists()); + assert!(view.checkpoint_candidates().unwrap().paths.is_empty()); + drop(view); + + db.replace_declared_workspace_layer( + "python-a", + &layer.layer_id, + ".venv", + "python-env", + "trail/python-test@1", + 1, + "test", + "builtin:python-test", + "dependency", + &layer.cache_key, + ) + .unwrap(); + assert!(!paths.generated_upper.join(".venv").exists()); + let overlap = db + .replace_declared_workspace_layer( + "python-a", + &layer.layer_id, + ".venv/lib", + "python-tools", + "trail/python-test@1", + 1, + "test", + "builtin:python-test", + "dependency", + &layer.cache_key, + ) + .unwrap_err(); + assert!(overlap.to_string().contains("overlaps")); + } + + #[test] + fn committed_binding_recovery_finishes_interrupted_private_upper_reset() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "reset-commit", + Some("main"), + mode, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let built = tempfile::tempdir().unwrap(); + fs::create_dir_all(built.path().join("pkg")).unwrap(); + fs::write(built.path().join("pkg/index.js"), "immutable\n").unwrap(); + let layer = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + let view_report = db.lane_workspace_view("reset-commit").unwrap().unwrap(); + let paths = db.workspace_view_paths_for_lane("reset-commit").unwrap(); + fs::create_dir_all(paths.generated_upper.join("node_modules/pkg")).unwrap(); + fs::write( + paths.generated_upper.join("node_modules/pkg/private.js"), + "private\n", + ) + .unwrap(); + let branch = db.lane_branch("reset-commit").unwrap(); + let head = db.get_ref(&branch.ref_name).unwrap(); + let mut core = ViewCore::new_lazy( + Trail::open(workspace.path()).unwrap(), + paths.source_upper.clone(), + head.root_id.clone(), + ) + .unwrap(); + let reset = core + .prepare_declared_layer_mount_path("node_modules", "dependency", &layer.layer_id) + .unwrap(); + drop(reset); // crash after the filesystem half, before local cleanup + drop(core); + db.conn + .execute( + "INSERT OR REPLACE INTO workspace_view_layers (view_id, layer_id, mount_path, priority, read_only) VALUES (?1, ?2, 'node_modules', 100, 1)", + params![view_report.view_id, layer.layer_id], + ) + .unwrap(); + + let mut reopened = ViewCore::new_lazy( + Trail::open(workspace.path()).unwrap(), + paths.source_upper, + head.root_id, + ) + .unwrap(); + assert!(!paths.generated_upper.join("node_modules").exists()); + let modules = reopened.lookup(VIEW_ROOT_INO, "node_modules").unwrap(); + let pkg = reopened.lookup(modules, "pkg").unwrap(); + let index = reopened.lookup(pkg, "index.js").unwrap(); + assert_eq!(reopened.read(index, 0, 64).unwrap().0, b"immutable\n"); + assert!(fs::read_dir(paths.meta_dir.join("layer-reset-intents")) + .unwrap() + .next() + .is_none()); } #[test] @@ -1630,8 +4706,10 @@ mod tests { let mut db = Trail::open(workspace.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "recover", @@ -1685,8 +4763,10 @@ mod tests { let mut db = Trail::open(workspace.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "pinned", @@ -1704,6 +4784,9 @@ mod tests { let layer = db .publish_workspace_layer_from_directory(&key(), built.path()) .unwrap(); + let verification_stamp = + workspace_layer_verification_stamp_path(Path::new(&layer.storage_path)); + assert!(verification_stamp.is_file()); db.attach_workspace_layer( "pinned", &layer.layer_id, @@ -1736,9 +4819,47 @@ mod tests { .iter() .any(|candidate| candidate.id == layer.layer_id)); assert!(!Path::new(&layer.storage_path).exists()); + assert!(!verification_stamp.exists()); assert!(db.list_workspace_layers().unwrap().is_empty()); } + #[test] + fn cache_gc_preserves_layers_referenced_only_by_retired_environment_generations() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let built = tempfile::tempdir().unwrap(); + fs::write(built.path().join("artifact"), "retained\n").unwrap(); + let layer = db + .publish_workspace_layer_from_directory(&key(), built.path()) + .unwrap(); + db.conn + .execute( + "INSERT INTO environment_generations + (generation_id, view_id, generation_sequence, source_root, specification_digest, predecessor_generation_id, state, created_at, activated_at, retired_at) + VALUES ('retired-generation', 'retired-view', 1, 'root', 'spec', NULL, 'retired', 1, 1, 2)", + [], + ) + .unwrap(); + db.conn + .execute( + "INSERT INTO environment_generation_components + (generation_id, component_id, adapter_identity, kind, component_key, layer_id, mount_path) + VALUES ('retired-generation', 'component', 'trail/test@1', 'dependency', 'key', ?1, 'vendor')", + params![layer.layer_id], + ) + .unwrap(); + + let report = db.workspace_cache_gc(false, Some(0)).unwrap(); + assert!(!report + .deleted + .iter() + .any(|entry| entry.id == layer.layer_id)); + assert!(Path::new(&layer.storage_path).is_dir()); + db.verify_workspace_layer(&layer.layer_id).unwrap(); + } + #[test] fn corrupt_bound_layer_is_an_exact_readiness_blocker() { let workspace = tempfile::tempdir().unwrap(); @@ -1747,8 +4868,10 @@ mod tests { let mut db = Trail::open(workspace.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "corrupt", diff --git a/trail/src/db/lane/workspace_node.rs b/trail/src/db/lane/workspace_node.rs index b85d715..3124038 100644 --- a/trail/src/db/lane/workspace_node.rs +++ b/trail/src/db/lane/workspace_node.rs @@ -1,202 +1,94 @@ +use super::workspace_environment::{ + resolve_workspace_tool_executable, WorkspaceEnvironmentAdapter, + WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentCacheAccess, + WorkspaceEnvironmentCacheProtocol, WorkspaceEnvironmentCommand, WorkspaceEnvironmentInput, + WorkspaceEnvironmentOutput, WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, + WorkspaceEnvironmentSandboxPolicy, +}; use super::*; -#[derive(Clone, Debug)] -struct NodeEnvironmentInputs { - package_root: String, - manager: String, - files: BTreeMap, - key: WorkspaceLayerKeyV1, -} +pub(crate) struct NodeWorkspaceAdapter; -impl Trail { - pub fn sync_node_dependencies( - &self, - lane: &str, - package_root: Option<&str>, - ) -> Result { - let branch = self.lane_branch(lane)?; - let head = self.get_ref(&branch.ref_name)?; - let inputs = self.node_environment_inputs(&head.root_id, package_root.unwrap_or(""))?; - let cache_key = self.workspace_layer_cache_key(&inputs.key)?; - let adapter_id = if inputs.package_root.is_empty() { +pub(crate) static NODE_WORKSPACE_ADAPTER: NodeWorkspaceAdapter = NodeWorkspaceAdapter; + +static NODE_WORKSPACE_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = + WorkspaceEnvironmentAdapterMetadata { + canonical_identity: "trail/node@1", + namespace: "trail", + name: "node", + contract_major: 1, + implementation_version: env!("CARGO_PKG_VERSION"), + distribution_digest: "builtin:node-plan-v1", + selectors: &["trail/node@1", "node"], + kind: "dependency", + layer_adapter_name: "node", + discovery_markers: &["package.json"], + supported_operating_systems: &["linux", "macos", "windows"], + supported_architectures: &["aarch64", "x86_64"], + stability: "stable", + description: + "Frozen npm, pnpm, Yarn, or Bun dependency tree with a private writable lane upper", + }; + +impl WorkspaceEnvironmentAdapter for NodeWorkspaceAdapter { + fn metadata(&self) -> &'static WorkspaceEnvironmentAdapterMetadata { + &NODE_WORKSPACE_ADAPTER_METADATA + } + + fn component_id(&self, component_root: &str) -> Result { + let root = normalize_package_root(component_root)?; + Ok(if root.is_empty() { "node".to_string() } else { - format!("node:{}", inputs.package_root) - }; - self.set_workspace_environment_state( - lane, - &adapter_id, - &cache_key, - None, - "building", - None, - )?; - let build = self.build_workspace_layer_singleflight(&inputs.key, |build_dir| { - let project = build_dir.join("project"); - fs::create_dir_all(&project)?; - for (path, entry) in &inputs.files { - let relative = strip_package_root(path, &inputs.package_root)?; - let destination = safe_join(&project, &relative)?; - if let Some(parent) = destination.parent() { - fs::create_dir_all(parent)?; - } - let projection = self.project_entry_file(entry)?; - fs::copy(projection, &destination)?; - let mut permissions = fs::metadata(&destination)?.permissions(); - permissions.set_readonly(false); - fs::set_permissions(&destination, permissions)?; - } - let cache = self.db_dir.join("cache/tool-home/node"); - fs::create_dir_all(&cache)?; - let mut command = Command::new(&inputs.manager); - command - .current_dir(&project) - .env("npm_config_cache", cache.join("npm")) - .env("PNPM_HOME", cache.join("pnpm-home")) - .env("PNPM_STORE_DIR", cache.join("pnpm-store")); - match inputs.manager.as_str() { - "npm" => { - command.args(["ci", "--ignore-scripts", "--no-audit", "--no-fund"]); - } - "pnpm" => { - command.args(["install", "--frozen-lockfile", "--ignore-scripts"]); - } - "yarn" => { - let version = inputs - .key - .tool_versions - .get("yarn") - .map(String::as_str) - .unwrap_or_default(); - if version.starts_with('1') { - command.args(["install", "--frozen-lockfile", "--ignore-scripts"]); - } else { - command.args(["install", "--immutable", "--mode=skip-build"]); - } - } - "bun" => { - command.args(["install", "--frozen-lockfile", "--ignore-scripts"]); - } - other => { - return Err(Error::InvalidInput(format!( - "unsupported Node package manager `{other}`" - ))) - } - } - let status = command.status().map_err(|err| { - Error::InvalidInput(format!( - "failed to launch `{}` for Node dependency layer: {err}", - inputs.manager - )) - })?; - if !status.success() { - return Err(Error::InvalidInput(format!( - "{} frozen install failed with {status}", - inputs.manager - ))); - } - let output = project.join("node_modules"); - fs::create_dir_all(&output)?; - Ok(output) - }); - let layer = match build { - Ok(layer) => layer, - Err(err) => { - self.set_workspace_environment_state( - lane, - &adapter_id, - &cache_key, - None, - "failed", - Some(&err.to_string()), - )?; - return Err(err); - } - }; - let mount_path = if inputs.package_root.is_empty() { - "node_modules".to_string() - } else { - format!("{}/node_modules", inputs.package_root) - }; - self.attach_workspace_layer(lane, &layer.layer_id, &mount_path, &adapter_id, &cache_key)?; - Ok(layer) + format!("node:{root}") + }) } - pub fn workspace_environment_status( - &self, - lane: &str, - ) -> Result> { - let branch = self.lane_branch(lane)?; - let head = self.get_ref(&branch.ref_name)?; - let mut reports = self.workspace_environment_rows(lane)?; - for report in &mut reports { - let package_root = if report.adapter == "node" { - "" - } else if let Some(root) = report.adapter.strip_prefix("node:") { - root - } else { - continue; - }; - let inputs = self.node_environment_inputs(&head.root_id, package_root)?; - let expected = self.workspace_layer_cache_key(&inputs.key)?; - if report.attached_key.as_deref() == Some(expected.as_str()) { - report.expected_key = expected; - report.status = "ready".to_string(); - report.reason = None; - } else if report.expected_key != expected { - report.expected_key = expected; - report.status = "stale".to_string(); - report.reason = Some("package or lock inputs changed".to_string()); + fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result { + let root = normalize_package_root(component_root)?; + if db + .root_file_entry(source_root, &join_repo_path(&root, "package.json"))? + .is_none() + { + return Ok(false); + } + for (name, _) in supported_lockfiles() { + if db + .root_file_entry(source_root, &join_repo_path(&root, name))? + .is_some() + { + return Ok(true); } } - Ok(reports) + Ok(false) } - fn workspace_environment_rows(&self, lane: &str) -> Result> { - let view = self.lane_workspace_view(lane)?.ok_or_else(|| { - Error::InvalidInput(format!( - "lane `{lane}` does not have a layered workspace view" - )) - })?; - let mut stmt = self.conn.prepare( - "SELECT view_id, adapter, expected_key, attached_key, status, reason, updated_at FROM workspace_environment_states WHERE view_id = ?1 ORDER BY adapter", - )?; - let reports = stmt - .query_map(params![view.view_id], |row| { - Ok(WorkspaceEnvironmentReport { - view_id: row.get(0)?, - adapter: row.get(1)?, - expected_key: row.get(2)?, - attached_key: row.get(3)?, - status: row.get(4)?, - reason: row.get(5)?, - updated_at: row.get(6)?, - }) - })? - .collect::, _>>()?; - Ok(reports) + fn plan( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result { + db.node_environment_plan(source_root, component_root) } +} - pub fn refresh_workspace_environment_staleness(&self, lane: &str) -> Result<()> { - let states = self.workspace_environment_status(lane)?; - for state in states { - self.set_workspace_environment_state( - lane, - &state.adapter, - &state.expected_key, - state.attached_key.as_deref(), - &state.status, - state.reason.as_deref(), - )?; - } - Ok(()) +impl Trail { + /// Compatibility entry point retained for existing `trail deps sync` + /// callers. All execution and persistence is owned by the generic host. + pub fn sync_node_dependencies( + &self, + lane: &str, + package_root: Option<&str>, + ) -> Result { + self.sync_workspace_environment(lane, "trail/node@1", package_root) } - fn node_environment_inputs( + fn node_environment_plan( &self, root_id: &ObjectId, package_root: &str, - ) -> Result { + ) -> Result { let package_root = if package_root.trim_matches('/').is_empty() { String::new() } else { @@ -215,16 +107,8 @@ impl Trail { } )) })?; - let lock_names = [ - ("pnpm-lock.yaml", "pnpm"), - ("yarn.lock", "yarn"), - ("bun.lock", "bun"), - ("bun.lockb", "bun"), - ("npm-shrinkwrap.json", "npm"), - ("package-lock.json", "npm"), - ]; let mut selected = None; - for (name, manager) in lock_names { + for (name, manager) in supported_lockfiles() { let path = join_repo_path(&package_root, name); if let Some(entry) = self.root_file_entry(root_id, &path)? { selected = Some((path, manager.to_string(), entry)); @@ -243,76 +127,246 @@ impl Trail { })?; let manager_version = tool_version(&manager)?; let node_version = tool_version("node")?; + let node_tool = resolve_workspace_tool_executable("node")?; + let manager_tool = resolve_workspace_tool_executable(&manager)?; + let package_projection = self.project_entry_file(&package_entry)?; + let package_text = fs::read_to_string(package_projection)?; + let package_value: serde_json::Value = serde_json::from_str(&package_text)?; + if package_value.get("workspaces").is_some() { + return Err(Error::InvalidInput(format!( + "Node component `{}` declares workspaces; synchronize a supported leaf package explicitly until the monorepo adapter is enabled", + display_package_root(&package_root) + ))); + } + if contains_local_node_dependency(&package_value) { + return Err(Error::InvalidInput(format!( + "Node component `{}` contains file:, link:, or workspace: dependencies that cannot be represented by an isolated node_modules layer", + display_package_root(&package_root) + ))); + } + if manager == "pnpm" + && self + .root_file_entry( + root_id, + &join_repo_path(&package_root, "pnpm-workspace.yaml"), + )? + .is_some() + { + return Err(Error::InvalidInput( + "pnpm workspace roots require the future monorepo environment adapter".to_string(), + )); + } + if manager == "yarn" + && (!manager_version.starts_with('1') + || self + .root_file_entry(root_id, &join_repo_path(&package_root, ".yarnrc.yml"))? + .is_some()) + { + return Err(Error::InvalidInput( + "Yarn Berry/PnP layouts are not node_modules layers; use Yarn Classic or wait for the PnP adapter" + .to_string(), + )); + } let mut files = BTreeMap::from([ (package_json.clone(), package_entry.clone()), (lock_path.clone(), lock_entry.clone()), ]); - // Workspace package manifests affect installation. Discover them in a - // streaming pass; this runs only during explicit dependency sync, not - // during view creation or lookup. - self.for_each_root_file_chunk(root_id, 1024, |chunk| { - for (path, entry) in chunk { - if Path::new(&path).file_name().and_then(|name| name.to_str()) - != Some("package.json") - { - continue; - } - if package_root.is_empty() - || path.starts_with(&format!("{package_root}/")) - || path == package_json - { - files.insert(path, entry); - } + for name in [ + ".npmrc", + ".yarnrc", + "pnpmfile.cjs", + ".node-version", + ".nvmrc", + ] { + let path = join_repo_path(&package_root, name); + if let Some(entry) = self.root_file_entry(root_id, &path)? { + files.insert(path, entry); } - Ok(()) - })?; + } + let implementation_version = env!("CARGO_PKG_VERSION").to_string(); + let distribution_digest = "builtin:node-plan-v1".to_string(); + let mut key_inputs = files + .iter() + .map(|(path, entry)| (path.clone(), entry.content_hash.clone())) + .collect::>(); + key_inputs.insert( + "adapter_implementation".to_string(), + implementation_version.clone(), + ); + key_inputs.insert( + "adapter_distribution_digest".to_string(), + distribution_digest.clone(), + ); let key = WorkspaceLayerKeyV1 { kind: "dependency".to_string(), adapter: "node".to_string(), adapter_version: 1, - inputs: files - .iter() - .map(|(path, entry)| (path.clone(), entry.content_hash.clone())) - .collect(), + inputs: key_inputs, tool_versions: BTreeMap::from([ ("node".to_string(), node_version), (manager.clone(), manager_version), + ("node-executable".to_string(), node_tool.identity), + ( + format!("{manager}-executable"), + manager_tool.identity.clone(), + ), ]), platform: std::env::consts::OS.to_string(), architecture: std::env::consts::ARCH.to_string(), portability_scope: "platform-architecture-node-abi".to_string(), strategy: format!("{manager}-frozen-ignore-scripts-v1"), }; - Ok(NodeEnvironmentInputs { - package_root, - manager, - files, - key, + let project = "project".to_string(); + let cache = self.declare_workspace_environment_cache( + NODE_WORKSPACE_ADAPTER.identity(), + "package-manager", + WorkspaceEnvironmentCacheProtocol::ContentStore, + WorkspaceEnvironmentCacheAccess::ToolConcurrent, + BTreeMap::from([ + ("manager".to_string(), manager.clone()), + ( + "manager_executable".to_string(), + manager_tool.identity.clone(), + ), + ("platform".to_string(), std::env::consts::OS.to_string()), + ( + "architecture".to_string(), + std::env::consts::ARCH.to_string(), + ), + ]), + )?; + let cache_root = &cache.storage_path; + let environment = BTreeMap::from([ + ( + "npm_config_cache".to_string(), + cache_root.join("npm").to_string_lossy().into_owned(), + ), + ( + "PNPM_HOME".to_string(), + cache_root.join("pnpm-home").to_string_lossy().into_owned(), + ), + ( + "PNPM_STORE_DIR".to_string(), + cache_root.join("pnpm-store").to_string_lossy().into_owned(), + ), + ]); + let args = match manager.as_str() { + "npm" => vec!["ci", "--ignore-scripts", "--no-audit", "--no-fund"], + "pnpm" => vec!["install", "--frozen-lockfile", "--ignore-scripts"], + "yarn" => vec!["install", "--frozen-lockfile", "--ignore-scripts"], + "bun" => vec!["install", "--frozen-lockfile", "--ignore-scripts"], + other => { + return Err(Error::InvalidInput(format!( + "unsupported Node package manager `{other}`" + ))); + } + } + .into_iter() + .map(str::to_string) + .collect(); + let inputs = files + .into_iter() + .map(|(source_path, entry)| { + let relative = strip_package_root(&source_path, &package_root)?; + Ok(WorkspaceEnvironmentInput { + source_path, + staging_path: format!("project/{relative}"), + entry, + }) + }) + .collect::>>()?; + let mount_path = if package_root.is_empty() { + "node_modules".to_string() + } else { + format!("{package_root}/node_modules") + }; + Ok(WorkspaceEnvironmentPlan { + component_id: NODE_WORKSPACE_ADAPTER.component_id(&package_root)?, + adapter_identity: NODE_WORKSPACE_ADAPTER.identity().to_string(), + adapter_version: 1, + implementation_version, + distribution_digest, + kind: "dependency".to_string(), + dependencies: Vec::new(), + resolved_dependencies: Vec::new(), + layer_key: key, + inputs, + source_projection: None, + pre_commands: Vec::new(), + command: Some(WorkspaceEnvironmentCommand { + program: manager, + resolved_program: manager_tool.path, + executable_identity: manager_tool.identity, + args, + working_directory: project.clone(), + environment, + remove_environment: Vec::new(), + cache_names: vec![cache.name.clone()], + }), + mounted_commands: Vec::new(), + caches: vec![cache], + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin, + outputs: vec![WorkspaceEnvironmentOutput { + name: "modules".to_string(), + output_path: format!("{project}/node_modules"), + mount_path, + policy: WorkspaceEnvironmentOutputPolicy::ImmutableSeedPrivate, + create_if_missing: true, + }], + stale_reason: + "package, lockfile, Node runtime, package manager, or adapter policy changed" + .to_string(), }) } +} - fn set_workspace_environment_state( - &self, - lane: &str, - adapter: &str, - expected_key: &str, - attached_key: Option<&str>, - status: &str, - reason: Option<&str>, - ) -> Result<()> { - let view = self.lane_workspace_view(lane)?.ok_or_else(|| { - Error::InvalidInput(format!( - "lane `{lane}` does not have a layered workspace view" - )) - })?; - self.conn.execute( - "INSERT OR REPLACE INTO workspace_environment_states (view_id, adapter, expected_key, attached_key, status, reason, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", - params![view.view_id, adapter, expected_key, attached_key, status, reason, now_ts()], - )?; - Ok(()) +fn supported_lockfiles() -> [(&'static str, &'static str); 6] { + [ + ("pnpm-lock.yaml", "pnpm"), + ("yarn.lock", "yarn"), + ("bun.lock", "bun"), + ("bun.lockb", "bun"), + ("npm-shrinkwrap.json", "npm"), + ("package-lock.json", "npm"), + ] +} + +fn normalize_package_root(package_root: &str) -> Result { + if package_root.trim_matches('/').is_empty() { + Ok(String::new()) + } else { + normalize_relative_path(package_root) } } +fn display_package_root(package_root: &str) -> &str { + if package_root.is_empty() { + "." + } else { + package_root + } +} + +fn contains_local_node_dependency(package: &serde_json::Value) -> bool { + [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies", + ] + .into_iter() + .filter_map(|name| package.get(name).and_then(serde_json::Value::as_object)) + .flat_map(|dependencies| dependencies.values()) + .filter_map(serde_json::Value::as_str) + .any(|value| { + ["file:", "link:", "workspace:"] + .iter() + .any(|prefix| value.starts_with(prefix)) + }) +} + fn tool_version(tool: &str) -> Result { let output = Command::new(tool) .arg("--version") @@ -375,8 +429,10 @@ mod tests { let mut db = Trail::open(workspace.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; for lane in ["node-one", "node-two"] { db.spawn_lane_with_workdir_mode_paths_and_neighbors( @@ -392,7 +448,9 @@ mod tests { .unwrap(); } let first = db.sync_node_dependencies("node-one", None).unwrap(); - let second = db.sync_node_dependencies("node-two", None).unwrap(); + let second = db + .sync_workspace_environment("node-two", "auto", None) + .unwrap(); assert_eq!(first.layer_id, second.layer_id); assert_eq!(first.cache_key, second.cache_key); assert_eq!(db.list_workspace_layers().unwrap().len(), 1); @@ -407,6 +465,49 @@ mod tests { ) .unwrap(); assert_eq!(bound, 2); + let generation_one = db + .active_environment_generation("node-one") + .unwrap() + .unwrap(); + let generation_two = db + .active_environment_generation("node-two") + .unwrap() + .unwrap(); + let cache_one = &generation_one.components[0].caches[0]; + let cache_two = &generation_two.components[0].caches[0]; + assert_eq!(cache_one.namespace_id, cache_two.namespace_id); + assert_eq!(cache_one.protocol, "content_store"); + assert_eq!(cache_one.access, "tool_concurrent"); + assert_eq!(cache_one.authority, "performance_only"); + assert!(db + .db_dir + .join("cache/namespaces") + .join(&cache_one.namespace_id) + .is_dir()); + assert!(!db.db_dir.join("cache/tool-home/node").exists()); + + db.conn + .execute( + "UPDATE workspace_views SET owner_pid = ?1, owner_start_token = ?2, status = 'mounted' WHERE view_id = ?3", + params![ + std::process::id(), + current_process_start_token(), + view_one.view_id + ], + ) + .unwrap(); + let mounted = db.sync_node_dependencies("node-one", None).unwrap_err(); + assert!(mounted.to_string().contains("trail lane unmount node-one")); + assert_eq!( + db.workspace_environment_rows("node-one").unwrap()[0].status, + "ready" + ); + db.conn + .execute( + "UPDATE workspace_views SET owner_pid = NULL, owner_start_token = NULL, status = 'unmounted' WHERE view_id = ?1", + params![view_one.view_id], + ) + .unwrap(); db.conn .execute( @@ -415,13 +516,38 @@ mod tests { ) .unwrap(); let dynamic = db.workspace_environment_status("node-one").unwrap(); - assert_eq!(dynamic[0].status, "ready"); + assert_eq!(dynamic[0].status, "building"); let persisted = db.workspace_environment_rows("node-one").unwrap().remove(0); assert_eq!(persisted.status, "building"); assert_eq!(persisted.reason.as_deref(), Some("sentinel")); + + let normalized = db + .enforce_read_only_mcp_call("trail.env_status", |db| { + db.environment_component_status("node-one") + }) + .unwrap(); + assert_eq!(normalized.len(), 1); + assert_eq!(normalized[0].component.component_id, "node"); + + let normalized = db + .enforce_read_only_mcp_call("trail.env_status", |db| { + db.environment_component_status("node-one") + }) + .unwrap(); + assert_eq!(normalized.len(), 1); + assert_eq!(normalized[0].component.component_id, "node"); assert_eq!(persisted.updated_at, -1); let paths = db.workspace_view_paths_for_lane("node-one").unwrap(); + let mut intent = + super::super::workdir::ViewMutationJournal::open(&paths.source_upper).unwrap(); + intent + .append( + super::super::workdir::ViewMutationKind::Write, + "package-lock.json", + None, + ) + .unwrap(); fs::write( paths.source_upper.join("package-lock.json"), r#"{"name":"trail-node-layer-test","version":"1.0.1","lockfileVersion":3,"requires":true,"packages":{"":{"name":"trail-node-layer-test","version":"1.0.1"}}}"#, @@ -434,5 +560,20 @@ mod tests { .blockers .iter() .any(|issue| issue.code == "dependency_environment_stale")); + let explanation = db + .explain_workspace_environment_staleness("node-one", "node") + .unwrap(); + assert!(explanation.complete); + assert_eq!(explanation.status, "stale"); + assert!(explanation.changes.iter().any(|change| { + change.dimension == "input" + && change.name == "package-lock.json" + && change.change == "modified" + })); + let state = db.environment_component_status("node-one").unwrap(); + assert!(state[0] + .reason + .as_deref() + .is_some_and(|reason| reason.contains("input:package-lock.json modified"))); } } diff --git a/trail/src/db/lane/workspace_oci.rs b/trail/src/db/lane/workspace_oci.rs new file mode 100644 index 0000000..daa8208 --- /dev/null +++ b/trail/src/db/lane/workspace_oci.rs @@ -0,0 +1,583 @@ +use serde::Deserialize; + +use super::workspace_environment::{ + workspace_external_artifacts_identity, workspace_runtime_resources_identity, + WorkspaceEnvironmentAdapter, WorkspaceEnvironmentAdapterMetadata, + WorkspaceEnvironmentExternalArtifact, WorkspaceEnvironmentPlan, + WorkspaceEnvironmentRuntimeResource, WorkspaceEnvironmentSandboxPolicy, + WorkspaceEnvironmentSecretReference, +}; +use super::*; + +const OCI_SPEC_SCHEMA: &str = "trail.oci-images/v1"; +const OCI_SPEC_FILE: &str = "trail.oci.toml"; +const MAX_OCI_SPEC_BYTES: u64 = 1024 * 1024; + +pub(crate) struct OciImageAdapter; + +pub(crate) static OCI_IMAGE_ADAPTER: OciImageAdapter = OciImageAdapter; + +static OCI_IMAGE_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = + WorkspaceEnvironmentAdapterMetadata { + canonical_identity: "trail/oci-image@1", + namespace: "trail", + name: "oci-image", + contract_major: 1, + implementation_version: env!("CARGO_PKG_VERSION"), + distribution_digest: "builtin:pinned-oci-image-plan-v1", + selectors: &["trail/oci-image@1", "oci-image", "oci"], + kind: "external", + layer_adapter_name: "oci-image", + discovery_markers: &[OCI_SPEC_FILE], + supported_operating_systems: &["linux", "macos", "windows"], + supported_architectures: &["aarch64", "x86_64"], + stability: "experimental", + description: "Digest-pinned OCI image identities recorded atomically with a lane environment generation", + }; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct OciImageSpecification { + schema: String, + #[serde(default, rename = "image")] + images: Vec, + #[serde(default, rename = "service")] + services: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct OciImageDeclaration { + name: String, + reference: String, + platform: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct OciServiceDeclaration { + name: String, + image: String, + container_port: u16, + #[serde(default = "default_health_timeout_ms")] + health_timeout_ms: u64, + #[serde(default = "default_restart_policy")] + restart_policy: String, + #[serde(default)] + volume_target: Option, + #[serde(default, rename = "secret")] + secrets: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct OciServiceSecretDeclaration { + name: String, + provider: String, + reference: String, + #[serde(default)] + version: Option, + purpose: String, + injection: String, + target: String, + #[serde(default)] + environment: Option, + #[serde(default = "default_secret_required")] + required: bool, +} + +fn default_secret_required() -> bool { + true +} + +fn default_health_timeout_ms() -> u64 { + 30_000 +} + +fn default_restart_policy() -> String { + "on_failure".to_string() +} + +impl WorkspaceEnvironmentAdapter for OciImageAdapter { + fn metadata(&self) -> &'static WorkspaceEnvironmentAdapterMetadata { + &OCI_IMAGE_ADAPTER_METADATA + } + + fn component_id(&self, component_root: &str) -> Result { + let root = normalize_component_root(component_root)?; + Ok(if root.is_empty() { + "oci-images".to_string() + } else { + format!("oci-images:{root}") + }) + } + + fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result { + let root = normalize_component_root(component_root)?; + Ok(db + .root_file_entry(source_root, &join_repo_path(&root, OCI_SPEC_FILE))? + .is_some()) + } + + fn plan( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result { + let component_root = normalize_component_root(component_root)?; + let specification_path = join_repo_path(&component_root, OCI_SPEC_FILE); + let entry = db + .root_file_entry(source_root, &specification_path)? + .ok_or_else(|| { + Error::InvalidInput(format!( + "OCI component `{}` has no {OCI_SPEC_FILE}", + display_component_root(&component_root) + )) + })?; + if entry.size_bytes > MAX_OCI_SPEC_BYTES { + return Err(Error::InvalidInput(format!( + "OCI specification `{specification_path}` is {} bytes; the maximum is {MAX_OCI_SPEC_BYTES}", + entry.size_bytes + ))); + } + let manifest_digest = entry.content_hash.clone(); + let mut bytes = + db.materialize_entries_bytes(&BTreeMap::from([(specification_path.clone(), entry)]))?; + let text = String::from_utf8(bytes.remove(&specification_path).ok_or_else(|| { + Error::Corrupt(format!( + "failed to read `{specification_path}` from the pinned source root" + )) + })?) + .map_err(|_| { + Error::InvalidInput(format!( + "OCI specification `{specification_path}` must be UTF-8" + )) + })?; + let specification: OciImageSpecification = toml::from_str(&text).map_err(|error| { + Error::InvalidInput(format!( + "invalid OCI specification `{specification_path}`: {error}" + )) + })?; + if specification.schema != OCI_SPEC_SCHEMA { + return Err(Error::InvalidInput(format!( + "OCI specification `{specification_path}` uses schema `{}`; expected `{OCI_SPEC_SCHEMA}`", + specification.schema + ))); + } + if specification.images.is_empty() { + return Err(Error::InvalidInput(format!( + "OCI specification `{specification_path}` must declare at least one [[image]]" + ))); + } + if specification.images.len() > 32 { + return Err(Error::InvalidInput(format!( + "OCI specification `{specification_path}` declares more than 32 images" + ))); + } + + let mut names = BTreeSet::new(); + let mut external_artifacts = Vec::with_capacity(specification.images.len()); + for image in specification.images { + if !names.insert(image.name.clone()) { + return Err(Error::InvalidInput(format!( + "OCI specification `{specification_path}` repeats image `{}`", + image.name + ))); + } + let digest = image + .reference + .rsplit_once('@') + .map(|(_, digest)| digest.to_string()) + .ok_or_else(|| { + Error::InvalidInput(format!( + "OCI image `{}` must use a digest-pinned reference such as registry.example/repository@sha256:", + image.name + )) + })?; + external_artifacts.push(WorkspaceEnvironmentExternalArtifact { + name: image.name, + artifact_type: "oci_image".to_string(), + provider: "oci".to_string(), + reference: image.reference, + digest, + platform: image.platform, + cleanup_owner: "external".to_string(), + }); + } + external_artifacts.sort_by(|left, right| left.name.cmp(&right.name)); + let external_artifact_contract = + workspace_external_artifacts_identity(&external_artifacts)?; + if specification.services.len() > 32 { + return Err(Error::InvalidInput(format!( + "OCI specification `{specification_path}` declares more than 32 services" + ))); + } + let artifact_names = external_artifacts + .iter() + .map(|artifact| artifact.name.as_str()) + .collect::>(); + let mut service_names = BTreeSet::new(); + let mut runtime_resources = Vec::with_capacity(specification.services.len()); + for service in specification.services { + if !service_names.insert(service.name.clone()) { + return Err(Error::InvalidInput(format!( + "OCI specification `{specification_path}` repeats service `{}`", + service.name + ))); + } + if !artifact_names.contains(service.image.as_str()) { + return Err(Error::InvalidInput(format!( + "OCI service `{}` references undeclared image `{}`", + service.name, service.image + ))); + } + runtime_resources.push(WorkspaceEnvironmentRuntimeResource { + name: service.name, + runtime_type: "container".to_string(), + provider: "oci".to_string(), + artifact_name: service.image, + container_port: service.container_port, + protocol: "tcp".to_string(), + health_type: "tcp".to_string(), + health_timeout_ms: service.health_timeout_ms, + restart_policy: service.restart_policy, + cleanup_owner: "trail".to_string(), + volume_target: service.volume_target, + secrets: service + .secrets + .into_iter() + .map(|secret| WorkspaceEnvironmentSecretReference { + name: secret.name, + provider: secret.provider, + reference: secret.reference, + version: secret.version, + purpose: secret.purpose, + injection: secret.injection, + target: secret.target, + environment: secret.environment, + required: secret.required, + }) + .collect(), + }); + } + runtime_resources.sort_by(|left, right| left.name.cmp(&right.name)); + let runtime_resource_contract = workspace_runtime_resources_identity(&runtime_resources)?; + let component_id = self.component_id(&component_root)?; + let implementation_version = env!("CARGO_PKG_VERSION").to_string(); + let distribution_digest = "builtin:pinned-oci-image-plan-v1".to_string(); + let mut layer_inputs = BTreeMap::from([ + ("component_id".to_string(), component_id.clone()), + ("component_root".to_string(), component_root.clone()), + ("source_root".to_string(), source_root.0.clone()), + ("manifest".to_string(), specification_path), + ("manifest_digest".to_string(), manifest_digest), + ( + "external_artifact_contract".to_string(), + external_artifact_contract, + ), + ( + "adapter_implementation".to_string(), + implementation_version.clone(), + ), + ( + "adapter_distribution_digest".to_string(), + distribution_digest.clone(), + ), + ]); + if !runtime_resources.is_empty() { + layer_inputs.insert( + "runtime_resource_contract".to_string(), + runtime_resource_contract, + ); + } + + Ok(WorkspaceEnvironmentPlan { + component_id: component_id.clone(), + adapter_identity: self.identity().to_string(), + adapter_version: 1, + implementation_version: implementation_version.clone(), + distribution_digest: distribution_digest.clone(), + kind: "external".to_string(), + dependencies: Vec::new(), + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: "external".to_string(), + adapter: self.layer_adapter_name().to_string(), + adapter_version: 1, + inputs: layer_inputs, + tool_versions: BTreeMap::new(), + platform: "oci".to_string(), + architecture: "declared-per-artifact".to_string(), + portability_scope: "external-oci-digest-platform".to_string(), + strategy: "pinned-oci-images-v1".to_string(), + }, + inputs: Vec::new(), + source_projection: None, + pre_commands: Vec::new(), + command: None, + mounted_commands: Vec::new(), + caches: Vec::new(), + external_artifacts, + runtime_resources, + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin, + outputs: Vec::new(), + stale_reason: "pinned OCI reference, digest, platform, or adapter policy changed" + .to_string(), + }) + } +} + +fn normalize_component_root(component_root: &str) -> Result { + if component_root.trim_matches('/').is_empty() { + Ok(String::new()) + } else { + normalize_relative_path(component_root) + } +} + +fn join_repo_path(root: &str, name: &str) -> String { + if root.is_empty() { + name.to_string() + } else { + format!("{root}/{name}") + } +} + +fn display_component_root(component_root: &str) -> &str { + if component_root.is_empty() { + "." + } else { + component_root + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const DIGEST: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + fn write_specification(root: &Path, reference: &str) { + fs::write( + root.join(OCI_SPEC_FILE), + format!( + "schema = \"{OCI_SPEC_SCHEMA}\"\n\n[[image]]\nname = \"web\"\nreference = \"{reference}\"\nplatform = \"linux/amd64\"\n" + ), + ) + .unwrap(); + } + + fn write_service_specification(root: &Path, reference: &str) { + fs::write( + root.join(OCI_SPEC_FILE), + format!( + "schema = \"{OCI_SPEC_SCHEMA}\"\n\n[[image]]\nname = \"database-image\"\nreference = \"{reference}\"\nplatform = \"linux/amd64\"\n\n[[service]]\nname = \"database\"\nimage = \"database-image\"\ncontainer_port = 5432\nhealth_timeout_ms = 45000\nrestart_policy = \"on_failure\"\nvolume_target = \"/var/lib/postgresql/data\"\n\n[[service.secret]]\nname = \"database-password\"\nprovider = \"environment_file\"\nreference = \"DATABASE_PASSWORD_FILE\"\nversion = \"rotation-7\"\npurpose = \"authenticate the database service\"\ninjection = \"file\"\ntarget = \"/run/secrets/database-password\"\nenvironment = \"POSTGRES_PASSWORD_FILE\"\nrequired = true\n" + ), + ) + .unwrap(); + } + + #[test] + fn discovery_and_planning_are_pinned_and_side_effect_free() { + let workspace = tempfile::tempdir().unwrap(); + write_specification(workspace.path(), &format!("ghcr.io/example/web@{DIGEST}")); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let root = db.resolve_branch_ref("main").unwrap().root_id; + let cache_root = workspace.path().join(".trail/cache/environments"); + let before = fs::read_dir(&cache_root) + .ok() + .map(|entries| entries.count()) + .unwrap_or_default(); + + assert!(OCI_IMAGE_ADAPTER.detect(&db, &root, "").unwrap()); + let plan = OCI_IMAGE_ADAPTER.plan(&db, &root, "").unwrap(); + assert_eq!(plan.component_id, "oci-images"); + assert_eq!(plan.kind, "external"); + assert!(plan.outputs.is_empty()); + assert!(plan.command.is_none()); + assert!(plan.caches.is_empty()); + assert_eq!(plan.external_artifacts.len(), 1); + assert_eq!(plan.external_artifacts[0].digest, DIGEST); + assert_eq!(plan.external_artifacts[0].platform, "linux/amd64"); + + let after = fs::read_dir(&cache_root) + .ok() + .map(|entries| entries.count()) + .unwrap_or_default(); + assert_eq!(after, before); + } + + #[test] + fn planning_rejects_tags_without_a_digest() { + let workspace = tempfile::tempdir().unwrap(); + write_specification(workspace.path(), "ghcr.io/example/web:latest"); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let root = db.resolve_branch_ref("main").unwrap().root_id; + let error = OCI_IMAGE_ADAPTER.plan(&db, &root, "").unwrap_err(); + assert!(error.to_string().contains("digest-pinned reference")); + } + + #[test] + fn two_lanes_activate_independent_metadata_generations_without_fake_layers() { + let workspace = tempfile::tempdir().unwrap(); + write_specification(workspace.path(), &format!("ghcr.io/example/web@{DIGEST}")); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + for lane in ["oci-a", "oci-b"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + } + + let first = db + .sync_workspace_environment_component("oci-a", OCI_IMAGE_ADAPTER.identity(), None, None) + .unwrap(); + let second = db + .sync_workspace_environment_component("oci-b", OCI_IMAGE_ADAPTER.identity(), None, None) + .unwrap(); + + assert!(first.layers.is_empty()); + assert!(second.layers.is_empty()); + assert_ne!( + first.generation.generation_id, + second.generation.generation_id + ); + assert_eq!( + first.generation.components[0].component_key, + second.generation.components[0].component_key + ); + assert!(first.generation.components[0].outputs.is_empty()); + assert_eq!( + first.generation.components[0].external_artifacts, + second.generation.components[0].external_artifacts + ); + assert_eq!( + first.generation.components[0].external_artifacts[0].digest, + DIGEST + ); + assert!(db.list_workspace_layers().unwrap().is_empty()); + } + + #[test] + fn service_declarations_allocate_distinct_pending_runtime_identities_per_lane() { + let workspace = tempfile::tempdir().unwrap(); + write_service_specification( + workspace.path(), + &format!("ghcr.io/example/postgres@{DIGEST}"), + ); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + for lane in ["runtime-a", "runtime-b"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + } + + let plan = db + .plan_workspace_environment_component( + "runtime-a", + OCI_IMAGE_ADAPTER.identity(), + None, + None, + ) + .unwrap(); + assert_eq!(plan.runtime_resources.len(), 1); + assert_eq!(plan.runtime_resources[0].name, "database"); + assert_eq!(plan.runtime_resources[0].container_port, 5432); + assert_eq!( + plan.runtime_resources[0].volume_target.as_deref(), + Some("/var/lib/postgresql/data") + ); + assert_eq!(plan.runtime_resources[0].secrets.len(), 1); + assert_eq!( + plan.runtime_resources[0].secrets[0].reference, + "DATABASE_PASSWORD_FILE" + ); + assert_eq!( + plan.runtime_resources[0].secrets[0].environment.as_deref(), + Some("POSTGRES_PASSWORD_FILE") + ); + assert!(plan.outputs.is_empty()); + + let first = db + .sync_workspace_environment_component( + "runtime-a", + OCI_IMAGE_ADAPTER.identity(), + None, + None, + ) + .unwrap(); + let second = db + .sync_workspace_environment_component( + "runtime-b", + OCI_IMAGE_ADAPTER.identity(), + None, + None, + ) + .unwrap(); + let first_resource = &first.generation.components[0].runtime_resources[0]; + let second_resource = &second.generation.components[0].runtime_resources[0]; + assert_eq!(first_resource.status, "pending"); + assert_eq!(first_resource.health_status, "pending"); + assert_eq!(first_resource.image_digest, DIGEST); + assert_eq!(first_resource.secret_statuses.len(), 1); + assert_eq!(first_resource.secret_statuses[0].status, "pending"); + assert_ne!(first_resource.allocation_id, second_resource.allocation_id); + assert_ne!( + first_resource.container_name, + second_resource.container_name + ); + assert_ne!(first_resource.network_name, second_resource.network_name); + assert_ne!(first_resource.volume_name, second_resource.volume_name); + assert!(first.layers.is_empty()); + assert!(second.layers.is_empty()); + assert!(db + .lane_readiness("runtime-a") + .unwrap() + .blockers + .iter() + .any(|blocker| blocker.code == "environment_runtime_unhealthy")); + + drop(db); + let reopened = Trail::open(workspace.path()).unwrap(); + let persisted = reopened + .active_environment_generation("runtime-a") + .unwrap() + .unwrap(); + assert_eq!( + persisted.components[0].runtime_resources[0], + first_resource.clone() + ); + } +} diff --git a/trail/src/db/lane/workspace_plugin.rs b/trail/src/db/lane/workspace_plugin.rs new file mode 100644 index 0000000..45607c7 --- /dev/null +++ b/trail/src/db/lane/workspace_plugin.rs @@ -0,0 +1,3345 @@ +use std::ffi::{OsStr, OsString}; +use std::io::{Cursor, Read, Write}; +use std::process::{Child, Command, Stdio}; +use std::thread; + +use ed25519_dalek::{Signature, VerifyingKey}; +use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; +use serde::{Deserialize, Serialize}; +use trail_environment_adapter_sdk::{ + read_frame, write_frame, AdapterAction, AdapterCache, AdapterCacheAccess, AdapterCacheProtocol, + AdapterCommand, AdapterDependencyType, AdapterExternalArtifact, AdapterHost, AdapterOperation, + AdapterOutput, AdapterOutputPolicy, AdapterPackageManifest, AdapterPackageSignature, + AdapterPermissions, AdapterPlan, AdapterPlanV2, AdapterPortability, AdapterPublisherKey, + AdapterRequest, AdapterResponse, AdapterResult, AdapterRuntimeResource, DiscoveredComponent, + PinnedFile, MAX_FRAME_BYTES, PACKAGE_SCHEMA_V1, PACKAGE_SIGNATURE_SCHEMA_V1, PROTOCOL_V1, + PROTOCOL_V2, TRUSTED_PUBLISHER_KEY_SCHEMA_V1, +}; + +use super::workspace_environment::{ + WorkspaceEnvironmentCache, WorkspaceEnvironmentCacheAccess, WorkspaceEnvironmentCacheProtocol, + WorkspaceEnvironmentCommand, WorkspaceEnvironmentDependency, WorkspaceEnvironmentEdgeType, + WorkspaceEnvironmentExternalArtifact, WorkspaceEnvironmentInput, WorkspaceEnvironmentOutput, + WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, + WorkspaceEnvironmentRuntimeResource, WorkspaceEnvironmentSandboxPolicy, + WorkspaceEnvironmentSecretReference, +}; +use super::*; + +const PLUGIN_STORE_VERSION: &str = "v1"; +const PLUGIN_PACKAGE_MANIFEST: &str = "trail-adapter.toml"; +const PLUGIN_PACKAGE_SIGNATURE: &str = "trail-adapter.sig"; +const INSTALLED_MANIFEST_FILE: &str = "manifest.cbor"; +const REGISTRY_RECORD_SCHEMA: &str = "trail.environment-adapter-registry/v1"; +const TRUST_RECORD_SCHEMA: &str = "trail.environment-adapter-publisher-trust/v1"; +const MAX_PACKAGE_MANIFEST_BYTES: u64 = 1024 * 1024; +const MAX_PUBLISHER_DOCUMENT_BYTES: u64 = 64 * 1024; +const MAX_PLUGIN_EXECUTABLE_BYTES: u64 = 256 * 1024 * 1024; +const HARD_MAX_INPUT_FILES: u32 = 100_000; +const HARD_MAX_INPUT_BYTES: u64 = 8 * 1024 * 1024; +const HARD_MAX_TIMEOUT_MS: u64 = 30_000; +const HARD_MAX_RESPONSE_BYTES: u64 = 16 * 1024 * 1024; +const PLUGIN_MEMORY_LIMIT_BYTES: u64 = 512 * 1024 * 1024; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct InstalledPluginManifest { + package: AdapterPackageManifest, + #[serde(default)] + payload_digest: String, + distribution_digest: String, + executable_digest: String, + executable_file: String, + #[serde(default)] + signature: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PluginRegistryRecord { + schema: String, + canonical_identity: String, + action: String, + distribution_digest: Option, + recorded_at: i64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PublisherTrustRecord { + schema: String, + key_id: String, + publisher: String, + public_key: String, + action: String, + recorded_at: i64, +} + +struct SelectedPluginFiles { + protocol_files: Vec, + entries: BTreeMap, +} + +struct ProposedPluginPlan { + component_id: String, + kind: String, + dependencies: Vec, + identity_inputs: Vec, + semantic_inputs: BTreeMap, + caches: Vec, + external_artifacts: Vec, + runtime_resources: Vec, + command: Option, + mounted_commands: Vec, + outputs: Vec, + portability: AdapterPortability, + stale_reason: String, +} + +impl From for ProposedPluginPlan { + fn from(plan: AdapterPlan) -> Self { + Self { + component_id: plan.component_id, + kind: plan.kind, + dependencies: plan + .dependencies + .into_iter() + .map(WorkspaceEnvironmentDependency::build_requires) + .collect(), + identity_inputs: plan.identity_inputs, + semantic_inputs: plan.semantic_inputs, + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + command: Some(plan.command), + mounted_commands: Vec::new(), + outputs: plan.outputs, + portability: plan.portability, + stale_reason: plan.stale_reason, + } + } +} + +impl ProposedPluginPlan { + fn from_v2(plan: AdapterPlanV2) -> Result { + let metadata_only = !plan.external_artifacts.is_empty(); + if (!metadata_only && plan.actions.is_empty()) || plan.actions.len() > 9 { + return Err(Error::InvalidInput( + "adapter protocol-v2 filesystem plan must declare between one and nine actions" + .to_string(), + )); + } + if metadata_only + && (plan.kind != "external" + || !plan.actions.is_empty() + || !plan.caches.is_empty() + || !plan.outputs.is_empty()) + { + return Err(Error::InvalidInput( + "adapter protocol-v2 external-artifact plan must use kind `external` and cannot mix actions, caches, or filesystem outputs" + .to_string(), + )); + } + let mut command = None; + let mut mounted_commands = Vec::new(); + for action in plan.actions { + match action { + AdapterAction::Staging(candidate) => { + if command.replace(candidate).is_some() { + return Err(Error::InvalidInput( + "adapter protocol-v2 plan may declare at most one staging action" + .to_string(), + )); + } + } + AdapterAction::MountedInitialization(candidate) => mounted_commands.push(candidate), + } + } + if mounted_commands.len() > 8 { + return Err(Error::InvalidInput( + "adapter protocol-v2 plan may declare at most eight mounted actions".to_string(), + )); + } + let dependencies = plan + .dependencies + .into_iter() + .map(WorkspaceEnvironmentDependency::build_requires) + .chain(plan.dependency_edges.into_iter().map(|dependency| { + WorkspaceEnvironmentDependency { + component_id: dependency.component_id, + edge_type: match dependency.edge_type { + AdapterDependencyType::BuildRequires => { + WorkspaceEnvironmentEdgeType::BuildRequires + } + AdapterDependencyType::RuntimeRequires => { + WorkspaceEnvironmentEdgeType::RuntimeRequires + } + AdapterDependencyType::BindsAfter => { + WorkspaceEnvironmentEdgeType::BindsAfter + } + AdapterDependencyType::InvalidatesWith => { + WorkspaceEnvironmentEdgeType::InvalidatesWith + } + }, + } + })) + .collect(); + Ok(Self { + component_id: plan.component_id, + kind: plan.kind, + dependencies, + identity_inputs: plan.identity_inputs, + semantic_inputs: plan.semantic_inputs, + caches: plan.caches, + external_artifacts: plan.external_artifacts, + runtime_resources: plan.runtime_resources, + command, + mounted_commands, + outputs: plan.outputs, + portability: plan.portability, + stale_reason: plan.stale_reason, + }) + } +} + +struct BoundedRead { + bytes: Vec, + overflow: bool, +} + +struct PreparedEnvironmentPluginPackage { + package: AdapterPackageManifest, + executable_bytes: Vec, + executable_permissions: fs::Permissions, + executable_digest: String, + payload_material: Vec, + payload_digest: String, + signature: Option, +} + +#[derive(Clone, Debug)] +pub(super) struct InstalledEnvironmentPlugin { + pub(super) manifest: AdapterPackageManifest, + pub(super) distribution_digest: String, + pub(super) executable_digest: String, + pub(super) executable_path: PathBuf, + pub(super) publisher: Option, + pub(super) publisher_key_id: Option, + pub(super) trust: String, + pub(super) certification_tier: String, +} + +impl Trail { + pub fn trust_environment_adapter_publisher_key( + &self, + key_document: impl AsRef, + ) -> Result { + let key_document = fs::canonicalize(key_document)?; + let metadata = fs::symlink_metadata(&key_document)?; + if !metadata.is_file() + || metadata.file_type().is_symlink() + || metadata.len() > MAX_PUBLISHER_DOCUMENT_BYTES + { + return Err(Error::InvalidInput(format!( + "publisher key document `{}` must be a bounded regular file", + key_document.display() + ))); + } + let document: AdapterPublisherKey = toml::from_str(&fs::read_to_string(&key_document)?) + .map_err(|error| { + Error::InvalidInput(format!( + "cannot parse publisher key document `{}`: {error}", + key_document.display() + )) + })?; + let (publisher, public_key, key_id) = validate_publisher_key_document(&document)?; + self.append_publisher_trust_record(PublisherTrustRecord { + schema: TRUST_RECORD_SCHEMA.to_string(), + key_id: key_id.clone(), + publisher: publisher.clone(), + public_key, + action: "trust".to_string(), + recorded_at: now_ts(), + })?; + Ok(EnvironmentPublisherTrustMutationReport { + publisher: Some(publisher), + key_id, + action: "trust".to_string(), + }) + } + + pub fn remove_environment_adapter_publisher_key( + &self, + key_id: &str, + ) -> Result { + validate_publisher_key_id(key_id)?; + let current = self.latest_publisher_trust_record(key_id)?; + let publisher = current + .as_ref() + .filter(|record| record.action == "trust") + .map(|record| record.publisher.clone()); + let public_key = current + .as_ref() + .map(|record| record.public_key.clone()) + .unwrap_or_default(); + self.append_publisher_trust_record(PublisherTrustRecord { + schema: TRUST_RECORD_SCHEMA.to_string(), + key_id: key_id.to_string(), + publisher: publisher.clone().unwrap_or_default(), + public_key, + action: "remove".to_string(), + recorded_at: now_ts(), + })?; + Ok(EnvironmentPublisherTrustMutationReport { + publisher, + key_id: key_id.to_string(), + action: "remove".to_string(), + }) + } + + pub fn environment_adapter_publisher_trust(&self) -> Result { + let root = self.environment_plugin_store_root().join("trust"); + if !root.exists() { + return Ok(EnvironmentPublisherTrustReport { keys: Vec::new() }); + } + let mut keys = Vec::new(); + for directory in fs::read_dir(&root)? { + let directory = directory?; + if !directory.file_type()?.is_dir() { + return Err(Error::Corrupt(format!( + "publisher trust store contains non-directory `{}`", + directory.path().display() + ))); + } + let mut records = + fs::read_dir(directory.path())?.collect::, _>>()?; + records.retain(|entry| { + entry.file_type().is_ok_and(|kind| kind.is_file()) + && entry.path().extension().and_then(|value| value.to_str()) == Some("cbor") + }); + records.sort_by_key(|entry| entry.file_name()); + let Some(latest) = records.last() else { + continue; + }; + let record: PublisherTrustRecord = serde_cbor::from_slice(&fs::read(latest.path())?) + .map_err(|error| { + Error::Corrupt(format!( + "cannot decode publisher trust record `{}`: {error}", + latest.path().display() + )) + })?; + validate_publisher_trust_record(&record, &directory.file_name())?; + if record.action == "trust" { + keys.push(EnvironmentPublisherTrustEntryReport { + publisher: record.publisher, + key_id: record.key_id, + public_key: record.public_key, + trusted_at: record.recorded_at, + }); + } + } + keys.sort_by(|left, right| left.key_id.cmp(&right.key_id)); + Ok(EnvironmentPublisherTrustReport { keys }) + } + + pub fn inspect_environment_adapter_plugin_package( + &self, + package_directory: impl AsRef, + ) -> Result { + let prepared = prepare_environment_plugin_package(package_directory)?; + let distribution_material = adapter_package_distribution_material( + &prepared.payload_material, + prepared.signature.as_ref(), + )?; + Ok(EnvironmentPluginPackageInspectionReport { + canonical_identity: prepared.package.adapter.canonical_identity, + payload_digest: prepared.payload_digest, + executable_digest: prepared.executable_digest, + distribution_digest: format!("sha256:{}", sha256_hex(&distribution_material)), + signature_present: prepared.signature.is_some(), + publisher: prepared + .signature + .as_ref() + .map(|signature| signature.publisher.clone()), + publisher_key_id: prepared.signature.map(|signature| signature.key_id), + }) + } + + pub fn install_environment_adapter_plugin( + &self, + package_directory: impl AsRef, + ) -> Result { + let PreparedEnvironmentPluginPackage { + package, + executable_bytes, + executable_permissions, + executable_digest, + payload_material, + payload_digest, + signature, + } = prepare_environment_plugin_package(package_directory)?; + let (publisher, publisher_key_id, trust, certification_tier) = + if let Some(signature) = &signature { + let trusted = self.verify_adapter_package_signature(signature, &payload_digest)?; + ( + Some(trusted.publisher), + Some(trusted.key_id), + "publisher_signed".to_string(), + "publisher-authenticated-experimental".to_string(), + ) + } else { + ( + None, + None, + "local_unsigned".to_string(), + "local-experimental".to_string(), + ) + }; + let distribution_material = + adapter_package_distribution_material(&payload_material, signature.as_ref())?; + let distribution_hex = sha256_hex(&distribution_material); + let distribution_digest = format!("sha256:{distribution_hex}"); + let executable_file = package.executable.path.clone(); + let installed = InstalledPluginManifest { + package: package.clone(), + payload_digest, + distribution_digest: distribution_digest.clone(), + executable_digest: executable_digest.clone(), + executable_file: executable_file.clone(), + signature, + }; + + for metadata in super::workspace_environment::registered_environment_adapter_metadata() { + if metadata.canonical_identity == package.adapter.canonical_identity { + return Err(Error::InvalidInput(format!( + "adapter identity `{}` is reserved by a built-in adapter", + package.adapter.canonical_identity + ))); + } + if let Some(selector) = package + .adapter + .selectors + .iter() + .find(|selector| metadata.selectors.contains(&selector.as_str())) + { + return Err(Error::InvalidInput(format!( + "adapter selector `{selector}` is already claimed by `{}`", + metadata.canonical_identity + ))); + } + } + for existing in + self.installed_environment_plugins_except(Some(&package.adapter.canonical_identity))? + { + if let Some(selector) = package + .adapter + .selectors + .iter() + .find(|selector| existing.manifest.adapter.selectors.contains(selector)) + { + return Err(Error::InvalidInput(format!( + "adapter selector `{selector}` is already claimed by `{}`", + existing.manifest.adapter.canonical_identity + ))); + } + } + let previous = self + .latest_environment_plugin_registry_record(&package.adapter.canonical_identity)? + .filter(|record| record.action == "install") + .and_then(|record| record.distribution_digest); + + let store = self.environment_plugin_store_root(); + let packages = store.join("packages"); + fs::create_dir_all(&packages)?; + let destination = packages.join(&distribution_hex); + let publish_needed = if destination.exists() { + match self.verify_environment_plugin_package(&destination, &distribution_digest) { + Ok(_) => false, + Err(_) => { + let quarantine = packages.join("quarantine"); + fs::create_dir_all(&quarantine)?; + let quarantined = + quarantine.join(format!("{}-{:032}", distribution_hex, now_nanos())); + fs::rename(&destination, &quarantined)?; + sync_directory(&quarantine); + sync_directory(&packages); + true + } + } + } else { + true + }; + if publish_needed { + let staging = packages.join(format!(".install-{}-{}", std::process::id(), now_nanos())); + fs::create_dir(&staging)?; + let publish = (|| -> Result<()> { + let executable_destination = staging.join(&executable_file); + fs::write(&executable_destination, &executable_bytes)?; + fs::set_permissions(&executable_destination, executable_permissions.clone())?; + OpenOptions::new() + .read(true) + .open(&executable_destination)? + .sync_all()?; + write_file_atomic( + &staging.join(INSTALLED_MANIFEST_FILE), + &cbor(&installed)?, + true, + )?; + sync_directory(&staging); + match fs::rename(&staging, &destination) { + Ok(()) => { + sync_directory(&packages); + Ok(()) + } + Err(_error) if destination.is_dir() => { + let _ = fs::remove_dir_all(&staging); + self.verify_environment_plugin_package(&destination, &distribution_digest) + .map(|_| ()) + } + Err(error) => Err(error.into()), + } + })(); + if publish.is_err() { + let _ = fs::remove_dir_all(&staging); + } + publish?; + } + self.verify_environment_plugin_package(&destination, &distribution_digest)?; + + self.append_environment_plugin_registry_record(PluginRegistryRecord { + schema: REGISTRY_RECORD_SCHEMA.to_string(), + canonical_identity: package.adapter.canonical_identity.clone(), + action: "install".to_string(), + distribution_digest: Some(distribution_digest.clone()), + recorded_at: now_ts(), + })?; + + Ok(EnvironmentPluginInstallReport { + canonical_identity: package.adapter.canonical_identity, + distribution_digest, + executable_digest, + package_path: destination.to_string_lossy().into_owned(), + replaced_distribution_digest: previous, + publisher, + publisher_key_id, + trust, + certification_tier, + }) + } + + pub fn remove_environment_adapter_plugin( + &self, + canonical_identity: &str, + ) -> Result { + validate_plugin_identity(canonical_identity)?; + let existing = self + .latest_environment_plugin_registry_record(canonical_identity)? + .filter(|record| record.action == "install") + .and_then(|record| record.distribution_digest); + self.append_environment_plugin_registry_record(PluginRegistryRecord { + schema: REGISTRY_RECORD_SCHEMA.to_string(), + canonical_identity: canonical_identity.to_string(), + action: "remove".to_string(), + distribution_digest: None, + recorded_at: now_ts(), + })?; + Ok(EnvironmentPluginRemoveReport { + canonical_identity: canonical_identity.to_string(), + removed_distribution_digest: existing, + }) + } + + pub(super) fn installed_environment_plugins(&self) -> Result> { + self.installed_environment_plugins_except(None) + } + + fn installed_environment_plugins_except( + &self, + excluded_identity: Option<&str>, + ) -> Result> { + let registry = self.environment_plugin_store_root().join("registry"); + if !registry.exists() { + return Ok(Vec::new()); + } + let mut plugins = Vec::new(); + for identity_directory in fs::read_dir(®istry)? { + let identity_directory = identity_directory?; + if !identity_directory.file_type()?.is_dir() { + return Err(Error::Corrupt(format!( + "adapter registry contains non-directory `{}`", + identity_directory.path().display() + ))); + } + let mut records = fs::read_dir(identity_directory.path())? + .collect::, _>>()?; + records.retain(|entry| { + entry.file_type().is_ok_and(|kind| kind.is_file()) + && entry.path().extension().and_then(|value| value.to_str()) == Some("cbor") + }); + records.sort_by_key(|entry| entry.file_name()); + let Some(latest) = records.last() else { + continue; + }; + let record: PluginRegistryRecord = serde_cbor::from_slice(&fs::read(latest.path())?) + .map_err(|error| { + Error::Corrupt(format!( + "cannot decode adapter registry record `{}`: {error}", + latest.path().display() + )) + })?; + validate_registry_record(&record, &identity_directory.file_name())?; + if record.action == "remove" { + continue; + } + if excluded_identity == Some(record.canonical_identity.as_str()) { + continue; + } + let distribution_digest = record.distribution_digest.ok_or_else(|| { + Error::Corrupt(format!( + "adapter install record for `{}` has no distribution digest", + record.canonical_identity + )) + })?; + let digest_hex = distribution_digest + .strip_prefix("sha256:") + .ok_or_else(|| Error::Corrupt("plugin digest is not sha256".to_string()))?; + let package_directory = self + .environment_plugin_store_root() + .join("packages") + .join(digest_hex); + plugins.push( + self.verify_environment_plugin_package(&package_directory, &distribution_digest)?, + ); + } + plugins.sort_by(|left, right| { + left.manifest + .adapter + .canonical_identity + .cmp(&right.manifest.adapter.canonical_identity) + }); + validate_plugin_catalog(&plugins)?; + Ok(plugins) + } + + fn latest_environment_plugin_registry_record( + &self, + canonical_identity: &str, + ) -> Result> { + validate_plugin_identity(canonical_identity)?; + let directory = self + .environment_plugin_store_root() + .join("registry") + .join(sha256_hex(canonical_identity.as_bytes())); + if !directory.exists() { + return Ok(None); + } + let mut records = fs::read_dir(&directory)?.collect::, _>>()?; + records.retain(|entry| { + entry.file_type().is_ok_and(|kind| kind.is_file()) + && entry.path().extension().and_then(|value| value.to_str()) == Some("cbor") + }); + records.sort_by_key(|entry| entry.file_name()); + let Some(latest) = records.last() else { + return Ok(None); + }; + let record: PluginRegistryRecord = serde_cbor::from_slice(&fs::read(latest.path())?) + .map_err(|error| { + Error::Corrupt(format!( + "cannot decode adapter registry record `{}`: {error}", + latest.path().display() + )) + })?; + validate_registry_record(&record, directory.file_name().unwrap_or_default())?; + if record.canonical_identity != canonical_identity { + return Err(Error::Corrupt(format!( + "adapter registry record for `{canonical_identity}` contains identity `{}`", + record.canonical_identity + ))); + } + Ok(Some(record)) + } + + fn environment_plugin_store_root(&self) -> PathBuf { + self.db_dir + .join("adapter-plugins") + .join(PLUGIN_STORE_VERSION) + } + + fn latest_publisher_trust_record(&self, key_id: &str) -> Result> { + validate_publisher_key_id(key_id)?; + let directory = self + .environment_plugin_store_root() + .join("trust") + .join(key_id.trim_start_matches("sha256:")); + if !directory.exists() { + return Ok(None); + } + let mut records = fs::read_dir(&directory)?.collect::, _>>()?; + records.retain(|entry| { + entry.file_type().is_ok_and(|kind| kind.is_file()) + && entry.path().extension().and_then(|value| value.to_str()) == Some("cbor") + }); + records.sort_by_key(|entry| entry.file_name()); + let Some(latest) = records.last() else { + return Ok(None); + }; + let record: PublisherTrustRecord = serde_cbor::from_slice(&fs::read(latest.path())?) + .map_err(|error| { + Error::Corrupt(format!( + "cannot decode publisher trust record `{}`: {error}", + latest.path().display() + )) + })?; + validate_publisher_trust_record(&record, directory.file_name().unwrap_or_default())?; + if record.key_id != key_id { + return Err(Error::Corrupt(format!( + "publisher trust record for `{key_id}` contains key `{}`", + record.key_id + ))); + } + Ok(Some(record)) + } + + fn append_publisher_trust_record(&self, record: PublisherTrustRecord) -> Result<()> { + validate_publisher_trust_record( + &record, + OsStr::new(record.key_id.trim_start_matches("sha256:")), + )?; + let directory = self + .environment_plugin_store_root() + .join("trust") + .join(record.key_id.trim_start_matches("sha256:")); + fs::create_dir_all(&directory)?; + let path = directory.join(format!( + "{:032}-{:010}-{}.cbor", + now_nanos(), + std::process::id(), + record.action + )); + write_file_atomic(&path, &cbor(&record)?, true)?; + sync_directory(&directory); + Ok(()) + } + + fn trusted_publisher_key(&self, key_id: &str) -> Result { + self.latest_publisher_trust_record(key_id)? + .filter(|record| record.action == "trust") + .ok_or_else(|| { + Error::InvalidInput(format!( + "adapter publisher key `{key_id}` is not trusted in this workspace" + )) + }) + } + + fn verify_adapter_package_signature( + &self, + signature: &AdapterPackageSignature, + payload_digest: &str, + ) -> Result { + validate_adapter_package_signature(signature)?; + if signature.payload_digest != payload_digest { + return Err(Error::InvalidInput(format!( + "adapter signature authenticates `{}` but package payload is `{payload_digest}`", + signature.payload_digest + ))); + } + let trusted = self.trusted_publisher_key(&signature.key_id)?; + if trusted.publisher != signature.publisher { + return Err(Error::InvalidInput(format!( + "adapter signature publisher `{}` does not match trusted key owner `{}`", + signature.publisher, trusted.publisher + ))); + } + let public_key = decode_fixed_hex::<32>(&trusted.public_key, "publisher public key")?; + let signature_bytes = decode_fixed_hex::<64>(&signature.signature, "adapter signature")?; + let verifying_key = VerifyingKey::from_bytes(&public_key).map_err(|error| { + Error::InvalidInput(format!("publisher public key is invalid: {error}")) + })?; + let signature_value = Signature::from_bytes(&signature_bytes); + verifying_key + .verify_strict( + &adapter_package_signature_message(payload_digest), + &signature_value, + ) + .map_err(|_| { + Error::InvalidInput(format!( + "adapter package signature from `{}` is invalid", + signature.publisher + )) + })?; + Ok(trusted) + } + + fn append_environment_plugin_registry_record( + &self, + record: PluginRegistryRecord, + ) -> Result<()> { + let identity_hash = sha256_hex(record.canonical_identity.as_bytes()); + let directory = self + .environment_plugin_store_root() + .join("registry") + .join(identity_hash); + fs::create_dir_all(&directory)?; + let path = directory.join(format!( + "{:032}-{:010}-{}.cbor", + now_nanos(), + std::process::id(), + record.action + )); + write_file_atomic(&path, &cbor(&record)?, true)?; + sync_directory(&directory); + Ok(()) + } + + fn verify_environment_plugin_package( + &self, + package_directory: &Path, + expected_distribution_digest: &str, + ) -> Result { + let store = fs::canonicalize(self.environment_plugin_store_root())?; + let package_directory = fs::canonicalize(package_directory).map_err(|error| { + Error::Corrupt(format!( + "installed adapter package `{}` is unavailable: {error}", + package_directory.display() + )) + })?; + if !package_directory.starts_with(&store) { + return Err(Error::Corrupt(format!( + "installed adapter package `{}` escapes its store", + package_directory.display() + ))); + } + let manifest_path = package_directory.join(INSTALLED_MANIFEST_FILE); + let metadata = fs::symlink_metadata(&manifest_path)?; + if !metadata.is_file() + || metadata.file_type().is_symlink() + || metadata.len() > MAX_PACKAGE_MANIFEST_BYTES + { + return Err(Error::Corrupt(format!( + "installed adapter manifest `{}` is not a bounded regular file", + manifest_path.display() + ))); + } + let mut installed: InstalledPluginManifest = + serde_cbor::from_slice(&fs::read(&manifest_path)?).map_err(|error| { + Error::Corrupt(format!( + "cannot decode installed adapter manifest `{}`: {error}", + manifest_path.display() + )) + })?; + canonicalize_and_validate_package(&mut installed.package)?; + if installed.executable_file != installed.package.executable.path { + return Err(Error::Corrupt( + "installed adapter executable file disagrees with its package manifest".to_string(), + )); + } + if installed.distribution_digest != expected_distribution_digest { + return Err(Error::Corrupt(format!( + "installed adapter package digest `{}` does not match registry digest `{expected_distribution_digest}`", + installed.distribution_digest + ))); + } + let executable_path = package_directory.join(&installed.executable_file); + let executable_metadata = fs::symlink_metadata(&executable_path)?; + if !executable_metadata.is_file() + || executable_metadata.file_type().is_symlink() + || executable_metadata.len() > MAX_PLUGIN_EXECUTABLE_BYTES + { + return Err(Error::Corrupt(format!( + "installed adapter executable `{}` is not a bounded regular file", + executable_path.display() + ))); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if executable_metadata.permissions().mode() & 0o111 == 0 { + return Err(Error::Corrupt(format!( + "installed adapter executable `{}` lost its executable mode", + executable_path.display() + ))); + } + } + let executable_digest = format!("sha256:{}", sha256_hex(&fs::read(&executable_path)?)); + if executable_digest != installed.executable_digest + || executable_digest != installed.package.executable.sha256 + { + return Err(Error::Corrupt(format!( + "installed adapter executable `{}` failed digest verification", + executable_path.display() + ))); + } + let executable_bytes = fs::read(&executable_path)?; + let (payload_material, payload_digest) = + adapter_package_payload(&installed.package, &executable_bytes)?; + if !installed.payload_digest.is_empty() && installed.payload_digest != payload_digest { + return Err(Error::Corrupt(format!( + "installed adapter package `{}` failed payload verification", + package_directory.display() + ))); + } + let (publisher, publisher_key_id, trust, certification_tier) = if let Some(signature) = + &installed.signature + { + let trusted = self + .verify_adapter_package_signature(signature, &payload_digest) + .map_err(|error| { + Error::Corrupt(format!( + "installed adapter package `{}` no longer has a valid trusted publisher: {error}", + package_directory.display() + )) + })?; + ( + Some(trusted.publisher), + Some(trusted.key_id), + "publisher_signed".to_string(), + "publisher-authenticated-experimental".to_string(), + ) + } else { + ( + None, + None, + "local_unsigned".to_string(), + "local-experimental".to_string(), + ) + }; + let distribution_material = + adapter_package_distribution_material(&payload_material, installed.signature.as_ref())?; + let actual_distribution = format!("sha256:{}", sha256_hex(&distribution_material)); + if actual_distribution != expected_distribution_digest { + return Err(Error::Corrupt(format!( + "installed adapter package `{}` failed distribution verification", + package_directory.display() + ))); + } + Ok(InstalledEnvironmentPlugin { + manifest: installed.package, + distribution_digest: installed.distribution_digest, + executable_digest, + executable_path, + publisher, + publisher_key_id, + trust, + certification_tier, + }) + } +} + +impl Trail { + pub(super) fn environment_plugin_for_selector( + &self, + selector: &str, + ) -> Result> { + Ok(self + .installed_environment_plugins()? + .into_iter() + .find(|plugin| { + plugin + .manifest + .adapter + .selectors + .iter() + .any(|candidate| candidate == selector) + })) + } + + pub(super) fn discover_environment_plugin_component( + &self, + plugin: &InstalledEnvironmentPlugin, + source_root: &ObjectId, + component_root: &str, + ) -> Result> { + ensure_environment_plugin_supports_current_host(plugin)?; + let protocol = selected_environment_plugin_protocol(plugin)?; + let selected = self.select_environment_plugin_files(plugin, source_root, component_root)?; + let request_id = plugin_request_id(plugin, source_root, component_root, "discover", None); + let request = AdapterRequest { + protocol: protocol.to_string(), + request_id: request_id.clone(), + adapter_identity: plugin.manifest.adapter.canonical_identity.clone(), + distribution_digest: plugin.distribution_digest.clone(), + host: AdapterHost { + operating_system: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + }, + source_root: source_root.0.clone(), + operation: AdapterOperation::Discover { + component_root: component_root.to_string(), + files: selected.protocol_files, + }, + }; + match self.invoke_environment_plugin(plugin, &request)? { + AdapterResult::Discovered { component: None } => Ok(None), + AdapterResult::Discovered { + component: Some(component), + } => { + validate_discovered_plugin_component(plugin, &component)?; + Ok(Some(EnvironmentDiscoveredComponentReport { + component_id: component.component_id, + component_root: component_root.to_string(), + kind: component.kind, + adapter_identity: plugin.manifest.adapter.canonical_identity.clone(), + })) + } + _ => Err(Error::InvalidInput(format!( + "adapter `{}` returned a non-discovery response to request `{request_id}`", + plugin.manifest.adapter.canonical_identity + ))), + } + } + + pub(super) fn plan_environment_plugin_component( + &self, + plugin: &InstalledEnvironmentPlugin, + source_root: &ObjectId, + component_root: &str, + component_id: &str, + ) -> Result { + ensure_environment_plugin_supports_current_host(plugin)?; + let protocol = selected_environment_plugin_protocol(plugin)?; + let selected = self.select_environment_plugin_files(plugin, source_root, component_root)?; + let request_id = plugin_request_id( + plugin, + source_root, + component_root, + "plan", + Some(component_id), + ); + let request = AdapterRequest { + protocol: protocol.to_string(), + request_id: request_id.clone(), + adapter_identity: plugin.manifest.adapter.canonical_identity.clone(), + distribution_digest: plugin.distribution_digest.clone(), + host: AdapterHost { + operating_system: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + }, + source_root: source_root.0.clone(), + operation: AdapterOperation::Plan { + component_id: component_id.to_string(), + component_root: component_root.to_string(), + files: selected.protocol_files, + }, + }; + match (protocol, self.invoke_environment_plugin(plugin, &request)?) { + (PROTOCOL_V1, AdapterResult::Planned { plan }) => self + .normalize_environment_plugin_plan( + plugin, + component_root, + component_id, + selected.entries, + protocol, + plan.into(), + ), + (PROTOCOL_V2, AdapterResult::PlannedV2 { plan }) => self + .normalize_environment_plugin_plan( + plugin, + component_root, + component_id, + selected.entries, + protocol, + ProposedPluginPlan::from_v2(plan)?, + ), + _ => Err(Error::InvalidInput(format!( + "adapter `{}` returned a non-plan response to request `{request_id}`", + plugin.manifest.adapter.canonical_identity + ))), + } + } + + fn select_environment_plugin_files( + &self, + plugin: &InstalledEnvironmentPlugin, + source_root: &ObjectId, + component_root: &str, + ) -> Result { + let component_root = normalize_plugin_component_root(component_root)?; + let patterns = compile_plugin_patterns(&plugin.manifest.permissions)?; + let prefix = if component_root.is_empty() { + String::new() + } else { + format!("{component_root}/") + }; + let mut entries = BTreeMap::new(); + let mut total_bytes = 0u64; + self.for_each_root_file_chunk(source_root, 1024, |chunk| { + for (repository_path, entry) in chunk { + let relative = if prefix.is_empty() { + repository_path.as_str() + } else if let Some(relative) = repository_path.strip_prefix(&prefix) { + relative + } else { + continue; + }; + if !patterns.is_match(relative) { + continue; + } + if entries.len() >= plugin.manifest.permissions.max_input_files as usize { + return Err(Error::InvalidInput(format!( + "adapter `{}` input selection exceeds {} files", + plugin.manifest.adapter.canonical_identity, + plugin.manifest.permissions.max_input_files + ))); + } + total_bytes = total_bytes.checked_add(entry.size_bytes).ok_or_else(|| { + Error::InvalidInput("adapter input byte count overflowed".to_string()) + })?; + if total_bytes > plugin.manifest.permissions.max_input_bytes { + return Err(Error::InvalidInput(format!( + "adapter `{}` input selection exceeds {} bytes", + plugin.manifest.adapter.canonical_identity, + plugin.manifest.permissions.max_input_bytes + ))); + } + entries.insert(relative.to_string(), (repository_path, entry)); + } + Ok(()) + })?; + let mut protocol_files = Vec::with_capacity(entries.len()); + for (relative, (_, entry)) in &entries { + let content = fs::read(self.project_entry_file(entry)?)?; + if content.len() as u64 != entry.size_bytes { + return Err(Error::Corrupt(format!( + "pinned adapter input `{relative}` has inconsistent size" + ))); + } + protocol_files.push(PinnedFile { + path: relative.clone(), + content_hash: entry.content_hash.clone(), + executable: entry.executable, + content, + }); + } + Ok(SelectedPluginFiles { + protocol_files, + entries, + }) + } + + fn invoke_environment_plugin( + &self, + plugin: &InstalledEnvironmentPlugin, + request: &AdapterRequest, + ) -> Result { + let mut request_bytes = Vec::new(); + write_frame(&mut request_bytes, request, MAX_FRAME_BYTES) + .map_err(|error| Error::Serialization(error.to_string()))?; + #[cfg(target_os = "windows")] + let sandbox_parent = std::env::temp_dir(); + #[cfg(not(target_os = "windows"))] + let sandbox_parent = PathBuf::from("/tmp"); + if !sandbox_parent.is_dir() { + return Err(Error::InvalidInput( + "adapter plugins require an available host temporary directory".to_string(), + )); + } + let sandbox = tempfile::Builder::new() + .prefix("trail-adapter-plugin-") + .tempdir_in(sandbox_parent)?; + let sandbox_root = fs::canonicalize(sandbox.path())?; + let executable_name = plugin + .executable_path + .file_name() + .ok_or_else(|| Error::Corrupt("installed adapter has no file name".to_string()))?; + let executable = sandbox_root.join(executable_name); + fs::copy(&plugin.executable_path, &executable)?; + fs::set_permissions( + &executable, + fs::metadata(&plugin.executable_path)?.permissions(), + )?; + let executable = fs::canonicalize(executable)?; + let copied_digest = format!("sha256:{}", sha256_hex(&fs::read(&executable)?)); + if copied_digest != plugin.executable_digest { + return Err(Error::Corrupt(format!( + "staged adapter executable for `{}` failed digest verification", + plugin.manifest.adapter.canonical_identity + ))); + } + let home = sandbox_root.join("home"); + let temporary = sandbox_root.join("tmp"); + fs::create_dir(&home)?; + fs::create_dir(&temporary)?; + let home = fs::canonicalize(home)?; + let temporary = fs::canonicalize(temporary)?; + let (launcher, launcher_args) = + sandboxed_plugin_launcher(&sandbox_root, &home, &temporary, &executable)?; + + let mut command = Command::new(launcher); + command + .args(launcher_args) + .current_dir(&sandbox_root) + .env_clear() + .env("HOME", &home) + .env("TMPDIR", &temporary) + .env("TMP", &temporary) + .env("TEMP", &temporary) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + #[cfg(windows)] + if let Some(system_root) = std::env::var_os("SystemRoot") { + command.env("SystemRoot", system_root); + } + configure_plugin_resource_limits(&mut command, plugin.manifest.permissions.timeout_ms)?; + let mut child = command.spawn().map_err(|error| { + Error::InvalidInput(format!( + "cannot launch adapter plugin `{}`: {error}", + plugin.manifest.adapter.canonical_identity + )) + })?; + let mut stdin = child.stdin.take().ok_or_else(|| { + Error::Corrupt("adapter plugin stdin pipe was unavailable".to_string()) + })?; + let stdout = child.stdout.take().ok_or_else(|| { + Error::Corrupt("adapter plugin stdout pipe was unavailable".to_string()) + })?; + let stderr = child.stderr.take().ok_or_else(|| { + Error::Corrupt("adapter plugin stderr pipe was unavailable".to_string()) + })?; + let writer = thread::spawn(move || -> std::io::Result<()> { + stdin.write_all(&request_bytes)?; + stdin.flush() + }); + let response_limit = usize::try_from(plugin.manifest.permissions.max_response_bytes) + .unwrap_or(MAX_FRAME_BYTES) + .min(MAX_FRAME_BYTES); + let stdout_reader = spawn_bounded_reader(stdout, response_limit.saturating_add(4)); + let stderr_reader = spawn_bounded_reader(stderr, 64 * 1024); + let status = wait_for_plugin( + &mut child, + Duration::from_millis(plugin.manifest.permissions.timeout_ms), + ); + let writer_result = writer + .join() + .map_err(|_| Error::Corrupt("adapter plugin stdin writer panicked".to_string()))?; + let stdout = stdout_reader + .join() + .map_err(|_| Error::Corrupt("adapter plugin stdout reader panicked".to_string()))??; + let stderr = stderr_reader + .join() + .map_err(|_| Error::Corrupt("adapter plugin stderr reader panicked".to_string()))??; + let status = status?; + writer_result.map_err(Error::Io)?; + if stdout.overflow { + return Err(Error::InvalidInput(format!( + "adapter `{}` response exceeded {response_limit} bytes", + plugin.manifest.adapter.canonical_identity + ))); + } + if stderr.overflow { + return Err(Error::InvalidInput(format!( + "adapter `{}` diagnostic output exceeded 65536 bytes", + plugin.manifest.adapter.canonical_identity + ))); + } + if !status.success() { + let diagnostic = redact_sensitive_text(&String::from_utf8_lossy(&stderr.bytes)); + return Err(Error::InvalidInput(format!( + "adapter `{}` exited with {status}: {}", + plugin.manifest.adapter.canonical_identity, + diagnostic.trim() + ))); + } + let mut cursor = Cursor::new(&stdout.bytes); + let response: AdapterResponse = read_frame(&mut cursor, response_limit) + .map_err(|error| Error::InvalidInput(format!("invalid adapter response: {error}")))?; + if cursor.position() != stdout.bytes.len() as u64 { + return Err(Error::InvalidInput(format!( + "adapter `{}` wrote trailing bytes after its response frame", + plugin.manifest.adapter.canonical_identity + ))); + } + if response.protocol != request.protocol || response.request_id != request.request_id { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned a mismatched protocol or request ID", + plugin.manifest.adapter.canonical_identity + ))); + } + match response.result { + AdapterResult::Error { code, message } => Err(Error::InvalidInput(format!( + "adapter `{}` rejected request with {}: {}", + plugin.manifest.adapter.canonical_identity, + redact_sensitive_text(&code), + redact_sensitive_text(&message) + ))), + result => Ok(result), + } + } + + fn normalize_environment_plugin_caches( + &self, + plugin: &InstalledEnvironmentPlugin, + protocol: &str, + caches: &[AdapterCache], + has_staging_action: bool, + ) -> Result<(Vec, BTreeMap)> { + if caches.is_empty() { + return Ok((Vec::new(), BTreeMap::new())); + } + if protocol != PROTOCOL_V2 || !has_staging_action { + return Err(Error::InvalidInput(format!( + "adapter `{}` may use host caches only from a protocol-v2 staging action", + plugin.manifest.adapter.canonical_identity + ))); + } + if caches.len() > 16 { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned more than sixteen cache declarations", + plugin.manifest.adapter.canonical_identity + ))); + } + + let mut proposed = caches.to_vec(); + proposed.sort_by(|left, right| left.name.cmp(&right.name)); + let mut names = BTreeSet::new(); + let mut environment = BTreeMap::new(); + let mut normalized = Vec::with_capacity(proposed.len()); + for cache in proposed { + if !names.insert(cache.name.clone()) { + return Err(Error::InvalidInput(format!( + "adapter `{}` repeats cache declaration `{}`", + plugin.manifest.adapter.canonical_identity, cache.name + ))); + } + if cache.access != AdapterCacheAccess::HostExclusive { + return Err(Error::InvalidInput(format!( + "adapter `{}` cache `{}` requests tool-concurrent access without independent cache certification; use host_exclusive", + plugin.manifest.adapter.canonical_identity, cache.name + ))); + } + if cache.compatibility.len() > 24 + || cache + .compatibility + .keys() + .any(|name| name.starts_with("trail.")) + { + return Err(Error::InvalidInput(format!( + "adapter `{}` cache `{}` has too many or host-reserved compatibility dimensions", + plugin.manifest.adapter.canonical_identity, cache.name + ))); + } + if cache.environment.is_empty() || cache.environment.len() > 16 { + return Err(Error::InvalidInput(format!( + "adapter `{}` cache `{}` must declare between one and sixteen environment bindings", + plugin.manifest.adapter.canonical_identity, cache.name + ))); + } + + let mut compatibility = cache.compatibility; + compatibility.insert( + "trail.adapter_distribution".to_string(), + plugin.distribution_digest.clone(), + ); + compatibility.insert("trail.protocol".to_string(), protocol.to_string()); + compatibility.insert( + "trail.operating_system".to_string(), + std::env::consts::OS.to_string(), + ); + compatibility.insert( + "trail.architecture".to_string(), + std::env::consts::ARCH.to_string(), + ); + let protocol = match cache.protocol { + AdapterCacheProtocol::ContentStore => { + WorkspaceEnvironmentCacheProtocol::ContentStore + } + AdapterCacheProtocol::CompilerCache => { + WorkspaceEnvironmentCacheProtocol::CompilerCache + } + AdapterCacheProtocol::LockedIndex => WorkspaceEnvironmentCacheProtocol::LockedIndex, + }; + let declared = self.declare_workspace_environment_cache( + &plugin.manifest.adapter.canonical_identity, + &cache.name, + protocol, + WorkspaceEnvironmentCacheAccess::HostExclusive, + compatibility, + )?; + for (name, subpath) in cache.environment { + super::workspace_recipe::validate_recipe_environment( + &name, + &subpath, + &plugin.manifest.adapter.canonical_identity, + )?; + let relative = normalize_plugin_path_allow_root(&subpath)?; + let value = if relative.is_empty() { + declared.storage_path.clone() + } else { + declared.storage_path.join(&relative) + }; + if environment + .insert(name.clone(), value.to_string_lossy().into_owned()) + .is_some() + { + return Err(Error::InvalidInput(format!( + "adapter `{}` binds cache environment variable `{name}` more than once", + plugin.manifest.adapter.canonical_identity + ))); + } + } + normalized.push(declared); + } + Ok((normalized, environment)) + } + + fn normalize_environment_plugin_plan( + &self, + plugin: &InstalledEnvironmentPlugin, + component_root: &str, + expected_component_id: &str, + entries: BTreeMap, + protocol: &str, + plan: ProposedPluginPlan, + ) -> Result { + validate_plugin_component_id(&plan.component_id)?; + if plan.component_id != expected_component_id { + return Err(Error::InvalidInput(format!( + "adapter `{}` planned component `{}` instead of requested `{expected_component_id}`", + plugin.manifest.adapter.canonical_identity, plan.component_id + ))); + } + if plan.kind != plugin.manifest.adapter.kind { + return Err(Error::InvalidInput(format!( + "adapter `{}` planned kind `{}` but its package declares `{}`", + plugin.manifest.adapter.canonical_identity, plan.kind, plugin.manifest.adapter.kind + ))); + } + let declared_inputs = plan + .identity_inputs + .iter() + .map(|path| normalize_relative_path(path)) + .collect::>>()?; + if declared_inputs.len() != plan.identity_inputs.len() { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned duplicate identity inputs", + plugin.manifest.adapter.canonical_identity + ))); + } + let supplied_inputs = entries.keys().cloned().collect::>(); + if declared_inputs != supplied_inputs { + return Err(Error::InvalidInput(format!( + "adapter `{}` must key every pinned file it was allowed to inspect", + plugin.manifest.adapter.canonical_identity + ))); + } + if supplied_inputs.is_empty() { + return Err(Error::InvalidInput(format!( + "adapter `{}` cannot plan a component without pinned identity inputs", + plugin.manifest.adapter.canonical_identity + ))); + } + if plan.semantic_inputs.len() > 256 { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned more than 256 semantic inputs", + plugin.manifest.adapter.canonical_identity + ))); + } + let mut semantic_bytes = 0usize; + for (name, value) in &plan.semantic_inputs { + semantic_bytes = semantic_bytes + .checked_add(name.len() + value.len()) + .ok_or_else(|| Error::InvalidInput("semantic input size overflowed".to_string()))?; + if name.is_empty() + || name.len() > 256 + || !name.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '_' | '-' | ':') + }) + || value.len() > 128 * 1024 + || value.contains('\0') + || contains_sensitive_text(name) + || contains_sensitive_text(value) + { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned invalid or sensitive semantic input `{name}`", + plugin.manifest.adapter.canonical_identity + ))); + } + } + if semantic_bytes > 1024 * 1024 { + return Err(Error::InvalidInput(format!( + "adapter `{}` semantic inputs exceed one MiB", + plugin.manifest.adapter.canonical_identity + ))); + } + let component_root = normalize_plugin_component_root(component_root)?; + match protocol { + PROTOCOL_V1 if plan.command.is_none() || !plan.mounted_commands.is_empty() => { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned protocol-v2 actions in a v1 plan", + plugin.manifest.adapter.canonical_identity + ))); + } + PROTOCOL_V1 => {} + PROTOCOL_V2 + if plan.command.is_none() + && plan.mounted_commands.is_empty() + && plan.external_artifacts.is_empty() + && plan.runtime_resources.is_empty() => + { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned a v2 filesystem plan without an action", + plugin.manifest.adapter.canonical_identity + ))); + } + PROTOCOL_V2 if plan.mounted_commands.len() > 8 => { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned more than eight mounted initialization actions", + plugin.manifest.adapter.canonical_identity + ))); + } + PROTOCOL_V2 => {} + other => { + return Err(Error::InvalidInput(format!( + "adapter `{}` used unsupported protocol `{other}`", + plugin.manifest.adapter.canonical_identity + ))); + } + } + + let cache_contract = serde_json::to_string(&plan.caches)?; + let (normalized_caches, cache_environment) = self.normalize_environment_plugin_caches( + plugin, + protocol, + &plan.caches, + plan.command.is_some(), + )?; + let cache_names = normalized_caches + .iter() + .map(|cache| cache.name.clone()) + .collect::>(); + + let normalize_command = |command: &trail_environment_adapter_sdk::AdapterCommand, + mounted: bool| + -> Result<(WorkspaceEnvironmentCommand, String)> { + if command.program.contains('/') + || command.program.contains('\\') + || super::workspace_recipe::is_shell_program(&command.program) + || command.args.len() > 4096 + || command.args.iter().any(|argument| { + argument.len() > 128 * 1024 + || argument.contains('\0') + || contains_sensitive_text(argument) + }) + { + return Err(Error::InvalidInput(format!( + "adapter `{}` proposed a shell, path-qualified executable, or invalid argument", + plugin.manifest.adapter.canonical_identity + ))); + } + let tool = + super::workspace_environment::resolve_workspace_tool_executable(&command.program)?; + super::workspace_recipe::validate_recipe_tool_path( + self, + &tool.path, + &plan.component_id, + )?; + for (name, value) in &command.environment { + super::workspace_recipe::validate_recipe_environment( + name, + value, + &plan.component_id, + )?; + } + let mut environment = command.environment.clone(); + if !mounted { + for (name, value) in &cache_environment { + if environment.insert(name.clone(), value.clone()).is_some() { + return Err(Error::InvalidInput(format!( + "adapter `{}` binds cache environment variable `{name}` more than once", + plugin.manifest.adapter.canonical_identity + ))); + } + } + } + let working_relative = normalize_plugin_path_allow_root(&command.working_directory)?; + let working_repository_path = join_plugin_path(&component_root, &working_relative); + let working_directory = if mounted { + working_repository_path + } else if working_repository_path.is_empty() { + "project".to_string() + } else { + format!("project/{working_repository_path}") + }; + Ok(( + WorkspaceEnvironmentCommand { + program: command.program.clone(), + resolved_program: tool.path, + executable_identity: tool.identity, + args: command.args.clone(), + working_directory, + environment, + remove_environment: Vec::new(), + cache_names: if mounted { + Vec::new() + } else { + cache_names.clone() + }, + }, + working_relative, + )) + }; + let normalized_command = plan + .command + .as_ref() + .map(|command| normalize_command(command, false)) + .transpose()?; + let normalized_mounted_commands = plan + .mounted_commands + .iter() + .map(|command| normalize_command(command, true)) + .collect::>>()?; + let mut external_artifacts = plan + .external_artifacts + .iter() + .map(|artifact| WorkspaceEnvironmentExternalArtifact { + name: artifact.name.clone(), + artifact_type: artifact.artifact_type.clone(), + provider: artifact.provider.clone(), + reference: artifact.reference.clone(), + digest: artifact.digest.clone(), + platform: artifact.platform.clone(), + cleanup_owner: artifact.cleanup_owner.clone(), + }) + .collect::>(); + external_artifacts.sort_by(|left, right| left.name.cmp(&right.name)); + let mut runtime_resources = plan + .runtime_resources + .iter() + .map(|resource| WorkspaceEnvironmentRuntimeResource { + name: resource.name.clone(), + runtime_type: resource.runtime_type.clone(), + provider: resource.provider.clone(), + artifact_name: resource.artifact_name.clone(), + container_port: resource.container_port, + protocol: resource.protocol.clone(), + health_type: resource.health_type.clone(), + health_timeout_ms: resource.health_timeout_ms, + restart_policy: resource.restart_policy.clone(), + cleanup_owner: resource.cleanup_owner.clone(), + volume_target: resource.volume_target.clone(), + secrets: resource + .secrets + .iter() + .map(|secret| WorkspaceEnvironmentSecretReference { + name: secret.name.clone(), + provider: secret.provider.clone(), + reference: secret.reference.clone(), + version: secret.version.clone(), + purpose: secret.purpose.clone(), + injection: secret.injection.clone(), + target: secret.target.clone(), + environment: secret.environment.clone(), + required: secret.required, + }) + .collect(), + }) + .collect::>(); + runtime_resources.sort_by(|left, right| left.name.cmp(&right.name)); + let working_repository_path = normalized_command + .as_ref() + .map(|(_, working_relative)| join_plugin_path(&component_root, working_relative)) + .unwrap_or_else(|| component_root.clone()); + if plan.external_artifacts.is_empty() + && (plan.outputs.is_empty() || plan.outputs.len() > 32) + { + return Err(Error::InvalidInput(format!( + "adapter `{}` must propose between one and 32 outputs", + plugin.manifest.adapter.canonical_identity + ))); + } + if !plan.external_artifacts.is_empty() && !plan.outputs.is_empty() { + return Err(Error::InvalidInput(format!( + "adapter `{}` cannot mix external artifacts with filesystem outputs", + plugin.manifest.adapter.canonical_identity + ))); + } + let mut output_names = BTreeSet::new(); + let mut output_paths = Vec::<(String, String)>::new(); + let mut mount_paths = Vec::<(String, String)>::new(); + let mut outputs = Vec::with_capacity(plan.outputs.len()); + for output in &plan.outputs { + if !output.create_if_missing { + return Err(Error::InvalidInput(format!( + "adapter `{}` must allow Trail to create output `{}` before sandbox execution", + plugin.manifest.adapter.canonical_identity, output.name + ))); + } + validate_plugin_output_name(&output.name)?; + if !output_names.insert(output.name.clone()) { + return Err(Error::InvalidInput(format!( + "adapter `{}` proposed duplicate output `{}`", + plugin.manifest.adapter.canonical_identity, output.name + ))); + } + let source = normalize_relative_path(&output.source)?; + let target = normalize_relative_path(&output.target)?; + let output_repository_path = join_plugin_path(&working_repository_path, &source); + let mount_path = join_plugin_path(&component_root, &target); + for (other_name, other_path) in &output_paths { + if plugin_paths_overlap(&output_repository_path, other_path) { + return Err(Error::InvalidInput(format!( + "adapter `{}` output `{}` overlaps output `{other_name}`", + plugin.manifest.adapter.canonical_identity, output.name + ))); + } + } + for (other_name, other_path) in &mount_paths { + if plugin_paths_overlap(&mount_path, other_path) { + return Err(Error::InvalidInput(format!( + "adapter `{}` mount `{}` overlaps output `{other_name}`", + plugin.manifest.adapter.canonical_identity, output.name + ))); + } + } + for (relative, (repository_path, _)) in &entries { + if plugin_paths_overlap(&output_repository_path, repository_path) + || plugin_paths_overlap(&mount_path, repository_path) + { + return Err(Error::InvalidInput(format!( + "adapter `{}` output `{}` overlaps pinned input `{relative}`", + plugin.manifest.adapter.canonical_identity, output.name + ))); + } + } + output_paths.push((output.name.clone(), output_repository_path.clone())); + mount_paths.push((output.name.clone(), mount_path.clone())); + outputs.push(WorkspaceEnvironmentOutput { + name: output.name.clone(), + output_path: format!("project/{output_repository_path}"), + mount_path, + policy: match output.policy { + AdapterOutputPolicy::ImmutableSeedPrivate => { + WorkspaceEnvironmentOutputPolicy::ImmutableSeedPrivate + } + AdapterOutputPolicy::WritablePrivate => { + WorkspaceEnvironmentOutputPolicy::WritablePrivate + } + }, + create_if_missing: output.create_if_missing, + }); + } + if !normalized_mounted_commands.is_empty() + && outputs + .iter() + .any(|output| output.policy != WorkspaceEnvironmentOutputPolicy::WritablePrivate) + { + return Err(Error::InvalidInput(format!( + "adapter `{}` may use mounted initialization only with writable-private outputs", + plugin.manifest.adapter.canonical_identity + ))); + } + let output_contract = serde_json::to_string( + &outputs + .iter() + .map(|output| { + ( + &output.name, + &output.output_path, + &output.mount_path, + output.policy.as_str(), + ) + }) + .collect::>(), + )?; + let external_artifact_contract = + super::workspace_environment::workspace_external_artifacts_identity( + &external_artifacts, + )?; + let runtime_resource_contract = + super::workspace_environment::workspace_runtime_resources_identity(&runtime_resources)?; + let mut layer_inputs = BTreeMap::from([ + ("component_id".to_string(), plan.component_id.clone()), + ("component_root".to_string(), component_root.clone()), + ("protocol".to_string(), protocol.to_string()), + ( + "adapter_implementation".to_string(), + plugin.manifest.adapter.implementation_version.clone(), + ), + ( + "adapter_distribution_digest".to_string(), + plugin.distribution_digest.clone(), + ), + ( + "adapter_executable_digest".to_string(), + plugin.executable_digest.clone(), + ), + ( + "staging_action".to_string(), + serde_json::to_string( + &plan.command.as_ref().zip( + normalized_command + .as_ref() + .map(|(_, working_relative)| working_relative), + ), + )?, + ), + ( + "mounted_actions".to_string(), + serde_json::to_string( + &plan + .mounted_commands + .iter() + .zip( + normalized_mounted_commands + .iter() + .map(|(_, working_relative)| working_relative), + ) + .collect::>(), + )?, + ), + ("output_contract".to_string(), output_contract), + ( + "capability_contract".to_string(), + if normalized_mounted_commands.is_empty() { + "plugin-plan:bounded-pinned-bytes;action:staging;fs-read:declared-inputs;fs-write:declared-outputs+isolated-home+tmp;process:exact-host-resolved-executable;child-exec:deny;network:deny;shell:deny;scripts:deny;secrets:deny" + .to_string() + } else { + "plugin-plan:bounded-pinned-bytes;action:staging+mounted-candidate;mount-authority:host-only;fs-read:declared-inputs;fs-write:declared-writable-private-outputs+isolated-home+tmp;source-write:deny;process:exact-host-resolved-executable;child-exec:deny;network:deny;shell:deny;scripts:deny;secrets:deny" + .to_string() + }, + ), + ]); + if !external_artifacts.is_empty() { + layer_inputs.insert( + "external_artifact_contract".to_string(), + external_artifact_contract, + ); + layer_inputs.insert( + "capability_contract".to_string(), + if runtime_resources + .iter() + .any(|resource| !resource.secrets.is_empty()) + { + "plugin-plan:bounded-pinned-bytes;action:none;filesystem:none;process:none;network:none;secrets:opaque-reference-only;secret-resolution:host-runtime-file-handle;authority:external-identity+runtime-declaration-only" + .to_string() + } else { + "plugin-plan:bounded-pinned-bytes;action:none;filesystem:none;process:none;network:none;secrets:none;authority:external-identity-only" + .to_string() + }, + ); + } + if !runtime_resources.is_empty() { + layer_inputs.insert( + "runtime_resource_contract".to_string(), + runtime_resource_contract, + ); + } + if !plan.caches.is_empty() { + layer_inputs.insert("cache_contract".to_string(), cache_contract); + } + if protocol == PROTOCOL_V1 { + let command = plan.command.as_ref().ok_or_else(|| { + Error::Corrupt("validated protocol-v1 plugin plan lost its command".to_string()) + })?; + let working_relative = normalized_command + .as_ref() + .map(|(_, working_relative)| working_relative) + .ok_or_else(|| { + Error::Corrupt( + "validated protocol-v1 plugin plan lost its working directory".to_string(), + ) + })?; + layer_inputs.remove("staging_action"); + layer_inputs.remove("mounted_actions"); + layer_inputs.insert( + "command".to_string(), + serde_json::to_string(&( + &command.program, + &command.args, + working_relative, + &command.environment, + ))?, + ); + layer_inputs.insert( + "capability_contract".to_string(), + "plugin-plan:bounded-pinned-bytes;fs-read:declared-inputs;fs-write:declared-outputs+isolated-home+tmp;process:exact-host-resolved-executable;child-exec:deny;network:deny;shell:deny;scripts:deny;secrets:deny" + .to_string(), + ); + } + let mounted_commands = normalized_mounted_commands + .iter() + .map(|(command, _)| command.clone()) + .collect::>(); + if !mounted_commands.is_empty() { + layer_inputs.insert( + "mounted_action".to_string(), + super::workspace_environment::workspace_mounted_commands_identity( + &mounted_commands, + )?, + ); + } + for (name, value) in plan.semantic_inputs { + layer_inputs.insert(format!("semantic:{name}"), value); + } + let mut inputs = Vec::with_capacity(entries.len()); + for (relative, (repository_path, entry)) in entries { + layer_inputs.insert( + format!("input:{repository_path}"), + entry.content_hash.clone(), + ); + inputs.push(WorkspaceEnvironmentInput { + source_path: repository_path.clone(), + staging_path: format!("project/{repository_path}"), + entry, + }); + let _ = relative; + } + let portability_scope = match plan.portability { + AdapterPortability::Host => "plugin-tool-host", + AdapterPortability::Platform => "plugin-tool-platform", + }; + if plan.stale_reason.trim().is_empty() + || plan.stale_reason.len() > 4096 + || contains_sensitive_text(&plan.stale_reason) + { + return Err(Error::InvalidInput(format!( + "adapter `{}` returned an invalid stale explanation", + plugin.manifest.adapter.canonical_identity + ))); + } + let mut tool_versions = BTreeMap::new(); + if let Some((command, _)) = &normalized_command { + let name = if protocol == PROTOCOL_V1 { + format!("executable:{}", command.program) + } else { + format!("staging-executable:{}", command.program) + }; + tool_versions.insert(name, command.executable_identity.clone()); + } + for (index, (command, _)) in normalized_mounted_commands.iter().enumerate() { + tool_versions.insert( + format!("mounted-executable:{index}:{}", command.program), + command.executable_identity.clone(), + ); + } + let command = normalized_command.map(|(command, _)| command); + let has_plugin_caches = !normalized_caches.is_empty(); + Ok(WorkspaceEnvironmentPlan { + component_id: plan.component_id, + adapter_identity: plugin.manifest.adapter.canonical_identity.clone(), + adapter_version: 1, + implementation_version: plugin.manifest.adapter.implementation_version.clone(), + distribution_digest: plugin.distribution_digest.clone(), + kind: plan.kind, + dependencies: plan.dependencies, + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: plugin.manifest.adapter.kind.clone(), + adapter: plugin.manifest.adapter.layer_adapter_name.clone(), + adapter_version: 1, + inputs: layer_inputs, + tool_versions, + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: portability_scope.to_string(), + strategy: if !external_artifacts.is_empty() { + "isolated-plugin-external-artifact-plan-v2" + } else if protocol == PROTOCOL_V1 { + "isolated-plugin-plan-v1" + } else if mounted_commands.is_empty() { + "isolated-plugin-plan-v2" + } else { + "isolated-plugin-mounted-candidate-v2" + } + .to_string(), + }, + inputs, + source_projection: None, + pre_commands: Vec::new(), + command, + mounted_commands, + caches: normalized_caches, + external_artifacts, + runtime_resources, + sandbox_policy: if protocol == PROTOCOL_V2 && !normalized_mounted_commands.is_empty() { + WorkspaceEnvironmentSandboxPolicy::RestrictedPluginMounted + } else if protocol == PROTOCOL_V2 && has_plugin_caches { + WorkspaceEnvironmentSandboxPolicy::RestrictedPluginStaging + } else { + WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + }, + outputs, + stale_reason: plan.stale_reason, + }) + } +} + +fn spawn_bounded_reader( + mut reader: impl Read + Send + 'static, + maximum: usize, +) -> thread::JoinHandle> { + thread::spawn(move || { + let mut bytes = Vec::with_capacity(maximum.min(64 * 1024)); + let mut overflow = false; + let mut buffer = [0u8; 16 * 1024]; + loop { + let read = reader.read(&mut buffer)?; + if read == 0 { + break; + } + let remaining = maximum.saturating_sub(bytes.len()); + let retained = remaining.min(read); + bytes.extend_from_slice(&buffer[..retained]); + if retained != read { + overflow = true; + } + } + Ok(BoundedRead { bytes, overflow }) + }) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn configure_plugin_resource_limits(command: &mut Command, timeout_ms: u64) -> Result<()> { + use std::os::unix::process::CommandExt; + + #[cfg(target_os = "linux")] + type RlimitResource = libc::__rlimit_resource_t; + #[cfg(target_os = "macos")] + type RlimitResource = libc::c_int; + + let cpu_seconds = timeout_ms.div_ceil(1000).saturating_add(1); + // SAFETY: the closure performs only async-signal-safe setrlimit calls and + // constructs errors from the already-captured errno before exec. + unsafe { + command.pre_exec(move || { + fn set( + name: &str, + resource: RlimitResource, + soft: u64, + hard: u64, + ) -> std::io::Result<()> { + let limit = libc::rlimit { + rlim_cur: soft as libc::rlim_t, + rlim_max: hard as libc::rlim_t, + }; + // SAFETY: `limit` is initialized for the requested resource. + if unsafe { libc::setrlimit(resource, &limit) } != 0 { + let source = std::io::Error::last_os_error(); + Err(std::io::Error::new( + source.kind(), + format!("cannot set plugin {name} resource limit: {source}"), + )) + } else { + Ok(()) + } + } + + #[cfg(target_os = "linux")] + set( + "address-space", + libc::RLIMIT_AS, + 512 * 1024 * 1024, + 512 * 1024 * 1024, + )?; + set("cpu", libc::RLIMIT_CPU, cpu_seconds, cpu_seconds)?; + set( + "file-size", + libc::RLIMIT_FSIZE, + 16 * 1024 * 1024, + 16 * 1024 * 1024, + )?; + set("open-files", libc::RLIMIT_NOFILE, 64, 64)?; + set("core", libc::RLIMIT_CORE, 0, 0)?; + Ok(()) + }); + } + Ok(()) +} + +#[cfg(windows)] +fn configure_plugin_resource_limits(_command: &mut Command, _timeout_ms: u64) -> Result<()> { + // The hidden Windows sandbox helper applies the process-memory limit to + // zero-output invocations (the adapter-plugin profile) on its Job Object. + Ok(()) +} + +#[cfg(all(unix, not(any(target_os = "linux", target_os = "macos"))))] +fn configure_plugin_resource_limits(_command: &mut Command, _timeout_ms: u64) -> Result<()> { + Err(Error::InvalidInput( + "adapter plugin resource limits are unavailable on this Unix platform".to_string(), + )) +} + +#[cfg(not(any(unix, windows)))] +fn configure_plugin_resource_limits(_command: &mut Command, _timeout_ms: u64) -> Result<()> { + Err(Error::InvalidInput( + "adapter plugin resource limits are unavailable on this platform".to_string(), + )) +} + +fn wait_for_plugin(child: &mut Child, timeout: Duration) -> Result { + let deadline = Instant::now() + timeout; + loop { + if let Some(status) = child.try_wait()? { + return Ok(status); + } + if plugin_resident_bytes(child.id()).is_some_and(|bytes| bytes > PLUGIN_MEMORY_LIMIT_BYTES) + { + let _ = child.kill(); + let _ = child.wait(); + return Err(Error::InvalidInput(format!( + "adapter plugin exceeded its {} byte resident-memory limit", + PLUGIN_MEMORY_LIMIT_BYTES + ))); + } + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + return Err(Error::InvalidInput(format!( + "adapter plugin exceeded its {} ms deadline", + timeout.as_millis() + ))); + } + thread::sleep(Duration::from_millis(10)); + } +} + +#[cfg(target_os = "macos")] +fn plugin_resident_bytes(pid: u32) -> Option { + // SAFETY: the structure is plain data and proc_pidinfo receives its exact + // size and a valid writable pointer. + let mut info: libc::proc_taskinfo = unsafe { std::mem::zeroed() }; + let size = std::mem::size_of::(); + let read = unsafe { + libc::proc_pidinfo( + pid as libc::c_int, + libc::PROC_PIDTASKINFO, + 0, + (&mut info as *mut libc::proc_taskinfo).cast(), + size as libc::c_int, + ) + }; + (read == size as libc::c_int).then_some(info.pti_resident_size) +} + +#[cfg(target_os = "linux")] +fn plugin_resident_bytes(pid: u32) -> Option { + let status = fs::read_to_string(format!("/proc/{pid}/status")).ok()?; + let kibibytes = status.lines().find_map(|line| { + line.strip_prefix("VmRSS:")? + .split_whitespace() + .next()? + .parse::() + .ok() + })?; + kibibytes.checked_mul(1024) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn plugin_resident_bytes(_pid: u32) -> Option { + None +} + +fn sandboxed_plugin_launcher( + root: &Path, + home: &Path, + temporary: &Path, + executable: &Path, +) -> Result<(PathBuf, Vec)> { + #[cfg(target_os = "macos")] + { + let launcher = PathBuf::from("/usr/bin/sandbox-exec"); + if !launcher.is_file() { + return Err(Error::InvalidInput( + "adapter plugins require `/usr/bin/sandbox-exec` on macOS".to_string(), + )); + } + let profile = format!( + "(version 1)\n\ + (deny default)\n\ + (import \"system.sb\")\n\ + (deny mach-lookup)\n\ + (deny mach-register)\n\ + (deny process-fork)\n\ + (deny process-exec)\n\ + (allow process-exec (literal \"{}\"))\n\ + (deny file-write*)\n\ + (allow file-write* (subpath \"{}\") (subpath \"{}\"))\n\ + (allow file-read* (subpath \"{}\") (literal \"{}\") (subpath \"/bin\") (subpath \"/usr\") (subpath \"/System\") (subpath \"/Library\") (subpath \"/opt/homebrew\") (subpath \"/nix/store\"))\n\ + (deny network*)", + super::workspace_environment::sandbox_profile_escape(executable), + super::workspace_environment::sandbox_profile_escape(home), + super::workspace_environment::sandbox_profile_escape(temporary), + super::workspace_environment::sandbox_profile_escape(root), + super::workspace_environment::sandbox_profile_escape(executable), + ); + Ok(( + launcher, + vec![ + OsString::from("-p"), + OsString::from(profile), + executable.as_os_str().to_owned(), + ], + )) + } + #[cfg(all(any(target_os = "linux", target_os = "windows"), not(test)))] + { + let launcher = std::env::current_exe()?; + Ok(( + launcher, + vec![ + OsString::from("__environment-sandbox"), + OsString::from("--root"), + root.as_os_str().to_owned(), + OsString::from("--home"), + home.as_os_str().to_owned(), + OsString::from("--tmp"), + temporary.as_os_str().to_owned(), + OsString::from("--program"), + executable.as_os_str().to_owned(), + OsString::from("--"), + ], + )) + } + #[cfg(any( + all(any(target_os = "linux", target_os = "windows"), test), + not(any(target_os = "macos", target_os = "linux", target_os = "windows")) + ))] + { + let _ = (root, home, temporary, executable); + Err(Error::InvalidInput(format!( + "adapter plugin sandboxing is unavailable on {}; Trail refuses to execute plugin code without native enforcement", + std::env::consts::OS + ))) + } +} + +fn plugin_request_id( + plugin: &InstalledEnvironmentPlugin, + source_root: &ObjectId, + component_root: &str, + method: &str, + component_id: Option<&str>, +) -> String { + sha256_hex( + format!( + "{}\0{}\0{}\0{}\0{}", + plugin.distribution_digest, + source_root.0, + component_root, + method, + component_id.unwrap_or_default() + ) + .as_bytes(), + ) +} + +fn validate_discovered_plugin_component( + plugin: &InstalledEnvironmentPlugin, + component: &DiscoveredComponent, +) -> Result<()> { + if component.kind != plugin.manifest.adapter.kind { + return Err(Error::InvalidInput(format!( + "adapter `{}` discovered kind `{}` but its package declares `{}`", + plugin.manifest.adapter.canonical_identity, + component.kind, + plugin.manifest.adapter.kind + ))); + } + validate_plugin_component_id(&component.component_id) +} + +fn validate_plugin_component_id(component_id: &str) -> Result<()> { + if component_id.is_empty() + || component_id.len() > 256 + || contains_sensitive_text(component_id) + || !component_id.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '_' | '-' | ':' | '/') + }) + { + return Err(Error::InvalidInput(format!( + "plugin component ID `{component_id}` is invalid or sensitive" + ))); + } + Ok(()) +} + +pub(super) fn environment_plugin_supports_current_host( + plugin: &InstalledEnvironmentPlugin, +) -> bool { + plugin + .manifest + .adapter + .supported_operating_systems + .iter() + .any(|value| value == std::env::consts::OS) + && plugin + .manifest + .adapter + .supported_architectures + .iter() + .any(|value| value == std::env::consts::ARCH) +} + +pub(super) fn selected_environment_plugin_protocol( + plugin: &InstalledEnvironmentPlugin, +) -> Result<&'static str> { + if plugin + .manifest + .adapter + .protocols + .iter() + .any(|protocol| protocol == PROTOCOL_V2) + { + Ok(PROTOCOL_V2) + } else if plugin + .manifest + .adapter + .protocols + .iter() + .any(|protocol| protocol == PROTOCOL_V1) + { + Ok(PROTOCOL_V1) + } else { + Err(Error::InvalidInput(format!( + "adapter `{}` shares no supported protocol with this Trail host", + plugin.manifest.adapter.canonical_identity + ))) + } +} + +fn ensure_environment_plugin_supports_current_host( + plugin: &InstalledEnvironmentPlugin, +) -> Result<()> { + if environment_plugin_supports_current_host(plugin) { + Ok(()) + } else { + Err(Error::InvalidInput(format!( + "adapter `{}` does not support {}/{}", + plugin.manifest.adapter.canonical_identity, + std::env::consts::OS, + std::env::consts::ARCH + ))) + } +} + +fn normalize_plugin_component_root(component_root: &str) -> Result { + if component_root.trim_matches('/').is_empty() || component_root == "." { + Ok(String::new()) + } else { + normalize_relative_path(component_root) + } +} + +fn normalize_plugin_path_allow_root(path: &str) -> Result { + if path.trim_matches('/').is_empty() || path == "." { + Ok(String::new()) + } else { + normalize_relative_path(path) + } +} + +fn join_plugin_path(root: &str, child: &str) -> String { + if root.is_empty() { + child.to_string() + } else if child.is_empty() { + root.to_string() + } else { + format!("{root}/{child}") + } +} + +fn plugin_paths_overlap(left: &str, right: &str) -> bool { + left == right + || left.starts_with(&format!("{right}/")) + || right.starts_with(&format!("{left}/")) +} + +fn validate_plugin_output_name(name: &str) -> Result<()> { + if name.is_empty() + || name.len() > 128 + || contains_sensitive_text(name) + || !name.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '_' | '-') + }) + { + return Err(Error::InvalidInput(format!( + "plugin output name `{name}` is invalid or sensitive" + ))); + } + Ok(()) +} + +fn read_adapter_package_signature( + package_directory: &Path, +) -> Result> { + let path = package_directory.join(PLUGIN_PACKAGE_SIGNATURE); + let metadata = match fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(Error::Io(error)), + }; + if !metadata.is_file() + || metadata.file_type().is_symlink() + || metadata.len() > MAX_PUBLISHER_DOCUMENT_BYTES + { + return Err(Error::InvalidInput(format!( + "adapter signature `{}` must be a bounded regular file", + path.display() + ))); + } + let signature = toml::from_str(&fs::read_to_string(&path)?).map_err(|error| { + Error::InvalidInput(format!( + "cannot parse adapter signature `{}`: {error}", + path.display() + )) + })?; + Ok(Some(signature)) +} + +fn prepare_environment_plugin_package( + package_directory: impl AsRef, +) -> Result { + let package_directory = fs::canonicalize(package_directory)?; + if !package_directory.is_dir() { + return Err(Error::InvalidInput(format!( + "adapter package `{}` is not a directory", + package_directory.display() + ))); + } + let manifest_path = package_directory.join(PLUGIN_PACKAGE_MANIFEST); + let metadata = fs::symlink_metadata(&manifest_path)?; + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(Error::InvalidInput(format!( + "adapter package manifest `{}` must be a regular file", + manifest_path.display() + ))); + } + if metadata.len() > MAX_PACKAGE_MANIFEST_BYTES { + return Err(Error::InvalidInput(format!( + "adapter package manifest exceeds {MAX_PACKAGE_MANIFEST_BYTES} bytes" + ))); + } + let manifest_text = fs::read_to_string(&manifest_path)?; + if contains_sensitive_text(&manifest_text) { + return Err(Error::InvalidInput( + "adapter package manifest appears to contain a secret".to_string(), + )); + } + let mut package: AdapterPackageManifest = toml::from_str(&manifest_text).map_err(|error| { + Error::InvalidInput(format!( + "cannot parse adapter package manifest `{}`: {error}", + manifest_path.display() + )) + })?; + canonicalize_and_validate_package(&mut package)?; + + let executable_source = package_directory.join(&package.executable.path); + let executable_metadata = fs::symlink_metadata(&executable_source)?; + if !executable_metadata.is_file() || executable_metadata.file_type().is_symlink() { + return Err(Error::InvalidInput(format!( + "adapter executable `{}` must be a regular file", + executable_source.display() + ))); + } + if executable_metadata.len() > MAX_PLUGIN_EXECUTABLE_BYTES { + return Err(Error::InvalidInput(format!( + "adapter executable exceeds {MAX_PLUGIN_EXECUTABLE_BYTES} bytes" + ))); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if executable_metadata.permissions().mode() & 0o111 == 0 { + return Err(Error::InvalidInput(format!( + "adapter executable `{}` is not executable", + executable_source.display() + ))); + } + } + let executable_bytes = fs::read(&executable_source)?; + let executable_digest = format!("sha256:{}", sha256_hex(&executable_bytes)); + if package.executable.sha256 != executable_digest { + return Err(Error::InvalidInput(format!( + "adapter executable digest mismatch: manifest declares `{}`, actual is `{executable_digest}`", + package.executable.sha256 + ))); + } + let (payload_material, payload_digest) = adapter_package_payload(&package, &executable_bytes)?; + let signature = read_adapter_package_signature(&package_directory)?; + if let Some(signature) = &signature { + validate_adapter_package_signature(signature)?; + if signature.payload_digest != payload_digest { + return Err(Error::InvalidInput(format!( + "adapter signature authenticates `{}` but package payload is `{payload_digest}`", + signature.payload_digest + ))); + } + } + Ok(PreparedEnvironmentPluginPackage { + package, + executable_bytes, + executable_permissions: executable_metadata.permissions(), + executable_digest, + payload_material, + payload_digest, + signature, + }) +} + +fn adapter_package_payload( + package: &AdapterPackageManifest, + executable_bytes: &[u8], +) -> Result<(Vec, String)> { + let canonical_package = cbor(package)?; + let mut material = Vec::with_capacity( + canonical_package.len() + executable_bytes.len() + std::mem::size_of::(), + ); + material.extend_from_slice(&(canonical_package.len() as u64).to_be_bytes()); + material.extend_from_slice(&canonical_package); + material.extend_from_slice(executable_bytes); + let digest = format!("sha256:{}", sha256_hex(&material)); + Ok((material, digest)) +} + +fn adapter_package_distribution_material( + payload_material: &[u8], + signature: Option<&AdapterPackageSignature>, +) -> Result> { + let Some(signature) = signature else { + // Preserve the v1 unsigned package identity for existing stores. + return Ok(payload_material.to_vec()); + }; + let signature_bytes = cbor(signature)?; + let mut material = Vec::with_capacity( + 64 + payload_material.len() + signature_bytes.len() + 2 * std::mem::size_of::(), + ); + material.extend_from_slice(b"trail.environment-adapter-distribution/v1\0"); + material.extend_from_slice(&(payload_material.len() as u64).to_be_bytes()); + material.extend_from_slice(payload_material); + material.extend_from_slice(&(signature_bytes.len() as u64).to_be_bytes()); + material.extend_from_slice(&signature_bytes); + Ok(material) +} + +fn adapter_package_signature_message(payload_digest: &str) -> Vec { + let mut message = + Vec::with_capacity(PACKAGE_SIGNATURE_SCHEMA_V1.len() + payload_digest.len() + 1); + message.extend_from_slice(PACKAGE_SIGNATURE_SCHEMA_V1.as_bytes()); + message.push(0); + message.extend_from_slice(payload_digest.as_bytes()); + message +} + +fn validate_publisher_name(publisher: &str) -> Result<()> { + if publisher.is_empty() + || publisher.len() > 128 + || contains_sensitive_text(publisher) + || !publisher.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '.' | '_' | '-') + }) + { + return Err(Error::InvalidInput(format!( + "adapter publisher `{publisher}` is invalid or sensitive" + ))); + } + Ok(()) +} + +fn validate_publisher_key_id(key_id: &str) -> Result<()> { + let digest = key_id + .strip_prefix("sha256:") + .ok_or_else(|| Error::InvalidInput("publisher key ID must use sha256:".to_string()))?; + decode_fixed_hex::<32>(digest, "publisher key ID").map(|_| ()) +} + +fn validate_publisher_key_document( + document: &AdapterPublisherKey, +) -> Result<(String, String, String)> { + if document.schema != TRUSTED_PUBLISHER_KEY_SCHEMA_V1 { + return Err(Error::InvalidInput(format!( + "unsupported publisher key schema `{}`; expected `{TRUSTED_PUBLISHER_KEY_SCHEMA_V1}`", + document.schema + ))); + } + validate_publisher_name(&document.publisher)?; + let public_key = decode_fixed_hex::<32>(&document.public_key, "publisher public key")?; + VerifyingKey::from_bytes(&public_key).map_err(|error| { + Error::InvalidInput(format!("publisher public key is invalid: {error}")) + })?; + let public_key_hex = hex::encode(public_key); + let key_id = format!("sha256:{}", sha256_hex(&public_key)); + Ok((document.publisher.clone(), public_key_hex, key_id)) +} + +fn validate_adapter_package_signature(signature: &AdapterPackageSignature) -> Result<()> { + if signature.schema != PACKAGE_SIGNATURE_SCHEMA_V1 { + return Err(Error::InvalidInput(format!( + "unsupported adapter signature schema `{}`; expected `{PACKAGE_SIGNATURE_SCHEMA_V1}`", + signature.schema + ))); + } + validate_publisher_name(&signature.publisher)?; + validate_publisher_key_id(&signature.key_id)?; + validate_sha256_digest(&signature.payload_digest, "adapter payload digest")?; + decode_fixed_hex::<64>(&signature.signature, "adapter signature").map(|_| ()) +} + +fn validate_publisher_trust_record(record: &PublisherTrustRecord, directory: &OsStr) -> Result<()> { + if record.schema != TRUST_RECORD_SCHEMA || !matches!(record.action.as_str(), "trust" | "remove") + { + return Err(Error::Corrupt( + "publisher trust record has an unsupported schema or action".to_string(), + )); + } + validate_publisher_key_id(&record.key_id) + .map_err(|error| Error::Corrupt(format!("invalid publisher trust key ID: {error}")))?; + if directory.to_string_lossy() != record.key_id.trim_start_matches("sha256:") { + return Err(Error::Corrupt( + "publisher trust record is stored under the wrong key directory".to_string(), + )); + } + if record.action == "trust" { + validate_publisher_name(&record.publisher) + .map_err(|error| Error::Corrupt(format!("invalid trusted publisher: {error}")))?; + let public_key = decode_fixed_hex::<32>(&record.public_key, "publisher public key") + .map_err(|error| Error::Corrupt(error.to_string()))?; + let expected_key_id = format!("sha256:{}", sha256_hex(&public_key)); + if expected_key_id != record.key_id { + return Err(Error::Corrupt( + "publisher trust key ID does not match its public key".to_string(), + )); + } + } + Ok(()) +} + +fn validate_sha256_digest(value: &str, label: &str) -> Result<()> { + let digest = value + .strip_prefix("sha256:") + .ok_or_else(|| Error::InvalidInput(format!("{label} must use sha256:")))?; + decode_fixed_hex::<32>(digest, label).map(|_| ()) +} + +fn decode_fixed_hex(value: &str, label: &str) -> Result<[u8; N]> { + if value.len() != N * 2 + || !value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(Error::InvalidInput(format!( + "{label} must be {} lowercase hexadecimal characters", + N * 2 + ))); + } + let bytes = hex::decode(value) + .map_err(|error| Error::InvalidInput(format!("cannot decode {label}: {error}")))?; + bytes + .try_into() + .map_err(|_| Error::InvalidInput(format!("{label} has the wrong decoded byte length"))) +} + +fn canonicalize_and_validate_package(package: &mut AdapterPackageManifest) -> Result<()> { + if package.schema != PACKAGE_SCHEMA_V1 { + return Err(Error::InvalidInput(format!( + "unsupported adapter package schema `{}`; expected `{PACKAGE_SCHEMA_V1}`", + package.schema + ))); + } + let (_, _, major) = validate_plugin_identity(&package.adapter.canonical_identity)?; + if major != 1 { + return Err(Error::InvalidInput(format!( + "adapter `{}` uses unsupported contract major {major}; this host supports major 1", + package.adapter.canonical_identity + ))); + } + if package.adapter.implementation_version.trim().is_empty() + || package.adapter.implementation_version.len() > 128 + || package.adapter.description.trim().is_empty() + || package.adapter.description.len() > 4096 + { + return Err(Error::InvalidInput( + "adapter implementation version and description must be non-empty and bounded" + .to_string(), + )); + } + if package.adapter.stability != "experimental" { + return Err(Error::InvalidInput( + "unsigned local adapter packages must declare stability = \"experimental\"".to_string(), + )); + } + if !matches!( + package.adapter.kind.as_str(), + "dependency" | "compiler-results" | "generated" + ) { + return Err(Error::InvalidInput(format!( + "adapter `{}` declares unsupported kind `{}`", + package.adapter.canonical_identity, package.adapter.kind + ))); + } + if package.adapter.layer_adapter_name.is_empty() + || package.adapter.layer_adapter_name.len() > 64 + || !package.adapter.layer_adapter_name.chars().all(|character| { + character.is_ascii_lowercase() || character.is_ascii_digit() || character == '-' + }) + { + return Err(Error::InvalidInput( + "adapter layer storage name must use 1-64 lowercase ASCII letters, digits, or hyphens" + .to_string(), + )); + } + sort_unique_bounded(&mut package.adapter.selectors, 32, "adapter selectors")?; + if !package + .adapter + .selectors + .contains(&package.adapter.canonical_identity) + { + return Err(Error::InvalidInput( + "adapter selectors must contain canonical_identity".to_string(), + )); + } + sort_unique_bounded( + &mut package.adapter.discovery_markers, + 32, + "adapter discovery markers", + )?; + sort_unique_bounded(&mut package.adapter.protocols, 4, "adapter protocols")?; + if package.adapter.protocols.is_empty() + || package + .adapter + .protocols + .iter() + .any(|protocol| !matches!(protocol.as_str(), PROTOCOL_V1 | PROTOCOL_V2)) + { + return Err(Error::InvalidInput(format!( + "adapter `{}` must declare one or more host-supported protocols", + package.adapter.canonical_identity + ))); + } + for marker in &package.adapter.discovery_markers { + if marker == "." + || marker == ".." + || marker.contains('/') + || marker.contains('\\') + || marker.contains('\0') + || contains_sensitive_text(marker) + { + return Err(Error::InvalidInput(format!( + "adapter discovery marker `{marker}` is not a safe file name" + ))); + } + } + sort_unique_bounded( + &mut package.adapter.supported_operating_systems, + 8, + "adapter supported operating systems", + )?; + if package + .adapter + .supported_operating_systems + .iter() + .any(|value| !matches!(value.as_str(), "linux" | "macos" | "windows")) + { + return Err(Error::InvalidInput( + "adapter supported operating systems contain an unknown value".to_string(), + )); + } + sort_unique_bounded( + &mut package.adapter.supported_architectures, + 8, + "adapter supported architectures", + )?; + if package + .adapter + .supported_architectures + .iter() + .any(|value| !matches!(value.as_str(), "aarch64" | "x86_64")) + { + return Err(Error::InvalidInput( + "adapter supported architectures contain an unknown value".to_string(), + )); + } + sort_unique_bounded( + &mut package.permissions.read_patterns, + 128, + "adapter read patterns", + )?; + if package.permissions.read_patterns.is_empty() { + return Err(Error::InvalidInput( + "adapter must declare at least one repository read pattern".to_string(), + )); + } + let patterns = compile_plugin_patterns(&package.permissions)?; + for marker in &package.adapter.discovery_markers { + if !patterns.is_match(marker) { + return Err(Error::InvalidInput(format!( + "adapter discovery marker `{marker}` is not covered by a declared read pattern" + ))); + } + } + if package.permissions.max_input_files == 0 + || package.permissions.max_input_files > HARD_MAX_INPUT_FILES + || package.permissions.max_input_bytes == 0 + || package.permissions.max_input_bytes > HARD_MAX_INPUT_BYTES + || package.permissions.timeout_ms == 0 + || package.permissions.timeout_ms > HARD_MAX_TIMEOUT_MS + || package.permissions.max_response_bytes == 0 + || package.permissions.max_response_bytes > HARD_MAX_RESPONSE_BYTES + { + return Err(Error::InvalidInput( + "adapter resource limits are zero or exceed Trail's hard limits".to_string(), + )); + } + let executable_path = Path::new(&package.executable.path); + if executable_path.components().count() != 1 + || !matches!( + executable_path.components().next(), + Some(std::path::Component::Normal(_)) + ) + { + return Err(Error::InvalidInput( + "adapter executable path must be one relative file name".to_string(), + )); + } + let declared_digest = package + .executable + .sha256 + .strip_prefix("sha256:") + .unwrap_or(&package.executable.sha256); + if declared_digest.len() != 64 + || !declared_digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(Error::InvalidInput( + "adapter executable sha256 must be 64 lowercase hexadecimal characters".to_string(), + )); + } + package.executable.sha256 = format!("sha256:{declared_digest}"); + Ok(()) +} + +pub(super) fn validate_plugin_identity(identity: &str) -> Result<(String, String, u32)> { + let (qualified, major) = identity.rsplit_once('@').ok_or_else(|| { + Error::InvalidInput(format!( + "adapter identity `{identity}` must be namespace/name@major" + )) + })?; + let (namespace, name) = qualified.split_once('/').ok_or_else(|| { + Error::InvalidInput(format!( + "adapter identity `{identity}` must be namespace/name@major" + )) + })?; + let valid_segment = |segment: &str| { + !segment.is_empty() + && segment.len() <= 64 + && segment.chars().all(|character| { + character.is_ascii_lowercase() + || character.is_ascii_digit() + || character == '-' + || character == '_' + }) + }; + if !valid_segment(namespace) || !valid_segment(name) { + return Err(Error::InvalidInput(format!( + "adapter identity `{identity}` contains an invalid namespace or name" + ))); + } + let major = major.parse::().map_err(|_| { + Error::InvalidInput(format!( + "adapter identity `{identity}` has an invalid contract major" + )) + })?; + Ok((namespace.to_string(), name.to_string(), major)) +} + +fn sort_unique_bounded(values: &mut Vec, maximum: usize, label: &str) -> Result<()> { + if values.is_empty() || values.len() > maximum { + return Err(Error::InvalidInput(format!( + "{label} must contain between 1 and {maximum} entries" + ))); + } + if values + .iter() + .any(|value| value.trim().is_empty() || value.len() > 4096 || value.contains('\0')) + { + return Err(Error::InvalidInput(format!( + "{label} contains an empty, oversized, or NUL-bearing value" + ))); + } + values.sort(); + values.dedup(); + Ok(()) +} + +fn compile_plugin_patterns(permissions: &AdapterPermissions) -> Result { + let mut builder = GlobSetBuilder::new(); + for pattern in &permissions.read_patterns { + if pattern.starts_with('/') + || pattern.starts_with("../") + || pattern.contains("/../") + || pattern.contains('\\') + || contains_sensitive_text(pattern) + { + return Err(Error::InvalidInput(format!( + "adapter read pattern `{pattern}` is unsafe" + ))); + } + let glob = GlobBuilder::new(pattern) + .literal_separator(true) + .backslash_escape(false) + .build() + .map_err(|error| { + Error::InvalidInput(format!( + "adapter read pattern `{pattern}` is invalid: {error}" + )) + })?; + builder.add(glob); + } + builder.build().map_err(|error| { + Error::InvalidInput(format!("cannot compile adapter read patterns: {error}")) + }) +} + +fn validate_registry_record(record: &PluginRegistryRecord, directory_name: &OsStr) -> Result<()> { + if record.schema != REGISTRY_RECORD_SCHEMA + || !matches!(record.action.as_str(), "install" | "remove") + { + return Err(Error::Corrupt( + "adapter registry record has an unsupported schema or action".to_string(), + )); + } + validate_plugin_identity(&record.canonical_identity).map_err(|error| { + Error::Corrupt(format!("adapter registry identity is invalid: {error}")) + })?; + let expected_directory = sha256_hex(record.canonical_identity.as_bytes()); + if directory_name != OsStr::new(&expected_directory) { + return Err(Error::Corrupt(format!( + "adapter registry identity directory does not match `{}`", + record.canonical_identity + ))); + } + if record.action == "install" + && !record + .distribution_digest + .as_deref() + .is_some_and(valid_sha256_digest) + { + return Err(Error::Corrupt( + "adapter install record has an invalid distribution digest".to_string(), + )); + } + if record.action == "remove" && record.distribution_digest.is_some() { + return Err(Error::Corrupt( + "adapter removal record unexpectedly names a distribution".to_string(), + )); + } + Ok(()) +} + +fn valid_sha256_digest(value: &str) -> bool { + value.strip_prefix("sha256:").is_some_and(|digest| { + digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) +} + +fn validate_plugin_catalog(plugins: &[InstalledEnvironmentPlugin]) -> Result<()> { + let mut identities = BTreeSet::new(); + let mut selectors = BTreeMap::<&str, &str>::new(); + for plugin in plugins { + let identity = plugin.manifest.adapter.canonical_identity.as_str(); + if !identities.insert(identity) { + return Err(Error::Corrupt(format!( + "adapter registry contains duplicate identity `{identity}`" + ))); + } + for selector in &plugin.manifest.adapter.selectors { + if let Some(other) = selectors.insert(selector, identity) { + return Err(Error::Corrupt(format!( + "adapter selector `{selector}` is claimed by both `{other}` and `{identity}`" + ))); + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use ed25519_dalek::{Signer, SigningKey}; + + #[test] + fn protocol_v2_typed_dependencies_normalize_to_host_edge_semantics() { + let plan = AdapterPlanV2::builder("application", "generated") + .build_requires("compiler") + .runtime_requires("database") + .binds_after("network") + .invalidates_with("configuration") + .mounted_command(AdapterCommand::new("initializer", ["prepare"])) + .output(AdapterOutput::writable_private("state", "state", "state")) + .stale_reason("typed dependency changed") + .build() + .unwrap(); + let normalized = ProposedPluginPlan::from_v2(plan).unwrap(); + assert_eq!( + normalized + .dependencies + .iter() + .map(|dependency| ( + dependency.component_id.as_str(), + dependency.edge_type.as_str() + )) + .collect::>(), + [ + ("compiler", "build_requires"), + ("configuration", "invalidates_with"), + ("database", "runtime_requires"), + ("network", "binds_after") + ] + ); + } + + #[test] + fn protocol_v2_runtime_resources_survive_host_protocol_normalization() { + let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let plan = AdapterPlanV2::builder("services", "external") + .external_artifact(AdapterExternalArtifact::pinned_oci_image( + "database-image", + format!("example.invalid/postgres@{digest}"), + "linux/amd64", + )) + .runtime_resource( + AdapterRuntimeResource::oci_container("database", "database-image", 5432) + .health_timeout_ms(45_000) + .volume_target("/var/lib/postgresql/data"), + ) + .stale_reason("service image or runtime contract changed") + .build() + .unwrap(); + + let normalized = ProposedPluginPlan::from_v2(plan).unwrap(); + assert_eq!(normalized.external_artifacts.len(), 1); + assert_eq!(normalized.runtime_resources.len(), 1); + assert_eq!(normalized.runtime_resources[0].name, "database"); + assert_eq!(normalized.runtime_resources[0].container_port, 5432); + assert_eq!( + normalized.runtime_resources[0].volume_target.as_deref(), + Some("/var/lib/postgresql/data") + ); + } + + #[test] + fn protocol_v2_plugin_caches_are_host_scoped_and_fail_closed() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let package = tempfile::tempdir().unwrap(); + write_test_package(package.path(), "example/cache-test@1"); + db.install_environment_adapter_plugin(package.path()) + .unwrap(); + let mut plugins = db.installed_environment_plugins().unwrap(); + let plugin = plugins.remove(0); + + let cache = + AdapterCache::host_exclusive("package-store", AdapterCacheProtocol::ContentStore) + .compatibility_dimension("tool", "fixture@1") + .environment_variable("FIXTURE_CACHE", "store"); + let (normalized, environment) = db + .normalize_environment_plugin_caches(&plugin, PROTOCOL_V2, &[cache], true) + .unwrap(); + assert_eq!(normalized.len(), 1); + assert_eq!( + normalized[0].access, + WorkspaceEnvironmentCacheAccess::HostExclusive + ); + assert_eq!( + normalized[0].compatibility["trail.adapter_distribution"], + plugin.distribution_digest + ); + assert_eq!( + environment["FIXTURE_CACHE"], + normalized[0] + .storage_path + .join("store") + .to_string_lossy() + .into_owned() + ); + assert!(!normalized[0].storage_path.exists()); + + let concurrent = + AdapterCache::tool_concurrent("unsafe-store", AdapterCacheProtocol::LockedIndex) + .environment_variable("FIXTURE_CACHE", "."); + assert!(db + .normalize_environment_plugin_caches(&plugin, PROTOCOL_V2, &[concurrent], true) + .unwrap_err() + .to_string() + .contains("without independent cache certification")); + + let sensitive = + AdapterCache::host_exclusive("secret-store", AdapterCacheProtocol::ContentStore) + .compatibility_dimension("access_token", "do-not-persist") + .environment_variable("FIXTURE_CACHE", "."); + assert!(db + .normalize_environment_plugin_caches(&plugin, PROTOCOL_V2, &[sensitive], true) + .unwrap_err() + .to_string() + .contains("secret-like data")); + } + + fn write_test_package(root: &Path, identity: &str) { + let executable = root.join("adapter-test"); + fs::write(&executable, b"test adapter executable\n").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap(); + } + let digest = sha256_hex(&fs::read(&executable).unwrap()); + fs::write( + root.join(PLUGIN_PACKAGE_MANIFEST), + format!( + r#"schema = "trail.environment-adapter-package/v1" + +[adapter] +canonical_identity = "{identity}" +implementation_version = "1.0.0" +selectors = ["{identity}", "test-plugin"] +kind = "generated" +layer_adapter_name = "test-plugin" +discovery_markers = ["test.plugin"] +stability = "experimental" +description = "Test adapter" + +[executable] +path = "adapter-test" +sha256 = "{digest}" + +[permissions] +read_patterns = ["test.plugin", "config/*.toml"] +max_input_files = 16 +max_input_bytes = 1048576 +timeout_ms = 1000 +max_response_bytes = 1048576 +"# + ), + ) + .unwrap(); + } + + fn sign_test_package(root: &Path, signing_key: &SigningKey, publisher: &str) -> PathBuf { + let mut package: AdapterPackageManifest = + toml::from_str(&fs::read_to_string(root.join(PLUGIN_PACKAGE_MANIFEST)).unwrap()) + .unwrap(); + canonicalize_and_validate_package(&mut package).unwrap(); + let executable = fs::read(root.join(&package.executable.path)).unwrap(); + let (_, payload_digest) = adapter_package_payload(&package, &executable).unwrap(); + let verifying_key = signing_key.verifying_key(); + let public_key = verifying_key.to_bytes(); + let key_id = format!("sha256:{}", sha256_hex(&public_key)); + let signature = signing_key.sign(&adapter_package_signature_message(&payload_digest)); + let signature_document = AdapterPackageSignature { + schema: PACKAGE_SIGNATURE_SCHEMA_V1.to_string(), + publisher: publisher.to_string(), + key_id: key_id.clone(), + payload_digest, + signature: hex::encode(signature.to_bytes()), + }; + fs::write( + root.join(PLUGIN_PACKAGE_SIGNATURE), + toml::to_string(&signature_document).unwrap(), + ) + .unwrap(); + let key_path = root.join("publisher-key.toml"); + fs::write( + &key_path, + toml::to_string(&AdapterPublisherKey { + schema: TRUSTED_PUBLISHER_KEY_SCHEMA_V1.to_string(), + publisher: publisher.to_string(), + public_key: hex::encode(public_key), + }) + .unwrap(), + ) + .unwrap(); + key_path + } + + #[test] + fn plugin_install_is_content_addressed_catalogued_and_tombstoned() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let package = tempfile::tempdir().unwrap(); + write_test_package(package.path(), "example/test@1"); + + let inspected = db + .inspect_environment_adapter_plugin_package(package.path()) + .unwrap(); + assert_eq!(inspected.canonical_identity, "example/test@1"); + assert!(!inspected.signature_present); + + let installed = db + .install_environment_adapter_plugin(package.path()) + .unwrap(); + assert_eq!(installed.canonical_identity, "example/test@1"); + assert_eq!(installed.distribution_digest, inspected.distribution_digest); + assert!(installed.distribution_digest.starts_with("sha256:")); + assert_eq!(installed.trust, "local_unsigned"); + assert_eq!(installed.certification_tier, "local-experimental"); + assert!(installed.publisher.is_none()); + assert!(Path::new(&installed.package_path).is_dir()); + let catalog = db.workspace_environment_adapters().unwrap(); + let entry = catalog + .adapters + .iter() + .find(|entry| entry.canonical_identity == "example/test@1") + .unwrap(); + assert_eq!(entry.source, "plugin"); + assert_eq!(entry.trust, "local_unsigned"); + assert_eq!(entry.certification_tier, "local-experimental"); + assert_eq!( + entry.identity.distribution_digest.as_deref(), + Some(installed.distribution_digest.as_str()) + ); + + let repeated = db + .install_environment_adapter_plugin(package.path()) + .unwrap(); + assert_eq!( + repeated.replaced_distribution_digest.as_deref(), + Some(installed.distribution_digest.as_str()) + ); + let removed = db + .remove_environment_adapter_plugin("example/test@1") + .unwrap(); + assert_eq!( + removed.removed_distribution_digest.as_deref(), + Some(installed.distribution_digest.as_str()) + ); + assert!(db + .workspace_environment_adapters() + .unwrap() + .adapters + .iter() + .all(|entry| entry.canonical_identity != "example/test@1")); + } + + #[test] + fn signed_plugin_requires_live_publisher_trust_and_reports_authentication() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let package = tempfile::tempdir().unwrap(); + write_test_package(package.path(), "example/signed@1"); + let signing_key = SigningKey::from_bytes(&[7_u8; 32]); + let key_document = sign_test_package(package.path(), &signing_key, "example-publisher"); + + let inspected = db + .inspect_environment_adapter_plugin_package(package.path()) + .unwrap(); + assert!(inspected.signature_present); + assert_eq!(inspected.publisher.as_deref(), Some("example-publisher")); + + let untrusted = db + .install_environment_adapter_plugin(package.path()) + .unwrap_err(); + assert!(untrusted.to_string().contains("is not trusted")); + + let trusted = db + .trust_environment_adapter_publisher_key(&key_document) + .unwrap(); + assert_eq!(trusted.action, "trust"); + assert_eq!(trusted.publisher.as_deref(), Some("example-publisher")); + let installed = db + .install_environment_adapter_plugin(package.path()) + .unwrap(); + assert_eq!(installed.publisher.as_deref(), Some("example-publisher")); + assert_eq!( + installed.publisher_key_id.as_deref(), + Some(trusted.key_id.as_str()) + ); + assert_eq!(installed.trust, "publisher_signed"); + assert_eq!( + installed.certification_tier, + "publisher-authenticated-experimental" + ); + let entry = db + .workspace_environment_adapters() + .unwrap() + .adapters + .into_iter() + .find(|entry| entry.canonical_identity == "example/signed@1") + .unwrap(); + assert_eq!(entry.publisher.as_deref(), Some("example-publisher")); + assert_eq!(entry.publisher_key_id, installed.publisher_key_id); + assert_eq!(entry.trust, "publisher_signed"); + + let trust = db.environment_adapter_publisher_trust().unwrap(); + assert_eq!(trust.keys.len(), 1); + assert_eq!(trust.keys[0].key_id, trusted.key_id); + db.remove_environment_adapter_publisher_key(&trusted.key_id) + .unwrap(); + assert!(db + .workspace_environment_adapters() + .unwrap_err() + .to_string() + .contains("no longer has a valid trusted publisher")); + assert!(db + .environment_adapter_publisher_trust() + .unwrap() + .keys + .is_empty()); + + db.trust_environment_adapter_publisher_key(&key_document) + .unwrap(); + assert!(db.workspace_environment_adapters().is_ok()); + let signature_path = package.path().join(PLUGIN_PACKAGE_SIGNATURE); + let mut signature: AdapterPackageSignature = + toml::from_str(&fs::read_to_string(&signature_path).unwrap()).unwrap(); + let mut signature_bytes = hex::decode(&signature.signature).unwrap(); + signature_bytes[0] ^= 1; + signature.signature = hex::encode(signature_bytes); + fs::write(&signature_path, toml::to_string(&signature).unwrap()).unwrap(); + let invalid = db + .install_environment_adapter_plugin(package.path()) + .unwrap_err(); + assert!(invalid.to_string().contains("signature")); + + let removed = db + .remove_environment_adapter_plugin("example/signed@1") + .unwrap(); + assert!(removed.removed_distribution_digest.is_some()); + } + + #[test] + fn plugin_catalog_fails_closed_after_executable_tampering() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let package = tempfile::tempdir().unwrap(); + write_test_package(package.path(), "example/test@1"); + let installed = db + .install_environment_adapter_plugin(package.path()) + .unwrap(); + fs::write( + Path::new(&installed.package_path).join("adapter-test"), + b"tampered\n", + ) + .unwrap(); + let error = db.workspace_environment_adapters().unwrap_err(); + assert!(error.to_string().contains("digest verification")); + + let repaired = db + .install_environment_adapter_plugin(package.path()) + .unwrap(); + assert_eq!(repaired.canonical_identity, "example/test@1"); + assert!(db.workspace_environment_adapters().is_ok()); + fs::write( + Path::new(&repaired.package_path).join("adapter-test"), + b"tampered again\n", + ) + .unwrap(); + let removed = db + .remove_environment_adapter_plugin("example/test@1") + .unwrap(); + assert!(removed.removed_distribution_digest.is_some()); + assert!(db.workspace_environment_adapters().is_ok()); + } +} diff --git a/trail/src/db/lane/workspace_python.rs b/trail/src/db/lane/workspace_python.rs new file mode 100644 index 0000000..2db3ee3 --- /dev/null +++ b/trail/src/db/lane/workspace_python.rs @@ -0,0 +1,698 @@ +use super::workspace_environment::{ + resolve_workspace_tool_executable, workspace_mounted_commands_identity, + WorkspaceEnvironmentAdapter, WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentCommand, + WorkspaceEnvironmentInput, WorkspaceEnvironmentOutput, WorkspaceEnvironmentOutputPolicy, + WorkspaceEnvironmentPlan, WorkspaceEnvironmentSandboxPolicy, +}; +use super::*; + +pub(crate) struct PythonVenvAdapter; + +pub(crate) static PYTHON_VENV_ADAPTER: PythonVenvAdapter = PythonVenvAdapter; + +const PYTHON_IDENTITY_FILES: [&str; 7] = [ + "pyproject.toml", + "uv.lock", + "poetry.lock", + "pdm.lock", + "Pipfile.lock", + "requirements.lock", + "requirements.txt", +]; + +static PYTHON_VENV_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = + WorkspaceEnvironmentAdapterMetadata { + canonical_identity: "trail/python-venv@1", + namespace: "trail", + name: "python-venv", + contract_major: 1, + implementation_version: env!("CARGO_PKG_VERSION"), + distribution_digest: "builtin:python-venv-plan-v2", + selectors: &["trail/python-venv@1", "python-venv", "python"], + kind: "dependency", + layer_adapter_name: "python-venv", + discovery_markers: &PYTHON_IDENTITY_FILES, + supported_operating_systems: &["linux", "macos", "windows"], + supported_architectures: &["aarch64", "x86_64"], + stability: "experimental", + description: "Automatically initialized lane-private Python virtual environment at the stable mounted lane path", + }; + +impl WorkspaceEnvironmentAdapter for PythonVenvAdapter { + fn metadata(&self) -> &'static WorkspaceEnvironmentAdapterMetadata { + &PYTHON_VENV_ADAPTER_METADATA + } + + fn component_id(&self, component_root: &str) -> Result { + let root = normalize_python_component_root(component_root)?; + Ok(if root.is_empty() { + "python-venv".to_string() + } else { + format!("python-venv:{root}") + }) + } + + fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result { + let root = normalize_python_component_root(component_root)?; + for file in PYTHON_IDENTITY_FILES { + if db + .root_file_entry(source_root, &join_python_path(&root, file))? + .is_some() + { + return Ok(true); + } + } + Ok(false) + } + + fn plan( + &self, + db: &Trail, + source_root: &ObjectId, + component_root: &str, + ) -> Result { + let component_root = normalize_python_component_root(component_root)?; + let python = resolve_python_executable()?; + let component_id = self.component_id(&component_root)?; + let mount_path = join_python_path(&component_root, ".venv"); + let implementation_version = env!("CARGO_PKG_VERSION").to_string(); + let distribution_digest = "builtin:python-venv-plan-v2".to_string(); + let mut mounted_args = vec![ + "-m".to_string(), + "venv".to_string(), + "--without-pip".to_string(), + ]; + // Python otherwise attempts interpreter symlinks first. macOS NFS + // clients can reject that operation and make venv print a warning + // before falling back to a copy, so request the supported copy mode + // directly for a clean first-run experience. + #[cfg(target_os = "macos")] + mounted_args.push("--copies".to_string()); + mounted_args.push(".venv".to_string()); + let mounted_command = WorkspaceEnvironmentCommand { + program: "python".to_string(), + resolved_program: python.path.clone(), + executable_identity: python.identity.clone(), + args: mounted_args, + working_directory: component_root.clone(), + environment: BTreeMap::new(), + remove_environment: Vec::new(), + cache_names: Vec::new(), + }; + let mut key_inputs = BTreeMap::from([ + ("component_id".to_string(), component_id.clone()), + ("component_root".to_string(), component_root.clone()), + ( + "adapter_implementation".to_string(), + implementation_version.clone(), + ), + ( + "adapter_distribution_digest".to_string(), + distribution_digest.clone(), + ), + ( + "output_contract".to_string(), + format!("writable-private:{mount_path}"), + ), + ( + "creation_phase".to_string(), + "host-mounted-initialization".to_string(), + ), + ( + "mounted_action".to_string(), + workspace_mounted_commands_identity(std::slice::from_ref(&mounted_command))?, + ), + ]); + let mut inputs = Vec::new(); + for file in PYTHON_IDENTITY_FILES { + let path = join_python_path(&component_root, file); + if let Some(entry) = db.root_file_entry(source_root, &path)? { + key_inputs.insert(format!("input:{path}"), entry.content_hash.clone()); + inputs.push(WorkspaceEnvironmentInput { + source_path: path.clone(), + staging_path: format!("project/{path}"), + entry, + }); + } + } + if inputs.is_empty() { + return Err(Error::InvalidInput(format!( + "Python component `{}` has no supported dependency manifest or lockfile", + display_python_root(&component_root) + ))); + } + inputs.sort_by(|left, right| left.source_path.cmp(&right.source_path)); + Ok(WorkspaceEnvironmentPlan { + component_id, + adapter_identity: self.identity().to_string(), + adapter_version: 1, + implementation_version, + distribution_digest, + kind: "dependency".to_string(), + dependencies: Vec::new(), + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: "dependency".to_string(), + adapter: self.layer_adapter_name().to_string(), + adapter_version: 1, + inputs: key_inputs, + tool_versions: BTreeMap::from([( + "python-executable".to_string(), + python.identity.clone(), + )]), + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: "lane-private-host-python".to_string(), + strategy: "python-venv-private-mounted-init-v2".to_string(), + }, + inputs, + source_projection: None, + pre_commands: Vec::new(), + // Python virtual environments commonly embed absolute interpreter + // and prefix paths, so the host initializes this output through + // an ephemeral candidate view mounted at the final lane path. + command: None, + mounted_commands: vec![mounted_command], + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::TrustedBuiltin, + outputs: vec![WorkspaceEnvironmentOutput { + name: "venv".to_string(), + output_path: "private/venv".to_string(), + mount_path, + policy: WorkspaceEnvironmentOutputPolicy::WritablePrivate, + create_if_missing: true, + }], + stale_reason: + "Python executable, dependency manifest or lockfile, component root, platform, or adapter policy changed" + .to_string(), + }) + } +} + +fn resolve_python_executable() -> Result { + #[cfg(windows)] + let candidates = ["python", "python3"]; + #[cfg(not(windows))] + let candidates = ["python3", "python"]; + let mut errors = Vec::new(); + for candidate in candidates { + match resolve_workspace_tool_executable(candidate) { + Ok(tool) => return Ok(tool), + Err(error) => errors.push(error.to_string()), + } + } + Err(Error::InvalidInput(format!( + "Python adapter requires `python3` or `python` on PATH: {}", + errors.join("; ") + ))) +} + +fn normalize_python_component_root(component_root: &str) -> Result { + if component_root.trim_matches('/').is_empty() { + Ok(String::new()) + } else { + normalize_relative_path(component_root) + } +} + +fn join_python_path(root: &str, child: &str) -> String { + if root.is_empty() { + child.to_string() + } else { + format!("{root}/{child}") + } +} + +fn display_python_root(root: &str) -> &str { + if root.is_empty() { + "." + } else { + root + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[cfg(any(target_os = "linux", target_os = "macos", windows))] + use std::ffi::OsStr; + + fn wait_for_mounted_crash_handshake( + child: &mut std::process::Child, + ready: &Path, + phase: &str, + ) { + for _ in 0..1_000 { + if ready.is_file() { + return; + } + if let Some(status) = child.try_wait().unwrap() { + panic!("mounted crash helper exited at {phase} before handshake: {status}"); + } + std::thread::sleep(Duration::from_millis(10)); + } + let _ = child.kill(); + panic!("timed out waiting for mounted crash helper at {phase}"); + } + + #[test] + fn python_venv_is_keyed_private_and_initialized_at_the_mounted_lane() { + if resolve_python_executable().is_err() { + return; + } + let workspace = tempfile::tempdir().unwrap(); + fs::write( + workspace.path().join("pyproject.toml"), + "[project]\nname = \"example\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + fs::write(workspace.path().join("uv.lock"), "version = 1\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "python", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + + let discovery = db.discover_workspace_environment("python", None).unwrap(); + let component = discovery + .components + .iter() + .find(|component| component.adapter_identity == "trail/python-venv@1") + .unwrap(); + assert_eq!(component.component_id, "python-venv"); + let plan = db + .plan_workspace_environment("python", "trail/python-venv@1", None) + .unwrap(); + assert_eq!(plan.commands.len(), 1); + assert_eq!(plan.commands[0].phase, "mounted_initialization"); + #[cfg(target_os = "macos")] + assert_eq!( + plan.commands[0].args, + ["-m", "venv", "--without-pip", "--copies", ".venv"] + ); + #[cfg(not(target_os = "macos"))] + assert_eq!( + plan.commands[0].args, + ["-m", "venv", "--without-pip", ".venv"] + ); + assert_eq!(plan.outputs[0].mount_path, ".venv"); + assert_eq!(plan.outputs[0].policy, "writable_private"); + assert_eq!( + plan.inputs + .iter() + .map(|input| input.source_path.as_str()) + .collect::>(), + ["pyproject.toml", "uv.lock"] + ); + #[cfg(target_os = "linux")] + if std::env::var_os("TRAIL_RUN_FUSE_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + #[cfg(target_os = "macos")] + if std::env::var_os("TRAIL_RUN_NFS_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + #[cfg(windows)] + if std::env::var_os("TRAIL_RUN_DOKAN_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + let synchronized = db + .sync_workspace_environment_component("python", "trail/python-venv@1", None, None) + .unwrap(); + assert!(synchronized.layers.is_empty()); + let output = &synchronized.generation.components[0].outputs[0]; + assert_eq!(output.policy, "writable_private"); + assert!(output.layer_id.is_none()); + assert!(db.list_workspace_layers().unwrap().is_empty()); + assert!(db + .workspace_view_paths_for_lane("python") + .unwrap() + .generated_upper + .join(".venv") + .is_dir()); + assert!(db + .workspace_view_paths_for_lane("python") + .unwrap() + .generated_upper + .join(".venv/pyvenv.cfg") + .is_file()); + } + + #[test] + fn mounted_python_initialization_crash_helper() { + let Some(workspace) = std::env::var_os("TRAIL_TEST_MOUNTED_PYTHON_WORKSPACE") else { + return; + }; + let db = Trail::open(PathBuf::from(workspace)).unwrap(); + let _ = db.sync_workspace_environment_component( + "python-crash", + "trail/python-venv@1", + None, + None, + ); + panic!("mounted Python crash helper passed its requested crash point"); + } + + #[cfg(any(target_os = "linux", target_os = "macos", windows))] + #[test] + fn killing_mounted_python_initialization_never_exposes_a_partial_generation() { + #[cfg(target_os = "linux")] + if std::env::var_os("TRAIL_RUN_FUSE_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + #[cfg(target_os = "macos")] + if std::env::var_os("TRAIL_RUN_NFS_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + #[cfg(windows)] + if std::env::var_os("TRAIL_RUN_DOKAN_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + if resolve_python_executable().is_err() { + return; + } + let workspace = tempfile::tempdir().unwrap(); + fs::write( + workspace.path().join("pyproject.toml"), + "[project]\nname = \"crash-venv\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "python-crash", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let paths = db.workspace_view_paths_for_lane("python-crash").unwrap(); + drop(db); + + let ready = workspace.path().join("mounted-python-crash.ready"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "db::lane::workspace_python::tests::mounted_python_initialization_crash_helper", + "--nocapture", + ]) + .env("RUST_TEST_THREADS", "1") + .env( + "TRAIL_TEST_CRASH_AT", + "environment_after_mounted_initialization", + ) + .env("TRAIL_TEST_CRASH_READY", &ready) + .env("TRAIL_TEST_MOUNTED_PYTHON_WORKSPACE", workspace.path()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap(); + wait_for_mounted_crash_handshake( + &mut child, + &ready, + "environment_after_mounted_initialization", + ); + child.kill().unwrap(); + let _ = child.wait().unwrap(); + + let reopened = Trail::open(workspace.path()).unwrap(); + assert!(reopened + .active_environment_generation("python-crash") + .unwrap() + .is_none()); + assert!(!paths.generated_upper.join(".venv").exists()); + let states = reopened.workspace_environment_rows("python-crash").unwrap(); + assert_eq!(states.len(), 1); + assert_eq!(states[0].status, "failed"); + assert!(!fs::read_dir(workspace.path().join(".trail/cache/staging")) + .unwrap() + .filter_map(std::result::Result::ok) + .any(|entry| entry + .file_name() + .to_string_lossy() + .starts_with("mounted-environment-envsync_"))); + } + + #[cfg(any(target_os = "linux", target_os = "macos", windows))] + #[test] + fn sync_all_initializes_nested_python_components_at_final_lane_paths() { + #[cfg(target_os = "linux")] + if std::env::var_os("TRAIL_RUN_FUSE_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + #[cfg(target_os = "macos")] + if std::env::var_os("TRAIL_RUN_NFS_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + #[cfg(windows)] + if std::env::var_os("TRAIL_RUN_DOKAN_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + if resolve_python_executable().is_err() { + return; + } + let workspace = tempfile::tempdir().unwrap(); + for component in ["services/api", "services/worker"] { + fs::create_dir_all(workspace.path().join(component)).unwrap(); + fs::write( + workspace.path().join(component).join("pyproject.toml"), + format!( + "[project]\nname = \"{}\"\nversion = \"0.1.0\"\n", + component.replace('/', "-") + ), + ) + .unwrap(); + } + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "python-all", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let report = db + .sync_all_workspace_environments("python-all", None) + .unwrap(); + assert!(report.layers.is_empty()); + assert_eq!(report.generation.components.len(), 2); + + #[cfg(target_os = "macos")] + let mounted = db.mount_nfs_cow_workdir_for_lane("python-all").unwrap(); + #[cfg(any(target_os = "linux", windows))] + let mounted = db.mount_fuse_cow_workdir_for_lane("python-all").unwrap(); + let workdir = PathBuf::from(db.lane_workdir("python-all").unwrap().workdir.unwrap()); + for component in ["services/api", "services/worker"] { + let venv = workdir.join(component).join(".venv"); + assert!(venv.join("pyvenv.cfg").is_file()); + #[cfg(windows)] + let executable = venv.join("Scripts/python.exe"); + #[cfg(not(windows))] + let executable = venv.join("bin/python"); + let prefix = Command::new(executable) + .args(["-c", "import sys; print(sys.prefix)"]) + .output() + .unwrap(); + assert!(prefix.status.success()); + #[cfg(windows)] + assert_eq!( + fs::canonicalize(String::from_utf8(prefix.stdout).unwrap().trim()).unwrap(), + fs::canonicalize(&venv).unwrap() + ); + #[cfg(not(windows))] + assert_eq!( + String::from_utf8(prefix.stdout).unwrap().trim(), + venv.to_string_lossy() + ); + } + drop(mounted); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn real_python_venvs_embed_lane_paths_and_remain_isolated() { + #[cfg(target_os = "linux")] + if std::env::var_os("TRAIL_RUN_FUSE_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + #[cfg(target_os = "macos")] + if std::env::var_os("TRAIL_RUN_NFS_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + if resolve_python_executable().is_err() { + return; + } + let workspace = tempfile::tempdir().unwrap(); + fs::write( + workspace.path().join("pyproject.toml"), + "[project]\nname = \"real-venv\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + for lane in ["python-a", "python-b"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let report = db + .sync_workspace_environment_component(lane, "trail/python-venv@1", None, None) + .unwrap(); + assert!(report.layers.is_empty()); + + #[cfg(target_os = "macos")] + let mounted = db.mount_nfs_cow_workdir_for_lane(lane).unwrap(); + #[cfg(target_os = "linux")] + let mounted = db.mount_fuse_cow_workdir_for_lane(lane).unwrap(); + let workdir = PathBuf::from(db.lane_workdir(lane).unwrap().workdir.unwrap()); + assert!(workdir.join(".venv/pyvenv.cfg").is_file()); + let venv_python = workdir.join(".venv/bin/python"); + let prefix = Command::new(&venv_python) + .args(["-c", "import sys; print(sys.prefix)"]) + .current_dir(&workdir) + .output() + .unwrap(); + assert!(prefix.status.success()); + assert_eq!( + String::from_utf8(prefix.stdout).unwrap().trim(), + workdir.join(".venv").to_string_lossy() + ); + if lane == "python-a" { + fs::write(workdir.join(".venv/lane-a.txt"), "private\n").unwrap(); + } else { + assert!(!workdir.join(".venv/lane-a.txt").exists()); + } + drop(mounted); + } + + let unchanged = db + .sync_workspace_environment_component("python-a", "trail/python-venv@1", None, None) + .unwrap(); + assert!(unchanged.layers.is_empty()); + #[cfg(target_os = "macos")] + let mounted = db.mount_nfs_cow_workdir_for_lane("python-a").unwrap(); + #[cfg(target_os = "linux")] + let mounted = db.mount_fuse_cow_workdir_for_lane("python-a").unwrap(); + let workdir = PathBuf::from(db.lane_workdir("python-a").unwrap().workdir.unwrap()); + assert!(workdir.join(".venv/lane-a.txt").is_file()); + drop(mounted); + assert!(db.list_workspace_layers().unwrap().is_empty()); + } + + #[cfg(windows)] + #[test] + fn real_windows_python_venvs_embed_lane_paths_and_remain_isolated() { + if std::env::var_os("TRAIL_RUN_DOKAN_COW_TESTS").as_deref() != Some(OsStr::new("1")) { + return; + } + if resolve_python_executable().is_err() { + return; + } + let workspace = tempfile::tempdir().unwrap(); + fs::write( + workspace.path().join("pyproject.toml"), + "[project]\nname = \"real-venv\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + for lane in ["python-a", "python-b"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + LaneWorkdirMode::FuseCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let report = db + .sync_workspace_environment_component(lane, "trail/python-venv@1", None, None) + .unwrap(); + assert!(report.layers.is_empty()); + + let mounted = db.mount_fuse_cow_workdir_for_lane(lane).unwrap(); + let workdir = PathBuf::from(db.lane_workdir(lane).unwrap().workdir.unwrap()); + assert!(workdir.join(".venv/pyvenv.cfg").is_file()); + let venv_python = workdir.join(".venv/Scripts/python.exe"); + let prefix = Command::new(&venv_python) + .args(["-c", "import sys; print(sys.prefix)"]) + .current_dir(&workdir) + .output() + .unwrap(); + assert!(prefix.status.success()); + let actual_prefix = + fs::canonicalize(String::from_utf8(prefix.stdout).unwrap().trim()).unwrap(); + let expected_prefix = fs::canonicalize(workdir.join(".venv")).unwrap(); + assert_eq!(actual_prefix, expected_prefix); + if lane == "python-a" { + fs::write(workdir.join(".venv/lane-a.txt"), "private\n").unwrap(); + } else { + assert!(!workdir.join(".venv/lane-a.txt").exists()); + } + drop(mounted); + } + + let unchanged = db + .sync_workspace_environment_component("python-a", "trail/python-venv@1", None, None) + .unwrap(); + assert!(unchanged.layers.is_empty()); + let mounted = db.mount_fuse_cow_workdir_for_lane("python-a").unwrap(); + let workdir = PathBuf::from(db.lane_workdir("python-a").unwrap().workdir.unwrap()); + assert!(workdir.join(".venv/lane-a.txt").is_file()); + drop(mounted); + assert!(db.list_workspace_layers().unwrap().is_empty()); + } +} diff --git a/trail/src/db/lane/workspace_recipe.rs b/trail/src/db/lane/workspace_recipe.rs new file mode 100644 index 0000000..dcd9c2c --- /dev/null +++ b/trail/src/db/lane/workspace_recipe.rs @@ -0,0 +1,2665 @@ +use globset::{GlobBuilder, GlobSetBuilder}; +use serde::{Deserialize, Serialize}; + +use super::workspace_environment::{ + resolve_workspace_tool_executable, ResolvedWorkspaceTool, WorkspaceEnvironmentAdapterMetadata, + WorkspaceEnvironmentCommand, WorkspaceEnvironmentDependency, WorkspaceEnvironmentEdgeType, + WorkspaceEnvironmentInput, WorkspaceEnvironmentOutput, WorkspaceEnvironmentOutputPolicy, + WorkspaceEnvironmentPlan, WorkspaceEnvironmentSandboxPolicy, +}; +use super::*; + +const RECIPE_SCHEMA: &str = "trail.environment/v1"; +const RECIPE_ADAPTER_IDENTITY: &str = "trail/command@1"; +const RECIPE_SPEC_PATHS: [&str; 2] = ["trail.environment.toml", ".trail/environment.toml"]; +const MAX_RECIPE_SPEC_BYTES: u64 = 1024 * 1024; +const MAX_RECIPE_TOTAL_SPEC_BYTES: u64 = 4 * 1024 * 1024; +const MAX_RECIPE_INCLUDE_FILES: usize = 32; +const MAX_RECIPE_INCLUDE_DEPTH: usize = 8; +const MAX_RECIPE_INPUT_FILES: usize = 100_000; +const MAX_RECIPE_INPUT_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +#[cfg(test)] +thread_local! { + static COMMAND_RECIPE_LOAD_COUNT: Cell = const { Cell::new(0) }; +} + +pub(crate) static COMMAND_RECIPE_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = + WorkspaceEnvironmentAdapterMetadata { + canonical_identity: RECIPE_ADAPTER_IDENTITY, + namespace: "trail", + name: "command", + contract_major: 1, + implementation_version: env!("CARGO_PKG_VERSION"), + distribution_digest: "builtin:command-recipe-plan-v1", + selectors: &[RECIPE_ADAPTER_IDENTITY, "command"], + kind: "generated", + layer_adapter_name: "command", + discovery_markers: &RECIPE_SPEC_PATHS, + supported_operating_systems: &["linux", "macos", "windows"], + supported_architectures: &["aarch64", "x86_64"], + stability: "experimental", + description: "Repository-declared argv command with exact inputs, a contained generated output, denied network, and host sandbox enforcement", + }; + +#[derive(Clone, Debug)] +struct CommandRecipe { + specification_digest: String, + specification_sources: BTreeMap, + profile_versions: BTreeMap, + defaults: RecipeEnvironment, + component: RecipeComponent, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RecipeSpecification { + schema: String, + #[serde(default)] + include: Vec, + #[serde(default)] + environment: RecipeEnvironment, + #[serde(default)] + profile: BTreeMap, + #[serde(default, rename = "component")] + components: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct RecipeEnvironment { + name: Option, + default_network: String, + default_scripts: String, +} + +impl Default for RecipeEnvironment { + fn default() -> Self { + Self { + name: None, + default_network: "deny".to_string(), + default_scripts: "deny".to_string(), + } + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RecipeComponentDefinition { + id: String, + #[serde(default)] + root: String, + #[serde(default)] + extends: Vec, + #[serde(default)] + adapter: Option, + #[serde(default)] + kind: Option, + #[serde(default, alias = "dependencies")] + depends_on: Vec, + #[serde(default, rename = "edge")] + edges: Vec, + #[serde(default, alias = "inputs", rename = "input")] + inputs: Vec, + #[serde(default, alias = "outputs", rename = "output")] + outputs: Vec, + #[serde(default)] + build: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RecipeProfile { + version: String, + #[serde(default)] + extends: Vec, + #[serde(default)] + adapter: Option, + #[serde(default)] + kind: Option, + #[serde(default, alias = "dependencies")] + depends_on: Vec, + #[serde(default, rename = "edge")] + edges: Vec, + #[serde(default, alias = "inputs", rename = "input")] + inputs: Vec, + #[serde(default, alias = "outputs", rename = "output")] + outputs: Vec, + #[serde(default)] + build: Option, +} + +#[derive(Clone, Debug, Default)] +struct RecipeFragment { + adapter: Option, + kind: Option, + dependencies: Vec, + edges: Vec, + inputs: Vec, + outputs: Vec, + build: Option, +} + +#[derive(Clone, Debug)] +struct ResolvedRecipeProfile { + fragment: RecipeFragment, + versions: BTreeMap, +} + +#[derive(Debug, Default)] +struct RecipeDocuments { + defaults: RecipeEnvironment, + profiles: BTreeMap, + components: Vec, + specification_sources: BTreeMap, +} + +#[derive(Clone, Debug, Serialize)] +struct RecipeComponent { + id: String, + adapter: String, + root: String, + kind: String, + dependencies: Vec, + edges: Vec, + inputs: Vec, + outputs: Vec, + build: RecipeBuild, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RecipeInput { + path: String, + #[serde(default = "default_identity_role")] + role: String, + #[serde(default = "default_bytes_format")] + format: String, + #[serde(default)] + optional: bool, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RecipeDependencyEdge { + component: String, + #[serde(rename = "type")] + edge_type: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RecipeOutput { + #[serde(default)] + name: Option, + source: String, + target: String, + #[serde(default = "default_private_seed_policy")] + policy: String, + #[serde(default = "default_host_portability")] + portability: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct RecipeBuild { + command: Vec, + #[serde(default)] + cwd: Option, + #[serde(default)] + network: Option, + #[serde(default)] + scripts: Option, + #[serde(default)] + environment: BTreeMap, +} + +fn default_recipe_kind() -> String { + "generated".to_string() +} + +fn default_identity_role() -> String { + "identity".to_string() +} + +fn default_bytes_format() -> String { + "bytes".to_string() +} + +fn default_private_seed_policy() -> String { + "immutable_seed_private".to_string() +} + +fn default_host_portability() -> String { + "host".to_string() +} + +impl Trail { + pub(crate) fn command_recipe_discovery( + &self, + source_root: &ObjectId, + component_root: Option<&str>, + ) -> Result> { + let requested_root = component_root + .map(normalize_recipe_path_allow_root) + .transpose()?; + let recipes = self.load_command_recipes(source_root)?; + Ok(recipes + .into_iter() + .filter(|recipe| { + requested_root + .as_ref() + .is_none_or(|root| root == &recipe.component.root) + }) + .map(|recipe| EnvironmentDiscoveredComponentReport { + component_id: recipe.component.id, + component_root: recipe.component.root, + kind: recipe.component.kind, + adapter_identity: RECIPE_ADAPTER_IDENTITY.to_string(), + }) + .collect()) + } + + pub(crate) fn command_recipe_plan( + &self, + source_root: &ObjectId, + component_id: &str, + ) -> Result { + let recipes = self.load_command_recipes(source_root)?; + let recipe = recipes + .into_iter() + .find(|recipe| recipe.component.id == component_id) + .ok_or_else(|| { + Error::InvalidInput(format!( + "no `{RECIPE_ADAPTER_IDENTITY}` component named `{component_id}` exists in the pinned environment specification" + )) + })?; + self.plan_command_recipe(source_root, recipe) + } + + pub(crate) fn command_recipe_plans( + &self, + source_root: &ObjectId, + component_ids: &BTreeSet, + ) -> Result> { + let recipes = self.load_command_recipes(source_root)?; + let mut plans = BTreeMap::new(); + let mut tools = BTreeMap::::new(); + for recipe in recipes { + if component_ids.contains(&recipe.component.id) { + let component_id = recipe.component.id.clone(); + let program = recipe + .component + .build + .command + .first() + .cloned() + .ok_or_else(|| { + Error::InvalidInput(format!( + "command component `{component_id}` has an empty build.command" + )) + })?; + let tool = if let Some(tool) = tools.get(&program) { + tool.clone() + } else { + let tool = resolve_workspace_tool_executable(&program)?; + tools.insert(program, tool.clone()); + tool + }; + plans.insert( + component_id, + self.plan_command_recipe_with_tool(source_root, recipe, Some(tool))?, + ); + } + } + if plans.len() != component_ids.len() { + let missing = component_ids + .iter() + .filter(|component_id| !plans.contains_key(*component_id)) + .cloned() + .collect::>(); + return Err(Error::InvalidInput(format!( + "pinned environment specification is missing command component(s): {}", + missing.join(", ") + ))); + } + Ok(plans) + } + + pub(crate) fn command_recipe_plan_for_root( + &self, + source_root: &ObjectId, + component_root: &str, + ) -> Result { + let component_root = normalize_recipe_path_allow_root(component_root)?; + let mut matching = self + .load_command_recipes(source_root)? + .into_iter() + .filter(|recipe| recipe.component.root == component_root) + .collect::>(); + match matching.len() { + 1 => self.plan_command_recipe(source_root, matching.remove(0)), + 0 => Err(Error::InvalidInput(format!( + "no `{RECIPE_ADAPTER_IDENTITY}` component is declared at `{}`", + display_recipe_root(&component_root) + ))), + count => Err(Error::InvalidInput(format!( + "{count} `{RECIPE_ADAPTER_IDENTITY}` components are declared at `{}`; synchronize all components or give each recipe a distinct root", + display_recipe_root(&component_root) + ))), + } + } + + fn load_command_recipes(&self, source_root: &ObjectId) -> Result> { + #[cfg(test)] + COMMAND_RECIPE_LOAD_COUNT.with(|count| count.set(count.get() + 1)); + let mut found = Vec::new(); + for path in RECIPE_SPEC_PATHS { + if self.root_file_entry(source_root, path)?.is_some() { + found.push(path.to_string()); + } + } + if found.len() > 1 { + return Err(Error::InvalidInput(format!( + "environment specification is ambiguous; keep only one of {}", + RECIPE_SPEC_PATHS.join(", ") + ))); + } + let Some(spec_path) = found.pop() else { + return Ok(Vec::new()); + }; + + let mut documents = RecipeDocuments::default(); + let mut visited = BTreeSet::new(); + let mut stack = Vec::new(); + let mut total_bytes = 0u64; + self.collect_recipe_document( + source_root, + &spec_path, + 0, + true, + &mut documents, + &mut visited, + &mut stack, + &mut total_bytes, + )?; + + let mut ids = BTreeSet::new(); + let mut targets = BTreeMap::::new(); + let mut profile_cache = BTreeMap::new(); + let mut recipes = Vec::with_capacity(documents.components.len()); + for definition in documents.components { + let (component, profile_versions) = + resolve_recipe_component(definition, &documents.profiles, &mut profile_cache)?; + validate_recipe_component_identity(&component.id)?; + if !ids.insert(component.id.clone()) { + return Err(Error::InvalidInput(format!( + "environment specification declares component `{}` more than once", + component.id + ))); + } + if component.adapter.as_str() != RECIPE_ADAPTER_IDENTITY { + return Err(Error::InvalidInput(format!( + "component `{}` uses unsupported declarative adapter `{}`; this specification host currently accepts only `{RECIPE_ADAPTER_IDENTITY}`", + component.id, component.adapter + ))); + } + if component.kind != "generated" { + return Err(Error::InvalidInput(format!( + "command component `{}` must use kind = \"generated\"", + component.id + ))); + } + if component.outputs.is_empty() || component.outputs.len() > 32 { + return Err(Error::InvalidInput(format!( + "command component `{}` must declare between 1 and 32 outputs", + component.id, + ))); + } + let mut output_names = BTreeSet::new(); + for (index, output) in component.outputs.iter().enumerate() { + let name = output + .name + .clone() + .unwrap_or_else(|| format!("output-{index}")); + validate_recipe_output_name(&name, &component.id)?; + if !output_names.insert(name.clone()) { + return Err(Error::InvalidInput(format!( + "command component `{}` declares output name `{name}` more than once", + component.id + ))); + } + let target = normalize_relative_path(&output.target)?; + if let Some((other_target, other_id)) = recipe_target_overlap(&targets, &target) { + return Err(Error::InvalidInput(format!( + "command component `{}` target `{target}` overlaps component `{other_id}` target `{other_target}`", + component.id + ))); + } + targets.insert(target, format!("{}:{name}", component.id)); + } + let canonical = serde_json::to_vec(&(RECIPE_SCHEMA, &component, &profile_versions))?; + recipes.push(CommandRecipe { + specification_digest: sha256_hex(&canonical), + specification_sources: documents.specification_sources.clone(), + profile_versions, + defaults: documents.defaults.clone(), + component, + }); + } + recipes.sort_by(|left, right| left.component.id.cmp(&right.component.id)); + Ok(recipes) + } + + #[allow(clippy::too_many_arguments)] + fn collect_recipe_document( + &self, + source_root: &ObjectId, + path: &str, + depth: usize, + is_root: bool, + documents: &mut RecipeDocuments, + visited: &mut BTreeSet, + stack: &mut Vec, + total_bytes: &mut u64, + ) -> Result<()> { + if depth > MAX_RECIPE_INCLUDE_DEPTH { + return Err(Error::InvalidInput(format!( + "environment specification include depth exceeds {MAX_RECIPE_INCLUDE_DEPTH} at `{path}`" + ))); + } + if let Some(index) = stack.iter().position(|candidate| candidate == path) { + let mut cycle = stack[index..].to_vec(); + cycle.push(path.to_string()); + return Err(Error::InvalidInput(format!( + "environment specification include cycle: {}", + cycle.join(" -> ") + ))); + } + if visited.contains(path) { + return Ok(()); + } + if visited.len().saturating_add(stack.len()) >= MAX_RECIPE_INCLUDE_FILES { + return Err(Error::InvalidInput(format!( + "environment specification includes more than {MAX_RECIPE_INCLUDE_FILES} files" + ))); + } + let entry = self.root_file_entry(source_root, path)?.ok_or_else(|| { + Error::InvalidInput(format!( + "environment specification include `{path}` does not exist in the pinned source root" + )) + })?; + if entry.size_bytes > MAX_RECIPE_SPEC_BYTES { + return Err(Error::InvalidInput(format!( + "environment specification `{path}` is {} bytes; the per-file maximum is {MAX_RECIPE_SPEC_BYTES}", + entry.size_bytes + ))); + } + *total_bytes = total_bytes.checked_add(entry.size_bytes).ok_or_else(|| { + Error::InvalidInput("environment specification size overflowed".to_string()) + })?; + if *total_bytes > MAX_RECIPE_TOTAL_SPEC_BYTES { + return Err(Error::InvalidInput(format!( + "environment specifications total more than {MAX_RECIPE_TOTAL_SPEC_BYTES} bytes" + ))); + } + let entries = BTreeMap::from([(path.to_string(), entry.clone())]); + let bytes = self + .materialize_entries_bytes(&entries)? + .remove(path) + .ok_or_else(|| Error::Corrupt(format!("failed to read `{path}` from source root")))?; + let text = String::from_utf8(bytes).map_err(|_| { + Error::InvalidInput(format!("environment specification `{path}` must be UTF-8")) + })?; + let specification: RecipeSpecification = toml::from_str(&text).map_err(|err| { + Error::InvalidInput(format!("invalid environment specification `{path}`: {err}")) + })?; + validate_recipe_specification_header(&specification, path)?; + + stack.push(path.to_string()); + for include in &specification.include { + let include_path = resolve_recipe_include_path(path, include)?; + self.collect_recipe_document( + source_root, + &include_path, + depth + 1, + false, + documents, + visited, + stack, + total_bytes, + )?; + } + stack.pop(); + + for (name, profile) in specification.profile { + let canonical_name = canonical_recipe_profile_name(&name)?; + if documents + .profiles + .insert(canonical_name.clone(), profile) + .is_some() + { + return Err(Error::InvalidInput(format!( + "environment specifications declare profile `{canonical_name}` more than once" + ))); + } + } + documents.components.extend(specification.components); + if is_root { + documents.defaults = specification.environment; + } + documents + .specification_sources + .insert(path.to_string(), entry.content_hash); + visited.insert(path.to_string()); + Ok(()) + } + + fn plan_command_recipe( + &self, + source_root: &ObjectId, + recipe: CommandRecipe, + ) -> Result { + self.plan_command_recipe_with_tool(source_root, recipe, None) + } + + fn plan_command_recipe_with_tool( + &self, + source_root: &ObjectId, + recipe: CommandRecipe, + resolved_tool: Option, + ) -> Result { + let component = recipe.component; + let network = component + .build + .network + .as_deref() + .unwrap_or(&recipe.defaults.default_network); + let scripts = component + .build + .scripts + .as_deref() + .unwrap_or(&recipe.defaults.default_scripts); + if network != "deny" || scripts != "deny" { + return Err(Error::InvalidInput(format!( + "command component `{}` requires network = \"deny\" and scripts = \"deny\"", + component.id + ))); + } + if component.build.command.is_empty() { + return Err(Error::InvalidInput(format!( + "command component `{}` has an empty build.command", + component.id + ))); + } + if component.build.command.len() > 4096 + || component.build.command.iter().any(|argument| { + argument.len() > 128 * 1024 + || argument.contains('\0') + || contains_sensitive_text(argument) + }) + { + return Err(Error::InvalidInput(format!( + "command component `{}` exceeds command argument limits", + component.id + ))); + } + let program = &component.build.command[0]; + if program.contains('/') || program.contains('\\') || is_shell_program(program) { + return Err(Error::InvalidInput(format!( + "command component `{}` must name a non-shell executable from PATH, not `{program}`", + component.id + ))); + } + let tool = resolved_tool + .map(Ok) + .unwrap_or_else(|| resolve_workspace_tool_executable(program))?; + validate_recipe_tool_path(self, &tool.path, &component.id)?; + let cwd = normalize_recipe_path_allow_root( + component.build.cwd.as_deref().unwrap_or(&component.root), + )?; + if !component.root.is_empty() + && cwd != component.root + && !cwd.starts_with(&format!("{}/", component.root)) + { + return Err(Error::InvalidInput(format!( + "command component `{}` cwd `{}` escapes its root `{}`", + component.id, cwd, component.root + ))); + } + let selected_inputs = self.expand_recipe_inputs(source_root, &component)?; + let mut outputs = Vec::with_capacity(component.outputs.len()); + let mut output_paths = Vec::<(String, String)>::new(); + let mut portability = None; + for (index, output) in component.outputs.iter().enumerate() { + let policy = match output.policy.as_str() { + "immutable_seed_private" => WorkspaceEnvironmentOutputPolicy::ImmutableSeedPrivate, + "writable_private" => WorkspaceEnvironmentOutputPolicy::WritablePrivate, + _ => { + return Err(Error::InvalidInput(format!( + "command component `{}` output policy must be `immutable_seed_private` or `writable_private`", + component.id + ))); + } + }; + if output.portability != "host" && output.portability != "platform" { + return Err(Error::InvalidInput(format!( + "command component `{}` output portability must be `host` or `platform`", + component.id + ))); + } + if portability + .as_deref() + .is_some_and(|value| value != output.portability) + { + return Err(Error::InvalidInput(format!( + "command component `{}` outputs must currently use one portability class", + component.id + ))); + } + portability = Some(output.portability.clone()); + let name = output + .name + .clone() + .unwrap_or_else(|| format!("output-{index}")); + validate_recipe_output_name(&name, &component.id)?; + let output_source = normalize_relative_path(&output.source)?; + let output_repository_path = join_recipe_path(&cwd, &output_source); + for (other_name, other_path) in &output_paths { + if recipe_paths_overlap(&output_repository_path, other_path) { + return Err(Error::InvalidInput(format!( + "command component `{}` output `{name}` path `{output_repository_path}` overlaps output `{other_name}` path `{other_path}`", + component.id + ))); + } + } + for path in selected_inputs.keys() { + if recipe_paths_overlap(path, &output_repository_path) { + return Err(Error::InvalidInput(format!( + "command component `{}` output `{output_repository_path}` overlaps declared input `{path}`", + component.id + ))); + } + } + let mount_path = normalize_relative_path(&output.target)?; + output_paths.push((name.clone(), output_repository_path.clone())); + outputs.push(WorkspaceEnvironmentOutput { + name, + output_path: format!("project/{output_repository_path}"), + mount_path, + policy, + create_if_missing: true, + }); + } + let output_contract = serde_json::to_string( + &outputs + .iter() + .map(|output| { + ( + &output.name, + &output.output_path, + &output.mount_path, + output.policy.as_str(), + ) + }) + .collect::>(), + )?; + let mut layer_inputs = BTreeMap::from([ + ( + "specification_digest".to_string(), + recipe.specification_digest.clone(), + ), + ("component_id".to_string(), component.id.clone()), + ("component_root".to_string(), component.root.clone()), + ( + "command".to_string(), + serde_json::to_string(&component.build.command)?, + ), + ("cwd".to_string(), cwd.clone()), + ("output_contract".to_string(), output_contract), + ("network".to_string(), "deny".to_string()), + ("scripts".to_string(), "deny".to_string()), + ( + "capability_contract".to_string(), + "fs-read:declared-inputs;fs-write:declared-outputs+isolated-home+tmp;process:exact-executable;child-exec:deny;network:deny;shell:deny;scripts:deny;secrets:deny" + .to_string(), + ), + ( + "adapter_implementation".to_string(), + env!("CARGO_PKG_VERSION").to_string(), + ), + ( + "adapter_distribution_digest".to_string(), + "builtin:command-recipe-plan-v1".to_string(), + ), + ]); + for (path, digest) in &recipe.specification_sources { + layer_inputs.insert(format!("specification_source:{path}"), digest.clone()); + } + for (profile, version) in &recipe.profile_versions { + layer_inputs.insert(format!("profile:{profile}"), version.clone()); + } + for (path, entry) in &selected_inputs { + layer_inputs.insert(format!("input:{path}"), entry.content_hash.clone()); + } + for (name, value) in &component.build.environment { + validate_recipe_environment(name, value, &component.id)?; + layer_inputs.insert(format!("environment:{name}"), value.clone()); + } + let inputs = selected_inputs + .into_iter() + .map(|(path, entry)| WorkspaceEnvironmentInput { + source_path: path.clone(), + staging_path: format!("project/{path}"), + entry, + }) + .collect::>(); + let portability_scope = if portability.as_deref() == Some("platform") { + "recipe-tool-platform" + } else { + "recipe-tool-host" + }; + Ok(WorkspaceEnvironmentPlan { + component_id: component.id, + adapter_identity: RECIPE_ADAPTER_IDENTITY.to_string(), + adapter_version: 1, + implementation_version: env!("CARGO_PKG_VERSION").to_string(), + distribution_digest: "builtin:command-recipe-plan-v1".to_string(), + kind: "generated".to_string(), + dependencies: component + .dependencies + .into_iter() + .map(|dependency| { + Ok(WorkspaceEnvironmentDependency::build_requires(dependency)) + }) + .chain(component.edges.into_iter().map(|edge| { + Ok(WorkspaceEnvironmentDependency { + component_id: edge.component, + edge_type: WorkspaceEnvironmentEdgeType::parse(&edge.edge_type)?, + }) + })) + .collect::>>()?, + resolved_dependencies: Vec::new(), + layer_key: WorkspaceLayerKeyV1 { + kind: "generated".to_string(), + adapter: "command".to_string(), + adapter_version: 1, + inputs: layer_inputs, + tool_versions: BTreeMap::from([( + format!("executable:{program}"), + tool.identity.clone(), + )]), + platform: std::env::consts::OS.to_string(), + architecture: std::env::consts::ARCH.to_string(), + portability_scope: portability_scope.to_string(), + strategy: "restricted-command-recipe-v1".to_string(), + }, + inputs, + source_projection: None, + pre_commands: Vec::new(), + command: Some(WorkspaceEnvironmentCommand { + program: program.clone(), + resolved_program: tool.path, + executable_identity: tool.identity, + args: component.build.command.into_iter().skip(1).collect(), + working_directory: if cwd.is_empty() { + "project".to_string() + } else { + format!("project/{cwd}") + }, + environment: component.build.environment, + remove_environment: Vec::new(), + cache_names: Vec::new(), + }), + mounted_commands: Vec::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + sandbox_policy: WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe, + outputs, + stale_reason: + "environment specification, declared input, executable, platform, or command policy changed" + .to_string(), + }) + } + + fn expand_recipe_inputs( + &self, + source_root: &ObjectId, + component: &RecipeComponent, + ) -> Result> { + if component.inputs.is_empty() { + return Err(Error::InvalidInput(format!( + "command component `{}` must declare at least one identity input", + component.id + ))); + } + let mut exact = Vec::new(); + let mut patterns = Vec::new(); + for input in &component.inputs { + if input.role != "identity" || input.format != "bytes" { + return Err(Error::InvalidInput(format!( + "command component `{}` currently accepts only role = \"identity\", format = \"bytes\" inputs", + component.id + ))); + } + validate_recipe_pattern(&input.path)?; + if contains_glob_meta(&input.path) { + patterns.push(input); + } else { + exact.push(input); + } + } + let exact_paths = exact + .iter() + .map(|input| normalize_relative_path(&input.path)) + .collect::>>()?; + let mut selected = self.load_root_files_for_selections(source_root, &exact_paths)?; + for input in exact { + let normalized = normalize_relative_path(&input.path)?; + let matched = selected + .keys() + .any(|path| path == &normalized || path.starts_with(&format!("{normalized}/"))); + if !matched && !input.optional { + return Err(Error::InvalidInput(format!( + "command component `{}` required input `{}` did not match a file or directory", + component.id, input.path + ))); + } + } + if !patterns.is_empty() { + let mut builder = GlobSetBuilder::new(); + for input in &patterns { + builder.add( + GlobBuilder::new(&input.path) + .literal_separator(true) + .backslash_escape(false) + .build() + .map_err(|err| { + Error::InvalidInput(format!( + "command component `{}` has invalid input glob `{}`: {err}", + component.id, input.path + )) + })?, + ); + } + let matcher = builder.build().map_err(|err| { + Error::InvalidInput(format!( + "command component `{}` input glob set is invalid: {err}", + component.id + )) + })?; + let mut matched_counts = vec![0usize; patterns.len()]; + self.for_each_root_file_chunk(source_root, 1024, |chunk| { + for (path, entry) in chunk { + let matches = matcher.matches(&path); + if matches.is_empty() { + continue; + } + for index in matches { + matched_counts[index] += 1; + } + selected.insert(path, entry); + } + Ok(()) + })?; + for (input, count) in patterns.iter().zip(matched_counts) { + if count == 0 && !input.optional { + return Err(Error::InvalidInput(format!( + "command component `{}` required input glob `{}` matched no files", + component.id, input.path + ))); + } + } + } + let total_bytes = selected.values().try_fold(0u64, |total, entry| { + total.checked_add(entry.size_bytes).ok_or_else(|| { + Error::InvalidInput(format!( + "command component `{}` input byte count overflowed", + component.id + )) + }) + })?; + if selected.len() > MAX_RECIPE_INPUT_FILES || total_bytes > MAX_RECIPE_INPUT_BYTES { + return Err(Error::InvalidInput(format!( + "command component `{}` selects {} files and {total_bytes} bytes; limits are {MAX_RECIPE_INPUT_FILES} files and {MAX_RECIPE_INPUT_BYTES} bytes", + component.id, + selected.len() + ))); + } + Ok(selected) + } +} + +fn validate_recipe_specification_header( + specification: &RecipeSpecification, + path: &str, +) -> Result<()> { + if specification.schema != RECIPE_SCHEMA { + return Err(Error::InvalidInput(format!( + "unsupported environment schema `{}` in `{path}`; expected `{RECIPE_SCHEMA}`", + specification.schema + ))); + } + if specification.environment.default_network != "deny" + || specification.environment.default_scripts != "deny" + { + return Err(Error::InvalidInput(format!( + "environment specification `{path}` must set default_network and default_scripts to `deny`" + ))); + } + let _environment_name = specification.environment.name.as_deref(); + Ok(()) +} + +fn resolve_recipe_include_path(including_path: &str, include: &str) -> Result { + if include.is_empty() + || include.starts_with('/') + || include.contains("://") + || include.contains('\\') + || contains_glob_meta(include) + || include + .split('/') + .any(|segment| segment.is_empty() || segment == "." || segment == "..") + { + return Err(Error::InvalidInput(format!( + "invalid local environment specification include `{include}` in `{including_path}`" + ))); + } + let parent = including_path + .rsplit_once('/') + .map(|(parent, _)| parent) + .unwrap_or(""); + normalize_relative_path(&join_recipe_path(parent, include)) +} + +fn validate_recipe_profile_name(name: &str) -> Result<()> { + let canonical = name.strip_prefix("profile.").unwrap_or(name); + if canonical.is_empty() + || canonical.len() > 256 + || !canonical + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_' | '/')) + || canonical + .split('/') + .any(|segment| segment.is_empty() || segment == "..") + { + return Err(Error::InvalidInput(format!( + "invalid environment recipe profile name `{name}`" + ))); + } + Ok(()) +} + +fn canonical_recipe_profile_name(name: &str) -> Result { + validate_recipe_profile_name(name)?; + Ok(name.strip_prefix("profile.").unwrap_or(name).to_string()) +} + +fn recipe_profile_fragment(profile: &RecipeProfile) -> RecipeFragment { + RecipeFragment { + adapter: profile.adapter.clone(), + kind: profile.kind.clone(), + dependencies: profile.depends_on.clone(), + edges: profile.edges.clone(), + inputs: profile.inputs.clone(), + outputs: profile.outputs.clone(), + build: profile.build.clone(), + } +} + +fn apply_recipe_fragment(target: &mut RecipeFragment, source: &RecipeFragment) { + if source.adapter.is_some() { + target.adapter.clone_from(&source.adapter); + } + if source.kind.is_some() { + target.kind.clone_from(&source.kind); + } + target.dependencies.extend(source.dependencies.clone()); + target.edges.extend(source.edges.clone()); + target.inputs.extend(source.inputs.clone()); + if !source.outputs.is_empty() { + target.outputs.clone_from(&source.outputs); + } + if source.build.is_some() { + target.build.clone_from(&source.build); + } +} + +fn resolve_recipe_profile( + requested_name: &str, + profiles: &BTreeMap, + cache: &mut BTreeMap, + stack: &mut Vec, +) -> Result { + let name = canonical_recipe_profile_name(requested_name)?; + if let Some(resolved) = cache.get(&name) { + return Ok(resolved.clone()); + } + if let Some(index) = stack.iter().position(|candidate| candidate == &name) { + let mut cycle = stack[index..].to_vec(); + cycle.push(name); + return Err(Error::InvalidInput(format!( + "environment recipe profile cycle: {}", + cycle.join(" -> ") + ))); + } + let profile = profiles.get(&name).ok_or_else(|| { + Error::InvalidInput(format!( + "environment recipe references unknown profile `{requested_name}`" + )) + })?; + if profile.version.is_empty() + || profile.version.len() > 128 + || profile.version.contains(char::is_whitespace) + || profile.version.contains('\0') + { + return Err(Error::InvalidInput(format!( + "environment recipe profile `{name}` has invalid version `{}`", + profile.version + ))); + } + + stack.push(name.clone()); + let mut fragment = RecipeFragment::default(); + let mut versions = BTreeMap::new(); + for parent in &profile.extends { + let resolved = resolve_recipe_profile(parent, profiles, cache, stack)?; + apply_recipe_fragment(&mut fragment, &resolved.fragment); + versions.extend(resolved.versions); + } + stack.pop(); + apply_recipe_fragment(&mut fragment, &recipe_profile_fragment(profile)); + versions.insert(name.clone(), profile.version.clone()); + let resolved = ResolvedRecipeProfile { fragment, versions }; + cache.insert(name, resolved.clone()); + Ok(resolved) +} + +fn resolve_recipe_component( + definition: RecipeComponentDefinition, + profiles: &BTreeMap, + cache: &mut BTreeMap, +) -> Result<(RecipeComponent, BTreeMap)> { + validate_recipe_component_identity(&definition.id)?; + let root = normalize_recipe_path_allow_root(&definition.root)?; + let mut fragment = RecipeFragment::default(); + let mut versions = BTreeMap::new(); + let mut stack = Vec::new(); + for profile_name in &definition.extends { + let resolved = resolve_recipe_profile(profile_name, profiles, cache, &mut stack)?; + apply_recipe_fragment(&mut fragment, &resolved.fragment); + versions.extend(resolved.versions); + } + apply_recipe_fragment( + &mut fragment, + &RecipeFragment { + adapter: definition.adapter, + kind: definition.kind, + dependencies: definition.depends_on, + edges: definition.edges, + inputs: definition.inputs, + outputs: definition.outputs, + build: definition.build, + }, + ); + let adapter = fragment.adapter.ok_or_else(|| { + Error::InvalidInput(format!( + "command component `{}` has no adapter after profile expansion", + definition.id + )) + })?; + let mut build = fragment.build.ok_or_else(|| { + Error::InvalidInput(format!( + "command component `{}` has no build declaration after profile expansion", + definition.id + )) + })?; + let mut inputs = fragment.inputs; + let mut outputs = fragment.outputs; + let mut dependencies = fragment.dependencies; + let edges = fragment.edges; + let mut seen_dependencies = BTreeSet::new(); + dependencies.retain(|dependency| seen_dependencies.insert(dependency.clone())); + for dependency in &dependencies { + validate_recipe_component_identity(dependency)?; + if dependency == &definition.id { + return Err(Error::InvalidInput(format!( + "environment component `{}` cannot depend on itself", + definition.id + ))); + } + } + let mut typed_edge_components = BTreeMap::new(); + for edge in edges { + if let Some(previous) = + typed_edge_components.insert(edge.component.clone(), edge.edge_type.clone()) + { + if previous != edge.edge_type { + return Err(Error::InvalidInput(format!( + "environment component `{}` declares conflicting edge types `{previous}` and `{}` for `{}`", + definition.id, edge.edge_type, edge.component + ))); + } + } + } + let edges = typed_edge_components + .into_iter() + .map(|(component, edge_type)| RecipeDependencyEdge { + component, + edge_type, + }) + .collect::>(); + for edge in &edges { + validate_recipe_component_identity(&edge.component)?; + WorkspaceEnvironmentEdgeType::parse(&edge.edge_type)?; + if edge.component == definition.id { + return Err(Error::InvalidInput(format!( + "environment component `{}` cannot depend on itself", + definition.id + ))); + } + if seen_dependencies.contains(&edge.component) { + return Err(Error::InvalidInput(format!( + "environment component `{}` declares both legacy depends_on and typed edge for `{}`", + definition.id, edge.component + ))); + } + } + for input in &mut inputs { + input.path = expand_recipe_root_template(&input.path, &root); + } + for output in &mut outputs { + output.source = expand_recipe_root_template(&output.source, &root); + output.target = expand_recipe_root_template(&output.target, &root); + } + for argument in &mut build.command { + *argument = expand_recipe_root_template(argument, &root); + } + if let Some(cwd) = &mut build.cwd { + *cwd = expand_recipe_root_template(cwd, &root); + } + for value in build.environment.values_mut() { + *value = expand_recipe_root_template(value, &root); + } + let mut seen_inputs = BTreeSet::new(); + inputs.retain(|input| { + seen_inputs.insert(( + input.path.clone(), + input.role.clone(), + input.format.clone(), + input.optional, + )) + }); + Ok(( + RecipeComponent { + id: definition.id, + adapter, + root, + kind: fragment.kind.unwrap_or_else(default_recipe_kind), + dependencies, + edges, + inputs, + outputs, + build, + }, + versions, + )) +} + +fn expand_recipe_root_template(value: &str, root: &str) -> String { + if root.is_empty() { + value.replace("{root}/", "").replace("{root}", ".") + } else { + value.replace("{root}", root) + } +} + +fn validate_recipe_component_identity(id: &str) -> Result<()> { + if id.is_empty() + || id.len() > 256 + || !id + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_' | ':' | '/')) + || id.starts_with('/') + || id.ends_with('/') + || id + .split('/') + .any(|segment| segment.is_empty() || segment == "..") + { + return Err(Error::InvalidInput(format!( + "invalid command component id `{id}`" + ))); + } + Ok(()) +} + +fn validate_recipe_output_name(name: &str, component_id: &str) -> Result<()> { + if name.is_empty() + || name.len() > 128 + || !name + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_')) + { + return Err(Error::InvalidInput(format!( + "command component `{component_id}` has invalid output name `{name}`" + ))); + } + Ok(()) +} + +fn validate_recipe_pattern(pattern: &str) -> Result<()> { + if pattern.is_empty() + || pattern.starts_with('/') + || pattern.contains('\\') + || pattern.split('/').any(|segment| segment == "..") + { + return Err(Error::InvalidInput(format!( + "invalid repository-relative recipe input `{pattern}`" + ))); + } + normalize_relative_path(pattern).map(|_| ()) +} + +fn contains_glob_meta(path: &str) -> bool { + path.bytes() + .any(|byte| matches!(byte, b'*' | b'?' | b'[' | b'{' | b'!')) +} + +pub(super) fn validate_recipe_environment( + name: &str, + value: &str, + component_id: &str, +) -> Result<()> { + let valid_name = !name.is_empty() + && name + .chars() + .all(|ch| ch == '_' || ch.is_ascii_alphanumeric()); + let upper = name.to_ascii_uppercase(); + let sensitive = [ + "TOKEN", + "SECRET", + "PASSWORD", + "PASSWD", + "CREDENTIAL", + "PRIVATE_KEY", + "AUTH", + ] + .iter() + .any(|needle| upper.contains(needle)); + if !valid_name + || sensitive + || matches!( + upper.as_str(), + "PATH" | "HOME" | "TMP" | "TMPDIR" | "TEMP" | "SHELL" | "DYLD_INSERT_LIBRARIES" + ) + || value.contains('\0') + || value.len() > 128 * 1024 + || contains_sensitive_text(value) + { + return Err(Error::InvalidInput(format!( + "command component `{component_id}` has forbidden environment entry `{name}`" + ))); + } + Ok(()) +} + +pub(super) fn validate_recipe_tool_path(db: &Trail, path: &Path, component_id: &str) -> Result<()> { + let canonical = fs::canonicalize(path)?; + let mut forbidden = vec![db.workspace_root.clone(), db.db_dir.clone()]; + if let Some(home) = std::env::var_os("HOME") { + forbidden.push(PathBuf::from(home)); + } + if forbidden.iter().any(|root| canonical.starts_with(root)) { + return Err(Error::InvalidInput(format!( + "command component `{component_id}` executable `{}` is under a mutable workspace or user home; bind a host-managed toolchain instead", + canonical.display() + ))); + } + Ok(()) +} + +pub(super) fn is_shell_program(program: &str) -> bool { + matches!( + program.to_ascii_lowercase().as_str(), + "sh" | "bash" + | "zsh" + | "fish" + | "dash" + | "ksh" + | "csh" + | "tcsh" + | "cmd" + | "cmd.exe" + | "powershell" + | "powershell.exe" + | "pwsh" + ) +} + +fn normalize_recipe_path_allow_root(path: &str) -> Result { + if path.trim_matches('/').is_empty() || path == "." { + Ok(String::new()) + } else { + normalize_relative_path(path) + } +} + +fn join_recipe_path(root: &str, child: &str) -> String { + if root.is_empty() { + child.to_string() + } else { + format!("{root}/{child}") + } +} + +fn recipe_paths_overlap(left: &str, right: &str) -> bool { + left == right + || left.starts_with(&format!("{right}/")) + || right.starts_with(&format!("{left}/")) +} + +fn recipe_target_overlap<'a>( + targets: &'a BTreeMap, + target: &str, +) -> Option<(&'a str, &'a str)> { + if let Some((stored, owner)) = targets.get_key_value(target) { + return Some((stored, owner)); + } + let mut prefix = String::new(); + let mut segments = target.split('/').peekable(); + while let Some(segment) = segments.next() { + if !prefix.is_empty() { + prefix.push('/'); + } + prefix.push_str(segment); + if segments.peek().is_some() { + if let Some((stored, owner)) = targets.get_key_value(&prefix) { + return Some((stored, owner)); + } + } + } + targets + .range(target.to_string()..) + .next() + .filter(|(stored, _)| stored.starts_with(&format!("{target}/"))) + .map(|(stored, owner)| (stored.as_str(), owner.as_str())) +} + +fn display_recipe_root(root: &str) -> &str { + if root.is_empty() { + "." + } else { + root + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ordered_recipe_target_overlap_finds_ancestors_and_descendants() { + let descendants = BTreeMap::from([("generated/nested".to_string(), "child".to_string())]); + assert_eq!( + recipe_target_overlap(&descendants, "generated").map(|(path, owner)| (path, owner)), + Some(("generated/nested", "child")) + ); + let ancestors = BTreeMap::from([("generated".to_string(), "parent".to_string())]); + assert_eq!( + recipe_target_overlap(&ancestors, "generated/nested") + .map(|(path, owner)| (path, owner)), + Some(("generated", "parent")) + ); + assert!(recipe_target_overlap(&ancestors, "generated-sibling").is_none()); + } + + fn write_recipe_workspace(workspace: &Path, command: &[&str]) { + write_recipe_workspace_with_policy(workspace, command, "immutable_seed_private"); + } + + fn write_recipe_workspace_with_policy(workspace: &Path, command: &[&str], policy: &str) { + fs::write(workspace.join("input.txt"), "declared input\n").unwrap(); + let command = command + .iter() + .map(|value| format!("{:?}", value)) + .collect::>() + .join(", "); + fs::write( + workspace.join("trail.environment.toml"), + format!( + r#"schema = "trail.environment/v1" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "generated.copy" +adapter = "trail/command@1" +root = "." +kind = "generated" + +[[component.input]] +path = "*.txt" +role = "identity" +format = "bytes" + +[component.build] +command = [{command}] +cwd = "." +network = "deny" +scripts = "deny" + +[[component.output]] +name = "generated" +source = "generated" +target = ".trail-generated/copy" +policy = "{policy}" +portability = "host" +"# + ), + ) + .unwrap(); + } + + fn open_recipe_lane(command: &[&str]) -> (tempfile::TempDir, Trail) { + let workspace = tempfile::tempdir().unwrap(); + write_recipe_workspace(workspace.path(), command); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + for lane in ["recipe-a", "recipe-b"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + mode.clone(), + None, + None, + None, + &[], + false, + ) + .unwrap(); + } + (workspace, db) + } + + fn open_recipe_graph(specification: &str) -> (tempfile::TempDir, Trail) { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("input.txt"), "graph\n").unwrap(); + fs::write( + workspace.path().join("trail.environment.toml"), + specification, + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "graph", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + (workspace, db) + } + + #[test] + fn command_recipe_discovery_and_plan_are_side_effect_free_and_exact() { + let (_workspace, db) = open_recipe_lane(&["cp", "input.txt", "generated/copied.txt"]); + let discovery = db.discover_workspace_environment("recipe-a", None).unwrap(); + assert_eq!(discovery.components.len(), 1); + assert_eq!(discovery.components[0].component_id, "generated.copy"); + assert_eq!( + discovery.components[0].adapter_identity, + RECIPE_ADAPTER_IDENTITY + ); + let plan = db + .command_recipe_plan(&discovery.source_root, "generated.copy") + .unwrap(); + assert_eq!( + plan.sandbox_policy, + WorkspaceEnvironmentSandboxPolicy::RestrictedRecipe + ); + assert_eq!(plan.outputs[0].mount_path, ".trail-generated/copy"); + assert_eq!(plan.inputs.len(), 1); + assert_eq!(plan.inputs[0].source_path, "input.txt"); + let report = db + .plan_workspace_environment("recipe-a", RECIPE_ADAPTER_IDENTITY, None) + .unwrap(); + assert_eq!(report.component_id, "generated.copy"); + assert_eq!(report.capabilities.network, "deny"); + assert_eq!(report.capabilities.shell, "deny"); + assert_eq!(report.capabilities.scripts, "deny"); + assert_eq!(report.capabilities.secrets, "deny"); + assert_eq!(report.capabilities.filesystem_read, vec!["input.txt"]); + assert_eq!( + report.capabilities.filesystem_write, + vec!["project/generated"] + ); + assert!(db.list_workspace_layers().unwrap().is_empty()); + } + + #[test] + fn local_include_and_versioned_profile_expand_into_a_canonical_plan() { + let workspace = tempfile::tempdir().unwrap(); + fs::create_dir_all(workspace.path().join("config")).unwrap(); + fs::create_dir_all(workspace.path().join("apps/api")).unwrap(); + fs::write( + workspace.path().join("apps/api/input.txt"), + "profile input\n", + ) + .unwrap(); + fs::write( + workspace.path().join("trail.environment.toml"), + r#"schema = "trail.environment/v1" +include = ["config/copy.toml"] + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "generated.profile-copy" +root = "apps/api" +extends = ["profile.copy"] +"#, + ) + .unwrap(); + fs::write( + workspace.path().join("config/copy.toml"), + r#"schema = "trail.environment/v1" + +[profile.copy] +version = "1.2.0" +adapter = "trail/command@1" +kind = "generated" +inputs = [{ path = "{root}/input.txt", role = "identity", format = "bytes" }] +outputs = [{ source = "generated", target = "{root}/generated", policy = "immutable_seed_private", portability = "host" }] + +[profile.copy.build] +command = ["cp", "input.txt", "generated/copied.txt"] +cwd = "{root}" +network = "deny" +scripts = "deny" +"#, + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let root = db.resolve_branch_ref("main").unwrap().root_id; + let plan = db + .command_recipe_plan(&root, "generated.profile-copy") + .unwrap(); + assert_eq!( + plan.command.as_ref().unwrap().working_directory, + "project/apps/api" + ); + assert_eq!(plan.outputs[0].mount_path, "apps/api/generated"); + assert_eq!(plan.inputs[0].source_path, "apps/api/input.txt"); + assert_eq!( + plan.layer_key + .inputs + .get("profile:copy") + .map(String::as_str), + Some("1.2.0") + ); + assert!(plan + .layer_key + .inputs + .contains_key("specification_source:trail.environment.toml")); + assert!(plan + .layer_key + .inputs + .contains_key("specification_source:config/copy.toml")); + assert!(db.list_workspace_layers().unwrap().is_empty()); + } + + #[test] + fn component_dependencies_finalize_in_topological_order_and_fail_closed() { + let chain = r#"schema = "trail.environment/v1" + +[[component]] +id = "c" +adapter = "trail/command@1" +kind = "generated" +depends_on = ["b"] +inputs = [{ path = "input.txt" }] +outputs = [{ source = "out-c", target = ".trail-generated/c" }] +[component.build] +command = ["cp", "input.txt", "out-c/value.txt"] + +[[component]] +id = "a" +adapter = "trail/command@1" +kind = "generated" +inputs = [{ path = "input.txt" }] +outputs = [{ source = "out-a", target = ".trail-generated/a" }] +[component.build] +command = ["cp", "input.txt", "out-a/value.txt"] + +[[component]] +id = "b" +adapter = "trail/command@1" +kind = "generated" +depends_on = ["a"] +inputs = [{ path = "input.txt" }] +outputs = [{ source = "out-b", target = ".trail-generated/b" }] +[component.build] +command = ["cp", "input.txt", "out-b/value.txt"] +"#; + let (_workspace, db) = open_recipe_graph(chain); + let discovery = db.discover_workspace_environment("graph", None).unwrap(); + let finalized = db + .plan_discovered_environment_graph(&discovery.source_root, &discovery.components) + .unwrap(); + assert_eq!( + finalized + .iter() + .map(|(plan, _)| plan.component_id.as_str()) + .collect::>(), + ["a", "b", "c"] + ); + assert_eq!( + finalized[1].0.layer_key.inputs["dependency:a"], + finalized[0].1 + ); + assert_eq!( + finalized[2].0.layer_key.inputs["dependency:b"], + finalized[1].1 + ); + let graph = db.workspace_environment_graph("graph", None).unwrap(); + assert_eq!( + graph + .nodes + .iter() + .map(|node| node.component_id.as_str()) + .collect::>(), + ["a", "b", "c"] + ); + assert_eq!(graph.edges.len(), 2); + assert_eq!(graph.edges[0].source_component_id, "a"); + assert_eq!(graph.edges[0].target_component_id, "b"); + assert_eq!(graph.edges[0].edge_type, "build_requires"); + assert_eq!( + graph.edges[0].source_component_key, + graph.nodes[0].component_key + ); + assert_eq!(graph.edges[1].source_component_id, "b"); + assert_eq!(graph.edges[1].target_component_id, "c"); + assert!(db.list_workspace_layers().unwrap().is_empty()); + let report = db + .plan_workspace_environment_component("graph", RECIPE_ADAPTER_IDENTITY, None, Some("c")) + .unwrap(); + assert_eq!(report.dependencies, ["b"]); + assert_eq!(report.component_key, finalized[2].1); + let error = db + .sync_workspace_environment_component("graph", RECIPE_ADAPTER_IDENTITY, None, Some("c")) + .unwrap_err(); + assert!(error + .to_string() + .contains("requires `b`, which is not attached")); + assert!(error.to_string().contains("env sync-all graph")); + + let missing = chain.replace("depends_on = [\"b\"]", "depends_on = [\"missing\"]"); + let (_workspace, db) = open_recipe_graph(&missing); + let discovery = db.discover_workspace_environment("graph", None).unwrap(); + let error = db + .plan_discovered_environment_graph(&discovery.source_root, &discovery.components) + .unwrap_err(); + assert!(error + .to_string() + .contains("component `c` requires missing component `missing`")); + + let cycle = chain.replace( + "id = \"a\"\nadapter = \"trail/command@1\"\nkind = \"generated\"", + "id = \"a\"\nadapter = \"trail/command@1\"\nkind = \"generated\"\ndepends_on = [\"c\"]", + ); + let (_workspace, db) = open_recipe_graph(&cycle); + let discovery = db.discover_workspace_environment("graph", None).unwrap(); + let error = db + .plan_discovered_environment_graph(&discovery.source_root, &discovery.components) + .unwrap_err(); + assert!(error + .to_string() + .contains("dependency cycle: a -> c -> b -> a")); + } + + #[test] + fn recipe_typed_edges_are_reported_and_only_identity_edges_change_keys() { + let specification = r#"schema = "trail.environment/v1" + +[[component]] +id = "source" +adapter = "trail/command@1" +kind = "generated" +inputs = [{ path = "input.txt" }] +outputs = [{ source = "out-source", target = ".trail-generated/source" }] +[component.build] +command = ["cp", "input.txt", "out-source/value.txt"] + +[[component]] +id = "runtime" +adapter = "trail/command@1" +kind = "generated" +inputs = [{ path = "input.txt" }] +outputs = [{ source = "out-runtime", target = ".trail-generated/runtime" }] +[[component.edge]] +component = "source" +type = "runtime_requires" +[component.build] +command = ["cp", "input.txt", "out-runtime/value.txt"] + +[[component]] +id = "configuration" +adapter = "trail/command@1" +kind = "generated" +inputs = [{ path = "input.txt" }] +outputs = [{ source = "out-configuration", target = ".trail-generated/configuration" }] +[[component.edge]] +component = "source" +type = "invalidates_with" +[component.build] +command = ["cp", "input.txt", "out-configuration/value.txt"] +"#; + let (_workspace, db) = open_recipe_graph(specification); + let discovery = db.discover_workspace_environment("graph", None).unwrap(); + let finalized = db + .plan_discovered_environment_graph(&discovery.source_root, &discovery.components) + .unwrap(); + let by_id = finalized + .iter() + .map(|(plan, key)| (plan.component_id.as_str(), (plan, key))) + .collect::>(); + assert!(!by_id["runtime"] + .0 + .layer_key + .inputs + .keys() + .any(|key| key.starts_with("dependency:"))); + assert_eq!( + by_id["configuration"].0.layer_key.inputs["dependency:invalidates_with:source"], + *by_id["source"].1 + ); + let graph = db.workspace_environment_graph("graph", None).unwrap(); + assert_eq!( + graph + .edges + .iter() + .map(|edge| (edge.target_component_id.as_str(), edge.edge_type.as_str())) + .collect::>(), + [ + ("configuration", "invalidates_with"), + ("runtime", "runtime_requires") + ] + ); + } + + #[test] + fn thousand_component_graph_parses_recipes_twice_not_once_per_component() { + let count = 1_000usize; + let program = if cfg!(windows) { "where" } else { "cp" }; + let mut specification = String::from("schema = \"trail.environment/v1\"\n"); + for index in (0..count).rev() { + let component_id = format!("component-{index:04}"); + let dependency = (index > 0) + .then(|| format!("depends_on = [\"component-{:04}\"]\n", index - 1)) + .unwrap_or_default(); + specification.push_str(&format!( + r#" +[[component]] +id = "{component_id}" +adapter = "trail/command@1" +kind = "generated" +{dependency}inputs = [{{ path = "input.txt" }}] +outputs = [{{ source = "out-{index:04}", target = ".trail-generated/{component_id}" }}] +[component.build] +command = ["{program}", "input.txt", "out-{index:04}/value.txt"] +"# + )); + } + let (_workspace, db) = open_recipe_graph(&specification); + COMMAND_RECIPE_LOAD_COUNT.with(|loads| loads.set(0)); + let graph = db.workspace_environment_graph("graph", None).unwrap(); + assert_eq!(graph.nodes.len(), count); + assert_eq!(graph.edges.len(), count - 1); + assert_eq!(graph.nodes[0].component_id, "component-0000"); + assert_eq!(graph.nodes[count - 1].component_id, "component-0999"); + COMMAND_RECIPE_LOAD_COUNT.with(|loads| assert_eq!(loads.get(), 2)); + let page = db + .workspace_environment_graph_page("graph", None, 400, 250) + .unwrap(); + assert_eq!(page.total_nodes, count as u64); + assert_eq!(page.total_edges, (count - 1) as u64); + assert_eq!(page.offset, 400); + assert_eq!(page.next_offset, Some(650)); + assert_eq!(page.nodes.len(), 250); + assert_eq!(page.edges.len(), 250); + assert_eq!(page.nodes[0].component_id, "component-0400"); + COMMAND_RECIPE_LOAD_COUNT.with(|loads| assert_eq!(loads.get(), 4)); + assert!(db.list_workspace_layers().unwrap().is_empty()); + } + + #[test] + fn recipe_include_and_profile_cycles_fail_with_the_full_chain() { + let include_workspace = tempfile::tempdir().unwrap(); + fs::create_dir_all(include_workspace.path().join("config")).unwrap(); + fs::write( + include_workspace.path().join("trail.environment.toml"), + "schema = \"trail.environment/v1\"\ninclude = [\"config/a.toml\"]\n", + ) + .unwrap(); + fs::write( + include_workspace.path().join("config/a.toml"), + "schema = \"trail.environment/v1\"\ninclude = [\"b.toml\"]\n", + ) + .unwrap(); + fs::write( + include_workspace.path().join("config/b.toml"), + "schema = \"trail.environment/v1\"\ninclude = [\"a.toml\"]\n", + ) + .unwrap(); + Trail::init( + include_workspace.path(), + "main", + InitImportMode::WorkingTree, + false, + ) + .unwrap(); + let include_db = Trail::open(include_workspace.path()).unwrap(); + let root = include_db.resolve_branch_ref("main").unwrap().root_id; + let error = include_db.load_command_recipes(&root).unwrap_err(); + assert!(error + .to_string() + .contains("config/a.toml -> config/b.toml -> config/a.toml")); + + let profile_workspace = tempfile::tempdir().unwrap(); + fs::write( + profile_workspace.path().join("trail.environment.toml"), + r#"schema = "trail.environment/v1" + +[profile.a] +version = "1" +extends = ["profile.b"] + +[profile.b] +version = "1" +extends = ["profile.a"] + +[[component]] +id = "generated.cycle" +extends = ["profile.a"] +"#, + ) + .unwrap(); + Trail::init( + profile_workspace.path(), + "main", + InitImportMode::WorkingTree, + false, + ) + .unwrap(); + let profile_db = Trail::open(profile_workspace.path()).unwrap(); + let root = profile_db.resolve_branch_ref("main").unwrap().root_id; + let error = profile_db.load_command_recipes(&root).unwrap_err(); + assert!(error.to_string().contains("a -> b -> a")); + } + + #[test] + fn recipe_includes_reject_remote_globbed_and_traversing_paths() { + for include in ["https://example.invalid/x.toml", "*.toml", "../x.toml"] { + let workspace = tempfile::tempdir().unwrap(); + fs::write( + workspace.path().join("trail.environment.toml"), + format!("schema = \"trail.environment/v1\"\ninclude = [{include:?}]\n"), + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let root = db.resolve_branch_ref("main").unwrap().root_id; + let error = db.load_command_recipes(&root).unwrap_err(); + assert!(error + .to_string() + .contains("invalid local environment specification include")); + } + } + + #[test] + fn command_recipe_rejects_shells_before_execution() { + let (_workspace, db) = open_recipe_lane(&["sh", "-c", "true"]); + let discovery = db.discover_workspace_environment("recipe-a", None).unwrap(); + let error = db + .command_recipe_plan(&discovery.source_root, "generated.copy") + .unwrap_err(); + assert!(error.to_string().contains("non-shell executable")); + assert!(db.list_workspace_layers().unwrap().is_empty()); + } + + #[test] + fn command_recipe_component_selector_disambiguates_shared_roots() { + let workspace = tempfile::tempdir().unwrap(); + write_recipe_workspace( + workspace.path(), + &["cp", "input.txt", "generated/copied.txt"], + ); + let mut specification = + fs::read_to_string(workspace.path().join("trail.environment.toml")).unwrap(); + specification.push_str( + r#" +[[component]] +id = "generated.second" +adapter = "trail/command@1" +root = "." +kind = "generated" +inputs = [{ path = "input.txt", role = "identity", format = "bytes" }] + +[component.build] +command = ["cp", "input.txt", "generated-second/copied.txt"] +cwd = "." +network = "deny" +scripts = "deny" + +[[component.output]] +source = "generated-second" +target = ".trail-generated/second" +policy = "immutable_seed_private" +portability = "host" +"#, + ); + fs::write( + workspace.path().join("trail.environment.toml"), + specification, + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "recipes", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let error = db + .plan_workspace_environment("recipes", RECIPE_ADAPTER_IDENTITY, None) + .unwrap_err(); + assert!(error.to_string().contains("2 `trail/command@1` components")); + let selected = db + .plan_workspace_environment_component( + "recipes", + RECIPE_ADAPTER_IDENTITY, + None, + Some("generated.second"), + ) + .unwrap(); + assert_eq!(selected.component_id, "generated.second"); + assert_eq!(selected.mount_path, ".trail-generated/second"); + } + + #[cfg(target_os = "macos")] + #[test] + fn restricted_command_recipe_builds_once_and_reuses_a_verified_layer() { + let (_workspace, db) = open_recipe_lane(&["cp", "input.txt", "generated/copied.txt"]); + let first_batch = db + .sync_all_workspace_environments("recipe-a", None) + .unwrap(); + assert_eq!(first_batch.generation.components.len(), 1); + assert_eq!( + first_batch.generation.components[0].component_id, + "generated.copy" + ); + let first = &first_batch.layers[0]; + let second = db + .sync_workspace_environment("recipe-b", "command", None) + .unwrap(); + assert_eq!(first.layer_id, second.layer_id); + assert_eq!(first.adapter, "command"); + assert_eq!( + fs::read(Path::new(&first.storage_path).join("copied.txt")).unwrap(), + b"declared input\n" + ); + assert_eq!(db.list_workspace_layers().unwrap().len(), 1); + for lane in ["recipe-a", "recipe-b"] { + let status = db.environment_component_status(lane).unwrap(); + assert_eq!(status[0].status, "ready"); + assert_eq!(status[0].component.kind, "generated"); + assert_eq!(status[0].adapter.name, "command"); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn writable_private_recipe_has_no_fake_layer_and_preserves_compatible_lane_state() { + let workspace = tempfile::tempdir().unwrap(); + write_recipe_workspace_with_policy( + workspace.path(), + &["cp", "input.txt", "generated/copied.txt"], + "writable_private", + ); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + for lane in ["private-a", "private-b"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + LaneWorkdirMode::NfsCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + } + + let plan = db + .plan_workspace_environment("private-a", RECIPE_ADAPTER_IDENTITY, None) + .unwrap(); + assert_eq!(plan.outputs[0].policy, "writable_private"); + let first = db + .sync_workspace_environment_component("private-a", RECIPE_ADAPTER_IDENTITY, None, None) + .unwrap(); + assert!(first.layers.is_empty()); + let output = &first.generation.components[0].outputs[0]; + assert_eq!(output.policy, "writable_private"); + assert!(output.layer_id.is_none()); + assert!(output.storage_identity.starts_with("private_")); + assert!(db.list_workspace_layers().unwrap().is_empty()); + assert_eq!( + db.workspace_layer_key_by_cache_key(&plan.component_key) + .unwrap() + .unwrap() + .strategy, + "restricted-command-recipe-v1" + ); + + let mounted = db.mount_nfs_cow_workdir_for_lane("private-a").unwrap(); + let workdir = PathBuf::from(db.lane_workdir("private-a").unwrap().workdir.unwrap()); + let copied = workdir.join(".trail-generated/copy/copied.txt"); + assert_eq!(fs::read(&copied).unwrap(), b"declared input\n"); + fs::write(&copied, "lane-private mutation\n").unwrap(); + drop(mounted); + + let second = db + .sync_workspace_environment_component("private-a", RECIPE_ADAPTER_IDENTITY, None, None) + .unwrap(); + assert!(second.layers.is_empty()); + assert_eq!( + second.generation.predecessor_generation_id.as_deref(), + Some(first.generation.generation_id.as_str()) + ); + let mounted = db.mount_nfs_cow_workdir_for_lane("private-a").unwrap(); + assert_eq!(fs::read(&copied).unwrap(), b"lane-private mutation\n"); + fs::write(workdir.join("input.txt"), "changed input\n").unwrap(); + drop(mounted); + db.checkpoint_lane_workspace("private-a", Some("change private input".to_string())) + .unwrap(); + let readiness = db.lane_readiness("private-a").unwrap(); + assert!(readiness + .blockers + .iter() + .any(|blocker| blocker.code == "dependency_environment_stale")); + let explanation = db + .explain_workspace_environment_staleness("private-a", "generated.copy") + .unwrap(); + assert!(explanation.provenance_complete); + assert!( + explanation.changes.iter().any(|change| { + change.dimension == "input" + && change.name == "input.txt" + && change.change == "modified" + }), + "{:?}", + explanation.changes + ); + let rebuilt = db + .sync_workspace_environment_component("private-a", RECIPE_ADAPTER_IDENTITY, None, None) + .unwrap(); + assert!(rebuilt.layers.is_empty()); + let mounted = db.mount_nfs_cow_workdir_for_lane("private-a").unwrap(); + assert_eq!(fs::read(&copied).unwrap(), b"changed input\n"); + fs::remove_dir_all(workdir.join(".trail-generated/copy")).unwrap(); + drop(mounted); + let restored = db + .sync_workspace_environment_component("private-a", RECIPE_ADAPTER_IDENTITY, None, None) + .unwrap(); + assert!(restored.layers.is_empty()); + let mounted = db.mount_nfs_cow_workdir_for_lane("private-a").unwrap(); + assert_eq!(fs::read(&copied).unwrap(), b"changed input\n"); + drop(mounted); + + let other = db + .sync_workspace_environment_component("private-b", RECIPE_ADAPTER_IDENTITY, None, None) + .unwrap(); + assert!(other.layers.is_empty()); + let mounted = db.mount_nfs_cow_workdir_for_lane("private-b").unwrap(); + let other_workdir = PathBuf::from(db.lane_workdir("private-b").unwrap().workdir.unwrap()); + assert_eq!( + fs::read(other_workdir.join(".trail-generated/copy/copied.txt")).unwrap(), + b"declared input\n" + ); + drop(mounted); + } + + #[cfg(target_os = "macos")] + #[test] + fn sync_all_atomically_composes_shared_and_private_components() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("input.txt"), "composed\n").unwrap(); + fs::write(workspace.path().join("private-input.txt"), "private base\n").unwrap(); + fs::write( + workspace.path().join("trail.environment.toml"), + r#"schema = "trail.environment/v1" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "generated.shared" +adapter = "trail/command@1" +root = "." +kind = "generated" +inputs = [{ path = "input.txt", role = "identity", format = "bytes" }] +outputs = [{ name = "shared", source = "shared", target = ".trail-generated/shared", policy = "immutable_seed_private", portability = "host" }] +[component.build] +command = ["cp", "input.txt", "shared/value.txt"] +cwd = "." +network = "deny" +scripts = "deny" + +[[component]] +id = "generated.private" +adapter = "trail/command@1" +root = "." +kind = "generated" +depends_on = ["generated.shared"] +inputs = [{ path = "private-input.txt", role = "identity", format = "bytes" }] +outputs = [{ name = "private", source = "private", target = ".trail-generated/private", policy = "writable_private", portability = "host" }] +[component.build] +command = ["cp", "private-input.txt", "private/value.txt"] +cwd = "." +network = "deny" +scripts = "deny" +"#, + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "composed", + Some("main"), + LaneWorkdirMode::NfsCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + + let first = db + .sync_all_workspace_environments("composed", None) + .unwrap(); + assert_eq!(first.layers.len(), 1); + assert_eq!(first.generation.components.len(), 2); + let private_component = first + .generation + .components + .iter() + .find(|component| component.component_id == "generated.private") + .unwrap(); + assert_eq!(private_component.dependencies.len(), 1); + assert_eq!( + private_component.dependencies[0].component_id, + "generated.shared" + ); + assert_eq!( + private_component.dependencies[0].edge_type, + "build_requires" + ); + assert_eq!( + private_component.dependencies[0].component_key, + first + .generation + .components + .iter() + .find(|component| component.component_id == "generated.shared") + .unwrap() + .component_key + ); + let policies = first + .generation + .components + .iter() + .flat_map(|component| &component.outputs) + .map(|output| output.policy.as_str()) + .collect::>(); + assert_eq!( + policies, + BTreeSet::from(["immutable_seed_private", "writable_private"]) + ); + let mounted = db.mount_nfs_cow_workdir_for_lane("composed").unwrap(); + let workdir = PathBuf::from(db.lane_workdir("composed").unwrap().workdir.unwrap()); + assert_eq!( + fs::read(workdir.join(".trail-generated/shared/value.txt")).unwrap(), + b"composed\n" + ); + let private = workdir.join(".trail-generated/private/value.txt"); + assert_eq!(fs::read(&private).unwrap(), b"private base\n"); + fs::write(&private, "preserved private\n").unwrap(); + drop(mounted); + + let second = db + .sync_all_workspace_environments("composed", None) + .unwrap(); + assert_eq!(second.layers.len(), 1); + assert_eq!(second.layers[0].layer_id, first.layers[0].layer_id); + let mounted = db.mount_nfs_cow_workdir_for_lane("composed").unwrap(); + assert_eq!(fs::read(&private).unwrap(), b"preserved private\n"); + drop(mounted); + + let mounted = db.mount_nfs_cow_workdir_for_lane("composed").unwrap(); + fs::write(workdir.join("input.txt"), "changed upstream\n").unwrap(); + drop(mounted); + db.checkpoint_lane_workspace("composed", Some("change upstream".to_string())) + .unwrap(); + let readiness = db.lane_readiness("composed").unwrap(); + assert!(readiness + .blockers + .iter() + .any(|blocker| blocker.code == "dependency_environment_stale")); + let explanation = db + .explain_workspace_environment_staleness("composed", "generated.private") + .unwrap(); + assert!(explanation.changes.iter().any(|change| { + change.dimension == "input" + && change.name == "dependency:generated.shared" + && change.change == "modified" + })); + let old_private_dependency_key = second + .generation + .components + .iter() + .find(|component| component.component_id == "generated.private") + .unwrap() + .dependencies[0] + .component_key + .clone(); + let upstream_only = db + .sync_workspace_environment_component( + "composed", + RECIPE_ADAPTER_IDENTITY, + None, + Some("generated.shared"), + ) + .unwrap(); + let private_after_upstream = upstream_only + .generation + .components + .iter() + .find(|component| component.component_id == "generated.private") + .unwrap(); + assert_eq!( + private_after_upstream.dependencies[0].component_key, + old_private_dependency_key + ); + assert_eq!( + db.environment_component_status("composed") + .unwrap() + .into_iter() + .find(|state| state.component.component_id == "generated.private") + .unwrap() + .status, + "stale" + ); + let rebuilt = db + .sync_all_workspace_environments("composed", None) + .unwrap(); + assert!(rebuilt + .generation + .components + .iter() + .any(|component| component.component_id == "generated.private")); + let mounted = db.mount_nfs_cow_workdir_for_lane("composed").unwrap(); + assert_eq!(fs::read(&private).unwrap(), b"private base\n"); + drop(mounted); + + let mounted = db.mount_nfs_cow_workdir_for_lane("composed").unwrap(); + let specification_path = workdir.join("trail.environment.toml"); + let specification = fs::read_to_string(&specification_path).unwrap(); + let retained = specification + .split_once("\n[[component]]\nid = \"generated.private\"") + .unwrap() + .0; + fs::write(&specification_path, format!("{retained}\n")).unwrap(); + drop(mounted); + db.checkpoint_lane_workspace("composed", Some("remove private component".to_string())) + .unwrap(); + let retired = db + .sync_all_workspace_environments("composed", None) + .unwrap(); + assert_eq!(retired.generation.components.len(), 1); + assert_eq!( + retired.generation.components[0].component_id, + "generated.shared" + ); + assert!(db + .environment_component_status("composed") + .unwrap() + .into_iter() + .all(|state| state.component.component_id != "generated.private")); + let mounted = db.mount_nfs_cow_workdir_for_lane("composed").unwrap(); + assert!(!workdir.join(".trail-generated/private").exists()); + drop(mounted); + + let mounted = db.mount_nfs_cow_workdir_for_lane("composed").unwrap(); + fs::remove_file(workdir.join("trail.environment.toml")).unwrap(); + drop(mounted); + db.checkpoint_lane_workspace("composed", Some("remove environment".to_string())) + .unwrap(); + let cleared = db + .sync_all_workspace_environments("composed", None) + .unwrap(); + assert!(cleared.generation.components.is_empty()); + assert!(cleared.layers.is_empty()); + assert!(db + .environment_component_status("composed") + .unwrap() + .is_empty()); + let mounted = db.mount_nfs_cow_workdir_for_lane("composed").unwrap(); + assert!(!workdir.join(".trail-generated/shared").exists()); + drop(mounted); + } + + #[cfg(target_os = "macos")] + #[test] + fn restricted_command_recipe_publishes_and_activates_multiple_outputs_atomically() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("input.txt"), "identity\n").unwrap(); + fs::write( + workspace.path().join("trail.environment.toml"), + r#"schema = "trail.environment/v1" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "generated.multi" +adapter = "trail/command@1" +kind = "generated" +inputs = [{ path = "input.txt", role = "identity", format = "bytes" }] + +[component.build] +command = ["touch", "generated-a/a.txt", "generated-b/b.txt"] +cwd = "." +network = "deny" +scripts = "deny" + +[[component.output]] +name = "alpha" +source = "generated-a" +target = ".trail-generated/alpha" +policy = "immutable_seed_private" +portability = "host" + +[[component.output]] +name = "beta" +source = "generated-b" +target = ".trail-generated/beta" +policy = "immutable_seed_private" +portability = "host" +"#, + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + for lane in ["multi-a", "multi-b"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + LaneWorkdirMode::NfsCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + } + + let planned = db + .plan_workspace_environment_component( + "multi-a", + RECIPE_ADAPTER_IDENTITY, + None, + Some("generated.multi"), + ) + .unwrap(); + assert_eq!(planned.outputs.len(), 2); + assert_eq!(planned.capabilities.filesystem_write.len(), 2); + + let first = db.sync_all_workspace_environments("multi-a", None).unwrap(); + assert_eq!(first.layers.len(), 1); + assert_eq!(first.generation.components.len(), 1); + let component = &first.generation.components[0]; + assert_eq!(component.outputs.len(), 2); + assert_eq!(component.outputs[0].name, "alpha"); + assert_eq!(component.outputs[1].name, "beta"); + assert_eq!(component.outputs[0].layer_id, component.outputs[1].layer_id); + let layer_root = Path::new(&first.layers[0].storage_path); + assert!(layer_root.join("outputs/0000/a.txt").is_file()); + assert!(layer_root.join("outputs/0001/b.txt").is_file()); + + let output_rows = db + .conn + .query_row( + "SELECT COUNT(*) FROM environment_component_output_bindings WHERE component_id = 'generated.multi'", + [], + |row| row.get::<_, usize>(0), + ) + .unwrap(); + assert_eq!(output_rows, 2); + let second = db + .sync_workspace_environment_component( + "multi-b", + RECIPE_ADAPTER_IDENTITY, + None, + Some("generated.multi"), + ) + .unwrap(); + assert_eq!(first.layers[0].layer_id, second.layers[0].layer_id); + let second_generation = db + .active_environment_generation("multi-b") + .unwrap() + .unwrap(); + assert_eq!(second_generation.components[0].outputs.len(), 2); + + let mount_a = db.mount_nfs_cow_workdir_for_lane("multi-a").unwrap(); + let mount_b = db.mount_nfs_cow_workdir_for_lane("multi-b").unwrap(); + let workdir_a = PathBuf::from(db.lane_workdir("multi-a").unwrap().workdir.unwrap()); + let workdir_b = PathBuf::from(db.lane_workdir("multi-b").unwrap().workdir.unwrap()); + let alpha_a = workdir_a.join(".trail-generated/alpha/a.txt"); + let beta_a = workdir_a.join(".trail-generated/beta/b.txt"); + let alpha_b = workdir_b.join(".trail-generated/alpha/a.txt"); + let beta_b = workdir_b.join(".trail-generated/beta/b.txt"); + assert_eq!(fs::read(&alpha_a).unwrap(), b""); + assert_eq!(fs::read(&beta_a).unwrap(), b""); + fs::write(&alpha_a, b"lane-a").unwrap(); + fs::write(&beta_a, b"private-beta").unwrap(); + assert_eq!(fs::read(&alpha_a).unwrap(), b"lane-a"); + assert_eq!(fs::read(&beta_a).unwrap(), b"private-beta"); + assert_eq!(fs::read(&alpha_b).unwrap(), b""); + assert_eq!(fs::read(&beta_b).unwrap(), b""); + drop(mount_a); + drop(mount_b); + db.replace_declared_workspace_layers( + "multi-a", + &[EnvironmentLayerActivation { + layer_id: Some(first.layers[0].layer_id.clone()), + outputs: vec![EnvironmentLayerOutputActivation { + name: "alpha".to_string(), + mount_path: ".trail-generated/alpha".to_string(), + policy: "immutable_seed_private".to_string(), + binding_identity: first.layers[0].layer_id.clone(), + private_seed: None, + layer_subpath: "outputs/0000".to_string(), + }], + component_id: "generated.multi".to_string(), + adapter_identity: RECIPE_ADAPTER_IDENTITY.to_string(), + adapter_version: 1, + implementation_version: env!("CARGO_PKG_VERSION").to_string(), + distribution_digest: "builtin:command-recipe-plan-v1".to_string(), + kind: "generated".to_string(), + dependencies: Vec::new(), + caches: Vec::new(), + external_artifacts: Vec::new(), + runtime_resources: Vec::new(), + expected_key: first.layers[0].cache_key.clone(), + canonical_key: db + .workspace_layer_key_by_cache_key(&first.layers[0].cache_key) + .unwrap() + .unwrap(), + }], + ) + .unwrap(); + let reduced = db + .active_environment_generation("multi-a") + .unwrap() + .unwrap(); + assert_eq!(reduced.components[0].outputs.len(), 1); + let view = db.lane_workspace_view("multi-a").unwrap().unwrap(); + let generated_upper = Path::new(&view.source_upper) + .parent() + .unwrap() + .join("generated-upper/.trail-generated/beta"); + assert!(!generated_upper.exists()); + } + + #[cfg(target_os = "macos")] + #[test] + fn restricted_command_recipe_denies_undeclared_host_reads() { + let (_workspace, db) = open_recipe_lane(&["cp", "/etc/passwd", "generated/copied.txt"]); + let error = db + .sync_workspace_environment("recipe-a", RECIPE_ADAPTER_IDENTITY, None) + .unwrap_err(); + assert!(error.to_string().contains("failed with")); + assert!(db.list_workspace_layers().unwrap().is_empty()); + let status = db.environment_component_status("recipe-a").unwrap(); + assert_eq!(status[0].status, "failed"); + assert_eq!(status[0].attached_key, None); + } + + #[cfg(target_os = "macos")] + #[test] + fn restricted_command_recipe_denies_writes_outside_declared_output() { + let (_workspace, db) = open_recipe_lane(&["cp", "input.txt", "escape.txt"]); + let error = db + .sync_workspace_environment("recipe-a", RECIPE_ADAPTER_IDENTITY, None) + .unwrap_err(); + assert!(error.to_string().contains("failed with")); + assert!(db.list_workspace_layers().unwrap().is_empty()); + assert!(db + .active_environment_generation("recipe-a") + .unwrap() + .is_none()); + } + + #[cfg(target_os = "macos")] + #[test] + fn restricted_command_recipe_denies_network_connections() { + use std::net::TcpListener; + use std::thread; + use std::time::{Duration, Instant}; + + if !Path::new("/usr/bin/nc").is_file() { + return; + } + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let address = listener.local_addr().unwrap(); + let observer = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(3); + while Instant::now() < deadline { + match listener.accept() { + Ok((_stream, _)) => return true, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(10)); + } + Err(_) => return false, + } + } + false + }); + let port = address.port().to_string(); + let (_workspace, db) = open_recipe_lane(&["nc", "-z", "-w", "1", "127.0.0.1", &port]); + let error = db + .sync_workspace_environment("recipe-a", RECIPE_ADAPTER_IDENTITY, None) + .unwrap_err(); + assert!(error.to_string().contains("failed with")); + assert!( + !observer.join().unwrap(), + "sandboxed netcat reached a host socket" + ); + assert!(db.list_workspace_layers().unwrap().is_empty()); + } + + #[cfg(target_os = "macos")] + #[test] + fn restricted_command_recipe_never_publishes_an_escaping_symlink() { + let (_workspace, db) = + open_recipe_lane(&["ln", "-s", "/etc/passwd", "generated/passwd-link"]); + let error = db + .sync_workspace_environment("recipe-a", RECIPE_ADAPTER_IDENTITY, None) + .unwrap_err(); + assert!(error.to_string().contains("symlink")); + assert!(db + .list_workspace_layers() + .unwrap() + .iter() + .all(|layer| layer.state != "available")); + assert!(db + .active_environment_generation("recipe-a") + .unwrap() + .is_none()); + } + + #[cfg(not(target_os = "macos"))] + #[test] + fn restricted_command_recipe_fails_closed_without_a_kernel_backend() { + let (_workspace, db) = open_recipe_lane(&["cp", "input.txt", "generated/copied.txt"]); + let error = db + .sync_workspace_environment("recipe-a", RECIPE_ADAPTER_IDENTITY, None) + .unwrap_err(); + assert!(error.to_string().contains("sandboxing is unavailable")); + assert!(db.list_workspace_layers().unwrap().is_empty()); + } +} diff --git a/trail/src/db/lane/workspace_runtime.rs b/trail/src/db/lane/workspace_runtime.rs new file mode 100644 index 0000000..4bca7f6 --- /dev/null +++ b/trail/src/db/lane/workspace_runtime.rs @@ -0,0 +1,1983 @@ +use std::collections::BTreeMap; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpStream}; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; +use std::path::PathBuf; +use std::process::{Command, Output}; +use std::thread; + +use rusqlite::params; +use serde_json::Value; + +use super::*; + +const TRAIL_MANAGED_LABEL: &str = "io.trail.managed"; +const TRAIL_WORKSPACE_LABEL: &str = "io.trail.workspace"; +const TRAIL_ALLOCATION_LABEL: &str = "io.trail.allocation"; +const TRAIL_CLEANUP_LABEL: &str = "io.trail.cleanup-token"; +const TRAIL_SECRET_BINDINGS_LABEL: &str = "io.trail.secret-bindings"; +const MAX_PROVIDER_DIAGNOSTIC_BYTES: usize = 8 * 1024; + +#[derive(Clone, Debug)] +struct RuntimeAllocation { + generation_id: String, + component_id: String, + resource_name: String, + image_reference: String, + image_digest: String, + container_port: u16, + health_timeout_ms: u64, + restart_policy: String, + volume_target: Option, + allocation_id: String, + provider_resource_id: Option, + container_name: String, + network_name: String, + volume_name: Option, + host_port: Option, + cleanup_token: String, + owner_pid: Option, + owner_start_token: Option, + secrets: Vec, +} + +#[derive(Clone, Debug)] +struct RuntimeSecretReference { + name: String, + provider: String, + reference: String, + target: String, + required: bool, + purpose: String, + environment: Option, +} + +#[derive(Clone, Debug)] +struct ResolvedRuntimeSecret { + source_path: PathBuf, + target: String, + environment: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RuntimeContainerObservation { + provider_resource_id: String, + running: bool, + host_port: Option, + labels: BTreeMap, +} + +trait RuntimeProvider { + fn provider_name(&self) -> &str; + fn ensure_image(&self, reference: &str, expected_digest: &str) -> Result<()>; + fn ensure_network(&self, allocation: &RuntimeAllocation) -> Result<()>; + fn ensure_volume(&self, allocation: &RuntimeAllocation) -> Result<()>; + fn inspect_container( + &self, + allocation: &RuntimeAllocation, + ) -> Result>; + fn create_container( + &self, + allocation: &RuntimeAllocation, + secrets: &[ResolvedRuntimeSecret], + ) -> Result; + fn start_container(&self, allocation: &RuntimeAllocation) -> Result<()>; + fn stop_container(&self, allocation: &RuntimeAllocation) -> Result<()>; + fn remove_container(&self, allocation: &RuntimeAllocation) -> Result<()>; + fn remove_network(&self, allocation: &RuntimeAllocation) -> Result<()>; + fn remove_volume(&self, allocation: &RuntimeAllocation) -> Result<()>; +} + +struct CliRuntimeProvider { + name: String, + executable: PathBuf, + workspace_id: String, +} + +impl CliRuntimeProvider { + fn detect(workspace_id: &str) -> Result { + let mut failures = Vec::new(); + for name in ["docker", "podman"] { + let tool = match super::workspace_environment::resolve_workspace_tool_executable(name) { + Ok(tool) => tool, + Err(err) => { + failures.push(err.to_string()); + continue; + } + }; + let output = Command::new(&tool.path) + .args(["info", "--format", "{{json .ServerVersion}}"]) + .output(); + match output { + Ok(output) if output.status.success() => { + return Ok(Self { + name: name.to_string(), + executable: tool.path, + workspace_id: workspace_id.to_string(), + }); + } + Ok(output) => { + failures.push(format!("{name}: {}", provider_output_diagnostic(&output))) + } + Err(err) => failures.push(format!("{name}: {err}")), + } + } + Err(Error::InvalidInput(format!( + "no usable OCI runtime provider was found; install and start Docker or Podman ({})", + failures.join("; ") + ))) + } + + fn run(&self, operation: &str, args: &[String]) -> Result { + let output = Command::new(&self.executable).args(args).output()?; + if output.status.success() { + Ok(output) + } else { + Err(Error::InvalidInput(format!( + "{} runtime {operation} failed: {}", + self.name, + provider_output_diagnostic(&output) + ))) + } + } + + fn inspect_resource_labels( + &self, + kind: &str, + name: &str, + ) -> Result>> { + let output = Command::new(&self.executable) + .args([kind, "inspect", name, "--format", "{{json .Labels}}"]) + .output()?; + if !output.status.success() { + return Ok(None); + } + let labels = + serde_json::from_slice::>(&output.stdout).map_err(|err| { + Error::Corrupt(format!( + "{} returned invalid {kind} label JSON for `{name}`: {err}", + self.name + )) + })?; + Ok(Some(labels)) + } + + fn ownership_labels(&self, allocation: &RuntimeAllocation) -> Vec { + self.label_args([ + format!("{TRAIL_MANAGED_LABEL}=true"), + format!("{TRAIL_WORKSPACE_LABEL}={}", self.workspace_id), + format!("{TRAIL_ALLOCATION_LABEL}={}", allocation.allocation_id), + format!("{TRAIL_CLEANUP_LABEL}={}", allocation.cleanup_token), + ]) + } + + fn network_labels(&self, allocation: &RuntimeAllocation) -> Vec { + self.label_args([ + format!("{TRAIL_MANAGED_LABEL}=true"), + format!("{TRAIL_WORKSPACE_LABEL}={}", self.workspace_id), + format!("io.trail.network={}", allocation.network_name), + ]) + } + + fn volume_labels(&self, volume_name: &str) -> Vec { + self.label_args([ + format!("{TRAIL_MANAGED_LABEL}=true"), + format!("{TRAIL_WORKSPACE_LABEL}={}", self.workspace_id), + format!("io.trail.volume={volume_name}"), + ]) + } + + fn label_args(&self, labels: [String; N]) -> Vec { + labels + .into_iter() + .flat_map(|label| ["--label".to_string(), label]) + .collect() + } + + fn validate_workspace_labels( + &self, + kind: &str, + name: &str, + labels: &BTreeMap, + ) -> Result<()> { + if labels.get(TRAIL_MANAGED_LABEL).map(String::as_str) != Some("true") + || labels.get(TRAIL_WORKSPACE_LABEL) != Some(&self.workspace_id) + { + return Err(Error::InvalidInput(format!( + "{kind} name `{name}` is occupied by a resource this Trail workspace does not own; refusing to adopt or replace it" + ))); + } + Ok(()) + } +} + +impl RuntimeProvider for CliRuntimeProvider { + fn provider_name(&self) -> &str { + &self.name + } + + fn ensure_image(&self, reference: &str, expected_digest: &str) -> Result<()> { + let inspect = Command::new(&self.executable) + .args([ + "image", + "inspect", + reference, + "--format", + "{{json .RepoDigests}}", + ]) + .output()?; + let output = if inspect.status.success() { + inspect + } else { + self.run("image pull", &["pull".to_string(), reference.to_string()])?; + self.run( + "image inspect", + &[ + "image".to_string(), + "inspect".to_string(), + reference.to_string(), + "--format".to_string(), + "{{json .RepoDigests}}".to_string(), + ], + )? + }; + let observed = String::from_utf8_lossy(&output.stdout); + if !observed.contains(expected_digest) { + return Err(Error::InvalidInput(format!( + "{} resolved image `{reference}` without expected digest `{expected_digest}`", + self.name + ))); + } + Ok(()) + } + + fn ensure_network(&self, allocation: &RuntimeAllocation) -> Result<()> { + if let Some(labels) = self.inspect_resource_labels("network", &allocation.network_name)? { + self.validate_workspace_labels("network", &allocation.network_name, &labels)?; + if labels.get("io.trail.network") != Some(&allocation.network_name) { + return Err(Error::InvalidInput(format!( + "network `{}` has incompatible Trail ownership metadata", + allocation.network_name + ))); + } + return Ok(()); + } + let mut args = vec!["network".to_string(), "create".to_string()]; + args.extend(self.network_labels(allocation)); + args.push(allocation.network_name.clone()); + self.run("network create", &args)?; + Ok(()) + } + + fn ensure_volume(&self, allocation: &RuntimeAllocation) -> Result<()> { + let Some(volume_name) = allocation.volume_name.as_deref() else { + return Ok(()); + }; + if let Some(labels) = self.inspect_resource_labels("volume", volume_name)? { + self.validate_workspace_labels("volume", volume_name, &labels)?; + if labels.get("io.trail.volume").map(String::as_str) != Some(volume_name) { + return Err(Error::InvalidInput(format!( + "volume `{volume_name}` has incompatible Trail ownership metadata" + ))); + } + return Ok(()); + } + let mut args = vec!["volume".to_string(), "create".to_string()]; + args.extend(self.volume_labels(volume_name)); + args.push(volume_name.to_string()); + self.run("volume create", &args)?; + Ok(()) + } + + fn inspect_container( + &self, + allocation: &RuntimeAllocation, + ) -> Result> { + let output = Command::new(&self.executable) + .args([ + "container", + "inspect", + &allocation.container_name, + "--format", + "{{json .}}", + ]) + .output()?; + if !output.status.success() { + return Ok(None); + } + let value: Value = serde_json::from_slice(&output.stdout).map_err(|err| { + Error::Corrupt(format!( + "{} returned invalid container inspection JSON: {err}", + self.name + )) + })?; + let provider_resource_id = value + .get("Id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .ok_or_else(|| Error::Corrupt("container inspection omitted Id".to_string()))? + .to_string(); + let running = value + .pointer("/State/Running") + .and_then(Value::as_bool) + .unwrap_or(false); + let labels = value + .pointer("/Config/Labels") + .and_then(Value::as_object) + .map(|labels| { + labels + .iter() + .filter_map(|(name, value)| { + value + .as_str() + .map(|value| (name.clone(), value.to_string())) + }) + .collect() + }) + .unwrap_or_default(); + let port_key = format!("{}/tcp", allocation.container_port); + let host_port = value + .pointer("/NetworkSettings/Ports") + .and_then(Value::as_object) + .and_then(|ports| ports.get(&port_key)) + .and_then(Value::as_array) + .and_then(|bindings| bindings.first()) + .and_then(|binding| binding.get("HostPort")) + .and_then(Value::as_str) + .and_then(|port| port.parse::().ok()); + Ok(Some(RuntimeContainerObservation { + provider_resource_id, + running, + host_port, + labels, + })) + } + + fn create_container( + &self, + allocation: &RuntimeAllocation, + secrets: &[ResolvedRuntimeSecret], + ) -> Result { + let host_port = allocation.host_port.ok_or_else(|| { + Error::Corrupt(format!( + "runtime allocation `{}` has no reserved host port", + allocation.allocation_id + )) + })?; + let mut args = vec![ + "container".to_string(), + "create".to_string(), + "--name".to_string(), + allocation.container_name.clone(), + "--network".to_string(), + allocation.network_name.clone(), + "--publish".to_string(), + format!("127.0.0.1:{host_port}:{}", allocation.container_port), + "--restart".to_string(), + match allocation.restart_policy.as_str() { + "never" => "no".to_string(), + "on_failure" => "on-failure".to_string(), + "always" => "always".to_string(), + other => { + return Err(Error::Corrupt(format!( + "unsupported persisted restart policy `{other}`" + ))); + } + }, + ]; + args.extend(self.ownership_labels(allocation)); + if !allocation.secrets.is_empty() { + args.extend(self.label_args([format!( + "{TRAIL_SECRET_BINDINGS_LABEL}={}", + runtime_secret_binding_digest(secrets) + )])); + } + if let (Some(volume_name), Some(volume_target)) = ( + allocation.volume_name.as_deref(), + allocation.volume_target.as_deref(), + ) { + args.extend([ + "--mount".to_string(), + format!("type=volume,src={volume_name},dst={volume_target}"), + ]); + } + for secret in secrets { + args.extend([ + "--mount".to_string(), + format!( + "type=bind,src={},dst={},readonly", + secret.source_path.to_string_lossy(), + secret.target + ), + ]); + if let Some(environment) = secret.environment.as_deref() { + args.extend([ + "--env".to_string(), + format!("{environment}={}", secret.target), + ]); + } + } + args.push(allocation.image_reference.clone()); + let output = self.run("container create", &args)?; + let id = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if id.is_empty() { + return Err(Error::Corrupt(format!( + "{} created container `{}` without returning an ID", + self.name, allocation.container_name + ))); + } + Ok(id) + } + + fn start_container(&self, allocation: &RuntimeAllocation) -> Result<()> { + self.run( + "container start", + &["start".to_string(), allocation.container_name.clone()], + )?; + Ok(()) + } + + fn stop_container(&self, allocation: &RuntimeAllocation) -> Result<()> { + if self.inspect_container(allocation)?.is_none() { + return Ok(()); + } + self.run( + "container stop", + &[ + "stop".to_string(), + "--time".to_string(), + "10".to_string(), + allocation.container_name.clone(), + ], + )?; + Ok(()) + } + + fn remove_container(&self, allocation: &RuntimeAllocation) -> Result<()> { + let Some(observation) = self.inspect_container(allocation)? else { + return Ok(()); + }; + validate_runtime_container_ownership(allocation, &observation)?; + self.run( + "container remove", + &[ + "rm".to_string(), + "--force".to_string(), + allocation.container_name.clone(), + ], + )?; + Ok(()) + } + + fn remove_network(&self, allocation: &RuntimeAllocation) -> Result<()> { + let Some(labels) = self.inspect_resource_labels("network", &allocation.network_name)? + else { + return Ok(()); + }; + self.validate_workspace_labels("network", &allocation.network_name, &labels)?; + if labels.get("io.trail.network") != Some(&allocation.network_name) { + return Err(Error::InvalidInput(format!( + "network `{}` has incompatible Trail ownership metadata", + allocation.network_name + ))); + } + self.run( + "network remove", + &[ + "network".to_string(), + "rm".to_string(), + allocation.network_name.clone(), + ], + )?; + Ok(()) + } + + fn remove_volume(&self, allocation: &RuntimeAllocation) -> Result<()> { + let Some(volume_name) = allocation.volume_name.as_deref() else { + return Ok(()); + }; + let Some(labels) = self.inspect_resource_labels("volume", volume_name)? else { + return Ok(()); + }; + self.validate_workspace_labels("volume", volume_name, &labels)?; + if labels.get("io.trail.volume").map(String::as_str) != Some(volume_name) { + return Err(Error::InvalidInput(format!( + "volume `{volume_name}` has incompatible Trail ownership metadata" + ))); + } + self.run( + "volume remove", + &[ + "volume".to_string(), + "rm".to_string(), + volume_name.to_string(), + ], + )?; + Ok(()) + } +} + +impl Trail { + pub(crate) fn recover_workspace_runtime_leases(&self) -> Result<()> { + let rows = self.runtime_allocations_with_live_statuses(&["allocating", "stopping"])?; + for allocation in rows { + let owner_is_live = allocation + .owner_pid + .zip(allocation.owner_start_token.as_deref()) + .is_some_and(|(pid, token)| process_matches_start_token(pid, token)); + if owner_is_live { + continue; + } + self.conn.execute( + "UPDATE environment_generation_runtime_resources + SET status = 'orphaned', health_status = 'unknown', + reason = ?1, owner_pid = NULL, owner_start_token = NULL, updated_at = ?2 + WHERE allocation_id = ?3 AND status IN ('allocating', 'stopping')", + params![ + "runtime lifecycle owner exited; reconcile will inspect and adopt matching provider resources", + now_ts(), + allocation.allocation_id + ], + )?; + } + Ok(()) + } + + pub fn reconcile_workspace_environment_runtime( + &self, + lane: &str, + ) -> Result { + let generation = self.active_environment_generation(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` has no active environment generation" + )) + })?; + let allocations = self.runtime_allocations_for_generation(&generation.generation_id)?; + if allocations.is_empty() { + return Ok(generation); + } + self.recover_workspace_runtime_leases()?; + let provider = CliRuntimeProvider::detect(&self.config.workspace.id.0)?; + self.reconcile_workspace_environment_runtime_with(&provider, &allocations)?; + self.active_environment_generation(lane)?.ok_or_else(|| { + Error::Corrupt("active generation disappeared after runtime reconcile".to_string()) + }) + } + + pub fn stop_workspace_environment_runtime( + &self, + lane: &str, + ) -> Result { + let generation = self.active_environment_generation(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` has no active environment generation" + )) + })?; + let allocations = self.runtime_allocations_for_generation(&generation.generation_id)?; + if allocations.is_empty() { + return Ok(generation); + } + self.recover_workspace_runtime_leases()?; + let provider = CliRuntimeProvider::detect(&self.config.workspace.id.0)?; + for allocation in allocations.iter().rev() { + self.claim_runtime_allocation(allocation, "stopping", "stopped")?; + let stopped = (|| -> Result<()> { + if let Some(observation) = provider.inspect_container(allocation)? { + validate_runtime_container_ownership(allocation, &observation)?; + } + provider.stop_container(allocation) + })(); + match stopped { + Ok(()) => self.finish_runtime_allocation( + allocation, + "stopped", + "stopped", + allocation.provider_resource_id.as_deref(), + allocation.host_port, + None, + )?, + Err(err) => { + self.finish_runtime_allocation( + allocation, + "failed", + "unknown", + allocation.provider_resource_id.as_deref(), + allocation.host_port, + Some(&err.to_string()), + )?; + return Err(err); + } + } + } + self.active_environment_generation(lane)?.ok_or_else(|| { + Error::Corrupt("active generation disappeared after runtime stop".to_string()) + }) + } + + /// Remove Trail-owned runtime resources belonging only to retired + /// generations. A logical lane-service volume is retained while any + /// active generation references it. + pub fn cleanup_retired_workspace_environment_runtime( + &self, + lane: &str, + ) -> Result { + let generation = self.active_environment_generation(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` has no active environment generation" + )) + })?; + self.recover_workspace_runtime_leases()?; + let allocations = self.runtime_allocations_where( + "WHERE generation_id IN ( + SELECT generation_id FROM environment_generations + WHERE view_id = ?1 AND state = 'retired' + ) AND status != 'stopped' + ORDER BY generation_id, component_id, resource_name", + [generation.view_id.as_str()], + )?; + if allocations.is_empty() { + return Ok(generation); + } + let provider = CliRuntimeProvider::detect(&self.config.workspace.id.0)?; + self.cleanup_retired_workspace_environment_runtime_with(&provider, &allocations)?; + Ok(generation) + } + + fn cleanup_retired_workspace_environment_runtime_with( + &self, + provider: &dyn RuntimeProvider, + allocations: &[RuntimeAllocation], + ) -> Result<()> { + for allocation in allocations { + self.claim_runtime_allocation(allocation, "stopping", "stopped")?; + let cleanup = (|| -> Result<()> { + provider.remove_container(allocation)?; + if let Some(volume_name) = allocation.volume_name.as_deref() { + let active_references = self.conn.query_row( + "SELECT COUNT(*) + FROM environment_generation_runtime_resources r + JOIN environment_generations g ON g.generation_id = r.generation_id + WHERE g.state = 'active' AND r.volume_name = ?1", + params![volume_name], + |row| row.get::<_, i64>(0), + )?; + if active_references == 0 { + provider.remove_volume(allocation)?; + } + } + Ok(()) + })(); + match cleanup { + Ok(()) => self.finish_runtime_allocation( + allocation, + "stopped", + "stopped", + allocation.provider_resource_id.as_deref(), + allocation.host_port, + Some("retired generation runtime resources were cleaned"), + )?, + Err(err) => { + self.finish_runtime_allocation( + allocation, + "failed", + "unknown", + allocation.provider_resource_id.as_deref(), + allocation.host_port, + Some(&err.to_string()), + )?; + return Err(err); + } + } + } + let mut networks = BTreeMap::::new(); + for allocation in allocations { + networks + .entry(allocation.network_name.clone()) + .or_insert(allocation); + } + for allocation in networks.into_values() { + let active_references = self.conn.query_row( + "SELECT COUNT(*) + FROM environment_generation_runtime_resources r + JOIN environment_generations g ON g.generation_id = r.generation_id + WHERE g.state = 'active' AND r.network_name = ?1", + params![&allocation.network_name], + |row| row.get::<_, i64>(0), + )?; + if active_references == 0 { + provider.remove_network(allocation)?; + } + } + Ok(()) + } + + fn reconcile_workspace_environment_runtime_with( + &self, + provider: &dyn RuntimeProvider, + allocations: &[RuntimeAllocation], + ) -> Result<()> { + let mut first_error = None; + for allocation in allocations { + if let Err(err) = self.claim_runtime_allocation(allocation, "allocating", "starting") { + if first_error.is_none() { + first_error = Some(err); + } + continue; + } + if let Err(err) = self.reconcile_claimed_runtime_allocation(provider, allocation) { + let mut reason = err.to_string(); + if let Ok(Some(observation)) = provider.inspect_container(allocation) { + if validate_runtime_container_ownership(allocation, &observation).is_ok() { + if let Err(stop_err) = provider.stop_container(allocation) { + reason = format!( + "{reason}; additionally failed to stop the unhealthy owned container: {stop_err}" + ); + } + } + } + self.finish_runtime_allocation( + allocation, + "failed", + "unhealthy", + allocation.provider_resource_id.as_deref(), + allocation.host_port, + Some(&reason), + )?; + if first_error.is_none() { + first_error = Some(err); + } + } + } + if let Some(err) = first_error { + Err(err) + } else { + Ok(()) + } + } + + fn reconcile_claimed_runtime_allocation( + &self, + provider: &dyn RuntimeProvider, + allocation: &RuntimeAllocation, + ) -> Result<()> { + let resolved_secrets = self.resolve_runtime_secrets(allocation)?; + provider.ensure_image(&allocation.image_reference, &allocation.image_digest)?; + provider.ensure_network(allocation)?; + provider.ensure_volume(allocation)?; + let mut allocation = allocation.clone(); + let mut observation = provider.inspect_container(&allocation)?; + if let Some(existing) = observation.as_ref() { + validate_runtime_container_ownership(&allocation, existing)?; + if !runtime_secret_bindings_match(&allocation, &resolved_secrets, existing) { + provider.remove_container(&allocation)?; + observation = None; + } + } + if observation.is_none() { + if allocation.host_port.is_none() { + allocation.host_port = Some(self.reserve_runtime_host_port(&allocation)?); + } + let provider_resource_id = provider.create_container(&allocation, &resolved_secrets)?; + self.update_runtime_provider_identity(&allocation, &provider_resource_id)?; + } + if !observation.as_ref().is_some_and(|state| state.running) { + provider.start_container(&allocation)?; + } + observation = provider.inspect_container(&allocation)?; + let observation = observation.ok_or_else(|| { + Error::InvalidInput(format!( + "{} started `{}` but it cannot be inspected", + provider.provider_name(), + allocation.container_name + )) + })?; + validate_runtime_container_ownership(&allocation, &observation)?; + let host_port = observation.host_port.ok_or_else(|| { + Error::InvalidInput(format!( + "{} container `{}` has no host port for {}/tcp", + provider.provider_name(), + allocation.container_name, + allocation.container_port + )) + })?; + wait_for_tcp_health(host_port, allocation.health_timeout_ms)?; + self.finish_runtime_allocation( + &allocation, + "running", + "healthy", + Some(&observation.provider_resource_id), + Some(host_port), + None, + ) + } + + fn resolve_runtime_secrets( + &self, + allocation: &RuntimeAllocation, + ) -> Result> { + let mut resolved = Vec::with_capacity(allocation.secrets.len()); + for secret in &allocation.secrets { + let path = match secret.provider.as_str() { + "file" => Some(PathBuf::from(&secret.reference)), + "environment_file" => std::env::var_os(&secret.reference).map(PathBuf::from), + _ => None, + }; + let Some(path) = path else { + self.update_runtime_secret_status( + allocation, + secret, + "unavailable", + Some("secret provider did not return a file handle"), + None, + )?; + if secret.required { + return Err(Error::InvalidInput(format!( + "required secret reference `{}` is unavailable from provider `{}`", + secret.name, secret.provider + ))); + } + continue; + }; + if let Err(err) = validate_runtime_secret_file(&path) { + self.update_runtime_secret_status( + allocation, + secret, + "unavailable", + Some("secret provider returned an unsafe or inaccessible file handle"), + None, + )?; + if secret.required { + return Err(Error::InvalidInput(format!( + "required secret reference `{}` failed file-handle validation: {err}", + secret.name + ))); + } + continue; + } + let canonical_path = fs::canonicalize(&path).map_err(|_| { + Error::InvalidInput(format!( + "required secret reference `{}` could not be canonicalized", + secret.name + )) + })?; + if canonical_path.starts_with(&self.workspace_root) + || canonical_path.starts_with(&self.db_dir) + { + self.update_runtime_secret_status( + allocation, + secret, + "unavailable", + Some("secret provider file handle points inside the Trail workspace"), + None, + )?; + if secret.required { + return Err(Error::InvalidInput(format!( + "secret reference `{}` must resolve outside the Trail workspace so it cannot enter checkpoints or artifacts", + secret.name + ))); + } + continue; + } + self.update_runtime_secret_status( + allocation, + secret, + "available", + None, + Some(now_ts()), + )?; + resolved.push(ResolvedRuntimeSecret { + source_path: canonical_path, + target: secret.target.clone(), + environment: secret.environment.clone(), + }); + } + Ok(resolved) + } + + fn update_runtime_secret_status( + &self, + allocation: &RuntimeAllocation, + secret: &RuntimeSecretReference, + status: &str, + reason: Option<&str>, + resolved_at: Option, + ) -> Result<()> { + self.conn + .execute_batch("SAVEPOINT trail_runtime_secret_access")?; + let result = (|| -> Result<()> { + let changed = self.conn.execute( + "UPDATE environment_generation_runtime_secrets + SET status = ?1, reason = ?2, resolved_at = ?3, updated_at = ?4 + WHERE generation_id = ?5 AND component_id = ?6 + AND resource_name = ?7 AND secret_name = ?8", + params![ + status, + reason, + resolved_at, + now_ts(), + allocation.generation_id, + allocation.component_id, + allocation.resource_name, + secret.name + ], + )?; + if changed != 1 { + return Err(Error::Corrupt(format!( + "runtime secret reference `{}` disappeared during resolution", + secret.name + ))); + } + let access_id = format!( + "secret_access_{}", + crate::ids::short_hash( + format!( + "{}:{}:{}:{}:{}", + allocation.allocation_id, + secret.name, + status, + now_nanos(), + std::process::id() + ) + .as_bytes(), + 16 + ) + ); + self.conn.execute( + "INSERT INTO environment_secret_access_audit + (access_id, generation_id, component_id, resource_name, secret_name, + provider, purpose, status, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![ + access_id, + allocation.generation_id, + allocation.component_id, + allocation.resource_name, + secret.name, + secret.provider, + secret.purpose, + status, + now_ts() + ], + )?; + Ok(()) + })(); + match result { + Ok(()) => { + self.conn + .execute_batch("RELEASE SAVEPOINT trail_runtime_secret_access")?; + Ok(()) + } + Err(err) => { + let _ = self.conn.execute_batch( + "ROLLBACK TO SAVEPOINT trail_runtime_secret_access; + RELEASE SAVEPOINT trail_runtime_secret_access", + ); + Err(err) + } + } + } + + fn claim_runtime_allocation( + &self, + allocation: &RuntimeAllocation, + status: &str, + health_status: &str, + ) -> Result<()> { + let token = current_process_start_token(); + let changed = self.conn.execute( + "UPDATE environment_generation_runtime_resources + SET status = ?1, health_status = ?2, reason = NULL, + owner_pid = ?3, owner_start_token = ?4, updated_at = ?5 + WHERE allocation_id = ?6 + AND (owner_pid IS NULL OR (owner_pid = ?3 AND owner_start_token = ?4))", + params![ + status, + health_status, + std::process::id(), + token, + now_ts(), + allocation.allocation_id + ], + )?; + if changed != 1 { + return Err(Error::InvalidInput(format!( + "runtime allocation `{}` is owned by another live process", + allocation.allocation_id + ))); + } + Ok(()) + } + + fn update_runtime_provider_identity( + &self, + allocation: &RuntimeAllocation, + provider_resource_id: &str, + ) -> Result<()> { + let changed = self.conn.execute( + "UPDATE environment_generation_runtime_resources + SET provider_resource_id = ?1, updated_at = ?2 + WHERE allocation_id = ?3 AND owner_pid = ?4 AND owner_start_token = ?5", + params![ + provider_resource_id, + now_ts(), + allocation.allocation_id, + std::process::id(), + current_process_start_token() + ], + )?; + if changed != 1 { + return Err(Error::InvalidInput(format!( + "runtime allocation `{}` ownership changed during create", + allocation.allocation_id + ))); + } + Ok(()) + } + + fn reserve_runtime_host_port(&self, allocation: &RuntimeAllocation) -> Result { + let listener = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0))?; + let host_port = listener.local_addr()?.port(); + let changed = self.conn.execute( + "UPDATE environment_generation_runtime_resources + SET host_port = ?1, updated_at = ?2 + WHERE allocation_id = ?3 AND owner_pid = ?4 AND owner_start_token = ?5", + params![ + host_port, + now_ts(), + allocation.allocation_id, + std::process::id(), + current_process_start_token() + ], + )?; + if changed != 1 { + return Err(Error::InvalidInput(format!( + "runtime allocation `{}` ownership changed while reserving a loopback port", + allocation.allocation_id + ))); + } + drop(listener); + Ok(host_port) + } + + fn finish_runtime_allocation( + &self, + allocation: &RuntimeAllocation, + status: &str, + health_status: &str, + provider_resource_id: Option<&str>, + host_port: Option, + reason: Option<&str>, + ) -> Result<()> { + let reason = reason.map(sanitize_provider_text); + let now = now_ts(); + let changed = self.conn.execute( + "UPDATE environment_generation_runtime_resources + SET status = ?1, health_status = ?2, reason = ?3, + provider_resource_id = COALESCE(?4, provider_resource_id), host_port = ?5, + owner_pid = NULL, owner_start_token = NULL, updated_at = ?6, + started_at = CASE WHEN ?1 = 'running' THEN COALESCE(started_at, ?6) ELSE started_at END, + stopped_at = CASE WHEN ?1 = 'stopped' THEN ?6 ELSE stopped_at END + WHERE allocation_id = ?7 AND owner_pid = ?8 AND owner_start_token = ?9", + params![ + status, + health_status, + reason, + provider_resource_id, + host_port, + now, + allocation.allocation_id, + std::process::id(), + current_process_start_token() + ], + )?; + if changed != 1 { + return Err(Error::InvalidInput(format!( + "runtime allocation `{}` ownership changed before lifecycle state could be recorded", + allocation.allocation_id + ))); + } + Ok(()) + } + + fn runtime_allocations_for_generation( + &self, + generation_id: &str, + ) -> Result> { + self.runtime_allocations_where( + "WHERE generation_id = ?1 ORDER BY component_id, resource_name", + [generation_id], + ) + } + + fn runtime_allocations_with_live_statuses( + &self, + statuses: &[&str], + ) -> Result> { + let mut allocations = Vec::new(); + for status in statuses { + allocations.extend(self.runtime_allocations_where( + "WHERE status = ?1 ORDER BY generation_id, component_id, resource_name", + [*status], + )?); + } + Ok(allocations) + } + + fn runtime_allocations_where( + &self, + clause: &str, + values: [&str; N], + ) -> Result> { + let sql = format!( + "SELECT generation_id, component_id, resource_name, + image_reference, image_digest, container_port, health_timeout_ms, restart_policy, + volume_target, allocation_id, provider_resource_id, container_name, + network_name, volume_name, host_port, cleanup_token, + owner_pid, owner_start_token + FROM environment_generation_runtime_resources {clause}" + ); + let mut stmt = self.conn.prepare(&sql)?; + let rows = stmt.query_map(rusqlite::params_from_iter(values), |row| { + let owner_pid = row + .get::<_, Option>(16)? + .and_then(|pid| u32::try_from(pid).ok()); + Ok(RuntimeAllocation { + generation_id: row.get(0)?, + component_id: row.get(1)?, + resource_name: row.get(2)?, + image_reference: row.get(3)?, + image_digest: row.get(4)?, + container_port: row.get(5)?, + health_timeout_ms: row.get(6)?, + restart_policy: row.get(7)?, + volume_target: row.get(8)?, + allocation_id: row.get(9)?, + provider_resource_id: row.get(10)?, + container_name: row.get(11)?, + network_name: row.get(12)?, + volume_name: row.get(13)?, + host_port: row.get(14)?, + cleanup_token: row.get(15)?, + owner_pid, + owner_start_token: row.get(17)?, + secrets: Vec::new(), + }) + })?; + let mut allocations = rows + .collect::, _>>() + .map_err(Error::from)?; + drop(stmt); + for allocation in &mut allocations { + let mut secret_stmt = self.conn.prepare( + "SELECT secret_name, provider, reference, target, required, purpose, environment + FROM environment_generation_runtime_secrets + WHERE generation_id = ?1 AND component_id = ?2 AND resource_name = ?3 + ORDER BY secret_name", + )?; + allocation.secrets = secret_stmt + .query_map( + params![ + &allocation.generation_id, + &allocation.component_id, + &allocation.resource_name + ], + |row| { + Ok(RuntimeSecretReference { + name: row.get(0)?, + provider: row.get(1)?, + reference: row.get(2)?, + target: row.get(3)?, + required: row.get(4)?, + purpose: row.get(5)?, + environment: row.get(6)?, + }) + }, + )? + .collect::, _>>()?; + } + Ok(allocations) + } +} + +fn validate_runtime_container_ownership( + allocation: &RuntimeAllocation, + observation: &RuntimeContainerObservation, +) -> Result<()> { + let managed = observation + .labels + .get(TRAIL_MANAGED_LABEL) + .is_some_and(|value| value == "true"); + let allocation_matches = observation + .labels + .get(TRAIL_ALLOCATION_LABEL) + .is_some_and(|value| value == &allocation.allocation_id); + let cleanup_matches = observation + .labels + .get(TRAIL_CLEANUP_LABEL) + .is_some_and(|value| value == &allocation.cleanup_token); + if !managed || !allocation_matches || !cleanup_matches { + return Err(Error::InvalidInput(format!( + "container name `{}` is occupied by a resource Trail does not own; refusing to adopt or replace it", + allocation.container_name + ))); + } + Ok(()) +} + +fn runtime_secret_binding_digest(secrets: &[ResolvedRuntimeSecret]) -> String { + let mut bindings = secrets + .iter() + .map(|secret| { + format!( + "{}\0{}\0{}", + secret.source_path.to_string_lossy(), + secret.target, + secret.environment.as_deref().unwrap_or_default() + ) + }) + .collect::>(); + bindings.sort(); + sha256_hex(bindings.join("\0\0").as_bytes()) +} + +fn runtime_secret_bindings_match( + allocation: &RuntimeAllocation, + secrets: &[ResolvedRuntimeSecret], + observation: &RuntimeContainerObservation, +) -> bool { + allocation.secrets.is_empty() + || observation + .labels + .get(TRAIL_SECRET_BINDINGS_LABEL) + .is_some_and(|digest| digest == &runtime_secret_binding_digest(secrets)) +} + +fn wait_for_tcp_health(host_port: u16, timeout_ms: u64) -> Result<()> { + let deadline = Instant::now() + Duration::from_millis(timeout_ms); + let address = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), host_port); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(Error::InvalidInput(format!( + "runtime TCP health check timed out after {timeout_ms} ms on 127.0.0.1:{host_port}" + ))); + } + if TcpStream::connect_timeout(&address, remaining.min(Duration::from_millis(500))).is_ok() { + return Ok(()); + } + thread::sleep(remaining.min(Duration::from_millis(100))); + } +} + +fn validate_runtime_secret_file(path: &std::path::Path) -> Result<()> { + if !path.is_absolute() + || path.to_str().map_or(true, |path| { + path.contains(',') || path.chars().any(char::is_control) + }) + { + return Err(Error::InvalidInput( + "secret provider file handle must be an absolute UTF-8 path without control characters or commas" + .to_string(), + )); + } + let metadata = fs::symlink_metadata(path).map_err(|_| { + Error::InvalidInput("secret provider file handle is inaccessible".to_string()) + })?; + if !metadata.is_file() || metadata.file_type().is_symlink() || metadata.len() == 0 { + return Err(Error::InvalidInput( + "secret provider file handle must be a non-empty regular file, not a symlink" + .to_string(), + )); + } + if metadata.len() > 1024 * 1024 { + return Err(Error::InvalidInput( + "secret provider file handle exceeds one MiB".to_string(), + )); + } + #[cfg(unix)] + if metadata.mode() & 0o077 != 0 { + return Err(Error::InvalidInput( + "secret provider file permissions grant group or other access".to_string(), + )); + } + Ok(()) +} + +fn provider_output_diagnostic(output: &Output) -> String { + let bytes = if output.stderr.is_empty() { + &output.stdout + } else { + &output.stderr + }; + let text = String::from_utf8_lossy(bytes); + let diagnostic = sanitize_provider_text(&text); + if diagnostic.is_empty() { + format!("process exited with {}", output.status) + } else { + diagnostic + } +} + +fn sanitize_provider_text(text: &str) -> String { + let mut sanitized = text + .chars() + .filter(|character| !character.is_control() || *character == '\n' || *character == '\t') + .collect::(); + for marker in ["token=", "password=", "secret=", "authorization:"] { + let mut cursor = 0; + while let Some(relative_start) = sanitized[cursor..].to_ascii_lowercase().find(marker) { + let start = cursor + relative_start; + let value_start = start + marker.len(); + let value_end = sanitized[value_start..] + .find(char::is_whitespace) + .map_or(sanitized.len(), |offset| value_start + offset); + sanitized.replace_range(value_start..value_end, "[REDACTED]"); + cursor = value_start + "[REDACTED]".len(); + } + } + if sanitized.len() > MAX_PROVIDER_DIAGNOSTIC_BYTES { + sanitized.truncate(MAX_PROVIDER_DIAGNOSTIC_BYTES); + sanitized.push('…'); + } + sanitized.trim().to_string() +} + +#[cfg(test)] +mod tests { + use std::net::TcpListener; + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt; + use std::path::Path; + use std::sync::Mutex; + + use super::*; + + const DIGEST: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + #[derive(Default)] + struct FakeState { + exists: bool, + running: bool, + stopped: bool, + create_count: usize, + remove_container_count: usize, + remove_network_count: usize, + remove_volume_count: usize, + secret_binding_digest: Option, + resolved_secret_mounts: Vec<(PathBuf, String, Option)>, + } + + struct FakeRuntimeProvider { + listener: TcpListener, + state: Mutex, + foreign_labels: bool, + } + + impl FakeRuntimeProvider { + fn new(foreign_labels: bool) -> Self { + Self { + listener: TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(), + state: Mutex::new(FakeState::default()), + foreign_labels, + } + } + + fn create_count(&self) -> usize { + self.state.lock().unwrap().create_count + } + } + + impl RuntimeProvider for FakeRuntimeProvider { + fn provider_name(&self) -> &str { + "fake" + } + + fn ensure_image(&self, _reference: &str, _expected_digest: &str) -> Result<()> { + Ok(()) + } + + fn ensure_network(&self, _allocation: &RuntimeAllocation) -> Result<()> { + Ok(()) + } + + fn ensure_volume(&self, _allocation: &RuntimeAllocation) -> Result<()> { + Ok(()) + } + + fn inspect_container( + &self, + allocation: &RuntimeAllocation, + ) -> Result> { + let state = self.state.lock().unwrap(); + if !state.exists { + return Ok(None); + } + let mut labels = if self.foreign_labels { + BTreeMap::new() + } else { + BTreeMap::from([ + (TRAIL_MANAGED_LABEL.to_string(), "true".to_string()), + ( + TRAIL_ALLOCATION_LABEL.to_string(), + allocation.allocation_id.clone(), + ), + ( + TRAIL_CLEANUP_LABEL.to_string(), + allocation.cleanup_token.clone(), + ), + ]) + }; + if !self.foreign_labels { + if let Some(digest) = state.secret_binding_digest.as_ref() { + labels.insert(TRAIL_SECRET_BINDINGS_LABEL.to_string(), digest.clone()); + } + } + Ok(Some(RuntimeContainerObservation { + provider_resource_id: "fake-container-id".to_string(), + running: state.running, + host_port: Some(self.listener.local_addr().unwrap().port()), + labels, + })) + } + + fn create_container( + &self, + allocation: &RuntimeAllocation, + secrets: &[ResolvedRuntimeSecret], + ) -> Result { + let mut state = self.state.lock().unwrap(); + state.exists = true; + state.create_count += 1; + state.secret_binding_digest = + (!allocation.secrets.is_empty()).then(|| runtime_secret_binding_digest(secrets)); + state.resolved_secret_mounts = secrets + .iter() + .map(|secret| { + ( + secret.source_path.clone(), + secret.target.clone(), + secret.environment.clone(), + ) + }) + .collect(); + Ok("fake-container-id".to_string()) + } + + fn start_container(&self, _allocation: &RuntimeAllocation) -> Result<()> { + self.state.lock().unwrap().running = true; + Ok(()) + } + + fn stop_container(&self, _allocation: &RuntimeAllocation) -> Result<()> { + let mut state = self.state.lock().unwrap(); + state.running = false; + state.stopped = true; + Ok(()) + } + + fn remove_container(&self, _allocation: &RuntimeAllocation) -> Result<()> { + let mut state = self.state.lock().unwrap(); + state.exists = false; + state.running = false; + state.remove_container_count += 1; + state.secret_binding_digest = None; + state.resolved_secret_mounts.clear(); + Ok(()) + } + + fn remove_network(&self, _allocation: &RuntimeAllocation) -> Result<()> { + self.state.lock().unwrap().remove_network_count += 1; + Ok(()) + } + + fn remove_volume(&self, _allocation: &RuntimeAllocation) -> Result<()> { + self.state.lock().unwrap().remove_volume_count += 1; + Ok(()) + } + } + + fn write_service_specification(root: &Path) { + fs::write( + root.join("trail.oci.toml"), + format!( + "schema = \"trail.oci-images/v1\"\n\n[[image]]\nname = \"database-image\"\nreference = \"example.invalid/postgres@{DIGEST}\"\nplatform = \"linux/amd64\"\n\n[[service]]\nname = \"database\"\nimage = \"database-image\"\ncontainer_port = 5432\nhealth_timeout_ms = 1000\nrestart_policy = \"on_failure\"\nvolume_target = \"/var/lib/postgresql/data\"\n" + ), + ) + .unwrap(); + } + + fn runtime_workspace(lane: &str) -> (tempfile::TempDir, Trail) { + let workspace = tempfile::tempdir().unwrap(); + write_service_specification(workspace.path()); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + db.sync_workspace_environment_component(lane, "oci", None, None) + .unwrap(); + (workspace, db) + } + + fn runtime_workspace_with_secret(lane: &str, secret_path: &Path) -> (tempfile::TempDir, Trail) { + let workspace = tempfile::tempdir().unwrap(); + let reference = secret_path.to_string_lossy(); + fs::write( + workspace.path().join("trail.oci.toml"), + format!( + "schema = \"trail.oci-images/v1\"\n\n[[image]]\nname = \"database-image\"\nreference = \"example.invalid/postgres@{DIGEST}\"\nplatform = \"linux/amd64\"\n\n[[service]]\nname = \"database\"\nimage = \"database-image\"\ncontainer_port = 5432\nhealth_timeout_ms = 1000\nrestart_policy = \"on_failure\"\n\n[[service.secret]]\nname = \"database-password\"\nprovider = \"file\"\nreference = {reference:?}\nversion = \"rotation-7\"\npurpose = \"authenticate the database service\"\ninjection = \"file\"\ntarget = \"/run/secrets/database-password\"\nenvironment = \"DATABASE_PASSWORD_FILE\"\nrequired = true\n" + ), + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); + db.sync_workspace_environment_component(lane, "oci", None, None) + .unwrap(); + (workspace, db) + } + + fn assert_tree_does_not_contain(root: &Path, needle: &[u8]) { + let mut pending = vec![root.to_path_buf()]; + while let Some(path) = pending.pop() { + let metadata = fs::symlink_metadata(&path).unwrap(); + if metadata.is_dir() { + pending.extend( + fs::read_dir(path) + .unwrap() + .map(|entry| entry.unwrap().path()), + ); + } else if metadata.is_file() { + let bytes = fs::read(&path).unwrap(); + assert!( + !bytes.windows(needle.len()).any(|window| window == needle), + "secret canary leaked into {}", + path.display() + ); + } + } + } + + #[test] + fn reconcile_is_idempotent_and_recovers_a_dead_lifecycle_owner() { + let lane = "runtime-reconcile"; + let (workspace, db) = runtime_workspace(lane); + let generation = db.active_environment_generation(lane).unwrap().unwrap(); + let allocations = db + .runtime_allocations_for_generation(&generation.generation_id) + .unwrap(); + let provider = FakeRuntimeProvider::new(false); + + db.reconcile_workspace_environment_runtime_with(&provider, &allocations) + .unwrap(); + db.reconcile_workspace_environment_runtime_with(&provider, &allocations) + .unwrap(); + assert_eq!(provider.create_count(), 1); + let running = db.active_environment_generation(lane).unwrap().unwrap(); + let resource = &running.components[0].runtime_resources[0]; + assert_eq!(resource.status, "running"); + assert_eq!(resource.health_status, "healthy"); + assert_eq!( + resource.provider_resource_id.as_deref(), + Some("fake-container-id") + ); + assert!(resource.host_port.is_some()); + let command_environment = db + .lane_workspace_environment(lane) + .unwrap() + .into_iter() + .collect::>(); + assert_eq!( + command_environment + .get("TRAIL_SERVICE_DATABASE_PORT") + .and_then(|port| port.parse::().ok()), + resource.host_port + ); + let services: Value = + serde_json::from_str(command_environment.get("TRAIL_SERVICES_JSON").unwrap()).unwrap(); + assert_eq!( + services["oci-images/database"]["port"].as_u64(), + resource.host_port.map(u64::from) + ); + assert!(!db + .lane_readiness(lane) + .unwrap() + .blockers + .iter() + .any(|blocker| blocker.code == "environment_runtime_unhealthy")); + + db.conn + .execute( + "UPDATE environment_generation_runtime_resources + SET status = 'allocating', health_status = 'starting', + owner_pid = ?1, owner_start_token = 'dead-owner'", + params![i64::from(u32::MAX)], + ) + .unwrap(); + drop(db); + + let reopened = Trail::open(workspace.path()).unwrap(); + let recovered = reopened + .active_environment_generation(lane) + .unwrap() + .unwrap(); + assert_eq!( + recovered.components[0].runtime_resources[0].status, + "orphaned" + ); + let allocations = reopened + .runtime_allocations_for_generation(&generation.generation_id) + .unwrap(); + reopened + .reconcile_workspace_environment_runtime_with(&provider, &allocations) + .unwrap(); + assert_eq!(provider.create_count(), 1); + assert_eq!( + reopened + .active_environment_generation(lane) + .unwrap() + .unwrap() + .components[0] + .runtime_resources[0] + .status, + "running" + ); + } + + #[test] + fn reconcile_rejects_foreign_container_name_collisions() { + let lane = "runtime-collision"; + let (_workspace, db) = runtime_workspace(lane); + let generation = db.active_environment_generation(lane).unwrap().unwrap(); + let allocations = db + .runtime_allocations_for_generation(&generation.generation_id) + .unwrap(); + let provider = FakeRuntimeProvider::new(true); + provider.state.lock().unwrap().exists = true; + + let error = db + .reconcile_workspace_environment_runtime_with(&provider, &allocations) + .unwrap_err(); + assert!(error.to_string().contains("does not own")); + let failed = db.active_environment_generation(lane).unwrap().unwrap(); + let resource = &failed.components[0].runtime_resources[0]; + assert_eq!(resource.status, "failed"); + assert_eq!(resource.health_status, "unhealthy"); + assert!(resource.reason.as_deref().unwrap().contains("does not own")); + } + + #[test] + fn opaque_file_secret_is_late_bound_revocable_and_never_persisted() { + let secret_dir = tempfile::tempdir().unwrap(); + let secret_path = secret_dir.path().join("database-password"); + let canary = b"trail-secret-canary-4b9d0f6d7c2a"; + fs::write(&secret_path, canary).unwrap(); + #[cfg(unix)] + fs::set_permissions(&secret_path, fs::Permissions::from_mode(0o600)).unwrap(); + + let lane = "runtime-secret"; + let (workspace, db) = runtime_workspace_with_secret(lane, &secret_path); + let generation = db.active_environment_generation(lane).unwrap().unwrap(); + let allocations = db + .runtime_allocations_for_generation(&generation.generation_id) + .unwrap(); + let provider = FakeRuntimeProvider::new(false); + db.reconcile_workspace_environment_runtime_with(&provider, &allocations) + .unwrap(); + + let running = db.active_environment_generation(lane).unwrap().unwrap(); + let resource = &running.components[0].runtime_resources[0]; + assert_eq!(resource.secret_statuses.len(), 1); + assert_eq!(resource.secret_statuses[0].status, "available"); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM environment_secret_access_audit + WHERE generation_id = ?1 AND secret_name = 'database-password' + AND status = 'available'", + params![generation.generation_id], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1 + ); + assert_eq!( + provider.state.lock().unwrap().resolved_secret_mounts, + vec![( + fs::canonicalize(&secret_path).unwrap(), + "/run/secrets/database-password".to_string(), + Some("DATABASE_PASSWORD_FILE".to_string()) + )] + ); + assert_tree_does_not_contain(&workspace.path().join(".trail"), canary); + + fs::remove_file(&secret_path).unwrap(); + let error = db + .reconcile_workspace_environment_runtime_with(&provider, &allocations) + .unwrap_err(); + assert!(error.to_string().contains("database-password")); + assert!(!error + .to_string() + .contains(&secret_path.to_string_lossy()[..])); + assert!(!error.to_string().contains("trail-secret-canary")); + assert!(provider.state.lock().unwrap().stopped); + let failed = db.active_environment_generation(lane).unwrap().unwrap(); + let resource = &failed.components[0].runtime_resources[0]; + assert_eq!(resource.status, "failed"); + assert_eq!(resource.secret_statuses[0].status, "unavailable"); + assert!(db + .lane_readiness(lane) + .unwrap() + .blockers + .iter() + .any(|blocker| blocker.code == "environment_secret_unavailable")); + assert!(db + .lane_workspace_environment(lane) + .unwrap_err() + .to_string() + .contains("env runtime reconcile")); + assert_tree_does_not_contain(&workspace.path().join(".trail"), canary); + } + + #[test] + fn rotating_a_secret_file_handle_recreates_the_owned_container() { + let first_dir = tempfile::tempdir().unwrap(); + let second_dir = tempfile::tempdir().unwrap(); + let first_path = first_dir.path().join("database-password"); + let second_path = second_dir.path().join("database-password"); + fs::write(&first_path, b"first-rotation-canary").unwrap(); + fs::write(&second_path, b"second-rotation-canary").unwrap(); + #[cfg(unix)] + { + fs::set_permissions(&first_path, fs::Permissions::from_mode(0o600)).unwrap(); + fs::set_permissions(&second_path, fs::Permissions::from_mode(0o600)).unwrap(); + } + + let lane = "runtime-secret-rotation"; + let (_workspace, db) = runtime_workspace_with_secret(lane, &first_path); + let generation = db.active_environment_generation(lane).unwrap().unwrap(); + let mut allocations = db + .runtime_allocations_for_generation(&generation.generation_id) + .unwrap(); + let provider = FakeRuntimeProvider::new(false); + db.reconcile_workspace_environment_runtime_with(&provider, &allocations) + .unwrap(); + + allocations[0].secrets[0].reference = second_path.to_string_lossy().into_owned(); + db.reconcile_workspace_environment_runtime_with(&provider, &allocations) + .unwrap(); + + let state = provider.state.lock().unwrap(); + assert_eq!(state.create_count, 2); + assert_eq!(state.remove_container_count, 1); + assert_eq!( + state.resolved_secret_mounts, + vec![( + fs::canonicalize(&second_path).unwrap(), + "/run/secrets/database-password".to_string(), + Some("DATABASE_PASSWORD_FILE".to_string()) + )] + ); + } + + #[test] + fn secret_provider_handles_inside_workspace_are_rejected_before_runtime_create() { + let external = tempfile::NamedTempFile::new().unwrap(); + fs::write(external.path(), b"external-placeholder").unwrap(); + #[cfg(unix)] + fs::set_permissions(external.path(), fs::Permissions::from_mode(0o600)).unwrap(); + let lane = "runtime-workspace-secret"; + let (workspace, db) = runtime_workspace_with_secret(lane, external.path()); + let in_workspace = workspace.path().join("accidental-secret"); + fs::write(&in_workspace, b"must-not-be-mounted").unwrap(); + #[cfg(unix)] + fs::set_permissions(&in_workspace, fs::Permissions::from_mode(0o600)).unwrap(); + let generation = db.active_environment_generation(lane).unwrap().unwrap(); + let mut allocations = db + .runtime_allocations_for_generation(&generation.generation_id) + .unwrap(); + allocations[0].secrets[0].reference = in_workspace.to_string_lossy().into_owned(); + let provider = FakeRuntimeProvider::new(false); + + let error = db + .reconcile_workspace_environment_runtime_with(&provider, &allocations) + .unwrap_err(); + assert!(error.to_string().contains("outside the Trail workspace")); + assert_eq!(provider.create_count(), 0); + assert_eq!( + db.active_environment_generation(lane) + .unwrap() + .unwrap() + .components[0] + .runtime_resources[0] + .secret_statuses[0] + .status, + "unavailable" + ); + } + + #[test] + fn new_generations_reuse_private_volume_and_cleanup_retired_ephemera() { + let lane = "runtime-roll-forward"; + let (_workspace, db) = runtime_workspace(lane); + let first_generation = db.active_environment_generation(lane).unwrap().unwrap(); + let first = db + .runtime_allocations_for_generation(&first_generation.generation_id) + .unwrap(); + let provider = FakeRuntimeProvider::new(false); + db.reconcile_workspace_environment_runtime_with(&provider, &first) + .unwrap(); + + { + let mut state = provider.state.lock().unwrap(); + state.exists = false; + state.running = false; + } + db.sync_workspace_environment_component(lane, "oci", None, None) + .unwrap(); + let second_generation = db.active_environment_generation(lane).unwrap().unwrap(); + let second = db + .runtime_allocations_for_generation(&second_generation.generation_id) + .unwrap(); + assert_ne!(first[0].allocation_id, second[0].allocation_id); + assert_ne!(first[0].container_name, second[0].container_name); + assert_ne!(first[0].network_name, second[0].network_name); + assert_eq!(first[0].volume_name, second[0].volume_name); + + db.reconcile_workspace_environment_runtime_with(&provider, &second) + .unwrap(); + let retired = db + .runtime_allocations_where( + "WHERE generation_id = ?1 ORDER BY component_id, resource_name", + [first_generation.generation_id.as_str()], + ) + .unwrap(); + db.cleanup_retired_workspace_environment_runtime_with(&provider, &retired) + .unwrap(); + + let state = provider.state.lock().unwrap(); + assert_eq!(state.remove_network_count, 1); + assert_eq!(state.remove_volume_count, 0); + drop(state); + let retired_status = db + .conn + .query_row( + "SELECT status FROM environment_generation_runtime_resources WHERE generation_id = ?1", + params![first_generation.generation_id], + |row| row.get::<_, String>(0), + ) + .unwrap(); + assert_eq!(retired_status, "stopped"); + } + + #[test] + fn provider_diagnostics_are_bounded_and_redact_common_secret_forms() { + let input = format!( + "token=abc password=hunter2 secret=value authorization:Bearer-value {}", + "x".repeat(MAX_PROVIDER_DIAGNOSTIC_BYTES * 2) + ); + let sanitized = sanitize_provider_text(&input); + assert!(!sanitized.contains("abc")); + assert!(!sanitized.contains("hunter2")); + assert!(!sanitized.contains("Bearer-value")); + assert!(sanitized.len() <= MAX_PROVIDER_DIAGNOSTIC_BYTES + '…'.len_utf8()); + } + + #[test] + fn secret_file_validation_rejects_empty_broad_and_symlink_handles() { + let directory = tempfile::tempdir().unwrap(); + let empty = directory.path().join("empty"); + fs::write(&empty, b"").unwrap(); + assert!(validate_runtime_secret_file(&empty).is_err()); + + let broad = directory.path().join("broad"); + fs::write(&broad, b"not-a-real-secret").unwrap(); + #[cfg(unix)] + { + fs::set_permissions(&broad, fs::Permissions::from_mode(0o644)).unwrap(); + assert!(validate_runtime_secret_file(&broad).is_err()); + fs::set_permissions(&broad, fs::Permissions::from_mode(0o600)).unwrap(); + let symlink = directory.path().join("symlink"); + std::os::unix::fs::symlink(&broad, &symlink).unwrap(); + assert!(validate_runtime_secret_file(&symlink).is_err()); + } + } + + #[test] + fn secret_status_and_access_audit_commit_or_rollback_together() { + let secret = tempfile::NamedTempFile::new().unwrap(); + fs::write(secret.path(), b"audit-rollback-canary").unwrap(); + #[cfg(unix)] + fs::set_permissions(secret.path(), fs::Permissions::from_mode(0o600)).unwrap(); + let lane = "runtime-secret-audit"; + let (_workspace, db) = runtime_workspace_with_secret(lane, secret.path()); + let generation = db.active_environment_generation(lane).unwrap().unwrap(); + let allocations = db + .runtime_allocations_for_generation(&generation.generation_id) + .unwrap(); + db.conn + .execute_batch( + "CREATE TRIGGER reject_secret_access_audit + BEFORE INSERT ON environment_secret_access_audit + BEGIN SELECT RAISE(ABORT, 'injected audit failure'); END;", + ) + .unwrap(); + let provider = FakeRuntimeProvider::new(false); + + assert!(db + .reconcile_workspace_environment_runtime_with(&provider, &allocations) + .is_err()); + assert_eq!( + db.conn + .query_row( + "SELECT status FROM environment_generation_runtime_secrets + WHERE generation_id = ?1", + params![generation.generation_id], + |row| row.get::<_, String>(0), + ) + .unwrap(), + "pending" + ); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM environment_secret_access_audit", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + assert_eq!(provider.create_count(), 0); + } +} diff --git a/trail/src/db/lane/workspace_view.rs b/trail/src/db/lane/workspace_view.rs index 6c6787c..fc65cb1 100644 --- a/trail/src/db/lane/workspace_view.rs +++ b/trail/src/db/lane/workspace_view.rs @@ -1,4 +1,4 @@ -use super::workdir::{ViewMutationBarrier, ViewMutationJournal}; +use super::workdir::{ViewIntentWriter, ViewJournalCut, ViewMutationBarrier, ViewMutationJournal}; use super::*; const VIEW_JOURNAL_FILE: &str = "mutation-journal.jsonl"; @@ -26,10 +26,16 @@ pub(crate) struct WorkspaceMountLease { impl WorkspaceMountLease { pub(crate) fn mark_mounted(&mut self) -> Result<()> { let db = Trail::open_with_db_dir(&self.workspace_root, &self.db_dir)?; - db.conn.execute( + let updated = db.conn.execute( "UPDATE workspace_views SET status = 'mounted', heartbeat_at = ?1, updated_at = ?1 WHERE view_id = ?2 AND owner_start_token = ?3", params![now_ts(), self.view_id, self.owner_start_token], )?; + if updated != 1 { + return Err(Error::InvalidInput(format!( + "workspace view `{}` mount lease was lost before the backend became ready", + self.view_id + ))); + } Ok(()) } @@ -121,13 +127,13 @@ impl Trail { return Err(Error::InvalidInput(format!( "workspace view `{}` mount worker stopped before becoming ready", view.view_id - ))) + ))); } Err(std::sync::mpsc::TryRecvError::Disconnected) => { return Err(Error::InvalidInput(format!( "workspace view `{}` mount worker exited unexpectedly", view.view_id - ))) + ))); } Err(std::sync::mpsc::TryRecvError::Empty) => {} } @@ -178,8 +184,8 @@ impl Trail { } match mode { - LaneWorkdirMode::OverlayCow => { - let mount = self.mount_overlay_cow_workdir_for_lane(lane)?; + LaneWorkdirMode::FuseCow => { + let mount = self.mount_fuse_cow_workdir_for_lane(lane)?; self.wait_for_workspace_unmount_request(lane, &view, &stop_path)?; drop(mount); } @@ -188,11 +194,23 @@ impl Trail { self.wait_for_workspace_unmount_request(lane, &view, &stop_path)?; drop(mount); } + LaneWorkdirMode::DokanCow => { + #[cfg(target_os = "windows")] + { + let mount = self.mount_dokan_cow_workdir_for_lane(lane)?; + self.wait_for_workspace_unmount_request(lane, &view, &stop_path)?; + drop(mount); + } + #[cfg(not(target_os = "windows"))] + return Err(Error::InvalidInput( + "dokan-cow workdirs are currently supported only on Windows".to_string(), + )); + } _ => { return Err(Error::InvalidInput(format!( "lane `{lane}` uses `{}` rather than a layered workspace view", mode.as_str() - ))) + ))); } } let _ = fs::remove_file(&stop_path); @@ -364,6 +382,7 @@ impl Trail { ] { fs::create_dir_all(dir)?; } + ViewMutationJournal::initialize_storage(&paths.source_upper)?; let now = now_ts(); self.conn.execute( "INSERT INTO workspace_views \ @@ -401,11 +420,27 @@ impl Trail { backend: &str, ) -> Result { let _lock = self.acquire_write_lock()?; - let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + let initial = self.lane_workspace_view(lane)?.ok_or_else(|| { Error::InvalidInput(format!( "lane `{lane}` does not have a layered workspace view" )) })?; + let _barrier = ViewMutationBarrier::shared(Path::new(&initial.meta_dir))?; + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` lost its layered workspace view while acquiring its mount lease" + )) + })?; + if view.view_id != initial.view_id + || view.meta_dir != initial.meta_dir + || view.generation != initial.generation + || view.base_root != initial.base_root + || view.base_change != initial.base_change + { + return Err(Error::InvalidInput(format!( + "workspace view for lane `{lane}` changed while acquiring its mount lease; retry the mount" + ))); + } if view.backend != backend { return Err(Error::InvalidInput(format!( "workspace view `{}` uses backend `{}`; `{backend}` cannot mount it", @@ -447,10 +482,25 @@ impl Trail { } } let owner_start_token = current_process_start_token(); - self.conn.execute( - "UPDATE workspace_views SET status = 'mounting', owner_pid = ?1, owner_start_token = ?2, heartbeat_at = ?3, updated_at = ?3 WHERE view_id = ?4", - params![std::process::id(), owner_start_token, now_ts(), view.view_id], + let installed = self.conn.execute( + "UPDATE workspace_views SET status = 'mounting', owner_pid = ?1, owner_start_token = ?2, heartbeat_at = ?3, updated_at = ?3 \ + WHERE view_id = ?4 AND generation = ?5 AND base_root = ?6 AND base_change = ?7 AND owner_pid IS NULL", + params![ + std::process::id(), + owner_start_token, + now_ts(), + view.view_id, + view.generation as i64, + view.base_root.0, + view.base_change.0, + ], )?; + if installed != 1 { + return Err(Error::InvalidInput(format!( + "workspace view `{}` changed before its mount lease could be installed; retry the mount", + view.view_id + ))); + } write_file_atomic( &Path::new(&view.meta_dir).join("mount.json"), &serde_json::to_vec_pretty(&serde_json::json!({ @@ -533,7 +583,7 @@ impl Trail { fn recover_workspace_checkpoint_markers(&self) -> Result<()> { let mut stmt = self.conn.prepare( - "SELECT v.view_id, v.meta_dir, v.source_upper, v.checkpoint_seq, b.ref_name \ + "SELECT v.view_id, v.meta_dir, v.source_upper, v.checkpoint_seq, v.generation, v.checkpoint_root, b.ref_name \ FROM workspace_views v JOIN lane_branches b ON b.lane_id = v.lane_id \ WHERE b.status != 'removed'", )?; @@ -544,51 +594,98 @@ impl Trail { row.get::<_, String>(1)?, row.get::<_, String>(2)?, row.get::<_, i64>(3)?.max(0) as u64, - row.get::<_, String>(4)?, + row.get::<_, i64>(4)?.max(0) as u64, + row.get::<_, String>(5)?, + row.get::<_, String>(6)?, )) })? .collect::, _>>()?; - for (view_id, meta_dir, source_upper, checkpoint_seq, ref_name) in rows { + for ( + view_id, + meta_dir, + source_upper, + checkpoint_seq, + generation, + checkpoint_root, + ref_name, + ) in rows + { let path = Path::new(&meta_dir).join("clean-checkpoint.json"); - let bytes = match fs::read(&path) { - Ok(bytes) => bytes, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, - Err(err) => return Err(Error::Io(err)), - }; - let marker: serde_json::Value = serde_json::from_slice(&bytes)?; - if marker["view_id"].as_str() != Some(view_id.as_str()) { + let read_barrier = ViewMutationBarrier::shared(Path::new(&meta_dir))?; + let marker = fs::read(&path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + let mirror_matches = marker.as_ref().is_some_and(|marker| { + marker["view_id"].as_str() == Some(view_id.as_str()) + && marker["root_id"].as_str() == Some(checkpoint_root.as_str()) + && marker["journal_sequence"].as_u64() == Some(checkpoint_seq) + && marker["generation"].as_u64() == Some(generation) + && marker["recovery_qualified"].as_bool() == Some(true) + }); + if mirror_matches + && read_barrier.checkpoint_sequence() == checkpoint_seq + && read_barrier.checkpoint_generation() == generation + { + continue; + } + drop(read_barrier); + + let mut barrier = ViewMutationBarrier::exclusive(Path::new(&meta_dir))?; + let journal = ViewMutationJournal::open(Path::new(&source_upper))?; + if !journal.recovery_is_qualified() { + return Err(Error::ChangeLedgerReconcileRequired { + scope: view_id, + state: "unqualified_view_journal".into(), + reason: + "workspace checkpoint recovery cannot qualify journal generation evidence" + .into(), + command: "trail ledger reconcile".into(), + }); + } + if checkpoint_seq > journal.last_sequence() { return Err(Error::Corrupt(format!( - "workspace checkpoint marker `{}` belongs to a different view", - path.display() + "SQLite workspace checkpoint {} is ahead of durable journal sequence {} for `{}`", + checkpoint_seq, + journal.last_sequence(), + view_id, ))); } - let sequence = marker["journal_sequence"].as_u64().ok_or_else(|| { - Error::Corrupt(format!( - "workspace checkpoint marker `{}` has no journal sequence", - path.display() - )) - })?; - let root = marker["root_id"].as_str().ok_or_else(|| { - Error::Corrupt(format!( - "workspace checkpoint marker `{}` has no root", - path.display() - )) - })?; - let journal_sequence = - ViewMutationJournal::open(Path::new(&source_upper))?.last_sequence(); - if sequence > journal_sequence { + let head = self.get_ref(&ref_name)?; + if head.root_id.0 != checkpoint_root { return Err(Error::Corrupt(format!( - "workspace checkpoint marker `{}` is ahead of its durable mutation journal", - path.display() + "workspace checkpoint root `{checkpoint_root}` is not lane head `{}` for `{view_id}`", + head.root_id.0 ))); } - let head = self.get_ref(&ref_name)?; - if sequence > checkpoint_seq && head.root_id.0 == root { - self.conn.execute( - "UPDATE workspace_views SET checkpoint_seq = ?1, checkpoint_root = ?2, updated_at = ?3 WHERE view_id = ?4", - params![sequence as i64, root, now_ts(), view_id], - )?; + let marker = fs::read(&path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + let mirror_matches = marker.as_ref().is_some_and(|marker| { + marker["view_id"].as_str() == Some(view_id.as_str()) + && marker["root_id"].as_str() == Some(checkpoint_root.as_str()) + && marker["journal_sequence"].as_u64() == Some(checkpoint_seq) + && marker["generation"].as_u64() == Some(generation) + && marker["recovery_qualified"].as_bool() == Some(true) + }); + if !mirror_matches { + let checkpoint = serde_json::json!({ + "view_id": view_id, + "root_id": checkpoint_root, + "operation": serde_json::Value::Null, + "journal_sequence": checkpoint_seq, + "journal_qualified": journal.is_qualified(), + "recovery_qualified": true, + "generation": generation, + "completed_at": now_ts(), + }); + write_file_atomic(&path, &serde_json::to_vec_pretty(&checkpoint)?, false)?; } + ViewMutationJournal::rotate_after_checkpoint( + Path::new(&source_upper), + checkpoint_seq, + generation, + )?; + barrier.record_checkpoint_cut(checkpoint_seq, generation)?; } Ok(()) } @@ -614,8 +711,8 @@ impl Trail { let head = self.get_ref(&branch.ref_name)?; let run = || self.run_workspace_command(&view, &head.root_id, command); let exit_code = match mode { - LaneWorkdirMode::OverlayCow => { - let mount = self.mount_overlay_cow_workdir_for_lane(lane)?; + LaneWorkdirMode::FuseCow => { + let mount = self.mount_fuse_cow_workdir_for_lane(lane)?; let result = run(); drop(mount); result? @@ -626,13 +723,34 @@ impl Trail { drop(mount); result? } + LaneWorkdirMode::DokanCow => { + #[cfg(target_os = "windows")] + { + let mount = self.mount_dokan_cow_workdir_for_lane(lane)?; + let result = run(); + drop(mount); + result? + } + #[cfg(not(target_os = "windows"))] + return Err(Error::InvalidInput( + "dokan-cow workdirs are currently supported only on Windows".to_string(), + )); + } _ => { return Err(Error::InvalidInput(format!( "lane `{lane}` uses `{}` rather than a layered workspace view", mode.as_str() - ))) + ))); } }; + let environment_generation = self + .conn + .query_row( + "SELECT generation_id FROM environment_view_generations WHERE view_id = ?1", + params![&view.view_id], + |row| row.get::<_, String>(0), + ) + .optional()?; self.insert_lane_event( &branch.lane_id, "workspace_view_exec_completed", @@ -642,6 +760,7 @@ impl Trail { "view_id": view.view_id, "source_root": head.root_id.0, "generation": view.generation, + "environment_generation": environment_generation, "command_fingerprint": sha256_hex(&serde_json::to_vec(command)?), "exit_code": exit_code, }), @@ -651,6 +770,7 @@ impl Trail { lane_id: branch.lane_id, source_root: head.root_id, generation: view.generation, + environment_generation, backend: view.backend, command: command.to_vec(), exit_code, @@ -730,6 +850,78 @@ impl Trail { node_cache.to_string_lossy().into_owned(), ), ]; + if let Some(generation_id) = self + .conn + .query_row( + "SELECT generation_id FROM environment_view_generations WHERE view_id = ?1", + params![&view.view_id], + |row| row.get::<_, String>(0), + ) + .optional()? + { + environment.push(("TRAIL_ENVIRONMENT_GENERATION".to_string(), generation_id)); + } + let mut runtime_stmt = self.conn.prepare( + "SELECT r.component_id, r.resource_name, r.host_port, r.status, r.health_status + FROM environment_view_generations active + JOIN environment_generation_runtime_resources r + ON r.generation_id = active.generation_id + WHERE active.view_id = ?1 + ORDER BY r.component_id, r.resource_name", + )?; + let runtime_resources = runtime_stmt + .query_map(params![&view.view_id], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + )) + })? + .collect::, _>>()?; + if !runtime_resources.is_empty() { + let mut alias_counts = BTreeMap::::new(); + for (_, resource_name, _, _, _) in &runtime_resources { + *alias_counts + .entry(runtime_service_environment_segment(resource_name)) + .or_default() += 1; + } + let mut services = serde_json::Map::new(); + for (component_id, resource_name, host_port, status, health_status) in runtime_resources + { + let host_port = host_port.filter(|_| { + status == "running" && health_status == "healthy" + }).ok_or_else(|| { + Error::InvalidInput(format!( + "runtime service `{component_id}/{resource_name}` is {status}/{health_status}; run `trail env runtime reconcile {}` before executing lane commands", + view.lane_id + )) + })?; + let address = format!("127.0.0.1:{host_port}"); + services.insert( + format!("{component_id}/{resource_name}"), + serde_json::json!({ + "host": "127.0.0.1", + "port": host_port, + "address": address, + }), + ); + let alias = runtime_service_environment_segment(&resource_name); + if alias_counts.get(&alias) == Some(&1) { + let prefix = format!("TRAIL_SERVICE_{alias}"); + environment.extend([ + (format!("{prefix}_HOST"), "127.0.0.1".to_string()), + (format!("{prefix}_PORT"), host_port.to_string()), + (format!("{prefix}_ADDRESS"), address), + ]); + } + } + environment.push(( + "TRAIL_SERVICES_JSON".to_string(), + serde_json::to_string(&services)?, + )); + } if command_available("sccache") { environment.extend([ ("RUSTC_WRAPPER".to_string(), "sccache".to_string()), @@ -900,17 +1092,92 @@ impl Trail { lane: &str, root_id: &ObjectId, operation: Option<&ChangeId>, + barrier: &mut ViewMutationBarrier, ) -> Result { + // TRAIL_FS_PRODUCER: cow_checkpoint CowPublication controlled let Some(view) = self.lane_workspace_view(lane)? else { return Ok(0); }; - let journal = ViewMutationJournal::open(Path::new(&view.source_upper))?; - let sequence = journal.last_sequence(); + let cut = checkpoint_view(Path::new(&view.source_upper))?; + if !cut.recovery_qualified { + return Err(Error::ChangeLedgerReconcileRequired { + scope: view.view_id, + state: "unqualified_view_journal".into(), + reason: "workspace checkpoint has neither a qualified changed-path journal nor a qualified independent whiteout journal".into(), + command: "trail ledger reconcile".into(), + }); + } + let sequence = cut.sequence; + let next_generation = cut.generation.saturating_add(1); + self.conn.execute( + "UPDATE workspace_views SET checkpoint_seq = ?1, checkpoint_root = ?2, generation = ?3, updated_at = ?4 WHERE view_id = ?5", + params![sequence as i64, root_id.0, next_generation as i64, now_ts(), view.view_id], + )?; + self.write_workspace_checkpoint_mirror( + &view, + root_id, + operation, + sequence, + next_generation, + cut.qualified, + barrier, + )?; + Ok(sequence) + } + + pub(crate) fn repair_workspace_checkpoint_mirror( + &self, + lane: &str, + root_id: &ObjectId, + operation: Option<&ChangeId>, + sequence: u64, + generation: u64, + journal_qualified: bool, + barrier: &mut ViewMutationBarrier, + ) -> Result<()> { + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::Corrupt(format!( + "layered lane `{lane}` has no persisted workspace view" + )) + })?; + if view.checkpoint_seq != sequence + || view.checkpoint_root.as_ref() != Some(root_id) + || view.generation != generation + { + return Err(Error::Corrupt(format!( + "workspace view `{}` checkpoint publication is not the committed SQLite generation", + view.view_id + ))); + } + self.write_workspace_checkpoint_mirror( + &view, + root_id, + operation, + sequence, + generation, + journal_qualified, + barrier, + ) + } + + fn write_workspace_checkpoint_mirror( + &self, + view: &LaneWorkspaceViewReport, + root_id: &ObjectId, + operation: Option<&ChangeId>, + sequence: u64, + generation: u64, + journal_qualified: bool, + barrier: &mut ViewMutationBarrier, + ) -> Result<()> { let checkpoint = serde_json::json!({ "view_id": view.view_id, "root_id": root_id.0, "operation": operation.map(|value| value.0.as_str()), "journal_sequence": sequence, + "journal_qualified": journal_qualified, + "recovery_qualified": true, + "generation": generation, "completed_at": now_ts(), }); write_file_atomic( @@ -919,12 +1186,13 @@ impl Trail { false, )?; test_crash_point("checkpoint_after_clean_marker"); - self.conn.execute( - "UPDATE workspace_views SET checkpoint_seq = ?1, checkpoint_root = ?2, updated_at = ?3 WHERE view_id = ?4", - params![sequence as i64, root_id.0, now_ts(), view.view_id], + ViewMutationJournal::rotate_after_checkpoint( + Path::new(&view.source_upper), + sequence, + generation, )?; - ViewMutationBarrier::record_checkpoint_sequence(Path::new(&view.meta_dir), sequence)?; - Ok(sequence) + barrier.record_checkpoint_cut(sequence, generation)?; + Ok(()) } pub(crate) fn workspace_view_last_journal_sequence( @@ -939,27 +1207,58 @@ impl Trail { lane: &str, message: Option, ) -> Result { - let record = self.record_lane_workdir(lane, message)?; - let view = self.lane_workspace_view(lane)?.ok_or_else(|| { - Error::InvalidInput(format!( - "lane `{lane}` does not have a layered workspace view" - )) - })?; - let sequence = view.checkpoint_seq; - let generated = directory_usage(Path::new(&view.generated_upper))?; - Ok(WorkspaceCheckpointReport { - view_id: view.view_id, - operation: record.operation, - root_id: record.root_id, - journal_sequence: sequence, - source_paths: record - .changed_paths - .into_iter() - .map(|item| item.path) - .collect(), - generated_dirty_paths: generated.file_count, + let metrics = self.operation_metrics.clone(); + profile_operation_metrics( + metrics.as_ref(), + OperationMetricsKind::CowCheckpoint, + || { + let record = self.record_lane_workdir(lane, message)?; + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::InvalidInput(format!( + "lane `{lane}` does not have a layered workspace view" + )) + })?; + let sequence = view.checkpoint_seq; + let report = WorkspaceCheckpointReport { + view_id: view.view_id, + operation: record.operation, + root_id: record.root_id, + journal_sequence: sequence, + source_paths: record + .changed_paths + .into_iter() + .map(|item| item.path) + .collect(), + generated_dirty_paths: record.generated_dirty_paths, + generated_path_accounting: "journal_interval".into(), + upper_recovery_walks: record.upper_recovery_walks, + }; + Ok(report) + }, + ) + } +} + +fn checkpoint_view(source_upper: &Path) -> Result { + let journal: ViewIntentWriter = ViewMutationJournal::open(source_upper)?; + Ok(journal.cut()) +} + +fn runtime_service_environment_segment(name: &str) -> String { + let mut segment = name + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() { + character.to_ascii_uppercase() + } else { + '_' + } }) + .collect::(); + if segment.as_bytes().first().is_some_and(u8::is_ascii_digit) { + segment.insert(0, '_'); } + segment } fn workspace_view_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { @@ -1081,9 +1380,14 @@ fn file_physical_bytes(metadata: &fs::Metadata) -> u64 { #[cfg(test)] mod tests { - use super::super::workdir::{ViewCore, ViewMutationBarrier, VIEW_ROOT_INO}; + use super::super::workdir::{ + ViewCore, ViewMutationBarrier, ViewMutationJournal, ViewUpperLayout, VIEW_ROOT_INO, + }; use super::*; use std::process::Stdio; + use std::sync::mpsc; + use std::thread; + use std::time::Duration; #[test] fn checkpoint_crash_helper() { @@ -1125,8 +1429,10 @@ mod tests { let mut db = Trail::open(workspace.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "checkpoint-crash", @@ -1172,6 +1478,18 @@ mod tests { "durable after kill\n" ); let mut reopened = Trail::open(workspace.path()).unwrap(); + if phase == "checkpoint_after_clean_marker" { + let view = reopened + .lane_workspace_view("checkpoint-crash") + .unwrap() + .unwrap(); + let journal = ViewMutationJournal::open(&source_upper).unwrap(); + assert_eq!(journal.generation(), view.generation); + assert_eq!(journal.last_sequence(), view.checkpoint_seq); + let barrier = ViewMutationBarrier::shared(Path::new(&view.meta_dir)).unwrap(); + assert_eq!(barrier.checkpoint_sequence(), view.checkpoint_seq); + assert_eq!(barrier.checkpoint_generation(), view.generation); + } let recovered = reopened .checkpoint_lane_workspace("checkpoint-crash", Some("recover".to_string())) .unwrap(); @@ -1194,6 +1512,47 @@ mod tests { .blockers .iter() .any(|issue| issue.code == "uncheckpointed_source_changes")); + + if phase == "checkpoint_after_clean_marker" { + for index in 0..6 { + let branch = reopened.lane_branch("checkpoint-crash").unwrap(); + let head = reopened.get_ref(&branch.ref_name).unwrap(); + let mut core = ViewCore::new_lazy( + Trail::open(workspace.path()).unwrap(), + source_upper.clone(), + head.root_id, + ) + .unwrap(); + let name = format!("post-recovery-{index}.txt"); + let file = core.create(VIEW_ROOT_INO, &name, 0o644, true).unwrap(); + core.write(file.ino, 0, b"generation recovery\n").unwrap(); + drop(core); + reopened + .checkpoint_lane_workspace( + "checkpoint-crash", + Some(format!("post-recovery-{index}")), + ) + .unwrap(); + } + + let view = reopened + .lane_workspace_view("checkpoint-crash") + .unwrap() + .unwrap(); + let journal = ViewMutationJournal::open(&source_upper).unwrap(); + assert_eq!(journal.generation(), view.generation); + assert_eq!(journal.last_sequence(), view.checkpoint_seq); + let layout = ViewUpperLayout::from_source_upper(source_upper.clone()); + let journal_files = fs::read_dir(layout.meta_dir) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_name().to_string_lossy().contains("journal.g")) + .count(); + assert!( + journal_files <= 4, + "recovered journal generations were not compacted" + ); + } } } @@ -1205,8 +1564,10 @@ mod tests { let mut db = Trail::open(workspace.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "barrier", @@ -1328,8 +1689,10 @@ mod tests { let mut db = Trail::open(temp.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "layered", @@ -1371,8 +1734,10 @@ mod tests { let mut db = Trail::open(temp.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "checkpoint", @@ -1407,6 +1772,7 @@ mod tests { .unwrap(); assert_eq!(checkpoint.source_paths, vec!["README.md"]); assert!(checkpoint.generated_dirty_paths >= 1); + assert_eq!(checkpoint.generated_path_accounting, "journal_interval"); assert!(checkpoint.journal_sequence > 0); let entry = db .root_file_entry(&checkpoint.root_id, "README.md") @@ -1484,7 +1850,7 @@ mod tests { .collect::>() ); - let change = ChangeId("ch_layered_native_equivalence".to_string()); + let change = ChangeId("change_layered_native_equivalence".to_string()); let layered = db .build_root_for_selected_disk_files_incremental( &root, @@ -1515,8 +1881,10 @@ mod tests { let mut db = Trail::open(temp.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "crash", @@ -1557,12 +1925,11 @@ mod tests { core.write(newer.ino, 0, b"uncheckpointed\n").unwrap(); drop(core); let view = db.lane_workspace_view("crash").unwrap().unwrap(); - db.conn - .execute( - "UPDATE workspace_views SET checkpoint_seq = 0, checkpoint_root = NULL WHERE view_id = ?1", - params![view.view_id], - ) - .unwrap(); + fs::write( + Path::new(&view.meta_dir).join("clean-checkpoint.json"), + b"stale postcommit mirror", + ) + .unwrap(); drop(db); let reopened = Trail::open(temp.path()).unwrap(); @@ -1590,8 +1957,10 @@ mod tests { let mut db = Trail::open(temp.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "retry", @@ -1621,12 +1990,6 @@ mod tests { assert!(recorded.operation.is_some()); let view = db.lane_workspace_view("retry").unwrap().unwrap(); fs::remove_file(Path::new(&view.meta_dir).join("clean-checkpoint.json")).unwrap(); - db.conn - .execute( - "UPDATE workspace_views SET checkpoint_seq = 0, checkpoint_root = NULL WHERE view_id = ?1", - params![view.view_id], - ) - .unwrap(); drop(db); let mut reopened = Trail::open(temp.path()).unwrap(); @@ -1652,8 +2015,10 @@ mod tests { let mut db = Trail::open(temp.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "lease", @@ -1700,6 +2065,101 @@ mod tests { ); } + #[test] + fn mark_mounted_fails_when_mount_lease_cas_is_lost() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "lease-cas", + Some("main"), + mode, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let view = db.lane_workspace_view("lease-cas").unwrap().unwrap(); + let mut lease = db + .acquire_workspace_mount_lease("lease-cas", &view.backend) + .unwrap(); + db.conn + .execute( + "UPDATE workspace_views SET owner_start_token = 'replacement-owner' WHERE view_id = ?1", + params![view.view_id], + ) + .unwrap(); + + assert!(lease.mark_mounted().is_err()); + assert_ne!( + db.lane_workspace_view("lease-cas").unwrap().unwrap().status, + "mounted" + ); + } + + #[test] + fn workspace_mount_lease_waits_for_exclusive_view_barrier() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "barrier-lease", + Some("main"), + mode, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let view = db.lane_workspace_view("barrier-lease").unwrap().unwrap(); + let meta_dir = PathBuf::from(&view.meta_dir); + let root = temp.path().to_path_buf(); + let backend = view.backend; + drop(db); + let (opened_tx, opened_rx) = mpsc::channel(); + let (go_tx, go_rx) = mpsc::channel(); + let (acquired_tx, acquired_rx) = mpsc::channel(); + let worker = thread::spawn(move || { + let db = Trail::open(root).unwrap(); + opened_tx.send(()).unwrap(); + go_rx.recv().unwrap(); + let lease = db + .acquire_workspace_mount_lease("barrier-lease", &backend) + .unwrap(); + acquired_tx.send(()).unwrap(); + drop(lease); + }); + opened_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let barrier = ViewMutationBarrier::exclusive(&meta_dir).unwrap(); + go_tx.send(()).unwrap(); + assert!(acquired_rx + .recv_timeout(Duration::from_millis(150)) + .is_err()); + drop(barrier); + acquired_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + worker.join().unwrap(); + } + #[test] fn unmount_request_is_acknowledged_by_the_mount_owner_before_lease_release() { let temp = tempfile::tempdir().unwrap(); @@ -1708,8 +2168,10 @@ mod tests { let mut db = Trail::open(temp.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "unmount-control", @@ -1768,8 +2230,10 @@ mod tests { let mut db = Trail::open(temp.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; for lane in ["cargo-a", "cargo-b"] { db.spawn_lane_with_workdir_mode_paths_and_neighbors( @@ -1836,8 +2300,10 @@ mod tests { let mut db = Trail::open(temp.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "quota", @@ -1894,6 +2360,7 @@ mod tests { let started = Instant::now(); let mut paths = SortedBatchBuilder::new(db.store.clone(), root_map_prolly_config()); let mut file_index = BatchBuilder::new(db.store.clone(), root_map_prolly_config()); + let mut case_fold = SortedBatchBuilder::new(db.store.clone(), root_map_prolly_config()); const PATHS: u64 = 1_000_000; for index in 0..PATHS { let path = format!("tree/{:04}/{:08}.txt", index / 10_000, index); @@ -1902,13 +2369,21 @@ mod tests { .add(path.as_bytes().to_vec(), cbor(&entry).unwrap()) .unwrap(); file_index.add(entry.file_id.encode_key(), path.as_bytes().to_vec()); + case_fold + .add( + case_insensitive_path_key(&path).into_bytes(), + path.as_bytes().to_vec(), + ) + .unwrap(); } let path_tree = paths.build().unwrap(); let file_index_tree = file_index.build().unwrap(); + let case_fold_tree = case_fold.build().unwrap(); let root = WorktreeRoot { version: ROOT_OBJECT_VERSION, path_map_root: tree_root_hex(&path_tree), file_index_map_root: tree_root_hex(&file_index_tree), + case_fold_map_root: tree_root_hex(&case_fold_tree), file_count: PATHS, total_text_bytes: PATHS * entry.size_bytes, created_by: change.clone(), @@ -1928,8 +2403,10 @@ mod tests { let view_started = Instant::now(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; let mut cores = Vec::new(); let mut exclusive_bytes = 0_u64; diff --git a/trail/src/db/merge/conflicts.rs b/trail/src/db/merge/conflicts.rs index 8ca76bf..a9ced02 100644 --- a/trail/src/db/merge/conflicts.rs +++ b/trail/src/db/merge/conflicts.rs @@ -165,9 +165,9 @@ impl Trail { "UPDATE conflict_sets SET status = 'resolved' WHERE conflict_set_id = ?1", params![conflict_set_id], )?; - if let Some(queue_id) = pending.queue_id { + if let Some(queue_id) = pending.lane_queue_id { self.conn.execute( - "UPDATE merge_queue SET status = 'merged', updated_at = ?1 WHERE queue_id = ?2", + "UPDATE lane_merge_queue SET status = 'merged', updated_at = ?1 WHERE queue_id = ?2", params![now_ts(), queue_id], )?; } @@ -266,7 +266,7 @@ impl Trail { Ok(Some(ConflictExplanation { merge: ConflictMergeContext { merge_id: pending.merge_id, - queue_id: pending.queue_id, + lane_queue_id: pending.lane_queue_id, source_ref: pending.source_ref, target_ref: pending.target_ref, base_change: pending.base_change, diff --git a/trail/src/db/merge/git_export.rs b/trail/src/db/merge/git_export.rs index 04e21bb..c239aeb 100644 --- a/trail/src/db/merge/git_export.rs +++ b/trail/src/db/merge/git_export.rs @@ -16,19 +16,59 @@ impl Trail { } pub fn git_export_commit(&mut self, range: &str, message: &str) -> Result { + self.reset_git_handoff_metrics(); let _lock = self.acquire_write_lock()?; + let git_state = self.current_git_state()?.ok_or_else(|| { + Error::Git(format!( + "git export requires a Git working tree at {}", + self.workspace_root.display() + )) + })?; + self.git_export_commit_with_state( + range, + message, + git_state, + GitExportPolicy::AllowFullSnapshot, + ) + } + + pub(crate) fn git_export_commit_mapped( + &mut self, + range: &str, + message: &str, + state: Option, + ) -> Result { + let _lock = self.acquire_write_lock()?; + let git_state = match state { + Some(state) => state, + None => self.current_git_state()?.ok_or_else(|| { + Error::Git(format!( + "git export requires a Git working tree at {}", + self.workspace_root.display() + )) + })?, + }; + self.git_export_commit_with_state( + range, + message, + git_state, + GitExportPolicy::RequireMappedDelta, + ) + } + + fn git_export_commit_with_state( + &mut self, + range: &str, + message: &str, + git_state: GitState, + policy: GitExportPolicy, + ) -> Result { let message = message.trim(); if message.is_empty() { return Err(Error::InvalidInput( "git export commit message cannot be empty".to_string(), )); } - let Some(git_state) = self.current_git_state()? else { - return Err(Error::Git(format!( - "git export requires a Git working tree at {}", - self.workspace_root.display() - ))); - }; let (left, right) = parse_range(range)?; let left_ref = self.resolve_refish(left)?; let right_ref = self.resolve_refish(right)?; @@ -52,34 +92,59 @@ impl Trail { } self.ensure_lane_merge_readiness(lane)?; } + let can_export_delta = match (git_state.head.as_deref(), git_state.dirty) { + (Some(head), false) => { + if self.git_clean_head_matches_root_mapping(head, &left_ref.root_id)? { + true + } else { + match policy { + GitExportPolicy::RequireMappedDelta => { + return Err(Error::GitMappingRequired(format!( + "clean Git HEAD `{head}` has no mapping for Trail root `{}`", + left_ref.root_id.0 + ))); + } + GitExportPolicy::AllowFullSnapshot => self + .ensure_git_clean_head_root_mapping( + &branch, + &left_ref.change_id, + &left_ref.root_id, + head, + )?, + } + } + } + _ => match policy { + GitExportPolicy::RequireMappedDelta => { + return Err(Error::GitMappingRequired(format!( + "mapped delta export requires a clean Git HEAD for Trail root `{}`", + left_ref.root_id.0 + ))); + } + GitExportPolicy::AllowFullSnapshot => false, + }, + }; let mut patch_left = BTreeMap::new(); let mut patch_right = BTreeMap::new(); - let _diff = self.diff_root_file_maps( + let diff = self.diff_root_file_maps( &left_ref.root_id, &right_ref.root_id, &mut patch_left, &mut patch_right, )?; - let can_export_delta = git_state - .head - .as_deref() - .filter(|_| !git_state.dirty) - .map(|head| { - self.ensure_git_clean_head_root_mapping( - &branch, - &left_ref.change_id, - &left_ref.root_id, - head, - ) - }) - .transpose()? - .unwrap_or(false); + self.set_git_changed_path_count(diff.summaries.len() as u64); let tree_oid = if let (true, Some(head)) = (can_export_delta, git_state.head.as_deref()) { + self.set_git_export_mode(GitExportMode::MappedDelta); self.git_write_tree_from_head_delta(head, &patch_left, &patch_right)? } else { let files = self.load_root_files(&right_ref.root_id)?; + self.set_git_export_mode(GitExportMode::FullSnapshot); + self.add_git_full_root_file_count(files.len() as u64); self.git_write_tree(&files)? }; + if can_export_delta { + self.record_git_plumbing_command(); + } let commit = self.git_commit_tree(&tree_oid, git_state.head.as_deref(), message)?; let mapping = self.insert_git_mapping_for_state( "export", @@ -97,6 +162,7 @@ impl Trail { commit, parent: git_state.head, mapping, + performance: self.git_handoff_metrics_report(), }) } @@ -218,3 +284,101 @@ impl Trail { Ok(true) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn run_git(root: &Path, args: &[&str]) { + let output = Command::new("git") + .arg("-C") + .arg(root) + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "git failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn mapped_git_export_requires_preexisting_clean_mapping() { + if Command::new("git").arg("--version").output().is_err() { + return; + } + let temp = tempfile::tempdir().unwrap(); + run_git(temp.path(), &["init"]); + run_git(temp.path(), &["config", "user.email", "trail@example.test"]); + run_git(temp.path(), &["config", "user.name", "Trail Test"]); + fs::write(temp.path().join("README.md"), "one\n").unwrap(); + run_git(temp.path(), &["add", "README.md"]); + run_git(temp.path(), &["commit", "-m", "initial"]); + let init = Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + fs::write(temp.path().join("README.md"), "one\ntwo\n").unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let record = db + .record(Some("main"), Some("change".into()), Actor::human(), false) + .unwrap(); + run_git(temp.path(), &["checkout", "--", "README.md"]); + let range = format!("{}..{}", init.operation.0, record.operation.unwrap().0); + let err = db + .git_export_commit_mapped(&range, "mapped", None) + .unwrap_err(); + assert!(matches!(err, Error::GitMappingRequired(_))); + assert!(db.git_mappings(10).unwrap().is_empty()); + } + + #[test] + fn mapped_git_export_reports_mapped_delta_mode() { + if Command::new("git").arg("--version").output().is_err() { + return; + } + let temp = tempfile::tempdir().unwrap(); + run_git(temp.path(), &["init"]); + run_git(temp.path(), &["config", "user.email", "trail@example.test"]); + run_git(temp.path(), &["config", "user.name", "Trail Test"]); + fs::write(temp.path().join("README.md"), "one\n").unwrap(); + run_git(temp.path(), &["add", "README.md"]); + run_git(temp.path(), &["commit", "-m", "initial"]); + let init = Trail::init(temp.path(), "main", InitImportMode::GitTracked, false).unwrap(); + fs::write(temp.path().join("README.md"), "one\ntwo\n").unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let record = db + .record(Some("main"), Some("change".into()), Actor::human(), false) + .unwrap(); + run_git(temp.path(), &["checkout", "--", "README.md"]); + let range = format!("{}..{}", init.operation.0, record.operation.unwrap().0); + + let report = db.git_export_commit_mapped(&range, "mapped", None).unwrap(); + + assert_eq!(report.performance.export_mode, "mapped_delta"); + } + + #[test] + fn general_git_export_reports_full_snapshot_mode_without_mapping() { + if Command::new("git").arg("--version").output().is_err() { + return; + } + let temp = tempfile::tempdir().unwrap(); + run_git(temp.path(), &["init"]); + run_git(temp.path(), &["config", "user.email", "trail@example.test"]); + run_git(temp.path(), &["config", "user.name", "Trail Test"]); + fs::write(temp.path().join("README.md"), "one\n").unwrap(); + run_git(temp.path(), &["add", "README.md"]); + run_git(temp.path(), &["commit", "-m", "initial"]); + let init = Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + fs::write(temp.path().join("README.md"), "one\ntwo\n").unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + assert!(db.git_mappings(10).unwrap().is_empty()); + let record = db + .record(Some("main"), Some("change".into()), Actor::human(), false) + .unwrap(); + let range = format!("{}..{}", init.operation.0, record.operation.unwrap().0); + + let report = db.git_export_commit(&range, "full snapshot").unwrap(); + + assert_eq!(report.performance.export_mode, "full_snapshot"); + } +} diff --git a/trail/src/db/merge/lane.rs b/trail/src/db/merge/lane.rs index c436eae..a51d3cd 100644 --- a/trail/src/db/merge/lane.rs +++ b/trail/src/db/merge/lane.rs @@ -1,6 +1,49 @@ use super::*; use crate::db::lane::ViewMutationBarrier; +#[cfg(debug_assertions)] +std::thread_local! { + static FAIL_LAYERED_UPDATE_POSTCOMMIT_BOUNDARY: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(test)] +fn set_layered_update_postcommit_failure_for_current_thread(boundary: Option<&'static str>) { + FAIL_LAYERED_UPDATE_POSTCOMMIT_BOUNDARY.with(|selected| *selected.borrow_mut() = boundary); +} + +#[cfg(debug_assertions)] +fn fail_layered_update_postcommit_if_requested(boundary: &'static str) -> Result<()> { + if FAIL_LAYERED_UPDATE_POSTCOMMIT_BOUNDARY.with(|selected| *selected.borrow() == Some(boundary)) + { + return Err(Error::Io(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + format!("injected layered update post-commit failure at {boundary}"), + ))); + } + Ok(()) +} + +#[cfg(not(debug_assertions))] +fn fail_layered_update_postcommit_if_requested(_boundary: &'static str) -> Result<()> { + Ok(()) +} + +fn layered_update_committed_repair( + operation: &ObjectId, + repair: &'static str, + error: Error, +) -> Error { + match error { + Error::CommittedRepairRequired { .. } => error, + error => Error::CommittedRepairRequired { + operation: operation.0.clone(), + repair: repair.into(), + reason: error.to_string(), + }, + } +} + impl Trail { pub fn merge_lane(&mut self, lane: &str, into: &str) -> Result { self.merge_lane_with_options(lane, into, false) @@ -62,13 +105,19 @@ impl Trail { source_branch: &str, checkpoint: bool, ) -> Result { + // TRAIL_FS_PRODUCER: layered_lane_update RestoreProjection controlled if checkpoint { self.checkpoint_lane_workspace( lane, Some(format!("Checkpoint before updating from {source_branch}")), )?; } - let _lock = self.acquire_write_lock()?; + let ledger_authority = crate::db::change_ledger::command_authority_enabled(); + let _lock = if ledger_authority { + None + } else { + Some(self.acquire_write_lock()?) + }; validate_ref_segment(lane)?; validate_ref_segment(source_branch)?; let lane_branch = self.lane_branch(lane)?; @@ -78,14 +127,25 @@ impl Trail { .is_transparent_cow() { return Err(Error::InvalidInput(format!( - "lane update requires a layered workspace view; lane {lane} is not overlay-cow or nfs-cow" + "lane update requires a layered workspace view; lane {lane} is not fuse-cow, nfs-cow, or dokan-cow" ))); } - let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + let view_hint = self.lane_workspace_view(lane)?.ok_or_else(|| { Error::Corrupt(format!( "layered lane {lane} has no persisted workspace view" )) })?; + let mut barrier = ViewMutationBarrier::exclusive(Path::new(&view_hint.meta_dir))?; + let view = self.lane_workspace_view(lane)?.ok_or_else(|| { + Error::Corrupt(format!( + "layered lane {lane} lost its persisted workspace view while acquiring the mutation barrier" + )) + })?; + if view.view_id != view_hint.view_id || view.meta_dir != view_hint.meta_dir { + return Err(Error::InvalidInput(format!( + "workspace view for lane {lane} changed while acquiring its mutation barrier" + ))); + } if let (Some(pid), Some(token)) = (view.owner_pid, view.owner_start_token.as_deref()) { if process_matches_start_token(pid, token) { return Err(Error::InvalidInput(format!( @@ -94,7 +154,6 @@ impl Trail { ))); } } - let _barrier = ViewMutationBarrier::exclusive(Path::new(&view.meta_dir))?; let lane_head = self.get_ref(&lane_branch.ref_name)?; self.ensure_lane_workdir_clean(&lane_branch, &lane_head)?; let source_ref_name = branch_ref(source_branch); @@ -139,42 +198,140 @@ impl Trail { after_root: root_id.clone(), branch: lane_branch.ref_name.clone(), actor, - session_id: lane_branch.session_id, + session_id: lane_branch.session_id.clone(), message: Some(format!("Update lane {lane} from {source_branch}")), changes: diff.changes, created_at: now_ts(), }; - let operation_id = self.store_operation(&operation)?; - self.advance_ref_cas(&lane_head, &change_id, &root_id, &operation_id)?; - self.conn.execute( - "UPDATE lane_branches SET base_change = ?1, base_root = ?2, head_change = ?3, head_root = ?4, status = 'active', updated_at = ?5 WHERE lane_id = ?6", - params![ - source_ref.change_id.0, - source_ref.root_id.0, - change_id.0, - root_id.0, - now_ts(), - lane_branch.lane_id - ], - )?; - self.conn.execute( - "UPDATE workspace_views SET base_change = ?1, base_root = ?2, generation = generation + 1, updated_at = ?3 WHERE view_id = ?4", - params![change_id.0, root_id.0, now_ts(), view.view_id], - )?; - self.complete_workspace_checkpoint(lane, &root_id, Some(&change_id))?; - self.refresh_workspace_environment_staleness(lane)?; - self.insert_lane_event( - &lane_branch.lane_id, - "workspace_view_updated", - Some(&change_id), - None, - &serde_json::json!({ - "view_id": view.view_id, - "source_branch": source_branch, - "source_change": source_ref.change_id.0, - "root_id": root_id.0, - }), - )?; + self.invalidate_lane_marker_if_materialized(&lane_branch)?; + let committed_operation_id = if ledger_authority { + let checkpoint_sequence = self.workspace_view_last_journal_sequence(&view)?; + let unaligned_scope = crate::db::change_ledger::ExpectedScope { + scope_id: crate::db::change_ledger::materialized_lane_scope_id( + &self.config.workspace.id.0, + &lane_branch.lane_id, + ), + epoch: 0, + ref_name: lane_head.name.clone(), + ref_generation: u64::try_from(lane_head.generation) + .map_err(|_| Error::Corrupt("negative lane ref generation".into()))?, + baseline_root: lane_head.root_id.clone(), + policy_fingerprint: [0; 32], + policy_generation: 0, + filesystem_identity: Vec::new(), + provider_identity: Vec::new(), + }; + let evidence = crate::db::change_ledger::IntentEvidence { + exact_paths: Vec::new(), + complete_prefixes: Vec::new(), + }; + crate::db::change_ledger::run_ref_advancing_projection( + self, + &unaligned_scope, + &lane_head, + &lane_branch.lane_id, + crate::db::change_ledger::IntentProducer::RestoreProjection, + &operation, + &evidence, + crate::db::change_ledger::RefAdvancingProjectionMode::LayeredUpdateReconcileRequired { + reason: "layered_view_update_requires_task12_reconciliation", + lane_base_change: &source_ref.change_id, + lane_base_root: &source_ref.root_id, + view_id: &view.view_id, + checkpoint_sequence, + }, + |_, _| { + Err(Error::InvalidInput( + "layered view update cannot use a native lane observer".into(), + )) + }, + |_, _| Ok(()), + )? + .operation_id + } else { + crate::db::change_ledger::commit_lane_operation_atomic( + self, + &lane_head, + &lane_branch.lane_id, + &operation, + None, + )? + }; + if !ledger_authority { + (|| -> Result<()> { + self.conn.execute( + "UPDATE lane_branches SET base_change=?1,base_root=?2,status='active',updated_at=?3 + WHERE lane_id=?4 AND head_change=?5 AND head_root=?6", + params![ + source_ref.change_id.0, + source_ref.root_id.0, + now_ts(), + lane_branch.lane_id, + change_id.0, + root_id.0, + ], + )?; + self.conn.execute( + "UPDATE workspace_views SET base_change = ?1, base_root = ?2, generation = generation + 1, updated_at = ?3 WHERE view_id = ?4", + params![change_id.0, root_id.0, now_ts(), view.view_id], + )?; + Ok(()) + })() + .map_err(|error| { + layered_update_committed_repair( + &committed_operation_id, + "layered update association", + error, + ) + })?; + } + (|| { + fail_layered_update_postcommit_if_requested("checkpoint")?; + self.complete_workspace_checkpoint(lane, &root_id, Some(&change_id), &mut barrier) + })() + .map_err(|error| { + layered_update_committed_repair( + &committed_operation_id, + "layered update workspace checkpoint", + error, + ) + })?; + (|| { + fail_layered_update_postcommit_if_requested("environment")?; + self.refresh_workspace_environment_staleness(lane) + })() + .map_err(|error| { + layered_update_committed_repair( + &committed_operation_id, + "layered update environment staleness", + error, + ) + })?; + (|| { + fail_layered_update_postcommit_if_requested("event")?; + self.insert_lane_event( + &lane_branch.lane_id, + "workspace_view_updated", + Some(&change_id), + None, + &serde_json::json!({ + "view_id": view.view_id, + "source_branch": source_branch, + "source_change": source_ref.change_id.0, + "root_id": root_id.0, + }), + ) + })() + .map_err(|error| { + layered_update_committed_repair(&committed_operation_id, "layered update event", error) + })?; + (|| { + fail_layered_update_postcommit_if_requested("marker")?; + self.publish_lane_marker_if_materialized(lane) + })() + .map_err(|error| { + layered_update_committed_repair(&committed_operation_id, "layered update marker", error) + })?; Ok(MergeReport { operation: change_id, source_ref: source_ref_name, @@ -421,11 +578,28 @@ impl Trail { if !workdir_path.is_dir() { return Err(Error::WorkspaceNotFound(workdir_path)); } - Ok(Some(self.lane_workdir_record_changed_paths( - branch, - head, - &workdir_path, - )?)) + let lane_record = self.lane_record(&branch.lane_id)?; + let workdir_mode = self.lane_workdir_mode_for(&lane_record, branch)?; + if workdir_mode.is_transparent_cow() { + return Ok(Some(self.lane_workdir_record_changed_paths( + branch, + head, + &workdir_path, + )?)); + } + if crate::db::change_ledger::command_authority_enabled() { + let (comparison, _) = self.compare_materialized_lane_candidates( + &branch.lane_id, + crate::db::change_ledger::CandidateMaterialization::ManifestOnly, + )?; + Ok(Some(comparison.summaries)) + } else { + Ok(Some(self.lane_workdir_record_changed_paths( + branch, + head, + &workdir_path, + )?)) + } } fn ensure_direct_lane_merge_allowed( @@ -439,7 +613,7 @@ impl Trail { return Ok(()); } Err(Error::InvalidInput(format!( - "direct merge into shared target `{into}` is disabled by default; run `trail merge-queue add {lane} --into {into}` and `trail merge-queue run`, or pass `--direct` to merge immediately" + "direct merge into shared target `{into}` is disabled by default; run `trail lane merge-queue add {lane} --into {into}` and `trail lane merge-queue run`, or pass `--direct` to merge immediately" ))) } } @@ -461,7 +635,7 @@ fn lane_refresh_preview_next_steps( )]; } vec![format!( - "Review the changed paths, then merge via `trail merge-queue add {lane} --into {target_branch}` when ready." + "Review the changed paths, then merge via `trail lane merge-queue add {lane} --into {target_branch}` when ready." )] } @@ -469,17 +643,33 @@ fn lane_refresh_preview_next_steps( mod tests { use super::*; + static LAYERED_UPDATE_AUTHORITY_TEST: OnceLock> = OnceLock::new(); + + struct AuthorityReset; + + impl Drop for AuthorityReset { + fn drop(&mut self) { + crate::db::set_command_authority_override(false); + } + } + + fn layered_mode() -> LaneWorkdirMode { + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + } + } + #[test] fn layered_lane_update_refuses_an_active_workspace_writer() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "baseline\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(temp.path()).unwrap(); - let mode = if cfg!(target_os = "macos") { - LaneWorkdirMode::NfsCow - } else { - LaneWorkdirMode::OverlayCow - }; + let mode = layered_mode(); db.spawn_lane_with_workdir_mode_paths_and_neighbors( "active-update", Some("main"), @@ -508,4 +698,59 @@ mod tests { db.update_layered_lane_from("active-update", "main", false) .unwrap(); } + + #[test] + fn layered_update_postcommit_failures_keep_the_committed_projection() { + let _lock = LAYERED_UPDATE_AUTHORITY_TEST + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + crate::db::set_command_authority_override(true); + let _authority_reset = AuthorityReset; + + for boundary in ["checkpoint", "environment", "event", "marker"] { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "baseline\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + &format!("repair-{boundary}"), + Some("main"), + layered_mode(), + None, + None, + None, + &[], + false, + ) + .unwrap(); + let lane = format!("repair-{boundary}"); + let before = db.get_ref(&lane_ref(&lane)).unwrap(); + let source = db.get_ref(&branch_ref("main")).unwrap(); + + set_layered_update_postcommit_failure_for_current_thread(Some(boundary)); + let result = db.update_layered_lane_from(&lane, "main", false); + set_layered_update_postcommit_failure_for_current_thread(None); + + let error = result.unwrap_err(); + let after = db.get_ref(&lane_ref(&lane)).unwrap(); + assert_ne!(after.change_id, before.change_id, "boundary {boundary}"); + let branch = db.lane_branch(&lane).unwrap(); + assert_eq!(branch.base_change, source.change_id, "boundary {boundary}"); + let view = db.lane_workspace_view(&lane).unwrap().unwrap(); + assert_eq!(view.base_change, after.change_id, "boundary {boundary}"); + match error { + Error::CommittedRepairRequired { + operation, + repair, + reason, + } => { + assert_eq!(operation, after.operation_id.0, "boundary {boundary}"); + assert!(repair.contains(boundary), "{repair}"); + assert!(reason.contains(boundary), "{reason}"); + } + other => panic!("expected committed repair error at {boundary}, got {other:?}"), + } + } + } } diff --git a/trail/src/db/merge/queue.rs b/trail/src/db/merge/queue.rs index 1113488..8d25272 100644 --- a/trail/src/db/merge/queue.rs +++ b/trail/src/db/merge/queue.rs @@ -1,45 +1,49 @@ use super::*; impl Trail { - pub fn enqueue_merge( + pub fn enqueue_lane_merge( &mut self, - source: &str, + lane: &str, target: &str, priority: i64, - ) -> Result { + ) -> Result { let _lock = self.acquire_write_lock()?; - let source_ref = self.normalize_merge_queue_source_ref(source)?; - let target_ref = self.normalize_merge_queue_target_ref(target)?; + let lane = self.lane_details(lane).map_err(|err| match err { + Error::RefNotFound(_) => Error::InvalidInput(format!("lane `{lane}` does not exist")), + other => other, + })?; + let target_ref = self.normalize_lane_merge_queue_target_ref(target)?; if let Some(entry) = self .conn .query_row( - "SELECT queue_id, source_ref, target_ref, status, priority, created_at, updated_at \ - FROM merge_queue \ - WHERE source_ref = ?1 AND target_ref = ?2 AND status IN ('queued', 'running') \ - ORDER BY created_at LIMIT 1", - params![source_ref, target_ref], - merge_queue_row, + "SELECT q.queue_id, q.lane_id, l.name, q.target_ref, q.status, q.priority, q.created_at, q.updated_at \ + FROM lane_merge_queue q JOIN lanes l ON l.lane_id = q.lane_id \ + WHERE q.lane_id = ?1 AND q.target_ref = ?2 AND q.status IN ('queued', 'running') \ + ORDER BY q.created_at LIMIT 1", + params![lane.record.lane_id, target_ref], + lane_merge_queue_row, ) .optional()? { - return Ok(MergeQueueAddReport { entry }); + return Ok(LaneMergeQueueAddReport { entry }); } let now = now_ts(); - let seed = format!("{source_ref}:{target_ref}:{priority}:{now}"); + let seed = format!("{}:{target_ref}:{priority}:{now}", lane.record.lane_id); let hash = sha256_hex(seed.as_bytes()); - let queue_id = format!("mq_{}", &hash[..16]); + let queue_id = format!("lmq_{}", &hash[..16]); self.conn.execute( - "INSERT INTO merge_queue \ - (queue_id, source_ref, target_ref, status, priority, created_at, updated_at) \ + "INSERT INTO lane_merge_queue \ + (queue_id, lane_id, target_ref, status, priority, created_at, updated_at) \ VALUES (?1, ?2, ?3, 'queued', ?4, ?5, ?5)", - params![queue_id, source_ref, target_ref, priority, now], + params![queue_id, lane.record.lane_id, target_ref, priority, now], )?; - Ok(MergeQueueAddReport { - entry: MergeQueueEntry { + Ok(LaneMergeQueueAddReport { + entry: LaneMergeQueueEntry { queue_id, - source_ref, + lane_id: lane.record.lane_id, + lane: lane.record.name, target_ref, status: "queued".to_string(), priority, @@ -49,40 +53,42 @@ impl Trail { }) } - pub fn list_merge_queue(&self) -> Result> { + pub fn list_lane_merge_queue(&self) -> Result> { let mut stmt = self.conn.prepare( - "SELECT queue_id, source_ref, target_ref, status, priority, created_at, updated_at \ - FROM merge_queue ORDER BY status = 'queued' DESC, priority DESC, created_at ASC", + "SELECT q.queue_id, q.lane_id, l.name, q.target_ref, q.status, q.priority, q.created_at, q.updated_at \ + FROM lane_merge_queue q JOIN lanes l ON l.lane_id = q.lane_id \ + ORDER BY q.status = 'queued' DESC, q.priority DESC, q.created_at ASC", )?; - let rows = stmt.query_map([], merge_queue_row)?; + let rows = stmt.query_map([], lane_merge_queue_row)?; rows.collect::, _>>() .map_err(Error::from) } - pub fn remove_merge_queue(&mut self, selector: &str) -> Result { + pub fn remove_lane_merge_queue( + &mut self, + selector: &str, + ) -> Result { let _lock = self.acquire_write_lock()?; - let lane_candidate = lane_ref(selector); - let branch_candidate = branch_ref(selector); let entry = self .conn .query_row( - "SELECT queue_id, source_ref, target_ref, status, priority, created_at, updated_at \ - FROM merge_queue \ - WHERE (queue_id = ?1 OR source_ref = ?1 OR source_ref = ?2 OR source_ref = ?3) \ - AND status NOT IN ('merged', 'cancelled') \ - ORDER BY priority DESC, created_at ASC LIMIT 1", - params![selector, lane_candidate, branch_candidate], - merge_queue_row, + "SELECT q.queue_id, q.lane_id, l.name, q.target_ref, q.status, q.priority, q.created_at, q.updated_at \ + FROM lane_merge_queue q JOIN lanes l ON l.lane_id = q.lane_id \ + WHERE (q.queue_id = ?1 OR q.lane_id = ?1 OR l.name = ?1) \ + AND q.status NOT IN ('merged', 'cancelled') \ + ORDER BY q.priority DESC, q.created_at ASC LIMIT 1", + params![selector], + lane_merge_queue_row, ) .optional()? .ok_or_else(|| Error::InvalidInput(format!("merge queue item `{selector}` not found")))?; let now = now_ts(); self.conn.execute( - "UPDATE merge_queue SET status = 'cancelled', updated_at = ?1 WHERE queue_id = ?2", + "UPDATE lane_merge_queue SET status = 'cancelled', updated_at = ?1 WHERE queue_id = ?2", params![now, entry.queue_id], )?; - Ok(MergeQueueRemoveReport { - entry: MergeQueueEntry { + Ok(LaneMergeQueueRemoveReport { + entry: LaneMergeQueueEntry { status: "cancelled".to_string(), updated_at: now, ..entry @@ -90,23 +96,23 @@ impl Trail { }) } - pub fn explain_merge_queue(&mut self, selector: &str) -> Result { + pub fn explain_lane_merge_queue( + &mut self, + selector: &str, + ) -> Result { let _lock = self.acquire_write_lock()?; - let entry = self.merge_queue_entry_by_selector(selector)?; - let mut readiness = None; + let entry = self.lane_merge_queue_entry_by_selector(selector)?; let mut blockers = Vec::new(); let mut warnings = Vec::new(); let mut next_steps = Vec::new(); - if let Some(lane) = entry.source_ref.strip_prefix(LANE_REF_PREFIX) { - let report = self.lane_readiness(lane)?; - blockers.extend(report.blockers.clone()); - warnings.extend(report.warnings.clone()); - next_steps.extend(merge_queue_readiness_next_steps(lane, &report)); - readiness = Some(report); - } + let report = self.lane_readiness(&entry.lane_id)?; + blockers.extend(report.blockers.clone()); + warnings.extend(report.warnings.clone()); + next_steps.extend(lane_merge_queue_readiness_next_steps(&entry.lane, &report)); + let readiness = Some(report); - let (dry_run, error) = match self.merge_queue_entry_dry_run(&entry) { + let (dry_run, error) = match self.lane_merge_queue_entry_dry_run(&entry) { Ok(report) => { if !report.conflicts.is_empty() { blockers.push(readiness_issue( @@ -119,7 +125,7 @@ impl Trail { "conflicts": report.conflicts.clone() })), )); - next_steps.push(merge_queue_dry_run_next_step(&entry)); + next_steps.push(lane_merge_queue_dry_run_next_step(&entry)); } (Some(report), None) } @@ -131,7 +137,7 @@ impl Trail { Some(serde_json::json!({ "error": message })), )); next_steps.push( - "Fix the preflight error, then run `trail merge-queue explain` again." + "Fix the preflight error, then run `trail lane merge-queue explain` again." .to_string(), ); (None, Some(message)) @@ -139,10 +145,10 @@ impl Trail { }; if blockers.is_empty() { - next_steps.push("Run `trail merge-queue run` to merge this item.".to_string()); + next_steps.push("Run `trail lane merge-queue run` to merge this item.".to_string()); } - Ok(MergeQueueExplainReport { + Ok(LaneMergeQueueExplainReport { entry, readiness, dry_run, @@ -153,22 +159,26 @@ impl Trail { }) } - pub fn run_merge_queue(&mut self, limit: Option) -> Result { + pub fn run_lane_merge_queue( + &mut self, + limit: Option, + ) -> Result { let _lock = self.acquire_write_lock()?; - let entries = self.queued_merge_entries(limit)?; + let entries = self.queued_lane_merge_entries(limit)?; let mut processed = Vec::new(); let mut stopped_on_conflict = false; let mut stopped_on_failure = false; for entry in entries { - self.set_merge_queue_status(&entry.queue_id, "running")?; - let context = match self.merge_queue_context(&entry.source_ref, &entry.target_ref) { + self.set_lane_merge_queue_status(&entry.queue_id, "running")?; + let context = match self.lane_merge_queue_context(&entry) { Ok(context) => context, Err(err) => { - self.set_merge_queue_status(&entry.queue_id, "failed")?; - processed.push(MergeQueueRunItem { + self.set_lane_merge_queue_status(&entry.queue_id, "failed")?; + processed.push(LaneMergeQueueRunItem { queue_id: entry.queue_id, - source_ref: entry.source_ref, + lane_id: entry.lane_id, + lane: entry.lane, target_ref: entry.target_ref, status: "failed".to_string(), operation: None, @@ -180,9 +190,9 @@ impl Trail { } }; - match self.merge_queue_entry(&entry) { + match self.lane_merge_queue_entry(&entry) { Ok(report) => { - self.set_merge_queue_status(&entry.queue_id, "merged")?; + self.set_lane_merge_queue_status(&entry.queue_id, "merged")?; self.insert_merge_result( &entry, &context, @@ -190,9 +200,10 @@ impl Trail { "merged", None, )?; - processed.push(MergeQueueRunItem { + processed.push(LaneMergeQueueRunItem { queue_id: entry.queue_id, - source_ref: report.source_ref, + lane_id: entry.lane_id, + lane: entry.lane, target_ref: report.target_ref, status: "merged".to_string(), operation: Some(report.operation), @@ -204,7 +215,7 @@ impl Trail { let is_conflict = matches!(err, Error::Conflict(_)); let status = if is_conflict { "conflicted" } else { "failed" }; let message = err.to_string(); - self.set_merge_queue_status(&entry.queue_id, status)?; + self.set_lane_merge_queue_status(&entry.queue_id, status)?; self.insert_merge_result( &entry, &context, @@ -212,9 +223,10 @@ impl Trail { status, is_conflict.then_some(message.as_str()), )?; - processed.push(MergeQueueRunItem { + processed.push(LaneMergeQueueRunItem { queue_id: entry.queue_id, - source_ref: entry.source_ref, + lane_id: entry.lane_id, + lane: entry.lane, target_ref: entry.target_ref, status: status.to_string(), operation: None, @@ -231,7 +243,7 @@ impl Trail { } } - Ok(MergeQueueRunReport { + Ok(LaneMergeQueueRunReport { processed, stopped_on_conflict, stopped_on_failure, @@ -239,26 +251,21 @@ impl Trail { } } -fn merge_queue_dry_run_next_step(entry: &MergeQueueEntry) -> String { +fn lane_merge_queue_dry_run_next_step(entry: &LaneMergeQueueEntry) -> String { let target = entry .target_ref .strip_prefix(MAIN_REF_PREFIX) .unwrap_or(&entry.target_ref); - if let Some(lane) = entry.source_ref.strip_prefix(LANE_REF_PREFIX) { - return format!( - "Inspect conflicts with `trail merge-lane {lane} --into {target} --dry-run` or run the queue to record a conflict set." - ); - } - let source = entry - .source_ref - .strip_prefix(MAIN_REF_PREFIX) - .unwrap_or(&entry.source_ref); format!( - "Inspect conflicts with `trail merge {source} --into {target} --dry-run` or run the queue to record a conflict set." + "Inspect conflicts with `trail lane merge {} --into {target} --dry-run` or run the queue to record a conflict set.", + entry.lane ) } -fn merge_queue_readiness_next_steps(lane: &str, readiness: &LaneReadinessReport) -> Vec { +fn lane_merge_queue_readiness_next_steps( + lane: &str, + readiness: &LaneReadinessReport, +) -> Vec { let mut steps = Vec::new(); for issue in &readiness.blockers { match issue.code.as_str() { diff --git a/trail/src/db/merge/queue_store.rs b/trail/src/db/merge/queue_store.rs index 7c0fa94..406d04e 100644 --- a/trail/src/db/merge/queue_store.rs +++ b/trail/src/db/merge/queue_store.rs @@ -1,21 +1,7 @@ use super::*; impl Trail { - pub(crate) fn normalize_merge_queue_source_ref(&self, source: &str) -> Result { - if source.starts_with("refs/") { - self.get_ref(source)?; - return Ok(source.to_string()); - } - let lane_ref_name = lane_ref(source); - if self.try_get_ref(&lane_ref_name)?.is_some() { - return Ok(lane_ref_name); - } - let branch_ref_name = branch_ref(source); - self.get_ref(&branch_ref_name)?; - Ok(branch_ref_name) - } - - pub(crate) fn normalize_merge_queue_target_ref(&self, target: &str) -> Result { + pub(crate) fn normalize_lane_merge_queue_target_ref(&self, target: &str) -> Result { let target_ref_name = branch_ref(target); if !target_ref_name.starts_with(MAIN_REF_PREFIX) { return Err(Error::InvalidInput( @@ -26,104 +12,87 @@ impl Trail { Ok(target_ref_name) } - pub(crate) fn queued_merge_entries( + pub(crate) fn queued_lane_merge_entries( &self, limit: Option, - ) -> Result> { - let sql = - "SELECT queue_id, source_ref, target_ref, status, priority, created_at, updated_at \ - FROM merge_queue WHERE status = 'queued' \ - ORDER BY priority DESC, created_at ASC"; + ) -> Result> { + let sql = "SELECT q.queue_id, q.lane_id, l.name, q.target_ref, q.status, q.priority, q.created_at, q.updated_at \ + FROM lane_merge_queue q JOIN lanes l ON l.lane_id = q.lane_id \ + WHERE q.status = 'queued' ORDER BY q.priority DESC, q.created_at ASC"; match limit { Some(limit) => { let mut stmt = self.conn.prepare(&format!("{sql} LIMIT ?1"))?; - let rows = stmt.query_map(params![limit as i64], merge_queue_row)?; + let rows = stmt.query_map(params![limit as i64], lane_merge_queue_row)?; rows.collect::, _>>() .map_err(Error::from) } None => { let mut stmt = self.conn.prepare(sql)?; - let rows = stmt.query_map([], merge_queue_row)?; + let rows = stmt.query_map([], lane_merge_queue_row)?; rows.collect::, _>>() .map_err(Error::from) } } } - pub(crate) fn set_merge_queue_status(&self, queue_id: &str, status: &str) -> Result<()> { + pub(crate) fn set_lane_merge_queue_status(&self, queue_id: &str, status: &str) -> Result<()> { self.conn.execute( - "UPDATE merge_queue SET status = ?1, updated_at = ?2 WHERE queue_id = ?3", + "UPDATE lane_merge_queue SET status = ?1, updated_at = ?2 WHERE queue_id = ?3", params![status, now_ts(), queue_id], )?; Ok(()) } - pub(crate) fn merge_queue_entry(&mut self, entry: &MergeQueueEntry) -> Result { + pub(crate) fn lane_merge_queue_entry( + &mut self, + entry: &LaneMergeQueueEntry, + ) -> Result { let target = entry .target_ref .strip_prefix(MAIN_REF_PREFIX) .unwrap_or(&entry.target_ref); - if let Some(lane) = entry.source_ref.strip_prefix(LANE_REF_PREFIX) { - return self.merge_lane_unlocked(lane, target, false, false); - } - if let Some(source) = entry.source_ref.strip_prefix(MAIN_REF_PREFIX) { - return self.merge_branches_unlocked(source, target, false); - } - Err(Error::InvalidInput(format!( - "merge queue source `{}` must be a lane or branch ref", - entry.source_ref - ))) + self.merge_lane_unlocked(&entry.lane_id, target, false, false) } - pub(crate) fn merge_queue_entry_dry_run( + pub(crate) fn lane_merge_queue_entry_dry_run( &mut self, - entry: &MergeQueueEntry, + entry: &LaneMergeQueueEntry, ) -> Result { let target = entry .target_ref .strip_prefix(MAIN_REF_PREFIX) .unwrap_or(&entry.target_ref); - if let Some(lane) = entry.source_ref.strip_prefix(LANE_REF_PREFIX) { - return self.merge_lane_unlocked(lane, target, true, false); - } - if let Some(source) = entry.source_ref.strip_prefix(MAIN_REF_PREFIX) { - return self.merge_branches_unlocked(source, target, true); - } - Err(Error::InvalidInput(format!( - "merge queue source `{}` must be a lane or branch ref", - entry.source_ref - ))) + self.merge_lane_unlocked(&entry.lane_id, target, true, false) } - pub(crate) fn merge_queue_entry_by_selector(&self, selector: &str) -> Result { - let lane_candidate = lane_ref(selector); - let branch_candidate = branch_ref(selector); + pub(crate) fn lane_merge_queue_entry_by_selector( + &self, + selector: &str, + ) -> Result { self.conn .query_row( - "SELECT queue_id, source_ref, target_ref, status, priority, created_at, updated_at \ - FROM merge_queue \ - WHERE (queue_id = ?1 OR source_ref = ?1 OR source_ref = ?2 OR source_ref = ?3) \ - AND status NOT IN ('merged', 'cancelled') \ - ORDER BY status = 'queued' DESC, priority DESC, created_at ASC LIMIT 1", - params![selector, lane_candidate, branch_candidate], - merge_queue_row, + "SELECT q.queue_id, q.lane_id, l.name, q.target_ref, q.status, q.priority, q.created_at, q.updated_at \ + FROM lane_merge_queue q JOIN lanes l ON l.lane_id = q.lane_id \ + WHERE (q.queue_id = ?1 OR q.lane_id = ?1 OR l.name = ?1) \ + AND q.status NOT IN ('merged', 'cancelled') \ + ORDER BY q.status = 'queued' DESC, q.priority DESC, q.created_at ASC LIMIT 1", + params![selector], + lane_merge_queue_row, ) .optional()? .ok_or_else(|| Error::InvalidInput(format!("merge queue item `{selector}` not found"))) } - pub(crate) fn merge_queue_context( + pub(crate) fn lane_merge_queue_context( &self, - source_ref_name: &str, - target_ref_name: &str, + entry: &LaneMergeQueueEntry, ) -> Result { + let lane = self.lane_details(&entry.lane_id)?; + let source_ref_name = &lane.branch.ref_name; + let target_ref_name = &entry.target_ref; let source_ref = self.get_ref(source_ref_name)?; let target_ref = self.get_ref(target_ref_name)?; - let base_change = if let Some(lane) = source_ref_name.strip_prefix(LANE_REF_PREFIX) { - self.lane_branch(lane)?.base_change - } else { - self.common_parent_hint(&source_ref.change_id, &target_ref.change_id)? - }; + let base_change = lane.branch.base_change; let base_ref = self.ref_from_change(&base_change)?; Ok(MergeContext { base_change, @@ -141,13 +110,13 @@ impl Trail { ) -> Result { self.conn .query_row( - "SELECT merge_id, queue_id, source_ref, target_ref, base_change, left_change, right_change, base_root, left_root, right_root \ + "SELECT merge_id, lane_queue_id, source_ref, target_ref, base_change, left_change, right_change, base_root, left_root, right_root \ FROM merge_results WHERE conflict_set = ?1 ORDER BY created_at DESC LIMIT 1", params![conflict_set_id], |row| { Ok(PendingConflictMerge { merge_id: row.get(0)?, - queue_id: row.get(1)?, + lane_queue_id: row.get(1)?, source_ref: row.get(2)?, target_ref: row.get(3)?, base_change: ChangeId(row.get(4)?), @@ -169,15 +138,16 @@ impl Trail { pub(crate) fn insert_merge_result( &self, - entry: &MergeQueueEntry, + entry: &LaneMergeQueueEntry, context: &MergeContext, result_change: Option<&ChangeId>, status: &str, conflict_detail: Option<&str>, ) -> Result<()> { + let lane = self.lane_details(&entry.lane_id)?; self.insert_merge_result_for_refs( Some(&entry.queue_id), - &entry.source_ref, + &lane.branch.ref_name, &entry.target_ref, context, result_change, @@ -189,7 +159,7 @@ impl Trail { pub(crate) fn insert_merge_result_for_refs( &self, - queue_id: Option<&str>, + lane_queue_id: Option<&str>, source_ref: &str, target_ref: &str, context: &MergeContext, @@ -200,7 +170,7 @@ impl Trail { let created_at = now_ts(); let seed = format!( "{}:{}:{}:{}:{}", - queue_id.unwrap_or("direct"), + lane_queue_id.unwrap_or("direct"), source_ref, target_ref, status, @@ -225,11 +195,11 @@ impl Trail { let result_change = result_change.map(|change| change.0.clone()); self.conn.execute( "INSERT INTO merge_results \ - (merge_id, queue_id, source_ref, target_ref, base_change, left_change, right_change, base_root, left_root, right_root, result_change, status, conflict_set, created_at) \ + (merge_id, lane_queue_id, source_ref, target_ref, base_change, left_change, right_change, base_root, left_root, right_root, result_change, status, conflict_set, created_at) \ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", params![ merge_id, - queue_id, + lane_queue_id, source_ref, target_ref, context.base_change.0, diff --git a/trail/src/db/mod.rs b/trail/src/db/mod.rs index 4ee2a39..b53dd89 100644 --- a/trail/src/db/mod.rs +++ b/trail/src/db/mod.rs @@ -1,23 +1,26 @@ use std::cell::Cell; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; use std::fs; -use std::fs::OpenOptions; -use std::io::{Read, Write}; +use std::fs::{File, OpenOptions}; +use std::io::{Read, Seek, SeekFrom, Write}; #[cfg(unix)] -use std::os::unix::fs::{symlink as symlink_file, MetadataExt, PermissionsExt}; +use std::os::unix::fs::{symlink as symlink_file, FileTypeExt, MetadataExt, PermissionsExt}; use std::path::{Component, Path, PathBuf}; use std::process::Command; use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, Mutex, + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, Condvar, Mutex, OnceLock, }; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use ignore::WalkBuilder; use prolly::{ - BatchBuilder, BatchOp, Cid, Config, Diff, Encoding, Prolly, SlateDbStore, SortedBatchBuilder, - SqliteStore, Store, Tree, + BatchBuilder, BatchOp, Cid, Config, Diff, Encoding, Prolly, SortedBatchBuilder, Store, Tree, }; +use prolly_store_slatedb::SlateDbStore; +#[cfg(unix)] +use prolly_store_sqlite::sqlite_main_file_identity; +use prolly_store_sqlite::{SqliteMainFileIdentity, SqliteStore}; use rusqlite::{params, params_from_iter, Connection, OptionalExtension}; use serde::{de::DeserializeOwned, Serialize}; use sha2::{Digest, Sha256}; @@ -34,7 +37,9 @@ use crate::model::*; const CONFIG_FILE: &str = "config.toml"; const HEAD_FILE: &str = "HEAD"; const DB_RELATIVE_PATH: &str = "index/trail.sqlite"; -const TRAIL_SCHEMA_VERSION: i64 = 4; +const SCHEMA_EXCLUSION_FILE: &str = "schema-exclusion.lock"; +const SCHEMA_VALIDATION_LEADER_FILE: &str = "schema-validation.lock"; +const TRAIL_SCHEMA_VERSION: i64 = 18; const SCHEMA_META_VERSION_KEY: &str = "schema.version"; const SCHEMA_META_APP_VERSION_KEY: &str = "app.version"; const MAIN_REF_PREFIX: &str = "refs/branches/"; @@ -42,737 +47,2857 @@ const LANE_REF_PREFIX: &str = "refs/lanes/"; const ROOT_OBJECT_VERSION: u16 = 1; const TEXT_OBJECT_VERSION: u16 = 1; -thread_local! { - static WRITE_LOCK_WAIT_DEADLINE: Cell> = const { Cell::new(None) }; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum SchemaOpenMode { + FreshCreate, + Existing, } -const OP_OBJECT_VERSION: u16 = 1; -const BLOB_OBJECT_VERSION: u16 = 1; -const MESSAGE_OBJECT_VERSION: u16 = 1; -const ANCHOR_OBJECT_VERSION: u16 = 1; -const WORKSPACE_LAYER_MANIFEST_KIND: &str = "workspace_layer_manifest"; -const WORKSPACE_LAYER_MANIFEST_VERSION: u16 = 1; -const OBJECT_CACHE_MAX_ENTRIES: usize = 4096; -const OBJECT_CACHE_MAX_BYTES: usize = 64 * 1024 * 1024; -const ORDER_KEY_STEP: u64 = 1024; -const LANE_TEST_OUTPUT_PREVIEW_BYTES: usize = 64 * 1024; -const DEFAULT_CRABIGNORE_PATTERNS: &[&str] = &[ - ".trail/", - ".git/", - ".env", - ".env.*", - "*.pem", - "*.key", - "*.p12", - "*.pfx", - "id_rsa", - "id_ed25519", - "node_modules/", - "target/", - "dist/", - "build/", - "coverage/", -]; -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct RootDirectoryChild { - pub(crate) name: String, - pub(crate) path: String, - pub(crate) entry: Option, +#[derive(Clone, Debug, Eq, PartialEq)] +struct SchemaFileGeneration { + suffix: &'static str, + present: bool, + device: u64, + inode: u64, + length: u64, + modified_seconds: i64, + modified_nanoseconds: i64, + changed_seconds: i64, + changed_nanoseconds: i64, } -pub struct Trail { - workspace_root: PathBuf, - db_dir: PathBuf, - conn: Connection, - store: TrailProllyStore, - prolly: Prolly, - root_prolly: Prolly, - config: TrailConfig, - object_cache: Mutex, - daemon_worktree_cache: Option, +#[derive(Clone, Debug, Eq, PartialEq)] +struct SchemaGeneration(Vec); + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[derive(Debug)] +enum CrossProcessSchemaValidationOutcome { + Success(String), + Failure(String), } -#[derive(Clone)] -enum TrailProllyStore { - Sqlite(Arc), - SlateDb(Arc), +#[derive(Default)] +struct SchemaValidationState { + validating: bool, + active_handoffs: u64, + round: u64, + validated: Option<(SchemaGeneration, String, String)>, + failed: Option<(u64, String)>, + validation_count: u64, } -#[derive(Debug)] -struct TrailProllyStoreError { - message: String, - source: Option>, +#[derive(Default)] +struct SchemaValidationEntry { + state: Mutex, + changed: Condvar, } -impl TrailProllyStoreError { - fn with_source( - message: impl Into, - source: impl std::error::Error + Send + Sync + 'static, - ) -> Self { - Self { - message: message.into(), - source: Some(Box::new(source)), - } - } +static SCHEMA_VALIDATIONS: OnceLock>>> = + OnceLock::new(); + +#[cfg(any(target_os = "linux", target_os = "macos"))] +static SCHEMA_VALIDATION_SERVERS: OnceLock>> = + OnceLock::new(); + +#[cfg(any(target_os = "linux", target_os = "macos"))] +static NEXT_SCHEMA_VALIDATION_SERVER_ID: AtomicU64 = AtomicU64::new(1); + +#[cfg(test)] +static SCHEMA_VALIDATION_FAILURES: OnceLock>> = OnceLock::new(); +#[cfg(test)] +type SchemaSnapshotGenerationHook = Box; +#[cfg(test)] +static NEXT_SCHEMA_SNAPSHOT_GENERATION_HOOKS: OnceLock< + Mutex>, +> = OnceLock::new(); + +#[cfg(test)] +static NEXT_SCHEMA_VALIDATION_SERVER_DELAYS: OnceLock>> = + OnceLock::new(); + +#[cfg(all(test, target_os = "linux"))] +type SchemaSocketAuthorityHook = Box; + +#[cfg(all(test, target_os = "linux"))] +static NEXT_SCHEMA_SOCKET_AUTHORITY_HOOKS: OnceLock< + Mutex>, +> = OnceLock::new(); + +#[cfg(all(test, unix))] +type SchemaWalDigestHook = Box; + +#[cfg(all(test, unix))] +static NEXT_SCHEMA_WAL_DIGEST_HOOKS: OnceLock>> = + OnceLock::new(); + +#[cfg(all(test, unix))] +type SchemaWalDigestAttemptHook = Box; + +#[cfg(all(test, unix))] +static SCHEMA_WAL_DIGEST_ATTEMPT_HOOKS: OnceLock< + Mutex>, +> = OnceLock::new(); + +#[cfg(all(test, target_os = "linux"))] +fn install_schema_socket_authority_hook(path: &Path, hook: SchemaSocketAuthorityHook) { + NEXT_SCHEMA_SOCKET_AUTHORITY_HOOKS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(path.to_path_buf(), hook); } -impl std::fmt::Display for TrailProllyStoreError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Trail prolly store error: {}", self.message) - } +#[cfg(all(test, unix))] +fn install_schema_wal_digest_hook(path: &Path, hook: SchemaWalDigestHook) { + NEXT_SCHEMA_WAL_DIGEST_HOOKS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(path.to_path_buf(), hook); } -impl std::error::Error for TrailProllyStoreError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - self.source - .as_ref() - .map(|e| e.as_ref() as &(dyn std::error::Error + 'static)) - } +#[cfg(all(test, unix))] +fn install_schema_wal_digest_attempt_hook(path: &Path, hook: SchemaWalDigestAttemptHook) { + SCHEMA_WAL_DIGEST_ATTEMPT_HOOKS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(path.to_path_buf(), hook); } -impl Store for TrailProllyStore { - type Error = TrailProllyStoreError; +#[cfg(all(test, unix))] +fn clear_schema_wal_digest_attempt_hook(path: &Path) { + SCHEMA_WAL_DIGEST_ATTEMPT_HOOKS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(path); +} - fn get(&self, key: &[u8]) -> std::result::Result>, Self::Error> { - match self { - TrailProllyStore::Sqlite(store) => store - .get(key) - .map_err(|err| TrailProllyStoreError::with_source("SQLite prolly get failed", err)), - TrailProllyStore::SlateDb(store) => store.get(key).map_err(|err| { - TrailProllyStoreError::with_source("SlateDB prolly get failed", err) - }), - } - } +#[cfg(test)] +fn fail_next_schema_validation(db_path: &Path) { + SCHEMA_VALIDATION_FAILURES + .get_or_init(|| Mutex::new(HashSet::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(db_path.to_path_buf()); +} - fn put(&self, key: &[u8], value: &[u8]) -> std::result::Result<(), Self::Error> { - match self { - TrailProllyStore::Sqlite(store) => store - .put(key, value) - .map_err(|err| TrailProllyStoreError::with_source("SQLite prolly put failed", err)), - TrailProllyStore::SlateDb(store) => store.put(key, value).map_err(|err| { - TrailProllyStoreError::with_source("SlateDB prolly put failed", err) - }), - } - } +#[cfg(test)] +fn install_schema_snapshot_generation_hook( + db_path: &Path, + hook: impl FnOnce(&Path) + Send + 'static, +) { + NEXT_SCHEMA_SNAPSHOT_GENERATION_HOOKS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(db_path.to_path_buf(), Box::new(hook)); +} - fn delete(&self, key: &[u8]) -> std::result::Result<(), Self::Error> { - match self { - TrailProllyStore::Sqlite(store) => store.delete(key).map_err(|err| { - TrailProllyStoreError::with_source("SQLite prolly delete failed", err) - }), - TrailProllyStore::SlateDb(store) => store.delete(key).map_err(|err| { - TrailProllyStoreError::with_source("SlateDB prolly delete failed", err) - }), - } +#[cfg(test)] +fn run_schema_snapshot_generation_hook(db_path: &Path) { + let hook = NEXT_SCHEMA_SNAPSHOT_GENERATION_HOOKS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(db_path); + if let Some(hook) = hook { + hook(db_path); } +} - fn batch(&self, ops: &[BatchOp]) -> std::result::Result<(), Self::Error> { - match self { - TrailProllyStore::Sqlite(store) => store.batch(ops).map_err(|err| { - TrailProllyStoreError::with_source("SQLite prolly batch failed", err) - }), - TrailProllyStore::SlateDb(store) => store.batch(ops).map_err(|err| { - TrailProllyStoreError::with_source("SlateDB prolly batch failed", err) - }), - } - } +#[cfg(test)] +fn delay_next_schema_validation_server_start_for_test(db_path: &Path, delay: Duration) { + NEXT_SCHEMA_VALIDATION_SERVER_DELAYS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .entry(db_path.to_path_buf()) + .or_default() + .0 = delay.as_millis() as u64; +} - fn batch_get( - &self, - keys: &[&[u8]], - ) -> std::result::Result, Vec>, Self::Error> { - match self { - TrailProllyStore::Sqlite(store) => store.batch_get(keys).map_err(|err| { - TrailProllyStoreError::with_source("SQLite prolly batch_get failed", err) - }), - TrailProllyStore::SlateDb(store) => store.batch_get(keys).map_err(|err| { - TrailProllyStoreError::with_source("SlateDB prolly batch_get failed", err) - }), - } - } +#[cfg(test)] +fn delay_next_schema_validation_server_shutdown_for_test(db_path: &Path, delay: Duration) { + NEXT_SCHEMA_VALIDATION_SERVER_DELAYS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .entry(db_path.to_path_buf()) + .or_default() + .1 = delay.as_millis() as u64; +} - fn batch_get_ordered( - &self, - keys: &[&[u8]], - ) -> std::result::Result>>, Self::Error> { - match self { - TrailProllyStore::Sqlite(store) => store.batch_get_ordered(keys).map_err(|err| { - TrailProllyStoreError::with_source("SQLite prolly batch_get_ordered failed", err) - }), - TrailProllyStore::SlateDb(store) => store.batch_get_ordered(keys).map_err(|err| { - TrailProllyStoreError::with_source("SlateDB prolly batch_get_ordered failed", err) - }), - } - } +#[cfg(test)] +fn take_schema_validation_server_delays_for_test(db_path: &Path) -> (Duration, Duration) { + let (start, shutdown) = NEXT_SCHEMA_VALIDATION_SERVER_DELAYS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(db_path) + .unwrap_or_default(); + ( + Duration::from_millis(start), + Duration::from_millis(shutdown), + ) +} - fn batch_put(&self, entries: &[(&[u8], &[u8])]) -> std::result::Result<(), Self::Error> { - match self { - TrailProllyStore::Sqlite(store) => store.batch_put(entries).map_err(|err| { - TrailProllyStoreError::with_source("SQLite prolly batch_put failed", err) - }), - TrailProllyStore::SlateDb(store) => store.batch_put(entries).map_err(|err| { - TrailProllyStoreError::with_source("SlateDB prolly batch_put failed", err) - }), - } - } +#[cfg(test)] +fn schema_validation_count(db_path: &Path) -> u64 { + SCHEMA_VALIDATIONS + .get() + .and_then(|entries| { + entries + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .get(db_path) + .cloned() + }) + .map(|entry| { + entry + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .validation_count + }) + .unwrap_or(0) +} - fn supports_hints(&self) -> bool { - match self { - TrailProllyStore::Sqlite(store) => store.supports_hints(), - TrailProllyStore::SlateDb(store) => store.supports_hints(), +pub(crate) struct ValidatedSchemaGeneration { + db_path: PathBuf, + generation: SchemaGeneration, + entry: Arc, + _parent_authority: File, + main_authority: File, + _shared_exclusion: SchemaSharedExclusion, + _leader_exclusion: Option, + authenticated_wal_digest: String, +} + +impl ValidatedSchemaGeneration { + pub(crate) fn verify_unchanged(&self) -> Result<()> { + let current = schema_generation(&self.db_path).map_err(schema_reinitialize_error)?; + let concurrent_handoffs = self + .entry + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .active_handoffs + > 1; + if current != self.generation + && !schema_generation_is_only_volatile_shm_presence_transition( + &self.generation, + ¤t, + concurrent_handoffs, + ) + && !(schema_generation_is_only_wal_ctime_transition(&self.generation, ¤t) + && schema_wal_digest_allowing_wal_ctime_transition(&self.db_path, ¤t) + .is_ok_and(|digest| digest == self.authenticated_wal_digest)) + { + return Err(schema_reinitialize_error( + "schema main/WAL/SHM generation changed during mutable handoff", + )); } + Ok(()) } - fn get_hint( - &self, - namespace: &[u8], - key: &[u8], - ) -> std::result::Result>, Self::Error> { - match self { - TrailProllyStore::Sqlite(store) => store.get_hint(namespace, key).map_err(|err| { - TrailProllyStoreError::with_source("SQLite prolly get_hint failed", err) - }), - TrailProllyStore::SlateDb(store) => store.get_hint(namespace, key).map_err(|err| { - TrailProllyStoreError::with_source("SlateDB prolly get_hint failed", err) - }), - } + pub(crate) fn verify_connection(&self, conn: &Connection) -> Result<()> { + #[cfg(unix)] + let identity = sqlite_main_file_identity(conn).map_err(schema_reinitialize_error)?; + #[cfg(not(unix))] + let identity = SqliteMainFileIdentity { + device: 0, + inode: 0, + length: 0, + }; + self.verify_main_identity(identity) } - fn put_hint( - &self, - namespace: &[u8], - key: &[u8], - value: &[u8], - ) -> std::result::Result<(), Self::Error> { - match self { - TrailProllyStore::Sqlite(store) => { - store.put_hint(namespace, key, value).map_err(|err| { - TrailProllyStoreError::with_source("SQLite prolly put_hint failed", err) - }) - } - TrailProllyStore::SlateDb(store) => { - store.put_hint(namespace, key, value).map_err(|err| { - TrailProllyStoreError::with_source("SlateDB prolly put_hint failed", err) - }) + pub(crate) fn verify_main_identity(&self, identity: SqliteMainFileIdentity) -> Result<()> { + let expected = self + .generation + .0 + .iter() + .find(|file| file.suffix.is_empty() && file.present) + .ok_or_else(|| schema_reinitialize_error("validated schema has no main database"))?; + #[cfg(unix)] + { + let retained = self + .main_authority + .metadata() + .map_err(schema_reinitialize_error)?; + if identity.device != expected.device + || identity.inode != expected.inode + || identity.length != expected.length + || retained.dev() != expected.device + || retained.ino() != expected.inode + || retained.len() != expected.length + { + return Err(schema_reinitialize_error( + "SQLite main-file handle does not match validated schema authority", + )); } } - } - - fn batch_put_with_hint( - &self, - entries: &[(&[u8], &[u8])], - namespace: &[u8], - key: &[u8], - value: &[u8], - ) -> std::result::Result<(), Self::Error> { - match self { - TrailProllyStore::Sqlite(store) => store - .batch_put_with_hint(entries, namespace, key, value) - .map_err(|err| { - TrailProllyStoreError::with_source( - "SQLite prolly batch_put_with_hint failed", - err, - ) - }), - TrailProllyStore::SlateDb(store) => store - .batch_put_with_hint(entries, namespace, key, value) - .map_err(|err| { - TrailProllyStoreError::with_source( - "SlateDB prolly batch_put_with_hint failed", - err, - ) - }), + #[cfg(not(unix))] + { + let _ = identity; + return Err(schema_reinitialize_error( + "verified SQLite main-file handles are unsupported on this platform", + )); } + Ok(()) } } -fn open_prolly_store(config: &TrailConfig, sqlite_path: &Path) -> Result { - match config.storage.prolly_backend.as_str() { - "sqlite" => Ok(TrailProllyStore::Sqlite(Arc::new(SqliteStore::open( - sqlite_path, - )?))), - "slatedb" => open_slatedb_prolly_store(&config.storage), - other => Err(Error::InvalidInput(format!( - "storage.prolly_backend must be sqlite or slatedb, got `{other}`" - ))), - } +fn schema_generation_is_only_wal_ctime_transition( + expected: &SchemaGeneration, + current: &SchemaGeneration, +) -> bool { + expected.0.len() == current.0.len() + && expected.0.iter().zip(¤t.0).all(|(left, right)| { + if left.suffix != right.suffix { + return false; + } + if left.suffix == "-wal" { + return left.present + && right.present + && left.device == right.device + && left.inode == right.inode + && left.length == right.length + && left.modified_seconds == right.modified_seconds + && left.modified_nanoseconds == right.modified_nanoseconds; + } + left == right + }) } -fn open_slatedb_prolly_store(storage: &StorageConfig) -> Result { - let path = storage.slatedb_path.trim().trim_matches('/'); - if path.is_empty() { - return Err(Error::InvalidInput( - "storage.slatedb_path must not be empty".to_string(), +fn schema_generation_is_only_volatile_shm_presence_transition( + expected: &SchemaGeneration, + current: &SchemaGeneration, + concurrent_handoffs: bool, +) -> bool { + expected.0.len() == current.0.len() + && expected + .0 + .iter() + .zip(¤t.0) + .all(|(expected, current)| { + if expected.suffix != current.suffix { + return false; + } + if expected.suffix == "-shm" { + // SQLite creates, removes, and recreates SHM as its first and + // last live connections cross the handoff. SHM is a rebuildable + // lock/index file, not durable schema authority. Permit only a + // presence transition or a same-device, same-length inode + // rotation; in-place byte/length mutation remains a hard failure. + return concurrent_handoffs + || expected.present != current.present + || (expected.present + && current.present + && expected.device == current.device + && expected.inode != current.inode + && expected.length == current.length); + } + expected == current + }) +} + +#[cfg(unix)] +fn open_schema_main_authority( + db_path: &Path, + generation: &SchemaGeneration, +) -> Result<(File, File)> { + use rustix::fs::{openat, Mode, OFlags, CWD}; + + let expected = generation + .0 + .iter() + .find(|file| file.suffix.is_empty() && file.present) + .ok_or_else(|| schema_reinitialize_error("schema main database is missing"))?; + let parent_path = db_path + .parent() + .ok_or_else(|| schema_reinitialize_error("schema database has no parent directory"))?; + let leaf = db_path + .file_name() + .ok_or_else(|| schema_reinitialize_error("schema database has no main-file leaf"))?; + let parent = File::from( + openat( + CWD, + parent_path, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(schema_reinitialize_error)?, + ); + let file = File::from( + openat( + &parent, + Path::new(leaf), + OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(schema_reinitialize_error)?, + ); + let metadata = file.metadata().map_err(schema_reinitialize_error)?; + if !metadata.is_file() + || metadata.dev() != expected.device + || metadata.ino() != expected.inode + || metadata.len() != expected.length + { + return Err(schema_reinitialize_error( + "schema main-file authority changed after validation", )); } + Ok((parent, file)) +} - let object_store = build_slatedb_object_store(storage)?; - let store = SlateDbStore::open(path, object_store)?; - Ok(TrailProllyStore::SlateDb(Arc::new(store))) +#[cfg(not(unix))] +fn open_schema_main_authority( + _db_path: &Path, + _generation: &SchemaGeneration, +) -> Result<(File, File)> { + Err(schema_reinitialize_error( + "verified SQLite main-file handles are unsupported on this platform", + )) } -fn build_slatedb_object_store(storage: &StorageConfig) -> Result> { - if storage.slatedb_s3_endpoint.trim().is_empty() { - return Err(Error::InvalidInput( - "storage.slatedb_s3_endpoint must not be empty".to_string(), - )); - } - if storage.slatedb_s3_bucket.trim().is_empty() { - return Err(Error::InvalidInput( - "storage.slatedb_s3_bucket must not be empty".to_string(), - )); +impl Drop for ValidatedSchemaGeneration { + fn drop(&mut self) { + let mut state = self + .entry + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + state.active_handoffs = state.active_handoffs.saturating_sub(1); + self.entry.changed.notify_all(); } - if storage.slatedb_s3_region.trim().is_empty() { - return Err(Error::InvalidInput( - "storage.slatedb_s3_region must not be empty".to_string(), - )); - } - - let store = AmazonS3Builder::new() - .with_endpoint(storage.slatedb_s3_endpoint.trim_end_matches('/')) - .with_bucket_name(storage.slatedb_s3_bucket.trim()) - .with_region(storage.slatedb_s3_region.trim()) - .with_access_key_id(&storage.slatedb_s3_access_key_id) - .with_secret_access_key(&storage.slatedb_s3_secret_access_key) - .with_allow_http(storage.slatedb_s3_allow_http) - .with_virtual_hosted_style_request(false) - .build() - .map_err(|err| { - Error::InvalidInput(format!( - "failed to configure SlateDB S3 object store: {err}" - )) - })?; - - Ok(Arc::new(store)) -} - -#[derive(Debug, Default)] -struct ObjectCache { - entries: HashMap, - order: VecDeque, - total_bytes: usize, } #[derive(Debug)] -struct ObjectCacheEntry { - kind: String, - bytes: Vec, +struct SchemaSharedExclusion { + _database: File, } -impl ObjectCache { - fn get(&self, kind: &str, object_id: &ObjectId) -> Option> { - self.entries.get(&object_id.0).and_then(|entry| { - if entry.kind == kind { - Some(entry.bytes.clone()) - } else { - None - } - }) - } +pub(crate) fn preflight_existing_schema( + db_path: &Path, + prolly_backend: &str, +) -> Result { + let shared_exclusion = acquire_schema_shared_exclusion(db_path)?; + let mut generation = schema_generation(db_path).map_err(schema_reinitialize_error)?; + let key = db_path.to_path_buf(); + let entry = { + let entries = SCHEMA_VALIDATIONS.get_or_init(|| Mutex::new(HashMap::new())); + let mut entries = entries + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + entries + .entry(key) + .or_insert_with(|| Arc::new(SchemaValidationEntry::default())) + .clone() + }; + let backend = prolly_backend.to_owned(); + let mut volatile_generation_retries = 0_usize; - fn insert(&mut self, object_id: &ObjectId, kind: &str, bytes: &[u8]) { - if bytes.len() > OBJECT_CACHE_MAX_BYTES { - return; + loop { + let mut state = entry + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if state + .validated + .as_ref() + .is_some_and(|(validated, validated_backend, _)| { + validated == &generation && validated_backend == &backend + }) + { + let (parent_authority, main_authority) = + open_schema_main_authority(db_path, &generation)?; + let authenticated_wal_digest = state.validated.as_ref().unwrap().2.clone(); + state.active_handoffs = state.active_handoffs.saturating_add(1); + drop(state); + return Ok(ValidatedSchemaGeneration { + db_path: db_path.to_path_buf(), + generation, + entry, + _parent_authority: parent_authority, + main_authority, + _shared_exclusion: shared_exclusion, + _leader_exclusion: None, + authenticated_wal_digest, + }); } - if self.entries.contains_key(&object_id.0) { - return; + if state.validating { + let waited_round = state.round; + while state.validating && state.round == waited_round { + state = entry + .changed + .wait(state) + .unwrap_or_else(|poisoned| poisoned.into_inner()); + } + if state + .failed + .as_ref() + .is_some_and(|(round, _)| *round == waited_round) + { + let message = state.failed.as_ref().unwrap().1.clone(); + return Err(schema_reinitialize_error(message)); + } + continue; } - self.entries.insert( - object_id.0.clone(), - ObjectCacheEntry { - kind: kind.to_string(), - bytes: bytes.to_vec(), - }, + if state.active_handoffs != 0 { + while state.active_handoffs != 0 { + state = entry + .changed + .wait(state) + .unwrap_or_else(|poisoned| poisoned.into_inner()); + } + drop(state); + generation = schema_generation(db_path).map_err(schema_reinitialize_error)?; + continue; + } + state.validating = true; + state.round = state.round.saturating_add(1); + state.validation_count = state.validation_count.saturating_add(1); + let round = state.round; + drop(state); + + let mut authenticated_wal_digest = None; + let (mut validation, leader_exclusion) = coordinate_schema_snapshot_validation( + db_path, + prolly_backend, + &mut generation, + &mut authenticated_wal_digest, ); - self.order.push_back(object_id.0.clone()); - self.total_bytes = self.total_bytes.saturating_add(bytes.len()); - while self.entries.len() > OBJECT_CACHE_MAX_ENTRIES - || self.total_bytes > OBJECT_CACHE_MAX_BYTES - { - let Some(evicted) = self.order.pop_front() else { - break; - }; - if let Some(entry) = self.entries.remove(&evicted) { - self.total_bytes = self.total_bytes.saturating_sub(entry.bytes.len()); + if validation.is_ok() && authenticated_wal_digest.is_none() { + match schema_wal_digest(db_path, &generation) { + Ok(digest) => authenticated_wal_digest = Some(digest), + Err(error) => validation = Err(error), + } + } + + // A live writer can advance the same WAL while the private snapshot + // is being copied. Depending on the exact torn-copy point SQLite may + // report a low-level snapshot error (including IOERR), not only one of + // our explicit generation-change diagnostics. Retry only when a fresh + // authority read proves that the main database is unchanged and the + // same WAL/SHM objects advanced in place. Stable corruption and every + // pathname replacement still fail closed on the first round. + let retry_generation = validation + .is_err() + .then(|| schema_generation(db_path).map_err(schema_reinitialize_error)) + .transpose()? + .filter(|current| { + schema_snapshot_generation_is_retryable_volatile_change(&generation, current) + }); + if let Some(current) = retry_generation { + let mut state = entry + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + state.validating = false; + state.validated = None; + state.failed = None; + entry.changed.notify_all(); + drop(state); + drop(leader_exclusion); + volatile_generation_retries = volatile_generation_retries.saturating_add(1); + if volatile_generation_retries >= 8 { + return Err(Error::WorkspaceLocked( + "SQLite WAL remained active throughout bounded schema snapshot validation; retry the command" + .into(), + )); } + generation = current; + std::thread::sleep(Duration::from_millis(2)); + continue; } + + let mut state = entry + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + state.validating = false; + match validation { + Ok(()) => { + state.validated = Some(( + generation.clone(), + backend.clone(), + authenticated_wal_digest.clone().unwrap(), + )); + state.failed = None; + } + Err(error) => { + state.validated = None; + state.failed = Some((round, schema_failure_message(&error))); + } + } + entry.changed.notify_all(); + if let Some((failed_round, message)) = &state.failed { + if *failed_round == round { + return Err(schema_reinitialize_error(message.clone())); + } + } + let (parent_authority, main_authority) = open_schema_main_authority(db_path, &generation)?; + state.active_handoffs = state.active_handoffs.saturating_add(1); + drop(state); + return Ok(ValidatedSchemaGeneration { + db_path: db_path.to_path_buf(), + generation, + entry, + _parent_authority: parent_authority, + main_authority, + _shared_exclusion: shared_exclusion, + _leader_exclusion: leader_exclusion, + authenticated_wal_digest: authenticated_wal_digest.unwrap(), + }); } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum InitImportMode { - Empty, - GitTracked, - WorkingTree, +fn schema_snapshot_generation_is_retryable_volatile_change( + before: &SchemaGeneration, + after: &SchemaGeneration, +) -> bool { + before != after + && before.0.len() == after.0.len() + && before.0.iter().zip(&after.0).all(|(left, right)| { + if left.suffix != right.suffix { + return false; + } + match left.suffix { + "-wal" | "-shm" => { + left.present == right.present + && (!left.present + || (left.device == right.device && left.inode == right.inode)) + } + // The main database and rollback journal are immutable + // authority for this retry classification. A pathname + // replacement, checkpoint rewrite, or any other change must + // fail this validation round rather than authenticate a new + // object under a stale snapshot. + _ => left == right, + } + }) } -#[derive(Debug, Clone)] -pub(crate) struct DiskFile { - path: String, - bytes: Vec, - executable: bool, +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn schema_generation_key(generation: &SchemaGeneration) -> String { + let mut digest = Sha256::new(); + digest.update(b"trail-schema-generation-v1\0"); + for file in &generation.0 { + digest.update((file.suffix.len() as u64).to_le_bytes()); + digest.update(file.suffix.as_bytes()); + digest.update([u8::from(file.present)]); + digest.update(file.device.to_le_bytes()); + digest.update(file.inode.to_le_bytes()); + digest.update(file.length.to_le_bytes()); + digest.update(file.modified_seconds.to_le_bytes()); + digest.update(file.modified_nanoseconds.to_le_bytes()); + digest.update(file.changed_seconds.to_le_bytes()); + digest.update(file.changed_nanoseconds.to_le_bytes()); + } + hex::encode(digest.finalize()) } -#[derive(Debug)] -pub(crate) struct WorktreePathScan { - paths: Vec, - total_bytes: u64, -} +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn schema_validation_runtime_namespace(db_path: &Path) -> Result> { + use std::os::unix::ffi::OsStrExt; + use std::os::unix::fs::DirBuilderExt; -#[derive(Debug)] -pub(crate) struct RootBuildResult { - root_id: ObjectId, - files: BTreeMap, - disk_manifest: BTreeMap, - stats: ImportStats, + let uid = rustix::process::getuid().as_raw(); + let runtime_dir = PathBuf::from(format!("/tmp/trail-sv-{uid}")); + let mut builder = fs::DirBuilder::new(); + builder.mode(0o700); + if let Err(error) = builder.create(&runtime_dir) { + if error.kind() != std::io::ErrorKind::AlreadyExists { + return Ok(None); + } + } + let metadata = match fs::symlink_metadata(&runtime_dir) { + Ok(metadata) => metadata, + Err(_) => return Ok(None), + }; + if !metadata.is_dir() || metadata.uid() != uid || metadata.mode() & 0o077 != 0 { + return Ok(None); + } + + let mut digest = Sha256::new(); + digest.update(b"trail-schema-validation-path-v1\0"); + digest.update(db_path.as_os_str().as_bytes()); + Ok(Some((runtime_dir, hex::encode(digest.finalize())))) } -#[derive(Debug)] -pub(crate) struct IncrementalRootBuildResult { - root_id: ObjectId, +#[cfg(any(target_os = "linux", target_os = "macos"))] +struct SchemaValidationRuntimeEntry { + path: PathBuf, + device: u64, + inode: u64, } -#[derive(Debug)] -pub(crate) struct GitTrackedRootBuildResult { - root_id: ObjectId, - disk_manifest: BTreeMap, - stats: ImportStats, +#[cfg(any(target_os = "linux", target_os = "macos"))] +struct SchemaValidationSocketAuthority { + #[cfg(target_os = "linux")] + _inode: File, } -#[derive(Debug)] -pub(crate) struct SelectedWorktreeSnapshot { - paths: Vec, - files: Vec, - summaries: Vec, +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn schema_validation_socket_identity_matches( + metadata: &fs::Metadata, + expected: &SchemaValidationRuntimeEntry, +) -> bool { + metadata.file_type().is_socket() + && metadata.uid() == rustix::process::getuid().as_raw() + && metadata.dev() == expected.device + && metadata.ino() == expected.inode + && metadata.mode() & 0o777 == 0o600 } -#[derive(Debug)] -pub(crate) struct FileBuildResult { - entry: FileEntry, - disk_manifest: DiskManifest, - line_changes: Vec, +#[cfg(target_os = "linux")] +fn secure_schema_validation_socket( + path: &Path, + expected: &SchemaValidationRuntimeEntry, +) -> std::io::Result { + use std::os::fd::AsRawFd; + use std::os::unix::fs::OpenOptionsExt; + + // Linux does not implement fchmodat(AT_SYMLINK_NOFOLLOW). Pin the bound + // socket inode with O_PATH|O_NOFOLLOW, then chmod through that descriptor's + // procfs handle. The pathname can be replaced without redirecting chmod; + // the final lstat comparison rejects any such substitution. + let inode = OpenOptions::new() + .read(true) + .custom_flags(libc::O_PATH | libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path)?; + let before = inode.metadata()?; + if !before.file_type().is_socket() + || before.uid() != rustix::process::getuid().as_raw() + || before.dev() != expected.device + || before.ino() != expected.inode + { + return Err(std::io::Error::other( + "schema validation socket inode changed before permission binding", + )); + } + let descriptor_path = PathBuf::from(format!("/proc/self/fd/{}", inode.as_raw_fd())); + fs::set_permissions(&descriptor_path, fs::Permissions::from_mode(0o600))?; + let pinned = inode.metadata()?; + let named = fs::symlink_metadata(path)?; + if !schema_validation_socket_identity_matches(&pinned, expected) + || !schema_validation_socket_identity_matches(&named, expected) + { + return Err(std::io::Error::other( + "schema validation socket inode changed during permission binding", + )); + } + #[cfg(test)] + if let Some(hook) = NEXT_SCHEMA_SOCKET_AUTHORITY_HOOKS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(path) + { + hook(path); + } + let pinned = inode.metadata()?; + let named = fs::symlink_metadata(path)?; + if !schema_validation_socket_identity_matches(&pinned, expected) + || !schema_validation_socket_identity_matches(&named, expected) + { + return Err(std::io::Error::other( + "schema validation socket pathname changed after permission binding", + )); + } + Ok(SchemaValidationSocketAuthority { _inode: inode }) } -#[derive(Debug)] -pub(crate) struct TextBuildResult { - object_id: ObjectId, - line_changes: Vec, +#[cfg(target_os = "macos")] +fn secure_schema_validation_socket( + path: &Path, + expected: &SchemaValidationRuntimeEntry, +) -> std::io::Result { + rustix::fs::chmodat( + rustix::fs::CWD, + path, + rustix::fs::Mode::from_raw_mode(0o600), + rustix::fs::AtFlags::SYMLINK_NOFOLLOW, + ) + .map_err(std::io::Error::from)?; + let named = fs::symlink_metadata(path)?; + if !schema_validation_socket_identity_matches(&named, expected) { + return Err(std::io::Error::other( + "schema validation socket inode changed during permission binding", + )); + } + Ok(SchemaValidationSocketAuthority {}) } -#[derive(Debug, Clone)] -pub(crate) struct RootDiff { - changes: Vec, - summaries: Vec, +#[cfg(any(target_os = "linux", target_os = "macos"))] +impl SchemaValidationRuntimeEntry { + fn capture(path: &Path) -> std::io::Result { + let metadata = fs::symlink_metadata(path)?; + Ok(Self { + path: path.to_path_buf(), + device: metadata.dev(), + inode: metadata.ino(), + }) + } } -#[derive(Debug)] -pub(crate) struct PathLocalMergeResult { - target_files: BTreeMap, - merged_files: BTreeMap, - conflicts: Vec, +#[cfg(any(target_os = "linux", target_os = "macos"))] +impl Drop for SchemaValidationRuntimeEntry { + fn drop(&mut self) { + if fs::symlink_metadata(&self.path) + .is_ok_and(|metadata| metadata.dev() == self.device && metadata.ino() == self.inode) + { + let _ = fs::remove_file(&self.path); + } + } } -#[derive(Debug)] -pub(crate) struct CommandRunResult { - success: bool, - exit_code: Option, - timed_out: bool, - duration_ms: u64, - stdout: Vec, - stderr: Vec, +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn cleanup_stale_schema_validation_runtime_entries(runtime_dir: &Path, namespace: &str) { + let prefix = format!("{}-", &namespace[..24]); + for path in fs::read_dir(runtime_dir) + .ok() + .into_iter() + .flatten() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| { + let name = entry.file_name(); + let name = name.to_string_lossy(); + (name.starts_with(&prefix) + && (name.ends_with(".socket") || name.ends_with(".announce"))) + .then(|| entry.path()) + }) + { + if let Ok(entry) = SchemaValidationRuntimeEntry::capture(&path) { + drop(entry); + } + } } -#[derive(Debug, Clone)] -pub(crate) struct ExternalMutationAuditInput { - pub(crate) actor: String, - pub(crate) surface: String, - pub(crate) command: String, - pub(crate) target_ref: Option, - pub(crate) lane_id: Option, - pub(crate) turn_id: Option, - pub(crate) status: String, - pub(crate) status_code: Option, - pub(crate) change_id: Option, - pub(crate) summary: Option, +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn schema_validation_peer_identity( + stream: &std::os::unix::net::UnixStream, +) -> std::io::Result<(u32, u32)> { + use std::os::fd::AsRawFd; + + #[cfg(target_os = "linux")] + { + let mut credentials = std::mem::MaybeUninit::::uninit(); + let mut length = std::mem::size_of::() as libc::socklen_t; + let status = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERCRED, + credentials.as_mut_ptr().cast(), + &mut length, + ) + }; + if status != 0 { + return Err(std::io::Error::last_os_error()); + } + let credentials = unsafe { credentials.assume_init() }; + return Ok((credentials.pid as u32, credentials.uid)); + } + #[cfg(target_os = "macos")] + { + let mut pid = 0_i32; + let mut length = std::mem::size_of::() as libc::socklen_t; + let status = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_LOCAL, + libc::LOCAL_PEERPID, + (&mut pid as *mut i32).cast(), + &mut length, + ) + }; + if status != 0 { + return Err(std::io::Error::last_os_error()); + } + let mut uid = 0_u32; + let mut gid = 0_u32; + if unsafe { libc::getpeereid(stream.as_raw_fd(), &mut uid, &mut gid) } != 0 { + return Err(std::io::Error::last_os_error()); + } + return Ok((pid as u32, uid)); + } } -#[derive(Debug, Clone)] -pub(crate) struct HttpIdempotencyEntry { - pub(crate) method: String, - pub(crate) path: String, - pub(crate) request_hash: String, - pub(crate) status: u16, - pub(crate) body: Vec, +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn schema_validation_wire_result( + nonce: &str, + generation: &str, + backend: &str, + outcome: &CrossProcessSchemaValidationOutcome, +) -> String { + let (kind, payload) = match outcome { + CrossProcessSchemaValidationOutcome::Success(wal_digest) => ("success", wal_digest.clone()), + CrossProcessSchemaValidationOutcome::Failure(message) => { + ("failure", hex::encode(message.as_bytes())) + } + }; + format!( + "trail-schema-validation-ipc-v1\n{nonce}\n{generation}\n{}\n{kind}\n{payload}\n", + hex::encode(backend.as_bytes()), + ) } -#[derive(Debug, Clone)] -pub(crate) struct HttpIdempotencyStoreInput { - pub(crate) key: String, - pub(crate) method: String, - pub(crate) path: String, - pub(crate) request_hash: String, - pub(crate) status: u16, - pub(crate) body: Vec, +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn parse_schema_validation_wire_result( + text: &str, + nonce: &str, + generation: &str, + backend: &str, +) -> Option { + let mut lines = text.lines(); + if lines.next()? != "trail-schema-validation-ipc-v1" + || lines.next()? != nonce + || lines.next()? != generation + || lines.next()? != hex::encode(backend.as_bytes()) + { + return None; + } + let kind = lines.next()?; + let payload = lines.next()?; + if lines.next().is_some() { + return None; + } + match kind { + "success" if payload.len() == 64 && hex::decode(payload).is_ok() => Some( + CrossProcessSchemaValidationOutcome::Success(payload.to_owned()), + ), + "failure" => String::from_utf8(hex::decode(payload).ok()?) + .ok() + .map(CrossProcessSchemaValidationOutcome::Failure), + _ => None, + } } -#[derive(Debug, Clone)] -pub(crate) struct LaneTraceSpanBuilder { - span_id: String, - trace_id: String, - lane_id: String, - session_id: Option, - turn_id: Option, - parent_span_id: Option, - span_type: String, - name: String, - started_event_id: String, - started_at: i64, - attributes: Option, - ended_event_id: Option, - ended_at: Option, - status: Option, - result: Option, +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn serve_schema_validation_result( + listener: &std::os::unix::net::UnixListener, + nonce: &str, + generation: &str, + backend: &str, + outcome: &CrossProcessSchemaValidationOutcome, + shutdown: &SchemaValidationServerShutdown, +) { + use std::net::Shutdown; + + let response = schema_validation_wire_result(nonce, generation, backend, outcome); + let expected_request = format!( + "trail-schema-validation-request-v1\n{nonce}\n{generation}\n{}\n", + hex::encode(backend.as_bytes()) + ); + let hard_deadline = Instant::now() + Duration::from_millis(300); + let mut idle_deadline = hard_deadline; + while !shutdown.is_stopping() + && Instant::now() < hard_deadline + && Instant::now() < idle_deadline + { + match listener.accept() { + Ok((mut stream, _)) => { + // Followers can be runnable yet unscheduled while the validating + // process publishes its result. Keep the detached server alive for + // the remaining absolute window; the foreground handoff is already + // free to continue and pays none of this grace period. + idle_deadline = (Instant::now() + Duration::from_millis(300)).min(hard_deadline); + if schema_validation_peer_identity(&stream) + .is_ok_and(|(_, uid)| uid == rustix::process::getuid().as_raw()) + && read_schema_validation_request(&mut stream, shutdown) + .is_some_and(|request| request == expected_request) + { + let _ = stream.set_nonblocking(false); + let _ = stream.set_write_timeout(Some(Duration::from_millis(10))); + let _ = stream.write_all(response.as_bytes()); + let _ = stream.shutdown(Shutdown::Both); + } + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + let remaining = hard_deadline.saturating_duration_since(Instant::now()); + if shutdown.wait(remaining.min(Duration::from_millis(1))) { + break; + } + } + Err(_) => break, + } + } } -#[derive(Debug, Serialize, serde::Deserialize)] -pub(crate) struct BackupManifest { - format_version: u16, - trail_version: String, - created_at: i64, - source_workspace: String, - source_db_dir: String, - workspace_id: WorkspaceId, - branch: String, - ref_count: u64, - operation_count: u64, - sqlite_bytes: u64, - sqlite_sha256: String, - worktree_bytes: u64, +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn read_schema_validation_request( + stream: &mut std::os::unix::net::UnixStream, + shutdown: &SchemaValidationServerShutdown, +) -> Option { + stream.set_nonblocking(true).ok()?; + let deadline = Instant::now() + Duration::from_secs(1); + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + match stream.read(&mut buffer) { + Ok(0) => return String::from_utf8(request).ok(), + Ok(read) => { + if request.len().saturating_add(read) > 8192 { + return None; + } + request.extend_from_slice(&buffer[..read]); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() || shutdown.wait(remaining.min(Duration::from_millis(1))) { + return None; + } + } + Err(_) => return None, + } + } } -#[derive(Debug)] -pub(crate) struct PendingLineMerge { - path: String, - target_entry: FileEntry, - lines: Vec, +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[derive(Default)] +struct SchemaValidationServerShutdown { + stopping: AtomicBool, + mutex: Mutex<()>, + changed: Condvar, } -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub(crate) struct LineGap { - previous: Option, - next: Option, +#[cfg(any(target_os = "linux", target_os = "macos"))] +impl SchemaValidationServerShutdown { + fn is_stopping(&self) -> bool { + self.stopping.load(Ordering::Acquire) + } + + fn stop(&self) { + self.stopping.store(true, Ordering::Release); + self.changed.notify_all(); + } + + fn wait(&self, timeout: Duration) -> bool { + if self.is_stopping() { + return true; + } + let guard = self + .mutex + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let _ = self + .changed + .wait_timeout_while(guard, timeout, |_| !self.is_stopping()) + .unwrap_or_else(|poisoned| poisoned.into_inner()); + self.is_stopping() + } } -#[derive(Debug, Clone)] -pub(crate) struct OperationObject { - object_id: ObjectId, - operation: Operation, +#[cfg(any(target_os = "linux", target_os = "macos"))] +struct SchemaValidationServer { + listener: std::os::unix::net::UnixListener, + nonce: String, + generation: String, + backend: String, + outcome: CrossProcessSchemaValidationOutcome, + _leader_exclusion: File, + _socket_authority: SchemaValidationSocketAuthority, + _socket_cleanup: SchemaValidationRuntimeEntry, + _announcement_cleanup: SchemaValidationRuntimeEntry, + #[cfg(test)] + panic_on_serve: bool, + #[cfg(test)] + start_delay: Duration, + #[cfg(test)] + shutdown_delay: Duration, } -#[derive(Debug, Clone)] -pub(crate) struct DiskManifest { - kind: FileKind, - executable: bool, - content_hash: String, +#[cfg(any(target_os = "linux", target_os = "macos"))] +impl SchemaValidationServer { + #[cfg(test)] + fn delay_start_for_test(&self) { + std::thread::sleep(self.start_delay); + } + + #[cfg(not(test))] + fn delay_start_for_test(&self) {} + + fn serve(self, shutdown: &SchemaValidationServerShutdown) { + #[cfg(test)] + if self.panic_on_serve { + panic!("injected schema validation server panic"); + } + serve_schema_validation_result( + &self.listener, + &self.nonce, + &self.generation, + &self.backend, + &self.outcome, + shutdown, + ); + #[cfg(test)] + std::thread::sleep(self.shutdown_delay); + } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct WorktreeFileStamp { - size_bytes: u64, - modified_ns: i64, - changed_ns: i64, - device_id: i64, - inode: i64, - executable: bool, +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[cfg(test)] +fn spawn_schema_validation_server( + server: SchemaValidationServer, + shutdown: Arc, +) -> std::io::Result> { + std::thread::Builder::new() + .name("trail-schema-validation".to_owned()) + .spawn(move || server.serve(&shutdown)) } -impl WorktreeFileStamp { - pub(crate) fn from_metadata(metadata: &fs::Metadata) -> Self { - Self { - size_bytes: metadata.len(), - modified_ns: metadata_modified_ns(metadata), - changed_ns: metadata_changed_ns(metadata), - device_id: metadata_device_id(metadata), - inode: metadata_inode(metadata), - executable: metadata_executable(metadata), +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[derive(Clone)] +struct ActiveSchemaValidationServer { + id: u64, + shutdown: Arc, +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +struct SchemaValidationServerRegistration { + db_path: PathBuf, + id: u64, +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +impl Drop for SchemaValidationServerRegistration { + fn drop(&mut self) { + let servers = SCHEMA_VALIDATION_SERVERS.get_or_init(|| Mutex::new(HashMap::new())); + let mut servers = servers + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if servers + .get(&self.db_path) + .is_some_and(|active| active.id == self.id) + { + servers.remove(&self.db_path); } } } -#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub(crate) struct WorkdirFileStamp { - size_bytes: u64, - modified_ns: i64, - changed_ns: i64, - #[serde(default)] - device_id: i64, - #[serde(default)] - inode: i64, - executable: bool, +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn stop_schema_validation_server(db_path: &Path) -> bool { + let active = SCHEMA_VALIDATION_SERVERS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .get(db_path) + .cloned(); + if let Some(active) = active { + active.shutdown.stop(); + true + } else { + false + } } -impl WorkdirFileStamp { - pub(crate) fn from_metadata(metadata: &fs::Metadata) -> Self { - Self { - size_bytes: metadata.len(), - modified_ns: metadata_modified_ns(metadata), - changed_ns: metadata_changed_ns(metadata), - device_id: metadata_device_id(metadata), - inode: metadata_inode(metadata), - executable: metadata_executable(metadata), - } +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn start_schema_validation_server(db_path: &Path, server: SchemaValidationServer) { + let id = NEXT_SCHEMA_VALIDATION_SERVER_ID.fetch_add(1, Ordering::Relaxed); + let shutdown = Arc::new(SchemaValidationServerShutdown::default()); + let registration = SchemaValidationServerRegistration { + db_path: db_path.to_path_buf(), + id, + }; + let servers = SCHEMA_VALIDATION_SERVERS.get_or_init(|| Mutex::new(HashMap::new())); + let mut servers = servers + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(active) = servers.get(db_path) { + active.shutdown.stop(); + return; } + servers.insert( + db_path.to_path_buf(), + ActiveSchemaValidationServer { + id, + shutdown: shutdown.clone(), + }, + ); + drop(servers); + let thread_shutdown = shutdown.clone(); + let _ = std::thread::Builder::new() + .name("trail-schema-validation".to_owned()) + .spawn(move || { + let _registration = registration; + server.delay_start_for_test(); + server.serve(&thread_shutdown); + }); } -#[derive(Debug, Clone, Default)] -pub(crate) struct MaterializedWorkdir { - files_written: usize, - stamps: BTreeMap, +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn request_schema_validation_result( + socket_path: &Path, + nonce: &str, + generation: &str, + backend: &str, + leader_pid: u32, +) -> Option { + use std::net::Shutdown; + use std::os::unix::net::UnixStream; + + let mut stream = UnixStream::connect(socket_path).ok()?; + let (peer_pid, peer_uid) = schema_validation_peer_identity(&stream).ok()?; + if peer_pid != leader_pid || peer_uid != rustix::process::getuid().as_raw() { + return None; + } + stream.set_read_timeout(Some(Duration::from_secs(5))).ok()?; + let request = format!( + "trail-schema-validation-request-v1\n{nonce}\n{generation}\n{}\n", + hex::encode(backend.as_bytes()) + ); + stream.write_all(request.as_bytes()).ok()?; + stream.shutdown(Shutdown::Write).ok()?; + let mut response = String::new(); + stream + .take(1024 * 1024) + .read_to_string(&mut response) + .ok()?; + parse_schema_validation_wire_result(&response, nonce, generation, backend) } -impl MaterializedWorkdir { - pub(crate) fn insert_stamp(&mut self, path: String, stamp: WorkdirFileStamp) { - self.files_written += 1; - self.stamps.insert(path, stamp); +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn read_schema_validation_announcement( + path: &Path, + leader_pid: u32, + expected_namespace: &str, +) -> Option<(String, PathBuf)> { + use std::os::unix::fs::OpenOptionsExt; + + let file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path) + .ok()?; + let metadata = file.metadata().ok()?; + if !metadata.is_file() + || metadata.uid() != rustix::process::getuid().as_raw() + || metadata.mode() & 0o077 != 0 + { + return None; + } + let mut text = String::new(); + file.take(8192).read_to_string(&mut text).ok()?; + let mut lines = text.lines(); + if lines.next()? != "trail-schema-validation-announce-v1" + || lines.next()?.parse::().ok()? != leader_pid + || lines.next()? != expected_namespace + { + return None; + } + let nonce = lines.next()?.to_owned(); + use std::os::unix::ffi::OsStringExt; + let socket = PathBuf::from(std::ffi::OsString::from_vec( + hex::decode(lines.next()?).ok()?, + )); + if nonce.len() != 64 || lines.next().is_some() { + return None; } + Some((nonce, socket)) +} - pub(crate) fn extend(&mut self, other: MaterializedWorkdir) { - self.files_written += other.files_written; - self.stamps.extend(other.stamps); +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn coordinate_schema_snapshot_validation( + db_path: &Path, + backend: &str, + generation: &mut SchemaGeneration, + authenticated_wal_digest: &mut Option, +) -> (Result<()>, Option) { + use rustix::process::{Flock, FlockType, Pid}; + use std::os::unix::fs::OpenOptionsExt; + use std::os::unix::net::UnixListener; + + *authenticated_wal_digest = None; + + // POSIX record locks are process-associated, so never overlap a replacement + // FD with an existing process-local server. Revoke the old server without + // waiting and conservatively perform this rare replacement validation + // locally; later rounds may elect a fresh cross-process authority after the + // old thread has released its FD. + if stop_schema_validation_server(db_path) { + return ( + validate_schema_snapshot_generation(db_path, backend, generation), + None, + ); + } + + let Some((runtime_dir, namespace)) = + schema_validation_runtime_namespace(db_path).ok().flatten() + else { + return ( + validate_schema_snapshot_generation(db_path, backend, generation), + None, + ); + }; + loop { + let file = match OpenOptions::new() + .read(true) + .write(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(schema_validation_leader_path(db_path)) + { + Ok(file) => file, + Err(error) => return (Err(schema_reinitialize_error(error)), None), + }; + match rustix::fs::fcntl_lock(&file, rustix::fs::FlockOperation::NonBlockingLockExclusive) { + Ok(()) => { + // Exclusive leader authority proves no live cross-process server + // owns an entry in this workspace namespace. Remove artifacts + // left by a process that exited without running Rust destructors. + cleanup_stale_schema_validation_runtime_entries(&runtime_dir, &namespace); + *generation = match schema_generation(db_path).map_err(schema_reinitialize_error) { + Ok(current) => current, + Err(error) => return (Err(error), Some(file)), + }; + let mut nonce = [0_u8; 32]; + if getrandom::getrandom(&mut nonce).is_err() { + return ( + validate_schema_snapshot_generation(db_path, backend, generation), + Some(file), + ); + } + let nonce = hex::encode(nonce); + let leaf = format!("{}-{}", &namespace[..24], &nonce[..24]); + let socket_path = runtime_dir.join(format!("{leaf}.socket")); + let announcement_path = runtime_dir.join(format!("{leaf}.announce")); + if socket_path.as_os_str().as_encoded_bytes().len() >= 100 { + return ( + validate_schema_snapshot_generation(db_path, backend, generation), + Some(file), + ); + } + let listener = match UnixListener::bind(&socket_path) { + Ok(listener) => listener, + Err(_) => { + return ( + validate_schema_snapshot_generation(db_path, backend, generation), + Some(file), + ) + } + }; + let socket_cleanup = match SchemaValidationRuntimeEntry::capture(&socket_path) { + Ok(identity) => identity, + Err(_) => { + return ( + validate_schema_snapshot_generation(db_path, backend, generation), + Some(file), + ) + } + }; + let socket_authority = + match secure_schema_validation_socket(&socket_path, &socket_cleanup) { + Ok(authority) => authority, + Err(_) => { + return ( + validate_schema_snapshot_generation(db_path, backend, generation), + Some(file), + ) + } + }; + let _ = listener.set_nonblocking(true); + let announcement = format!( + "trail-schema-validation-announce-v1\n{}\n{}\n{}\n{}\n", + std::process::id(), + namespace, + nonce, + hex::encode(socket_path.as_os_str().as_encoded_bytes()), + ); + let mut announcement_file = match OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(&announcement_path) + { + Ok(file) => file, + Err(_) => { + return ( + validate_schema_snapshot_generation(db_path, backend, generation), + Some(file), + ) + } + }; + let announcement_cleanup = + match SchemaValidationRuntimeEntry::capture(&announcement_path) { + Ok(identity) => identity, + Err(_) => { + return ( + validate_schema_snapshot_generation(db_path, backend, generation), + Some(file), + ) + } + }; + if announcement_file + .write_all(announcement.as_bytes()) + .is_err() + || announcement_file.sync_all().is_err() + { + return ( + validate_schema_snapshot_generation(db_path, backend, generation), + Some(file), + ); + } + *generation = match schema_generation(db_path).map_err(schema_reinitialize_error) { + Ok(current) => current, + Err(error) => return (Err(error), Some(file)), + }; + let generation_key = schema_generation_key(generation); + let mut validation = + validate_schema_snapshot_generation(db_path, backend, generation); + let outcome = if validation.is_ok() { + match schema_wal_digest(db_path, generation) { + Ok(digest) => { + *authenticated_wal_digest = Some(digest.clone()); + CrossProcessSchemaValidationOutcome::Success(digest) + } + Err(error) => { + validation = Err(error); + CrossProcessSchemaValidationOutcome::Failure(schema_failure_message( + validation.as_ref().unwrap_err(), + )) + } + } + } else { + CrossProcessSchemaValidationOutcome::Failure(schema_failure_message( + validation.as_ref().unwrap_err(), + )) + }; + #[cfg(test)] + let (start_delay, shutdown_delay) = + take_schema_validation_server_delays_for_test(db_path); + let server = SchemaValidationServer { + listener, + nonce, + generation: generation_key, + backend: backend.to_owned(), + outcome, + _leader_exclusion: file, + _socket_authority: socket_authority, + _socket_cleanup: socket_cleanup, + _announcement_cleanup: announcement_cleanup, + #[cfg(test)] + panic_on_serve: false, + #[cfg(test)] + start_delay, + #[cfg(test)] + shutdown_delay, + }; + start_schema_validation_server(db_path, server); + return (validation, None); + } + Err(error) if error == rustix::io::Errno::AGAIN => { + let requested = Flock::from(FlockType::WriteLock); + let leader_pid = rustix::process::fcntl_getlk(&file, &requested) + .ok() + .flatten() + .map(|lock| Pid::as_raw(lock.pid) as u32); + if let Some(leader_pid) = leader_pid { + let generation_key = schema_generation_key(generation); + let prefix = format!("{}-", &namespace[..24]); + let announcements = fs::read_dir(&runtime_dir) + .ok() + .into_iter() + .flatten() + .filter_map(|entry| entry.ok()) + .filter(|entry| { + let name = entry.file_name(); + let name = name.to_string_lossy(); + name.starts_with(&prefix) && name.ends_with(".announce") + }); + for announcement in announcements { + if let Some((nonce, announced_socket)) = read_schema_validation_announcement( + &announcement.path(), + leader_pid, + &namespace, + ) { + let expected_socket = runtime_dir.join(format!( + "{}-{}.socket", + &namespace[..24], + &nonce[..24] + )); + if announced_socket != expected_socket { + continue; + } + if let Some(outcome) = request_schema_validation_result( + &announced_socket, + &nonce, + &generation_key, + backend, + leader_pid, + ) { + return ( + match outcome { + CrossProcessSchemaValidationOutcome::Success(digest) => { + *authenticated_wal_digest = Some(digest); + Ok(()) + } + CrossProcessSchemaValidationOutcome::Failure(message) => { + Err(schema_reinitialize_error(message)) + } + }, + None, + ); + } + } + } + } + std::thread::sleep(Duration::from_millis(2)); + } + Err(error) => return (Err(Error::Io(error.into())), None), + } } } -#[derive(Debug, Clone, Default)] -pub(crate) struct RootMaterializationReport { - file_count: u64, - disk_manifest: BTreeMap, - materialized: MaterializedWorkdir, +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn coordinate_schema_snapshot_validation( + db_path: &Path, + backend: &str, + generation: &mut SchemaGeneration, + authenticated_wal_digest: &mut Option, +) -> (Result<()>, Option) { + *authenticated_wal_digest = None; + ( + validate_schema_snapshot_generation(db_path, backend, generation), + None, + ) } -#[derive(Debug, Clone)] -pub(crate) struct IndexedDiskManifest { - manifest: DiskManifest, - stamp: WorktreeFileStamp, +fn validate_schema_snapshot_generation( + db_path: &Path, + prolly_backend: &str, + generation: &SchemaGeneration, +) -> Result<()> { + validate_schema_snapshot(db_path, prolly_backend)?; + #[cfg(test)] + run_schema_snapshot_generation_hook(db_path); + let after = schema_generation(db_path).map_err(schema_reinitialize_error)?; + if &after != generation { + return Err(schema_reinitialize_error( + "schema main/WAL/SHM generation changed during snapshot validation", + )); + } + Ok(()) } -fn metadata_modified_ns(metadata: &fs::Metadata) -> i64 { - metadata - .modified() - .ok() - .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) - .map(duration_ns) - .unwrap_or(0) +fn schema_failure_message(error: &Error) -> String { + match error { + Error::SchemaReinitializeRequired { found, .. } => found.clone(), + other => other.to_string(), + } } -#[cfg(unix)] -fn metadata_changed_ns(metadata: &fs::Metadata) -> i64 { - metadata - .ctime() - .saturating_mul(1_000_000_000) - .saturating_add(metadata.ctime_nsec()) +fn validate_schema_snapshot(db_path: &Path, prolly_backend: &str) -> Result<()> { + #[cfg(test)] + schema_validation_process_test_probe()?; + #[cfg(test)] + if SCHEMA_VALIDATION_FAILURES + .get_or_init(|| Mutex::new(HashSet::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(db_path) + { + std::thread::sleep(Duration::from_millis(100)); + return Err(schema_reinitialize_error( + "injected schema validation leader failure", + )); + } + let snapshot = tempfile::Builder::new() + .prefix("trail-schema-preflight-") + .tempdir() + .map_err(schema_reinitialize_error)?; + let snapshot_db = snapshot.path().join("trail.sqlite"); + // The WAL is part of the durable database image. The SHM file is only a + // rebuildable shared-memory index and copying it can transplant stale reader + // marks into the private snapshot. + for suffix in ["", "-wal", "-journal"] { + let mut source = db_path.as_os_str().to_os_string(); + source.push(suffix); + let source = PathBuf::from(source); + if source.exists() { + let mut destination = snapshot_db.as_os_str().to_os_string(); + destination.push(suffix); + let destination = PathBuf::from(destination); + fs::copy(&source, &destination).map_err(schema_reinitialize_error)?; + } + } + let conn = rusqlite::Connection::open(&snapshot_db).map_err(schema_reinitialize_error)?; + conn.pragma_update(None, "foreign_keys", true) + .map_err(schema_reinitialize_error)?; + Trail::validate_schema_v18(&conn).map_err(schema_reinitialize_error)?; + match prolly_backend { + "sqlite" => { + storage::validate_prolly_sqlite_schema_v18(&conn).map_err(schema_reinitialize_error) + } + "slatedb" => { + storage::validate_no_prolly_sqlite_schema_v18(&conn).map_err(schema_reinitialize_error) + } + other => Err(Error::InvalidInput(format!( + "storage.prolly_backend must be sqlite or slatedb, got `{other}`" + ))), + } } -#[cfg(not(unix))] -fn metadata_changed_ns(_metadata: &fs::Metadata) -> i64 { - 0 +#[cfg(test)] +fn schema_validation_process_test_probe() -> Result<()> { + use std::io::Write as _; + + if let Some(counter_path) = std::env::var_os("TRAIL_TEST_SCHEMA_VALIDATION_COUNTER") { + let mut counter = OpenOptions::new() + .create(true) + .append(true) + .open(counter_path)?; + #[cfg(any(target_os = "linux", target_os = "macos"))] + rustix::fs::flock(&counter, rustix::fs::FlockOperation::LockExclusive) + .map_err(|error| Error::Io(error.into()))?; + writeln!(counter, "{}", std::process::id())?; + counter.sync_all()?; + } + if let Some(started_path) = std::env::var_os("TRAIL_TEST_SCHEMA_VALIDATION_STARTED") { + let _ = OpenOptions::new() + .write(true) + .create_new(true) + .open(started_path); + } + if let Some(crash_path) = std::env::var_os("TRAIL_TEST_SCHEMA_VALIDATION_CRASH_ONCE") { + if let Ok(mut crash) = OpenOptions::new() + .write(true) + .create_new(true) + .open(crash_path) + { + writeln!(crash, "{}", std::process::id())?; + crash.sync_all()?; + loop { + std::thread::park(); + } + } + } + if let Some(delay) = std::env::var_os("TRAIL_TEST_SCHEMA_VALIDATION_DELAY_MS") { + let delay = delay + .to_string_lossy() + .parse::() + .map_err(|error| Error::InvalidInput(error.to_string()))?; + std::thread::sleep(Duration::from_millis(delay)); + } + if let Ok(message) = std::env::var("TRAIL_TEST_SCHEMA_VALIDATION_FAIL") { + return Err(schema_reinitialize_error(message)); + } + Ok(()) } #[cfg(unix)] -fn metadata_device_id(metadata: &fs::Metadata) -> i64 { - metadata.dev().min(i64::MAX as u64) as i64 +fn schema_wal_digest(db_path: &Path, expected: &SchemaGeneration) -> Result { + schema_wal_digest_inner(db_path, expected, false)?.ok_or_else(|| { + schema_reinitialize_error("schema WAL unexpectedly lacked a stable digest snapshot") + }) } -#[cfg(not(unix))] -fn metadata_device_id(_metadata: &fs::Metadata) -> i64 { - 0 +#[cfg(unix)] +fn schema_wal_digest_allowing_wal_ctime_transition( + db_path: &Path, + expected: &SchemaGeneration, +) -> Result { + const MAX_STABLE_SNAPSHOT_ATTEMPTS: usize = 16; + + for attempt in 0..MAX_STABLE_SNAPSHOT_ATTEMPTS { + if let Some(digest) = schema_wal_digest_inner(db_path, expected, true)? { + return Ok(digest); + } + if attempt + 1 < MAX_STABLE_SNAPSHOT_ATTEMPTS { + std::thread::sleep(Duration::from_millis(1)); + } + } + Err(schema_reinitialize_error( + "schema WAL did not reach a stable descriptor snapshot for digest", + )) } #[cfg(unix)] -fn metadata_inode(metadata: &fs::Metadata) -> i64 { - metadata.ino().min(i64::MAX as u64) as i64 +fn schema_wal_digest_inner( + db_path: &Path, + expected: &SchemaGeneration, + allow_wal_ctime_transition: bool, +) -> Result> { + use std::os::unix::fs::OpenOptionsExt; + + let before = schema_generation(db_path).map_err(schema_reinitialize_error)?; + if &before != expected + && !(allow_wal_ctime_transition + && schema_generation_is_only_wal_ctime_transition(expected, &before)) + { + return Err(schema_reinitialize_error( + "schema generation changed before WAL digest", + )); + } + let wal = before + .0 + .iter() + .find(|file| file.suffix == "-wal") + .ok_or_else(|| schema_reinitialize_error("schema generation omitted WAL authority"))?; + let mut digest = Sha256::new(); + digest.update(b"trail-schema-wal-v1\0"); + digest.update([u8::from(wal.present)]); + if wal.present { + let mut path = db_path.as_os_str().to_os_string(); + path.push("-wal"); + let path = PathBuf::from(path); + #[cfg(test)] + if let Some(hook) = NEXT_SCHEMA_WAL_DIGEST_HOOKS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(&path) + { + hook(&path); + } + let mut file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(&path) + .map_err(schema_reinitialize_error)?; + let opened = file.metadata().map_err(schema_reinitialize_error)?; + let opened = schema_wal_generation_from_metadata(&opened); + if !schema_wal_generation_matches_authority(&opened, wal) { + return Err(schema_reinitialize_error( + "schema WAL descriptor changed before digest", + )); + } + #[cfg(test)] + if let Some(hook) = SCHEMA_WAL_DIGEST_ATTEMPT_HOOKS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .get_mut(&path) + { + hook(&path); + } + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer).map_err(schema_reinitialize_error)?; + if read == 0 { + break; + } + digest.update(&buffer[..read]); + } + let hashed = file.metadata().map_err(schema_reinitialize_error)?; + let hashed = schema_wal_generation_from_metadata(&hashed); + if hashed != opened { + if allow_wal_ctime_transition { + return Ok(None); + } + return Err(schema_reinitialize_error( + "schema WAL descriptor changed during digest", + )); + } + } + let after = schema_generation(db_path).map_err(schema_reinitialize_error)?; + if after != before { + if allow_wal_ctime_transition + && schema_generation_is_only_wal_ctime_transition(&before, &after) + { + return Ok(None); + } + return Err(schema_reinitialize_error( + "schema generation changed during WAL digest", + )); + } + Ok(Some(hex::encode(digest.finalize()))) } -#[cfg(not(unix))] -fn metadata_inode(_metadata: &fs::Metadata) -> i64 { - 0 +#[cfg(unix)] +fn schema_wal_generation_from_metadata(metadata: &fs::Metadata) -> SchemaFileGeneration { + SchemaFileGeneration { + suffix: "-wal", + present: metadata.file_type().is_file(), + device: metadata.dev(), + inode: metadata.ino(), + length: metadata.len(), + modified_seconds: metadata.mtime(), + modified_nanoseconds: metadata.mtime_nsec(), + changed_seconds: metadata.ctime(), + changed_nanoseconds: metadata.ctime_nsec(), + } } #[cfg(unix)] -fn metadata_executable(metadata: &fs::Metadata) -> bool { - metadata.permissions().mode() & 0o111 != 0 +fn schema_wal_generation_matches_authority( + observed: &SchemaFileGeneration, + expected: &SchemaFileGeneration, +) -> bool { + observed.present + && observed.device == expected.device + && observed.inode == expected.inode + && observed.length == expected.length + && observed.modified_seconds == expected.modified_seconds + && observed.modified_nanoseconds == expected.modified_nanoseconds } #[cfg(not(unix))] -fn metadata_executable(_metadata: &fs::Metadata) -> bool { - false +fn schema_wal_digest(_db_path: &Path, _expected: &SchemaGeneration) -> Result { + Err(schema_reinitialize_error( + "schema WAL digest is unsupported on this platform", + )) } -fn duration_ns(duration: Duration) -> i64 { - let ns = (duration.as_secs() as u128) - .saturating_mul(1_000_000_000) - .saturating_add(duration.subsec_nanos() as u128); - ns.min(i64::MAX as u128) as i64 +#[cfg(unix)] +fn schema_generation(db_path: &Path) -> std::io::Result { + let mut files = Vec::with_capacity(4); + for suffix in ["", "-wal", "-shm", "-journal"] { + let mut path = db_path.as_os_str().to_os_string(); + path.push(suffix); + let path = PathBuf::from(path); + match fs::metadata(&path) { + Ok(metadata) => files.push(SchemaFileGeneration { + suffix, + present: true, + device: metadata.dev(), + inode: metadata.ino(), + length: metadata.len(), + modified_seconds: (suffix != "-shm").then_some(metadata.mtime()).unwrap_or(0), + modified_nanoseconds: (suffix != "-shm") + .then_some(metadata.mtime_nsec()) + .unwrap_or(0), + changed_seconds: (suffix != "-shm").then_some(metadata.ctime()).unwrap_or(0), + changed_nanoseconds: (suffix != "-shm") + .then_some(metadata.ctime_nsec()) + .unwrap_or(0), + }), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + files.push(SchemaFileGeneration { + suffix, + present: false, + device: 0, + inode: 0, + length: 0, + modified_seconds: 0, + modified_nanoseconds: 0, + changed_seconds: 0, + changed_nanoseconds: 0, + }); + } + Err(error) => return Err(error), + } + } + Ok(SchemaGeneration(files)) } -pub(crate) struct DaemonWorktreeCache { - state: Arc>, - persist: Option, - watcher: Option, +fn schema_lock_waiting_is_enabled() -> bool { + WRITE_LOCK_WAIT_DEADLINE + .with(|deadline| deadline.get()) + .is_some_and(|deadline| Instant::now() < deadline) } -#[derive(Clone, Debug)] -pub(crate) struct DaemonWorktreeCachePersist { - path: PathBuf, - workspace_root: PathBuf, - pid: u32, - active: Arc, +fn schema_exclusion_path(_db_dir: &Path, db_path: &Path) -> PathBuf { + db_path + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(SCHEMA_EXCLUSION_FILE) } -#[derive(Debug)] -pub struct DaemonWorktreeCacheWarmup { - workspace_root: PathBuf, - db_dir: PathBuf, - state: Arc>, - persist: Option, - generation: u64, +fn schema_validation_leader_path(db_path: &Path) -> PathBuf { + db_path + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(SCHEMA_VALIDATION_LEADER_FILE) } -#[derive(Debug, Default)] -pub(crate) struct DaemonWorktreeCacheState { - dirty_paths: BTreeSet, - overflow: bool, - initialized: bool, - baseline_root_id: Option, - generation: u64, +fn schema_db_dir(db_path: &Path) -> Result<&Path> { + let parent = db_path.parent().ok_or_else(|| { + Error::InvalidInput("schema database path has no workspace directory".into()) + })?; + if parent.file_name().is_some_and(|name| name == "index") { + return parent.parent().ok_or_else(|| { + Error::InvalidInput("schema database index has no workspace directory".into()) + }); + } + Ok(parent) } -#[derive(Debug)] -pub(crate) enum DaemonWorktreeSnapshot { - Clean { - generation: u64, - root_id: Option, - }, - Dirty { - generation: u64, - paths: Vec, - }, - Overflow { - generation: u64, - }, +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn acquire_schema_shared_exclusion(db_path: &Path) -> Result { + let lock_path = schema_exclusion_path(schema_db_dir(db_path)?, db_path); + let database = File::open(lock_path).map_err(schema_reinitialize_error)?; + let mut delay = Duration::from_millis(2); + loop { + match rustix::fs::flock(&database, rustix::fs::FlockOperation::NonBlockingLockShared) { + Ok(()) => { + return Ok(SchemaSharedExclusion { + _database: database, + }) + } + Err(error) if error == rustix::io::Errno::AGAIN => { + if !schema_lock_waiting_is_enabled() { + return Err(Error::WorkspaceLocked( + "workspace schema writer is active".into(), + )); + } + std::thread::sleep(delay); + delay = (delay * 2).min(Duration::from_millis(50)); + } + Err(error) => return Err(Error::Io(error.into())), + } + } } -pub(crate) enum CachedWorkdirManifestStatus { - Clean, - Dirty { - disk_manifest: BTreeMap, - candidate_paths: Option>, - }, - Missing, +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn acquire_schema_shared_exclusion(db_path: &Path) -> Result { + let lock_path = schema_exclusion_path(schema_db_dir(db_path)?, db_path); + let database = File::open(lock_path).map_err(schema_reinitialize_error)?; + Ok(SchemaSharedExclusion { + _database: database, + }) } -#[derive(Debug, Clone)] -pub(crate) struct MergeContext { - base_change: ChangeId, +#[cfg(not(unix))] +fn schema_generation(db_path: &Path) -> std::io::Result { + let mut files = Vec::with_capacity(4); + for suffix in ["", "-wal", "-shm", "-journal"] { + let mut path = db_path.as_os_str().to_os_string(); + path.push(suffix); + let path = PathBuf::from(path); + match fs::metadata(&path) { + Ok(metadata) => files.push(SchemaFileGeneration { + suffix, + present: true, + device: 0, + inode: 0, + length: metadata.len(), + modified_seconds: if suffix == "-shm" { + 0 + } else { + metadata + .modified()? + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + .min(i64::MAX as u64) as i64 + }, + modified_nanoseconds: if suffix == "-shm" { + 0 + } else { + metadata + .modified()? + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .subsec_nanos() as i64 + }, + changed_seconds: 0, + changed_nanoseconds: 0, + }), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + files.push(SchemaFileGeneration { + suffix, + present: false, + device: 0, + inode: 0, + length: 0, + modified_seconds: 0, + modified_nanoseconds: 0, + changed_seconds: 0, + changed_nanoseconds: 0, + }); + } + Err(error) => return Err(error), + } + } + Ok(SchemaGeneration(files)) +} + +fn schema_reinitialize_error(err: impl std::fmt::Display) -> Error { + Error::SchemaReinitializeRequired { + found: err.to_string(), + guidance: "back up this workspace, then run `trail init --force` to create schema v18" + .into(), + } +} + +thread_local! { + static WRITE_LOCK_WAIT_DEADLINE: Cell> = const { Cell::new(None) }; +} +const OP_OBJECT_VERSION: u16 = 1; +const BLOB_OBJECT_VERSION: u16 = 1; +const MESSAGE_OBJECT_VERSION: u16 = 1; +const ANCHOR_OBJECT_VERSION: u16 = 1; +const WORKSPACE_LAYER_MANIFEST_KIND: &str = "workspace_layer_manifest"; +const WORKSPACE_LAYER_MANIFEST_VERSION: u16 = 1; +const OBJECT_CACHE_MAX_ENTRIES: usize = 4096; +const OBJECT_CACHE_MAX_BYTES: usize = 64 * 1024 * 1024; +const ORDER_KEY_STEP: u64 = 1024; +const LANE_TEST_OUTPUT_PREVIEW_BYTES: usize = 64 * 1024; +const DEFAULT_CRABIGNORE_PATTERNS: &[&str] = &[ + ".trail/", + ".git/", + ".env", + ".env.*", + "*.pem", + "*.key", + "*.p12", + "*.pfx", + "id_rsa", + "id_ed25519", + "node_modules/", + "target/", + "dist/", + "build/", + "coverage/", +]; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct RootDirectoryChild { + pub(crate) name: String, + pub(crate) path: String, + pub(crate) entry: Option, +} + +pub struct Trail { + workspace_root: PathBuf, + db_dir: PathBuf, + sqlite_path: PathBuf, + store: TrailProllyStore, + prolly: Prolly, + root_prolly: Prolly, + // Keep the connection configured with NO_CKPT_ON_CLOSE after every cloned + // SQLite Prolly store handle so it is the last SQLite connection dropped. + conn: Connection, + config: TrailConfig, + object_cache: Mutex, + daemon_worktree_cache: Option, + changed_path_daemon_registry: Mutex, + git_handoff_metrics: Cell, + case_fold_index_metrics: Cell, + operation_metrics: Option>, +} + +pub(crate) struct WorkspaceIgnorePolicySnapshot { + workspace_root: PathBuf, + metrics: Option>, + matcher: OnceLock>, +} + +#[derive(Clone)] +struct TrailProllyStore { + backend: TrailProllyStoreBackend, + metrics: Option>, +} + +#[derive(Clone)] +enum TrailProllyStoreBackend { + Sqlite(Arc), + SlateDb(Arc), +} + +impl TrailProllyStore { + fn new(backend: TrailProllyStoreBackend, metrics: Option>) -> Self { + Self { backend, metrics } + } + + fn note_prolly_read_call(&self, key_count: usize) { + if let Some(metrics) = &self.metrics { + metrics.note_prolly_read_call(key_count); + } + } + + fn note_prolly_read_values<'a, I>(&self, values: I) + where + I: IntoIterator>, + { + if let Some(metrics) = &self.metrics { + metrics.note_prolly_read_values(values); + } + } + + fn note_prolly_write_call(&self, key_count: usize, value_bytes: usize) { + if let Some(metrics) = &self.metrics { + metrics.note_prolly_write_call(key_count, value_bytes); + } + } +} + +#[derive(Debug)] +struct TrailProllyStoreError { + message: String, + source: Option>, +} + +impl TrailProllyStoreError { + fn with_source( + message: impl Into, + source: impl std::error::Error + Send + Sync + 'static, + ) -> Self { + Self { + message: message.into(), + source: Some(Box::new(source)), + } + } +} + +impl std::fmt::Display for TrailProllyStoreError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Trail prolly store error: {}", self.message) + } +} + +impl std::error::Error for TrailProllyStoreError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.source + .as_ref() + .map(|e| e.as_ref() as &(dyn std::error::Error + 'static)) + } +} + +impl Store for TrailProllyStore { + type Error = TrailProllyStoreError; + + fn get(&self, key: &[u8]) -> std::result::Result>, Self::Error> { + self.note_prolly_read_call(1); + let result = match &self.backend { + TrailProllyStoreBackend::Sqlite(store) => store + .get(key) + .map_err(|err| TrailProllyStoreError::with_source("SQLite prolly get failed", err)), + TrailProllyStoreBackend::SlateDb(store) => store.get(key).map_err(|err| { + TrailProllyStoreError::with_source("SlateDB prolly get failed", err) + }), + }; + if let Ok(Some(value)) = &result { + self.note_prolly_read_values(std::iter::once(value)); + } + result + } + + fn put(&self, key: &[u8], value: &[u8]) -> std::result::Result<(), Self::Error> { + self.note_prolly_write_call(1, value.len()); + match &self.backend { + TrailProllyStoreBackend::Sqlite(store) => store + .put(key, value) + .map_err(|err| TrailProllyStoreError::with_source("SQLite prolly put failed", err)), + TrailProllyStoreBackend::SlateDb(store) => store.put(key, value).map_err(|err| { + TrailProllyStoreError::with_source("SlateDB prolly put failed", err) + }), + } + } + + fn delete(&self, key: &[u8]) -> std::result::Result<(), Self::Error> { + self.note_prolly_write_call(1, 0); + match &self.backend { + TrailProllyStoreBackend::Sqlite(store) => store.delete(key).map_err(|err| { + TrailProllyStoreError::with_source("SQLite prolly delete failed", err) + }), + TrailProllyStoreBackend::SlateDb(store) => store.delete(key).map_err(|err| { + TrailProllyStoreError::with_source("SlateDB prolly delete failed", err) + }), + } + } + + fn batch(&self, ops: &[BatchOp]) -> std::result::Result<(), Self::Error> { + let value_bytes = ops + .iter() + .map(|op| match op { + BatchOp::Upsert { value, .. } => value.len(), + BatchOp::Delete { .. } => 0, + }) + .fold(0usize, usize::saturating_add); + self.note_prolly_write_call(ops.len(), value_bytes); + match &self.backend { + TrailProllyStoreBackend::Sqlite(store) => store.batch(ops).map_err(|err| { + TrailProllyStoreError::with_source("SQLite prolly batch failed", err) + }), + TrailProllyStoreBackend::SlateDb(store) => store.batch(ops).map_err(|err| { + TrailProllyStoreError::with_source("SlateDB prolly batch failed", err) + }), + } + } + + fn batch_get( + &self, + keys: &[&[u8]], + ) -> std::result::Result, Vec>, Self::Error> { + self.note_prolly_read_call(keys.len()); + let result = match &self.backend { + TrailProllyStoreBackend::Sqlite(store) => store.batch_get(keys).map_err(|err| { + TrailProllyStoreError::with_source("SQLite prolly batch_get failed", err) + }), + TrailProllyStoreBackend::SlateDb(store) => store.batch_get(keys).map_err(|err| { + TrailProllyStoreError::with_source("SlateDB prolly batch_get failed", err) + }), + }; + if let Ok(values) = &result { + self.note_prolly_read_values(values.values()); + } + result + } + + fn batch_get_ordered( + &self, + keys: &[&[u8]], + ) -> std::result::Result>>, Self::Error> { + self.note_prolly_read_call(keys.len()); + let result = match &self.backend { + TrailProllyStoreBackend::Sqlite(store) => { + store.batch_get_ordered(keys).map_err(|err| { + TrailProllyStoreError::with_source( + "SQLite prolly batch_get_ordered failed", + err, + ) + }) + } + TrailProllyStoreBackend::SlateDb(store) => { + store.batch_get_ordered(keys).map_err(|err| { + TrailProllyStoreError::with_source( + "SlateDB prolly batch_get_ordered failed", + err, + ) + }) + } + }; + if let Ok(values) = &result { + self.note_prolly_read_values(values.iter().filter_map(Option::as_ref)); + } + result + } + + fn batch_put(&self, entries: &[(&[u8], &[u8])]) -> std::result::Result<(), Self::Error> { + let value_bytes = entries + .iter() + .map(|(_, value)| value.len()) + .fold(0usize, usize::saturating_add); + self.note_prolly_write_call(entries.len(), value_bytes); + match &self.backend { + TrailProllyStoreBackend::Sqlite(store) => store.batch_put(entries).map_err(|err| { + TrailProllyStoreError::with_source("SQLite prolly batch_put failed", err) + }), + TrailProllyStoreBackend::SlateDb(store) => store.batch_put(entries).map_err(|err| { + TrailProllyStoreError::with_source("SlateDB prolly batch_put failed", err) + }), + } + } + + fn supports_hints(&self) -> bool { + match &self.backend { + TrailProllyStoreBackend::Sqlite(store) => store.supports_hints(), + TrailProllyStoreBackend::SlateDb(store) => store.supports_hints(), + } + } + + fn get_hint( + &self, + namespace: &[u8], + key: &[u8], + ) -> std::result::Result>, Self::Error> { + match &self.backend { + TrailProllyStoreBackend::Sqlite(store) => { + store.get_hint(namespace, key).map_err(|err| { + TrailProllyStoreError::with_source("SQLite prolly get_hint failed", err) + }) + } + TrailProllyStoreBackend::SlateDb(store) => { + store.get_hint(namespace, key).map_err(|err| { + TrailProllyStoreError::with_source("SlateDB prolly get_hint failed", err) + }) + } + } + } + + fn put_hint( + &self, + namespace: &[u8], + key: &[u8], + value: &[u8], + ) -> std::result::Result<(), Self::Error> { + match &self.backend { + TrailProllyStoreBackend::Sqlite(store) => { + store.put_hint(namespace, key, value).map_err(|err| { + TrailProllyStoreError::with_source("SQLite prolly put_hint failed", err) + }) + } + TrailProllyStoreBackend::SlateDb(store) => { + store.put_hint(namespace, key, value).map_err(|err| { + TrailProllyStoreError::with_source("SlateDB prolly put_hint failed", err) + }) + } + } + } + + fn batch_put_with_hint( + &self, + entries: &[(&[u8], &[u8])], + namespace: &[u8], + key: &[u8], + value: &[u8], + ) -> std::result::Result<(), Self::Error> { + let value_bytes = entries + .iter() + .map(|(_, value)| value.len()) + .fold(0usize, usize::saturating_add); + self.note_prolly_write_call(entries.len(), value_bytes); + match &self.backend { + TrailProllyStoreBackend::Sqlite(store) => store + .batch_put_with_hint(entries, namespace, key, value) + .map_err(|err| { + TrailProllyStoreError::with_source( + "SQLite prolly batch_put_with_hint failed", + err, + ) + }), + TrailProllyStoreBackend::SlateDb(store) => store + .batch_put_with_hint(entries, namespace, key, value) + .map_err(|err| { + TrailProllyStoreError::with_source( + "SlateDB prolly batch_put_with_hint failed", + err, + ) + }), + } + } +} + +fn open_prolly_store( + config: &TrailConfig, + sqlite_path: &Path, + metrics: Option>, + schema_mode: SchemaOpenMode, + validated_schema: Option<&ValidatedSchemaGeneration>, +) -> Result { + let backend = match config.storage.prolly_backend.as_str() { + "sqlite" => { + let store = match schema_mode { + SchemaOpenMode::FreshCreate => SqliteStore::open(sqlite_path)?, + SchemaOpenMode::Existing => { + if let Some(validated) = validated_schema { + SqliteStore::open_existing_verified(sqlite_path, |identity| { + validated.verify_main_identity(identity).map_err(|error| { + prolly_store_sqlite::SqliteStoreError::new(error.to_string()) + }) + }) + .map_err(schema_reinitialize_error)? + } else { + // The only unverified existing-open path is an internal clone made + // while the caller already owns the workspace writer exclusion and + // a fully validated Trail handle. + SqliteStore::open_existing(sqlite_path)? + } + } + }; + TrailProllyStoreBackend::Sqlite(Arc::new(store)) + } + "slatedb" => open_slatedb_prolly_store(&config.storage)?, + other => Err(Error::InvalidInput(format!( + "storage.prolly_backend must be sqlite or slatedb, got `{other}`" + )))?, + }; + Ok(TrailProllyStore::new(backend, metrics)) +} + +fn open_slatedb_prolly_store(storage: &StorageConfig) -> Result { + let path = storage.slatedb_path.trim().trim_matches('/'); + if path.is_empty() { + return Err(Error::InvalidInput( + "storage.slatedb_path must not be empty".to_string(), + )); + } + + let object_store = build_slatedb_object_store(storage)?; + let store = SlateDbStore::open(path, object_store)?; + Ok(TrailProllyStoreBackend::SlateDb(Arc::new(store))) +} + +fn build_slatedb_object_store(storage: &StorageConfig) -> Result> { + if storage.slatedb_s3_endpoint.trim().is_empty() { + return Err(Error::InvalidInput( + "storage.slatedb_s3_endpoint must not be empty".to_string(), + )); + } + if storage.slatedb_s3_bucket.trim().is_empty() { + return Err(Error::InvalidInput( + "storage.slatedb_s3_bucket must not be empty".to_string(), + )); + } + if storage.slatedb_s3_region.trim().is_empty() { + return Err(Error::InvalidInput( + "storage.slatedb_s3_region must not be empty".to_string(), + )); + } + + let store = AmazonS3Builder::new() + .with_endpoint(storage.slatedb_s3_endpoint.trim_end_matches('/')) + .with_bucket_name(storage.slatedb_s3_bucket.trim()) + .with_region(storage.slatedb_s3_region.trim()) + .with_access_key_id(&storage.slatedb_s3_access_key_id) + .with_secret_access_key(&storage.slatedb_s3_secret_access_key) + .with_allow_http(storage.slatedb_s3_allow_http) + .with_virtual_hosted_style_request(false) + .build() + .map_err(|err| { + Error::InvalidInput(format!( + "failed to configure SlateDB S3 object store: {err}" + )) + })?; + + Ok(Arc::new(store)) +} + +#[derive(Debug, Default)] +struct ObjectCache { + entries: HashMap, + order: VecDeque, + total_bytes: usize, +} + +#[derive(Debug)] +struct ObjectCacheEntry { + kind: String, + bytes: Vec, +} + +impl ObjectCache { + fn get(&self, kind: &str, object_id: &ObjectId) -> Option> { + self.entries.get(&object_id.0).and_then(|entry| { + if entry.kind == kind { + Some(entry.bytes.clone()) + } else { + None + } + }) + } + + fn insert(&mut self, object_id: &ObjectId, kind: &str, bytes: &[u8]) { + if bytes.len() > OBJECT_CACHE_MAX_BYTES { + return; + } + if self.entries.contains_key(&object_id.0) { + return; + } + self.entries.insert( + object_id.0.clone(), + ObjectCacheEntry { + kind: kind.to_string(), + bytes: bytes.to_vec(), + }, + ); + self.order.push_back(object_id.0.clone()); + self.total_bytes = self.total_bytes.saturating_add(bytes.len()); + while self.entries.len() > OBJECT_CACHE_MAX_ENTRIES + || self.total_bytes > OBJECT_CACHE_MAX_BYTES + { + let Some(evicted) = self.order.pop_front() else { + break; + }; + if let Some(entry) = self.entries.remove(&evicted) { + self.total_bytes = self.total_bytes.saturating_sub(entry.bytes.len()); + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InitImportMode { + Empty, + GitTracked, + WorkingTree, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum GitExportPolicy { + RequireMappedDelta, + AllowFullSnapshot, +} + +#[derive(Debug, Clone)] +pub(crate) struct DiskFile { + path: String, + bytes: Vec, + executable: bool, +} + +#[derive(Debug)] +pub(crate) struct WorktreePathScan { + paths: Vec, + total_bytes: u64, +} + +#[derive(Debug)] +pub(crate) struct RootBuildResult { + root_id: ObjectId, + files: BTreeMap, + disk_manifest: BTreeMap, + stats: ImportStats, +} + +#[derive(Debug)] +pub(crate) struct IncrementalRootBuildResult { + root_id: ObjectId, +} + +#[derive(Debug)] +pub(crate) enum RecordCaseFoldResolutionState { + Indexed { + previous_tree: Tree, + mutations: Vec, + }, + LegacyUnavailable, + Collision { + path: String, + previous: String, + }, +} + +#[derive(Debug)] +pub(crate) struct RecordCaseFoldResolution { + selected_paths: Vec, + expected_final_present_paths: BTreeSet, + expected_observed_present_paths: BTreeSet, + expected_absent_paths: BTreeSet, + state: RecordCaseFoldResolutionState, +} + +#[derive(Debug)] +pub(crate) struct RecordCaseFoldPreflight { + selected_paths: Vec, + expected_final_present_paths: BTreeSet, + expected_observed_present_paths: BTreeSet, + expected_absent_paths: BTreeSet, + case_fold_tree: Tree, +} + +#[derive(Debug)] +pub(crate) struct GitTrackedRootBuildResult { + root_id: ObjectId, + disk_manifest: BTreeMap, + stats: ImportStats, +} + +#[derive(Debug)] +pub(crate) struct SelectedWorktreeSnapshot { + paths: Vec, + files: Vec, + summaries: Vec, +} + +#[derive(Debug)] +pub(crate) struct FileBuildResult { + entry: FileEntry, + disk_manifest: DiskManifest, + line_changes: Vec, +} + +#[derive(Debug)] +pub(crate) struct TextBuildResult { + object_id: ObjectId, + line_changes: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct RootDiff { + changes: Vec, + summaries: Vec, +} + +#[derive(Debug)] +pub(crate) struct PathLocalMergeResult { + target_files: BTreeMap, + merged_files: BTreeMap, + conflicts: Vec, +} + +#[derive(Debug)] +pub(crate) struct CommandRunResult { + success: bool, + exit_code: Option, + timed_out: bool, + duration_ms: u64, + stdout: Vec, + stderr: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct ExternalMutationAuditInput { + pub(crate) actor: String, + pub(crate) surface: String, + pub(crate) command: String, + pub(crate) target_ref: Option, + pub(crate) lane_id: Option, + pub(crate) turn_id: Option, + pub(crate) status: String, + pub(crate) status_code: Option, + pub(crate) change_id: Option, + pub(crate) summary: Option, +} + +#[derive(Debug, Clone)] +pub(crate) struct HttpIdempotencyEntry { + pub(crate) method: String, + pub(crate) path: String, + pub(crate) request_hash: String, + pub(crate) status: u16, + pub(crate) body: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct HttpIdempotencyStoreInput { + pub(crate) key: String, + pub(crate) method: String, + pub(crate) path: String, + pub(crate) request_hash: String, + pub(crate) status: u16, + pub(crate) body: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct LaneTraceSpanBuilder { + span_id: String, + trace_id: String, + lane_id: String, + session_id: Option, + turn_id: Option, + parent_span_id: Option, + span_type: String, + name: String, + started_event_id: String, + started_at: i64, + attributes: Option, + ended_event_id: Option, + ended_at: Option, + status: Option, + result: Option, +} + +#[derive(Debug, Serialize, serde::Deserialize)] +pub(crate) struct BackupManifest { + format_version: u16, + trail_version: String, + created_at: i64, + source_workspace: String, + source_db_dir: String, + workspace_id: WorkspaceId, + branch: String, + ref_count: u64, + operation_count: u64, + sqlite_bytes: u64, + sqlite_sha256: String, + worktree_bytes: u64, +} + +#[derive(Debug)] +pub(crate) struct PendingLineMerge { + path: String, + target_entry: FileEntry, + lines: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) struct LineGap { + previous: Option, + next: Option, +} + +#[derive(Debug, Clone)] +pub(crate) struct OperationObject { + object_id: ObjectId, + operation: Operation, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DiskManifest { + kind: FileKind, + executable: bool, + content_hash: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct WorktreeFileStamp { + size_bytes: u64, + modified_ns: i64, + changed_ns: i64, + device_id: i64, + inode: i64, + executable: bool, +} + +impl WorktreeFileStamp { + pub(crate) fn from_metadata(metadata: &fs::Metadata) -> Self { + Self { + size_bytes: metadata.len(), + modified_ns: metadata_modified_ns(metadata), + changed_ns: metadata_changed_ns(metadata), + device_id: metadata_device_id(metadata), + inode: metadata_inode(metadata), + executable: metadata_executable(metadata), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub(crate) struct WorkdirFileStamp { + size_bytes: u64, + modified_ns: i64, + changed_ns: i64, + #[serde(default)] + device_id: i64, + #[serde(default)] + inode: i64, + executable: bool, +} + +impl WorkdirFileStamp { + pub(crate) fn from_metadata(metadata: &fs::Metadata) -> Self { + Self { + size_bytes: metadata.len(), + modified_ns: metadata_modified_ns(metadata), + changed_ns: metadata_changed_ns(metadata), + device_id: metadata_device_id(metadata), + inode: metadata_inode(metadata), + executable: metadata_executable(metadata), + } + } +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct MaterializedWorkdir { + files_written: usize, + stamps: BTreeMap, +} + +impl MaterializedWorkdir { + pub(crate) fn insert_stamp(&mut self, path: String, stamp: WorkdirFileStamp) { + self.files_written += 1; + self.stamps.insert(path, stamp); + } + + pub(crate) fn extend(&mut self, other: MaterializedWorkdir) { + self.files_written += other.files_written; + self.stamps.extend(other.stamps); + } +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct RootMaterializationReport { + file_count: u64, + disk_manifest: BTreeMap, + materialized: MaterializedWorkdir, +} + +#[derive(Debug, Clone)] +pub(crate) struct IndexedDiskManifest { + manifest: DiskManifest, + stamp: WorktreeFileStamp, +} + +fn metadata_modified_ns(metadata: &fs::Metadata) -> i64 { + metadata + .modified() + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map(duration_ns) + .unwrap_or(0) +} + +#[cfg(unix)] +fn metadata_changed_ns(metadata: &fs::Metadata) -> i64 { + metadata + .ctime() + .saturating_mul(1_000_000_000) + .saturating_add(metadata.ctime_nsec()) +} + +#[cfg(not(unix))] +fn metadata_changed_ns(_metadata: &fs::Metadata) -> i64 { + 0 +} + +#[cfg(unix)] +fn metadata_device_id(metadata: &fs::Metadata) -> i64 { + metadata.dev().min(i64::MAX as u64) as i64 +} + +#[cfg(not(unix))] +fn metadata_device_id(_metadata: &fs::Metadata) -> i64 { + 0 +} + +#[cfg(unix)] +fn metadata_inode(metadata: &fs::Metadata) -> i64 { + metadata.ino().min(i64::MAX as u64) as i64 +} + +#[cfg(not(unix))] +fn metadata_inode(_metadata: &fs::Metadata) -> i64 { + 0 +} + +#[cfg(unix)] +fn metadata_executable(metadata: &fs::Metadata) -> bool { + metadata.permissions().mode() & 0o111 != 0 +} + +#[cfg(not(unix))] +fn metadata_executable(_metadata: &fs::Metadata) -> bool { + false +} + +fn duration_ns(duration: Duration) -> i64 { + let ns = (duration.as_secs() as u128) + .saturating_mul(1_000_000_000) + .saturating_add(duration.subsec_nanos() as u128); + ns.min(i64::MAX as u128) as i64 +} + +pub(crate) struct DaemonWorktreeCache { + state: Arc>, + persist: Option, + watcher: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct DaemonWorktreeCachePersist; + +#[derive(Debug)] +pub struct DaemonWorktreeCacheWarmup { + workspace_root: PathBuf, + db_dir: PathBuf, + state: Arc>, + persist: Option, + generation: u64, +} + +#[derive(Debug, Default)] +pub(crate) struct DaemonWorktreeCacheState { + dirty_paths: BTreeSet, + overflow: bool, + initialized: bool, + baseline_root_id: Option, + generation: u64, + policy_invalidation_index: Option, +} + +#[derive(Debug)] +pub(crate) enum DaemonWorktreeSnapshot { + Clean { + generation: u64, + root_id: Option, + }, + Dirty { + generation: u64, + paths: Vec, + }, + Overflow { + generation: u64, + }, +} + +pub(crate) enum CachedWorkdirManifestStatus { + Clean, + Dirty { + disk_manifest: BTreeMap, + candidate_paths: Option>, + }, + Missing, +} + +#[derive(Debug, Clone)] +pub(crate) struct MergeContext { + base_change: ChangeId, left_change: ChangeId, right_change: ChangeId, base_root: ObjectId, @@ -783,7 +2908,7 @@ pub(crate) struct MergeContext { #[derive(Debug, Clone)] pub(crate) struct PendingConflictMerge { merge_id: String, - queue_id: Option, + lane_queue_id: Option, source_ref: String, target_ref: String, base_change: ChangeId, @@ -794,44 +2919,1118 @@ pub(crate) struct PendingConflictMerge { right_root: Option, } -#[derive(Debug, Clone)] -pub(crate) struct GitState { - head: Option, - dirty: bool, -} +#[derive(Debug, Clone)] +pub(crate) struct GitState { + head: Option, + dirty: bool, +} + +#[derive(Debug, Clone)] +pub(crate) struct GitIdentity { + head: String, + branch: Option, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct GitHandoffMetrics { + export_mode: GitExportMode, + changed_path_count: u64, + blob_write_count: u64, + git_plumbing_command_count: u64, + tracked_status_count: u64, + full_root_file_count: u64, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct CaseFoldIndexMetrics { + mode: CaseFoldIndexMode, + lookup_count: u64, + full_root_path_load_count: u64, + full_filesystem_path_scan_count: u64, +} + +#[derive(Clone, Copy, Debug, Default)] +enum CaseFoldIndexMode { + #[default] + Unknown, + Indexed, +} + +#[allow(dead_code)] // Reported by Task 5's scale harness; tests use it in this slice. +impl CaseFoldIndexMode { + fn as_str(self) -> &'static str { + match self { + Self::Unknown => "unknown", + Self::Indexed => "indexed", + } + } +} + +pub(crate) type CaseFoldIndexMetricsReport = PathIndexMetricsReport; + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) enum GitExportMode { + #[default] + Unknown, + MappedDelta, + FullSnapshot, +} + +impl GitExportMode { + fn as_str(self) -> &'static str { + match self { + Self::Unknown => "unknown", + Self::MappedDelta => "mapped_delta", + Self::FullSnapshot => "full_snapshot", + } + } +} + +impl From for GitHandoffMetricsReport { + fn from(metrics: GitHandoffMetrics) -> Self { + Self { + export_mode: metrics.export_mode.as_str().to_string(), + changed_path_count: metrics.changed_path_count, + blob_write_count: metrics.blob_write_count, + git_plumbing_command_count: metrics.git_plumbing_command_count, + tracked_status_count: metrics.tracked_status_count, + full_root_file_count: metrics.full_root_file_count, + } + } +} + +pub(crate) fn validate_git_publication_state(expected_head: &str, state: &GitState) -> Result<()> { + if state.head.as_deref() != Some(expected_head) { + return Err(Error::GitHeadChanged(format!( + "expected Git HEAD `{expected_head}`, found `{}`", + state.head.as_deref().unwrap_or("") + ))); + } + if state.dirty { + return Err(Error::GitWorktreeDirty( + "current Git worktree has tracked changes; commit, stash, or revert them before `trail agent apply`" + .to_string(), + )); + } + Ok(()) +} + +#[derive(Debug, Default)] +pub(crate) struct GitTreeNode { + blobs: BTreeMap, + dirs: BTreeMap, +} + +#[derive(Debug)] +pub(crate) struct GitBlobEntry { + mode: &'static str, + oid: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ConflictTake { + Source, + Target, +} + +#[derive(Debug)] +pub(crate) enum ConflictResolution { + Take(ConflictTake), + Manual(ConflictManualResolution), +} + +#[derive(Debug)] +pub(crate) struct WorkspaceLock { + path: PathBuf, + owner_record: String, + candidate_residue: Option, + _owner_exclusion: File, + _schema_exclusion: File, + observer_coordination: Option, + observer_write_exclusion: Option, +} + +#[derive(Default)] +struct AuthorizedObserverExclusionEntry { + active: usize, + observer_lock_active: bool, + release_generation: u64, +} + +#[derive(Default)] +struct AuthorizedObserverExclusionState { + entries: HashMap, +} + +static AUTHORIZED_OBSERVER_EXCLUSIONS: OnceLock<( + Mutex, + Condvar, +)> = OnceLock::new(); + +#[derive(Debug)] +pub(crate) struct AuthorizedObserverWriteExclusion { + db_dir: PathBuf, +} + +pub(crate) fn begin_authorized_observer_write_exclusion( + db_dir: &Path, +) -> AuthorizedObserverWriteExclusion { + let (state, _) = AUTHORIZED_OBSERVER_EXCLUSIONS.get_or_init(|| { + ( + Mutex::new(AuthorizedObserverExclusionState::default()), + Condvar::new(), + ) + }); + let mut state = state.lock().unwrap_or_else(|poison| poison.into_inner()); + state + .entries + .entry(db_dir.to_path_buf()) + .or_default() + .active += 1; + AuthorizedObserverWriteExclusion { + db_dir: db_dir.to_path_buf(), + } +} + +fn wait_for_observer_workspace_lock(db_dir: &Path) { + let (state, changed) = AUTHORIZED_OBSERVER_EXCLUSIONS.get_or_init(|| { + ( + Mutex::new(AuthorizedObserverExclusionState::default()), + Condvar::new(), + ) + }); + let mut state = state.lock().unwrap_or_else(|poison| poison.into_inner()); + while state + .entries + .entry(db_dir.to_path_buf()) + .or_default() + .observer_lock_active + { + state = changed + .wait(state) + .unwrap_or_else(|poison| poison.into_inner()); + } +} + +/// Active observer durability participates in an explicit same-process +/// handoff with observed-record publication. It waits without an arbitrary +/// timeout only while that exact authorized handoff is live; unrelated writer +/// contention and every durability/integrity failure remain terminal. +pub(crate) fn acquire_workspace_lock_for_observer( + db_dir: &Path, + schema_path: &Path, +) -> Result { + let coordination = AUTHORIZED_OBSERVER_EXCLUSIONS.get_or_init(|| { + ( + Mutex::new(AuthorizedObserverExclusionState::default()), + Condvar::new(), + ) + }); + loop { + let (generation, mut reservation) = { + let (state, changed) = coordination; + let mut state = state.lock().unwrap_or_else(|poison| poison.into_inner()); + loop { + let entry = state.entries.entry(db_dir.to_path_buf()).or_default(); + if entry.active == 0 && !entry.observer_lock_active { + entry.observer_lock_active = true; + break ( + entry.release_generation, + ObserverWorkspaceLockReservation { + db_dir: db_dir.to_path_buf(), + armed: true, + }, + ); + } + state = changed + .wait(state) + .unwrap_or_else(|poison| poison.into_inner()); + } + }; + match acquire_workspace_lock_for_database(db_dir, schema_path) { + Ok(mut lock) => { + lock.observer_coordination = Some(reservation.disarm()); + return Ok(lock); + } + Err(error @ Error::WorkspaceLocked(_)) => { + drop(reservation); + let (state, _) = coordination; + let mut state = state.lock().unwrap_or_else(|poison| poison.into_inner()); + let entry = state.entries.entry(db_dir.to_path_buf()).or_default(); + if entry.active > 0 || entry.release_generation != generation { + continue; + } + return Err(error); + } + Err(error) => { + drop(reservation); + return Err(error); + } + } + } +} + +struct ObserverWorkspaceLockReservation { + db_dir: PathBuf, + armed: bool, +} + +impl ObserverWorkspaceLockReservation { + fn disarm(&mut self) -> PathBuf { + self.armed = false; + self.db_dir.clone() + } +} + +impl Drop for ObserverWorkspaceLockReservation { + fn drop(&mut self) { + if self.armed { + release_observer_workspace_lock(&self.db_dir); + } + } +} + +fn release_observer_workspace_lock(db_dir: &Path) { + let Some((state, changed)) = AUTHORIZED_OBSERVER_EXCLUSIONS.get() else { + return; + }; + let mut state = state.lock().unwrap_or_else(|poison| poison.into_inner()); + let entry = state.entries.entry(db_dir.to_path_buf()).or_default(); + debug_assert!(entry.observer_lock_active); + entry.observer_lock_active = false; + changed.notify_all(); +} + +pub(crate) fn acquire_workspace_lock(db_dir: &Path) -> Result { + acquire_workspace_lock_for_database(db_dir, &db_dir.join(DB_RELATIVE_PATH)) +} + +pub(crate) fn acquire_workspace_lock_for_database( + db_dir: &Path, + schema_path: &Path, +) -> Result { + let path = db_dir.join("lock"); + let mut delay = Duration::from_millis(2); + cleanup_abandoned_workspace_lock_candidates(db_dir)?; + let owner = WorkspaceLockOwner::current()?; + let publication; + loop { + match publish_workspace_lock(db_dir, &path, &owner) { + Ok(value) => { + publication = value; + break; + } + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { + let (reaped, holder) = inspect_existing_workspace_lock(&path)?; + if reaped { + continue; + } + let should_wait = WRITE_LOCK_WAIT_DEADLINE + .with(|deadline| deadline.get()) + .is_some_and(|deadline| Instant::now() < deadline); + if should_wait { + std::thread::sleep(delay); + delay = (delay * 2).min(Duration::from_millis(50)); + continue; + } + return Err(Error::WorkspaceLocked(holder.trim().to_string())); + } + Err(err) => return Err(Error::Io(err)), + } + } + let WorkspaceLockPublication { + file: owner_exclusion, + candidate_residue, + } = publication; + let exclusion_path = schema_exclusion_path(db_dir, schema_path); + let schema_exclusion = match OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(exclusion_path) + { + Ok(file) => file, + Err(error) => { + cleanup_owned_workspace_lock( + &path, + candidate_residue.as_deref(), + &owner.encode(), + &owner_exclusion, + ); + return Err(Error::Io(error)); + } + }; + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + let mut delay = Duration::from_millis(2); + let reader_drain_deadline = Instant::now() + Duration::from_secs(1); + loop { + match rustix::fs::flock( + &schema_exclusion, + rustix::fs::FlockOperation::NonBlockingLockExclusive, + ) { + Ok(()) => break, + Err(error) + if error == rustix::io::Errno::AGAIN + && (schema_lock_waiting_is_enabled() + || Instant::now() < reader_drain_deadline) => + { + std::thread::sleep(delay); + delay = (delay * 2).min(Duration::from_millis(50)); + } + Err(error) if error == rustix::io::Errno::AGAIN => { + cleanup_owned_workspace_lock( + &path, + candidate_residue.as_deref(), + &owner.encode(), + &owner_exclusion, + ); + return Err(Error::WorkspaceLocked( + "workspace schema reader is active".into(), + )); + } + Err(error) => { + cleanup_owned_workspace_lock( + &path, + candidate_residue.as_deref(), + &owner.encode(), + &owner_exclusion, + ); + return Err(Error::Io(error.into())); + } + } + } + } + Ok(WorkspaceLock { + path, + owner_record: owner.encode(), + candidate_residue, + _owner_exclusion: owner_exclusion, + _schema_exclusion: schema_exclusion, + observer_coordination: None, + observer_write_exclusion: None, + }) +} + +impl Drop for WorkspaceLock { + fn drop(&mut self) { + cleanup_owned_workspace_lock( + &self.path, + self.candidate_residue.as_deref(), + &self.owner_record, + &self._owner_exclusion, + ); + if let Some(db_dir) = self.observer_coordination.take() { + release_observer_workspace_lock(&db_dir); + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct WorkspaceLockOwner { + pid: u32, + process_start_identity: String, + nonce: String, + created_at: i64, +} + +#[cfg(test)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum WorkspaceLockPublishPhase { + BeforePublish, + AfterPublish, +} + +#[cfg(test)] +static WORKSPACE_LOCK_PUBLISH_CRASHES: OnceLock< + Mutex>, +> = OnceLock::new(); +#[cfg(test)] +static WORKSPACE_LOCK_CANDIDATE_UNLINK_FAILURES: OnceLock>> = + OnceLock::new(); +#[cfg(test)] +static WORKSPACE_LOCK_ROLLBACK_UNLINK_FAILURES: OnceLock>> = OnceLock::new(); + +#[cfg(test)] +fn crash_next_workspace_lock_publish(lock_path: &Path, phase: WorkspaceLockPublishPhase) { + WORKSPACE_LOCK_PUBLISH_CRASHES + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .insert(lock_path.to_path_buf(), phase); +} + +#[cfg(test)] +fn run_workspace_lock_publish_crash_hook(lock_path: &Path, phase: WorkspaceLockPublishPhase) { + let crashes = WORKSPACE_LOCK_PUBLISH_CRASHES.get_or_init(|| Mutex::new(HashMap::new())); + let should_crash = { + let mut crashes = crashes.lock().unwrap_or_else(|poison| poison.into_inner()); + if crashes.get(lock_path) == Some(&phase) { + crashes.remove(lock_path); + true + } else { + false + } + }; + if should_crash { + panic!("simulated workspace-lock crash at {phase:?}"); + } +} + +#[cfg(test)] +fn fail_next_workspace_lock_candidate_unlink(lock_path: &Path) { + WORKSPACE_LOCK_CANDIDATE_UNLINK_FAILURES + .get_or_init(|| Mutex::new(HashSet::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .insert(lock_path.to_path_buf()); +} + +#[cfg(test)] +fn fail_next_workspace_lock_rollback_unlink(lock_path: &Path) { + WORKSPACE_LOCK_ROLLBACK_UNLINK_FAILURES + .get_or_init(|| Mutex::new(HashSet::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .insert(lock_path.to_path_buf()); +} + +#[cfg(test)] +fn workspace_lock_rollback_unlink_result(lock_path: &Path) -> std::io::Result<()> { + let fail = WORKSPACE_LOCK_ROLLBACK_UNLINK_FAILURES + .get_or_init(|| Mutex::new(HashSet::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .remove(lock_path); + if fail { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected workspace-lock rollback unlink failure", + )) + } else { + fs::remove_file(lock_path) + } +} + +#[cfg(not(test))] +fn workspace_lock_rollback_unlink_result(lock_path: &Path) -> std::io::Result<()> { + fs::remove_file(lock_path) +} + +#[cfg(test)] +fn workspace_lock_candidate_unlink_result( + lock_path: &Path, + candidate: &Path, +) -> std::io::Result<()> { + let fail = WORKSPACE_LOCK_CANDIDATE_UNLINK_FAILURES + .get_or_init(|| Mutex::new(HashSet::new())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .remove(lock_path); + if fail { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected workspace-lock candidate unlink failure", + )) + } else { + fs::remove_file(candidate) + } +} + +#[cfg(not(test))] +fn workspace_lock_candidate_unlink_result( + _lock_path: &Path, + candidate: &Path, +) -> std::io::Result<()> { + fs::remove_file(candidate) +} + +impl WorkspaceLockOwner { + fn current() -> std::io::Result { + let mut nonce = [0_u8; 32]; + getrandom::getrandom(&mut nonce).map_err(|error| { + std::io::Error::new( + std::io::ErrorKind::Other, + format!("failed to generate workspace-lock nonce: {error}"), + ) + })?; + Ok(Self { + pid: std::process::id(), + process_start_identity: self::util::current_process_start_token(), + nonce: hex::encode(nonce), + created_at: self::util::now_ts(), + }) + } + + fn encode(&self) -> String { + format!( + "trail-workspace-lock-v1\npid={}\nprocess_start_identity={}\nnonce={}\ncreated_at={}\n", + self.pid, + hex::encode(self.process_start_identity.as_bytes()), + self.nonce, + self.created_at + ) + } + + fn parse(record: &str) -> Option { + let mut lines = record.lines(); + if lines.next()? != "trail-workspace-lock-v1" { + return None; + } + let pid = lines.next()?.strip_prefix("pid=")?.parse().ok()?; + let process_start_identity = String::from_utf8( + hex::decode(lines.next()?.strip_prefix("process_start_identity=")?).ok()?, + ) + .ok()?; + let nonce = lines.next()?.strip_prefix("nonce=")?.to_owned(); + let created_at = lines.next()?.strip_prefix("created_at=")?.parse().ok()?; + if pid == 0 + || process_start_identity.is_empty() + || nonce.len() != 64 + || !nonce.bytes().all(|byte| byte.is_ascii_hexdigit()) + || lines.next().is_some() + { + return None; + } + Some(Self { + pid, + process_start_identity, + nonce, + created_at, + }) + } +} + +fn create_workspace_lock_candidate( + db_dir: &Path, + owner: &WorkspaceLockOwner, +) -> std::io::Result<(PathBuf, File)> { + let candidate = db_dir.join(format!(".workspace-lock-candidate-{}", owner.nonce)); + let mut options = OpenOptions::new(); + options.read(true).write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let result = (|| { + let mut file = options.open(&candidate)?; + file.write_all(owner.encode().as_bytes())?; + // The record must be complete before its inode becomes authoritative. + // The directory is intentionally not fsynced: this lock is ephemeral, + // so losing either a publication or an unlink across a machine crash is + // safe (no owner survives, and a reappearing record is reaped). Avoiding + // directory barriers also keeps every workspace command off that hot + // durability path. + file.sync_all()?; + #[cfg(any(target_os = "linux", target_os = "macos"))] + rustix::fs::flock(&file, rustix::fs::FlockOperation::LockExclusive) + .map_err(std::io::Error::from)?; + Ok(file) + })(); + match result { + Ok(file) => Ok((candidate, file)), + Err(error) => { + let _ = fs::remove_file(&candidate); + Err(error) + } + } +} + +struct WorkspaceLockPublication { + file: File, + candidate_residue: Option, +} + +fn publish_workspace_lock( + db_dir: &Path, + lock_path: &Path, + owner: &WorkspaceLockOwner, +) -> std::io::Result { + let (candidate, file) = create_workspace_lock_candidate(db_dir, owner)?; + #[cfg(test)] + run_workspace_lock_publish_crash_hook(lock_path, WorkspaceLockPublishPhase::BeforePublish); + match fs::hard_link(&candidate, lock_path) { + Ok(()) => { + #[cfg(test)] + run_workspace_lock_publish_crash_hook( + lock_path, + WorkspaceLockPublishPhase::AfterPublish, + ); + match workspace_lock_candidate_unlink_result(lock_path, &candidate) { + Ok(()) => Ok(WorkspaceLockPublication { + file, + candidate_residue: None, + }), + Err(error) => { + let link_count = file.metadata().map(|metadata| { + #[cfg(unix)] + { + metadata.nlink() + } + #[cfg(not(unix))] + { + 2 + } + }); + if matches!(link_count, Ok(1)) { + // The original candidate name disappeared despite the + // unlink error (for example, a concurrent same-uid + // substitution). The published inode is already in its + // valid one-link state. + return Ok(WorkspaceLockPublication { + file, + candidate_residue: None, + }); + } + if rollback_workspace_lock_publication( + lock_path, + &candidate, + &owner.encode(), + &file, + ) { + Err(error) + } else { + // Returning an ordinary error here would drop the only + // flock capability while leaving a current-live-PID + // record. Keep ownership explicit so mutual exclusion + // remains fail-closed and Drop can retry exact cleanup. + Ok(WorkspaceLockPublication { + file, + candidate_residue: Some(candidate), + }) + } + } + } + } + Err(error) => { + let _ = fs::remove_file(&candidate); + Err(error) + } + } +} + +const MAX_WORKSPACE_LOCK_RECORD_BYTES: u64 = 4096; + +fn read_workspace_lock_record(file: &File) -> std::io::Result> { + let before = file.metadata()?; + if before.len() > MAX_WORKSPACE_LOCK_RECORD_BYTES { + return Ok(None); + } + let mut bytes = vec![0_u8; before.len() as usize]; + #[cfg(unix)] + { + use std::os::unix::fs::FileExt; + let mut offset = 0; + while offset < bytes.len() { + let read = file.read_at(&mut bytes[offset..], offset as u64)?; + if read == 0 { + return Ok(None); + } + offset += read; + } + } + #[cfg(not(unix))] + { + let mut clone = file.try_clone()?; + clone.seek(SeekFrom::Start(0))?; + clone.read_exact(&mut bytes)?; + } + let after = file.metadata()?; + if after.len() != before.len() { + return Ok(None); + } + #[cfg(unix)] + if after.dev() != before.dev() || after.ino() != before.ino() { + return Ok(None); + } + Ok(String::from_utf8(bytes).ok()) +} + +fn rollback_workspace_lock_publication( + lock_path: &Path, + candidate_path: &Path, + owner_record: &str, + owner_file: &File, +) -> bool { + #[cfg(unix)] + { + let Ok(descriptor) = owner_file.metadata() else { + return false; + }; + let Ok(lock) = fs::symlink_metadata(lock_path) else { + return false; + }; + let Ok(candidate) = fs::symlink_metadata(candidate_path) else { + return false; + }; + if !workspace_lock_metadata_is_private_regular(&descriptor) + || !workspace_lock_metadata_is_private_regular(&lock) + || !workspace_lock_metadata_is_private_regular(&candidate) + || descriptor.nlink() != 2 + || lock.nlink() != 2 + || candidate.nlink() != 2 + || lock.dev() != descriptor.dev() + || lock.ino() != descriptor.ino() + || candidate.dev() != descriptor.dev() + || candidate.ino() != descriptor.ino() + { + return false; + } + } + if read_workspace_lock_record(owner_file) + .ok() + .flatten() + .as_deref() + != Some(owner_record) + { + return false; + } + if workspace_lock_rollback_unlink_result(lock_path).is_ok() { + remove_workspace_lock_candidate_if_matches(candidate_path, owner_record, owner_file); + true + } else { + false + } +} + +fn cleanup_owned_workspace_lock( + lock_path: &Path, + candidate_residue: Option<&Path>, + owner_record: &str, + owner_file: &File, +) { + if let Some(candidate) = candidate_residue { + if rollback_workspace_lock_publication(lock_path, candidate, owner_record, owner_file) { + return; + } + } + remove_owned_workspace_lock(lock_path, owner_record, owner_file); +} + +fn remove_owned_workspace_lock(lock_path: &Path, owner_record: &str, owner_file: &File) { + #[cfg(unix)] + { + let Ok(descriptor) = owner_file.metadata() else { + return; + }; + let Ok(named) = fs::symlink_metadata(lock_path) else { + return; + }; + if !workspace_lock_metadata_is_private_regular(&descriptor) + || !workspace_lock_metadata_is_private_regular(&named) + || descriptor.nlink() != 1 + || named.nlink() != 1 + || descriptor.dev() != named.dev() + || descriptor.ino() != named.ino() + || read_workspace_lock_record(owner_file) + .ok() + .flatten() + .as_deref() + != Some(owner_record) + { + return; + } + } + #[cfg(not(unix))] + if read_workspace_lock_record(owner_file) + .ok() + .flatten() + .as_deref() + != Some(owner_record) + { + return; + } + // POSIX has no unlink-if-inode-matches primitive. The retained flock makes + // this conditional unlink safe among Trail processes; a hostile same-uid + // process could still swap the name in the final syscall window, but such a + // process can already delete or rewrite Trail's private state and is outside + // the workspace-lock threat boundary. + let _ = fs::remove_file(lock_path); +} + +fn inspect_existing_workspace_lock(lock_path: &Path) -> std::io::Result<(bool, String)> { + #[cfg(unix)] + { + let metadata = match fs::symlink_metadata(lock_path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok((true, String::new())); + } + Err(error) => return Err(error), + }; + if !workspace_lock_metadata_is_private_regular(&metadata) { + return Ok(( + false, + "workspace lock is not a private regular file owned by this user".into(), + )); + } + if metadata.len() > MAX_WORKSPACE_LOCK_RECORD_BYTES { + return Ok(( + false, + "workspace lock record exceeds the safe size limit".into(), + )); + } + } + let mut options = OpenOptions::new(); + options.read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW); + } + let file = match options.open(lock_path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok((true, String::new())); + } + Err(error) => { + return Ok((false, format!("unreadable workspace lock: {error}"))); + } + }; -#[derive(Debug, Default)] -pub(crate) struct GitTreeNode { - blobs: BTreeMap, - dirs: BTreeMap, + #[cfg(any(target_os = "linux", target_os = "macos"))] + if let Err(error) = + rustix::fs::flock(&file, rustix::fs::FlockOperation::NonBlockingLockExclusive) + { + if error == rustix::io::Errno::AGAIN { + return Ok(( + false, + read_workspace_lock_record(&file)?.unwrap_or_else(|| { + "workspace lock record is invalid or exceeds the safe size limit".into() + }), + )); + } + return Err(error.into()); + } + + let Some(holder) = read_workspace_lock_record(&file)? else { + return Ok(( + false, + "workspace lock record is invalid or exceeds the safe size limit".into(), + )); + }; + let Some(owner) = WorkspaceLockOwner::parse(&holder) else { + return Ok((false, holder)); + }; + if self::util::process_matches_start_token(owner.pid, &owner.process_start_identity) { + return Ok((false, holder)); + } + + #[cfg(unix)] + { + let descriptor = file.metadata()?; + let named = match fs::symlink_metadata(lock_path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok((true, String::new())); + } + Err(error) => return Err(error), + }; + if descriptor.dev() != named.dev() || descriptor.ino() != named.ino() { + return Ok(( + false, + "workspace lock pathname changed during validation".into(), + )); + } + if !workspace_lock_metadata_is_private_regular(&descriptor) + || !workspace_lock_link_topology_is_valid(lock_path, &descriptor, &owner) + { + return Ok(( + false, + "workspace lock has an untrusted hard-link topology".into(), + )); + } + } + if read_workspace_lock_record(&file)?.as_deref() != Some(&holder) { + return Ok((false, "workspace lock changed during validation".into())); + } + + match fs::remove_file(lock_path) { + Ok(()) => { + remove_workspace_lock_candidate_if_matches( + &lock_path.with_file_name(format!(".workspace-lock-candidate-{}", owner.nonce)), + &holder, + &file, + ); + Ok((true, holder)) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok((true, holder)), + Err(error) => Err(error), + } } -#[derive(Debug)] -pub(crate) struct GitBlobEntry { - mode: &'static str, - oid: String, +fn remove_workspace_lock_candidate_if_matches( + candidate_path: &Path, + owner_record: &str, + owner_file: &File, +) { + #[cfg(unix)] + { + let Ok(descriptor) = owner_file.metadata() else { + return; + }; + let Ok(named) = fs::symlink_metadata(candidate_path) else { + return; + }; + if !workspace_lock_metadata_is_private_regular(&descriptor) + || !workspace_lock_metadata_is_private_regular(&named) + || descriptor.nlink() != 1 + || named.nlink() != 1 + || descriptor.dev() != named.dev() + || descriptor.ino() != named.ino() + { + return; + } + } + if read_workspace_lock_record(owner_file) + .ok() + .flatten() + .as_deref() + != Some(owner_record) + { + return; + } + let _ = fs::remove_file(candidate_path); } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ConflictTake { - Source, - Target, +#[cfg(unix)] +fn workspace_lock_metadata_is_private_regular(metadata: &fs::Metadata) -> bool { + metadata.file_type().is_file() + && metadata.mode() & 0o777 == 0o600 + // SAFETY: geteuid has no preconditions and does not dereference memory. + && metadata.uid() == unsafe { libc::geteuid() } } -#[derive(Debug)] -pub(crate) enum ConflictResolution { - Take(ConflictTake), - Manual(ConflictManualResolution), +#[cfg(unix)] +fn workspace_lock_link_topology_is_valid( + lock_path: &Path, + metadata: &fs::Metadata, + owner: &WorkspaceLockOwner, +) -> bool { + match metadata.nlink() { + 1 => true, + 2 => { + let Some(parent) = lock_path.parent() else { + return false; + }; + let candidate = parent.join(format!(".workspace-lock-candidate-{}", owner.nonce)); + fs::symlink_metadata(candidate).is_ok_and(|candidate| { + workspace_lock_metadata_is_private_regular(&candidate) + && candidate.dev() == metadata.dev() + && candidate.ino() == metadata.ino() + && candidate.nlink() == 2 + }) + } + _ => false, + } } -#[derive(Debug)] -pub(crate) struct WorkspaceLock { - path: PathBuf, +fn cleanup_abandoned_workspace_lock_candidates(db_dir: &Path) -> std::io::Result<()> { + const MAX_DIRECTORY_ENTRIES: usize = 4096; + let entries = fs::read_dir(db_dir)?; + for (index, entry) in entries.enumerate() { + if index >= MAX_DIRECTORY_ENTRIES { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "workspace lock directory exceeds the bounded cleanup scan; remove unexpected entries and retry", + )); + } + let entry = entry?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + let Some(nonce) = name.strip_prefix(".workspace-lock-candidate-") else { + continue; + }; + if nonce.len() != 64 || !nonce.bytes().all(|byte| byte.is_ascii_hexdigit()) { + continue; + } + let path = entry.path(); + #[cfg(unix)] + { + let Ok(named) = fs::symlink_metadata(&path) else { + continue; + }; + if !workspace_lock_metadata_is_private_regular(&named) || named.nlink() != 1 { + continue; + } + } + let mut options = OpenOptions::new(); + options.read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW); + } + let Ok(file) = options.open(&path) else { + continue; + }; + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + let mut locked = false; + for _ in 0..3 { + if rustix::fs::flock(&file, rustix::fs::FlockOperation::NonBlockingLockExclusive) + .is_ok() + { + locked = true; + break; + } + std::thread::yield_now(); + } + if !locked { + continue; + } + } + let Some(record) = read_workspace_lock_record(&file)? else { + continue; + }; + let Some(owner) = WorkspaceLockOwner::parse(&record) else { + continue; + }; + if owner.nonce != nonce + || self::util::process_matches_start_token(owner.pid, &owner.process_start_identity) + { + continue; + } + #[cfg(unix)] + { + let Ok(metadata) = file.metadata() else { + continue; + }; + let Ok(named) = fs::symlink_metadata(&path) else { + continue; + }; + if !workspace_lock_metadata_is_private_regular(&metadata) + || metadata.nlink() != 1 + || named.dev() != metadata.dev() + || named.ino() != metadata.ino() + { + continue; + } + } + if read_workspace_lock_record(&file)?.as_deref() != Some(&record) { + continue; + } + let _ = fs::remove_file(path); + } + Ok(()) } -impl Drop for WorkspaceLock { +impl Drop for AuthorizedObserverWriteExclusion { fn drop(&mut self) { - let _ = fs::remove_file(&self.path); + let Some((state, changed)) = AUTHORIZED_OBSERVER_EXCLUSIONS.get() else { + return; + }; + let mut state = state.lock().unwrap_or_else(|poison| poison.into_inner()); + let entry = state.entries.entry(self.db_dir.clone()).or_default(); + debug_assert!(entry.active > 0); + entry.active = entry.active.saturating_sub(1); + if entry.active == 0 { + entry.release_generation = entry.release_generation.wrapping_add(1); + changed.notify_all(); + } } } @@ -846,18 +4045,865 @@ impl Drop for WriteLockWaitGuard { } mod agent; +mod change_ledger; +#[cfg(debug_assertions)] +pub(crate) use change_ledger::run_command_flow; +#[cfg(debug_assertions)] +pub(crate) use change_ledger::run_command_long_lock_flow; +#[cfg(debug_assertions)] +pub(crate) use change_ledger::run_materialized_candidate_lifecycle_flow; +#[cfg(debug_assertions)] +pub(crate) use change_ledger::run_materialized_lane_snapshot_flow; +#[cfg(all(debug_assertions, unix))] +pub(crate) use change_ledger::run_non_utf_database_path_mark_recover_and_retire; +#[cfg(debug_assertions)] +#[allow(unused_imports)] +pub(crate) use change_ledger::set_command_authority_override; +pub(crate) use change_ledger::{ + command_authority_enabled, prepare_workspace_daemon_launch, + verified_stale_workspace_owner_for_launch, workspace_daemon_fence, + workspace_daemon_ready_proof, workspace_daemon_reconcile, VerifiedStaleWorkspaceOwner, + WorkspaceDaemonLaunchIdentity, WorkspaceDaemonProof, +}; +#[cfg(debug_assertions)] +pub(crate) use change_ledger::{ + ledger_authority_enabled_for, prepare_workspace_daemon, ActivationEvidence, + LEDGER_AUTHORITY_ENABLED, +}; +pub(crate) use change_ledger::{ + materialized_lane_daemon_full_reconcile, workspace_daemon_full_reconcile, +}; +#[cfg(debug_assertions)] +pub(crate) use change_ledger::{ + run_acknowledgement_race, run_advanced_prefix_recovery, run_ambiguous_recovery_gate, + run_backup_overwrite_rollback, run_backup_restore_rotation, run_callback_spool, + run_crash_matrix, run_deletion_normal_retry_idempotence, + run_deletion_parent_substitution_rejection, + run_deletion_post_quarantine_verification_substitution_rejection, + run_deletion_post_verification_substitution_rejection, + run_deletion_quiesced_missing_quarantine_rejection, + run_deletion_quiesced_reappeared_original_rejection, + run_deletion_retry_hostile_quarantine_replacement_rejection, + run_exact_interval_bridge_rejection, run_gc_root_lifecycle, run_lane_deletion_retirement, + run_missing_sidecar_rejection, run_oracle, run_prefix_interval_bridge_rejection, + run_qualified_proof_revalidation, run_races, run_restored_nullable_provider_lane_deletion, + run_retained_writer_quiescence, run_retirement_barrier, run_valid_prefix_interval_recovery, +}; +#[cfg(all(debug_assertions, target_os = "linux"))] +pub(crate) use change_ledger::{ + run_authenticated_fence_rejections, run_complete_prefix_publication_races, + run_content_mode_create_delete, run_controlled_fence_queue_ordering, run_delayed_backlog, + run_fault_revocation_matrix, run_fence_ordering, run_owner_death_and_root_replacement, + run_policy_dependency_observation, run_process_owner_child, run_raw_decoder_faults, + run_reconciliation_interval_qualification, run_recursive_coverage, run_rename_matrix, + run_rename_storm_and_cookie_expiry, run_segment_writer_reconcile_publication, + run_unsupported_filesystem_rejection, +}; +#[cfg(all(debug_assertions, unix))] +pub(crate) use change_ledger::{ + run_deletion_leaf_substitution_rejection, run_mark_ancestor_substitution_rejection, + run_recovery_ancestor_substitution_rejection, +}; +#[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] +pub(crate) use change_ledger::{ + run_empty_orphan_quarantine_rejection, run_no_orphan_quarantine_allocation, + run_orphan_quarantine_substitution_rejection, +}; +#[cfg(all(debug_assertions, target_os = "macos"))] +pub(crate) use change_ledger::{ + run_macos_continuity_fault_matrix, run_macos_fence_ordering, run_macos_gap_flag_matrix, + run_macos_history_authority, run_macos_malformed_callbacks, run_macos_null_context_generation, + run_macos_paused_callback_fence, run_macos_real_apfs_file_events, + run_macos_root_revalidation_failures, run_macos_startup_cancellation, + run_macos_unsupported_filesystem_rejection, run_macos_uuid_revalidation, +}; mod core; mod lane; +#[cfg(debug_assertions)] +pub(crate) use lane::{ + install_lane_record_after_c2_write_for_current_thread, run_changed_path_view_flow, + set_lane_record_postcommit_failure_for_current_thread, + set_sparse_selection_write_failure_for_current_thread, +}; mod merge; +mod performance; mod record; +#[cfg(debug_assertions)] +pub(crate) use record::{ + install_observed_record_after_compare_hook, install_observed_record_with_lock_hook, +}; mod storage; +use self::performance::*; +#[cfg(debug_assertions)] +pub(crate) use storage::{ + install_git_qualification_after_c2_hook, install_git_qualification_after_porcelain_hook, +}; +pub(crate) use storage::{observed_exact_paths_for_candidates, ObservedPathKind}; mod util; +#[doc(hidden)] +pub use self::util::process_liveness::run_internal_process_watchdog; +pub(crate) use self::util::{redact_sensitive_json, redact_sensitive_text}; + #[cfg(test)] mod tests { use super::util::*; use super::*; + fn dead_workspace_lock_owner(nonce_byte: u8) -> WorkspaceLockOwner { + WorkspaceLockOwner { + pid: u32::MAX, + process_start_identity: "definitely-not-a-live-process".into(), + nonce: format!("{nonce_byte:02x}").repeat(32), + created_at: 1, + } + } + + #[test] + fn workspace_lock_crash_before_atomic_publish_leaves_no_lock_name() { + let temp = tempfile::tempdir().unwrap(); + let owner = dead_workspace_lock_owner(0x11); + let lock_path = temp.path().join("lock"); + crash_next_workspace_lock_publish(&lock_path, WorkspaceLockPublishPhase::BeforePublish); + + let crashed = std::panic::catch_unwind(|| { + publish_workspace_lock(temp.path(), &lock_path, &owner).unwrap(); + }); + + assert!(crashed.is_err()); + assert!(!lock_path.exists()); + let candidate = temp + .path() + .join(format!(".workspace-lock-candidate-{}", owner.nonce)); + assert_eq!( + WorkspaceLockOwner::parse(&fs::read_to_string(&candidate).unwrap()), + Some(owner.clone()) + ); + #[cfg(unix)] + { + let metadata = fs::symlink_metadata(&candidate).unwrap(); + assert!(workspace_lock_metadata_is_private_regular(&metadata)); + assert_eq!(metadata.nlink(), 1); + } + assert!(!self::util::process_matches_start_token( + owner.pid, + &owner.process_start_identity + )); + let lock = + acquire_workspace_lock_for_database(temp.path(), &temp.path().join("trail.sqlite")) + .unwrap(); + assert!(!candidate.exists()); + drop(lock); + } + + #[test] + fn workspace_lock_crash_after_atomic_publish_is_reaped_by_verified_identity() { + let temp = tempfile::tempdir().unwrap(); + let lock_path = temp.path().join("lock"); + let owner = dead_workspace_lock_owner(0x22); + crash_next_workspace_lock_publish(&lock_path, WorkspaceLockPublishPhase::AfterPublish); + let crashed = std::panic::catch_unwind(|| { + publish_workspace_lock(temp.path(), &lock_path, &owner).unwrap(); + }); + assert!(crashed.is_err()); + let candidate = temp + .path() + .join(format!(".workspace-lock-candidate-{}", owner.nonce)); + assert!(candidate.exists()); + assert_eq!(fs::read_to_string(&lock_path).unwrap(), owner.encode()); + + let lock = + acquire_workspace_lock_for_database(temp.path(), &temp.path().join("trail.sqlite")) + .unwrap(); + + let replacement = WorkspaceLockOwner::parse(&fs::read_to_string(&lock_path).unwrap()) + .expect("replacement lock has an authenticated owner record"); + assert_ne!(replacement.nonce, owner.nonce); + assert_eq!(replacement.pid, std::process::id()); + assert!(!candidate.exists()); + drop(lock); + assert!(!lock_path.exists()); + } + + #[test] + fn workspace_lock_atomic_publication_has_one_winner_under_parallel_contention() { + use std::sync::atomic::AtomicUsize; + use std::sync::Barrier; + + const CONTENDERS: usize = 16; + let temp = tempfile::tempdir().unwrap(); + let db_dir = Arc::new(temp.path().to_path_buf()); + let start = Arc::new(Barrier::new(CONTENDERS + 1)); + let acquired = Arc::new(Barrier::new(CONTENDERS + 1)); + let release = Arc::new(Barrier::new(CONTENDERS + 1)); + let winners = Arc::new(AtomicUsize::new(0)); + let mut threads = Vec::new(); + for _ in 0..CONTENDERS { + let db_dir = Arc::clone(&db_dir); + let start = Arc::clone(&start); + let acquired = Arc::clone(&acquired); + let release = Arc::clone(&release); + let winners = Arc::clone(&winners); + threads.push(std::thread::spawn(move || { + start.wait(); + let result = + acquire_workspace_lock_for_database(&db_dir, &db_dir.join("trail.sqlite")); + if result.is_ok() { + winners.fetch_add(1, Ordering::SeqCst); + } else { + assert!(matches!(result, Err(Error::WorkspaceLocked(_)))); + } + acquired.wait(); + release.wait(); + drop(result); + })); + } + + start.wait(); + acquired.wait(); + assert_eq!(winners.load(Ordering::SeqCst), 1); + release.wait(); + for thread in threads { + thread.join().unwrap(); + } + assert!(!temp.path().join("lock").exists()); + } + + #[test] + fn legacy_empty_workspace_lock_fails_closed() { + let temp = tempfile::tempdir().unwrap(); + let lock_path = temp.path().join("lock"); + File::create(&lock_path).unwrap(); + + let result = + acquire_workspace_lock_for_database(temp.path(), &temp.path().join("trail.sqlite")); + + assert!(matches!(result, Err(Error::WorkspaceLocked(_)))); + assert!(lock_path.exists()); + } + + #[cfg(unix)] + #[test] + fn workspace_lock_candidate_cleanup_retains_hostile_symlink_type_mode_and_hardlink() { + use std::os::unix::fs::{symlink, PermissionsExt}; + + let temp = tempfile::tempdir().unwrap(); + + let symlink_owner = dead_workspace_lock_owner(0x31); + let symlink_victim = temp.path().join("symlink-victim"); + fs::write(&symlink_victim, symlink_owner.encode()).unwrap(); + let symlink_candidate = temp + .path() + .join(format!(".workspace-lock-candidate-{}", symlink_owner.nonce)); + symlink(&symlink_victim, &symlink_candidate).unwrap(); + + let mode_owner = dead_workspace_lock_owner(0x32); + let mode_candidate = temp + .path() + .join(format!(".workspace-lock-candidate-{}", mode_owner.nonce)); + fs::write(&mode_candidate, mode_owner.encode()).unwrap(); + fs::set_permissions(&mode_candidate, fs::Permissions::from_mode(0o644)).unwrap(); + + let type_owner = dead_workspace_lock_owner(0x33); + let type_candidate = temp + .path() + .join(format!(".workspace-lock-candidate-{}", type_owner.nonce)); + fs::create_dir(&type_candidate).unwrap(); + + let hardlink_owner = dead_workspace_lock_owner(0x34); + let hardlink_victim = temp.path().join("hardlink-victim"); + fs::write(&hardlink_victim, hardlink_owner.encode()).unwrap(); + fs::set_permissions(&hardlink_victim, fs::Permissions::from_mode(0o600)).unwrap(); + let hardlink_candidate = temp.path().join(format!( + ".workspace-lock-candidate-{}", + hardlink_owner.nonce + )); + fs::hard_link(&hardlink_victim, &hardlink_candidate).unwrap(); + + let lock = + acquire_workspace_lock_for_database(temp.path(), &temp.path().join("trail.sqlite")) + .unwrap(); + + assert!(symlink_candidate + .symlink_metadata() + .unwrap() + .file_type() + .is_symlink()); + assert!(symlink_victim.exists()); + assert!(mode_candidate.exists()); + assert!(type_candidate.is_dir()); + assert!(hardlink_candidate.exists()); + assert!(hardlink_victim.exists()); + drop(lock); + } + + #[cfg(unix)] + #[test] + fn workspace_lock_hostile_symlink_type_mode_and_hardlink_fail_closed() { + use std::os::unix::fs::{symlink, PermissionsExt}; + + for kind in ["symlink", "directory", "mode", "hardlink"] { + let temp = tempfile::tempdir().unwrap(); + let lock_path = temp.path().join("lock"); + let owner = dead_workspace_lock_owner(match kind { + "symlink" => 0x41, + "directory" => 0x42, + "mode" => 0x43, + _ => 0x44, + }); + match kind { + "symlink" => { + let victim = temp.path().join("victim"); + fs::write(&victim, owner.encode()).unwrap(); + symlink(victim, &lock_path).unwrap(); + } + "directory" => fs::create_dir(&lock_path).unwrap(), + "mode" => { + fs::write(&lock_path, owner.encode()).unwrap(); + fs::set_permissions(&lock_path, fs::Permissions::from_mode(0o644)).unwrap(); + } + "hardlink" => { + let victim = temp.path().join("victim"); + fs::write(&victim, owner.encode()).unwrap(); + fs::set_permissions(&victim, fs::Permissions::from_mode(0o600)).unwrap(); + fs::hard_link(victim, &lock_path).unwrap(); + } + _ => unreachable!(), + } + + let result = + acquire_workspace_lock_for_database(temp.path(), &temp.path().join("trail.sqlite")); + + assert!( + matches!(result, Err(Error::WorkspaceLocked(_))), + "hostile {kind} lock must fail closed: {result:?}" + ); + assert!(lock_path.symlink_metadata().is_ok()); + } + } + + #[cfg(unix)] + #[test] + fn workspace_lock_oversized_record_fails_closed_without_unbounded_read() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let lock_path = temp.path().join("lock"); + fs::write( + &lock_path, + vec![b'x'; MAX_WORKSPACE_LOCK_RECORD_BYTES as usize + 1], + ) + .unwrap(); + fs::set_permissions(&lock_path, fs::Permissions::from_mode(0o600)).unwrap(); + + let result = + acquire_workspace_lock_for_database(temp.path(), &temp.path().join("trail.sqlite")); + + assert!( + matches!(result, Err(Error::WorkspaceLocked(message)) if message.contains("size limit")) + ); + assert_eq!( + fs::metadata(lock_path).unwrap().len(), + MAX_WORKSPACE_LOCK_RECORD_BYTES + 1 + ); + } + + #[test] + fn workspace_lock_owner_unlink_does_not_remove_substituted_successor() { + let temp = tempfile::tempdir().unwrap(); + let lock_path = temp.path().join("lock"); + let owner = WorkspaceLockOwner::current().unwrap(); + let publication = publish_workspace_lock(temp.path(), &lock_path, &owner).unwrap(); + fs::remove_file(&lock_path).unwrap(); + let successor = WorkspaceLockOwner::current().unwrap(); + let successor_publication = + publish_workspace_lock(temp.path(), &lock_path, &successor).unwrap(); + + remove_owned_workspace_lock(&lock_path, &owner.encode(), &publication.file); + + assert_eq!(fs::read_to_string(&lock_path).unwrap(), successor.encode()); + remove_owned_workspace_lock(&lock_path, &successor.encode(), &successor_publication.file); + assert!(!lock_path.exists()); + } + + #[test] + fn workspace_lock_candidate_unlink_failure_rolls_back_and_allows_same_process_reacquire() { + let temp = tempfile::tempdir().unwrap(); + let lock_path = temp.path().join("lock"); + fail_next_workspace_lock_candidate_unlink(&lock_path); + + let failed = + acquire_workspace_lock_for_database(temp.path(), &temp.path().join("trail.sqlite")); + + assert!( + matches!(failed, Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::PermissionDenied) + ); + assert!(!lock_path.exists()); + assert!(!fs::read_dir(temp.path()).unwrap().any(|entry| { + entry + .ok() + .and_then(|entry| entry.file_name().into_string().ok()) + .is_some_and(|name| name.starts_with(".workspace-lock-candidate-")) + })); + + let lock = + acquire_workspace_lock_for_database(temp.path(), &temp.path().join("trail.sqlite")) + .unwrap(); + drop(lock); + assert!(!lock_path.exists()); + } + + #[test] + fn workspace_lock_double_unlink_failure_retains_raii_owner_until_drop_then_reacquires() { + let temp = tempfile::tempdir().unwrap(); + let lock_path = temp.path().join("lock"); + fail_next_workspace_lock_candidate_unlink(&lock_path); + fail_next_workspace_lock_rollback_unlink(&lock_path); + + let retained = + acquire_workspace_lock_for_database(temp.path(), &temp.path().join("trail.sqlite")) + .unwrap(); + + assert!(lock_path.exists()); + assert!(retained + .candidate_residue + .as_ref() + .is_some_and(|path| path.exists())); + let contended = + acquire_workspace_lock_for_database(temp.path(), &temp.path().join("trail.sqlite")); + assert!(matches!(contended, Err(Error::WorkspaceLocked(_)))); + + drop(retained); + assert!(!lock_path.exists()); + let reacquired = + acquire_workspace_lock_for_database(temp.path(), &temp.path().join("trail.sqlite")) + .unwrap(); + drop(reacquired); + assert!(!lock_path.exists()); + } + + #[test] + fn operation_metrics_scope_nests_and_resets_after_errors_retries_and_cancellation() { + let metrics = Arc::new(OperationMetricsState::default()); + + let first: Result<()> = metrics.profile(OperationMetricsKind::Status, || { + metrics.add(OperationMetricsDelta { + input_path_count: 3, + ..OperationMetricsDelta::default() + }); + metrics.profile(OperationMetricsKind::Diff, || { + metrics.add(OperationMetricsDelta { + final_path_count: 2, + ..OperationMetricsDelta::default() + }); + Ok::<(), Error>(()) + })?; + Err(Error::InvalidInput( + "expected metric test failure".to_string(), + )) + }); + assert!(first.is_err()); + let failed = metrics.last_report(); + assert_eq!(failed.generation, 1); + assert_eq!(failed.operation, "status"); + assert_eq!(failed.outcome, OperationMetricsOutcome::Error); + assert_eq!(failed.input_path_count, 3); + assert_eq!(failed.final_path_count, 2); + + metrics + .profile(OperationMetricsKind::Status, || { + metrics.add(OperationMetricsDelta { + input_path_count: 1, + ..OperationMetricsDelta::default() + }); + Ok::<(), Error>(()) + }) + .unwrap(); + let retry = metrics.last_report(); + assert_eq!(retry.generation, 2); + assert_eq!(retry.outcome, OperationMetricsOutcome::Success); + assert_eq!(retry.input_path_count, 1); + assert_eq!(retry.final_path_count, 0); + + let cancelled = std::panic::catch_unwind(std::panic::AssertUnwindSafe({ + let metrics = Arc::clone(&metrics); + move || { + metrics.profile(OperationMetricsKind::Record, || -> Result<()> { + metrics.add(OperationMetricsDelta { + expanded_path_count: 7, + ..OperationMetricsDelta::default() + }); + panic!("cancel metric scope") + }) + } + })); + assert!(cancelled.is_err()); + let cancelled = metrics.last_report(); + assert_eq!(cancelled.generation, 3); + assert_eq!(cancelled.operation, "record"); + assert_eq!( + cancelled.outcome, + OperationMetricsOutcome::CancelledOrUnclassified + ); + assert_eq!(cancelled.expanded_path_count, 7); + assert_eq!(cancelled.input_path_count, 0); + } + + #[test] + fn trail_prolly_store_reports_calls_requested_keys_found_values_and_bytes_across_clones() { + let metrics = Arc::new(OperationMetricsState::default()); + let store = TrailProllyStore::new( + TrailProllyStoreBackend::Sqlite(Arc::new(SqliteStore::open_in_memory().unwrap())), + Some(Arc::clone(&metrics)), + ); + store.put(b"present", b"abc").unwrap(); + let clone = store.clone(); + + metrics + .profile(OperationMetricsKind::Diff, || { + store.put(b"written", b"xyz").unwrap(); + store + .batch(&[ + BatchOp::Upsert { + key: b"batch-written", + value: b"de", + }, + BatchOp::Delete { + key: b"batch-missing", + }, + ]) + .unwrap(); + store + .batch_put(&[(b"batch-put".as_slice(), b"fgh".as_slice())]) + .unwrap(); + store.delete(b"delete-missing").unwrap(); + store + .batch_put_with_hint( + &[(b"hinted-node".as_slice(), b"ijkl".as_slice())], + b"test-namespace", + b"test-key", + b"performance-hint-not-a-node", + ) + .unwrap(); + assert_eq!(store.get(b"present").unwrap(), Some(b"abc".to_vec())); + assert_eq!(store.get(b"missing").unwrap(), None); + let unordered = clone.batch_get(&[b"present", b"missing", b"present"])?; + assert_eq!(unordered.len(), 1); + let ordered = store.batch_get_ordered(&[b"present", b"missing", b"present"])?; + assert_eq!(ordered.iter().filter(|value| value.is_some()).count(), 2); + Ok::<(), TrailProllyStoreError>(()) + }) + .unwrap(); + + let report = metrics.last_report(); + assert_eq!(report.prolly_read_call_count, 4); + assert_eq!(report.prolly_read_key_count, 8); + assert_eq!(report.prolly_read_value_count, 4); + assert_eq!(report.prolly_read_value_bytes, 12); + assert_eq!(report.prolly_write_call_count, 5); + assert_eq!(report.prolly_write_key_count, 6); + assert_eq!(report.prolly_write_value_bytes, 12); + } + + #[test] + #[ignore = "reproducible release-mode microbenchmark; run explicitly for performance evidence"] + fn operation_metrics_store_read_overhead_benchmark() { + const READS_PER_SAMPLE: u64 = 50_000; + const SAMPLES: usize = 7; + + let raw = SqliteStore::open_in_memory().unwrap(); + raw.put(b"present", b"abc").unwrap(); + let disabled = TrailProllyStore::new( + TrailProllyStoreBackend::Sqlite(Arc::new(SqliteStore::open_in_memory().unwrap())), + None, + ); + disabled.put(b"present", b"abc").unwrap(); + let metrics = Arc::new(OperationMetricsState::default()); + let measured = TrailProllyStore::new( + TrailProllyStoreBackend::Sqlite(Arc::new(SqliteStore::open_in_memory().unwrap())), + Some(Arc::clone(&metrics)), + ); + measured.put(b"present", b"abc").unwrap(); + + let mut raw_samples = Vec::with_capacity(SAMPLES); + let mut disabled_samples = Vec::with_capacity(SAMPLES); + let mut measured_samples = Vec::with_capacity(SAMPLES); + for sample in 0..SAMPLES { + let run_raw = || { + let started = Instant::now(); + for _ in 0..READS_PER_SAMPLE { + std::hint::black_box(raw.get(b"present").unwrap()); + } + started.elapsed().as_nanos() as u64 + }; + let run_measured = || { + let started = Instant::now(); + for _ in 0..READS_PER_SAMPLE { + std::hint::black_box(measured.get(b"present").unwrap()); + } + started.elapsed().as_nanos() as u64 + }; + let run_disabled = || { + let started = Instant::now(); + for _ in 0..READS_PER_SAMPLE { + std::hint::black_box(disabled.get(b"present").unwrap()); + } + started.elapsed().as_nanos() as u64 + }; + match sample % 3 { + 0 => { + raw_samples.push(run_raw()); + disabled_samples.push(run_disabled()); + measured_samples.push(run_measured()); + } + 1 => { + disabled_samples.push(run_disabled()); + measured_samples.push(run_measured()); + raw_samples.push(run_raw()); + } + _ => { + measured_samples.push(run_measured()); + raw_samples.push(run_raw()); + disabled_samples.push(run_disabled()); + } + } + } + raw_samples.sort_unstable(); + disabled_samples.sort_unstable(); + measured_samples.sort_unstable(); + let raw_ns_per_read = raw_samples[SAMPLES / 2] as f64 / READS_PER_SAMPLE as f64; + let disabled_ns_per_read = disabled_samples[SAMPLES / 2] as f64 / READS_PER_SAMPLE as f64; + let measured_ns_per_read = measured_samples[SAMPLES / 2] as f64 / READS_PER_SAMPLE as f64; + let disabled_overhead_percent = + ((disabled_ns_per_read / raw_ns_per_read) - 1.0).mul_add(100.0, 0.0); + let enabled_overhead_percent = + ((measured_ns_per_read / raw_ns_per_read) - 1.0).mul_add(100.0, 0.0); + println!( + "operation_metrics_store_read raw_ns_per_read={raw_ns_per_read:.2} disabled_ns_per_read={disabled_ns_per_read:.2} enabled_ns_per_read={measured_ns_per_read:.2} disabled_overhead_percent={disabled_overhead_percent:.2} enabled_overhead_percent={enabled_overhead_percent:.2} samples={SAMPLES} reads_per_sample={READS_PER_SAMPLE}" + ); + } + + #[test] + fn disabled_operation_metrics_skip_scopes_reports_and_store_counters() { + let disabled = None; + let result = + profile_operation_metrics(disabled.as_ref(), OperationMetricsKind::Status, || { + Ok::<_, Error>("unchanged") + }) + .unwrap(); + assert_eq!(result, "unchanged"); + assert_eq!(operation_metrics_report(disabled.as_ref()), None); + + let untouched = Arc::new(OperationMetricsState::default()); + let store = TrailProllyStore::new( + TrailProllyStoreBackend::Sqlite(Arc::new(SqliteStore::open_in_memory().unwrap())), + None, + ); + store.put(b"present", b"abc").unwrap(); + assert_eq!(store.get(b"present").unwrap(), Some(b"abc".to_vec())); + untouched + .profile(OperationMetricsKind::Diff, || Ok::<(), Error>(())) + .unwrap(); + let report = untouched.last_report(); + assert_eq!(report.prolly_read_call_count, 0); + assert_eq!(report.prolly_write_call_count, 0); + } + + #[test] + fn operation_metrics_env_parser_accepts_only_documented_truthy_values() { + for value in ["1", "true", "TRUE", "yes", "YES", "on", "ON"] { + assert!(operation_metrics_env_value_is_truthy(value), "{value}"); + } + for value in ["", "0", "false", "enabled", " true", "on ", "2"] { + assert!(!operation_metrics_env_value_is_truthy(value), "{value}"); + } + } + + #[test] + fn operation_metrics_expose_truthful_structural_surface_and_daemon_cumulative_totals() { + let metrics = Arc::new(OperationMetricsState::default()); + metrics.note_daemon_cumulative_rewrite(11); + + metrics + .profile(OperationMetricsKind::Record, || { + metrics.add(OperationMetricsDelta { + input_path_count: 1, + canonical_path_count: 2, + expanded_path_count: 3, + final_path_count: 4, + full_filesystem_walk_count: 5, + bounded_filesystem_walk_count: 6, + filesystem_entry_count: 7, + filesystem_stat_count: 8, + filesystem_read_count: 9, + filesystem_read_bytes: 10, + filesystem_hash_count: 11, + filesystem_hash_bytes: 12, + full_root_range_count: 13, + bounded_root_range_count: 14, + root_range_row_count: 15, + root_point_key_count: 16, + prolly_tree_batch_call_count: 17, + prolly_tree_batch_mutation_count: 18, + selected_worktree_index_sqlite_envelope_count: 1, + selected_worktree_index_sqlite_full_scan_count: 19, + selected_worktree_index_sqlite_row_read_count: 20, + selected_worktree_index_sqlite_row_delete_count: 21, + selected_worktree_index_sqlite_row_upsert_count: 22, + selected_worktree_index_sqlite_statement_count: 23, + selected_worktree_index_sqlite_transaction_count: 24, + selection_comparison_count: 25, + policy_build_count: 26, + policy_dependency_full_discovery: 39, + policy_dependency_bytes: 27, + policy_dependency_file_count: 28, + git_subprocess_count: 29, + git_global_work_count: 30, + git_output_bytes: 31, + git_output_record_count: 32, + daemon_snapshot_bytes: 33, + daemon_snapshot_path_count: 34, + manifest_bytes: 35, + manifest_key_comparison_count: 36, + journal_bytes: 37, + upper_work_count: 38, + ..OperationMetricsDelta::default() + }); + metrics.note_daemon_cumulative_rewrite(13); + Ok::<(), Error>(()) + }) + .unwrap(); + + let report = metrics.last_report(); + assert_eq!(report.input_path_count, 1); + assert_eq!(report.canonical_path_count, 2); + assert_eq!(report.expanded_path_count, 3); + assert_eq!(report.final_path_count, 4); + assert_eq!(report.full_filesystem_walk_count, 5); + assert_eq!(report.bounded_filesystem_walk_count, 6); + assert_eq!(report.filesystem_entry_count, 7); + assert_eq!(report.filesystem_stat_count, 8); + assert_eq!(report.filesystem_read_count, 9); + assert_eq!(report.filesystem_read_bytes, 10); + assert_eq!(report.filesystem_hash_count, 11); + assert_eq!(report.filesystem_hash_bytes, 12); + assert_eq!(report.full_root_range_count, 13); + assert_eq!(report.bounded_root_range_count, 14); + assert_eq!(report.root_range_row_count, 15); + assert_eq!(report.root_point_key_count, 16); + assert_eq!(report.prolly_tree_batch_call_count, 17); + assert_eq!(report.prolly_tree_batch_mutation_count, 18); + assert!(report.selected_worktree_index_sqlite_accounting_complete); + assert_eq!( + report.selected_worktree_index_sqlite_accounting_disposition, + "complete" + ); + assert_eq!(report.selected_worktree_index_sqlite_envelope_count, 1); + assert_eq!( + report.selected_worktree_index_sqlite_not_applicable_count, + 0 + ); + assert_eq!(report.selected_worktree_index_sqlite_full_scan_count, 19); + assert_eq!(report.selected_worktree_index_sqlite_row_read_count, 20); + assert_eq!(report.selected_worktree_index_sqlite_row_delete_count, 21); + assert_eq!(report.selected_worktree_index_sqlite_row_upsert_count, 22); + assert_eq!(report.selected_worktree_index_sqlite_statement_count, 23); + assert_eq!(report.selected_worktree_index_sqlite_transaction_count, 24); + assert_eq!(report.selection_comparison_count, 25); + assert_eq!(report.policy_build_count, 26); + assert_eq!(report.policy_dependency_full_discovery, 39); + assert_eq!(report.policy_dependency_bytes, 27); + assert_eq!(report.policy_dependency_file_count, 28); + assert_eq!(report.git_subprocess_count, 29); + assert_eq!(report.git_global_work_count, 30); + assert_eq!(report.git_output_bytes, 31); + assert_eq!(report.git_output_record_count, 32); + assert_eq!(report.daemon_snapshot_bytes, 33); + assert_eq!(report.daemon_snapshot_path_count, 34); + assert_eq!(report.manifest_bytes, 35); + assert_eq!(report.manifest_key_comparison_count, 36); + assert_eq!(report.journal_bytes, 37); + assert_eq!(report.upper_work_count, 38); + assert_eq!(report.daemon_cumulative_rewrite_count, 1); + assert_eq!(report.daemon_cumulative_rewrite_bytes, 13); + assert_eq!(report.daemon_cumulative_rewrite_count_total, 2); + assert_eq!(report.daemon_cumulative_rewrite_bytes_total, 24); + assert!(report.wall_time_ns > 0); + assert!(report.rss_end_bytes <= report.rss_lifetime_high_water_bytes); + assert!(report.rss_start_bytes <= report.rss_lifetime_high_water_bytes); + } + + #[test] + fn operation_metrics_distinguish_not_applicable_from_ambiguous_sqlite_accounting() { + let metrics = Arc::new(OperationMetricsState::default()); + metrics + .profile(OperationMetricsKind::Status, || { + metrics.add(OperationMetricsDelta { + selected_worktree_index_sqlite_not_applicable_count: 1, + ..OperationMetricsDelta::default() + }); + Ok::<(), Error>(()) + }) + .unwrap(); + let not_applicable = metrics.last_report(); + assert!(!not_applicable.selected_worktree_index_sqlite_accounting_complete); + assert_eq!( + not_applicable.selected_worktree_index_sqlite_accounting_disposition, + "not_applicable" + ); + assert_eq!( + not_applicable.selected_worktree_index_sqlite_not_applicable_count, + 1 + ); + + metrics + .profile(OperationMetricsKind::Diff, || { + metrics.add(OperationMetricsDelta { + selected_worktree_index_sqlite_envelope_count: 1, + selected_worktree_index_sqlite_not_applicable_count: 1, + ..OperationMetricsDelta::default() + }); + Ok::<(), Error>(()) + }) + .unwrap(); + let ambiguous = metrics.last_report(); + assert!(!ambiguous.selected_worktree_index_sqlite_accounting_complete); + assert_eq!( + ambiguous.selected_worktree_index_sqlite_accounting_disposition, + "ambiguous" + ); + } + + #[test] + fn daemon_rewrite_count_and_bytes_are_snapshotted_as_one_event() { + const REWRITES: usize = 20_000; + const BYTES_PER_REWRITE: u64 = 7; + let metrics = Arc::new(OperationMetricsState::default()); + let writer_metrics = Arc::clone(&metrics); + let writer = std::thread::spawn(move || { + for _ in 0..REWRITES { + writer_metrics.note_daemon_cumulative_rewrite(BYTES_PER_REWRITE as usize); + } + }); + + while !writer.is_finished() { + let snapshot = metrics.snapshot(); + assert_eq!( + snapshot.daemon_cumulative_rewrite_bytes, + snapshot + .daemon_cumulative_rewrite_count + .saturating_mul(BYTES_PER_REWRITE) + ); + } + writer.join().unwrap(); + let snapshot = metrics.snapshot(); + assert_eq!(snapshot.daemon_cumulative_rewrite_count, REWRITES as u64); + assert_eq!( + snapshot.daemon_cumulative_rewrite_bytes, + (REWRITES as u64).saturating_mul(BYTES_PER_REWRITE) + ); + } + #[test] fn case_fold_collision_validation_rejects_ambiguous_paths() { let paths = [ @@ -1112,7 +5158,7 @@ mod tests { "op": op, "path": path, "line_id": if seed % 2 == 0 { - serde_json::json!("change_abc:1") + serde_json::json!("line_abc:1") } else { serde_json::json!(1) }, diff --git a/trail/src/db/performance.rs b/trail/src/db/performance.rs new file mode 100644 index 0000000..d9c4b60 --- /dev/null +++ b/trail/src/db/performance.rs @@ -0,0 +1,773 @@ +use serde::Serialize; +use std::io::Write; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Instant; + +const PERFORMANCE_METRICS_ENV: &str = "TRAIL_PERFORMANCE_METRICS"; +const PERFORMANCE_METRICS_FILE_ENV: &str = "TRAIL_PERFORMANCE_METRICS_FILE"; +static PERFORMANCE_METRICS_FILE_LOCK: Mutex<()> = Mutex::new(()); + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum OperationMetricsOutcome { + Success, + Error, + #[default] + CancelledOrUnclassified, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum OperationMetricsKind { + Status, + StatusReadOnly, + Diff, + Record, + MaterializedLaneRecord, + StructuredPatch, + CowCheckpoint, +} + +impl OperationMetricsKind { + fn as_str(self) -> &'static str { + match self { + Self::Status => "status", + Self::StatusReadOnly => "status_read_only", + Self::Diff => "diff", + Self::Record => "record", + Self::MaterializedLaneRecord => "materialized_lane_record", + Self::StructuredPatch => "structured_patch", + Self::CowCheckpoint => "cow_checkpoint", + } + } +} + +macro_rules! define_operation_metric_counters { + ($($field:ident),+ $(,)?) => { + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] + pub(crate) struct OperationMetricsDelta { + $(pub(crate) $field: u64,)+ + } + + #[derive(Debug, Default)] + struct AtomicOperationMetrics { + $($field: AtomicU64,)+ + } + + impl AtomicOperationMetrics { + fn add(&self, delta: OperationMetricsDelta) { + $(saturating_atomic_add(&self.$field, delta.$field);)+ + } + + fn snapshot(&self) -> OperationMetricsDelta { + OperationMetricsDelta { + $($field: self.$field.load(Ordering::Relaxed),)+ + } + } + } + + impl OperationMetricsDelta { + fn saturating_sub(self, earlier: Self) -> Self { + Self { + $($field: self.$field.saturating_sub(earlier.$field),)+ + } + } + } + }; +} + +define_operation_metric_counters!( + input_path_count, + canonical_path_count, + expanded_path_count, + final_path_count, + full_filesystem_walk_count, + bounded_filesystem_walk_count, + filesystem_entry_count, + filesystem_stat_count, + filesystem_read_count, + filesystem_read_bytes, + filesystem_hash_count, + filesystem_hash_bytes, + full_root_range_count, + bounded_root_range_count, + root_range_row_count, + root_point_key_count, + prolly_read_call_count, + prolly_read_key_count, + prolly_read_value_count, + prolly_read_value_bytes, + prolly_write_call_count, + prolly_write_key_count, + prolly_write_value_bytes, + prolly_tree_batch_call_count, + prolly_tree_batch_mutation_count, + selected_worktree_index_sqlite_envelope_count, + selected_worktree_index_sqlite_not_applicable_count, + selected_worktree_index_sqlite_full_scan_count, + selected_worktree_index_sqlite_row_read_count, + selected_worktree_index_sqlite_row_delete_count, + selected_worktree_index_sqlite_row_upsert_count, + selected_worktree_index_sqlite_statement_count, + selected_worktree_index_sqlite_transaction_count, + selection_comparison_count, + policy_build_count, + policy_dependency_full_discovery, + policy_dependency_bytes, + policy_dependency_file_count, + git_subprocess_count, + git_global_work_count, + git_index_refresh_count, + git_trace2_region_count, + git_trace2_bytes, + git_fsmonitor_qualification_count, + git_untracked_cache_qualification_count, + external_adapter_global_work, + git_index_read_count, + git_index_bytes, + git_shared_index_read_count, + git_shared_index_bytes, + git_output_bytes, + git_output_record_count, + daemon_snapshot_bytes, + daemon_snapshot_path_count, + daemon_cumulative_rewrite_count, + daemon_cumulative_rewrite_bytes, + authoritative_candidate_count, + ledger_row_touch_count, + observer_tail_record_fold_count, + reconciliation_run_count, + manifest_bytes, + manifest_key_comparison_count, + journal_bytes, + upper_work_count, +); + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub(crate) struct OperationMetricsReport { + pub(crate) generation: u64, + pub(crate) operation: String, + pub(crate) outcome: OperationMetricsOutcome, + pub(crate) input_path_count: u64, + pub(crate) canonical_path_count: u64, + pub(crate) expanded_path_count: u64, + pub(crate) final_path_count: u64, + pub(crate) full_filesystem_walk_count: u64, + pub(crate) bounded_filesystem_walk_count: u64, + pub(crate) filesystem_entry_count: u64, + pub(crate) filesystem_stat_count: u64, + pub(crate) filesystem_read_count: u64, + pub(crate) filesystem_read_bytes: u64, + pub(crate) filesystem_hash_count: u64, + pub(crate) filesystem_hash_bytes: u64, + pub(crate) full_root_range_count: u64, + pub(crate) bounded_root_range_count: u64, + pub(crate) root_range_row_count: u64, + pub(crate) root_point_key_count: u64, + /// Store read calls and requested keys are attempted work, including + /// backend errors. Found values and bytes count only successful results. + pub(crate) prolly_read_call_count: u64, + pub(crate) prolly_read_key_count: u64, + pub(crate) prolly_read_value_count: u64, + pub(crate) prolly_read_value_bytes: u64, + /// Store write calls, keys, and value bytes are attempted work, including + /// backend errors. Deletes contribute a key and zero value bytes. + pub(crate) prolly_write_call_count: u64, + pub(crate) prolly_write_key_count: u64, + pub(crate) prolly_write_value_bytes: u64, + pub(crate) prolly_tree_batch_call_count: u64, + pub(crate) prolly_tree_batch_mutation_count: u64, + /// This completeness claim covers only the selected worktree-index sync + /// envelope. It does not claim that every SQLite statement issued by the + /// containing status/diff/record operation is instrumented. + pub(crate) selected_worktree_index_sqlite_accounting_complete: bool, + /// Typed disposition for the selected worktree-index SQL surface. A + /// `not_applicable` report is emitted only when the authoritative command + /// path explicitly declares that it cannot access that index. + pub(crate) selected_worktree_index_sqlite_accounting_disposition: String, + pub(crate) selected_worktree_index_sqlite_envelope_count: u64, + pub(crate) selected_worktree_index_sqlite_not_applicable_count: u64, + /// Executions for which SQLite reported at least one FULLSCAN_STEP. + pub(crate) selected_worktree_index_sqlite_full_scan_count: u64, + /// Worktree-index rows decoded by exact/descendant candidate queries. + /// The schema_meta baseline row is deliberately excluded. + pub(crate) selected_worktree_index_sqlite_row_read_count: u64, + /// Worktree-index row mutations made durable by a successful COMMIT. + pub(crate) selected_worktree_index_sqlite_row_delete_count: u64, + pub(crate) selected_worktree_index_sqlite_row_upsert_count: u64, + /// Attempted SQL executions, including transaction control and failed + /// mutation/COMMIT/ROLLBACK attempts. + pub(crate) selected_worktree_index_sqlite_statement_count: u64, + /// Selected-sync transactions whose BEGIN IMMEDIATE succeeded. + pub(crate) selected_worktree_index_sqlite_transaction_count: u64, + pub(crate) selection_comparison_count: u64, + pub(crate) policy_build_count: u64, + pub(crate) policy_dependency_full_discovery: u64, + pub(crate) policy_dependency_bytes: u64, + pub(crate) policy_dependency_file_count: u64, + pub(crate) git_subprocess_count: u64, + pub(crate) git_global_work_count: u64, + pub(crate) git_index_refresh_count: u64, + pub(crate) git_trace2_region_count: u64, + pub(crate) git_trace2_bytes: u64, + pub(crate) git_fsmonitor_qualification_count: u64, + pub(crate) git_untracked_cache_qualification_count: u64, + pub(crate) external_adapter_global_work: u64, + pub(crate) git_index_read_count: u64, + pub(crate) git_index_bytes: u64, + pub(crate) git_shared_index_read_count: u64, + pub(crate) git_shared_index_bytes: u64, + pub(crate) git_output_bytes: u64, + pub(crate) git_output_record_count: u64, + /// Bytes physically read from the durable daemon snapshot. In-memory + /// snapshots copy typed state and therefore contribute paths but zero bytes. + pub(crate) daemon_snapshot_bytes: u64, + pub(crate) daemon_snapshot_path_count: u64, + /// Full serialized daemon snapshot rewrite work. These counters are + /// cumulative outside request scopes as well as reported as scope deltas. + pub(crate) daemon_cumulative_rewrite_count: u64, + pub(crate) daemon_cumulative_rewrite_bytes: u64, + pub(crate) daemon_cumulative_rewrite_count_total: u64, + pub(crate) daemon_cumulative_rewrite_bytes_total: u64, + pub(crate) authoritative_candidate_count: u64, + pub(crate) ledger_row_touch_count: u64, + pub(crate) observer_tail_record_fold_count: u64, + pub(crate) reconciliation_run_count: u64, + pub(crate) manifest_bytes: u64, + pub(crate) manifest_key_comparison_count: u64, + pub(crate) journal_bytes: u64, + pub(crate) upper_work_count: u64, + pub(crate) wall_time_ns: u64, + pub(crate) rss_start_bytes: u64, + pub(crate) rss_end_bytes: u64, + pub(crate) rss_lifetime_high_water_bytes: u64, +} + +#[derive(Debug, Default)] +pub(crate) struct OperationMetricsState { + counters: AtomicOperationMetrics, + daemon_rewrites: Mutex, + scope: Mutex, +} + +#[derive(Clone, Copy, Debug, Default)] +struct DaemonRewriteTotals { + count: u64, + bytes: u64, +} + +pub(crate) struct OperationMetricsAccumulator { + metrics: Option>, + pub(crate) delta: OperationMetricsDelta, +} + +#[derive(Debug, Default)] +struct OperationScopeTracker { + generation: u64, + depth: u64, + active: Option, + last_report: OperationMetricsReport, +} + +#[derive(Debug)] +struct ActiveOperationScope { + generation: u64, + operation: OperationMetricsKind, + started: Instant, + rss_start_bytes: u64, + counters_start: OperationMetricsDelta, +} + +pub(crate) struct OperationMetricsScope { + state: Arc, + generation: u64, + finished: bool, +} + +impl OperationMetricsState { + pub(crate) fn add(&self, mut delta: OperationMetricsDelta) { + if delta.daemon_cumulative_rewrite_count != 0 || delta.daemon_cumulative_rewrite_bytes != 0 + { + let mut daemon = lock_unpoisoned(&self.daemon_rewrites); + daemon.count = daemon + .count + .saturating_add(delta.daemon_cumulative_rewrite_count); + daemon.bytes = daemon + .bytes + .saturating_add(delta.daemon_cumulative_rewrite_bytes); + delta.daemon_cumulative_rewrite_count = 0; + delta.daemon_cumulative_rewrite_bytes = 0; + } + self.counters.add(delta); + } + + pub(crate) fn note_prolly_read_call(&self, key_count: usize) { + saturating_atomic_add(&self.counters.prolly_read_call_count, 1); + saturating_atomic_add( + &self.counters.prolly_read_key_count, + saturating_u64_from_usize(key_count), + ); + } + + pub(crate) fn note_prolly_read_values<'a, I>(&self, values: I) + where + I: IntoIterator>, + { + let mut count = 0u64; + let mut bytes = 0u64; + for value in values { + count = count.saturating_add(1); + bytes = bytes.saturating_add(saturating_u64_from_usize(value.len())); + } + saturating_atomic_add(&self.counters.prolly_read_value_count, count); + saturating_atomic_add(&self.counters.prolly_read_value_bytes, bytes); + } + + pub(crate) fn note_prolly_write_call(&self, key_count: usize, value_bytes: usize) { + saturating_atomic_add(&self.counters.prolly_write_call_count, 1); + saturating_atomic_add( + &self.counters.prolly_write_key_count, + saturating_u64_from_usize(key_count), + ); + saturating_atomic_add( + &self.counters.prolly_write_value_bytes, + saturating_u64_from_usize(value_bytes), + ); + } + + #[cfg(test)] + pub(crate) fn note_daemon_cumulative_rewrite(&self, bytes: usize) { + let mut daemon = lock_unpoisoned(&self.daemon_rewrites); + daemon.count = daemon.count.saturating_add(1); + daemon.bytes = daemon + .bytes + .saturating_add(saturating_u64_from_usize(bytes)); + } + + pub(crate) fn profile( + self: &Arc, + kind: OperationMetricsKind, + operation: impl FnOnce() -> std::result::Result, + ) -> std::result::Result { + let scope = self.begin(kind); + let result = operation(); + let outcome = if result.is_ok() { + OperationMetricsOutcome::Success + } else { + OperationMetricsOutcome::Error + }; + scope.finish(outcome); + result + } + + fn begin(self: &Arc, kind: OperationMetricsKind) -> OperationMetricsScope { + let mut tracker = lock_unpoisoned(&self.scope); + if tracker.depth == 0 { + tracker.generation = tracker.generation.saturating_add(1); + let (rss_start_bytes, _) = process_rss_snapshot(); + tracker.active = Some(ActiveOperationScope { + generation: tracker.generation, + operation: kind, + started: Instant::now(), + rss_start_bytes, + counters_start: self.snapshot(), + }); + } + tracker.depth = tracker.depth.saturating_add(1); + let generation = tracker + .active + .as_ref() + .map(|active| active.generation) + .unwrap_or(tracker.generation); + drop(tracker); + OperationMetricsScope { + state: Arc::clone(self), + generation, + finished: false, + } + } + + pub(crate) fn last_report(&self) -> OperationMetricsReport { + lock_unpoisoned(&self.scope).last_report.clone() + } + + fn finish_scope(&self, generation: u64, outcome: OperationMetricsOutcome) { + let mut tracker = lock_unpoisoned(&self.scope); + if tracker.active.as_ref().map(|active| active.generation) != Some(generation) { + return; + } + tracker.depth = tracker.depth.saturating_sub(1); + if tracker.depth != 0 { + return; + } + let Some(active) = tracker.active.take() else { + return; + }; + let counters_end = self.snapshot(); + let delta = counters_end.saturating_sub(active.counters_start); + let (rss_end_bytes, rss_lifetime_high_water_bytes) = process_rss_snapshot(); + let report = OperationMetricsReport::from_delta( + active, + outcome, + delta, + counters_end, + rss_end_bytes, + rss_lifetime_high_water_bytes, + ); + tracker.last_report = report.clone(); + drop(tracker); + emit_operation_metrics_report(&report); + } + + pub(crate) fn snapshot(&self) -> OperationMetricsDelta { + let mut snapshot = self.counters.snapshot(); + let daemon = lock_unpoisoned(&self.daemon_rewrites); + snapshot.daemon_cumulative_rewrite_count = daemon.count; + snapshot.daemon_cumulative_rewrite_bytes = daemon.bytes; + snapshot + } +} + +impl OperationMetricsAccumulator { + pub(crate) fn new( + metrics: Option<&Arc>, + delta: OperationMetricsDelta, + ) -> Self { + Self { + metrics: metrics.cloned(), + delta, + } + } +} + +impl Drop for OperationMetricsAccumulator { + fn drop(&mut self) { + if let Some(metrics) = &self.metrics { + metrics.add(self.delta); + } + } +} + +pub(crate) fn profile_operation_metrics( + metrics: Option<&Arc>, + kind: OperationMetricsKind, + operation: impl FnOnce() -> std::result::Result, +) -> std::result::Result { + match metrics { + Some(metrics) => metrics.profile(kind, || { + // Materialized-lane record and structured patch bypass the + // workspace worktree-index by design. They still cross this one + // audited empty envelope so request metrics prove that the SQL + // surface performed zero work instead of reporting an ambiguous + // absence of instrumentation. Any real selected-index entry point + // contributes its own envelope and SQL counters in addition. + if matches!( + kind, + OperationMetricsKind::MaterializedLaneRecord + | OperationMetricsKind::StructuredPatch + ) { + metrics.add(OperationMetricsDelta { + selected_worktree_index_sqlite_envelope_count: 1, + ..OperationMetricsDelta::default() + }); + } + operation() + }), + None => operation(), + } +} + +pub(crate) fn operation_metrics_report( + metrics: Option<&Arc>, +) -> Option { + metrics.map(|metrics| metrics.last_report()) +} + +/// Accepted opt-in values are `1`, `true`, `yes`, and `on`, compared with +/// ASCII case folding and without trimming. Every other value is disabled. +pub(crate) fn operation_metrics_env_value_is_truthy(value: &str) -> bool { + value == "1" + || value.eq_ignore_ascii_case("true") + || value.eq_ignore_ascii_case("yes") + || value.eq_ignore_ascii_case("on") +} + +pub(crate) fn operation_metrics_are_enabled() -> bool { + if cfg!(test) { + return true; + } + if std::env::var_os(PERFORMANCE_METRICS_FILE_ENV).is_some_and(|path| !path.is_empty()) { + return true; + } + std::env::var_os(PERFORMANCE_METRICS_ENV) + .and_then(|value| value.into_string().ok()) + .as_deref() + .is_some_and(operation_metrics_env_value_is_truthy) +} + +fn emit_operation_metrics_report(report: &OperationMetricsReport) { + // An auto-started daemon must return the report on the authenticated RPC + // response. Its inherited sidecar path belongs to the command that first + // launched it and is not request-scoped; writing there would either lose a + // later command's report or duplicate the copy emitted by that command's + // CLI process. + if std::env::var_os("TRAIL_WORKSPACE_DAEMON").is_some() { + return; + } + let Some(path) = std::env::var_os(PERFORMANCE_METRICS_FILE_ENV).filter(|path| !path.is_empty()) + else { + return; + }; + let _guard = lock_unpoisoned(&PERFORMANCE_METRICS_FILE_LOCK); + // Metrics must never change the command result after its operation has + // completed. Emit one O_APPEND write so concurrent inherited daemon + // processes cannot interleave a JSON line. The scale gate treats a + // missing or malformed report as a hard failure. + let result = (|| -> std::result::Result<(), Box> { + let mut line = serde_json::to_vec(report)?; + line.push(b'\n'); + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + let written = file.write(&line)?; + if written != line.len() { + return Err(format!( + "short performance metrics append: wrote {written} of {} bytes", + line.len() + ) + .into()); + } + file.flush()?; + Ok(()) + })(); + if let Err(error) = result { + eprintln!("trail: failed to emit performance metrics JSONL: {error}"); + } +} + +impl super::Trail { + pub(crate) fn note_operation_metrics(&self, delta: OperationMetricsDelta) { + if let Some(metrics) = &self.operation_metrics { + metrics.add(delta); + } + } + + pub(crate) fn operation_metrics_generation(&self) -> Option { + operation_metrics_report(self.operation_metrics.as_ref()).map(|report| report.generation) + } + + pub(crate) fn operation_metrics_json_after(&self, generation: u64) -> Option { + let report = operation_metrics_report(self.operation_metrics.as_ref())?; + (report.generation > generation) + .then(|| serde_json::to_string(&report).ok()) + .flatten() + } +} + +impl OperationMetricsScope { + fn finish(mut self, outcome: OperationMetricsOutcome) { + self.state.finish_scope(self.generation, outcome); + self.finished = true; + } +} + +impl Drop for OperationMetricsScope { + fn drop(&mut self) { + if !self.finished { + self.state.finish_scope( + self.generation, + OperationMetricsOutcome::CancelledOrUnclassified, + ); + self.finished = true; + } + } +} + +impl OperationMetricsReport { + fn from_delta( + active: ActiveOperationScope, + outcome: OperationMetricsOutcome, + delta: OperationMetricsDelta, + totals: OperationMetricsDelta, + rss_end_bytes: u64, + rss_lifetime_high_water_bytes: u64, + ) -> Self { + Self { + generation: active.generation, + operation: active.operation.as_str().to_string(), + outcome, + input_path_count: delta.input_path_count, + canonical_path_count: delta.canonical_path_count, + expanded_path_count: delta.expanded_path_count, + final_path_count: delta.final_path_count, + full_filesystem_walk_count: delta.full_filesystem_walk_count, + bounded_filesystem_walk_count: delta.bounded_filesystem_walk_count, + filesystem_entry_count: delta.filesystem_entry_count, + filesystem_stat_count: delta.filesystem_stat_count, + filesystem_read_count: delta.filesystem_read_count, + filesystem_read_bytes: delta.filesystem_read_bytes, + filesystem_hash_count: delta.filesystem_hash_count, + filesystem_hash_bytes: delta.filesystem_hash_bytes, + full_root_range_count: delta.full_root_range_count, + bounded_root_range_count: delta.bounded_root_range_count, + root_range_row_count: delta.root_range_row_count, + root_point_key_count: delta.root_point_key_count, + prolly_read_call_count: delta.prolly_read_call_count, + prolly_read_key_count: delta.prolly_read_key_count, + prolly_read_value_count: delta.prolly_read_value_count, + prolly_read_value_bytes: delta.prolly_read_value_bytes, + prolly_write_call_count: delta.prolly_write_call_count, + prolly_write_key_count: delta.prolly_write_key_count, + prolly_write_value_bytes: delta.prolly_write_value_bytes, + prolly_tree_batch_call_count: delta.prolly_tree_batch_call_count, + prolly_tree_batch_mutation_count: delta.prolly_tree_batch_mutation_count, + selected_worktree_index_sqlite_accounting_complete: delta + .selected_worktree_index_sqlite_envelope_count + > 0 + && delta.selected_worktree_index_sqlite_not_applicable_count == 0, + selected_worktree_index_sqlite_accounting_disposition: match ( + delta.selected_worktree_index_sqlite_envelope_count > 0, + delta.selected_worktree_index_sqlite_not_applicable_count > 0, + ) { + (true, false) => "complete", + (false, true) => "not_applicable", + _ => "ambiguous", + } + .to_string(), + selected_worktree_index_sqlite_envelope_count: delta + .selected_worktree_index_sqlite_envelope_count, + selected_worktree_index_sqlite_not_applicable_count: delta + .selected_worktree_index_sqlite_not_applicable_count, + selected_worktree_index_sqlite_full_scan_count: delta + .selected_worktree_index_sqlite_full_scan_count, + selected_worktree_index_sqlite_row_read_count: delta + .selected_worktree_index_sqlite_row_read_count, + selected_worktree_index_sqlite_row_delete_count: delta + .selected_worktree_index_sqlite_row_delete_count, + selected_worktree_index_sqlite_row_upsert_count: delta + .selected_worktree_index_sqlite_row_upsert_count, + selected_worktree_index_sqlite_statement_count: delta + .selected_worktree_index_sqlite_statement_count, + selected_worktree_index_sqlite_transaction_count: delta + .selected_worktree_index_sqlite_transaction_count, + selection_comparison_count: delta.selection_comparison_count, + policy_build_count: delta.policy_build_count, + policy_dependency_full_discovery: delta.policy_dependency_full_discovery, + policy_dependency_bytes: delta.policy_dependency_bytes, + policy_dependency_file_count: delta.policy_dependency_file_count, + git_subprocess_count: delta.git_subprocess_count, + git_global_work_count: delta.git_global_work_count, + git_index_refresh_count: delta.git_index_refresh_count, + git_trace2_region_count: delta.git_trace2_region_count, + git_trace2_bytes: delta.git_trace2_bytes, + git_fsmonitor_qualification_count: delta.git_fsmonitor_qualification_count, + git_untracked_cache_qualification_count: delta.git_untracked_cache_qualification_count, + external_adapter_global_work: delta.external_adapter_global_work, + git_index_read_count: delta.git_index_read_count, + git_index_bytes: delta.git_index_bytes, + git_shared_index_read_count: delta.git_shared_index_read_count, + git_shared_index_bytes: delta.git_shared_index_bytes, + git_output_bytes: delta.git_output_bytes, + git_output_record_count: delta.git_output_record_count, + daemon_snapshot_bytes: delta.daemon_snapshot_bytes, + daemon_snapshot_path_count: delta.daemon_snapshot_path_count, + daemon_cumulative_rewrite_count: delta.daemon_cumulative_rewrite_count, + daemon_cumulative_rewrite_bytes: delta.daemon_cumulative_rewrite_bytes, + daemon_cumulative_rewrite_count_total: totals.daemon_cumulative_rewrite_count, + daemon_cumulative_rewrite_bytes_total: totals.daemon_cumulative_rewrite_bytes, + authoritative_candidate_count: delta.authoritative_candidate_count, + ledger_row_touch_count: delta.ledger_row_touch_count, + observer_tail_record_fold_count: delta.observer_tail_record_fold_count, + reconciliation_run_count: delta.reconciliation_run_count, + manifest_bytes: delta.manifest_bytes, + manifest_key_comparison_count: delta.manifest_key_comparison_count, + journal_bytes: delta.journal_bytes, + upper_work_count: delta.upper_work_count, + wall_time_ns: active.started.elapsed().as_nanos().min(u64::MAX as u128) as u64, + rss_start_bytes: active.rss_start_bytes, + rss_end_bytes, + rss_lifetime_high_water_bytes, + } + } +} + +fn saturating_atomic_add(counter: &AtomicU64, delta: u64) { + if delta == 0 { + return; + } + let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| { + Some(value.saturating_add(delta)) + }); +} + +pub(crate) fn saturating_u64_from_usize(value: usize) -> u64 { + u64::try_from(value).unwrap_or(u64::MAX) +} + +fn lock_unpoisoned(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +#[cfg(target_os = "linux")] +fn process_rss_snapshot() -> (u64, u64) { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + let page_size = u64::try_from(page_size).unwrap_or(0); + let current = std::fs::read_to_string("/proc/self/statm") + .ok() + .and_then(|value| value.split_whitespace().nth(1)?.parse::().ok()) + .map(|pages| pages.saturating_mul(page_size)) + .unwrap_or(0); + let mut usage = std::mem::MaybeUninit::::uninit(); + let high_water = if unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) } == 0 { + unsafe { usage.assume_init().ru_maxrss as u64 }.saturating_mul(1024) + } else { + 0 + }; + (current, high_water.max(current)) +} + +#[cfg(target_os = "macos")] +fn process_rss_snapshot() -> (u64, u64) { + let mut usage = std::mem::MaybeUninit::::uninit(); + let high_water = if unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) } == 0 { + unsafe { usage.assume_init().ru_maxrss as u64 } + } else { + 0 + }; + // `getrusage` exposes lifetime high-water RSS on macOS, not boundary RSS. + // Leave the boundary values unknown rather than mislabeling high-water data. + (0, high_water) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn process_rss_snapshot() -> (u64, u64) { + (0, 0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn controlled_lane_scopes_prove_an_empty_selected_index_envelope() { + for kind in [ + OperationMetricsKind::MaterializedLaneRecord, + OperationMetricsKind::StructuredPatch, + ] { + let metrics = Arc::new(OperationMetricsState::default()); + profile_operation_metrics(Some(&metrics), kind, || Ok::<(), ()>(())).unwrap(); + let report = metrics.last_report(); + assert!(report.selected_worktree_index_sqlite_accounting_complete); + assert_eq!( + report.selected_worktree_index_sqlite_accounting_disposition, + "complete" + ); + assert_eq!(report.selected_worktree_index_sqlite_envelope_count, 1); + assert_eq!(report.selected_worktree_index_sqlite_statement_count, 0); + assert_eq!(report.selected_worktree_index_sqlite_transaction_count, 0); + } + } +} diff --git a/trail/src/db/record/checkout.rs b/trail/src/db/record/checkout.rs index fd5b3ae..5d1d908 100644 --- a/trail/src/db/record/checkout.rs +++ b/trail/src/db/record/checkout.rs @@ -13,7 +13,14 @@ impl Trail { workdir: Option<&Path>, record_dirty: bool, ) -> Result { - let _lock = self.acquire_write_lock()?; + // TRAIL_FS_PRODUCER: primary_workspace_checkout Checkout controlled + let ledger_authority = + crate::db::change_ledger::command_authority_enabled() && workdir.is_none() && !dry_run; + let _lock = if ledger_authority { + None + } else { + Some(self.acquire_write_lock()?) + }; if dry_run && record_dirty { return Err(Error::InvalidInput( "checkout --record-dirty cannot be combined with --dry-run".to_string(), @@ -48,6 +55,7 @@ impl Trail { .transpose()?; if target.root_id == current.root_id { if let Some(output_root) = &output_root { + // TRAIL_FS_PRODUCER: alternate_checkout_destination exempt_alternate_output exempt let written_files = if dry_run { 0 } else { @@ -97,6 +105,73 @@ impl Trail { if !cloned_from_workspace { self.materialize_files_at(output_root, &BTreeMap::new(), &target_files)?; } + } else if ledger_authority { + let expected = + crate::db::change_ledger::prepare_workspace_controlled_projection(self)?; + let mut evidence_paths = diff + .summaries + .iter() + .flat_map(|summary| { + std::iter::once(summary.path.as_str()).chain(summary.old_path.as_deref()) + }) + .map(crate::db::change_ledger::LedgerPath::parse) + .collect::>>()?; + evidence_paths.sort(); + evidence_paths.dedup(); + let evidence = crate::db::change_ledger::IntentEvidence { + exact_paths: evidence_paths, + complete_prefixes: Vec::new(), + }; + let alignment = if target.root_id == current.root_id { + crate::db::change_ledger::ProjectionAlignmentMode::Aligned + } else { + crate::db::change_ledger::ProjectionAlignmentMode::RetainDirty { + target: crate::db::change_ledger::IntentTarget { + change_id: target.change_id.clone(), + root_id: target.root_id.clone(), + operation_id: None, + }, + } + }; + crate::db::change_ledger::run_projection_alignment( + self, + &expected, + crate::db::change_ledger::IntentProducer::Checkout, + &evidence, + alignment, + |db, intent| { + crate::db::change_ledger::with_workspace_controlled_interval( + db, + intent, + &evidence, + |db| { + if force { + db.remove_visible_files_absent_from_target(&target_files)?; + } + db.materialize_files(¤t_files, &target_files) + }, + |db, policy, candidates| { + let comparison = db.compare_controlled_projection_target( + policy, + candidates, + &target.root_id, + crate::db::change_ledger::CandidateMaterialization::ManifestOnly, + )?; + if comparison.summaries.is_empty() { + Ok(()) + } else { + Err(Error::ChangeLedgerReconcileRequired { + scope: expected.scope_id.to_text(), + state: "stale_baseline".into(), + reason: "checkout pinned verification did not match its target root".into(), + command: "trail status".into(), + }) + } + }, + ) + }, + |_| Ok(()), + )?; } else { if force { self.remove_visible_files_absent_from_target(&target_files)?; @@ -123,6 +198,7 @@ impl Trail { &self, target_files: &BTreeMap, ) -> Result<()> { + let mut changed_directories = BTreeSet::new(); for path in self.scan_worktree_file_paths()?.paths { if target_files.contains_key(&path) { continue; @@ -136,7 +212,18 @@ impl Trail { if metadata.file_type().is_symlink() || !metadata.is_file() { continue; } - fs::remove_file(abs)?; + fs::remove_file(&abs)?; + let parent = abs.parent().ok_or_else(|| Error::InvalidPath { + path: path.clone(), + reason: "removed workspace file has no parent directory".into(), + })?; + changed_directories.insert(parent.to_path_buf()); + } + // The controlled projection proof is not allowed to advance until + // forced deletions are durable. Sync each directory whose namespace + // changed so a successful return means the removals survive a crash. + for directory in changed_directories { + sync_directory_strict(&directory)?; } Ok(()) } diff --git a/trail/src/db/record/diff.rs b/trail/src/db/record/diff.rs index bc23797..444502a 100644 --- a/trail/src/db/record/diff.rs +++ b/trail/src/db/record/diff.rs @@ -1,8 +1,65 @@ use super::*; impl Trail { + pub(crate) fn diff_dirty_from_changed_path_ledger( + &mut self, + patches: bool, + line_changes: bool, + ) -> Result { + let branch = self.current_branch()?; + let head = self.resolve_branch_ref(&branch)?; + let materialization = if patches || line_changes { + crate::db::change_ledger::CandidateMaterialization::RecordBytes + } else { + crate::db::change_ledger::CandidateMaterialization::ManifestOnly + }; + let branch_for_diff = branch.clone(); + let (summary, _fenced) = + self.with_workspace_authoritative_command_snapshot(|db, policy, candidates, _git| { + let comparison = db.compare_authoritative_candidates( + policy, + candidates, + &head.root_id, + materialization, + )?; + if !patches && !line_changes { + debug_assert!(comparison.disk_files.is_none()); + return Ok(DiffSummary { + from: branch_for_diff.clone(), + to: "dirty".into(), + files: comparison.summaries, + }); + } + let disk_files = comparison.disk_files.as_ref().ok_or_else(|| { + Error::Corrupt( + "authoritative dirty diff omitted pinned candidate contents".into(), + ) + })?; + let change_id = db.allocate_change_id("trail", "dirty-diff")?; + let built = db.build_root_for_selected_disk_files_incremental( + &head.root_id, + &comparison.baseline_files, + disk_files, + &comparison.selections, + &change_id, + )?; + db.diff_files( + branch_for_diff.clone(), + "dirty".into(), + &comparison.baseline_files, + &built.files, + patches, + line_changes, + ) + })?; + Ok(summary) + } + pub fn diff_range(&self, spec: &str, patches: bool) -> Result { - self.diff_range_with_options(spec, patches, false) + let metrics = self.operation_metrics.clone(); + profile_operation_metrics(metrics.as_ref(), OperationMetricsKind::Diff, || { + self.diff_range_with_options(spec, patches, false) + }) } pub fn diff_range_with_options( @@ -11,12 +68,18 @@ impl Trail { patches: bool, line_changes: bool, ) -> Result { - let (left, right) = parse_range(spec)?; - self.diff_refs_with_options(left, right, patches, line_changes) + let metrics = self.operation_metrics.clone(); + profile_operation_metrics(metrics.as_ref(), OperationMetricsKind::Diff, || { + let (left, right) = parse_range(spec)?; + self.diff_refs_with_options(left, right, patches, line_changes) + }) } pub fn diff_refs(&self, left: &str, right: &str, patches: bool) -> Result { - self.diff_refs_with_options(left, right, patches, false) + let metrics = self.operation_metrics.clone(); + profile_operation_metrics(metrics.as_ref(), OperationMetricsKind::Diff, || { + self.diff_refs_with_options(left, right, patches, false) + }) } pub fn diff_refs_with_options( @@ -26,33 +89,78 @@ impl Trail { patches: bool, line_changes: bool, ) -> Result { - let left_ref = self.resolve_refish(left)?; - let right_ref = self.resolve_refish(right)?; - self.diff_root_files( - left.to_string(), - right.to_string(), - &left_ref.root_id, - &right_ref.root_id, - patches, - line_changes, - ) + let metrics = self.operation_metrics.clone(); + let result_metrics = metrics.clone(); + profile_operation_metrics(metrics.as_ref(), OperationMetricsKind::Diff, || { + let left_ref = self.resolve_refish(left)?; + let right_ref = self.resolve_refish(right)?; + let result = self.diff_root_files( + left.to_string(), + right.to_string(), + &left_ref.root_id, + &right_ref.root_id, + patches, + line_changes, + ); + if let (Some(metrics), Ok(summary)) = (&result_metrics, &result) { + metrics.add(OperationMetricsDelta { + final_path_count: saturating_u64_from_usize(summary.files.len()), + ..OperationMetricsDelta::default() + }); + } + result + }) } pub fn diff_roots(&self, spec: &str, patches: bool, line_changes: bool) -> Result { - let (left, right) = parse_range(spec)?; - let left_id = ObjectId(left.to_string()); - let right_id = ObjectId(right.to_string()); - self.diff_root_files( - left.to_string(), - right.to_string(), - &left_id, - &right_id, - patches, - line_changes, - ) + let metrics = self.operation_metrics.clone(); + let result_metrics = metrics.clone(); + profile_operation_metrics(metrics.as_ref(), OperationMetricsKind::Diff, || { + let (left, right) = parse_range(spec)?; + let left_id = ObjectId(left.to_string()); + let right_id = ObjectId(right.to_string()); + let result = self.diff_root_files( + left.to_string(), + right.to_string(), + &left_id, + &right_id, + patches, + line_changes, + ); + if let (Some(metrics), Ok(summary)) = (&result_metrics, &result) { + metrics.add(OperationMetricsDelta { + final_path_count: saturating_u64_from_usize(summary.files.len()), + ..OperationMetricsDelta::default() + }); + } + result + }) } pub fn diff_dirty(&mut self, patches: bool, line_changes: bool) -> Result { + let metrics = self.operation_metrics.clone(); + let result_metrics = metrics.clone(); + profile_operation_metrics(metrics.as_ref(), OperationMetricsKind::Diff, || { + let result = if crate::db::command_authority_enabled() { + self.note_operation_metrics(OperationMetricsDelta { + selected_worktree_index_sqlite_not_applicable_count: 1, + ..OperationMetricsDelta::default() + }); + self.diff_dirty_from_changed_path_ledger(patches, line_changes) + } else { + self.diff_dirty_profiled(patches, line_changes) + }; + if let (Some(metrics), Ok(summary)) = (&result_metrics, &result) { + metrics.add(OperationMetricsDelta { + final_path_count: saturating_u64_from_usize(summary.files.len()), + ..OperationMetricsDelta::default() + }); + } + result + }) + } + + fn diff_dirty_profiled(&mut self, patches: bool, line_changes: bool) -> Result { let _lock = self.acquire_write_lock()?; let branch = self.current_branch()?; let head = self.resolve_branch_ref(&branch)?; @@ -61,7 +169,8 @@ impl Trail { { return Ok(diff); } - let fast_dirty_paths = self.scan_git_dirty_tracked_paths()?; + let git_policy = self.workspace_ignore_policy_snapshot(); + let fast_dirty_paths = self.scan_git_dirty_tracked_paths_with_policy(&git_policy)?; let disk_files; let build_selected_paths; let previous_files; @@ -74,7 +183,8 @@ impl Trail { }); } previous_files = self.load_root_files_for_paths(&head.root_id, &paths)?; - let snapshot = self.selected_worktree_snapshot(&previous_files, &paths)?; + let snapshot = + self.selected_worktree_snapshot_with_policy(&previous_files, &paths, &git_policy)?; if snapshot.paths.is_empty() { return Ok(DiffSummary { from: branch, @@ -86,10 +196,9 @@ impl Trail { build_selected_paths = Some(snapshot.paths); } else { let refresh = self.refresh_worktree_index_streaming_report()?; + let baseline = self.worktree_index_baseline_root()?; if !refresh.changed - && self - .worktree_index_baseline_root()? - .is_some_and(|baseline| baseline == head.root_id.clone()) + && self.clean_baseline_matches_visible_root(baseline.as_ref(), &head.root_id) { return Ok(DiffSummary { from: branch, @@ -151,12 +260,15 @@ impl Trail { DaemonWorktreeSnapshot::Clean { generation: _, root_id: Some(clean_root), - } if clean_root == *root_id => { - return Ok(Some(DiffSummary { - from: branch.to_string(), - to: "dirty".to_string(), - files: Vec::new(), - })); + } => { + if self.clean_baseline_matches_visible_root(Some(&clean_root), root_id) { + return Ok(Some(DiffSummary { + from: branch.to_string(), + to: "dirty".to_string(), + files: Vec::new(), + })); + } + return Ok(None); } DaemonWorktreeSnapshot::Dirty { generation, paths } if paths.len() <= self.daemon_dirty_path_limit() => @@ -169,7 +281,9 @@ impl Trail { }; let previous_files = self.load_root_files_for_selections(root_id, &paths)?; - let snapshot = self.selected_worktree_snapshot(&previous_files, &paths)?; + let policy = self.workspace_ignore_policy_snapshot(); + let snapshot = + self.selected_worktree_snapshot_with_policy(&previous_files, &paths, &policy)?; self.reconcile_daemon_status_paths(root_id, &paths, &snapshot.summaries, generation); if snapshot.paths.is_empty() { return Ok(Some(DiffSummary { diff --git a/trail/src/db/record/inspection/history.rs b/trail/src/db/record/inspection/history.rs index 85c50d8..8f31738 100644 --- a/trail/src/db/record/inspection/history.rs +++ b/trail/src/db/record/inspection/history.rs @@ -23,10 +23,11 @@ impl Trail { } pub fn history_for_line_id(&self, line_id: &str) -> Result { + let storage_line_id = line_id_key_value(&parse_line_id_key(line_id)?); Ok(HistoryResult { selector: line_id.to_string(), file_history: Vec::new(), - line_history: self.line_history_by_line_id(line_id)?, + line_history: self.line_history_by_line_id(&storage_line_id)?, }) } @@ -34,7 +35,7 @@ impl Trail { let mut changes = Vec::new(); if let Some(lane) = selector.strip_prefix("lane:") { changes.extend(self.lane_change_ids(lane)?); - } else if selector.starts_with("msg_") { + } else if selector.starts_with(crate::ids::MESSAGE_ID_PREFIX) { let change_id: Option = self .conn .query_row( @@ -49,14 +50,24 @@ impl Trail { ))); }; changes.push(ChangeId(change_id)); - } else if selector.starts_with("ch_") { - changes.push(ChangeId(selector.to_string())); + } else if crate::ids::is_change_id(selector) + || ChangeId::from_checkpoint_alias(selector).is_some() + { + changes.push( + ChangeId::from_checkpoint_alias(selector) + .unwrap_or_else(|| ChangeId(selector.to_string())), + ); } else if selector.starts_with("session_") { changes.extend(self.session_change_ids(selector)?); - } else if let Ok(lane) = self.lane_branch(selector) { - changes.extend(self.lane_change_ids(&lane.lane_id)?); } else { - changes.extend(self.session_change_ids(selector)?); + match self.lane_branch(selector) { + Ok(lane) => { + changes.extend(self.lane_change_ids(&lane.lane_id)?); + } + _ => { + changes.extend(self.session_change_ids(selector)?); + } + } } let mut operations = Vec::new(); diff --git a/trail/src/db/record/inspection/objects.rs b/trail/src/db/record/inspection/objects.rs index 37c9bff..691302b 100644 --- a/trail/src/db/record/inspection/objects.rs +++ b/trail/src/db/record/inspection/objects.rs @@ -7,8 +7,11 @@ impl Trail { value: self.lane_branch(lane)?, }); } - if selector.starts_with("ch_") { - let operation = self.operation(&ChangeId(selector.to_string()))?; + if crate::ids::is_change_id(selector) || ChangeId::from_checkpoint_alias(selector).is_some() + { + let change_id = ChangeId::from_checkpoint_alias(selector) + .unwrap_or_else(|| ChangeId(selector.to_string())); + let operation = self.operation(&change_id)?; return Ok(ShowResult::Operation { value: OperationShow { changed_paths: summarize_file_changes(&operation.changes), @@ -17,12 +20,12 @@ impl Trail { }, }); } - if selector.starts_with("msg_") { + if selector.starts_with(crate::ids::MESSAGE_ID_PREFIX) { return Ok(ShowResult::Message { value: self.message(selector)?, }); } - if selector.starts_with("obj_") { + if crate::ids::is_object_id(selector) { return Ok(ShowResult::Object { value: self.object_info(selector)?, }); @@ -102,7 +105,7 @@ impl Trail { "anchor_id": anchor.id, "label": anchor.label, "file_id": file_id_key(&anchor.file_id), - "line_id": line_id_key_value(&anchor.line_id), + "line_id": anchor.line_id.alias(), "created_path": anchor.created_path, "created_line": anchor.created_line, "created_change": anchor.created_change, @@ -149,7 +152,7 @@ impl Trail { .enumerate() .map(|(idx, line)| TextLineInspect { line_number: idx as u64 + 1, - line_id: line.line_id_key(), + line_id: line.line_id.alias(), text_hash: line.text_hash, text: String::from_utf8_lossy(&line.text).into_owned(), newline: line.newline, diff --git a/trail/src/db/record/mod.rs b/trail/src/db/record/mod.rs index 7ca7afb..64f1c2b 100644 --- a/trail/src/db/record/mod.rs +++ b/trail/src/db/record/mod.rs @@ -7,3 +7,7 @@ mod checkout; mod diff; mod inspection; mod recording; +#[cfg(debug_assertions)] +pub(crate) use recording::{ + install_observed_record_after_compare_hook, install_observed_record_with_lock_hook, +}; diff --git a/trail/src/db/record/recording.rs b/trail/src/db/record/recording.rs index 7ae61cb..419aeab 100644 --- a/trail/src/db/record/recording.rs +++ b/trail/src/db/record/recording.rs @@ -2,4 +2,8 @@ use super::*; mod git; mod manual; +#[cfg(debug_assertions)] +pub(crate) use manual::{ + install_observed_record_after_compare_hook, install_observed_record_with_lock_hook, +}; mod timeline; diff --git a/trail/src/db/record/recording/git.rs b/trail/src/db/record/recording/git.rs index 44e6112..35bde13 100644 --- a/trail/src/db/record/recording/git.rs +++ b/trail/src/db/record/recording/git.rs @@ -1,6 +1,26 @@ use super::*; impl Trail { + /// Return a clean Git mapping only when the mapped commit is the exact + /// commit being qualified. Callers must still bind the returned Trail + /// change/root to their fenced ledger baseline; a matching HEAD alone is + /// never sufficient evidence that the workspace is clean. + pub(crate) fn clean_git_mapping_for_evidence( + &self, + head_oid: &str, + ) -> Result> { + self.conn + .query_row( + "SELECT crab_change,crab_root FROM git_mappings + WHERE git_head=?1 AND git_dirty=0 + ORDER BY created_at DESC,rowid DESC LIMIT 1", + [head_oid], + |row| Ok((row.get(0)?, ObjectId(row.get(1)?))), + ) + .optional() + .map_err(Error::from) + } + pub fn git_import_update( &mut self, branch: Option<&str>, diff --git a/trail/src/db/record/recording/manual.rs b/trail/src/db/record/recording/manual.rs index 6d183f9..f709d9b 100644 --- a/trail/src/db/record/recording/manual.rs +++ b/trail/src/db/record/recording/manual.rs @@ -1,5 +1,64 @@ use super::*; +#[cfg(debug_assertions)] +type ObservedRecordAfterCompareHook = Box Result<()> + Send>; + +#[cfg(debug_assertions)] +static OBSERVED_RECORD_AFTER_COMPARE_HOOK: std::sync::OnceLock< + std::sync::Mutex>, +> = std::sync::OnceLock::new(); + +#[cfg(debug_assertions)] +static OBSERVED_RECORD_WITH_LOCK_HOOK: std::sync::OnceLock< + std::sync::Mutex>, +> = std::sync::OnceLock::new(); + +#[cfg(debug_assertions)] +pub(crate) fn install_observed_record_after_compare_hook( + hook: impl FnOnce() -> Result<()> + Send + 'static, +) { + *OBSERVED_RECORD_AFTER_COMPARE_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Some(Box::new(hook)); +} + +#[cfg(debug_assertions)] +pub(crate) fn install_observed_record_with_lock_hook( + hook: impl FnOnce() -> Result<()> + Send + 'static, +) { + *OBSERVED_RECORD_WITH_LOCK_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Some(Box::new(hook)); +} + +#[cfg(debug_assertions)] +fn run_observed_record_after_compare_hook() -> Result<()> { + let hook = OBSERVED_RECORD_AFTER_COMPARE_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .take(); + if let Some(hook) = hook { + hook()?; + } + Ok(()) +} + +#[cfg(debug_assertions)] +fn run_observed_record_with_lock_hook() -> Result<()> { + let hook = OBSERVED_RECORD_WITH_LOCK_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .take(); + if let Some(hook) = hook { + hook()?; + } + Ok(()) +} + impl Trail { pub fn record( &mut self, @@ -8,19 +67,22 @@ impl Trail { actor: Actor, watch: bool, ) -> Result { - self.record_with_options( - branch, - message, - actor, - RecordOptions { - kind: Some(if watch { - OperationKind::WatchRecord - } else { - OperationKind::ManualRecord - }), - ..RecordOptions::default() - }, - ) + let metrics = self.operation_metrics.clone(); + profile_operation_metrics(metrics.as_ref(), OperationMetricsKind::Record, || { + self.record_with_options( + branch, + message, + actor, + RecordOptions { + kind: Some(if watch { + OperationKind::WatchRecord + } else { + OperationKind::ManualRecord + }), + ..RecordOptions::default() + }, + ) + }) } pub fn record_with_options( @@ -30,8 +92,18 @@ impl Trail { actor: Actor, options: RecordOptions, ) -> Result { - let _lock = self.acquire_write_lock()?; - self.record_with_options_unlocked(branch, message, actor, options) + let metrics = self.operation_metrics.clone(); + profile_operation_metrics(metrics.as_ref(), OperationMetricsKind::Record, || { + if crate::db::change_ledger::command_authority_enabled() { + // Native fence durability uses the same workspace writer + // exclusion. The observed path obtains c1/c2 first, then + // takes the lock for its single CAS transaction below. + self.record_with_options_unlocked(branch, message, actor, options) + } else { + let _lock = self.acquire_write_lock()?; + self.record_with_options_unlocked(branch, message, actor, options) + } + }) } pub(crate) fn record_with_options_unlocked( @@ -40,19 +112,68 @@ impl Trail { message: Option, actor: Actor, options: RecordOptions, + ) -> Result { + let metrics = self.operation_metrics.clone(); + let result_metrics = metrics.clone(); + profile_operation_metrics(metrics.as_ref(), OperationMetricsKind::Record, || { + let result = + self.record_with_options_unlocked_profiled(branch, message, actor, options); + if let (Some(metrics), Ok(report)) = (&result_metrics, &result) { + metrics.add(OperationMetricsDelta { + final_path_count: saturating_u64_from_usize(report.changed_paths.len()), + ..OperationMetricsDelta::default() + }); + } + result + }) + } + + fn record_with_options_unlocked_profiled( + &mut self, + branch: Option<&str>, + message: Option, + actor: Actor, + options: RecordOptions, ) -> Result { let branch = branch.map(str::to_string).unwrap_or(self.current_branch()?); let ref_name = branch_ref(&branch); let head = self.get_ref(&ref_name)?; + self.note_operation_metrics(OperationMetricsDelta { + input_path_count: saturating_u64_from_usize(options.paths.len()), + ..OperationMetricsDelta::default() + }); let selected_paths = normalize_record_paths(&options.paths)?; + let explicit_selections = (!selected_paths.is_empty()) + .then(|| SelectionSet::from_paths(&selected_paths)) + .transpose()?; + self.note_operation_metrics(OperationMetricsDelta { + canonical_path_count: explicit_selections + .as_ref() + .map(|selections| saturating_u64_from_usize(selections.as_slice().len())) + .unwrap_or(0), + ..OperationMetricsDelta::default() + }); let session_id = options .session_id + .clone() .map(|session_id| { validate_session_id(&session_id)?; self.lane_session(&session_id)?; Ok::(session_id) }) .transpose()?; + if crate::db::change_ledger::command_authority_enabled() + && selected_paths.is_empty() + && branch == self.current_branch()? + { + self.note_operation_metrics(OperationMetricsDelta { + selected_worktree_index_sqlite_not_applicable_count: 1, + ..OperationMetricsDelta::default() + }); + return self.record_from_changed_path_ledger( + branch, head, message, actor, options, session_id, + ); + } let daemon_snapshot = if selected_paths.is_empty() { self.daemon_worktree_snapshot() } else { @@ -70,13 +191,16 @@ impl Trail { Some(DaemonWorktreeSnapshot::Clean { generation: _, root_id: Some(clean_root), - }) if clean_root == head.root_id => { - return Ok(RecordReport { - branch, - operation: None, - root_id: head.root_id, - changed_paths: Vec::new(), - }); + }) => { + if self.clean_baseline_matches_visible_root(Some(&clean_root), &head.root_id) { + return Ok(RecordReport { + branch, + operation: None, + root_id: head.root_id, + changed_paths: Vec::new(), + }); + } + None } Some(DaemonWorktreeSnapshot::Dirty { generation, paths }) if paths.len() <= self.daemon_dirty_path_limit() => @@ -85,18 +209,25 @@ impl Trail { } _ => None, }; - let fast_dirty_paths = if selected_paths.is_empty() && daemon_dirty_paths.is_none() { - self.scan_git_dirty_tracked_paths()? + let git_policy = if selected_paths.is_empty() && daemon_dirty_paths.is_none() { + Some(self.workspace_ignore_policy_snapshot()) } else { None }; + let fast_dirty_paths = match git_policy.as_ref() { + Some(policy) => self.scan_git_dirty_tracked_paths_with_policy(policy)?, + None => None, + }; let disk_files; let build_selected_paths; + let build_selection_set; let previous_files; let record_generation; if let Some((generation, paths)) = daemon_dirty_paths { previous_files = self.load_root_files_for_selections(&head.root_id, &paths)?; - let snapshot = self.selected_worktree_snapshot(&previous_files, &paths)?; + let policy = self.workspace_ignore_policy_snapshot(); + let snapshot = + self.selected_worktree_snapshot_with_policy(&previous_files, &paths, &policy)?; self.reconcile_daemon_status_paths( &head.root_id, &paths, @@ -113,6 +244,7 @@ impl Trail { } disk_files = snapshot.files; build_selected_paths = Some(snapshot.paths); + build_selection_set = None; record_generation = Some(generation); } else if let Some(paths) = fast_dirty_paths { if paths.is_empty() { @@ -124,7 +256,13 @@ impl Trail { }); } previous_files = self.load_root_files_for_paths(&head.root_id, &paths)?; - let snapshot = self.selected_worktree_snapshot(&previous_files, &paths)?; + let snapshot = self.selected_worktree_snapshot_with_policy( + &previous_files, + &paths, + git_policy + .as_ref() + .expect("Git candidates have an operation policy snapshot"), + )?; if snapshot.paths.is_empty() { return Ok(RecordReport { branch, @@ -135,19 +273,28 @@ impl Trail { } disk_files = snapshot.files; build_selected_paths = Some(snapshot.paths); + build_selection_set = None; record_generation = fallback_generation; } else if !selected_paths.is_empty() { + let selections = explicit_selections + .as_ref() + .expect("nonempty explicit record paths have a selection set"); previous_files = self.load_root_files_for_selections(&head.root_id, &selected_paths)?; - disk_files = - self.scan_record_selection_files(&selected_paths, options.allow_ignored)?; + let policy = self.workspace_ignore_policy_snapshot(); + disk_files = self.scan_record_selection_files_with_policy( + &selected_paths, + selections, + options.allow_ignored, + &policy, + )?; build_selected_paths = Some(selected_paths.clone()); + build_selection_set = Some(selections.clone()); record_generation = None; } else { let refresh = self.refresh_worktree_index_streaming_report()?; + let baseline = self.worktree_index_baseline_root()?; if !refresh.changed - && self - .worktree_index_baseline_root()? - .is_some_and(|baseline| baseline == head.root_id.clone()) + && self.clean_baseline_matches_visible_root(baseline.as_ref(), &head.root_id) { return Ok(RecordReport { branch, @@ -173,18 +320,31 @@ impl Trail { disk_files = self.scan_visible_files_for_paths(&paths)?; previous_files = self.load_root_files_for_paths(&head.root_id, &paths)?; build_selected_paths = Some(paths); + build_selection_set = None; record_generation = fallback_generation; } let change_id = self.allocate_change_id(&actor.id, "record")?; let built = if let Some(paths) = build_selected_paths.as_deref() { - self.build_root_for_selected_record_incremental( - &head.root_id, - &previous_files, - &disk_files, - paths, - options.allow_ignored, - &change_id, - )? + if let Some(selections) = build_selection_set.as_ref() { + self.build_root_for_selected_record_incremental_with_selection_set( + &head.root_id, + &previous_files, + &disk_files, + paths, + selections, + options.allow_ignored, + &change_id, + )? + } else { + self.build_root_for_selected_record_incremental( + &head.root_id, + &previous_files, + &disk_files, + paths, + options.allow_ignored, + &change_id, + )? + } } else { self.build_root_from_disk_files(&disk_files, &change_id, Some(&previous_files))? }; @@ -226,4 +386,127 @@ impl Trail { changed_paths: diff.summaries, }) } + + fn record_from_changed_path_ledger( + &mut self, + branch: String, + head: RefRecord, + message: Option, + actor: Actor, + options: RecordOptions, + session_id: Option, + ) -> Result { + let branch_for_build = branch.clone(); + let actor_for_build = actor.clone(); + let message_for_build = message.clone(); + let session_for_build = session_id.clone(); + let kind = options.kind.unwrap_or(OperationKind::ManualRecord); + let (built_record, fenced) = + self.with_workspace_authoritative_command_snapshot(|db, policy, candidates, _git| { + let comparison = db.compare_authoritative_candidates( + policy, + candidates, + &head.root_id, + crate::db::change_ledger::CandidateMaterialization::RecordBytes, + )?; + if comparison.summaries.is_empty() { + return Ok(None); + } + #[cfg(debug_assertions)] + run_observed_record_after_compare_hook()?; + let change_id = db.allocate_change_id(&actor_for_build.id, "record")?; + let disk_files = comparison.disk_files.as_ref().ok_or_else(|| { + crate::Error::Corrupt( + "record candidate comparison omitted pinned contents".into(), + ) + })?; + let built = db.build_root_for_selected_disk_files_incremental( + &head.root_id, + &comparison.baseline_files, + disk_files, + &comparison.selections, + &change_id, + )?; + let built_manifest = built + .files + .iter() + .map(|(path, entry)| { + ( + path.clone(), + DiskManifest { + kind: entry.kind.clone(), + executable: entry.executable, + content_hash: entry.content_hash.clone(), + }, + ) + }) + .collect::>(); + if built_manifest != comparison.disk_manifest { + return Err(crate::Error::ChangeLedgerReconcileRequired { + scope: candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "authorized candidate contents changed during root construction" + .into(), + command: "trail record".into(), + }); + } + let diff = db.diff_file_maps(&comparison.baseline_files, &built.files)?; + if diff.changes.is_empty() { + return Ok(None); + } + Ok(Some(( + Operation { + version: OP_OBJECT_VERSION, + change_id: change_id.clone(), + kind: kind.clone(), + parents: vec![head.change_id.clone()], + before_root: Some(head.root_id.clone()), + after_root: built.root_id.clone(), + branch: branch_for_build.clone(), + actor: actor_for_build.clone(), + session_id: session_for_build.clone(), + message: message_for_build + .clone() + .map(|message| redact_sensitive_text(&message)), + changes: diff.changes, + created_at: now_ts(), + }, + built.root_id, + diff.summaries, + ))) + })?; + let Some((operation, root_id, changed_paths)) = built_record else { + return Ok(RecordReport { + branch, + operation: None, + root_id: head.root_id, + changed_paths: Vec::new(), + }); + }; + let observed = crate::db::change_ledger::ObservedRecordCut { + expected: fenced.candidates.expected.clone(), + c1: fenced.candidates.cut.clone(), + c2: fenced.c2, + acknowledgement_tokens: fenced.candidates.acknowledgement_tokens, + }; + let _observer_exclusion = + crate::db::begin_authorized_observer_write_exclusion(&self.db_dir); + let _lock = Self::with_write_lock_wait(std::time::Duration::from_secs(5), || { + self.acquire_write_lock() + })?; + // Keep exclusion through mirror repair, daemon baseline rebind, and + // worktree-index baseline publication. Narrowing it to the SQLite + // COMMIT would expose those shared persistent transitions to another + // writer even though the authoritative SQL transaction had finished. + #[cfg(debug_assertions)] + run_observed_record_with_lock_hook()?; + self.commit_observed_record(&operation, &head, &observed, true, None)?; + self.set_worktree_index_baseline(&root_id)?; + Ok(RecordReport { + branch, + operation: Some(operation.change_id), + root_id, + changed_paths, + }) + } } diff --git a/trail/src/db/storage/content.rs b/trail/src/db/storage/content.rs index 250ed81..0d30a9a 100644 --- a/trail/src/db/storage/content.rs +++ b/trail/src/db/storage/content.rs @@ -120,10 +120,23 @@ impl Trail { ) -> Result> { let root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, root_id)?; let tree = root_map_tree_from_root_hex(root.path_map_root.as_deref())?; + let paths = paths + .iter() + .map(|path| normalize_relative_path(path)) + .collect::>>()? + .into_iter() + .collect::>(); let mut out = BTreeMap::new(); - for path in paths { - let path = normalize_relative_path(path)?; - if let Some(value) = self.root_prolly.get(&tree, path.as_bytes())? { + if paths.is_empty() { + return Ok(out); + } + self.note_operation_metrics(OperationMetricsDelta { + root_point_key_count: saturating_u64_from_usize(paths.len()), + ..OperationMetricsDelta::default() + }); + let values = self.root_prolly.get_many(&tree, &paths)?; + for (path, value) in paths.into_iter().zip(values) { + if let Some(value) = value { out.insert(path, from_cbor(&value)?); } } @@ -137,25 +150,43 @@ impl Trail { ) -> Result> { let root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, root_id)?; let tree = root_map_tree_from_root_hex(root.path_map_root.as_deref())?; + let selections = SelectionSet::from_paths(selections)?; let mut out = BTreeMap::new(); - for selection in selections { - let selection = normalize_relative_path(selection)?; - if let Some(value) = self.root_prolly.get(&tree, selection.as_bytes())? { + if selections.as_slice().is_empty() { + return Ok(out); + } + self.note_operation_metrics(OperationMetricsDelta { + root_point_key_count: saturating_u64_from_usize(selections.as_slice().len()), + ..OperationMetricsDelta::default() + }); + let exact_values = self.root_prolly.get_many(&tree, selections.as_slice())?; + for (selection, value) in selections.as_slice().iter().zip(exact_values) { + if let Some(value) = value { out.insert(selection.clone(), from_cbor(&value)?); } + } + let mut root_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta::default(), + ); + for selection in selections.as_slice() { let prefix = format!("{selection}/"); let end = prefix_upper_bound(prefix.as_bytes()); + root_metrics.delta.bounded_root_range_count = root_metrics + .delta + .bounded_root_range_count + .saturating_add(1); let iter = self .root_prolly .range(&tree, prefix.as_bytes(), end.as_deref())?; for item in iter { let (key, value) = item?; + root_metrics.delta.root_range_row_count = + root_metrics.delta.root_range_row_count.saturating_add(1); let path = String::from_utf8(key) .map_err(|err| Error::Corrupt(format!("non UTF-8 path key: {err}")))?; - if path_matches_selection(&path, &selection) { - out.insert(path, from_cbor(&value)?); - } + out.insert(path, from_cbor(&value)?); } } Ok(out) @@ -166,6 +197,10 @@ impl Trail { root_id: &ObjectId, selections: &[String], ) -> Result> { + self.note_operation_metrics(OperationMetricsDelta { + root_point_key_count: saturating_u64_from_usize(selections.len()), + ..OperationMetricsDelta::default() + }); let root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, root_id)?; let tree = root_map_tree_from_root_hex(root.path_map_root.as_deref())?; let mut out = BTreeMap::new(); @@ -218,12 +253,21 @@ impl Trail { prefix: &str, out: &mut BTreeMap, ) -> Result<()> { + let mut root_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta { + bounded_root_range_count: 1, + ..OperationMetricsDelta::default() + }, + ); let end = prefix_upper_bound(prefix.as_bytes()); let iter = self .root_prolly .range(tree, prefix.as_bytes(), end.as_deref())?; for item in iter { let (key, value) = item?; + root_metrics.delta.root_range_row_count = + root_metrics.delta.root_range_row_count.saturating_add(1); let path = String::from_utf8(key) .map_err(|err| Error::Corrupt(format!("non UTF-8 path key: {err}")))?; out.insert(path, from_cbor(&value)?); @@ -244,12 +288,22 @@ impl Trail { } pub(crate) fn load_root_paths(&self, root_id: &ObjectId) -> Result> { + self.note_full_root_path_load(); + let mut root_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta { + full_root_range_count: 1, + ..OperationMetricsDelta::default() + }, + ); let root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, root_id)?; let tree = root_map_tree_from_root_hex(root.path_map_root.as_deref())?; let iter = self.root_prolly.range(&tree, &[], None)?; let mut paths = Vec::new(); for item in iter { let (key, _) = item?; + root_metrics.delta.root_range_row_count = + root_metrics.delta.root_range_row_count.saturating_add(1); let path = String::from_utf8(key) .map_err(|err| Error::Corrupt(format!("non UTF-8 path key: {err}")))?; paths.push(path); @@ -266,6 +320,14 @@ impl Trail { where F: FnMut(BTreeMap) -> Result<()>, { + self.note_full_root_path_load(); + let mut root_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta { + full_root_range_count: 1, + ..OperationMetricsDelta::default() + }, + ); let root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, root_id)?; let tree = root_map_tree_from_root_hex(root.path_map_root.as_deref())?; let iter = self.root_prolly.range(&tree, &[], None)?; @@ -273,6 +335,8 @@ impl Trail { let mut chunk = BTreeMap::new(); for item in iter { let (key, value) = item?; + root_metrics.delta.root_range_row_count = + root_metrics.delta.root_range_row_count.saturating_add(1); let path = String::from_utf8(key) .map_err(|err| Error::Corrupt(format!("non UTF-8 path key: {err}")))?; let entry: FileEntry = from_cbor(&value)?; @@ -603,7 +667,7 @@ impl Trail { continue; } WorkspaceCowMaterializeStatus::Skipped => {} - WorkspaceCowMaterializeStatus::Unavailable => { + WorkspaceCowMaterializeStatus::Unavailable(_) => { cow_available = false; } } @@ -626,7 +690,9 @@ impl Trail { ) -> Result { self.validate_streaming_root_case_collisions(root_id, output_root)?; let empty = BTreeMap::new(); - let clean_index_available = self.worktree_index_baseline_root()?.as_ref() == Some(root_id); + let baseline = self.worktree_index_baseline_root()?; + let clean_index_available = + self.clean_baseline_matches_visible_root(baseline.as_ref(), root_id); let mut report = RootMaterializationReport::default(); self.for_each_root_file_chunk(root_id, MATERIALIZE_BATCH_FILES, |chunk| { let mut chunk_report = None; @@ -891,6 +957,35 @@ mod tests { (workspace, db, head) } + fn root_with_path_values( + db: &Trail, + source_root_id: &ObjectId, + values: BTreeMap>, + ) -> ObjectId { + let source: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, source_root_id).unwrap(); + let mut builder = SortedBatchBuilder::new(db.store.clone(), root_map_prolly_config()); + for (path, value) in &values { + builder + .add(path.as_bytes().to_vec(), value.clone()) + .unwrap(); + } + let path_tree = builder.build().unwrap(); + let root = WorktreeRoot { + path_map_root: tree_root_hex(&path_tree), + file_count: values.len() as u64, + ..source + }; + db.put_object(WORKTREE_ROOT_KIND, ROOT_OBJECT_VERSION, &root) + .unwrap() + } + + fn fixture_entry(db: &Trail, root_id: &ObjectId) -> FileEntry { + db.load_root_files_for_paths(root_id, &["a.txt".to_string()]) + .unwrap() + .remove("a.txt") + .unwrap() + } + #[test] fn streaming_root_materialization_chunks_visit_sorted_files_once() { let (_workspace, db, head) = streaming_root_materialization_fixture(); @@ -968,6 +1063,153 @@ mod tests { .is_some()); } + #[test] + fn exact_root_load_batches_unique_keys_without_parent_collapsing_or_case_folding() { + let (_workspace, db, head) = streaming_root_materialization_fixture(); + let entry = fixture_entry(&db, &head.root_id); + let encoded = cbor(&entry).unwrap(); + let root_id = root_with_path_values( + &db, + &head.root_id, + BTreeMap::from([ + ("README.md".to_string(), encoded.clone()), + ("docs/child.txt".to_string(), encoded), + ]), + ); + let metrics = db.operation_metrics.as_ref().unwrap(); + let before = metrics.snapshot(); + + let files = db + .load_root_files_for_paths( + &root_id, + &[ + "README.md".to_string(), + "README.md".to_string(), + "readme.md".to_string(), + "docs".to_string(), + "docs/child.txt".to_string(), + "missing.txt".to_string(), + ], + ) + .unwrap(); + let after = metrics.snapshot(); + + assert_eq!( + files.keys().cloned().collect::>(), + ["README.md", "docs/child.txt"] + ); + assert_eq!(after.root_point_key_count - before.root_point_key_count, 5); + assert_eq!( + after.prolly_read_call_count - before.prolly_read_call_count, + 1 + ); + assert_eq!( + after.full_root_range_count - before.full_root_range_count, + 0 + ); + assert_eq!( + after.bounded_root_range_count - before.bounded_root_range_count, + 0 + ); + } + + #[test] + fn selected_root_load_collapses_overlap_and_ignores_ten_thousand_decoys() { + let (_workspace, db, head) = streaming_root_materialization_fixture(); + let entry = fixture_entry(&db, &head.root_id); + let encoded = cbor(&entry).unwrap(); + let mut values = BTreeMap::new(); + for idx in 0..10_000 { + values.insert(format!("decoy/{idx:05}.txt"), encoded.clone()); + } + values.insert("docs/a.txt".to_string(), encoded.clone()); + values.insert("docs/sub/b.txt".to_string(), encoded.clone()); + values.insert("docs-z/sibling.txt".to_string(), encoded); + let root_id = root_with_path_values(&db, &head.root_id, values); + let metrics = db.operation_metrics.as_ref().unwrap(); + let before = metrics.snapshot(); + + let files = db + .load_root_files_for_selections( + &root_id, + &[ + "docs/sub".to_string(), + "docs".to_string(), + "docs/a.txt".to_string(), + "docs".to_string(), + ], + ) + .unwrap(); + let after = metrics.snapshot(); + + assert_eq!( + files.keys().cloned().collect::>(), + ["docs/a.txt", "docs/sub/b.txt"] + ); + assert_eq!( + after.full_root_range_count - before.full_root_range_count, + 0 + ); + assert_eq!(after.root_point_key_count - before.root_point_key_count, 1); + assert_eq!( + after.bounded_root_range_count - before.bounded_root_range_count, + 1 + ); + assert_eq!(after.root_range_row_count - before.root_range_row_count, 2); + } + + #[test] + fn selected_root_load_does_not_report_work_when_root_resolution_fails() { + let (_workspace, db, _head) = streaming_root_materialization_fixture(); + let metrics = db.operation_metrics.as_ref().unwrap(); + let before = metrics.snapshot(); + + let err = db + .load_root_files_for_selections( + &ObjectId("missing-root".to_string()), + &["docs".to_string(), "docs/sub".to_string()], + ) + .unwrap_err(); + let after = metrics.snapshot(); + + assert!(matches!(err, Error::ObjectNotFound { .. })); + assert_eq!(after.root_point_key_count - before.root_point_key_count, 0); + assert_eq!( + after.bounded_root_range_count - before.bounded_root_range_count, + 0 + ); + assert_eq!(after.root_range_row_count - before.root_range_row_count, 0); + } + + #[test] + fn selected_root_load_counts_rows_consumed_before_decode_error() { + let (_workspace, db, head) = streaming_root_materialization_fixture(); + let entry = fixture_entry(&db, &head.root_id); + let root_id = root_with_path_values( + &db, + &head.root_id, + BTreeMap::from([ + ("docs/a.txt".to_string(), cbor(&entry).unwrap()), + ("docs/b.txt".to_string(), vec![0xff]), + ]), + ); + let metrics = db.operation_metrics.as_ref().unwrap(); + let before = metrics.snapshot(); + + let err = db + .load_root_files_for_selections(&root_id, &["docs".to_string()]) + .unwrap_err(); + let after = metrics.snapshot(); + + assert!(matches!(err, Error::Serialization(_))); + assert_eq!(after.root_point_key_count - before.root_point_key_count, 1); + assert_eq!( + after.bounded_root_range_count - before.bounded_root_range_count, + 1 + ); + assert_eq!(after.root_range_row_count - before.root_range_row_count, 2); + } + #[test] fn blob_projection_is_content_addressed_and_reused() { let (_workspace, db, head) = streaming_root_materialization_fixture(); diff --git a/trail/src/db/storage/diff.rs b/trail/src/db/storage/diff.rs index 29e1733..db5902b 100644 --- a/trail/src/db/storage/diff.rs +++ b/trail/src/db/storage/diff.rs @@ -1,4 +1,49 @@ use super::*; +use std::collections::VecDeque; + +#[derive(Default)] +pub(super) struct RenameMatchIndex { + by_hash_and_file_id: HashMap>>, +} + +impl RenameMatchIndex { + pub(super) fn insert(&mut self, path: String, entry: FileEntry) { + self.by_hash_and_file_id + .entry(entry.content_hash.clone()) + .or_default() + .entry(entry.file_id.clone()) + .or_default() + .push_back((path, entry)); + } + + pub(super) fn take(&mut self, entry: &FileEntry) -> Option<(String, FileEntry)> { + self.by_hash_and_file_id + .get_mut(&entry.content_hash) + .and_then(|by_file_id| by_file_id.get_mut(&entry.file_id)) + .and_then(VecDeque::pop_front) + } +} + +#[derive(Default)] +pub(super) struct RenameLookupProbes { + #[cfg(test)] + count: u64, +} + +impl RenameLookupProbes { + #[inline] + pub(super) fn note_lookup(&mut self) { + #[cfg(test)] + { + self.count = self.count.saturating_add(1); + } + } + + #[cfg(test)] + pub(super) fn count(&self) -> u64 { + self.count + } +} impl Trail { pub(crate) fn diff_root_files( @@ -58,11 +103,32 @@ impl Trail { &self, left: &BTreeMap, right: &BTreeMap, + ) -> Result { + let mut probes = RenameLookupProbes::default(); + self.diff_file_maps_indexed(left, right, &mut probes) + } + + #[cfg(test)] + fn diff_file_maps_with_rename_probe_count( + &self, + left: &BTreeMap, + right: &BTreeMap, + ) -> Result<(RootDiff, u64)> { + let mut probes = RenameLookupProbes::default(); + let diff = self.diff_file_maps_indexed(left, right, &mut probes)?; + Ok((diff, probes.count())) + } + + fn diff_file_maps_indexed( + &self, + left: &BTreeMap, + right: &BTreeMap, + probes: &mut RenameLookupProbes, ) -> Result { let mut added = Vec::new(); let mut removed = Vec::new(); let mut changed = Vec::new(); - let mut removed_by_hash: HashMap> = HashMap::new(); + let mut removed_by_identity = RenameMatchIndex::default(); for (path, entry) in left { match right.get(path) { Some(new_entry) @@ -75,10 +141,7 @@ impl Trail { Some(_) => {} None => { removed.push((path.clone(), entry.clone())); - removed_by_hash - .entry(entry.content_hash.clone()) - .or_default() - .push((path.clone(), entry.clone())); + removed_by_identity.insert(path.clone(), entry.clone()); } } } @@ -91,22 +154,16 @@ impl Trail { let mut changes = Vec::new(); let mut consumed_removed = HashSet::new(); for (path, new_entry) in added { - let rename = removed_by_hash - .get(&new_entry.content_hash) - .and_then(|candidates| { - candidates.iter().find(|(old_path, old_entry)| { - !consumed_removed.contains(old_path) - && old_entry.file_id == new_entry.file_id - }) - }); + probes.note_lookup(); + let rename = removed_by_identity.take(&new_entry); if let Some((old_path, old_entry)) = rename { consumed_removed.insert(old_path.clone()); changes.push(FileChange { path, - old_path: Some(old_path.clone()), + old_path: Some(old_path), file_id: Some(new_entry.file_id), kind: FileChangeKind::Renamed, - before_hash: Some(old_entry.content_hash.clone()), + before_hash: Some(old_entry.content_hash), after_hash: Some(new_entry.content_hash), line_changes: Vec::new(), }); @@ -167,3 +224,94 @@ impl Trail { Ok(RootDiff { changes, summaries }) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture_entry(db: &Trail, root_id: &ObjectId) -> FileEntry { + db.load_root_files(root_id) + .unwrap() + .into_values() + .next() + .unwrap() + } + + #[test] + fn indexed_rename_matching_is_linear_for_same_content_and_preserves_identity() { + const FILE_COUNT: usize = 1_000; + + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("seed.bin"), [0, 1, 2, 3]).unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let template = fixture_entry(&db, &head.root_id); + let mut left = BTreeMap::new(); + let mut right = BTreeMap::new(); + for index in 0..FILE_COUNT { + let mut entry = template.clone(); + entry.file_id = FileId::new(ChangeId("change_rename_source".to_string()), index as u64); + left.insert(format!("old/{index:04}.bin"), entry.clone()); + right.insert(format!("new/{index:04}.bin"), entry); + } + + let (diff, rename_identity_lookup_count) = db + .diff_file_maps_with_rename_probe_count(&left, &right) + .unwrap(); + + assert_eq!(diff.changes.len(), FILE_COUNT); + assert_eq!(diff.summaries.len(), FILE_COUNT); + assert_eq!(rename_identity_lookup_count, FILE_COUNT as u64); + for (index, change) in diff.changes.iter().enumerate() { + assert_eq!(change.kind, FileChangeKind::Renamed); + assert_eq!(change.path, format!("new/{index:04}.bin")); + assert_eq!( + change.old_path.as_deref(), + Some(format!("old/{index:04}.bin").as_str()) + ); + assert_eq!( + change.file_id, + Some(FileId::new( + ChangeId("change_rename_source".to_string()), + index as u64, + )) + ); + assert_eq!(diff.summaries[index].path, change.path); + assert_eq!(diff.summaries[index].old_path, change.old_path); + assert_eq!(diff.summaries[index].kind, FileChangeKind::Renamed); + } + } + + #[test] + fn indexed_rename_matching_preserves_first_unconsumed_path_order() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("seed.bin"), [0, 1, 2, 3]).unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let entry = fixture_entry(&db, &head.root_id); + let left = [ + ("old/a.bin".to_string(), entry.clone()), + ("old/b.bin".to_string(), entry.clone()), + ] + .into_iter() + .collect(); + let right = [ + ("new/a.bin".to_string(), entry.clone()), + ("new/b.bin".to_string(), entry), + ] + .into_iter() + .collect(); + + let diff = db.diff_file_maps(&left, &right).unwrap(); + + assert_eq!(diff.changes.len(), 2); + assert_eq!(diff.changes[0].path, "new/a.bin"); + assert_eq!(diff.changes[0].old_path.as_deref(), Some("old/a.bin")); + assert_eq!(diff.changes[1].path, "new/b.bin"); + assert_eq!(diff.changes[1].old_path.as_deref(), Some("old/b.bin")); + } +} diff --git a/trail/src/db/storage/files.rs b/trail/src/db/storage/files.rs index 880ebc8..2c81fd7 100644 --- a/trail/src/db/storage/files.rs +++ b/trail/src/db/storage/files.rs @@ -11,6 +11,369 @@ struct PathFileRead { } impl Trail { + /// Rejects agent/integration startup when any mutable live ref still uses a + /// non-empty legacy root without the persistent path-invariant index. + /// + /// This is deliberately read-only. The explicit recovery boundary remains + /// `trail index rebuild`, which repairs all live branch and lane refs in one + /// auditable operation. + pub fn ensure_live_path_invariant_indexes(&self) -> Result<()> { + let mut examined_roots = BTreeSet::new(); + for reference in self.all_refs()?.into_iter().filter(|reference| { + reference.name.starts_with(MAIN_REF_PREFIX) + || reference.name.starts_with(LANE_REF_PREFIX) + }) { + if !examined_roots.insert(reference.root_id.clone()) { + continue; + } + let root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, &reference.root_id)?; + if root.file_count > 0 && root.case_fold_map_root.is_none() { + return Err(Error::PathIndexRequired(format!( + "live ref `{}` has a legacy root with no case-fold index; run `trail index rebuild`", + reference.name + ))); + } + } + Ok(()) + } + + #[allow(dead_code)] // Task 5 resets metrics around scale scenarios. + pub(crate) fn reset_case_fold_index_metrics(&self) { + self.case_fold_index_metrics + .set(CaseFoldIndexMetrics::default()); + } + + #[allow(dead_code)] // Task 5 reports these metrics from scale scenarios. + pub(crate) fn case_fold_index_metrics_report(&self) -> CaseFoldIndexMetricsReport { + let metrics = self.case_fold_index_metrics.get(); + CaseFoldIndexMetricsReport { + mode: metrics.mode.as_str().to_string(), + lookup_count: metrics.lookup_count, + full_root_path_load_count: metrics.full_root_path_load_count, + full_filesystem_path_scan_count: metrics.full_filesystem_path_scan_count, + } + } + + pub(crate) fn note_full_root_path_load(&self) { + let mut metrics = self.case_fold_index_metrics.get(); + metrics.full_root_path_load_count = metrics.full_root_path_load_count.saturating_add(1); + self.case_fold_index_metrics.set(metrics); + } + + pub(crate) fn note_full_filesystem_path_scan(&self) { + let mut metrics = self.case_fold_index_metrics.get(); + metrics.full_filesystem_path_scan_count = + metrics.full_filesystem_path_scan_count.saturating_add(1); + self.case_fold_index_metrics.set(metrics); + } + + fn note_case_fold_index_lookups(&self, lookup_count: usize) { + let mut metrics = self.case_fold_index_metrics.get(); + metrics.mode = CaseFoldIndexMode::Indexed; + metrics.lookup_count = metrics.lookup_count.saturating_add(lookup_count as u64); + self.case_fold_index_metrics.set(metrics); + } + + pub(crate) fn build_case_fold_map_tree<'a, I>(&self, paths: I) -> Result + where + I: IntoIterator, + { + let mut mappings = BTreeMap::new(); + for path in paths { + insert_case_fold_mapping(&mut mappings, path)?; + } + + self.build_case_fold_map_tree_from_sorted_mappings(mappings) + } + + fn build_case_fold_map_tree_from_sorted_mappings( + &self, + mappings: BTreeMap, + ) -> Result { + let mut builder = SortedBatchBuilder::new(self.store.clone(), root_map_prolly_config()); + for (folded, canonical) in mappings { + builder.add(folded.into_bytes(), canonical.into_bytes())?; + } + Ok(builder.build()?) + } + + pub(crate) fn validate_and_update_case_fold_index( + &self, + previous_root: &WorktreeRoot, + removals: &[String], + additions: &[String], + ) -> Result { + let (previous_tree, mutations) = + self.plan_case_fold_index_update(previous_root, removals, additions)?; + if mutations.is_empty() { + return Ok(previous_tree); + } + self.note_operation_metrics(OperationMetricsDelta { + prolly_tree_batch_call_count: 1, + prolly_tree_batch_mutation_count: saturating_u64_from_usize(mutations.len()), + ..OperationMetricsDelta::default() + }); + Ok(self.root_prolly.batch(&previous_tree, mutations)?) + } + + pub(crate) fn validate_case_fold_index_update_read_only( + &self, + previous_root: &WorktreeRoot, + removals: &[String], + additions: &[String], + ) -> Result<()> { + self.plan_case_fold_index_update(previous_root, removals, additions)?; + Ok(()) + } + + fn plan_case_fold_index_update( + &self, + previous_root: &WorktreeRoot, + removals: &[String], + additions: &[String], + ) -> Result<(Tree, Vec)> { + for path in removals.iter().chain(additions) { + let normalized = normalize_relative_path(path)?; + if normalized != *path { + return Err(Error::InvalidPath { + path: path.clone(), + reason: format!("path must be normalized as `{normalized}`"), + }); + } + } + let previous_tree = match previous_root.case_fold_map_root.as_deref() { + Some(case_fold_root) => root_map_tree_from_root_hex(Some(case_fold_root))?, + None if previous_root.file_count == 0 => self.root_prolly.create(), + None => { + return Err(Error::PathIndexRequired( + "legacy root has no case-fold index; run `trail index rebuild`".to_string(), + )); + } + }; + let touched_keys = removals + .iter() + .chain(additions) + .map(|path| case_insensitive_path_key(path)) + .collect::>() + .into_iter() + .collect::>(); + self.note_case_fold_index_lookups(touched_keys.len()); + let existing = self.root_prolly.get_many(&previous_tree, &touched_keys)?; + let mut before = BTreeMap::new(); + for (folded, canonical) in touched_keys.into_iter().zip(existing) { + let canonical = canonical + .map(|value| validate_case_fold_index_value(&folded, value)) + .transpose()?; + before.insert(folded, canonical); + } + let mut after = before.clone(); + + for path in removals { + let folded = case_insensitive_path_key(path); + if after.get(&folded).and_then(Option::as_deref) == Some(path.as_str()) { + after.insert(folded, None); + } + } + for path in additions { + let folded = case_insensitive_path_key(path); + if let Some(previous) = after.get(&folded).and_then(Option::as_deref) { + if previous != path { + return Err(Error::InvalidPath { + path: path.clone(), + reason: format!("case-insensitive path collision with `{previous}`"), + }); + } + } + after.insert(folded, Some(path.clone())); + } + + let mutations = after + .into_iter() + .filter_map(|(folded, canonical)| { + (before.get(&folded) != Some(&canonical)).then(|| { + let key = folded.into_bytes(); + match canonical { + Some(canonical) => prolly::Mutation::Upsert { + key, + val: canonical.into_bytes(), + }, + None => prolly::Mutation::Delete { key }, + } + }) + }) + .collect::>(); + Ok((previous_tree, mutations)) + } + + pub(crate) fn resolve_record_case_fold_candidates_read_only( + &self, + previous_root_id: &ObjectId, + candidate_paths: &[String], + disk_manifest: &BTreeMap, + ) -> Result { + let previous_root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, previous_root_id)?; + let mut candidates_by_folded = BTreeMap::>::new(); + for path in candidate_paths { + let path = normalize_relative_path(path)?; + candidates_by_folded + .entry(case_insensitive_path_key(&path)) + .or_default() + .insert(path); + } + let mut selected_paths = candidates_by_folded + .values() + .flat_map(|paths| paths.iter().cloned()) + .collect::>(); + let candidate_present_paths = selected_paths + .iter() + .filter(|path| disk_manifest.contains_key(*path)) + .cloned() + .collect::>(); + + let previous_tree = match previous_root.case_fold_map_root.as_deref() { + Some(case_fold_root) => root_map_tree_from_root_hex(Some(case_fold_root))?, + None if previous_root.file_count == 0 => self.root_prolly.create(), + None => { + return Ok(RecordCaseFoldResolution { + expected_absent_paths: selected_paths + .difference(&candidate_present_paths) + .cloned() + .collect(), + selected_paths: selected_paths.into_iter().collect(), + expected_final_present_paths: candidate_present_paths.clone(), + expected_observed_present_paths: candidate_present_paths, + state: RecordCaseFoldResolutionState::LegacyUnavailable, + }); + } + }; + + let touched_keys = candidates_by_folded.keys().cloned().collect::>(); + self.note_case_fold_index_lookups(touched_keys.len()); + let existing = self.root_prolly.get_many(&previous_tree, &touched_keys)?; + + let mut mutations = Vec::new(); + let mut expected_final_present_paths = BTreeSet::new(); + let mut collision = None; + for ((folded, candidates), existing) in + candidates_by_folded.into_iter().zip(existing.into_iter()) + { + let previous = existing + .map(|value| validate_case_fold_index_value(&folded, value)) + .transpose()?; + if let Some(previous) = &previous { + selected_paths.insert(previous.clone()); + } + + let mut present = candidates + .iter() + .filter(|path| disk_manifest.contains_key(*path)) + .cloned() + .collect::>(); + if let Some(previous) = &previous { + if disk_manifest.contains_key(previous) || !candidates.contains(previous) { + present.insert(previous.clone()); + } + } + if present.len() > 1 { + let mut paths = present.into_iter(); + let previous = paths.next().expect("present path exists"); + let path = paths.next().expect("collision path exists"); + collision.get_or_insert((path, previous)); + continue; + } + let final_path = present.into_iter().next(); + if let Some(path) = &final_path { + expected_final_present_paths.insert(path.clone()); + } + if previous == final_path { + continue; + } + let key = folded.into_bytes(); + mutations.push(match final_path { + Some(path) => prolly::Mutation::Upsert { + key, + val: path.into_bytes(), + }, + None => prolly::Mutation::Delete { key }, + }); + } + + let state = if let Some((path, previous)) = collision { + RecordCaseFoldResolutionState::Collision { path, previous } + } else { + RecordCaseFoldResolutionState::Indexed { + previous_tree, + mutations, + } + }; + let expected_observed_present_paths = selected_paths + .iter() + .filter(|path| disk_manifest.contains_key(*path)) + .cloned() + .collect::>(); + let expected_absent_paths = selected_paths + .difference(&expected_observed_present_paths) + .cloned() + .collect(); + Ok(RecordCaseFoldResolution { + selected_paths: selected_paths.into_iter().collect(), + expected_final_present_paths, + expected_observed_present_paths, + expected_absent_paths, + state, + }) + } + + pub(crate) fn finalize_record_case_fold_resolution( + &self, + resolution: RecordCaseFoldResolution, + ) -> Result { + let RecordCaseFoldResolution { + selected_paths, + expected_final_present_paths, + expected_observed_present_paths, + expected_absent_paths, + state, + } = resolution; + let case_fold_tree = match state { + RecordCaseFoldResolutionState::Indexed { + previous_tree, + mutations, + } => { + if mutations.is_empty() { + previous_tree + } else { + self.note_operation_metrics(OperationMetricsDelta { + prolly_tree_batch_call_count: 1, + prolly_tree_batch_mutation_count: saturating_u64_from_usize( + mutations.len(), + ), + ..OperationMetricsDelta::default() + }); + self.root_prolly.batch(&previous_tree, mutations)? + } + } + RecordCaseFoldResolutionState::LegacyUnavailable => { + return Err(Error::PathIndexRequired( + "legacy root has no case-fold index; run `trail index rebuild`".to_string(), + )); + } + RecordCaseFoldResolutionState::Collision { path, previous } => { + return Err(Error::InvalidPath { + path, + reason: format!("case-insensitive path collision with `{previous}`"), + }); + } + }; + Ok(RecordCaseFoldPreflight { + selected_paths, + expected_final_present_paths, + expected_observed_present_paths, + expected_absent_paths, + case_fold_tree, + }) + } + pub(crate) fn build_root_from_git_tracked_paths( &self, paths: &[String], @@ -37,6 +400,7 @@ impl Trail { let mut files = BTreeMap::new(); let mut disk_manifest = BTreeMap::new(); + let mut case_fold_mappings = BTreeMap::new(); let mut path_builder = SortedBatchBuilder::new(self.store.clone(), root_map_prolly_config()); let mut file_index_builder = @@ -49,6 +413,7 @@ impl Trail { for chunk in paths.chunks(PATH_READ_BATCH) { let reads = read_path_file_batch(&self.workspace_root, chunk)?; for read in reads { + insert_case_fold_mapping(&mut case_fold_mappings, &read.path)?; let built = self.build_file_entry( &read.path, read.bytes, @@ -73,12 +438,15 @@ impl Trail { } } + let case_fold_tree = + self.build_case_fold_map_tree_from_sorted_mappings(case_fold_mappings)?; let path_tree = path_builder.build()?; let file_index_tree = file_index_builder.build()?; let root = WorktreeRoot { version: ROOT_OBJECT_VERSION, path_map_root: tree_root_hex(&path_tree), file_index_map_root: tree_root_hex(&file_index_tree), + case_fold_map_root: tree_root_hex(&case_fold_tree), file_count: files.len() as u64, total_text_bytes, created_by: change_id.clone(), @@ -99,27 +467,68 @@ impl Trail { change_id: &ChangeId, ) -> Result { let paths = normalize_root_build_paths(paths)?; - self.ensure_git_tracked_incremental_final_paths_safe(previous_root_id, &paths)?; + let mut accepted_paths = Vec::new(); + for path in paths { + let abs = self.workspace_root.join(path_from_rel(&path)); + let metadata = match fs::symlink_metadata(&abs) { + Ok(metadata) => metadata, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, + Err(err) => return Err(Error::Io(err)), + }; + if metadata.file_type().is_symlink() || !metadata.is_file() { + continue; + } + accepted_paths.push(path); + } let previous_root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, previous_root_id)?; let mut path_tree = root_map_tree_from_root_hex(previous_root.path_map_root.as_deref())?; let mut file_index_tree = root_map_tree_from_root_hex(previous_root.file_index_map_root.as_deref())?; let previous_tree = path_tree.clone(); - let new_paths = paths.iter().map(String::as_str).collect::>(); + let new_paths = accepted_paths + .iter() + .map(String::as_str) + .collect::>(); let mut previous_by_hash: HashMap> = HashMap::new(); let mut file_count = previous_root.file_count as i128; let mut total_text_bytes = previous_root.total_text_bytes as i128; let mut stats = ImportStats::default(); + let mut removed_entries = Vec::new(); + let mut existing_accepted_paths = HashSet::new(); + let mut root_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta { + full_root_range_count: 1, + ..OperationMetricsDelta::default() + }, + ); for item in self.root_prolly.range(&previous_tree, &[], None)? { let (key, value) = item?; + root_metrics.delta.root_range_row_count = + root_metrics.delta.root_range_row_count.saturating_add(1); let path = String::from_utf8(key) .map_err(|err| Error::Corrupt(format!("non UTF-8 path key: {err}")))?; let entry: FileEntry = from_cbor(&value)?; add_entry_import_stats(&mut stats, &entry); if new_paths.contains(path.as_str()) { + existing_accepted_paths.insert(path); continue; } + removed_entries.push((path, entry)); + } + let removals = removed_entries + .iter() + .map(|(path, _)| path.clone()) + .collect::>(); + let additions = accepted_paths + .iter() + .filter(|path| !existing_accepted_paths.contains(path.as_str())) + .cloned() + .collect::>(); + let case_fold_tree = + self.validate_and_update_case_fold_index(&previous_root, &removals, &additions)?; + for (path, entry) in removed_entries { path_tree = self.root_prolly.delete(&path_tree, path.as_bytes())?; file_index_tree = self .root_prolly @@ -136,7 +545,7 @@ impl Trail { let mut disk_manifest = BTreeMap::new(); let mut file_seq = 1; let mut line_seq = 1; - for chunk in paths.chunks(PATH_READ_BATCH) { + for chunk in accepted_paths.chunks(PATH_READ_BATCH) { let normalized_paths = chunk.to_vec(); let mut read_paths = Vec::new(); for path in normalized_paths { @@ -145,37 +554,11 @@ impl Trail { .get(&previous_tree, path.as_bytes())? .map(|value| from_cbor::(&value)) .transpose()?; - let abs = self.workspace_root.join(path_from_rel(&path)); - let metadata = match fs::symlink_metadata(&abs) { - Ok(metadata) => metadata, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - if let Some(previous_entry) = previous_same_path { - path_tree = self.root_prolly.delete(&path_tree, path.as_bytes())?; - file_index_tree = self - .root_prolly - .delete(&file_index_tree, &previous_entry.file_id.encode_key())?; - file_count -= 1; - total_text_bytes -= entry_text_bytes(&previous_entry) as i128; - remove_entry_import_stats(&mut stats, &previous_entry); - } - continue; - } - Err(err) => return Err(Error::Io(err)), - }; - if metadata.file_type().is_symlink() || !metadata.is_file() { - if let Some(previous_entry) = previous_same_path { - path_tree = self.root_prolly.delete(&path_tree, path.as_bytes())?; - file_index_tree = self - .root_prolly - .delete(&file_index_tree, &previous_entry.file_id.encode_key())?; - file_count -= 1; - total_text_bytes -= entry_text_bytes(&previous_entry) as i128; - remove_entry_import_stats(&mut stats, &previous_entry); - } - continue; - } - if let Some(previous_entry) = previous_same_path { + // The first stat preceded the previous-root range. Refresh + // metadata before trusting the unchanged-file cache, while + // still avoiding a content read for a true cache hit. + let metadata = fresh_regular_file_metadata(&self.workspace_root, &path)?; if previous_entry.executable == executable_from_metadata(&metadata) { if let Some(cached_manifest) = self.cached_worktree_manifest_for_metadata(&path, &metadata)? @@ -193,7 +576,7 @@ impl Trail { read_paths.push(path); } - let reads = read_path_file_batch(&self.workspace_root, &read_paths)?; + let reads = read_known_regular_path_file_batch(&self.workspace_root, &read_paths)?; for read in reads { let path = read.path; let content_hash = read.content_hash; @@ -269,6 +652,7 @@ impl Trail { version: ROOT_OBJECT_VERSION, path_map_root: tree_root_hex(&path_tree), file_index_map_root: tree_root_hex(&file_index_tree), + case_fold_map_root: tree_root_hex(&case_fold_tree), file_count: file_count as u64, total_text_bytes: total_text_bytes as u64, created_by: change_id.clone(), @@ -281,31 +665,6 @@ impl Trail { }) } - fn ensure_git_tracked_incremental_final_paths_safe( - &self, - previous_root_id: &ObjectId, - paths: &[String], - ) -> Result<()> { - let mut final_paths = self - .load_root_paths(previous_root_id)? - .into_iter() - .collect::>(); - for path in paths { - final_paths.remove(path); - let abs = self.workspace_root.join(path_from_rel(path)); - let metadata = match fs::symlink_metadata(&abs) { - Ok(metadata) => metadata, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, - Err(err) => return Err(Error::Io(err)), - }; - if metadata.file_type().is_symlink() || !metadata.is_file() { - continue; - } - final_paths.insert(path.clone()); - } - validate_no_case_fold_collisions(final_paths.iter()) - } - pub(crate) fn build_root_from_disk_files( &self, disk_files: &[DiskFile], @@ -385,13 +744,39 @@ impl Trail { allow_ignored: bool, change_id: &ChangeId, ) -> Result { - let selected_disk_files = - self.selected_record_disk_files(disk_files, selected_paths, allow_ignored)?; - self.build_root_for_selected_disk_files_incremental( + let selections = SelectionSet::from_paths(selected_paths)?; + self.build_root_for_selected_record_incremental_with_selection_set( previous_root_id, previous, - &selected_disk_files, + disk_files, + selected_paths, + &selections, + allow_ignored, + change_id, + ) + } + + pub(crate) fn build_root_for_selected_record_incremental_with_selection_set( + &self, + previous_root_id: &ObjectId, + previous: &BTreeMap, + disk_files: &[DiskFile], + selected_paths: &[String], + selections: &SelectionSet, + allow_ignored: bool, + change_id: &ChangeId, + ) -> Result { + let selected_disk_files = self.selected_record_disk_files_with_selection_set( + disk_files, selected_paths, + &selections, + allow_ignored, + )?; + self.build_root_for_selected_disk_files_incremental_with_selected_inputs( + previous_root_id, + previous, + selected_disk_files, + &selections, change_id, ) } @@ -404,59 +789,179 @@ impl Trail { selected_paths: &[String], change_id: &ChangeId, ) -> Result { - let selected_disk_files = disk_files + let selections = SelectionSet::from_paths(selected_paths)?; + let selected_disk_files = + self.selected_disk_files_for_selection_set(disk_files, &selections); + self.build_root_for_selected_disk_files_incremental_with_selected_inputs( + previous_root_id, + previous, + selected_disk_files, + &selections, + change_id, + ) + } + + fn build_root_for_selected_disk_files_incremental_with_selected_inputs( + &self, + previous_root_id: &ObjectId, + previous: &BTreeMap, + selected_disk_files: Vec, + selections: &SelectionSet, + change_id: &ChangeId, + ) -> Result { + validate_disk_file_root_paths(&selected_disk_files)?; + let removed_entries = self.selected_previous_entries(previous, selections); + let previous_root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, previous_root_id)?; + let removals = removed_entries .iter() - .filter(|file| { - selected_paths - .iter() - .any(|selected| path_matches_selection(&file.path, selected)) - }) - .cloned() + .map(|(path, _)| path.clone()) + .collect::>(); + let additions = selected_disk_files + .iter() + .map(|file| file.path.clone()) .collect::>(); - self.ensure_selected_disk_final_root_paths_safe( + let case_fold_tree = + self.validate_and_update_case_fold_index(&previous_root, &removals, &additions)?; + self.build_root_for_selected_disk_files_incremental_from_selected_inputs( previous_root_id, - selected_paths, - &selected_disk_files, + previous, + selected_disk_files, + removed_entries, + change_id, + case_fold_tree, + ) + } + + pub(crate) fn build_root_for_selected_disk_files_incremental_with_record_preflight( + &self, + worktree_root: &Path, + previous_root_id: &ObjectId, + previous: &BTreeMap, + disk_files: &[DiskFile], + selected_paths: &[String], + change_id: &ChangeId, + preflight: RecordCaseFoldPreflight, + ) -> Result { + let mut expected_domain = preflight.expected_observed_present_paths.clone(); + expected_domain.extend(preflight.expected_absent_paths.iter().cloned()); + if expected_domain + != preflight + .selected_paths + .iter() + .cloned() + .collect::>() + { + return Err(Error::Corrupt( + "record case-fold preflight has an inconsistent path domain".to_string(), + )); + } + let observed = observed_exact_paths_for_candidates( + worktree_root, + &preflight.selected_paths, + is_case_insensitive_filesystem(worktree_root)?, )?; + let observed_paths = observed.keys().cloned().collect::>(); + if observed_paths != preflight.expected_observed_present_paths + || observed + .values() + .any(|kind| *kind != ObservedPathKind::RegularFile) + { + return Err(Error::InvalidInput( + "record candidate paths changed after case-fold preflight (directory spellings)" + .to_string(), + )); + } + + let selections = SelectionSet::from_paths(selected_paths)?; + validate_disk_file_root_paths(disk_files)?; + let scanned_present_paths = disk_files + .iter() + .map(|file| file.path.clone()) + .collect::>(); + if scanned_present_paths != preflight.expected_observed_present_paths { + return Err(Error::InvalidInput( + "record candidate paths changed after case-fold preflight (selected scan)" + .to_string(), + )); + } + let selected_disk_files = + self.selected_disk_files_for_selection_set(disk_files, &selections); + validate_disk_file_root_paths(&selected_disk_files)?; + let actual_present_paths = selected_disk_files + .iter() + .map(|file| file.path.clone()) + .collect::>(); + let removed_entries = self.selected_previous_entries(previous, &selections); + let removed_paths = removed_entries + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(); + let mut actual_final_present_paths = previous + .keys() + .filter(|path| expected_domain.contains(*path)) + .filter(|path| !removed_paths.contains(path.as_str())) + .cloned() + .collect::>(); + actual_final_present_paths.extend(actual_present_paths); + if actual_final_present_paths != preflight.expected_final_present_paths { + return Err(Error::InvalidInput( + "record candidate paths changed after case-fold preflight (final domain)" + .to_string(), + )); + } + + self.build_root_for_selected_disk_files_incremental_from_selected_inputs( + previous_root_id, + previous, + selected_disk_files, + removed_entries, + change_id, + preflight.case_fold_tree, + ) + } + fn build_root_for_selected_disk_files_incremental_from_selected_inputs( + &self, + previous_root_id: &ObjectId, + previous: &BTreeMap, + mut selected_disk_files: Vec, + removed_entries: Vec<(String, FileEntry)>, + change_id: &ChangeId, + case_fold_tree: Tree, + ) -> Result { let previous_root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, previous_root_id)?; - let mut path_tree = root_map_tree_from_root_hex(previous_root.path_map_root.as_deref())?; - let mut file_index_tree = + let path_tree = root_map_tree_from_root_hex(previous_root.path_map_root.as_deref())?; + let file_index_tree = root_map_tree_from_root_hex(previous_root.file_index_map_root.as_deref())?; let mut file_count = previous_root.file_count as i128; let mut total_text_bytes = previous_root.total_text_bytes as i128; let mut files = previous.clone(); - let mut removed_entries = Vec::new(); - for selected in selected_paths { - let removed_paths = files - .keys() - .filter(|path| path_matches_selection(path, selected)) - .cloned() - .collect::>(); - for path in removed_paths { - if let Some(entry) = files.remove(&path) { - path_tree = self.root_prolly.delete(&path_tree, path.as_bytes())?; - file_index_tree = self - .root_prolly - .delete(&file_index_tree, &entry.file_id.encode_key())?; - file_count -= 1; - total_text_bytes -= entry_text_bytes(&entry) as i128; - removed_entries.push((path, entry)); - } + let mut previous_by_hash: HashMap> = HashMap::new(); + let mut path_mutation_intents = BTreeMap::, Option>>::new(); + let mut file_index_mutation_intents = BTreeMap::, Option>>::new(); + let selected_disk_paths = selected_disk_files + .iter() + .map(|file| file.path.clone()) + .collect::>(); + for (path, entry) in removed_entries { + let is_rename_candidate = !selected_disk_paths.contains(&path); + files.remove(&path); + path_mutation_intents.insert(path.as_bytes().to_vec(), None); + file_index_mutation_intents.insert(entry.file_id.encode_key(), None); + file_count -= 1; + total_text_bytes -= entry_text_bytes(&entry) as i128; + if is_rename_candidate { + previous_by_hash + .entry(entry.content_hash.clone()) + .or_default() + .push_back((path, entry)); } } - let mut previous_by_hash: HashMap> = HashMap::new(); - for (path, entry) in removed_entries { - previous_by_hash - .entry(entry.content_hash.clone()) - .or_default() - .push((path, entry)); - } - let mut file_seq = 1; let mut line_seq = 1; + selected_disk_files.sort_by(|left, right| left.path.cmp(&right.path)); for disk_file in selected_disk_files { let DiskFile { path, @@ -484,28 +989,24 @@ impl Trail { } } else { let previous_entry = previous_by_hash - .get(&content_hash) - .and_then(|matches| matches.first().map(|(_, entry)| entry)); + .get_mut(&content_hash) + .and_then(VecDeque::pop_front) + .map(|(_, entry)| entry); let built = self.build_file_entry( &path, bytes, content_hash, executable, change_id, - previous_entry, + previous_entry.as_ref(), &mut file_seq, &mut line_seq, )?; built.entry }; - path_tree = - self.root_prolly - .put(&path_tree, path.as_bytes().to_vec(), cbor(&entry)?)?; - file_index_tree = self.root_prolly.put( - &file_index_tree, - entry.file_id.encode_key(), - path.as_bytes().to_vec(), - )?; + path_mutation_intents.insert(path.as_bytes().to_vec(), Some(cbor(&entry)?)); + file_index_mutation_intents + .insert(entry.file_id.encode_key(), Some(path.as_bytes().to_vec())); file_count += 1; total_text_bytes += entry_text_bytes(&entry) as i128; files.insert(path, entry); @@ -517,11 +1018,16 @@ impl Trail { )); } + let path_tree = self.apply_root_tree_mutation_intents(path_tree, path_mutation_intents)?; + let file_index_tree = + self.apply_root_tree_mutation_intents(file_index_tree, file_index_mutation_intents)?; + let (stats, _) = root_stats(&files); let root = WorktreeRoot { version: ROOT_OBJECT_VERSION, path_map_root: tree_root_hex(&path_tree), file_index_map_root: tree_root_hex(&file_index_tree), + case_fold_map_root: tree_root_hex(&case_fold_tree), file_count: file_count as u64, total_text_bytes: total_text_bytes as u64, created_by: change_id.clone(), @@ -535,22 +1041,77 @@ impl Trail { }) } - fn ensure_selected_disk_final_root_paths_safe( + fn selected_disk_files_for_selection_set( &self, - previous_root_id: &ObjectId, - selected_paths: &[String], - selected_disk_files: &[DiskFile], - ) -> Result<()> { - let mut paths = self - .load_root_paths(previous_root_id)? - .into_iter() - .collect::>(); - for selected in selected_paths { - let selected = normalize_relative_path(selected)?; - paths.retain(|path| !path_matches_selection(path, &selected)); + disk_files: &[DiskFile], + selections: &SelectionSet, + ) -> Vec { + if selections.is_empty() { + return Vec::new(); + } + let mut comparison_count = 0u64; + let selected = disk_files + .iter() + .filter(|file| { + let (matches, comparisons) = selections.contains_counted(&file.path); + comparison_count = comparison_count.saturating_add(comparisons); + matches + }) + .cloned() + .collect(); + self.note_operation_metrics(OperationMetricsDelta { + selection_comparison_count: comparison_count, + ..OperationMetricsDelta::default() + }); + selected + } + + fn selected_previous_entries( + &self, + previous: &BTreeMap, + selections: &SelectionSet, + ) -> Vec<(String, FileEntry)> { + if selections.is_empty() { + return Vec::new(); + } + let mut comparison_count = 0u64; + let selected = previous + .iter() + .filter(|(path, _)| { + let (matches, comparisons) = selections.contains_counted(path); + comparison_count = comparison_count.saturating_add(comparisons); + matches + }) + .map(|(path, entry)| (path.clone(), entry.clone())) + .collect(); + self.note_operation_metrics(OperationMetricsDelta { + selection_comparison_count: comparison_count, + ..OperationMetricsDelta::default() + }); + selected + } + + fn apply_root_tree_mutation_intents( + &self, + tree: Tree, + intents: BTreeMap, Option>>, + ) -> Result { + if intents.is_empty() { + return Ok(tree); } - paths.extend(selected_disk_files.iter().map(|file| file.path.clone())); - validate_no_case_fold_collisions(paths.iter()) + let mutations = intents + .into_iter() + .map(|(key, value)| match value { + Some(val) => prolly::Mutation::Upsert { key, val }, + None => prolly::Mutation::Delete { key }, + }) + .collect::>(); + self.note_operation_metrics(OperationMetricsDelta { + prolly_tree_batch_call_count: 1, + prolly_tree_batch_mutation_count: saturating_u64_from_usize(mutations.len()), + ..OperationMetricsDelta::default() + }); + Ok(self.root_prolly.batch(&tree, mutations)?) } pub(crate) fn build_root_from_touched_file_entries_incremental( @@ -560,8 +1121,27 @@ impl Trail { target: &BTreeMap, change_id: &ChangeId, ) -> Result { - self.ensure_touched_final_root_paths_safe(previous_root_id, previous, target)?; + let previous_root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, previous_root_id)?; + let (removals, additions) = touched_path_map_changes(previous, target); + let case_fold_tree = + self.validate_and_update_case_fold_index(&previous_root, &removals, &additions)?; + self.build_root_from_touched_file_entries_incremental_with_case_fold_tree( + previous_root_id, + previous, + target, + change_id, + case_fold_tree, + ) + } + pub(crate) fn build_root_from_touched_file_entries_incremental_with_case_fold_tree( + &self, + previous_root_id: &ObjectId, + previous: &BTreeMap, + target: &BTreeMap, + change_id: &ChangeId, + case_fold_tree: Tree, + ) -> Result { let previous_root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, previous_root_id)?; let mut path_tree = root_map_tree_from_root_hex(previous_root.path_map_root.as_deref())?; let mut file_index_tree = @@ -613,6 +1193,7 @@ impl Trail { version: ROOT_OBJECT_VERSION, path_map_root: tree_root_hex(&path_tree), file_index_map_root: tree_root_hex(&file_index_tree), + case_fold_map_root: tree_root_hex(&case_fold_tree), file_count: file_count as u64, total_text_bytes: total_text_bytes as u64, created_by: change_id.clone(), @@ -621,28 +1202,6 @@ impl Trail { Ok(IncrementalRootBuildResult { root_id }) } - fn ensure_touched_final_root_paths_safe( - &self, - previous_root_id: &ObjectId, - previous: &BTreeMap, - target: &BTreeMap, - ) -> Result<()> { - validate_no_case_fold_collisions(target.keys())?; - if target.keys().all(|path| previous.contains_key(path)) { - return Ok(()); - } - - let mut paths = self - .load_root_paths(previous_root_id)? - .into_iter() - .collect::>(); - for path in previous.keys() { - paths.remove(path); - } - paths.extend(target.keys().cloned()); - validate_no_case_fold_collisions(paths.iter()) - } - pub(crate) fn build_root_from_file_entries( &self, files: BTreeMap, @@ -672,6 +1231,7 @@ impl Trail { ) -> Result { validate_no_case_fold_collisions(files.keys())?; validate_no_case_fold_collisions(disk_manifest.keys())?; + let case_fold_tree = self.build_case_fold_map_tree(files.keys())?; let mut path_builder = SortedBatchBuilder::new(self.store.clone(), root_map_prolly_config()); @@ -688,6 +1248,7 @@ impl Trail { version: ROOT_OBJECT_VERSION, path_map_root: tree_root_hex(&path_tree), file_index_map_root: tree_root_hex(&file_index_tree), + case_fold_map_root: tree_root_hex(&case_fold_tree), file_count: files.len() as u64, total_text_bytes, created_by: change_id.clone(), @@ -702,6 +1263,61 @@ impl Trail { } } +fn validate_case_fold_index_value(folded: &str, value: Vec) -> Result { + let canonical = String::from_utf8(value).map_err(|err| { + Error::Corrupt(format!( + "case-fold index key {folded:?} stores non UTF-8 canonical path bytes: {err}" + )) + })?; + let normalized = normalize_relative_path(&canonical).map_err(|err| { + Error::Corrupt(format!( + "case-fold index key {folded:?} stores invalid canonical path {canonical:?}: {err}" + )) + })?; + if normalized != canonical { + return Err(Error::Corrupt(format!( + "case-fold index key {folded:?} stores noncanonical path {canonical:?}; canonical path must be normalized as {normalized:?}" + ))); + } + let canonical_folded = case_insensitive_path_key(&canonical); + if canonical_folded != folded { + return Err(Error::Corrupt(format!( + "case-fold index key {folded:?} stores canonical path {canonical:?}, which folds to {canonical_folded:?}" + ))); + } + Ok(canonical) +} + +fn insert_case_fold_mapping(mappings: &mut BTreeMap, path: &str) -> Result<()> { + let folded = case_insensitive_path_key(path); + if let Some(previous) = mappings.insert(folded, path.to_string()) { + if previous != path { + return Err(Error::InvalidPath { + path: path.to_string(), + reason: format!("case-insensitive path collision with `{previous}`"), + }); + } + } + Ok(()) +} + +fn touched_path_map_changes( + previous: &BTreeMap, + target: &BTreeMap, +) -> (Vec, Vec) { + let removals = previous + .keys() + .filter(|path| !target.contains_key(*path)) + .cloned() + .collect(); + let additions = target + .keys() + .filter(|path| !previous.contains_key(*path)) + .cloned() + .collect(); + (removals, additions) +} + fn normalize_root_build_paths(paths: &[String]) -> Result> { let mut normalized = BTreeSet::new(); for path in paths { @@ -738,6 +1354,66 @@ fn read_path_file_batch(root: &Path, paths: &[String]) -> Result Result> { + paths + .par_iter() + .map(|path| { + let abs = root.join(path_from_rel(path)); + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC); + } + #[cfg(not(unix))] + { + let metadata = fs::symlink_metadata(&abs)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(Error::InvalidInput(format!( + "tracked path `{path}` changed file type during import" + ))); + } + } + let mut file = options.open(&abs)?; + let metadata = file.metadata()?; + if !metadata.is_file() { + return Err(Error::InvalidInput(format!( + "tracked path `{path}` changed file type during import" + ))); + } + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + #[cfg(not(unix))] + { + let final_metadata = fs::symlink_metadata(&abs)?; + if final_metadata.file_type().is_symlink() || !final_metadata.is_file() { + return Err(Error::InvalidInput(format!( + "tracked path `{path}` changed file type during import" + ))); + } + } + let content_hash = sha256_hex(&bytes); + Ok(PathFileRead { + path: path.clone(), + bytes, + executable: executable_from_metadata(&metadata), + content_hash, + }) + }) + .collect() +} + +fn fresh_regular_file_metadata(root: &Path, path: &str) -> Result { + let metadata = fs::symlink_metadata(root.join(path_from_rel(path)))?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(Error::InvalidInput(format!( + "tracked path `{path}` changed file type during import" + ))); + } + Ok(metadata) +} + fn read_path_file(root: &Path, path: &str) -> Result> { let abs = root.join(path_from_rel(path)); let metadata = match fs::symlink_metadata(&abs) { @@ -800,191 +1476,1935 @@ fn root_stats(files: &BTreeMap) -> (ImportStats, u64) { mod tests { use super::*; + fn assert_case_fold_mapping( + db: &Trail, + root_id: &ObjectId, + folded_path: &str, + canonical_path: &str, + ) { + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, root_id).unwrap(); + let tree = root_map_tree_from_root_hex(root.case_fold_map_root.as_deref()).unwrap(); + assert_eq!( + db.root_prolly.get(&tree, folded_path.as_bytes()).unwrap(), + Some(canonical_path.as_bytes().to_vec()) + ); + } + + fn root_map_entries(db: &Trail, root_hex: Option<&str>) -> Vec<(Vec, Vec)> { + let tree = root_map_tree_from_root_hex(root_hex).unwrap(); + db.root_prolly + .range(&tree, &[], None) + .unwrap() + .map(|item| item.unwrap()) + .collect() + } + + fn root_with_case_fold_value( + db: &Trail, + root: &WorktreeRoot, + folded: &str, + canonical: &str, + ) -> WorktreeRoot { + let tree = root_map_tree_from_root_hex(root.case_fold_map_root.as_deref()).unwrap(); + let tree = db + .root_prolly + .put( + &tree, + folded.as_bytes().to_vec(), + canonical.as_bytes().to_vec(), + ) + .unwrap(); + let mut corrupt = root.clone(); + corrupt.case_fold_map_root = tree_root_hex(&tree); + corrupt + } + #[test] - fn touched_incremental_root_rejects_final_case_fold_collisions_before_objects() { + fn legacy_worktree_root_deserialization_defaults_case_fold_index_to_none() { + #[derive(Serialize)] + struct LegacyWorktreeRoot { + version: u16, + path_map_root: Option, + file_index_map_root: Option, + file_count: u64, + total_text_bytes: u64, + created_by: ChangeId, + } + + let bytes = cbor(&LegacyWorktreeRoot { + version: ROOT_OBJECT_VERSION, + path_map_root: Some("path-root".to_string()), + file_index_map_root: Some("file-index-root".to_string()), + file_count: 2, + total_text_bytes: 12, + created_by: ChangeId("change_legacy_root".to_string()), + }) + .unwrap(); + + let root: WorktreeRoot = from_cbor(&bytes).unwrap(); + assert_eq!(root.case_fold_map_root, None); + } + + #[test] + fn indexed_case_fold_update_renames_a_path_and_preserves_untouched_entries() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + fs::write(temp.path().join("LICENSE"), "license\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let db = Trail::open(temp.path()).unwrap(); let head = db.get_ref("refs/branches/main").unwrap(); - let previous = db - .load_root_files_for_paths(&head.root_id, &["README.md".to_string()]) - .unwrap(); - let mut target = previous.clone(); - target.insert( - "readme.md".to_string(), - previous.get("README.md").unwrap().clone(), - ); - let count_rows = |table: &str| -> i64 { - db.conn - .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { - row.get(0) - }) - .unwrap() - }; - let objects_before = count_rows("objects"); - let prolly_nodes_before = count_rows("prolly_nodes"); - - let err = db - .build_root_from_touched_file_entries_incremental( - &head.root_id, - &previous, - &target, - &ChangeId("ch_collision_test".to_string()), + let mut root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + root.path_map_root = Some("path-tree-must-not-be-loaded".to_string()); + root.file_index_map_root = Some("file-index-tree-must-not-be-loaded".to_string()); + + let next = db + .validate_and_update_case_fold_index( + &root, + &["README.md".to_string()], + &["docs/Guide.md".to_string()], ) - .unwrap_err(); + .unwrap(); - assert!( - matches!(err, Error::InvalidPath { ref reason, .. } if reason.contains("case-insensitive path collision")), - "expected final root collision, got {err:?}" + assert_eq!(db.root_prolly.get(&next, b"readme.md").unwrap(), None); + assert_eq!( + db.root_prolly.get(&next, b"docs/guide.md").unwrap(), + Some(b"docs/Guide.md".to_vec()) + ); + assert_eq!( + db.root_prolly.get(&next, b"license").unwrap(), + Some(b"LICENSE".to_vec()) ); - assert_eq!(count_rows("objects"), objects_before); - assert_eq!(count_rows("prolly_nodes"), prolly_nodes_before); } #[test] - fn full_file_entry_root_rejects_case_fold_collisions_before_objects() { + fn indexed_case_fold_update_applies_removal_before_case_only_addition() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let db = Trail::open(temp.path()).unwrap(); let head = db.get_ref("refs/branches/main").unwrap(); - let mut files = db.load_root_files(&head.root_id).unwrap(); - let entry = files.get("README.md").unwrap().clone(); - files.insert("README.md".to_string(), entry); - let count_rows = |table: &str| -> i64 { - db.conn - .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { - row.get(0) - }) - .unwrap() - }; - let objects_before = count_rows("objects"); - let prolly_nodes_before = count_rows("prolly_nodes"); + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); - let err = db - .build_root_from_file_entries(files, &ChangeId("ch_full_collision_test".to_string())) - .unwrap_err(); + let next = db + .validate_and_update_case_fold_index( + &root, + &["README.md".to_string()], + &["readme.md".to_string()], + ) + .unwrap(); - assert!( - matches!(err, Error::InvalidPath { ref reason, .. } if reason.contains("case-insensitive path collision")), - "expected full root collision, got {err:?}" + assert_eq!( + db.root_prolly.get(&next, b"readme.md").unwrap(), + Some(b"readme.md".to_vec()) ); - assert_eq!(count_rows("objects"), objects_before); - assert_eq!(count_rows("prolly_nodes"), prolly_nodes_before); } #[test] - fn disk_file_root_rejects_case_fold_collisions_before_objects() { + fn indexed_case_fold_update_adds_the_first_path_to_an_empty_root() { let temp = tempfile::tempdir().unwrap(); Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); let db = Trail::open(temp.path()).unwrap(); - let disk_files = vec![ - DiskFile { - path: "README.md".to_string(), - bytes: b"hello\n".to_vec(), - executable: false, - }, - DiskFile { - path: "readme.md".to_string(), - bytes: b"collision\n".to_vec(), - executable: false, - }, - ]; - let count_rows = |table: &str| -> i64 { - db.conn - .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { - row.get(0) - }) - .unwrap() - }; - let objects_before = count_rows("objects"); - let prolly_nodes_before = count_rows("prolly_nodes"); + let head = db.get_ref("refs/branches/main").unwrap(); + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + assert_eq!(root.file_count, 0); + assert_eq!(root.case_fold_map_root, None); - let err = db - .build_root_from_disk_files( - &disk_files, - &ChangeId("ch_disk_file_collision_test".to_string()), - None, - ) - .unwrap_err(); + let next = db + .validate_and_update_case_fold_index(&root, &[], &["docs/ReadMe.md".to_string()]) + .unwrap(); - assert!( - matches!(err, Error::InvalidPath { ref reason, .. } if reason.contains("case-insensitive path collision")), - "expected disk-file root collision, got {err:?}" + assert_eq!( + db.root_prolly.get(&next, b"docs/readme.md").unwrap(), + Some(b"docs/ReadMe.md".to_vec()) ); - assert_eq!(count_rows("objects"), objects_before); - assert_eq!(count_rows("prolly_nodes"), prolly_nodes_before); } #[test] - fn path_list_root_build_rejects_case_fold_collisions_before_objects() { + fn indexed_case_fold_update_empty_root_collision_writes_no_nodes() { let temp = tempfile::tempdir().unwrap(); Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); - fs::write(temp.path().join("README.md"), "hello\n").unwrap(); - fs::write(temp.path().join("readme.md"), "collision\n").unwrap(); let db = Trail::open(temp.path()).unwrap(); - let count_rows = |table: &str| -> i64 { - db.conn - .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { - row.get(0) - }) - .unwrap() - }; - let objects_before = count_rows("objects"); - let prolly_nodes_before = count_rows("prolly_nodes"); + let head = db.get_ref("refs/branches/main").unwrap(); + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + let prolly_nodes_before: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row.get(0)) + .unwrap(); let err = db - .build_root_from_worktree_paths( + .validate_and_update_case_fold_index( + &root, + &[], &["README.md".to_string(), "readme.md".to_string()], - &ChangeId("ch_path_list_collision_test".to_string()), ) .unwrap_err(); - assert!( - matches!(err, Error::InvalidPath { ref reason, .. } if reason.contains("case-insensitive path collision")), - "expected path-list root collision, got {err:?}" + assert!(matches!( + err, + Error::InvalidPath { ref path, ref reason } + if path == "readme.md" + && reason == "case-insensitive path collision with `README.md`" + )); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + prolly_nodes_before ); - assert_eq!(count_rows("objects"), objects_before); - assert_eq!(count_rows("prolly_nodes"), prolly_nodes_before); } #[test] - fn git_tracked_incremental_root_rejects_final_case_fold_collisions_before_objects() { + fn indexed_case_fold_update_rejects_traversal_and_non_nfc_paths_without_writes() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); - fs::write(temp.path().join("readme.md"), "collision\n").unwrap(); let db = Trail::open(temp.path()).unwrap(); let head = db.get_ref("refs/branches/main").unwrap(); - let count_rows = |table: &str| -> i64 { + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + let prolly_nodes_before: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row.get(0)) + .unwrap(); + + let traversal = db + .validate_and_update_case_fold_index(&root, &["../README.md".to_string()], &[]) + .unwrap_err(); + assert!(matches!( + traversal, + Error::InvalidPath { ref reason, .. } if reason.contains("inside the workspace") + )); + + let non_nfc = db + .validate_and_update_case_fold_index(&root, &[], &["docs/cafe\u{0301}.md".to_string()]) + .unwrap_err(); + assert!(matches!( + non_nfc, + Error::InvalidPath { ref reason, .. } if reason.contains("Unicode NFC") + )); + assert_eq!( db.conn - .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { - row.get(0) - }) - .unwrap() - }; - let objects_before = count_rows("objects"); - let prolly_nodes_before = count_rows("prolly_nodes"); + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + prolly_nodes_before + ); + } - let err = db - .build_root_from_git_tracked_paths_incremental( - &["readme.md".to_string()], - &head.root_id, - &ChangeId("ch_git_incremental_collision_test".to_string()), + #[test] + fn indexed_case_fold_update_rejects_ascii_and_unicode_within_batch_collisions() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("LICENSE"), "license\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + + let ascii = db + .validate_and_update_case_fold_index( + &root, + &[], + &["README.md".to_string(), "readme.md".to_string()], + ) + .unwrap_err(); + assert!(matches!( + ascii, + Error::InvalidPath { ref path, ref reason } + if path == "readme.md" + && reason == "case-insensitive path collision with `README.md`" + )); + + let unicode = db + .validate_and_update_case_fold_index( + &root, + &[], + &["src/Kernel.rs".to_string(), "src/kernel.rs".to_string()], ) .unwrap_err(); + assert!(matches!( + unicode, + Error::InvalidPath { ref path, ref reason } + if path == "src/kernel.rs" + && reason == "case-insensitive path collision with `src/Kernel.rs`" + )); + } - assert!( - matches!(err, Error::InvalidPath { ref reason, .. } if reason.contains("case-insensitive path collision")), - "expected git-tracked incremental collision, got {err:?}" - ); + #[test] + fn indexed_case_fold_update_rejects_collision_with_existing_root_without_writes() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + let prolly_nodes_before: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row.get(0)) + .unwrap(); + + let err = db + .validate_and_update_case_fold_index(&root, &[], &["readme.md".to_string()]) + .unwrap_err(); + + assert!(matches!( + err, + Error::InvalidPath { ref path, ref reason } + if path == "readme.md" + && reason == "case-insensitive path collision with `README.md`" + )); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + prolly_nodes_before + ); + } + + #[test] + fn indexed_case_fold_update_rejects_stored_key_value_mismatch_without_writes() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + let corrupt = root_with_case_fold_value(&db, &root, "readme.md", "Other.md"); + let objects_before: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM objects", [], |row| row.get(0)) + .unwrap(); + let prolly_nodes_before: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row.get(0)) + .unwrap(); + + let err = db + .validate_and_update_case_fold_index(&corrupt, &["README.md".to_string()], &[]) + .unwrap_err(); + + assert!(matches!( + err, + Error::Corrupt(ref message) + if message.contains("readme.md") + && message.contains("Other.md") + && message.contains("other.md") + )); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM objects", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + objects_before + ); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + prolly_nodes_before + ); + } + + #[test] + fn indexed_case_fold_update_rejects_invalid_stored_canonical_paths_without_writes() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + + for (stored, expected_reason) in [ + ("../README.md", "inside the workspace"), + ("docs/./ReadMe.md", "must be normalized"), + ("docs/cafe\u{0301}.md", "Unicode NFC"), + ] { + let corrupt = root_with_case_fold_value(&db, &root, "readme.md", stored); + let objects_before: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM objects", [], |row| row.get(0)) + .unwrap(); + let prolly_nodes_before: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row.get(0)) + .unwrap(); + + let err = db + .validate_and_update_case_fold_index(&corrupt, &["README.md".to_string()], &[]) + .unwrap_err(); + + assert!(matches!( + err, + Error::Corrupt(ref message) + if message.contains("readme.md") + && message.contains(expected_reason) + && message.contains("canonical path") + )); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM objects", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + objects_before, + "stored value {stored:?} wrote an object" + ); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + prolly_nodes_before, + "stored value {stored:?} wrote a Prolly node" + ); + } + } + + #[test] + fn indexed_case_fold_update_duplicate_and_noop_inputs_reuse_previous_tree() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + let previous_tree = + root_map_tree_from_root_hex(root.case_fold_map_root.as_deref()).unwrap(); + let prolly_nodes_before: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row.get(0)) + .unwrap(); + + let duplicate = db + .validate_and_update_case_fold_index( + &root, + &[], + &["README.md".to_string(), "README.md".to_string()], + ) + .unwrap(); + let wrong_case_removal = db + .validate_and_update_case_fold_index(&root, &["readme.md".to_string()], &[]) + .unwrap(); + let delete_then_add = db + .validate_and_update_case_fold_index( + &root, + &["README.md".to_string()], + &["README.md".to_string()], + ) + .unwrap(); + let empty = db + .validate_and_update_case_fold_index(&root, &[], &[]) + .unwrap(); + + assert_eq!(tree_root_hex(&duplicate), tree_root_hex(&previous_tree)); + assert_eq!( + tree_root_hex(&wrong_case_removal), + tree_root_hex(&previous_tree) + ); + assert_eq!( + tree_root_hex(&delete_then_add), + tree_root_hex(&previous_tree) + ); + assert_eq!(tree_root_hex(&empty), tree_root_hex(&previous_tree)); + assert_eq!( + db.root_prolly + .get(&wrong_case_removal, b"readme.md") + .unwrap(), + Some(b"README.md".to_vec()) + ); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + prolly_nodes_before + ); + } + + #[test] + fn indexed_case_fold_update_legacy_root_requires_rebuild_without_reads_or_writes() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let mut root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + root.case_fold_map_root = None; + root.path_map_root = Some("invalid-path-tree-that-must-not-be-loaded".to_string()); + let objects_before: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM objects", [], |row| row.get(0)) + .unwrap(); + let prolly_nodes_before: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row.get(0)) + .unwrap(); + + let err = db + .validate_and_update_case_fold_index(&root, &[], &["new.md".to_string()]) + .unwrap_err(); + + assert!(matches!( + err, + Error::PathIndexRequired(ref message) + if message.contains("trail index rebuild") + )); + assert_eq!(err.code(), "PATH_INDEX_REQUIRED"); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM objects", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + objects_before + ); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + prolly_nodes_before + ); + } + + #[test] + fn full_path_list_root_build_persists_ascii_and_unicode_case_fold_keys() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + fs::create_dir_all(temp.path().join("src")).unwrap(); + fs::create_dir_all(temp.path().join("docs")).unwrap(); + fs::write(temp.path().join("src/Kernel.rs"), "kernel\n").unwrap(); + fs::write(temp.path().join("docs/ReadMe.md"), "readme\n").unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let built = db + .build_root_from_worktree_paths( + &["src/Kernel.rs".to_string(), "docs/ReadMe.md".to_string()], + &ChangeId("change_case_fold_path_root".to_string()), + ) + .unwrap(); + + assert_case_fold_mapping(&db, &built.root_id, "docs/readme.md", "docs/ReadMe.md"); + assert_case_fold_mapping(&db, &built.root_id, "src/kernel.rs", "src/Kernel.rs"); + } + + #[cfg(unix)] + #[test] + fn full_path_list_root_case_fold_domain_excludes_filtered_paths() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + fs::create_dir_all(temp.path().join("docs")).unwrap(); + fs::create_dir_all(temp.path().join("directory-only")).unwrap(); + fs::write(temp.path().join("docs/ReadMe.md"), "readme\n").unwrap(); + symlink_file("docs/ReadMe.md", temp.path().join("readme-link.md")).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let built = db + .build_root_from_worktree_paths( + &[ + "docs/ReadMe.md".to_string(), + "directory-only".to_string(), + "missing.md".to_string(), + "readme-link.md".to_string(), + ], + &ChangeId("change_case_fold_filtered_paths".to_string()), + ) + .unwrap(); + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &built.root_id).unwrap(); + + let path_entries = root_map_entries(&db, root.path_map_root.as_deref()); + let case_fold_entries = root_map_entries(&db, root.case_fold_map_root.as_deref()); + let path_domain = path_entries + .iter() + .map(|(path, _)| String::from_utf8(path.clone()).unwrap()) + .collect::>(); + let case_fold_domain = case_fold_entries + .iter() + .map(|(_, canonical)| String::from_utf8(canonical.clone()).unwrap()) + .collect::>(); + + assert_eq!(path_domain, vec!["docs/ReadMe.md"]); + assert_eq!(case_fold_domain, path_domain); + assert_eq!( + case_fold_entries, + vec![(b"docs/readme.md".to_vec(), b"docs/ReadMe.md".to_vec())] + ); + } + + #[test] + fn full_file_entry_root_build_persists_ascii_and_unicode_case_fold_keys() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir_all(temp.path().join("src")).unwrap(); + fs::create_dir_all(temp.path().join("docs")).unwrap(); + fs::write(temp.path().join("src/Kernel.rs"), "kernel\n").unwrap(); + fs::write(temp.path().join("docs/ReadMe.md"), "readme\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let files = db.load_root_files(&head.root_id).unwrap(); + let built = db + .build_root_from_file_entries( + files, + &ChangeId("change_case_fold_file_entry_root".to_string()), + ) + .unwrap(); + + assert_case_fold_mapping(&db, &built.root_id, "docs/readme.md", "docs/ReadMe.md"); + assert_case_fold_mapping(&db, &built.root_id, "src/kernel.rs", "src/Kernel.rs"); + } + + #[test] + fn touched_incremental_root_rejects_final_case_fold_collisions_before_objects() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let previous = db + .load_root_files_for_paths(&head.root_id, &["README.md".to_string()]) + .unwrap(); + let mut target = previous.clone(); + target.insert( + "readme.md".to_string(), + previous.get("README.md").unwrap().clone(), + ); + let count_rows = |table: &str| -> i64 { + db.conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() + }; + let objects_before = count_rows("objects"); + let prolly_nodes_before = count_rows("prolly_nodes"); + + let err = db + .build_root_from_touched_file_entries_incremental( + &head.root_id, + &previous, + &target, + &ChangeId("change_collision_test".to_string()), + ) + .unwrap_err(); + + assert!( + matches!(err, Error::InvalidPath { ref reason, .. } if reason.contains("case-insensitive path collision")), + "expected final root collision, got {err:?}" + ); + assert_eq!(count_rows("objects"), objects_before); + assert_eq!(count_rows("prolly_nodes"), prolly_nodes_before); + } + + #[test] + fn full_file_entry_root_rejects_case_fold_collisions_before_objects() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let mut files = db.load_root_files(&head.root_id).unwrap(); + let entry = files.get("README.md").unwrap().clone(); + files.insert("README.md".to_string(), entry); + let count_rows = |table: &str| -> i64 { + db.conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() + }; + let objects_before = count_rows("objects"); + let prolly_nodes_before = count_rows("prolly_nodes"); + + let err = db + .build_root_from_file_entries( + files, + &ChangeId("change_full_collision_test".to_string()), + ) + .unwrap_err(); + + assert!( + matches!(err, Error::InvalidPath { ref reason, .. } if reason.contains("case-insensitive path collision")), + "expected full root collision, got {err:?}" + ); + assert_eq!(count_rows("objects"), objects_before); + assert_eq!(count_rows("prolly_nodes"), prolly_nodes_before); + } + + #[test] + fn disk_file_root_rejects_case_fold_collisions_before_objects() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let disk_files = vec![ + DiskFile { + path: "README.md".to_string(), + bytes: b"hello\n".to_vec(), + executable: false, + }, + DiskFile { + path: "readme.md".to_string(), + bytes: b"collision\n".to_vec(), + executable: false, + }, + ]; + let count_rows = |table: &str| -> i64 { + db.conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() + }; + let objects_before = count_rows("objects"); + let prolly_nodes_before = count_rows("prolly_nodes"); + + let err = db + .build_root_from_disk_files( + &disk_files, + &ChangeId("change_disk_file_collision_test".to_string()), + None, + ) + .unwrap_err(); + + assert!( + matches!(err, Error::InvalidPath { ref reason, .. } if reason.contains("case-insensitive path collision")), + "expected disk-file root collision, got {err:?}" + ); + assert_eq!(count_rows("objects"), objects_before); + assert_eq!(count_rows("prolly_nodes"), prolly_nodes_before); + } + + #[test] + fn path_list_root_build_rejects_case_fold_collisions_before_objects() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + fs::write(temp.path().join("readme.md"), "collision\n").unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let count_rows = |table: &str| -> i64 { + db.conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() + }; + let objects_before = count_rows("objects"); + let prolly_nodes_before = count_rows("prolly_nodes"); + + let err = db + .build_root_from_worktree_paths( + &["README.md".to_string(), "readme.md".to_string()], + &ChangeId("change_path_list_collision_test".to_string()), + ) + .unwrap_err(); + + assert!( + matches!(err, Error::InvalidPath { ref reason, .. } if reason.contains("case-insensitive path collision")), + "expected path-list root collision, got {err:?}" + ); + assert_eq!(count_rows("objects"), objects_before); + assert_eq!(count_rows("prolly_nodes"), prolly_nodes_before); + } + + #[test] + fn git_tracked_incremental_root_rejects_final_case_fold_collisions_before_objects() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + fs::write(temp.path().join("readme.md"), "collision\n").unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let count_rows = |table: &str| -> i64 { + db.conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() + }; + let objects_before = count_rows("objects"); + let prolly_nodes_before = count_rows("prolly_nodes"); + + let err = db + .build_root_from_git_tracked_paths_incremental( + &["README.md".to_string(), "readme.md".to_string()], + &head.root_id, + &ChangeId("change_git_incremental_collision_test".to_string()), + ) + .unwrap_err(); + + assert!( + matches!(err, Error::InvalidPath { ref reason, .. } if reason.contains("case-insensitive path collision")), + "expected git-tracked incremental collision, got {err:?}" + ); + assert_eq!(count_rows("objects"), objects_before); + assert_eq!(count_rows("prolly_nodes"), prolly_nodes_before); + } + + #[test] + fn full_root_metrics_preserve_rows_consumed_before_decode_error() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("a.txt"), "a\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let mut root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + let tree = root_map_tree_from_root_hex(root.path_map_root.as_deref()).unwrap(); + let tree = db + .root_prolly + .put(&tree, b"zz-corrupt".to_vec(), vec![0xff]) + .unwrap(); + root.path_map_root = tree_root_hex(&tree); + let corrupt_root_id = db + .put_object(WORKTREE_ROOT_KIND, ROOT_OBJECT_VERSION, &root) + .unwrap(); + let metrics = db.operation_metrics.clone().unwrap(); + + let result = metrics.profile(OperationMetricsKind::Diff, || { + db.diff_root_to_disk_manifest(&corrupt_root_id, &BTreeMap::new()) + }); + + assert!(result.is_err()); + let report = metrics.last_report(); + assert_eq!(report.outcome, OperationMetricsOutcome::Error); + assert_eq!(report.full_root_range_count, 1); + assert_eq!(report.root_range_row_count, 2); + } + + fn assert_indexed_case_fold_metrics(db: &Trail, max_lookups: u64) { + let metrics = db.case_fold_index_metrics_report(); + assert_eq!(metrics.mode, "indexed"); + assert!( + metrics.lookup_count <= max_lookups, + "expected at most {max_lookups} indexed lookups, got {metrics:?}" + ); + assert_eq!(metrics.full_root_path_load_count, 0); + } + + #[test] + fn git_tracked_incremental_root_updates_index_for_accepted_regular_file_domain() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("old.txt"), "old\n").unwrap(); + fs::write(temp.path().join("keep.txt"), "keep\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + fs::remove_file(temp.path().join("old.txt")).unwrap(); + fs::write(temp.path().join("new.txt"), "new\n").unwrap(); + #[cfg(unix)] + std::os::unix::fs::symlink("keep.txt", temp.path().join("linked.txt")).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + db.reset_case_fold_index_metrics(); + let mut paths = vec![ + "keep.txt".to_string(), + "missing.txt".to_string(), + "new.txt".to_string(), + ]; + #[cfg(unix)] + paths.push("linked.txt".to_string()); + let built = db + .build_root_from_git_tracked_paths_incremental( + &paths, + &head.root_id, + &ChangeId("change_git_indexed_domain".to_string()), + ) + .unwrap(); + + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &built.root_id).unwrap(); + let entries = root_map_entries(&db, root.case_fold_map_root.as_deref()); + assert_eq!( + entries, + vec![ + (b"keep.txt".to_vec(), b"keep.txt".to_vec()), + (b"new.txt".to_vec(), b"new.txt".to_vec()), + ] + ); + assert_indexed_case_fold_metrics(&db, 2); + } + + #[test] + fn selected_directory_incremental_root_indexes_actual_removed_and_added_files() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir_all(temp.path().join("docs")).unwrap(); + fs::write(temp.path().join("docs/old.md"), "old\n").unwrap(); + fs::write(temp.path().join("keep.md"), "keep\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let previous = db + .load_root_files_for_selections(&head.root_id, &["docs".to_string()]) + .unwrap(); + let disk_files = vec![DiskFile { + path: "docs/New.md".to_string(), + bytes: b"new\n".to_vec(), + executable: false, + }]; + db.reset_case_fold_index_metrics(); + let built = db + .build_root_for_selected_disk_files_incremental( + &head.root_id, + &previous, + &disk_files, + &["docs".to_string()], + &ChangeId("change_selected_directory_index".to_string()), + ) + .unwrap(); + + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &built.root_id).unwrap(); + let entries = root_map_entries(&db, root.case_fold_map_root.as_deref()); + assert_eq!( + entries, + vec![ + (b"docs/new.md".to_vec(), b"docs/New.md".to_vec()), + (b"keep.md".to_vec(), b"keep.md".to_vec()), + ] + ); + assert_indexed_case_fold_metrics(&db, 2); + } + + #[test] + fn selected_same_content_multi_rename_consumes_unique_file_identities() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir(temp.path().join("old")).unwrap(); + fs::write(temp.path().join("old/a.bin"), [0, 1, 2, 3]).unwrap(); + fs::write(temp.path().join("old/b.bin"), [0, 1, 2, 3]).unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let selections = vec![ + "old/a.bin".to_string(), + "old/b.bin".to_string(), + "new/a.bin".to_string(), + "new/b.bin".to_string(), + ]; + let previous = db + .load_root_files_for_paths(&head.root_id, &selections) + .unwrap(); + let old_ids = previous + .values() + .map(|entry| entry.file_id.clone()) + .collect::>(); + let disk_files = vec![ + DiskFile { + path: "new/b.bin".to_string(), + bytes: vec![0, 1, 2, 3], + executable: false, + }, + DiskFile { + path: "new/a.bin".to_string(), + bytes: vec![0, 1, 2, 3], + executable: false, + }, + ]; + + let built = db + .build_root_for_selected_disk_files_incremental( + &head.root_id, + &previous, + &disk_files, + &selections, + &ChangeId("change_selected_same_content_renames".to_string()), + ) + .unwrap(); + let built_root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &built.root_id).unwrap(); + + db.validate_worktree_root(&built_root).unwrap(); + let new_ids = built + .files + .values() + .map(|entry| entry.file_id.clone()) + .collect::>(); + assert_eq!(new_ids, old_ids); + assert_eq!(new_ids.len(), 2); + let diff = db.diff_file_maps(&previous, &built.files).unwrap(); + assert_eq!(diff.changes.len(), 2); + assert_eq!(diff.summaries.len(), 2); + for (index, change) in diff.changes.iter().enumerate() { + let name = if index == 0 { "a" } else { "b" }; + assert_eq!(change.kind, FileChangeKind::Renamed); + assert_eq!(change.path, format!("new/{name}.bin")); + assert_eq!( + change.old_path.as_deref(), + Some(format!("old/{name}.bin").as_str()) + ); + } + } + + #[test] + fn selected_same_content_survivor_is_not_a_rename_source() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("a.bin"), [0, 1, 2, 3]).unwrap(); + fs::write(temp.path().join("b.bin"), [0, 1, 2, 3]).unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let selections = vec![ + "a.bin".to_string(), + "b.bin".to_string(), + "c.bin".to_string(), + ]; + let previous = db + .load_root_files_for_paths(&head.root_id, &selections) + .unwrap(); + let a_id = previous["a.bin"].file_id.clone(); + let b_id = previous["b.bin"].file_id.clone(); + let disk_files = vec![ + DiskFile { + path: "c.bin".to_string(), + bytes: vec![0, 1, 2, 3], + executable: false, + }, + DiskFile { + path: "a.bin".to_string(), + bytes: vec![0, 1, 2, 3], + executable: false, + }, + ]; + + let built = db + .build_root_for_selected_disk_files_incremental( + &head.root_id, + &previous, + &disk_files, + &selections, + &ChangeId("change_selected_survivor_and_rename".to_string()), + ) + .unwrap(); + let built_root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &built.root_id).unwrap(); + + db.validate_worktree_root(&built_root).unwrap(); + assert_eq!(built.files["a.bin"].file_id, a_id); + assert_eq!(built.files["c.bin"].file_id, b_id); + assert_ne!(built.files["a.bin"].file_id, built.files["c.bin"].file_id); + } + + #[test] + fn selected_record_1000_candidates_use_linear_membership_and_two_tree_batches() { + const SELECTED_COUNT: usize = 1_000; + const DECOY_COUNT: usize = 1_000; + + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("seed.bin"), [0, 1, 2, 3]).unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let template = db + .load_root_files(&head.root_id) + .unwrap() + .into_values() + .next() + .unwrap(); + let mut previous = BTreeMap::new(); + let mut selected_paths = Vec::with_capacity(SELECTED_COUNT); + let mut disk_files = Vec::with_capacity(SELECTED_COUNT + DECOY_COUNT); + for index in 0..SELECTED_COUNT { + let path = format!("selected/{index:04}.bin"); + let mut entry = template.clone(); + entry.file_id = FileId::new( + ChangeId("change_selected_scale_source".to_string()), + index as u64, + ); + previous.insert(path.clone(), entry); + selected_paths.push(path.clone()); + let mut bytes = vec![0, 0xff]; + bytes.extend_from_slice(&(index as u32).to_be_bytes()); + disk_files.push(DiskFile { + path, + bytes, + executable: false, + }); + } + for index in 0..DECOY_COUNT { + let path = format!("unrelated/{index:04}.bin"); + let mut entry = template.clone(); + entry.file_id = FileId::new( + ChangeId("change_selected_scale_decoy".to_string()), + index as u64, + ); + previous.insert(path.clone(), entry); + disk_files.push(DiskFile { + path, + bytes: vec![0, 0xfe, (index % 251) as u8], + executable: false, + }); + } + let previous_build = db + .build_root_from_file_entries( + previous.clone(), + &ChangeId("change_selected_scale_root".to_string()), + ) + .unwrap(); + let previous_root: WorktreeRoot = db + .get_object(WORKTREE_ROOT_KIND, &previous_build.root_id) + .unwrap(); + let case_fold_tree = + root_map_tree_from_root_hex(previous_root.case_fold_map_root.as_deref()).unwrap(); + let selections = SelectionSet::from_paths(&selected_paths).unwrap(); + let metrics = db.operation_metrics.clone().unwrap(); + + let built = metrics + .profile(OperationMetricsKind::Record, || { + let selected_disk_files = + db.selected_disk_files_for_selection_set(&disk_files, &selections); + let removed_entries = db.selected_previous_entries(&previous, &selections); + db.build_root_for_selected_disk_files_incremental_from_selected_inputs( + &previous_build.root_id, + &previous, + selected_disk_files, + removed_entries, + &ChangeId("change_selected_scale_update".to_string()), + case_fold_tree, + ) + }) + .unwrap(); + let report = metrics.last_report(); + + assert!(report.selection_comparison_count > 0); + assert!( + report.selection_comparison_count <= 2 * (previous.len() + disk_files.len()) as u64, + "selected membership work was not linear: {report:?}" + ); + assert_eq!(report.prolly_tree_batch_call_count, 2); + assert!( + report.prolly_tree_batch_mutation_count <= 4 * SELECTED_COUNT as u64, + "selected tree mutations were not proportional: {report:?}" + ); + + let diff = db.diff_file_maps(&previous, &built.files).unwrap(); + assert_eq!(built.files.len(), SELECTED_COUNT + DECOY_COUNT); + assert_eq!(diff.changes.len(), SELECTED_COUNT); + assert_eq!(diff.summaries.len(), SELECTED_COUNT); + for (index, change) in diff.changes.iter().enumerate() { + assert_eq!(change.path, format!("selected/{index:04}.bin")); + assert_eq!(change.kind, FileChangeKind::Modified); + assert_eq!( + change.file_id, + Some(FileId::new( + ChangeId("change_selected_scale_source".to_string()), + index as u64, + )) + ); + assert_eq!(diff.summaries[index].path, change.path); + assert_eq!(diff.summaries[index].kind, FileChangeKind::Modified); + } + } + + #[test] + fn touched_incremental_root_handles_case_only_rename_and_reuses_unchanged_index() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let previous = db + .load_root_files_for_paths(&head.root_id, &["README.md".to_string()]) + .unwrap(); + let mut renamed = BTreeMap::new(); + renamed.insert( + "readme.md".to_string(), + previous.get("README.md").unwrap().clone(), + ); + db.reset_case_fold_index_metrics(); + let renamed_root = db + .build_root_from_touched_file_entries_incremental( + &head.root_id, + &previous, + &renamed, + &ChangeId("change_case_only_rename_index".to_string()), + ) + .unwrap(); + assert_case_fold_mapping(&db, &renamed_root.root_id, "readme.md", "readme.md"); + assert_indexed_case_fold_metrics(&db, 1); + + let renamed_root_value: WorktreeRoot = db + .get_object(WORKTREE_ROOT_KIND, &renamed_root.root_id) + .unwrap(); + let renamed_files = db.load_root_files(&renamed_root.root_id).unwrap(); + let mut content_only = renamed_files.clone(); + content_only.get_mut("readme.md").unwrap().content_hash = "changed".to_string(); + db.reset_case_fold_index_metrics(); + let content_root = db + .build_root_from_touched_file_entries_incremental( + &renamed_root.root_id, + &renamed_files, + &content_only, + &ChangeId("change_content_only_index_reuse".to_string()), + ) + .unwrap(); + let content_root_value: WorktreeRoot = db + .get_object(WORKTREE_ROOT_KIND, &content_root.root_id) + .unwrap(); + assert_eq!( + content_root_value.case_fold_map_root, + renamed_root_value.case_fold_map_root + ); + assert_indexed_case_fold_metrics(&db, 0); + } + + #[test] + fn patch_and_record_policy_preflights_return_indexed_trees_without_full_root_loads() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let previous = db + .load_root_files_for_paths(&head.root_id, &["README.md".to_string()]) + .unwrap(); + db.reset_case_fold_index_metrics(); + let patch_tree = db + .ensure_patch_final_root_paths_safe( + &head.root_id, + &previous, + &[PatchEdit::Rename { + from: "README.md".to_string(), + to: "readme.md".to_string(), + }], + ) + .unwrap(); + assert!(tree_root_hex(&patch_tree).is_some()); + assert_indexed_case_fold_metrics(&db, 1); + + let summary = FileDiffSummary { + path: "readme.md".to_string(), + old_path: Some("README.md".to_string()), + kind: FileChangeKind::Renamed, + before_hash: None, + after_hash: None, + additions: 0, + deletions: 0, + line_changes: Vec::new(), + patch: None, + }; + db.reset_case_fold_index_metrics(); + let prolly_before_record_preview = db + .conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(); + db.ensure_record_final_root_paths_safe_from_summaries(&head.root_id, &[summary]) + .unwrap(); + assert_indexed_case_fold_metrics(&db, 1); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + prolly_before_record_preview + ); + } + + #[test] + fn touched_incremental_deletion_to_empty_root_keeps_empty_index_semantics() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let previous = db.load_root_files(&head.root_id).unwrap(); + db.reset_case_fold_index_metrics(); + let built = db + .build_root_from_touched_file_entries_incremental( + &head.root_id, + &previous, + &BTreeMap::new(), + &ChangeId("change_delete_to_empty_index".to_string()), + ) + .unwrap(); + + let root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &built.root_id).unwrap(); + assert_eq!(root.file_count, 0); + assert_eq!(root.case_fold_map_root, None); + assert_indexed_case_fold_metrics(&db, 1); + } + + #[test] + fn touched_incremental_legacy_root_fails_before_tree_or_object_writes() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let mut legacy: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + legacy.case_fold_map_root = None; + let legacy_root_id = db + .put_object(WORKTREE_ROOT_KIND, ROOT_OBJECT_VERSION, &legacy) + .unwrap(); + let previous = db.load_root_files(&legacy_root_id).unwrap(); + let mut target = previous.clone(); + target.insert( + "new.md".to_string(), + previous.get("README.md").unwrap().clone(), + ); + let count_rows = |table: &str| -> i64 { + db.conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() + }; + let objects_before = count_rows("objects"); + let prolly_nodes_before = count_rows("prolly_nodes"); + + let err = db + .build_root_from_touched_file_entries_incremental( + &legacy_root_id, + &previous, + &target, + &ChangeId("change_legacy_index_failure".to_string()), + ) + .unwrap_err(); + assert_eq!(err.code(), "PATH_INDEX_REQUIRED"); assert_eq!(count_rows("objects"), objects_before); assert_eq!(count_rows("prolly_nodes"), prolly_nodes_before); } + + #[test] + fn high_level_patch_and_record_use_one_bounded_index_preflight() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("metric-patch", Some("main"), false, None, None) + .unwrap(); + db.reset_case_fold_index_metrics(); + let patch_report = db + .apply_lane_patch( + "metric-patch", + PatchDocument { + base_change: None, + allow_stale: true, + allow_ignored: false, + session_id: None, + message: None, + edits: vec![PatchEdit::Write { + path: "notes.md".to_string(), + content: "notes\n".to_string(), + executable: false, + }], + }, + ) + .unwrap(); + assert_eq!(patch_report.path_index.mode, "indexed"); + assert_eq!(patch_report.path_index.lookup_count, 1); + assert_eq!(patch_report.path_index.full_root_path_load_count, 0); + assert_eq!(patch_report.path_index.full_filesystem_path_scan_count, 0); + assert_indexed_case_fold_metrics(&db, 1); + + let spawned = db + .spawn_lane("metric-record", Some("main"), true, None, None) + .unwrap(); + let workdir = PathBuf::from(spawned.workdir.unwrap()); + fs::write(workdir.join("recorded.md"), "recorded\n").unwrap(); + let count_rows = |table: &str| -> i64 { + db.conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() + }; + let objects_before_preview = count_rows("objects"); + let prolly_before_preview = count_rows("prolly_nodes"); + db.reset_case_fold_index_metrics(); + let preview = db.preview_lane_workdir_record("metric-record").unwrap(); + assert!(preview.policy.allowed, "{:?}", preview.policy.error); + assert_indexed_case_fold_metrics(&db, 1); + assert_eq!(count_rows("objects"), objects_before_preview); + assert_eq!(count_rows("prolly_nodes"), prolly_before_preview); + + db.reset_case_fold_index_metrics(); + let record_report = db.record_lane_workdir("metric-record", None).unwrap(); + assert_eq!(record_report.path_index.mode, "indexed"); + assert_eq!(record_report.path_index.lookup_count, 1); + assert_eq!(record_report.path_index.full_root_path_load_count, 0); + assert_eq!(record_report.path_index.full_filesystem_path_scan_count, 1); + assert_indexed_case_fold_metrics(&db, 1); + + let sparse = db + .spawn_lane_with_workdir_paths( + "metric-sparse-record", + Some("main"), + true, + None, + None, + None, + &["README.md".to_string()], + ) + .unwrap(); + let sparse_workdir = PathBuf::from(sparse.workdir.unwrap()); + fs::write(sparse_workdir.join("README.md"), "sparse\n").unwrap(); + let sparse_report = db + .record_lane_workdir("metric-sparse-record", None) + .unwrap(); + assert_eq!(sparse_report.path_index.mode, "indexed"); + assert_eq!(sparse_report.path_index.lookup_count, 1); + assert_eq!(sparse_report.path_index.full_root_path_load_count, 0); + assert_eq!(sparse_report.path_index.full_filesystem_path_scan_count, 0); + } + + #[test] + fn operation_reports_count_each_unbounded_root_traversal() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + fs::write(temp.path().join("untouched.md"), "stable\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + let patch_lane = db + .spawn_lane("metric-full-patch", Some("main"), true, None, None) + .unwrap(); + let patch_workdir = PathBuf::from(patch_lane.workdir.unwrap()); + fs::remove_file(patch_workdir.join(".trail/workdir-manifest.json")).unwrap(); + + let patch = db + .apply_lane_patch( + "metric-full-patch", + PatchDocument { + base_change: None, + allow_stale: true, + allow_ignored: false, + session_id: None, + message: None, + edits: vec![PatchEdit::Write { + path: "README.md".to_string(), + content: "patched\n".to_string(), + executable: false, + }], + }, + ) + .unwrap(); + assert_eq!(patch.path_index.mode, "indexed"); + assert_eq!(patch.path_index.lookup_count, 0); + assert_eq!(patch.path_index.full_root_path_load_count, 2); + assert_eq!(patch.path_index.full_filesystem_path_scan_count, 0); + assert_eq!( + fs::read_to_string(patch_workdir.join("README.md")).unwrap(), + "patched\n" + ); + + let record_lane = db + .spawn_lane("metric-full-record", Some("main"), true, None, None) + .unwrap(); + let record_workdir = PathBuf::from(record_lane.workdir.unwrap()); + fs::remove_file(record_workdir.join(".trail/workdir-manifest.json")).unwrap(); + fs::write(record_workdir.join("README.md"), "recorded\n").unwrap(); + + let record = db.record_lane_workdir("metric-full-record", None).unwrap(); + assert_eq!(record.path_index.mode, "indexed"); + assert_eq!(record.path_index.lookup_count, 1); + assert_eq!(record.path_index.full_root_path_load_count, 1); + assert_eq!(record.path_index.full_filesystem_path_scan_count, 1); + } + + #[test] + fn operation_reports_reset_case_fold_metrics_after_error_retry_and_noop() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("metric-reset", Some("main"), false, None, None) + .unwrap(); + let first = db + .apply_lane_patch( + "metric-reset", + PatchDocument { + base_change: None, + allow_stale: true, + allow_ignored: false, + session_id: None, + message: None, + edits: vec![ + PatchEdit::Write { + path: "first.md".to_string(), + content: "first\n".to_string(), + executable: false, + }, + PatchEdit::Write { + path: "second.md".to_string(), + content: "second\n".to_string(), + executable: false, + }, + ], + }, + ) + .unwrap(); + assert_eq!(first.path_index.mode, "indexed"); + assert_eq!(first.path_index.lookup_count, 2); + + let held_lock = db.acquire_write_lock().unwrap(); + let lock_err = db + .apply_lane_patch( + "metric-reset", + PatchDocument { + base_change: None, + allow_stale: true, + allow_ignored: false, + session_id: None, + message: None, + edits: vec![PatchEdit::Write { + path: "locked.md".to_string(), + content: "locked\n".to_string(), + executable: false, + }], + }, + ) + .unwrap_err(); + assert!(matches!(lock_err, Error::WorkspaceLocked(_))); + drop(held_lock); + assert_eq!( + db.case_fold_index_metrics_report(), + CaseFoldIndexMetricsReport::default() + ); + + let err = db + .apply_lane_patch( + "metric-reset", + PatchDocument { + base_change: None, + allow_stale: true, + allow_ignored: false, + session_id: None, + message: None, + edits: vec![PatchEdit::Write { + path: "../invalid.md".to_string(), + content: "invalid\n".to_string(), + executable: false, + }], + }, + ) + .unwrap_err(); + assert!(matches!(err, Error::InvalidPath { .. })); + assert_eq!( + db.case_fold_index_metrics_report(), + CaseFoldIndexMetricsReport { + mode: "unknown".to_string(), + lookup_count: 0, + full_root_path_load_count: 0, + full_filesystem_path_scan_count: 0, + } + ); + + let retry = db + .apply_lane_patch( + "metric-reset", + PatchDocument { + base_change: None, + allow_stale: true, + allow_ignored: false, + session_id: None, + message: None, + edits: vec![PatchEdit::Write { + path: "third.md".to_string(), + content: "third\n".to_string(), + executable: false, + }], + }, + ) + .unwrap(); + assert_eq!(retry.path_index.mode, "indexed"); + assert_eq!(retry.path_index.lookup_count, 1); + assert_eq!(retry.path_index.full_root_path_load_count, 0); + assert_eq!(retry.path_index.full_filesystem_path_scan_count, 0); + + let spawned = db + .spawn_lane("metric-noop", Some("main"), true, None, None) + .unwrap(); + let workdir = PathBuf::from(spawned.workdir.unwrap()); + fs::write(workdir.join("recorded.md"), "recorded\n").unwrap(); + let recorded = db.record_lane_workdir("metric-noop", None).unwrap(); + assert_eq!(recorded.path_index.mode, "indexed"); + assert_eq!(recorded.path_index.lookup_count, 1); + assert_eq!(recorded.path_index.full_filesystem_path_scan_count, 1); + + let noop = db.record_lane_workdir("metric-noop", None).unwrap(); + assert!(noop.operation.is_none()); + assert_eq!(noop.path_index.mode, "unknown"); + assert_eq!(noop.path_index.lookup_count, 0); + assert_eq!(noop.path_index.full_root_path_load_count, 0); + assert_eq!(noop.path_index.full_filesystem_path_scan_count, 1); + } + + #[test] + fn missing_manifest_record_preview_validates_without_persisting_index_nodes() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + let spawned = db + .spawn_lane("missing-manifest-preview", Some("main"), true, None, None) + .unwrap(); + let workdir = PathBuf::from(spawned.workdir.unwrap()); + fs::remove_file(workdir.join(".trail/workdir-manifest.json")).unwrap(); + fs::write(workdir.join("added.md"), "added\n").unwrap(); + let count_rows = |table: &str| -> i64 { + db.conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() + }; + let objects_before = count_rows("objects"); + let prolly_before = count_rows("prolly_nodes"); + db.reset_case_fold_index_metrics(); + + let preview = db + .preview_lane_workdir_record("missing-manifest-preview") + .unwrap(); + + assert!(preview.policy.allowed, "{:?}", preview.policy.error); + assert!(preview + .changed_paths + .iter() + .any(|summary| summary.path == "added.md")); + let metrics = db.case_fold_index_metrics_report(); + assert_eq!(metrics.mode, "indexed"); + assert!(metrics.lookup_count <= 1, "{metrics:?}"); + assert_eq!(metrics.full_root_path_load_count, 1); + assert_eq!(count_rows("objects"), objects_before); + assert_eq!(count_rows("prolly_nodes"), prolly_before); + } + + #[test] + fn record_candidate_preflight_distinguishes_partial_collision_from_explicit_rename() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let disk_manifest = [( + "readme.md".to_string(), + DiskManifest { + kind: FileKind::Text, + executable: false, + content_hash: sha256_hex(b"hello\n"), + }, + )] + .into_iter() + .collect::>(); + + let count_rows = |table: &str| -> i64 { + db.conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() + }; + let objects_before = count_rows("objects"); + let prolly_before = count_rows("prolly_nodes"); + let resolution = db + .resolve_record_case_fold_candidates_read_only( + &head.root_id, + &["readme.md".to_string()], + &disk_manifest, + ) + .unwrap(); + assert!(matches!( + resolution.state, + RecordCaseFoldResolutionState::Collision { .. } + )); + assert_eq!(count_rows("objects"), objects_before); + assert_eq!(count_rows("prolly_nodes"), prolly_before); + + let resolution = db + .resolve_record_case_fold_candidates_read_only( + &head.root_id, + &["README.md".to_string(), "readme.md".to_string()], + &disk_manifest, + ) + .unwrap(); + let preflight = db.finalize_record_case_fold_resolution(resolution).unwrap(); + assert_eq!( + preflight.selected_paths, + vec!["README.md".to_string(), "readme.md".to_string()] + ); + assert_eq!( + db.root_prolly + .get(&preflight.case_fold_tree, b"readme.md") + .unwrap(), + Some(b"readme.md".to_vec()) + ); + } + + #[test] + fn legacy_record_status_read_does_not_require_or_write_case_fold_index() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let spawned = db + .spawn_lane("legacy-read", Some("main"), true, None, None) + .unwrap(); + let workdir = PathBuf::from(spawned.workdir.unwrap()); + let branch = db.lane_branch("legacy-read").unwrap(); + let mut head = db.get_ref(&branch.ref_name).unwrap(); + let mut legacy: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + legacy.case_fold_map_root = None; + let legacy_root_id = db + .put_object(WORKTREE_ROOT_KIND, ROOT_OBJECT_VERSION, &legacy) + .unwrap(); + head.root_id = legacy_root_id.clone(); + let files = db.load_root_files(&legacy_root_id).unwrap(); + db.write_clean_workdir_manifest(&workdir, &legacy_root_id, &files, files.keys()) + .unwrap(); + fs::write(workdir.join("README.md"), "changed\n").unwrap(); + + let count_rows = |table: &str| -> i64 { + db.conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() + }; + let objects_before = count_rows("objects"); + let prolly_before = count_rows("prolly_nodes"); + db.reset_case_fold_index_metrics(); + + let summaries = db + .lane_workdir_record_changed_paths(&branch, &head, &workdir) + .unwrap(); + + assert_eq!(summaries.len(), 1); + assert_eq!(summaries[0].path, "README.md"); + assert_eq!(db.case_fold_index_metrics_report().lookup_count, 0); + assert_eq!(count_rows("objects"), objects_before); + assert_eq!(count_rows("prolly_nodes"), prolly_before); + + db.set_ref( + &head.name, + &head.change_id, + &legacy_root_id, + &head.operation_id, + ) + .unwrap(); + let objects_before_preview = count_rows("objects"); + let prolly_before_preview = count_rows("prolly_nodes"); + db.reset_case_fold_index_metrics(); + let preview = db.preview_lane_workdir_record("legacy-read").unwrap(); + assert!(!preview.policy.allowed); + assert!(preview + .policy + .error + .as_deref() + .is_some_and(|error| error.contains("case-fold index"))); + assert_eq!(db.case_fold_index_metrics_report().lookup_count, 0); + assert_eq!(count_rows("objects"), objects_before_preview); + assert_eq!(count_rows("prolly_nodes"), prolly_before_preview); + } + + #[test] + fn record_preflight_rejects_final_disk_domain_mismatch_before_objects() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let disk_manifest = [( + "readme.md".to_string(), + DiskManifest { + kind: FileKind::Text, + executable: false, + content_hash: sha256_hex(b"hello\n"), + }, + )] + .into_iter() + .collect::>(); + let resolution = db + .resolve_record_case_fold_candidates_read_only( + &head.root_id, + &["README.md".to_string(), "readme.md".to_string()], + &disk_manifest, + ) + .unwrap(); + let preflight = db.finalize_record_case_fold_resolution(resolution).unwrap(); + let previous = db + .load_root_files_for_paths( + &head.root_id, + &["README.md".to_string(), "readme.md".to_string()], + ) + .unwrap(); + let count_rows = |table: &str| -> i64 { + db.conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() + }; + let objects_before = count_rows("objects"); + + let err = db + .build_root_for_selected_disk_files_incremental_with_record_preflight( + temp.path(), + &head.root_id, + &previous, + &[], + &["README.md".to_string(), "readme.md".to_string()], + &ChangeId("change_record_domain_mismatch".to_string()), + preflight, + ) + .unwrap_err(); + assert!(err + .to_string() + .contains("changed after case-fold preflight")); + assert_eq!(count_rows("objects"), objects_before); + } + + #[test] + fn record_preflight_rejects_reappearance_disappearance_and_type_change() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let disk_manifest = [( + "readme.md".to_string(), + DiskManifest { + kind: FileKind::Text, + executable: false, + content_hash: sha256_hex(b"hello\n"), + }, + )] + .into_iter() + .collect::>(); + let selected_paths = vec!["README.md".to_string(), "readme.md".to_string()]; + let previous = db + .load_root_files_for_paths(&head.root_id, &selected_paths) + .unwrap(); + let stale_disk_file = DiskFile { + path: "readme.md".to_string(), + bytes: b"hello\n".to_vec(), + executable: false, + }; + let count_rows = |table: &str| -> i64 { + db.conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() + }; + let objects_before = count_rows("objects"); + + let renamed = temp.path().join("rename-tmp"); + fs::rename(temp.path().join("README.md"), &renamed).unwrap(); + fs::rename(&renamed, temp.path().join("readme.md")).unwrap(); + + if !is_case_insensitive_filesystem(temp.path()).unwrap() { + let resolution = db + .resolve_record_case_fold_candidates_read_only( + &head.root_id, + &selected_paths, + &disk_manifest, + ) + .unwrap(); + let preflight = db.finalize_record_case_fold_resolution(resolution).unwrap(); + fs::write(temp.path().join("README.md"), "reappeared\n").unwrap(); + let err = db + .build_root_for_selected_disk_files_incremental_with_record_preflight( + temp.path(), + &head.root_id, + &previous, + std::slice::from_ref(&stale_disk_file), + &selected_paths, + &ChangeId("change_record_reappearance".to_string()), + preflight, + ) + .unwrap_err(); + assert!(err + .to_string() + .contains("changed after case-fold preflight")); + fs::remove_file(temp.path().join("README.md")).unwrap(); + } + + let resolution = db + .resolve_record_case_fold_candidates_read_only( + &head.root_id, + &selected_paths, + &disk_manifest, + ) + .unwrap(); + let preflight = db.finalize_record_case_fold_resolution(resolution).unwrap(); + fs::remove_file(temp.path().join("readme.md")).unwrap(); + let err = db + .build_root_for_selected_disk_files_incremental_with_record_preflight( + temp.path(), + &head.root_id, + &previous, + std::slice::from_ref(&stale_disk_file), + &selected_paths, + &ChangeId("change_record_disappearance".to_string()), + preflight, + ) + .unwrap_err(); + assert!(err + .to_string() + .contains("changed after case-fold preflight")); + + let resolution = db + .resolve_record_case_fold_candidates_read_only( + &head.root_id, + &selected_paths, + &disk_manifest, + ) + .unwrap(); + let preflight = db.finalize_record_case_fold_resolution(resolution).unwrap(); + fs::create_dir(temp.path().join("readme.md")).unwrap(); + let err = db + .build_root_for_selected_disk_files_incremental_with_record_preflight( + temp.path(), + &head.root_id, + &previous, + &[stale_disk_file], + &selected_paths, + &ChangeId("change_record_type_change".to_string()), + preflight, + ) + .unwrap_err(); + assert!(err + .to_string() + .contains("changed after case-fold preflight")); + assert_eq!(count_rows("objects"), objects_before); + } + + #[cfg(unix)] + #[test] + fn known_regular_file_batch_rejects_symlink_swap_and_disappearance() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("tracked.txt"), "tracked\n").unwrap(); + fs::write(temp.path().join("target.txt"), "target\n").unwrap(); + let paths = vec!["tracked.txt".to_string()]; + + fs::remove_file(temp.path().join("tracked.txt")).unwrap(); + std::os::unix::fs::symlink("target.txt", temp.path().join("tracked.txt")).unwrap(); + assert!(fresh_regular_file_metadata(temp.path(), "tracked.txt").is_err()); + assert!(read_known_regular_path_file_batch(temp.path(), &paths).is_err()); + + fs::remove_file(temp.path().join("tracked.txt")).unwrap(); + assert!(read_known_regular_path_file_batch(temp.path(), &paths).is_err()); + } } diff --git a/trail/src/db/storage/git.rs b/trail/src/db/storage/git.rs index 9402b9d..e442067 100644 --- a/trail/src/db/storage/git.rs +++ b/trail/src/db/storage/git.rs @@ -1,34 +1,169 @@ use super::*; +#[cfg(unix)] +use std::os::unix::ffi::OsStrExt; impl Trail { - pub(crate) fn current_git_state(&self) -> Result> { - let inside = Command::new("git") + pub(crate) fn with_workspace_authoritative_command_snapshot( + &self, + mut consume: F, + ) -> Result<(T, crate::db::change_ledger::FencedCandidateSnapshot)> + where + F: FnMut( + &Trail, + &crate::db::change_ledger::CompiledPolicy, + &crate::db::change_ledger::CandidateSnapshot, + Option<&crate::db::change_ledger::QualifiedGitCandidates>, + ) -> Result, + { + let git_bound: bool = self.conn.query_row( + "SELECT EXISTS(SELECT 1 FROM git_mappings LIMIT 1)", + [], + |row| row.get(0), + )?; + if git_bound { + // The changed-path ledger is the command-path candidate authority. + // Git contributes only a bounded structural fence here: running + // `status` or `ls-files` would reintroduce O(repository files) work + // on every warm status/diff/record operation. + let captured = std::cell::RefCell::new(None); + let held = std::cell::RefCell::new(None); + let result = self.with_workspace_authoritative_snapshot(|db, policy, candidates| { + let identity = db.capture_git_command_identity()?; + *held.borrow_mut() = Some(GitCommandStructuralHold::open(db, policy, &identity)?); + *captured.borrow_mut() = Some(identity); + consume(db, policy, candidates, None) + })?; + #[cfg(debug_assertions)] + run_git_qualification_after_c2_hook()?; + let observed = self.capture_git_command_identity()?; + let expected = + captured + .into_inner() + .ok_or_else(|| Error::ChangeLedgerReconcileRequired { + scope: result.1.candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "Git command fence omitted its c1 structural identity".into(), + command: "trail index reconcile".into(), + })?; + let path_matches = git_command_identity_matches(&observed, &expected); + let held_matches = held + .into_inner() + .ok_or_else(|| Error::ChangeLedgerReconcileRequired { + scope: result.1.candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "Git command fence omitted its held descriptors".into(), + command: "trail index reconcile".into(), + })? + .verify(self)?; + if !path_matches || !held_matches { + return Err(Error::ChangeLedgerReconcileRequired { + scope: result.1.candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "Git structural identity changed across ledger c2".into(), + command: "trail index reconcile".into(), + }); + } + Ok(result) + } else { + self.with_workspace_authoritative_snapshot(|db, policy, candidates| { + consume(db, policy, candidates, None) + }) + } + } + + pub(crate) fn reset_git_handoff_metrics(&self) { + self.git_handoff_metrics.set(GitHandoffMetrics::default()); + } + + pub(crate) fn git_handoff_metrics_report(&self) -> GitHandoffMetricsReport { + self.git_handoff_metrics.get().into() + } + + pub(crate) fn set_git_export_mode(&self, export_mode: GitExportMode) { + let mut metrics = self.git_handoff_metrics.get(); + metrics.export_mode = export_mode; + self.git_handoff_metrics.set(metrics); + } + + pub(crate) fn set_git_changed_path_count(&self, changed_path_count: u64) { + let mut metrics = self.git_handoff_metrics.get(); + metrics.changed_path_count = changed_path_count; + self.git_handoff_metrics.set(metrics); + } + + pub(crate) fn add_git_full_root_file_count(&self, full_root_file_count: u64) { + let mut metrics = self.git_handoff_metrics.get(); + metrics.full_root_file_count = metrics + .full_root_file_count + .saturating_add(full_root_file_count); + self.git_handoff_metrics.set(metrics); + } + + fn record_git_blob_write(&self) { + let mut metrics = self.git_handoff_metrics.get(); + metrics.blob_write_count = metrics.blob_write_count.saturating_add(1); + self.git_handoff_metrics.set(metrics); + } + + fn add_git_blob_writes(&self, count: u64) { + let mut metrics = self.git_handoff_metrics.get(); + metrics.blob_write_count = metrics.blob_write_count.saturating_add(count); + self.git_handoff_metrics.set(metrics); + } + + pub(crate) fn record_git_plumbing_command(&self) { + let mut metrics = self.git_handoff_metrics.get(); + metrics.git_plumbing_command_count = metrics.git_plumbing_command_count.saturating_add(1); + self.git_handoff_metrics.set(metrics); + } + + fn record_tracked_git_status(&self) { + let mut metrics = self.git_handoff_metrics.get(); + metrics.tracked_status_count = metrics.tracked_status_count.saturating_add(1); + self.git_handoff_metrics.set(metrics); + } + + pub(crate) fn current_git_identity(&self) -> Result> { + let head_output = Command::new("git") .arg("-C") .arg(&self.workspace_root) - .args(["rev-parse", "--is-inside-work-tree"]) + .args(["rev-parse", "--verify", "HEAD"]) .output() .map_err(|err| Error::Git(err.to_string()))?; - if !inside.status.success() { + if !head_output.status.success() { + return Ok(None); + } + let head = String::from_utf8_lossy(&head_output.stdout) + .trim() + .to_string(); + if head.is_empty() { return Ok(None); } - let head_output = Command::new("git") + let branch_output = Command::new("git") .arg("-C") .arg(&self.workspace_root) - .args(["rev-parse", "--verify", "HEAD"]) + .args(["symbolic-ref", "--quiet", "--short", "HEAD"]) .output() .map_err(|err| Error::Git(err.to_string()))?; - let head = if head_output.status.success() { - Some( - String::from_utf8_lossy(&head_output.stdout) + let branch = branch_output + .status + .success() + .then(|| { + String::from_utf8_lossy(&branch_output.stdout) .trim() - .to_string(), - ) - .filter(|head| !head.is_empty()) - } else { - None - }; + .to_string() + }) + .filter(|branch| !branch.is_empty()); + Ok(Some(GitIdentity { head, branch })) + } + + pub(crate) fn tracked_git_state(&self, identity: &GitIdentity) -> Result { + self.tracked_git_state_for_head(Some(identity.head.clone())) + } + fn tracked_git_state_for_head(&self, head: Option) -> Result { + self.record_tracked_git_status(); let status = Command::new("git") .arg("-C") .arg(&self.workspace_root) @@ -43,11 +178,26 @@ impl Trail { stderr.trim() ))); } - - Ok(Some(GitState { + Ok(GitState { head, dirty: !status.stdout.is_empty(), - })) + }) + } + + pub(crate) fn current_git_state(&self) -> Result> { + if let Some(identity) = self.current_git_identity()? { + return self.tracked_git_state(&identity).map(Some); + } + let inside = Command::new("git") + .arg("-C") + .arg(&self.workspace_root) + .args(["rev-parse", "--is-inside-work-tree"]) + .output() + .map_err(|err| Error::Git(err.to_string()))?; + if !inside.status.success() { + return Ok(None); + } + self.tracked_git_state_for_head(None).map(Some) } pub(crate) fn git_write_tree(&self, files: &BTreeMap) -> Result { @@ -55,6 +205,7 @@ impl Trail { for (path, entry) in files { let bytes = self.materialize_entry_bytes(entry)?; let oid = self.git_output_with_input(&["hash-object", "-w", "--stdin"], &bytes)?; + self.record_git_blob_write(); let blob = GitBlobEntry { mode: if entry.executable { "100755" } else { "100644" }, oid, @@ -132,45 +283,98 @@ impl Trail { let tmp_dir = self.db_dir.join("tmp"); fs::create_dir_all(&tmp_dir)?; - let index_path = tmp_dir.join(format!("git-index-{}-{}", std::process::id(), now_ts())); + let batch_dir = tempfile::Builder::new() + .prefix("git-delta-") + .tempdir_in(&tmp_dir)?; + let blob_dir = batch_dir.path().join("blobs"); + fs::create_dir(&blob_dir)?; + let index_path = batch_dir.path().join("index"); let result = (|| { + self.record_git_plumbing_command(); self.git_output_with_index(&["read-tree".to_string(), head.to_string()], &index_path)?; - for path in changed_paths { - if let Some(entry) = patch_right.get(&path) { + + let mut additions = Vec::new(); + for (ordinal, path) in changed_paths.iter().enumerate() { + if let Some(entry) = patch_right.get(path) { let bytes = self.materialize_entry_bytes(entry)?; - let oid = - self.git_output_with_input(&["hash-object", "-w", "--stdin"], &bytes)?; - let mode = if entry.executable { "100755" } else { "100644" }; - self.git_output_with_index( - &[ - "update-index".to_string(), - "--add".to_string(), - "--cacheinfo".to_string(), - mode.to_string(), - oid, - path, - ], - &index_path, - )?; + let synthetic_name = format!("blob-{ordinal:020}"); + let synthetic_path = blob_dir.join(&synthetic_name); + fs::write(&synthetic_path, bytes)?; + additions.push(( + path.clone(), + if entry.executable { "100755" } else { "100644" }, + synthetic_path, + )); + } + } + + let oids = if additions.is_empty() { + Vec::new() + } else { + let mut hash_input = Vec::new(); + for (_, _, synthetic_path) in &additions { + append_git_stdin_path(&mut hash_input, synthetic_path)?; + } + self.record_git_plumbing_command(); + let output = self.git_output_bytes_with_input( + &["hash-object", "-w", "--no-filters", "--stdin-paths"], + &hash_input, + None, + )?; + let oids = parse_git_hash_object_oids(&output, additions.len())?; + self.add_git_blob_writes(oids.len() as u64); + oids + }; + + let oid_length = oids.first().map_or(head.len(), String::len); + if !matches!(oid_length, 40 | 64) { + return Err(Error::Git(format!( + "unsupported Git object ID length {oid_length}" + ))); + } + let zero_oid = "0".repeat(oid_length); + let oid_by_path = additions + .iter() + .zip(oids) + .map(|((path, mode, _), oid)| (path.as_str(), (*mode, oid))) + .collect::>(); + let mut index_input = Vec::new(); + for path in &changed_paths { + if path.as_bytes().contains(&0) { + return Err(Error::InvalidPath { + path: path.clone(), + reason: "Git index paths cannot contain NUL bytes".to_string(), + }); + } + if let Some((mode, oid)) = oid_by_path.get(path.as_str()) { + index_input.extend_from_slice(mode.as_bytes()); + index_input.push(b' '); + index_input.extend_from_slice(oid.as_bytes()); } else { - self.git_output_with_index( - &[ - "update-index".to_string(), - "--force-remove".to_string(), - "--".to_string(), - path, - ], - &index_path, - )?; + index_input.extend_from_slice(b"0 "); + index_input.extend_from_slice(zero_oid.as_bytes()); } + index_input.push(b'\t'); + index_input.extend_from_slice(path.as_bytes()); + index_input.push(0); } + + self.record_git_plumbing_command(); + self.git_output_bytes_with_input( + &["update-index", "-z", "--index-info"], + &index_input, + Some(&index_path), + )?; + self.record_git_plumbing_command(); self.git_output_with_index(&["write-tree".to_string()], &index_path) })(); - - let _ = fs::remove_file(&index_path); - let _ = fs::remove_file(index_path.with_extension("lock")); - result + let cleanup = batch_dir.close(); + match (result, cleanup) { + (Ok(tree), Ok(())) => Ok(tree), + (Ok(_), Err(err)) => Err(Error::Io(err)), + (Err(err), _) => Err(err), + } } pub(crate) fn git_insert_tree_path( @@ -297,6 +501,38 @@ impl Trail { self.git_checked_output(&args, output) } + fn git_output_bytes_with_input( + &self, + args: &[&str], + input: &[u8], + index_path: Option<&Path>, + ) -> Result> { + let mut command = Command::new("git"); + command.arg("-C").arg(&self.workspace_root).args(args); + if let Some(index_path) = index_path { + command.env("GIT_INDEX_FILE", index_path); + } + let mut stdin = tempfile::tempfile()?; + stdin.write_all(input)?; + stdin.seek(SeekFrom::Start(0))?; + let output = command + .stdin(std::process::Stdio::from(stdin)) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .output() + .map_err(|err| Error::Git(err.to_string()))?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(Error::Git(format!( + "git {} failed in {}: {}", + args.join(" "), + self.workspace_root.display(), + stderr.trim() + ))); + } + Ok(output.stdout) + } + pub(crate) fn git_checked_output( &self, args: &[String], @@ -313,4 +549,1640 @@ impl Trail { } Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) } + + /// Git evidence is obtainable only while consuming the same fenced + /// workspace snapshot used by status/diff/record. The private raw + /// qualifier cannot be called by another producer with a fabricated or + /// stale baseline. + pub(crate) fn qualified_git_candidates( + &self, + ) -> Result { + let (_, _, qualified) = + self.with_workspace_authoritative_git_snapshot(None, |_, _, _, _qualified| Ok(()))?; + Ok(qualified) + } + + pub(crate) fn with_workspace_authoritative_git_snapshot( + &self, + policy_fingerprint_override: Option<[u8; 32]>, + mut consume: F, + ) -> Result<( + T, + crate::db::change_ledger::FencedCandidateSnapshot, + crate::db::change_ledger::QualifiedGitCandidates, + )> + where + F: FnMut( + &Trail, + &crate::db::change_ledger::CompiledPolicy, + &crate::db::change_ledger::CandidateSnapshot, + &crate::db::change_ledger::QualifiedGitCandidates, + ) -> Result, + { + let final_identity = std::cell::RefCell::new(None); + let held_files = std::cell::RefCell::new(None); + let final_qualified = std::cell::RefCell::new(None); + let result = self.with_workspace_authoritative_snapshot(|db, policy, candidates| { + let (qualified, captured, pinned_worktree) = + db.qualify_git_snapshot(policy, candidates, policy_fingerprint_override)?; + *held_files.borrow_mut() = + Some(GitStructuralHold::open(db, &captured, pinned_worktree)?); + *final_identity.borrow_mut() = Some(qualified.qualification.clone()); + *final_qualified.borrow_mut() = Some(qualified.clone()); + let mut augmented = candidates.clone(); + for path in &qualified.exact_paths { + augmented + .exact_paths + .push(crate::db::change_ledger::LedgerPath::parse(path)?); + } + augmented.exact_paths.sort(); + augmented.exact_paths.dedup(); + consume(db, policy, &augmented, &qualified) + })?; + #[cfg(debug_assertions)] + run_git_qualification_after_c2_hook()?; + let mut metrics = crate::db::change_ledger::GitStructuralMetrics::default(); + let observed = self.capture_git_repository_identity(&mut metrics)?; + let expected = + final_identity + .into_inner() + .ok_or_else(|| Error::ChangeLedgerReconcileRequired { + scope: result.1.candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "Git structural qualification omitted its c1 identity".into(), + command: "trail index reconcile".into(), + })?; + let identity_mismatches = git_identity_mismatches(&observed, &expected); + let held_matches = held_files + .into_inner() + .ok_or_else(|| Error::ChangeLedgerReconcileRequired { + scope: result.1.candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "Git structural descriptors were not retained through c2".into(), + command: "trail index reconcile".into(), + })? + .verify(self, &mut metrics)?; + if !identity_mismatches.is_empty() || !held_matches { + let mut reasons = identity_mismatches; + if !held_matches { + reasons.push("held_descriptors"); + } + return Err(Error::ChangeLedgerReconcileRequired { + scope: result.1.candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: format!( + "Git structural identity changed across ledger c2: {}", + reasons.join(", ") + ), + command: "trail index reconcile".into(), + }); + } + let mut qualified = + final_qualified + .into_inner() + .ok_or_else(|| Error::ChangeLedgerReconcileRequired { + scope: result.1.candidates.expected.scope_id.to_text(), + state: "untrusted_gap".into(), + reason: "Git structural qualification omitted its returned evidence".into(), + command: "trail index reconcile".into(), + })?; + merge_git_structural_metrics(&mut qualified.metrics, &metrics); + Ok((result.0, result.1, qualified)) + } + + #[cfg(debug_assertions)] + pub(crate) fn qualified_git_candidates_for_test( + &self, + force_policy_mismatch: bool, + ) -> Result { + if force_policy_mismatch { + let (_, _, qualified) = self + .with_workspace_authoritative_git_snapshot(Some([0x5a; 32]), |_, _, _, _| Ok(()))?; + Ok(qualified) + } else { + self.qualified_git_candidates() + } + } + + #[cfg(debug_assertions)] + pub(crate) fn git_qualification_full_scan_oracle_for_test(&self) -> Result> { + let branch = self.current_branch()?; + let head = self.resolve_branch_ref(&branch)?; + let disk_files = self.scan_files_under(&self.workspace_root)?; + let manifest = self.disk_manifest(&disk_files); + Ok(self + .diff_root_to_disk_manifest(&head.root_id, &manifest)? + .into_iter() + .map(|summary| summary.path) + .collect()) + } + + fn qualify_git_snapshot( + &self, + policy: &crate::db::change_ledger::CompiledPolicy, + snapshot: &crate::db::change_ledger::CandidateSnapshot, + policy_fingerprint_override: Option<[u8; 32]>, + ) -> Result<( + crate::db::change_ledger::QualifiedGitCandidates, + GitRepositoryQualificationIdentity, + PinnedWorktreeRoot, + )> { + use crate::db::change_ledger::{ + GitEvidenceQualification, GitStructuralMetrics, QualifiedGitCandidates, TrustState, + }; + + let mut metrics = GitStructuralMetrics::default(); + let before = self.capture_git_repository_identity(&mut metrics)?; + let index_bytes = + self.git_read_structural_index(&before.index_path, false, &mut metrics)?; + let index = self.git_index_semantics(&mut metrics, &index_bytes)?; + let config = self.git_qualification_config(!policy.case_sensitive(), &mut metrics)?; + let matcher = policy.recording_matcher()?; + let porcelain = self.git_porcelain_v2(&matcher, &mut metrics)?; + #[cfg(debug_assertions)] + run_git_qualification_after_porcelain_hook()?; + let after = self.capture_git_repository_identity(&mut metrics)?; + + let mapped = self.clean_git_mapping_for_evidence(&after.head_oid)?; + let current_ref = self.conn.query_row( + "SELECT change_id,root_id,generation FROM refs WHERE name=?1", + [&snapshot.expected.ref_name], + |row| { + Ok(( + row.get::<_, String>(0)?, + ObjectId(row.get(1)?), + row.get::<_, i64>(2)?, + )) + }, + )?; + let mapped_trail_root = mapped + .as_ref() + .map(|(_, root)| root.clone()) + .unwrap_or_else(|| ObjectId(String::new())); + let head_equivalent = before.head_oid == after.head_oid + && before.head_identity == after.head_identity + && mapped.as_ref().is_some_and(|(change, root)| { + *root == snapshot.expected.baseline_root + && *change == current_ref.0 + && *root == current_ref.1 + }) + && current_ref.1 == snapshot.expected.baseline_root + && u64::try_from(current_ref.2).ok() == Some(snapshot.expected.ref_generation); + let index_equivalent = before.index_path == after.index_path + && before.index_identity == after.index_identity + && before.shared_index_path == after.shared_index_path + && before.shared_index_identity == after.shared_index_identity + && (!index.split_index + || (before.shared_index_identity.is_some() + && after.shared_index_identity.is_some())) + && !index.assume_unchanged + && !index.skip_worktree + && !index.unresolved; + let policy_fingerprint = + policy_fingerprint_override.unwrap_or_else(|| policy.fingerprint()); + let policy_equivalent = snapshot.expected.policy_fingerprint == policy_fingerprint; + let pinned = self.open_pinned_worktree_root(policy)?; + let filesystem_identity = self.pinned_worktree_root_identity(&pinned); + let worktree_equivalent = before.worktree_top_level == after.worktree_top_level + && before.worktree_identity == after.worktree_identity + && after.worktree_identity == filesystem_identity; + let filesystem_equivalent = filesystem_identity == snapshot.expected.filesystem_identity + && self.verify_pinned_worktree_root(&pinned)? + && worktree_equivalent; + let expected_case_insensitive = !policy.case_sensitive(); + let mode_equivalent = config.file_mode; + // Trail records regular files only. A tracked Git symlink is therefore + // structurally advisory even when Git itself has native symlink mode. + let symlink_equivalent = config.symlinks && !index.symlink; + let sparse_equivalent = + !config.sparse_checkout && !index.skip_worktree && !index.sparse_index; + let submodule_equivalent = !index.submodule && !porcelain.unresolved_submodule; + let ignore_equivalent = self.config.recording.ignore_gitignored && policy_equivalent; + let case_equivalent = config.ignore_case == expected_case_insensitive; + let fsmonitor_qualified = !config.fsmonitor && !index.fsmonitor; + let untracked_cache_qualified = !config.untracked_cache && !index.untracked_cache; + metrics.fsmonitor_qualification_count = u64::from(fsmonitor_qualified); + metrics.untracked_cache_qualification_count = u64::from(untracked_cache_qualified); + self.note_operation_metrics(OperationMetricsDelta { + git_fsmonitor_qualification_count: metrics.fsmonitor_qualification_count, + git_untracked_cache_qualification_count: metrics.untracked_cache_qualification_count, + ..OperationMetricsDelta::default() + }); + + let mut advisory_reasons = Vec::new(); + push_git_advisory(&mut advisory_reasons, !head_equivalent, "head_or_mapping"); + push_git_advisory( + &mut advisory_reasons, + !index_equivalent, + "index_identity_or_flags", + ); + push_git_advisory( + &mut advisory_reasons, + !filesystem_equivalent, + "filesystem_identity", + ); + push_git_advisory( + &mut advisory_reasons, + !worktree_equivalent, + "git_worktree_identity", + ); + push_git_advisory( + &mut advisory_reasons, + !policy_equivalent, + "policy_fingerprint", + ); + push_git_advisory(&mut advisory_reasons, !mode_equivalent, "file_mode"); + push_git_advisory(&mut advisory_reasons, !symlink_equivalent, "symlink"); + push_git_advisory( + &mut advisory_reasons, + !sparse_equivalent, + "sparse_or_skip_worktree", + ); + push_git_advisory(&mut advisory_reasons, !submodule_equivalent, "submodule"); + push_git_advisory(&mut advisory_reasons, !ignore_equivalent, "ignore_policy"); + push_git_advisory(&mut advisory_reasons, !case_equivalent, "case_policy"); + push_git_advisory(&mut advisory_reasons, !fsmonitor_qualified, "fsmonitor"); + push_git_advisory( + &mut advisory_reasons, + !untracked_cache_qualified, + "untracked_cache", + ); + push_git_advisory( + &mut advisory_reasons, + !porcelain.complete, + "porcelain_incomplete", + ); + push_git_advisory( + &mut advisory_reasons, + snapshot.trust != TrustState::Trusted, + "ledger_not_trusted", + ); + let clean_proof_allowed = advisory_reasons.is_empty(); + let qualified = QualifiedGitCandidates { + qualification: GitEvidenceQualification { + head_oid: after.head_oid.clone(), + head_identity: after.head_identity.clone(), + worktree_top_level: after.worktree_top_level.to_string_lossy().into_owned(), + worktree_identity: after.worktree_identity.clone(), + index_identity: after.index_identity.clone(), + split_index_identity: index.split_index.then_some(after.index_identity.clone()), + shared_index_identity: after.shared_index_identity.clone(), + mapped_trail_root, + ledger_baseline_root: snapshot.expected.baseline_root.clone(), + filesystem_identity, + policy_fingerprint, + filesystem_equivalent, + worktree_equivalent, + policy_equivalent, + head_equivalent, + index_equivalent, + mode_equivalent, + symlink_equivalent, + sparse_equivalent, + submodule_equivalent, + ignore_equivalent, + case_equivalent, + fsmonitor_qualified, + untracked_cache_qualified, + clean_proof_allowed, + advisory_reasons, + }, + exact_paths: porcelain.paths.into_iter().collect(), + rename_pairs: porcelain.rename_pairs, + metrics, + }; + Ok((qualified, after, pinned)) + } + + fn capture_git_command_identity(&self) -> Result { + let mut metrics = crate::db::change_ledger::GitStructuralMetrics::default(); + let head_oid = + self.git_qualification_text(&["rev-parse", "--verify", "HEAD"], &mut metrics)?; + let worktree_top_level = self + .git_qualification_path(&["rev-parse", "--show-toplevel"], &mut metrics)? + .canonicalize()?; + let worktree_identity = self.pinned_worktree_identity_for_path(&worktree_top_level)?; + let index_path = + self.git_qualification_path(&["rev-parse", "--git-path", "index"], &mut metrics)?; + // Deliberately do not invoke `rev-parse --shared-index-path` here: Git + // may inspect the repository index to answer it. The command path does + // not consume index contents, and every usable split-index transition + // rewrites/replaces the main index that this bounded metadata fence + // already holds. Full qualification retains the shared-index check. + let head_path = + self.git_qualification_path(&["rev-parse", "--git-path", "HEAD"], &mut metrics)?; + let packed_refs_path = + self.git_qualification_path(&["rev-parse", "--git-path", "packed-refs"], &mut metrics)?; + let symbolic_ref = self + .git_qualification_optional_text(&["symbolic-ref", "--quiet", "HEAD"], &mut metrics)?; + let symbolic_ref_path = match symbolic_ref { + Some(reference) => Some(self.git_qualification_path( + &["rev-parse", "--git-path", reference.as_str()], + &mut metrics, + )?), + None => None, + }; + let index_identity = + git_file_metadata_identity_optional(&index_path)?.ok_or_else(|| { + Error::Git(format!("Git index `{}` is missing", index_path.display())) + })?; + Ok(GitCommandStructuralIdentity { + head_oid, + head_path: head_path.clone(), + head_identity: git_file_metadata_identity_optional(&head_path)?, + symbolic_ref_path: symbolic_ref_path.clone(), + symbolic_ref_identity: symbolic_ref_path + .as_deref() + .map(git_file_metadata_identity_optional) + .transpose()? + .flatten(), + packed_refs_path: packed_refs_path.clone(), + packed_refs_identity: git_file_metadata_identity_optional(&packed_refs_path)?, + worktree_top_level, + worktree_identity, + index_path, + index_identity, + }) + } + + fn capture_git_repository_identity( + &self, + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, + ) -> Result { + let head_oid = self.git_qualification_text(&["rev-parse", "--verify", "HEAD"], metrics)?; + let worktree_top_level = self + .git_qualification_path(&["rev-parse", "--show-toplevel"], metrics)? + .canonicalize()?; + let worktree_identity = self.pinned_worktree_identity_for_path(&worktree_top_level)?; + let index_path = + self.git_qualification_path(&["rev-parse", "--git-path", "index"], metrics)?; + let shared_index_path = self + .git_qualification_optional_text(&["rev-parse", "--shared-index-path"], metrics) + .map_err(|error| Error::Git(format!("Git shared_index discovery failed: {error}")))? + .filter(|path| !path.is_empty()) + .map(|path| self.absolute_git_path(&path)); + let head_path = + self.git_qualification_path(&["rev-parse", "--git-path", "HEAD"], metrics)?; + let packed_refs = + self.git_qualification_path(&["rev-parse", "--git-path", "packed-refs"], metrics)?; + let symbolic_ref = + self.git_qualification_optional_text(&["symbolic-ref", "--quiet", "HEAD"], metrics)?; + let symbolic_ref_path = match symbolic_ref { + Some(reference) => { + let path = self.git_qualification_path( + &["rev-parse", "--git-path", reference.as_str()], + metrics, + )?; + Some(path) + } + None => None, + }; + let head_file_identity = git_file_identity_optional(&head_path)?; + let symbolic_ref_identity = symbolic_ref_path + .as_deref() + .map(git_file_identity_optional) + .transpose()? + .flatten(); + let packed_refs_identity = git_file_identity_optional(&packed_refs)?; + let mut head_identity = head_oid.as_bytes().to_vec(); + for identity in [ + head_file_identity.as_ref(), + symbolic_ref_identity.as_ref(), + packed_refs_identity.as_ref(), + ] { + head_identity.push(0); + if let Some(identity) = identity { + head_identity.extend(identity); + } + } + let index_identity = self + .git_structural_index_identity(&index_path, false, metrics)? + .ok_or_else(|| { + Error::Git(format!("Git index `{}` is missing", index_path.display())) + })?; + let shared_index_identity = shared_index_path + .as_deref() + .map(|path| self.git_structural_index_identity(path, true, metrics)) + .transpose()? + .flatten(); + Ok(GitRepositoryQualificationIdentity { + head_oid, + head_identity, + head_path, + head_file_identity, + symbolic_ref_path, + symbolic_ref_identity, + packed_refs_path: packed_refs, + packed_refs_identity, + worktree_top_level, + worktree_identity, + index_path, + index_identity, + shared_index_path, + shared_index_identity, + }) + } + + fn git_read_structural_index( + &self, + path: &Path, + shared: bool, + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, + ) -> Result> { + let mut file = open_git_structural_file_no_follow(path)?; + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + let byte_count = saturating_u64_from_usize(bytes.len()); + self.note_git_structural_read(shared, byte_count, metrics); + Ok(bytes) + } + + fn note_git_structural_read( + &self, + shared: bool, + byte_count: u64, + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, + ) { + if shared { + metrics.shared_index_read_count = metrics.shared_index_read_count.saturating_add(1); + metrics.shared_index_bytes = metrics.shared_index_bytes.saturating_add(byte_count); + } else { + metrics.index_read_count = metrics.index_read_count.saturating_add(1); + metrics.index_bytes = metrics.index_bytes.saturating_add(byte_count); + } + self.note_operation_metrics(OperationMetricsDelta { + git_index_read_count: u64::from(!shared), + git_index_bytes: if shared { 0 } else { byte_count }, + git_shared_index_read_count: u64::from(shared), + git_shared_index_bytes: if shared { byte_count } else { 0 }, + filesystem_read_count: 1, + filesystem_read_bytes: byte_count, + ..OperationMetricsDelta::default() + }); + } + + fn git_structural_index_identity( + &self, + path: &Path, + shared: bool, + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, + ) -> Result>> { + let mut file = match open_git_structural_file_no_follow(path) { + Ok(file) => file, + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(None) + } + Err(error) => return Err(error), + }; + let metadata = file.metadata()?; + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + self.note_git_structural_read(shared, saturating_u64_from_usize(bytes.len()), metrics); + Ok(Some(git_file_identity(&metadata, &bytes))) + } + + fn git_qualification_config( + &self, + default_ignore_case: bool, + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, + ) -> Result { + Ok(GitQualificationConfig { + file_mode: self.git_qualification_bool("core.filemode", true, metrics)?, + symlinks: self.git_qualification_bool("core.symlinks", true, metrics)?, + ignore_case: self.git_qualification_bool( + "core.ignorecase", + default_ignore_case, + metrics, + )?, + sparse_checkout: self.git_qualification_bool("core.sparsecheckout", false, metrics)?, + fsmonitor: self + .git_qualification_optional_text(&["config", "--get", "core.fsmonitor"], metrics)? + .is_some_and(|value| !matches!(value.as_str(), "" | "false" | "0")), + untracked_cache: self + .git_qualification_optional_text( + &["config", "--get", "core.untrackedcache"], + metrics, + )? + .is_some_and(|value| !matches!(value.as_str(), "" | "false" | "0")), + }) + } + + fn git_qualification_bool( + &self, + key: &str, + default: bool, + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, + ) -> Result { + let Some(value) = + self.git_qualification_optional_text(&["config", "--bool", "--get", key], metrics)? + else { + return Ok(default); + }; + match value.as_str() { + "true" => Ok(true), + "false" => Ok(false), + _ => Err(Error::Git(format!( + "Git config `{key}` returned non-boolean value `{value}`" + ))), + } + } + + fn git_index_semantics( + &self, + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, + index_bytes: &[u8], + ) -> Result { + let records_before = metrics.output_record_count; + let staged = self.git_qualification_output( + &["ls-files", "--stage", "-t", "-z"], + metrics, + true, + None, + )?; + let mut semantics = GitIndexSemantics { + fsmonitor: bytes_contain(index_bytes, b"FSMN"), + untracked_cache: bytes_contain(index_bytes, b"UNTR"), + sparse_index: bytes_contain(index_bytes, b"sdir"), + split_index: bytes_contain(index_bytes, b"link"), + ..GitIndexSemantics::default() + }; + for record in staged + .stdout + .split(|byte| *byte == 0) + .filter(|raw| !raw.is_empty()) + { + metrics.output_record_count = metrics.output_record_count.saturating_add(1); + let Some(tab) = record.iter().position(|byte| *byte == b'\t') else { + semantics.unresolved = true; + continue; + }; + let header = &record[..tab]; + let tag = header.first().copied().unwrap_or_default(); + semantics.skip_worktree |= tag == b'S'; + let mut fields = header[1..] + .split(|byte| byte.is_ascii_whitespace()) + .filter(|part| !part.is_empty()); + let mode = fields.next().unwrap_or_default(); + let _oid = fields.next(); + let stage = fields.next().unwrap_or_default(); + semantics.unresolved |= stage != b"0"; + semantics.symlink |= mode == b"120000"; + semantics.submodule |= mode == b"160000"; + } + let verbose = + self.git_qualification_output(&["ls-files", "-v", "-z"], metrics, true, None)?; + for record in verbose + .stdout + .split(|byte| *byte == 0) + .filter(|raw| !raw.is_empty()) + { + metrics.output_record_count = metrics.output_record_count.saturating_add(1); + semantics.assume_unchanged |= record + .first() + .copied() + .is_some_and(|tag| tag.is_ascii_lowercase()); + } + self.note_operation_metrics(OperationMetricsDelta { + git_output_record_count: metrics.output_record_count.saturating_sub(records_before), + ..OperationMetricsDelta::default() + }); + Ok(semantics) + } + + fn git_porcelain_v2( + &self, + matcher: &crate::db::change_ledger::CompiledRecordingMatcher, + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, + ) -> Result { + let trace = tempfile::NamedTempFile::new()?; + let output = self.git_qualification_output( + &[ + "status", + "--porcelain=v2", + "-z", + "--untracked-files=all", + "--ignore-submodules=none", + ], + metrics, + true, + Some(trace.path()), + )?; + let trace_bytes = fs::read(trace.path())?; + metrics.trace2_bytes = metrics + .trace2_bytes + .saturating_add(saturating_u64_from_usize(trace_bytes.len())); + for line in trace_bytes.split(|byte| *byte == b'\n') { + if bytes_contain(line, b"\"event\":\"region_enter\"") { + metrics.trace2_region_count = metrics.trace2_region_count.saturating_add(1); + } + if bytes_contain(line, b"\"label\":\"refresh\"") { + metrics.index_refresh_count = metrics.index_refresh_count.saturating_add(1); + } + } + self.note_operation_metrics(OperationMetricsDelta { + git_trace2_region_count: metrics.trace2_region_count, + git_trace2_bytes: metrics.trace2_bytes, + git_index_refresh_count: metrics.index_refresh_count, + ..OperationMetricsDelta::default() + }); + let records_before = metrics.output_record_count; + let evidence = parse_git_porcelain_v2(&output.stdout, matcher, metrics)?; + self.note_operation_metrics(OperationMetricsDelta { + git_output_record_count: metrics.output_record_count.saturating_sub(records_before), + ..OperationMetricsDelta::default() + }); + Ok(evidence) + } + + fn git_qualification_path( + &self, + args: &[&str], + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, + ) -> Result { + let value = self.git_qualification_text(args, metrics)?; + Ok(self.absolute_git_path(&value)) + } + + fn absolute_git_path(&self, value: &str) -> PathBuf { + let path = PathBuf::from(value); + if path.is_absolute() { + path + } else { + self.workspace_root.join(path) + } + } + + fn git_qualification_text( + &self, + args: &[&str], + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, + ) -> Result { + let output = self.git_qualification_output(args, metrics, false, None)?; + if !output.status.success() { + return Err(git_qualification_command_error( + args, + &self.workspace_root, + &output, + )); + } + String::from_utf8(output.stdout) + .map(|value| value.trim().to_string()) + .map_err(|error| { + Error::Git(format!( + "git {} returned non-UTF-8: {error}", + args.join(" ") + )) + }) + } + + fn git_qualification_optional_text( + &self, + args: &[&str], + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, + ) -> Result> { + let output = self.git_qualification_output(args, metrics, false, None)?; + if output.status.success() { + return String::from_utf8(output.stdout) + .map(|value| Some(value.trim().to_string())) + .map_err(|error| { + Error::Git(format!( + "git {} returned non-UTF-8: {error}", + args.join(" ") + )) + }); + } + if output.status.code() == Some(1) { + return Ok(None); + } + Err(git_qualification_command_error( + args, + &self.workspace_root, + &output, + )) + } + + fn git_qualification_output( + &self, + args: &[&str], + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, + global_work: bool, + trace2: Option<&Path>, + ) -> Result { + let mut command = Command::new("git"); + command + .arg("-C") + .arg(&self.workspace_root) + .args(args) + .env("GIT_OPTIONAL_LOCKS", "0"); + for selector in GIT_REPOSITORY_SELECTOR_ENVIRONMENT { + command.env_remove(selector); + } + if let Some(trace2) = trace2 { + command.env("GIT_TRACE2_EVENT", trace2); + } + let output = command + .output() + .map_err(|error| Error::Git(error.to_string()))?; + let bytes = output.stdout.len().saturating_add(output.stderr.len()); + metrics.subprocess_count = metrics.subprocess_count.saturating_add(1); + metrics.output_bytes = metrics + .output_bytes + .saturating_add(saturating_u64_from_usize(bytes)); + if global_work { + metrics.external_adapter_global_work = + metrics.external_adapter_global_work.saturating_add(1); + } + self.note_operation_metrics(OperationMetricsDelta { + git_subprocess_count: 1, + git_global_work_count: u64::from(global_work), + external_adapter_global_work: u64::from(global_work), + git_output_bytes: saturating_u64_from_usize(bytes), + ..OperationMetricsDelta::default() + }); + if !output.status.success() && global_work { + return Err(git_qualification_command_error( + args, + &self.workspace_root, + &output, + )); + } + Ok(output) + } +} + +const GIT_REPOSITORY_SELECTOR_ENVIRONMENT: &[&str] = &[ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_COMMON_DIR", + "GIT_INDEX_FILE", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_CEILING_DIRECTORIES", + "GIT_DISCOVERY_ACROSS_FILESYSTEM", + "GIT_PREFIX", +]; + +#[derive(Debug)] +struct GitCommandStructuralIdentity { + head_oid: String, + head_path: PathBuf, + head_identity: Option>, + symbolic_ref_path: Option, + symbolic_ref_identity: Option>, + packed_refs_path: PathBuf, + packed_refs_identity: Option>, + worktree_top_level: PathBuf, + worktree_identity: Vec, + index_path: PathBuf, + index_identity: Vec, +} + +#[derive(Debug)] +struct GitRepositoryQualificationIdentity { + head_oid: String, + head_identity: Vec, + head_path: PathBuf, + head_file_identity: Option>, + symbolic_ref_path: Option, + symbolic_ref_identity: Option>, + packed_refs_path: PathBuf, + packed_refs_identity: Option>, + worktree_top_level: PathBuf, + worktree_identity: Vec, + index_path: PathBuf, + index_identity: Vec, + shared_index_path: Option, + shared_index_identity: Option>, +} + +struct GitCommandStructuralHold { + index: HeldGitMetadataFile, + head: Option, + symbolic_ref: Option, + packed_refs: Option, + worktree: PinnedWorktreeRoot, +} + +impl GitCommandStructuralHold { + fn open( + db: &Trail, + policy: &crate::db::change_ledger::CompiledPolicy, + identity: &GitCommandStructuralIdentity, + ) -> Result { + let index = + HeldGitMetadataFile::open_required(&identity.index_path, &identity.index_identity)?; + let head = HeldGitMetadataFile::open_optional( + &identity.head_path, + identity.head_identity.as_deref(), + )?; + let symbolic_ref = HeldGitMetadataFile::open_optional_pair( + identity.symbolic_ref_path.as_deref(), + identity.symbolic_ref_identity.as_deref(), + "symbolic-ref", + )?; + let packed_refs = HeldGitMetadataFile::open_optional( + &identity.packed_refs_path, + identity.packed_refs_identity.as_deref(), + )?; + let worktree = db.open_pinned_worktree_root(policy)?; + if db.pinned_worktree_root_identity(&worktree) != identity.worktree_identity + || !db.verify_pinned_worktree_root(&worktree)? + { + return Err(Error::ChangeLedgerReconcileRequired { + scope: "workspace".into(), + state: "untrusted_gap".into(), + reason: "Git worktree root changed before ledger command consumption".into(), + command: "trail index reconcile".into(), + }); + } + Ok(Self { + index, + head, + symbolic_ref, + packed_refs, + worktree, + }) + } + + fn verify(mut self, db: &Trail) -> Result { + if !self.index.verify()? { + return Ok(false); + } + for held in [ + &mut self.head, + &mut self.symbolic_ref, + &mut self.packed_refs, + ] + .into_iter() + .flatten() + { + if !held.verify()? { + return Ok(false); + } + } + db.verify_pinned_worktree_root(&self.worktree) + } +} + +struct GitStructuralHold { + index: HeldGitStructuralFile, + shared_index: Option, + head: Option, + symbolic_ref: Option, + packed_refs: Option, + worktree: PinnedWorktreeRoot, +} + +impl GitStructuralHold { + fn open( + db: &Trail, + identity: &GitRepositoryQualificationIdentity, + worktree: PinnedWorktreeRoot, + ) -> Result { + let index = + HeldGitStructuralFile::open_required(&identity.index_path, &identity.index_identity)?; + let shared_index = HeldGitStructuralFile::open_optional_pair( + identity.shared_index_path.as_deref(), + identity.shared_index_identity.as_deref(), + "shared-index", + )?; + let head = HeldGitStructuralFile::open_optional( + &identity.head_path, + identity.head_file_identity.as_deref(), + )?; + let symbolic_ref = match ( + identity.symbolic_ref_path.as_deref(), + identity.symbolic_ref_identity.as_deref(), + ) { + (Some(path), expected) => HeldGitStructuralFile::open_optional(path, expected)?, + (None, None) => None, + (None, Some(_)) => { + return Err(Error::Git( + "Git symbolic-ref identity has no structural path".into(), + )); + } + }; + let packed_refs = HeldGitStructuralFile::open_optional( + &identity.packed_refs_path, + identity.packed_refs_identity.as_deref(), + )?; + if db.pinned_worktree_root_identity(&worktree) != identity.worktree_identity + || !db.verify_pinned_worktree_root(&worktree)? + { + return Err(Error::ChangeLedgerReconcileRequired { + scope: "workspace".into(), + state: "untrusted_gap".into(), + reason: "Git worktree root changed before ledger consumption".into(), + command: "trail index reconcile".into(), + }); + } + Ok(Self { + index, + shared_index, + head, + symbolic_ref, + packed_refs, + worktree, + }) + } + + fn verify( + mut self, + db: &Trail, + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, + ) -> Result { + let (index_matches, index_bytes) = self.index.verify()?; + db.note_git_structural_read(false, index_bytes, metrics); + if !index_matches { + return Ok(false); + } + if let Some(mut shared) = self.shared_index { + let (shared_matches, shared_bytes) = shared.verify()?; + db.note_git_structural_read(true, shared_bytes, metrics); + if !shared_matches { + return Ok(false); + } + } + for held in [ + &mut self.head, + &mut self.symbolic_ref, + &mut self.packed_refs, + ] + .into_iter() + .flatten() + { + if !held.verify()?.0 { + return Ok(false); + } + } + db.verify_pinned_worktree_root(&self.worktree) + } +} + +struct HeldGitMetadataFile { + file: File, + expected_identity: Vec, +} + +impl HeldGitMetadataFile { + fn open_required(path: &Path, expected_identity: &[u8]) -> Result { + let file = open_git_structural_file_no_follow(path)?; + if git_file_metadata_identity(&file.metadata()?) != expected_identity { + return Err(Error::ChangeLedgerReconcileRequired { + scope: "workspace".into(), + state: "untrusted_gap".into(), + reason: format!( + "Git structural file `{}` changed before ledger command consumption", + path.display() + ), + command: "trail index reconcile".into(), + }); + } + Ok(Self { + file, + expected_identity: expected_identity.to_vec(), + }) + } + + fn open_optional(path: &Path, expected_identity: Option<&[u8]>) -> Result> { + match expected_identity { + Some(expected) => Self::open_required(path, expected).map(Some), + None => match open_git_structural_file_no_follow(path) { + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + Ok(_) => Err(Error::ChangeLedgerReconcileRequired { + scope: "workspace".into(), + state: "untrusted_gap".into(), + reason: format!( + "Git structural file `{}` appeared before ledger command consumption", + path.display() + ), + command: "trail index reconcile".into(), + }), + }, + } + } + + fn open_optional_pair( + path: Option<&Path>, + expected_identity: Option<&[u8]>, + label: &str, + ) -> Result> { + match (path, expected_identity) { + (Some(path), expected) => Self::open_optional(path, expected), + (None, None) => Ok(None), + (None, Some(_)) => Err(Error::Git(format!( + "Git {label} identity has no structural path" + ))), + } + } + + fn verify(&mut self) -> Result { + Ok(git_file_metadata_identity(&self.file.metadata()?) == self.expected_identity) + } +} + +struct HeldGitStructuralFile { + file: File, + expected_identity: Vec, +} + +impl HeldGitStructuralFile { + fn open_required(path: &Path, expected_identity: &[u8]) -> Result { + let mut file = open_git_structural_file_no_follow(path)?; + let (observed, _) = held_git_file_identity(&mut file)?; + if observed != expected_identity { + return Err(Error::ChangeLedgerReconcileRequired { + scope: "workspace".into(), + state: "untrusted_gap".into(), + reason: format!( + "Git structural file `{}` changed before ledger consumption", + path.display() + ), + command: "trail index reconcile".into(), + }); + } + Ok(Self { + file, + expected_identity: expected_identity.to_vec(), + }) + } + + fn open_optional(path: &Path, expected_identity: Option<&[u8]>) -> Result> { + match expected_identity { + Some(expected) => Self::open_required(path, expected).map(Some), + None => match open_git_structural_file_no_follow(path) { + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + Ok(_) => Err(Error::ChangeLedgerReconcileRequired { + scope: "workspace".into(), + state: "untrusted_gap".into(), + reason: format!( + "Git structural file `{}` appeared before ledger consumption", + path.display() + ), + command: "trail index reconcile".into(), + }), + }, + } + } + + fn open_optional_pair( + path: Option<&Path>, + expected_identity: Option<&[u8]>, + label: &str, + ) -> Result> { + match (path, expected_identity) { + (Some(path), expected) => Self::open_optional(path, expected), + (None, None) => Ok(None), + (None, Some(_)) => Err(Error::Git(format!( + "Git {label} identity has no structural path" + ))), + } + } + + fn verify(&mut self) -> Result<(bool, u64)> { + let (observed, bytes) = held_git_file_identity(&mut self.file)?; + Ok((observed == self.expected_identity, bytes)) + } +} + +#[derive(Debug)] +struct GitQualificationConfig { + file_mode: bool, + symlinks: bool, + ignore_case: bool, + sparse_checkout: bool, + fsmonitor: bool, + untracked_cache: bool, +} + +#[derive(Debug, Default)] +struct GitIndexSemantics { + assume_unchanged: bool, + skip_worktree: bool, + unresolved: bool, + symlink: bool, + submodule: bool, + fsmonitor: bool, + untracked_cache: bool, + sparse_index: bool, + split_index: bool, +} + +#[derive(Debug, Default)] +struct GitPorcelainEvidence { + paths: BTreeSet, + rename_pairs: Vec<(String, String)>, + unresolved_submodule: bool, + complete: bool, +} + +fn parse_git_porcelain_v2( + output: &[u8], + matcher: &crate::db::change_ledger::CompiledRecordingMatcher, + metrics: &mut crate::db::change_ledger::GitStructuralMetrics, +) -> Result { + let records = output + .split(|byte| *byte == 0) + .filter(|record| !record.is_empty()) + .collect::>(); + let mut evidence = GitPorcelainEvidence { + complete: true, + ..GitPorcelainEvidence::default() + }; + let mut index = 0; + while index < records.len() { + let record = records[index]; + metrics.output_record_count = metrics.output_record_count.saturating_add(1); + let parsed = match record.first().copied() { + Some(b'1') => split_git_record(record, 9).map(|fields| { + evidence.unresolved_submodule |= fields[2] != b"N..."; + vec![fields[8]] + }), + Some(b'2') => split_git_record(record, 10).and_then(|fields| { + evidence.unresolved_submodule |= fields[2] != b"N..."; + index = index.saturating_add(1); + records.get(index).map(|old| vec![fields[9], *old]) + }), + Some(b'u') => split_git_record(record, 11).map(|fields| { + evidence.complete = false; + evidence.unresolved_submodule = true; + vec![fields[10]] + }), + Some(b'?') if record.get(1) == Some(&b' ') => Some(vec![&record[2..]]), + Some(b'#') => Some(Vec::new()), + _ => None, + }; + let Some(paths) = parsed else { + evidence.complete = false; + index = index.saturating_add(1); + continue; + }; + let mut normalized = Vec::new(); + for raw in paths { + let path = match std::str::from_utf8(raw) { + Ok(path) => match normalize_relative_path(path) { + Ok(path) => path, + Err(_) => { + evidence.complete = false; + continue; + } + }, + Err(_) => { + evidence.complete = false; + continue; + } + }; + if !matcher.is_ignored(&path, false)? { + evidence.paths.insert(path.clone()); + } + normalized.push(path); + } + if record.first() == Some(&b'2') && normalized.len() == 2 { + evidence + .rename_pairs + .push((normalized[1].clone(), normalized[0].clone())); + } + index = index.saturating_add(1); + } + Ok(evidence) +} + +fn split_git_record(record: &[u8], expected_fields: usize) -> Option> { + let fields = record + .splitn(expected_fields, |byte| *byte == b' ') + .collect::>(); + (fields.len() == expected_fields).then_some(fields) +} + +fn git_file_identity_optional(path: &Path) -> Result>> { + let mut file = match open_git_structural_file_no_follow(path) { + Ok(file) => file, + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + held_git_file_identity(&mut file).map(|(identity, _)| Some(identity)) +} + +fn git_file_metadata_identity_optional(path: &Path) -> Result>> { + let file = match open_git_structural_file_no_follow(path) { + Ok(file) => file, + Err(Error::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + Ok(Some(git_file_metadata_identity(&file.metadata()?))) +} + +fn open_git_structural_file_no_follow(path: &Path) -> Result { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC); + } + let file = options.open(path)?; + let metadata = file.metadata()?; + if !metadata.is_file() { + return Err(Error::Git(format!( + "Git structural file `{}` is not a regular file", + path.display() + ))); + } + Ok(file) +} + +fn git_file_metadata_identity(metadata: &fs::Metadata) -> Vec { + let mut digest = Sha256::new(); + digest.update(b"trail-git-file-metadata-identity-v1\0"); + #[cfg(unix)] + { + digest.update(metadata.dev().to_le_bytes()); + digest.update(metadata.ino().to_le_bytes()); + digest.update(metadata.mode().to_le_bytes()); + digest.update(metadata.mtime().to_le_bytes()); + digest.update(metadata.mtime_nsec().to_le_bytes()); + digest.update(metadata.ctime().to_le_bytes()); + digest.update(metadata.ctime_nsec().to_le_bytes()); + } + #[cfg(not(unix))] + if let Ok(modified) = metadata.modified() { + if let Ok(duration) = modified.duration_since(std::time::UNIX_EPOCH) { + digest.update(duration.as_nanos().to_le_bytes()); + } + } + digest.update(metadata.len().to_le_bytes()); + digest.finalize().to_vec() +} + +fn git_file_identity(metadata: &fs::Metadata, bytes: &[u8]) -> Vec { + let mut digest = Sha256::new(); + digest.update(b"trail-git-file-identity-v1\0"); + #[cfg(unix)] + { + digest.update(metadata.dev().to_le_bytes()); + digest.update(metadata.ino().to_le_bytes()); + digest.update(metadata.mode().to_le_bytes()); + } + digest.update(metadata.len().to_le_bytes()); + digest.update(Sha256::digest(bytes)); + digest.finalize().to_vec() +} + +fn held_git_file_identity(file: &mut File) -> Result<(Vec, u64)> { + file.seek(SeekFrom::Start(0))?; + let metadata = file.metadata()?; + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + file.seek(SeekFrom::Start(0))?; + Ok(( + git_file_identity(&metadata, &bytes), + saturating_u64_from_usize(bytes.len()), + )) +} + +fn merge_git_structural_metrics( + target: &mut crate::db::change_ledger::GitStructuralMetrics, + additional: &crate::db::change_ledger::GitStructuralMetrics, +) { + target.subprocess_count = target + .subprocess_count + .saturating_add(additional.subprocess_count); + target.index_refresh_count = target + .index_refresh_count + .saturating_add(additional.index_refresh_count); + target.trace2_region_count = target + .trace2_region_count + .saturating_add(additional.trace2_region_count); + target.trace2_bytes = target.trace2_bytes.saturating_add(additional.trace2_bytes); + target.output_bytes = target.output_bytes.saturating_add(additional.output_bytes); + target.output_record_count = target + .output_record_count + .saturating_add(additional.output_record_count); + target.index_read_count = target + .index_read_count + .saturating_add(additional.index_read_count); + target.index_bytes = target.index_bytes.saturating_add(additional.index_bytes); + target.shared_index_read_count = target + .shared_index_read_count + .saturating_add(additional.shared_index_read_count); + target.shared_index_bytes = target + .shared_index_bytes + .saturating_add(additional.shared_index_bytes); + target.fsmonitor_qualification_count = target + .fsmonitor_qualification_count + .saturating_add(additional.fsmonitor_qualification_count); + target.untracked_cache_qualification_count = target + .untracked_cache_qualification_count + .saturating_add(additional.untracked_cache_qualification_count); + target.external_adapter_global_work = target + .external_adapter_global_work + .saturating_add(additional.external_adapter_global_work); +} + +fn git_identity_mismatches( + observed: &GitRepositoryQualificationIdentity, + expected: &crate::db::change_ledger::GitEvidenceQualification, +) -> Vec<&'static str> { + let mut mismatches = Vec::new(); + if observed.head_oid != expected.head_oid || observed.head_identity != expected.head_identity { + mismatches.push("HEAD"); + } + if observed.worktree_top_level.to_string_lossy() != expected.worktree_top_level + || observed.worktree_identity != expected.worktree_identity + { + mismatches.push("worktree"); + } + if observed.index_identity != expected.index_identity { + mismatches.push("index"); + } + if observed.shared_index_identity != expected.shared_index_identity { + mismatches.push("shared_index"); + } + mismatches +} + +fn git_command_identity_matches( + observed: &GitCommandStructuralIdentity, + expected: &GitCommandStructuralIdentity, +) -> bool { + observed.head_oid == expected.head_oid + && observed.head_path == expected.head_path + && observed.head_identity == expected.head_identity + && observed.symbolic_ref_path == expected.symbolic_ref_path + && observed.symbolic_ref_identity == expected.symbolic_ref_identity + && observed.packed_refs_path == expected.packed_refs_path + && observed.packed_refs_identity == expected.packed_refs_identity + && observed.worktree_top_level == expected.worktree_top_level + && observed.worktree_identity == expected.worktree_identity + && observed.index_path == expected.index_path + && observed.index_identity == expected.index_identity +} + +fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() + && haystack + .windows(needle.len()) + .any(|window| window == needle) +} + +fn push_git_advisory(reasons: &mut Vec, condition: bool, reason: &str) { + if condition { + reasons.push(reason.to_string()); + } +} + +fn git_qualification_command_error( + args: &[&str], + workspace: &Path, + output: &std::process::Output, +) -> Error { + Error::Git(format!( + "git {} failed in {}: {}", + args.join(" "), + workspace.display(), + String::from_utf8_lossy(&output.stderr).trim() + )) +} + +#[cfg(debug_assertions)] +type GitQualificationHook = Box Result<()> + Send>; + +#[cfg(debug_assertions)] +static GIT_QUALIFICATION_AFTER_PORCELAIN_HOOK: std::sync::OnceLock< + std::sync::Mutex>, +> = std::sync::OnceLock::new(); + +#[cfg(debug_assertions)] +static GIT_QUALIFICATION_AFTER_C2_HOOK: std::sync::OnceLock< + std::sync::Mutex>, +> = std::sync::OnceLock::new(); + +#[cfg(debug_assertions)] +pub(crate) fn install_git_qualification_after_porcelain_hook( + hook: impl FnOnce() -> Result<()> + Send + 'static, +) { + *GIT_QUALIFICATION_AFTER_PORCELAIN_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Some(Box::new(hook)); +} + +#[cfg(debug_assertions)] +pub(crate) fn install_git_qualification_after_c2_hook( + hook: impl FnOnce() -> Result<()> + Send + 'static, +) { + *GIT_QUALIFICATION_AFTER_C2_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) = Some(Box::new(hook)); +} + +#[cfg(debug_assertions)] +fn run_git_qualification_after_porcelain_hook() -> Result<()> { + let hook = GIT_QUALIFICATION_AFTER_PORCELAIN_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .take(); + match hook { + Some(hook) => hook(), + None => Ok(()), + } +} + +#[cfg(debug_assertions)] +fn run_git_qualification_after_c2_hook() -> Result<()> { + let hook = GIT_QUALIFICATION_AFTER_C2_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .take(); + match hook { + Some(hook) => hook(), + None => Ok(()), + } +} + +fn append_git_stdin_path(input: &mut Vec, path: &Path) -> Result<()> { + #[cfg(unix)] + let bytes = path.as_os_str().as_bytes(); + #[cfg(not(unix))] + let bytes = path + .to_str() + .ok_or_else(|| Error::Git("Git blob batch path is not valid UTF-8".to_string()))? + .as_bytes(); + if bytes.contains(&b'\n') || bytes.contains(&b'\r') { + return Err(Error::Git( + "Git blob batch path contains a line separator".to_string(), + )); + } + input.extend_from_slice(bytes); + input.push(b'\n'); + Ok(()) +} + +fn parse_git_hash_object_oids(output: &[u8], expected_count: usize) -> Result> { + let output = std::str::from_utf8(output) + .map_err(|err| Error::Git(format!("git hash-object returned non-UTF-8 output: {err}")))?; + let mut lines = output.split('\n').collect::>(); + if lines.last() == Some(&"") { + lines.pop(); + } + if lines.len() != expected_count { + return Err(Error::Git(format!( + "git hash-object returned {} object IDs for {expected_count} paths", + lines.len() + ))); + } + let mut oid_length = None; + let mut oids = Vec::with_capacity(lines.len()); + for (index, oid) in lines.into_iter().enumerate() { + if !matches!(oid.len(), 40 | 64) || !oid.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(Error::Git(format!( + "git hash-object returned invalid object ID at position {index}: `{oid}`" + ))); + } + if oid_length + .replace(oid.len()) + .is_some_and(|length| length != oid.len()) + { + return Err(Error::Git( + "git hash-object returned mixed object ID lengths".to_string(), + )); + } + oids.push(oid.to_string()); + } + Ok(oids) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(unix)] + #[test] + fn git_stdin_path_preserves_non_utf8_bytes() { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + + let path = PathBuf::from(OsString::from_vec(b"/tmp/trail-\xff/blob".to_vec())); + let mut input = Vec::new(); + + append_git_stdin_path(&mut input, &path).unwrap(); + + assert_eq!(input, b"/tmp/trail-\xff/blob\n"); + } + + #[test] + fn git_hash_batch_drains_output_larger_than_pipe_capacity() { + const CHILD_ENV: &str = "TRAIL_GIT_HASH_BATCH_DEADLOCK_CHILD"; + const TEST_NAME: &str = + "db::storage::git::tests::git_hash_batch_drains_output_larger_than_pipe_capacity"; + if std::env::var_os(CHILD_ENV).is_none() { + let mut child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST_NAME, "--nocapture"]) + .env(CHILD_ENV, "1") + .spawn() + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if let Some(status) = child.try_wait().unwrap() { + assert!(status.success(), "bounded Git hash batch child failed"); + return; + } + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("Git hash batch deadlocked while stdout exceeded pipe capacity"); + } + std::thread::sleep(Duration::from_millis(25)); + } + } + + if Command::new("git").arg("--version").output().is_err() { + return; + } + let temp = tempfile::tempdir().unwrap(); + Command::new("git") + .arg("-C") + .arg(temp.path()) + .arg("init") + .output() + .unwrap(); + fs::write(temp.path().join("batch-source"), b"batch bytes\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let mut input = Vec::with_capacity(1_300_000); + for _ in 0..100_000 { + input.extend_from_slice(b"batch-source\n"); + } + + let output = db + .git_output_bytes_with_input(&["hash-object", "-w", "--stdin-paths"], &input, None) + .unwrap(); + + assert!(output.len() > 4_000_000); + assert_eq!( + parse_git_hash_object_oids(&output, 100_000).unwrap().len(), + 100_000 + ); + } + + #[test] + fn git_hash_batch_output_preserves_order_and_validates_count() { + let first = "1".repeat(40); + let second = "a".repeat(40); + let output = format!("{first}\n{second}\n"); + assert_eq!( + parse_git_hash_object_oids(output.as_bytes(), 2).unwrap(), + vec![first, second] + ); + assert!(matches!( + parse_git_hash_object_oids(output.as_bytes(), 1), + Err(Error::Git(message)) if message.contains("2 object IDs for 1 paths") + )); + } + + #[test] + fn git_hash_batch_output_rejects_invalid_or_mixed_oids() { + assert!(matches!( + parse_git_hash_object_oids(b"not-an-oid\n", 1), + Err(Error::Git(message)) if message.contains("invalid object ID") + )); + let mixed = format!("{}\n{}\n", "1".repeat(40), "2".repeat(64)); + assert!(matches!( + parse_git_hash_object_oids(mixed.as_bytes(), 2), + Err(Error::Git(message)) if message.contains("mixed object ID lengths") + )); + } + + #[test] + fn git_publication_state_rejects_changed_head() { + assert!(matches!( + validate_git_publication_state( + "old", + &GitState { + head: Some("new".into()), + dirty: false, + } + ), + Err(Error::GitHeadChanged(_)) + )); + } + + #[test] + fn git_publication_state_rejects_dirty_worktree() { + assert!(matches!( + validate_git_publication_state( + "head", + &GitState { + head: Some("head".into()), + dirty: true, + } + ), + Err(Error::GitWorktreeDirty(_)) + )); + } } diff --git a/trail/src/db/storage/lifecycle/gc.rs b/trail/src/db/storage/lifecycle/gc.rs index 92579cb..049f98f 100644 --- a/trail/src/db/storage/lifecycle/gc.rs +++ b/trail/src/db/storage/lifecycle/gc.rs @@ -1,9 +1,17 @@ use super::*; +use crate::db::change_ledger::{ledger_gc_roots, IntentGcRoot}; impl Trail { pub fn gc(&mut self, dry_run: bool) -> Result { let _lock = self.acquire_write_lock()?; - let reachable = self.reachable_object_ids()?; + // Capture roots before recovery terminalizes an intent. This makes the + // recovery boundary itself conservative: the next GC, not this one, + // may collect a target that recovery proved terminal. + let intent_roots = ledger_gc_roots(&self.conn)?; + if !dry_run { + self.changed_path_ledger().recover()?; + } + let reachable = self.reachable_object_ids_with_intent_roots(&intent_roots)?; let known_kinds = known_gc_object_kinds(); let mut stmt = self .conn @@ -48,7 +56,10 @@ impl Trail { Ok(report) } - pub(crate) fn reachable_object_ids(&self) -> Result> { + fn reachable_object_ids_with_intent_roots( + &self, + intent_roots: &[IntentGcRoot], + ) -> Result> { let (operation_objects, mut errors) = self.operation_objects()?; let reachable_changes = self.reachable_operation_changes(&operation_objects, &mut errors)?; @@ -58,6 +69,60 @@ impl Trail { .collect::>(); let mut reachable = HashSet::new(); + let by_object = operation_objects + .iter() + .map(|object| (object.object_id.0.as_str(), object)) + .collect::>(); + for intent in intent_roots { + self.collect_root_reachable(&intent.root_id, &mut reachable, &mut errors); + let Some(operation_id) = &intent.operation_id else { + continue; + }; + let Some(target_operation) = by_object.get(operation_id.0.as_str()) else { + errors.push(format!( + "intent operation {} is missing or is not a valid operation object", + operation_id.0 + )); + continue; + }; + if target_operation.operation.change_id != intent.change_id + || target_operation.operation.after_root != intent.root_id + { + errors.push(format!( + "intent operation {} does not match target change/root", + operation_id.0 + )); + continue; + } + let mut pending = vec![intent.change_id.0.clone()]; + while let Some(change_id) = pending.pop() { + let Some(object) = by_change.get(&change_id) else { + errors.push(format!( + "intent operation ancestry is missing change {change_id}" + )); + continue; + }; + if !reachable.insert(object.object_id.0.clone()) { + continue; + } + if let Some(before) = &object.operation.before_root { + self.collect_root_reachable(before, &mut reachable, &mut errors); + } + self.collect_root_reachable( + &object.operation.after_root, + &mut reachable, + &mut errors, + ); + pending.extend( + object + .operation + .parents + .iter() + .map(|parent| parent.0.clone()), + ); + } + } + for reference in self.all_refs()? { reachable.insert(reference.root_id.0.clone()); reachable.insert(reference.operation_id.0.clone()); diff --git a/trail/src/db/storage/lifecycle/rebuild.rs b/trail/src/db/storage/lifecycle/rebuild.rs index 867ce9f..7a8a04e 100644 --- a/trail/src/db/storage/lifecycle/rebuild.rs +++ b/trail/src/db/storage/lifecycle/rebuild.rs @@ -1,5 +1,114 @@ use super::*; +const PENDING_REPAIR_LANE_MANIFEST: &str = "lane_manifest"; +const PENDING_REPAIR_WORKSPACE_CHECKPOINT: &str = "workspace_checkpoint"; + +struct PreparedPathIndexRepair { + old_root: ObjectId, + new_root: ObjectId, + case_fold_map_root: String, +} + +#[derive(Clone)] +struct CleanGitMappingRepairSource { + direction: String, + branch: String, + git_head: Option, +} + +struct PreparedLaneRepair { + branch: LaneBranch, + retarget_clean_manifest: bool, + checkpoint_view_id: Option, + retarget_checkpoint_marker: bool, +} + +struct PendingPathIndexDerivedRepair { + ref_name: String, + repair_kind: String, + old_root: ObjectId, + new_root: ObjectId, + new_change: ChangeId, +} + +fn remove_path_index_derived_mirror(path: &Path) -> bool { + match fs::remove_file(path) { + Ok(()) => true, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => true, + Err(_) => false, + } +} + +struct PreparedRefRepairPublication { + reference: RefRecord, + new_root: ObjectId, + change_id: ChangeId, + operation_id: ObjectId, + operation: Operation, + git_mappings: Vec, + lane: Option, + retarget_current_worktree_baseline: bool, +} + +struct PreflightRefRepair { + reference: RefRecord, + new_root: ObjectId, + git_mappings: Vec, + lane: Option, + retarget_current_worktree_baseline: bool, +} + +#[derive(Default)] +struct PathIndexRepairOutcome { + roots: Vec, + refs: Vec, +} + +#[cfg(test)] +#[derive(Clone, Copy, Default)] +struct PathIndexRebuildMemoryMetrics { + live_paths: usize, + peak_live_paths: usize, + observed_paths: usize, +} + +#[cfg(test)] +thread_local! { + static PATH_INDEX_REBUILD_MEMORY_METRICS: Cell = + Cell::new(PathIndexRebuildMemoryMetrics::default()); +} + +#[cfg(test)] +fn reset_path_index_rebuild_memory_metrics() { + PATH_INDEX_REBUILD_MEMORY_METRICS + .with(|metrics| metrics.set(PathIndexRebuildMemoryMetrics::default())); +} + +#[cfg(test)] +fn retain_path_index_rebuild_paths(count: usize) { + PATH_INDEX_REBUILD_MEMORY_METRICS.with(|metrics| { + let mut next = metrics.get(); + next.live_paths += count; + next.observed_paths += count; + next.peak_live_paths = next.peak_live_paths.max(next.live_paths); + metrics.set(next); + }); +} + +#[cfg(test)] +fn release_path_index_rebuild_paths(count: usize) { + PATH_INDEX_REBUILD_MEMORY_METRICS.with(|metrics| { + let mut next = metrics.get(); + next.live_paths = next.live_paths.saturating_sub(count); + metrics.set(next); + }); +} + +#[cfg(test)] +fn path_index_rebuild_memory_metrics() -> PathIndexRebuildMemoryMetrics { + PATH_INDEX_REBUILD_MEMORY_METRICS.with(Cell::get) +} + impl Trail { pub fn rebuild_indexes(&mut self) -> Result { let _lock = self.acquire_write_lock()?; @@ -66,6 +175,15 @@ impl Trail { } pub(crate) fn rebuild_indexes_unlocked(&self) -> Result { + self.drain_pending_path_index_derived_repairs()?; + // Repair immutable root state before deleting/rebuilding the derived + // operation indexes. The maintenance operations published below must + // participate in reachability and be indexed by this same command. + let path_index_repairs = self.rebuild_live_path_invariant_indexes_unlocked()?; + // Ref files are derived mirrors of the authoritative SQLite refs. Run + // this on every rebuild so an interrupted/permission-blocked mirror + // write after a prior committed repair remains retryable. + self.reconcile_live_ref_files_best_effort(); let (operation_objects, mut errors) = self.operation_objects()?; let reachable_changes = self.reachable_operation_changes(&operation_objects, &mut errors)?; @@ -88,6 +206,8 @@ impl Trail { let mut report = IndexRebuildReport { errors, + path_index_repaired_roots: path_index_repairs.roots, + path_index_repaired_refs: path_index_repairs.refs, ..IndexRebuildReport::default() }; for change_id in changes { @@ -118,6 +238,638 @@ impl Trail { Ok(report) } + fn rebuild_live_path_invariant_indexes_unlocked(&self) -> Result { + let live_refs = self + .all_refs()? + .into_iter() + .filter(|reference| { + reference.name.starts_with(MAIN_REF_PREFIX) + || reference.name.starts_with(LANE_REF_PREFIX) + }) + .collect::>(); + + // First validate every distinct legacy root. No operation, ref, lane, + // baseline, or Git-mapping metadata is published until all roots pass. + let mut examined_roots = BTreeSet::new(); + let mut legacy_roots = BTreeMap::::new(); + for reference in &live_refs { + if !examined_roots.insert(reference.root_id.clone()) { + continue; + } + let root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, &reference.root_id)?; + if root.case_fold_map_root.is_some() || root.file_count == 0 { + continue; + } + self.with_validated_legacy_root_paths(&reference.root_id, root.file_count, |_| Ok(()))?; + legacy_roots.insert(reference.root_id.clone(), root.file_count); + } + + // Building may write content-addressed Prolly nodes and root objects, + // but only after every legacy root has passed path validation. + let mut prepared = BTreeMap::::new(); + for (old_root_id, file_count) in legacy_roots { + let mut root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, &old_root_id)?; + let case_fold_map_root = + self.with_validated_legacy_root_paths(&old_root_id, file_count, |paths| { + let case_fold_tree = self.build_case_fold_map_tree(paths.iter())?; + tree_root_hex(&case_fold_tree).ok_or_else(|| { + Error::Corrupt(format!( + "non-empty legacy root {} produced an empty path-invariant index", + old_root_id.0 + )) + }) + })?; + root.case_fold_map_root = Some(case_fold_map_root.clone()); + let new_root = self.put_object(WORKTREE_ROOT_KIND, ROOT_OBJECT_VERSION, &root)?; + prepared.insert( + old_root_id.clone(), + PreparedPathIndexRepair { + old_root: old_root_id, + new_root, + case_fold_map_root, + }, + ); + } + + if prepared.is_empty() { + return Ok(PathIndexRepairOutcome::default()); + } + + let current_branch_ref = branch_ref(&self.current_branch()?); + let current_worktree_baseline = self.worktree_index_baseline_root()?; + let mut preflight_refs = Vec::new(); + + // Preflight every ref's derived metadata before creating maintenance + // operations or advancing the first ref. This keeps a corrupt later + // lane from partially publishing repairs for earlier refs. + for reference in live_refs { + let Some(repair) = prepared.get(&reference.root_id) else { + continue; + }; + let lane = if let Some(lane_name) = reference.name.strip_prefix(LANE_REF_PREFIX) { + Some(self.preflight_lane_path_index_repair(&reference, lane_name)?) + } else { + None + }; + let git_mappings = self.clean_git_mapping_sources_for_path_index_repair(&reference)?; + preflight_refs.push(PreflightRefRepair { + retarget_current_worktree_baseline: reference.name == current_branch_ref + && current_worktree_baseline.as_ref() == Some(&reference.root_id), + reference, + new_root: repair.new_root.clone(), + git_mappings, + lane, + }); + } + + let mut publications = Vec::new(); + for preflight in preflight_refs { + let actor = Actor::system(); + let change_id = self.allocate_change_id(&actor.id, "path-index-rebuild")?; + let operation = Operation { + version: OP_OBJECT_VERSION, + change_id: change_id.clone(), + kind: OperationKind::ManualCheckpoint, + parents: vec![preflight.reference.change_id.clone()], + before_root: Some(preflight.reference.root_id.clone()), + after_root: preflight.new_root.clone(), + branch: preflight + .reference + .name + .strip_prefix(MAIN_REF_PREFIX) + .unwrap_or(&preflight.reference.name) + .to_string(), + actor, + session_id: None, + message: Some("Rebuild path invariant index".to_string()), + changes: Vec::new(), + created_at: now_ts(), + }; + // The object is immutable and may safely become orphaned if the + // later atomic SQLite publication fails. + let operation_id = self.put_object(OPERATION_KIND, OP_OBJECT_VERSION, &operation)?; + publications.push(PreparedRefRepairPublication { + retarget_current_worktree_baseline: preflight.retarget_current_worktree_baseline, + reference: preflight.reference, + new_root: preflight.new_root, + change_id, + operation_id, + operation, + git_mappings: preflight.git_mappings, + lane: preflight.lane, + }); + } + + // Publish all authoritative SQLite metadata as one unit. Ref files and + // clean manifest/checkpoint files are derived mirrors and are refreshed + // after commit using their existing recovery semantics. + self.conn.execute_batch("BEGIN IMMEDIATE;")?; + let publication_result = (|| -> Result<()> { + for publication in &publications { + self.index_operation_in_transaction( + &publication.operation, + &publication.operation_id, + )?; + if let Some(lane) = &publication.lane { + if lane.retarget_clean_manifest { + self.insert_pending_path_index_derived_repair( + &publication.reference.name, + PENDING_REPAIR_LANE_MANIFEST, + &publication.reference.root_id, + &publication.new_root, + &publication.change_id, + )?; + } + if lane.retarget_checkpoint_marker { + self.insert_pending_path_index_derived_repair( + &publication.reference.name, + PENDING_REPAIR_WORKSPACE_CHECKPOINT, + &publication.reference.root_id, + &publication.new_root, + &publication.change_id, + )?; + } + } + let generation = publication.reference.generation + 1; + let updated = self.conn.execute( + "UPDATE refs SET change_id = ?1, root_id = ?2, operation_id = ?3, generation = ?4, updated_at = ?5 \ + WHERE name = ?6 AND generation = ?7 AND change_id = ?8 AND root_id = ?9", + params![ + publication.change_id.0, + publication.new_root.0, + publication.operation_id.0, + generation, + now_ts(), + publication.reference.name, + publication.reference.generation, + publication.reference.change_id.0, + publication.reference.root_id.0 + ], + )?; + if updated != 1 { + return Err(Error::StaleBranch(publication.reference.name.clone())); + } + + if let Some(lane) = &publication.lane { + let updated = self.conn.execute( + "UPDATE lane_branches SET head_change = ?1, head_root = ?2, updated_at = ?3 \ + WHERE lane_id = ?4 AND head_change = ?5 AND head_root = ?6", + params![ + publication.change_id.0, + publication.new_root.0, + now_ts(), + lane.branch.lane_id, + publication.reference.change_id.0, + publication.reference.root_id.0 + ], + )?; + if updated != 1 { + return Err(Error::Corrupt(format!( + "lane branch {} changed during path-index repair", + lane.branch.ref_name + ))); + } + if let Some(view_id) = &lane.checkpoint_view_id { + let updated = self.conn.execute( + "UPDATE workspace_views SET checkpoint_root = ?1, updated_at = ?2 WHERE view_id = ?3 AND checkpoint_root = ?4", + params![ + publication.new_root.0, + now_ts(), + view_id, + publication.reference.root_id.0 + ], + )?; + if updated != 1 { + return Err(Error::Corrupt(format!( + "workspace view {view_id} changed during path-index repair" + ))); + } + } + } + + if publication.retarget_current_worktree_baseline { + self.conn.execute( + "INSERT OR REPLACE INTO schema_meta (key, value, updated_at) VALUES (?1, ?2, ?3)", + params![ + "worktree.index.baseline_root", + publication.new_root.0, + now_ts() + ], + )?; + } + for mapping in &publication.git_mappings { + self.insert_git_mapping_for_state( + &mapping.direction, + &mapping.branch, + &publication.change_id, + &publication.new_root, + mapping.git_head.clone(), + false, + )?; + } + } + Ok(()) + })(); + match publication_result { + Ok(()) => { + if let Err(err) = self.conn.execute_batch("COMMIT;") { + let _ = self.conn.execute_batch("ROLLBACK;"); + return Err(Error::from(err)); + } + } + Err(err) => { + let _ = self.conn.execute_batch("ROLLBACK;"); + return Err(err); + } + } + test_crash_point("path_index_rebuild_after_publication_commit"); + self.drain_pending_path_index_derived_repairs()?; + + let mut outcome = PathIndexRepairOutcome { + roots: prepared + .values() + .map(|repair| PathIndexRootRepair { + old_root: repair.old_root.clone(), + new_root: repair.new_root.clone(), + case_fold_map_root: repair.case_fold_map_root.clone(), + }) + .collect(), + refs: Vec::new(), + }; + for publication in publications { + outcome.refs.push(PathIndexRefRepair { + name: publication.reference.name, + old_change: publication.reference.change_id, + new_change: publication.change_id, + old_root: publication.reference.root_id, + new_root: publication.new_root, + }); + } + Ok(outcome) + } + + fn with_validated_legacy_root_paths( + &self, + root_id: &ObjectId, + expected_file_count: u64, + consume: impl FnOnce(&[String]) -> Result, + ) -> Result { + let root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, root_id)?; + let path_tree = root_map_tree_from_root_hex(root.path_map_root.as_deref())?; + let mut paths = Vec::new(); + for item in self.root_prolly.range(&path_tree, &[], None)? { + let (key, _) = item?; + let path = String::from_utf8(key).map_err(|err| { + Error::Corrupt(format!( + "legacy root {} has a non UTF-8 path-map key: {err}", + root_id.0 + )) + })?; + let normalized = normalize_relative_path(&path).map_err(|err| { + Error::Corrupt(format!( + "legacy root {} has invalid path-map key {path:?}: {err}", + root_id.0 + )) + })?; + if normalized != path { + return Err(Error::Corrupt(format!( + "legacy root {} has noncanonical path-map key {path:?}; path must be normalized as {normalized:?}", + root_id.0 + ))); + } + paths.push(path); + } + if paths.len() as u64 != expected_file_count { + return Err(Error::Corrupt(format!( + "legacy root {} declares {} files but its path map contains {} entries", + root_id.0, + expected_file_count, + paths.len() + ))); + } + validate_no_case_fold_collisions(paths.iter()).map_err(|err| match err { + Error::InvalidPath { path, reason } => Error::InvalidPath { + path, + reason: format!("legacy root {}: {reason}", root_id.0), + }, + other => other, + })?; + + #[cfg(test)] + retain_path_index_rebuild_paths(paths.len()); + let result = consume(&paths); + #[cfg(test)] + release_path_index_rebuild_paths(paths.len()); + result + } + + fn insert_pending_path_index_derived_repair( + &self, + ref_name: &str, + repair_kind: &str, + old_root: &ObjectId, + new_root: &ObjectId, + new_change: &ChangeId, + ) -> Result<()> { + self.conn.execute( + "INSERT INTO pending_path_index_derived_repairs \ + (ref_name, repair_kind, old_root, new_root, new_change, created_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6) \ + ON CONFLICT(ref_name, repair_kind) DO UPDATE SET \ + old_root = excluded.old_root, new_root = excluded.new_root, \ + new_change = excluded.new_change, created_at = excluded.created_at", + params![ + ref_name, + repair_kind, + old_root.0, + new_root.0, + new_change.0, + now_ts() + ], + )?; + Ok(()) + } + + pub(crate) fn has_pending_path_index_derived_repairs(&self) -> Result { + self.conn + .query_row( + "SELECT EXISTS(SELECT 1 FROM pending_path_index_derived_repairs LIMIT 1)", + [], + |row| row.get(0), + ) + .map_err(Error::from) + } + + pub(crate) fn drain_pending_path_index_derived_repairs(&self) -> Result<()> { + self.drain_pending_path_index_derived_repairs_with_db_mapping(None) + } + + pub(crate) fn drain_pending_path_index_derived_repairs_from_restore_stage( + &self, + published_db_dir: &Path, + ) -> Result<()> { + self.drain_pending_path_index_derived_repairs_with_db_mapping(Some(published_db_dir)) + } + + fn drain_pending_path_index_derived_repairs_with_db_mapping( + &self, + published_db_dir: Option<&Path>, + ) -> Result<()> { + let pending = { + let mut stmt = self.conn.prepare( + "SELECT ref_name, repair_kind, old_root, new_root, new_change \ + FROM pending_path_index_derived_repairs \ + ORDER BY ref_name, repair_kind", + )?; + let repairs = stmt + .query_map([], |row| { + Ok(PendingPathIndexDerivedRepair { + ref_name: row.get(0)?, + repair_kind: row.get(1)?, + old_root: ObjectId(row.get(2)?), + new_root: ObjectId(row.get(3)?), + new_change: ChangeId(row.get(4)?), + }) + })? + .collect::, _>>()?; + repairs + }; + + for repair in pending { + if !self.repair_pending_path_index_derived_mirror(&repair, published_db_dir)? { + continue; + } + self.conn.execute( + "DELETE FROM pending_path_index_derived_repairs \ + WHERE ref_name = ?1 AND repair_kind = ?2 AND old_root = ?3 \ + AND new_root = ?4 AND new_change = ?5", + params![ + repair.ref_name, + repair.repair_kind, + repair.old_root.0, + repair.new_root.0, + repair.new_change.0 + ], + )?; + } + Ok(()) + } + + fn repair_pending_path_index_derived_mirror( + &self, + repair: &PendingPathIndexDerivedRepair, + published_db_dir: Option<&Path>, + ) -> Result { + let Some(reference) = self.try_get_ref(&repair.ref_name)? else { + return Ok(true); + }; + if reference.root_id != repair.new_root || reference.change_id != repair.new_change { + return Ok(true); + } + let Some(lane_name) = repair.ref_name.strip_prefix(LANE_REF_PREFIX) else { + return Ok(true); + }; + let branch = match self.lane_branch(lane_name) { + Ok(branch) => branch, + Err(Error::RefNotFound(_)) => return Ok(true), + Err(err) => return Err(err), + }; + if branch.status == "removed" + || branch.ref_name != repair.ref_name + || branch.head_root != repair.new_root + || branch.head_change != repair.new_change + { + return Ok(true); + } + let lane = match self.lane_record(&branch.lane_id) { + Ok(lane) => lane, + Err(Error::RefNotFound(_)) => return Ok(true), + Err(err) => return Err(err), + }; + + match repair.repair_kind.as_str() { + PENDING_REPAIR_LANE_MANIFEST => { + let Some(workdir) = branch.workdir.as_deref() else { + return Ok(true); + }; + let path = self + .lane_layered_clean_manifest_path(&branch)? + .unwrap_or_else(|| { + Path::new(workdir) + .join(".trail") + .join("workdir-manifest.json") + }); + let path = published_db_dir + .and_then(|published| path.strip_prefix(published).ok()) + .map_or(path.clone(), |relative| self.db_dir.join(relative)); + Ok(self.repair_clean_workdir_manifest_root_mirror( + &path, + &repair.old_root, + &repair.new_root, + )) + } + PENDING_REPAIR_WORKSPACE_CHECKPOINT => { + let Some(view) = self.lane_workspace_view(&lane.name)? else { + return Ok(true); + }; + if view.checkpoint_root.as_ref() != Some(&repair.new_root) { + return Ok(true); + } + let current_meta = self.workspace_view_paths_for_lane_name(&lane.name).meta_dir; + if Path::new(&view.meta_dir) != current_meta { + return Ok(false); + } + Ok(self.repair_workspace_checkpoint_marker_mirror( + ¤t_meta.join("clean-checkpoint.json"), + &view, + repair, + )) + } + unknown => Err(Error::Corrupt(format!( + "pending path-index derived repair has unknown kind `{unknown}`" + ))), + } + } + + fn repair_workspace_checkpoint_marker_mirror( + &self, + path: &Path, + view: &LaneWorkspaceViewReport, + repair: &PendingPathIndexDerivedRepair, + ) -> bool { + let bytes = match fs::read(path) { + Ok(bytes) => bytes, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return true, + Err(_) => return false, + }; + let mut marker: serde_json::Value = match serde_json::from_slice(&bytes) { + Ok(marker) => marker, + Err(_) => return remove_path_index_derived_mirror(path), + }; + if marker["view_id"].as_str() != Some(view.view_id.as_str()) + || marker["journal_sequence"].as_u64() != Some(view.checkpoint_seq) + { + return remove_path_index_derived_mirror(path); + } + let marker_root = marker["root_id"].as_str(); + if marker_root == Some(repair.new_root.0.as_str()) + && marker["operation"].as_str() == Some(repair.new_change.0.as_str()) + { + return true; + } + if marker_root != Some(repair.old_root.0.as_str()) { + return remove_path_index_derived_mirror(path); + } + marker["root_id"] = serde_json::Value::String(repair.new_root.0.clone()); + marker["operation"] = serde_json::Value::String(repair.new_change.0.clone()); + match serde_json::to_vec_pretty(&marker) { + Ok(bytes) if write_file_atomic(path, &bytes, false).is_ok() => true, + _ => remove_path_index_derived_mirror(path), + } + } + + fn reconcile_live_ref_files_best_effort(&self) { + let Ok(references) = self.all_refs() else { + return; + }; + for reference in references { + let _ = write_ref_file( + &self.db_dir, + &reference.name, + &reference.change_id, + &reference.root_id, + &reference.operation_id, + reference.generation, + ); + } + } + + fn clean_git_mapping_sources_for_path_index_repair( + &self, + reference: &RefRecord, + ) -> Result> { + let mut stmt = self.conn.prepare( + "SELECT DISTINCT direction, branch, git_head FROM git_mappings \ + WHERE crab_root = ?1 AND git_dirty = 0 \ + ORDER BY direction, branch, git_head", + )?; + let rows = stmt.query_map(params![reference.root_id.0], |row| { + Ok(CleanGitMappingRepairSource { + direction: row.get(0)?, + branch: row.get(1)?, + git_head: row.get(2)?, + }) + })?; + rows.collect::, _>>() + .map_err(Error::from) + } + + fn preflight_lane_path_index_repair( + &self, + reference: &RefRecord, + lane_name: &str, + ) -> Result { + let branch = self.lane_branch(lane_name)?; + if branch.ref_name != reference.name + || branch.head_change != reference.change_id + || branch.head_root != reference.root_id + { + return Err(Error::Corrupt(format!( + "lane branch {} does not match its mutable ref head before path-index repair", + reference.name + ))); + } + let layered_manifest_path = self.lane_layered_clean_manifest_path(&branch)?; + let retarget_clean_manifest = if let Some(workdir) = &branch.workdir { + self.preflight_clean_workdir_manifest_root_retarget( + Path::new(workdir), + layered_manifest_path.as_deref(), + &reference.root_id, + )? + } else { + false + }; + + let lane = self.lane_record(&branch.lane_id)?; + let mut checkpoint_view_id = None; + let mut retarget_checkpoint_marker = false; + if let Some(view) = self.lane_workspace_view(&lane.name)? { + if view.checkpoint_root.as_ref() == Some(&reference.root_id) { + checkpoint_view_id = Some(view.view_id.clone()); + let path = Path::new(&view.meta_dir).join("clean-checkpoint.json"); + match fs::read(&path) { + Ok(bytes) => { + let marker: serde_json::Value = + serde_json::from_slice(&bytes).map_err(|err| { + Error::Corrupt(format!( + "workspace checkpoint marker `{}` cannot be retargeted: {err}", + path.display() + )) + })?; + if marker["view_id"].as_str() != Some(view.view_id.as_str()) + || marker["root_id"].as_str() != Some(reference.root_id.0.as_str()) + || marker["journal_sequence"].as_u64() != Some(view.checkpoint_seq) + { + return Err(Error::Corrupt(format!( + "workspace checkpoint marker `{}` does not match its clean lane baseline", + path.display() + ))); + } + retarget_checkpoint_marker = true; + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(Error::Io(err)), + } + } + } + Ok(PreparedLaneRepair { + branch, + retarget_clean_manifest, + checkpoint_view_id, + retarget_checkpoint_marker, + }) + } + pub(crate) fn operation_objects(&self) -> Result<(Vec, Vec)> { let mut stmt = self .conn @@ -213,3 +965,876 @@ impl Trail { Ok(reachable) } } + +#[cfg(test)] +mod path_index_rebuild_tests { + use super::*; + use std::process::Stdio; + + fn publish_legacy_root(db: &Trail, head: &RefRecord) -> (ObjectId, ChangeId) { + let mut legacy: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &head.root_id).unwrap(); + assert!(legacy.case_fold_map_root.take().is_some()); + let legacy_root_id = db + .put_object(WORKTREE_ROOT_KIND, ROOT_OBJECT_VERSION, &legacy) + .unwrap(); + let change_id = db + .allocate_change_id("trail-test", "legacy-path-index") + .unwrap(); + let operation = Operation { + version: OP_OBJECT_VERSION, + change_id: change_id.clone(), + kind: OperationKind::ManualCheckpoint, + parents: vec![head.change_id.clone()], + before_root: Some(head.root_id.clone()), + after_root: legacy_root_id.clone(), + branch: head.name.clone(), + actor: Actor::system(), + session_id: None, + message: Some("Simulate legacy root".to_string()), + changes: Vec::new(), + created_at: now_ts(), + }; + let operation_id = db.store_operation(&operation).unwrap(); + db.advance_ref_cas(head, &change_id, &legacy_root_id, &operation_id) + .unwrap(); + (legacy_root_id, change_id) + } + + fn write_patch(path: &str, content: &str, base_change: &ChangeId) -> PatchDocument { + serde_json::from_value(serde_json::json!({ + "base_change": base_change.0, + "edits": [{"op": "write", "path": path, "content": content}] + })) + .unwrap() + } + + #[test] + fn rebuild_repairs_shared_branch_and_lane_legacy_heads_once() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "hello\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let modern = db.resolve_branch_ref("main").unwrap(); + let modern_files = db.load_root_files(&modern.root_id).unwrap(); + let (legacy_root_id, legacy_change_id) = publish_legacy_root(&db, &modern); + db.set_worktree_index_baseline(&legacy_root_id).unwrap(); + db.spawn_lane("legacy-lane", Some("main"), true, None, None) + .unwrap(); + + // Compatibility reads and materialization stay available before repair. + assert_eq!(db.load_root_files(&legacy_root_id).unwrap(), modern_files); + assert!(db + .diff_root_file_summaries(&modern.root_id, &legacy_root_id) + .unwrap() + .is_empty()); + let materialized = tempfile::tempdir().unwrap(); + db.materialize_files_at(materialized.path(), &BTreeMap::new(), &modern_files) + .unwrap(); + assert_eq!( + fs::read(materialized.path().join("README.md")).unwrap(), + b"hello\n" + ); + + let object_count_before: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM objects", [], |row| row.get(0)) + .unwrap(); + let operation_count_before: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM operations", [], |row| row.get(0)) + .unwrap(); + let prolly_count_before: i64 = db + .conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row.get(0)) + .unwrap(); + let err = db + .apply_lane_patch( + "legacy-lane", + write_patch("new.txt", "new\n", &legacy_change_id), + ) + .unwrap_err(); + assert!(matches!(err, Error::PathIndexRequired(_))); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM objects", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + object_count_before + ); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM operations", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + operation_count_before + ); + assert_eq!( + db.conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + prolly_count_before + ); + + let report = db.rebuild_indexes().unwrap(); + assert_eq!(report.path_index_repaired_roots.len(), 1); + assert_eq!(report.path_index_repaired_refs.len(), 2); + let repaired_branch = db.resolve_branch_ref("main").unwrap(); + let repaired_lane = db.get_ref(&lane_ref("legacy-lane")).unwrap(); + assert_eq!(repaired_branch.root_id, repaired_lane.root_id); + assert_ne!(repaired_branch.root_id, legacy_root_id); + let repaired_root: WorktreeRoot = db + .get_object(WORKTREE_ROOT_KIND, &repaired_branch.root_id) + .unwrap(); + let legacy_root: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, &legacy_root_id).unwrap(); + assert!(repaired_root.case_fold_map_root.is_some()); + assert_eq!(repaired_root.path_map_root, legacy_root.path_map_root); + assert_eq!( + repaired_root.file_index_map_root, + legacy_root.file_index_map_root + ); + assert_eq!(repaired_root.file_count, legacy_root.file_count); + assert_eq!(repaired_root.total_text_bytes, legacy_root.total_text_bytes); + assert_eq!(repaired_root.created_by, legacy_root.created_by); + assert_eq!( + db.load_root_files(&repaired_branch.root_id).unwrap(), + modern_files + ); + assert_eq!( + db.worktree_index_baseline_root().unwrap(), + Some(repaired_branch.root_id.clone()) + ); + let lane_row = db.lane_branch("legacy-lane").unwrap(); + assert_eq!(lane_row.head_change, repaired_lane.change_id); + assert_eq!(lane_row.head_root, repaired_lane.root_id); + assert!(db + .preview_lane_workdir_record("legacy-lane") + .unwrap() + .changed_paths + .is_empty()); + for repaired_ref in &report.path_index_repaired_refs { + let operation = db.operation(&repaired_ref.new_change).unwrap(); + assert!(operation.changes.is_empty()); + assert_eq!(operation.before_root, Some(repaired_ref.old_root.clone())); + assert_eq!(operation.after_root, repaired_ref.new_root); + assert_eq!( + operation.message.as_deref(), + Some("Rebuild path invariant index") + ); + } + + let applied = db + .apply_lane_patch( + "legacy-lane", + write_patch("new.txt", "new\n", &repaired_lane.change_id), + ) + .unwrap(); + assert_eq!(applied.changed_paths.len(), 1); + + let branch_before_second = db.resolve_branch_ref("main").unwrap(); + let lane_before_second = db.get_ref(&lane_ref("legacy-lane")).unwrap(); + let second = db.rebuild_indexes().unwrap(); + assert!(second.path_index_repaired_roots.is_empty()); + assert!(second.path_index_repaired_refs.is_empty()); + let branch_after_second = db.resolve_branch_ref("main").unwrap(); + assert_eq!( + branch_after_second.change_id, + branch_before_second.change_id + ); + assert_eq!(branch_after_second.root_id, branch_before_second.root_id); + assert_eq!( + branch_after_second.generation, + branch_before_second.generation + ); + let lane_after_second = db.get_ref(&lane_ref("legacy-lane")).unwrap(); + assert_eq!(lane_after_second.change_id, lane_before_second.change_id); + assert_eq!(lane_after_second.root_id, lane_before_second.root_id); + assert_eq!(lane_after_second.generation, lane_before_second.generation); + } + + #[test] + fn rebuild_preserves_clean_git_mapping_for_repaired_branch() { + let workspace = tempfile::tempdir().unwrap(); + let git = |args: &[&str]| { + let output = Command::new("git") + .arg("-C") + .arg(workspace.path()) + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() + }; + git(&["init"]); + git(&["config", "user.email", "trail@example.test"]); + git(&["config", "user.name", "Trail Test"]); + fs::write(workspace.path().join("README.md"), "hello\n").unwrap(); + git(&["add", "README.md"]); + git(&["commit", "-m", "initial"]); + let git_head = git(&["rev-parse", "HEAD"]); + Trail::init(workspace.path(), "main", InitImportMode::GitTracked, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let modern = db.resolve_branch_ref("main").unwrap(); + let (legacy_root_id, _) = publish_legacy_root(&db, &modern); + let historical_change = modern.change_id.clone(); + db.insert_git_mapping_for_state( + "import", + "main", + &historical_change, + &legacy_root_id, + Some(git_head.clone()), + false, + ) + .unwrap(); + // Historical operations can share a root, and repeated verification + // can produce duplicate clean mapping facts for that visible state. + db.insert_git_mapping_for_state( + "import", + "main", + &ChangeId("change_historical_duplicate".to_string()), + &legacy_root_id, + Some(git_head.clone()), + false, + ) + .unwrap(); + db.insert_git_mapping_for_state( + "export", + "release", + &ChangeId("change_historical_release".to_string()), + &legacy_root_id, + Some(git_head.clone()), + false, + ) + .unwrap(); + db.spawn_lane("mapped-lane", Some("main"), false, None, None) + .unwrap(); + + let report = db.rebuild_indexes().unwrap(); + let repaired = db.resolve_branch_ref("main").unwrap(); + assert_eq!(report.path_index_repaired_refs.len(), 2); + assert!(db + .git_clean_head_matches_root_mapping(&git_head, &repaired.root_id) + .unwrap()); + let mapping = db + .git_mappings(20) + .unwrap() + .into_iter() + .find(|mapping| { + mapping.crab_root == repaired.root_id + && mapping.crab_change == repaired.change_id + && mapping.git_head.as_deref() == Some(git_head.as_str()) + }) + .unwrap(); + assert_eq!(mapping.direction, "import"); + assert_eq!(mapping.branch, "main"); + assert!(!mapping.git_dirty); + let copied = db + .git_mappings(50) + .unwrap() + .into_iter() + .filter(|mapping| { + mapping.crab_root == repaired.root_id && mapping.crab_change == repaired.change_id + }) + .collect::>(); + assert_eq!( + copied + .iter() + .filter(|mapping| mapping.direction == "import" && mapping.branch == "main") + .count(), + 1 + ); + assert_eq!( + copied + .iter() + .filter(|mapping| mapping.direction == "export" && mapping.branch == "release") + .count(), + 1 + ); + + let repaired_lane = db.get_ref(&lane_ref("mapped-lane")).unwrap(); + let applied = db + .apply_lane_patch( + "mapped-lane", + write_patch("agent.txt", "agent\n", &repaired_lane.change_id), + ) + .unwrap(); + db.agent_mark_reviewed("mapped-lane", None).unwrap(); + let range = format!("{}..{}", repaired_lane.change_id.0, applied.operation.0); + db.reset_git_handoff_metrics(); + let exported = db + .git_export_commit_mapped( + &range, + "mapped delta after index repair", + Some(GitState { + head: Some(git_head), + dirty: false, + }), + ) + .unwrap(); + assert_eq!(exported.performance.export_mode, "mapped_delta"); + assert_eq!(exported.performance.full_root_file_count, 0); + } + + #[test] + fn equivalent_repaired_root_reuses_process_local_clean_daemon_snapshot_but_not_dirty_state() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "hello\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut daemon_db = Trail::open(workspace.path()).unwrap(); + let modern = daemon_db.resolve_branch_ref("main").unwrap(); + let (legacy_root, _) = publish_legacy_root(&daemon_db, &modern); + daemon_db.enable_daemon_worktree_cache().unwrap(); + // This regression exercises process-local snapshot reuse, not + // asynchronous OS event delivery. Stop the watcher after warmup so + // delayed backend events cannot turn the fixed baseline fixture dirty + // while the index-repair assertions are running. + drop( + daemon_db + .daemon_worktree_cache + .as_mut() + .unwrap() + .watcher + .take(), + ); + let isolated_reader = Trail::open(workspace.path()).unwrap(); + assert!(isolated_reader.daemon_worktree_snapshot().is_none()); + drop(isolated_reader); + + daemon_db.rebuild_indexes().unwrap(); + let repaired = daemon_db.resolve_branch_ref("main").unwrap(); + assert_ne!(repaired.root_id, legacy_root); + let repaired_snapshot = daemon_db.daemon_worktree_snapshot(); + assert!( + matches!( + repaired_snapshot, + Some(DaemonWorktreeSnapshot::Clean { root_id: Some(ref root), .. }) if root == &legacy_root + ), + "unexpected repaired daemon snapshot: {repaired_snapshot:?}" + ); + daemon_db + .conn + .execute("DELETE FROM worktree_file_index", []) + .unwrap(); + + let clean = daemon_db.status(None).unwrap(); + + assert_eq!(clean.worktree_state, WorktreeState::Clean); + assert_eq!( + daemon_db + .conn + .query_row("SELECT COUNT(*) FROM worktree_file_index", [], |row| { + row.get::<_, i64>(0) + }) + .unwrap(), + 0, + "clean equivalent daemon state must avoid the O(N) worktree refresh" + ); + + fs::write(workspace.path().join("README.md"), "dirty\n").unwrap(); + { + let cache = daemon_db.daemon_worktree_cache.as_ref().unwrap(); + let mut state = cache.state.lock().unwrap(); + state.dirty_paths.insert("README.md".to_string()); + state.generation = state.generation.saturating_add(1); + } + let dirty = daemon_db.status(None).unwrap(); + assert_eq!(dirty.worktree_state, WorktreeState::DirtyTracked); + assert_eq!(dirty.changed_paths.len(), 1); + } + + #[test] + fn empty_and_modern_roots_do_not_publish_path_index_repairs() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::Empty, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let head = db.resolve_branch_ref("main").unwrap(); + + let report = db.rebuild_indexes().unwrap(); + + assert!(report.path_index_repaired_roots.is_empty()); + assert!(report.path_index_repaired_refs.is_empty()); + let after = db.resolve_branch_ref("main").unwrap(); + assert_eq!(after.change_id, head.change_id); + assert_eq!(after.root_id, head.root_id); + assert_eq!(after.generation, head.generation); + } + + #[test] + fn rebuild_repairs_distinct_noncurrent_branch_and_lane_roots() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "hello\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.create_branch("other", Some("main")).unwrap(); + fs::write(workspace.path().join("other.txt"), "other\n").unwrap(); + db.record( + Some("other"), + Some("different root".to_string()), + Actor::human(), + false, + ) + .unwrap(); + let main = db.resolve_branch_ref("main").unwrap(); + let other = db.resolve_branch_ref("other").unwrap(); + assert_ne!(main.root_id, other.root_id); + let (main_legacy, _) = publish_legacy_root(&db, &main); + let (other_legacy, _) = publish_legacy_root(&db, &other); + assert_ne!(main_legacy, other_legacy); + db.spawn_lane("other-lane", Some("other"), false, None, None) + .unwrap(); + + let report = db.rebuild_indexes().unwrap(); + + assert_eq!(report.path_index_repaired_roots.len(), 2); + assert_eq!(report.path_index_repaired_refs.len(), 3); + let roots = report + .path_index_repaired_roots + .iter() + .map(|repair| repair.old_root.clone()) + .collect::>(); + assert_eq!(roots, BTreeSet::from([main_legacy, other_legacy])); + assert_eq!( + db.resolve_branch_ref("other").unwrap().root_id, + db.get_ref(&lane_ref("other-lane")).unwrap().root_id + ); + } + + fn legacy_root_with_path_keys( + db: &Trail, + source_root_id: &ObjectId, + keys: Vec>, + ) -> ObjectId { + let source: WorktreeRoot = db.get_object(WORKTREE_ROOT_KIND, source_root_id).unwrap(); + let source_files = db.load_root_files(source_root_id).unwrap(); + let entry = source_files.values().next().unwrap(); + let file_count = keys.len() as u64; + let mut builder = SortedBatchBuilder::new(db.store.clone(), root_map_prolly_config()); + for key in keys { + builder.add(key, cbor(entry).unwrap()).unwrap(); + } + let path_tree = builder.build().unwrap(); + let legacy = WorktreeRoot { + path_map_root: tree_root_hex(&path_tree), + case_fold_map_root: None, + file_count, + ..source + }; + db.put_object(WORKTREE_ROOT_KIND, ROOT_OBJECT_VERSION, &legacy) + .unwrap() + } + + fn assert_rebuild_preflight_does_not_publish(db: &mut Trail, protected_ref: &RefRecord) { + fn count(db: &Trail, table: &str) -> i64 { + db.conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() + } + let before_ref = db.get_ref(&protected_ref.name).unwrap(); + let objects = count(db, "objects"); + let prolly_nodes = count(db, "prolly_nodes"); + let operations = count(db, "operations"); + let git_mappings = count(db, "git_mappings"); + let lane_branches = count(db, "lane_branches"); + + assert!(db.rebuild_indexes().is_err()); + + let after_ref = db.get_ref(&protected_ref.name).unwrap(); + assert_eq!(after_ref.change_id, before_ref.change_id); + assert_eq!(after_ref.root_id, before_ref.root_id); + assert_eq!(after_ref.generation, before_ref.generation); + assert_eq!(count(db, "objects"), objects); + assert_eq!(count(db, "prolly_nodes"), prolly_nodes); + assert_eq!(count(db, "operations"), operations); + assert_eq!(count(db, "git_mappings"), git_mappings); + assert_eq!(count(db, "lane_branches"), lane_branches); + } + + #[test] + fn corrupt_later_collision_root_does_not_advance_earlier_valid_ref() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "hello\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.create_branch("a-good", Some("main")).unwrap(); + let good = db.resolve_branch_ref("a-good").unwrap(); + publish_legacy_root(&db, &good); + let good_legacy = db.resolve_branch_ref("a-good").unwrap(); + let main = db.resolve_branch_ref("main").unwrap(); + let bad_root = legacy_root_with_path_keys( + &db, + &main.root_id, + vec![b"README.md".to_vec(), b"readme.md".to_vec()], + ); + db.set_ref( + &branch_ref("z-bad"), + &main.change_id, + &bad_root, + &main.operation_id, + ) + .unwrap(); + + assert_rebuild_preflight_does_not_publish(&mut db, &good_legacy); + } + + #[test] + fn malformed_legacy_path_key_does_not_publish_any_ref_metadata() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "hello\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.create_branch("a-good", Some("main")).unwrap(); + let good = db.resolve_branch_ref("a-good").unwrap(); + publish_legacy_root(&db, &good); + let good_legacy = db.resolve_branch_ref("a-good").unwrap(); + let main = db.resolve_branch_ref("main").unwrap(); + let bad_root = legacy_root_with_path_keys(&db, &main.root_id, vec![vec![0xff, 0xfe]]); + db.set_ref( + &branch_ref("z-malformed"), + &main.change_id, + &bad_root, + &main.operation_id, + ) + .unwrap(); + + assert_rebuild_preflight_does_not_publish(&mut db, &good_legacy); + } + + #[test] + fn corrupt_later_lane_manifest_does_not_create_maintenance_operation_or_advance_refs() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "hello\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let modern = db.resolve_branch_ref("main").unwrap(); + publish_legacy_root(&db, &modern); + db.spawn_lane("z-corrupt", Some("main"), true, None, None) + .unwrap(); + + let main_before = db.resolve_branch_ref("main").unwrap(); + let lane_before = db.get_ref(&lane_ref("z-corrupt")).unwrap(); + let lane_row_before = db.lane_branch("z-corrupt").unwrap(); + let manifest_path = Path::new(lane_row_before.workdir.as_ref().unwrap()) + .join(".trail") + .join("workdir-manifest.json"); + assert!(manifest_path.is_file()); + fs::write(&manifest_path, b"{not-json").unwrap(); + let operation_objects_before: i64 = db + .conn + .query_row( + "SELECT COUNT(*) FROM objects WHERE kind = ?1", + params![OPERATION_KIND], + |row| row.get(0), + ) + .unwrap(); + + let err = db.rebuild_indexes().unwrap_err(); + + assert!(matches!(err, Error::Corrupt(message) if message.contains("cannot be retargeted"))); + let main_after = db.resolve_branch_ref("main").unwrap(); + let lane_after = db.get_ref(&lane_ref("z-corrupt")).unwrap(); + let lane_row_after = db.lane_branch("z-corrupt").unwrap(); + assert_eq!(main_after.change_id, main_before.change_id); + assert_eq!(main_after.root_id, main_before.root_id); + assert_eq!(main_after.generation, main_before.generation); + assert_eq!(lane_after.change_id, lane_before.change_id); + assert_eq!(lane_after.root_id, lane_before.root_id); + assert_eq!(lane_after.generation, lane_before.generation); + assert_eq!(lane_row_after.head_change, lane_row_before.head_change); + assert_eq!(lane_row_after.head_root, lane_row_before.head_root); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM objects WHERE kind = ?1", + params![OPERATION_KIND], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + operation_objects_before + ); + } + + #[test] + fn path_index_rebuild_publication_crash_helper() { + let Some(workspace) = std::env::var_os("TRAIL_TEST_PATH_INDEX_CRASH_WORKSPACE") else { + return; + }; + let mut db = Trail::open(PathBuf::from(workspace)).unwrap(); + let _ = db.rebuild_indexes(); + panic!("path-index rebuild crash helper passed its requested crash point"); + } + + fn wait_for_path_index_crash_handshake( + child: &mut std::process::Child, + ready: &Path, + phase: &str, + ) { + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + if ready.is_file() { + return; + } + if let Some(status) = child.try_wait().unwrap() { + panic!("path-index crash helper exited at {phase} before handshake: {status}"); + } + std::thread::sleep(Duration::from_millis(25)); + } + panic!("timed out waiting for path-index crash handshake at {phase}"); + } + + #[test] + fn publication_commit_survives_crash_with_operation_graph_and_lane_mirror_recovery() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "hello\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let modern = db.resolve_branch_ref("main").unwrap(); + publish_legacy_root(&db, &modern); + let spawned = db + .spawn_lane("crash-lane", Some("main"), true, None, None) + .unwrap(); + let lane_before = db.get_ref(&lane_ref("crash-lane")).unwrap(); + let lane_row = db.lane_branch("crash-lane").unwrap(); + let workdir = PathBuf::from(spawned.workdir.unwrap()); + let manifest_path = workdir.join(".trail/workdir-manifest.json"); + let lane = db.lane_record(&lane_row.lane_id).unwrap(); + let view = db + .create_workspace_view( + &lane.lane_id, + &lane_before.change_id, + &lane_before.root_id, + "test-cow", + &workdir, + ) + .unwrap(); + let marker_path = Path::new(&view.meta_dir).join("clean-checkpoint.json"); + write_file_atomic( + &marker_path, + &serde_json::to_vec_pretty(&serde_json::json!({ + "view_id": view.view_id, + "operation": lane_before.change_id.0, + "root_id": lane_before.root_id.0, + "journal_sequence": view.checkpoint_seq, + })) + .unwrap(), + false, + ) + .unwrap(); + drop(db); + + let phase = "path_index_rebuild_after_publication_commit"; + let ready = workspace.path().join(".trail/tmp/path-index-commit.ready"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "db::storage::lifecycle::rebuild::path_index_rebuild_tests::path_index_rebuild_publication_crash_helper", + "--nocapture", + ]) + .env("RUST_TEST_THREADS", "1") + .env("TRAIL_TEST_CRASH_AT", phase) + .env("TRAIL_TEST_CRASH_READY", &ready) + .env("TRAIL_TEST_PATH_INDEX_CRASH_WORKSPACE", workspace.path()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + wait_for_path_index_crash_handshake(&mut child, &ready, phase); + child.kill().unwrap(); + let _ = child.wait().unwrap(); + + let reopened = Trail::open(workspace.path()).unwrap(); + let lane_after = reopened.get_ref(&lane_ref("crash-lane")).unwrap(); + assert_ne!(lane_after.change_id, lane_before.change_id); + let operation = reopened.operation(&lane_after.change_id).unwrap(); + assert_eq!(operation.parents, vec![lane_before.change_id.clone()]); + assert_eq!( + reopened.parents(&lane_after.change_id).unwrap(), + vec![lane_before.change_id] + ); + let manifest: serde_json::Value = + serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap(); + assert_eq!(manifest["root_id"], lane_after.root_id.0); + let marker: serde_json::Value = + serde_json::from_slice(&fs::read(&marker_path).unwrap()).unwrap(); + assert_eq!(marker["root_id"], lane_after.root_id.0); + assert_eq!(marker["operation"], lane_after.change_id.0); + let pending: i64 = reopened + .conn + .query_row( + "SELECT COUNT(*) FROM pending_path_index_derived_repairs", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(pending, 0); + } + + #[test] + fn failed_pending_manifest_repair_stays_durable_and_retries_after_invalidation() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "hello\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let spawned = db + .spawn_lane("pending-lane", Some("main"), true, None, None) + .unwrap(); + let head = db.get_ref(&lane_ref("pending-lane")).unwrap(); + let manifest = PathBuf::from(spawned.workdir.unwrap()) + .join(".trail") + .join("workdir-manifest.json"); + fs::remove_file(&manifest).unwrap(); + fs::create_dir(&manifest).unwrap(); + db.conn + .execute( + "INSERT INTO pending_path_index_derived_repairs \ + (ref_name, repair_kind, old_root, new_root, new_change, created_at) \ + VALUES (?1, 'lane_manifest', ?2, ?2, ?3, ?4)", + params![head.name, head.root_id.0, head.change_id.0, now_ts()], + ) + .unwrap(); + + db.rebuild_indexes().unwrap(); + let pending = |db: &Trail| { + db.conn + .query_row( + "SELECT COUNT(*) FROM pending_path_index_derived_repairs", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap() + }; + assert_eq!(pending(&db), 1); + + fs::remove_dir(&manifest).unwrap(); + db.rebuild_indexes().unwrap(); + assert_eq!(pending(&db), 0); + } + + #[test] + fn schema_preflight_waits_for_writer_exclusion_even_with_empty_pending_queue() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::Empty, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let lock = db.acquire_write_lock().unwrap(); + let releaser = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(50)); + drop(lock); + }); + + Trail::open(workspace.path()).unwrap(); + releaser.join().unwrap(); + } + + #[test] + fn pending_repair_for_removed_lane_scope_clears_without_using_stale_paths() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::Empty, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane("removed-lane", Some("main"), false, None, None) + .unwrap(); + let head = db.get_ref(&lane_ref("removed-lane")).unwrap(); + db.conn + .execute( + "INSERT INTO pending_path_index_derived_repairs \ + (ref_name, repair_kind, old_root, new_root, new_change, created_at) \ + VALUES (?1, 'workspace_checkpoint', ?2, ?2, ?3, ?4)", + params![head.name, head.root_id.0, head.change_id.0, now_ts()], + ) + .unwrap(); + db.conn + .execute("DELETE FROM refs WHERE name = ?1", params![head.name]) + .unwrap(); + db.conn + .execute( + "DELETE FROM lane_branches WHERE ref_name = ?1", + params![head.name], + ) + .unwrap(); + + db.rebuild_indexes().unwrap(); + + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM pending_path_index_derived_repairs", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + } + + #[test] + fn rebuild_retains_paths_for_at_most_one_distinct_legacy_root() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "hello\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let modern = db.resolve_branch_ref("main").unwrap(); + let one = legacy_root_with_path_keys(&db, &modern.root_id, vec![b"one.txt".to_vec()]); + let two = legacy_root_with_path_keys( + &db, + &modern.root_id, + vec![b"one.txt".to_vec(), b"two.txt".to_vec()], + ); + let three = legacy_root_with_path_keys( + &db, + &modern.root_id, + vec![ + b"one.txt".to_vec(), + b"three.txt".to_vec(), + b"two.txt".to_vec(), + ], + ); + for (name, root) in [("main", one), ("two", two), ("three", three)] { + db.set_ref( + &branch_ref(name), + &modern.change_id, + &root, + &modern.operation_id, + ) + .unwrap(); + } + + reset_path_index_rebuild_memory_metrics(); + let report = db.rebuild_indexes().unwrap(); + let metrics = path_index_rebuild_memory_metrics(); + + assert_eq!(report.path_index_repaired_roots.len(), 3); + assert!(metrics.observed_paths >= 6); + assert_eq!(metrics.live_paths, 0); + assert!( + metrics.peak_live_paths <= 3, + "peak retained paths {} exceeded largest root", + metrics.peak_live_paths + ); + } + + #[test] + fn rebuild_reconciles_stale_ref_file_even_without_root_repairs() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::Empty, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let head = db.resolve_branch_ref("main").unwrap(); + let ref_path = db.db_dir.join(&head.name); + fs::write(&ref_path, br#"{"root_id":"stale"}"#).unwrap(); + + let report = db.rebuild_indexes().unwrap(); + + assert!(report.path_index_repaired_refs.is_empty()); + let mirrored: serde_json::Value = + serde_json::from_slice(&fs::read(ref_path).unwrap()).unwrap(); + assert_eq!(mirrored["root_id"], head.root_id.0); + assert_eq!(mirrored["change_id"], head.change_id.0); + assert_eq!(mirrored["generation"], head.generation); + } +} diff --git a/trail/src/db/storage/manifest.rs b/trail/src/db/storage/manifest.rs index 63fb464..a2a6ed4 100644 --- a/trail/src/db/storage/manifest.rs +++ b/trail/src/db/storage/manifest.rs @@ -6,6 +6,14 @@ impl Trail { &self, root_id: &ObjectId, ) -> Result> { + self.note_full_root_path_load(); + let mut root_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta { + full_root_range_count: 1, + ..OperationMetricsDelta::default() + }, + ); let root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, root_id)?; let tree = root_map_tree_from_root_hex(root.path_map_root.as_deref())?; let mut root_iter = self.root_prolly.range(&tree, &[], None)?; @@ -14,7 +22,7 @@ impl Trail { FROM worktree_file_index ORDER BY path ASC", )?; let mut index_rows = stmt.query([])?; - let mut left = next_root_file(&mut root_iter)?; + let mut left = next_root_file(&mut root_iter, &mut root_metrics)?; let mut right = next_index_file(&mut index_rows)?; let mut summaries = Vec::new(); @@ -24,7 +32,7 @@ impl Trail { match left_path.cmp(right_path) { std::cmp::Ordering::Less => { summaries.push(deleted_manifest_summary(left_path.clone(), left_entry)); - left = next_root_file(&mut root_iter)?; + left = next_root_file(&mut root_iter, &mut root_metrics)?; } std::cmp::Ordering::Greater => { summaries.push(added_manifest_summary(right_path.clone(), right_entry)); @@ -41,14 +49,14 @@ impl Trail { right_entry, )); } - left = next_root_file(&mut root_iter)?; + left = next_root_file(&mut root_iter, &mut root_metrics)?; right = next_index_file(&mut index_rows)?; } } } (Some((left_path, left_entry)), None) => { summaries.push(deleted_manifest_summary(left_path.clone(), left_entry)); - left = next_root_file(&mut root_iter)?; + left = next_root_file(&mut root_iter, &mut root_metrics)?; } (None, Some((right_path, right_entry))) => { summaries.push(added_manifest_summary(right_path.clone(), right_entry)); @@ -65,11 +73,19 @@ impl Trail { root_id: &ObjectId, manifest: &BTreeMap, ) -> Result> { + self.note_full_root_path_load(); + let mut root_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta { + full_root_range_count: 1, + ..OperationMetricsDelta::default() + }, + ); let root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, root_id)?; let tree = root_map_tree_from_root_hex(root.path_map_root.as_deref())?; let mut root_iter = self.root_prolly.range(&tree, &[], None)?; let mut manifest_iter = manifest.iter(); - let mut left = next_root_file(&mut root_iter)?; + let mut left = next_root_file(&mut root_iter, &mut root_metrics)?; let mut right = manifest_iter.next(); let mut summaries = Vec::new(); @@ -79,7 +95,7 @@ impl Trail { match left_path.cmp(right_path) { std::cmp::Ordering::Less => { summaries.push(deleted_manifest_summary(left_path.clone(), left_entry)); - left = next_root_file(&mut root_iter)?; + left = next_root_file(&mut root_iter, &mut root_metrics)?; } std::cmp::Ordering::Greater => { summaries.push(added_manifest_summary(right_path.clone(), right_entry)); @@ -96,14 +112,14 @@ impl Trail { right_entry, )); } - left = next_root_file(&mut root_iter)?; + left = next_root_file(&mut root_iter, &mut root_metrics)?; right = manifest_iter.next(); } } } (Some((left_path, left_entry)), None) => { summaries.push(deleted_manifest_summary(left_path.clone(), left_entry)); - left = next_root_file(&mut root_iter)?; + left = next_root_file(&mut root_iter, &mut root_metrics)?; } (None, Some((right_path, right_entry))) => { summaries.push(added_manifest_summary(right_path.clone(), right_entry)); @@ -116,6 +132,13 @@ impl Trail { } pub(crate) fn disk_manifest(&self, disk_files: &[DiskFile]) -> BTreeMap { + self.note_operation_metrics(OperationMetricsDelta { + filesystem_hash_count: saturating_u64_from_usize(disk_files.len()), + filesystem_hash_bytes: disk_files.iter().fold(0u64, |total, file| { + total.saturating_add(saturating_u64_from_usize(file.bytes.len())) + }), + ..OperationMetricsDelta::default() + }); disk_files .iter() .map(|file| { @@ -139,6 +162,10 @@ impl Trail { let mut paths = BTreeSet::new(); paths.extend(left.keys().cloned()); paths.extend(right.keys().cloned()); + self.note_operation_metrics(OperationMetricsDelta { + manifest_key_comparison_count: saturating_u64_from_usize(paths.len()), + ..OperationMetricsDelta::default() + }); let mut summaries = Vec::new(); for path in paths { match (left.get(&path), right.get(&path)) { @@ -193,21 +220,21 @@ impl Trail { summaries } - pub(crate) fn selected_worktree_snapshot( + pub(crate) fn selected_worktree_snapshot_with_policy( &self, head_files: &BTreeMap, candidate_paths: &[String], + policy: &WorkspaceIgnorePolicySnapshot, ) -> Result { - let disk_files = self.scan_visible_files_for_paths(candidate_paths)?; + let disk_files = self.scan_visible_files_for_paths_with_policy(candidate_paths, policy)?; let disk_paths = disk_files .iter() .map(|file| file.path.clone()) - .collect::>(); - self.prune_worktree_index_for_selections(candidate_paths, &disk_paths)?; + .collect::>(); let disk_manifest = self.disk_manifest(&disk_files); - self.update_worktree_index_from_disk_files_and_manifest(&disk_files, &disk_manifest)?; + self.sync_selected_worktree_index(candidate_paths, &disk_paths, &disk_manifest)?; let summaries = - self.diff_file_maps_to_manifest_for_paths(head_files, &disk_manifest, candidate_paths); + self.diff_file_maps_to_manifest_for_paths(head_files, &disk_manifest, candidate_paths)?; let paths = summaries .iter() .map(|summary| summary.path.clone()) @@ -224,15 +251,16 @@ impl Trail { }) } - pub(crate) fn selected_worktree_snapshot_read_only( + pub(crate) fn selected_worktree_snapshot_read_only_with_policy( &self, head_files: &BTreeMap, candidate_paths: &[String], + policy: &WorkspaceIgnorePolicySnapshot, ) -> Result { - let disk_files = self.scan_visible_files_for_paths(candidate_paths)?; + let disk_files = self.scan_visible_files_for_paths_with_policy(candidate_paths, policy)?; let disk_manifest = self.disk_manifest(&disk_files); let summaries = - self.diff_file_maps_to_manifest_for_paths(head_files, &disk_manifest, candidate_paths); + self.diff_file_maps_to_manifest_for_paths(head_files, &disk_manifest, candidate_paths)?; let paths = summaries .iter() .map(|summary| summary.path.clone()) @@ -249,22 +277,34 @@ impl Trail { }) } + #[cfg(test)] pub(crate) fn selected_worktree_snapshot_for_root( &self, root_id: &ObjectId, candidate_paths: &[String], + ) -> Result { + let policy = self.workspace_ignore_policy_snapshot(); + self.selected_worktree_snapshot_for_root_with_policy(root_id, candidate_paths, &policy) + } + + pub(crate) fn selected_worktree_snapshot_for_root_with_policy( + &self, + root_id: &ObjectId, + candidate_paths: &[String], + policy: &WorkspaceIgnorePolicySnapshot, ) -> Result { let head_files = self.load_root_files_for_selections(root_id, candidate_paths)?; - self.selected_worktree_snapshot(&head_files, candidate_paths) + self.selected_worktree_snapshot_with_policy(&head_files, candidate_paths, policy) } - pub(crate) fn selected_worktree_snapshot_for_root_read_only( + pub(crate) fn selected_worktree_snapshot_for_root_read_only_with_policy( &self, root_id: &ObjectId, candidate_paths: &[String], + policy: &WorkspaceIgnorePolicySnapshot, ) -> Result { let head_files = self.load_root_files_for_selections(root_id, candidate_paths)?; - self.selected_worktree_snapshot_read_only(&head_files, candidate_paths) + self.selected_worktree_snapshot_read_only_with_policy(&head_files, candidate_paths, policy) } pub(crate) fn diff_file_maps_to_manifest_for_paths( @@ -272,22 +312,31 @@ impl Trail { left: &BTreeMap, right: &BTreeMap, selected_paths: &[String], - ) -> Vec { + ) -> Result> { + let selections = SelectionSet::from_paths(selected_paths)?; + if selections.as_slice().is_empty() { + return Ok(Vec::new()); + } let mut paths = BTreeSet::new(); - for selected in selected_paths { - paths.insert(selected.clone()); - paths.extend( - left.keys() - .filter(|path| path_matches_selection(path, selected)) - .cloned(), - ); - paths.extend( - right - .keys() - .filter(|path| path_matches_selection(path, selected)) - .cloned(), - ); + paths.extend(left.keys().cloned()); + paths.extend(right.keys().cloned()); + let mut selection_comparison_count = 0u64; + paths.retain(|path| { + let (selected, comparisons) = selections.contains_counted(path); + selection_comparison_count = selection_comparison_count.saturating_add(comparisons); + selected + }); + self.note_operation_metrics(OperationMetricsDelta { + selection_comparison_count, + ..OperationMetricsDelta::default() + }); + if paths.is_empty() { + return Ok(Vec::new()); } + self.note_operation_metrics(OperationMetricsDelta { + manifest_key_comparison_count: saturating_u64_from_usize(paths.len()), + ..OperationMetricsDelta::default() + }); let mut summaries = Vec::new(); for path in paths { match (left.get(&path), right.get(&path)) { @@ -339,7 +388,7 @@ impl Trail { (None, None) => {} } } - summaries + Ok(summaries) } } @@ -357,7 +406,10 @@ fn added_manifest_summary(path: String, entry: &DiskManifest) -> FileDiffSummary } } -fn next_root_file(iter: &mut prolly::RangeIter<'_, S>) -> Result> +fn next_root_file( + iter: &mut prolly::RangeIter<'_, S>, + metrics: &mut OperationMetricsAccumulator, +) -> Result> where S: prolly::Store, { @@ -365,6 +417,7 @@ where return Ok(None); }; let (key, value) = item?; + metrics.delta.root_range_row_count = metrics.delta.root_range_row_count.saturating_add(1); let path = String::from_utf8(key) .map_err(|err| Error::Corrupt(format!("non UTF-8 path key: {err}")))?; Ok(Some((path, from_cbor::(&value)?))) @@ -423,3 +476,133 @@ fn changed_manifest_summary( patch: None, } } + +#[cfg(test)] +mod tests { + use super::*; + + fn manifest_fixture() -> (tempfile::TempDir, Trail, FileEntry) { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "hello\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(workspace.path()).unwrap(); + let head = db.resolve_branch_ref("main").unwrap(); + let entry = db + .load_root_files_for_paths(&head.root_id, &["README.md".to_string()]) + .unwrap() + .remove("README.md") + .unwrap(); + (workspace, db, entry) + } + + fn disk_manifest(entry: &FileEntry) -> DiskManifest { + DiskManifest { + kind: entry.kind.clone(), + executable: entry.executable, + content_hash: entry.content_hash.clone(), + } + } + + #[test] + fn selected_manifest_key_union_preserves_both_exact_rename_endpoints() { + let (_workspace, db, entry) = manifest_fixture(); + let left = BTreeMap::from([("old.txt".to_string(), entry.clone())]); + let right = BTreeMap::from([("new.txt".to_string(), disk_manifest(&entry))]); + + let summaries = db + .diff_file_maps_to_manifest_for_paths( + &left, + &right, + &["old.txt".to_string(), "new.txt".to_string()], + ) + .unwrap(); + + assert_eq!( + summaries + .iter() + .map(|summary| (summary.path.as_str(), summary.kind.clone())) + .collect::>(), + [ + ("new.txt", FileChangeKind::Added), + ("old.txt", FileChangeKind::Deleted), + ] + ); + } + + #[test] + fn selected_manifest_key_union_filters_once_with_overlap_and_case_aliases() { + let (_workspace, db, entry) = manifest_fixture(); + let mut changed_manifest = disk_manifest(&entry); + changed_manifest.content_hash = "different".to_string(); + let left = BTreeMap::from([ + ("README.md".to_string(), entry.clone()), + ("docs/old.txt".to_string(), entry.clone()), + ("unrelated/left.txt".to_string(), entry.clone()), + ]); + let right = BTreeMap::from([ + ("docs/new.txt".to_string(), disk_manifest(&entry)), + ("readme.md".to_string(), disk_manifest(&entry)), + ("unrelated/right.txt".to_string(), changed_manifest), + ]); + let metrics = db.operation_metrics.as_ref().unwrap(); + let before = metrics.snapshot(); + + let summaries = db + .diff_file_maps_to_manifest_for_paths( + &left, + &right, + &[ + "docs/new.txt".to_string(), + "docs".to_string(), + "README.md".to_string(), + "readme.md".to_string(), + ], + ) + .unwrap(); + let after = metrics.snapshot(); + + assert_eq!( + summaries + .iter() + .map(|summary| summary.path.as_str()) + .collect::>(), + ["README.md", "docs/new.txt", "docs/old.txt", "readme.md"] + ); + assert!( + after.selection_comparison_count - before.selection_comparison_count <= 12, + "each of six union keys should need at most exact plus interval membership" + ); + assert_eq!( + after.manifest_key_comparison_count - before.manifest_key_comparison_count, + 4 + ); + } + + #[test] + fn selected_manifest_empty_selection_does_not_scan_maps_or_report_membership_work() { + let (_workspace, db, entry) = manifest_fixture(); + let left = (0..1_000) + .map(|idx| (format!("left/{idx:04}.txt"), entry.clone())) + .collect::>(); + let right = (0..1_000) + .map(|idx| (format!("right/{idx:04}.txt"), disk_manifest(&entry))) + .collect::>(); + let metrics = db.operation_metrics.as_ref().unwrap(); + let before = metrics.snapshot(); + + let summaries = db + .diff_file_maps_to_manifest_for_paths(&left, &right, &[]) + .unwrap(); + let after = metrics.snapshot(); + + assert!(summaries.is_empty()); + assert_eq!( + after.selection_comparison_count - before.selection_comparison_count, + 0 + ); + assert_eq!( + after.manifest_key_comparison_count - before.manifest_key_comparison_count, + 0 + ); + } +} diff --git a/trail/src/db/storage/mod.rs b/trail/src/db/storage/mod.rs index b8d159b..4ce0bd8 100644 --- a/trail/src/db/storage/mod.rs +++ b/trail/src/db/storage/mod.rs @@ -25,3 +25,14 @@ mod schema; mod validation; mod worktree_index; mod worktree_scan; + +#[cfg(debug_assertions)] +pub(crate) use git::{ + install_git_qualification_after_c2_hook, install_git_qualification_after_porcelain_hook, +}; +pub(crate) use schema::{validate_no_prolly_sqlite_schema_v18, validate_prolly_sqlite_schema_v18}; +pub(crate) use worktree_index::{ + file_kind_from_index, PinnedWorktreeRoot, ReconciliationDirectory, ReconciliationFile, + ReconciliationScanEntry, +}; +pub(crate) use worktree_scan::{observed_exact_paths_for_candidates, ObservedPathKind}; diff --git a/trail/src/db/storage/objects/messages.rs b/trail/src/db/storage/objects/messages.rs index c00d31c..b17c834 100644 --- a/trail/src/db/storage/objects/messages.rs +++ b/trail/src/db/storage/objects/messages.rs @@ -20,7 +20,7 @@ impl Trail { now_nanos() ); ChangeId(format!( - "msg_seed_{}", + "change_message_seed_{}", crate::ids::short_hash(seed.as_bytes(), 16) )) }); diff --git a/trail/src/db/storage/objects/operations.rs b/trail/src/db/storage/objects/operations.rs index eac8185..36975b9 100644 --- a/trail/src/db/storage/objects/operations.rs +++ b/trail/src/db/storage/objects/operations.rs @@ -2,12 +2,20 @@ use super::*; impl Trail { pub(crate) fn store_operation(&self, operation: &Operation) -> Result { - let operation = redact_operation_for_storage(operation); - let operation_id = self.put_object(OPERATION_KIND, OP_OBJECT_VERSION, &operation)?; + let (operation, operation_id) = self.store_operation_object_unindexed(operation)?; self.index_operation(&operation, &operation_id)?; Ok(operation_id) } + pub(crate) fn store_operation_object_unindexed( + &self, + operation: &Operation, + ) -> Result<(Operation, ObjectId)> { + let operation = redact_operation_for_storage(operation); + let operation_id = self.put_object(OPERATION_KIND, OP_OBJECT_VERSION, &operation)?; + Ok((operation, operation_id)) + } + pub(crate) fn index_operation( &self, operation: &Operation, @@ -23,7 +31,7 @@ impl Trail { result } - fn index_operation_in_transaction( + pub(crate) fn index_operation_in_transaction( &self, operation: &Operation, operation_id: &ObjectId, @@ -134,7 +142,7 @@ mod tests { let db = Trail::open(temp.path()).unwrap(); let operation = Operation { version: OP_OBJECT_VERSION, - change_id: ChangeId("ch_secret_message".to_string()), + change_id: ChangeId("change_secret_message".to_string()), kind: OperationKind::ManualCheckpoint, parents: Vec::new(), before_root: None, @@ -155,7 +163,7 @@ mod tests { .conn .query_row( "SELECT message FROM operations WHERE change_id = ?1", - params!["ch_secret_message"], + params!["change_secret_message"], |row| row.get(0), ) .unwrap(); diff --git a/trail/src/db/storage/record_selection.rs b/trail/src/db/storage/record_selection.rs index 6e653c9..92dd2fc 100644 --- a/trail/src/db/storage/record_selection.rs +++ b/trail/src/db/storage/record_selection.rs @@ -1,22 +1,70 @@ use super::*; impl Trail { + #[cfg(test)] pub(crate) fn scan_record_selection_files( &self, selected_paths: &[String], + selections: &SelectionSet, allow_ignored: bool, ) -> Result> { - if !allow_ignored - && selected_paths - .iter() - .any(|path| self.workspace_root.join(path_from_rel(path)).is_dir()) - { - let disk_files = self.scan_visible_files_for_paths(selected_paths)?; - return self.selected_record_disk_files(&disk_files, selected_paths, false); + let policy = self.workspace_ignore_policy_snapshot(); + self.scan_record_selection_files_with_policy( + selected_paths, + selections, + allow_ignored, + &policy, + ) + } + + pub(crate) fn scan_record_selection_files_with_policy( + &self, + selected_paths: &[String], + selections: &SelectionSet, + allow_ignored: bool, + policy: &WorkspaceIgnorePolicySnapshot, + ) -> Result> { + if allow_ignored { + if let Some(path) = selected_paths.iter().find(|path| is_internal_path(path)) { + return Err(Error::IgnoredPath(path.clone())); + } + } + let mut filesystem_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta::default(), + ); + let mut selected_directory = false; + if !allow_ignored { + for path in selections.as_slice() { + let abs = safe_join(&self.workspace_root, path)?; + filesystem_metrics.delta.filesystem_stat_count = filesystem_metrics + .delta + .filesystem_stat_count + .saturating_add(1); + match fs::symlink_metadata(abs) { + Ok(metadata) if metadata.is_dir() => { + selected_directory = true; + break; + } + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(Error::Io(err)), + } + } + } + if selected_directory { + let disk_files = + self.scan_visible_files_for_paths_with_policy(selections.as_slice(), policy)?; + return self.selected_record_disk_files_with_selection_set( + &disk_files, + selected_paths, + selections, + false, + ); } let mut selected = BTreeMap::new(); - for path in selected_paths { + for path in selections.as_slice() { if allow_ignored { for file in self.read_record_selection_unfiltered(path)? { selected.insert(file.path.clone(), file); @@ -24,7 +72,11 @@ impl Trail { continue; } - let abs = self.workspace_root.join(path_from_rel(path)); + let abs = safe_join(&self.workspace_root, path)?; + filesystem_metrics.delta.filesystem_stat_count = filesystem_metrics + .delta + .filesystem_stat_count + .saturating_add(1); let metadata = match fs::symlink_metadata(&abs) { Ok(metadata) => metadata, Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, @@ -32,16 +84,25 @@ impl Trail { }; if is_internal_path(path) || metadata.file_type().is_symlink() - || self.ignore_check(path)?.ignored + || policy.check_with_is_dir(path, metadata.is_dir())?.ignored { return Err(Error::IgnoredPath(path.clone())); } if metadata.is_file() { + filesystem_metrics.delta.filesystem_read_count = filesystem_metrics + .delta + .filesystem_read_count + .saturating_add(1); + let bytes = fs::read(&abs)?; + filesystem_metrics.delta.filesystem_read_bytes = filesystem_metrics + .delta + .filesystem_read_bytes + .saturating_add(saturating_u64_from_usize(bytes.len())); selected.insert( path.clone(), DiskFile { path: path.clone(), - bytes: fs::read(&abs)?, + bytes, executable: executable_from_metadata(&metadata), }, ); @@ -50,32 +111,52 @@ impl Trail { Ok(selected.into_values().collect()) } - pub(crate) fn selected_record_disk_files( + pub(crate) fn selected_record_disk_files_with_selection_set( &self, disk_files: &[DiskFile], - selected_paths: &[String], + explicit_paths: &[String], + selections: &SelectionSet, allow_ignored: bool, ) -> Result> { + if allow_ignored { + if let Some(path) = explicit_paths.iter().find(|path| is_internal_path(path)) { + return Err(Error::IgnoredPath(path.clone())); + } + } + let mut filesystem_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta::default(), + ); let mut selected = BTreeMap::new(); + let mut selection_comparison_count = 0u64; for file in disk_files { - if selected_paths - .iter() - .any(|path| path_matches_selection(&file.path, path)) - { + let (matches, comparisons) = selections.contains_counted(&file.path); + selection_comparison_count = selection_comparison_count.saturating_add(comparisons); + if matches { selected.insert(file.path.clone(), file.clone()); } } + self.note_operation_metrics(OperationMetricsDelta { + selection_comparison_count, + ..OperationMetricsDelta::default() + }); - for path in selected_paths { - let had_visible_match = selected - .keys() - .any(|candidate| path_matches_selection(candidate, path)); - if allow_ignored { + if allow_ignored { + for path in selections.as_slice() { for file in self.read_record_selection_unfiltered(path)? { selected.insert(file.path.clone(), file); } - } else if !had_visible_match { - let abs = self.workspace_root.join(path_from_rel(path)); + } + } else { + for path in explicit_paths { + if selected_map_has_selection_match(&selected, path) { + continue; + } + let abs = safe_join(&self.workspace_root, path)?; + filesystem_metrics.delta.filesystem_stat_count = filesystem_metrics + .delta + .filesystem_stat_count + .saturating_add(1); if abs.exists() { return Err(Error::IgnoredPath(path.clone())); } @@ -86,57 +167,97 @@ impl Trail { } pub(crate) fn read_record_selection_unfiltered(&self, path: &str) -> Result> { + let mut filesystem_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta::default(), + ); if is_internal_path(path) { return Err(Error::IgnoredPath(path.to_string())); } - let abs = self.workspace_root.join(path_from_rel(path)); - if !abs.exists() { - return Ok(Vec::new()); - } - let metadata = fs::symlink_metadata(&abs)?; + let abs = safe_join(&self.workspace_root, path)?; + filesystem_metrics.delta.filesystem_stat_count = filesystem_metrics + .delta + .filesystem_stat_count + .saturating_add(1); + let metadata = match fs::symlink_metadata(&abs) { + Ok(metadata) => metadata, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(err) => return Err(Error::Io(err)), + }; if metadata.file_type().is_symlink() { return Ok(Vec::new()); } if metadata.is_file() { + filesystem_metrics.delta.filesystem_read_count = filesystem_metrics + .delta + .filesystem_read_count + .saturating_add(1); + let bytes = fs::read(&abs)?; + filesystem_metrics.delta.filesystem_read_bytes = filesystem_metrics + .delta + .filesystem_read_bytes + .saturating_add(saturating_u64_from_usize(bytes.len())); return Ok(vec![DiskFile { path: path.to_string(), - bytes: fs::read(&abs)?, + bytes, executable: executable_from_metadata(&metadata), }]); } if !metadata.is_dir() { return Ok(Vec::new()); } + filesystem_metrics.delta.bounded_filesystem_walk_count = filesystem_metrics + .delta + .bounded_filesystem_walk_count + .saturating_add(1); let mut files = Vec::new(); - self.read_record_dir_unfiltered(&abs, path, &mut files)?; + self.read_record_dir_unfiltered_profiled(&abs, path, &mut files, &mut filesystem_metrics)?; files.sort_by(|left, right| left.path.cmp(&right.path)); Ok(files) } - pub(crate) fn read_record_dir_unfiltered( + fn read_record_dir_unfiltered_profiled( &self, dir: &Path, rel_dir: &str, files: &mut Vec, + filesystem_metrics: &mut OperationMetricsAccumulator, ) -> Result<()> { for entry in fs::read_dir(dir)? { let entry = entry?; + filesystem_metrics.delta.filesystem_entry_count = filesystem_metrics + .delta + .filesystem_entry_count + .saturating_add(1); let name = entry.file_name().to_string_lossy().to_string(); let rel = format!("{rel_dir}/{name}"); if is_internal_path(&rel) { continue; } let path = entry.path(); + filesystem_metrics.delta.filesystem_stat_count = filesystem_metrics + .delta + .filesystem_stat_count + .saturating_add(1); let metadata = fs::symlink_metadata(&path)?; if metadata.file_type().is_symlink() { continue; } if metadata.is_dir() { - self.read_record_dir_unfiltered(&path, &rel, files)?; + self.read_record_dir_unfiltered_profiled(&path, &rel, files, filesystem_metrics)?; } else if metadata.is_file() { + filesystem_metrics.delta.filesystem_read_count = filesystem_metrics + .delta + .filesystem_read_count + .saturating_add(1); + let bytes = fs::read(&path)?; + filesystem_metrics.delta.filesystem_read_bytes = filesystem_metrics + .delta + .filesystem_read_bytes + .saturating_add(saturating_u64_from_usize(bytes.len())); files.push(DiskFile { path: rel, - bytes: fs::read(&path)?, + bytes, executable: executable_from_metadata(&metadata), }); } @@ -144,3 +265,135 @@ impl Trail { Ok(()) } } + +fn selected_map_has_selection_match(selected: &BTreeMap, path: &str) -> bool { + if selected.contains_key(path) { + return true; + } + let lower = format!("{path}/"); + let upper = format!("{path}0"); + selected.range(lower..upper).next().is_some() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(unix)] + fn assert_selected_record_rejects_symlink_ancestor( + selected_path: &str, + outside_path: &str, + allow_ignored: bool, + ) { + let workspace = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + fs::create_dir_all(outside.path().join("docs")).unwrap(); + fs::write(outside.path().join("secret.txt"), "outside secret\n").unwrap(); + fs::write( + outside.path().join("docs/secret.txt"), + "outside directory secret\n", + ) + .unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::Empty, false).unwrap(); + std::os::unix::fs::symlink(outside.path(), workspace.path().join("link")).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + + let error = db + .record_with_options( + Some("main"), + Some("reject symlink ancestor".to_string()), + Actor::human(), + RecordOptions { + paths: vec![selected_path.to_string()], + allow_ignored, + ..RecordOptions::default() + }, + ) + .unwrap_err(); + + assert!( + matches!( + &error, + Error::InvalidPath { path, reason } + if path == selected_path && reason == "path uses a symlink ancestor" + ), + "selected {outside_path} through an ancestor symlink returned {error}" + ); + } + + #[test] + fn canonical_parent_selection_preserves_explicit_ignored_child_error() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir(temp.path().join("src")).unwrap(); + fs::write(temp.path().join("src/visible.txt"), "visible\n").unwrap(); + fs::write(temp.path().join("src/ignored.txt"), "ignored\n").unwrap(); + fs::write(temp.path().join(".trailignore"), "src/ignored.txt\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let paths = vec!["src".to_string(), "src/ignored.txt".to_string()]; + let selections = SelectionSet::from_paths(&paths).unwrap(); + + let error = db + .scan_record_selection_files(&paths, &selections, false) + .unwrap_err(); + + assert!(matches!(error, Error::IgnoredPath(path) if path == "src/ignored.txt")); + } + + #[cfg(unix)] + #[test] + fn selected_record_rejects_exact_file_through_symlink_ancestor() { + assert_selected_record_rejects_symlink_ancestor("link/secret.txt", "exact file", false); + } + + #[cfg(unix)] + #[test] + fn selected_record_rejects_directory_through_symlink_ancestor() { + assert_selected_record_rejects_symlink_ancestor("link/docs", "selected directory", false); + } + + #[cfg(unix)] + #[test] + fn selected_record_allow_ignored_rejects_symlink_ancestors() { + assert_selected_record_rejects_symlink_ancestor( + "link/secret.txt", + "allow-ignored exact file", + true, + ); + assert_selected_record_rejects_symlink_ancestor( + "link/docs", + "allow-ignored selected directory", + true, + ); + } + + #[cfg(unix)] + #[test] + fn selected_record_final_component_symlink_remains_ignored() { + let workspace = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + fs::write(outside.path().join("secret.txt"), "outside secret\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::Empty, false).unwrap(); + std::os::unix::fs::symlink( + outside.path().join("secret.txt"), + workspace.path().join("secret-link"), + ) + .unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + + let error = db + .record_with_options( + Some("main"), + Some("retain final symlink behavior".to_string()), + Actor::human(), + RecordOptions { + paths: vec!["secret-link".to_string()], + ..RecordOptions::default() + }, + ) + .unwrap_err(); + + assert!(matches!(error, Error::IgnoredPath(path) if path == "secret-link")); + } +} diff --git a/trail/src/db/storage/refs.rs b/trail/src/db/storage/refs.rs index 84a9c73..23d9417 100644 --- a/trail/src/db/storage/refs.rs +++ b/trail/src/db/storage/refs.rs @@ -1,6 +1,32 @@ use super::*; impl Trail { + pub(crate) fn insert_new_ref_database_only( + &self, + name: &str, + change_id: &ChangeId, + root_id: &ObjectId, + operation_id: &ObjectId, + ) -> Result<()> { + self.conn.execute( + "INSERT INTO refs + (name,change_id,root_id,operation_id,generation,updated_at) + VALUES (?1,?2,?3,?4,1,?5)", + params![name, change_id.0, root_id.0, operation_id.0, now_ts()], + )?; + Ok(()) + } + + pub(crate) fn repair_new_ref_mirror( + &self, + name: &str, + change_id: &ChangeId, + root_id: &ObjectId, + operation_id: &ObjectId, + ) -> Result<()> { + write_ref_file(&self.db_dir, name, change_id, root_id, operation_id, 1) + } + pub(crate) fn set_ref( &self, name: &str, @@ -46,9 +72,19 @@ impl Trail { change_id: &ChangeId, root_id: &ObjectId, operation_id: &ObjectId, + ) -> Result<()> { + self.advance_ref_cas_in_transaction(expected, change_id, root_id, operation_id)?; + self.repair_ref_mirror(expected, change_id, root_id, operation_id) + } + + pub(crate) fn advance_ref_cas_in_transaction( + &self, + expected: &RefRecord, + change_id: &ChangeId, + root_id: &ObjectId, + operation_id: &ObjectId, ) -> Result<()> { let generation = expected.generation + 1; - let now = now_ts(); let updated = self.conn.execute( "UPDATE refs SET change_id = ?1, root_id = ?2, operation_id = ?3, generation = ?4, updated_at = ?5 \ WHERE name = ?6 AND generation = ?7 AND change_id = ?8", @@ -57,7 +93,7 @@ impl Trail { root_id.0, operation_id.0, generation, - now, + now_ts(), expected.name.clone(), expected.generation, expected.change_id.0.clone() @@ -66,6 +102,17 @@ impl Trail { if updated != 1 { return Err(Error::StaleBranch(expected.name.clone())); } + Ok(()) + } + + pub(crate) fn repair_ref_mirror( + &self, + expected: &RefRecord, + change_id: &ChangeId, + root_id: &ObjectId, + operation_id: &ObjectId, + ) -> Result<()> { + let generation = expected.generation + 1; write_ref_file( &self.db_dir, &expected.name, @@ -120,7 +167,10 @@ impl Trail { if let Some(root_id) = refish.strip_prefix("root:") { return self.ref_from_root(&ObjectId(root_id.to_string())); } - if refish.starts_with("ch_") { + if let Some(change_id) = ChangeId::from_checkpoint_alias(refish) { + return self.ref_from_change(&change_id); + } + if crate::ids::is_change_id(refish) { return self.ref_from_change(&ChangeId(refish.to_string())); } if refish.starts_with("refs/") { @@ -132,7 +182,7 @@ impl Trail { if let Ok(record) = self.get_ref(&lane_ref(refish)) { return Ok(record); } - if refish.starts_with("obj_") { + if crate::ids::is_object_id(refish) { return self.ref_from_root(&ObjectId(refish.to_string())); } Err(Error::RefNotFound(refish.to_string())) diff --git a/trail/src/db/storage/root_diff.rs b/trail/src/db/storage/root_diff.rs index ad7aabd..8928261 100644 --- a/trail/src/db/storage/root_diff.rs +++ b/trail/src/db/storage/root_diff.rs @@ -1,10 +1,69 @@ +use super::diff::{RenameLookupProbes, RenameMatchIndex}; use super::*; impl Trail { + pub(crate) fn visit_root_file_entries( + &self, + root_id: &ObjectId, + prefixes: &[String], + mut visitor: F, + ) -> Result<()> + where + F: FnMut(String, FileEntry) -> Result<()>, + { + let root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, root_id)?; + let tree = root_map_tree_from_root_hex(root.path_map_root.as_deref())?; + if prefixes.is_empty() { + for item in self.root_prolly.range(&tree, &[], None)? { + let (key, value) = item?; + visitor(path_from_key(key)?, from_cbor::(&value)?)?; + } + return Ok(()); + } + + for prefix in prefixes { + if let Some(value) = self.root_prolly.get(&tree, prefix.as_bytes())? { + visitor(prefix.clone(), from_cbor::(&value)?)?; + } + let start = format!("{prefix}/"); + let end = format!("{prefix}0"); + for item in self + .root_prolly + .range(&tree, start.as_bytes(), Some(end.as_bytes()))? + { + let (key, value) = item?; + visitor(path_from_key(key)?, from_cbor::(&value)?)?; + } + } + Ok(()) + } + pub(crate) fn diff_root_file_summaries( &self, left_root_id: &ObjectId, right_root_id: &ObjectId, + ) -> Result> { + let mut probes = RenameLookupProbes::default(); + self.diff_root_file_summaries_indexed(left_root_id, right_root_id, &mut probes) + } + + #[cfg(test)] + fn diff_root_file_summaries_with_rename_probe_count( + &self, + left_root_id: &ObjectId, + right_root_id: &ObjectId, + ) -> Result<(Vec, u64)> { + let mut probes = RenameLookupProbes::default(); + let summaries = + self.diff_root_file_summaries_indexed(left_root_id, right_root_id, &mut probes)?; + Ok((summaries, probes.count())) + } + + fn diff_root_file_summaries_indexed( + &self, + left_root_id: &ObjectId, + right_root_id: &ObjectId, + probes: &mut RenameLookupProbes, ) -> Result> { let left_root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, left_root_id)?; let right_root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, right_root_id)?; @@ -40,31 +99,22 @@ impl Trail { changed.sort_by(|left, right| left.0.cmp(&right.0)); let mut summaries = Vec::new(); - let mut removed_by_hash: HashMap> = HashMap::new(); + let mut removed_by_identity = RenameMatchIndex::default(); for (path, entry) in &removed { - removed_by_hash - .entry(entry.content_hash.clone()) - .or_default() - .push((path.clone(), entry.clone())); + removed_by_identity.insert(path.clone(), entry.clone()); } let mut consumed_removed = HashSet::new(); for (path, new_entry) in added { - let rename = removed_by_hash - .get(&new_entry.content_hash) - .and_then(|candidates| { - candidates.iter().find(|(old_path, old_entry)| { - !consumed_removed.contains(old_path) - && old_entry.file_id == new_entry.file_id - }) - }); + probes.note_lookup(); + let rename = removed_by_identity.take(&new_entry); if let Some((old_path, old_entry)) = rename { consumed_removed.insert(old_path.clone()); summaries.push(file_diff_summary( path, - Some(old_path.clone()), + Some(old_path), FileChangeKind::Renamed, - Some(old_entry.content_hash.clone()), + Some(old_entry.content_hash), Some(new_entry.content_hash), )); continue; @@ -127,6 +177,43 @@ impl Trail { right_root_id: &ObjectId, patch_left: &mut BTreeMap, patch_right: &mut BTreeMap, + ) -> Result { + let mut probes = RenameLookupProbes::default(); + self.diff_root_file_maps_indexed( + left_root_id, + right_root_id, + patch_left, + patch_right, + &mut probes, + ) + } + + #[cfg(test)] + fn diff_root_file_maps_with_rename_probe_count( + &self, + left_root_id: &ObjectId, + right_root_id: &ObjectId, + patch_left: &mut BTreeMap, + patch_right: &mut BTreeMap, + ) -> Result<(RootDiff, u64)> { + let mut probes = RenameLookupProbes::default(); + let diff = self.diff_root_file_maps_indexed( + left_root_id, + right_root_id, + patch_left, + patch_right, + &mut probes, + )?; + Ok((diff, probes.count())) + } + + fn diff_root_file_maps_indexed( + &self, + left_root_id: &ObjectId, + right_root_id: &ObjectId, + patch_left: &mut BTreeMap, + patch_right: &mut BTreeMap, + probes: &mut RenameLookupProbes, ) -> Result { let left_root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, left_root_id)?; let right_root: WorktreeRoot = self.get_object(WORKTREE_ROOT_KIND, right_root_id)?; @@ -162,34 +249,25 @@ impl Trail { changed.sort_by(|left, right| left.0.cmp(&right.0)); let mut changes = Vec::new(); - let mut removed_by_hash: HashMap> = HashMap::new(); + let mut removed_by_identity = RenameMatchIndex::default(); for (path, entry) in &removed { - removed_by_hash - .entry(entry.content_hash.clone()) - .or_default() - .push((path.clone(), entry.clone())); + removed_by_identity.insert(path.clone(), entry.clone()); } let mut consumed_removed = HashSet::new(); for (path, new_entry) in added { - let rename = removed_by_hash - .get(&new_entry.content_hash) - .and_then(|candidates| { - candidates.iter().find(|(old_path, old_entry)| { - !consumed_removed.contains(old_path) - && old_entry.file_id == new_entry.file_id - }) - }); + probes.note_lookup(); + let rename = removed_by_identity.take(&new_entry); if let Some((old_path, old_entry)) = rename { consumed_removed.insert(old_path.clone()); patch_left.insert(old_path.clone(), old_entry.clone()); patch_right.insert(path.clone(), new_entry.clone()); changes.push(FileChange { path, - old_path: Some(old_path.clone()), + old_path: Some(old_path), file_id: Some(new_entry.file_id), kind: FileChangeKind::Renamed, - before_hash: Some(old_entry.content_hash.clone()), + before_hash: Some(old_entry.content_hash), after_hash: Some(new_entry.content_hash), line_changes: Vec::new(), }); @@ -284,3 +362,78 @@ fn file_diff_summary( patch: None, } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn root_diff_rename_matching_is_linear_for_same_content() { + const FILE_COUNT: usize = 1_000; + + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("seed.bin"), [0, 1, 2, 3]).unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let db = Trail::open(temp.path()).unwrap(); + let head = db.get_ref("refs/branches/main").unwrap(); + let template = db + .load_root_files(&head.root_id) + .unwrap() + .into_values() + .next() + .unwrap(); + let mut left = BTreeMap::new(); + let mut right = BTreeMap::new(); + for index in 0..FILE_COUNT { + let mut entry = template.clone(); + entry.file_id = FileId::new( + ChangeId("change_root_diff_source".to_string()), + index as u64, + ); + left.insert(format!("old/{index:04}.bin"), entry.clone()); + right.insert(format!("new/{index:04}.bin"), entry); + } + let left_root = db + .build_root_from_file_entries(left, &ChangeId("change_root_diff_left".to_string())) + .unwrap(); + let right_root = db + .build_root_from_file_entries(right, &ChangeId("change_root_diff_right".to_string())) + .unwrap(); + + let (summaries, summary_lookups) = db + .diff_root_file_summaries_with_rename_probe_count( + &left_root.root_id, + &right_root.root_id, + ) + .unwrap(); + let mut patch_left = BTreeMap::new(); + let mut patch_right = BTreeMap::new(); + let (diff, map_lookups) = db + .diff_root_file_maps_with_rename_probe_count( + &left_root.root_id, + &right_root.root_id, + &mut patch_left, + &mut patch_right, + ) + .unwrap(); + + assert_eq!(summary_lookups, FILE_COUNT as u64); + assert_eq!(map_lookups, FILE_COUNT as u64); + assert_eq!(summaries.len(), FILE_COUNT); + assert_eq!(diff.changes.len(), FILE_COUNT); + assert_eq!(diff.summaries.len(), summaries.len()); + for (map_summary, summary) in diff.summaries.iter().zip(&summaries) { + assert_eq!(map_summary.path, summary.path); + assert_eq!(map_summary.old_path, summary.old_path); + assert_eq!(map_summary.kind, summary.kind); + assert_eq!(map_summary.before_hash, summary.before_hash); + assert_eq!(map_summary.after_hash, summary.after_hash); + } + assert_eq!(patch_left.len(), FILE_COUNT); + assert_eq!(patch_right.len(), FILE_COUNT); + assert!(summaries + .iter() + .all(|summary| summary.kind == FileChangeKind::Renamed)); + } +} diff --git a/trail/src/db/storage/schema.rs b/trail/src/db/storage/schema.rs index 9b772e9..db733d4 100644 --- a/trail/src/db/storage/schema.rs +++ b/trail/src/db/storage/schema.rs @@ -1,4 +1,154 @@ use super::*; +mod agent_capture; +mod changed_path_ledger; mod ddl; mod version; + +const PROLLY_SQLITE_SCHEMA: &str = " +CREATE TABLE IF NOT EXISTS prolly_nodes ( + cid BLOB PRIMARY KEY NOT NULL, + node BLOB NOT NULL +) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS prolly_hints ( + namespace BLOB NOT NULL, + key BLOB NOT NULL, + value BLOB NOT NULL, + PRIMARY KEY (namespace, key) +) WITHOUT ROWID; +CREATE TABLE IF NOT EXISTS prolly_roots ( + name BLOB PRIMARY KEY NOT NULL, + manifest BLOB NOT NULL +) WITHOUT ROWID;"; + +impl Trail { + pub(crate) fn validate_schema_v18(conn: &Connection) -> Result<()> { + validate_schema_v18(conn) + } +} + +pub(crate) fn validate_schema_v18(conn: &Connection) -> Result<()> { + let user_version = conn.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?; + if user_version != TRAIL_SCHEMA_VERSION { + return Err(Error::Corrupt(format!( + "found version {user_version}; expected version {TRAIL_SCHEMA_VERSION}" + ))); + } + if !ddl::base_schema_v18_complete(conn)? { + return Err(Error::Corrupt("base schema v18 shape is incomplete".into())); + } + if !agent_capture::agent_capture_schema_complete(conn)? { + return Err(Error::Corrupt( + "agent capture schema v18 shape is incomplete".into(), + )); + } + if !changed_path_ledger::changed_path_ledger_schema_complete(conn)? { + return Err(Error::Corrupt( + "changed-path ledger schema v18 shape is incomplete".into(), + )); + } + Ok(()) +} + +pub(crate) fn validate_prolly_sqlite_schema_v18(conn: &Connection) -> Result<()> { + let expected = Connection::open_in_memory()?; + expected.execute_batch(PROLLY_SQLITE_SCHEMA)?; + if prolly_sqlite_objects(conn)? != prolly_sqlite_objects(&expected)? + || prolly_sqlite_structure(conn)? != prolly_sqlite_structure(&expected)? + { + return Err(Error::Corrupt( + "SQLite Prolly schema v18 shape is incomplete".into(), + )); + } + Ok(()) +} + +fn prolly_sqlite_structure( + conn: &Connection, +) -> Result, Vec<(String, Vec)>)>> { + let mut structure = Vec::new(); + for table in ["prolly_hints", "prolly_nodes", "prolly_roots"] { + let columns = conn + .prepare( + "SELECT cid, name, type, [notnull], COALESCE(dflt_value, ''), pk, hidden + FROM pragma_table_xinfo(?1) + ORDER BY cid", + )? + .query_map([table], |row| { + Ok(format!( + "{}|{}|{}|{}|{}|{}|{}", + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, String>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, i64>(6)?, + )) + })? + .collect::, _>>()?; + + let indexes = conn + .prepare( + "SELECT name FROM pragma_index_list(?1) + ORDER BY name", + )? + .query_map([table], |row| row.get::<_, String>(0))? + .collect::, _>>()? + .into_iter() + .map(|index| { + let entries = conn + .prepare( + "SELECT seqno, cid, COALESCE(name, ''), [desc], coll, key + FROM pragma_index_xinfo(?1) + ORDER BY seqno", + )? + .query_map([&index], |row| { + Ok(format!( + "{}|{}|{}|{}|{}|{}", + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, String>(4)?, + row.get::<_, i64>(5)?, + )) + })? + .collect::, _>>()?; + Ok((index, entries)) + }) + .collect::>>()?; + structure.push((table.to_string(), columns, indexes)); + } + Ok(structure) +} + +pub(crate) fn validate_no_prolly_sqlite_schema_v18(conn: &Connection) -> Result<()> { + if !prolly_sqlite_objects(conn)?.is_empty() { + return Err(Error::Corrupt( + "SlateDB workspace contains unexpected SQLite Prolly objects".into(), + )); + } + Ok(()) +} + +fn prolly_sqlite_objects(conn: &Connection) -> Result> { + let mut statement = conn.prepare( + "SELECT type, name, sql FROM sqlite_master + WHERE name LIKE 'prolly_%' + AND name NOT LIKE 'sqlite_%' + ORDER BY type, name", + )?; + let objects = statement + .query_map([], |row| { + let sql = row.get::<_, Option>(2)?.unwrap_or_default(); + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + sql.split_whitespace().collect::>().join(" "), + )) + })? + .collect::, _>>() + .map_err(Error::from)?; + Ok(objects) +} diff --git a/trail/src/db/storage/schema/agent_capture.rs b/trail/src/db/storage/schema/agent_capture.rs new file mode 100644 index 0000000..9eb7651 --- /dev/null +++ b/trail/src/db/storage/schema/agent_capture.rs @@ -0,0 +1,55 @@ +use super::*; +pub(in crate::db::storage) fn agent_capture_schema_complete(conn: &Connection) -> Result { + const REQUIRED_TABLES: [&str; 14] = [ + "agent_hook_installations", + "agent_capture_runs", + "lane_agent_sessions", + "lane_agent_session_aliases", + "lane_artifacts", + "agent_hook_receipts", + "lane_turn_evidence_manifests", + "lane_provenance_nodes", + "lane_provenance_edges", + "lane_session_attestations", + "agent_attestation_key_revocations", + "lane_session_attestation_turns", + "lane_learnings", + "git_agent_links", + ]; + for table in REQUIRED_TABLES { + let exists = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)", + params![table], + |row| row.get::<_, bool>(0), + )?; + if !exists { + return Ok(false); + } + } + + let mapping_columns = schema_table_columns(conn, "lane_agent_sessions")?; + let receipt_columns = schema_table_columns(conn, "agent_hook_receipts")?; + let artifact_columns = schema_table_columns(conn, "lane_artifacts")?; + Ok(mapping_columns.contains("capture_epoch") + && mapping_columns.contains("finalization_owner") + && mapping_columns.contains("finalization_lease_expires_at") + && mapping_columns.contains("next_receive_sequence") + && receipt_columns.contains("receive_sequence") + && receipt_columns.contains("raw_object_id") + && receipt_columns.contains("attempt_count") + && receipt_columns.contains("next_attempt_at") + && receipt_columns.contains("connection_id") + && receipt_columns.contains("direction") + && receipt_columns.contains("connection_sequence") + && artifact_columns.contains("retention_status") + && artifact_columns.contains("trust")) +} + +fn schema_table_columns(conn: &Connection, table: &str) -> Result> { + let mut stmt = conn.prepare(&format!("PRAGMA table_info({table})"))?; + let columns = stmt + .query_map([], |row| row.get::<_, String>(1))? + .collect::, _>>() + .map_err(Error::from)?; + Ok(columns) +} diff --git a/trail/src/db/storage/schema/changed_path_ledger.rs b/trail/src/db/storage/schema/changed_path_ledger.rs new file mode 100644 index 0000000..32ebb7a --- /dev/null +++ b/trail/src/db/storage/schema/changed_path_ledger.rs @@ -0,0 +1,1899 @@ +use super::*; + +const CHANGED_PATH_LEDGER_SCHEMA_VERSION: i64 = 1; +const CHANGED_PATH_OBSERVER_LOG_FORMAT_VERSION: i64 = 1; + +pub(super) const CHANGED_PATH_LEDGER_SCHEMA_V18: &str = + "CREATE TABLE changed_path_scopes ( + scope_id TEXT NOT NULL PRIMARY KEY CHECK (length(scope_id) > 0), + schema_version INTEGER NOT NULL DEFAULT 1 CHECK (schema_version = 1), + scope_kind TEXT NOT NULL CHECK (scope_kind IN ('workspace', 'materialized_lane', 'workspace_view')), + owner_id TEXT NOT NULL CHECK (length(owner_id) > 0), + scope_root TEXT NOT NULL, + scope_root_identity TEXT NOT NULL, + filesystem_identity TEXT NOT NULL, + filesystem_kind TEXT NOT NULL, + case_sensitive INTEGER NOT NULL CHECK (case_sensitive IN (0, 1)), + ref_name TEXT NOT NULL, + ref_generation INTEGER NOT NULL CHECK (ref_generation >= 0), + change_id TEXT NOT NULL, + baseline_root_id TEXT NOT NULL, + policy_fingerprint TEXT NOT NULL, + policy_dependency_generation INTEGER NOT NULL CHECK (policy_dependency_generation >= 0), + trust_state TEXT NOT NULL DEFAULT 'reconciling' + CHECK (trust_state IN ('trusted', 'reconciling', 'overflow', 'untrusted_gap', 'stale_baseline', 'corrupt')), + trust_reason TEXT NOT NULL DEFAULT 'fresh_create', + continuity_generation INTEGER NOT NULL DEFAULT 1 CHECK (continuity_generation >= 1), + epoch INTEGER NOT NULL DEFAULT 1 CHECK (epoch >= 1), + max_candidate_rows INTEGER NOT NULL DEFAULT 250000 CHECK(max_candidate_rows>0), + max_prefix_rows INTEGER NOT NULL DEFAULT 16384 CHECK(max_prefix_rows>0), + max_observer_log_bytes INTEGER NOT NULL DEFAULT 268435456 CHECK(max_observer_log_bytes>0), + max_segment_bytes INTEGER NOT NULL DEFAULT 16777216 CHECK(max_segment_bytes>0 AND max_segment_bytes<=max_observer_log_bytes), + max_unfolded_tail_records INTEGER NOT NULL DEFAULT 65536 CHECK(max_unfolded_tail_records>0), + provider_id TEXT, + provider_identity TEXT, + durable_cursor INTEGER NOT NULL DEFAULT 0 CHECK (durable_cursor IN (0, 1)), + linearizable_fence INTEGER NOT NULL DEFAULT 0 CHECK (linearizable_fence IN (0, 1)), + rename_pairing INTEGER NOT NULL DEFAULT 0 CHECK (rename_pairing IN (0, 1)), + overflow_scope INTEGER NOT NULL DEFAULT 0 CHECK (overflow_scope IN (0, 1)), + filesystem_supported INTEGER NOT NULL DEFAULT 0 CHECK (filesystem_supported IN (0, 1)), + clean_proof_allowed INTEGER NOT NULL DEFAULT 0 CHECK (clean_proof_allowed IN (0, 1)), + power_loss_durability INTEGER NOT NULL DEFAULT 0 CHECK (power_loss_durability IN (0, 1)), + provider_cursor BLOB, + provider_fence BLOB, + durable_offset INTEGER NOT NULL DEFAULT 0 CHECK (durable_offset >= 0), + folded_offset INTEGER NOT NULL DEFAULT 0 CHECK (folded_offset >= 0), + observer_owner_token TEXT, + observer_heartbeat_at INTEGER, + observer_error_state TEXT, + observer_error_at INTEGER, + retired_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE (scope_kind, owner_id), + CHECK (folded_offset <= durable_offset), + CHECK ((observer_error_state IS NULL AND observer_error_at IS NULL) + OR (observer_error_state IS NOT NULL AND observer_error_at IS NOT NULL)) + ); + CREATE TABLE changed_path_intents ( + intent_id TEXT NOT NULL PRIMARY KEY CHECK (length(intent_id) > 0), + schema_version INTEGER NOT NULL DEFAULT 1 CHECK (schema_version = 1), + scope_id TEXT NOT NULL + REFERENCES changed_path_scopes(scope_id) ON UPDATE CASCADE ON DELETE CASCADE, + producer TEXT NOT NULL, + expected_scope_epoch INTEGER NOT NULL CHECK (expected_scope_epoch >= 1), + expected_ref_name TEXT NOT NULL, + expected_ref_generation INTEGER NOT NULL CHECK (expected_ref_generation >= 0), + expected_change_id TEXT NOT NULL, + expected_root_id TEXT NOT NULL, + target_change_id TEXT NOT NULL, + target_root_id TEXT NOT NULL, + target_operation_id TEXT, + start_cursor BLOB, + lifecycle_state TEXT NOT NULL DEFAULT 'prepared' + CHECK (lifecycle_state IN ('prepared', 'filesystem_applied', 'published', 'acknowledged', 'aborted')), + verified_cut BLOB, + failure_reason TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE INDEX changed_path_intents_scope_state_idx + ON changed_path_intents(scope_id, lifecycle_state, updated_at); + CREATE INDEX changed_path_intents_gc_idx + ON changed_path_intents(lifecycle_state, target_root_id, target_operation_id); + CREATE TABLE changed_path_entries ( + scope_id TEXT NOT NULL + REFERENCES changed_path_scopes(scope_id) ON UPDATE CASCADE ON DELETE CASCADE, + normalized_path TEXT COLLATE BINARY NOT NULL CHECK (length(normalized_path) > 0), + event_flags INTEGER NOT NULL CHECK (event_flags >= 0), + source_mask INTEGER NOT NULL CHECK (source_mask >= 0), + first_sequence INTEGER NOT NULL CHECK (first_sequence >= 0), + last_sequence INTEGER NOT NULL CHECK (last_sequence >= first_sequence), + provider_id TEXT, + provider_sequence INTEGER CHECK (provider_sequence IS NULL OR provider_sequence >= 0), + intent_id TEXT + REFERENCES changed_path_intents(intent_id) ON UPDATE CASCADE ON DELETE SET NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (scope_id, normalized_path) + ); + CREATE INDEX changed_path_entries_sequence_idx + ON changed_path_entries(scope_id, last_sequence); + CREATE INDEX changed_path_entries_provider_idx + ON changed_path_entries(scope_id, provider_id, provider_sequence); + CREATE INDEX changed_path_entries_intent_idx + ON changed_path_entries(intent_id); + CREATE TABLE changed_path_prefixes ( + scope_id TEXT NOT NULL + REFERENCES changed_path_scopes(scope_id) ON UPDATE CASCADE ON DELETE CASCADE, + normalized_prefix TEXT COLLATE BINARY NOT NULL CHECK (length(normalized_prefix) > 0), + completeness_reason TEXT NOT NULL, + event_flags INTEGER NOT NULL CHECK (event_flags >= 0), + source_mask INTEGER NOT NULL CHECK (source_mask >= 0), + first_sequence INTEGER NOT NULL CHECK (first_sequence >= 0), + last_sequence INTEGER NOT NULL CHECK (last_sequence >= first_sequence), + provider_id TEXT, + provider_sequence INTEGER CHECK (provider_sequence IS NULL OR provider_sequence >= 0), + intent_id TEXT + REFERENCES changed_path_intents(intent_id) ON UPDATE CASCADE ON DELETE SET NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (scope_id, normalized_prefix) + ); + CREATE INDEX changed_path_prefixes_sequence_idx + ON changed_path_prefixes(scope_id, last_sequence); + CREATE INDEX changed_path_prefixes_provider_idx + ON changed_path_prefixes(scope_id, provider_id, provider_sequence); + CREATE INDEX changed_path_prefixes_intent_idx + ON changed_path_prefixes(intent_id); + CREATE TABLE changed_path_policy_dependencies ( + scope_id TEXT NOT NULL + REFERENCES changed_path_scopes(scope_id) ON UPDATE CASCADE ON DELETE CASCADE, + dependency_identity TEXT COLLATE BINARY NOT NULL CHECK(length(dependency_identity)>0), + dependency_kind TEXT NOT NULL + CHECK(dependency_kind IN ('builtin','trail_config','ignore','trailignore','gitignore','git_info_exclude','git_excludes_file','git_config','normalization','mode','case_policy')), + content_identity BLOB NOT NULL + CHECK(typeof(content_identity)='blob' AND length(content_identity)=32), + metadata_identity BLOB NOT NULL, + observable INTEGER NOT NULL CHECK(observable IN (0,1)), + generation INTEGER NOT NULL CHECK(generation>=0), + last_source_sequence INTEGER NOT NULL DEFAULT 0 CHECK(last_source_sequence>=0), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY(scope_id,dependency_identity,dependency_kind) + ) WITHOUT ROWID; + CREATE INDEX changed_path_policy_dependencies_generation_idx + ON changed_path_policy_dependencies(scope_id,generation,last_source_sequence); + CREATE TABLE changed_path_intent_paths ( + intent_id TEXT NOT NULL + REFERENCES changed_path_intents(intent_id) ON UPDATE CASCADE ON DELETE CASCADE, + normalized_path TEXT COLLATE BINARY NOT NULL CHECK (length(normalized_path) > 0), + event_flags INTEGER NOT NULL CHECK (event_flags >= 0), + PRIMARY KEY (intent_id, normalized_path) + ); + CREATE TABLE changed_path_intent_prefixes ( + intent_id TEXT NOT NULL + REFERENCES changed_path_intents(intent_id) ON UPDATE CASCADE ON DELETE CASCADE, + normalized_prefix TEXT COLLATE BINARY NOT NULL CHECK (length(normalized_prefix) > 0), + completeness_reason TEXT NOT NULL, + event_flags INTEGER NOT NULL CHECK (event_flags >= 0), + PRIMARY KEY (intent_id, normalized_prefix) + ); + CREATE TABLE changed_path_reconciliations ( + attempt_id TEXT NOT NULL PRIMARY KEY CHECK (length(attempt_id) > 0), + schema_version INTEGER NOT NULL DEFAULT 1 CHECK (schema_version = 1), + scope_id TEXT NOT NULL + REFERENCES changed_path_scopes(scope_id) ON UPDATE CASCADE ON DELETE CASCADE, + expected_scope_epoch INTEGER NOT NULL CHECK (expected_scope_epoch >= 1), + expected_ref_name TEXT NOT NULL, + expected_ref_generation INTEGER NOT NULL CHECK (expected_ref_generation >= 0), + expected_change_id TEXT NOT NULL, + expected_root_id TEXT NOT NULL, + filesystem_identity TEXT NOT NULL, + policy_fingerprint TEXT NOT NULL, + policy_dependency_generation INTEGER NOT NULL CHECK (policy_dependency_generation >= 0), + provider_id TEXT, + provider_identity TEXT, + start_cursor BLOB, + start_fence BLOB, + mode TEXT NOT NULL CHECK (mode IN ('full', 'prefix')), + reason TEXT NOT NULL, + completeness_class TEXT NOT NULL + CHECK (completeness_class IN ('complete', 'provider_complete_prefix', 'point_in_time_untrusted')), + staged_store_location TEXT NOT NULL DEFAULT 'sqlite', + state TEXT NOT NULL DEFAULT 'prepared' + CHECK (state IN ('prepared', 'staging', 'ready', 'published', 'abandoned', 'failed')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE INDEX changed_path_reconciliations_scope_state_idx + ON changed_path_reconciliations(scope_id, state, updated_at); + CREATE TABLE changed_path_reconciliation_rows ( + attempt_id TEXT NOT NULL + REFERENCES changed_path_reconciliations(attempt_id) ON UPDATE CASCADE ON DELETE CASCADE, + normalized_path TEXT COLLATE BINARY NOT NULL CHECK (length(normalized_path) > 0), + row_kind TEXT NOT NULL CHECK (row_kind IN ('entry', 'deletion')), + file_kind TEXT, + content_hash TEXT, + executable INTEGER CHECK (executable IS NULL OR executable IN (0, 1)), + size_bytes INTEGER CHECK (size_bytes IS NULL OR size_bytes >= 0), + before_identity TEXT, + after_identity TEXT, + source_sequence INTEGER CHECK (source_sequence IS NULL OR source_sequence >= 0), + staged_at INTEGER NOT NULL, + PRIMARY KEY (attempt_id, normalized_path) + ); + CREATE INDEX changed_path_reconciliation_rows_kind_idx + ON changed_path_reconciliation_rows(attempt_id, row_kind, normalized_path COLLATE BINARY); + CREATE TABLE changed_path_reconciliation_guards ( + attempt_id TEXT NOT NULL + REFERENCES changed_path_reconciliations(attempt_id) ON UPDATE CASCADE ON DELETE CASCADE, + relative_path BLOB NOT NULL CHECK (length(relative_path) > 0), + directory_identity BLOB NOT NULL CHECK (length(directory_identity) > 0), + staged_at INTEGER NOT NULL, + PRIMARY KEY (attempt_id, relative_path) + ) WITHOUT ROWID; + CREATE TABLE changed_path_observer_owners ( + scope_id TEXT NOT NULL PRIMARY KEY + REFERENCES changed_path_scopes(scope_id) ON UPDATE CASCADE ON DELETE CASCADE, + epoch INTEGER NOT NULL CHECK (epoch >= 1), + owner_token TEXT NOT NULL UNIQUE CHECK (length(owner_token) > 0), + provider_id TEXT NOT NULL, + provider_identity TEXT NOT NULL, + lease_state TEXT NOT NULL DEFAULT 'active' + CHECK (lease_state IN ('active', 'revoked', 'expired', 'error')), + fence_nonce BLOB, + acquired_at INTEGER NOT NULL, + heartbeat_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + error_state TEXT, + error_at INTEGER, + daemon_launch_nonce TEXT UNIQUE, + daemon_pid INTEGER, + daemon_process_start_identity TEXT, + updated_at INTEGER NOT NULL, + CHECK (acquired_at <= heartbeat_at AND heartbeat_at <= expires_at), + CHECK ((error_state IS NULL AND error_at IS NULL) + OR (error_state IS NOT NULL AND error_at IS NOT NULL)), + CHECK ((daemon_launch_nonce IS NULL AND daemon_pid IS NULL AND daemon_process_start_identity IS NULL) + OR (length(daemon_launch_nonce) = 64 AND daemon_pid > 0 AND length(daemon_process_start_identity) > 0)) + ); + CREATE TRIGGER changed_path_observer_owner_fail_closed + AFTER UPDATE OF lease_state ON changed_path_observer_owners + WHEN OLD.lease_state = 'active' AND NEW.lease_state <> 'active' + BEGIN + UPDATE changed_path_scopes + SET trust_state = CASE + WHEN trust_state IN ('trusted', 'reconciling') THEN 'untrusted_gap' + ELSE trust_state + END, + trust_reason = CASE + WHEN trust_state IN ('trusted', 'reconciling') + THEN 'observer_owner_' || NEW.lease_state + ELSE trust_reason + END, + continuity_generation = continuity_generation + 1, + updated_at = NEW.updated_at + WHERE scope_id = NEW.scope_id AND epoch = NEW.epoch; + END; + CREATE INDEX changed_path_observer_owners_state_idx + ON changed_path_observer_owners(lease_state, expires_at); + CREATE TABLE changed_path_observer_segments ( + scope_id TEXT NOT NULL + REFERENCES changed_path_scopes(scope_id) ON UPDATE CASCADE ON DELETE CASCADE, + epoch INTEGER NOT NULL CHECK (epoch >= 1), + segment_id TEXT NOT NULL CHECK (length(segment_id) > 0), + log_format_version INTEGER NOT NULL DEFAULT 1 CHECK (log_format_version = 1), + owner_token TEXT NOT NULL, + provider_id TEXT NOT NULL, + first_sequence INTEGER NOT NULL CHECK (first_sequence >= 1), + last_sequence INTEGER CHECK (last_sequence IS NULL OR last_sequence >= first_sequence), + durable_end_offset INTEGER NOT NULL DEFAULT 0 CHECK (durable_end_offset >= 0), + folded_end_offset INTEGER NOT NULL DEFAULT 0 CHECK (folded_end_offset >= 0), + previous_segment_id TEXT, + previous_segment_hash TEXT, + segment_hash TEXT, + segment_path TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'open' + CHECK (state IN ('open', 'sealed', 'retiring', 'retired', 'corrupt')), + retirement_source_state TEXT + CHECK (retirement_source_state IS NULL OR + retirement_source_state IN ('open','sealed')), + retirement_file_length INTEGER CHECK (retirement_file_length IS NULL OR retirement_file_length>=0), + retirement_file_hash TEXT CHECK (retirement_file_hash IS NULL OR length(retirement_file_hash)=64), + retirement_durable_hash TEXT CHECK (retirement_durable_hash IS NULL OR length(retirement_durable_hash)=64), + retirement_source_device TEXT, + retirement_source_inode TEXT, + created_at INTEGER NOT NULL, + sealed_at INTEGER, + updated_at INTEGER NOT NULL, + PRIMARY KEY (scope_id, epoch, segment_id), + UNIQUE (scope_id, epoch, first_sequence), + CHECK (folded_end_offset <= durable_end_offset), + CHECK ((retirement_source_device IS NULL)=(retirement_source_inode IS NULL)), + CHECK (state<>'retired' OR + (retirement_source_state IS NOT NULL AND retirement_file_length IS NOT NULL + AND retirement_file_hash IS NOT NULL AND retirement_durable_hash IS NOT NULL + AND retirement_source_device IS NOT NULL)) + ); + CREATE INDEX changed_path_observer_segments_state_idx + ON changed_path_observer_segments(scope_id, epoch, state, last_sequence); + CREATE TABLE changed_path_segment_quarantine_allocations ( + attempt_nonce TEXT PRIMARY KEY CHECK (length(attempt_nonce) = 64), + scope_id TEXT NOT NULL, + epoch INTEGER NOT NULL CHECK (epoch >= 1), + segment_id TEXT NOT NULL CHECK (length(segment_id) > 0), + quarantine_leaf TEXT NOT NULL UNIQUE CHECK (length(quarantine_leaf) > 0), + scope_directory_device TEXT NOT NULL CHECK (length(scope_directory_device) > 0), + scope_directory_inode TEXT NOT NULL CHECK (length(scope_directory_inode) > 0), + identity_policy TEXT NOT NULL + CHECK (identity_policy = 'direct_noreplace_same_directory_v1'), + source_segment_device TEXT NOT NULL CHECK (length(source_segment_device) > 0), + source_segment_inode TEXT NOT NULL CHECK (length(source_segment_inode) > 0), + quarantine_device TEXT, + quarantine_inode TEXT, + observed_conflict_device TEXT, + observed_conflict_inode TEXT, + retained_reason TEXT, + state TEXT NOT NULL DEFAULT 'allocating' + CHECK (state IN ('allocating', 'allocated', 'bound', 'abandoned')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + allocated_at INTEGER, + bound_at INTEGER, + abandoned_at INTEGER, + FOREIGN KEY (scope_id, epoch, segment_id) + REFERENCES changed_path_observer_segments(scope_id, epoch, segment_id) + ON UPDATE CASCADE ON DELETE CASCADE, + CHECK ((quarantine_device IS NULL) = (quarantine_inode IS NULL)), + CHECK (quarantine_device IS NULL OR + (quarantine_device=source_segment_device AND + quarantine_inode=source_segment_inode)), + CHECK ((observed_conflict_device IS NULL) = + (observed_conflict_inode IS NULL)), + CHECK (state NOT IN ('allocated', 'bound') OR + (quarantine_device IS NOT NULL AND allocated_at IS NOT NULL)), + CHECK ((state = 'bound' AND bound_at IS NOT NULL) OR + (state <> 'bound' AND bound_at IS NULL)), + CHECK ((state = 'abandoned' AND retained_reason IS NOT NULL + AND abandoned_at IS NOT NULL) OR + (state <> 'abandoned' AND retained_reason IS NULL + AND abandoned_at IS NULL)) + ); + CREATE INDEX changed_path_segment_quarantine_allocations_state_idx + ON changed_path_segment_quarantine_allocations(scope_id, epoch, segment_id, state); + CREATE UNIQUE INDEX changed_path_segment_quarantine_allocations_active_idx + ON changed_path_segment_quarantine_allocations(scope_id, epoch, segment_id) + WHERE state IN ('allocating', 'allocated', 'bound'); + CREATE TABLE changed_path_segment_deletions ( + scope_id TEXT NOT NULL, + epoch INTEGER NOT NULL CHECK (epoch >= 1), + segment_id TEXT NOT NULL CHECK (length(segment_id) > 0), + original_leaf TEXT NOT NULL CHECK (length(original_leaf) > 0), + quarantine_leaf TEXT NOT NULL CHECK (length(quarantine_leaf) > 0), + allocation_nonce TEXT NOT NULL UNIQUE CHECK (length(allocation_nonce) = 64), + log_format_version INTEGER NOT NULL CHECK (log_format_version=1), + provider_id TEXT NOT NULL CHECK (length(provider_id)>0), + folded_end_offset INTEGER NOT NULL CHECK (folded_end_offset>=0), + retirement_continuity_generation INTEGER NOT NULL CHECK (retirement_continuity_generation>=1), + retirement_fence_nonce BLOB NOT NULL CHECK (length(retirement_fence_nonce)=32), + scope_directory_device TEXT NOT NULL CHECK (length(scope_directory_device) > 0), + scope_directory_inode TEXT NOT NULL CHECK (length(scope_directory_inode) > 0), + quarantine_device TEXT NOT NULL CHECK (length(quarantine_device) > 0), + quarantine_inode TEXT NOT NULL CHECK (length(quarantine_inode) > 0), + segment_device TEXT NOT NULL CHECK (length(segment_device) > 0), + segment_inode TEXT NOT NULL CHECK (length(segment_inode) > 0), + file_length INTEGER NOT NULL CHECK (file_length >= 0), + file_hash TEXT NOT NULL CHECK (length(file_hash) = 64), + durable_end_offset INTEGER NOT NULL CHECK (durable_end_offset >= 0), + durable_hash TEXT NOT NULL CHECK (length(durable_hash) = 64), + max_observer_log_bytes INTEGER NOT NULL CHECK (max_observer_log_bytes > 0), + max_segment_bytes INTEGER NOT NULL CHECK (max_segment_bytes > 0), + max_unfolded_tail_records INTEGER NOT NULL CHECK (max_unfolded_tail_records > 0), + owner_token TEXT NOT NULL CHECK (length(owner_token) = 64), + first_sequence INTEGER NOT NULL CHECK (first_sequence >= 1), + last_sequence INTEGER, + previous_segment_id TEXT, + previous_segment_hash TEXT NOT NULL CHECK (length(previous_segment_hash) = 64), + source_state TEXT NOT NULL CHECK (source_state IN ('open', 'sealed')), + state TEXT NOT NULL DEFAULT 'quiesced' CHECK (state = 'quiesced'), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + completed_at INTEGER, + PRIMARY KEY (scope_id, epoch, segment_id), + UNIQUE (scope_id, epoch, original_leaf), + UNIQUE (scope_id, epoch, quarantine_leaf), + FOREIGN KEY (scope_id, epoch, segment_id) + REFERENCES changed_path_observer_segments(scope_id, epoch, segment_id) + ON UPDATE CASCADE ON DELETE CASCADE, + FOREIGN KEY (allocation_nonce) + REFERENCES changed_path_segment_quarantine_allocations(attempt_nonce) + ON UPDATE CASCADE ON DELETE CASCADE, + CHECK (quarantine_device=segment_device AND quarantine_inode=segment_inode), + CHECK (completed_at IS NOT NULL) + ); + CREATE INDEX changed_path_segment_deletions_state_idx + ON changed_path_segment_deletions(scope_id, epoch, state);"; + +pub(super) fn create_changed_path_ledger_schema(conn: &Connection) -> Result<()> { + conn.execute_batch(CHANGED_PATH_LEDGER_SCHEMA_V18) + .map_err(Into::into) +} + +pub(super) fn changed_path_ledger_schema_complete(conn: &Connection) -> Result { + let foreign_keys = conn.query_row("PRAGMA foreign_keys", [], |row| row.get::<_, i64>(0))?; + if foreign_keys != 1 + || CHANGED_PATH_LEDGER_SCHEMA_VERSION != 1 + || CHANGED_PATH_OBSERVER_LOG_FORMAT_VERSION != 1 + { + return Ok(false); + } + let user_version = conn.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))?; + if user_version != TRAIL_SCHEMA_VERSION { + return Ok(false); + } + let schema_meta_exists = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'schema_meta')", + [], + |row| row.get::<_, bool>(0), + )?; + if !schema_meta_exists { + return Ok(false); + } + let expected_version = TRAIL_SCHEMA_VERSION.to_string(); + let meta_version = conn + .query_row( + "SELECT value FROM schema_meta WHERE key = ?1", + params![SCHEMA_META_VERSION_KEY], + |row| row.get::<_, String>(0), + ) + .optional()?; + if meta_version.as_deref() != Some(expected_version.as_str()) { + return Ok(false); + } + if !ledger_master_matches(conn)? { + return Ok(false); + } + schema_structure_complete(conn) +} + +fn ledger_master_matches(conn: &Connection) -> Result { + let expected = Connection::open_in_memory()?; + expected.execute_batch(CHANGED_PATH_LEDGER_SCHEMA_V18)?; + Ok(ledger_schema_objects(conn)? == ledger_schema_objects(&expected)?) +} + +fn ledger_schema_objects(conn: &Connection) -> Result> { + let mut statement = conn.prepare( + "SELECT type, name, COALESCE(sql, '') FROM sqlite_master + WHERE name LIKE 'changed_path_%' + ORDER BY type, name", + )?; + let objects = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + normalized_sql(&row.get::<_, String>(2)?), + )) + })? + .collect::, _>>()?; + Ok(objects) +} + +fn schema_structure_complete(conn: &Connection) -> Result { + type Column = (&'static str, &'static str, bool, Option<&'static str>, i64); + let tables: [(&str, &[Column]); 14] = [ + ( + "changed_path_scopes", + &[ + ("scope_id", "TEXT", true, None, 1), + ("schema_version", "INTEGER", true, Some("1"), 0), + ("scope_kind", "TEXT", true, None, 0), + ("owner_id", "TEXT", true, None, 0), + ("scope_root", "TEXT", true, None, 0), + ("scope_root_identity", "TEXT", true, None, 0), + ("filesystem_identity", "TEXT", true, None, 0), + ("filesystem_kind", "TEXT", true, None, 0), + ("case_sensitive", "INTEGER", true, None, 0), + ("ref_name", "TEXT", true, None, 0), + ("ref_generation", "INTEGER", true, None, 0), + ("change_id", "TEXT", true, None, 0), + ("baseline_root_id", "TEXT", true, None, 0), + ("policy_fingerprint", "TEXT", true, None, 0), + ("policy_dependency_generation", "INTEGER", true, None, 0), + ("trust_state", "TEXT", true, Some("'reconciling'"), 0), + ("trust_reason", "TEXT", true, Some("'fresh_create'"), 0), + ("continuity_generation", "INTEGER", true, Some("1"), 0), + ("epoch", "INTEGER", true, Some("1"), 0), + ("max_candidate_rows", "INTEGER", true, Some("250000"), 0), + ("max_prefix_rows", "INTEGER", true, Some("16384"), 0), + ( + "max_observer_log_bytes", + "INTEGER", + true, + Some("268435456"), + 0, + ), + ("max_segment_bytes", "INTEGER", true, Some("16777216"), 0), + ( + "max_unfolded_tail_records", + "INTEGER", + true, + Some("65536"), + 0, + ), + ("provider_id", "TEXT", false, None, 0), + ("provider_identity", "TEXT", false, None, 0), + ("durable_cursor", "INTEGER", true, Some("0"), 0), + ("linearizable_fence", "INTEGER", true, Some("0"), 0), + ("rename_pairing", "INTEGER", true, Some("0"), 0), + ("overflow_scope", "INTEGER", true, Some("0"), 0), + ("filesystem_supported", "INTEGER", true, Some("0"), 0), + ("clean_proof_allowed", "INTEGER", true, Some("0"), 0), + ("power_loss_durability", "INTEGER", true, Some("0"), 0), + ("provider_cursor", "BLOB", false, None, 0), + ("provider_fence", "BLOB", false, None, 0), + ("durable_offset", "INTEGER", true, Some("0"), 0), + ("folded_offset", "INTEGER", true, Some("0"), 0), + ("observer_owner_token", "TEXT", false, None, 0), + ("observer_heartbeat_at", "INTEGER", false, None, 0), + ("observer_error_state", "TEXT", false, None, 0), + ("observer_error_at", "INTEGER", false, None, 0), + ("retired_at", "INTEGER", false, None, 0), + ("created_at", "INTEGER", true, None, 0), + ("updated_at", "INTEGER", true, None, 0), + ], + ), + ( + "changed_path_entries", + &[ + ("scope_id", "TEXT", true, None, 1), + ("normalized_path", "TEXT", true, None, 2), + ("event_flags", "INTEGER", true, None, 0), + ("source_mask", "INTEGER", true, None, 0), + ("first_sequence", "INTEGER", true, None, 0), + ("last_sequence", "INTEGER", true, None, 0), + ("provider_id", "TEXT", false, None, 0), + ("provider_sequence", "INTEGER", false, None, 0), + ("intent_id", "TEXT", false, None, 0), + ("created_at", "INTEGER", true, None, 0), + ("updated_at", "INTEGER", true, None, 0), + ], + ), + ( + "changed_path_prefixes", + &[ + ("scope_id", "TEXT", true, None, 1), + ("normalized_prefix", "TEXT", true, None, 2), + ("completeness_reason", "TEXT", true, None, 0), + ("event_flags", "INTEGER", true, None, 0), + ("source_mask", "INTEGER", true, None, 0), + ("first_sequence", "INTEGER", true, None, 0), + ("last_sequence", "INTEGER", true, None, 0), + ("provider_id", "TEXT", false, None, 0), + ("provider_sequence", "INTEGER", false, None, 0), + ("intent_id", "TEXT", false, None, 0), + ("created_at", "INTEGER", true, None, 0), + ("updated_at", "INTEGER", true, None, 0), + ], + ), + ( + "changed_path_policy_dependencies", + &[ + ("scope_id", "TEXT", true, None, 1), + ("dependency_identity", "TEXT", true, None, 2), + ("dependency_kind", "TEXT", true, None, 3), + ("content_identity", "BLOB", true, None, 0), + ("metadata_identity", "BLOB", true, None, 0), + ("observable", "INTEGER", true, None, 0), + ("generation", "INTEGER", true, None, 0), + ("last_source_sequence", "INTEGER", true, Some("0"), 0), + ("created_at", "INTEGER", true, None, 0), + ("updated_at", "INTEGER", true, None, 0), + ], + ), + ( + "changed_path_intents", + &[ + ("intent_id", "TEXT", true, None, 1), + ("schema_version", "INTEGER", true, Some("1"), 0), + ("scope_id", "TEXT", true, None, 0), + ("producer", "TEXT", true, None, 0), + ("expected_scope_epoch", "INTEGER", true, None, 0), + ("expected_ref_name", "TEXT", true, None, 0), + ("expected_ref_generation", "INTEGER", true, None, 0), + ("expected_change_id", "TEXT", true, None, 0), + ("expected_root_id", "TEXT", true, None, 0), + ("target_change_id", "TEXT", true, None, 0), + ("target_root_id", "TEXT", true, None, 0), + ("target_operation_id", "TEXT", false, None, 0), + ("start_cursor", "BLOB", false, None, 0), + ("lifecycle_state", "TEXT", true, Some("'prepared'"), 0), + ("verified_cut", "BLOB", false, None, 0), + ("failure_reason", "TEXT", false, None, 0), + ("created_at", "INTEGER", true, None, 0), + ("updated_at", "INTEGER", true, None, 0), + ], + ), + ( + "changed_path_intent_paths", + &[ + ("intent_id", "TEXT", true, None, 1), + ("normalized_path", "TEXT", true, None, 2), + ("event_flags", "INTEGER", true, None, 0), + ], + ), + ( + "changed_path_intent_prefixes", + &[ + ("intent_id", "TEXT", true, None, 1), + ("normalized_prefix", "TEXT", true, None, 2), + ("completeness_reason", "TEXT", true, None, 0), + ("event_flags", "INTEGER", true, None, 0), + ], + ), + ( + "changed_path_reconciliations", + &[ + ("attempt_id", "TEXT", true, None, 1), + ("schema_version", "INTEGER", true, Some("1"), 0), + ("scope_id", "TEXT", true, None, 0), + ("expected_scope_epoch", "INTEGER", true, None, 0), + ("expected_ref_name", "TEXT", true, None, 0), + ("expected_ref_generation", "INTEGER", true, None, 0), + ("expected_change_id", "TEXT", true, None, 0), + ("expected_root_id", "TEXT", true, None, 0), + ("filesystem_identity", "TEXT", true, None, 0), + ("policy_fingerprint", "TEXT", true, None, 0), + ("policy_dependency_generation", "INTEGER", true, None, 0), + ("provider_id", "TEXT", false, None, 0), + ("provider_identity", "TEXT", false, None, 0), + ("start_cursor", "BLOB", false, None, 0), + ("start_fence", "BLOB", false, None, 0), + ("mode", "TEXT", true, None, 0), + ("reason", "TEXT", true, None, 0), + ("completeness_class", "TEXT", true, None, 0), + ("staged_store_location", "TEXT", true, Some("'sqlite'"), 0), + ("state", "TEXT", true, Some("'prepared'"), 0), + ("created_at", "INTEGER", true, None, 0), + ("updated_at", "INTEGER", true, None, 0), + ], + ), + ( + "changed_path_reconciliation_rows", + &[ + ("attempt_id", "TEXT", true, None, 1), + ("normalized_path", "TEXT", true, None, 2), + ("row_kind", "TEXT", true, None, 0), + ("file_kind", "TEXT", false, None, 0), + ("content_hash", "TEXT", false, None, 0), + ("executable", "INTEGER", false, None, 0), + ("size_bytes", "INTEGER", false, None, 0), + ("before_identity", "TEXT", false, None, 0), + ("after_identity", "TEXT", false, None, 0), + ("source_sequence", "INTEGER", false, None, 0), + ("staged_at", "INTEGER", true, None, 0), + ], + ), + ( + "changed_path_reconciliation_guards", + &[ + ("attempt_id", "TEXT", true, None, 1), + ("relative_path", "BLOB", true, None, 2), + ("directory_identity", "BLOB", true, None, 0), + ("staged_at", "INTEGER", true, None, 0), + ], + ), + ( + "changed_path_observer_segments", + &[ + ("scope_id", "TEXT", true, None, 1), + ("epoch", "INTEGER", true, None, 2), + ("segment_id", "TEXT", true, None, 3), + ("log_format_version", "INTEGER", true, Some("1"), 0), + ("owner_token", "TEXT", true, None, 0), + ("provider_id", "TEXT", true, None, 0), + ("first_sequence", "INTEGER", true, None, 0), + ("last_sequence", "INTEGER", false, None, 0), + ("durable_end_offset", "INTEGER", true, Some("0"), 0), + ("folded_end_offset", "INTEGER", true, Some("0"), 0), + ("previous_segment_id", "TEXT", false, None, 0), + ("previous_segment_hash", "TEXT", false, None, 0), + ("segment_hash", "TEXT", false, None, 0), + ("segment_path", "TEXT", true, None, 0), + ("state", "TEXT", true, Some("'open'"), 0), + ("retirement_source_state", "TEXT", false, None, 0), + ("retirement_file_length", "INTEGER", false, None, 0), + ("retirement_file_hash", "TEXT", false, None, 0), + ("retirement_durable_hash", "TEXT", false, None, 0), + ("retirement_source_device", "TEXT", false, None, 0), + ("retirement_source_inode", "TEXT", false, None, 0), + ("created_at", "INTEGER", true, None, 0), + ("sealed_at", "INTEGER", false, None, 0), + ("updated_at", "INTEGER", true, None, 0), + ], + ), + ( + "changed_path_observer_owners", + &[ + ("scope_id", "TEXT", true, None, 1), + ("epoch", "INTEGER", true, None, 0), + ("owner_token", "TEXT", true, None, 0), + ("provider_id", "TEXT", true, None, 0), + ("provider_identity", "TEXT", true, None, 0), + ("lease_state", "TEXT", true, Some("'active'"), 0), + ("fence_nonce", "BLOB", false, None, 0), + ("acquired_at", "INTEGER", true, None, 0), + ("heartbeat_at", "INTEGER", true, None, 0), + ("expires_at", "INTEGER", true, None, 0), + ("error_state", "TEXT", false, None, 0), + ("error_at", "INTEGER", false, None, 0), + ("daemon_launch_nonce", "TEXT", false, None, 0), + ("daemon_pid", "INTEGER", false, None, 0), + ("daemon_process_start_identity", "TEXT", false, None, 0), + ("updated_at", "INTEGER", true, None, 0), + ], + ), + ( + "changed_path_segment_quarantine_allocations", + &[ + ("attempt_nonce", "TEXT", false, None, 1), + ("scope_id", "TEXT", true, None, 0), + ("epoch", "INTEGER", true, None, 0), + ("segment_id", "TEXT", true, None, 0), + ("quarantine_leaf", "TEXT", true, None, 0), + ("scope_directory_device", "TEXT", true, None, 0), + ("scope_directory_inode", "TEXT", true, None, 0), + ("identity_policy", "TEXT", true, None, 0), + ("source_segment_device", "TEXT", true, None, 0), + ("source_segment_inode", "TEXT", true, None, 0), + ("quarantine_device", "TEXT", false, None, 0), + ("quarantine_inode", "TEXT", false, None, 0), + ("observed_conflict_device", "TEXT", false, None, 0), + ("observed_conflict_inode", "TEXT", false, None, 0), + ("retained_reason", "TEXT", false, None, 0), + ("state", "TEXT", true, Some("'allocating'"), 0), + ("created_at", "INTEGER", true, None, 0), + ("updated_at", "INTEGER", true, None, 0), + ("allocated_at", "INTEGER", false, None, 0), + ("bound_at", "INTEGER", false, None, 0), + ("abandoned_at", "INTEGER", false, None, 0), + ], + ), + ( + "changed_path_segment_deletions", + &[ + ("scope_id", "TEXT", true, None, 1), + ("epoch", "INTEGER", true, None, 2), + ("segment_id", "TEXT", true, None, 3), + ("original_leaf", "TEXT", true, None, 0), + ("quarantine_leaf", "TEXT", true, None, 0), + ("allocation_nonce", "TEXT", true, None, 0), + ("log_format_version", "INTEGER", true, None, 0), + ("provider_id", "TEXT", true, None, 0), + ("folded_end_offset", "INTEGER", true, None, 0), + ("retirement_continuity_generation", "INTEGER", true, None, 0), + ("retirement_fence_nonce", "BLOB", true, None, 0), + ("scope_directory_device", "TEXT", true, None, 0), + ("scope_directory_inode", "TEXT", true, None, 0), + ("quarantine_device", "TEXT", true, None, 0), + ("quarantine_inode", "TEXT", true, None, 0), + ("segment_device", "TEXT", true, None, 0), + ("segment_inode", "TEXT", true, None, 0), + ("file_length", "INTEGER", true, None, 0), + ("file_hash", "TEXT", true, None, 0), + ("durable_end_offset", "INTEGER", true, None, 0), + ("durable_hash", "TEXT", true, None, 0), + ("max_observer_log_bytes", "INTEGER", true, None, 0), + ("max_segment_bytes", "INTEGER", true, None, 0), + ("max_unfolded_tail_records", "INTEGER", true, None, 0), + ("owner_token", "TEXT", true, None, 0), + ("first_sequence", "INTEGER", true, None, 0), + ("last_sequence", "INTEGER", false, None, 0), + ("previous_segment_id", "TEXT", false, None, 0), + ("previous_segment_hash", "TEXT", true, None, 0), + ("source_state", "TEXT", true, None, 0), + ("state", "TEXT", true, Some("'quiesced'"), 0), + ("created_at", "INTEGER", true, None, 0), + ("updated_at", "INTEGER", true, None, 0), + ("completed_at", "INTEGER", false, None, 0), + ], + ), + ]; + for (table, expected) in tables { + if !table_columns_match(conn, table, expected)? { + return Ok(false); + } + } + + let table_fragments: [(&str, &[&str]); 14] = [ + ( + "changed_path_scopes", + &[ + "CHECK (length(scope_id) > 0)", + "CHECK (schema_version = 1)", + "CHECK (scope_kind IN ('workspace', 'materialized_lane', 'workspace_view'))", + "CHECK (length(owner_id) > 0)", + "CHECK (case_sensitive IN (0, 1))", + "CHECK (ref_generation >= 0)", + "CHECK (policy_dependency_generation >= 0)", + "CHECK (trust_state IN ('trusted', 'reconciling', 'overflow', 'untrusted_gap', 'stale_baseline', 'corrupt'))", + "CHECK (continuity_generation >= 1)", + "CHECK (epoch >= 1)", + "CHECK(max_candidate_rows>0)", + "CHECK(max_prefix_rows>0)", + "CHECK(max_observer_log_bytes>0)", + "CHECK(max_segment_bytes>0 AND max_segment_bytes<=max_observer_log_bytes)", + "CHECK(max_unfolded_tail_records>0)", + "CHECK (durable_cursor IN (0, 1))", + "CHECK (linearizable_fence IN (0, 1))", + "CHECK (rename_pairing IN (0, 1))", + "CHECK (overflow_scope IN (0, 1))", + "CHECK (filesystem_supported IN (0, 1))", + "CHECK (clean_proof_allowed IN (0, 1))", + "CHECK (power_loss_durability IN (0, 1))", + "CHECK (durable_offset >= 0)", + "CHECK (folded_offset >= 0)", + "UNIQUE (scope_kind, owner_id)", + "CHECK (folded_offset <= durable_offset)", + "CHECK ((observer_error_state IS NULL AND observer_error_at IS NULL) OR (observer_error_state IS NOT NULL AND observer_error_at IS NOT NULL))", + ], + ), + ( + "changed_path_entries", + &[ + "CHECK (length(normalized_path) > 0)", + "CHECK (event_flags >= 0)", + "CHECK (source_mask >= 0)", + "CHECK (first_sequence >= 0)", + "CHECK (provider_sequence IS NULL OR provider_sequence >= 0)", + "CHECK (last_sequence >= first_sequence)", + ], + ), + ( + "changed_path_prefixes", + &[ + "CHECK (length(normalized_prefix) > 0)", + "CHECK (event_flags >= 0)", + "CHECK (source_mask >= 0)", + "CHECK (first_sequence >= 0)", + "CHECK (provider_sequence IS NULL OR provider_sequence >= 0)", + "CHECK (last_sequence >= first_sequence)", + ], + ), + ( + "changed_path_policy_dependencies", + &[ + "dependency_identity TEXT COLLATE BINARY NOT NULL", + "CHECK(dependency_kind IN ('builtin','trail_config','ignore','trailignore','gitignore','git_info_exclude','git_excludes_file','git_config','normalization','mode','case_policy'))", + "CHECK(typeof(content_identity)='blob' AND length(content_identity)=32)", + "CHECK(observable IN (0,1))", + "PRIMARY KEY(scope_id,dependency_identity,dependency_kind)", + "WITHOUT ROWID", + ], + ), + ( + "changed_path_intents", + &[ + "CHECK (length(intent_id) > 0)", + "CHECK (schema_version = 1)", + "CHECK (expected_scope_epoch >= 1)", + "CHECK (expected_ref_generation >= 0)", + "CHECK (lifecycle_state IN ('prepared', 'filesystem_applied', 'published', 'acknowledged', 'aborted'))", + ], + ), + ( + "changed_path_intent_paths", + &[ + "CHECK (length(normalized_path) > 0)", + "CHECK (event_flags >= 0)", + ], + ), + ( + "changed_path_intent_prefixes", + &[ + "CHECK (length(normalized_prefix) > 0)", + "CHECK (event_flags >= 0)", + ], + ), + ( + "changed_path_reconciliations", + &[ + "CHECK (length(attempt_id) > 0)", + "CHECK (schema_version = 1)", + "CHECK (expected_scope_epoch >= 1)", + "CHECK (expected_ref_generation >= 0)", + "CHECK (policy_dependency_generation >= 0)", + "CHECK (mode IN ('full', 'prefix'))", + "CHECK (completeness_class IN ('complete', 'provider_complete_prefix', 'point_in_time_untrusted'))", + "CHECK (state IN ('prepared', 'staging', 'ready', 'published', 'abandoned', 'failed'))", + ], + ), + ( + "changed_path_reconciliation_rows", + &[ + "CHECK (length(normalized_path) > 0)", + "CHECK (row_kind IN ('entry', 'deletion'))", + "CHECK (executable IS NULL OR executable IN (0, 1))", + "CHECK (size_bytes IS NULL OR size_bytes >= 0)", + "CHECK (source_sequence IS NULL OR source_sequence >= 0)", + ], + ), + ( + "changed_path_reconciliation_guards", + &[ + "CHECK (length(relative_path) > 0)", + "CHECK (length(directory_identity) > 0)", + "WITHOUT ROWID", + ], + ), + ( + "changed_path_observer_segments", + &[ + "CHECK (length(segment_id) > 0)", + "CHECK (epoch >= 1)", + "CHECK (log_format_version = 1)", + "CHECK (first_sequence >= 1)", + "CHECK (last_sequence IS NULL OR last_sequence >= first_sequence)", + "CHECK (durable_end_offset >= 0)", + "CHECK (folded_end_offset >= 0)", + "CHECK (state IN ('open', 'sealed', 'retiring', 'retired', 'corrupt'))", + "UNIQUE (scope_id, epoch, first_sequence)", + "CHECK (folded_end_offset <= durable_end_offset)", + "CHECK ((retirement_source_device IS NULL)=(retirement_source_inode IS NULL))", + ], + ), + ( + "changed_path_observer_owners", + &[ + "CHECK (length(owner_token) > 0)", + "CHECK (epoch >= 1)", + "CHECK (lease_state IN ('active', 'revoked', 'expired', 'error'))", + "CHECK (acquired_at <= heartbeat_at AND heartbeat_at <= expires_at)", + "CHECK ((error_state IS NULL AND error_at IS NULL) OR (error_state IS NOT NULL AND error_at IS NOT NULL))", + "CHECK ((daemon_launch_nonce IS NULL AND daemon_pid IS NULL AND daemon_process_start_identity IS NULL) OR (length(daemon_launch_nonce) = 64 AND daemon_pid > 0 AND length(daemon_process_start_identity) > 0))", + "owner_token TEXT NOT NULL UNIQUE", + "daemon_launch_nonce TEXT UNIQUE", + ], + ), + ( + "changed_path_segment_quarantine_allocations", + &[ + "CHECK (length(attempt_nonce) = 64)", + "CHECK (epoch >= 1)", + "CHECK (identity_policy = 'direct_noreplace_same_directory_v1')", + "CHECK (state IN ('allocating', 'allocated', 'bound', 'abandoned'))", + "CHECK (quarantine_device IS NULL OR (quarantine_device=source_segment_device AND quarantine_inode=source_segment_inode))", + ], + ), + ( + "changed_path_segment_deletions", + &[ + "CHECK (epoch >= 1)", + "CHECK (length(allocation_nonce) = 64)", + "CHECK (log_format_version=1)", + "CHECK (folded_end_offset>=0)", + "CHECK (retirement_continuity_generation>=1)", + "CHECK (length(retirement_fence_nonce)=32)", + "CHECK (state = 'quiesced')", + "CHECK (quarantine_device=segment_device AND quarantine_inode=segment_inode)", + "UNIQUE (scope_id, epoch, original_leaf)", + "UNIQUE (scope_id, epoch, quarantine_leaf)", + ], + ), + ]; + for (table, fragments) in table_fragments { + if !table_sql_contains(conn, table, fragments)? { + return Ok(false); + } + } + + let primary_keys: [(&str, &[&str]); 14] = [ + ("changed_path_scopes", &["scope_id"]), + ("changed_path_entries", &["scope_id", "normalized_path"]), + ("changed_path_prefixes", &["scope_id", "normalized_prefix"]), + ( + "changed_path_policy_dependencies", + &["scope_id", "dependency_identity", "dependency_kind"], + ), + ("changed_path_intents", &["intent_id"]), + ( + "changed_path_intent_paths", + &["intent_id", "normalized_path"], + ), + ( + "changed_path_intent_prefixes", + &["intent_id", "normalized_prefix"], + ), + ("changed_path_reconciliations", &["attempt_id"]), + ( + "changed_path_reconciliation_rows", + &["attempt_id", "normalized_path"], + ), + ( + "changed_path_reconciliation_guards", + &["attempt_id", "relative_path"], + ), + ( + "changed_path_observer_segments", + &["scope_id", "epoch", "segment_id"], + ), + ("changed_path_observer_owners", &["scope_id"]), + ( + "changed_path_segment_quarantine_allocations", + &["attempt_nonce"], + ), + ( + "changed_path_segment_deletions", + &["scope_id", "epoch", "segment_id"], + ), + ]; + for (table, columns) in primary_keys { + if !origin_index_matches(conn, table, "pk", columns)? { + return Ok(false); + } + } + for (table, columns) in [ + ( + "changed_path_scopes", + &["scope_kind", "owner_id"].as_slice(), + ), + ( + "changed_path_observer_segments", + &["scope_id", "epoch", "first_sequence"].as_slice(), + ), + ] { + if !origin_index_matches(conn, table, "u", columns)? { + return Ok(false); + } + } + if !origin_indexes_match( + conn, + "changed_path_observer_owners", + "u", + &[&["owner_token"], &["daemon_launch_nonce"]], + )? || !origin_indexes_match( + conn, + "changed_path_segment_quarantine_allocations", + "u", + &[&["quarantine_leaf"]], + )? || !origin_indexes_match( + conn, + "changed_path_segment_deletions", + "u", + &[ + &["allocation_nonce"], + &["scope_id", "epoch", "original_leaf"], + &["scope_id", "epoch", "quarantine_leaf"], + ], + )? { + return Ok(false); + } + for table in [ + "changed_path_entries", + "changed_path_prefixes", + "changed_path_intents", + "changed_path_intent_paths", + "changed_path_intent_prefixes", + "changed_path_reconciliations", + "changed_path_reconciliation_rows", + "changed_path_reconciliation_guards", + ] { + if origin_index_count(conn, table, "u")? != 0 { + return Ok(false); + } + } + + let named_indexes: [(&str, &str, &[&str]); 15] = [ + ( + "changed_path_intents", + "changed_path_intents_scope_state_idx", + &["scope_id", "lifecycle_state", "updated_at"], + ), + ( + "changed_path_intents", + "changed_path_intents_gc_idx", + &["lifecycle_state", "target_root_id", "target_operation_id"], + ), + ( + "changed_path_entries", + "changed_path_entries_sequence_idx", + &["scope_id", "last_sequence"], + ), + ( + "changed_path_entries", + "changed_path_entries_provider_idx", + &["scope_id", "provider_id", "provider_sequence"], + ), + ( + "changed_path_entries", + "changed_path_entries_intent_idx", + &["intent_id"], + ), + ( + "changed_path_prefixes", + "changed_path_prefixes_sequence_idx", + &["scope_id", "last_sequence"], + ), + ( + "changed_path_prefixes", + "changed_path_prefixes_provider_idx", + &["scope_id", "provider_id", "provider_sequence"], + ), + ( + "changed_path_prefixes", + "changed_path_prefixes_intent_idx", + &["intent_id"], + ), + ( + "changed_path_policy_dependencies", + "changed_path_policy_dependencies_generation_idx", + &["scope_id", "generation", "last_source_sequence"], + ), + ( + "changed_path_reconciliations", + "changed_path_reconciliations_scope_state_idx", + &["scope_id", "state", "updated_at"], + ), + ( + "changed_path_reconciliation_rows", + "changed_path_reconciliation_rows_kind_idx", + &["attempt_id", "row_kind", "normalized_path"], + ), + ( + "changed_path_observer_owners", + "changed_path_observer_owners_state_idx", + &["lease_state", "expires_at"], + ), + ( + "changed_path_observer_segments", + "changed_path_observer_segments_state_idx", + &["scope_id", "epoch", "state", "last_sequence"], + ), + ( + "changed_path_segment_quarantine_allocations", + "changed_path_segment_quarantine_allocations_state_idx", + &["scope_id", "epoch", "segment_id", "state"], + ), + ( + "changed_path_segment_deletions", + "changed_path_segment_deletions_state_idx", + &["scope_id", "epoch", "state"], + ), + ]; + for (table, index, columns) in named_indexes { + if !named_index_matches(conn, table, index, columns)? { + return Ok(false); + } + } + if !named_unique_partial_index_matches( + conn, + "changed_path_segment_quarantine_allocations", + "changed_path_segment_quarantine_allocations_active_idx", + &["scope_id", "epoch", "segment_id"], + )? { + return Ok(false); + } + + let expected_foreign_keys: [(&str, &[(&str, &str, &str, &str, &str)]); 14] = [ + ("changed_path_scopes", &[]), + ( + "changed_path_entries", + &[ + ( + "scope_id", + "changed_path_scopes", + "scope_id", + "CASCADE", + "CASCADE", + ), + ( + "intent_id", + "changed_path_intents", + "intent_id", + "CASCADE", + "SET NULL", + ), + ], + ), + ( + "changed_path_prefixes", + &[ + ( + "scope_id", + "changed_path_scopes", + "scope_id", + "CASCADE", + "CASCADE", + ), + ( + "intent_id", + "changed_path_intents", + "intent_id", + "CASCADE", + "SET NULL", + ), + ], + ), + ( + "changed_path_policy_dependencies", + &[( + "scope_id", + "changed_path_scopes", + "scope_id", + "CASCADE", + "CASCADE", + )], + ), + ( + "changed_path_intents", + &[( + "scope_id", + "changed_path_scopes", + "scope_id", + "CASCADE", + "CASCADE", + )], + ), + ( + "changed_path_intent_paths", + &[( + "intent_id", + "changed_path_intents", + "intent_id", + "CASCADE", + "CASCADE", + )], + ), + ( + "changed_path_intent_prefixes", + &[( + "intent_id", + "changed_path_intents", + "intent_id", + "CASCADE", + "CASCADE", + )], + ), + ( + "changed_path_reconciliations", + &[( + "scope_id", + "changed_path_scopes", + "scope_id", + "CASCADE", + "CASCADE", + )], + ), + ( + "changed_path_reconciliation_rows", + &[( + "attempt_id", + "changed_path_reconciliations", + "attempt_id", + "CASCADE", + "CASCADE", + )], + ), + ( + "changed_path_reconciliation_guards", + &[( + "attempt_id", + "changed_path_reconciliations", + "attempt_id", + "CASCADE", + "CASCADE", + )], + ), + ( + "changed_path_observer_segments", + &[( + "scope_id", + "changed_path_scopes", + "scope_id", + "CASCADE", + "CASCADE", + )], + ), + ( + "changed_path_observer_owners", + &[( + "scope_id", + "changed_path_scopes", + "scope_id", + "CASCADE", + "CASCADE", + )], + ), + ( + "changed_path_segment_quarantine_allocations", + &[ + ( + "scope_id", + "changed_path_observer_segments", + "scope_id", + "CASCADE", + "CASCADE", + ), + ( + "epoch", + "changed_path_observer_segments", + "epoch", + "CASCADE", + "CASCADE", + ), + ( + "segment_id", + "changed_path_observer_segments", + "segment_id", + "CASCADE", + "CASCADE", + ), + ], + ), + ( + "changed_path_segment_deletions", + &[ + ( + "scope_id", + "changed_path_observer_segments", + "scope_id", + "CASCADE", + "CASCADE", + ), + ( + "epoch", + "changed_path_observer_segments", + "epoch", + "CASCADE", + "CASCADE", + ), + ( + "segment_id", + "changed_path_observer_segments", + "segment_id", + "CASCADE", + "CASCADE", + ), + ( + "allocation_nonce", + "changed_path_segment_quarantine_allocations", + "attempt_nonce", + "CASCADE", + "CASCADE", + ), + ], + ), + ]; + for (table, expected) in expected_foreign_keys { + if !foreign_keys_match(conn, table, expected)? { + return Ok(false); + } + } + + for (table, _) in tables { + let mut statement = conn.prepare(&format!("PRAGMA foreign_key_check({table})"))?; + let mut rows = statement.query([])?; + if rows.next()?.is_some() { + return Ok(false); + } + } + let invalid_retirement_graph: bool = conn.query_row( + "WITH invalid AS ( + SELECT allocation.attempt_nonce + FROM changed_path_segment_quarantine_allocations allocation + LEFT JOIN changed_path_segment_deletions deletion + ON deletion.allocation_nonce=allocation.attempt_nonce + GROUP BY allocation.attempt_nonce + HAVING COUNT(deletion.allocation_nonce)<>CASE WHEN allocation.state='bound' THEN 1 ELSE 0 END + OR (allocation.state='bound' AND ( + MIN(deletion.state)<>'quiesced' OR MIN(deletion.completed_at) IS NULL + OR MIN(deletion.scope_id)<>allocation.scope_id + OR MIN(deletion.epoch)<>allocation.epoch + OR MIN(deletion.segment_id)<>allocation.segment_id + OR MIN(deletion.quarantine_leaf)<>allocation.quarantine_leaf + OR MIN(deletion.scope_directory_device)<>allocation.scope_directory_device + OR MIN(deletion.scope_directory_inode)<>allocation.scope_directory_inode + OR MIN(deletion.segment_device)<>allocation.source_segment_device + OR MIN(deletion.segment_inode)<>allocation.source_segment_inode + OR MIN(deletion.quarantine_device)<>allocation.quarantine_device + OR MIN(deletion.quarantine_inode)<>allocation.quarantine_inode)) + UNION ALL + SELECT deletion.allocation_nonce + FROM changed_path_segment_deletions deletion + LEFT JOIN changed_path_segment_quarantine_allocations allocation + ON allocation.attempt_nonce=deletion.allocation_nonce + WHERE allocation.attempt_nonce IS NULL OR allocation.state<>'bound' + OR deletion.state<>'quiesced' OR deletion.completed_at IS NULL + UNION ALL + SELECT segment_id + FROM changed_path_segment_quarantine_allocations + WHERE state IN ('allocating','allocated','bound') + GROUP BY scope_id,epoch,segment_id HAVING COUNT(*)<>1 + UNION ALL + SELECT attempt_nonce + FROM changed_path_segment_quarantine_allocations + WHERE quarantine_device IS NOT NULL + AND (quarantine_device<>source_segment_device + OR quarantine_inode<>source_segment_inode) + UNION ALL + SELECT segment_id + FROM changed_path_segment_deletions + WHERE quarantine_device<>segment_device OR quarantine_inode<>segment_inode + UNION ALL + SELECT scope.scope_id + FROM changed_path_scopes scope + WHERE scope.retired_at IS NULL + AND scope.trust_reason<>'scope_retiring' + AND (EXISTS( + SELECT 1 FROM changed_path_segment_quarantine_allocations allocation + WHERE allocation.scope_id=scope.scope_id) + OR EXISTS( + SELECT 1 FROM changed_path_segment_deletions deletion + WHERE deletion.scope_id=scope.scope_id) + OR EXISTS( + SELECT 1 FROM changed_path_observer_segments segment + WHERE segment.scope_id=scope.scope_id + AND segment.state IN ('retiring','retired'))) + UNION ALL + SELECT scope.scope_id + FROM changed_path_scopes scope + WHERE scope.retired_at IS NOT NULL + AND (scope.trust_state<>'untrusted_gap' OR scope.trust_reason<>'scope_retired' + OR EXISTS( + SELECT 1 FROM changed_path_observer_segments segment + WHERE segment.scope_id=scope.scope_id + AND (segment.epoch<>scope.epoch OR segment.state<>'retired')) + OR EXISTS( + SELECT 1 FROM changed_path_observer_segments segment + WHERE segment.scope_id=scope.scope_id + AND ((SELECT COUNT(*) + FROM changed_path_segment_quarantine_allocations allocation + WHERE allocation.scope_id=segment.scope_id + AND allocation.epoch=segment.epoch + AND allocation.segment_id=segment.segment_id + AND allocation.state='bound')<>1 + OR (SELECT COUNT(*) + FROM changed_path_segment_deletions deletion + WHERE deletion.scope_id=segment.scope_id + AND deletion.epoch=segment.epoch + AND deletion.segment_id=segment.segment_id)<>1)) + OR (EXISTS(SELECT 1 FROM changed_path_observer_segments segment + WHERE segment.scope_id=scope.scope_id) + AND NOT EXISTS(SELECT 1 FROM changed_path_observer_owners owner + WHERE owner.scope_id=scope.scope_id + AND owner.epoch=scope.epoch + AND owner.lease_state='revoked' + AND length(owner.fence_nonce)=32))) + UNION ALL + SELECT scope.scope_id + FROM changed_path_scopes scope + WHERE scope.retired_at IS NULL AND scope.trust_reason='scope_retiring' + AND (scope.trust_state<>'untrusted_gap' + OR EXISTS( + SELECT 1 FROM changed_path_observer_segments segment + WHERE segment.scope_id=scope.scope_id + AND (segment.epoch<>scope.epoch OR segment.state<>'retiring' + OR segment.retirement_source_state IS NULL)) + OR EXISTS( + SELECT 1 FROM changed_path_observer_owners owner + WHERE owner.scope_id=scope.scope_id + AND (owner.epoch<>scope.epoch OR owner.lease_state<>'revoked' + OR length(owner.fence_nonce)<>32)) + OR (EXISTS(SELECT 1 FROM changed_path_observer_segments segment + WHERE segment.scope_id=scope.scope_id) + AND NOT EXISTS(SELECT 1 FROM changed_path_observer_owners owner + WHERE owner.scope_id=scope.scope_id + AND owner.epoch=scope.epoch + AND owner.lease_state='revoked' + AND length(owner.fence_nonce)=32)) + OR EXISTS( + SELECT 1 FROM changed_path_segment_deletions deletion + WHERE deletion.scope_id=scope.scope_id)) + UNION ALL + SELECT deletion.segment_id + FROM changed_path_segment_deletions deletion + JOIN changed_path_observer_segments segment + ON segment.scope_id=deletion.scope_id AND segment.epoch=deletion.epoch + AND segment.segment_id=deletion.segment_id + JOIN changed_path_segment_quarantine_allocations allocation + ON allocation.attempt_nonce=deletion.allocation_nonce + JOIN changed_path_scopes scope ON scope.scope_id=segment.scope_id + JOIN changed_path_observer_owners owner ON owner.scope_id=segment.scope_id + WHERE scope.retired_at IS NULL OR scope.trust_reason<>'scope_retired' + OR segment.state<>'retired' OR allocation.state<>'bound' + OR owner.epoch<>segment.epoch OR owner.lease_state<>'revoked' + OR deletion.retirement_continuity_generation<>scope.continuity_generation + OR deletion.retirement_fence_nonce<>owner.fence_nonce + OR deletion.original_leaf<>segment.segment_path + OR deletion.log_format_version<>segment.log_format_version + OR deletion.owner_token<>segment.owner_token + OR deletion.owner_token<>owner.owner_token + OR deletion.provider_id<>segment.provider_id + OR deletion.provider_id<>owner.provider_id + OR deletion.provider_id IS NOT scope.provider_id + OR deletion.first_sequence<>segment.first_sequence + OR deletion.last_sequence IS NOT segment.last_sequence + OR deletion.durable_end_offset<>segment.durable_end_offset + OR deletion.folded_end_offset<>segment.folded_end_offset + OR deletion.previous_segment_id IS NOT segment.previous_segment_id + OR deletion.previous_segment_hash<> + COALESCE(segment.previous_segment_hash, + '0000000000000000000000000000000000000000000000000000000000000000') + OR (segment.segment_hash IS NOT NULL + AND deletion.file_hash<>segment.segment_hash) + OR deletion.source_state<>segment.retirement_source_state + OR deletion.file_length<>segment.retirement_file_length + OR deletion.file_hash<>segment.retirement_file_hash + OR deletion.durable_hash<>segment.retirement_durable_hash + OR deletion.segment_device<>segment.retirement_source_device + OR deletion.segment_inode<>segment.retirement_source_inode + OR allocation.source_segment_device<>segment.retirement_source_device + OR allocation.source_segment_inode<>segment.retirement_source_inode + ) SELECT EXISTS(SELECT 1 FROM invalid)", + [], + |row| row.get(0), + )?; + if invalid_retirement_graph { + return Ok(false); + } + policy_dependencies_canonical(conn) +} + +fn policy_dependencies_canonical(conn: &Connection) -> Result { + let invalid_row: bool = conn.query_row( + "SELECT EXISTS( + SELECT 1 + FROM changed_path_policy_dependencies dependency + JOIN changed_path_scopes scope ON scope.scope_id=dependency.scope_id + WHERE typeof(dependency.content_identity)<>'blob' + OR length(dependency.content_identity)<>32 + OR dependency.generation<>scope.policy_dependency_generation + )", + [], + |row| row.get(0), + )?; + if invalid_row { + return Ok(false); + } + + let mut statement = conn.prepare( + "SELECT dependency_identity,dependency_kind + FROM changed_path_policy_dependencies + ORDER BY scope_id,dependency_kind COLLATE BINARY,dependency_identity COLLATE BINARY", + )?; + let mut rows = statement.query([])?; + while let Some(row) = rows.next()? { + let identity = row.get::<_, String>(0)?; + let kind = row.get::<_, String>(1)?; + if !policy_dependency_identity_is_canonical(&identity, &kind) { + return Ok(false); + } + } + Ok(true) +} + +fn policy_dependency_identity_is_canonical(identity: &str, kind: &str) -> bool { + if let Some(encoded_path) = identity.strip_prefix("path:") { + if !matches!( + kind, + "trail_config" + | "ignore" + | "trailignore" + | "gitignore" + | "git_info_exclude" + | "git_excludes_file" + | "git_config" + ) || encoded_path.is_empty() + || encoded_path.len() % 2 != 0 + || !encoded_path + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return false; + } + let Ok(path_bytes) = hex::decode(encoded_path) else { + return false; + }; + let path = policy_path_from_bytes(path_bytes); + return path.is_absolute() + && lexical_normalize_policy_path(&path) == path + && policy_path_identity(&path) == identity; + } + + match kind { + "builtin" => identity == "builtin:recording-policy", + "trail_config" => identity == "trail-config:recording", + "normalization" => identity == "normalization:path", + "mode" => identity == "mode:filesystem-entry", + "case_policy" => identity == "case-policy:scope", + "git_config" => identity + .strip_prefix("git-env:") + .is_some_and(canonical_nonempty_hex), + "ignore" | "trailignore" | "gitignore" | "git_info_exclude" | "git_excludes_file" => false, + _ => false, + } +} + +fn canonical_nonempty_hex(value: &str) -> bool { + !value.is_empty() + && value.len() % 2 == 0 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +#[cfg(unix)] +fn policy_path_from_bytes(bytes: Vec) -> PathBuf { + use std::os::unix::ffi::OsStringExt; + PathBuf::from(std::ffi::OsString::from_vec(bytes)) +} + +#[cfg(not(unix))] +fn policy_path_from_bytes(bytes: Vec) -> PathBuf { + PathBuf::from(String::from_utf8_lossy(&bytes).into_owned()) +} + +#[cfg(unix)] +fn policy_path_identity(path: &Path) -> String { + use std::os::unix::ffi::OsStrExt; + format!("path:{}", hex::encode(path.as_os_str().as_bytes())) +} + +#[cfg(not(unix))] +fn policy_path_identity(path: &Path) -> String { + format!("path:{}", hex::encode(path.to_string_lossy().as_bytes())) +} + +fn lexical_normalize_policy_path(path: &Path) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + if !normalized.pop() { + normalized.push(component.as_os_str()); + } + } + other => normalized.push(other.as_os_str()), + } + } + normalized +} + +fn table_columns_match( + conn: &Connection, + table: &str, + expected: &[(&str, &str, bool, Option<&str>, i64)], +) -> Result { + let mut statement = conn.prepare(&format!("PRAGMA table_info({table})"))?; + let actual = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, bool>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, i64>(5)?, + )) + })? + .collect::, _>>()?; + Ok(actual.len() == expected.len() + && actual.iter().zip(expected).all( + |((name, ty, not_null, default_value, pk), expected)| { + name == expected.0 + && ty == expected.1 + && not_null == &expected.2 + && default_value.as_deref() == expected.3 + && pk == &expected.4 + }, + )) +} + +fn normalized_sql(sql: &str) -> String { + sql.split_whitespace().collect::>().join(" ") +} + +fn table_sql_contains(conn: &Connection, table: &str, fragments: &[&str]) -> Result { + let sql = conn + .query_row( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?1", + params![table], + |row| row.get::<_, String>(0), + ) + .optional()?; + let Some(sql) = sql else { + return Ok(false); + }; + let sql = normalized_sql(&sql); + Ok(fragments + .iter() + .all(|fragment| sql.contains(&normalized_sql(fragment)))) +} + +fn index_key_columns(conn: &Connection, index: &str) -> Result>> { + let mut statement = + conn.prepare("SELECT name, coll FROM pragma_index_xinfo(?1) WHERE key = 1 ORDER BY seqno")?; + let columns = statement + .query_map(params![index], |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + )) + })? + .collect::, _>>() + .map_err(Error::from)?; + let columns = columns + .into_iter() + .map(|(name, collation)| Some((name?, collation?))) + .collect::>>(); + Ok(columns) +} + +fn origin_index_matches( + conn: &Connection, + table: &str, + origin: &str, + expected_columns: &[&str], +) -> Result { + let mut statement = conn.prepare( + "SELECT name FROM pragma_index_list(?1) + WHERE origin = ?2 AND [unique] = 1 AND partial = 0", + )?; + let names = statement + .query_map(params![table, origin], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + if names.len() != 1 { + return Ok(false); + } + for name in names { + let Some(columns) = index_key_columns(conn, &name)? else { + return Ok(false); + }; + if columns.len() == expected_columns.len() + && columns + .iter() + .zip(expected_columns) + .all(|((actual, collation), expected)| actual == expected && collation == "BINARY") + { + return Ok(true); + } + } + Ok(false) +} + +fn origin_index_count(conn: &Connection, table: &str, origin: &str) -> Result { + conn.query_row( + "SELECT COUNT(*) FROM pragma_index_list(?1) WHERE origin = ?2", + params![table, origin], + |row| row.get::<_, i64>(0), + ) + .map_err(Error::from) +} + +fn origin_indexes_match( + conn: &Connection, + table: &str, + origin: &str, + expected: &[&[&str]], +) -> Result { + let mut statement = conn.prepare( + "SELECT name FROM pragma_index_list(?1) + WHERE origin=?2 AND [unique]=1 AND partial=0", + )?; + let names = statement + .query_map(params![table, origin], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + let mut actual = names + .iter() + .map(|name| { + Ok(index_key_columns(conn, name)? + .unwrap_or_default() + .into_iter() + .map(|(column, _)| column) + .collect::>()) + }) + .collect::>>()?; + actual.sort(); + let mut expected = expected + .iter() + .map(|columns| { + columns + .iter() + .map(|column| (*column).to_string()) + .collect::>() + }) + .collect::>(); + expected.sort(); + Ok(actual == expected) +} + +fn named_index_matches( + conn: &Connection, + table: &str, + index: &str, + expected_columns: &[&str], +) -> Result { + let metadata = conn + .query_row( + "SELECT [unique], origin, partial FROM pragma_index_list(?1) WHERE name = ?2", + params![table, index], + |row| { + Ok(( + row.get::<_, bool>(0)?, + row.get::<_, String>(1)?, + row.get::<_, bool>(2)?, + )) + }, + ) + .optional()?; + if metadata != Some((false, "c".to_string(), false)) { + return Ok(false); + } + let Some(columns) = index_key_columns(conn, index)? else { + return Ok(false); + }; + Ok(columns.len() == expected_columns.len() + && columns + .iter() + .zip(expected_columns) + .all(|((actual, collation), expected)| actual == expected && collation == "BINARY")) +} + +fn named_unique_partial_index_matches( + conn: &Connection, + table: &str, + index: &str, + expected_columns: &[&str], +) -> Result { + let metadata = conn + .query_row( + "SELECT [unique],origin,partial FROM pragma_index_list(?1) WHERE name=?2", + params![table, index], + |row| { + Ok(( + row.get::<_, bool>(0)?, + row.get::<_, String>(1)?, + row.get::<_, bool>(2)?, + )) + }, + ) + .optional()?; + if metadata != Some((true, "c".into(), true)) { + return Ok(false); + } + let Some(columns) = index_key_columns(conn, index)? else { + return Ok(false); + }; + Ok(columns.len() == expected_columns.len() + && columns + .iter() + .zip(expected_columns) + .all(|((actual, collation), expected)| actual == expected && collation == "BINARY")) +} + +fn foreign_keys_match( + conn: &Connection, + table: &str, + expected: &[(&str, &str, &str, &str, &str)], +) -> Result { + let mut statement = conn.prepare(&format!("PRAGMA foreign_key_list({table})"))?; + let actual = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(3)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(4)?, + row.get::<_, String>(5)?, + row.get::<_, String>(6)?, + row.get::<_, String>(7)?, + )) + })? + .collect::, _>>()?; + let Some(mut actual) = actual + .into_iter() + .map(|(from, table, to, on_update, on_delete, match_kind)| { + Some((from, table, to?, on_update, on_delete, match_kind)) + }) + .collect::>>() + else { + return Ok(false); + }; + actual.sort(); + let mut expected = expected + .iter() + .map(|(from, target, to, on_update, on_delete)| { + ( + (*from).to_string(), + (*target).to_string(), + (*to).to_string(), + (*on_update).to_string(), + (*on_delete).to_string(), + "NONE".to_string(), + ) + }) + .collect::>(); + expected.sort(); + Ok(actual == expected) +} diff --git a/trail/src/db/storage/schema/ddl.rs b/trail/src/db/storage/schema/ddl.rs index 28372b5..c79997a 100644 --- a/trail/src/db/storage/schema/ddl.rs +++ b/trail/src/db/storage/schema/ddl.rs @@ -1,138 +1,443 @@ use super::*; -impl Trail { - pub(crate) fn init_schema(&self) -> Result<()> { - let user_version = self.schema_user_version()?; - if user_version > TRAIL_SCHEMA_VERSION { - return Err(Error::InvalidInput(format!( - "Trail schema version {user_version} is newer than supported version {TRAIL_SCHEMA_VERSION}; upgrade this binary before opening the workspace" - ))); - } - self.conn.execute_batch( - "\ - CREATE TABLE IF NOT EXISTS schema_meta ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL, - updated_at INTEGER NOT NULL +pub(super) const BASE_SCHEMA_V18: &str = r#" +CREATE TABLE agent_attestation_key_revocations ( + key_id TEXT PRIMARY KEY, + public_key_hex TEXT NOT NULL, + reason TEXT NOT NULL, + revoked_at INTEGER NOT NULL, + metadata_json TEXT + ); +CREATE TABLE agent_capture_runs ( + capture_run_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + lane_id TEXT, + workdir TEXT NOT NULL, + canonical_workdir TEXT NOT NULL, + owner_agent TEXT NOT NULL, + owner_session_id TEXT NOT NULL, + executor_agent TEXT, + work_item_id TEXT, + status TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + ended_at INTEGER, + metadata_json TEXT + ); +CREATE TABLE agent_hook_installations ( + installation_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + provider TEXT NOT NULL, + scope TEXT NOT NULL, + config_path TEXT NOT NULL, + lane_id TEXT, + manifest_digest TEXT NOT NULL, + manifest_signature_json TEXT, + ownership_inventory_json TEXT NOT NULL, + config_before_digest TEXT, + config_after_digest TEXT NOT NULL, + adapter_version TEXT NOT NULL, + provider_version_range TEXT, + detected_provider_version TEXT, + capability_status TEXT NOT NULL, + status TEXT NOT NULL, + installed_at INTEGER NOT NULL, + verified_at INTEGER, + last_receipt_at INTEGER, + metadata_json TEXT + ); +CREATE TABLE agent_hook_receipts ( + receipt_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + installation_id TEXT REFERENCES agent_hook_installations(installation_id), + mapping_id TEXT REFERENCES lane_agent_sessions(mapping_id), + provider TEXT NOT NULL, + native_event TEXT NOT NULL, + native_session_id TEXT, + native_turn_id TEXT, + transport TEXT NOT NULL, + dedupe_key TEXT NOT NULL, + payload_digest TEXT NOT NULL, + raw_object_id TEXT NOT NULL REFERENCES objects(object_id), + raw_artifact_id TEXT REFERENCES lane_artifacts(artifact_id), + receive_sequence INTEGER, + connection_id TEXT, + direction TEXT, + connection_sequence INTEGER, + status TEXT NOT NULL, + attempt_count INTEGER NOT NULL DEFAULT 0, + next_attempt_at INTEGER, + diagnostic TEXT, + occurred_at INTEGER, + received_at INTEGER NOT NULL, + processed_at INTEGER, + updated_at INTEGER NOT NULL, + UNIQUE(workspace_id, provider, dedupe_key), + UNIQUE(mapping_id, receive_sequence) + ); +CREATE TABLE anchors ( + anchor_id TEXT PRIMARY KEY, + label TEXT NOT NULL, + file_id TEXT NOT NULL, + line_id TEXT NOT NULL, + object_id TEXT NOT NULL, + created_path TEXT NOT NULL, + created_line INTEGER NOT NULL, + created_change TEXT NOT NULL, + created_at INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS objects ( - object_id TEXT PRIMARY KEY, - kind TEXT NOT NULL, - version INTEGER NOT NULL, - codec TEXT NOT NULL, - hash_alg TEXT NOT NULL, - size_bytes INTEGER NOT NULL, - bytes BLOB NOT NULL, +CREATE TABLE conflict_resolution_suggestions ( + suggestion_id TEXT PRIMARY KEY, + signature TEXT NOT NULL, + path TEXT NOT NULL, + conflict_class TEXT NOT NULL, + resolution TEXT NOT NULL, + conflict_set_id TEXT NOT NULL, + operation TEXT NOT NULL, + source_ref TEXT, + target_ref TEXT, created_at INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS refs ( - name TEXT PRIMARY KEY, - change_id TEXT NOT NULL, - root_id TEXT NOT NULL, - operation_id TEXT NOT NULL, - generation INTEGER NOT NULL, - updated_at INTEGER NOT NULL +CREATE TABLE conflict_sets ( + conflict_set_id TEXT PRIMARY KEY, + merge_id TEXT, + source_ref TEXT, + target_ref TEXT, + status TEXT NOT NULL, + details_json TEXT, + created_at INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS operations ( - change_id TEXT PRIMARY KEY, - operation_id TEXT NOT NULL, +CREATE TABLE environment_cache_namespaces ( + namespace_id TEXT PRIMARY KEY, + adapter_identity TEXT NOT NULL, + cache_name TEXT NOT NULL, + protocol TEXT NOT NULL, + access TEXT NOT NULL, + authority TEXT NOT NULL DEFAULT 'performance_only', + scope TEXT NOT NULL DEFAULT 'workspace', + compatibility_json BLOB NOT NULL, + storage_path TEXT NOT NULL, + last_used_at INTEGER NOT NULL, + created_at INTEGER NOT NULL + ); +CREATE TABLE environment_component_bindings ( + view_id TEXT NOT NULL, + component_id TEXT NOT NULL, + mount_path TEXT NOT NULL, kind TEXT NOT NULL, - branch TEXT NOT NULL, - before_root TEXT, - after_root TEXT NOT NULL, - actor_kind TEXT NOT NULL, - actor_id TEXT NOT NULL, - session_id TEXT, - message TEXT, - path_count INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (view_id, component_id), + UNIQUE (view_id, mount_path) + ); +CREATE TABLE environment_component_caches ( + view_id TEXT NOT NULL, + component_id TEXT NOT NULL, + cache_name TEXT NOT NULL, + namespace_id TEXT NOT NULL, + protocol TEXT NOT NULL, + access TEXT NOT NULL, + compatibility_json BLOB NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (view_id, component_id, cache_name) + ); +CREATE TABLE environment_component_dependencies ( + view_id TEXT NOT NULL, + component_id TEXT NOT NULL, + dependency_component_id TEXT NOT NULL, + dependency_component_key TEXT NOT NULL DEFAULT '', + edge_type TEXT NOT NULL DEFAULT 'build_requires', + updated_at INTEGER NOT NULL, + PRIMARY KEY (view_id, component_id, dependency_component_id) + ); +CREATE TABLE environment_component_external_artifacts ( + view_id TEXT NOT NULL, + component_id TEXT NOT NULL, + artifact_name TEXT NOT NULL, + artifact_type TEXT NOT NULL, + provider TEXT NOT NULL, + reference TEXT NOT NULL, + digest TEXT NOT NULL, + platform TEXT NOT NULL, + cleanup_owner TEXT NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (view_id, component_id, artifact_name) + ); +CREATE TABLE environment_component_key_provenance ( + component_key TEXT PRIMARY KEY, + canonical_key_json BLOB NOT NULL, created_at INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS operations_branch_created_idx ON operations(branch, created_at); - CREATE INDEX IF NOT EXISTS operations_session_created_idx ON operations(session_id, created_at); - CREATE TABLE IF NOT EXISTS operation_parents ( - change_id TEXT NOT NULL, - parent_change_id TEXT NOT NULL, - position INTEGER NOT NULL, - PRIMARY KEY (change_id, position) +CREATE TABLE environment_component_output_bindings ( + view_id TEXT NOT NULL, + component_id TEXT NOT NULL, + output_name TEXT NOT NULL, + mount_path TEXT NOT NULL, + layer_subpath TEXT NOT NULL DEFAULT '', + policy TEXT NOT NULL DEFAULT 'immutable_seed_private', + binding_identity TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (view_id, component_id, output_name), + UNIQUE (view_id, mount_path) ); - CREATE TABLE IF NOT EXISTS file_history ( - file_id TEXT NOT NULL, - change_id TEXT NOT NULL, - path TEXT NOT NULL, - old_path TEXT, +CREATE TABLE environment_component_runtime_resources ( + view_id TEXT NOT NULL, + component_id TEXT NOT NULL, + resource_name TEXT NOT NULL, + runtime_type TEXT NOT NULL, + provider TEXT NOT NULL, + artifact_name TEXT NOT NULL, + container_port INTEGER NOT NULL, + protocol TEXT NOT NULL, + health_type TEXT NOT NULL, + health_timeout_ms INTEGER NOT NULL, + restart_policy TEXT NOT NULL, + cleanup_owner TEXT NOT NULL, + volume_target TEXT, + updated_at INTEGER NOT NULL, + PRIMARY KEY (view_id, component_id, resource_name) + ); +CREATE TABLE environment_component_runtime_secrets ( + view_id TEXT NOT NULL, + component_id TEXT NOT NULL, + resource_name TEXT NOT NULL, + secret_name TEXT NOT NULL, + provider TEXT NOT NULL, + reference TEXT NOT NULL, + version TEXT, + purpose TEXT NOT NULL, + injection TEXT NOT NULL, + target TEXT NOT NULL, + environment TEXT, + required INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (view_id, component_id, resource_name, secret_name), + UNIQUE (view_id, component_id, resource_name, target) + ); +CREATE TABLE environment_component_states ( + view_id TEXT NOT NULL, + component_id TEXT NOT NULL, + adapter_identity TEXT NOT NULL, + adapter_version INTEGER NOT NULL, + implementation_version TEXT NOT NULL, + distribution_digest TEXT, kind TEXT NOT NULL, - before_hash TEXT, - after_hash TEXT, - created_at INTEGER NOT NULL + expected_key TEXT NOT NULL, + attached_key TEXT, + status TEXT NOT NULL, + reason TEXT, + updated_at INTEGER NOT NULL, + PRIMARY KEY (view_id, component_id) ); - CREATE INDEX IF NOT EXISTS file_history_file_idx ON file_history(file_id, created_at); - CREATE INDEX IF NOT EXISTS file_history_path_idx ON file_history(path, created_at); - CREATE TABLE IF NOT EXISTS line_history ( - line_id TEXT NOT NULL, - file_id TEXT NOT NULL, - change_id TEXT NOT NULL, - path TEXT NOT NULL, - line_number INTEGER, +CREATE TABLE environment_generation_caches ( + generation_id TEXT NOT NULL, + component_id TEXT NOT NULL, + cache_name TEXT NOT NULL, + namespace_id TEXT NOT NULL, + protocol TEXT NOT NULL, + access TEXT NOT NULL, + compatibility_json BLOB NOT NULL, + PRIMARY KEY (generation_id, component_id, cache_name) + ); +CREATE TABLE environment_generation_components ( + generation_id TEXT NOT NULL, + component_id TEXT NOT NULL, + adapter_identity TEXT NOT NULL, kind TEXT NOT NULL, - text_hash TEXT, + component_key TEXT NOT NULL, + layer_id TEXT, + mount_path TEXT, + PRIMARY KEY (generation_id, component_id) + ); +CREATE TABLE environment_generation_edges ( + generation_id TEXT NOT NULL, + component_id TEXT NOT NULL, + dependency_component_id TEXT NOT NULL, + dependency_component_key TEXT NOT NULL, + edge_type TEXT NOT NULL DEFAULT 'build_requires', + PRIMARY KEY (generation_id, component_id, dependency_component_id) + ); +CREATE TABLE environment_generation_external_artifacts ( + generation_id TEXT NOT NULL, + component_id TEXT NOT NULL, + artifact_name TEXT NOT NULL, + artifact_type TEXT NOT NULL, + provider TEXT NOT NULL, + reference TEXT NOT NULL, + digest TEXT NOT NULL, + platform TEXT NOT NULL, + cleanup_owner TEXT NOT NULL, + PRIMARY KEY (generation_id, component_id, artifact_name) + ); +CREATE TABLE environment_generation_outputs ( + generation_id TEXT NOT NULL, + component_id TEXT NOT NULL, + output_name TEXT NOT NULL, + policy TEXT NOT NULL DEFAULT 'immutable_seed_private', + storage_identity TEXT NOT NULL, + layer_id TEXT, + mount_path TEXT NOT NULL, + layer_subpath TEXT NOT NULL DEFAULT '', + PRIMARY KEY (generation_id, component_id, output_name), + UNIQUE (generation_id, mount_path) + ); +CREATE TABLE environment_generation_runtime_resources ( + generation_id TEXT NOT NULL, + component_id TEXT NOT NULL, + resource_name TEXT NOT NULL, + runtime_type TEXT NOT NULL, + provider TEXT NOT NULL, + artifact_name TEXT NOT NULL, + image_reference TEXT NOT NULL, + image_digest TEXT NOT NULL, + image_platform TEXT NOT NULL, + container_port INTEGER NOT NULL, + protocol TEXT NOT NULL, + health_type TEXT NOT NULL, + health_timeout_ms INTEGER NOT NULL, + restart_policy TEXT NOT NULL, + cleanup_owner TEXT NOT NULL, + volume_target TEXT, + allocation_id TEXT NOT NULL, + provider_resource_id TEXT, + container_name TEXT NOT NULL, + network_name TEXT NOT NULL, + volume_name TEXT, + host_port INTEGER, + status TEXT NOT NULL, + health_status TEXT NOT NULL, + reason TEXT, + cleanup_token TEXT NOT NULL, + owner_pid INTEGER, + owner_start_token TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + started_at INTEGER, + stopped_at INTEGER, + PRIMARY KEY (generation_id, component_id, resource_name), + UNIQUE (allocation_id), + UNIQUE (container_name) + ); +CREATE TABLE environment_generation_runtime_secrets ( + generation_id TEXT NOT NULL, + component_id TEXT NOT NULL, + resource_name TEXT NOT NULL, + secret_name TEXT NOT NULL, + provider TEXT NOT NULL, + reference TEXT NOT NULL, + version TEXT, + purpose TEXT NOT NULL, + injection TEXT NOT NULL, + target TEXT NOT NULL, + environment TEXT, + required INTEGER NOT NULL, + status TEXT NOT NULL, + reason TEXT, + resolved_at INTEGER, + updated_at INTEGER NOT NULL, + PRIMARY KEY (generation_id, component_id, resource_name, secret_name), + UNIQUE (generation_id, component_id, resource_name, target) + ); +CREATE TABLE environment_generations ( + generation_id TEXT PRIMARY KEY, + view_id TEXT NOT NULL, + generation_sequence INTEGER NOT NULL, + source_root TEXT NOT NULL, + specification_digest TEXT NOT NULL, + predecessor_generation_id TEXT, + state TEXT NOT NULL, + created_at INTEGER NOT NULL, + activated_at INTEGER, + retired_at INTEGER, + UNIQUE (view_id, generation_sequence) + ); +CREATE TABLE environment_secret_access_audit ( + access_id TEXT PRIMARY KEY, + generation_id TEXT NOT NULL, + component_id TEXT NOT NULL, + resource_name TEXT NOT NULL, + secret_name TEXT NOT NULL, + provider TEXT NOT NULL, + purpose TEXT NOT NULL, + status TEXT NOT NULL, created_at INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS line_history_line_idx ON line_history(line_id, created_at); - CREATE TABLE IF NOT EXISTS messages ( - message_id TEXT PRIMARY KEY, - role TEXT NOT NULL, - body TEXT NOT NULL, +CREATE TABLE environment_sync_attempts ( + attempt_id TEXT PRIMARY KEY, + view_id TEXT NOT NULL, + source_root TEXT NOT NULL, + mode TEXT NOT NULL, + owner_pid INTEGER NOT NULL, + owner_start_token TEXT NOT NULL, + status TEXT NOT NULL, + reason TEXT, + started_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + finished_at INTEGER + ); +CREATE TABLE environment_view_generations ( + view_id TEXT PRIMARY KEY, + generation_id TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); +CREATE TABLE external_mutation_audit ( + audit_id TEXT PRIMARY KEY, + actor TEXT NOT NULL DEFAULT 'unknown', + surface TEXT NOT NULL, + command TEXT NOT NULL, + target_ref TEXT, lane_id TEXT, - session_id TEXT, + status TEXT NOT NULL, + status_code INTEGER, change_id TEXT, - object_id TEXT NOT NULL, + summary_json TEXT, created_at INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS anchors ( - anchor_id TEXT PRIMARY KEY, - label TEXT NOT NULL, +CREATE TABLE file_history ( file_id TEXT NOT NULL, - line_id TEXT NOT NULL, - object_id TEXT NOT NULL, - created_path TEXT NOT NULL, - created_line INTEGER NOT NULL, - created_change TEXT NOT NULL, + change_id TEXT NOT NULL, + path TEXT NOT NULL, + old_path TEXT, + kind TEXT NOT NULL, + before_hash TEXT, + after_hash TEXT, created_at INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS anchors_file_idx ON anchors(file_id, created_at); - CREATE INDEX IF NOT EXISTS anchors_line_idx ON anchors(line_id, created_at); - CREATE TABLE IF NOT EXISTS lanes ( - lane_id TEXT PRIMARY KEY, - name TEXT NOT NULL UNIQUE, - kind TEXT, - provider TEXT, - model TEXT, - created_at INTEGER NOT NULL, - metadata_json TEXT +CREATE TABLE git_agent_links ( + git_agent_link_id TEXT PRIMARY KEY, + git_commit TEXT NOT NULL, + lane_id TEXT NOT NULL REFERENCES lanes(lane_id), + session_id TEXT NOT NULL REFERENCES lane_sessions(session_id), + turn_id TEXT REFERENCES lane_turns(turn_id), + from_change TEXT, + through_change TEXT, + confidence TEXT NOT NULL, + source TEXT NOT NULL, + created_at INTEGER NOT NULL, + metadata_json TEXT + ); +CREATE TABLE git_mappings ( + mapping_id TEXT PRIMARY KEY, + direction TEXT NOT NULL, + branch TEXT NOT NULL, + git_head TEXT, + git_dirty INTEGER NOT NULL, + crab_change TEXT NOT NULL, + crab_root TEXT NOT NULL, + created_at INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS lane_branches ( - lane_id TEXT PRIMARY KEY, - ref_name TEXT NOT NULL UNIQUE, - base_change TEXT NOT NULL, - head_change TEXT NOT NULL, - base_root TEXT NOT NULL, - head_root TEXT NOT NULL, - session_id TEXT, - workdir TEXT, - status TEXT NOT NULL, +CREATE TABLE http_idempotency_keys ( + key TEXT PRIMARY KEY, + method TEXT NOT NULL, + path TEXT NOT NULL, + request_hash TEXT NOT NULL, + status INTEGER NOT NULL, + body BLOB NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS lane_sessions ( - session_id TEXT PRIMARY KEY, - lane_id TEXT NOT NULL, - title TEXT, - status TEXT NOT NULL, - started_at INTEGER NOT NULL, - ended_at INTEGER, - metadata_json TEXT - ); - CREATE TABLE IF NOT EXISTS lane_acp_sessions ( +CREATE TABLE lane_acp_sessions ( acp_session_id TEXT PRIMARY KEY, upstream_session_id TEXT, lane_id TEXT NOT NULL, @@ -141,27 +446,98 @@ impl Trail { provider TEXT, model TEXT, upstream_command_json TEXT, + path_mappings_json TEXT NOT NULL DEFAULT '[]', + current_mode_id TEXT, + config_options_json TEXT NOT NULL DEFAULT '{}', status TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS lane_acp_sessions_lane_idx ON lane_acp_sessions(lane_id, updated_at); - CREATE INDEX IF NOT EXISTS lane_acp_sessions_trail_session_idx ON lane_acp_sessions(trail_session_id); - CREATE TABLE IF NOT EXISTS lane_turns ( - turn_id TEXT PRIMARY KEY, +CREATE TABLE lane_agent_session_aliases ( + workspace_id TEXT NOT NULL, + provider TEXT NOT NULL, + native_session_alias TEXT NOT NULL, + mapping_id TEXT NOT NULL REFERENCES lane_agent_sessions(mapping_id), + reason TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY(workspace_id, provider, native_session_alias) + ); +CREATE TABLE lane_agent_sessions ( + mapping_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + provider TEXT NOT NULL, + native_session_id TEXT NOT NULL, + parent_native_session_id TEXT, + trail_session_id TEXT NOT NULL REFERENCES lane_sessions(session_id), + lane_id TEXT NOT NULL REFERENCES lanes(lane_id), + capture_run_id TEXT REFERENCES agent_capture_runs(capture_run_id), + primary_transport TEXT NOT NULL, + transcript_identity TEXT, + transcript_offset INTEGER, + resume_json TEXT, + last_attestation_id TEXT, + status TEXT NOT NULL, + pending_turn_outcome TEXT, + session_close_requested INTEGER NOT NULL DEFAULT 0, + capture_epoch INTEGER NOT NULL DEFAULT 1, + finalization_owner TEXT, + finalization_lease_expires_at INTEGER, + next_receive_sequence INTEGER NOT NULL DEFAULT 1, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(workspace_id, provider, native_session_id) + ); +CREATE TABLE lane_approvals ( + approval_id TEXT PRIMARY KEY, lane_id TEXT NOT NULL, session_id TEXT, + turn_id TEXT, + action TEXT NOT NULL, + summary TEXT NOT NULL, + payload_json TEXT, + status TEXT NOT NULL, + requested_at INTEGER NOT NULL, + decided_at INTEGER, + reviewer TEXT, + note TEXT + ); +CREATE TABLE lane_artifacts ( + artifact_id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + lane_id TEXT NOT NULL REFERENCES lanes(lane_id), + session_id TEXT NOT NULL REFERENCES lane_sessions(session_id), + turn_id TEXT REFERENCES lane_turns(turn_id), + provider TEXT NOT NULL, + artifact_kind TEXT NOT NULL, + format TEXT NOT NULL, + source TEXT NOT NULL, + source_locator_redacted TEXT, + content_object_id TEXT, + content_digest TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + start_offset INTEGER, + end_offset INTEGER, + redaction_profile TEXT, + retention_status TEXT NOT NULL, + trust TEXT NOT NULL, + supersedes_artifact_id TEXT REFERENCES lane_artifacts(artifact_id), + created_at INTEGER NOT NULL, + metadata_json TEXT + ); +CREATE TABLE lane_branches ( + lane_id TEXT PRIMARY KEY, + ref_name TEXT NOT NULL UNIQUE, base_change TEXT NOT NULL, - before_change TEXT NOT NULL, - after_change TEXT, + head_change TEXT NOT NULL, + base_root TEXT NOT NULL, + head_root TEXT NOT NULL, + session_id TEXT, + workdir TEXT, status TEXT NOT NULL, - started_at INTEGER NOT NULL, - ended_at INTEGER, - metadata_json TEXT + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS lane_turns_session_started_idx ON lane_turns(session_id, started_at); - CREATE INDEX IF NOT EXISTS lane_turns_lane_started_idx ON lane_turns(lane_id, started_at); - CREATE TABLE IF NOT EXISTS lane_events ( +CREATE TABLE lane_events ( event_id TEXT PRIMARY KEY, lane_id TEXT NOT NULL, turn_id TEXT, @@ -172,42 +548,63 @@ impl Trail { payload_json TEXT, created_at INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS lane_events_lane_created_idx ON lane_events(lane_id, created_at); - CREATE INDEX IF NOT EXISTS lane_events_session_created_idx ON lane_events(session_id, created_at); - CREATE INDEX IF NOT EXISTS lane_events_turn_created_idx ON lane_events(turn_id, created_at); - CREATE INDEX IF NOT EXISTS lane_events_type_created_idx ON lane_events(event_type, created_at); - CREATE INDEX IF NOT EXISTS lane_events_lane_type_created_idx ON lane_events(lane_id, event_type, created_at); - CREATE INDEX IF NOT EXISTS lane_events_session_type_created_idx ON lane_events(session_id, event_type, created_at); - CREATE INDEX IF NOT EXISTS lane_events_turn_type_created_idx ON lane_events(turn_id, event_type, created_at); - CREATE TABLE IF NOT EXISTS lane_trace_span_events ( - span_id TEXT NOT NULL, - event_id TEXT PRIMARY KEY, - event_type TEXT NOT NULL, - trace_id TEXT, - lane_id TEXT NOT NULL, - session_id TEXT, - turn_id TEXT, - created_at INTEGER NOT NULL - ); - CREATE INDEX IF NOT EXISTS lane_trace_span_events_span_created_idx ON lane_trace_span_events(span_id, created_at); - CREATE INDEX IF NOT EXISTS lane_trace_span_events_trace_created_idx ON lane_trace_span_events(trace_id, created_at); - CREATE TABLE IF NOT EXISTS lane_approvals ( - approval_id TEXT PRIMARY KEY, +CREATE TABLE lane_learnings ( + learning_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL REFERENCES lanes(lane_id), + session_id TEXT NOT NULL REFERENCES lane_sessions(session_id), + turn_id TEXT REFERENCES lane_turns(turn_id), + scope TEXT NOT NULL, + body TEXT NOT NULL, + status TEXT NOT NULL, + confidence REAL, + source_artifact_id TEXT REFERENCES lane_artifacts(artifact_id), + anchor_json TEXT, + created_at INTEGER NOT NULL, + reviewed_at INTEGER, + reviewer TEXT, + expires_at INTEGER, + superseded_by TEXT REFERENCES lane_learnings(learning_id), + metadata_json TEXT + ); +CREATE TABLE lane_merge_queue ( + queue_id TEXT PRIMARY KEY, lane_id TEXT NOT NULL, - session_id TEXT, - turn_id TEXT, - action TEXT NOT NULL, - summary TEXT NOT NULL, - payload_json TEXT, + target_ref TEXT NOT NULL, status TEXT NOT NULL, - requested_at INTEGER NOT NULL, - decided_at INTEGER, - reviewer TEXT, - note TEXT + priority INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS lane_approvals_status_idx ON lane_approvals(status, requested_at); - CREATE INDEX IF NOT EXISTS lane_approvals_lane_idx ON lane_approvals(lane_id, requested_at); - CREATE TABLE IF NOT EXISTS lane_run_states ( +CREATE TABLE lane_provenance_edges ( + provenance_edge_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL REFERENCES lanes(lane_id), + session_id TEXT NOT NULL REFERENCES lane_sessions(session_id), + from_node_id TEXT NOT NULL REFERENCES lane_provenance_nodes(provenance_node_id), + to_node_id TEXT NOT NULL REFERENCES lane_provenance_nodes(provenance_node_id), + relation TEXT NOT NULL, + source_confidence TEXT NOT NULL, + receipt_id TEXT REFERENCES agent_hook_receipts(receipt_id), + created_at INTEGER NOT NULL, + attributes_json TEXT + ); +CREATE TABLE lane_provenance_nodes ( + provenance_node_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL REFERENCES lanes(lane_id), + session_id TEXT NOT NULL REFERENCES lane_sessions(session_id), + turn_id TEXT REFERENCES lane_turns(turn_id), + node_kind TEXT NOT NULL, + summary TEXT NOT NULL, + event_id TEXT, + span_id TEXT, + message_id TEXT, + change_id TEXT, + artifact_id TEXT REFERENCES lane_artifacts(artifact_id), + source_confidence TEXT NOT NULL, + classifier_version TEXT, + created_at INTEGER NOT NULL, + attributes_json TEXT + ); +CREATE TABLE lane_run_states ( run_id TEXT PRIMARY KEY, lane_id TEXT NOT NULL, session_id TEXT, @@ -224,31 +621,162 @@ impl Trail { reviewer TEXT, note TEXT ); - CREATE INDEX IF NOT EXISTS lane_run_states_lane_idx ON lane_run_states(lane_id, updated_at); - CREATE INDEX IF NOT EXISTS lane_run_states_status_idx ON lane_run_states(status, updated_at); - CREATE INDEX IF NOT EXISTS lane_run_states_approval_idx ON lane_run_states(approval_id); - CREATE TABLE IF NOT EXISTS leases ( +CREATE TABLE lane_session_attestation_turns ( + attestation_id TEXT NOT NULL REFERENCES lane_session_attestations(attestation_id), + turn_id TEXT NOT NULL REFERENCES lane_turns(turn_id), + change_id TEXT, + evidence_manifest_id TEXT NOT NULL REFERENCES lane_turn_evidence_manifests(manifest_id), + PRIMARY KEY(attestation_id, turn_id) + ); +CREATE TABLE lane_session_attestations ( + attestation_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL REFERENCES lanes(lane_id), + session_id TEXT NOT NULL REFERENCES lane_sessions(session_id), + capture_run_id TEXT REFERENCES agent_capture_runs(capture_run_id), + previous_attestation_id TEXT REFERENCES lane_session_attestations(attestation_id), + statement_object_id TEXT NOT NULL, + statement_digest TEXT NOT NULL UNIQUE, + signature_json TEXT, + status TEXT NOT NULL, + created_at INTEGER NOT NULL, + superseded_by TEXT REFERENCES lane_session_attestations(attestation_id), + metadata_json TEXT + ); +CREATE TABLE lane_sessions ( + session_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL, + title TEXT, + status TEXT NOT NULL, + started_at INTEGER NOT NULL, + ended_at INTEGER, + metadata_json TEXT + ); +CREATE TABLE lane_trace_span_events ( + span_id TEXT NOT NULL, + event_id TEXT PRIMARY KEY, + event_type TEXT NOT NULL, + trace_id TEXT, + lane_id TEXT NOT NULL, + session_id TEXT, + turn_id TEXT, + created_at INTEGER NOT NULL + ); +CREATE TABLE lane_turn_evidence_manifests ( + manifest_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL REFERENCES lanes(lane_id), + session_id TEXT NOT NULL REFERENCES lane_sessions(session_id), + turn_id TEXT NOT NULL UNIQUE REFERENCES lane_turns(turn_id), + schema_version INTEGER NOT NULL, + object_id TEXT NOT NULL, + digest TEXT NOT NULL UNIQUE, + created_at INTEGER NOT NULL + ); +CREATE TABLE lane_turns ( + turn_id TEXT PRIMARY KEY, + lane_id TEXT NOT NULL, + session_id TEXT, + base_change TEXT NOT NULL, + before_change TEXT NOT NULL, + after_change TEXT, + status TEXT NOT NULL, + started_at INTEGER NOT NULL, + ended_at INTEGER, + metadata_json TEXT + ); +CREATE TABLE lanes ( + lane_id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + kind TEXT, + provider TEXT, + model TEXT, + created_at INTEGER NOT NULL, + metadata_json TEXT + ); +CREATE TABLE leases ( lease_id TEXT PRIMARY KEY, lane_id TEXT NOT NULL, ref_name TEXT NOT NULL, path TEXT, - file_id TEXT, - mode TEXT NOT NULL, - expires_at INTEGER NOT NULL, - created_at INTEGER NOT NULL - ); - CREATE TABLE IF NOT EXISTS merge_queue ( - queue_id TEXT PRIMARY KEY, - source_ref TEXT NOT NULL, - target_ref TEXT NOT NULL, + file_id TEXT, + mode TEXT NOT NULL, + expires_at INTEGER NOT NULL, + created_at INTEGER NOT NULL + ); +CREATE TABLE line_history ( + line_id TEXT NOT NULL, + file_id TEXT NOT NULL, + change_id TEXT NOT NULL, + path TEXT NOT NULL, + line_number INTEGER, + kind TEXT NOT NULL, + text_hash TEXT, + created_at INTEGER NOT NULL + ); +CREATE TABLE memory_embedding_indexes ( + index_id TEXT PRIMARY KEY, + backend TEXT NOT NULL, + provider TEXT NOT NULL, + model TEXT NOT NULL, + dims INTEGER NOT NULL, + table_name TEXT NOT NULL UNIQUE, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(backend, provider, model, dims) + ); +CREATE TABLE memory_embeddings ( + memory_id TEXT PRIMARY KEY REFERENCES memory_items(memory_id) ON DELETE CASCADE, + memory_ord INTEGER NOT NULL UNIQUE, + provider TEXT NOT NULL, + model TEXT NOT NULL, + dims INTEGER NOT NULL, + embedding BLOB NOT NULL, + embedding_hash TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); +CREATE TABLE memory_items ( + memory_ord INTEGER PRIMARY KEY AUTOINCREMENT, + memory_id TEXT NOT NULL UNIQUE, + scope_type TEXT NOT NULL, + scope_id TEXT NOT NULL, + kind TEXT NOT NULL, + path TEXT, + title TEXT, + body TEXT NOT NULL, + status TEXT NOT NULL, + source_ref TEXT, + source_change TEXT, + source_root TEXT, + metadata_json TEXT NOT NULL, + created_by TEXT NOT NULL, + updated_by TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + archived_at INTEGER + ); +CREATE TABLE memory_revisions ( + revision_id TEXT PRIMARY KEY, + memory_id TEXT NOT NULL, + version INTEGER NOT NULL, + operation TEXT NOT NULL, + scope_type TEXT NOT NULL, + scope_id TEXT NOT NULL, + kind TEXT NOT NULL, + path TEXT, + title TEXT, + body TEXT NOT NULL, status TEXT NOT NULL, - priority INTEGER NOT NULL DEFAULT 0, + source_ref TEXT, + source_change TEXT, + source_root TEXT, + metadata_json TEXT NOT NULL, + embedding_hash TEXT, + actor_id TEXT NOT NULL, created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL + UNIQUE(memory_id, version) ); - CREATE TABLE IF NOT EXISTS merge_results ( +CREATE TABLE merge_results ( merge_id TEXT PRIMARY KEY, - queue_id TEXT, + lane_queue_id TEXT, source_ref TEXT NOT NULL, target_ref TEXT NOT NULL, base_change TEXT NOT NULL, @@ -262,104 +790,89 @@ impl Trail { conflict_set TEXT, created_at INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS conflict_sets ( - conflict_set_id TEXT PRIMARY KEY, - merge_id TEXT, - source_ref TEXT, - target_ref TEXT, - status TEXT NOT NULL, - details_json TEXT, +CREATE TABLE messages ( + message_id TEXT PRIMARY KEY, + role TEXT NOT NULL, + body TEXT NOT NULL, + lane_id TEXT, + session_id TEXT, + change_id TEXT, + object_id TEXT NOT NULL, created_at INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS conflict_resolution_suggestions ( - suggestion_id TEXT PRIMARY KEY, - signature TEXT NOT NULL, - path TEXT NOT NULL, - conflict_class TEXT NOT NULL, - resolution TEXT NOT NULL, - conflict_set_id TEXT NOT NULL, - operation TEXT NOT NULL, - source_ref TEXT, - target_ref TEXT, +CREATE TABLE objects ( + object_id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + version INTEGER NOT NULL, + codec TEXT NOT NULL, + hash_alg TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + bytes BLOB NOT NULL, created_at INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS conflict_resolution_suggestions_signature_idx ON conflict_resolution_suggestions(signature, created_at); - CREATE TABLE IF NOT EXISTS external_mutation_audit ( - audit_id TEXT PRIMARY KEY, - actor TEXT NOT NULL DEFAULT 'unknown', - surface TEXT NOT NULL, - command TEXT NOT NULL, - target_ref TEXT, - lane_id TEXT, - status TEXT NOT NULL, - status_code INTEGER, - change_id TEXT, - summary_json TEXT, +CREATE TABLE operation_parents ( + change_id TEXT NOT NULL, + parent_change_id TEXT NOT NULL, + position INTEGER NOT NULL, + PRIMARY KEY (change_id, position) + ); +CREATE TABLE operations ( + change_id TEXT PRIMARY KEY, + operation_id TEXT NOT NULL, + kind TEXT NOT NULL, + branch TEXT NOT NULL, + before_root TEXT, + after_root TEXT NOT NULL, + actor_kind TEXT NOT NULL, + actor_id TEXT NOT NULL, + session_id TEXT, + message TEXT, + path_count INTEGER NOT NULL, created_at INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS external_mutation_audit_created_idx ON external_mutation_audit(created_at); - CREATE INDEX IF NOT EXISTS external_mutation_audit_surface_created_idx ON external_mutation_audit(surface, created_at); - CREATE INDEX IF NOT EXISTS external_mutation_audit_lane_created_idx ON external_mutation_audit(lane_id, created_at); - CREATE TABLE IF NOT EXISTS http_idempotency_keys ( - key TEXT PRIMARY KEY, - method TEXT NOT NULL, - path TEXT NOT NULL, - request_hash TEXT NOT NULL, - status INTEGER NOT NULL, - body BLOB NOT NULL, +CREATE TABLE pending_path_index_derived_repairs ( + ref_name TEXT NOT NULL, + repair_kind TEXT NOT NULL CHECK (repair_kind IN ('lane_manifest', 'workspace_checkpoint')), + old_root TEXT NOT NULL, + new_root TEXT NOT NULL, + new_change TEXT NOT NULL, created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL + PRIMARY KEY (ref_name, repair_kind) ); - CREATE INDEX IF NOT EXISTS http_idempotency_keys_updated_idx ON http_idempotency_keys(updated_at); - CREATE TABLE IF NOT EXISTS git_mappings ( - mapping_id TEXT PRIMARY KEY, - direction TEXT NOT NULL, - branch TEXT NOT NULL, - git_head TEXT, - git_dirty INTEGER NOT NULL, - crab_change TEXT NOT NULL, - crab_root TEXT NOT NULL, - created_at INTEGER NOT NULL +CREATE TABLE refs ( + name TEXT PRIMARY KEY, + change_id TEXT NOT NULL, + root_id TEXT NOT NULL, + operation_id TEXT NOT NULL, + generation INTEGER NOT NULL, + updated_at INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS git_mappings_change_idx ON git_mappings(crab_change); - CREATE INDEX IF NOT EXISTS git_mappings_head_idx ON git_mappings(git_head); - CREATE TABLE IF NOT EXISTS worktree_file_index ( - path TEXT PRIMARY KEY, - size_bytes INTEGER NOT NULL, - modified_ns INTEGER NOT NULL, - changed_ns INTEGER NOT NULL, - device_id INTEGER NOT NULL DEFAULT 0, - inode INTEGER NOT NULL DEFAULT 0, - executable INTEGER NOT NULL, - kind TEXT NOT NULL, - content_hash TEXT NOT NULL, - last_seen_scan INTEGER NOT NULL DEFAULT 0, +CREATE TABLE schema_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, updated_at INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS workspace_views ( +CREATE TABLE workspace_environment_states ( + view_id TEXT NOT NULL, + adapter TEXT NOT NULL, + expected_key TEXT NOT NULL, + attached_key TEXT, + status TEXT NOT NULL, + reason TEXT, + updated_at INTEGER NOT NULL, + PRIMARY KEY (view_id, adapter) + ); +CREATE TABLE workspace_git_shadows ( view_id TEXT PRIMARY KEY, - lane_id TEXT NOT NULL UNIQUE, - base_change TEXT NOT NULL, - base_root TEXT NOT NULL, - backend TEXT NOT NULL, - mountpoint TEXT NOT NULL, - source_upper TEXT NOT NULL, - generated_upper TEXT NOT NULL, - scratch_upper TEXT NOT NULL, - meta_dir TEXT NOT NULL, - journal_path TEXT NOT NULL, - generation INTEGER NOT NULL, - checkpoint_seq INTEGER NOT NULL, - checkpoint_root TEXT, + git_dir TEXT NOT NULL, + policy TEXT NOT NULL, + pinned_head TEXT NOT NULL, + current_head TEXT NOT NULL, status TEXT NOT NULL, - owner_pid INTEGER, - owner_start_token TEXT, - heartbeat_at INTEGER, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS workspace_views_status_idx ON workspace_views(status, updated_at); - CREATE TABLE IF NOT EXISTS workspace_layers ( +CREATE TABLE workspace_layers ( layer_id TEXT PRIMARY KEY, kind TEXT NOT NULL, cache_key TEXT NOT NULL UNIQUE, @@ -377,137 +890,356 @@ impl Trail { last_used_at INTEGER NOT NULL, created_at INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS workspace_layers_state_used_idx ON workspace_layers(state, last_used_at); - CREATE TABLE IF NOT EXISTS workspace_view_layers ( +CREATE TABLE workspace_view_layers ( view_id TEXT NOT NULL, layer_id TEXT NOT NULL, mount_path TEXT NOT NULL, priority INTEGER NOT NULL, read_only INTEGER NOT NULL, + source_path TEXT NOT NULL DEFAULT '', PRIMARY KEY (view_id, mount_path, priority) ); - CREATE INDEX IF NOT EXISTS workspace_view_layers_layer_idx ON workspace_view_layers(layer_id); - CREATE TABLE IF NOT EXISTS workspace_environment_states ( - view_id TEXT NOT NULL, - adapter TEXT NOT NULL, - expected_key TEXT NOT NULL, - attached_key TEXT, - status TEXT NOT NULL, - reason TEXT, - updated_at INTEGER NOT NULL, - PRIMARY KEY (view_id, adapter) - ); - CREATE TABLE IF NOT EXISTS workspace_git_shadows ( +CREATE TABLE workspace_views ( view_id TEXT PRIMARY KEY, - git_dir TEXT NOT NULL, - policy TEXT NOT NULL, - pinned_head TEXT NOT NULL, - current_head TEXT NOT NULL, + lane_id TEXT NOT NULL UNIQUE, + base_change TEXT NOT NULL, + base_root TEXT NOT NULL, + backend TEXT NOT NULL, + mountpoint TEXT NOT NULL, + source_upper TEXT NOT NULL, + generated_upper TEXT NOT NULL, + scratch_upper TEXT NOT NULL, + meta_dir TEXT NOT NULL, + journal_path TEXT NOT NULL, + generation INTEGER NOT NULL, + checkpoint_seq INTEGER NOT NULL, + checkpoint_root TEXT, status TEXT NOT NULL, + owner_pid INTEGER, + owner_start_token TEXT, + heartbeat_at INTEGER, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); - CREATE TABLE IF NOT EXISTS memory_items ( - memory_ord INTEGER PRIMARY KEY AUTOINCREMENT, - memory_id TEXT NOT NULL UNIQUE, - scope_type TEXT NOT NULL, - scope_id TEXT NOT NULL, +CREATE TABLE worktree_file_index ( + path TEXT PRIMARY KEY, + size_bytes INTEGER NOT NULL, + modified_ns INTEGER NOT NULL, + changed_ns INTEGER NOT NULL, + device_id INTEGER NOT NULL DEFAULT 0, + inode INTEGER NOT NULL DEFAULT 0, + executable INTEGER NOT NULL, kind TEXT NOT NULL, - path TEXT, - title TEXT, - body TEXT NOT NULL, - status TEXT NOT NULL, - source_ref TEXT, - source_change TEXT, - source_root TEXT, - metadata_json TEXT NOT NULL, - created_by TEXT NOT NULL, - updated_by TEXT NOT NULL, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - archived_at INTEGER - ); - CREATE INDEX IF NOT EXISTS memory_items_scope_idx ON memory_items(scope_type, scope_id, status, updated_at); - CREATE INDEX IF NOT EXISTS memory_items_kind_idx ON memory_items(kind, status, updated_at); - CREATE INDEX IF NOT EXISTS memory_items_path_idx ON memory_items(path, status, updated_at); - CREATE INDEX IF NOT EXISTS memory_items_source_change_idx ON memory_items(source_change, updated_at); - CREATE TABLE IF NOT EXISTS memory_embeddings ( - memory_id TEXT PRIMARY KEY REFERENCES memory_items(memory_id) ON DELETE CASCADE, - memory_ord INTEGER NOT NULL UNIQUE, - provider TEXT NOT NULL, - model TEXT NOT NULL, - dims INTEGER NOT NULL, - embedding BLOB NOT NULL, - embedding_hash TEXT NOT NULL, + content_hash TEXT NOT NULL, + last_seen_scan INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS memory_embeddings_model_idx ON memory_embeddings(provider, model, dims); - CREATE TABLE IF NOT EXISTS memory_embedding_indexes ( - index_id TEXT PRIMARY KEY, - backend TEXT NOT NULL, - provider TEXT NOT NULL, - model TEXT NOT NULL, - dims INTEGER NOT NULL, - table_name TEXT NOT NULL UNIQUE, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - UNIQUE(backend, provider, model, dims) - ); - CREATE TABLE IF NOT EXISTS memory_revisions ( - revision_id TEXT PRIMARY KEY, - memory_id TEXT NOT NULL, - version INTEGER NOT NULL, - operation TEXT NOT NULL, - scope_type TEXT NOT NULL, - scope_id TEXT NOT NULL, - kind TEXT NOT NULL, - path TEXT, - title TEXT, - body TEXT NOT NULL, - status TEXT NOT NULL, - source_ref TEXT, - source_change TEXT, - source_root TEXT, - metadata_json TEXT NOT NULL, - embedding_hash TEXT, - actor_id TEXT NOT NULL, - created_at INTEGER NOT NULL, - UNIQUE(memory_id, version) - ); - CREATE INDEX IF NOT EXISTS memory_revisions_memory_idx ON memory_revisions(memory_id, version); - CREATE INDEX IF NOT EXISTS memory_revisions_source_change_idx ON memory_revisions(source_change, created_at); - ", - )?; - ensure_column(&self.conn, "conflict_sets", "details_json", "TEXT")?; - ensure_column(&self.conn, "merge_results", "base_root", "TEXT")?; - ensure_column(&self.conn, "merge_results", "left_root", "TEXT")?; - ensure_column(&self.conn, "merge_results", "right_root", "TEXT")?; - ensure_column( - &self.conn, - "external_mutation_audit", - "actor", - "TEXT NOT NULL DEFAULT 'unknown'", - )?; - ensure_column(&self.conn, "lane_events", "session_id", "TEXT")?; - ensure_column( - &self.conn, - "worktree_file_index", - "last_seen_scan", - "INTEGER NOT NULL DEFAULT 0", - )?; - ensure_column( - &self.conn, - "worktree_file_index", - "device_id", - "INTEGER NOT NULL DEFAULT 0", - )?; - ensure_column( - &self.conn, - "worktree_file_index", - "inode", - "INTEGER NOT NULL DEFAULT 0", - )?; - self.record_schema_version()?; - Ok(()) +CREATE INDEX agent_attestation_key_revocations_time_idx + ON agent_attestation_key_revocations(revoked_at, key_id); +CREATE INDEX agent_capture_runs_active_workdir_idx + ON agent_capture_runs(workspace_id, canonical_workdir, expires_at) + WHERE status = 'active'; +CREATE INDEX agent_capture_runs_owner_idx + ON agent_capture_runs(owner_agent, owner_session_id, updated_at); +CREATE INDEX agent_hook_installations_status_idx + ON agent_hook_installations(provider, status, verified_at); +CREATE UNIQUE INDEX agent_hook_installations_target_idx + ON agent_hook_installations(workspace_id, provider, scope, config_path); +CREATE INDEX agent_hook_receipts_replay_idx + ON agent_hook_receipts(status, next_attempt_at, received_at); +CREATE INDEX agent_hook_receipts_session_idx + ON agent_hook_receipts(workspace_id, provider, native_session_id, received_at); +CREATE INDEX agent_hook_receipts_turn_idx + ON agent_hook_receipts(native_turn_id, received_at); +CREATE UNIQUE INDEX agent_hook_receipts_connection_sequence_idx + ON agent_hook_receipts(connection_id, direction, connection_sequence) + WHERE connection_id IS NOT NULL + AND direction IS NOT NULL + AND connection_sequence IS NOT NULL; +CREATE INDEX anchors_file_idx ON anchors(file_id, created_at); +CREATE INDEX anchors_line_idx ON anchors(line_id, created_at); +CREATE INDEX conflict_resolution_suggestions_signature_idx ON conflict_resolution_suggestions(signature, created_at); +CREATE INDEX environment_cache_namespaces_lru_idx + ON environment_cache_namespaces(last_used_at, namespace_id); +CREATE INDEX environment_component_dependencies_dependency_idx + ON environment_component_dependencies(view_id, dependency_component_id, component_id); +CREATE INDEX environment_component_states_adapter_idx + ON environment_component_states(adapter_identity, status, updated_at); +CREATE INDEX environment_generation_caches_namespace_idx + ON environment_generation_caches(namespace_id, generation_id); +CREATE INDEX environment_generation_components_layer_idx + ON environment_generation_components(layer_id); +CREATE INDEX environment_generation_edges_dependency_idx + ON environment_generation_edges(generation_id, dependency_component_id, component_id); +CREATE INDEX environment_generation_external_artifacts_digest_idx + ON environment_generation_external_artifacts(provider, digest, generation_id); +CREATE INDEX environment_generation_outputs_layer_idx + ON environment_generation_outputs(layer_id); +CREATE INDEX environment_generation_runtime_resources_status_idx + ON environment_generation_runtime_resources(status, updated_at, generation_id); +CREATE INDEX environment_generation_runtime_secrets_status_idx + ON environment_generation_runtime_secrets(status, updated_at, generation_id); +CREATE INDEX environment_generations_view_state_idx + ON environment_generations(view_id, state, generation_sequence); +CREATE INDEX environment_secret_access_audit_generation_idx + ON environment_secret_access_audit(generation_id, created_at); +CREATE UNIQUE INDEX environment_sync_attempts_running_view_idx + ON environment_sync_attempts(view_id) WHERE status = 'running'; +CREATE INDEX environment_sync_attempts_status_idx + ON environment_sync_attempts(status, updated_at); +CREATE INDEX external_mutation_audit_created_idx ON external_mutation_audit(created_at); +CREATE INDEX external_mutation_audit_lane_created_idx ON external_mutation_audit(lane_id, created_at); +CREATE INDEX external_mutation_audit_surface_created_idx ON external_mutation_audit(surface, created_at); +CREATE INDEX file_history_file_idx ON file_history(file_id, created_at); +CREATE INDEX file_history_path_idx ON file_history(path, created_at); +CREATE UNIQUE INDEX git_agent_links_identity_idx + ON git_agent_links(git_commit, session_id, COALESCE(turn_id, ''), source); +CREATE INDEX git_agent_links_session_idx + ON git_agent_links(session_id, created_at); +CREATE INDEX git_agent_links_turn_idx + ON git_agent_links(turn_id, created_at); +CREATE INDEX git_mappings_change_idx ON git_mappings(crab_change); +CREATE INDEX git_mappings_head_idx ON git_mappings(git_head); +CREATE INDEX http_idempotency_keys_updated_idx ON http_idempotency_keys(updated_at); +CREATE INDEX lane_acp_sessions_lane_idx ON lane_acp_sessions(lane_id, updated_at); +CREATE INDEX lane_acp_sessions_trail_session_idx ON lane_acp_sessions(trail_session_id); +CREATE INDEX lane_agent_session_aliases_mapping_idx + ON lane_agent_session_aliases(mapping_id); +CREATE INDEX lane_agent_sessions_finalization_idx + ON lane_agent_sessions(status, finalization_lease_expires_at) + WHERE status = 'finalizing'; +CREATE INDEX lane_agent_sessions_lane_idx + ON lane_agent_sessions(lane_id, updated_at); +CREATE INDEX lane_agent_sessions_run_idx + ON lane_agent_sessions(capture_run_id, updated_at); +CREATE INDEX lane_agent_sessions_trail_session_idx + ON lane_agent_sessions(trail_session_id, updated_at); +CREATE INDEX lane_approvals_lane_idx ON lane_approvals(lane_id, requested_at); +CREATE INDEX lane_approvals_status_idx ON lane_approvals(status, requested_at); +CREATE INDEX lane_artifacts_digest_idx + ON lane_artifacts(content_digest, artifact_kind); +CREATE INDEX lane_artifacts_session_idx + ON lane_artifacts(session_id, created_at, artifact_id); +CREATE INDEX lane_artifacts_turn_idx + ON lane_artifacts(turn_id, created_at, artifact_id); +CREATE INDEX lane_events_lane_created_idx ON lane_events(lane_id, created_at); +CREATE INDEX lane_events_lane_type_created_idx ON lane_events(lane_id, event_type, created_at); +CREATE INDEX lane_events_session_created_idx ON lane_events(session_id, created_at); +CREATE INDEX lane_events_session_type_created_idx ON lane_events(session_id, event_type, created_at); +CREATE INDEX lane_events_turn_created_idx ON lane_events(turn_id, created_at); +CREATE INDEX lane_events_turn_type_created_idx ON lane_events(turn_id, event_type, created_at); +CREATE INDEX lane_events_type_created_idx ON lane_events(event_type, created_at); +CREATE INDEX lane_learnings_scope_idx + ON lane_learnings(lane_id, scope, status, created_at); +CREATE INDEX lane_learnings_session_idx + ON lane_learnings(session_id, turn_id, created_at); +CREATE INDEX lane_merge_queue_active_idx + ON lane_merge_queue(lane_id, target_ref, status); +CREATE INDEX lane_merge_queue_run_idx + ON lane_merge_queue(status, priority DESC, created_at ASC); +CREATE INDEX lane_provenance_edges_from_idx + ON lane_provenance_edges(from_node_id, relation); +CREATE UNIQUE INDEX lane_provenance_edges_identity_idx + ON lane_provenance_edges( + from_node_id, to_node_id, relation, COALESCE(receipt_id, '') + ); +CREATE INDEX lane_provenance_edges_to_idx + ON lane_provenance_edges(to_node_id, relation); +CREATE INDEX lane_provenance_nodes_change_idx + ON lane_provenance_nodes(change_id, created_at); +CREATE INDEX lane_provenance_nodes_session_idx + ON lane_provenance_nodes(session_id, turn_id, node_kind, created_at); +CREATE INDEX lane_run_states_approval_idx ON lane_run_states(approval_id); +CREATE INDEX lane_run_states_lane_idx ON lane_run_states(lane_id, updated_at); +CREATE INDEX lane_run_states_status_idx ON lane_run_states(status, updated_at); +CREATE INDEX lane_session_attestations_session_idx + ON lane_session_attestations(session_id, created_at); +CREATE INDEX lane_trace_span_events_span_created_idx ON lane_trace_span_events(span_id, created_at); +CREATE INDEX lane_trace_span_events_trace_created_idx ON lane_trace_span_events(trace_id, created_at); +CREATE INDEX lane_turn_evidence_manifests_session_idx + ON lane_turn_evidence_manifests(session_id, created_at); +CREATE INDEX lane_turns_lane_started_idx ON lane_turns(lane_id, started_at); +CREATE INDEX lane_turns_session_started_idx ON lane_turns(session_id, started_at); +CREATE INDEX line_history_line_idx ON line_history(line_id, created_at); +CREATE INDEX memory_embeddings_model_idx ON memory_embeddings(provider, model, dims); +CREATE INDEX memory_items_kind_idx ON memory_items(kind, status, updated_at); +CREATE INDEX memory_items_path_idx ON memory_items(path, status, updated_at); +CREATE INDEX memory_items_scope_idx ON memory_items(scope_type, scope_id, status, updated_at); +CREATE INDEX memory_items_source_change_idx ON memory_items(source_change, updated_at); +CREATE INDEX memory_revisions_memory_idx ON memory_revisions(memory_id, version); +CREATE INDEX memory_revisions_source_change_idx ON memory_revisions(source_change, created_at); +CREATE INDEX operations_branch_created_idx ON operations(branch, created_at); +CREATE INDEX operations_session_created_idx ON operations(session_id, created_at); +CREATE INDEX pending_path_index_derived_repairs_root_idx + ON pending_path_index_derived_repairs(new_root); +CREATE INDEX workspace_layers_state_used_idx ON workspace_layers(state, last_used_at); +CREATE INDEX workspace_view_layers_layer_idx ON workspace_view_layers(layer_id); +CREATE INDEX workspace_views_status_idx ON workspace_views(status, updated_at); +"#; + +impl Trail { + pub(crate) fn create_schema_v18(&self) -> Result<()> { + create_schema_v18(&self.conn) + } +} + +pub(crate) fn create_schema_v18(conn: &Connection) -> Result<()> { + if conn.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0))? != 0 { + return Err(Error::Corrupt( + "fresh schema connection is not empty".into(), + )); + } + conn.execute_batch("SAVEPOINT create_v18;")?; + let result = (|| { + conn.execute_batch(BASE_SCHEMA_V18)?; + super::changed_path_ledger::create_changed_path_ledger_schema(conn)?; + validate_schema_v18_shape(conn)?; + let now = now_ts(); + for (key, value) in [ + (SCHEMA_META_VERSION_KEY, TRAIL_SCHEMA_VERSION.to_string()), + ( + SCHEMA_META_APP_VERSION_KEY, + env!("CARGO_PKG_VERSION").to_string(), + ), + ("changed_path.observer_log_format_min", "1".to_string()), + ("changed_path.observer_log_format_max", "1".to_string()), + ] { + conn.execute( + "INSERT INTO schema_meta(key, value, updated_at) VALUES(?1, ?2, ?3)", + params![key, value, now], + )?; + } + conn.pragma_update(None, "user_version", TRAIL_SCHEMA_VERSION)?; + Trail::validate_schema_v18(conn) + })(); + match result { + Ok(()) => conn + .execute_batch("RELEASE create_v18;") + .map_err(Into::into), + Err(err) => { + let _ = conn.execute_batch("ROLLBACK TO create_v18; RELEASE create_v18;"); + Err(err) + } + } +} + +pub(super) fn validate_schema_v18_shape(conn: &Connection) -> Result<()> { + let expected = Connection::open_in_memory()?; + expected.pragma_update(None, "foreign_keys", true)?; + expected.execute_batch(BASE_SCHEMA_V18)?; + if schema_objects(conn)? != schema_objects(&expected)? { + return Err(Error::Corrupt( + "base schema v18 sqlite_master shape does not match".into(), + )); + } + Ok(()) +} + +pub(super) fn base_schema_v18_complete(conn: &Connection) -> Result { + if validate_schema_v18_shape(conn).is_err() { + return Ok(false); + } + let mut statement = conn.prepare( + "SELECT key, value FROM schema_meta + WHERE key IN ( + 'schema.version', + 'changed_path.observer_log_format_min', + 'changed_path.observer_log_format_max' + ) ORDER BY key", + )?; + let metadata = statement + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()?; + Ok(metadata + == vec![ + ( + "changed_path.observer_log_format_max".to_string(), + "1".to_string(), + ), + ( + "changed_path.observer_log_format_min".to_string(), + "1".to_string(), + ), + ( + SCHEMA_META_VERSION_KEY.to_string(), + TRAIL_SCHEMA_VERSION.to_string(), + ), + ] + && conn + .query_row( + "SELECT length(value) > 0 FROM schema_meta WHERE key = 'app.version'", + [], + |row| row.get::<_, bool>(0), + ) + .optional()? + == Some(true)) +} + +fn schema_objects(conn: &Connection) -> Result> { + let mut statement = conn.prepare( + "SELECT type, name, COALESCE(sql, '') FROM sqlite_master + WHERE name NOT LIKE 'sqlite_%' + AND name NOT LIKE 'prolly_%' + AND name NOT LIKE 'changed_path_%' + ORDER BY type, name", + )?; + let objects = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + normalize_sql(&row.get::<_, String>(2)?), + )) + })? + .collect::, _>>() + .map_err(Error::from)?; + Ok(objects) +} + +fn normalize_sql(sql: &str) -> String { + sql.split_whitespace().collect::>().join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn master_objects(conn: &Connection) -> Vec<(String, String, Option)> { + conn.prepare( + "SELECT type, name, sql FROM sqlite_master + WHERE name NOT LIKE 'sqlite_%' + ORDER BY type, name", + ) + .unwrap() + .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?))) + .unwrap() + .collect::, _>>() + .unwrap() + } + + #[test] + fn late_ledger_ddl_conflict_rolls_back_entire_fresh_creation() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE changed_path_observer_owners ( + sentinel TEXT NOT NULL + );", + ) + .unwrap(); + let before = master_objects(&conn); + let before_user_version: i64 = conn + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .unwrap(); + + assert!(create_schema_v18(&conn).is_err()); + + assert_eq!(master_objects(&conn), before); + assert_eq!( + conn.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0)) + .unwrap(), + before_user_version + ); } } diff --git a/trail/src/db/storage/schema/version.rs b/trail/src/db/storage/schema/version.rs index f7f5dd7..e34f542 100644 --- a/trail/src/db/storage/schema/version.rs +++ b/trail/src/db/storage/schema/version.rs @@ -7,30 +7,6 @@ impl Trail { .map_err(Error::from) } - pub(crate) fn set_schema_user_version(&self, version: i64) -> Result<()> { - self.conn - .execute_batch(&format!("PRAGMA user_version = {version};"))?; - Ok(()) - } - - pub(crate) fn record_schema_version(&self) -> Result<()> { - self.set_schema_user_version(TRAIL_SCHEMA_VERSION)?; - let now = now_ts(); - for (key, value) in [ - (SCHEMA_META_VERSION_KEY, TRAIL_SCHEMA_VERSION.to_string()), - ( - SCHEMA_META_APP_VERSION_KEY, - env!("CARGO_PKG_VERSION").to_string(), - ), - ] { - self.conn.execute( - "INSERT OR REPLACE INTO schema_meta (key, value, updated_at) VALUES (?1, ?2, ?3)", - params![key, value, now], - )?; - } - Ok(()) - } - pub(crate) fn schema_meta_value(&self, key: &str) -> Result> { self.conn .query_row( diff --git a/trail/src/db/storage/worktree_index.rs b/trail/src/db/storage/worktree_index.rs index 601b67a..520d7ad 100644 --- a/trail/src/db/storage/worktree_index.rs +++ b/trail/src/db/storage/worktree_index.rs @@ -1,15 +1,65 @@ use super::*; +use crate::db::change_ledger::CompiledRecordingMatcher; +#[cfg(test)] +use crate::db::change_ledger::PolicyInvalidationIndex; +use crate::db::change_ledger::{raw_path_may_invalidate_policy, CompiledPolicy}; use notify::event::{CreateKind, ModifyKind, RemoveKind, RenameMode}; use notify::{ Config as NotifyConfig, Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher, }; use rayon::prelude::*; +use rusqlite::{Params, Statement, StatementStatus}; +use sha2::{Digest, Sha256}; +use walkdir::WalkDir; const DAEMON_STATUS_DIRTY_PATH_LIMIT: usize = 16_384; const WORKTREE_INDEX_STAMP_LOOKUP_CHUNK: usize = 512; const WORKTREE_INDEX_BASELINE_ROOT_KEY: &str = "worktree.index.baseline_root"; -const DAEMON_WORKTREE_SNAPSHOT_FILE: &str = "worktree-daemon-cache.json"; -const DAEMON_WORKTREE_SNAPSHOT_VERSION: u32 = 1; +const SELECT_WORKTREE_INDEX_EXACT_SQL: &str = + "SELECT path FROM worktree_file_index WHERE path COLLATE BINARY = ?1"; +const SELECT_WORKTREE_INDEX_DESCENDANTS_SQL: &str = "SELECT path FROM worktree_file_index \ + WHERE path COLLATE BINARY >= ?1 AND path COLLATE BINARY < ?2 \ + ORDER BY path COLLATE BINARY"; +const DELETE_WORKTREE_INDEX_PATH_SQL: &str = + "DELETE FROM worktree_file_index WHERE path COLLATE BINARY = ?1"; +const UPSERT_WORKTREE_INDEX_PATH_SQL: &str = + "INSERT OR REPLACE INTO worktree_file_index \ + (path, size_bytes, modified_ns, changed_ns, device_id, inode, executable, kind, content_hash, last_seen_scan, updated_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + +const RECONCILE_READ_BUFFER_BYTES: usize = 64 * 1024; + +pub(crate) struct PinnedWorktreeRoot { + path: PathBuf, + descriptor: fs::File, + identity: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ReconciliationFile { + pub(crate) path: String, + pub(crate) file_kind: String, + pub(crate) content_hash: String, + pub(crate) executable: bool, + pub(crate) size_bytes: u64, + pub(crate) identity: Vec, + pub(crate) peak_buffer_bytes: u64, + /// Present only for command candidate materialization. Full reconciliation + /// remains streaming and never retains complete file contents. + pub(crate) bytes: Option>, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ReconciliationDirectory { + pub(crate) path: String, + pub(crate) identity: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ReconciliationScanEntry { + Directory(ReconciliationDirectory), + File(ReconciliationFile), +} #[derive(Debug)] struct WorktreeIndexReadCandidate { @@ -31,27 +81,325 @@ pub(crate) struct WorktreeIndexRefresh { pub(crate) changed: bool, } -#[derive(Debug, serde::Serialize, serde::Deserialize)] -struct PersistedDaemonWorktreeSnapshot { - version: u32, - pid: u32, - workspace_root: String, - generation: u64, - initialized: bool, - overflow: bool, - baseline_root_id: Option, - dirty_paths: Vec, - updated_ns: i64, -} - impl Trail { + pub(crate) fn open_pinned_worktree_root( + &self, + policy: &CompiledPolicy, + ) -> Result { + // A compiled policy is rooted at the exact authoritative scope it was + // compiled for. That is normally the primary workspace, but qualified + // materialized lanes use their own pinned workdir root. Never silently + // substitute Trail's primary workspace here: doing so would compare a + // lane's evidence against the wrong filesystem tree. + let scope_root = policy.workspace_root(); + let descriptor = open_absolute_directory_no_follow(scope_root)?; + let identity = root_descriptor_identity(&descriptor)?; + Ok(PinnedWorktreeRoot { + path: scope_root.to_path_buf(), + descriptor, + identity, + }) + } + + pub(crate) fn pinned_worktree_root_identity(&self, root: &PinnedWorktreeRoot) -> Vec { + root.identity.clone() + } + + pub(crate) fn verify_pinned_worktree_root(&self, root: &PinnedWorktreeRoot) -> Result { + let current = open_absolute_directory_no_follow(&root.path)?; + Ok(root_descriptor_identity(¤t)? == root.identity + && root_descriptor_identity(&root.descriptor)? == root.identity) + } + + pub(crate) fn pinned_worktree_identity_for_path(&self, path: &Path) -> Result> { + let descriptor = open_absolute_directory_no_follow(path)?; + root_descriptor_identity(&descriptor) + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn pinned_worktree_path_is_visible( + &self, + root: &PinnedWorktreeRoot, + path: &str, + ) -> Result { + let path = normalize_relative_path(path)?; + if reconcile_path_ignored(&path) { + return Ok(false); + } + if read_reconciliation_file_no_follow(&root.descriptor, &path, &self.config.text, false)? + .is_some() + { + return Ok(true); + } + Ok(open_relative_directory_no_follow(&root.descriptor, &path)?.is_some()) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn pinned_worktree_path_is_visible( + &self, + _root: &PinnedWorktreeRoot, + _path: &str, + ) -> Result { + Err(Error::InvalidInput( + "pinned changed-path visibility requires Linux or macOS".into(), + )) + } + + pub(crate) fn visit_pinned_worktree_files( + &self, + root: &PinnedWorktreeRoot, + _policy: &CompiledPolicy, + prefixes: &[String], + mut visitor: F, + ) -> Result<()> + where + F: FnMut(ReconciliationScanEntry) -> Result<()>, + { + let mut walker = WalkDir::new(&root.path).follow_links(false).into_iter(); + while let Some(item) = walker.next() { + let entry = item.map_err(|err| Error::InvalidInput(err.to_string()))?; + if entry.path() == root.path { + continue; + } + let relative = entry + .path() + .strip_prefix(&root.path) + .map_err(|err| Error::InvalidInput(err.to_string()))?; + let relative = relative.to_str().ok_or_else(|| { + Error::InvalidInput("reconciliation does not support non-UTF-8 paths".into()) + })?; + let relative = normalize_relative_path(relative)?; + let is_dir = entry.file_type().is_dir(); + if !path_intersects_reconcile_scope(&relative, is_dir, prefixes) + || reconcile_path_ignored(&relative) + { + if is_dir { + walker.skip_current_dir(); + } + continue; + } + if is_dir { + let directory = + read_reconciliation_directory_no_follow(&root.descriptor, &relative)? + .ok_or_else(|| { + Error::InvalidInput(format!( + "directory identity changed during reconciliation: `{relative}`" + )) + })?; + visitor(ReconciliationScanEntry::Directory(directory))?; + continue; + } + if !entry.file_type().is_file() { + continue; + } + if let Some(file) = read_reconciliation_file_no_follow( + &root.descriptor, + &relative, + &self.config.text, + false, + )? { + visitor(ReconciliationScanEntry::File(file))?; + } + } + Ok(()) + } + + pub(crate) fn verify_pinned_worktree_directory( + &self, + root: &PinnedWorktreeRoot, + path: &str, + expected_identity: &[u8], + ) -> Result { + Ok( + read_reconciliation_directory_no_follow(&root.descriptor, path)? + .is_some_and(|directory| directory.identity == expected_identity), + ) + } + + pub(crate) fn read_pinned_worktree_path( + &self, + root: &PinnedWorktreeRoot, + _policy: &CompiledPolicy, + path: &str, + ) -> Result> { + let path = normalize_relative_path(path)?; + if reconcile_path_ignored(&path) { + return Ok(None); + } + read_reconciliation_file_no_follow(&root.descriptor, &path, &self.config.text, false) + } + + pub(crate) fn read_pinned_candidate_path( + &self, + root: &PinnedWorktreeRoot, + _policy: &CompiledPolicy, + path: &str, + retain_bytes: bool, + ) -> Result> { + let path = normalize_relative_path(path)?; + if reconcile_path_ignored(&path) { + return Ok(None); + } + read_reconciliation_file_no_follow(&root.descriptor, &path, &self.config.text, retain_bytes) + } + + /// Walk only the selected complete prefixes from descriptor-relative, + /// no-follow handles. Unlike `visit_pinned_worktree_files`, this never + /// starts a `WalkDir` at the workspace root and is therefore O(k+affected + /// subtree), not O(N), on the authoritative command path. + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub(crate) fn visit_pinned_worktree_prefix_files( + &self, + root: &PinnedWorktreeRoot, + matcher: &CompiledRecordingMatcher, + prefixes: &[String], + retain_bytes: bool, + mut visitor: F, + ) -> Result<()> + where + F: FnMut(ReconciliationFile) -> Result<()>, + { + use rustix::fs::{openat, statat, AtFlags, Dir, FileType, Mode, OFlags}; + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + let prefixes = minimal_component_selections(prefixes); + for prefix in prefixes { + if matcher.is_ignored(&prefix, false)? { + continue; + } + if let Some(file) = read_reconciliation_file_no_follow( + &root.descriptor, + &prefix, + &self.config.text, + retain_bytes, + )? { + visitor(file)?; + continue; + } + let Some(prefix_dir) = open_relative_directory_no_follow(&root.descriptor, &prefix)? + else { + continue; + }; + let mut pending = vec![(prefix_dir, prefix.clone())]; + while let Some((directory, relative_dir)) = pending.pop() { + let mut entries = + Dir::read_from(&directory).map_err(|error| Error::Io(error.into()))?; + while let Some(entry) = entries.next() { + let entry = entry.map_err(|error| Error::Io(error.into()))?; + let name_bytes = entry.file_name().to_bytes(); + if matches!(name_bytes, b"." | b"..") { + continue; + } + let name = std::str::from_utf8(name_bytes).map_err(|_| { + Error::InvalidInput( + "authoritative candidate walk does not support non-UTF-8 paths".into(), + ) + })?; + let relative = format!("{relative_dir}/{name}"); + let stat = match statat( + &directory, + OsStr::from_bytes(name_bytes), + AtFlags::SYMLINK_NOFOLLOW, + ) { + Ok(stat) => stat, + Err(error) if error == rustix::io::Errno::NOENT => continue, + Err(error) => return Err(Error::Io(error.into())), + }; + match FileType::from_raw_mode(stat.st_mode) { + FileType::Directory => { + if matcher.is_ignored(&relative, true)? { + continue; + } + let child = openat( + &directory, + OsStr::from_bytes(name_bytes), + OFlags::RDONLY + | OFlags::DIRECTORY + | OFlags::NOFOLLOW + | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(|error| Error::Io(error.into()))?; + pending.push((fs::File::from(child), relative)); + } + FileType::RegularFile => { + if matcher.is_ignored(&relative, false)? { + continue; + } + if let Some(file) = read_reconciliation_file_no_follow( + &root.descriptor, + &relative, + &self.config.text, + retain_bytes, + )? { + visitor(file)?; + } + } + _ => {} + } + } + } + } + Ok(()) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + pub(crate) fn visit_pinned_worktree_prefix_files( + &self, + _root: &PinnedWorktreeRoot, + _matcher: &CompiledRecordingMatcher, + _prefixes: &[String], + _retain_bytes: bool, + _visitor: F, + ) -> Result<()> + where + F: FnMut(ReconciliationFile) -> Result<()>, + { + Err(Error::InvalidInput( + "authoritative changed-path candidates require Linux or macOS".into(), + )) + } + + /// Returns true only when a clean baseline identifies the same immutable + /// visible file state as `target_root_id`. Path-invariant indexes and + /// creator metadata are deliberately excluded because they do not change + /// materialized bytes, paths, modes, or file identities. + pub(crate) fn clean_baseline_matches_visible_root( + &self, + baseline_root_id: Option<&ObjectId>, + target_root_id: &ObjectId, + ) -> bool { + let Some(baseline_root_id) = baseline_root_id else { + return false; + }; + if baseline_root_id == target_root_id { + return true; + } + let Ok(baseline) = self.get_object::(WORKTREE_ROOT_KIND, baseline_root_id) + else { + return false; + }; + let Ok(target) = self.get_object::(WORKTREE_ROOT_KIND, target_root_id) else { + return false; + }; + baseline.path_map_root == target.path_map_root + && baseline.file_index_map_root == target.file_index_map_root + && baseline.file_count == target.file_count + && baseline.total_text_bytes == target.total_text_bytes + } + pub fn enable_daemon_worktree_cache(&mut self) -> Result<()> { let warmup = self.start_daemon_worktree_cache()?; warmup.run() } pub fn start_daemon_worktree_cache(&mut self) -> Result { - let cache = DaemonWorktreeCache::start(&self.workspace_root, &self.db_dir)?; + let cache = DaemonWorktreeCache::start( + &self.workspace_root, + &self.db_dir, + self.operation_metrics.clone(), + )?; let warmup = DaemonWorktreeCacheWarmup { workspace_root: self.workspace_root.clone(), db_dir: self.db_dir.clone(), @@ -83,55 +431,37 @@ impl Trail { } pub(crate) fn daemon_worktree_snapshot(&self) -> Option { - if let Some(snapshot) = self + let snapshot = self .daemon_worktree_cache .as_ref() - .map(DaemonWorktreeCache::snapshot) - { - return Some(snapshot); + .map(DaemonWorktreeCache::snapshot); + if let Some(snapshot) = &snapshot { + let (input_path_count, canonical_path_count) = match snapshot { + DaemonWorktreeSnapshot::Dirty { paths, .. } => { + let input_path_count = saturating_u64_from_usize(paths.len()); + let canonical_path_count = SelectionSet::from_paths(paths) + .map(|selections| saturating_u64_from_usize(selections.as_slice().len())) + .unwrap_or(0); + (input_path_count, canonical_path_count) + } + DaemonWorktreeSnapshot::Clean { .. } | DaemonWorktreeSnapshot::Overflow { .. } => { + (0, 0) + } + }; + self.note_operation_metrics(OperationMetricsDelta { + input_path_count, + canonical_path_count, + daemon_snapshot_path_count: input_path_count, + ..OperationMetricsDelta::default() + }); } - self.persisted_daemon_worktree_snapshot().ok().flatten() + snapshot } pub(crate) fn daemon_dirty_path_limit(&self) -> usize { DAEMON_STATUS_DIRTY_PATH_LIMIT } - fn persisted_daemon_worktree_snapshot(&self) -> Result> { - let path = daemon_worktree_snapshot_path(&self.db_dir); - let bytes = match fs::read(&path) { - Ok(bytes) => bytes, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(err) => return Err(Error::Io(err)), - }; - let snapshot: PersistedDaemonWorktreeSnapshot = serde_json::from_slice(&bytes)?; - if snapshot.version != DAEMON_WORKTREE_SNAPSHOT_VERSION - || snapshot.workspace_root != self.workspace_root.to_string_lossy() - || !snapshot.initialized - { - return Ok(None); - } - if !process_is_alive(snapshot.pid) { - let _ = fs::remove_file(path); - return Ok(None); - } - if snapshot.overflow { - return Ok(Some(DaemonWorktreeSnapshot::Overflow { - generation: snapshot.generation, - })); - } - if snapshot.dirty_paths.is_empty() { - return Ok(Some(DaemonWorktreeSnapshot::Clean { - generation: snapshot.generation, - root_id: snapshot.baseline_root_id.map(ObjectId), - })); - } - Ok(Some(DaemonWorktreeSnapshot::Dirty { - generation: snapshot.generation, - paths: snapshot.dirty_paths, - })) - } - pub(crate) fn reconcile_daemon_status_paths( &self, root_id: &ObjectId, @@ -185,6 +515,13 @@ impl Trail { &self, scan_id: i64, ) -> Result { + let mut filesystem_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta { + full_filesystem_walk_count: 1, + ..OperationMetricsDelta::default() + }, + ); let indexed_entries = self.worktree_index_count()?; let root = self.workspace_root.canonicalize()?; let mut builder = WalkBuilder::new(&root); @@ -195,6 +532,7 @@ impl Trail { .git_global(self.config.recording.ignore_gitignored) .add_custom_ignore_filename(".trailignore"); + note_walkbuilder_policy_build(self.operation_metrics.as_ref()); let walker = builder.build(); let mut count = 0u64; let mut indexed_seen = 0u64; @@ -209,6 +547,10 @@ impl Trail { if path == root { continue; } + filesystem_metrics.delta.filesystem_entry_count = filesystem_metrics + .delta + .filesystem_entry_count + .saturating_add(1); let rel = path .strip_prefix(&root) .map_err(|err| Error::InvalidInput(err.to_string()))?; @@ -223,6 +565,10 @@ impl Trail { continue; } + filesystem_metrics.delta.filesystem_stat_count = filesystem_metrics + .delta + .filesystem_stat_count + .saturating_add(1); let metadata = fs::symlink_metadata(path)?; let stamp = WorktreeFileStamp::from_metadata(&metadata); if let Some(cached_stamp) = cached_worktree_file_stamp(&mut cached_stmt, &rel)? { @@ -244,7 +590,11 @@ impl Trail { let has_deleted_index_entries = indexed_seen < indexed_entries; let changed = !read_candidates.is_empty() || has_deleted_index_entries; - let updates = read_worktree_index_candidates(&read_candidates, &self.config.text)?; + let updates = read_worktree_index_candidates( + &read_candidates, + &self.config.text, + self.operation_metrics.as_ref(), + )?; for update in updates { self.upsert_worktree_index_manifest_for_scan( &update.path, @@ -269,6 +619,13 @@ impl Trail { pub(crate) fn scan_worktree_manifest_indexed_with_stamps( &self, ) -> Result> { + let mut filesystem_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta { + full_filesystem_walk_count: 1, + ..OperationMetricsDelta::default() + }, + ); let root = self.workspace_root.canonicalize()?; let mut builder = WalkBuilder::new(&root); builder @@ -278,6 +635,7 @@ impl Trail { .git_global(self.config.recording.ignore_gitignored) .add_custom_ignore_filename(".trailignore"); + note_walkbuilder_policy_build(self.operation_metrics.as_ref()); let walker = builder.build(); let mut manifest = BTreeMap::new(); let mut seen = BTreeSet::new(); @@ -287,6 +645,10 @@ impl Trail { if path == root { continue; } + filesystem_metrics.delta.filesystem_entry_count = filesystem_metrics + .delta + .filesystem_entry_count + .saturating_add(1); let rel = path .strip_prefix(&root) .map_err(|err| Error::InvalidInput(err.to_string()))?; @@ -301,12 +663,33 @@ impl Trail { continue; } + filesystem_metrics.delta.filesystem_stat_count = filesystem_metrics + .delta + .filesystem_stat_count + .saturating_add(1); let metadata = fs::symlink_metadata(path)?; let stamp = WorktreeFileStamp::from_metadata(&metadata); let disk_manifest = if let Some(cached) = self.cached_worktree_manifest(&rel, stamp)? { cached } else { + filesystem_metrics.delta.filesystem_read_count = filesystem_metrics + .delta + .filesystem_read_count + .saturating_add(1); let bytes = fs::read(path)?; + let bytes_len = saturating_u64_from_usize(bytes.len()); + filesystem_metrics.delta.filesystem_read_bytes = filesystem_metrics + .delta + .filesystem_read_bytes + .saturating_add(bytes_len); + filesystem_metrics.delta.filesystem_hash_count = filesystem_metrics + .delta + .filesystem_hash_count + .saturating_add(1); + filesystem_metrics.delta.filesystem_hash_bytes = filesystem_metrics + .delta + .filesystem_hash_bytes + .saturating_add(bytes_len); let disk_manifest = DiskManifest { kind: classify_file_kind(&bytes, &self.config.text), executable: stamp.executable, @@ -333,17 +716,28 @@ impl Trail { files: &BTreeMap, ) -> Result>> { let indexed_manifest = self.scan_worktree_manifest_indexed_with_stamps()?; - let manifest = indexed_manifest - .iter() - .map(|(path, indexed)| (path.clone(), indexed.manifest.clone())) + // A native source only needs every immutable-root file to be present + // and exact. Unrelated untracked files are never cloned and therefore + // do not make an otherwise complete source unsafe. + let manifest = files + .keys() + .filter_map(|path| { + indexed_manifest + .get(path) + .map(|indexed| (path.clone(), indexed.manifest.clone())) + }) .collect::>(); if !self.diff_file_maps_to_manifest(files, &manifest).is_empty() { return Ok(None); } Ok(Some( - indexed_manifest - .into_iter() - .map(|(path, indexed)| (path, indexed.stamp)) + files + .keys() + .filter_map(|path| { + indexed_manifest + .get(path) + .map(|indexed| (path.clone(), indexed.stamp)) + }) .collect(), )) } @@ -353,7 +747,8 @@ impl Trail { root_id: &ObjectId, files: &BTreeMap, ) -> Result>> { - if self.worktree_index_baseline_root()?.as_ref() != Some(root_id) { + let baseline = self.worktree_index_baseline_root()?; + if !self.clean_baseline_matches_visible_root(baseline.as_ref(), root_id) { return Ok(None); } if files.is_empty() { @@ -439,6 +834,13 @@ impl Trail { } fn scan_visible_worktree_paths(&self) -> Result> { + let mut filesystem_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta { + full_filesystem_walk_count: 1, + ..OperationMetricsDelta::default() + }, + ); let root = self.workspace_root.canonicalize()?; let mut builder = WalkBuilder::new(&root); builder @@ -448,6 +850,7 @@ impl Trail { .git_global(self.config.recording.ignore_gitignored) .add_custom_ignore_filename(".trailignore"); + note_walkbuilder_policy_build(self.operation_metrics.as_ref()); let walker = builder.build(); let mut paths = BTreeSet::new(); for item in walker { @@ -456,6 +859,10 @@ impl Trail { if path == root { continue; } + filesystem_metrics.delta.filesystem_entry_count = filesystem_metrics + .delta + .filesystem_entry_count + .saturating_add(1); let rel = path .strip_prefix(&root) .map_err(|err| Error::InvalidInput(err.to_string()))?; @@ -552,6 +959,171 @@ impl Trail { Ok(()) } + /// Synchronize only the selected portion of the worktree cache. The + /// metrics emitted here are complete for this SQL envelope, not for every + /// SQLite statement issued by the containing Trail operation. + pub(crate) fn sync_selected_worktree_index( + &self, + selections: &[String], + paths: &[String], + manifests: &BTreeMap, + ) -> Result<()> { + let mut sqlite_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta { + selected_worktree_index_sqlite_envelope_count: 1, + ..OperationMetricsDelta::default() + }, + ); + // Even a true empty selection crosses the single audited entry point. + // Recording that empty envelope distinguishes "proved zero SQL work" + // from an operation that never established accounting at all. + if selections.is_empty() && paths.is_empty() { + return Ok(()); + } + + note_selected_index_statement(&mut sqlite_metrics); + self.conn.execute_batch("BEGIN IMMEDIATE;")?; + sqlite_metrics + .delta + .selected_worktree_index_sqlite_transaction_count = sqlite_metrics + .delta + .selected_worktree_index_sqlite_transaction_count + .saturating_add(1); + + let mut pending_row_deletes = 0u64; + let mut pending_row_upserts = 0u64; + let result = (|| { + let minimal_selections = minimal_component_selections(selections); + let mut cached_paths = BTreeSet::new(); + { + let mut exact = self.conn.prepare(SELECT_WORKTREE_INDEX_EXACT_SQL)?; + let mut descendants = self.conn.prepare(SELECT_WORKTREE_INDEX_DESCENDANTS_SQL)?; + for selection in minimal_selections { + cached_paths.extend(query_selected_index_paths( + &mut exact, + params![selection.as_str()], + &mut sqlite_metrics, + )?); + let (lower, upper) = selected_path_descendant_bounds(&selection); + cached_paths.extend(query_selected_index_paths( + &mut descendants, + params![lower, upper], + &mut sqlite_metrics, + )?); + } + } + + let seen = paths.iter().map(String::as_str).collect::>(); + let paths_to_delete = cached_paths + .into_iter() + .filter(|path| !seen.contains(path.as_str())) + .collect::>(); + if paths_to_delete.is_empty() && paths.is_empty() { + return Ok(()); + } + + { + let mut clear_baseline = self + .conn + .prepare("DELETE FROM schema_meta WHERE key = ?1")?; + execute_selected_index_statement( + &mut clear_baseline, + params![WORKTREE_INDEX_BASELINE_ROOT_KEY], + &mut sqlite_metrics, + )?; + } + + let scan_id = worktree_scan_id(); + let now = now_ts(); + let mut delete = self.conn.prepare(DELETE_WORKTREE_INDEX_PATH_SQL)?; + let mut upsert = self.conn.prepare(UPSERT_WORKTREE_INDEX_PATH_SQL)?; + for path in paths_to_delete { + let deleted = execute_selected_index_statement( + &mut delete, + params![path], + &mut sqlite_metrics, + )?; + pending_row_deletes = + pending_row_deletes.saturating_add(saturating_u64_from_usize(deleted)); + } + for path in paths { + let abs = self.workspace_root.join(path_from_rel(path)); + let metadata = match fs::symlink_metadata(&abs) { + Ok(metadata) => metadata, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + let deleted = execute_selected_index_statement( + &mut delete, + params![path], + &mut sqlite_metrics, + )?; + pending_row_deletes = + pending_row_deletes.saturating_add(saturating_u64_from_usize(deleted)); + continue; + } + Err(err) => return Err(Error::Io(err)), + }; + if !metadata.is_file() || metadata.file_type().is_symlink() { + let deleted = execute_selected_index_statement( + &mut delete, + params![path], + &mut sqlite_metrics, + )?; + pending_row_deletes = + pending_row_deletes.saturating_add(saturating_u64_from_usize(deleted)); + continue; + } + let stamp = WorktreeFileStamp::from_metadata(&metadata); + let disk_manifest = manifests.get(path).ok_or_else(|| { + Error::Corrupt(format!("missing computed disk manifest for `{}`", path)) + })?; + let upserted = execute_selected_index_statement( + &mut upsert, + params![ + path.as_str(), + stamp.size_bytes as i64, + stamp.modified_ns, + stamp.changed_ns, + stamp.device_id, + stamp.inode, + i64::from(stamp.executable), + file_kind_index_label(&disk_manifest.kind), + disk_manifest.content_hash.as_str(), + scan_id, + now + ], + &mut sqlite_metrics, + )?; + pending_row_upserts = + pending_row_upserts.saturating_add(saturating_u64_from_usize(upserted)); + } + Ok(()) + })(); + + match result { + Ok(()) => { + note_selected_index_statement(&mut sqlite_metrics); + if let Err(err) = self.conn.execute_batch("COMMIT;") { + note_selected_index_statement(&mut sqlite_metrics); + let _ = self.conn.execute_batch("ROLLBACK;"); + return Err(Error::from(err)); + } + sqlite_metrics + .delta + .selected_worktree_index_sqlite_row_delete_count = pending_row_deletes; + sqlite_metrics + .delta + .selected_worktree_index_sqlite_row_upsert_count = pending_row_upserts; + Ok(()) + } + Err(err) => { + note_selected_index_statement(&mut sqlite_metrics); + let _ = self.conn.execute_batch("ROLLBACK;"); + Err(err) + } + } + } + pub(crate) fn delete_worktree_index_path(&self, path: &str) -> Result<()> { self.clear_worktree_index_baseline()?; self.delete_worktree_index_path_row(path) @@ -673,31 +1245,6 @@ impl Trail { Ok(()) } - pub(crate) fn prune_worktree_index_for_selections( - &self, - selections: &[String], - seen: &BTreeSet, - ) -> Result<()> { - let mut stmt = self.conn.prepare("SELECT path FROM worktree_file_index")?; - let rows = stmt.query_map([], |row| row.get::<_, String>(0))?; - let cached_paths = rows - .collect::, _>>() - .map_err(Error::from)?; - drop(stmt); - for path in cached_paths { - if seen.contains(&path) { - continue; - } - if selections - .iter() - .any(|selection| path_matches_selection(&path, selection)) - { - self.delete_worktree_index_path(&path)?; - } - } - Ok(()) - } - pub(crate) fn worktree_index_baseline_root(&self) -> Result> { Ok(self .schema_meta_value(WORKTREE_INDEX_BASELINE_ROOT_KEY)? @@ -721,74 +1268,521 @@ impl Trail { } } -fn cached_worktree_file_stamp( - stmt: &mut rusqlite::Statement<'_>, - path: &str, -) -> Result> { - stmt.query_row(params![path], |row| { - Ok(WorktreeFileStamp { - size_bytes: row.get::<_, i64>(0)?.max(0) as u64, - modified_ns: row.get(1)?, - changed_ns: row.get(2)?, - device_id: row.get(3)?, - inode: row.get(4)?, - executable: row.get::<_, i64>(5)? != 0, - }) - }) - .optional() - .map_err(Error::from) +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn open_relative_directory_no_follow(root: &fs::File, relative: &str) -> Result> { + use rustix::fs::{openat, Mode, OFlags}; + let path = path_from_rel(relative); + let mut directory = root.try_clone().map_err(Error::Io)?; + for component in path.components() { + let Component::Normal(name) = component else { + return Err(Error::InvalidInput(format!( + "candidate prefix `{relative}` is not normalized" + ))); + }; + directory = match openat( + &directory, + Path::new(name), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) { + Ok(fd) => fs::File::from(fd), + Err(error) + if error == rustix::io::Errno::NOENT || error == rustix::io::Errno::NOTDIR => + { + return Ok(None); + } + Err(error) => return Err(Error::Io(error.into())), + }; + } + Ok(Some(directory)) } -fn read_worktree_index_candidates( - candidates: &[WorktreeIndexReadCandidate], - text_config: &TextConfig, -) -> Result> { - if candidates.len() <= 1 { - return candidates - .iter() - .map(|candidate| read_worktree_index_candidate(candidate, text_config)) - .collect(); - } +fn reconcile_path_ignored(relative: &str) -> bool { + // Task 4's flattened matcher cannot prove exact nested ignore semantics. + // Reconciliation therefore over-enumerates and excludes only Trail's + // hardcoded internal/default-denied paths. Git-ignored files may be false + // positive candidates, but no visible regular file can be omitted. + is_default_ignored(relative) +} - candidates - .par_iter() - .map(|candidate| read_worktree_index_candidate(candidate, text_config)) - .collect() +fn path_intersects_reconcile_scope(relative: &str, is_dir: bool, prefixes: &[String]) -> bool { + prefixes.is_empty() + || prefixes.iter().any(|prefix| { + relative == prefix + || relative.starts_with(&format!("{prefix}/")) + || (is_dir && prefix.starts_with(&format!("{relative}/"))) + }) } -fn read_worktree_index_candidate( - candidate: &WorktreeIndexReadCandidate, - text_config: &TextConfig, -) -> Result { - let bytes = fs::read(&candidate.abs_path)?; - Ok(WorktreeIndexUpdate { - path: candidate.path.clone(), - stamp: candidate.stamp, - disk_manifest: DiskManifest { - kind: classify_file_kind(&bytes, text_config), - executable: candidate.stamp.executable, - content_hash: sha256_hex(&bytes), - }, - }) +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn open_absolute_directory_no_follow(path: &Path) -> Result { + use rustix::fs::{openat, Mode, OFlags, CWD}; + + if !path.is_absolute() { + return Err(Error::InvalidInput(format!( + "reconciliation root `{}` is not absolute", + path.display() + ))); + } + let mut descriptor = fs::File::from( + openat( + CWD, + Path::new("/"), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(|err| Error::Io(err.into()))?, + ); + for component in path + .strip_prefix(Path::new("/")) + .map_err(|err| Error::InvalidInput(err.to_string()))? + .components() + { + let Component::Normal(name) = component else { + return Err(Error::InvalidInput(format!( + "reconciliation root `{}` is not normalized", + path.display() + ))); + }; + descriptor = fs::File::from( + openat( + &descriptor, + Path::new(name), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(|err| Error::Io(err.into()))?, + ); + } + Ok(descriptor) } -fn duration_ns(duration: Duration) -> i64 { - let ns = (duration.as_secs() as u128) - .saturating_mul(1_000_000_000) - .saturating_add(duration.subsec_nanos() as u128); - ns.min(i64::MAX as u128) as i64 +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn open_absolute_directory_no_follow(path: &Path) -> Result { + Err(Error::InvalidInput(format!( + "qualified changed-path reconciliation is unsupported for `{}` on this platform", + path.display() + ))) } -fn worktree_scan_id() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(duration_ns) - .unwrap_or(0) +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn root_descriptor_identity(file: &fs::File) -> Result> { + use rustix::fs::fstat; + + let stat = fstat(file).map_err(|err| Error::Io(err.into()))?; + Ok(format!( + "root-v1:dev={};ino={};mode={};uid={};gid={}", + stat.st_dev, stat.st_ino, stat.st_mode, stat.st_uid, stat.st_gid + ) + .into_bytes()) } -fn file_kind_index_label(kind: &FileKind) -> &'static str { - match kind { - FileKind::Text => "Text", +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn read_reconciliation_directory_no_follow( + root: &fs::File, + relative: &str, +) -> Result> { + use rustix::fs::{fstat, openat, Mode, OFlags}; + + let path = path_from_rel(relative); + let mut descriptor = root.try_clone().map_err(Error::Io)?; + for component in path.components() { + let Component::Normal(name) = component else { + return Err(Error::InvalidInput(format!( + "reconciliation directory `{relative}` is not normalized" + ))); + }; + descriptor = match openat( + &descriptor, + Path::new(name), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) { + Ok(fd) => fs::File::from(fd), + Err(err) if err == rustix::io::Errno::NOENT => return Ok(None), + Err(err) => return Err(Error::Io(err.into())), + }; + } + let stat = fstat(&descriptor).map_err(|err| Error::Io(err.into()))?; + Ok(Some(ReconciliationDirectory { + path: relative.to_string(), + identity: format!( + "directory-v1:dev={};ino={};mode={};ctime={};ctime_nsec={}", + stat.st_dev, stat.st_ino, stat.st_mode, stat.st_ctime, stat.st_ctime_nsec + ) + .into_bytes(), + })) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn read_reconciliation_directory_no_follow( + _root: &fs::File, + relative: &str, +) -> Result> { + Err(Error::InvalidInput(format!( + "qualified reconciliation of directory `{relative}` is unsupported on this platform" + ))) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn root_descriptor_identity(_file: &fs::File) -> Result> { + Err(Error::InvalidInput( + "qualified changed-path reconciliation is unsupported on this platform".into(), + )) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn read_reconciliation_file_no_follow( + root: &fs::File, + relative: &str, + text: &TextConfig, + retain_bytes: bool, +) -> Result> { + use rustix::fs::{fstat, openat, statat, AtFlags, FileType, Mode, OFlags}; + + let path = path_from_rel(relative); + let components = path.components().collect::>(); + if components.is_empty() + || components + .iter() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(Error::InvalidInput(format!( + "reconciliation path `{relative}` is not normalized" + ))); + } + for _ in 0..2 { + let mut directory = root.try_clone().map_err(Error::Io)?; + for component in &components[..components.len() - 1] { + let Component::Normal(name) = component else { + unreachable!(); + }; + directory = match openat( + &directory, + Path::new(name), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) { + Ok(fd) => fs::File::from(fd), + Err(err) if err == rustix::io::Errno::NOENT => return Ok(None), + Err(err) => return Err(Error::Io(err.into())), + }; + } + let Component::Normal(name) = components[components.len() - 1] else { + unreachable!(); + }; + let before_path = match statat(&directory, Path::new(name), AtFlags::SYMLINK_NOFOLLOW) { + Ok(stat) => stat, + Err(err) if err == rustix::io::Errno::NOENT => return Ok(None), + Err(err) => return Err(Error::Io(err.into())), + }; + if FileType::from_raw_mode(before_path.st_mode) != FileType::RegularFile { + return Ok(None); + } + let descriptor = match openat( + &directory, + Path::new(name), + OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) { + Ok(fd) => fd, + Err(err) if err == rustix::io::Errno::NOENT => continue, + Err(err) => return Err(Error::Io(err.into())), + }; + let mut file = fs::File::from(descriptor); + let before_open = fstat(&file).map_err(|err| Error::Io(err.into()))?; + if stat_identity(&before_path) != stat_identity(&before_open) + || FileType::from_raw_mode(before_open.st_mode) != FileType::RegularFile + { + continue; + } + let mut hasher = Sha256::new(); + let mut buffer = [0u8; RECONCILE_READ_BUFFER_BYTES]; + let mut utf8_validation = Vec::with_capacity(RECONCILE_READ_BUFFER_BYTES + 3); + let mut utf8_tail = Vec::with_capacity(3); + let mut utf8_valid = true; + let mut binary = false; + let mut binary_bytes_seen = 0usize; + let mut current_line_bytes = 0u64; + let mut max_line_bytes = 0u64; + let mut size = 0u64; + let mut retained = retain_bytes.then(Vec::new); + loop { + let read = file.read(&mut buffer).map_err(Error::Io)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + if let Some(retained) = retained.as_mut() { + retained.extend_from_slice(&buffer[..read]); + } + size = size.saturating_add(read as u64); + if binary_bytes_seen < 8192 { + let inspected = read.min(8192 - binary_bytes_seen); + binary |= buffer[..inspected].contains(&0); + binary_bytes_seen += inspected; + } + for byte in &buffer[..read] { + if *byte == b'\n' { + max_line_bytes = max_line_bytes.max(current_line_bytes); + current_line_bytes = 0; + } else { + current_line_bytes = current_line_bytes.saturating_add(1); + } + } + if utf8_valid { + utf8_validation.clear(); + utf8_validation.extend_from_slice(&utf8_tail); + utf8_validation.extend_from_slice(&buffer[..read]); + utf8_tail.clear(); + if let Err(err) = std::str::from_utf8(&utf8_validation) { + if err.error_len().is_some() { + utf8_valid = false; + } else { + utf8_tail.extend_from_slice(&utf8_validation[err.valid_up_to()..]); + } + } + } + } + let after_open = fstat(&file).map_err(|err| Error::Io(err.into()))?; + let after_path = match statat(&directory, Path::new(name), AtFlags::SYMLINK_NOFOLLOW) { + Ok(stat) => stat, + Err(_) => continue, + }; + if stat_identity(&before_open) != stat_identity(&after_open) + || stat_identity(&after_open) != stat_identity(&after_path) + || i64::try_from(size).ok() != Some(after_open.st_size) + { + continue; + } + max_line_bytes = max_line_bytes.max(current_line_bytes); + utf8_valid &= utf8_tail.is_empty(); + let kind = if binary { + FileKind::Binary + } else if size > text.opaque_text_max_bytes + || !utf8_valid + || max_line_bytes > text.max_line_bytes + { + FileKind::OpaqueText + } else { + FileKind::Text + }; + return Ok(Some(ReconciliationFile { + path: relative.to_string(), + file_kind: file_kind_index_label(&kind).to_string(), + content_hash: hex::encode(hasher.finalize()), + executable: before_open.st_mode & 0o111 != 0, + size_bytes: size, + identity: stat_identity(&after_open), + peak_buffer_bytes: (buffer.len() + utf8_validation.capacity() + utf8_tail.capacity()) + .saturating_add(retained.as_ref().map_or(0, Vec::capacity)) + as u64, + bytes: retained, + })); + } + Err(Error::InvalidInput(format!( + "reconciliation path `{relative}` changed while it was read" + ))) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn read_reconciliation_file_no_follow( + _root: &fs::File, + relative: &str, + _text: &TextConfig, + _retain_bytes: bool, +) -> Result> { + Err(Error::InvalidInput(format!( + "qualified reconciliation of `{relative}` is unsupported on this platform" + ))) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn stat_identity(stat: &rustix::fs::Stat) -> Vec { + format!( + "file-v1:dev={};ino={};mode={};len={};mtime={};mtime_nsec={};ctime={};ctime_nsec={}", + stat.st_dev, + stat.st_ino, + stat.st_mode, + stat.st_size, + stat.st_mtime, + stat.st_mtime_nsec, + stat.st_ctime, + stat.st_ctime_nsec + ) + .into_bytes() +} + +fn minimal_component_selections(selections: &[String]) -> Vec { + let unique = selections + .iter() + .map(String::as_str) + .collect::>(); + let mut minimal = BTreeSet::new(); + for selection in unique { + let covered = selection + .match_indices('/') + .any(|(separator, _)| minimal.contains(&selection[..separator])); + if !covered { + minimal.insert(selection.to_string()); + } + } + minimal.into_iter().collect() +} + +fn selected_path_descendant_bounds(selection: &str) -> (String, String) { + let lower = format!("{selection}/"); + let mut upper = lower.as_bytes().to_vec(); + let terminal_separator = upper + .last_mut() + .expect("selected descendant lower bound always ends in slash"); + debug_assert_eq!(*terminal_separator, b'/'); + *terminal_separator = b'0'; + let upper = String::from_utf8(upper) + .expect("incrementing an ASCII path separator preserves valid UTF-8"); + (lower, upper) +} + +fn note_selected_index_statement(metrics: &mut OperationMetricsAccumulator) { + metrics.delta.selected_worktree_index_sqlite_statement_count = metrics + .delta + .selected_worktree_index_sqlite_statement_count + .saturating_add(1); +} + +fn note_selected_index_full_scan( + statement: &Statement<'_>, + metrics: &mut OperationMetricsAccumulator, +) { + if statement.get_status(StatementStatus::FullscanStep) > 0 { + metrics.delta.selected_worktree_index_sqlite_full_scan_count = metrics + .delta + .selected_worktree_index_sqlite_full_scan_count + .saturating_add(1); + } +} + +fn execute_selected_index_statement( + statement: &mut Statement<'_>, + params: P, + metrics: &mut OperationMetricsAccumulator, +) -> Result { + statement.reset_status(StatementStatus::FullscanStep); + note_selected_index_statement(metrics); + let result = statement.execute(params).map_err(Error::from); + note_selected_index_full_scan(statement, metrics); + result +} + +fn query_selected_index_paths( + statement: &mut Statement<'_>, + params: P, + metrics: &mut OperationMetricsAccumulator, +) -> Result> { + statement.reset_status(StatementStatus::FullscanStep); + note_selected_index_statement(metrics); + let result = (|| -> rusqlite::Result> { + let mut rows = statement.query(params)?; + let mut paths = Vec::new(); + while let Some(row) = rows.next()? { + paths.push(row.get::<_, String>(0)?); + metrics.delta.selected_worktree_index_sqlite_row_read_count = metrics + .delta + .selected_worktree_index_sqlite_row_read_count + .saturating_add(1); + } + Ok(paths) + })() + .map_err(Error::from); + note_selected_index_full_scan(statement, metrics); + result +} + +fn cached_worktree_file_stamp( + stmt: &mut rusqlite::Statement<'_>, + path: &str, +) -> Result> { + stmt.query_row(params![path], |row| { + Ok(WorktreeFileStamp { + size_bytes: row.get::<_, i64>(0)?.max(0) as u64, + modified_ns: row.get(1)?, + changed_ns: row.get(2)?, + device_id: row.get(3)?, + inode: row.get(4)?, + executable: row.get::<_, i64>(5)? != 0, + }) + }) + .optional() + .map_err(Error::from) +} + +fn read_worktree_index_candidates( + candidates: &[WorktreeIndexReadCandidate], + text_config: &TextConfig, + metrics: Option<&Arc>, +) -> Result> { + if candidates.len() <= 1 { + return candidates + .iter() + .map(|candidate| read_worktree_index_candidate(candidate, text_config, metrics)) + .collect(); + } + + candidates + .par_iter() + .map(|candidate| read_worktree_index_candidate(candidate, text_config, metrics)) + .collect() +} + +fn read_worktree_index_candidate( + candidate: &WorktreeIndexReadCandidate, + text_config: &TextConfig, + metrics: Option<&Arc>, +) -> Result { + if let Some(metrics) = metrics { + metrics.add(OperationMetricsDelta { + filesystem_read_count: 1, + ..OperationMetricsDelta::default() + }); + } + let bytes = fs::read(&candidate.abs_path)?; + if let Some(metrics) = metrics { + let bytes_len = saturating_u64_from_usize(bytes.len()); + metrics.add(OperationMetricsDelta { + filesystem_read_bytes: bytes_len, + filesystem_hash_count: 1, + filesystem_hash_bytes: bytes_len, + ..OperationMetricsDelta::default() + }); + } + Ok(WorktreeIndexUpdate { + path: candidate.path.clone(), + stamp: candidate.stamp, + disk_manifest: DiskManifest { + kind: classify_file_kind(&bytes, text_config), + executable: candidate.stamp.executable, + content_hash: sha256_hex(&bytes), + }, + }) +} + +fn duration_ns(duration: Duration) -> i64 { + let ns = (duration.as_secs() as u128) + .saturating_mul(1_000_000_000) + .saturating_add(duration.subsec_nanos() as u128); + ns.min(i64::MAX as u128) as i64 +} + +fn worktree_scan_id() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(duration_ns) + .unwrap_or(0) +} + +fn file_kind_index_label(kind: &FileKind) -> &'static str { + match kind { + FileKind::Text => "Text", FileKind::OpaqueText => "OpaqueText", FileKind::Binary => "Binary", } @@ -806,27 +1800,17 @@ pub(crate) fn file_kind_from_index(value: &str) -> std::result::Result Result { + fn start( + workspace_root: &Path, + db_dir: &Path, + metrics: Option>, + ) -> Result { let state = Arc::new(Mutex::new(DaemonWorktreeCacheState::default())); let root = workspace_root.to_path_buf(); - let persist = DaemonWorktreeCachePersist { - path: daemon_worktree_snapshot_path(db_dir), - workspace_root: workspace_root.to_path_buf(), - pid: std::process::id(), - active: Arc::new(AtomicBool::new(true)), - }; - persist_daemon_worktree_state(&persist, &state); + let _ = (db_dir, metrics); let state_for_watcher = Arc::clone(&state); - let persist_for_watcher = persist.clone(); let mut watcher = RecommendedWatcher::new( - move |event| { - handle_daemon_watch_event( - &root, - &state_for_watcher, - Some(&persist_for_watcher), - event, - ) - }, + move |event| handle_daemon_watch_event(&root, &state_for_watcher, None, event), NotifyConfig::default(), ) .map_err(notify_error)?; @@ -835,7 +1819,7 @@ impl DaemonWorktreeCache { .map_err(notify_error)?; Ok(Self { state, - persist: Some(persist), + persist: None, watcher: Some(watcher), }) } @@ -969,11 +1953,11 @@ fn handle_daemon_watch_event( if matches!(event.kind, EventKind::Access(_)) { return; } - if daemon_event_paths_all_default_ignored(root, &event.paths) { + if daemon_event_touches_policy_dependency(root, state, &event.paths) { + mark_daemon_cache_overflow(state, persist); return; } - if daemon_event_touches_ignore_file(root, &event.paths) { - mark_daemon_cache_overflow(state, persist); + if daemon_event_paths_all_default_ignored(root, &event.paths) { return; } if matches!( @@ -1021,11 +2005,22 @@ fn daemon_event_paths_all_default_ignored(root: &Path, paths: &[PathBuf]) -> boo }) } -fn daemon_event_touches_ignore_file(root: &Path, paths: &[PathBuf]) -> bool { - paths.iter().any(|path| { +fn daemon_event_touches_policy_dependency( + root: &Path, + state: &Arc>, + paths: &[PathBuf], +) -> bool { + if paths.iter().any(|path| { daemon_event_relative_path(root, path) - .is_some_and(|path| path == ".trailignore" || path == ".gitignore") - }) + .is_some_and(|path| raw_path_may_invalidate_policy(&path_from_rel(&path))) + }) { + return true; + } + let state = state.lock().expect("daemon worktree cache poisoned"); + state + .policy_invalidation_index + .as_ref() + .is_some_and(|index| paths.iter().any(|path| index.matches(root, path))) } fn handle_daemon_rename_both_event( @@ -1142,73 +2137,15 @@ fn notify_error(err: notify::Error) -> Error { Error::InvalidInput(format!("daemon worktree watcher failed: {err}")) } -fn daemon_worktree_snapshot_path(db_dir: &Path) -> PathBuf { - db_dir.join(DAEMON_WORKTREE_SNAPSHOT_FILE) -} - fn persist_daemon_worktree_state( - persist: &DaemonWorktreeCachePersist, - state: &Arc>, + _persist: &DaemonWorktreeCachePersist, + _state: &Arc>, ) { - if !persist.active.load(Ordering::Acquire) { - return; - } - let snapshot = { - let state = state.lock().expect("daemon worktree cache poisoned"); - PersistedDaemonWorktreeSnapshot { - version: DAEMON_WORKTREE_SNAPSHOT_VERSION, - pid: persist.pid, - workspace_root: persist.workspace_root.to_string_lossy().to_string(), - generation: state.generation, - initialized: state.initialized, - overflow: state.overflow, - baseline_root_id: state.baseline_root_id.as_ref().map(|id| id.0.clone()), - dirty_paths: state.dirty_paths.iter().cloned().collect(), - updated_ns: worktree_scan_id(), - } - }; - let tmp = persist.path.with_file_name(format!( - "{DAEMON_WORKTREE_SNAPSHOT_FILE}.{}.tmp", - persist.pid - )); - let Ok(bytes) = serde_json::to_vec(&snapshot) else { - return; - }; - if fs::write(&tmp, bytes).is_err() { - return; - } - if !persist.active.load(Ordering::Acquire) { - let _ = fs::remove_file(tmp); - return; - } - let _ = fs::rename(tmp, &persist.path); -} - -#[cfg(all(test, unix))] -fn write_persisted_daemon_worktree_snapshot( - path: &Path, - snapshot: &PersistedDaemonWorktreeSnapshot, - pid: u32, -) -> Result<()> { - let tmp = path.with_file_name(format!("{DAEMON_WORKTREE_SNAPSHOT_FILE}.{pid}.tmp")); - fs::write(&tmp, serde_json::to_vec(snapshot)?)?; - fs::rename(tmp, path)?; - Ok(()) } impl Drop for DaemonWorktreeCache { fn drop(&mut self) { - if let Some(persist) = &self.persist { - persist.active.store(false, Ordering::Release); - } drop(self.watcher.take()); - if let Some(persist) = &self.persist { - let _ = fs::remove_file(&persist.path); - let _ = fs::remove_file(persist.path.with_file_name(format!( - "{DAEMON_WORKTREE_SNAPSHOT_FILE}.{}.tmp", - persist.pid - ))); - } } } @@ -1216,6 +2153,367 @@ impl Drop for DaemonWorktreeCache { mod tests { use super::*; + fn seed_worktree_index_paths(db: &Trail, paths: &[String]) { + db.conn.execute_batch("BEGIN IMMEDIATE;").unwrap(); + { + let mut insert = db + .conn + .prepare( + "INSERT OR REPLACE INTO worktree_file_index \ + (path, size_bytes, modified_ns, changed_ns, device_id, inode, executable, kind, content_hash, last_seen_scan, updated_at) \ + VALUES (?1, 1, 1, 1, 1, 1, 0, 'Text', 'seed', 1, 1)", + ) + .unwrap(); + for path in paths { + insert.execute(params![path]).unwrap(); + } + } + db.conn.execute_batch("COMMIT;").unwrap(); + } + + fn selected_sync_manifest(path: &str, bytes: &[u8]) -> BTreeMap { + BTreeMap::from([( + path.to_string(), + DiskManifest { + kind: FileKind::Text, + executable: false, + content_hash: sha256_hex(bytes), + }, + )]) + } + + fn profile_selected_worktree_index_sync( + db: &Trail, + selections: &[String], + paths: &[String], + manifests: &BTreeMap, + ) -> Result { + let metrics = Arc::clone( + db.operation_metrics + .as_ref() + .expect("test operation metrics should be enabled"), + ); + metrics.profile(OperationMetricsKind::Diff, || { + db.sync_selected_worktree_index(selections, paths, manifests) + })?; + Ok(metrics.last_report()) + } + + fn selected_sync_scale_fixture( + decoy_count: usize, + ) -> (OperationMetricsReport, u64, Vec, Option) { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir_all(temp.path().join("live-dir")).unwrap(); + fs::write(temp.path().join("live-dir/keep.txt"), b"live\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + + let mut seeded = (0..decoy_count) + .map(|index| format!("decoy/{index:05}.txt")) + .collect::>(); + seeded.extend([ + "exact.txt".to_string(), + "deleted-dir/a.txt".to_string(), + "deleted-dir/nested/b.txt".to_string(), + "live-dir/keep.txt".to_string(), + ]); + seed_worktree_index_paths(&db, &seeded); + db.set_worktree_index_baseline(&ObjectId("selected-sync-baseline".to_string())) + .unwrap(); + + let selections = ["exact.txt", "deleted-dir", "live-dir"] + .into_iter() + .map(str::to_string) + .collect::>(); + let live_path = "live-dir/keep.txt".to_string(); + let paths = vec![live_path.clone()]; + let manifests = selected_sync_manifest(&live_path, b"live\n"); + let report = + profile_selected_worktree_index_sync(&db, &selections, &paths, &manifests).unwrap(); + + let decoys = db + .conn + .query_row( + "SELECT COUNT(*) FROM worktree_file_index WHERE path >= 'decoy/' AND path < 'decoy0'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap() + .max(0) as u64; + let selected_rows = db + .conn + .prepare( + "SELECT path FROM worktree_file_index \ + WHERE path = 'exact.txt' \ + OR (path >= 'deleted-dir/' AND path < 'deleted-dir0') \ + OR (path >= 'live-dir/' AND path < 'live-dir0') \ + ORDER BY path", + ) + .unwrap() + .query_map([], |row| row.get::<_, String>(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + let baseline = db.worktree_index_baseline_root().unwrap(); + (report, decoys, selected_rows, baseline) + } + + #[test] + fn selected_worktree_index_sync_is_bounded_independent_of_repository_rows() { + let (small, small_decoys, small_rows, small_baseline) = selected_sync_scale_fixture(0); + let (large, large_decoys, large_rows, large_baseline) = selected_sync_scale_fixture(10_000); + + assert_eq!(small_decoys, 0); + assert_eq!(large_decoys, 10_000); + assert_eq!(small_rows, vec!["live-dir/keep.txt"]); + assert_eq!(large_rows, small_rows); + assert_eq!(small_baseline, None); + assert_eq!(large_baseline, None); + for report in [&small, &large] { + assert!(report.selected_worktree_index_sqlite_accounting_complete); + assert_eq!(report.selected_worktree_index_sqlite_envelope_count, 1); + assert_eq!(report.selected_worktree_index_sqlite_full_scan_count, 0); + assert_eq!(report.selected_worktree_index_sqlite_row_read_count, 4); + assert_eq!(report.selected_worktree_index_sqlite_row_delete_count, 3); + assert_eq!(report.selected_worktree_index_sqlite_row_upsert_count, 1); + assert_eq!(report.selected_worktree_index_sqlite_statement_count, 13); + assert_eq!(report.selected_worktree_index_sqlite_transaction_count, 1); + } + } + + #[test] + fn selected_worktree_index_true_empty_input_records_a_complete_empty_envelope() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let baseline = ObjectId("empty-sync-baseline".to_string()); + db.set_worktree_index_baseline(&baseline).unwrap(); + + let report = profile_selected_worktree_index_sync(&db, &[], &[], &BTreeMap::new()).unwrap(); + + assert_eq!(db.worktree_index_baseline_root().unwrap(), Some(baseline)); + assert!(report.selected_worktree_index_sqlite_accounting_complete); + assert_eq!( + report.selected_worktree_index_sqlite_accounting_disposition, + "complete" + ); + assert_eq!(report.selected_worktree_index_sqlite_envelope_count, 1); + assert_eq!( + report.selected_worktree_index_sqlite_not_applicable_count, + 0 + ); + assert_eq!(report.selected_worktree_index_sqlite_statement_count, 0); + assert_eq!(report.selected_worktree_index_sqlite_transaction_count, 0); + } + + #[test] + fn selected_worktree_index_queries_use_binary_primary_key_searches() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + + let explain = |sql: &str, parameters: &[&str]| { + let sql = format!("EXPLAIN QUERY PLAN {sql}"); + db.conn + .prepare(&sql) + .unwrap() + .query_map(params_from_iter(parameters.iter().copied()), |row| { + row.get::<_, String>(3) + }) + .unwrap() + .collect::, _>>() + .unwrap() + }; + let exact = explain(SELECT_WORKTREE_INDEX_EXACT_SQL, &["src"]); + let range = explain(SELECT_WORKTREE_INDEX_DESCENDANTS_SQL, &["src/", "src0"]); + + assert!(exact + .iter() + .any(|detail| detail.contains("SEARCH worktree_file_index"))); + assert!(exact.iter().any(|detail| detail.contains("path=?"))); + assert!(exact + .iter() + .all(|detail| !detail.contains("SCAN worktree_file_index"))); + assert!(range + .iter() + .any(|detail| detail.contains("SEARCH worktree_file_index"))); + assert!(range + .iter() + .any(|detail| { detail.contains("path>?") && detail.contains("path(0), + ) + .unwrap(), + 0, + "selected path must be deleted" + ); + } + for path in sibling_rows { + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM worktree_file_index WHERE path = ?1", + params![path], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1, + "binary sibling must remain" + ); + } + assert_eq!(report.selected_worktree_index_sqlite_full_scan_count, 0); + assert_eq!(report.selected_worktree_index_sqlite_row_read_count, 3); + assert_eq!(report.selected_worktree_index_sqlite_row_delete_count, 3); + } + + #[test] + fn selected_worktree_index_sync_deduplicates_overlapping_component_selections() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + seed_worktree_index_paths( + &db, + &[ + "tree/a.txt".to_string(), + "tree/sub/b.txt".to_string(), + "treehouse/keep.txt".to_string(), + ], + ); + let selections = ["tree/sub", "tree", "tree/sub/b.txt", "tree"] + .into_iter() + .map(str::to_string) + .collect::>(); + + let report = + profile_selected_worktree_index_sync(&db, &selections, &[], &BTreeMap::new()).unwrap(); + + assert_eq!(report.selected_worktree_index_sqlite_row_read_count, 2); + assert_eq!(report.selected_worktree_index_sqlite_row_delete_count, 2); + assert_eq!(report.selected_worktree_index_sqlite_statement_count, 7); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM worktree_file_index WHERE path = 'treehouse/keep.txt'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1 + ); + } + + #[test] + fn selected_worktree_index_commit_failure_rolls_back_baseline_and_rows() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + fs::create_dir_all(temp.path().join("live")).unwrap(); + fs::write(temp.path().join("live/upsert.txt"), b"live\n").unwrap(); + let db = Trail::open(temp.path()).unwrap(); + seed_worktree_index_paths(&db, &["gone.txt".to_string()]); + let baseline = ObjectId("rollback-baseline".to_string()); + db.set_worktree_index_baseline(&baseline).unwrap(); + db.conn + .execute_batch( + "CREATE TABLE selected_sync_commit_parent (id INTEGER PRIMARY KEY); + CREATE TABLE selected_sync_commit_child ( + parent_id INTEGER REFERENCES selected_sync_commit_parent(id) + DEFERRABLE INITIALLY DEFERRED + ); + CREATE TRIGGER selected_sync_fail_commit + AFTER INSERT ON worktree_file_index + WHEN NEW.path = 'live/upsert.txt' + BEGIN + INSERT INTO selected_sync_commit_child(parent_id) VALUES (1); + END;", + ) + .unwrap(); + let live_path = "live/upsert.txt".to_string(); + let manifests = selected_sync_manifest(&live_path, b"live\n"); + let metrics = Arc::clone(db.operation_metrics.as_ref().unwrap()); + + let result = metrics.profile(OperationMetricsKind::Diff, || { + db.sync_selected_worktree_index( + &["gone.txt".to_string(), "live".to_string()], + &[live_path], + &manifests, + ) + }); + + assert!(result.is_err()); + let report = metrics.last_report(); + assert!(report.selected_worktree_index_sqlite_accounting_complete); + assert_eq!(report.selected_worktree_index_sqlite_transaction_count, 1); + assert_eq!(report.selected_worktree_index_sqlite_row_read_count, 1); + assert_eq!(report.selected_worktree_index_sqlite_row_delete_count, 0); + assert_eq!(report.selected_worktree_index_sqlite_row_upsert_count, 0); + assert_eq!(report.selected_worktree_index_sqlite_statement_count, 10); + assert_eq!(db.worktree_index_baseline_root().unwrap(), Some(baseline)); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM worktree_file_index WHERE path = 'gone.txt'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 1 + ); + assert_eq!( + db.conn + .query_row( + "SELECT COUNT(*) FROM worktree_file_index WHERE path = 'live/upsert.txt'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + } + #[test] fn daemon_rename_both_tracks_file_paths_without_overflow() { let temp = tempfile::tempdir().unwrap(); @@ -1293,92 +2591,204 @@ mod tests { } #[test] - fn persisted_daemon_worktree_snapshot_is_available_to_second_db_handle() { + fn daemon_root_and_nested_policy_file_events_mark_overflow() { let temp = tempfile::tempdir().unwrap(); - fs::write(temp.path().join("README.md"), "hello\n").unwrap(); - Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + for path in [ + ".trailignore", + ".gitignore", + "nested/.trailignore", + "nested/.gitignore", + ".trail/config.toml", + ".git/info/exclude", + ".git/config", + ".git/config.worktree", + ] { + let state = Arc::new(Mutex::new(DaemonWorktreeCacheState { + initialized: true, + baseline_root_id: Some(ObjectId("root".to_string())), + ..DaemonWorktreeCacheState::default() + })); + handle_daemon_watch_event( + temp.path(), + &state, + None, + Ok(Event::new(EventKind::Modify(ModifyKind::Data( + notify::event::DataChange::Content, + ))) + .add_path(temp.path().join(path))), + ); - let mut daemon_db = Trail::open(temp.path()).unwrap(); - daemon_db.enable_daemon_worktree_cache().unwrap(); - let head = daemon_db.resolve_branch_ref("main").unwrap(); - let reader = Trail::open(temp.path()).unwrap(); - match reader.daemon_worktree_snapshot().unwrap() { - DaemonWorktreeSnapshot::Clean { - root_id: Some(root_id), - .. - } => assert_eq!(root_id, head.root_id), - other => panic!("expected persisted clean snapshot, got {other:?}"), + let state = state.lock().unwrap(); + assert!(state.overflow, "policy event {path} did not overflow"); + assert_eq!(state.baseline_root_id, None, "{path}"); + assert!(state.dirty_paths.is_empty(), "{path}"); } + } + + #[test] + fn daemon_exact_policy_index_runs_before_default_ignore_filter_case_insensitively() { + let temp = tempfile::tempdir().unwrap(); + let dependency = temp.path().join(".trail/cache/Arbitrary.Rules"); + let index = PolicyInvalidationIndex::from_paths(temp.path(), false, [&dependency]); + let state = Arc::new(Mutex::new(DaemonWorktreeCacheState { + initialized: true, + baseline_root_id: Some(ObjectId("root".to_string())), + policy_invalidation_index: Some(index), + ..DaemonWorktreeCacheState::default() + })); - fs::write(temp.path().join("README.md"), "hello\ndirty\n").unwrap(); - let cache = daemon_db.daemon_worktree_cache.as_ref().unwrap(); handle_daemon_watch_event( temp.path(), - &cache.state, - cache.persist.as_ref(), + &state, + None, Ok(Event::new(EventKind::Modify(ModifyKind::Data( notify::event::DataChange::Content, ))) - .add_path(temp.path().join("README.md"))), + .add_path(temp.path().join(".TRAIL/CACHE/arbitrary.rules"))), ); - let reader = Trail::open(temp.path()).unwrap(); - match reader.daemon_worktree_snapshot().unwrap() { - DaemonWorktreeSnapshot::Dirty { paths, .. } => { - assert_eq!(paths, vec!["README.md".to_string()]); - } - other => panic!("expected persisted dirty snapshot, got {other:?}"), + assert!(state.lock().unwrap().overflow); + } + + #[test] + fn daemon_non_policy_suffixes_remain_bounded_dirty_paths() { + let temp = tempfile::tempdir().unwrap(); + let state = Arc::new(Mutex::new(DaemonWorktreeCacheState { + initialized: true, + baseline_root_id: Some(ObjectId("root".to_string())), + ..DaemonWorktreeCacheState::default() + })); + for path in ["nested/not.trailignore", "nested/.gitignore.bak"] { + handle_daemon_watch_event( + temp.path(), + &state, + None, + Ok(Event::new(EventKind::Modify(ModifyKind::Data( + notify::event::DataChange::Content, + ))) + .add_path(temp.path().join(path))), + ); } - drop(daemon_db); - let reader = Trail::open(temp.path()).unwrap(); - assert!(reader.daemon_worktree_snapshot().is_none()); + let state = state.lock().unwrap(); + assert!(!state.overflow); + assert!(state.dirty_paths.contains("nested/not.trailignore")); + assert!(state.dirty_paths.contains("nested/.gitignore.bak")); } - #[cfg(unix)] #[test] - fn stale_persisted_daemon_worktree_snapshot_is_ignored_and_removed() { + fn daemon_nested_trailignore_event_cannot_leave_status_clean() { let temp = tempfile::tempdir().unwrap(); - fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + fs::create_dir_all(temp.path().join("nested")).unwrap(); + fs::write(temp.path().join("nested/.trailignore"), "hidden.txt\n").unwrap(); + fs::write(temp.path().join("nested/hidden.txt"), "hidden baseline\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); - let db = Trail::open(temp.path()).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); let head = db.resolve_branch_ref("main").unwrap(); - let path = daemon_worktree_snapshot_path(db.db_dir()); - - write_persisted_daemon_worktree_snapshot( - &path, - &PersistedDaemonWorktreeSnapshot { - version: DAEMON_WORKTREE_SNAPSHOT_VERSION, - pid: u32::MAX, - workspace_root: db.workspace_root.to_string_lossy().to_string(), - generation: 42, + db.daemon_worktree_cache = Some(DaemonWorktreeCache { + state: Arc::new(Mutex::new(DaemonWorktreeCacheState { initialized: true, - overflow: false, - baseline_root_id: Some(head.root_id.0), - dirty_paths: Vec::new(), - updated_ns: worktree_scan_id(), - }, - u32::MAX, - ) + baseline_root_id: Some(head.root_id), + generation: 1, + ..DaemonWorktreeCacheState::default() + })), + persist: None, + watcher: None, + }); + fs::write(temp.path().join("nested/.trailignore"), "").unwrap(); + let cache = db.daemon_worktree_cache.as_ref().unwrap(); + handle_daemon_watch_event( + temp.path(), + &cache.state, + None, + Ok(Event::new(EventKind::Modify(ModifyKind::Data( + notify::event::DataChange::Content, + ))) + .add_path(temp.path().join("nested/.trailignore"))), + ); + + let status = db.status(None).unwrap(); + + assert!( + status + .changed_paths + .iter() + .any(|change| change.path == "nested/hidden.txt"), + "nested policy change incorrectly produced {:?}", + status.changed_paths + ); + let report = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_eq!(report.git_global_work_count, 1); + assert_eq!(report.full_filesystem_walk_count, 1); + } + + #[test] + fn invalid_daemon_selection_reports_zero_canonical_candidates() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.daemon_worktree_cache = Some(DaemonWorktreeCache { + state: Arc::new(Mutex::new(DaemonWorktreeCacheState { + initialized: true, + dirty_paths: BTreeSet::from(["../outside".to_string()]), + generation: 1, + ..DaemonWorktreeCacheState::default() + })), + persist: None, + watcher: None, + }); + let metrics = db.operation_metrics.as_ref().unwrap(); + + profile_operation_metrics(Some(metrics), OperationMetricsKind::Status, || { + assert!(matches!( + db.daemon_worktree_snapshot(), + Some(DaemonWorktreeSnapshot::Dirty { .. }) + )); + Ok::<(), Error>(()) + }) .unwrap(); - let reader = Trail::open(temp.path()).unwrap(); - assert!(reader.daemon_worktree_snapshot().is_none()); - assert!(!path.exists()); + let report = operation_metrics_report(Some(metrics)).unwrap(); + assert_eq!(report.input_path_count, 1); + assert_eq!(report.canonical_path_count, 0); + assert_eq!(report.daemon_snapshot_path_count, 1); } #[test] - fn corrupt_persisted_daemon_worktree_snapshot_is_ignored() { + fn oversized_daemon_status_accounts_snapshot_once_before_fallback() { let temp = tempfile::tempdir().unwrap(); - fs::write(temp.path().join("README.md"), "hello\n").unwrap(); - Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); - let db = Trail::open(temp.path()).unwrap(); - let path = daemon_worktree_snapshot_path(db.db_dir()); - fs::write(&path, b"not json").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let raw_count = db.daemon_dirty_path_limit() + 1; + let dirty_paths = (0..raw_count) + .map(|index| format!("dirty/file_{index:05}.txt")) + .collect::>(); + db.daemon_worktree_cache = Some(DaemonWorktreeCache { + state: Arc::new(Mutex::new(DaemonWorktreeCacheState { + initialized: true, + dirty_paths, + generation: 1, + ..DaemonWorktreeCacheState::default() + })), + persist: None, + watcher: None, + }); + + let _status = db.status(None).unwrap(); - let reader = Trail::open(temp.path()).unwrap(); - assert!(reader.daemon_worktree_snapshot().is_none()); - assert!(path.exists()); + let report = operation_metrics_report(db.operation_metrics.as_ref()).unwrap(); + assert_eq!( + report.input_path_count, + saturating_u64_from_usize(raw_count) + ); + assert_eq!( + report.canonical_path_count, + saturating_u64_from_usize(raw_count) + ); + assert_eq!( + report.daemon_snapshot_path_count, + saturating_u64_from_usize(raw_count) + ); } #[test] @@ -1434,7 +2844,7 @@ mod tests { preserve_similarity: 0.0, }; - let updates = read_worktree_index_candidates(&candidates, &text_config).unwrap(); + let updates = read_worktree_index_candidates(&candidates, &text_config, None).unwrap(); let updates = updates .into_iter() @@ -1587,6 +2997,7 @@ mod tests { initialized: true, baseline_root_id: None, generation: 1, + policy_invalidation_index: None, })), persist: None, watcher: Some(watcher), diff --git a/trail/src/db/storage/worktree_scan.rs b/trail/src/db/storage/worktree_scan.rs index 1509fd5..2c586d7 100644 --- a/trail/src/db/storage/worktree_scan.rs +++ b/trail/src/db/storage/worktree_scan.rs @@ -1,13 +1,24 @@ use super::*; impl Trail { - pub(crate) fn scan_git_dirty_tracked_paths(&self) -> Result>> { + pub(crate) fn scan_git_dirty_tracked_paths_with_policy( + &self, + policy: &WorkspaceIgnorePolicySnapshot, + ) -> Result>> { let output = Command::new("git") .arg("-C") .arg(&self.workspace_root) .args(["status", "--porcelain=v1", "-z", "--untracked-files=all"]) .output() .map_err(|err| Error::Git(err.to_string()))?; + self.note_operation_metrics(OperationMetricsDelta { + git_subprocess_count: 1, + git_global_work_count: 1, + git_output_bytes: saturating_u64_from_usize( + output.stdout.len().saturating_add(output.stderr.len()), + ), + ..OperationMetricsDelta::default() + }); if !output.status.success() { return Ok(None); } @@ -18,6 +29,11 @@ impl Trail { .split(|byte| *byte == 0) .filter(|record| !record.is_empty()) .collect::>(); + self.note_operation_metrics(OperationMetricsDelta { + git_output_record_count: saturating_u64_from_usize(records.len()), + input_path_count: saturating_u64_from_usize(records.len()), + ..OperationMetricsDelta::default() + }); let mut idx = 0; while idx < records.len() { let record = records[idx]; @@ -26,10 +42,10 @@ impl Trail { } let status = &record[..2]; let path = normalize_relative_path(&String::from_utf8_lossy(&record[3..]))?; - if path == ".trailignore" || path == ".gitignore" { + if is_ignore_policy_path(&path) { return Ok(None); } - if !self.ignore_check(&path)?.ignored { + if !policy.check(&path)?.ignored { paths.insert(path); } if status == b"??" { @@ -42,40 +58,72 @@ impl Trail { return Ok(None); }; let old_path = normalize_relative_path(&String::from_utf8_lossy(old_record))?; - if old_path == ".trailignore" || old_path == ".gitignore" { + if is_ignore_policy_path(&old_path) { return Ok(None); } - if !self.ignore_check(&old_path)?.ignored { + if !policy.check(&old_path)?.ignored { paths.insert(old_path); } } idx += 1; } - Ok(Some(paths.into_iter().collect())) + let exact_paths = paths.into_iter().collect::>(); + let canonical_paths = SelectionSet::from_paths(&exact_paths)?; + self.note_operation_metrics(OperationMetricsDelta { + canonical_path_count: saturating_u64_from_usize(canonical_paths.as_slice().len()), + ..OperationMetricsDelta::default() + }); + Ok(Some(exact_paths)) } pub(crate) fn scan_visible_files_for_paths(&self, paths: &[String]) -> Result> { + let policy = self.workspace_ignore_policy_snapshot(); + self.scan_visible_files_for_paths_with_policy(paths, &policy) + } + + pub(crate) fn scan_visible_files_for_paths_with_policy( + &self, + paths: &[String], + policy: &WorkspaceIgnorePolicySnapshot, + ) -> Result> { let root = self.workspace_root.canonicalize()?; + let selections = SelectionSet::from_paths(paths)?; let mut files = BTreeMap::new(); - for path in paths { - if self.ignore_check(path)?.ignored { - continue; - } - let abs = self.workspace_root.join(path_from_rel(path)); + let mut filesystem_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta::default(), + ); + for path in selections.as_slice() { + let abs = safe_join(&root, path)?; + filesystem_metrics.delta.filesystem_stat_count = filesystem_metrics + .delta + .filesystem_stat_count + .saturating_add(1); let metadata = match fs::symlink_metadata(&abs) { - Ok(metadata) => metadata, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, + Ok(metadata) => Some(metadata), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => None, Err(err) => return Err(Error::Io(err)), }; + if policy + .check_with_is_dir(path, metadata.as_ref().is_some_and(fs::Metadata::is_dir))? + .ignored + { + continue; + } + let Some(metadata) = metadata else { + continue; + }; if metadata.file_type().is_symlink() { continue; } if metadata.is_file() { + let (bytes, metadata) = + read_selected_regular_file(abs, self.operation_metrics.as_ref())?; files.insert( path.clone(), DiskFile { path: path.clone(), - bytes: fs::read(&abs)?, + bytes, executable: executable_from_metadata(&metadata), }, ); @@ -84,10 +132,12 @@ impl Trail { &root, &abs, self.config.recording.ignore_gitignored, + self.operation_metrics.as_ref(), &mut files, )?; } } + filesystem_metrics.delta.expanded_path_count = saturating_u64_from_usize(files.len()); Ok(files.into_values().collect()) } @@ -102,6 +152,14 @@ impl Trail { .arg("-z") .output() .map_err(|err| Error::Git(err.to_string()))?; + self.note_operation_metrics(OperationMetricsDelta { + git_subprocess_count: 1, + git_global_work_count: 1, + git_output_bytes: saturating_u64_from_usize( + output.stdout.len().saturating_add(output.stderr.len()), + ), + ..OperationMetricsDelta::default() + }); if !output.status.success() { if required { let stderr = String::from_utf8_lossy(&output.stderr); @@ -113,6 +171,15 @@ impl Trail { } return Ok(None); } + let output_record_count = output + .stdout + .split(|byte| *byte == 0) + .filter(|raw| !raw.is_empty()) + .count(); + self.note_operation_metrics(OperationMetricsDelta { + git_output_record_count: saturating_u64_from_usize(output_record_count), + ..OperationMetricsDelta::default() + }); let mut paths = Vec::new(); for raw in output.stdout.split(|byte| *byte == 0) { if raw.is_empty() { @@ -126,6 +193,11 @@ impl Trail { paths.push(path); } paths.sort(); + self.note_operation_metrics(OperationMetricsDelta { + input_path_count: saturating_u64_from_usize(output_record_count), + canonical_path_count: saturating_u64_from_usize(paths.len()), + ..OperationMetricsDelta::default() + }); Ok(Some(paths)) } @@ -140,12 +212,20 @@ impl Trail { ) -> Result> { let root = root.canonicalize()?; let mut files = BTreeMap::new(); + let mut filesystem_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta::default(), + ); for path in paths { let path = normalize_relative_path(path)?; if is_default_ignored(&path) { continue; } let abs = safe_join(&root, &path)?; + filesystem_metrics.delta.filesystem_stat_count = filesystem_metrics + .delta + .filesystem_stat_count + .saturating_add(1); let metadata = match fs::symlink_metadata(&abs) { Ok(metadata) => metadata, Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, @@ -155,11 +235,13 @@ impl Trail { continue; } if metadata.is_file() { + let (bytes, metadata) = + read_selected_regular_file(abs, self.operation_metrics.as_ref())?; files.insert( path.clone(), DiskFile { path, - bytes: fs::read(&abs)?, + bytes, executable: executable_from_metadata(&metadata), }, ); @@ -168,14 +250,23 @@ impl Trail { &root, &abs, self.config.recording.ignore_gitignored, + self.operation_metrics.as_ref(), &mut files, )?; } } + filesystem_metrics.delta.expanded_path_count = saturating_u64_from_usize(files.len()); Ok(files.into_values().collect()) } pub(crate) fn scan_files_under(&self, root: &Path) -> Result> { + let mut filesystem_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta { + full_filesystem_walk_count: 1, + ..OperationMetricsDelta::default() + }, + ); let root = root.canonicalize()?; let mut builder = WalkBuilder::new(&root); builder @@ -184,6 +275,7 @@ impl Trail { .git_exclude(self.config.recording.ignore_gitignored) .git_global(self.config.recording.ignore_gitignored) .add_custom_ignore_filename(".trailignore"); + note_walkbuilder_policy_build(self.operation_metrics.as_ref()); let walker = builder.build(); let mut files = Vec::new(); for item in walker { @@ -192,6 +284,10 @@ impl Trail { if path == root { continue; } + filesystem_metrics.delta.filesystem_entry_count = filesystem_metrics + .delta + .filesystem_entry_count + .saturating_add(1); let rel = path .strip_prefix(&root) .map_err(|err| Error::InvalidInput(err.to_string()))?; @@ -205,17 +301,27 @@ impl Trail { if is_default_ignored(&rel) { continue; } + let (bytes, metadata) = + read_selected_regular_file(path.to_path_buf(), self.operation_metrics.as_ref())?; files.push(DiskFile { path: rel, - bytes: fs::read(path)?, - executable: executable(path)?, + bytes, + executable: executable_from_metadata(&metadata), }); } files.sort_by(|left, right| left.path.cmp(&right.path)); + filesystem_metrics.delta.expanded_path_count = saturating_u64_from_usize(files.len()); Ok(files) } fn scan_file_paths_under(&self, root: &Path) -> Result { + let mut filesystem_metrics = OperationMetricsAccumulator::new( + self.operation_metrics.as_ref(), + OperationMetricsDelta { + full_filesystem_walk_count: 1, + ..OperationMetricsDelta::default() + }, + ); let root = root.canonicalize()?; let mut builder = WalkBuilder::new(&root); builder @@ -224,6 +330,7 @@ impl Trail { .git_exclude(self.config.recording.ignore_gitignored) .git_global(self.config.recording.ignore_gitignored) .add_custom_ignore_filename(".trailignore"); + note_walkbuilder_policy_build(self.operation_metrics.as_ref()); let walker = builder.build(); let mut paths = Vec::new(); let mut total_bytes = 0u64; @@ -233,6 +340,10 @@ impl Trail { if path == root { continue; } + filesystem_metrics.delta.filesystem_entry_count = filesystem_metrics + .delta + .filesystem_entry_count + .saturating_add(1); let rel = path .strip_prefix(&root) .map_err(|err| Error::InvalidInput(err.to_string()))?; @@ -246,6 +357,10 @@ impl Trail { if is_default_ignored(&rel) { continue; } + filesystem_metrics.delta.filesystem_stat_count = filesystem_metrics + .delta + .filesystem_stat_count + .saturating_add(1); let metadata = match fs::symlink_metadata(path) { Ok(metadata) => metadata, Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, @@ -258,6 +373,7 @@ impl Trail { paths.push(rel); } paths.sort(); + filesystem_metrics.delta.expanded_path_count = saturating_u64_from_usize(paths.len()); Ok(WorktreePathScan { paths, total_bytes }) } } @@ -266,8 +382,16 @@ fn scan_files_under_selection( root: &Path, selected_root: &Path, use_git_ignores: bool, + metrics: Option<&Arc>, files: &mut BTreeMap, ) -> Result<()> { + let mut filesystem_metrics = OperationMetricsAccumulator::new( + metrics, + OperationMetricsDelta { + bounded_filesystem_walk_count: 1, + ..OperationMetricsDelta::default() + }, + ); let mut builder = WalkBuilder::new(selected_root); builder .hidden(false) @@ -275,6 +399,7 @@ fn scan_files_under_selection( .git_exclude(use_git_ignores) .git_global(use_git_ignores) .add_custom_ignore_filename(".trailignore"); + note_walkbuilder_policy_build(metrics); let walker = builder.build(); for item in walker { let entry = item.map_err(|err| Error::InvalidInput(err.to_string()))?; @@ -282,6 +407,10 @@ fn scan_files_under_selection( if path == selected_root { continue; } + filesystem_metrics.delta.filesystem_entry_count = filesystem_metrics + .delta + .filesystem_entry_count + .saturating_add(1); let rel = path .strip_prefix(root) .map_err(|err| Error::InvalidInput(err.to_string()))?; @@ -295,14 +424,354 @@ fn scan_files_under_selection( if is_default_ignored(&rel) { continue; } + let (bytes, metadata) = read_selected_regular_file(path.to_path_buf(), metrics)?; files.insert( rel.clone(), DiskFile { path: rel, - bytes: fs::read(path)?, - executable: executable(path)?, + bytes, + executable: executable_from_metadata(&metadata), }, ); } Ok(()) } + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ObservedPathKind { + RegularFile, + Directory, + Symlink, + Other, +} + +pub(crate) fn observed_exact_paths_for_candidates( + root: &Path, + candidate_paths: &[String], + case_insensitive: bool, +) -> Result> { + observed_exact_paths_for_candidates_impl(root, candidate_paths, case_insensitive) + .map(|(observed, _)| observed) +} + +#[cfg(test)] +pub(crate) fn observed_exact_paths_for_candidates_with_scan_count( + root: &Path, + candidate_paths: &[String], + case_insensitive: bool, +) -> Result<(BTreeMap, usize)> { + observed_exact_paths_for_candidates_impl(root, candidate_paths, case_insensitive) +} + +#[derive(Clone)] +struct CachedDirectoryEntry { + name: String, + path: PathBuf, + file_type: fs::FileType, +} + +#[derive(Default)] +struct CachedDirectory { + exact: BTreeMap>, + folded: BTreeMap>, +} + +fn cached_directory<'a>( + cache: &'a mut BTreeMap, + dir: &Path, + directory_scans: &mut usize, +) -> Result<&'a CachedDirectory> { + if !cache.contains_key(dir) { + *directory_scans += 1; + let mut cached = CachedDirectory::default(); + match fs::read_dir(dir) { + Ok(entries) => { + for entry in entries { + let entry = entry?; + let cached_entry = CachedDirectoryEntry { + name: entry.file_name().to_string_lossy().to_string(), + path: entry.path(), + file_type: entry.file_type()?, + }; + cached + .exact + .entry(cached_entry.name.clone()) + .or_default() + .push(cached_entry.clone()); + cached + .folded + .entry(case_insensitive_path_key(&cached_entry.name)) + .or_default() + .push(cached_entry); + } + for entries in cached.exact.values_mut() { + entries.sort_by(|left, right| left.name.cmp(&right.name)); + } + for entries in cached.folded.values_mut() { + entries.sort_by(|left, right| left.name.cmp(&right.name)); + } + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(Error::Io(err)), + } + cache.insert(dir.to_path_buf(), cached); + } + Ok(cache.get(dir).expect("cached directory exists")) +} + +fn observed_exact_paths_for_candidates_impl( + root: &Path, + candidate_paths: &[String], + case_insensitive: bool, +) -> Result<(BTreeMap, usize)> { + let root = root.canonicalize()?; + let mut observed = BTreeMap::new(); + let mut directory_cache = BTreeMap::new(); + let mut directory_scans = 0; + for candidate in candidate_paths { + let candidate = normalize_relative_path(candidate)?; + let components = path_from_rel(&candidate) + .components() + .map(|component| component.as_os_str().to_string_lossy().to_string()) + .collect::>(); + let mut parents = vec![(root.clone(), Vec::::new())]; + for (index, expected) in components.iter().enumerate() { + let is_final = index + 1 == components.len(); + let mut next_parents = Vec::new(); + for (dir, actual_components) in parents { + let directory = cached_directory(&mut directory_cache, &dir, &mut directory_scans)?; + let matches = if case_insensitive { + directory.folded.get(&case_insensitive_path_key(expected)) + } else { + directory.exact.get(expected) + } + .into_iter() + .flatten() + .map(|entry| (entry.name.clone(), entry.path.clone(), entry.file_type)) + .collect::>(); + if is_final { + for (name, _, file_type) in matches { + let mut actual = actual_components.clone(); + actual.push(name); + let path = normalize_relative_path(&actual.join("/"))?; + let kind = if file_type.is_file() { + ObservedPathKind::RegularFile + } else if file_type.is_dir() { + ObservedPathKind::Directory + } else if file_type.is_symlink() { + ObservedPathKind::Symlink + } else { + ObservedPathKind::Other + }; + observed.insert(path, kind); + } + continue; + } + + for (name, next, file_type) in matches { + if file_type.is_symlink() || !file_type.is_dir() { + return Err(Error::InvalidPath { + path: candidate.clone(), + reason: format!("parent component `{name}` is not a safe directory"), + }); + } + let canonical = next.canonicalize()?; + if !canonical.starts_with(&root) { + return Err(Error::InvalidPath { + path: candidate.clone(), + reason: "parent directory escapes the worktree".to_string(), + }); + } + let mut actual = actual_components.clone(); + actual.push(name); + next_parents.push((canonical, actual)); + } + } + if is_final { + break; + } + parents = next_parents; + if parents.is_empty() { + break; + } + } + } + Ok((observed, directory_scans)) +} + +fn read_selected_regular_file( + path: PathBuf, + metrics: Option<&Arc>, +) -> Result<(Vec, fs::Metadata)> { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC); + } + #[cfg(not(unix))] + { + if let Some(metrics) = metrics { + metrics.add(OperationMetricsDelta { + filesystem_stat_count: 1, + ..OperationMetricsDelta::default() + }); + } + let metadata = fs::symlink_metadata(&path)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(Error::InvalidInput(format!( + "selected path `{}` changed file type during scan", + path.display() + ))); + } + } + let mut file = options.open(&path)?; + if let Some(metrics) = metrics { + metrics.add(OperationMetricsDelta { + filesystem_stat_count: 1, + ..OperationMetricsDelta::default() + }); + } + let metadata = file.metadata()?; + if !metadata.is_file() { + return Err(Error::InvalidInput(format!( + "selected path `{}` changed file type during scan", + path.display() + ))); + } + let mut bytes = Vec::new(); + if let Some(metrics) = metrics { + metrics.add(OperationMetricsDelta { + filesystem_read_count: 1, + ..OperationMetricsDelta::default() + }); + } + let read_result = file.read_to_end(&mut bytes); + if let Some(metrics) = metrics { + metrics.add(OperationMetricsDelta { + filesystem_read_bytes: saturating_u64_from_usize(bytes.len()), + ..OperationMetricsDelta::default() + }); + } + read_result?; + #[cfg(not(unix))] + { + if let Some(metrics) = metrics { + metrics.add(OperationMetricsDelta { + filesystem_stat_count: 1, + ..OperationMetricsDelta::default() + }); + } + let final_metadata = fs::symlink_metadata(&path)?; + if final_metadata.file_type().is_symlink() || !final_metadata.is_file() { + return Err(Error::InvalidInput(format!( + "selected path `{}` changed file type during scan", + path.display() + ))); + } + } + Ok((bytes, metadata)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(unix)] + #[test] + fn selected_regular_file_read_rejects_symlink_swap() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("target.txt"), "target\n").unwrap(); + std::os::unix::fs::symlink("target.txt", temp.path().join("selected.txt")).unwrap(); + + assert!(read_selected_regular_file(temp.path().join("selected.txt"), None).is_err()); + } + + #[test] + fn exact_path_observation_enumerates_one_shared_parent_once() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("present.txt"), "present\n").unwrap(); + let candidates = (0..10_000) + .map(|index| format!("candidate-{index}.txt")) + .chain(std::iter::once("present.txt".to_string())) + .collect::>(); + + let (observed, directory_scans) = + observed_exact_paths_for_candidates_with_scan_count(temp.path(), &candidates, false) + .unwrap(); + + assert_eq!(directory_scans, 1); + assert_eq!( + observed.get("present.txt"), + Some(&ObservedPathKind::RegularFile) + ); + } + + #[test] + fn exact_path_observation_preserves_nested_parent_and_file_spelling() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir(temp.path().join("Src")).unwrap(); + fs::write(temp.path().join("Src/readme.md"), "present\n").unwrap(); + + let observed = + observed_exact_paths_for_candidates(temp.path(), &["src/README.md".to_string()], true) + .unwrap(); + + assert_eq!( + observed.get("Src/readme.md"), + Some(&ObservedPathKind::RegularFile) + ); + assert_eq!(observed.len(), 1); + } + + #[cfg(unix)] + #[test] + fn exact_path_observation_rejects_symlinked_parent() { + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + fs::write(outside.path().join("file.txt"), "outside\n").unwrap(); + std::os::unix::fs::symlink(outside.path(), root.path().join("linked")).unwrap(); + + let err = observed_exact_paths_for_candidates( + root.path(), + &["linked/file.txt".to_string()], + false, + ) + .unwrap_err(); + assert!(err.to_string().contains("safe directory")); + } + + #[test] + fn walkbuilder_scans_report_each_policy_build() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir_all(temp.path().join("src/nested")).unwrap(); + fs::write(temp.path().join("src/nested/file.txt"), "visible\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + let metrics = db.operation_metrics.as_ref().unwrap(); + + profile_operation_metrics(Some(metrics), OperationMetricsKind::StatusReadOnly, || { + db.scan_files_under(temp.path())?; + Ok::<(), Error>(()) + }) + .unwrap(); + let full = operation_metrics_report(Some(metrics)).unwrap(); + assert_eq!(full.full_filesystem_walk_count, 1); + assert_eq!(full.policy_build_count, 1); + assert_eq!(full.policy_dependency_file_count, 0); + assert_eq!(full.policy_dependency_bytes, 0); + + profile_operation_metrics(Some(metrics), OperationMetricsKind::Status, || { + let policy = db.workspace_ignore_policy_snapshot(); + db.scan_visible_files_for_paths_with_policy(&["src".to_string()], &policy)?; + Ok::<(), Error>(()) + }) + .unwrap(); + let bounded = operation_metrics_report(Some(metrics)).unwrap(); + assert_eq!(bounded.bounded_filesystem_walk_count, 1); + assert_eq!(bounded.policy_build_count, 2); + assert_eq!(bounded.policy_dependency_file_count, 1); + assert!(bounded.policy_dependency_bytes > 0); + } +} diff --git a/trail/src/db/util/config/entries.rs b/trail/src/db/util/config/entries.rs index a5585e3..1b411e7 100644 --- a/trail/src/db/util/config/entries.rs +++ b/trail/src/db/util/config/entries.rs @@ -3,6 +3,12 @@ use super::*; pub(crate) fn config_entries_from(config: &TrailConfig) -> Vec { vec![ config_entry("workspace.id", &config.workspace.id.0, "string", true), + config_entry( + "agent.default_provider", + config.agent.default_provider.as_deref().unwrap_or_default(), + "string", + false, + ), config_entry( "workspace.default_branch", &config.workspace.default_branch, diff --git a/trail/src/db/util/config/set.rs b/trail/src/db/util/config/set.rs index c0e1ef1..7368493 100644 --- a/trail/src/db/util/config/set.rs +++ b/trail/src/db/util/config/set.rs @@ -10,6 +10,21 @@ pub(crate) fn set_config_value( "workspace.id" => Err(Error::InvalidInput( "config key `workspace.id` is read-only".to_string(), )), + "agent.default_provider" => { + let provider = value.trim(); + if provider.is_empty() + || !provider + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + { + return Err(Error::InvalidInput( + "agent.default_provider must use lowercase ASCII letters, digits, and hyphens" + .to_string(), + )); + } + config.agent.default_provider = Some(provider.to_string()); + Ok(()) + } "workspace.default_branch" => { validate_ref_segment(value)?; if db.try_get_ref(&branch_ref(value))?.is_none() { diff --git a/trail/src/db/util/executable.rs b/trail/src/db/util/executable.rs index 56f7a46..0e44192 100644 --- a/trail/src/db/util/executable.rs +++ b/trail/src/db/util/executable.rs @@ -1,22 +1,11 @@ use super::*; -#[cfg(unix)] -pub(crate) fn executable(path: &Path) -> Result { - use std::os::unix::fs::PermissionsExt; - Ok(fs::metadata(path)?.permissions().mode() & 0o111 != 0) -} - #[cfg(unix)] pub(crate) fn executable_from_metadata(metadata: &fs::Metadata) -> bool { use std::os::unix::fs::PermissionsExt; metadata.permissions().mode() & 0o111 != 0 } -#[cfg(not(unix))] -pub(crate) fn executable(_path: &Path) -> Result { - Ok(false) -} - #[cfg(not(unix))] pub(crate) fn executable_from_metadata(_metadata: &fs::Metadata) -> bool { false diff --git a/trail/src/db/util/file_identity.rs b/trail/src/db/util/file_identity.rs new file mode 100644 index 0000000..9780e64 --- /dev/null +++ b/trail/src/db/util/file_identity.rs @@ -0,0 +1,67 @@ +use super::*; +use std::os::windows::ffi::OsStrExt; +use winapi::um::fileapi::{ + CreateFileW, GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION, OPEN_EXISTING, +}; +use winapi::um::handleapi::{CloseHandle, INVALID_HANDLE_VALUE}; +use winapi::um::winbase::{FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT}; +use winapi::um::winnt::{ + FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct WindowsFileIdentity { + pub(crate) attributes: u32, + pub(crate) volume_serial_number: u32, + pub(crate) file_index: u64, + pub(crate) number_of_links: u32, + pub(crate) creation_time: u64, + pub(crate) last_write_time: u64, +} + +pub(crate) fn windows_file_identity(path: &Path) -> Result { + let wide = path + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect::>(); + let handle = unsafe { + CreateFileW( + wide.as_ptr(), + FILE_READ_ATTRIBUTES, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + std::ptr::null_mut(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, + std::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(Error::Io(std::io::Error::last_os_error())); + } + let mut information = unsafe { std::mem::zeroed::() }; + let succeeded = unsafe { GetFileInformationByHandle(handle, &mut information) } != 0; + let get_error = (!succeeded).then(std::io::Error::last_os_error); + let close_result = unsafe { CloseHandle(handle) }; + if let Some(error) = get_error { + return Err(Error::Io(error)); + } + if close_result == 0 { + return Err(Error::Io(std::io::Error::last_os_error())); + } + let filetime = |high: u32, low: u32| (u64::from(high) << 32) | u64::from(low); + Ok(WindowsFileIdentity { + attributes: information.dwFileAttributes, + volume_serial_number: information.dwVolumeSerialNumber, + file_index: filetime(information.nFileIndexHigh, information.nFileIndexLow), + number_of_links: information.nNumberOfLinks, + creation_time: filetime( + information.ftCreationTime.dwHighDateTime, + information.ftCreationTime.dwLowDateTime, + ), + last_write_time: filetime( + information.ftLastWriteTime.dwHighDateTime, + information.ftLastWriteTime.dwLowDateTime, + ), + }) +} diff --git a/trail/src/db/util/fs_cow.rs b/trail/src/db/util/fs_cow.rs index 9852c21..4d350d1 100644 --- a/trail/src/db/util/fs_cow.rs +++ b/trail/src/db/util/fs_cow.rs @@ -5,7 +5,19 @@ use sha2::{Digest, Sha256}; pub(crate) enum WorkspaceCowMaterializeStatus { Cloned(WorkdirFileStamp), Skipped, - Unavailable, + Unavailable(NativeCloneUnavailable), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum NativeCloneUnavailable { + Unsupported, + CrossDevice, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum NativeCloneOutcome { + Cloned, + Unavailable(NativeCloneUnavailable), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -23,17 +35,12 @@ pub(crate) fn clone_or_copy_projected_file( if let Some(parent) = destination.parent() { fs::create_dir_all(parent)?; } - match cow_clone_file(source, destination) { - Ok(true) => Ok(FileProjectionCopy::Cloned), - Ok(false) => { - let _ = fs::remove_file(destination); + match clone_file_native(source, destination)? { + NativeCloneOutcome::Cloned => Ok(FileProjectionCopy::Cloned), + NativeCloneOutcome::Unavailable(_) => { fs::copy(source, destination)?; Ok(FileProjectionCopy::Copied) } - Err(err) => { - let _ = fs::remove_file(destination); - Err(Error::Io(err)) - } } } @@ -82,7 +89,7 @@ pub(crate) fn materialize_from_workspace_cow_report( WorkspaceCowMaterializeStatus::Cloned(stamp) => { report.insert_stamp(first_path.clone(), stamp); } - WorkspaceCowMaterializeStatus::Skipped | WorkspaceCowMaterializeStatus::Unavailable => { + WorkspaceCowMaterializeStatus::Skipped | WorkspaceCowMaterializeStatus::Unavailable(_) => { return Ok(None); } } @@ -118,7 +125,8 @@ pub(crate) fn materialize_from_workspace_cow_report( } Ok(( _, - WorkspaceCowMaterializeStatus::Skipped | WorkspaceCowMaterializeStatus::Unavailable, + WorkspaceCowMaterializeStatus::Skipped + | WorkspaceCowMaterializeStatus::Unavailable(_), )) => { rejected = true; } @@ -144,6 +152,22 @@ pub(crate) fn materialize_workspace_file_cow_status_if_matching( output_root: &Path, path: &str, entry: &FileEntry, +) -> Result { + materialize_workspace_file_cow_status_if_matching_with_durability( + workspace_root, + output_root, + path, + entry, + false, + ) +} + +pub(crate) fn materialize_workspace_file_cow_status_if_matching_with_durability( + workspace_root: &Path, + output_root: &Path, + path: &str, + entry: &FileEntry, + durable: bool, ) -> Result { let source = safe_join(workspace_root, path)?; let destination = safe_join(output_root, path)?; @@ -156,12 +180,22 @@ pub(crate) fn materialize_workspace_file_cow_status_if_matching( return Ok(WorkspaceCowMaterializeStatus::Skipped); } if let Some(parent) = destination.parent() { - fs::create_dir_all(parent)?; + if durable { + create_dir_all_durable(parent)?; + } else { + fs::create_dir_all(parent)?; + } } - clone_file_cow_clean(&source, &destination, entry.executable, false) + clone_file_cow_clean( + &source, + &destination, + entry.executable, + &entry.content_hash, + durable, + ) } -fn materialize_workspace_file_cow_status_if_stamp_matches( +pub(crate) fn materialize_workspace_file_cow_status_if_stamp_matches( workspace_root: &Path, output_root: &Path, path: &str, @@ -180,9 +214,19 @@ fn materialize_workspace_file_cow_status_if_stamp_matches( return Ok(WorkspaceCowMaterializeStatus::Skipped); } if let Some(parent) = destination.parent() { - fs::create_dir_all(parent)?; + if durable { + create_dir_all_durable(parent)?; + } else { + fs::create_dir_all(parent)?; + } } - clone_file_cow_clean(&source, &destination, entry.executable, durable) + clone_file_cow_clean( + &source, + &destination, + entry.executable, + &entry.content_hash, + durable, + ) } fn workspace_file_matches_entry(source: &Path, entry: &FileEntry) -> Result { @@ -228,17 +272,24 @@ fn clone_file_cow_clean( source: &Path, destination: &Path, executable: bool, + expected_content_hash: &str, durable: bool, ) -> Result { - match cow_clone_file(source, destination) { - Ok(true) => { + match clone_file_native(source, destination)? { + NativeCloneOutcome::Cloned => { let result = (|| -> Result { set_executable(destination, executable)?; let clean = clear_cloned_xattrs(destination)?; + if clean && sha256_file_hex(destination)? != expected_content_hash { + return Err(Error::Corrupt(format!( + "native clone `{}` does not match the immutable root", + destination.display() + ))); + } if clean && durable { sync_cloned_file(destination)?; if let Some(parent) = destination.parent() { - sync_directory(parent); + sync_directory_strict(parent)?; } } Ok(clean) @@ -252,16 +303,13 @@ fn clone_file_cow_clean( WorkdirFileStamp::from_metadata(&metadata), )) } else { - Ok(WorkspaceCowMaterializeStatus::Unavailable) + Ok(WorkspaceCowMaterializeStatus::Unavailable( + NativeCloneUnavailable::Unsupported, + )) } } - Ok(false) => { - let _ = fs::remove_file(destination); - Ok(WorkspaceCowMaterializeStatus::Unavailable) - } - Err(err) => { - let _ = fs::remove_file(destination); - Err(Error::Io(err)) + NativeCloneOutcome::Unavailable(reason) => { + Ok(WorkspaceCowMaterializeStatus::Unavailable(reason)) } } } @@ -274,7 +322,7 @@ fn remove_cow_attempt_files(output_root: &Path, paths: &[String]) { } } -fn sha256_file_hex(path: &Path) -> Result { +pub(crate) fn sha256_file_hex(path: &Path) -> Result { let mut file = fs::File::open(path)?; let mut hasher = Sha256::new(); let mut buffer = [0_u8; 64 * 1024]; @@ -289,7 +337,7 @@ fn sha256_file_hex(path: &Path) -> Result { } #[cfg(any(target_os = "macos", target_os = "ios"))] -fn cow_clone_file(source: &Path, destination: &Path) -> std::io::Result { +pub(crate) fn clone_file_native(source: &Path, destination: &Path) -> Result { use rustix::fs::{fclonefileat, CloneFlags}; let source_file = fs::File::open(source)?; @@ -306,23 +354,29 @@ fn cow_clone_file(source: &Path, destination: &Path) -> std::io::Result { ) })?; let parent_dir = fs::File::open(parent)?; - match fclonefileat( + let result = match fclonefileat( &source_file, &parent_dir, file_name, CloneFlags::NOFOLLOW | CloneFlags::NOOWNERCOPY, ) { - Ok(()) => Ok(true), - Err(err) if cow_clone_unavailable(err) => Ok(false), - Err(err) => Err(err.into()), + Ok(()) => Ok(NativeCloneOutcome::Cloned), + Err(err) => match native_clone_unavailable(err) { + Some(reason) => Ok(NativeCloneOutcome::Unavailable(reason)), + None => Err(Error::Io(err.into())), + }, + }; + if !matches!(result, Ok(NativeCloneOutcome::Cloned)) { + let _ = fs::remove_file(destination); } + result } #[cfg(all( target_os = "linux", not(any(target_arch = "sparc", target_arch = "sparc64")) ))] -fn cow_clone_file(source: &Path, destination: &Path) -> std::io::Result { +pub(crate) fn clone_file_native(source: &Path, destination: &Path) -> Result { use rustix::fs::ioctl_ficlone; let source_file = fs::File::open(source)?; @@ -330,11 +384,18 @@ fn cow_clone_file(source: &Path, destination: &Path) -> std::io::Result { .write(true) .create_new(true) .open(destination)?; - match ioctl_ficlone(&destination_file, &source_file) { - Ok(()) => Ok(true), - Err(err) if cow_clone_unavailable(err) => Ok(false), - Err(err) => Err(err.into()), + let result = match ioctl_ficlone(&destination_file, &source_file) { + Ok(()) => Ok(NativeCloneOutcome::Cloned), + Err(err) => match native_clone_unavailable(err) { + Some(reason) => Ok(NativeCloneOutcome::Unavailable(reason)), + None => Err(Error::Io(err.into())), + }, + }; + if !matches!(result, Ok(NativeCloneOutcome::Cloned)) { + drop(destination_file); + let _ = fs::remove_file(destination); } + result } #[cfg(not(any( @@ -345,8 +406,10 @@ fn cow_clone_file(source: &Path, destination: &Path) -> std::io::Result { not(any(target_arch = "sparc", target_arch = "sparc64")) ) )))] -fn cow_clone_file(_source: &Path, _destination: &Path) -> std::io::Result { - Ok(false) +pub(crate) fn clone_file_native(_source: &Path, _destination: &Path) -> Result { + Ok(NativeCloneOutcome::Unavailable( + NativeCloneUnavailable::Unsupported, + )) } #[cfg(any( @@ -365,7 +428,7 @@ fn clear_cloned_xattrs(path: &Path) -> std::io::Result { let mut empty: [u8; 0] = []; let size = match listxattr(path, &mut empty) { Ok(size) => size, - Err(err) if cow_clone_unavailable(err) => return Ok(false), + Err(err) if native_clone_unavailable(err).is_some() => return Ok(false), Err(err) => return Err(err.into()), }; if size == 0 { @@ -375,7 +438,7 @@ fn clear_cloned_xattrs(path: &Path) -> std::io::Result { let mut names = vec![0; size]; let size = match listxattr(path, &mut names) { Ok(size) => size, - Err(err) if cow_clone_unavailable(err) => return Ok(false), + Err(err) if native_clone_unavailable(err).is_some() => return Ok(false), Err(err) => return Err(err.into()), }; names.truncate(size); @@ -391,7 +454,7 @@ fn clear_cloned_xattrs(path: &Path) -> std::io::Result { })?; match removexattr(path, name.as_c_str()) { Ok(()) => {} - Err(err) if cow_clone_unavailable(err) => return Ok(false), + Err(err) if native_clone_unavailable(err).is_some() => return Ok(false), Err(err) => return Err(err.into()), } } @@ -418,25 +481,50 @@ fn clear_cloned_xattrs(_path: &Path) -> std::io::Result { not(any(target_arch = "sparc", target_arch = "sparc64")) ) ))] -fn cow_clone_unavailable(err: rustix::io::Errno) -> bool { - err == rustix::io::Errno::NOTSUP +fn native_clone_unavailable(err: rustix::io::Errno) -> Option { + if err == rustix::io::Errno::XDEV { + return Some(NativeCloneUnavailable::CrossDevice); + } + if err == rustix::io::Errno::NOTSUP || err == rustix::io::Errno::OPNOTSUPP || matches!( err, - rustix::io::Errno::NOSYS - | rustix::io::Errno::XDEV - | rustix::io::Errno::INVAL - | rustix::io::Errno::PERM - | rustix::io::Errno::ACCESS + rustix::io::Errno::NOSYS | rustix::io::Errno::INVAL | rustix::io::Errno::NOTTY ) + { + return Some(NativeCloneUnavailable::Unsupported); + } + None } #[cfg(test)] mod tests { use super::*; + #[cfg(any( + target_os = "macos", + target_os = "ios", + all( + target_os = "linux", + not(any(target_arch = "sparc", target_arch = "sparc64")) + ) + ))] + #[test] + fn clone_classifier_keeps_permissions_as_hard_errors() { + assert_eq!(native_clone_unavailable(rustix::io::Errno::PERM), None); + assert_eq!(native_clone_unavailable(rustix::io::Errno::ACCESS), None); + assert_eq!( + native_clone_unavailable(rustix::io::Errno::XDEV), + Some(NativeCloneUnavailable::CrossDevice) + ); + assert_eq!( + native_clone_unavailable(rustix::io::Errno::NOTTY), + Some(NativeCloneUnavailable::Unsupported) + ); + } + fn parallel_cow_clone_test_entry(bytes: &[u8]) -> FileEntry { - let change = ChangeId("ch_test".to_string()); + let change = ChangeId("change_test".to_string()); FileEntry { file_id: FileId::new(change.clone(), 1), kind: FileKind::Text, @@ -530,4 +618,51 @@ mod tests { assert!(!output.path().join("a.txt").exists()); assert!(!output.path().join("b.txt").exists()); } + + #[test] + fn durable_matching_clone_creates_nested_directories() { + let workspace = tempfile::tempdir().unwrap(); + let output = tempfile::tempdir().unwrap(); + let bytes = b"nested contents\n"; + fs::create_dir_all(workspace.path().join("one/two")).unwrap(); + fs::write(workspace.path().join("one/two/file.txt"), bytes).unwrap(); + let entry = parallel_cow_clone_test_entry(bytes); + + let _ = materialize_workspace_file_cow_status_if_matching_with_durability( + workspace.path(), + output.path(), + "one/two/file.txt", + &entry, + true, + ) + .unwrap(); + + assert!(output.path().join("one/two").is_dir()); + } + + #[cfg(unix)] + #[test] + fn durable_matching_clone_rejects_symlinked_directory_component() { + use std::os::unix::fs::symlink; + + let workspace = tempfile::tempdir().unwrap(); + let output = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let bytes = b"nested contents\n"; + fs::create_dir_all(workspace.path().join("nested")).unwrap(); + fs::write(workspace.path().join("nested/file.txt"), bytes).unwrap(); + symlink(outside.path(), output.path().join("nested")).unwrap(); + let entry = parallel_cow_clone_test_entry(bytes); + + let result = materialize_workspace_file_cow_status_if_matching_with_durability( + workspace.path(), + output.path(), + "nested/file.txt", + &entry, + true, + ); + + assert!(result.is_err()); + assert!(!outside.path().join("file.txt").exists()); + } } diff --git a/trail/src/db/util/ids.rs b/trail/src/db/util/ids.rs index 7e029f8..0bbea7c 100644 --- a/trail/src/db/util/ids.rs +++ b/trail/src/db/util/ids.rs @@ -33,18 +33,9 @@ pub(crate) fn line_id_key_value(line_id: &LineId) -> String { } pub(crate) fn parse_line_id_key(value: &str) -> Result { - let (change_id, local_seq) = value.rsplit_once(':').ok_or_else(|| { - Error::InvalidInput("line id must look like `ch_...:`".to_string()) - })?; - if !change_id.starts_with("ch_") { - return Err(Error::InvalidInput(format!( - "line id change id must start with `ch_`, got `{change_id}`" - ))); - } - let local_seq = local_seq.parse::().map_err(|_| { - Error::InvalidInput(format!("invalid line id local sequence `{local_seq}`")) - })?; - Ok(LineId::new(ChangeId(change_id.to_string()), local_seq)) + LineId::from_alias(value).ok_or_else(|| { + Error::InvalidInput("line id must look like `line_...:`".to_string()) + }) } pub(crate) trait LineChangeExt { diff --git a/trail/src/db/util/ignore.rs b/trail/src/db/util/ignore.rs index 068544f..33a7b4b 100644 --- a/trail/src/db/util/ignore.rs +++ b/trail/src/db/util/ignore.rs @@ -1,5 +1,25 @@ use super::*; +pub(crate) fn is_ignore_policy_path(path: &str) -> bool { + path_from_rel(path) + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| matches!(name, ".trailignore" | ".gitignore")) +} + +/// Counts construction of a configured ignore walker. The ignore crate does +/// not expose the dynamically discovered ignore files or bytes loaded by a +/// walker, so dependency-file and dependency-byte metrics remain exact only +/// for the root files explicitly probed by `WorkspaceIgnorePolicySnapshot`. +pub(crate) fn note_walkbuilder_policy_build(metrics: Option<&Arc>) { + if let Some(metrics) = metrics { + metrics.add(OperationMetricsDelta { + policy_build_count: 1, + ..OperationMetricsDelta::default() + }); + } +} + pub(crate) fn write_default_trailignore(workspace_root: &Path) -> Result<()> { let path = workspace_root.join(".trailignore"); if path.exists() { diff --git a/trail/src/db/util/line_ops.rs b/trail/src/db/util/line_ops.rs index 7f1f29c..06b867c 100644 --- a/trail/src/db/util/line_ops.rs +++ b/trail/src/db/util/line_ops.rs @@ -61,7 +61,7 @@ pub(crate) fn materialize_lines(lines: &[LineEntry]) -> Vec { pub(crate) fn encode_small_text_table(lines: &[LineEntry]) -> Vec { let default_origin_change = most_common_origin_change(lines) - .unwrap_or_else(|| ChangeId("ch_empty_small_text_table".to_string())); + .unwrap_or_else(|| ChangeId("change_empty_small_text_table".to_string())); let mut out = Vec::new(); out.push(SMALL_TEXT_TABLE_VERSION); write_change_id(&mut out, &default_origin_change); @@ -231,7 +231,7 @@ fn decode_newline(code: u8) -> Result { } fn write_change_id(out: &mut Vec, change_id: &ChangeId) { - if let Some(hex_id) = change_id.0.strip_prefix("ch_") { + if let Some(hex_id) = change_id.0.strip_prefix(crate::ids::CHANGE_ID_PREFIX) { if hex_id.len() == 64 { if let Ok(bytes) = hex::decode(hex_id) { out.push(0); @@ -248,7 +248,11 @@ fn read_change_id(data: &[u8], cursor: &mut usize) -> Result { match read_u8(data, cursor)? { 0 => { let bytes = read_exact(data, cursor, 32)?; - Ok(ChangeId(format!("ch_{}", hex::encode(bytes)))) + Ok(ChangeId(format!( + "{}{}", + crate::ids::CHANGE_ID_PREFIX, + hex::encode(bytes) + ))) } 1 => { let bytes = read_len_bytes(data, cursor)?; @@ -525,7 +529,7 @@ mod tests { } fn change_id(byte: u8) -> ChangeId { - ChangeId(format!("ch_{}", hex::encode([byte; 32]))) + ChangeId(format!("change_{}", hex::encode([byte; 32]))) } fn line( diff --git a/trail/src/db/util/map_inspect.rs b/trail/src/db/util/map_inspect.rs index c82b749..1a290c9 100644 --- a/trail/src/db/util/map_inspect.rs +++ b/trail/src/db/util/map_inspect.rs @@ -65,11 +65,11 @@ pub(crate) fn parse_map_key_spec(spec: &str) -> Result> { pub(crate) fn parse_compound_map_key(spec: &str) -> Result> { let (change_id, local_seq) = spec.rsplit_once(':').ok_or_else(|| { - Error::InvalidInput("compound map key must look like id:ch_...:".to_string()) + Error::InvalidInput("compound map key must look like id:change_...:".to_string()) })?; - if !change_id.starts_with("ch_") { + if !crate::ids::is_change_id(change_id) { return Err(Error::InvalidInput( - "compound map key change id must start with ch_".to_string(), + "compound map key change id must start with change_".to_string(), )); } let local_seq = local_seq.parse::().map_err(|_| { @@ -160,7 +160,7 @@ pub(crate) fn path_map_value_summary(value: &[u8]) -> serde_json::Value { pub(crate) fn text_order_value_summary(value: &[u8]) -> serde_json::Value { match decode_cbor_value::(value) { Ok(entry) => serde_json::json!({ - "line_id": line_id_key_value(&entry.line_id), + "line_id": entry.line_id.alias(), "text_hash": entry.text_hash, "text": utf8_preview(&entry.text, 240), "newline": entry.newline, diff --git a/trail/src/db/util/materialize.rs b/trail/src/db/util/materialize.rs index 7ece551..d542e9f 100644 --- a/trail/src/db/util/materialize.rs +++ b/trail/src/db/util/materialize.rs @@ -95,8 +95,25 @@ where for path in previous.keys() { if !target.contains_key(path) { let abs = safe_join(output_root, path)?; - if abs.exists() { - fs::remove_file(abs)?; + match fs::symlink_metadata(&abs) { + Ok(metadata) => { + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(Error::InvalidPath { + path: path.clone(), + reason: "path selected for deletion is not a regular file".into(), + }); + } + fs::remove_file(&abs)?; + if durable { + let parent = abs.parent().ok_or_else(|| Error::InvalidPath { + path: path.clone(), + reason: "path has no parent directory".into(), + })?; + sync_directory_strict(parent)?; + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(Error::Io(error)), } } } @@ -142,7 +159,11 @@ where for (path, entry) in batch { let abs = safe_join(output_root, path)?; if let Some(parent) = abs.parent() { - fs::create_dir_all(parent)?; + if durable { + create_dir_all_durable(parent)?; + } else { + fs::create_dir_all(parent)?; + } } let Some(file_bytes) = bytes.get(path) else { return Err(Error::Corrupt(format!( @@ -190,7 +211,7 @@ where Ok(()) } -fn case_insensitive_path_key(path: &str) -> String { +pub(crate) fn case_insensitive_path_key(path: &str) -> String { let folded: String = path.nfkc().flat_map(char::to_lowercase).collect(); folded.nfc().collect() } @@ -200,7 +221,11 @@ pub(crate) fn write_file_atomic(path: &Path, bytes: &[u8], durable: bool) -> Res path: path.to_string_lossy().to_string(), reason: "path has no parent directory".to_string(), })?; - fs::create_dir_all(parent)?; + if durable { + create_dir_all_durable(parent)?; + } else { + fs::create_dir_all(parent)?; + } let (tmp, mut file) = create_materialize_temp_file(parent, path)?; let result = (|| -> Result<()> { @@ -211,7 +236,7 @@ pub(crate) fn write_file_atomic(path: &Path, bytes: &[u8], durable: bool) -> Res drop(file); fs::rename(&tmp, path)?; if durable { - sync_directory(parent); + sync_directory_strict(parent)?; } Ok(()) })(); @@ -329,19 +354,26 @@ fn write_materialized_file_with_durability( path: rel.to_string(), reason: "path has no parent directory".to_string(), })?; - fs::create_dir_all(parent)?; + if durable { + create_dir_all_durable(parent)?; + } else { + fs::create_dir_all(parent)?; + } let (tmp, mut file) = create_materialize_temp_file(parent, path)?; let result = (|| -> Result { file.write_all(bytes)?; + // Mode is part of the authoritative file image. Apply it before the + // file durability barrier so a crash cannot publish bytes with the + // previous/default executable bit. + set_executable(&tmp, executable)?; if durable { file.sync_all()?; } drop(file); - set_executable(&tmp, executable)?; fs::rename(&tmp, path)?; if durable { - sync_directory(parent); + sync_directory_strict(parent)?; } let metadata = fs::symlink_metadata(path)?; Ok(WorkdirFileStamp::from_metadata(&metadata)) @@ -378,3 +410,59 @@ pub(crate) fn sync_directory(path: &Path) { let _ = dir.sync_all(); } } + +pub(crate) fn sync_directory_strict(path: &Path) -> Result<()> { + OpenOptions::new().read(true).open(path)?.sync_all()?; + Ok(()) +} + +/// Create every missing path component and durably publish each directory +/// entry. This is intentionally used only by strict materialization paths; +/// legacy best-effort projections retain their previous behavior. +pub(crate) fn create_dir_all_durable(path: &Path) -> Result<()> { + let mut missing = Vec::new(); + let mut cursor = path; + loop { + match fs::symlink_metadata(cursor) { + Ok(metadata) => { + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(Error::InvalidPath { + path: path.to_string_lossy().to_string(), + reason: "directory path contains a non-directory or symlink".into(), + }); + } + break; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + missing.push(cursor.to_path_buf()); + cursor = cursor.parent().ok_or_else(|| Error::InvalidPath { + path: path.to_string_lossy().to_string(), + reason: "directory path has no existing ancestor".into(), + })?; + } + Err(error) => return Err(Error::Io(error)), + } + } + for directory in missing.into_iter().rev() { + match fs::create_dir(&directory) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + let metadata = fs::symlink_metadata(&directory)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(Error::InvalidPath { + path: directory.to_string_lossy().to_string(), + reason: "concurrently created path is not a directory".into(), + }); + } + } + Err(error) => return Err(Error::Io(error)), + } + sync_directory_strict(&directory)?; + let parent = directory.parent().ok_or_else(|| Error::InvalidPath { + path: directory.to_string_lossy().to_string(), + reason: "created directory has no parent".into(), + })?; + sync_directory_strict(parent)?; + } + Ok(()) +} diff --git a/trail/src/db/util/mod.rs b/trail/src/db/util/mod.rs index 13070d4..93af14f 100644 --- a/trail/src/db/util/mod.rs +++ b/trail/src/db/util/mod.rs @@ -8,6 +8,8 @@ mod config_parse; mod conflicts; mod diff_summary; mod executable; +#[cfg(windows)] +mod file_identity; mod file_kind; mod fs_cow; mod gates; @@ -25,7 +27,8 @@ mod parsing; mod patch; mod path; mod previews; -mod process_liveness; +#[doc(hidden)] +pub mod process_liveness; mod prolly; mod readiness_reports; mod redaction; @@ -44,6 +47,8 @@ pub(crate) use self::config_parse::*; pub(crate) use self::conflicts::*; pub(crate) use self::diff_summary::*; pub(crate) use self::executable::*; +#[cfg(windows)] +pub(crate) use self::file_identity::*; pub(crate) use self::file_kind::*; pub(crate) use self::fs_cow::*; pub(crate) use self::gates::*; diff --git a/trail/src/db/util/parsing.rs b/trail/src/db/util/parsing.rs index 8c02a51..7ec659d 100644 --- a/trail/src/db/util/parsing.rs +++ b/trail/src/db/util/parsing.rs @@ -34,9 +34,10 @@ pub(crate) fn parse_session_end_status(value: &str) -> Result<&'static str> { "completed" => Ok("completed"), "failed" => Ok("failed"), "cancelled" => Ok("cancelled"), + "interrupted" => Ok("interrupted"), "archived" => Ok("archived"), other => Err(Error::InvalidInput(format!( - "session end status must be completed, failed, cancelled, or archived, got `{other}`" + "session end status must be completed, failed, cancelled, interrupted, or archived, got `{other}`" ))), } } diff --git a/trail/src/db/util/path.rs b/trail/src/db/util/path.rs index cccf68a..fbc8673 100644 --- a/trail/src/db/util/path.rs +++ b/trail/src/db/util/path.rs @@ -1,6 +1,29 @@ use super::*; use unicode_normalization::UnicodeNormalization; +pub(crate) fn canonicalize_lossless(path: &Path) -> Result { + match path.canonicalize() { + Ok(path) => Ok(path), + #[cfg(unix)] + Err(error) if path.to_str().is_none() && error.raw_os_error() == Some(libc::EILSEQ) => { + // macOS realpath(3) rejects existing names containing arbitrary + // filesystem bytes. Canonicalize the parent so symlinks above the + // opaque component are still resolved, then preserve that component + // byte-for-byte. Recursing also handles an opaque ancestor. + let name = path.file_name().ok_or_else(|| Error::Io(error))?; + let parent = path.parent().ok_or_else(|| Error::InvalidPath { + path: path.to_string_lossy().into_owned(), + reason: "path has no parent to canonicalize".into(), + })?; + let canonical_parent = canonicalize_lossless(parent)?; + let resolved = canonical_parent.join(name); + fs::metadata(&resolved)?; + Ok(resolved) + } + Err(error) => Err(Error::Io(error)), + } +} + pub(crate) fn normalize_relative_path(path: &str) -> Result { if path.as_bytes().contains(&0) { return Err(Error::InvalidPath { @@ -293,6 +316,81 @@ pub(crate) fn path_matches_selection(path: &str, selected: &str) -> bool { .is_some_and(|rest| rest.starts_with('/')) } +/// Canonical component-prefix selections used by bounded root reads and +/// selected-map membership checks. +/// +/// Stored paths and membership candidates are expected to already use Trail's +/// normalized relative-path representation. Construction normalizes user or +/// provider candidates, deduplicates them, and removes a selection only when +/// an accepted component ancestor covers it. Case-distinct paths remain +/// distinct. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct SelectionSet { + paths: Vec, + exact: BTreeSet, + descendant_ranges: Vec<(String, String)>, +} + +impl SelectionSet { + pub(crate) fn from_paths(paths: &[String]) -> Result { + let normalized = paths + .iter() + .map(|path| normalize_relative_path(path)) + .collect::>>()?; + let mut exact = BTreeSet::new(); + for path in normalized { + let covered = path + .match_indices('/') + .any(|(separator, _)| exact.contains(&path[..separator])); + if !covered { + exact.insert(path); + } + } + let paths = exact.iter().cloned().collect::>(); + let mut descendant_ranges = paths + .iter() + .map(|path| (format!("{path}/"), format!("{path}0"))) + .collect::>(); + descendant_ranges.sort_unstable_by(|left, right| left.0.cmp(&right.0)); + Ok(Self { + paths, + exact, + descendant_ranges, + }) + } + + pub(crate) fn as_slice(&self) -> &[String] { + &self.paths + } + + pub(crate) fn is_empty(&self) -> bool { + self.paths.is_empty() + } + + #[cfg(test)] + pub(crate) fn contains(&self, normalized_path: &str) -> bool { + self.contains_counted(normalized_path).0 + } + + /// Returns logical membership probes separately from manifest-entry + /// comparisons. The exact set and non-overlapping interval index each + /// contribute at most one probe; their internal tree/binary-search work is + /// logarithmic in the canonical selection count. + pub(crate) fn contains_counted(&self, normalized_path: &str) -> (bool, u64) { + if self.exact.contains(normalized_path) { + return (true, 1); + } + let interval_index = self + .descendant_ranges + .partition_point(|(lower, _)| lower.as_str() <= normalized_path); + let matches = interval_index + .checked_sub(1) + .and_then(|index| self.descendant_ranges.get(index)) + .is_some_and(|(_, upper)| normalized_path < upper.as_str()); + (matches, 2) + } +} + pub(crate) fn validate_ref_segment(name: &str) -> Result<()> { if name.is_empty() || name.contains("..") @@ -337,3 +435,56 @@ pub(crate) fn is_default_ignored(path: &str) -> bool { || file_name == "id_rsa" || file_name == "id_ed25519" } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn selection_set_normalizes_deduplicates_and_collapses_component_descendants() { + let selections = SelectionSet::from_paths(&[ + "docs/sub/deep.txt".to_string(), + "docs/./sub".to_string(), + "docs".to_string(), + "docs".to_string(), + "docs-z".to_string(), + "README.md".to_string(), + "readme.md".to_string(), + ]) + .unwrap(); + + assert_eq!( + selections.as_slice(), + ["README.md", "docs", "docs-z", "readme.md"] + ); + assert!(selections.contains("docs/sub/deep.txt")); + assert!(selections.contains("README.md")); + assert!(selections.contains("readme.md")); + assert!(!selections.contains("documentation/file.txt")); + } + + #[test] + fn selection_set_descendant_intervals_survive_lexical_sibling_boundaries() { + let selections = + SelectionSet::from_paths(&["foo".to_string(), "foo-z".to_string()]).unwrap(); + + let (foo_child, foo_comparisons) = selections.contains_counted("foo/bar.txt"); + let (foo_z_child, foo_z_comparisons) = selections.contains_counted("foo-z/bar.txt"); + + assert!(foo_child); + assert!(foo_z_child); + assert!(!selections.contains("foo-zebra/bar.txt")); + assert!(!selections.contains("foobar.txt")); + assert!(foo_comparisons <= 2); + assert!(foo_z_comparisons <= 2); + } + + #[test] + fn selection_set_rejects_invalid_paths_without_partial_output() { + let err = + SelectionSet::from_paths(&["valid/path.txt".to_string(), "../outside.txt".to_string()]) + .unwrap_err(); + + assert!(matches!(err, Error::InvalidPath { .. })); + } +} diff --git a/trail/src/db/util/process_liveness.rs b/trail/src/db/util/process_liveness.rs index 32b9e94..dc9161c 100644 --- a/trail/src/db/util/process_liveness.rs +++ b/trail/src/db/util/process_liveness.rs @@ -120,6 +120,127 @@ pub(crate) fn process_matches_start_token(pid: u32, token: &str) -> bool { process_start_token(pid).map_or(true, |actual| actual == token) } +/// Internal helper entry point used by the CLI's hidden process-watchdog mode. +/// The watchdog is a direct child of the Trail process and owns no workspace +/// state. It terminates one authenticated sandbox-helper process if its parent +/// disappears, preventing an adapter action from surviving host death. +#[doc(hidden)] +pub fn run_internal_process_watchdog( + parent_pid: u32, + child_pid: u32, + child_start_token: &str, +) -> std::result::Result<(), String> { + if child_pid == 0 || child_start_token.is_empty() { + return Err("process watchdog received an invalid child identity".to_string()); + } + if !process_matches_start_token(child_pid, child_start_token) { + return Ok(()); + } + + #[cfg(unix)] + { + let parent_pid = i32::try_from(parent_pid) + .map_err(|_| "process watchdog parent PID is out of range".to_string())?; + let child_pid = i32::try_from(child_pid) + .map_err(|_| "process watchdog child PID is out of range".to_string())?; + loop { + if !process_matches_start_token(child_pid as u32, child_start_token) { + return Ok(()); + } + // The watchdog is spawned directly by the owning Trail process. + // Unix reparents it immediately if that process exits, avoiding + // repeated process-table probes and PID-reuse ambiguity. + if unsafe { libc::getppid() } != parent_pid { + // SAFETY: the PID was range-checked and still matches the + // captured start token immediately before this signal. + if unsafe { libc::kill(child_pid, libc::SIGKILL) } != 0 { + let error = std::io::Error::last_os_error(); + if error.kind() != std::io::ErrorKind::NotFound { + return Err(format!( + "process watchdog could not terminate child {child_pid}: {error}" + )); + } + } + return Ok(()); + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + } + + #[cfg(target_os = "windows")] + { + return windows_watch_parent_and_terminate_child(parent_pid, child_pid, child_start_token); + } + + #[cfg(not(any(unix, target_os = "windows")))] + { + let _ = parent_pid; + Err("process watchdog is unavailable on this platform".to_string()) + } +} + +#[cfg(target_os = "windows")] +fn windows_watch_parent_and_terminate_child( + parent_pid: u32, + child_pid: u32, + child_start_token: &str, +) -> std::result::Result<(), String> { + use winapi::shared::minwindef::FALSE; + use winapi::shared::winerror::WAIT_TIMEOUT; + use winapi::um::handleapi::CloseHandle; + use winapi::um::processthreadsapi::{OpenProcess, TerminateProcess}; + use winapi::um::synchapi::WaitForSingleObject; + use winapi::um::winbase::WAIT_OBJECT_0; + use winapi::um::winnt::{PROCESS_QUERY_LIMITED_INFORMATION, PROCESS_TERMINATE, SYNCHRONIZE}; + + // SAFETY: handles are checked before use and closed on every return path. + unsafe { + let parent = OpenProcess(SYNCHRONIZE, FALSE, parent_pid); + if parent.is_null() { + return Err(format!( + "process watchdog cannot open parent {parent_pid}: {}", + std::io::Error::last_os_error() + )); + } + let child = OpenProcess( + SYNCHRONIZE | PROCESS_TERMINATE | PROCESS_QUERY_LIMITED_INFORMATION, + FALSE, + child_pid, + ); + if child.is_null() { + CloseHandle(parent); + return Ok(()); + } + if windows_process_start_token(child_pid).as_deref() != Some(child_start_token) { + CloseHandle(child); + CloseHandle(parent); + return Ok(()); + } + loop { + if WaitForSingleObject(child, 0) == WAIT_OBJECT_0 { + CloseHandle(child); + CloseHandle(parent); + return Ok(()); + } + match WaitForSingleObject(parent, 50) { + WAIT_OBJECT_0 => { + let _ = TerminateProcess(child, 137); + CloseHandle(child); + CloseHandle(parent); + return Ok(()); + } + WAIT_TIMEOUT => {} + _ => { + let error = std::io::Error::last_os_error(); + CloseHandle(child); + CloseHandle(parent); + return Err(format!("process watchdog wait failed: {error}")); + } + } + } + } +} + pub(crate) fn test_crash_point(name: &str) { #[cfg(test)] { diff --git a/trail/src/db/util/redaction.rs b/trail/src/db/util/redaction.rs index 9bcd4ad..425604c 100644 --- a/trail/src/db/util/redaction.rs +++ b/trail/src/db/util/redaction.rs @@ -21,7 +21,6 @@ pub(crate) fn redact_sensitive_json(value: serde_json::Value) -> serde_json::Val } } -#[cfg(test)] pub(crate) fn contains_sensitive_json(value: &serde_json::Value) -> bool { match value { serde_json::Value::Object(map) => map.iter().any(|(key, value)| { @@ -121,7 +120,6 @@ fn split_line_ending(chunk: &str) -> (&str, &str) { } } -#[cfg(test)] fn json_value_has_payload(value: &serde_json::Value) -> bool { match value { serde_json::Value::Null => false, diff --git a/trail/src/db/util/rows.rs b/trail/src/db/util/rows.rs index e68693f..ef0f7c9 100644 --- a/trail/src/db/util/rows.rs +++ b/trail/src/db/util/rows.rs @@ -62,15 +62,18 @@ pub(crate) fn lane_details_row(row: &rusqlite::Row<'_>) -> rusqlite::Result) -> rusqlite::Result { - Ok(MergeQueueEntry { +pub(crate) fn lane_merge_queue_row( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + Ok(LaneMergeQueueEntry { queue_id: row.get(0)?, - source_ref: row.get(1)?, - target_ref: row.get(2)?, - status: row.get(3)?, - priority: row.get(4)?, - created_at: row.get(5)?, - updated_at: row.get(6)?, + lane_id: row.get(1)?, + lane: row.get(2)?, + target_ref: row.get(3)?, + status: row.get(4)?, + priority: row.get(5)?, + created_at: row.get(6)?, + updated_at: row.get(7)?, }) } diff --git a/trail/src/db/util/sqlite.rs b/trail/src/db/util/sqlite.rs index cea760d..8aed24b 100644 --- a/trail/src/db/util/sqlite.rs +++ b/trail/src/db/util/sqlite.rs @@ -22,28 +22,49 @@ pub(crate) fn register_sqlite_vec_extension() -> Result<()> { } pub(crate) fn apply_sqlite_pragmas(conn: &Connection) -> Result<()> { - conn.pragma_update(None, "journal_mode", "WAL")?; - conn.pragma_update(None, "synchronous", "NORMAL")?; - conn.pragma_update(None, "foreign_keys", "ON")?; - conn.pragma_update(None, "temp_store", "MEMORY")?; - Ok(()) + // WAL mode persists in the database header. Reissuing the mutating + // journal-mode pragma on every observer control connection can require an + // exclusive transition while the daemon's primary connection and WAL are + // live; native Linux filesystems can report that failed transition as + // IOERR. New databases still opt into WAL, while runtime connections only + // verify and reuse the already-persisted mode. + let journal_mode: String = conn + .query_row("PRAGMA journal_mode", [], |row| row.get(0)) + .map_err(|error| { + Error::DaemonUnavailable(format!("SQLite journal-mode verification failed: {error}")) + })?; + if !journal_mode.eq_ignore_ascii_case("wal") { + conn.pragma_update(None, "journal_mode", "WAL") + .map_err(|error| { + Error::DaemonUnavailable(format!("SQLite WAL-mode initialization failed: {error}")) + })?; + } + conn.pragma_update(None, "synchronous", "NORMAL") + .map_err(|error| { + Error::DaemonUnavailable(format!( + "SQLite synchronous-mode initialization failed: {error}" + )) + })?; + apply_sqlite_runtime_pragmas(conn) } -pub(crate) fn ensure_column( - conn: &Connection, - table: &'static str, - column: &'static str, - definition: &'static str, -) -> Result<()> { - let mut stmt = conn.prepare(&format!("PRAGMA table_info({table})"))?; - let columns = stmt - .query_map([], |row| row.get::<_, String>(1))? - .collect::, _>>()?; - if !columns.iter().any(|existing| existing == column) { - conn.execute( - &format!("ALTER TABLE {table} ADD COLUMN {column} {definition}"), - [], - )?; - } +/// Configure connection-local behavior without inspecting or mutating the +/// database-wide journal mode. Runtime observer connections open only after +/// Trail's hard-cutover initializer has established WAL mode. +pub(crate) fn apply_sqlite_runtime_pragmas(conn: &Connection) -> Result<()> { + // Do not lower `synchronous` on a live database. New SQLite connections + // default to FULL, which is stronger than Trail's initialized NORMAL mode; + // attempting to change it while the primary daemon transaction is live can + // fail with IOERR on native Linux. + conn.pragma_update(None, "foreign_keys", "ON") + .map_err(|error| { + Error::DaemonUnavailable(format!("SQLite foreign-key initialization failed: {error}")) + })?; + conn.pragma_update(None, "temp_store", "MEMORY") + .map_err(|error| { + Error::DaemonUnavailable(format!( + "SQLite temporary-store initialization failed: {error}" + )) + })?; Ok(()) } diff --git a/trail/src/db/util/time.rs b/trail/src/db/util/time.rs index b68dc75..6a08e0a 100644 --- a/trail/src/db/util/time.rs +++ b/trail/src/db/util/time.rs @@ -14,6 +14,13 @@ pub(crate) fn now_nanos() -> u128 { .unwrap_or_default() } +pub(crate) fn now_millis() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_millis().try_into().unwrap_or(i64::MAX)) + .unwrap_or_default() +} + pub(crate) fn elapsed_ms(duration: Duration) -> u64 { duration.as_millis().try_into().unwrap_or(u64::MAX) } diff --git a/trail/src/error.rs b/trail/src/error.rs index d03ad40..289ee18 100644 --- a/trail/src/error.rs +++ b/trail/src/error.rs @@ -12,6 +12,23 @@ pub enum Error { WorkspaceExists(PathBuf), #[error("invalid path `{path}`: {reason}")] InvalidPath { path: String, reason: String }, + #[error("path invariant index is required: {0}")] + PathIndexRequired(String), + #[error("workspace schema {found} cannot be opened; {guidance}")] + SchemaReinitializeRequired { found: String, guidance: String }, + #[error("changed-path ledger reconciliation required for {scope}: {reason}; run `{command}`")] + ChangeLedgerReconcileRequired { + scope: String, + state: String, + reason: String, + command: String, + }, + #[error("operation {operation} committed but {repair} repair is required: {reason}")] + CommittedRepairRequired { + operation: String, + repair: String, + reason: String, + }, #[error("ignored path `{0}`")] IgnoredPath(String), #[error("ref not found: {0}")] @@ -38,8 +55,22 @@ pub enum Error { Corrupt(String), #[error("git interop failed: {0}")] Git(String), + #[error("Git baseline mapping is required: {0}")] + GitMappingRequired(String), + #[error("Git HEAD changed during handoff: {0}")] + GitHeadChanged(String), + #[error("Git tracked worktree is dirty: {0}")] + GitWorktreeDirty(String), + #[error("mapped Git delta export is required: {0}")] + GitDeltaExportRequired(String), #[error("invalid input: {0}")] InvalidInput(String), + #[error("native COW is unsupported for this source and destination")] + CloneUnsupported, + #[error("native COW source and destination are on different filesystems")] + CloneCrossDevice, + #[error("no complete validated filesystem source is available for native COW")] + NativeCowSourceUnavailable, #[error("I/O error: {0}")] Io(#[from] std::io::Error), #[error("SQLite error: {0}")] @@ -49,9 +80,9 @@ pub enum Error { #[error("prolly error: {0}")] Prolly(#[from] prolly::Error), #[error("prolly SQLite error: {0}")] - ProllySqlite(#[from] prolly::SqliteStoreError), + ProllySqlite(#[from] prolly_store_sqlite::SqliteStoreError), #[error("prolly SlateDB error: {0}")] - ProllySlateDb(#[from] prolly::SlateDbStoreError), + ProllySlateDb(#[from] prolly_store_slatedb::SlateDbStoreError), #[error("JSON error: {0}")] Json(#[from] serde_json::Error), #[error("TOML error: {0}")] @@ -71,6 +102,10 @@ impl Error { Error::WorkspaceNotFound(_) => "WORKSPACE_NOT_FOUND", Error::WorkspaceExists(_) => "WORKSPACE_EXISTS", Error::InvalidPath { .. } => "INVALID_PATH", + Error::PathIndexRequired(_) => "PATH_INDEX_REQUIRED", + Error::SchemaReinitializeRequired { .. } => "SCHEMA_REINITIALIZE_REQUIRED", + Error::ChangeLedgerReconcileRequired { .. } => "CHANGE_LEDGER_RECONCILE_REQUIRED", + Error::CommittedRepairRequired { .. } => "COMMITTED_REPAIR_REQUIRED", Error::IgnoredPath(_) => "IGNORED_PATH", Error::RefNotFound(_) => "REF_NOT_FOUND", Error::OperationNotFound(_) => "OPERATION_NOT_FOUND", @@ -83,7 +118,14 @@ impl Error { Error::StaleBranch(_) => "STALE_BRANCH", Error::Corrupt(_) => "DATABASE_CORRUPT", Error::Git(_) => "GIT_ERROR", + Error::GitMappingRequired(_) => "GIT_MAPPING_REQUIRED", + Error::GitHeadChanged(_) => "GIT_HEAD_CHANGED", + Error::GitWorktreeDirty(_) => "GIT_WORKTREE_DIRTY", + Error::GitDeltaExportRequired(_) => "GIT_DELTA_EXPORT_REQUIRED", Error::InvalidInput(_) => "INVALID_INPUT", + Error::CloneUnsupported => "CLONE_UNSUPPORTED", + Error::CloneCrossDevice => "CLONE_CROSS_DEVICE", + Error::NativeCowSourceUnavailable => "NATIVE_COW_SOURCE_UNAVAILABLE", Error::Io(_) => "IO_ERROR", Error::Sqlite(_) => "SQLITE_ERROR", Error::Serialization(_) => "SERIALIZATION_ERROR", @@ -106,12 +148,23 @@ impl Error { Error::Conflict(_) => 6, Error::PatchRejected(_) => 7, Error::StaleBranch(_) | Error::WorkspaceLocked(_) => 8, - Error::InvalidPath { .. } => 9, - Error::Git(_) => 10, + Error::InvalidPath { .. } | Error::PathIndexRequired(_) => 9, + Error::Git(_) + | Error::GitMappingRequired(_) + | Error::GitHeadChanged(_) + | Error::GitWorktreeDirty(_) + | Error::GitDeltaExportRequired(_) => 10, Error::OperationNotFound(_) => 12, Error::RefNotFound(_) => 13, Error::IgnoredPath(_) => 14, - Error::InvalidInput(_) | Error::WorkspaceExists(_) => 2, + Error::SchemaReinitializeRequired { .. } => 15, + Error::ChangeLedgerReconcileRequired { .. } => 16, + Error::CommittedRepairRequired { .. } => 16, + Error::InvalidInput(_) + | Error::WorkspaceExists(_) + | Error::CloneUnsupported + | Error::CloneCrossDevice + | Error::NativeCowSourceUnavailable => 2, Error::DaemonUnavailable(_) => 11, Error::DaemonError { exit_code, .. } => *exit_code, _ => 1, @@ -126,3 +179,81 @@ pub(crate) fn cbor(value: &T) -> Result> { pub(crate) fn from_cbor(bytes: &[u8]) -> Result { serde_cbor::from_slice(bytes).map_err(|err| Error::Serialization(err.to_string())) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn git_handoff_errors_have_stable_codes_and_exit_status() { + let errors = [ + ( + Error::GitMappingRequired("missing mapping".into()), + "GIT_MAPPING_REQUIRED", + ), + ( + Error::GitHeadChanged("head changed".into()), + "GIT_HEAD_CHANGED", + ), + ( + Error::GitWorktreeDirty("tracked changes".into()), + "GIT_WORKTREE_DIRTY", + ), + ( + Error::GitDeltaExportRequired("mapped delta required".into()), + "GIT_DELTA_EXPORT_REQUIRED", + ), + ]; + for (error, code) in errors { + assert_eq!(error.code(), code); + assert_eq!(error.exit_code(), 10); + } + } + + #[test] + fn path_index_required_has_stable_code_and_exit_status() { + let error = Error::PathIndexRequired( + "legacy root has no case-fold index; run `trail index rebuild`".into(), + ); + + assert_eq!(error.code(), "PATH_INDEX_REQUIRED"); + assert_eq!(error.exit_code(), 9); + } + + #[test] + fn schema_and_ledger_recovery_errors_have_stable_contracts() { + let schema = Error::SchemaReinitializeRequired { + found: "version 17".into(), + guidance: "back up this workspace, then run `trail init --force` to create schema v18" + .into(), + }; + assert_eq!(schema.code(), "SCHEMA_REINITIALIZE_REQUIRED"); + assert_eq!(schema.exit_code(), 15); + assert_eq!( + schema.to_string(), + "workspace schema version 17 cannot be opened; back up this workspace, then run `trail init --force` to create schema v18" + ); + + let reconcile = Error::ChangeLedgerReconcileRequired { + scope: "workspace:main".into(), + state: "untrusted_gap".into(), + reason: "observer startup failed".into(), + command: "trail status".into(), + }; + assert_eq!(reconcile.code(), "CHANGE_LEDGER_RECONCILE_REQUIRED"); + assert_eq!(reconcile.exit_code(), 16); + assert_eq!( + reconcile.to_string(), + "changed-path ledger reconciliation required for workspace:main: observer startup failed; run `trail status`" + ); + + let committed = Error::CommittedRepairRequired { + operation: "op-1".into(), + repair: "ref mirror".into(), + reason: "injected failure".into(), + }; + assert_eq!(committed.code(), "COMMITTED_REPAIR_REQUIRED"); + assert_eq!(committed.exit_code(), 16); + assert!(committed.to_string().contains("operation op-1 committed")); + } +} diff --git a/trail/src/ids.rs b/trail/src/ids.rs index f0cbfd3..7883bd5 100644 --- a/trail/src/ids.rs +++ b/trail/src/ids.rs @@ -7,6 +7,14 @@ use sha2::{Digest, Sha256}; static NONCE: AtomicU64 = AtomicU64::new(1); +pub(crate) const WORKSPACE_ID_PREFIX: &str = "workspace_"; +pub(crate) const CHANGE_ID_PREFIX: &str = "change_"; +pub(crate) const OBJECT_ID_PREFIX: &str = "object_"; +pub(crate) const MESSAGE_ID_PREFIX: &str = "message_"; +pub(crate) const ANCHOR_ID_PREFIX: &str = "anchor_"; +pub(crate) const CHECKPOINT_ALIAS_PREFIX: &str = "checkpoint_"; +pub(crate) const LINE_ALIAS_PREFIX: &str = "line_"; + #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct WorkspaceId(pub String); @@ -36,7 +44,7 @@ pub struct AnchorId(pub String); impl WorkspaceId { pub fn new(seed: &[u8]) -> Self { - Self(format!("wk_{}", short_hash(seed, 16))) + Self(format!("{WORKSPACE_ID_PREFIX}{}", short_hash(seed, 16))) } } @@ -54,7 +62,23 @@ impl ChangeId { hasher.update(now.to_be_bytes()); hasher.update(nonce.to_be_bytes()); hasher.update(hint.as_bytes()); - Self(format!("ch_{}", hex::encode(hasher.finalize()))) + Self(format!( + "{CHANGE_ID_PREFIX}{}", + hex::encode(hasher.finalize()) + )) + } + + pub fn checkpoint_alias(&self) -> String { + format!( + "{CHECKPOINT_ALIAS_PREFIX}{}", + self.0.strip_prefix(CHANGE_ID_PREFIX).unwrap_or(&self.0) + ) + } + + pub fn from_checkpoint_alias(value: &str) -> Option { + value + .strip_prefix(CHECKPOINT_ALIAS_PREFIX) + .map(|hash| Self(format!("{CHANGE_ID_PREFIX}{hash}"))) } } @@ -82,6 +106,25 @@ impl LineId { pub fn encode_key(&self) -> Vec { encode_compound_id(&self.origin_change.0, self.local_seq) } + + pub fn alias(&self) -> String { + let origin = self + .origin_change + .0 + .strip_prefix(CHANGE_ID_PREFIX) + .unwrap_or(&self.origin_change.0); + format!("{LINE_ALIAS_PREFIX}{origin}:{}", self.local_seq) + } + + pub fn from_alias(value: &str) -> Option { + let (origin, local_seq) = value.rsplit_once(':')?; + let origin = origin.strip_prefix(LINE_ALIAS_PREFIX)?; + let local_seq = local_seq.parse::().ok()?; + Some(Self::new( + ChangeId(format!("{CHANGE_ID_PREFIX}{origin}")), + local_seq, + )) + } } impl MessageId { @@ -90,7 +133,10 @@ impl MessageId { hasher.update(change_id.0.as_bytes()); hasher.update(role.as_bytes()); hasher.update(body.as_bytes()); - Self(format!("msg_{}", hex::encode(hasher.finalize()))) + Self(format!( + "{MESSAGE_ID_PREFIX}{}", + hex::encode(hasher.finalize()) + )) } } @@ -100,7 +146,10 @@ impl AnchorId { hasher.update(file_id.encode_key()); hasher.update(line_id.encode_key()); hasher.update(label.as_bytes()); - Self(format!("anc_{}", hex::encode(hasher.finalize()))) + Self(format!( + "{ANCHOR_ID_PREFIX}{}", + hex::encode(hasher.finalize()) + )) } } @@ -112,7 +161,10 @@ impl ObjectId { hasher.update(version.to_le_bytes()); hasher.update((bytes.len() as u64).to_le_bytes()); hasher.update(bytes); - Self(format!("obj_{}", hex::encode(hasher.finalize()))) + Self(format!( + "{OBJECT_ID_PREFIX}{}", + hex::encode(hasher.finalize()) + )) } } @@ -142,7 +194,7 @@ pub(crate) fn short_hash(seed: &[u8], bytes: usize) -> String { } fn encode_compound_id(change_id: &str, local_seq: u64) -> Vec { - let digest = if let Some(hex_id) = change_id.strip_prefix("ch_") { + let digest = if let Some(hex_id) = change_id_hash(change_id) { hex::decode(hex_id).unwrap_or_else(|_| Sha256::digest(change_id.as_bytes()).to_vec()) } else { Sha256::digest(change_id.as_bytes()).to_vec() @@ -156,6 +208,54 @@ fn encode_compound_id(change_id: &str, local_seq: u64) -> Vec { out } +pub(crate) fn is_change_id(value: &str) -> bool { + value.starts_with(CHANGE_ID_PREFIX) +} + +pub(crate) fn change_id_hash(value: &str) -> Option<&str> { + value.strip_prefix(CHANGE_ID_PREFIX) +} + +pub(crate) fn is_object_id(value: &str) -> bool { + value.starts_with(OBJECT_ID_PREFIX) +} + pub(crate) fn sha256_hex(bytes: &[u8]) -> String { hex::encode(Sha256::digest(bytes)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generated_ids_use_full_entity_prefixes() { + let workspace = WorkspaceId::new(b"workspace"); + let change = ChangeId::allocate(&workspace, "actor", 1, "test"); + let object = ObjectId::for_bytes("Blob", 1, b"content"); + let message = MessageId::new(&change, "assistant", "done"); + let file = FileId::new(change.clone(), 1); + let line = LineId::new(change.clone(), 2); + let anchor = AnchorId::new(&file, &line, "example"); + + assert!(workspace.0.starts_with(WORKSPACE_ID_PREFIX)); + assert!(change.0.starts_with(CHANGE_ID_PREFIX)); + assert!(object.0.starts_with(OBJECT_ID_PREFIX)); + assert!(message.0.starts_with(MESSAGE_ID_PREFIX)); + assert!(anchor.0.starts_with(ANCHOR_ID_PREFIX)); + assert_eq!( + ChangeId::from_checkpoint_alias(&change.checkpoint_alias()), + Some(change.clone()) + ); + assert_eq!(LineId::from_alias(&line.alias()), Some(line)); + } + + #[test] + fn canonical_prefixes_are_recognized() { + let digest = "ab".repeat(32); + assert!(is_change_id(&format!("change_{digest}"))); + assert!(is_object_id("object_current")); + assert!(!is_change_id(&format!("ch_{digest}"))); + assert!(!is_object_id("obj_legacy")); + } +} diff --git a/trail/src/lib.rs b/trail/src/lib.rs index 1bf424c..258d9a1 100644 --- a/trail/src/lib.rs +++ b/trail/src/lib.rs @@ -1,4 +1,4 @@ -#![recursion_limit = "256"] +#![recursion_limit = "512"] //! Trail core library. //! @@ -7,6 +7,7 @@ //! gives humans and coding lanes a safe branch/provenance layer above Git. pub mod acp; +pub mod agent_hooks; pub mod db; pub mod error; pub mod ids; @@ -19,6 +20,439 @@ pub use error::{Error, Result}; pub use ids::{AnchorId, ChangeId, FileId, LineId, MessageId, ObjectId, WorkspaceId}; pub use model::*; +#[cfg(debug_assertions)] +#[doc(hidden)] +pub mod test_support { + pub fn changed_path_command_flow() -> std::result::Result<(), String> { + crate::db::run_command_flow() + } + + pub fn changed_path_command_long_lock_flow() -> std::result::Result<(), String> { + crate::db::run_command_long_lock_flow() + } + + pub fn changed_path_materialized_lane_snapshot_flow() -> std::result::Result<(), String> { + crate::db::run_materialized_lane_snapshot_flow() + } + + pub fn changed_path_materialized_candidate_lifecycle_flow() -> std::result::Result<(), String> { + crate::db::run_materialized_candidate_lifecycle_flow() + } + + #[cfg(unix)] + pub fn changed_path_view_flow() -> std::result::Result<(), String> { + crate::db::run_changed_path_view_flow() + } + + pub fn set_sparse_selection_write_failure_for_current_thread(enabled: bool) { + crate::db::set_sparse_selection_write_failure_for_current_thread(enabled); + } + + pub fn install_lane_record_after_c2_write_for_current_thread( + path: std::path::PathBuf, + bytes: Vec, + ) { + crate::db::install_lane_record_after_c2_write_for_current_thread(path, bytes); + } + + pub fn set_lane_record_postcommit_failure_for_current_thread(boundary: Option<&'static str>) { + crate::db::set_lane_record_postcommit_failure_for_current_thread(boundary); + } + + pub fn set_changed_path_authority_override(enabled: bool) { + crate::db::set_command_authority_override(enabled); + } + + pub fn changed_path_activation_evidence() -> std::result::Result { + let evidence = crate::db::ActivationEvidence::from_checked_build()?; + serde_json::to_value(evidence).map_err(|error| error.to_string()) + } + + pub fn changed_path_authority_enabled_for(platform: &str) -> std::result::Result { + let evidence = crate::db::ActivationEvidence::from_checked_build()?; + Ok(crate::db::ledger_authority_enabled_for(platform, &evidence)) + } + + pub fn changed_path_production_authority_default() -> bool { + crate::db::LEDGER_AUTHORITY_ENABLED + } + + pub fn changed_path_git_qualification( + db: &crate::Trail, + force_policy_mismatch: bool, + ) -> std::result::Result { + crate::db::prepare_workspace_daemon(db, true).map_err(|error| error.to_string())?; + let qualified = db + .qualified_git_candidates_for_test(force_policy_mismatch) + .map_err(|error| error.to_string())?; + serde_json::to_value(qualified).map_err(|error| error.to_string()) + } + + pub fn changed_path_git_full_scan_oracle( + db: &crate::Trail, + ) -> std::result::Result, String> { + db.git_qualification_full_scan_oracle_for_test() + .map_err(|error| error.to_string()) + } + + pub fn changed_path_git_command_flow( + db: &mut crate::Trail, + ) -> std::result::Result { + crate::db::prepare_workspace_daemon(db, true).map_err(|error| error.to_string())?; + crate::db::set_command_authority_override(true); + let result = (|| { + let status = db.status(None).map_err(|error| error.to_string())?; + let diff = db + .diff_dirty(false, false) + .map_err(|error| error.to_string())?; + let record = db + .record( + Some("main"), + Some("qualified Git evidence command flow".to_string()), + crate::Actor::human(), + false, + ) + .map_err(|error| error.to_string())?; + Ok(serde_json::json!({ + "status": status.changed_paths.into_iter().map(|change| change.path).collect::>(), + "diff": diff.files.into_iter().map(|change| change.path).collect::>(), + "record": record.changed_paths.into_iter().map(|change| change.path).collect::>() + })) + })(); + crate::db::set_command_authority_override(false); + result + } + + pub fn install_git_qualification_after_porcelain_hook( + hook: impl FnOnce() -> std::result::Result<(), String> + Send + 'static, + ) { + crate::db::install_git_qualification_after_porcelain_hook(move || { + hook().map_err(crate::Error::InvalidInput) + }); + } + + pub fn install_git_qualification_after_c2_hook( + hook: impl FnOnce() -> std::result::Result<(), String> + Send + 'static, + ) { + crate::db::install_git_qualification_after_c2_hook(move || { + hook().map_err(crate::Error::InvalidInput) + }); + } + #[cfg(target_os = "macos")] + fn run_macos_integration( + test: fn() -> std::result::Result<(), String>, + ) -> std::result::Result<(), String> { + use std::sync::{Mutex, OnceLock}; + + static MACOS_INTEGRATION: OnceLock> = OnceLock::new(); + let _guard = MACOS_INTEGRATION + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()); + test() + } + + #[cfg(target_os = "macos")] + pub fn changed_path_macos_real_apfs_file_events() -> std::result::Result<(), String> { + run_macos_integration(crate::db::run_macos_real_apfs_file_events) + } + + #[cfg(target_os = "macos")] + pub fn changed_path_macos_gap_flag_matrix() -> std::result::Result<(), String> { + run_macos_integration(crate::db::run_macos_gap_flag_matrix) + } + + #[cfg(target_os = "macos")] + pub fn changed_path_macos_continuity_fault_matrix() -> std::result::Result<(), String> { + run_macos_integration(crate::db::run_macos_continuity_fault_matrix) + } + + #[cfg(target_os = "macos")] + pub fn changed_path_macos_fence_ordering() -> std::result::Result<(), String> { + run_macos_integration(crate::db::run_macos_fence_ordering) + } + + #[cfg(target_os = "macos")] + pub fn changed_path_macos_paused_callback_fence() -> std::result::Result<(), String> { + run_macos_integration(crate::db::run_macos_paused_callback_fence) + } + + #[cfg(target_os = "macos")] + pub fn changed_path_macos_history_authority() -> std::result::Result<(), String> { + run_macos_integration(crate::db::run_macos_history_authority) + } + + #[cfg(target_os = "macos")] + pub fn changed_path_macos_startup_cancellation() -> std::result::Result<(), String> { + run_macos_integration(crate::db::run_macos_startup_cancellation) + } + + #[cfg(target_os = "macos")] + pub fn changed_path_macos_malformed_callbacks() -> std::result::Result<(), String> { + run_macos_integration(crate::db::run_macos_malformed_callbacks) + } + + #[cfg(target_os = "macos")] + pub fn changed_path_macos_root_revalidation_failures() -> std::result::Result<(), String> { + run_macos_integration(crate::db::run_macos_root_revalidation_failures) + } + + #[cfg(target_os = "macos")] + pub fn changed_path_macos_null_context_generation() -> std::result::Result<(), String> { + run_macos_integration(crate::db::run_macos_null_context_generation) + } + + #[cfg(target_os = "macos")] + pub fn changed_path_macos_uuid_revalidation() -> std::result::Result<(), String> { + run_macos_integration(crate::db::run_macos_uuid_revalidation) + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_recursive_coverage() -> std::result::Result<(), String> { + crate::db::run_recursive_coverage() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_reconciliation_interval_qualification( + ) -> std::result::Result<(), String> { + crate::db::run_reconciliation_interval_qualification() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_content_mode_create_delete() -> std::result::Result<(), String> { + crate::db::run_content_mode_create_delete() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_rename_matrix() -> std::result::Result<(), String> { + crate::db::run_rename_matrix() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_rename_storm_and_cookie_expiry() -> std::result::Result<(), String> { + crate::db::run_rename_storm_and_cookie_expiry() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_delayed_backlog() -> std::result::Result<(), String> { + crate::db::run_delayed_backlog() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_controlled_fence_queue_ordering() -> std::result::Result<(), String> { + crate::db::run_controlled_fence_queue_ordering() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_fence_ordering() -> std::result::Result<(), String> { + crate::db::run_fence_ordering() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_fault_revocation_matrix() -> std::result::Result<(), String> { + crate::db::run_fault_revocation_matrix() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_owner_death_and_root_replacement() -> std::result::Result<(), String> + { + crate::db::run_owner_death_and_root_replacement() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_process_owner_child(root: &str) -> std::result::Result<(), String> { + crate::db::run_process_owner_child(root) + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_complete_prefix_publication_races() -> std::result::Result<(), String> + { + crate::db::run_complete_prefix_publication_races() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_authenticated_fence_rejections() -> std::result::Result<(), String> { + crate::db::run_authenticated_fence_rejections() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_segment_writer_reconcile_publication( + ) -> std::result::Result<(), String> { + crate::db::run_segment_writer_reconcile_publication() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_raw_decoder_faults() -> std::result::Result<(), String> { + crate::db::run_raw_decoder_faults() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_policy_dependency_observation() -> std::result::Result<(), String> { + crate::db::run_policy_dependency_observation() + } + + #[cfg(target_os = "linux")] + pub fn changed_path_linux_unsupported_filesystem_rejection() -> std::result::Result<(), String> + { + crate::db::run_unsupported_filesystem_rejection() + } + + #[cfg(target_os = "macos")] + pub fn changed_path_macos_unsupported_filesystem_rejection() -> std::result::Result<(), String> + { + run_macos_integration(crate::db::run_macos_unsupported_filesystem_rejection) + } + + pub fn changed_path_reconciliation_oracle() -> std::result::Result<(), String> { + crate::db::run_oracle() + } + + pub fn changed_path_reconciliation_races() -> std::result::Result<(), String> { + crate::db::run_races() + } + + pub fn changed_path_reconciliation_callback_spool() -> std::result::Result<(), String> { + crate::db::run_callback_spool() + } + + pub fn changed_path_intent_acknowledgement_race() -> std::result::Result<(), String> { + crate::db::run_acknowledgement_race() + } + + pub fn changed_path_intent_gc_root_lifecycle() -> std::result::Result<(), String> { + crate::db::run_gc_root_lifecycle() + } + + pub fn changed_path_intent_crash_matrix() -> std::result::Result<(), String> { + crate::db::run_crash_matrix() + } + + pub fn changed_path_backup_restore_rotation() -> std::result::Result<(), String> { + crate::db::run_backup_restore_rotation() + } + + pub fn changed_path_qualified_proof_revalidation() -> std::result::Result<(), String> { + crate::db::run_qualified_proof_revalidation() + } + + pub fn changed_path_ambiguous_recovery_gate() -> std::result::Result<(), String> { + crate::db::run_ambiguous_recovery_gate() + } + + pub fn changed_path_backup_overwrite_rollback() -> std::result::Result<(), String> { + crate::db::run_backup_overwrite_rollback() + } + + pub fn changed_path_retirement_barrier() -> std::result::Result<(), String> { + crate::db::run_retirement_barrier() + } + + pub fn changed_path_lane_deletion_retirement() -> std::result::Result<(), String> { + crate::db::run_lane_deletion_retirement() + } + + pub fn changed_path_missing_sidecar_rejection() -> std::result::Result<(), String> { + crate::db::run_missing_sidecar_rejection() + } + + pub fn changed_path_advanced_prefix_recovery() -> std::result::Result<(), String> { + crate::db::run_advanced_prefix_recovery() + } + + pub fn changed_path_exact_interval_bridge_rejection() -> std::result::Result<(), String> { + crate::db::run_exact_interval_bridge_rejection() + } + + pub fn changed_path_prefix_interval_bridge_rejection() -> std::result::Result<(), String> { + crate::db::run_prefix_interval_bridge_rejection() + } + + pub fn changed_path_valid_prefix_interval_recovery() -> std::result::Result<(), String> { + crate::db::run_valid_prefix_interval_recovery() + } + + #[cfg(unix)] + pub fn changed_path_mark_ancestor_substitution_rejection() -> std::result::Result<(), String> { + crate::db::run_mark_ancestor_substitution_rejection() + } + + #[cfg(unix)] + pub fn changed_path_recovery_ancestor_substitution_rejection() -> std::result::Result<(), String> + { + crate::db::run_recovery_ancestor_substitution_rejection() + } + + pub fn changed_path_deletion_parent_substitution_rejection() -> std::result::Result<(), String> + { + crate::db::run_deletion_parent_substitution_rejection() + } + + pub fn changed_path_deletion_post_verification_substitution_rejection( + ) -> std::result::Result<(), String> { + crate::db::run_deletion_post_verification_substitution_rejection() + } + + pub fn changed_path_deletion_post_quarantine_verification_substitution_rejection( + ) -> std::result::Result<(), String> { + crate::db::run_deletion_post_quarantine_verification_substitution_rejection() + } + + pub fn changed_path_deletion_retry_hostile_quarantine_replacement_rejection( + ) -> std::result::Result<(), String> { + crate::db::run_deletion_retry_hostile_quarantine_replacement_rejection() + } + + pub fn changed_path_deletion_normal_retry_idempotence() -> std::result::Result<(), String> { + crate::db::run_deletion_normal_retry_idempotence() + } + + pub fn changed_path_retained_writer_quiescence() -> std::result::Result<(), String> { + crate::db::run_retained_writer_quiescence() + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub fn changed_path_orphan_quarantine_substitution_rejection() -> std::result::Result<(), String> + { + crate::db::run_orphan_quarantine_substitution_rejection() + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub fn changed_path_empty_orphan_quarantine_rejection() -> std::result::Result<(), String> { + crate::db::run_empty_orphan_quarantine_rejection() + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + pub fn changed_path_no_orphan_quarantine_allocation() -> std::result::Result<(), String> { + crate::db::run_no_orphan_quarantine_allocation() + } + + pub fn changed_path_deletion_quiesced_missing_quarantine_rejection( + ) -> std::result::Result<(), String> { + crate::db::run_deletion_quiesced_missing_quarantine_rejection() + } + + pub fn changed_path_deletion_quiesced_reappeared_original_rejection( + ) -> std::result::Result<(), String> { + crate::db::run_deletion_quiesced_reappeared_original_rejection() + } + + pub fn changed_path_restored_nullable_provider_lane_deletion() -> std::result::Result<(), String> + { + crate::db::run_restored_nullable_provider_lane_deletion() + } + + #[cfg(unix)] + pub fn changed_path_non_utf_database_path_mark_recover_and_retire( + ) -> std::result::Result<(), String> { + crate::db::run_non_utf_database_path_mark_recover_and_retire() + } + + #[cfg(unix)] + pub fn changed_path_deletion_leaf_substitution_rejection() -> std::result::Result<(), String> { + crate::db::run_deletion_leaf_substitution_rejection() + } +} + /// Re-export the prolly crate as a Trail module namespace. pub use ::prolly; diff --git a/trail/src/mcp/audit.rs b/trail/src/mcp/audit.rs index 5f1d635..f418353 100644 --- a/trail/src/mcp/audit.rs +++ b/trail/src/mcp/audit.rs @@ -72,16 +72,15 @@ impl McpMutationAudit { fn argument_lane_for_tool(name: &str, arguments: &Value) -> Option { match name { - "trail.merge_queue_add" => top_level_string_for_keys(arguments, &["source"]) - .filter(|source| source.starts_with("refs/lanes/")), + "trail.lane_merge_queue_add" => top_level_string_for_keys(arguments, &["lane"]), _ => first_string_for_keys(arguments, &["lane", "lane_or_id"]), } } fn argument_target_ref_for_tool(name: &str, arguments: &Value) -> Option { let keys: &[&str] = match name { - "trail.merge_queue_add" => { - return top_level_string_for_keys(arguments, &["target", "target_branch", "into"]) + "trail.lane_merge_queue_add" => { + return top_level_string_for_keys(arguments, &["target"]) .map(|target| mcp_branch_ref(&target)); } "trail.begin_turn" | "trail.lane_spawn" => &[], diff --git a/trail/src/mcp/capabilities/resources.rs b/trail/src/mcp/capabilities/resources.rs index 53d8e9c..d9fa9b0 100644 --- a/trail/src/mcp/capabilities/resources.rs +++ b/trail/src/mcp/capabilities/resources.rs @@ -246,10 +246,10 @@ pub(crate) fn resources() -> Value { "mimeType": "application/json" }, { - "uri": RESOURCE_MERGE_QUEUE, - "name": "merge-queue", - "title": "Merge Queue", - "description": "Current serialized merge queue entries.", + "uri": RESOURCE_LANE_MERGE_QUEUE, + "name": "lane-merge-queue", + "title": "Lane Merge Queue", + "description": "Current serialized lane merge queue entries.", "mimeType": "application/json" }, { @@ -273,6 +273,20 @@ pub(crate) fn resources() -> Value { "description": "Grouped agent tasks and the one next useful action.", "mimeType": "application/json" }, + { + "uri": RESOURCE_AGENT_INTEGRATIONS, + "name": "agent-integrations", + "title": "Agent Integration Capabilities", + "description": "Built-in provider hook, transcript/export, and ACP capability contracts.", + "mimeType": "application/json" + }, + { + "uri": RESOURCE_AGENT_HOOK_RECEIPTS, + "name": "agent-hook-receipts", + "title": "Agent Hook Receipt Diagnostics", + "description": "The latest durable redacted native hook receipt journal rows.", + "mimeType": "application/json" + }, { "uri": RESOURCE_AGENT_LATEST_SUMMARY, "name": "latest-agent-summary", diff --git a/trail/src/mcp/prompt.rs b/trail/src/mcp/prompt.rs index 92f195e..6775572 100644 --- a/trail/src/mcp/prompt.rs +++ b/trail/src/mcp/prompt.rs @@ -48,7 +48,7 @@ Checklist:\n\ 6. Call `trail.diff_lane` with patches and line ids; inspect provenance with `trail.why`, `trail.history`, and `trail.code_from` when a change is unclear.\n\ 7. Confirm latest tests and evals passed or explain why warnings are acceptable.\n\ 8. Use `trail.approval_request` for any unresolved human decision and inspect linked paused runs with `trail.run_list`.\n\ -9. Prefer `trail.merge_queue_add` plus `trail.merge_queue_run` for shared target branches; use direct `merge-lane` only for one-off merges.\n\ +9. Prefer `trail.lane_merge_queue_add` plus `trail.lane_merge_queue_run` for shared target branches; use direct `trail lane merge` only for one-off merges.\n\ 10. If the work should be abandoned, use `trail.lane_rewind` with `record_current = true` instead of silently moving refs.\n\ 11. If conflicts exist, stop review and switch to the `{PROMPT_RESOLVE_CONFLICT}` prompt." ), diff --git a/trail/src/mcp/resource.rs b/trail/src/mcp/resource.rs index bb3ee33..fb91464 100644 --- a/trail/src/mcp/resource.rs +++ b/trail/src/mcp/resource.rs @@ -18,13 +18,24 @@ fn resource_read_response(db: &mut Trail, args: ResourceReadArgs) -> Result ("application/json", pretty_json(&db.doctor()?)?), RESOURCE_LANES => ("application/json", pretty_json(&db.list_lanes()?)?), - RESOURCE_MERGE_QUEUE => ("application/json", pretty_json(&db.list_merge_queue()?)?), + RESOURCE_LANE_MERGE_QUEUE => ( + "application/json", + pretty_json(&db.list_lane_merge_queue()?)?, + ), RESOURCE_CONFLICTS => ("application/json", pretty_json(&db.list_conflicts()?)?), RESOURCE_OPENAPI => ( "application/json", serde_json::to_string_pretty(&crate::server::openapi_spec())?, ), RESOURCE_AGENT_INBOX => ("application/json", pretty_json(&db.agent_inbox()?)?), + RESOURCE_AGENT_INTEGRATIONS => { + let registry = crate::agent_hooks::AgentProviderRegistry::built_in()?; + ("application/json", pretty_json(®istry.list())?) + } + RESOURCE_AGENT_HOOK_RECEIPTS => ( + "application/json", + pretty_json(&db.list_agent_hook_receipts(None, None, 100)?)?, + ), RESOURCE_AGENT_LATEST_SUMMARY => ( "application/json", pretty_json(&db.agent_summary("latest")?)?, diff --git a/trail/src/mcp/response.rs b/trail/src/mcp/response.rs index 5307d81..bfc79d1 100644 --- a/trail/src/mcp/response.rs +++ b/trail/src/mcp/response.rs @@ -27,10 +27,12 @@ pub(crate) fn tool_result(value: T) -> Result { } pub(crate) fn tool_error_result(err: &Error) -> Value { - let structured = json!({ - "message": err.to_string(), - "code": err.exit_code() - }); + let mut structured = + serde_json::to_value(crate::model::StructuredErrorEnvelope::from_error(err)) + .unwrap_or_else(|_| json!({ "error": { "message": err.to_string() } })); + if let Some(object) = structured.as_object_mut() { + object.insert("message".to_string(), Value::String(err.to_string())); + } json!({ "resultType": "complete", "content": [ diff --git a/trail/src/mcp/tool_call/agent_hooks.rs b/trail/src/mcp/tool_call/agent_hooks.rs new file mode 100644 index 0000000..e40d249 --- /dev/null +++ b/trail/src/mcp/tool_call/agent_hooks.rs @@ -0,0 +1,94 @@ +use serde_json::{json, Value}; + +use crate::agent_hooks::AgentProviderRegistry; +use crate::{Result, Trail}; + +use super::{super::response::tool_result, super::types::*, parse_args}; + +pub(super) fn handle(db: &mut Trail, name: &str, arguments: &Value) -> Result> { + let value = match name { + "trail.agent_integrations" => { + let args: AgentIntegrationArgs = parse_args(arguments)?; + let registry = AgentProviderRegistry::built_in()?; + if let Some(provider) = args.provider { + tool_result(registry.resolve(&provider)?) + } else { + tool_result(registry.list()) + } + } + "trail.agent_hook_installations" => { + let args: AgentIntegrationArgs = parse_args(arguments)?; + tool_result(db.list_agent_hook_installations(args.provider.as_deref())?) + } + "trail.agent_hook_receipts" => { + let args: AgentReceiptListArgs = parse_args(arguments)?; + tool_result(db.list_agent_hook_receipts_page( + args.provider.as_deref(), + args.status.as_deref(), + args.offset.unwrap_or(0), + args.limit.unwrap_or(100), + )?) + } + "trail.agent_capture_runs" => { + let args: AgentCaptureRunListArgs = parse_args(arguments)?; + tool_result(db.list_agent_capture_runs_page( + args.active_only, + args.offset.unwrap_or(0), + args.limit.unwrap_or(100), + )?) + } + "trail.agent_artifacts" => { + let args: AgentSessionEvidenceArgs = parse_args(arguments)?; + tool_result(db.list_lane_artifacts_page( + &args.session_id, + args.turn_id.as_deref(), + args.offset.unwrap_or(0), + args.limit.unwrap_or(100), + )?) + } + "trail.agent_provenance" => { + let args: AgentSessionEvidenceArgs = parse_args(arguments)?; + let (nodes, edges) = db.list_session_provenance_page( + &args.session_id, + args.offset.unwrap_or(0), + args.limit.unwrap_or(1_000), + )?; + tool_result(json!({"session_id": args.session_id, "nodes": nodes, "edges": edges})) + } + "trail.agent_attestations" => { + let args: AgentSessionEvidenceArgs = parse_args(arguments)?; + tool_result(db.list_session_attestations_page( + &args.session_id, + args.offset.unwrap_or(0), + args.limit.unwrap_or(100), + )?) + } + "trail.agent_attestation_verify" => { + let args: AgentAttestationArgs = parse_args(arguments)?; + tool_result(db.verify_session_attestation(&args.attestation_id)?) + } + "trail.agent_learnings" => { + let args: AgentLearningListArgs = parse_args(arguments)?; + tool_result(db.list_learnings_page( + args.session_id.as_deref(), + args.status.as_deref(), + args.offset.unwrap_or(0), + args.limit.unwrap_or(100), + )?) + } + "trail.agent_git_links" => { + let args: AgentSessionEvidenceArgs = parse_args(arguments)?; + tool_result(db.list_git_agent_links_page( + &args.session_id, + args.offset.unwrap_or(0), + args.limit.unwrap_or(100), + )?) + } + "trail.agent_trace" => { + let args: AgentTraceArgs = parse_args(arguments)?; + tool_result(db.export_agent_trace(&args.session_id, args.attachments)?) + } + _ => return Ok(None), + }; + Ok(Some(value?)) +} diff --git a/trail/src/mcp/tool_call/core.rs b/trail/src/mcp/tool_call/core.rs index a8a3886..17fd142 100644 --- a/trail/src/mcp/tool_call/core.rs +++ b/trail/src/mcp/tool_call/core.rs @@ -7,9 +7,16 @@ use super::{super::response::tool_result, super::types::*, parse_args}; pub(super) fn handle(db: &mut Trail, name: &str, arguments: &Value) -> Result> { let value = match name { "trail.doctor" => tool_result(db.doctor()?), + "trail.index_reconcile" => { + let args: IndexReconcileArgs = parse_args(arguments)?; + tool_result(crate::server::reconcile_changed_path_ledger( + db, + args.lane.as_deref(), + )?) + } "trail.status" => { let args: StatusArgs = parse_args(arguments)?; - tool_result(db.status_read_only(args.branch.as_deref())?) + tool_result(db.status(args.branch.as_deref())?) } "trail.diff" => { let args: DiffArgs = parse_args(arguments)?; diff --git a/trail/src/mcp/tool_call/lane.rs b/trail/src/mcp/tool_call/lane.rs index e5274a4..5bbd9d9 100644 --- a/trail/src/mcp/tool_call/lane.rs +++ b/trail/src/mcp/tool_call/lane.rs @@ -129,6 +129,84 @@ pub(super) fn handle(db: &mut Trail, name: &str, arguments: &Value) -> Result tool_result(db.workspace_environment_adapters()?), + "trail.env_status" => { + let args: LaneHandleArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + tool_result(db.environment_component_status(&lane)?) + } + "trail.env_discover" => { + let args: DependencySyncArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + tool_result(db.discover_workspace_environment(&lane, args.path.as_deref())?) + } + "trail.env_graph" => { + let args: EnvironmentGraphArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + tool_result(db.workspace_environment_graph_page( + &lane, + args.path.as_deref(), + args.offset, + args.limit, + )?) + } + "trail.env_generation" => { + let args: LaneHandleArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + tool_result(db.active_environment_generation(&lane)?) + } + "trail.env_explain" => { + let args: EnvironmentExplainArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + tool_result(db.explain_workspace_environment_staleness_page( + &lane, + &args.component, + args.offset, + args.limit, + )?) + } + "trail.env_plan" => { + let args: EnvironmentSyncArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + tool_result(db.plan_workspace_environment_component( + &lane, + args.adapter.as_deref().unwrap_or("auto"), + args.path.as_deref(), + args.component.as_deref(), + )?) + } + "trail.env_sync" => { + let args: EnvironmentSyncArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + tool_result(db.sync_workspace_environment_component_with_runtime( + &lane, + args.adapter.as_deref().unwrap_or("auto"), + args.path.as_deref(), + args.component.as_deref(), + )?) + } + "trail.env_sync_all" => { + let args: DependencySyncArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + tool_result( + db.sync_all_workspace_environments_with_runtime(&lane, args.path.as_deref())?, + ) + } + "trail.env_runtime_status" => { + let args: LaneHandleArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + tool_result(db.active_environment_generation(&lane)?) + } + "trail.env_runtime_reconcile" => { + let args: LaneHandleArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + tool_result(db.reconcile_workspace_environment_runtime(&lane)?) + } + "trail.env_runtime_stop" => { + let args: LaneHandleArgs = parse_args(arguments)?; + let lane = db.resolve_lane_handle(&args.lane)?; + tool_result(db.stop_workspace_environment_runtime(&lane)?) + } "trail.cache_list" => tool_result(db.list_workspace_layers()?), "trail.cache_inspect" | "trail.cache_verify" => { let args: CacheLayerArgs = parse_args(arguments)?; diff --git a/trail/src/mcp/tool_call/merge.rs b/trail/src/mcp/tool_call/merge.rs index a03bd7f..d503e77 100644 --- a/trail/src/mcp/tool_call/merge.rs +++ b/trail/src/mcp/tool_call/merge.rs @@ -6,22 +6,22 @@ use super::{super::response::tool_result, super::types::*, parse_args}; pub(super) fn handle(db: &mut Trail, name: &str, arguments: &Value) -> Result> { let value = match name { - "trail.merge_queue_add" => { - let args: MergeQueueAddArgs = parse_args(arguments)?; - tool_result(db.enqueue_merge(&args.source, &args.target, args.priority)?) + "trail.lane_merge_queue_add" => { + let args: LaneMergeQueueAddArgs = parse_args(arguments)?; + tool_result(db.enqueue_lane_merge(&args.lane, &args.target, args.priority)?) } - "trail.merge_queue_list" => tool_result(db.list_merge_queue()?), - "trail.merge_queue_run" => { - let args: MergeQueueRunArgs = parse_args(arguments)?; - tool_result(db.run_merge_queue(args.limit)?) + "trail.lane_merge_queue_list" => tool_result(db.list_lane_merge_queue()?), + "trail.lane_merge_queue_run" => { + let args: LaneMergeQueueRunArgs = parse_args(arguments)?; + tool_result(db.run_lane_merge_queue(args.limit)?) } - "trail.merge_queue_explain" => { - let args: MergeQueueExplainArgs = parse_args(arguments)?; - tool_result(db.explain_merge_queue(&args.selector)?) + "trail.lane_merge_queue_explain" => { + let args: LaneMergeQueueExplainArgs = parse_args(arguments)?; + tool_result(db.explain_lane_merge_queue(&args.selector)?) } - "trail.merge_queue_remove" => { - let args: MergeQueueRemoveArgs = parse_args(arguments)?; - tool_result(db.remove_merge_queue(&args.selector)?) + "trail.lane_merge_queue_remove" => { + let args: LaneMergeQueueRemoveArgs = parse_args(arguments)?; + tool_result(db.remove_lane_merge_queue(&args.selector)?) } "trail.conflict_list" => tool_result(db.list_conflicts()?), "trail.conflict_show" => { diff --git a/trail/src/mcp/tool_call/mod.rs b/trail/src/mcp/tool_call/mod.rs index 15704bc..c8f5005 100644 --- a/trail/src/mcp/tool_call/mod.rs +++ b/trail/src/mcp/tool_call/mod.rs @@ -5,6 +5,7 @@ use crate::{Error, Result, Trail}; use super::{tools::tool_is_read_only, types::*, utils::from_arguments}; +mod agent_hooks; mod collaboration; mod core; mod lane; @@ -23,6 +24,9 @@ fn dispatch_tool_call(db: &mut Trail, call: &ToolCall) -> Result { if let Some(value) = core::handle(db, &call.name, &call.arguments)? { return Ok(value); } + if let Some(value) = agent_hooks::handle(db, &call.name, &call.arguments)? { + return Ok(value); + } if let Some(value) = lane::handle(db, &call.name, &call.arguments)? { return Ok(value); } diff --git a/trail/src/mcp/tool_call/turns.rs b/trail/src/mcp/tool_call/turns.rs index d1d6cfd..5b8d45a 100644 --- a/trail/src/mcp/tool_call/turns.rs +++ b/trail/src/mcp/tool_call/turns.rs @@ -102,9 +102,13 @@ pub(super) fn handle(db: &mut Trail, name: &str, arguments: &Value) -> Result Result { - validate_external_patch_edit_sources("patch request", args.edits.len(), args.files.len())?; - let mut edits = args.edits; - for file in args.files { + validate_external_patch_edit_sources( + "patch request", + args.edits.is_some(), + args.files.is_some(), + )?; + let mut edits = args.edits.unwrap_or_default(); + for file in args.files.unwrap_or_default() { match file { ApiPatchFile::AddText { path, @@ -162,16 +166,24 @@ mod tests { use super::*; #[test] - fn apply_patch_args_reject_empty_or_ambiguous_edit_sources() { + fn apply_patch_args_accept_explicit_empty_source_and_reject_missing_or_ambiguous_sources() { let empty: ApplyPatchArgs = serde_json::from_value(serde_json::json!({ "turn_id": "turn_1", - "message": "empty" + "message": "empty", + "edits": [] })) .unwrap(); - let empty_err = patch_document_from_args(empty).unwrap_err(); - assert!(empty_err + assert!(patch_document_from_args(empty).unwrap().edits.is_empty()); + + let missing: ApplyPatchArgs = serde_json::from_value(serde_json::json!({ + "turn_id": "turn_1", + "message": "missing" + })) + .unwrap(); + let missing_err = patch_document_from_args(missing).unwrap_err(); + assert!(missing_err .to_string() - .contains("requires at least one edit in `edits` or `files`")); + .contains("requires exactly one explicit edit source")); let ambiguous: ApplyPatchArgs = serde_json::from_value(serde_json::json!({ "turn_id": "turn_1", @@ -196,7 +208,7 @@ mod tests { "path": "README.md", "edits": [{ "type": "modify_line", - "line_id": "ch_seed:1", + "line_id": "line_seed:1", "expected_text": "old", "new_text": "new" }] @@ -214,7 +226,7 @@ mod tests { new_text, } => { assert_eq!(path, "README.md"); - assert_eq!(line_id, "ch_seed:1"); + assert_eq!(line_id, "line_seed:1"); assert_eq!(expected_text.as_deref(), Some("old")); assert_eq!(new_text, "new"); } diff --git a/trail/src/mcp/tools.rs b/trail/src/mcp/tools.rs index b13cc18..540cf98 100644 --- a/trail/src/mcp/tools.rs +++ b/trail/src/mcp/tools.rs @@ -1,5 +1,6 @@ use serde_json::Value; +mod agent_hooks; mod annotations; mod collaboration; mod core; @@ -11,6 +12,7 @@ pub(crate) use annotations::tool_is_read_only; pub(crate) fn tools() -> Value { let mut tools = core::tools(); + append_tools(&mut tools, agent_hooks::tools()); append_tools(&mut tools, lane::tools()); append_tools(&mut tools, collaboration::tools()); append_tools(&mut tools, merge::tools()); diff --git a/trail/src/mcp/tools/agent_hooks.rs b/trail/src/mcp/tools/agent_hooks.rs new file mode 100644 index 0000000..41d620c --- /dev/null +++ b/trail/src/mcp/tools/agent_hooks.rs @@ -0,0 +1,114 @@ +use serde_json::{json, Value}; + +use crate::mcp::response::object_schema; + +pub(super) fn tools() -> Value { + json!([ + { + "name": "trail.agent_integrations", + "title": "Agent Integration Capabilities", + "description": "Read provider hook, transcript/export, and ACP capability contracts without invoking native capture.", + "inputSchema": object_schema(json!({ + "provider": {"type": "string", "description": "Optional canonical provider name or alias."} + }), vec![]) + }, + { + "name": "trail.agent_hook_installations", + "title": "Agent Hook Installations", + "description": "List persisted ownership and drift metadata for native hook installations.", + "inputSchema": object_schema(json!({ + "provider": {"type": "string"} + }), vec![]) + }, + { + "name": "trail.agent_hook_receipts", + "title": "Agent Hook Receipt Diagnostics", + "description": "List durable redacted hook receipts by provider or processing status.", + "inputSchema": object_schema(json!({ + "provider": {"type": "string"}, + "status": {"type": "string"}, + "offset": {"type": "integer", "minimum": 0, "maximum": 1000000}, + "limit": {"type": "integer", "minimum": 1, "maximum": 1000} + }), vec![]) + }, + { + "name": "trail.agent_capture_runs", + "title": "Agent Capture Runs", + "description": "List managed capture-run correlation leases.", + "inputSchema": object_schema(json!({ + "active_only": {"type": "boolean", "default": true}, + "offset": {"type": "integer", "minimum": 0, "maximum": 1000000}, + "limit": {"type": "integer", "minimum": 1, "maximum": 1000} + }), vec![]) + }, + { + "name": "trail.agent_artifacts", + "title": "Agent Session Artifacts", + "description": "List immutable native transcript, export, and evidence artifacts for a session or turn.", + "inputSchema": object_schema(json!({ + "session_id": {"type": "string"}, + "turn_id": {"type": "string"}, + "offset": {"type": "integer", "minimum": 0, "maximum": 1000000}, + "limit": {"type": "integer", "minimum": 1, "maximum": 1000} + }), vec!["session_id"]) + }, + { + "name": "trail.agent_provenance", + "title": "Agent Session Provenance", + "description": "Read the causal provenance graph for one captured agent session.", + "inputSchema": object_schema(json!({ + "session_id": {"type": "string"}, + "offset": {"type": "integer", "minimum": 0, "maximum": 1000000}, + "limit": {"type": "integer", "minimum": 1, "maximum": 10000} + }), vec!["session_id"]) + }, + { + "name": "trail.agent_attestations", + "title": "Agent Session Attestations", + "description": "List immutable chained attestation segments and exact turn coverage for one session.", + "inputSchema": object_schema(json!({ + "session_id": {"type": "string"}, + "offset": {"type": "integer", "minimum": 0, "maximum": 1000000}, + "limit": {"type": "integer", "minimum": 1, "maximum": 1000} + }), vec!["session_id"]) + }, + { + "name": "trail.agent_attestation_verify", + "title": "Verify Agent Attestation", + "description": "Verify statement, evidence, predecessor chain, signature, and key revocation status.", + "inputSchema": object_schema(json!({ + "attestation_id": {"type": "string"} + }), vec!["attestation_id"]) + }, + { + "name": "trail.agent_learnings", + "title": "Agent Learnings", + "description": "List proposed, accepted, or rejected reusable findings without changing provider context files.", + "inputSchema": object_schema(json!({ + "session_id": {"type": "string"}, + "status": {"type": "string"}, + "offset": {"type": "integer", "minimum": 0, "maximum": 1000000}, + "limit": {"type": "integer", "minimum": 1, "maximum": 1000} + }), vec![]) + }, + { + "name": "trail.agent_git_links", + "title": "Agent Session Git Links", + "description": "List explicit mappings between exact Trail session changes and Git commits.", + "inputSchema": object_schema(json!({ + "session_id": {"type": "string"}, + "offset": {"type": "integer", "minimum": 0, "maximum": 1000000}, + "limit": {"type": "integer", "minimum": 1, "maximum": 1000} + }), vec!["session_id"]) + }, + { + "name": "trail.agent_trace", + "title": "Portable Agent Trace", + "description": "Read a verified vendor-neutral trace projection for one session.", + "inputSchema": object_schema(json!({ + "session_id": {"type": "string"}, + "attachments": {"type": "boolean", "default": false} + }), vec!["session_id"]) + } + ]) +} diff --git a/trail/src/mcp/tools/annotations.rs b/trail/src/mcp/tools/annotations.rs index 86f422d..5e34dcd 100644 --- a/trail/src/mcp/tools/annotations.rs +++ b/trail/src/mcp/tools/annotations.rs @@ -46,6 +46,12 @@ fn tool_annotations(name: &str) -> Value { "idempotentHint": false, "openWorldHint": true }), + ToolRiskClass::OpenWorldDestructiveWrite => json!({ + "readOnlyHint": false, + "destructiveHint": true, + "idempotentHint": true, + "openWorldHint": true + }), } } @@ -60,6 +66,7 @@ enum ToolRiskClass { IdempotentWrite, DestructiveWrite, OpenWorldWrite, + OpenWorldDestructiveWrite, } fn tool_risk_class(name: &str) -> ToolRiskClass { @@ -69,8 +76,6 @@ fn tool_risk_class(name: &str) -> ToolRiskClass { fn classified_tool_risk_class(name: &str) -> Option { match name { "trail.doctor" - | "trail.status" - | "trail.diff" | "trail.timeline" | "trail.why" | "trail.history" @@ -117,6 +122,17 @@ fn classified_tool_risk_class(name: &str) -> Option { | "trail.agent_diff" | "trail.agent_review" | "trail.agent_focus" + | "trail.agent_integrations" + | "trail.agent_hook_installations" + | "trail.agent_hook_receipts" + | "trail.agent_capture_runs" + | "trail.agent_artifacts" + | "trail.agent_provenance" + | "trail.agent_attestations" + | "trail.agent_attestation_verify" + | "trail.agent_learnings" + | "trail.agent_git_links" + | "trail.agent_trace" | "trail.lane_list" | "trail.lane_show" | "trail.lane_status" @@ -129,6 +145,14 @@ fn classified_tool_risk_class(name: &str) -> Option { | "trail.lane_workspace" | "trail.lane_space" | "trail.deps_status" + | "trail.env_adapters" + | "trail.env_status" + | "trail.env_discover" + | "trail.env_graph" + | "trail.env_generation" + | "trail.env_runtime_status" + | "trail.env_explain" + | "trail.env_plan" | "trail.cache_list" | "trail.cache_inspect" | "trail.cache_verify" @@ -145,8 +169,8 @@ fn classified_tool_risk_class(name: &str) -> Option { | "trail.lease_list" | "trail.anchor_list" | "trail.anchor_resolve" - | "trail.merge_queue_list" - | "trail.merge_queue_explain" + | "trail.lane_merge_queue_list" + | "trail.lane_merge_queue_explain" | "trail.conflict_list" | "trail.conflict_show" | "trail.event_list" @@ -158,9 +182,13 @@ fn classified_tool_risk_class(name: &str) -> Option { | "trail.ignore_list" | "trail.ignore_check" | "trail.guardrail_check" => Some(ToolRiskClass::ReadOnly), - "trail.config_set" | "trail.ignore_add" | "trail.ignore_remove" | "trail.lane_unmount" => { - Some(ToolRiskClass::IdempotentWrite) - } + "trail.status" + | "trail.diff" + | "trail.config_set" + | "trail.ignore_add" + | "trail.ignore_remove" + | "trail.lane_unmount" + | "trail.index_reconcile" => Some(ToolRiskClass::IdempotentWrite), "trail.session_start" | "trail.session_end" | "trail.agent_mark_reviewed" @@ -179,7 +207,7 @@ fn classified_tool_risk_class(name: &str) -> Option { | "trail.lane_checkpoint" | "trail.lane_update" | "trail.lane_mount" - | "trail.merge_queue_add" + | "trail.lane_merge_queue_add" | "trail.begin_turn" | "trail.add_message" | "trail.add_event" @@ -193,16 +221,24 @@ fn classified_tool_risk_class(name: &str) -> Option { | "trail.agent_undo" | "trail.lane_rewind" | "trail.anchor_delete" - | "trail.merge_queue_run" - | "trail.merge_queue_remove" + | "trail.lane_merge_queue_run" + | "trail.lane_merge_queue_remove" | "trail.conflict_resolve" | "trail.apply_patch" | "trail.read_file" | "trail.lane_hydrate" | "trail.sync_workdir" => Some(ToolRiskClass::DestructiveWrite), "trail.cache_gc" => Some(ToolRiskClass::DestructiveWrite), - "trail.agent_test" | "trail.agent_eval" | "trail.run_test" | "trail.run_eval" - | "trail.lane_exec" | "trail.deps_sync" => Some(ToolRiskClass::OpenWorldWrite), + "trail.agent_test" + | "trail.agent_eval" + | "trail.run_test" + | "trail.run_eval" + | "trail.lane_exec" + | "trail.deps_sync" + | "trail.env_sync" + | "trail.env_sync_all" + | "trail.env_runtime_reconcile" => Some(ToolRiskClass::OpenWorldWrite), + "trail.env_runtime_stop" => Some(ToolRiskClass::OpenWorldDestructiveWrite), _ => None, } } diff --git a/trail/src/mcp/tools/collaboration.rs b/trail/src/mcp/tools/collaboration.rs index 81da3e8..6825431 100644 --- a/trail/src/mcp/tools/collaboration.rs +++ b/trail/src/mcp/tools/collaboration.rs @@ -50,7 +50,7 @@ pub(super) fn tools() -> Value { { "name": "trail.session_end", "title": "End Lane Session", - "description": "End a durable lane session with completed, failed, cancelled, or archived status.", + "description": "End a durable lane session with completed, failed, cancelled, interrupted, or archived status.", "inputSchema": object_schema(json!({ "session_id": { "type": "string" }, "status": { "type": "string", "enum": ["completed", "failed", "cancelled", "archived"] } diff --git a/trail/src/mcp/tools/core.rs b/trail/src/mcp/tools/core.rs index 8fbc287..275629b 100644 --- a/trail/src/mcp/tools/core.rs +++ b/trail/src/mcp/tools/core.rs @@ -10,6 +10,14 @@ pub(super) fn tools() -> Value { "description": "Run read-only operational diagnostics for workspace health, locks, fsck, approvals, leases, merge queue, conflicts, and lane workdirs.", "inputSchema": object_schema(json!({}), vec![]) }, + { + "name": "trail.index_reconcile", + "title": "Reconcile Changed-Path Ledger", + "description": "Start the scope observer when necessary and run a complete workspace or materialized-lane filesystem reconciliation.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string", "description": "Optional materialized lane name or id." } + }), vec![]) + }, { "name": "trail.status", "title": "Trail Status", @@ -23,8 +31,8 @@ pub(super) fn tools() -> Value { "title": "Trail Diff", "description": "Show a ref range, root range, or dirty worktree diff with optional patches and stable line ids.", "inputSchema": object_schema(json!({ - "range": { "type": "string", "description": "Ref range such as main..feature or ch_a..ch_b." }, - "root": { "type": "string", "description": "Root id range such as obj_a..obj_b." }, + "range": { "type": "string", "description": "Ref range such as main..feature or change_a..change_b." }, + "root": { "type": "string", "description": "Root id range such as object_a..object_b." }, "dirty": { "type": "boolean", "description": "Diff the current branch head against the materialized worktree." }, "patch": { "type": "boolean" }, "show_line_ids": { "type": "boolean" }, diff --git a/trail/src/mcp/tools/lane.rs b/trail/src/mcp/tools/lane.rs index f3f4962..d4a76af 100644 --- a/trail/src/mcp/tools/lane.rs +++ b/trail/src/mcp/tools/lane.rs @@ -12,7 +12,7 @@ pub(super) fn tools() -> Value { "name": { "type": "string" }, "from_ref": { "type": "string" }, "materialize": { "type": "boolean" }, - "workdir_mode": { "type": "string", "enum": ["auto", "virtual", "sparse", "full-cow", "overlay-cow", "nfs-cow"] }, + "workdir_mode": { "type": "string", "enum": ["auto", "virtual", "sparse", "native-cow", "portable-copy", "fuse-cow", "nfs-cow", "dokan-cow"] }, "workdir": { "type": "string" }, "workdir_path": { "type": "string" }, "paths": { "type": "array", "items": { "type": "string" } }, @@ -214,12 +214,120 @@ pub(super) fn tools() -> Value { { "name": "trail.deps_sync", "title": "Synchronize Dependencies", - "description": "Build or reuse a frozen dependency layer and attach it to the lane.", + "description": "Build or reuse a frozen dependency layer, then bulk-replace private dependency state in an unmounted lane.", "inputSchema": object_schema(json!({ "lane": { "type": "string" }, "path": { "type": "string" } }), vec!["lane"]) }, + { + "name": "trail.env_adapters", + "title": "Workspace Environment Adapters", + "description": "List registered adapters, selectors, component kinds, discovery markers, provenance, and stability without probing tools or repository code.", + "inputSchema": object_schema(json!({}), vec![]) + }, + { + "name": "trail.env_status", + "title": "Workspace Environment Status", + "description": "Show normalized component and versioned adapter state for one layered lane.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" } + }), vec!["lane"]) + }, + { + "name": "trail.env_discover", + "title": "Discover Workspace Environments", + "description": "Detect built-in environment components without running package managers, compilers, network providers, or repository code.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" }, + "path": { "type": "string" } + }), vec!["lane"]) + }, + { + "name": "trail.env_graph", + "title": "Workspace Environment Graph", + "description": "Return the validated desired component DAG, deterministic topological order, output ownership, component keys, and ordering/invalidation edges without executing tools or mutating state.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" }, + "path": { "type": "string" }, + "offset": { "type": "integer", "minimum": 0, "default": 0 }, + "limit": { "type": "integer", "minimum": 1, "maximum": 1000, "default": 256 } + }), vec!["lane"]) + }, + { + "name": "trail.env_generation", + "title": "Active Environment Generation", + "description": "Show the exact component keys, layers, mounts, source root, and predecessor active for one lane.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" } + }), vec!["lane"]) + }, + { + "name": "trail.env_explain", + "title": "Explain Workspace Environment Staleness", + "description": "List every canonical input, tool, platform, architecture, portability, and policy edge that differs from the attached component artifact.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" }, + "component": { "type": "string" }, + "offset": { "type": "integer", "minimum": 0, "default": 0 }, + "limit": { "type": "integer", "minimum": 1, "maximum": 1000, "default": 256 } + }), vec!["lane", "component"]) + }, + { + "name": "trail.env_plan", + "title": "Plan Workspace Environment", + "description": "Return the normalized component key, declared inputs, argv actions, output, and capability grants without executing or mutating state.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" }, + "adapter": { "type": "string", "default": "auto" }, + "component": { "type": "string" }, + "path": { "type": "string" } + }), vec!["lane"]) + }, + { + "name": "trail.env_sync", + "title": "Synchronize Workspace Environment", + "description": "Prepare one adapter-owned environment component and atomically activate its shared and/or writable-private outputs for an unmounted lane.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" }, + "adapter": { "type": "string", "default": "auto" }, + "component": { "type": "string" }, + "path": { "type": "string" } + }), vec!["lane"]) + }, + { + "name": "trail.env_sync_all", + "title": "Synchronize All Workspace Environments", + "description": "Build every discovered component first, then atomically activate all mounts as one environment generation.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" }, + "path": { "type": "string" } + }), vec!["lane"]) + }, + { + "name": "trail.env_runtime_status", + "title": "Environment Runtime Status", + "description": "Show persisted container, network, volume, port, lifecycle, and health state for the active lane generation without contacting a provider.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" } + }), vec!["lane"]) + }, + { + "name": "trail.env_runtime_reconcile", + "title": "Reconcile Environment Runtime", + "description": "Idempotently create or adopt declared lane-private OCI resources and wait for their health contracts.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" } + }), vec!["lane"]) + }, + { + "name": "trail.env_runtime_stop", + "title": "Stop Environment Runtime", + "description": "Stop the active generation's Trail-owned containers while retaining private networks and volumes for restart.", + "inputSchema": object_schema(json!({ + "lane": { "type": "string" } + }), vec!["lane"]) + }, { "name": "trail.cache_list", "title": "List Workspace Cache", @@ -253,3 +361,29 @@ pub(super) fn tools() -> Value { } ]) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lane_spawn_schema_uses_the_hard_cutover_modes() { + let declarations = tools(); + let lane_spawn = declarations + .as_array() + .unwrap() + .iter() + .find(|tool| tool["name"] == "trail.lane_spawn") + .unwrap(); + let modes = lane_spawn["inputSchema"]["properties"]["workdir_mode"]["enum"] + .as_array() + .unwrap(); + + assert!(modes.iter().any(|mode| mode == "native-cow")); + assert!(modes.iter().any(|mode| mode == "portable-copy")); + assert!(modes.iter().any(|mode| mode == "fuse-cow")); + assert!(modes.iter().any(|mode| mode == "dokan-cow")); + assert!(!modes.iter().any(|mode| mode == "full-cow")); + assert!(!modes.iter().any(|mode| mode == "overlay-cow")); + } +} diff --git a/trail/src/mcp/tools/merge.rs b/trail/src/mcp/tools/merge.rs index 7d479b7..38d7ced 100644 --- a/trail/src/mcp/tools/merge.rs +++ b/trail/src/mcp/tools/merge.rs @@ -48,41 +48,41 @@ fn conflict_resolve_schema() -> Value { pub(super) fn tools() -> Value { json!([ { - "name": "trail.merge_queue_add", - "title": "Queue Merge", - "description": "Queue a lane or branch ref for serialized merge into a target branch.", + "name": "trail.lane_merge_queue_add", + "title": "Queue Lane Merge", + "description": "Queue a lane for serialized merge into a target branch.", "inputSchema": object_schema(json!({ - "source": { "type": "string" }, + "lane": { "type": "string" }, "target": { "type": "string" }, "priority": { "type": "integer" } - }), vec!["source", "target"]) + }), vec!["lane", "target"]) }, { - "name": "trail.merge_queue_list", - "title": "List Merge Queue", - "description": "List queued, running, merged, cancelled, failed, and conflicted merge queue entries.", + "name": "trail.lane_merge_queue_list", + "title": "List Lane Merge Queue", + "description": "List queued, running, merged, cancelled, failed, and conflicted lane merge queue entries.", "inputSchema": object_schema(json!({}), vec![]) }, { - "name": "trail.merge_queue_run", - "title": "Run Merge Queue", - "description": "Run queued merges serially, pausing on the first conflict or failure.", + "name": "trail.lane_merge_queue_run", + "title": "Run Lane Merge Queue", + "description": "Run queued lane merges serially, pausing on the first conflict or failure.", "inputSchema": object_schema(json!({ "limit": { "type": "integer", "minimum": 1 } }), vec![]) }, { - "name": "trail.merge_queue_explain", - "title": "Explain Merge Queue Entry", + "name": "trail.lane_merge_queue_explain", + "title": "Explain Lane Merge Queue Entry", "description": "Explain why one queued merge is ready or blocked, including readiness blockers, dry-run conflicts, preflight errors, warnings, and next-step commands.", "inputSchema": object_schema(json!({ "selector": { "type": "string" } }), vec!["selector"]) }, { - "name": "trail.merge_queue_remove", - "title": "Remove Merge Queue Entry", - "description": "Cancel a queued or conflicted merge queue entry by queue id, lane, branch, or ref.", + "name": "trail.lane_merge_queue_remove", + "title": "Remove Lane Merge Queue Entry", + "description": "Cancel a queued or conflicted lane merge queue entry by queue id, lane id, or lane name.", "inputSchema": object_schema(json!({ "selector": { "type": "string" } }), vec!["selector"]) diff --git a/trail/src/mcp/tools/turns.rs b/trail/src/mcp/tools/turns.rs index 6395e59..c2a0884 100644 --- a/trail/src/mcp/tools/turns.rs +++ b/trail/src/mcp/tools/turns.rs @@ -107,12 +107,10 @@ fn apply_patch_schema() -> Value { "allow_stale": { "type": "boolean" }, "edits": { "type": "array", - "minItems": 1, "items": patch_edit_schema() }, "files": { "type": "array", - "minItems": 1, "items": api_patch_file_schema() } }), @@ -238,13 +236,13 @@ pub(super) fn tools() -> Value { { "name": "trail.apply_patch", "title": "Apply Lane Patch", - "description": "Apply a native Trail patch or design-style files patch to a turn's lane branch. Provide exactly one non-empty edit source: edits or files.", + "description": "Apply a native Trail patch or design-style files patch to a turn's lane branch. Explicitly provide exactly one source, edits or files; an empty array records a genuine zero-change patch operation.", "inputSchema": apply_patch_schema() }, { "name": "trail.end_turn", "title": "End Lane Turn", - "description": "Close a durable lane turn with completed, failed, cancelled, or archived status.", + "description": "Close a durable lane turn with completed, failed, cancelled, interrupted, or archived status.", "inputSchema": object_schema(json!({ "turn_id": { "type": "string" }, "status": { "type": "string", "enum": ["completed", "failed", "cancelled", "archived"] } diff --git a/trail/src/mcp/types/agent_hooks.rs b/trail/src/mcp/types/agent_hooks.rs new file mode 100644 index 0000000..aae2c89 --- /dev/null +++ b/trail/src/mcp/types/agent_hooks.rs @@ -0,0 +1,75 @@ +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AgentIntegrationArgs { + #[serde(default)] + pub(crate) provider: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AgentReceiptListArgs { + #[serde(default)] + pub(crate) provider: Option, + #[serde(default)] + pub(crate) status: Option, + #[serde(default)] + pub(crate) offset: Option, + #[serde(default)] + pub(crate) limit: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AgentCaptureRunListArgs { + #[serde(default = "default_true")] + pub(crate) active_only: bool, + #[serde(default)] + pub(crate) offset: Option, + #[serde(default)] + pub(crate) limit: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AgentSessionEvidenceArgs { + pub(crate) session_id: String, + #[serde(default)] + pub(crate) turn_id: Option, + #[serde(default)] + pub(crate) offset: Option, + #[serde(default)] + pub(crate) limit: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AgentAttestationArgs { + pub(crate) attestation_id: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AgentLearningListArgs { + #[serde(default)] + pub(crate) session_id: Option, + #[serde(default)] + pub(crate) status: Option, + #[serde(default)] + pub(crate) offset: Option, + #[serde(default)] + pub(crate) limit: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AgentTraceArgs { + pub(crate) session_id: String, + #[serde(default)] + pub(crate) attachments: bool, +} + +fn default_true() -> bool { + true +} diff --git a/trail/src/mcp/types/constants.rs b/trail/src/mcp/types/constants.rs index 980ec68..b75f7e0 100644 --- a/trail/src/mcp/types/constants.rs +++ b/trail/src/mcp/types/constants.rs @@ -4,10 +4,12 @@ pub(crate) const MCP_PROTOCOL_VERSION: &str = "2025-11-25"; pub(crate) const RESOURCE_STATUS: &str = "trail://workspace/status"; pub(crate) const RESOURCE_DOCTOR: &str = "trail://workspace/doctor"; pub(crate) const RESOURCE_LANES: &str = "trail://workspace/lanes"; -pub(crate) const RESOURCE_MERGE_QUEUE: &str = "trail://workspace/merge-queue"; +pub(crate) const RESOURCE_LANE_MERGE_QUEUE: &str = "trail://workspace/lane-merge-queue"; pub(crate) const RESOURCE_CONFLICTS: &str = "trail://workspace/conflicts"; pub(crate) const RESOURCE_OPENAPI: &str = "trail://workspace/openapi"; pub(crate) const RESOURCE_AGENT_INBOX: &str = "trail://workspace/agent-tasks"; +pub(crate) const RESOURCE_AGENT_INTEGRATIONS: &str = "trail://workspace/agent-integrations"; +pub(crate) const RESOURCE_AGENT_HOOK_RECEIPTS: &str = "trail://workspace/agent-hooks/receipts"; pub(crate) const RESOURCE_AGENT_LATEST_SUMMARY: &str = "trail://workspace/agent-tasks/latest/summary"; pub(crate) const RESOURCE_AGENT_LATEST_DIAGNOSE: &str = diff --git a/trail/src/mcp/types/core.rs b/trail/src/mcp/types/core.rs index 3299ed3..0d59ac1 100644 --- a/trail/src/mcp/types/core.rs +++ b/trail/src/mcp/types/core.rs @@ -8,6 +8,13 @@ pub(crate) struct StatusArgs { pub(crate) branch: Option, } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct IndexReconcileArgs { + #[serde(default)] + pub(crate) lane: Option, +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct DiffArgs { diff --git a/trail/src/mcp/types/lane.rs b/trail/src/mcp/types/lane.rs index 62f1187..78ccdd7 100644 --- a/trail/src/mcp/types/lane.rs +++ b/trail/src/mcp/types/lane.rs @@ -202,6 +202,53 @@ pub(crate) struct DependencySyncArgs { pub(crate) path: Option, } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct EnvironmentGraphArgs { + #[serde(alias = "lane_or_id", alias = "name")] + pub(crate) lane: String, + #[serde(default)] + pub(crate) path: Option, + #[serde(default)] + pub(crate) offset: u64, + #[serde(default = "default_environment_graph_limit")] + pub(crate) limit: u64, +} + +fn default_environment_graph_limit() -> u64 { + 256 +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct EnvironmentSyncArgs { + #[serde(alias = "lane_or_id", alias = "name")] + pub(crate) lane: String, + #[serde(default)] + pub(crate) adapter: Option, + #[serde(default, alias = "component_id")] + pub(crate) component: Option, + #[serde(default, alias = "component_root")] + pub(crate) path: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct EnvironmentExplainArgs { + #[serde(alias = "lane_or_id", alias = "name")] + pub(crate) lane: String, + #[serde(alias = "component_id")] + pub(crate) component: String, + #[serde(default)] + pub(crate) offset: u64, + #[serde(default = "default_environment_explain_limit")] + pub(crate) limit: u64, +} + +fn default_environment_explain_limit() -> u64 { + 256 +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct CacheLayerArgs { diff --git a/trail/src/mcp/types/merge.rs b/trail/src/mcp/types/merge.rs index 994b6bc..a31c2c3 100644 --- a/trail/src/mcp/types/merge.rs +++ b/trail/src/mcp/types/merge.rs @@ -4,9 +4,8 @@ use crate::model::ConflictManualResolution; #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -pub(crate) struct MergeQueueAddArgs { - pub(crate) source: String, - #[serde(alias = "into", alias = "target_branch")] +pub(crate) struct LaneMergeQueueAddArgs { + pub(crate) lane: String, pub(crate) target: String, #[serde(default)] pub(crate) priority: i64, @@ -14,20 +13,20 @@ pub(crate) struct MergeQueueAddArgs { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -pub(crate) struct MergeQueueRunArgs { +pub(crate) struct LaneMergeQueueRunArgs { #[serde(default)] pub(crate) limit: Option, } #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -pub(crate) struct MergeQueueExplainArgs { +pub(crate) struct LaneMergeQueueExplainArgs { pub(crate) selector: String, } #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -pub(crate) struct MergeQueueRemoveArgs { +pub(crate) struct LaneMergeQueueRemoveArgs { pub(crate) selector: String, } diff --git a/trail/src/mcp/types/mod.rs b/trail/src/mcp/types/mod.rs index df5bdb1..50e8bb5 100644 --- a/trail/src/mcp/types/mod.rs +++ b/trail/src/mcp/types/mod.rs @@ -1,3 +1,4 @@ +mod agent_hooks; mod collaboration; mod constants; mod core; @@ -6,6 +7,7 @@ mod merge; mod protocol; mod turns; +pub(crate) use self::agent_hooks::*; pub(crate) use self::collaboration::*; pub(crate) use self::constants::*; pub(crate) use self::core::*; diff --git a/trail/src/mcp/types/turns.rs b/trail/src/mcp/types/turns.rs index 869587d..7ae6e1f 100644 --- a/trail/src/mcp/types/turns.rs +++ b/trail/src/mcp/types/turns.rs @@ -144,10 +144,8 @@ pub(crate) struct ApplyPatchArgs { pub(crate) allow_ignored: bool, #[serde(default)] pub(crate) allow_stale: bool, - #[serde(default)] - pub(crate) edits: Vec, - #[serde(default)] - pub(crate) files: Vec, + pub(crate) edits: Option>, + pub(crate) files: Option>, } #[derive(Debug, Deserialize)] diff --git a/trail/src/model.rs b/trail/src/model.rs index 28df341..6b87603 100644 --- a/trail/src/model.rs +++ b/trail/src/model.rs @@ -1,4 +1,6 @@ include!("model/domain.rs"); +include!("model/agent_capture.rs"); +include!("model/agent_evidence.rs"); include!("model/lane.rs"); include!("model/inspect.rs"); include!("model/reports.rs"); diff --git a/trail/src/model/agent_capture.rs b/trail/src/model/agent_capture.rs new file mode 100644 index 0000000..6dfcdc4 --- /dev/null +++ b/trail/src/model/agent_capture.rs @@ -0,0 +1,1645 @@ +use std::fmt; + +/// Stable schema identifier for adapter-neutral agent lifecycle events. +pub const AGENT_LIFECYCLE_EVENT_SCHEMA: &str = "trail.agent_lifecycle_event"; +/// Current normalized lifecycle event version. +pub const AGENT_LIFECYCLE_EVENT_VERSION: u16 = 1; +/// Maximum serialized provider payload accepted at a mutation boundary. +pub const AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES: usize = 1024 * 1024; + +/// Transport that supplied evidence to Trail's shared capture coordinator. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum AgentCaptureTransport { + Acp, + NativeHooks, + Terminal, + Hybrid, +} + +/// How directly an evidence field was observed. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum AgentEvidenceConfidence { + ProtocolStructured, + NativeStructured, + NativeTranscript, + CanonicalExport, + WorktreeObserved, + Heuristic, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentEvidenceSource { + Acp, + NativeHook, + NativeTranscript, + CanonicalExport, + WorkdirObserved, + Reconstructed, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentEvidenceFactKind { + SessionLifecycle, + TurnBoundary, + UserMessage, + AssistantMessage, + ToolCall, + Approval, + Usage, + WorkspaceChange, + TranscriptLocator, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct AgentEvidenceFact { + pub kind: AgentEvidenceFactKind, + pub key: String, + pub value: serde_json::Value, + pub source: AgentEvidenceSource, + pub confidence: AgentEvidenceConfidence, + pub observed_at: Option, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentEvidenceMergeDecision { + Insert, + Enrich, + IgnoreDuplicate, + PreserveHigherPrecedence, + PreserveConflict, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct AgentEvidenceMergeResult { + pub fact: AgentEvidenceFact, + pub decision: AgentEvidenceMergeDecision, + pub conflict: Option, +} + +/// Merge the same factual identity monotonically; lower-precedence evidence can never +/// overwrite stronger evidence, and equal-precedence conflicts remain explicit. +pub fn merge_agent_evidence_fact( + existing: Option<&AgentEvidenceFact>, + incoming: AgentEvidenceFact, +) -> crate::Result { + validate_agent_capture_token("evidence fact key", &incoming.key, 512)?; + if incoming.observed_at.is_some_and(|value| value < 0) { + return Err(crate::Error::InvalidInput( + "agent evidence observed_at must be non-negative".to_string(), + )); + } + let Some(existing) = existing else { + return Ok(AgentEvidenceMergeResult { + fact: incoming, + decision: AgentEvidenceMergeDecision::Insert, + conflict: None, + }); + }; + if existing.kind != incoming.kind || existing.key != incoming.key { + return Err(crate::Error::InvalidInput( + "agent evidence merge requires the same fact kind and key".to_string(), + )); + } + if existing.value == incoming.value { + let incoming_rank = agent_evidence_precedence(incoming.kind, incoming.source); + let existing_rank = agent_evidence_precedence(existing.kind, existing.source); + return Ok(if incoming_rank > existing_rank { + AgentEvidenceMergeResult { + fact: incoming, + decision: AgentEvidenceMergeDecision::Enrich, + conflict: None, + } + } else { + AgentEvidenceMergeResult { + fact: existing.clone(), + decision: AgentEvidenceMergeDecision::IgnoreDuplicate, + conflict: None, + } + }); + } + let incoming_rank = agent_evidence_precedence(incoming.kind, incoming.source); + let existing_rank = agent_evidence_precedence(existing.kind, existing.source); + if incoming_rank > existing_rank { + Ok(AgentEvidenceMergeResult { + fact: incoming, + decision: AgentEvidenceMergeDecision::Enrich, + conflict: Some(existing.clone()), + }) + } else if incoming_rank < existing_rank { + Ok(AgentEvidenceMergeResult { + fact: existing.clone(), + decision: AgentEvidenceMergeDecision::PreserveHigherPrecedence, + conflict: Some(incoming), + }) + } else { + Ok(AgentEvidenceMergeResult { + fact: existing.clone(), + decision: AgentEvidenceMergeDecision::PreserveConflict, + conflict: Some(incoming), + }) + } +} + +pub fn agent_evidence_precedence(kind: AgentEvidenceFactKind, source: AgentEvidenceSource) -> u8 { + use AgentEvidenceFactKind as Kind; + use AgentEvidenceSource as Source; + match (kind, source) { + (Kind::WorkspaceChange, Source::WorkdirObserved) => 100, + (Kind::WorkspaceChange, Source::CanonicalExport) => 80, + (Kind::UserMessage | Kind::AssistantMessage, Source::NativeTranscript) => 100, + (Kind::UserMessage | Kind::AssistantMessage, Source::CanonicalExport) => 95, + (Kind::UserMessage | Kind::AssistantMessage, Source::Acp) => 90, + (Kind::ToolCall | Kind::Approval, Source::Acp) => 100, + (Kind::ToolCall | Kind::Approval, Source::NativeHook) => 90, + (Kind::Usage, Source::CanonicalExport) => 100, + (Kind::Usage, Source::Acp | Source::NativeHook) => 90, + (Kind::SessionLifecycle | Kind::TurnBoundary, Source::NativeHook) => 100, + (Kind::SessionLifecycle | Kind::TurnBoundary, Source::Acp) => 90, + (Kind::TranscriptLocator, Source::NativeHook) => 100, + (_, Source::CanonicalExport) => 85, + (_, Source::NativeTranscript) => 80, + (_, Source::Acp) => 75, + (_, Source::NativeHook) => 70, + (_, Source::WorkdirObserved) => 65, + (_, Source::Reconstructed) => 10, + } +} + +/// Provider-native identifiers carried without assigning them Trail semantics. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentNativeEventIdentity { + pub session_id: Option, + pub turn_id: Option, + pub message_id: Option, + pub tool_id: Option, + pub subagent_id: Option, + pub event_name: String, + pub sequence: Option, +} + +/// Trace and causal identifiers supplied by a provider or allocated by Trail. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentEventCorrelation { + pub parent_event_id: Option, + pub trace_id: Option, + pub span_id: Option, + pub parent_span_id: Option, +} + +/// Durable receipt and artifact provenance for one normalized event. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentEventEvidence { + pub receipt_id: String, + pub raw_digest: Option, + pub transcript_offset: Option, + pub confidence: AgentEvidenceConfidence, +} + +/// A forward-compatible event type encoded exactly as the public wire string. +#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(transparent)] +pub struct AgentLifecycleEventType(pub String); + +impl AgentLifecycleEventType { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Classify a wire event for the pure lifecycle state machine. + /// + /// Unrecognized values deliberately remain inert evidence. Adapters may retain + /// `provider..` values without granting them lifecycle authority. + pub fn kind(&self) -> AgentLifecycleEventKind { + match self.0.as_str() { + "session.started" => AgentLifecycleEventKind::SessionStarted, + "session.resumed" => AgentLifecycleEventKind::SessionResumed, + "session.updated" => AgentLifecycleEventKind::SessionUpdated, + "session.ended" => AgentLifecycleEventKind::SessionEnded, + "turn.started" => AgentLifecycleEventKind::TurnStarted, + "turn.completed" => AgentLifecycleEventKind::TurnCompleted, + "turn.failed" => AgentLifecycleEventKind::TurnFailed, + "turn.cancelled" => AgentLifecycleEventKind::TurnCancelled, + "message.user" => AgentLifecycleEventKind::MessageUser, + "message.assistant.delta" => AgentLifecycleEventKind::MessageAssistantDelta, + "message.assistant.completed" => AgentLifecycleEventKind::MessageAssistantCompleted, + "plan.updated" => AgentLifecycleEventKind::PlanUpdated, + "tool.started" => AgentLifecycleEventKind::ToolStarted, + "tool.completed" => AgentLifecycleEventKind::ToolCompleted, + "tool.failed" => AgentLifecycleEventKind::ToolFailed, + "approval.requested" => AgentLifecycleEventKind::ApprovalRequested, + "approval.decided" => AgentLifecycleEventKind::ApprovalDecided, + "subagent.started" => AgentLifecycleEventKind::SubagentStarted, + "subagent.completed" => AgentLifecycleEventKind::SubagentCompleted, + "subagent.failed" => AgentLifecycleEventKind::SubagentFailed, + "compaction.started" => AgentLifecycleEventKind::CompactionStarted, + "compaction.completed" => AgentLifecycleEventKind::CompactionCompleted, + "usage.updated" => AgentLifecycleEventKind::UsageUpdated, + "model.updated" => AgentLifecycleEventKind::ModelUpdated, + "workspace.diff" => AgentLifecycleEventKind::WorkspaceDiff, + "workspace.file_changed" => AgentLifecycleEventKind::WorkspaceFileChanged, + "workspace.checkpoint" => AgentLifecycleEventKind::WorkspaceCheckpoint, + "context.injected" => AgentLifecycleEventKind::ContextInjected, + "diagnostic" => AgentLifecycleEventKind::Diagnostic, + _ => AgentLifecycleEventKind::Unknown, + } + } +} + +impl From for AgentLifecycleEventType { + fn from(value: AgentLifecycleEventKind) -> Self { + Self(value.wire_name().to_string()) + } +} + +/// Adapter-neutral lifecycle evidence accepted by ACP and native hook transports. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct AgentLifecycleEvent { + pub schema: String, + pub version: u16, + pub event_id: String, + pub event_type: AgentLifecycleEventType, + pub occurred_at: Option, + pub received_at: i64, + pub provider: String, + pub provider_version: Option, + pub transport: AgentCaptureTransport, + pub workspace_id: String, + pub lane_id: Option, + pub capture_run_id: Option, + pub native: AgentNativeEventIdentity, + pub correlation: AgentEventCorrelation, + pub payload: serde_json::Value, + pub evidence: AgentEventEvidence, +} + +/// Untrusted, provider-native receipt accepted by durable capture ingress. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct AgentHookReceiptInput { + pub installation_id: Option, + pub provider: String, + pub native_event: String, + pub native_session_id: Option, + pub native_turn_id: Option, + pub transport: AgentCaptureTransport, + pub connection_id: Option, + pub direction: Option, + pub connection_sequence: Option, + pub dedupe_key: String, + pub payload: serde_json::Value, + pub occurred_at: Option, +} + +/// Durable receipt journal row. Large raw data is held by `raw_object_id`. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentHookReceipt { + pub receipt_id: String, + pub workspace_id: String, + pub installation_id: Option, + pub mapping_id: Option, + pub provider: String, + pub native_event: String, + pub native_session_id: Option, + pub native_turn_id: Option, + pub transport: AgentCaptureTransport, + pub dedupe_key: String, + pub payload_digest: String, + pub raw_object_id: ObjectId, + pub raw_artifact_id: Option, + pub receive_sequence: Option, + pub connection_id: Option, + pub direction: Option, + pub connection_sequence: Option, + pub status: String, + pub attempt_count: u32, + pub next_attempt_at: Option, + pub diagnostic: Option, + pub occurred_at: Option, + pub received_at: i64, + pub processed_at: Option, + pub updated_at: i64, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentHookReceiptIngestReport { + pub receipt: AgentHookReceipt, + pub duplicate: bool, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct AgentHookReplayReport { + pub receipt: AgentHookReceipt, + pub mapping: Option, + pub normalized_events: Vec, + pub actions: Vec, + pub diagnostics: Vec, + pub replayed: bool, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentHookReplayFailure { + pub receipt_id: String, + pub code: String, + pub message: String, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct AgentHookReplayBatchReport { + pub recovered_stale_receipts: usize, + pub replayed: Vec, + pub failures: Vec, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentCaptureRecoveryReport { + pub expired_run_ids: Vec, + pub interrupted_mapping_ids: Vec, + pub interrupted_turn_ids: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct LaneArtifactInput { + pub lane: String, + pub session_id: String, + pub turn_id: Option, + pub provider: String, + pub artifact_kind: String, + pub format: String, + pub source: AgentEvidenceSource, + pub source_locator_redacted: Option, + pub content: Vec, + pub start_offset: Option, + pub end_offset: Option, + pub redaction_profile: Option, + pub trust: String, + pub supersedes_artifact_id: Option, + pub metadata_json: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct LaneArtifact { + pub artifact_id: String, + pub workspace_id: String, + pub lane_id: String, + pub session_id: String, + pub turn_id: Option, + pub provider: String, + pub artifact_kind: String, + pub format: String, + pub source: AgentEvidenceSource, + pub source_locator_redacted: Option, + pub content_object_id: Option, + pub content_digest: String, + pub size_bytes: u64, + pub start_offset: Option, + pub end_offset: Option, + pub redaction_profile: Option, + pub retention_status: String, + pub trust: String, + pub supersedes_artifact_id: Option, + pub created_at: i64, + pub metadata_json: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct LaneAgentSessionInput { + pub provider: String, + pub native_session_id: String, + pub parent_native_session_id: Option, + pub lane: String, + pub trail_session_id: String, + pub capture_run_id: Option, + pub primary_transport: AgentCaptureTransport, + pub transcript_identity: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct LaneAgentSession { + pub mapping_id: String, + pub workspace_id: String, + pub provider: String, + pub native_session_id: String, + pub parent_native_session_id: Option, + pub trail_session_id: String, + pub lane_id: String, + pub capture_run_id: Option, + pub primary_transport: AgentCaptureTransport, + pub transcript_identity: Option, + pub transcript_offset: Option, + pub resume_json: Option, + pub last_attestation_id: Option, + pub status: AgentCapturePhase, + pub pending_turn_outcome: Option, + pub session_close_requested: bool, + pub capture_epoch: u64, + pub finalization_owner: Option, + pub finalization_lease_expires_at: Option, + pub next_receive_sequence: u64, + pub created_at: i64, + pub updated_at: i64, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentFinalizationLeaseReport { + pub mapping: LaneAgentSession, + pub acquired: bool, + pub owner: String, + pub expires_at: i64, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentCaptureRunInput { + pub lane: Option, + pub workdir: String, + pub owner_agent: String, + pub owner_session_id: String, + pub executor_agent: Option, + pub work_item_id: Option, + pub lease_ms: u64, + pub metadata_json: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentCaptureRun { + pub capture_run_id: String, + pub workspace_id: String, + pub lane_id: Option, + pub workdir: String, + pub canonical_workdir: String, + pub owner_agent: String, + pub owner_session_id: String, + pub executor_agent: Option, + pub work_item_id: Option, + pub status: String, + pub created_at: i64, + pub updated_at: i64, + pub expires_at: i64, + pub ended_at: Option, + pub metadata_json: Option, +} + +impl AgentLifecycleEvent { + /// Validate untrusted adapter output before it enters durable semantic storage. + pub fn validate(&self) -> crate::Result<()> { + if self.schema != AGENT_LIFECYCLE_EVENT_SCHEMA { + return Err(crate::Error::InvalidInput(format!( + "unsupported agent lifecycle schema `{}`", + self.schema + ))); + } + if self.version != AGENT_LIFECYCLE_EVENT_VERSION { + return Err(crate::Error::InvalidInput(format!( + "unsupported agent lifecycle event version {}; supported version is {}", + self.version, AGENT_LIFECYCLE_EVENT_VERSION + ))); + } + validate_agent_capture_token("event_id", &self.event_id, 256)?; + validate_agent_capture_token("event_type", self.event_type.as_str(), 128)?; + validate_agent_provider_name(&self.provider)?; + validate_agent_capture_token("workspace_id", &self.workspace_id, 256)?; + validate_agent_capture_token("receipt_id", &self.evidence.receipt_id, 256)?; + validate_optional_agent_capture_token("lane_id", self.lane_id.as_deref(), 256)?; + validate_optional_agent_capture_token( + "capture_run_id", + self.capture_run_id.as_deref(), + 256, + )?; + validate_optional_agent_capture_token( + "native.session_id", + self.native.session_id.as_deref(), + 1024, + )?; + validate_optional_agent_capture_token( + "native.turn_id", + self.native.turn_id.as_deref(), + 1024, + )?; + validate_optional_agent_capture_token( + "native.message_id", + self.native.message_id.as_deref(), + 1024, + )?; + validate_optional_agent_capture_token( + "native.tool_id", + self.native.tool_id.as_deref(), + 1024, + )?; + validate_optional_agent_capture_token( + "native.subagent_id", + self.native.subagent_id.as_deref(), + 1024, + )?; + validate_agent_capture_token("native.event_name", &self.native.event_name, 256)?; + validate_optional_agent_capture_token( + "correlation.parent_event_id", + self.correlation.parent_event_id.as_deref(), + 256, + )?; + validate_optional_agent_capture_token( + "correlation.trace_id", + self.correlation.trace_id.as_deref(), + 256, + )?; + validate_optional_agent_capture_token( + "correlation.span_id", + self.correlation.span_id.as_deref(), + 256, + )?; + validate_optional_agent_capture_token( + "correlation.parent_span_id", + self.correlation.parent_span_id.as_deref(), + 256, + )?; + if self.received_at < 0 || self.occurred_at.is_some_and(|value| value < 0) { + return Err(crate::Error::InvalidInput( + "agent lifecycle timestamps must be non-negative milliseconds".to_string(), + )); + } + if self.event_type.kind() == AgentLifecycleEventKind::Unknown + && !self + .event_type + .as_str() + .starts_with(&format!("provider.{}.", self.provider)) + { + return Err(crate::Error::InvalidInput(format!( + "unknown lifecycle event `{}` must use the `provider.{}.` namespace", + self.event_type, self.provider + ))); + } + let payload_bytes = serde_json::to_vec(&self.payload)?; + if payload_bytes.len() > AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES { + return Err(crate::Error::InvalidInput(format!( + "agent lifecycle payload is {} bytes; maximum is {}", + payload_bytes.len(), + AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES + ))); + } + Ok(()) + } +} + +fn validate_agent_provider_name(value: &str) -> crate::Result<()> { + validate_agent_capture_token("provider", value, 64)?; + if !value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + { + return Err(crate::Error::InvalidInput( + "agent provider must use lowercase ASCII letters, digits, and hyphens".to_string(), + )); + } + Ok(()) +} + +fn validate_optional_agent_capture_token( + field: &str, + value: Option<&str>, + max_len: usize, +) -> crate::Result<()> { + if let Some(value) = value { + validate_agent_capture_token(field, value, max_len)?; + } + Ok(()) +} + +fn validate_agent_capture_token(field: &str, value: &str, max_len: usize) -> crate::Result<()> { + if value.is_empty() { + return Err(crate::Error::InvalidInput(format!( + "agent lifecycle field `{field}` must not be empty" + ))); + } + if value.len() > max_len { + return Err(crate::Error::InvalidInput(format!( + "agent lifecycle field `{field}` exceeds {max_len} bytes" + ))); + } + if value.chars().any(char::is_control) { + return Err(crate::Error::InvalidInput(format!( + "agent lifecycle field `{field}` contains control characters" + ))); + } + Ok(()) +} + +/// Persisted lifecycle phase for one provider-native session mapping. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentCapturePhase { + Idle, + Active, + Finalizing, + Ended, + Interrupted, +} + +/// Closed-turn outcome chosen by the pure transition function. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentTurnOutcome { + Completed, + Failed, + Cancelled, + Interrupted, +} + +/// Exhaustive semantic vocabulary consumed by the lifecycle transition function. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentLifecycleEventKind { + SessionStarted, + SessionResumed, + SessionUpdated, + SessionEnded, + TurnStarted, + TurnCompleted, + TurnFailed, + TurnCancelled, + MessageUser, + MessageAssistantDelta, + MessageAssistantCompleted, + PlanUpdated, + ToolStarted, + ToolCompleted, + ToolFailed, + ApprovalRequested, + ApprovalDecided, + SubagentStarted, + SubagentCompleted, + SubagentFailed, + CompactionStarted, + CompactionCompleted, + UsageUpdated, + ModelUpdated, + WorkspaceDiff, + WorkspaceFileChanged, + WorkspaceCheckpoint, + ContextInjected, + Diagnostic, + Unknown, + /// Internal durable signal emitted after all finalization actions commit. + FinalizationCompleted, + /// Internal recovery signal emitted when finalization cannot be completed. + FinalizationInterrupted, +} + +impl AgentLifecycleEventKind { + pub const ALL: [Self; 32] = [ + Self::SessionStarted, + Self::SessionResumed, + Self::SessionUpdated, + Self::SessionEnded, + Self::TurnStarted, + Self::TurnCompleted, + Self::TurnFailed, + Self::TurnCancelled, + Self::MessageUser, + Self::MessageAssistantDelta, + Self::MessageAssistantCompleted, + Self::PlanUpdated, + Self::ToolStarted, + Self::ToolCompleted, + Self::ToolFailed, + Self::ApprovalRequested, + Self::ApprovalDecided, + Self::SubagentStarted, + Self::SubagentCompleted, + Self::SubagentFailed, + Self::CompactionStarted, + Self::CompactionCompleted, + Self::UsageUpdated, + Self::ModelUpdated, + Self::WorkspaceDiff, + Self::WorkspaceFileChanged, + Self::WorkspaceCheckpoint, + Self::ContextInjected, + Self::Diagnostic, + Self::Unknown, + Self::FinalizationCompleted, + Self::FinalizationInterrupted, + ]; + + pub fn wire_name(self) -> &'static str { + match self { + Self::SessionStarted => "session.started", + Self::SessionResumed => "session.resumed", + Self::SessionUpdated => "session.updated", + Self::SessionEnded => "session.ended", + Self::TurnStarted => "turn.started", + Self::TurnCompleted => "turn.completed", + Self::TurnFailed => "turn.failed", + Self::TurnCancelled => "turn.cancelled", + Self::MessageUser => "message.user", + Self::MessageAssistantDelta => "message.assistant.delta", + Self::MessageAssistantCompleted => "message.assistant.completed", + Self::PlanUpdated => "plan.updated", + Self::ToolStarted => "tool.started", + Self::ToolCompleted => "tool.completed", + Self::ToolFailed => "tool.failed", + Self::ApprovalRequested => "approval.requested", + Self::ApprovalDecided => "approval.decided", + Self::SubagentStarted => "subagent.started", + Self::SubagentCompleted => "subagent.completed", + Self::SubagentFailed => "subagent.failed", + Self::CompactionStarted => "compaction.started", + Self::CompactionCompleted => "compaction.completed", + Self::UsageUpdated => "usage.updated", + Self::ModelUpdated => "model.updated", + Self::WorkspaceDiff => "workspace.diff", + Self::WorkspaceFileChanged => "workspace.file_changed", + Self::WorkspaceCheckpoint => "workspace.checkpoint", + Self::ContextInjected => "context.injected", + Self::Diagnostic => "diagnostic", + Self::Unknown => "provider.unknown.unknown", + Self::FinalizationCompleted => "trail.finalization.completed", + Self::FinalizationInterrupted => "trail.finalization.interrupted", + } + } + + pub(crate) fn terminal_outcome(self) -> Option { + match self { + Self::TurnCompleted => Some(AgentTurnOutcome::Completed), + Self::TurnFailed => Some(AgentTurnOutcome::Failed), + Self::TurnCancelled => Some(AgentTurnOutcome::Cancelled), + _ => None, + } + } + + fn starts_or_implies_turn(self) -> bool { + matches!( + self, + Self::TurnStarted + | Self::MessageUser + | Self::MessageAssistantDelta + | Self::PlanUpdated + | Self::ToolStarted + | Self::ApprovalRequested + | Self::SubagentStarted + | Self::CompactionStarted + ) + } + + fn completes_existing_turn_activity(self) -> bool { + matches!( + self, + Self::MessageAssistantCompleted + | Self::ToolCompleted + | Self::ToolFailed + | Self::ApprovalDecided + | Self::SubagentCompleted + | Self::SubagentFailed + | Self::CompactionCompleted + | Self::WorkspaceDiff + | Self::WorkspaceFileChanged + | Self::WorkspaceCheckpoint + ) + } +} + +/// Bounded persisted facts used by the pure state transition. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentTransitionContext { + pub has_session: bool, + pub has_open_turn: bool, + pub duplicate: bool, + pub has_new_evidence: bool, + pub session_close_requested: bool, + /// Outcome stored when the session entered `finalizing`. + pub pending_turn_outcome: Option, +} + +/// Ordered, idempotently keyed side effects selected by the pure transition. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case", tag = "action")] +pub enum AgentCaptureAction { + EnsureSession, + CreateCaptureEpoch, + BeginTurn { synthetic: bool }, + CaptureBaseline, + AppendEvidence, + StartRootSpan, + StartToolSpan { synthetic: bool }, + EndToolSpan, + StartSubagentSpan { synthetic: bool }, + EndSubagentSpan, + StartCompactionSpan, + EndCompactionSpan, + RequestTurnFinalization { outcome: AgentTurnOutcome }, + ReconcileWorkdir, + ImportTranscript, + CloseTurn { outcome: AgentTurnOutcome }, + RequestSessionClose, + FinalizeAttestation, + CloseSession { interrupted: bool }, + WarnDuplicate, + RecoverInterruptedTurn, + DeferUntilFinalized, +} + +/// Stable diagnostic produced by lifecycle decisions, not by side effects. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AgentTransitionDiagnostic { + DuplicateEvent, + SyntheticTurnStart, + PriorTurnInterrupted, + EventDeferredDuringFinalization, + LateEventAfterSessionClose, + FinalizationInterrupted, +} + +/// Complete pure decision for one lifecycle event. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentTransitionResult { + pub new_phase: AgentCapturePhase, + pub actions: Vec, + pub diagnostics: Vec, +} + +/// Side-effect boundary used by every lifecycle transport after the pure decision step. +pub trait AgentCaptureActionExecutor { + fn execute_agent_capture_actions( + &mut self, + event: &AgentLifecycleEvent, + actions: &[AgentCaptureAction], + ) -> crate::Result<()>; +} + +/// Adapter-neutral coordinator entrypoint. Adapters emit events; executors own I/O. +#[derive(Clone, Copy, Debug, Default)] +pub struct AgentCaptureCoordinator; + +impl AgentCaptureCoordinator { + pub fn decide( + phase: AgentCapturePhase, + event: AgentLifecycleEventKind, + context: AgentTransitionContext, + ) -> AgentTransitionResult { + transition_agent_capture(phase, event, context) + } + + pub fn dispatch( + executor: &mut E, + phase: AgentCapturePhase, + event: &AgentLifecycleEvent, + context: AgentTransitionContext, + ) -> crate::Result { + let result = Self::decide(phase, event.event_type.kind(), context); + executor.execute_agent_capture_actions(event, &result.actions)?; + Ok(result) + } +} + +impl AgentTransitionResult { + fn unchanged(phase: AgentCapturePhase) -> Self { + Self { + new_phase: phase, + actions: Vec::new(), + diagnostics: Vec::new(), + } + } + + fn append(mut self, action: AgentCaptureAction) -> Self { + self.actions.push(action); + self + } + + fn diagnose(mut self, diagnostic: AgentTransitionDiagnostic) -> Self { + self.diagnostics.push(diagnostic); + self + } +} + +/// Choose lifecycle mutations without performing database or filesystem I/O. +pub fn transition_agent_capture( + phase: AgentCapturePhase, + event: AgentLifecycleEventKind, + context: AgentTransitionContext, +) -> AgentTransitionResult { + if context.duplicate { + let mut result = AgentTransitionResult::unchanged(phase); + if context.has_new_evidence { + result.actions.push(AgentCaptureAction::AppendEvidence); + } + result.actions.push(AgentCaptureAction::WarnDuplicate); + result + .diagnostics + .push(AgentTransitionDiagnostic::DuplicateEvent); + return result; + } + + if event == AgentLifecycleEventKind::Unknown { + return AgentTransitionResult::unchanged(phase).append(AgentCaptureAction::AppendEvidence); + } + + match phase { + AgentCapturePhase::Idle => transition_idle(event, context), + AgentCapturePhase::Active => transition_active(event), + AgentCapturePhase::Finalizing => transition_finalizing(event, context), + AgentCapturePhase::Ended | AgentCapturePhase::Interrupted => { + transition_closed(phase, event) + } + } +} + +fn transition_idle( + event: AgentLifecycleEventKind, + context: AgentTransitionContext, +) -> AgentTransitionResult { + if matches!( + event, + AgentLifecycleEventKind::SessionStarted + | AgentLifecycleEventKind::SessionResumed + | AgentLifecycleEventKind::SessionUpdated + ) { + let mut result = AgentTransitionResult::unchanged(AgentCapturePhase::Idle); + if !context.has_session { + result.actions.push(AgentCaptureAction::EnsureSession); + } + result.actions.push(AgentCaptureAction::AppendEvidence); + return result; + } + + if event == AgentLifecycleEventKind::SessionEnded { + return AgentTransitionResult { + new_phase: AgentCapturePhase::Ended, + actions: vec![ + AgentCaptureAction::EnsureSession, + AgentCaptureAction::AppendEvidence, + AgentCaptureAction::ImportTranscript, + AgentCaptureAction::FinalizeAttestation, + AgentCaptureAction::CloseSession { interrupted: false }, + ], + diagnostics: Vec::new(), + }; + } + + if let Some(outcome) = event.terminal_outcome() { + return begin_synthetic_finalization(outcome); + } + + // Providers may run hooks concurrently, so a tool/subagent/completion callback can + // arrive after Stop finalized the turn. Keep the evidence, but never manufacture a + // new active turn from a completion-only event. A corresponding start event still + // recovers a missing turn and later completion callbacks close its spans normally. + if event.completes_existing_turn_activity() { + return AgentTransitionResult::unchanged(AgentCapturePhase::Idle) + .append(AgentCaptureAction::EnsureSession) + .append(AgentCaptureAction::AppendEvidence); + } + + if event.starts_or_implies_turn() { + let synthetic = event != AgentLifecycleEventKind::TurnStarted; + let mut result = AgentTransitionResult { + new_phase: AgentCapturePhase::Active, + actions: vec![ + AgentCaptureAction::EnsureSession, + AgentCaptureAction::CaptureBaseline, + AgentCaptureAction::BeginTurn { synthetic }, + AgentCaptureAction::StartRootSpan, + AgentCaptureAction::AppendEvidence, + ], + diagnostics: Vec::new(), + }; + if synthetic { + result + .diagnostics + .push(AgentTransitionDiagnostic::SyntheticTurnStart); + } + append_event_span_actions(&mut result.actions, event, true); + return result; + } + + match event { + AgentLifecycleEventKind::FinalizationInterrupted => AgentTransitionResult { + new_phase: AgentCapturePhase::Interrupted, + actions: vec![AgentCaptureAction::CloseSession { interrupted: true }], + diagnostics: vec![AgentTransitionDiagnostic::FinalizationInterrupted], + }, + AgentLifecycleEventKind::FinalizationCompleted => { + AgentTransitionResult::unchanged(AgentCapturePhase::Idle) + } + AgentLifecycleEventKind::UsageUpdated + | AgentLifecycleEventKind::ModelUpdated + | AgentLifecycleEventKind::ContextInjected + | AgentLifecycleEventKind::Diagnostic => { + AgentTransitionResult::unchanged(AgentCapturePhase::Idle) + .append(AgentCaptureAction::EnsureSession) + .append(AgentCaptureAction::AppendEvidence) + } + AgentLifecycleEventKind::Unknown => unreachable!("unknown events return above"), + _ => AgentTransitionResult::unchanged(AgentCapturePhase::Idle) + .append(AgentCaptureAction::AppendEvidence), + } +} + +fn begin_synthetic_finalization(outcome: AgentTurnOutcome) -> AgentTransitionResult { + AgentTransitionResult { + new_phase: AgentCapturePhase::Finalizing, + actions: vec![ + AgentCaptureAction::EnsureSession, + AgentCaptureAction::CaptureBaseline, + AgentCaptureAction::BeginTurn { synthetic: true }, + AgentCaptureAction::StartRootSpan, + AgentCaptureAction::AppendEvidence, + AgentCaptureAction::RequestTurnFinalization { outcome }, + AgentCaptureAction::ReconcileWorkdir, + AgentCaptureAction::ImportTranscript, + ], + diagnostics: vec![AgentTransitionDiagnostic::SyntheticTurnStart], + } +} + +fn transition_active(event: AgentLifecycleEventKind) -> AgentTransitionResult { + if event == AgentLifecycleEventKind::TurnStarted { + return AgentTransitionResult { + new_phase: AgentCapturePhase::Active, + actions: vec![ + AgentCaptureAction::AppendEvidence, + AgentCaptureAction::RecoverInterruptedTurn, + AgentCaptureAction::CloseTurn { + outcome: AgentTurnOutcome::Interrupted, + }, + AgentCaptureAction::CaptureBaseline, + AgentCaptureAction::BeginTurn { synthetic: false }, + AgentCaptureAction::StartRootSpan, + ], + diagnostics: vec![AgentTransitionDiagnostic::PriorTurnInterrupted], + }; + } + + if let Some(outcome) = event.terminal_outcome() { + return AgentTransitionResult { + new_phase: AgentCapturePhase::Finalizing, + actions: vec![ + AgentCaptureAction::AppendEvidence, + AgentCaptureAction::RequestTurnFinalization { outcome }, + AgentCaptureAction::ReconcileWorkdir, + AgentCaptureAction::ImportTranscript, + ], + diagnostics: Vec::new(), + }; + } + + if event == AgentLifecycleEventKind::SessionEnded { + return AgentTransitionResult { + new_phase: AgentCapturePhase::Finalizing, + actions: vec![ + AgentCaptureAction::AppendEvidence, + AgentCaptureAction::RequestSessionClose, + AgentCaptureAction::RequestTurnFinalization { + outcome: AgentTurnOutcome::Interrupted, + }, + AgentCaptureAction::ReconcileWorkdir, + AgentCaptureAction::ImportTranscript, + ], + diagnostics: Vec::new(), + }; + } + + if event == AgentLifecycleEventKind::FinalizationInterrupted { + return AgentTransitionResult { + new_phase: AgentCapturePhase::Interrupted, + actions: vec![ + AgentCaptureAction::CloseTurn { + outcome: AgentTurnOutcome::Interrupted, + }, + AgentCaptureAction::CloseSession { interrupted: true }, + ], + diagnostics: vec![AgentTransitionDiagnostic::FinalizationInterrupted], + }; + } + + if event == AgentLifecycleEventKind::FinalizationCompleted { + return AgentTransitionResult::unchanged(AgentCapturePhase::Active); + } + + let mut result = AgentTransitionResult::unchanged(AgentCapturePhase::Active) + .append(AgentCaptureAction::AppendEvidence); + append_event_span_actions(&mut result.actions, event, false); + result +} + +fn transition_finalizing( + event: AgentLifecycleEventKind, + context: AgentTransitionContext, +) -> AgentTransitionResult { + if event == AgentLifecycleEventKind::FinalizationCompleted { + let outcome = context + .pending_turn_outcome + .unwrap_or(AgentTurnOutcome::Interrupted); + let mut actions = vec![AgentCaptureAction::CloseTurn { outcome }]; + let new_phase = if context.session_close_requested { + actions.push(AgentCaptureAction::FinalizeAttestation); + actions.push(AgentCaptureAction::CloseSession { interrupted: false }); + AgentCapturePhase::Ended + } else { + AgentCapturePhase::Idle + }; + return AgentTransitionResult { + new_phase, + actions, + diagnostics: Vec::new(), + }; + } + + if event == AgentLifecycleEventKind::FinalizationInterrupted { + let mut actions = vec![AgentCaptureAction::CloseTurn { + outcome: AgentTurnOutcome::Interrupted, + }]; + if context.session_close_requested { + actions.push(AgentCaptureAction::CloseSession { interrupted: true }); + } + return AgentTransitionResult { + new_phase: AgentCapturePhase::Interrupted, + actions, + diagnostics: vec![AgentTransitionDiagnostic::FinalizationInterrupted], + }; + } + + if event == AgentLifecycleEventKind::SessionEnded { + return AgentTransitionResult::unchanged(AgentCapturePhase::Finalizing) + .append(AgentCaptureAction::AppendEvidence) + .append(AgentCaptureAction::RequestSessionClose) + .append(AgentCaptureAction::ImportTranscript); + } + + if event == AgentLifecycleEventKind::TurnStarted + || event == AgentLifecycleEventKind::MessageUser + { + return AgentTransitionResult::unchanged(AgentCapturePhase::Finalizing) + .append(AgentCaptureAction::AppendEvidence) + .append(AgentCaptureAction::DeferUntilFinalized) + .diagnose(AgentTransitionDiagnostic::EventDeferredDuringFinalization); + } + + let mut result = AgentTransitionResult::unchanged(AgentCapturePhase::Finalizing) + .append(AgentCaptureAction::AppendEvidence); + append_event_span_actions(&mut result.actions, event, false); + result +} + +fn transition_closed( + phase: AgentCapturePhase, + event: AgentLifecycleEventKind, +) -> AgentTransitionResult { + if matches!( + event, + AgentLifecycleEventKind::SessionStarted | AgentLifecycleEventKind::SessionResumed + ) { + return AgentTransitionResult { + new_phase: AgentCapturePhase::Idle, + actions: vec![ + AgentCaptureAction::CreateCaptureEpoch, + AgentCaptureAction::AppendEvidence, + ], + diagnostics: Vec::new(), + }; + } + + AgentTransitionResult::unchanged(phase) + .append(AgentCaptureAction::AppendEvidence) + .diagnose(AgentTransitionDiagnostic::LateEventAfterSessionClose) +} + +fn append_event_span_actions( + actions: &mut Vec, + event: AgentLifecycleEventKind, + synthetic: bool, +) { + match event { + AgentLifecycleEventKind::ToolStarted => { + actions.push(AgentCaptureAction::StartToolSpan { synthetic }); + } + AgentLifecycleEventKind::ToolCompleted | AgentLifecycleEventKind::ToolFailed => { + if synthetic { + actions.push(AgentCaptureAction::StartToolSpan { synthetic: true }); + } + actions.push(AgentCaptureAction::EndToolSpan); + } + AgentLifecycleEventKind::SubagentStarted => { + actions.push(AgentCaptureAction::StartSubagentSpan { synthetic }); + } + AgentLifecycleEventKind::SubagentCompleted | AgentLifecycleEventKind::SubagentFailed => { + if synthetic { + actions.push(AgentCaptureAction::StartSubagentSpan { synthetic: true }); + } + actions.push(AgentCaptureAction::EndSubagentSpan); + } + AgentLifecycleEventKind::CompactionStarted => { + actions.push(AgentCaptureAction::StartCompactionSpan); + } + AgentLifecycleEventKind::CompactionCompleted => { + if synthetic { + actions.push(AgentCaptureAction::StartCompactionSpan); + } + actions.push(AgentCaptureAction::EndCompactionSpan); + } + _ => {} + } +} + +impl fmt::Display for AgentLifecycleEventType { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +#[cfg(test)] +mod agent_capture_model_tests { + use super::*; + + const PHASES: [AgentCapturePhase; 5] = [ + AgentCapturePhase::Idle, + AgentCapturePhase::Active, + AgentCapturePhase::Finalizing, + AgentCapturePhase::Ended, + AgentCapturePhase::Interrupted, + ]; + + fn context() -> AgentTransitionContext { + AgentTransitionContext { + has_session: true, + has_open_turn: true, + duplicate: false, + has_new_evidence: true, + session_close_requested: false, + pending_turn_outcome: Some(AgentTurnOutcome::Completed), + } + } + + #[test] + fn normalized_vocabulary_round_trips_to_kinds() { + for kind in AgentLifecycleEventKind::ALL { + if matches!( + kind, + AgentLifecycleEventKind::Unknown + | AgentLifecycleEventKind::FinalizationCompleted + | AgentLifecycleEventKind::FinalizationInterrupted + ) { + continue; + } + let wire = AgentLifecycleEventType::from(kind); + assert_eq!(wire.kind(), kind, "{}", wire.as_str()); + } + assert_eq!( + AgentLifecycleEventType::new("provider.codex.future_event").kind(), + AgentLifecycleEventKind::Unknown + ); + } + + #[test] + fn lifecycle_cross_product_is_total_and_bounded() { + for phase in PHASES { + for event in AgentLifecycleEventKind::ALL { + let result = transition_agent_capture(phase, event, context()); + assert!(result.actions.len() <= 10, "{phase:?} + {event:?}"); + assert!(result.diagnostics.len() <= 2, "{phase:?} + {event:?}"); + } + } + } + + #[test] + fn idle_turn_start_captures_baseline_and_root_span() { + let result = transition_agent_capture( + AgentCapturePhase::Idle, + AgentLifecycleEventKind::TurnStarted, + AgentTransitionContext::default(), + ); + assert_eq!(result.new_phase, AgentCapturePhase::Active); + assert_eq!( + result.actions, + vec![ + AgentCaptureAction::EnsureSession, + AgentCaptureAction::CaptureBaseline, + AgentCaptureAction::BeginTurn { synthetic: false }, + AgentCaptureAction::StartRootSpan, + AgentCaptureAction::AppendEvidence, + ] + ); + } + + #[test] + fn missing_turn_start_is_recovered_before_terminal_finalization() { + let result = transition_agent_capture( + AgentCapturePhase::Idle, + AgentLifecycleEventKind::TurnFailed, + AgentTransitionContext::default(), + ); + assert_eq!(result.new_phase, AgentCapturePhase::Finalizing); + assert!(result + .actions + .contains(&AgentCaptureAction::BeginTurn { synthetic: true })); + assert!(result + .actions + .contains(&AgentCaptureAction::RequestTurnFinalization { + outcome: AgentTurnOutcome::Failed + })); + assert_eq!( + result.diagnostics, + vec![AgentTransitionDiagnostic::SyntheticTurnStart] + ); + } + + #[test] + fn idle_completion_callbacks_never_reopen_a_finalized_turn() { + for event in [ + AgentLifecycleEventKind::MessageAssistantCompleted, + AgentLifecycleEventKind::ToolCompleted, + AgentLifecycleEventKind::ToolFailed, + AgentLifecycleEventKind::ApprovalDecided, + AgentLifecycleEventKind::SubagentCompleted, + AgentLifecycleEventKind::SubagentFailed, + AgentLifecycleEventKind::CompactionCompleted, + AgentLifecycleEventKind::WorkspaceDiff, + AgentLifecycleEventKind::WorkspaceFileChanged, + AgentLifecycleEventKind::WorkspaceCheckpoint, + ] { + let result = transition_agent_capture( + AgentCapturePhase::Idle, + event, + AgentTransitionContext { + has_session: true, + ..AgentTransitionContext::default() + }, + ); + assert_eq!(result.new_phase, AgentCapturePhase::Idle, "{event:?}"); + assert_eq!( + result.actions, + vec![ + AgentCaptureAction::EnsureSession, + AgentCaptureAction::AppendEvidence, + ], + "{event:?}" + ); + assert!(result.diagnostics.is_empty(), "{event:?}"); + } + } + + #[test] + fn active_turn_start_interrupts_prior_turn() { + let result = transition_agent_capture( + AgentCapturePhase::Active, + AgentLifecycleEventKind::TurnStarted, + context(), + ); + assert_eq!(result.new_phase, AgentCapturePhase::Active); + assert!(result + .actions + .contains(&AgentCaptureAction::RecoverInterruptedTurn)); + assert!(result.actions.contains(&AgentCaptureAction::CloseTurn { + outcome: AgentTurnOutcome::Interrupted + })); + } + + #[test] + fn session_end_waits_for_active_turn_finalization() { + let result = transition_agent_capture( + AgentCapturePhase::Active, + AgentLifecycleEventKind::SessionEnded, + context(), + ); + assert_eq!(result.new_phase, AgentCapturePhase::Finalizing); + assert!(result + .actions + .contains(&AgentCaptureAction::RequestSessionClose)); + assert!(!result + .actions + .contains(&AgentCaptureAction::CloseSession { interrupted: false })); + } + + #[test] + fn finalization_completion_closes_session_only_when_requested() { + let open = transition_agent_capture( + AgentCapturePhase::Finalizing, + AgentLifecycleEventKind::FinalizationCompleted, + context(), + ); + assert_eq!(open.new_phase, AgentCapturePhase::Idle); + + let closed = transition_agent_capture( + AgentCapturePhase::Finalizing, + AgentLifecycleEventKind::FinalizationCompleted, + AgentTransitionContext { + session_close_requested: true, + ..context() + }, + ); + assert_eq!(closed.new_phase, AgentCapturePhase::Ended); + assert!(closed + .actions + .contains(&AgentCaptureAction::CloseSession { interrupted: false })); + } + + #[test] + fn duplicate_terminal_event_never_refinalizes() { + let result = transition_agent_capture( + AgentCapturePhase::Finalizing, + AgentLifecycleEventKind::TurnCompleted, + AgentTransitionContext { + duplicate: true, + has_new_evidence: true, + ..context() + }, + ); + assert_eq!(result.new_phase, AgentCapturePhase::Finalizing); + assert_eq!( + result.actions, + vec![ + AgentCaptureAction::AppendEvidence, + AgentCaptureAction::WarnDuplicate + ] + ); + } + + #[test] + fn finalization_preserves_pending_failure_outcome() { + let result = transition_agent_capture( + AgentCapturePhase::Finalizing, + AgentLifecycleEventKind::FinalizationCompleted, + AgentTransitionContext { + pending_turn_outcome: Some(AgentTurnOutcome::Failed), + ..context() + }, + ); + assert_eq!(result.new_phase, AgentCapturePhase::Idle); + assert_eq!( + result.actions, + vec![AgentCaptureAction::CloseTurn { + outcome: AgentTurnOutcome::Failed + }] + ); + } + + #[test] + fn duplicate_events_never_schedule_semantic_mutations() { + for phase in PHASES { + for event in AgentLifecycleEventKind::ALL { + let result = transition_agent_capture( + phase, + event, + AgentTransitionContext { + duplicate: true, + has_new_evidence: false, + ..context() + }, + ); + assert_eq!(result.new_phase, phase); + assert_eq!(result.actions, vec![AgentCaptureAction::WarnDuplicate]); + } + } + } + + #[test] + fn unknown_event_is_inert_in_every_phase() { + for phase in PHASES { + let result = + transition_agent_capture(phase, AgentLifecycleEventKind::Unknown, context()); + assert_eq!(result.new_phase, phase); + assert_eq!(result.actions, vec![AgentCaptureAction::AppendEvidence]); + } + } + + #[test] + fn closed_session_resumes_as_new_capture_epoch() { + for phase in [AgentCapturePhase::Ended, AgentCapturePhase::Interrupted] { + let result = + transition_agent_capture(phase, AgentLifecycleEventKind::SessionResumed, context()); + assert_eq!(result.new_phase, AgentCapturePhase::Idle); + assert_eq!( + result.actions, + vec![ + AgentCaptureAction::CreateCaptureEpoch, + AgentCaptureAction::AppendEvidence + ] + ); + } + } + + #[test] + fn event_validation_rejects_wrong_schema_and_oversized_payload() { + let mut event = AgentLifecycleEvent { + schema: AGENT_LIFECYCLE_EVENT_SCHEMA.to_string(), + version: AGENT_LIFECYCLE_EVENT_VERSION, + event_id: "evt_1".to_string(), + event_type: AgentLifecycleEventType::from(AgentLifecycleEventKind::ToolCompleted), + occurred_at: Some(1), + received_at: 2, + provider: "codex".to_string(), + provider_version: Some("1.0".to_string()), + transport: AgentCaptureTransport::NativeHooks, + workspace_id: "workspace_1".to_string(), + lane_id: Some("lane_1".to_string()), + capture_run_id: None, + native: AgentNativeEventIdentity { + event_name: "PostToolUse".to_string(), + ..AgentNativeEventIdentity::default() + }, + correlation: AgentEventCorrelation::default(), + payload: serde_json::json!({"ok": true}), + evidence: AgentEventEvidence { + receipt_id: "receipt_1".to_string(), + raw_digest: None, + transcript_offset: None, + confidence: AgentEvidenceConfidence::NativeStructured, + }, + }; + event.validate().unwrap(); + + event.schema = "wrong".to_string(); + assert!(event.validate().is_err()); + event.schema = AGENT_LIFECYCLE_EVENT_SCHEMA.to_string(); + event.payload = + serde_json::Value::String("x".repeat(AGENT_LIFECYCLE_MAX_PAYLOAD_BYTES + 1)); + assert!(event.validate().is_err()); + } + + #[test] + fn unknown_event_requires_provider_namespace() { + let event_type = AgentLifecycleEventType::new("future.event"); + assert_eq!(event_type.kind(), AgentLifecycleEventKind::Unknown); + + let mut event = AgentLifecycleEvent { + schema: AGENT_LIFECYCLE_EVENT_SCHEMA.to_string(), + version: AGENT_LIFECYCLE_EVENT_VERSION, + event_id: "evt_1".to_string(), + event_type, + occurred_at: None, + received_at: 1, + provider: "codex".to_string(), + provider_version: None, + transport: AgentCaptureTransport::NativeHooks, + workspace_id: "workspace_1".to_string(), + lane_id: None, + capture_run_id: None, + native: AgentNativeEventIdentity { + event_name: "FutureEvent".to_string(), + ..AgentNativeEventIdentity::default() + }, + correlation: AgentEventCorrelation::default(), + payload: serde_json::Value::Null, + evidence: AgentEventEvidence { + receipt_id: "receipt_1".to_string(), + raw_digest: None, + transcript_offset: None, + confidence: AgentEvidenceConfidence::Heuristic, + }, + }; + assert!(event.validate().is_err()); + event.event_type = AgentLifecycleEventType::new("provider.codex.future_event"); + event.validate().unwrap(); + } + + #[test] + fn evidence_merge_is_monotonic_and_preserves_equal_rank_conflicts() { + let reconstructed = AgentEvidenceFact { + kind: AgentEvidenceFactKind::AssistantMessage, + key: "message-1".to_string(), + value: serde_json::json!("guessed"), + source: AgentEvidenceSource::Reconstructed, + confidence: AgentEvidenceConfidence::Heuristic, + observed_at: Some(1), + }; + let transcript = AgentEvidenceFact { + value: serde_json::json!("canonical"), + source: AgentEvidenceSource::NativeTranscript, + confidence: AgentEvidenceConfidence::NativeStructured, + observed_at: Some(2), + ..reconstructed.clone() + }; + let enriched = merge_agent_evidence_fact(Some(&reconstructed), transcript.clone()).unwrap(); + assert_eq!(enriched.decision, AgentEvidenceMergeDecision::Enrich); + assert_eq!(enriched.fact, transcript); + + let conflict = AgentEvidenceFact { + value: serde_json::json!("different canonical"), + ..transcript.clone() + }; + let preserved = merge_agent_evidence_fact(Some(&transcript), conflict).unwrap(); + assert_eq!( + preserved.decision, + AgentEvidenceMergeDecision::PreserveConflict + ); + assert!(preserved.conflict.is_some()); + } + + #[test] + fn observed_workdir_beats_protocol_diff_for_workspace_facts() { + assert!( + agent_evidence_precedence( + AgentEvidenceFactKind::WorkspaceChange, + AgentEvidenceSource::WorkdirObserved, + ) > agent_evidence_precedence( + AgentEvidenceFactKind::WorkspaceChange, + AgentEvidenceSource::Acp, + ) + ); + } +} diff --git a/trail/src/model/agent_evidence.rs b/trail/src/model/agent_evidence.rs new file mode 100644 index 0000000..2b04bfe --- /dev/null +++ b/trail/src/model/agent_evidence.rs @@ -0,0 +1,397 @@ +pub const TURN_EVIDENCE_MANIFEST_SCHEMA: &str = "trail.turn_evidence_manifest"; +pub const TURN_EVIDENCE_MANIFEST_VERSION: u16 = 1; +pub const SESSION_ATTESTATION_SCHEMA: &str = "trail.session_attestation"; +pub const SESSION_ATTESTATION_VERSION: u16 = 1; +pub const AGENT_TRACE_SCHEMA: &str = "trail.agent_trace"; +pub const AGENT_TRACE_VERSION: u16 = 1; + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct TurnEvidenceCoverage { + pub receipt_ids: Vec, + pub event_ids: Vec, + pub message_ids: Vec, + pub artifact_ids: Vec, + pub change_ids: Vec, + pub tool_span_ids: Vec, + pub approval_ids: Vec, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct TurnEvidenceStatement { + pub schema: String, + pub version: u16, + pub workspace_id: String, + pub lane_id: String, + pub session_id: String, + pub turn_id: String, + pub before_change: ChangeId, + pub after_change: ChangeId, + pub turn_status: String, + pub coverage: TurnEvidenceCoverage, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct TurnEvidenceManifest { + pub manifest_id: String, + pub lane_id: String, + pub session_id: String, + pub turn_id: String, + pub schema_version: u16, + pub object_id: ObjectId, + pub digest: String, + pub created_at: i64, + pub statement: TurnEvidenceStatement, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct ProvenanceNodeInput { + pub session_id: String, + pub turn_id: Option, + pub node_kind: String, + pub summary: String, + pub event_id: Option, + pub span_id: Option, + pub message_id: Option, + pub change_id: Option, + pub artifact_id: Option, + pub source_confidence: String, + pub classifier_version: Option, + pub attributes: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct ProvenanceNode { + pub provenance_node_id: String, + pub lane_id: String, + pub session_id: String, + pub turn_id: Option, + pub node_kind: String, + pub summary: String, + pub event_id: Option, + pub span_id: Option, + pub message_id: Option, + pub change_id: Option, + pub artifact_id: Option, + pub source_confidence: String, + pub classifier_version: Option, + pub created_at: i64, + pub attributes: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct ProvenanceEdgeInput { + pub from_node_id: String, + pub to_node_id: String, + pub relation: String, + pub source_confidence: String, + pub receipt_id: Option, + pub attributes: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct ProvenanceEdge { + pub provenance_edge_id: String, + pub lane_id: String, + pub session_id: String, + pub from_node_id: String, + pub to_node_id: String, + pub relation: String, + pub source_confidence: String, + pub receipt_id: Option, + pub created_at: i64, + pub attributes: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct ActivityClassificationReport { + pub session_id: String, + pub classifier_version: String, + pub nodes: Vec, + pub edges: Vec, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct SessionAttestationTurn { + pub turn_id: String, + pub change_id: Option, + pub evidence_manifest_id: String, + pub evidence_digest: String, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct SessionAttestationStatement { + pub schema: String, + pub version: u16, + pub workspace_id: String, + pub lane_id: String, + pub session_id: String, + pub capture_run_id: Option, + pub previous_attestation_id: Option, + pub turns: Vec, + pub capture_policy: String, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct SessionAttestation { + pub attestation_id: String, + pub lane_id: String, + pub session_id: String, + pub capture_run_id: Option, + pub previous_attestation_id: Option, + pub statement_object_id: ObjectId, + pub statement_digest: String, + pub signature: Option, + pub status: String, + pub created_at: i64, + pub superseded_by: Option, + pub metadata: Option, + pub turns: Vec, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AttestationSignature { + pub algorithm: String, + pub key_id: String, + pub public_key_hex: String, + pub signature_hex: String, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AttestationKeyRevocation { + pub key_id: String, + pub public_key_hex: String, + pub reason: String, + pub revoked_at: i64, + pub metadata: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AttestationVerificationReport { + pub attestation_id: String, + pub statement_digest_valid: bool, + pub evidence_digests_valid: bool, + pub chain_valid: bool, + pub signature_status: String, + pub valid: bool, + pub diagnostics: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct LearningInput { + pub session_id: String, + pub turn_id: Option, + pub scope: String, + pub body: String, + pub confidence: Option, + pub source_artifact_id: Option, + pub anchor: Option, + pub expires_at: Option, + pub metadata: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct Learning { + pub learning_id: String, + pub lane_id: String, + pub session_id: String, + pub turn_id: Option, + pub scope: String, + pub body: String, + pub status: String, + pub confidence: Option, + pub source_artifact_id: Option, + pub anchor: Option, + pub created_at: i64, + pub reviewed_at: Option, + pub reviewer: Option, + pub expires_at: Option, + pub superseded_by: Option, + pub metadata: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct GitAgentLinkInput { + pub session_id: String, + pub turn_id: Option, + pub git_commit: String, + pub from_change: Option, + pub through_change: Option, + pub confidence: String, + pub source: String, + pub metadata: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct GitAgentLink { + pub git_agent_link_id: String, + pub git_commit: String, + pub lane_id: String, + pub session_id: String, + pub turn_id: Option, + pub from_change: Option, + pub through_change: Option, + pub confidence: String, + pub source: String, + pub created_at: i64, + pub metadata: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct PortableAgentArtifact { + pub artifact_id: String, + pub provider: String, + pub artifact_kind: String, + pub format: String, + pub source: AgentEvidenceSource, + pub content_digest: String, + pub size_bytes: u64, + pub start_offset: Option, + pub end_offset: Option, + pub trust: String, + pub retention_status: String, + pub content: Option>, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct PortableAgentTrace { + pub schema: String, + pub version: u16, + pub source_workspace_id: String, + pub lane_id: String, + pub session_id: String, + pub session_status: String, + pub evidence_manifests: Vec, + pub artifacts: Vec, + pub provenance_nodes: Vec, + pub provenance_edges: Vec, + pub attestations: Vec, + pub learnings: Vec, + pub git_links: Vec, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AgentTraceVerificationReport { + pub schema_valid: bool, + pub ordering_valid: bool, + pub artifact_digests_valid: bool, + pub evidence_references_valid: bool, + pub attestation_references_valid: bool, + pub valid: bool, + pub diagnostics: Vec, +} + +impl PortableAgentTrace { + pub fn to_canonical_json(&self) -> crate::Result> { + let mut bytes = serde_json::to_vec(self)?; + bytes.push(b'\n'); + Ok(bytes) + } + + pub fn from_json(bytes: &[u8]) -> crate::Result { + const MAX_TRACE_BYTES: usize = 256 * 1024 * 1024; + if bytes.len() > MAX_TRACE_BYTES { + return Err(crate::Error::InvalidInput(format!( + "portable agent trace is {} bytes; maximum is {MAX_TRACE_BYTES}", + bytes.len() + ))); + } + let trace: Self = serde_json::from_slice(bytes)?; + let verification = trace.verify(); + if !verification.valid { + return Err(crate::Error::InvalidInput(format!( + "portable agent trace failed verification: {}", + verification.diagnostics.join("; ") + ))); + } + Ok(trace) + } + + pub fn verify(&self) -> AgentTraceVerificationReport { + use sha2::{Digest, Sha256}; + + let schema_valid = self.schema == AGENT_TRACE_SCHEMA && self.version == AGENT_TRACE_VERSION; + let mut diagnostics = Vec::new(); + if !schema_valid { + diagnostics.push(format!( + "unsupported trace schema `{}` version {}", + self.schema, self.version + )); + } + let ordering_valid = sorted_unique_by(&self.evidence_manifests, |value| &value.turn_id) + && sorted_unique_by(&self.artifacts, |value| &value.artifact_id) + && sorted_unique_by(&self.provenance_nodes, |value| &value.provenance_node_id) + && sorted_unique_by(&self.provenance_edges, |value| &value.provenance_edge_id) + && sorted_unique_by(&self.attestations, |value| &value.attestation_id) + && sorted_unique_by(&self.learnings, |value| &value.learning_id) + && sorted_unique_by(&self.git_links, |value| &value.git_agent_link_id); + if !ordering_valid { + diagnostics.push("trace collections are not canonically sorted and unique".to_string()); + } + let artifact_digests_valid = self.artifacts.iter().all(|artifact| { + artifact.content.as_ref().is_none_or(|content| { + artifact.size_bytes == content.len() as u64 + && artifact.content_digest + == format!("sha256:{}", hex::encode(Sha256::digest(content))) + }) + }); + if !artifact_digests_valid { + diagnostics.push("one or more attachment digests are invalid".to_string()); + } + let artifact_ids = self + .artifacts + .iter() + .map(|artifact| artifact.artifact_id.as_str()) + .collect::>(); + let evidence_references_valid = self.evidence_manifests.iter().all(|manifest| { + manifest.session_id == self.session_id + && manifest.lane_id == self.lane_id + && manifest + .statement + .coverage + .artifact_ids + .iter() + .all(|artifact_id| artifact_ids.contains(artifact_id.as_str())) + }); + if !evidence_references_valid { + diagnostics + .push("an evidence manifest has a missing or cross-session reference".to_string()); + } + let manifests = self + .evidence_manifests + .iter() + .map(|manifest| (manifest.manifest_id.as_str(), manifest.digest.as_str())) + .collect::>(); + let attestation_references_valid = self.attestations.iter().all(|attestation| { + attestation.session_id == self.session_id + && attestation.lane_id == self.lane_id + && attestation.turns.iter().all(|turn| { + manifests + .get(turn.evidence_manifest_id.as_str()) + .is_some_and(|digest| *digest == turn.evidence_digest) + }) + }); + if !attestation_references_valid { + diagnostics + .push("an attestation references missing or mismatched evidence".to_string()); + } + AgentTraceVerificationReport { + schema_valid, + ordering_valid, + artifact_digests_valid, + evidence_references_valid, + attestation_references_valid, + valid: schema_valid + && ordering_valid + && artifact_digests_valid + && evidence_references_valid + && attestation_references_valid, + diagnostics, + } + } +} + +fn sorted_unique_by(values: &[T], key: impl Fn(&T) -> &str) -> bool { + values.windows(2).all(|pair| key(&pair[0]) < key(&pair[1])) +} diff --git a/trail/src/model/domain/config.rs b/trail/src/model/domain/config.rs index a3d8306..f0d8647 100644 --- a/trail/src/model/domain/config.rs +++ b/trail/src/model/domain/config.rs @@ -9,6 +9,8 @@ pub const ANCHOR_KIND: &str = "Anchor"; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct TrailConfig { pub workspace: WorkspaceConfig, + #[serde(default)] + pub agent: AgentConfig, pub recording: RecordingConfig, pub text: TextConfig, pub lane: LaneConfig, @@ -21,6 +23,12 @@ pub struct TrailConfig { pub workspace_views: WorkspaceViewsConfig, } +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct AgentConfig { + #[serde(default)] + pub default_provider: Option, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct WorkspaceConfig { pub id: WorkspaceId, @@ -271,6 +279,7 @@ impl TrailConfig { id: workspace_id, default_branch: default_branch.into(), }, + agent: AgentConfig::default(), recording: RecordingConfig { mode: "save".to_string(), debounce_ms: 500, diff --git a/trail/src/model/domain/objects.rs b/trail/src/model/domain/objects.rs index 374a342..c48c241 100644 --- a/trail/src/model/domain/objects.rs +++ b/trail/src/model/domain/objects.rs @@ -11,6 +11,8 @@ pub struct WorktreeRoot { pub version: u16, pub path_map_root: Option, pub file_index_map_root: Option, + #[serde(default)] + pub case_fold_map_root: Option, pub file_count: u64, pub total_text_bytes: u64, pub created_by: ChangeId, diff --git a/trail/src/model/inspect/patch.rs b/trail/src/model/inspect/patch.rs index 0e6c5a9..bb0b7e5 100644 --- a/trail/src/model/inspect/patch.rs +++ b/trail/src/model/inspect/patch.rs @@ -45,13 +45,13 @@ pub enum PatchEdit { pub(crate) fn validate_external_patch_edit_sources( label: &str, - edits_len: usize, - files_len: usize, + edits_present: bool, + files_present: bool, ) -> crate::Result<()> { - match (edits_len > 0, files_len > 0) { + match (edits_present, files_present) { (true, false) | (false, true) => Ok(()), (false, false) => Err(crate::Error::InvalidInput(format!( - "{label} requires at least one edit in `edits` or `files`" + "{label} requires exactly one explicit edit source: `edits` or `files`" ))), (true, true) => Err(crate::Error::InvalidInput(format!( "{label} must use either `edits` or `files`, not both" diff --git a/trail/src/model/lane/activity.rs b/trail/src/model/lane/activity.rs index 8656f5e..4d8f885 100644 --- a/trail/src/model/lane/activity.rs +++ b/trail/src/model/lane/activity.rs @@ -22,6 +22,13 @@ pub struct LaneSessionContextReport { pub recent_operations: Vec, } +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct AcpPathMapping { + pub original: String, + pub effective: String, + pub isolated: bool, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct LaneAcpSession { pub acp_session_id: String, @@ -29,10 +36,13 @@ pub struct LaneAcpSession { pub lane_id: String, pub trail_session_id: String, pub cwd: String, + pub path_mappings: Vec, pub provider: Option, pub model: Option, pub upstream_command_json: Option, pub status: String, + pub current_mode_id: Option, + pub config_options: serde_json::Value, pub created_at: i64, pub updated_at: i64, } @@ -50,17 +60,6 @@ pub struct AcpProviderProfile { pub default_terminal_command: Option>, } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct AcpInstallReport { - pub agent: String, - pub editor: String, - pub dry_run: bool, - pub relay_command: Vec, - pub snippet: String, - pub detected: bool, - pub warnings: Vec, -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct AcpDoctorCheck { pub name: String, @@ -68,6 +67,19 @@ pub struct AcpDoctorCheck { pub message: String, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AcpConformanceEvidence { + pub wire_version: u16, + pub schema_commit: String, + pub schema_sha256: String, + pub meta_sha256: String, + pub transport: String, + pub method_count: u16, + pub evidence_status: String, + pub build_identifier: String, + pub exclusions: Vec, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct AcpDoctorReport { pub status: String, @@ -75,6 +87,7 @@ pub struct AcpDoctorReport { pub relay_command: Vec, pub lane: Option, pub session_id: Option, + pub conformance: AcpConformanceEvidence, pub checks: Vec, pub warnings: Vec, } @@ -1191,6 +1204,8 @@ pub struct AgentApplyReport { pub merge: Option, pub git_export: Option, pub fast_forwarded: bool, + #[serde(default)] + pub performance: GitHandoffMetricsReport, pub warnings: Vec, pub suggestions: Vec, } @@ -1206,21 +1221,6 @@ pub struct AgentFinishReport { pub suggestions: Vec, } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct AgentSetupReport { - pub provider: String, - pub editor: String, - pub mode: String, - pub command: Vec, - pub snippet: String, - pub detected: bool, - pub supports_acp: bool, - pub supports_mcp: bool, - pub supports_terminal: bool, - pub warnings: Vec, - pub suggestions: Vec, -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct AgentRunReport { pub task: AgentTaskReport, @@ -1255,7 +1255,7 @@ pub struct LaneTurn { } pub const TURN_ENVELOPE_SCHEMA: &str = "trail.turn_envelope"; -pub const TURN_ENVELOPE_VERSION: u16 = 1; +pub const TURN_ENVELOPE_VERSION: u16 = 2; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct TurnEnvelope { @@ -1263,6 +1263,12 @@ pub struct TurnEnvelope { pub version: u16, pub kind: String, pub protocol: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub host: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub adapter: Option, pub provider: Option, pub model: Option, pub session: TurnEnvelopeSession, @@ -1331,13 +1337,38 @@ pub struct TurnEnvelopeReview { } impl TurnEnvelope { - pub fn new_acp_prompt(input: TurnEnvelopeAcpPromptInput) -> Self { + pub fn new_agent_turn(input: TurnEnvelopeInput) -> Self { Self { schema: TURN_ENVELOPE_SCHEMA.to_string(), version: TURN_ENVELOPE_VERSION, + kind: input.kind, + protocol: input.protocol, + host: input.host, + agent: input.agent, + adapter: input.adapter, + provider: input.provider, + model: input.model, + session: input.session, + prompt: input.prompt, + workspace: input.workspace, + usage: TurnEnvelopeUsage::default(), + capture: TurnEnvelopeCapture::default(), + outcome: TurnEnvelopeOutcome::default(), + review: TurnEnvelopeReview::default(), + } + } + + pub fn new_acp_prompt(input: TurnEnvelopeAcpPromptInput) -> Self { + let provider = input.provider.clone(); + Self::new_agent_turn(TurnEnvelopeInput { kind: "acp_prompt".to_string(), protocol: "acp".to_string(), - provider: input.provider, + host: Some("acp-relay".to_string()), + agent: provider.clone(), + adapter: provider + .as_ref() + .map(|provider| format!("trail/{provider}@acp")), + provider, model: input.model, session: TurnEnvelopeSession { trail_session_id: Some(input.trail_session_id), @@ -1358,11 +1389,7 @@ impl TurnEnvelope { base_change: Some(input.base_change), before_change: Some(input.before_change), }, - usage: TurnEnvelopeUsage::default(), - capture: TurnEnvelopeCapture::default(), - outcome: TurnEnvelopeOutcome::default(), - review: TurnEnvelopeReview::default(), - } + }) } pub fn from_metadata_json(metadata: Option<&str>) -> Option { @@ -1374,7 +1401,7 @@ impl TurnEnvelope { && value .get("version") .and_then(serde_json::Value::as_u64) - .is_some_and(|version| version == u64::from(TURN_ENVELOPE_VERSION)) + .is_some_and(|version| matches!(version, 1 | 2)) { serde_json::from_value(value).ok() } else { @@ -1410,6 +1437,20 @@ impl TurnEnvelope { } } +#[derive(Clone, Debug)] +pub struct TurnEnvelopeInput { + pub kind: String, + pub protocol: String, + pub host: Option, + pub agent: Option, + pub adapter: Option, + pub provider: Option, + pub model: Option, + pub session: TurnEnvelopeSession, + pub prompt: TurnEnvelopePrompt, + pub workspace: TurnEnvelopeWorkspace, +} + #[derive(Clone, Debug)] pub struct TurnEnvelopeAcpPromptInput { pub provider: Option, diff --git a/trail/src/model/reports/lane.rs b/trail/src/model/reports/lane.rs index c344879..33119ff 100644 --- a/trail/src/model/reports/lane.rs +++ b/trail/src/model/reports/lane.rs @@ -1,31 +1,40 @@ #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub enum LaneWorkdirMode { + Auto, Virtual, Sparse, - FullCow, - OverlayCow, + NativeCow, + PortableCopy, + FuseCow, NfsCow, + DokanCow, } impl LaneWorkdirMode { pub fn as_str(&self) -> &'static str { match self { + LaneWorkdirMode::Auto => "auto", LaneWorkdirMode::Virtual => "virtual", LaneWorkdirMode::Sparse => "sparse", - LaneWorkdirMode::FullCow => "full-cow", - LaneWorkdirMode::OverlayCow => "overlay-cow", + LaneWorkdirMode::NativeCow => "native-cow", + LaneWorkdirMode::PortableCopy => "portable-copy", + LaneWorkdirMode::FuseCow => "fuse-cow", LaneWorkdirMode::NfsCow => "nfs-cow", + LaneWorkdirMode::DokanCow => "dokan-cow", } } pub fn parse(value: &str) -> Option { match value { + "auto" => Some(LaneWorkdirMode::Auto), "virtual" => Some(LaneWorkdirMode::Virtual), "sparse" => Some(LaneWorkdirMode::Sparse), - "full-cow" | "full_cow" => Some(LaneWorkdirMode::FullCow), - "overlay-cow" | "overlay_cow" => Some(LaneWorkdirMode::OverlayCow), + "native-cow" | "native_cow" => Some(LaneWorkdirMode::NativeCow), + "portable-copy" | "portable_copy" => Some(LaneWorkdirMode::PortableCopy), + "fuse-cow" | "fuse_cow" => Some(LaneWorkdirMode::FuseCow), "nfs-cow" | "nfs_cow" => Some(LaneWorkdirMode::NfsCow), + "dokan-cow" | "dokan_cow" => Some(LaneWorkdirMode::DokanCow), _ => None, } } @@ -34,17 +43,88 @@ impl LaneWorkdirMode { !matches!(self, LaneWorkdirMode::Virtual) } - pub fn cow_backend(&self) -> Option<&'static str> { + pub fn default_backend(&self) -> Option { match self { - LaneWorkdirMode::Virtual => None, - LaneWorkdirMode::Sparse | LaneWorkdirMode::FullCow => Some("filesystem-clone"), - LaneWorkdirMode::OverlayCow => Some("overlay"), - LaneWorkdirMode::NfsCow => Some("nfs-overlay"), + LaneWorkdirMode::Auto | LaneWorkdirMode::PortableCopy => None, + LaneWorkdirMode::Virtual => Some(WorkdirBackend::Virtual), + LaneWorkdirMode::Sparse => None, + LaneWorkdirMode::NativeCow => Some(WorkdirBackend::Clone), + LaneWorkdirMode::FuseCow => Some(WorkdirBackend::Fuse), + LaneWorkdirMode::NfsCow => Some(WorkdirBackend::Nfs), + LaneWorkdirMode::DokanCow => Some(WorkdirBackend::Dokan), } } pub fn is_transparent_cow(&self) -> bool { - matches!(self, LaneWorkdirMode::OverlayCow | LaneWorkdirMode::NfsCow) + matches!( + self, + LaneWorkdirMode::FuseCow | LaneWorkdirMode::NfsCow | LaneWorkdirMode::DokanCow + ) + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum WorkdirBackend { + Clone, + Mixed, + Copy, + Fuse, + Nfs, + Dokan, + Virtual, +} + +impl WorkdirBackend { + pub fn as_str(self) -> &'static str { + match self { + WorkdirBackend::Clone => "clone", + WorkdirBackend::Mixed => "mixed", + WorkdirBackend::Copy => "copy", + WorkdirBackend::Fuse => "fuse", + WorkdirBackend::Nfs => "nfs", + WorkdirBackend::Dokan => "dokan", + WorkdirBackend::Virtual => "virtual", + } + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum MaterializationFallbackReason { + CloneUnsupported, + CrossDevice, + NativeSourceUnavailable, +} + +impl MaterializationFallbackReason { + pub fn as_str(self) -> &'static str { + match self { + MaterializationFallbackReason::CloneUnsupported => "clone-unsupported", + MaterializationFallbackReason::CrossDevice => "cross-device", + MaterializationFallbackReason::NativeSourceUnavailable => "native-source-unavailable", + } + } +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct MaterializationReport { + pub cloned_files: u64, + pub cloned_bytes: u64, + pub copied_files: u64, + pub copied_bytes: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fallback_reason: Option, +} + +impl MaterializationReport { + pub fn backend(&self) -> WorkdirBackend { + match (self.cloned_files > 0, self.copied_files > 0) { + (true, true) => WorkdirBackend::Mixed, + (true, false) => WorkdirBackend::Clone, + (false, true) => WorkdirBackend::Copy, + (false, false) => WorkdirBackend::Clone, + } } } @@ -54,10 +134,13 @@ pub struct LaneSpawnReport { pub ref_name: String, pub base_change: ChangeId, pub workdir: Option, + pub requested_workdir_mode: LaneWorkdirMode, pub workdir_mode: LaneWorkdirMode, - pub cow_backend: Option, + pub workdir_backend: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub materialization: Option, pub sparse_paths: Vec, - pub overlay_available: bool, + pub transparent_cow_available: bool, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -109,6 +192,430 @@ pub struct WorkspaceEnvironmentReport { pub updated_at: i64, } +/// The stable, repository-local identity of an environment graph component. +/// +/// Component identity is deliberately independent from the adapter that currently +/// implements it. This lets Trail upgrade or replace an adapter without changing +/// references to the logical component in status, policy, and dependency edges. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentComponentIdentityReport { + pub component_id: String, + pub kind: String, +} + +/// The versioned identity of the adapter implementation responsible for a component. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentAdapterIdentityReport { + pub namespace: String, + pub name: String, + pub contract_major: u32, + pub implementation_version: String, + pub distribution_digest: Option, +} + +/// One adapter available to the environment host. +/// +/// Catalog entries describe discovery and compatibility only. They never grant +/// an adapter permission to execute commands or mutate a lane. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentAdapterCatalogEntryReport { + pub identity: EnvironmentAdapterIdentityReport, + pub canonical_identity: String, + pub selectors: Vec, + pub kind: String, + pub layer_adapter_name: String, + pub discovery_markers: Vec, + /// External planner protocols supported by the packaged executable. + /// Built-ins and repository recipes use the in-process host contract and + /// therefore report an empty list. + pub protocols: Vec, + pub supported_operating_systems: Vec, + pub supported_architectures: Vec, + pub source: String, + pub publisher: Option, + pub publisher_key_id: Option, + pub trust: String, + pub certification_tier: String, + pub stability: String, + pub description: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentAdapterCatalogReport { + pub contract_major: u32, + pub adapters: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentPluginInstallReport { + pub canonical_identity: String, + pub distribution_digest: String, + pub executable_digest: String, + pub package_path: String, + pub replaced_distribution_digest: Option, + pub publisher: Option, + pub publisher_key_id: Option, + pub trust: String, + pub certification_tier: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentPluginPackageInspectionReport { + pub canonical_identity: String, + pub payload_digest: String, + pub executable_digest: String, + pub distribution_digest: String, + pub signature_present: bool, + pub publisher: Option, + pub publisher_key_id: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentPluginRemoveReport { + pub canonical_identity: String, + pub removed_distribution_digest: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentStaleChangeReport { + pub dimension: String, + pub name: String, + pub change: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentStaleExplanationReport { + pub component_id: String, + pub status: String, + pub expected_key: String, + pub attached_key: Option, + pub complete: bool, + pub provenance_complete: bool, + pub total_changes: u64, + pub offset: u64, + pub next_offset: Option, + pub changes: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentPublisherTrustEntryReport { + pub publisher: String, + pub key_id: String, + pub public_key: String, + pub trusted_at: i64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentPublisherTrustReport { + pub keys: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentPublisherTrustMutationReport { + pub publisher: Option, + pub key_id: String, + pub action: String, +} + +/// Normalized environment state for one logical component in a workspace view. +/// +/// `expected_key`, `attached_key`, and the status fields intentionally mirror +/// [`WorkspaceEnvironmentReport`] so legacy dependency state has a lossless report +/// projection while clients move to component-oriented APIs. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentComponentStateReport { + pub view_id: String, + pub component: EnvironmentComponentIdentityReport, + pub adapter: EnvironmentAdapterIdentityReport, + pub expected_key: String, + pub attached_key: Option, + pub status: String, + pub reason: Option, + pub updated_at: i64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentGenerationOutputReport { + pub name: String, + pub policy: String, + pub storage_identity: String, + pub layer_id: Option, + pub mount_path: String, + pub layer_subpath: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentCacheReport { + pub name: String, + pub namespace_id: String, + pub protocol: String, + pub access: String, + pub authority: String, + pub scope: String, + pub compatibility: std::collections::BTreeMap, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentExternalArtifactReport { + pub name: String, + pub artifact_type: String, + pub provider: String, + pub reference: String, + pub digest: String, + pub platform: String, + pub cleanup_owner: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentRuntimeDeclarationReport { + pub name: String, + pub runtime_type: String, + pub provider: String, + pub artifact_name: String, + pub container_port: u16, + pub protocol: String, + pub health_type: String, + pub health_timeout_ms: u64, + pub restart_policy: String, + pub cleanup_owner: String, + pub volume_target: Option, + #[serde(default)] + pub secrets: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentSecretReferenceReport { + pub name: String, + pub provider: String, + pub reference: String, + pub version: Option, + pub purpose: String, + pub injection: String, + pub target: String, + pub environment: Option, + pub required: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentSecretStatusReport { + #[serde(flatten)] + pub reference: EnvironmentSecretReferenceReport, + pub status: String, + pub reason: Option, + pub resolved_at: Option, + pub updated_at: i64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentRuntimeResourceReport { + #[serde(flatten)] + pub declaration: EnvironmentRuntimeDeclarationReport, + pub image_reference: String, + pub image_digest: String, + pub image_platform: String, + pub allocation_id: String, + pub provider_resource_id: Option, + pub container_name: String, + pub network_name: String, + pub volume_name: Option, + pub host_port: Option, + pub status: String, + pub health_status: String, + pub reason: Option, + pub created_at: i64, + pub updated_at: i64, + pub started_at: Option, + pub stopped_at: Option, + #[serde(default)] + pub secret_statuses: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentGenerationDependencyReport { + pub component_id: String, + pub component_key: String, + #[serde(default = "default_environment_edge_type")] + pub edge_type: String, +} + +fn default_environment_edge_type() -> String { + "build_requires".to_string() +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentGenerationComponentReport { + pub component_id: String, + pub adapter_identity: String, + pub kind: String, + pub component_key: String, + pub layer_id: Option, + pub mount_path: Option, + #[serde(default)] + pub dependencies: Vec, + #[serde(default)] + pub outputs: Vec, + #[serde(default)] + pub caches: Vec, + #[serde(default)] + pub external_artifacts: Vec, + #[serde(default)] + pub runtime_resources: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentGenerationReport { + pub generation_id: String, + pub view_id: String, + pub generation_sequence: u64, + pub source_root: ObjectId, + pub specification_digest: String, + pub predecessor_generation_id: Option, + pub state: String, + pub components: Vec, + pub created_at: i64, + pub activated_at: Option, + pub retired_at: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentDiscoveredComponentReport { + pub component_id: String, + pub component_root: String, + pub kind: String, + pub adapter_identity: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentDiscoveryConflictReport { + pub component_root: String, + pub adapter_identities: Vec, + pub reason: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentDiscoveryReport { + pub source_root: ObjectId, + pub components: Vec, + pub conflicts: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentGraphNodeReport { + pub topological_index: u64, + pub component_id: String, + pub component_root: String, + pub kind: String, + pub adapter_identity: String, + pub component_key: String, + pub dependencies: Vec, + #[serde(default)] + pub caches: Vec, + #[serde(default)] + pub external_artifacts: Vec, + #[serde(default)] + pub runtime_resources: Vec, + pub outputs: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentGraphEdgeReport { + pub source_component_id: String, + pub source_component_key: String, + pub target_component_id: String, + pub edge_type: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentGraphReport { + pub source_root: ObjectId, + pub total_nodes: u64, + pub total_edges: u64, + pub offset: u64, + pub next_offset: Option, + pub nodes: Vec, + pub edges: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentPlanInputReport { + pub source_path: String, + pub staging_path: String, + pub content_hash: String, + pub size_bytes: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentPlanCommandReport { + /// `staging` or `mounted_initialization`. + #[serde(default = "default_environment_command_phase")] + pub phase: String, + pub program: String, + pub resolved_program: String, + pub executable_identity: String, + pub args: Vec, + pub working_directory: String, + pub environment_names: Vec, +} + +fn default_environment_command_phase() -> String { + "staging".to_string() +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentCapabilityReport { + pub filesystem_read: Vec, + pub filesystem_write: Vec, + pub process: Vec, + pub network: String, + pub shell: String, + pub scripts: String, + pub secrets: String, + pub sandbox: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentPlanOutputReport { + pub name: String, + pub output_path: String, + pub mount_path: String, + pub policy: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentPlanReport { + pub source_root: ObjectId, + pub component_id: String, + pub adapter_identity: String, + pub kind: String, + pub component_key: String, + #[serde(default)] + pub dependencies: Vec, + #[serde(default)] + pub dependency_edges: Vec, + #[serde(default)] + pub caches: Vec, + #[serde(default)] + pub external_artifacts: Vec, + #[serde(default)] + pub runtime_resources: Vec, + pub inputs: Vec, + pub tools: std::collections::BTreeMap, + pub commands: Vec, + pub outputs: Vec, + /// Compatibility projection of the first output. + pub output_path: String, + /// Compatibility projection of the first output. + pub mount_path: String, + pub portability_scope: String, + pub capabilities: EnvironmentCapabilityReport, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct EnvironmentSyncReport { + pub generation: EnvironmentGenerationReport, + pub layers: Vec, +} + #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct WorkspaceCheckpointReport { pub view_id: String, @@ -116,7 +623,12 @@ pub struct WorkspaceCheckpointReport { pub root_id: ObjectId, pub journal_sequence: u64, pub source_paths: Vec, + /// Generated/dependency paths changed in this authenticated view-journal + /// interval. This is not a recursive inventory of the retained upper. pub generated_dirty_paths: u64, + pub generated_path_accounting: String, + #[serde(default)] + pub upper_recovery_walks: u64, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -150,6 +662,7 @@ pub struct WorkspaceExecReport { pub lane_id: String, pub source_root: ObjectId, pub generation: u64, + pub environment_generation: Option, pub backend: String, pub command: Vec, pub exit_code: i32, @@ -218,6 +731,8 @@ pub struct LanePatchReport { pub operation: ChangeId, pub root_id: ObjectId, pub changed_paths: Vec, + #[serde(default)] + pub path_index: PathIndexMetricsReport, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -226,6 +741,44 @@ pub struct LaneRecordReport { pub operation: Option, pub root_id: ObjectId, pub changed_paths: Vec, + #[serde(default)] + pub path_index: PathIndexMetricsReport, + #[serde(default)] + pub upper_recovery_walks: u64, + #[serde(default)] + pub generated_dirty_paths: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct PathIndexMetricsReport { + /// Path resolution used by this operation (`indexed` or `unknown`). + #[serde(default = "default_path_index_mode")] + pub mode: String, + /// Number of unique folded keys looked up in the persisted path index. + #[serde(default)] + pub lookup_count: u64, + /// Number of unbounded traversals that enumerate every persisted root path. + #[serde(default)] + pub full_root_path_load_count: u64, + /// Number of unbounded repository-shaped filesystem validation walks. + /// Explicitly selected sparse materializations are bounded and excluded. + #[serde(default)] + pub full_filesystem_path_scan_count: u64, +} + +impl Default for PathIndexMetricsReport { + fn default() -> Self { + Self { + mode: default_path_index_mode(), + lookup_count: 0, + full_root_path_load_count: 0, + full_filesystem_path_scan_count: 0, + } + } +} + +fn default_path_index_mode() -> String { + "unknown".to_string() } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -318,10 +871,13 @@ pub struct LaneRewindReport { pub struct LaneWorkdirReport { pub lane_id: String, pub workdir: Option, + pub requested_workdir_mode: LaneWorkdirMode, pub workdir_mode: LaneWorkdirMode, - pub cow_backend: Option, + pub workdir_backend: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub materialization: Option, pub sparse_paths: Vec, - pub overlay_available: bool, + pub transparent_cow_available: bool, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -330,6 +886,11 @@ pub struct LaneWorkdirSyncReport { pub workdir: String, pub head_change: ChangeId, pub root_id: ObjectId, + pub requested_workdir_mode: LaneWorkdirMode, + pub workdir_mode: LaneWorkdirMode, + pub workdir_backend: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub materialization: Option, pub forced: bool, #[serde(default, skip_serializing_if = "Option::is_none")] pub rescue_workdir: Option, @@ -437,3 +998,100 @@ pub struct LaneGateOptions { #[serde(default, skip_serializing_if = "Option::is_none")] pub threshold: Option, } + +#[cfg(test)] +mod workdir_mode_tests { + use super::*; + + #[test] + fn cow_modes_use_backend_specific_names_and_reject_removed_aliases() { + assert_eq!( + LaneWorkdirMode::parse("native-cow"), + Some(LaneWorkdirMode::NativeCow) + ); + assert_eq!( + LaneWorkdirMode::parse("native_cow"), + Some(LaneWorkdirMode::NativeCow) + ); + assert_eq!( + LaneWorkdirMode::parse("fuse-cow"), + Some(LaneWorkdirMode::FuseCow) + ); + assert_eq!( + LaneWorkdirMode::parse("fuse_cow"), + Some(LaneWorkdirMode::FuseCow) + ); + assert_eq!( + LaneWorkdirMode::parse("dokan-cow"), + Some(LaneWorkdirMode::DokanCow) + ); + assert_eq!( + LaneWorkdirMode::parse("dokan_cow"), + Some(LaneWorkdirMode::DokanCow) + ); + assert_eq!(LaneWorkdirMode::parse("overlay-cow"), None); + assert_eq!(LaneWorkdirMode::parse("overlay_cow"), None); + assert_eq!(LaneWorkdirMode::parse("full-cow"), None); + assert_eq!(LaneWorkdirMode::parse("full_cow"), None); + assert_eq!(LaneWorkdirMode::parse("auto"), Some(LaneWorkdirMode::Auto)); + assert_eq!( + LaneWorkdirMode::parse("portable-copy"), + Some(LaneWorkdirMode::PortableCopy) + ); + assert_eq!( + LaneWorkdirMode::parse("portable_copy"), + Some(LaneWorkdirMode::PortableCopy) + ); + assert_eq!( + LaneWorkdirMode::NativeCow.default_backend(), + Some(WorkdirBackend::Clone) + ); + assert_eq!(LaneWorkdirMode::Auto.default_backend(), None); + assert_eq!(LaneWorkdirMode::PortableCopy.default_backend(), None); + assert_eq!( + LaneWorkdirMode::FuseCow.default_backend(), + Some(WorkdirBackend::Fuse) + ); + assert_eq!( + LaneWorkdirMode::NfsCow.default_backend(), + Some(WorkdirBackend::Nfs) + ); + assert_eq!( + LaneWorkdirMode::DokanCow.default_backend(), + Some(WorkdirBackend::Dokan) + ); + } + + #[test] + fn materialization_report_derives_actual_backend() { + let mut report = MaterializationReport::default(); + assert_eq!(report.backend(), WorkdirBackend::Clone); + report.copied_files = 1; + assert_eq!(report.backend(), WorkdirBackend::Copy); + report.cloned_files = 1; + assert_eq!(report.backend(), WorkdirBackend::Mixed); + report.copied_files = 0; + assert_eq!(report.backend(), WorkdirBackend::Clone); + } + + #[test] + fn legacy_patch_and_record_reports_default_path_index_metrics() { + let patch: LanePatchReport = serde_json::from_value(serde_json::json!({ + "lane_id": "lane-1", + "operation": "change-1", + "root_id": "root-1", + "changed_paths": [] + })) + .unwrap(); + assert_eq!(patch.path_index, PathIndexMetricsReport::default()); + + let record: LaneRecordReport = serde_json::from_value(serde_json::json!({ + "lane_id": "lane-1", + "operation": null, + "root_id": "root-1", + "changed_paths": [] + })) + .unwrap(); + assert_eq!(record.path_index, PathIndexMetricsReport::default()); + } +} diff --git a/trail/src/model/reports/maintenance.rs b/trail/src/model/reports/maintenance.rs index 9ae5c6b..d1150e7 100644 --- a/trail/src/model/reports/maintenance.rs +++ b/trail/src/model/reports/maintenance.rs @@ -29,9 +29,31 @@ pub struct IndexRebuildReport { pub messages: u64, #[serde(default)] pub rich_text_hydrated: u64, + /// Immutable roots upgraded with the persistent path-invariant index. + #[serde(default)] + pub path_index_repaired_roots: Vec, + /// Mutable branch/lane refs advanced to equivalent indexed roots. + #[serde(default)] + pub path_index_repaired_refs: Vec, pub errors: Vec, } +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct PathIndexRootRepair { + pub old_root: ObjectId, + pub new_root: ObjectId, + pub case_fold_map_root: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct PathIndexRefRepair { + pub name: String, + pub old_change: ChangeId, + pub new_change: ChangeId, + pub old_root: ObjectId, + pub new_root: ObjectId, +} + #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct WorktreeIndexReport { pub files: u64, @@ -39,6 +61,128 @@ pub struct WorktreeIndexReport { pub duration_ms: u64, } +/// Result of a requested full changed-path ledger reconciliation. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct ChangeLedgerReconcileReport { + pub scope_id: String, + pub scope_kind: String, + pub previous_state: String, + pub reason: String, + pub observed_paths: u64, + pub candidates: u64, + pub resulting_epoch: u64, + pub resulting_state: String, + #[serde(skip)] + pub(crate) mode: String, + #[serde(skip)] + pub(crate) observed_files: u64, + #[serde(skip)] + pub(crate) staged_rows: u64, + #[serde(skip)] + pub(crate) observed_candidates: u64, + #[serde(skip)] + pub(crate) candidate_rows: u64, + #[serde(skip)] + pub(crate) hashed_bytes: u64, + #[serde(skip)] + pub(crate) peak_batch_rows: u64, + #[serde(skip)] + pub(crate) peak_buffer_bytes: u64, + #[serde(skip)] + pub(crate) start_sequence: u64, + #[serde(skip)] + pub(crate) end_sequence: u64, + #[serde(skip)] + pub(crate) start_durable_offset: u64, + #[serde(skip)] + pub(crate) end_durable_offset: u64, + #[serde(skip)] + pub(crate) refreshed: bool, + #[serde(skip)] + pub(crate) published: bool, + #[serde(skip)] + pub(crate) trust_state: String, + #[serde(skip)] + pub(crate) retries: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct StructuredRecovery { + pub command: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct StructuredErrorDetails { + pub code: String, + pub status: u16, + pub exit: i32, + pub message: String, + pub scope: Option, + pub state: Option, + pub reason: Option, + pub recovery: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct StructuredErrorEnvelope { + pub error: StructuredErrorDetails, +} + +impl StructuredErrorEnvelope { + pub fn from_error(error: &crate::Error) -> Self { + let status = match error { + crate::Error::RefNotFound(_) + | crate::Error::OperationNotFound(_) + | crate::Error::RootNotFound(_) => 404, + crate::Error::Conflict(_) + | crate::Error::DirtyWorktree + | crate::Error::DirtyWorktreeWithMessage(_) + | crate::Error::PatchRejected(_) + | crate::Error::StaleBranch(_) + | crate::Error::WorkspaceLocked(_) + | crate::Error::SchemaReinitializeRequired { .. } + | crate::Error::ChangeLedgerReconcileRequired { .. } => 409, + crate::Error::InvalidInput(_) + | crate::Error::InvalidPath { .. } + | crate::Error::IgnoredPath(_) + | crate::Error::Json(_) => 400, + _ => 500, + }; + let (scope, state, reason, command) = match error { + crate::Error::ChangeLedgerReconcileRequired { + scope, + state, + reason, + command, + } => ( + Some(scope.clone()), + Some(state.clone()), + Some(reason.clone()), + Some(command.clone()), + ), + crate::Error::SchemaReinitializeRequired { found, guidance } => ( + None, + Some("reinitialize_required".to_string()), + Some(format!("{found}; {guidance}")), + Some("trail init --force".to_string()), + ), + _ => (None, None, None, None), + }; + Self { + error: StructuredErrorDetails { + code: error.code().to_string(), + status, + exit: error.exit_code(), + message: error.to_string(), + scope, + state, + reason, + recovery: command.map(|command| StructuredRecovery { command }), + }, + } + } +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ExternalMutationAuditRecord { pub audit_id: String, @@ -108,3 +252,24 @@ pub struct BackupRestoreReport { pub checked_roots: u64, pub checked_texts: u64, } + +#[cfg(test)] +mod maintenance_tests { + use super::*; + + #[test] + fn legacy_index_rebuild_report_defaults_path_index_repairs() { + let report: IndexRebuildReport = serde_json::from_value(serde_json::json!({ + "operations": 1, + "operation_parents": 0, + "file_history_rows": 0, + "line_history_rows": 0, + "messages": 0, + "errors": [] + })) + .unwrap(); + + assert!(report.path_index_repaired_roots.is_empty()); + assert!(report.path_index_repaired_refs.is_empty()); + } +} diff --git a/trail/src/model/reports/merge.rs b/trail/src/model/reports/merge.rs index 9f28476..16c1093 100644 --- a/trail/src/model/reports/merge.rs +++ b/trail/src/model/reports/merge.rs @@ -12,9 +12,10 @@ pub struct MergeReport { } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct MergeQueueEntry { +pub struct LaneMergeQueueEntry { pub queue_id: String, - pub source_ref: String, + pub lane_id: String, + pub lane: String, pub target_ref: String, pub status: String, pub priority: i64, @@ -23,18 +24,18 @@ pub struct MergeQueueEntry { } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct MergeQueueAddReport { - pub entry: MergeQueueEntry, +pub struct LaneMergeQueueAddReport { + pub entry: LaneMergeQueueEntry, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct MergeQueueRemoveReport { - pub entry: MergeQueueEntry, +pub struct LaneMergeQueueRemoveReport { + pub entry: LaneMergeQueueEntry, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct MergeQueueExplainReport { - pub entry: MergeQueueEntry, +pub struct LaneMergeQueueExplainReport { + pub entry: LaneMergeQueueEntry, pub readiness: Option, pub dry_run: Option, pub blockers: Vec, @@ -44,16 +45,17 @@ pub struct MergeQueueExplainReport { } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct MergeQueueRunReport { - pub processed: Vec, +pub struct LaneMergeQueueRunReport { + pub processed: Vec, pub stopped_on_conflict: bool, pub stopped_on_failure: bool, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct MergeQueueRunItem { +pub struct LaneMergeQueueRunItem { pub queue_id: String, - pub source_ref: String, + pub lane_id: String, + pub lane: String, pub target_ref: String, pub status: String, pub operation: Option, @@ -85,7 +87,7 @@ pub struct ConflictExplanation { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ConflictMergeContext { pub merge_id: String, - pub queue_id: Option, + pub lane_queue_id: Option, pub source_ref: String, pub target_ref: String, pub base_change: ChangeId, diff --git a/trail/src/model/reports/worktree.rs b/trail/src/model/reports/worktree.rs index e3da5a7..0af6967 100644 --- a/trail/src/model/reports/worktree.rs +++ b/trail/src/model/reports/worktree.rs @@ -50,6 +50,19 @@ pub struct GitExportReport { pub commit: String, pub parent: Option, pub mapping: Option, + #[serde(default)] + pub performance: GitHandoffMetricsReport, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct GitHandoffMetricsReport { + pub export_mode: String, + pub changed_path_count: u64, + pub blob_write_count: u64, + #[serde(default)] + pub git_plumbing_command_count: u64, + pub tracked_status_count: u64, + pub full_root_file_count: u64, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -109,3 +122,22 @@ pub struct CheckoutReport { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub changed_paths: Vec, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn git_handoff_metrics_defaults_new_plumbing_counter_for_legacy_json() { + let report: GitHandoffMetricsReport = serde_json::from_value(serde_json::json!({ + "export_mode": "mapped_delta", + "changed_path_count": 2, + "blob_write_count": 2, + "tracked_status_count": 1, + "full_root_file_count": 0 + })) + .unwrap(); + + assert_eq!(report.git_plumbing_command_count, 0); + } +} diff --git a/trail/src/server.rs b/trail/src/server.rs index 6674a84..f9d20bf 100644 --- a/trail/src/server.rs +++ b/trail/src/server.rs @@ -7,5 +7,232 @@ pub use openapi::openapi_spec; pub use transport::{ handle_http_request, handle_http_request_with_auth, serve_listener, serve_listener_with_auth, serve_listener_with_auth_and_rate_limit, serve_listener_with_auth_rate_limit_and_timeout, - HttpResponse, ServerAuth, ServerRateLimit, + DaemonServerIdentity, HttpResponse, ServerAuth, ServerRateLimit, }; + +#[cfg(unix)] +pub use transport::serve_unix_listener_with_auth_and_timeout; + +#[derive(Clone, Debug, serde::Serialize)] +pub struct WorkspaceLedgerProof { + pub scope_id: String, + pub epoch: u64, + pub daemon_launch_nonce: String, + pub sequence: u64, + pub durable_offset: u64, + pub folded_offset: u64, +} + +fn public_workspace_proof( + proof: crate::db::WorkspaceDaemonProof, +) -> crate::Result { + Ok(WorkspaceLedgerProof { + scope_id: proof.scope_id, + epoch: proof.epoch, + daemon_launch_nonce: proof.daemon_launch_nonce.ok_or_else(|| { + crate::Error::DaemonUnavailable( + "workspace daemon proof is missing its persisted launch binding".into(), + ) + })?, + sequence: proof.cut.sequence, + durable_offset: proof.cut.durable_offset, + folded_offset: proof.cut.folded_offset, + }) +} + +#[derive(serde::Deserialize)] +struct StaleDaemonPublicationEvidence { + stale_pid: u32, + process_start_identity: String, + daemon_launch_nonce: String, +} + +#[doc(hidden)] +pub fn prepare_workspace_changed_path_daemon( + db: &mut crate::Trail, +) -> crate::Result { + let daemon_launch_nonce = + std::env::var("TRAIL_WORKSPACE_DAEMON_LAUNCH_NONCE").map_err(|_| { + crate::Error::DaemonUnavailable("workspace daemon launch nonce is absent".into()) + })?; + let pid = std::process::id(); + let process_start_identity = workspace_daemon_process_start_identity(pid).ok_or_else(|| { + crate::Error::DaemonUnavailable( + "workspace daemon process start identity is unavailable".into(), + ) + })?; + if daemon_launch_nonce.len() != 64 { + return Err(crate::Error::DaemonUnavailable( + "workspace daemon launch nonce is malformed".into(), + )); + } + let stale = std::env::var("TRAIL_WORKSPACE_DAEMON_VERIFIED_STALE_OWNER") + .ok() + .map(|value| serde_json::from_str::(&value)) + .transpose() + .map_err(|_| { + crate::Error::DaemonUnavailable( + "workspace daemon stale publication evidence is malformed".into(), + ) + })?; + let verified_stale_owner = match stale { + Some(stale) => verify_stale_workspace_owner_publication(db, stale)?, + None => None, + }; + crate::db::prepare_workspace_daemon_launch( + db, + crate::db::WorkspaceDaemonLaunchIdentity { + nonce: daemon_launch_nonce, + pid, + process_start_identity, + }, + verified_stale_owner, + ) + .and_then(public_workspace_proof) +} + +fn verify_stale_workspace_owner_publication( + db: &crate::Trail, + stale: StaleDaemonPublicationEvidence, +) -> crate::Result> { + let stale_pid = stale.stale_pid; + let process_start_identity = stale.process_start_identity; + if stale_pid == 0 + || stale_pid > i32::MAX as u32 + || process_start_identity.is_empty() + || stale.daemon_launch_nonce.len() != 64 + { + return Err(crate::Error::DaemonUnavailable( + "verified stale workspace owner capability is malformed".into(), + )); + } + if workspace_daemon_process_is_alive(stale_pid) { + match workspace_daemon_process_start_identity(stale_pid) { + Some(actual) if actual == process_start_identity => { + return Err(crate::Error::DaemonUnavailable( + "verified stale workspace daemon process is still live; refusing owner replacement" + .into(), + )); + } + Some(_) => {} + None => { + return Err(crate::Error::DaemonUnavailable( + "workspace daemon PID is live but its start identity cannot be verified; refusing owner replacement" + .into(), + )); + } + } + } + crate::db::verified_stale_workspace_owner_for_launch( + db, + stale_pid, + &process_start_identity, + &stale.daemon_launch_nonce, + ) +} + +fn workspace_daemon_process_is_alive(pid: u32) -> bool { + let result = unsafe { libc::kill(pid as i32, 0) }; + result == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) +} + +fn workspace_daemon_process_start_identity(pid: u32) -> Option { + #[cfg(target_os = "linux")] + { + let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + let end = stat.rfind(')')?; + return stat + .get(end + 2..)? + .split_whitespace() + .nth(19) + .map(|value| format!("linux:{value}")); + } + #[cfg(target_os = "macos")] + { + let mut info = unsafe { std::mem::zeroed::() }; + let expected = std::mem::size_of::() as i32; + let read = unsafe { + libc::proc_pidinfo( + pid as i32, + libc::PROC_PIDTBSDINFO, + 0, + (&mut info as *mut libc::proc_bsdinfo).cast(), + expected, + ) + }; + if read != expected || info.pbi_pid != pid { + return None; + } + return Some(format!( + "macos:{}:{}:{}", + info.pbi_pid, info.pbi_start_tvsec, info.pbi_start_tvusec + )); + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = pid; + None + } +} + +pub(crate) fn workspace_changed_path_fence( + db: &mut crate::Trail, + scope_id: Option<&str>, + epoch: Option, +) -> crate::Result { + crate::db::workspace_daemon_fence(db, scope_id, epoch).and_then(public_workspace_proof) +} + +pub(crate) fn workspace_changed_path_reconcile( + db: &mut crate::Trail, + scope_id: Option<&str>, + epoch: Option, +) -> crate::Result { + crate::db::workspace_daemon_reconcile(db, scope_id, epoch).and_then(public_workspace_proof) +} + +pub(crate) fn workspace_changed_path_ready_proof( + db: &crate::Trail, +) -> crate::Result { + crate::db::workspace_daemon_ready_proof(db).and_then(public_workspace_proof) +} + +/// Start the scope observer when necessary and perform one full filesystem +/// reconciliation for either the workspace or a materialized lane. +pub fn reconcile_changed_path_ledger( + db: &mut crate::Trail, + lane: Option<&str>, +) -> crate::Result { + let result = match lane { + Some(lane) => crate::db::materialized_lane_daemon_full_reconcile(db, lane), + None => crate::db::workspace_daemon_full_reconcile(db), + }; + result.map_err(|error| match error { + crate::Error::ChangeLedgerReconcileRequired { + scope, + state, + reason, + .. + } => crate::Error::ChangeLedgerReconcileRequired { + scope, + state, + reason, + command: lane.map_or_else( + || "trail index reconcile".to_string(), + |lane| format!("trail index reconcile --lane {}", shell_quote(lane)), + ), + }, + error => error, + }) +} + +fn shell_quote(value: &str) -> String { + if value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.' | b'/')) + { + value.to_string() + } else { + format!("'{}'", value.replace('\'', "'\"'\"'")) + } +} diff --git a/trail/src/server/openapi.rs b/trail/src/server/openapi.rs index 17763f4..3bff1d4 100644 --- a/trail/src/server/openapi.rs +++ b/trail/src/server/openapi.rs @@ -53,3 +53,80 @@ pub fn openapi_spec() -> Value { } }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn patch_and_record_reports_document_path_index_metrics() { + let spec = openapi_spec(); + let schemas = &spec["components"]["schemas"]; + assert_eq!( + schemas["LanePatchReport"]["properties"]["path_index"]["$ref"], + "#/components/schemas/PathIndexMetricsReport" + ); + assert_eq!( + schemas["LaneRecordReport"]["properties"]["path_index"]["$ref"], + "#/components/schemas/PathIndexMetricsReport" + ); + assert_eq!( + schemas["PathIndexMetricsReport"]["required"], + serde_json::json!([ + "mode", + "lookup_count", + "full_root_path_load_count", + "full_filesystem_path_scan_count" + ]) + ); + assert_eq!( + spec["paths"]["/v1/lanes/{lane_or_id}/patches"]["post"]["responses"]["200"]["content"] + ["application/json"]["schema"]["$ref"], + "#/components/schemas/LanePatchReport" + ); + assert_eq!( + spec["paths"]["/v1/lane/turns/{turn_id}/patches"]["post"]["responses"]["200"] + ["content"]["application/json"]["schema"]["$ref"], + "#/components/schemas/LanePatchReport" + ); + } + + #[test] + fn changed_path_reconcile_contract_has_resolving_refs() { + let spec = openapi_spec(); + assert_eq!( + spec["paths"]["/v1/index/reconcile"]["post"]["requestBody"]["content"] + ["application/json"]["schema"]["$ref"], + "#/components/schemas/IndexReconcileRequest" + ); + assert_eq!( + spec["paths"]["/v1/index/reconcile"]["post"]["responses"]["200"]["content"] + ["application/json"]["schema"]["$ref"], + "#/components/schemas/ChangeLedgerReconcileReport" + ); + + fn check_refs(root: &Value, value: &Value) { + match value { + Value::Object(object) => { + if let Some(reference) = object.get("$ref").and_then(Value::as_str) { + let pointer = reference.strip_prefix('#').expect("local OpenAPI ref"); + assert!( + root.pointer(pointer).is_some(), + "unresolved OpenAPI reference {reference}" + ); + } + for child in object.values() { + check_refs(root, child); + } + } + Value::Array(values) => { + for child in values { + check_refs(root, child); + } + } + _ => {} + } + } + check_refs(&spec, &spec); + } +} diff --git a/trail/src/server/openapi/paths.rs b/trail/src/server/openapi/paths.rs index c07858b..2c8f6c9 100644 --- a/trail/src/server/openapi/paths.rs +++ b/trail/src/server/openapi/paths.rs @@ -1,5 +1,6 @@ use serde_json::{Map, Value}; +mod agent_hooks; mod collaboration; mod core; mod lanes; @@ -13,6 +14,7 @@ use super::helpers::{ pub(super) fn openapi_paths() -> Value { let mut paths = Map::new(); append_paths(&mut paths, core::core_paths()); + append_paths(&mut paths, agent_hooks::agent_hook_paths()); append_paths(&mut paths, collaboration::collaboration_paths()); append_paths(&mut paths, lanes::lane_paths()); append_paths(&mut paths, turns::turn_paths()); diff --git a/trail/src/server/openapi/paths/agent_hooks.rs b/trail/src/server/openapi/paths/agent_hooks.rs new file mode 100644 index 0000000..a7baece --- /dev/null +++ b/trail/src/server/openapi/paths/agent_hooks.rs @@ -0,0 +1,111 @@ +use serde_json::{json, Value}; + +use super::{openapi_operation, openapi_path_param, openapi_query}; + +pub(super) fn agent_hook_paths() -> Value { + json!({ + "/v1/agent-integrations/capabilities": { + "get": openapi_operation("agentIntegrationCapabilities", "Agent integration capabilities", "List the built-in native hook and ACP capability contracts for every supported provider.", vec![], None, true) + }, + "/v1/agent-hooks/{provider}/{event}": { + "post": openapi_operation("agentHookIngest", "Ingest native agent hook", "Durably journal one arbitrary provider payload before asynchronous semantic replay. This endpoint additionally requires the daemon to have token authentication configured.", vec![ + openapi_path_param("provider", "string"), + openapi_path_param("event", "string"), + openapi_query("installation", "string"), + openapi_query("dedupe_key", "string") + ], Some("AgentHookProviderPayload"), true) + }, + "/v1/agent-hooks/installations": { + "get": openapi_operation("agentHookInstallations", "Agent hook installations", "List Trail-owned native hook installations and their persisted ownership metadata.", vec![openapi_query("provider", "string")], None, true) + }, + "/v1/agent-hooks/installations/{id}": { + "get": openapi_operation("agentHookInstallation", "Agent hook installation", "Show one native hook installation record.", vec![openapi_path_param("id", "string")], None, true) + }, + "/v1/agent-hooks/receipts": { + "get": openapi_operation("agentHookReceipts", "Agent hook receipts", "List durable, redacted receipt journal rows for diagnostics and recovery.", vec![ + openapi_query("provider", "string"), + openapi_query("status", "string"), + openapi_query("offset", "integer"), + openapi_query("limit", "integer") + ], None, true) + }, + "/v1/agent-hooks/receipts/{id}": { + "get": openapi_operation("agentHookReceipt", "Agent hook receipt", "Show one durable redacted receipt journal row.", vec![openapi_path_param("id", "string")], None, true) + }, + "/v1/agent-hooks/receipts/{id}/replay": { + "post": openapi_operation("agentHookReceiptReplay", "Replay agent hook receipt", "Replay one durable receipt through the provider parser and shared lifecycle coordinator.", vec![openapi_path_param("id", "string")], None, true) + }, + "/v1/agent-hooks/receipts/{id}/retry": { + "post": openapi_operation("agentHookReceiptRetry", "Retry agent hook receipt", "Move one retrying or quarantined receipt back to the replay queue.", vec![openapi_path_param("id", "string")], None, true) + }, + "/v1/agent-hooks/receipts/{id}/discard": { + "post": openapi_operation("agentHookReceiptDiscard", "Discard agent hook receipt", "Explicitly discard one received, retrying, or quarantined receipt while retaining its audit row.", vec![openapi_path_param("id", "string")], None, true) + }, + "/v1/agent-capture-runs": { + "get": openapi_operation("agentCaptureRuns", "Agent capture runs", "List active or historical managed capture runs.", vec![openapi_query("active_only", "boolean"), openapi_query("offset", "integer"), openapi_query("limit", "integer")], None, true), + "post": openapi_operation("agentCaptureRunBegin", "Begin agent capture run", "Declare a leased managed run used to correlate nested native or ACP sessions by owner, executor, and canonical workdir.", vec![], Some("AgentCaptureRunRequest"), true) + }, + "/v1/agent-capture-runs/{id}/renew": { + "post": openapi_operation("agentCaptureRunRenew", "Renew agent capture run", "Renew a managed capture run using its exact owner identity.", vec![openapi_path_param("id", "string")], Some("AgentCaptureRunLeaseRequest"), true) + }, + "/v1/agent-capture-runs/{id}": { + "get": openapi_operation("agentCaptureRun", "Agent capture run", "Show one managed capture-run lease and ownership record.", vec![openapi_path_param("id", "string")], None, true) + }, + "/v1/agent-capture-runs/reconcile": { + "post": openapi_operation("agentCaptureRunReconcile", "Reconcile expired capture runs", "Expire abandoned managed-run leases and close their open turns and sessions as interrupted.", vec![], None, true) + }, + "/v1/agent-capture-runs/{id}/end": { + "post": openapi_operation("agentCaptureRunEnd", "End agent capture run", "Idempotently end a managed capture run using its exact owner identity.", vec![openapi_path_param("id", "string")], Some("AgentCaptureRunLeaseRequest"), true) + }, + "/v1/agent-sessions/{id}/artifacts": { + "get": openapi_operation("agentSessionArtifacts", "Agent session artifacts", "List immutable transcript, export, and evidence artifacts for one session.", vec![openapi_path_param("id", "string"), openapi_query("turn", "string"), openapi_query("offset", "integer"), openapi_query("limit", "integer")], None, true) + }, + "/v1/agent-artifacts/{id}": { + "get": openapi_operation("agentArtifact", "Agent artifact", "Show immutable artifact metadata without returning its potentially sensitive attachment bytes.", vec![openapi_path_param("id", "string")], None, true) + }, + "/v1/agent-artifacts/{id}/redact": { + "post": openapi_operation("agentArtifactRedact", "Redact agent artifact attachment", "Remove attachment access while preserving immutable digest, manifest, and attestation identity.", vec![openapi_path_param("id", "string")], Some("AgentArtifactRedactRequest"), true) + }, + "/v1/agent-turns/{id}/evidence": { + "get": openapi_operation("agentTurnEvidence", "Agent turn evidence manifest", "Show and verify the deterministic immutable evidence manifest for one completed turn.", vec![openapi_path_param("id", "string")], None, true) + }, + "/v1/agent-sessions/{id}/provenance": { + "get": openapi_operation("agentSessionProvenance", "Agent session provenance", "Return the queryable causal provenance nodes and edges for one session.", vec![openapi_path_param("id", "string"), openapi_query("offset", "integer"), openapi_query("limit", "integer")], None, true) + }, + "/v1/agent-sessions/{id}/attestations": { + "get": openapi_operation("agentSessionAttestations", "Agent session attestations", "List immutable chained attestation segments for one session.", vec![openapi_path_param("id", "string"), openapi_query("offset", "integer"), openapi_query("limit", "integer")], None, true), + "post": openapi_operation("agentSessionAttestationCreate", "Create agent session attestation", "Create the next idempotent attestation segment over completed turns not covered by the predecessor.", vec![openapi_path_param("id", "string")], Some("AgentAttestationCreateRequest"), true) + }, + "/v1/agent-attestations/{id}": { + "get": openapi_operation("agentAttestation", "Agent attestation", "Show one immutable session attestation and its exact turn coverage.", vec![openapi_path_param("id", "string")], None, true) + }, + "/v1/agent-attestations/{id}/verify": { + "post": openapi_operation("agentAttestationVerify", "Verify agent attestation", "Verify statement, evidence, predecessor chain, signature, and revocation status.", vec![openapi_path_param("id", "string")], None, true) + }, + "/v1/agent-sessions/{id}/export": { + "get": openapi_operation("agentSessionExport", "Export portable agent trace", "Project one session into the verified vendor-neutral agent-trace representation.", vec![openapi_path_param("id", "string"), openapi_query("format", "string"), openapi_query("attachments", "boolean")], None, true) + }, + "/v1/agent-learnings": { + "get": openapi_operation("agentLearnings", "Agent learnings", "List reviewable reusable findings without injecting them into provider files.", vec![openapi_query("session", "string"), openapi_query("status", "string"), openapi_query("offset", "integer"), openapi_query("limit", "integer")], None, true), + "post": openapi_operation("agentLearningPropose", "Propose agent learning", "Create a redacted, evidence-linked learning proposal requiring explicit review before context use.", vec![], Some("AgentLearningRequest"), true) + }, + "/v1/agent-learnings/{id}": { + "get": openapi_operation("agentLearning", "Agent learning", "Show one reviewable learning record.", vec![openapi_path_param("id", "string")], None, true) + }, + "/v1/agent-learnings/{id}/accept": { + "post": openapi_operation("agentLearningAccept", "Accept agent learning", "Accept one proposed learning for explicit, bounded context use.", vec![openapi_path_param("id", "string")], Some("AgentLearningReviewRequest"), true) + }, + "/v1/agent-learnings/{id}/reject": { + "post": openapi_operation("agentLearningReject", "Reject agent learning", "Reject one proposed learning without deleting its audit record.", vec![openapi_path_param("id", "string")], Some("AgentLearningReviewRequest"), true) + }, + "/v1/agent-sessions/{id}/git-links": { + "get": openapi_operation("agentSessionGitLinks", "Agent session Git links", "List explicit mappings between a session's exact Trail changes and Git commits.", vec![openapi_path_param("id", "string"), openapi_query("offset", "integer"), openapi_query("limit", "integer")], None, true) + }, + "/v1/agent-git-links": { + "post": openapi_operation("agentGitLinkCreate", "Create agent Git link", "Create an explicit exact association between a Git commit, Trail session, turn, and change boundary.", vec![], Some("AgentGitLinkRequest"), true) + }, + "/v1/agent-git-links/{id}": { + "get": openapi_operation("agentGitLink", "Agent Git link", "Show one exact Git-to-Trail association.", vec![openapi_path_param("id", "string")], None, true) + } + }) +} diff --git a/trail/src/server/openapi/paths/collaboration.rs b/trail/src/server/openapi/paths/collaboration.rs index 6ef29c7..4649537 100644 --- a/trail/src/server/openapi/paths/collaboration.rs +++ b/trail/src/server/openapi/paths/collaboration.rs @@ -79,25 +79,20 @@ pub(super) fn collaboration_paths() -> Value { openapi_path_param("anchor_id", "string") ], None, true) }, - "/v1/merge-queue": { - "get": openapi_operation("mergeQueueList", "List merge queue", "List merge queue entries.", vec![], None, true), - "post": openapi_operation("mergeQueueAdd", "Queue merge", "Queue a lane or branch for serialized merge.", vec![], Some("MergeQueueAddRequest"), true) + "/v1/lanes/merges/queue": { + "get": openapi_operation("laneMergeQueueList", "List lane merge queue", "List lane merge queue entries.", vec![], None, true), + "post": openapi_operation("laneMergeQueueAdd", "Queue lane merge", "Queue a lane for serialized merge.", vec![], Some("LaneMergeQueueAddRequest"), true) }, - "/v1/merge-queue/run": { - "post": openapi_operation("mergeQueueRun", "Run merge queue", "Run queued merges serially.", vec![], Some("MergeQueueRunRequest"), true) + "/v1/lanes/merges/queue/run": { + "post": openapi_operation("laneMergeQueueRun", "Run lane merge queue", "Run queued lane merges serially.", vec![], Some("LaneMergeQueueRunRequest"), true) }, - "/v1/merge-queue/explain": { - "get": openapi_operation("mergeQueueExplainByQuery", "Explain merge queue entry", "Explain why a queued merge is ready or blocked.", vec![ - openapi_query("selector", "string") - ], None, true) - }, - "/v1/merge-queue/{selector}": { - "delete": openapi_operation("mergeQueueRemove", "Remove queue entry", "Cancel a queued or conflicted merge queue entry.", vec![ + "/v1/lanes/merges/queue/{selector}": { + "delete": openapi_operation("laneMergeQueueRemove", "Remove lane queue entry", "Cancel a queued or conflicted lane merge queue entry.", vec![ openapi_path_param("selector", "string") ], None, true) }, - "/v1/merge-queue/{selector}/explain": { - "get": openapi_operation("mergeQueueExplain", "Explain merge queue entry", "Explain why a queued merge is ready or blocked.", vec![ + "/v1/lanes/merges/queue/{selector}/explain": { + "get": openapi_operation("laneMergeQueueExplain", "Explain lane merge queue entry", "Explain why a queued lane merge is ready or blocked.", vec![ openapi_path_param("selector", "string") ], None, true) }, @@ -115,10 +110,10 @@ pub(super) fn collaboration_paths() -> Value { openapi_path_param("conflict_set_id", "string") ], Some("ConflictResolveRequest"), true) }, - "/v1/branches/{branch}/merge-lane": { - "post": openapi_operation("branchMergeLane", "Merge lane", "Dry-run a lane merge or explicitly direct-merge a lane branch into a target branch.", vec![ - openapi_path_param("branch", "string") - ], Some("MergeLaneRequest"), true) + "/v1/lanes/{lane}/merge": { + "post": openapi_operation("laneMerge", "Merge lane", "Dry-run a lane merge or explicitly direct-merge this lane into the request target branch.", vec![ + openapi_path_param("lane", "string") + ], Some("LaneMergeRequest"), true) } }) } diff --git a/trail/src/server/openapi/paths/core.rs b/trail/src/server/openapi/paths/core.rs index de26beb..344aec1 100644 --- a/trail/src/server/openapi/paths/core.rs +++ b/trail/src/server/openapi/paths/core.rs @@ -1,8 +1,23 @@ use serde_json::{json, Value}; -use super::{openapi_operation, openapi_path_param, openapi_query, openapi_required_query}; +use super::{ + openapi_operation, openapi_operation_with_response_schema, openapi_path_param, openapi_query, + openapi_required_query, +}; pub(super) fn core_paths() -> Value { + let mut index_reconcile = openapi_operation_with_response_schema( + "indexReconcile", + "Reconcile changed-path ledger", + "Start the scope daemon if necessary and run a complete filesystem reconciliation.", + vec![], + Some("IndexReconcileRequest"), + "ChangeLedgerReconcileReport", + true, + ); + index_reconcile["requestBody"]["required"] = json!(false); + index_reconcile["responses"]["409"] = json!({ "$ref": "#/components/responses/Error" }); + json!({ "/v1/health": { "get": openapi_operation("health", "Health check", "Return service liveness without authentication.", vec![], None, false) @@ -16,6 +31,9 @@ pub(super) fn core_paths() -> Value { "/v1/status": { "get": openapi_operation("status", "Workspace status", "Return current branch status and changed paths.", vec![], None, true) }, + "/v1/index/reconcile": { + "post": index_reconcile + }, "/v1/record": { "post": openapi_operation("record", "Record workspace changes", "Record current workspace changes into a branch.", vec![], Some("RecordRequest"), true) }, diff --git a/trail/src/server/openapi/paths/lanes.rs b/trail/src/server/openapi/paths/lanes.rs index 16166bd..f40532f 100644 --- a/trail/src/server/openapi/paths/lanes.rs +++ b/trail/src/server/openapi/paths/lanes.rs @@ -2,10 +2,14 @@ use serde_json::{json, Value}; use super::{ openapi_operation, openapi_operation_with_response_schema, openapi_path_param, openapi_query, + openapi_required_query, }; pub(super) fn lane_paths() -> Value { json!({ + "/v1/environment/adapters": { + "get": openapi_operation_with_response_schema("environmentAdapterCatalog", "Workspace environment adapters", "List registered adapters and their side-effect-free discovery metadata, provenance, and stability.", vec![], None, "EnvironmentAdapterCatalogReport", true) + }, "/v1/lanes": { "get": openapi_operation("laneList", "List lanes", "List lane branches with metadata and branch state.", vec![], None, true), "post": openapi_operation("laneSpawn", "Spawn lane", "Create or reuse a lane branch.", vec![], Some("SpawnLaneRequest"), true) @@ -96,10 +100,75 @@ pub(super) fn lane_paths() -> Value { ], None, true) }, "/v1/lanes/{lane_or_id}/dependencies/sync": { - "post": openapi_operation("laneDependencySync", "Synchronize dependencies", "Build or reuse a frozen dependency layer and attach it to the lane.", vec![ + "post": openapi_operation("laneDependencySync", "Synchronize dependencies", "Build or reuse a frozen dependency layer, then bulk-replace private dependency state in an unmounted lane.", vec![ openapi_path_param("lane_or_id", "string") ], Some("DependencySyncRequest"), true) }, + "/v1/lanes/{lane_or_id}/environment": { + "get": openapi_operation_with_response_schema("laneEnvironmentStatus", "Workspace environment status", "Return normalized logical component and versioned adapter state for a layered lane.", vec![ + openapi_path_param("lane_or_id", "string") + ], None, "EnvironmentComponentStateReportList", true) + }, + "/v1/lanes/{lane_or_id}/environment/discover": { + "get": openapi_operation_with_response_schema("laneEnvironmentDiscover", "Discover workspace environments", "Detect built-in environment components without executing tools, network providers, or repository code.", vec![ + openapi_path_param("lane_or_id", "string"), + openapi_query("path", "string") + ], None, "EnvironmentDiscoveryReport", true) + }, + "/v1/lanes/{lane_or_id}/environment/graph": { + "get": openapi_operation_with_response_schema("laneEnvironmentGraph", "Desired environment graph", "Return the validated component DAG, deterministic topological order, output ownership, component keys, and ordering/invalidation edges without executing tools or mutating state.", vec![ + openapi_path_param("lane_or_id", "string"), + openapi_query("path", "string"), + openapi_query("offset", "integer"), + openapi_query("limit", "integer") + ], None, "EnvironmentGraphReport", true) + }, + "/v1/lanes/{lane_or_id}/environment/generation": { + "get": openapi_operation_with_response_schema("laneEnvironmentGeneration", "Active environment generation", "Return the exact source root, component keys, layers, mounts, and predecessor atomically active for a lane.", vec![ + openapi_path_param("lane_or_id", "string") + ], None, "EnvironmentGenerationReportNullable", true) + }, + "/v1/lanes/{lane_or_id}/environment/explain": { + "get": openapi_operation_with_response_schema("laneEnvironmentExplain", "Explain environment staleness", "Return every canonical input, tool, platform, and policy edge that differs from the attached component artifact.", vec![ + openapi_path_param("lane_or_id", "string"), + openapi_required_query("component", "string"), + openapi_query("offset", "integer"), + openapi_query("limit", "integer") + ], None, "EnvironmentStaleExplanationReport", true) + }, + "/v1/lanes/{lane_or_id}/environment/plan": { + "get": openapi_operation_with_response_schema("laneEnvironmentPlan", "Plan workspace environment", "Return the normalized key, inputs, argv actions, output, and capability grants without executing or mutating state.", vec![ + openapi_path_param("lane_or_id", "string"), + openapi_query("adapter", "string"), + openapi_query("component", "string"), + openapi_query("path", "string") + ], None, "EnvironmentPlanReport", true) + }, + "/v1/lanes/{lane_or_id}/environment/sync": { + "post": openapi_operation_with_response_schema("laneEnvironmentSync", "Synchronize workspace environment", "Prepare one adapter-owned environment component and atomically activate its shared and/or writable-private outputs for an unmounted lane.", vec![ + openapi_path_param("lane_or_id", "string") + ], Some("EnvironmentSyncRequest"), "EnvironmentSyncReport", true) + }, + "/v1/lanes/{lane_or_id}/environment/sync-all": { + "post": openapi_operation_with_response_schema("laneEnvironmentSyncAll", "Synchronize all workspace environments", "Build all discovered components before atomically activating one complete generation.", vec![ + openapi_path_param("lane_or_id", "string") + ], Some("DependencySyncRequest"), "EnvironmentSyncReport", true) + }, + "/v1/lanes/{lane_or_id}/environment/runtime/status": { + "get": openapi_operation_with_response_schema("laneEnvironmentRuntimeStatus", "Environment runtime status", "Return persisted container, network, volume, port, lifecycle, and health state without contacting the runtime provider.", vec![ + openapi_path_param("lane_or_id", "string") + ], None, "EnvironmentGenerationReportNullable", true) + }, + "/v1/lanes/{lane_or_id}/environment/runtime/reconcile": { + "post": openapi_operation_with_response_schema("laneEnvironmentRuntimeReconcile", "Reconcile environment runtime", "Idempotently create or adopt declared lane-private OCI resources and wait for health.", vec![ + openapi_path_param("lane_or_id", "string") + ], None, "EnvironmentGenerationReport", true) + }, + "/v1/lanes/{lane_or_id}/environment/runtime/stop": { + "post": openapi_operation_with_response_schema("laneEnvironmentRuntimeStop", "Stop environment runtime", "Stop active Trail-owned containers while retaining private networks and volumes for restart.", vec![ + openapi_path_param("lane_or_id", "string") + ], None, "EnvironmentGenerationReport", true) + }, "/v1/lanes/{lane_or_id}/checkpoint": { "post": openapi_operation("laneWorkspaceCheckpoint", "Checkpoint lane workspace", "Checkpoint source-upper mutations into the lane ref under a mutation barrier.", vec![ openapi_path_param("lane_or_id", "string") @@ -154,9 +223,9 @@ pub(super) fn lane_paths() -> Value { ], Some("LaneTestRequest"), true) }, "/v1/lanes/{lane_or_id}/patches": { - "post": openapi_operation("laneApplyPatch", "Apply lane patch", "Apply a patch directly to a lane branch.", vec![ + "post": openapi_operation_with_response_schema("laneApplyPatch", "Apply lane patch", "Apply a patch directly to a lane branch.", vec![ openapi_path_param("lane_or_id", "string") - ], Some("PatchRequest"), true) + ], Some("PatchRequest"), "LanePatchReport", true) } }) } diff --git a/trail/src/server/openapi/paths/turns.rs b/trail/src/server/openapi/paths/turns.rs index e1411bb..46e4454 100644 --- a/trail/src/server/openapi/paths/turns.rs +++ b/trail/src/server/openapi/paths/turns.rs @@ -1,6 +1,8 @@ use serde_json::{json, Value}; -use super::{openapi_operation, openapi_path_param, openapi_query}; +use super::{ + openapi_operation, openapi_operation_with_response_schema, openapi_path_param, openapi_query, +}; pub(super) fn turn_paths() -> Value { json!({ @@ -88,9 +90,9 @@ pub(super) fn turn_paths() -> Value { ], Some("StartSpanRequest"), true) }, "/v1/lane/turns/{turn_id}/patches": { - "post": openapi_operation("turnApplyPatch", "Apply turn patch", "Apply a patch linked to a durable turn.", vec![ + "post": openapi_operation_with_response_schema("turnApplyPatch", "Apply turn patch", "Apply a patch linked to a durable turn.", vec![ openapi_path_param("turn_id", "string") - ], Some("PatchRequest"), true) + ], Some("PatchRequest"), "LanePatchReport", true) }, "/v1/lane/turns/{turn_id}/end": { "post": openapi_operation("turnEnd", "End turn", "End a durable lane turn.", vec![ diff --git a/trail/src/server/openapi/schemas.rs b/trail/src/server/openapi/schemas.rs index 1be3e6d..a5c8d9d 100644 --- a/trail/src/server/openapi/schemas.rs +++ b/trail/src/server/openapi/schemas.rs @@ -1,5 +1,6 @@ use serde_json::{Map, Value}; +mod agent_hooks; mod collaboration; mod core; mod lane; @@ -8,6 +9,7 @@ mod patches; pub(super) fn openapi_schemas() -> Value { let mut schemas = Map::new(); append_schemas(&mut schemas, core::core_schemas()); + append_schemas(&mut schemas, agent_hooks::agent_hook_schemas()); append_schemas(&mut schemas, lane::lane_schemas()); append_schemas(&mut schemas, collaboration::collaboration_schemas()); append_schemas(&mut schemas, patches::patch_schemas()); diff --git a/trail/src/server/openapi/schemas/agent_hooks.rs b/trail/src/server/openapi/schemas/agent_hooks.rs new file mode 100644 index 0000000..2ba14c4 --- /dev/null +++ b/trail/src/server/openapi/schemas/agent_hooks.rs @@ -0,0 +1,80 @@ +use serde_json::{json, Value}; + +pub(super) fn agent_hook_schemas() -> Value { + json!({ + "AgentHookProviderPayload": { + "type": "object", + "description": "Arbitrary provider-native hook payload. Provider contracts determine its fields.", + "additionalProperties": true + }, + "AgentCaptureRunRequest": { + "type": "object", + "required": ["workdir", "owner_agent", "owner_session_id", "lease_ms"], + "properties": { + "lane": {"type": ["string", "null"]}, + "workdir": {"type": "string"}, + "owner_agent": {"type": "string"}, + "owner_session_id": {"type": "string"}, + "executor_agent": {"type": ["string", "null"]}, + "work_item_id": {"type": ["string", "null"]}, + "lease_ms": {"type": "integer", "minimum": 1000}, + "metadata": {"type": ["object", "null"], "additionalProperties": true} + } + }, + "AgentCaptureRunLeaseRequest": { + "type": "object", + "required": ["owner_agent", "owner_session_id"], + "properties": { + "owner_agent": {"type": "string"}, + "owner_session_id": {"type": "string"}, + "lease_ms": {"type": ["integer", "null"], "minimum": 1000} + } + }, + "AgentAttestationCreateRequest": { + "type": "object", + "properties": { + "capture_policy": {"type": "string", "default": "native-agent-hooks/v1"}, + "metadata": {"type": ["object", "null"], "additionalProperties": true} + } + }, + "AgentLearningReviewRequest": { + "type": "object", + "required": ["reviewer"], + "properties": {"reviewer": {"type": "string"}} + }, + "AgentArtifactRedactRequest": { + "type": "object", + "required": ["reason"], + "properties": {"reason": {"type": "string", "maxLength": 512}} + }, + "AgentLearningRequest": { + "type": "object", + "required": ["session_id", "scope", "body"], + "properties": { + "session_id": {"type": "string"}, + "turn_id": {"type": ["string", "null"]}, + "scope": {"type": "string"}, + "body": {"type": "string"}, + "confidence": {"type": ["number", "null"], "minimum": 0, "maximum": 1}, + "source_artifact_id": {"type": ["string", "null"]}, + "anchor": {"type": ["object", "null"], "additionalProperties": true}, + "expires_at": {"type": ["integer", "null"]}, + "metadata": {"type": ["object", "null"], "additionalProperties": true} + } + }, + "AgentGitLinkRequest": { + "type": "object", + "required": ["session_id", "git_commit", "confidence", "source"], + "properties": { + "session_id": {"type": "string"}, + "turn_id": {"type": ["string", "null"]}, + "git_commit": {"type": "string"}, + "from_change": {"type": ["string", "null"]}, + "through_change": {"type": ["string", "null"]}, + "confidence": {"type": "string"}, + "source": {"type": "string"}, + "metadata": {"type": ["object", "null"], "additionalProperties": true} + } + } + }) +} diff --git a/trail/src/server/openapi/schemas/collaboration.rs b/trail/src/server/openapi/schemas/collaboration.rs index d59c3d8..f765a89 100644 --- a/trail/src/server/openapi/schemas/collaboration.rs +++ b/trail/src/server/openapi/schemas/collaboration.rs @@ -2,15 +2,13 @@ use serde_json::{json, Value}; pub(super) fn collaboration_schemas() -> Value { json!({ - "MergeLaneRequest": { + "LaneMergeRequest": { "type": "object", + "required": ["into"], "properties": { - "lane_id": { "type": "string" }, - "lane": { "type": "string" }, - "name": { "type": "string" }, + "into": { "type": "string" }, "strategy": { "type": "string" }, "dry_run": { "type": "boolean" }, - "dry-run": { "type": "boolean" }, "direct": { "type": "boolean" } } }, @@ -80,19 +78,19 @@ pub(super) fn collaboration_schemas() -> Value { "branch": { "type": "string" } } }, - "MergeQueueAddRequest": { + "LaneMergeQueueAddRequest": { "type": "object", - "required": ["source", "target"], + "required": ["lane", "into"], + "additionalProperties": false, "properties": { - "source": { "type": "string" }, - "target": { "type": "string" }, + "lane": { "type": "string" }, "into": { "type": "string" }, - "target_branch": { "type": "string" }, "priority": { "type": "integer" } } }, - "MergeQueueRunRequest": { + "LaneMergeQueueRunRequest": { "type": "object", + "additionalProperties": false, "properties": { "limit": { "type": "integer", "minimum": 1 } } }, "ConflictSetSummary": { @@ -124,7 +122,7 @@ pub(super) fn collaboration_schemas() -> Value { "required": ["merge_id", "source_ref", "target_ref", "base_change", "target_change", "source_change", "base_root", "target_root", "source_root"], "properties": { "merge_id": { "type": "string" }, - "queue_id": { "type": "string" }, + "lane_queue_id": { "type": "string" }, "source_ref": { "type": "string" }, "target_ref": { "type": "string" }, "base_change": { "type": "string" }, diff --git a/trail/src/server/openapi/schemas/core.rs b/trail/src/server/openapi/schemas/core.rs index 7518338..b62522f 100644 --- a/trail/src/server/openapi/schemas/core.rs +++ b/trail/src/server/openapi/schemas/core.rs @@ -38,17 +38,56 @@ pub(super) fn core_schemas() -> Value { "ErrorBody": { "type": "object", "required": ["error"], + "additionalProperties": false, "properties": { "error": { "type": "object", - "required": ["message", "code"], + "required": ["code", "status", "exit", "message", "scope", "state", "reason", "recovery"], + "additionalProperties": false, "properties": { "message": { "type": "string" }, - "code": { "type": "integer" } + "code": { "type": "string" }, + "status": { "type": "integer" }, + "exit": { "type": "integer" }, + "scope": { "type": ["string", "null"] }, + "state": { "type": ["string", "null"] }, + "reason": { "type": ["string", "null"] }, + "recovery": { + "oneOf": [ + { "$ref": "#/components/schemas/StructuredRecovery" }, + { "type": "null" } + ] + } } } } }, + "StructuredRecovery": { + "type": "object", + "required": ["command"], + "additionalProperties": false, + "properties": { "command": { "type": "string" } } + }, + "IndexReconcileRequest": { + "type": "object", + "additionalProperties": false, + "properties": { "lane": { "type": ["string", "null"] } } + }, + "ChangeLedgerReconcileReport": { + "type": "object", + "required": ["scope_id", "scope_kind", "previous_state", "reason", "observed_paths", "candidates", "resulting_epoch", "resulting_state"], + "additionalProperties": false, + "properties": { + "scope_id": { "type": "string" }, + "scope_kind": { "type": "string", "enum": ["workspace", "materialized_lane"] }, + "previous_state": { "type": "string" }, + "reason": { "type": "string" }, + "observed_paths": { "type": "integer", "minimum": 0 }, + "candidates": { "type": "integer", "minimum": 0 }, + "resulting_epoch": { "type": "integer", "minimum": 0 }, + "resulting_state": { "type": "string" } + } + }, "ConfigSetRequest": { "type": "object", "required": ["key", "value"], diff --git a/trail/src/server/openapi/schemas/lane.rs b/trail/src/server/openapi/schemas/lane.rs index 23c0b31..7f091d6 100644 --- a/trail/src/server/openapi/schemas/lane.rs +++ b/trail/src/server/openapi/schemas/lane.rs @@ -103,15 +103,39 @@ pub(super) fn lane_schemas() -> Value { { "$ref": "#/components/schemas/LaneRecordPreviewReport" } ] }, + "PathIndexMetricsReport": { + "type": "object", + "required": ["mode", "lookup_count", "full_root_path_load_count", "full_filesystem_path_scan_count"], + "additionalProperties": false, + "properties": { + "mode": { "type": "string", "enum": ["unknown", "indexed"], "description": "Path resolution used by this operation." }, + "lookup_count": { "type": "integer", "minimum": 0, "description": "Unique folded keys looked up in the persisted path index." }, + "full_root_path_load_count": { "type": "integer", "minimum": 0, "description": "Unbounded traversals that enumerate every path in a persisted root." }, + "full_filesystem_path_scan_count": { "type": "integer", "minimum": 0, "description": "Unbounded repository-shaped filesystem validation walks; explicitly selected sparse materializations are excluded." } + } + }, + "LanePatchReport": { + "type": "object", + "required": ["lane_id", "operation", "root_id", "changed_paths", "path_index"], + "additionalProperties": false, + "properties": { + "lane_id": { "type": "string" }, + "operation": { "type": "string" }, + "root_id": { "type": "string" }, + "changed_paths": { "type": "array", "items": { "$ref": "#/components/schemas/FileDiffSummary" } }, + "path_index": { "$ref": "#/components/schemas/PathIndexMetricsReport" } + } + }, "LaneRecordReport": { "type": "object", - "required": ["lane_id", "operation", "root_id", "changed_paths"], + "required": ["lane_id", "operation", "root_id", "changed_paths", "path_index"], "additionalProperties": false, "properties": { "lane_id": { "type": "string" }, "operation": { "type": ["string", "null"] }, "root_id": { "type": "string" }, - "changed_paths": { "type": "array", "items": { "$ref": "#/components/schemas/FileDiffSummary" } } + "changed_paths": { "type": "array", "items": { "$ref": "#/components/schemas/FileDiffSummary" } }, + "path_index": { "$ref": "#/components/schemas/PathIndexMetricsReport" } } }, "LaneRecordPreviewReport": { @@ -193,7 +217,7 @@ pub(super) fn lane_schemas() -> Value { "from_ref": { "type": "string" }, "branch": { "type": "string" }, "materialize": { "type": "boolean" }, - "workdir_mode": { "type": "string", "enum": ["auto", "virtual", "sparse", "full-cow", "overlay-cow", "nfs-cow"] }, + "workdir_mode": { "type": "string", "enum": ["auto", "virtual", "sparse", "native-cow", "portable-copy", "fuse-cow", "nfs-cow", "dokan-cow"] }, "workdir": { "type": "string" }, "workdir_path": { "type": "string" }, "paths": { "type": "array", "items": { "type": "string" } }, @@ -355,6 +379,494 @@ pub(super) fn lane_schemas() -> Value { "path": { "type": "string" } } }, + "EnvironmentSyncRequest": { + "type": "object", + "properties": { + "adapter": { "type": "string", "default": "auto" }, + "component": { "type": "string" }, + "path": { "type": "string" } + } + }, + "EnvironmentAdapterIdentityReport": { + "type": "object", + "required": ["namespace", "name", "contract_major", "implementation_version"], + "additionalProperties": false, + "properties": { + "namespace": { "type": "string" }, + "name": { "type": "string" }, + "contract_major": { "type": "integer", "minimum": 1 }, + "implementation_version": { "type": "string" }, + "distribution_digest": { "type": ["string", "null"] } + } + }, + "EnvironmentAdapterCatalogEntryReport": { + "type": "object", + "required": ["identity", "canonical_identity", "selectors", "kind", "layer_adapter_name", "discovery_markers", "protocols", "supported_operating_systems", "supported_architectures", "source", "publisher", "publisher_key_id", "trust", "certification_tier", "stability", "description"], + "additionalProperties": false, + "properties": { + "identity": { "$ref": "#/components/schemas/EnvironmentAdapterIdentityReport" }, + "canonical_identity": { "type": "string" }, + "selectors": { "type": "array", "items": { "type": "string" } }, + "kind": { "type": "string" }, + "layer_adapter_name": { "type": "string" }, + "discovery_markers": { "type": "array", "items": { "type": "string" } }, + "protocols": { "type": "array", "items": { "type": "string", "enum": ["trail.environment-adapter/v1", "trail.environment-adapter/v2"] } }, + "supported_operating_systems": { "type": "array", "items": { "type": "string", "enum": ["linux", "macos", "windows"] } }, + "supported_architectures": { "type": "array", "items": { "type": "string", "enum": ["aarch64", "x86_64"] } }, + "source": { "type": "string", "enum": ["builtin", "recipe", "plugin"] }, + "publisher": { "type": ["string", "null"] }, + "publisher_key_id": { "type": ["string", "null"] }, + "trust": { "type": "string", "enum": ["builtin", "local_unsigned", "publisher_signed"] }, + "certification_tier": { "type": "string" }, + "stability": { "type": "string" }, + "description": { "type": "string" } + } + }, + "EnvironmentAdapterCatalogReport": { + "type": "object", + "required": ["contract_major", "adapters"], + "additionalProperties": false, + "properties": { + "contract_major": { "type": "integer", "minimum": 1 }, + "adapters": { + "type": "array", + "items": { "$ref": "#/components/schemas/EnvironmentAdapterCatalogEntryReport" } + } + } + }, + "EnvironmentComponentStateReport": { + "type": "object", + "required": ["view_id", "component", "adapter", "expected_key", "status", "updated_at"], + "properties": { + "view_id": { "type": "string" }, + "component": { + "type": "object", + "required": ["component_id", "kind"], + "properties": { + "component_id": { "type": "string" }, + "kind": { "type": "string" } + } + }, + "adapter": { + "type": "object", + "required": ["namespace", "name", "contract_major", "implementation_version"], + "properties": { + "namespace": { "type": "string" }, + "name": { "type": "string" }, + "contract_major": { "type": "integer", "minimum": 0 }, + "implementation_version": { "type": "string" }, + "distribution_digest": { "type": ["string", "null"] } + } + }, + "expected_key": { "type": "string" }, + "attached_key": { "type": ["string", "null"] }, + "status": { "type": "string", "enum": ["building", "ready", "stale", "failed"] }, + "reason": { "type": ["string", "null"] }, + "updated_at": { "type": "integer" } + } + }, + "EnvironmentComponentStateReportList": { + "type": "array", + "items": { "$ref": "#/components/schemas/EnvironmentComponentStateReport" } + }, + "EnvironmentStaleChangeReport": { + "type": "object", + "required": ["dimension", "name", "change"], + "additionalProperties": false, + "properties": { + "dimension": { "type": "string", "enum": ["input", "tool", "policy", "canonical_key", "provenance", "component", "attachment"] }, + "name": { "type": "string" }, + "change": { "type": "string", "enum": ["added", "removed", "modified", "not_attached", "removed_or_adapter_unavailable", "unavailable_for_legacy_or_missing_layer"] } + } + }, + "EnvironmentStaleExplanationReport": { + "type": "object", + "required": ["component_id", "status", "expected_key", "attached_key", "complete", "provenance_complete", "total_changes", "offset", "next_offset", "changes"], + "additionalProperties": false, + "properties": { + "component_id": { "type": "string" }, + "status": { "type": "string", "enum": ["ready", "stale"] }, + "expected_key": { "type": "string" }, + "attached_key": { "type": ["string", "null"] }, + "complete": { "type": "boolean" }, + "provenance_complete": { "type": "boolean" }, + "total_changes": { "type": "integer", "minimum": 0 }, + "offset": { "type": "integer", "minimum": 0 }, + "next_offset": { "type": ["integer", "null"], "minimum": 0 }, + "changes": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentStaleChangeReport" } } + } + }, + "WorkspaceLayerReport": { + "type": "object", + "required": ["layer_id", "kind", "cache_key", "adapter", "state", "storage_path", "logical_bytes", "entry_count", "portability_scope"], + "properties": { + "layer_id": { "type": "string" }, + "kind": { "type": "string" }, + "cache_key": { "type": "string" }, + "adapter": { "type": "string" }, + "state": { "type": "string" }, + "storage_path": { "type": "string" }, + "logical_bytes": { "type": "integer", "minimum": 0 }, + "physical_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "entry_count": { "type": "integer", "minimum": 0 }, + "portability_scope": { "type": "string" } + } + }, + "EnvironmentDiscoveredComponentReport": { + "type": "object", + "required": ["component_id", "component_root", "kind", "adapter_identity"], + "additionalProperties": false, + "properties": { + "component_id": { "type": "string" }, + "component_root": { "type": "string" }, + "kind": { "type": "string" }, + "adapter_identity": { "type": "string" } + } + }, + "EnvironmentDiscoveryConflictReport": { + "type": "object", + "required": ["component_root", "adapter_identities", "reason"], + "additionalProperties": false, + "properties": { + "component_root": { "type": "string" }, + "adapter_identities": { "type": "array", "items": { "type": "string" } }, + "reason": { "type": "string" } + } + }, + "EnvironmentDiscoveryReport": { + "type": "object", + "required": ["source_root", "components", "conflicts"], + "additionalProperties": false, + "properties": { + "source_root": { "type": "string" }, + "components": { + "type": "array", + "items": { "$ref": "#/components/schemas/EnvironmentDiscoveredComponentReport" } + }, + "conflicts": { + "type": "array", + "items": { "$ref": "#/components/schemas/EnvironmentDiscoveryConflictReport" } + } + } + }, + "EnvironmentGraphNodeReport": { + "type": "object", + "required": ["topological_index", "component_id", "component_root", "kind", "adapter_identity", "component_key", "dependencies", "caches", "external_artifacts", "runtime_resources", "outputs"], + "additionalProperties": false, + "properties": { + "topological_index": { "type": "integer", "minimum": 0 }, + "component_id": { "type": "string" }, + "component_root": { "type": "string" }, + "kind": { "type": "string" }, + "adapter_identity": { "type": "string" }, + "component_key": { "type": "string" }, + "dependencies": { "type": "array", "items": { "type": "string" } }, + "caches": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentCacheReport" } }, + "external_artifacts": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentExternalArtifactReport" } }, + "runtime_resources": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentRuntimeDeclarationReport" } }, + "outputs": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentPlanOutputReport" } } + } + }, + "EnvironmentGraphEdgeReport": { + "type": "object", + "required": ["source_component_id", "source_component_key", "target_component_id", "edge_type"], + "additionalProperties": false, + "properties": { + "source_component_id": { "type": "string" }, + "source_component_key": { "type": "string" }, + "target_component_id": { "type": "string" }, + "edge_type": { "type": "string", "enum": ["build_requires", "runtime_requires", "binds_after", "invalidates_with"] } + } + }, + "EnvironmentGraphReport": { + "type": "object", + "required": ["source_root", "total_nodes", "total_edges", "offset", "next_offset", "nodes", "edges"], + "additionalProperties": false, + "properties": { + "source_root": { "type": "string" }, + "total_nodes": { "type": "integer", "minimum": 0 }, + "total_edges": { "type": "integer", "minimum": 0 }, + "offset": { "type": "integer", "minimum": 0 }, + "next_offset": { "type": ["integer", "null"], "minimum": 0 }, + "nodes": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentGraphNodeReport" } }, + "edges": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentGraphEdgeReport" } } + } + }, + "EnvironmentPlanInputReport": { + "type": "object", + "required": ["source_path", "staging_path", "content_hash", "size_bytes"], + "additionalProperties": false, + "properties": { + "source_path": { "type": "string" }, + "staging_path": { "type": "string" }, + "content_hash": { "type": "string" }, + "size_bytes": { "type": "integer", "minimum": 0 } + } + }, + "EnvironmentPlanCommandReport": { + "type": "object", + "required": ["phase", "program", "resolved_program", "executable_identity", "args", "working_directory", "environment_names"], + "additionalProperties": false, + "properties": { + "phase": { "type": "string", "enum": ["staging", "mounted_initialization"] }, + "program": { "type": "string" }, + "resolved_program": { "type": "string" }, + "executable_identity": { "type": "string" }, + "args": { "type": "array", "items": { "type": "string" } }, + "working_directory": { "type": "string" }, + "environment_names": { "type": "array", "items": { "type": "string" } } + } + }, + "EnvironmentCapabilityReport": { + "type": "object", + "required": ["filesystem_read", "filesystem_write", "process", "network", "shell", "scripts", "secrets", "sandbox"], + "additionalProperties": false, + "properties": { + "filesystem_read": { "type": "array", "items": { "type": "string" } }, + "filesystem_write": { "type": "array", "items": { "type": "string" } }, + "process": { "type": "array", "items": { "type": "string" } }, + "network": { "type": "string" }, + "shell": { "type": "string" }, + "scripts": { "type": "string" }, + "secrets": { "type": "string" }, + "sandbox": { "type": "string" } + } + }, + "EnvironmentPlanReport": { + "type": "object", + "required": ["source_root", "component_id", "adapter_identity", "kind", "component_key", "dependencies", "dependency_edges", "caches", "external_artifacts", "runtime_resources", "inputs", "tools", "commands", "outputs", "output_path", "mount_path", "portability_scope", "capabilities"], + "additionalProperties": false, + "properties": { + "source_root": { "type": "string" }, + "component_id": { "type": "string" }, + "adapter_identity": { "type": "string" }, + "kind": { "type": "string" }, + "component_key": { "type": "string" }, + "dependencies": { "type": "array", "items": { "type": "string" } }, + "dependency_edges": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentGenerationDependencyReport" } }, + "caches": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentCacheReport" } }, + "external_artifacts": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentExternalArtifactReport" } }, + "runtime_resources": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentRuntimeDeclarationReport" } }, + "inputs": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentPlanInputReport" } }, + "tools": { "type": "object", "additionalProperties": { "type": "string" } }, + "commands": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentPlanCommandReport" } }, + "outputs": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentPlanOutputReport" } }, + "output_path": { "type": "string" }, + "mount_path": { "type": "string" }, + "portability_scope": { "type": "string" }, + "capabilities": { "$ref": "#/components/schemas/EnvironmentCapabilityReport" } + } + }, + "EnvironmentPlanOutputReport": { + "type": "object", + "required": ["name", "output_path", "mount_path", "policy"], + "additionalProperties": false, + "properties": { + "name": { "type": "string" }, + "output_path": { "type": "string" }, + "mount_path": { "type": "string" }, + "policy": { "type": "string", "enum": ["immutable_seed_private", "writable_private"] } + } + }, + "EnvironmentGenerationOutputReport": { + "type": "object", + "required": ["name", "policy", "storage_identity", "layer_id", "mount_path", "layer_subpath"], + "additionalProperties": false, + "properties": { + "name": { "type": "string" }, + "policy": { "type": "string", "enum": ["immutable_seed_private", "writable_private"] }, + "storage_identity": { "type": "string" }, + "layer_id": { "type": ["string", "null"] }, + "mount_path": { "type": "string" }, + "layer_subpath": { "type": "string" } + } + }, + "EnvironmentCacheReport": { + "type": "object", + "required": ["name", "namespace_id", "protocol", "access", "authority", "scope", "compatibility"], + "additionalProperties": false, + "properties": { + "name": { "type": "string" }, + "namespace_id": { "type": "string", "pattern": "^cache_[0-9a-f]{64}$" }, + "protocol": { "type": "string", "enum": ["content_store", "compiler_cache", "locked_index"] }, + "access": { "type": "string", "enum": ["tool_concurrent", "host_exclusive"] }, + "authority": { "type": "string", "enum": ["performance_only"] }, + "scope": { "type": "string", "enum": ["workspace"] }, + "compatibility": { "type": "object", "additionalProperties": { "type": "string" } } + } + }, + "EnvironmentExternalArtifactReport": { + "type": "object", + "required": ["name", "artifact_type", "provider", "reference", "digest", "platform", "cleanup_owner"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[A-Za-z0-9._-]+$" }, + "artifact_type": { "type": "string", "enum": ["oci_image"] }, + "provider": { "type": "string", "enum": ["oci"] }, + "reference": { "type": "string", "minLength": 1, "maxLength": 2120 }, + "digest": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }, + "platform": { "type": "string", "enum": ["linux/amd64", "linux/arm64", "windows/amd64", "windows/arm64"] }, + "cleanup_owner": { "type": "string", "enum": ["external"] } + } + }, + "EnvironmentRuntimeDeclarationReport": { + "type": "object", + "required": ["name", "runtime_type", "provider", "artifact_name", "container_port", "protocol", "health_type", "health_timeout_ms", "restart_policy", "cleanup_owner", "volume_target", "secrets"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[A-Za-z0-9._-]+$" }, + "runtime_type": { "type": "string", "enum": ["container"] }, + "provider": { "type": "string", "enum": ["oci"] }, + "artifact_name": { "type": "string", "minLength": 1 }, + "container_port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "protocol": { "type": "string", "enum": ["tcp"] }, + "health_type": { "type": "string", "enum": ["tcp"] }, + "health_timeout_ms": { "type": "integer", "minimum": 1000, "maximum": 300000 }, + "restart_policy": { "type": "string", "enum": ["never", "on_failure", "always"] }, + "cleanup_owner": { "type": "string", "enum": ["trail"] }, + "volume_target": { "type": ["string", "null"] }, + "secrets": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentSecretReferenceReport" }, "maxItems": 16 } + } + }, + "EnvironmentSecretReferenceReport": { + "type": "object", + "required": ["name", "provider", "reference", "version", "purpose", "injection", "target", "environment", "required"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[A-Za-z0-9._-]+$" }, + "provider": { "type": "string", "enum": ["file", "environment_file"] }, + "reference": { "type": "string", "minLength": 1, "maxLength": 4096 }, + "version": { "type": ["string", "null"], "minLength": 1, "maxLength": 256 }, + "purpose": { "type": "string", "minLength": 1, "maxLength": 256 }, + "injection": { "type": "string", "enum": ["file"] }, + "target": { "type": "string", "pattern": "^/run/secrets/[^/]+(?:/[^/]+)*$" }, + "environment": { "type": ["string", "null"], "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" }, + "required": { "type": "boolean" } + } + }, + "EnvironmentSecretStatusReport": { + "type": "object", + "required": ["name", "provider", "reference", "version", "purpose", "injection", "target", "environment", "required", "status", "reason", "resolved_at", "updated_at"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[A-Za-z0-9._-]+$" }, + "provider": { "type": "string", "enum": ["file", "environment_file"] }, + "reference": { "type": "string", "minLength": 1, "maxLength": 4096 }, + "version": { "type": ["string", "null"], "minLength": 1, "maxLength": 256 }, + "purpose": { "type": "string", "minLength": 1, "maxLength": 256 }, + "injection": { "type": "string", "enum": ["file"] }, + "target": { "type": "string", "pattern": "^/run/secrets/[^/]+(?:/[^/]+)*$" }, + "environment": { "type": ["string", "null"], "pattern": "^[A-Z_][A-Z0-9_]{0,127}$" }, + "required": { "type": "boolean" }, + "status": { "type": "string", "enum": ["pending", "available", "unavailable"] }, + "reason": { "type": ["string", "null"] }, + "resolved_at": { "type": ["integer", "null"] }, + "updated_at": { "type": "integer" } + } + }, + "EnvironmentRuntimeResourceReport": { + "type": "object", + "required": ["name", "runtime_type", "provider", "artifact_name", "container_port", "protocol", "health_type", "health_timeout_ms", "restart_policy", "cleanup_owner", "volume_target", "secrets", "image_reference", "image_digest", "image_platform", "allocation_id", "provider_resource_id", "container_name", "network_name", "volume_name", "host_port", "status", "health_status", "reason", "created_at", "updated_at", "started_at", "stopped_at", "secret_statuses"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[A-Za-z0-9._-]+$" }, + "runtime_type": { "type": "string", "enum": ["container"] }, + "provider": { "type": "string", "enum": ["oci"] }, + "artifact_name": { "type": "string", "minLength": 1 }, + "container_port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "protocol": { "type": "string", "enum": ["tcp"] }, + "health_type": { "type": "string", "enum": ["tcp"] }, + "health_timeout_ms": { "type": "integer", "minimum": 1000, "maximum": 300000 }, + "restart_policy": { "type": "string", "enum": ["never", "on_failure", "always"] }, + "cleanup_owner": { "type": "string", "enum": ["trail"] }, + "volume_target": { "type": ["string", "null"] }, + "secrets": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentSecretReferenceReport" }, "maxItems": 16 }, + "image_reference": { "type": "string" }, + "image_digest": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }, + "image_platform": { "type": "string" }, + "allocation_id": { "type": "string", "pattern": "^runtime_[0-9a-f]{32}$" }, + "provider_resource_id": { "type": ["string", "null"] }, + "container_name": { "type": "string" }, + "network_name": { "type": "string" }, + "volume_name": { "type": ["string", "null"] }, + "host_port": { "type": ["integer", "null"], "minimum": 1, "maximum": 65535 }, + "status": { "type": "string", "enum": ["pending", "allocating", "running", "failed", "stopping", "stopped", "orphaned"] }, + "health_status": { "type": "string", "enum": ["pending", "starting", "healthy", "unhealthy", "stopped", "unknown"] }, + "reason": { "type": ["string", "null"] }, + "created_at": { "type": "integer" }, + "updated_at": { "type": "integer" }, + "started_at": { "type": ["integer", "null"] }, + "stopped_at": { "type": ["integer", "null"] }, + "secret_statuses": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentSecretStatusReport" }, "maxItems": 16 } + } + }, + "EnvironmentGenerationDependencyReport": { + "type": "object", + "required": ["component_id", "component_key", "edge_type"], + "additionalProperties": false, + "properties": { + "component_id": { "type": "string" }, + "component_key": { "type": "string" }, + "edge_type": { "type": "string", "enum": ["build_requires", "runtime_requires", "binds_after", "invalidates_with"] } + } + }, + "EnvironmentGenerationComponentReport": { + "type": "object", + "required": ["component_id", "adapter_identity", "kind", "component_key", "dependencies", "outputs", "caches", "external_artifacts", "runtime_resources"], + "additionalProperties": false, + "properties": { + "component_id": { "type": "string" }, + "adapter_identity": { "type": "string" }, + "kind": { "type": "string" }, + "component_key": { "type": "string" }, + "layer_id": { "type": ["string", "null"] }, + "mount_path": { "type": ["string", "null"] }, + "dependencies": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentGenerationDependencyReport" } }, + "outputs": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentGenerationOutputReport" } }, + "caches": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentCacheReport" } }, + "external_artifacts": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentExternalArtifactReport" } }, + "runtime_resources": { "type": "array", "items": { "$ref": "#/components/schemas/EnvironmentRuntimeResourceReport" } } + } + }, + "EnvironmentGenerationReport": { + "type": "object", + "required": ["generation_id", "view_id", "generation_sequence", "source_root", "specification_digest", "state", "components", "created_at"], + "additionalProperties": false, + "properties": { + "generation_id": { "type": "string" }, + "view_id": { "type": "string" }, + "generation_sequence": { "type": "integer", "minimum": 1 }, + "source_root": { "type": "string" }, + "specification_digest": { "type": "string" }, + "predecessor_generation_id": { "type": ["string", "null"] }, + "state": { "type": "string", "enum": ["active", "retired", "failed"] }, + "components": { + "type": "array", + "items": { "$ref": "#/components/schemas/EnvironmentGenerationComponentReport" } + }, + "created_at": { "type": "integer" }, + "activated_at": { "type": ["integer", "null"] }, + "retired_at": { "type": ["integer", "null"] } + } + }, + "EnvironmentGenerationReportNullable": { + "oneOf": [ + { "$ref": "#/components/schemas/EnvironmentGenerationReport" }, + { "type": "null" } + ] + }, + "EnvironmentSyncReport": { + "type": "object", + "required": ["generation", "layers"], + "additionalProperties": false, + "properties": { + "generation": { "$ref": "#/components/schemas/EnvironmentGenerationReport" }, + "layers": { + "type": "array", + "items": { "$ref": "#/components/schemas/WorkspaceLayerReport" } + } + } + }, "CacheGcRequest": { "type": "object", "properties": { diff --git a/trail/src/server/openapi/schemas/patches.rs b/trail/src/server/openapi/schemas/patches.rs index 9907f1e..1a26354 100644 --- a/trail/src/server/openapi/schemas/patches.rs +++ b/trail/src/server/openapi/schemas/patches.rs @@ -4,7 +4,7 @@ pub(super) fn patch_schemas() -> Value { json!({ "PatchRequest": { "type": "object", - "description": "Native Trail PatchDocument or design-style files patch. Provide exactly one non-empty edit source: edits or files. replace_line/modify_line edits must include expected_text.", + "description": "Native Trail PatchDocument or design-style files patch. Explicitly provide exactly one source, edits or files; an empty array records a genuine zero-change patch operation. replace_line/modify_line edits must include expected_text.", "oneOf": [ { "required": ["edits"], "not": { "required": ["files"] } }, { "required": ["files"], "not": { "required": ["edits"] } } @@ -15,8 +15,8 @@ pub(super) fn patch_schemas() -> Value { "session_id": { "type": "string" }, "allow_ignored": { "type": "boolean" }, "allow_stale": { "type": "boolean" }, - "edits": { "type": "array", "minItems": 1, "items": { "$ref": "#/components/schemas/PatchEdit" } }, - "files": { "type": "array", "minItems": 1, "items": { "$ref": "#/components/schemas/ApiPatchFile" } } + "edits": { "type": "array", "items": { "$ref": "#/components/schemas/PatchEdit" } }, + "files": { "type": "array", "items": { "$ref": "#/components/schemas/ApiPatchFile" } } } }, "PatchEdit": { diff --git a/trail/src/server/request_types.rs b/trail/src/server/request_types.rs index 743c1dc..90a43eb 100644 --- a/trail/src/server/request_types.rs +++ b/trail/src/server/request_types.rs @@ -1,8 +1,10 @@ +mod agent_hooks; mod collaboration; mod core; mod lane; mod patches; +pub(crate) use agent_hooks::*; pub(crate) use collaboration::*; pub(crate) use core::*; pub(crate) use lane::*; diff --git a/trail/src/server/request_types/agent_hooks.rs b/trail/src/server/request_types/agent_hooks.rs new file mode 100644 index 0000000..516488d --- /dev/null +++ b/trail/src/server/request_types/agent_hooks.rs @@ -0,0 +1,52 @@ +use serde::Deserialize; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AgentCaptureRunRequest { + #[serde(default)] + pub(crate) lane: Option, + pub(crate) workdir: String, + pub(crate) owner_agent: String, + pub(crate) owner_session_id: String, + #[serde(default)] + pub(crate) executor_agent: Option, + #[serde(default)] + pub(crate) work_item_id: Option, + pub(crate) lease_ms: u64, + #[serde(default)] + pub(crate) metadata: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AgentCaptureRunLeaseRequest { + pub(crate) owner_agent: String, + pub(crate) owner_session_id: String, + #[serde(default)] + pub(crate) lease_ms: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AgentAttestationCreateRequest { + #[serde(default = "default_capture_policy")] + pub(crate) capture_policy: String, + #[serde(default)] + pub(crate) metadata: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AgentLearningReviewRequest { + pub(crate) reviewer: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AgentArtifactRedactRequest { + pub(crate) reason: String, +} + +fn default_capture_policy() -> String { + "native-agent-hooks/v1".to_string() +} diff --git a/trail/src/server/request_types/collaboration.rs b/trail/src/server/request_types/collaboration.rs index 4ed3365..cd42755 100644 --- a/trail/src/server/request_types/collaboration.rs +++ b/trail/src/server/request_types/collaboration.rs @@ -6,12 +6,11 @@ use super::default_completed_status; #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -pub(crate) struct MergeLaneRequest { - #[serde(default, alias = "lane", alias = "name")] - pub(crate) lane_id: Option, +pub(crate) struct LaneMergeRequest { + pub(crate) into: String, #[serde(default)] pub(crate) strategy: Option, - #[serde(default, alias = "dry-run")] + #[serde(default)] pub(crate) dry_run: bool, #[serde(default)] pub(crate) direct: bool, @@ -106,17 +105,16 @@ pub(crate) struct AnchorCreateRequest { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -pub(crate) struct MergeQueueAddRequest { - pub(crate) source: String, - #[serde(alias = "into", alias = "target_branch")] - pub(crate) target: String, +pub(crate) struct LaneMergeQueueAddRequest { + pub(crate) lane: String, + pub(crate) into: String, #[serde(default)] pub(crate) priority: i64, } #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -pub(crate) struct MergeQueueRunRequest { +pub(crate) struct LaneMergeQueueRunRequest { #[serde(default)] pub(crate) limit: Option, } diff --git a/trail/src/server/request_types/core.rs b/trail/src/server/request_types/core.rs index 7894df9..76649c3 100644 --- a/trail/src/server/request_types/core.rs +++ b/trail/src/server/request_types/core.rs @@ -17,6 +17,13 @@ pub(crate) struct RecordRequest { pub(crate) allow_ignored: bool, } +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct IndexReconcileRequest { + #[serde(default)] + pub(crate) lane: Option, +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct IgnorePatternRequest { diff --git a/trail/src/server/request_types/lane.rs b/trail/src/server/request_types/lane.rs index 3abeebd..c2c378e 100644 --- a/trail/src/server/request_types/lane.rs +++ b/trail/src/server/request_types/lane.rs @@ -182,13 +182,24 @@ pub(crate) struct WorkspaceExecRequest { pub(crate) command: Vec, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Default, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct DependencySyncRequest { #[serde(default)] pub(crate) path: Option, } +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct EnvironmentSyncRequest { + #[serde(default)] + pub(crate) adapter: Option, + #[serde(default, alias = "component_id")] + pub(crate) component: Option, + #[serde(default, alias = "component_root")] + pub(crate) path: Option, +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct CacheGcRequest { diff --git a/trail/src/server/request_types/patches.rs b/trail/src/server/request_types/patches.rs index 5d89892..9828464 100644 --- a/trail/src/server/request_types/patches.rs +++ b/trail/src/server/request_types/patches.rs @@ -13,10 +13,8 @@ pub(crate) struct ApiPatchRequest { pub(crate) allow_ignored: bool, #[serde(default)] pub(crate) allow_stale: bool, - #[serde(default)] - pub(crate) edits: Vec, - #[serde(default)] - pub(crate) files: Vec, + pub(crate) edits: Option>, + pub(crate) files: Option>, } #[derive(Debug, Deserialize)] diff --git a/trail/src/server/route.rs b/trail/src/server/route.rs index 3827b82..e96cdb2 100644 --- a/trail/src/server/route.rs +++ b/trail/src/server/route.rs @@ -1,3 +1,4 @@ +mod agent_hooks; mod audit; mod dispatch; mod idempotency; @@ -15,6 +16,27 @@ pub(crate) fn route_request( request: HttpRequest, auth: &ServerAuth, ) -> HttpResponse { + let metrics_requested = auth.daemon_identity.is_some() + && request + .headers + .get("x-trail-operation-metrics") + .is_some_and(|value| value == "1") + && utils::authorized(&request, auth); + let metrics_generation = metrics_requested + .then(|| db.operation_metrics_generation()) + .flatten(); + let mut response = route_request_inner(db, request, auth); + if let Some(generation) = metrics_generation { + if let Some(report) = db.operation_metrics_json_after(generation) { + response + .extra_headers + .push(("X-Trail-Operation-Metrics", report)); + } + } + response +} + +fn route_request_inner(db: &mut Trail, request: HttpRequest, auth: &ServerAuth) -> HttpResponse { let audit = audit::HttpMutationAudit::from_request(&request); let idempotency = if utils::host_allowed(&request) && utils::origin_allowed(&request) @@ -63,3 +85,47 @@ pub(crate) fn route_request( } response } + +#[cfg(test)] +mod tests { + use super::*; + use crate::InitImportMode; + use std::collections::BTreeMap; + + #[test] + fn authenticated_daemon_route_returns_only_its_new_operation_report() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::Empty, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + let auth = ServerAuth::bearer("route-metrics-token") + .unwrap() + .with_daemon_identity(super::super::transport::DaemonServerIdentity::new( + "owner", + "workspace", + "executable", + "process", + )); + let request = HttpRequest { + method: "GET".into(), + path: "/v1/status".into(), + headers: BTreeMap::from([ + ("host".into(), "localhost".into()), + ("authorization".into(), "Bearer route-metrics-token".into()), + ("x-trail-operation-metrics".into(), "1".into()), + ]), + body: Vec::new(), + }; + let response = route_request(&mut db, request, &auth); + assert_eq!(response.status, 200); + let reports = response + .extra_headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("x-trail-operation-metrics")) + .collect::>(); + assert_eq!(reports.len(), 1); + let report: serde_json::Value = serde_json::from_str(&reports[0].1).unwrap(); + assert_eq!(report["generation"], 1); + assert_eq!(report["operation"], "status"); + assert_eq!(report["outcome"], "success"); + } +} diff --git a/trail/src/server/route/agent_hooks.rs b/trail/src/server/route/agent_hooks.rs new file mode 100644 index 0000000..668ba17 --- /dev/null +++ b/trail/src/server/route/agent_hooks.rs @@ -0,0 +1,395 @@ +use sha2::{Digest, Sha256}; + +use crate::agent_hooks::AgentProviderRegistry; +use crate::server::request_types::{ + AgentArtifactRedactRequest, AgentAttestationCreateRequest, AgentCaptureRunLeaseRequest, + AgentCaptureRunRequest, AgentLearningReviewRequest, +}; +use crate::server::route::utils::{json_response, query_usize, query_value}; +use crate::server::transport::{HttpRequest, HttpResponse}; +use crate::{ + AgentCaptureRunInput, AgentCaptureTransport, AgentHookReceiptInput, GitAgentLinkInput, + LearningInput, Result, Trail, +}; + +pub(super) fn handle_agent_hook_route( + db: &mut Trail, + request: &HttpRequest, + path: &str, + query: &str, + parts: &[&str], +) -> Result> { + if request.method == "GET" && path == "/v1/agent-integrations/capabilities" { + let registry = AgentProviderRegistry::built_in()?; + return Ok(Some(json_response(200, "OK", ®istry.list())?)); + } + + if request.method == "GET" && path == "/v1/agent-hooks/installations" { + let reports = db.list_agent_hook_installations(query_value(query, "provider"))?; + return Ok(Some(json_response(200, "OK", &reports)?)); + } + if request.method == "GET" && parts_match(parts, &["v1", "agent-hooks", "installations", "*"]) { + return Ok(Some(json_response( + 200, + "OK", + &db.agent_hook_installation(parts[3])?, + )?)); + } + if request.method == "GET" && path == "/v1/agent-hooks/receipts" { + let reports = db.list_agent_hook_receipts_page( + query_value(query, "provider"), + query_value(query, "status"), + query_usize(query, "offset", 0)?, + query_usize(query, "limit", 100)?, + )?; + return Ok(Some(json_response(200, "OK", &reports)?)); + } + if request.method == "GET" && parts_match(parts, &["v1", "agent-hooks", "receipts", "*"]) { + return Ok(Some(json_response( + 200, + "OK", + &db.agent_hook_receipt(parts[3])?, + )?)); + } + if request.method == "POST" + && parts_match(parts, &["v1", "agent-hooks", "receipts", "*", "replay"]) + { + return Ok(Some(json_response( + 200, + "OK", + &db.replay_agent_hook_receipt(parts[3])?, + )?)); + } + if request.method == "POST" + && parts_match(parts, &["v1", "agent-hooks", "receipts", "*", "retry"]) + { + return Ok(Some(json_response( + 200, + "OK", + &db.retry_agent_hook_receipt(parts[3])?, + )?)); + } + if request.method == "POST" + && parts_match(parts, &["v1", "agent-hooks", "receipts", "*", "discard"]) + { + return Ok(Some(json_response( + 200, + "OK", + &db.discard_agent_hook_receipt(parts[3])?, + )?)); + } + if request.method == "POST" && parts.len() == 4 && parts[0] == "v1" && parts[1] == "agent-hooks" + { + let registry = AgentProviderRegistry::built_in()?; + let provider = registry.resolve(parts[2])?.provider.clone(); + let payload: serde_json::Value = serde_json::from_slice(&request.body)?; + let native_session_id = payload_string( + &payload, + &[ + "session_id", + "sessionId", + "sessionID", + "conversation_id", + "conversationId", + "thread_id", + "threadId", + ], + ); + let native_turn_id = payload_string(&payload, &["turn_id", "turnId"]); + let occurred_at = payload_i64(&payload, &["timestamp", "occurred_at", "occurredAt"]); + let dedupe_key = query_value(query, "dedupe_key") + .map(ToString::to_string) + .unwrap_or_else(|| { + hook_dedupe_key( + &provider, + parts[3], + native_session_id.as_deref(), + native_turn_id.as_deref(), + &payload, + ) + }); + let response_provider = provider.clone(); + let report = db.persist_agent_hook_receipt(AgentHookReceiptInput { + installation_id: query_value(query, "installation").map(ToString::to_string), + provider, + native_event: parts[3].to_string(), + native_session_id, + native_turn_id, + transport: AgentCaptureTransport::NativeHooks, + connection_id: None, + direction: None, + connection_sequence: None, + dedupe_key, + payload, + occurred_at, + })?; + let response_body = + if response_provider == "codex" && matches!(parts[3], "Stop" | "SubagentStop") { + serde_json::json!({"continue": true}) + } else if response_provider == "gemini" { + serde_json::json!({}) + } else { + serde_json::to_value(&report)? + }; + let mut response = json_response(200, "OK", &response_body)?; + response + .extra_headers + .push(("X-Trail-Receipt-Id", report.receipt.receipt_id.clone())); + return Ok(Some(response)); + } + + if request.method == "GET" && path == "/v1/agent-capture-runs" { + let active_only = query_value(query, "active_only") + .map(|value| !matches!(value, "0" | "false" | "no")) + .unwrap_or(true); + let reports = db.list_agent_capture_runs_page( + active_only, + query_usize(query, "offset", 0)?, + query_usize(query, "limit", 100)?, + )?; + return Ok(Some(json_response(200, "OK", &reports)?)); + } + if request.method == "GET" && parts_match(parts, &["v1", "agent-capture-runs", "*"]) { + return Ok(Some(json_response( + 200, + "OK", + &db.agent_capture_run(parts[2])?, + )?)); + } + if request.method == "POST" && path == "/v1/agent-capture-runs" { + let body: AgentCaptureRunRequest = serde_json::from_slice(&request.body)?; + let report = db.begin_agent_capture_run(AgentCaptureRunInput { + lane: body.lane, + workdir: body.workdir, + owner_agent: body.owner_agent, + owner_session_id: body.owner_session_id, + executor_agent: body.executor_agent, + work_item_id: body.work_item_id, + lease_ms: body.lease_ms, + metadata_json: body.metadata.map(|value| value.to_string()), + })?; + return Ok(Some(json_response(201, "Created", &report)?)); + } + if request.method == "POST" && path == "/v1/agent-capture-runs/reconcile" { + return Ok(Some(json_response( + 200, + "OK", + &db.reconcile_expired_agent_capture_runs()?, + )?)); + } + if request.method == "POST" && parts_match(parts, &["v1", "agent-capture-runs", "*", "renew"]) { + let body: AgentCaptureRunLeaseRequest = serde_json::from_slice(&request.body)?; + let report = db.renew_agent_capture_run( + parts[2], + &body.owner_agent, + &body.owner_session_id, + body.lease_ms.unwrap_or(300_000), + )?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + if request.method == "POST" && parts_match(parts, &["v1", "agent-capture-runs", "*", "end"]) { + let body: AgentCaptureRunLeaseRequest = serde_json::from_slice(&request.body)?; + let report = + db.end_agent_capture_run(parts[2], &body.owner_agent, &body.owner_session_id)?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + + if request.method == "GET" && parts_match(parts, &["v1", "agent-sessions", "*", "artifacts"]) { + let reports = db.list_lane_artifacts_page( + parts[2], + query_value(query, "turn"), + query_usize(query, "offset", 0)?, + query_usize(query, "limit", 100)?, + )?; + return Ok(Some(json_response(200, "OK", &reports)?)); + } + if request.method == "GET" && parts_match(parts, &["v1", "agent-artifacts", "*"]) { + return Ok(Some(json_response( + 200, + "OK", + &db.lane_artifact(parts[2])?, + )?)); + } + if request.method == "POST" && parts_match(parts, &["v1", "agent-artifacts", "*", "redact"]) { + let body: AgentArtifactRedactRequest = serde_json::from_slice(&request.body)?; + return Ok(Some(json_response( + 200, + "OK", + &db.redact_lane_artifact(parts[2], &body.reason)?, + )?)); + } + if request.method == "GET" && parts_match(parts, &["v1", "agent-turns", "*", "evidence"]) { + return Ok(Some(json_response( + 200, + "OK", + &db.turn_evidence_manifest(parts[2])?, + )?)); + } + if request.method == "GET" && parts_match(parts, &["v1", "agent-sessions", "*", "provenance"]) { + let (nodes, edges) = db.list_session_provenance_page( + parts[2], + query_usize(query, "offset", 0)?, + query_usize(query, "limit", 1_000)?, + )?; + return Ok(Some(json_response( + 200, + "OK", + &serde_json::json!({"session_id": parts[2], "nodes": nodes, "edges": edges}), + )?)); + } + if request.method == "GET" && parts_match(parts, &["v1", "agent-sessions", "*", "attestations"]) + { + return Ok(Some(json_response( + 200, + "OK", + &db.list_session_attestations_page( + parts[2], + query_usize(query, "offset", 0)?, + query_usize(query, "limit", 100)?, + )?, + )?)); + } + if request.method == "POST" + && parts_match(parts, &["v1", "agent-sessions", "*", "attestations"]) + { + let body: AgentAttestationCreateRequest = if request.body.is_empty() { + serde_json::from_value(serde_json::json!({}))? + } else { + serde_json::from_slice(&request.body)? + }; + let report = + db.create_session_attestation(parts[2], &body.capture_policy, body.metadata)?; + return Ok(Some(json_response(201, "Created", &report)?)); + } + if request.method == "POST" && parts_match(parts, &["v1", "agent-attestations", "*", "verify"]) + { + return Ok(Some(json_response( + 200, + "OK", + &db.verify_session_attestation(parts[2])?, + )?)); + } + if request.method == "GET" && parts_match(parts, &["v1", "agent-attestations", "*"]) { + return Ok(Some(json_response( + 200, + "OK", + &db.session_attestation(parts[2])?, + )?)); + } + if request.method == "GET" && parts_match(parts, &["v1", "agent-sessions", "*", "export"]) { + if query_value(query, "format").is_some_and(|format| format != "agent-trace") { + return Err(crate::Error::InvalidInput( + "only `format=agent-trace` is supported".to_string(), + )); + } + let attachments = query_value(query, "attachments") + .is_some_and(|value| matches!(value, "1" | "true" | "yes")); + return Ok(Some(json_response( + 200, + "OK", + &db.export_agent_trace(parts[2], attachments)?, + )?)); + } + + if request.method == "GET" && path == "/v1/agent-learnings" { + let reports = db.list_learnings_page( + query_value(query, "session"), + query_value(query, "status"), + query_usize(query, "offset", 0)?, + query_usize(query, "limit", 100)?, + )?; + return Ok(Some(json_response(200, "OK", &reports)?)); + } + if request.method == "POST" && path == "/v1/agent-learnings" { + let body: LearningInput = serde_json::from_slice(&request.body)?; + return Ok(Some(json_response( + 201, + "Created", + &db.propose_learning(body)?, + )?)); + } + if request.method == "GET" && parts_match(parts, &["v1", "agent-learnings", "*"]) { + return Ok(Some(json_response(200, "OK", &db.learning(parts[2])?)?)); + } + if request.method == "POST" + && parts.len() == 4 + && parts[0] == "v1" + && parts[1] == "agent-learnings" + && matches!(parts[3], "accept" | "reject") + { + let body: AgentLearningReviewRequest = serde_json::from_slice(&request.body)?; + let report = db.review_learning(parts[2], parts[3] == "accept", &body.reviewer)?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + + if request.method == "GET" && parts_match(parts, &["v1", "agent-sessions", "*", "git-links"]) { + return Ok(Some(json_response( + 200, + "OK", + &db.list_git_agent_links_page( + parts[2], + query_usize(query, "offset", 0)?, + query_usize(query, "limit", 100)?, + )?, + )?)); + } + if request.method == "POST" && path == "/v1/agent-git-links" { + let body: GitAgentLinkInput = serde_json::from_slice(&request.body)?; + return Ok(Some(json_response( + 201, + "Created", + &db.link_git_commit_to_agent(body)?, + )?)); + } + if request.method == "GET" && parts_match(parts, &["v1", "agent-git-links", "*"]) { + return Ok(Some(json_response( + 200, + "OK", + &db.git_agent_link(parts[2])?, + )?)); + } + + Ok(None) +} + +fn parts_match(parts: &[&str], pattern: &[&str]) -> bool { + parts.len() == pattern.len() + && parts + .iter() + .zip(pattern) + .all(|(part, expected)| *expected == "*" || part == expected) +} + +fn payload_string(payload: &serde_json::Value, names: &[&str]) -> Option { + names.iter().find_map(|name| { + payload + .get(*name) + .and_then(serde_json::Value::as_str) + .map(ToString::to_string) + }) +} + +fn payload_i64(payload: &serde_json::Value, names: &[&str]) -> Option { + names.iter().find_map(|name| { + let value = payload.get(*name)?; + value + .as_i64() + .or_else(|| value.as_str().and_then(|raw| raw.parse().ok())) + }) +} + +fn hook_dedupe_key( + provider: &str, + native_event: &str, + session_id: Option<&str>, + turn_id: Option<&str>, + payload: &serde_json::Value, +) -> String { + let bytes = serde_json::to_vec(payload).unwrap_or_default(); + let digest = hex::encode(Sha256::digest(bytes)); + format!( + "http:{provider}:{native_event}:{}:{}:{digest}", + session_id.unwrap_or("none"), + turn_id.unwrap_or("none") + ) +} diff --git a/trail/src/server/route/audit.rs b/trail/src/server/route/audit.rs index f331fe4..484f7df 100644 --- a/trail/src/server/route/audit.rs +++ b/trail/src/server/route/audit.rs @@ -92,17 +92,16 @@ fn http_actor(request: &HttpRequest) -> String { } fn http_request_lane(path: &str, parts: &[&str], body: Option<&Value>) -> Option { + if path == "/v1/lanes/merges/queue" { + return body.and_then(|value| top_level_string_for_keys(value, &["lane"])); + } if parts.len() >= 3 && parts[0] == "v1" && parts[1] == "lanes" { return Some(parts[2].to_string()); } - if parts.len() == 4 && parts[0] == "v1" && parts[1] == "branches" && parts[3] == "merge-lane" { - return body.and_then(|value| top_level_string_for_keys(value, &["lane_id", "lane"])); - } match path { "/v1/lane/turns" | "/v1/sessions" | "/v1/leases" | "/v1/approvals" | "/v1/lane/runs" => { body.and_then(|value| top_level_string_for_keys(value, &["lane"])) } - "/v1/merge-queue" => body.and_then(|value| top_level_string_for_keys(value, &["source"])), _ => None, } } @@ -115,14 +114,14 @@ fn http_request_turn_id(path: &str, body: Option<&Value>) -> Option { } fn http_request_target_ref(path: &str, parts: &[&str], body: Option<&Value>) -> Option { - if parts.len() == 4 && parts[0] == "v1" && parts[1] == "branches" && parts[3] == "merge-lane" { - return Some(http_branch_ref(parts[2])); + if parts.len() == 4 && parts[0] == "v1" && parts[1] == "lanes" && parts[3] == "merge" { + return body + .and_then(|value| top_level_string_for_keys(value, &["into"])) + .map(|target| http_branch_ref(&target)); } - if path == "/v1/merge-queue" { + if path == "/v1/lanes/merges/queue" { return body - .and_then(|value| { - top_level_string_for_keys(value, &["target", "target_branch", "into"]) - }) + .and_then(|value| top_level_string_for_keys(value, &["into"])) .map(|target| http_branch_ref(&target)); } None diff --git a/trail/src/server/route/dispatch.rs b/trail/src/server/route/dispatch.rs index c86db30..970c7fe 100644 --- a/trail/src/server/route/dispatch.rs +++ b/trail/src/server/route/dispatch.rs @@ -2,7 +2,7 @@ use crate::server::transport::{HttpRequest, HttpResponse, ServerAuth}; use crate::{Error, Result}; use super::utils; -use super::{lane, system}; +use super::{agent_hooks, lane, system}; pub(crate) fn route_request_result( db: &mut crate::Trail, @@ -38,7 +38,21 @@ pub(crate) fn route_request_result( let parts: Vec<&str> = path.split('/').filter(|part| !part.is_empty()).collect(); - if let Some(response) = system::handle_system_route(db, &request, path, query, &parts)? { + if request.method == "POST" + && parts.len() == 4 + && parts[0] == "v1" + && parts[1] == "agent-hooks" + && !auth.is_required() + { + return Ok(utils::unauthorized_response()); + } + + if let Some(response) = agent_hooks::handle_agent_hook_route(db, &request, path, query, &parts)? + { + return Ok(response); + } + + if let Some(response) = system::handle_system_route(db, &request, auth, path, query, &parts)? { return Ok(response); } diff --git a/trail/src/server/route/lane/collaboration.rs b/trail/src/server/route/lane/collaboration.rs index 731427c..69704f3 100644 --- a/trail/src/server/route/lane/collaboration.rs +++ b/trail/src/server/route/lane/collaboration.rs @@ -1,13 +1,13 @@ use crate::server::request_types::{ - default_lease_mode, AnchorCreateRequest, ConflictResolveRequest, LeaseAcquireRequest, - MergeLaneRequest, MergeQueueAddRequest, MergeQueueRunRequest, + default_lease_mode, AnchorCreateRequest, ConflictResolveRequest, LaneMergeQueueAddRequest, + LaneMergeQueueRunRequest, LaneMergeRequest, LeaseAcquireRequest, }; use crate::server::route::utils::{ - json_response, query_flag, query_usize, query_value, reject_unexpected_body, required_query, + json_response, query_flag, query_usize, query_value, reject_unexpected_body, resolve_conflict_request, validate_merge_strategy, }; use crate::server::transport::{HttpRequest, HttpResponse}; -use crate::{Error, Result, Trail}; +use crate::{Result, Trail}; pub(super) fn handle_collaboration_routes( db: &mut Trail, @@ -44,29 +44,24 @@ pub(super) fn handle_collaboration_routes( return Ok(Some(json_response(201, "Created", &report)?)); } - if request.method == "GET" && path == "/v1/merge-queue" { - let entries = db.list_merge_queue()?; + if request.method == "GET" && path == "/v1/lanes/merges/queue" { + let entries = db.list_lane_merge_queue()?; return Ok(Some(json_response(200, "OK", &entries)?)); } - if request.method == "POST" && path == "/v1/merge-queue" { - let body: MergeQueueAddRequest = serde_json::from_slice(&request.body)?; - let report = db.enqueue_merge(&body.source, &body.target, body.priority)?; + if request.method == "POST" && path == "/v1/lanes/merges/queue" { + let body: LaneMergeQueueAddRequest = serde_json::from_slice(&request.body)?; + let report = db.enqueue_lane_merge(&body.lane, &body.into, body.priority)?; return Ok(Some(json_response(201, "Created", &report)?)); } - if request.method == "POST" && path == "/v1/merge-queue/run" { - let body: MergeQueueRunRequest = if request.body.is_empty() { - MergeQueueRunRequest { limit: None } + if request.method == "POST" && path == "/v1/lanes/merges/queue/run" { + let body: LaneMergeQueueRunRequest = if request.body.is_empty() { + LaneMergeQueueRunRequest { limit: None } } else { serde_json::from_slice(&request.body)? }; - let report = db.run_merge_queue(body.limit)?; - return Ok(Some(json_response(200, "OK", &report)?)); - } - - if request.method == "GET" && path == "/v1/merge-queue/explain" { - let report = db.explain_merge_queue(required_query(query, "selector")?)?; + let report = db.run_lane_merge_queue(body.limit)?; return Ok(Some(json_response(200, "OK", &report)?)); } @@ -93,23 +88,27 @@ pub(super) fn handle_collaboration_routes( } } - if parts.len() == 3 + if parts.len() == 5 && parts[0] == "v1" - && parts[1] == "merge-queue" + && parts[1] == "lanes" + && parts[2] == "merges" + && parts[3] == "queue" && request.method == "DELETE" { - reject_unexpected_body(request, "DELETE /v1/merge-queue/{queue_id}")?; - let report = db.remove_merge_queue(parts[2])?; + reject_unexpected_body(request, "DELETE /v1/lanes/merges/queue/{selector}")?; + let report = db.remove_lane_merge_queue(parts[4])?; return Ok(Some(json_response(200, "OK", &report)?)); } - if parts.len() == 4 + if parts.len() == 6 && parts[0] == "v1" - && parts[1] == "merge-queue" - && parts[3] == "explain" + && parts[1] == "lanes" + && parts[2] == "merges" + && parts[3] == "queue" + && parts[5] == "explain" && request.method == "GET" { - let report = db.explain_merge_queue(parts[2])?; + let report = db.explain_lane_merge_queue(parts[4])?; return Ok(Some(json_response(200, "OK", &report)?)); } @@ -131,17 +130,15 @@ pub(super) fn handle_collaboration_routes( if parts.len() == 4 && parts[0] == "v1" - && parts[1] == "branches" - && parts[3] == "merge-lane" + && parts[1] == "lanes" + && parts[3] == "merge" && request.method == "POST" { - let body: MergeLaneRequest = serde_json::from_slice(&request.body)?; + let body: LaneMergeRequest = serde_json::from_slice(&request.body)?; validate_merge_strategy(body.strategy.as_deref())?; - let lane = body.lane_id.ok_or_else(|| { - Error::InvalidInput("merge-lane request requires `lane_id`".to_string()) - })?; - let lane = db.resolve_lane_handle(&lane)?; - let report = db.merge_lane_user_with_options(&lane, parts[2], body.dry_run, body.direct)?; + let lane = db.resolve_lane_handle(parts[2])?; + let report = + db.merge_lane_user_with_options(&lane, &body.into, body.dry_run, body.direct)?; return Ok(Some(json_response(200, "OK", &report)?)); } diff --git a/trail/src/server/route/lane/lanes.rs b/trail/src/server/route/lane/lanes.rs index bb00145..e850dd8 100644 --- a/trail/src/server/route/lane/lanes.rs +++ b/trail/src/server/route/lane/lanes.rs @@ -2,9 +2,9 @@ use std::path::PathBuf; use crate::model::LaneGateOptions; use crate::server::request_types::{ - DependencySyncRequest, LaneClaimRequest, LaneReadFileRequest, LaneRecordRequest, - LaneRewindRequest, LaneTestRequest, LaneUpdateRequest, SpawnLaneRequest, SyncWorkdirRequest, - WorkspaceCheckpointRequest, WorkspaceExecRequest, + DependencySyncRequest, EnvironmentSyncRequest, LaneClaimRequest, LaneReadFileRequest, + LaneRecordRequest, LaneRewindRequest, LaneTestRequest, LaneUpdateRequest, SpawnLaneRequest, + SyncWorkdirRequest, WorkspaceCheckpointRequest, WorkspaceExecRequest, }; use crate::server::route::utils::{ json_response, parse_patch_request, query_flag, query_line_ids_flag, query_usize, query_value, @@ -20,6 +20,11 @@ pub(super) fn handle_lane_resources( query: &str, parts: &[&str], ) -> Result> { + if request.method == "GET" && path == "/v1/environment/adapters" { + let report = db.workspace_environment_adapters()?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + if request.method == "GET" && path == "/v1/lanes" { let lanes = db.list_lanes()?; return Ok(Some(json_response(200, "OK", &lanes)?)); @@ -73,7 +78,12 @@ pub(super) fn handle_lane_resources( return Ok(Some(json_response(200, "OK", &report)?)); } - if parts.len() == 4 && parts[0] == "v1" && parts[1] == "lanes" && request.method == "GET" { + if parts.len() == 4 + && parts[0] == "v1" + && parts[1] == "lanes" + && path != "/v1/lanes/merges/queue" + && request.method == "GET" + { let lane = db.resolve_lane_handle(parts[2])?; return Ok(Some(match parts[3] { "status" => { @@ -125,6 +135,10 @@ pub(super) fn handle_lane_resources( let report = db.workspace_environment_status(&lane)?; json_response(200, "OK", &report)? } + "environment" => { + let report = db.environment_component_status(&lane)?; + json_response(200, "OK", &report)? + } "diff" => { let diff = db.diff_lane_with_options( &lane, @@ -137,11 +151,138 @@ pub(super) fn handle_lane_resources( return Err(Error::InvalidInput(format!( "unknown API endpoint `{}`", request.path - ))) + ))); } })); } + if parts.len() == 5 + && parts[0] == "v1" + && parts[1] == "lanes" + && parts[3] == "environment" + && parts[4] == "explain" + && request.method == "GET" + { + let lane = db.resolve_lane_handle(parts[2])?; + let component = query_value(query, "component").ok_or_else(|| { + Error::InvalidInput("environment explain requires `component`".to_string()) + })?; + let report = db.explain_workspace_environment_staleness_page( + &lane, + component, + query_usize(query, "offset", 0)? as u64, + query_usize(query, "limit", 256)? as u64, + )?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + + if parts.len() == 6 + && parts[0] == "v1" + && parts[1] == "lanes" + && parts[3] == "environment" + && parts[4] == "runtime" + && parts[5] == "status" + && request.method == "GET" + { + let lane = db.resolve_lane_handle(parts[2])?; + let report = db.active_environment_generation(&lane)?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + + if parts.len() == 6 + && parts[0] == "v1" + && parts[1] == "lanes" + && parts[3] == "environment" + && parts[4] == "runtime" + && matches!(parts[5], "reconcile" | "stop") + && request.method == "POST" + { + reject_unexpected_body(request, "environment runtime lifecycle")?; + let lane = db.resolve_lane_handle(parts[2])?; + let report = if parts[5] == "reconcile" { + db.reconcile_workspace_environment_runtime(&lane)? + } else { + db.stop_workspace_environment_runtime(&lane)? + }; + return Ok(Some(json_response(200, "OK", &report)?)); + } + + if parts.len() == 5 + && parts[0] == "v1" + && parts[1] == "lanes" + && parts[3] == "environment" + && parts[4] == "plan" + && request.method == "GET" + { + let lane = db.resolve_lane_handle(parts[2])?; + let report = db.plan_workspace_environment_component( + &lane, + query_value(query, "adapter").unwrap_or("auto"), + query_value(query, "path"), + query_value(query, "component"), + )?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + + if parts.len() == 5 + && parts[0] == "v1" + && parts[1] == "lanes" + && parts[3] == "environment" + && parts[4] == "graph" + && request.method == "GET" + { + let lane = db.resolve_lane_handle(parts[2])?; + let report = db.workspace_environment_graph_page( + &lane, + query_value(query, "path"), + query_usize(query, "offset", 0)? as u64, + query_usize(query, "limit", 256)? as u64, + )?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + + if parts.len() == 5 + && parts[0] == "v1" + && parts[1] == "lanes" + && parts[3] == "environment" + && parts[4] == "generation" + && request.method == "GET" + { + let lane = db.resolve_lane_handle(parts[2])?; + let report = db.active_environment_generation(&lane)?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + + if parts.len() == 5 + && parts[0] == "v1" + && parts[1] == "lanes" + && parts[3] == "environment" + && parts[4] == "discover" + && request.method == "GET" + { + let lane = db.resolve_lane_handle(parts[2])?; + let report = db.discover_workspace_environment(&lane, query_value(query, "path"))?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + + if parts.len() == 5 + && parts[0] == "v1" + && parts[1] == "lanes" + && parts[3] == "environment" + && parts[4] == "sync-all" + && request.method == "POST" + { + let lane = db.resolve_lane_handle(parts[2])?; + let body: DependencySyncRequest = if request.body.is_empty() { + DependencySyncRequest::default() + } else { + serde_json::from_slice(&request.body)? + }; + let report = + db.sync_all_workspace_environments_with_runtime(&lane, body.path.as_deref())?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + if parts.len() == 4 && parts[0] == "v1" && parts[1] == "lanes" @@ -234,7 +375,7 @@ pub(super) fn handle_lane_resources( { let lane = db.resolve_lane_handle(parts[2])?; let body: DependencySyncRequest = if request.body.is_empty() { - DependencySyncRequest { path: None } + DependencySyncRequest::default() } else { serde_json::from_slice(&request.body)? }; @@ -242,6 +383,24 @@ pub(super) fn handle_lane_resources( return Ok(Some(json_response(200, "OK", &report)?)); } + if parts.len() == 5 + && parts[0] == "v1" + && parts[1] == "lanes" + && parts[3] == "environment" + && parts[4] == "sync" + && request.method == "POST" + { + let lane = db.resolve_lane_handle(parts[2])?; + let body: EnvironmentSyncRequest = serde_json::from_slice(&request.body)?; + let report = db.sync_workspace_environment_component_with_runtime( + &lane, + body.adapter.as_deref().unwrap_or("auto"), + body.path.as_deref(), + body.component.as_deref(), + )?; + return Ok(Some(json_response(200, "OK", &report)?)); + } + if parts.len() == 4 && parts[0] == "v1" && parts[1] == "lanes" diff --git a/trail/src/server/route/lane/turns.rs b/trail/src/server/route/lane/turns.rs index 3578c4b..1d774c1 100644 --- a/trail/src/server/route/lane/turns.rs +++ b/trail/src/server/route/lane/turns.rs @@ -105,7 +105,7 @@ pub(super) fn handle_turn_routes( return Err(Error::InvalidInput(format!( "unknown API endpoint `{}`", request.path - ))) + ))); } })); } diff --git a/trail/src/server/route/system.rs b/trail/src/server/route/system.rs index 2a86239..ddb0759 100644 --- a/trail/src/server/route/system.rs +++ b/trail/src/server/route/system.rs @@ -1,12 +1,23 @@ use crate::model::{Actor, OperationKind, RecordOptions}; -use crate::server::transport::{HttpRequest, HttpResponse}; +use crate::server::transport::{HttpRequest, HttpResponse, ServerAuth}; use crate::{Error, Result}; use super::utils; +#[derive(serde::Deserialize)] +struct LedgerFenceRequest { + protocol_version: u16, + owner_nonce: String, + workspace_identity: String, + executable_identity: String, + scope_id: String, + expected_epoch: u64, +} + pub(super) fn handle_system_route( db: &mut crate::Trail, request: &HttpRequest, + auth: &ServerAuth, path: &str, query: &str, parts: &[&str], @@ -25,10 +36,87 @@ pub(super) fn handle_system_route( } if request.method == "GET" && path == "/v1/status" { + // Public status performs the authority dispatch. Never turn a missing + // structural dependency into a legacy full-scan success: qualification + // failures must remain fail-closed. let report = db.status(None)?; return Ok(Some(utils::json_response(200, "OK", &report)?)); } + if request.method == "POST" && path == "/v1/index/reconcile" { + let body: crate::server::request_types::IndexReconcileRequest = if request.body.is_empty() { + Default::default() + } else { + serde_json::from_slice(&request.body)? + }; + let report = super::super::reconcile_changed_path_ledger(db, body.lane.as_deref())?; + return Ok(Some(utils::json_response(200, "OK", &report)?)); + } + + if request.method == "POST" + && matches!( + path, + "/v1/ledger/challenge" | "/v1/ledger/fence" | "/v1/ledger/reconcile" + ) + { + let body: LedgerFenceRequest = serde_json::from_slice(&request.body)?; + let expected_identity = auth.daemon_identity.as_ref(); + if body.protocol_version != 2 + || expected_identity.is_none_or(|identity| { + identity.owner_nonce != body.owner_nonce + || identity.workspace_identity != body.workspace_identity + || identity.executable_identity != body.executable_identity + }) + { + return Err(Error::DaemonUnavailable( + "changed-path ledger RPC identity mismatch".into(), + )); + } + let proof = if path == "/v1/ledger/challenge" { + let proof = super::super::workspace_changed_path_ready_proof(db)?; + if proof.scope_id != body.scope_id || proof.epoch != body.expected_epoch { + return Err(Error::DaemonUnavailable( + "changed-path daemon challenge scope or epoch mismatch".into(), + )); + } + proof + } else if path == "/v1/ledger/reconcile" { + super::super::workspace_changed_path_reconcile( + db, + Some(&body.scope_id), + Some(body.expected_epoch), + )? + } else { + super::super::workspace_changed_path_fence( + db, + Some(&body.scope_id), + Some(body.expected_epoch), + )? + }; + return Ok(Some(utils::json_response( + 200, + "OK", + &serde_json::json!({ + "pid": std::process::id(), + "process_start_identity": expected_identity + .map(|identity| identity.process_start_identity.as_str()) + .unwrap_or_default(), + "executable_identity": expected_identity + .map(|identity| identity.executable_identity.as_str()) + .unwrap_or_default(), + "owner_nonce": body.owner_nonce, + "workspace_identity": body.workspace_identity, + "scope_id": proof.scope_id, + "epoch": proof.epoch, + "daemon_launch_nonce": proof.daemon_launch_nonce, + "live_fence_sequence": proof.sequence, + "durable_offset": proof.durable_offset, + "folded_offset": proof.folded_offset, + "protocol_version": 2 + }), + )?)); + } + if request.method == "POST" && path == "/v1/record" { use crate::server::request_types::RecordRequest; let body: RecordRequest = if request.body.is_empty() { diff --git a/trail/src/server/route/utils.rs b/trail/src/server/route/utils.rs index e70b90d..001b38b 100644 --- a/trail/src/server/route/utils.rs +++ b/trail/src/server/route/utils.rs @@ -38,11 +38,11 @@ pub(crate) fn parse_patch_request(body: &[u8]) -> Result { let request: ApiPatchRequest = serde_json::from_slice(body)?; validate_external_patch_edit_sources( "patch request", - request.edits.len(), - request.files.len(), + request.edits.is_some(), + request.files.is_some(), )?; - let mut edits = request.edits; - for file in request.files { + let mut edits = request.edits.unwrap_or_default(); + for file in request.files.unwrap_or_default() { match file { ApiPatchFile::AddText { path, @@ -100,11 +100,15 @@ mod tests { use super::*; #[test] - fn parse_patch_request_rejects_empty_or_ambiguous_edit_sources() { - let empty = parse_patch_request(br#"{"message":"empty"}"#).unwrap_err(); - assert!(empty + fn parse_patch_request_accepts_explicit_empty_source_and_rejects_missing_or_ambiguous_sources() + { + let empty = parse_patch_request(br#"{"message":"empty","edits":[]}"#).unwrap(); + assert!(empty.edits.is_empty()); + + let missing = parse_patch_request(br#"{"message":"missing"}"#).unwrap_err(); + assert!(missing .to_string() - .contains("requires at least one edit in `edits` or `files`")); + .contains("requires exactly one explicit edit source")); let ambiguous = parse_patch_request( br#"{ @@ -148,7 +152,7 @@ mod tests { || message.contains("unknown variant") || message.contains("missing field") || message.contains("invalid type") - || message.contains("requires at least one edit") + || message.contains("requires exactly one explicit edit source") || message.contains("must use either `edits` or `files`"), "unexpected parse error for seed {seed}: {message}" ); @@ -228,7 +232,7 @@ mod tests { "path": "README.md", "edits": [{ "type": "modify_line", - "line_id": format!("ch_seed_{seed}:1"), + "line_id": format!("line_seed_{seed}:1"), "expected_text": format!("old-{seed}"), "new_text": format!("new-{seed}") }] @@ -450,28 +454,12 @@ fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { } pub(crate) fn error_response(err: &Error) -> HttpResponse { - let status = match err { - Error::RefNotFound(_) | Error::OperationNotFound(_) | Error::RootNotFound(_) => 404, - Error::Conflict(_) - | Error::DirtyWorktree - | Error::DirtyWorktreeWithMessage(_) - | Error::PatchRejected(_) - | Error::StaleBranch(_) - | Error::WorkspaceLocked(_) => 409, - Error::InvalidInput(_) - | Error::InvalidPath { .. } - | Error::IgnoredPath(_) - | Error::Json(_) => 400, - _ => 500, - }; + let structured = crate::model::StructuredErrorEnvelope::from_error(err); + let status = structured.error.status; let reason = reason_for_status(status); - let body = serde_json::to_vec(&ErrorBody { - error: ErrorDetails { - message: err.to_string(), - code: err.exit_code(), - }, - }) - .unwrap_or_else(|_| b"{\"error\":{\"message\":\"serialization failed\",\"code\":1}}".to_vec()); + let body = serde_json::to_vec(&structured).unwrap_or_else(|_| { + b"{\"error\":{\"message\":\"serialization failed\",\"code\":1}}".to_vec() + }); HttpResponse { status, reason, @@ -480,61 +468,50 @@ pub(crate) fn error_response(err: &Error) -> HttpResponse { } } -#[derive(Serialize)] -pub(crate) struct ErrorBody { - error: ErrorDetails, -} - -#[derive(Debug, Serialize)] -pub(crate) struct ErrorDetails { - pub(crate) message: String, - pub(crate) code: i32, -} - pub(crate) fn unauthorized_response() -> HttpResponse { - let body = serde_json::to_vec(&ErrorBody { - error: ErrorDetails { - message: "unauthorized: missing or invalid Trail daemon token".to_string(), - code: 11, - }, - }) - .unwrap_or_else(|_| b"{\"error\":{\"message\":\"unauthorized\",\"code\":11}}".to_vec()); - HttpResponse { - status: 401, - reason: "Unauthorized", - extra_headers: Vec::new(), - body, - } + static_error_response( + 401, + "UNAUTHORIZED", + 11, + "unauthorized: missing or invalid Trail daemon token", + ) } pub(crate) fn forbidden_origin_response() -> HttpResponse { - let body = serde_json::to_vec(&ErrorBody { - error: ErrorDetails { - message: "forbidden: request origin is not a local loopback origin".to_string(), - code: 11, - }, - }) - .unwrap_or_else(|_| b"{\"error\":{\"message\":\"forbidden\",\"code\":11}}".to_vec()); - HttpResponse { - status: 403, - reason: "Forbidden", - extra_headers: Vec::new(), - body, - } + static_error_response( + 403, + "FORBIDDEN_ORIGIN", + 11, + "forbidden: request origin is not a local loopback origin", + ) } pub(crate) fn forbidden_host_response() -> HttpResponse { - let body = serde_json::to_vec(&ErrorBody { - error: ErrorDetails { - message: "forbidden: request host is missing or is not a local loopback host" - .to_string(), - code: 11, - }, - }) - .unwrap_or_else(|_| b"{\"error\":{\"message\":\"forbidden\",\"code\":11}}".to_vec()); + static_error_response( + 403, + "FORBIDDEN_HOST", + 11, + "forbidden: request host is missing or is not a local loopback host", + ) +} + +fn static_error_response(status: u16, code: &str, exit: i32, message: &str) -> HttpResponse { + let body = serde_json::to_vec(&serde_json::json!({ + "error": { + "code": code, + "status": status, + "exit": exit, + "message": message, + "scope": null, + "state": null, + "reason": null, + "recovery": null + } + })) + .unwrap_or_else(|_| b"{\"error\":{\"code\":\"SERIALIZATION_ERROR\"}}".to_vec()); HttpResponse { - status: 403, - reason: "Forbidden", + status, + reason: reason_for_status(status), extra_headers: Vec::new(), body, } diff --git a/trail/src/server/transport.rs b/trail/src/server/transport.rs index 4401214..8d7ba95 100644 --- a/trail/src/server/transport.rs +++ b/trail/src/server/transport.rs @@ -1,8 +1,13 @@ use std::collections::BTreeMap; use std::io::{BufReader, ErrorKind, Read, Write}; use std::net::{IpAddr, SocketAddr, TcpListener, TcpStream}; +#[cfg(unix)] +use std::os::unix::net::{UnixListener, UnixStream}; use std::time::{Duration, Instant}; +#[cfg(debug_assertions)] +use std::sync::atomic::{AtomicBool, Ordering}; + use serde::Deserialize; use serde_json::json; @@ -15,6 +20,9 @@ const HTTP_CONNECTION_TIMEOUT: Duration = Duration::from_secs(30); const DEFAULT_RATE_LIMIT_REQUESTS: usize = 600; const DEFAULT_RATE_LIMIT_WINDOW: Duration = Duration::from_secs(60); +#[cfg(debug_assertions)] +static TEST_DAEMON_RESPONSE_DELAY_USED: AtomicBool = AtomicBool::new(false); + #[derive(Debug)] pub(crate) struct HttpRequest { pub(crate) method: String, @@ -23,14 +31,39 @@ pub(crate) struct HttpRequest { pub(crate) body: Vec, } +#[derive(Clone, Debug)] +pub struct DaemonServerIdentity { + pub(crate) owner_nonce: String, + pub(crate) workspace_identity: String, + pub(crate) executable_identity: String, + pub(crate) process_start_identity: String, +} + +impl DaemonServerIdentity { + pub fn new( + owner_nonce: impl Into, + workspace_identity: impl Into, + executable_identity: impl Into, + process_start_identity: impl Into, + ) -> Self { + Self { + owner_nonce: owner_nonce.into(), + workspace_identity: workspace_identity.into(), + executable_identity: executable_identity.into(), + process_start_identity: process_start_identity.into(), + } + } +} + #[derive(Clone, Debug, Default)] pub struct ServerAuth { pub(crate) token: Option, + pub(crate) daemon_identity: Option, } impl ServerAuth { pub fn disabled() -> Self { - Self { token: None } + Self::default() } pub fn bearer(token: impl Into) -> Result { @@ -40,7 +73,15 @@ impl ServerAuth { "daemon auth token cannot be empty".to_string(), )); } - Ok(Self { token: Some(token) }) + Ok(Self { + token: Some(token), + daemon_identity: None, + }) + } + + pub fn with_daemon_identity(mut self, identity: DaemonServerIdentity) -> Self { + self.daemon_identity = Some(identity); + self } pub fn is_required(&self) -> bool { @@ -170,6 +211,65 @@ pub fn serve_listener_with_auth_rate_limit_and_timeout( Ok(()) } +#[cfg(unix)] +pub fn serve_unix_listener_with_auth_and_timeout( + db: &mut Trail, + listener: UnixListener, + auth: ServerAuth, + connection_timeout: Duration, +) -> Result<()> { + if connection_timeout.is_zero() { + return Err(Error::InvalidInput( + "connection timeout must be greater than zero".into(), + )); + } + loop { + let (mut stream, _) = listener.accept()?; + let _ = handle_unix_connection(db, &mut stream, &auth, connection_timeout); + if std::env::var_os("TRAIL_WORKSPACE_DAEMON").is_some() { + if super::workspace_changed_path_ready_proof(db).is_err() { + std::thread::sleep(Duration::from_millis(10)); + if super::workspace_changed_path_ready_proof(db).is_err() { + return Err(Error::DaemonUnavailable( + "workspace daemon observer health retirement requested".into(), + )); + } + } + } + } +} + +#[cfg(unix)] +fn handle_unix_connection( + db: &mut Trail, + stream: &mut UnixStream, + auth: &ServerAuth, + connection_timeout: Duration, +) -> Result<()> { + stream.set_read_timeout(Some(connection_timeout))?; + stream.set_write_timeout(Some(connection_timeout))?; + let request = match read_unix_request(stream) { + Ok(request) => request, + Err(err) => { + let response = request_read_error_response(&err, connection_timeout); + let _ = stream + .write_all(&response.to_http_bytes()) + .and_then(|_| stream.flush()); + return Ok(()); + } + }; + #[cfg(debug_assertions)] + let request_method = request.method.clone(); + #[cfg(debug_assertions)] + let request_path = request.path.clone(); + let response = route::route_request(db, request, auth); + #[cfg(debug_assertions)] + delay_daemon_response_for_test(&request_method, &request_path); + stream.write_all(&response.to_http_bytes())?; + stream.flush()?; + Ok(()) +} + pub fn handle_http_request(db: &mut Trail, raw: &[u8]) -> HttpResponse { handle_http_request_with_auth(db, raw, &ServerAuth::disabled()) } @@ -211,12 +311,36 @@ fn handle_connection( stream.flush()?; return Ok(()); } + #[cfg(debug_assertions)] + let request_method = request.method.clone(); + #[cfg(debug_assertions)] + let request_path = request.path.clone(); let response = route::route_request(db, request, auth); + #[cfg(debug_assertions)] + delay_daemon_response_for_test(&request_method, &request_path); stream.write_all(&response.to_http_bytes())?; stream.flush()?; Ok(()) } +#[cfg(debug_assertions)] +fn delay_daemon_response_for_test(request_method: &str, request_path: &str) { + let Some(expected_path) = std::env::var_os("TRAIL_TEST_DAEMON_RESPONSE_DELAY_PATH") else { + return; + }; + if request_method != "POST" + || request_path != expected_path + || TEST_DAEMON_RESPONSE_DELAY_USED.swap(true, Ordering::AcqRel) + { + return; + } + let millis = std::env::var("TRAIL_TEST_DAEMON_RESPONSE_DELAY_MS") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + std::thread::sleep(Duration::from_millis(millis)); +} + struct HttpRateLimiter { config: ServerRateLimit, peers: BTreeMap, @@ -341,6 +465,67 @@ fn read_request(stream: &mut TcpStream) -> Result { parse_request_parts(&first_line, headers, body) } +#[cfg(unix)] +fn read_unix_request(stream: &mut UnixStream) -> Result { + let mut reader = BufReader::new(stream.try_clone()?); + read_request_from(&mut reader) +} + +fn read_request_from(reader: &mut R) -> Result { + let mut bytes_read = 0usize; + let first_line = read_http_line_limited(reader, &mut bytes_read)?; + if first_line.trim().is_empty() { + return Err(Error::InvalidInput("empty HTTP request".to_string())); + } + let mut content_length = 0usize; + let mut headers = BTreeMap::new(); + loop { + let line = read_http_line_limited(reader, &mut bytes_read)?; + if line == "\r\n" || line == "\n" || line.is_empty() { + break; + } + let (name, value) = line + .split_once(':') + .ok_or_else(|| Error::InvalidInput("malformed HTTP request header".to_string()))?; + let name = name.trim().to_ascii_lowercase(); + let value = value.trim().to_string(); + if name == "content-length" { + content_length = value + .parse::() + .map_err(|_| Error::InvalidInput("invalid HTTP Content-Length".to_string()))?; + } + headers.insert(name, value); + } + if bytes_read.saturating_add(content_length) > MAX_HTTP_REQUEST_BYTES { + return Err(Error::InvalidInput( + "HTTP request exceeds size limit".to_string(), + )); + } + let mut body = vec![0_u8; content_length]; + reader.read_exact(&mut body)?; + let mut parts = first_line.split_whitespace(); + let method = parts + .next() + .ok_or_else(|| Error::InvalidInput("HTTP request method is missing".to_string()))?; + let path = parts + .next() + .ok_or_else(|| Error::InvalidInput("HTTP request path is missing".to_string()))?; + let version = parts + .next() + .ok_or_else(|| Error::InvalidInput("HTTP version is missing".to_string()))?; + if parts.next().is_some() || !version.starts_with("HTTP/") { + return Err(Error::InvalidInput( + "malformed HTTP request line".to_string(), + )); + } + Ok(HttpRequest { + method: method.to_string(), + path: path.to_string(), + headers, + body, + }) +} + fn read_http_line_limited(reader: &mut R, bytes_read: &mut usize) -> Result { let mut line = Vec::new(); let mut terminated = false; diff --git a/trail/tests/acp_capture_journal.rs b/trail/tests/acp_capture_journal.rs new file mode 100644 index 0000000..42c7fa6 --- /dev/null +++ b/trail/tests/acp_capture_journal.rs @@ -0,0 +1,328 @@ +#![cfg(unix)] + +use std::fs; +use std::io::{BufRead, BufReader, Read, Write}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use trail::{AgentCaptureTransport, AgentHookReceiptInput, InitImportMode, Trail}; + +fn trail_bin() -> PathBuf { + std::env::var_os("TRAIL_TEST_BIN") + .map(PathBuf::from) + .or_else(|| option_env!("CARGO_BIN_EXE_trail").map(PathBuf::from)) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/debug/trail")) +} + +fn write_echo_agent(workspace: &Path) -> PathBuf { + let agent = workspace.join("capture-echo-agent.sh"); + fs::write( + &agent, + r#"#!/bin/sh +set -eu +IFS= read -r initialize +printf '%s\n' '{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{}}}' +IFS= read -r session +printf '%s\n' '{"jsonrpc":"2.0","id":2,"result":{"sessionId":"capture-session"}}' +IFS= read -r prompt +printf '%s\n' "$prompt" +exec /bin/cat +"#, + ) + .unwrap(); + let mut permissions = fs::metadata(&agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&agent, permissions).unwrap(); + agent +} + +fn write_initialize_agent(workspace: &Path) -> PathBuf { + let agent = workspace.join("capture-initialize-agent.sh"); + fs::write( + &agent, + r#"#!/bin/sh +set -eu +IFS= read -r initialize +printf '%s\n' '{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{}}}' +exec /bin/cat +"#, + ) + .unwrap(); + let mut permissions = fs::metadata(&agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&agent, permissions).unwrap(); + agent +} + +fn receipt_payload(connection_id: &str, sequence: u64) -> serde_json::Value { + serde_json::json!({ + "connection_id": connection_id, + "direction": "agent_to_client", + "sequence": sequence, + "received_at": i64::try_from(sequence).unwrap() + 1, + "message": { + "jsonrpc": "2.0", + "method": "session/update", + "params": {"sequence": sequence} + }, + "project": false + }) +} + +fn connection_receipts(workspace: &Path, connection_id: &str) -> Vec { + let db = Trail::open(workspace).unwrap(); + let mut receipts = Vec::new(); + for offset in (0..3_000).step_by(1_000) { + let page = db + .list_agent_hook_receipts_page(Some("trail-acp"), None, offset, 1_000) + .unwrap(); + let page_len = page.len(); + receipts.extend( + page.into_iter() + .filter(|receipt| receipt.connection_id.as_deref() == Some(connection_id)), + ); + if page_len < 1_000 { + break; + } + } + receipts +} + +#[test] +fn forwarding_never_waits_for_database_writer() { + const FRAME_COUNT: usize = 1_000; + + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "capture fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let agent = write_echo_agent(temp.path()); + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["acp", "relay", "--"]) + .arg(agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + writeln!(stdin, r#"{{"jsonrpc":"2.0","id":1,"method":"initialize","params":{{"protocolVersion":1,"clientCapabilities":{{}}}}}}"#).unwrap(); + let mut initialize_response = String::new(); + stdout.read_line(&mut initialize_response).unwrap(); + assert!(initialize_response.contains(r#""protocolVersion":1"#)); + writeln!( + stdin, + r#"{{"jsonrpc":"2.0","id":2,"method":"session/new","params":{{"cwd":"{}","mcpServers":[]}}}}"#, + temp.path().display() + ) + .unwrap(); + let mut session_response = String::new(); + stdout.read_line(&mut session_response).unwrap(); + assert!(session_response.contains("capture-session")); + + let lock = temp.path().join(".trail/lock"); + let lock_deadline = Instant::now() + Duration::from_secs(5); + loop { + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&lock) + { + Ok(mut file) => { + writeln!( + file, + "pid={} created_at={}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + ) + .unwrap(); + break; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + assert!( + Instant::now() < lock_deadline, + "writer lock never became idle" + ); + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("failed to acquire test writer lock: {error}"), + } + } + writeln!(stdin, r#"{{"jsonrpc":"2.0","id":3,"method":"session/prompt","params":{{"sessionId":"capture-session","prompt":[{{"type":"text","text":"stress capture"}}]}}}}"#).unwrap(); + let mut prompt_echo = String::new(); + stdout.read_line(&mut prompt_echo).unwrap(); + assert!(prompt_echo.contains(r#""id":3"#)); + + let started = Instant::now(); + let writer = thread::spawn(move || { + for sequence in 0..FRAME_COUNT { + writeln!( + stdin, + r#"{{"jsonrpc":"2.0","method":"session/update","params":{{"sessionId":"capture-session","update":{{"sessionUpdate":"tool_call_update","toolCallId":"tool-{sequence}","status":"completed"}}}}}}"# + ) + .unwrap(); + } + stdin + }); + let (done_tx, done_rx) = mpsc::channel(); + let reader = thread::spawn(move || { + let mut line = String::new(); + for _ in 0..FRAME_COUNT { + line.clear(); + stdout.read_line(&mut line).unwrap(); + assert!(!line.is_empty()); + } + done_tx.send(started.elapsed()).unwrap(); + }); + + let within_budget = done_rx.recv_timeout(Duration::from_millis(250)).ok(); + fs::remove_file(&lock).unwrap(); + let stdin = writer.join().unwrap(); + reader.join().unwrap(); + let shutdown_started = Instant::now(); + drop(stdin); + let mut stderr = String::new(); + child + .stderr + .take() + .unwrap() + .read_to_string(&mut stderr) + .unwrap(); + let status = child.wait().unwrap(); + assert!(status.success(), "relay failed\nstderr:\n{stderr}"); + assert!( + shutdown_started.elapsed() < Duration::from_millis(2_500), + "capture shutdown exceeded its bounded drain budget: {:?}", + shutdown_started.elapsed() + ); + let elapsed = within_budget.unwrap_or_else(|| started.elapsed()); + assert!( + elapsed < Duration::from_millis(250), + "forwarding waited for Trail's writer lock: {elapsed:?}" + ); +} + +#[test] +fn pending_receipts_and_spill_replay_once_in_connection_order() { + const DURABLE_COUNT: u64 = 500; + const SPILL_COUNT: u64 = 500; + const TOTAL: usize = (DURABLE_COUNT + SPILL_COUNT) as usize; + const CONNECTION_ID: &str = "recovery-connection"; + + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "capture recovery fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + for sequence in 0..DURABLE_COUNT { + db.persist_agent_hook_receipt(AgentHookReceiptInput { + installation_id: None, + provider: "trail-acp".to_string(), + native_event: "acp/frame".to_string(), + native_session_id: None, + native_turn_id: None, + transport: AgentCaptureTransport::Acp, + connection_id: Some(CONNECTION_ID.to_string()), + direction: Some("agent_to_client".to_string()), + connection_sequence: Some(sequence), + dedupe_key: format!("acp:{CONNECTION_ID}:agent_to_client:{sequence}"), + payload: receipt_payload(CONNECTION_ID, sequence), + occurred_at: Some(i64::try_from(sequence).unwrap() + 1), + }) + .unwrap(); + } + drop(db); + + let spill_dir = temp.path().join(".trail/acp-ingress"); + fs::create_dir_all(&spill_dir).unwrap(); + let mut spill = String::new(); + for sequence in DURABLE_COUNT..(DURABLE_COUNT + SPILL_COUNT) { + let frame = serde_json::json!({ + "connection_id": CONNECTION_ID, + "direction": "AgentToClient", + "sequence": sequence, + "received_at": i64::try_from(sequence).unwrap() + 1, + "redacted_message": { + "jsonrpc": "2.0", + "method": "session/update", + "params": {"sequence": sequence} + }, + "project": false + }); + let line = serde_json::to_string(&frame).unwrap(); + spill.push_str(&line); + spill.push('\n'); + spill.push_str(&line); + spill.push('\n'); + } + fs::write(spill_dir.join(format!("{CONNECTION_ID}.jsonl")), spill).unwrap(); + + let agent = write_initialize_agent(temp.path()); + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["acp", "relay", "--"]) + .arg(agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + writeln!(stdin, r#"{{"jsonrpc":"2.0","id":1,"method":"initialize","params":{{"protocolVersion":1,"clientCapabilities":{{}}}}}}"#).unwrap(); + let mut initialize_response = String::new(); + stdout.read_line(&mut initialize_response).unwrap(); + assert!(initialize_response.contains(r#""protocolVersion":1"#)); + + let deadline = Instant::now() + Duration::from_secs(45); + let receipts = loop { + let receipts = connection_receipts(temp.path(), CONNECTION_ID); + if receipts.len() == TOTAL && receipts.iter().all(|receipt| receipt.status == "processed") { + break receipts; + } + assert!( + Instant::now() < deadline, + "timed out replaying capture journal: {} of {TOTAL} receipts", + receipts.len() + ); + thread::sleep(Duration::from_millis(50)); + }; + let mut sequences = receipts + .iter() + .map(|receipt| receipt.connection_sequence.unwrap()) + .collect::>(); + sequences.sort_unstable(); + assert_eq!( + sequences, + (0..u64::try_from(TOTAL).unwrap()).collect::>() + ); + assert_eq!( + receipts + .iter() + .map(|receipt| receipt.dedupe_key.as_str()) + .collect::>() + .len(), + TOTAL + ); + + drop(stdin); + assert!(child.wait().unwrap().success()); + let remaining_spill = fs::read_dir(spill_dir) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.metadata().is_ok_and(|metadata| metadata.len() > 0)) + .count(); + assert_eq!( + remaining_spill, 0, + "acknowledged spill files remain non-empty" + ); +} diff --git a/trail/tests/acp_client_callbacks.rs b/trail/tests/acp_client_callbacks.rs new file mode 100644 index 0000000..c9a359a --- /dev/null +++ b/trail/tests/acp_client_callbacks.rs @@ -0,0 +1,403 @@ +#![cfg(unix)] + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use serde_json::Value; +use trail::{InitImportMode, Trail}; + +fn trail_bin() -> PathBuf { + std::env::var_os("TRAIL_TEST_BIN") + .map(PathBuf::from) + .or_else(|| option_env!("CARGO_BIN_EXE_trail").map(PathBuf::from)) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/debug/trail")) +} + +fn write_callback_agent(workspace: &Path) -> PathBuf { + let agent = workspace.join("callback-agent.py"); + fs::write( + &agent, + r#"#!/usr/bin/env python3 +import json +import sys + +session_id = "callback-session" +effective_cwd = None +prompt_id = None +phase_one = {"perm-selected", "perm-cancelled", "perm-error", "read-ok", "read-error", "write-ok", "write-error", "create-ok", "create-error", "create-abandoned"} +phase_two = {"output-running", "output-exited", "output-signal", "output-error", "wait-ok", "wait-error", "kill-ok", "kill-error", "release-ok", "release-error"} +seen_one = set() +seen_two = set() + +def emit(request_id, method, params): + print(json.dumps({"jsonrpc":"2.0","id":request_id,"method":method,"params":params}, separators=(",", ":")), flush=True) + +for line in sys.stdin: + message = json.loads(line) + method = message.get("method") + request_id = message.get("id") + if method == "initialize": + print(json.dumps({"jsonrpc":"2.0","id":request_id,"result":{"protocolVersion":1,"agentCapabilities":{}}}, separators=(",", ":")), flush=True) + elif method == "session/new": + effective_cwd = message["params"]["cwd"] + print(json.dumps({"jsonrpc":"2.0","id":request_id,"result":{"sessionId":session_id}}, separators=(",", ":")), flush=True) + elif method == "session/prompt": + prompt_id = request_id + options = [ + {"optionId":"allow-once","name":"Allow once","kind":"allow_once"}, + {"optionId":"allow-always","name":"Allow always","kind":"allow_always"}, + {"optionId":"reject-once","name":"Reject once","kind":"reject_once"}, + {"optionId":"reject-always","name":"Reject always","kind":"reject_always"} + ] + tool = {"toolCallId":"permission-tool","title":"Permission matrix"} + emit("perm-selected", "session/request_permission", {"sessionId":session_id,"toolCall":tool,"options":options}) + emit(101, "session/request_permission", {"sessionId":session_id,"toolCall":tool,"options":options}) + emit("perm-error", "session/request_permission", {"sessionId":session_id,"toolCall":tool,"options":options}) + emit("read-ok", "fs/read_text_file", {"sessionId":session_id,"path":effective_cwd + "/read.txt","line":2,"limit":5}) + emit(202, "fs/read_text_file", {"sessionId":session_id,"path":effective_cwd + "/missing.txt"}) + emit("write-ok", "fs/write_text_file", {"sessionId":session_id,"path":effective_cwd + "/write.txt","content":"safe\napi_key=write-secret\n"}) + emit(303, "fs/write_text_file", {"sessionId":session_id,"path":effective_cwd + "/denied.txt","content":"password=denied-secret"}) + terminal = {"sessionId":session_id,"command":"printf","args":["--","hello"],"env":[{"name":"API_TOKEN","value":"terminal-secret"}],"cwd":effective_cwd,"outputByteLimit":16} + emit("create-ok", "terminal/create", terminal) + emit(404, "terminal/create", terminal) + emit("create-abandoned", "terminal/create", terminal) + print(json.dumps({"jsonrpc":"2.0","method":"session/update","params":{"sessionId":session_id,"update":{"sessionUpdate":"usage_update","used":1,"size":100}}}, separators=(",", ":")), flush=True) + elif method is None: + key = str(request_id) + aliases = {"101":"perm-cancelled", "202":"read-error", "303":"write-error", "404":"create-error"} + key = aliases.get(key, key) + if key in phase_one: + seen_one.add(key) + if seen_one == phase_one: + emit("output-running", "terminal/output", {"sessionId":session_id,"terminalId":"terminal-1"}) + emit(505, "terminal/output", {"sessionId":session_id,"terminalId":"terminal-1"}) + emit("output-signal", "terminal/output", {"sessionId":session_id,"terminalId":"terminal-1"}) + emit("output-error", "terminal/output", {"sessionId":session_id,"terminalId":"terminal-1"}) + emit("kill-ok", "terminal/kill", {"sessionId":session_id,"terminalId":"terminal-1"}) + emit(707, "terminal/kill", {"sessionId":session_id,"terminalId":"terminal-1"}) + emit("wait-ok", "terminal/wait_for_exit", {"sessionId":session_id,"terminalId":"terminal-1"}) + emit(606, "terminal/wait_for_exit", {"sessionId":session_id,"terminalId":"terminal-1"}) + emit("release-ok", "terminal/release", {"sessionId":session_id,"terminalId":"terminal-1"}) + emit(808, "terminal/release", {"sessionId":session_id,"terminalId":"terminal-1"}) + else: + aliases = {"505":"output-exited", "606":"wait-error", "707":"kill-error", "808":"release-error"} + key = aliases.get(key, key) + if key in phase_two: + seen_two.add(key) + if seen_two == phase_two: + emit("shutdown-pending", "fs/read_text_file", {"sessionId":session_id,"path":effective_cwd + "/pending.txt"}) + print(json.dumps({"jsonrpc":"2.0","id":prompt_id,"result":{"stopReason":"end_turn"}}, separators=(",", ":")), flush=True) + sys.exit(0) +"#, + ) + .unwrap(); + let mut permissions = fs::metadata(&agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&agent, permissions).unwrap(); + agent +} + +fn send(stdin: &mut impl Write, message: &Value) { + serde_json::to_writer(&mut *stdin, message).unwrap(); + stdin.write_all(b"\n").unwrap(); + stdin.flush().unwrap(); +} + +fn receive(stdout: &mut impl BufRead) -> Value { + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + assert!(!line.is_empty()); + serde_json::from_str(line.trim()).unwrap() +} + +fn exchange(stdin: &mut impl Write, stdout: &mut impl BufRead, message: Value) -> Value { + send(stdin, &message); + receive(stdout) +} + +fn error_response(id: Value) -> Value { + serde_json::json!({"jsonrpc":"2.0","id":id,"error":{"code":-32001,"message":"fixture rejection"}}) +} + +#[test] +fn all_client_callbacks_preserve_correlation_paths_artifacts_and_terminal_lifecycle() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "callback fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let agent = write_callback_agent(temp.path()); + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["acp", "relay", "--"]) + .arg(agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + + exchange( + &mut stdin, + &mut stdout, + serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":1,"clientCapabilities":{"fs":{"readTextFile":true,"writeTextFile":true},"terminal":true}}}), + ); + exchange( + &mut stdin, + &mut stdout, + serde_json::json!({"jsonrpc":"2.0","id":2,"method":"session/new","params":{"cwd":temp.path(),"mcpServers":[]}}), + ); + send( + &mut stdin, + &serde_json::json!({"jsonrpc":"2.0","id":3,"method":"session/prompt","params":{"sessionId":"callback-session","prompt":[{"type":"text","text":"Run callback matrix"}]}}), + ); + + let mut seen_methods = BTreeSet::new(); + loop { + let message = receive(&mut stdout); + if message.get("id") == Some(&Value::from(3)) { + break; + } + let method = message["method"].as_str().unwrap(); + if method == "session/update" { + continue; + } + seen_methods.insert(method.to_string()); + let id = message["id"].clone(); + if matches!(method, "fs/read_text_file" | "fs/write_text_file") { + assert!(message["params"]["path"] + .as_str() + .unwrap() + .starts_with(temp.path().to_string_lossy().as_ref())); + } + if method == "terminal/create" { + assert_eq!( + message["params"]["cwd"], + temp.path().to_string_lossy().as_ref() + ); + assert_eq!(message["params"]["env"][0]["value"], "terminal-secret"); + } + let response = match (method, id.as_str(), id.as_i64()) { + ("session/request_permission", Some("perm-selected"), _) => { + serde_json::json!({"jsonrpc":"2.0","id":id,"result":{"outcome":{"outcome":"selected","optionId":"allow-once"}}}) + } + ("session/request_permission", _, Some(101)) => { + serde_json::json!({"jsonrpc":"2.0","id":id,"result":{"outcome":{"outcome":"cancelled"}}}) + } + ("session/request_permission", _, _) => error_response(id), + ("fs/read_text_file", Some("read-ok"), _) => { + serde_json::json!({"jsonrpc":"2.0","id":id,"result":{"content":"line 2\ntoken=read-secret\n"}}) + } + ("fs/read_text_file", Some("shutdown-pending"), _) => continue, + ("fs/read_text_file", _, _) => error_response(id), + ("fs/write_text_file", Some("write-ok"), _) => { + serde_json::json!({"jsonrpc":"2.0","id":id,"result":{}}) + } + ("fs/write_text_file", _, _) => error_response(id), + ("terminal/create", Some("create-ok"), _) => { + serde_json::json!({"jsonrpc":"2.0","id":id,"result":{"terminalId":"terminal-1"}}) + } + ("terminal/create", Some("create-abandoned"), _) => { + serde_json::json!({"jsonrpc":"2.0","id":id,"result":{"terminalId":"terminal-2"}}) + } + ("terminal/create", _, _) => error_response(id), + ("terminal/output", Some("output-running"), _) => { + serde_json::json!({"jsonrpc":"2.0","id":id,"result":{"output":"0123456789abcdef","truncated":true,"exitStatus":null}}) + } + ("terminal/output", _, Some(505)) => { + serde_json::json!({"jsonrpc":"2.0","id":id,"result":{"output":"done","truncated":false,"exitStatus":{"exitCode":0,"signal":null}}}) + } + ("terminal/output", Some("output-signal"), _) => { + serde_json::json!({"jsonrpc":"2.0","id":id,"result":{"output":"killed","truncated":false,"exitStatus":{"exitCode":null,"signal":"SIGTERM"}}}) + } + ("terminal/output", _, _) => error_response(id), + ("terminal/wait_for_exit", Some("wait-ok"), _) => { + serde_json::json!({"jsonrpc":"2.0","id":id,"result":{"exitCode":null,"signal":"SIGKILL"}}) + } + ("terminal/wait_for_exit", _, _) => error_response(id), + ("terminal/kill", Some("kill-ok"), _) => { + serde_json::json!({"jsonrpc":"2.0","id":id,"result":{}}) + } + ("terminal/kill", _, _) => error_response(id), + ("terminal/release", Some("release-ok"), _) => { + serde_json::json!({"jsonrpc":"2.0","id":id,"result":{}}) + } + ("terminal/release", _, _) => error_response(id), + _ => panic!("unexpected callback: {message}"), + }; + send(&mut stdin, &response); + } + drop(stdin); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "relay failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + seen_methods, + [ + "fs/read_text_file", + "fs/write_text_file", + "session/request_permission", + "terminal/create", + "terminal/kill", + "terminal/output", + "terminal/release", + "terminal/wait_for_exit", + ] + .into_iter() + .map(str::to_string) + .collect() + ); + + let db = Trail::open(temp.path()).unwrap(); + let mapping = db.lane_acp_session("callback-session").unwrap(); + let session = db.show_lane_session(&mapping.trail_session_id).unwrap(); + let requested = session + .events + .iter() + .filter(|event| event.event_type == "acp_client_callback_requested") + .filter_map(|event| event.payload.as_ref()) + .collect::>(); + let finished = session + .events + .iter() + .filter(|event| event.event_type == "acp_client_callback_finished") + .filter_map(|event| event.payload.as_ref()) + .collect::>(); + let requested_methods = requested + .iter() + .filter_map(|payload| payload["method"].as_str()) + .collect::>(); + assert_eq!(requested_methods.len(), 8); + assert!(requested + .iter() + .any(|payload| payload["request_id"].is_string())); + assert!(requested + .iter() + .any(|payload| payload["request_id"].is_number())); + let permission_request = requested + .iter() + .find(|payload| payload["method"] == "session/request_permission") + .unwrap(); + assert_eq!( + permission_request["details"]["option_kinds"] + .as_object() + .unwrap() + .values() + .filter_map(Value::as_str) + .collect::>(), + ["allow_always", "allow_once", "reject_always", "reject_once"] + .into_iter() + .collect() + ); + let terminal_request = requested + .iter() + .find(|payload| payload["method"] == "terminal/create") + .unwrap(); + assert_eq!( + terminal_request["details"]["env_names"], + serde_json::json!(["API_TOKEN"]) + ); + assert!(finished.iter().any(|payload| payload["success"] == true)); + assert!(finished.iter().any(|payload| payload["success"] == false)); + assert!(finished.iter().any(|payload| { + payload["error"]["message"] == "relay shutdown with callback in flight" + })); + assert!(session + .events + .iter() + .any(|event| event.event_type == "acp_terminal_abandoned")); + assert!(session + .events + .iter() + .any(|event| event.event_type == "acp_usage_update")); + + let by_method = finished.iter().fold( + BTreeMap::<&str, (usize, usize)>::new(), + |mut counts, payload| { + let method = payload["method"].as_str().unwrap(); + let entry = counts.entry(method).or_default(); + if payload["success"] == true { + entry.0 += 1; + } else { + entry.1 += 1; + } + counts + }, + ); + for method in [ + "session/request_permission", + "fs/read_text_file", + "fs/write_text_file", + "terminal/create", + "terminal/output", + "terminal/wait_for_exit", + "terminal/kill", + "terminal/release", + ] { + let (success, error) = by_method[method]; + assert!(success > 0, "missing {method} success: {by_method:?}"); + assert!(error > 0, "missing {method} error: {by_method:?}"); + } + + let path_event = requested + .iter() + .find(|payload| payload["method"] == "fs/read_text_file") + .unwrap(); + assert_ne!( + path_event["details"]["effective_path"], + path_event["details"]["forwarded_path"] + ); + let output_event = finished + .iter() + .find(|payload| { + payload["method"] == "terminal/output" && payload["result"]["truncated"] == true + }) + .unwrap(); + assert_eq!(output_event["result"]["output_byte_len"], 16); + assert!(!output_event.to_string().contains("0123456789abcdef")); + let kill_position = finished + .iter() + .position(|payload| { + payload["method"] == "terminal/kill" && payload["request_id"] == "kill-ok" + }) + .unwrap(); + let wait_position = finished + .iter() + .position(|payload| { + payload["method"] == "terminal/wait_for_exit" && payload["request_id"] == "wait-ok" + }) + .unwrap(); + let release_position = finished + .iter() + .position(|payload| { + payload["method"] == "terminal/release" && payload["request_id"] == "release-ok" + }) + .unwrap(); + assert!(kill_position < wait_position); + assert!(wait_position < release_position); + + let artifacts = db + .list_lane_artifacts(&mapping.trail_session_id, None, 20) + .unwrap(); + assert_eq!(artifacts.len(), 2); + for artifact in artifacts { + assert_eq!(artifact.artifact_kind, "acp_fs_write"); + let content = + String::from_utf8(db.lane_artifact_content(&artifact.artifact_id).unwrap()).unwrap(); + assert!(!content.contains("write-secret")); + assert!(!content.contains("denied-secret")); + } + assert!(!session + .events + .iter() + .filter_map(|event| event.payload.as_ref()) + .any(|payload| payload.to_string().contains("terminal-secret"))); +} diff --git a/trail/tests/acp_conformance.rs b/trail/tests/acp_conformance.rs new file mode 100644 index 0000000..7c8fe7e --- /dev/null +++ b/trail/tests/acp_conformance.rs @@ -0,0 +1,893 @@ +#![cfg(any(unix, windows))] + +#[path = "support/acp_harness.rs"] +mod acp_harness; + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::time::{Duration, Instant}; + +use serde::Deserialize; +use serde_json::{json, Map, Value}; + +const META: &str = include_str!("fixtures/acp/v1/meta.json"); +const SCHEMA: &str = include_str!("fixtures/acp/v1/schema.json"); +const METHOD_CASES: &str = include_str!("fixtures/acp/v1/method_cases.json"); +const VARIANT_CASES: &str = include_str!("fixtures/acp/v1/variant_cases.json"); + +#[derive(Debug, Deserialize)] +struct MethodCase { + method: String, + side: String, + envelope: String, +} + +#[derive(Debug, Deserialize)] +struct VariantCase { + union: String, + name: String, + value: Value, +} + +fn strings(values: impl IntoIterator) -> BTreeSet { + values.into_iter().map(str::to_string).collect() +} + +#[test] +fn method_manifest_exactly_matches_the_pinned_v1_metadata() { + let meta: Value = serde_json::from_str(META).unwrap(); + let cases: Vec = serde_json::from_str(METHOD_CASES).unwrap(); + assert_eq!(cases.len(), 23, "ACP v1 has exactly 23 protocol methods"); + + let expected_groups = [ + ("agentMethods", "agent"), + ("clientMethods", "client"), + ("protocolMethods", "protocol"), + ]; + let mut pinned = BTreeMap::new(); + for (group, side) in expected_groups { + for method in meta[group].as_object().unwrap().values() { + let method = method.as_str().unwrap().to_string(); + assert!( + pinned.insert(method, side).is_none(), + "duplicate pinned method" + ); + } + } + + let notification_methods = strings(["session/cancel", "session/update", "$/cancel_request"]); + let mut covered = BTreeMap::new(); + for case in cases { + let expected_side = pinned + .get(&case.method) + .unwrap_or_else(|| panic!("fixture has unpinned method {}", case.method)); + assert_eq!(&case.side, expected_side, "wrong side for {}", case.method); + let expected_envelope = if notification_methods.contains(&case.method) { + "notification" + } else { + "request" + }; + assert_eq!( + case.envelope, expected_envelope, + "wrong envelope for {}", + case.method + ); + assert!( + covered.insert(case.method.clone(), case.side).is_none(), + "duplicate method fixture {}", + case.method + ); + eprintln!("ACP v1 method: {}", case.method); + } + assert_eq!( + covered.keys().collect::>(), + pinned.keys().collect::>() + ); +} + +fn discriminator_names(definition: &Value, property: &str, alternatives: &str) -> BTreeSet { + definition[alternatives] + .as_array() + .unwrap() + .iter() + .map(|branch| { + branch["properties"][property]["const"] + .as_str() + .unwrap() + .to_string() + }) + .collect() +} + +fn const_names(definition: &Value) -> BTreeSet { + definition["oneOf"] + .as_array() + .unwrap() + .iter() + .map(|branch| branch["const"].as_str().unwrap().to_string()) + .collect() +} + +fn pinned_variant_names(schema: &Value, union: &str) -> BTreeSet { + let definition = &schema["$defs"][union]; + match union { + "ContentBlock" | "ToolCallContent" | "SessionConfigOption" => { + discriminator_names(definition, "type", "oneOf") + } + "RequestPermissionOutcome" => discriminator_names(definition, "outcome", "oneOf"), + "SessionUpdate" => discriminator_names(definition, "sessionUpdate", "oneOf"), + "EmbeddedResourceResource" => definition["anyOf"] + .as_array() + .unwrap() + .iter() + .map(|branch| branch["title"].as_str().unwrap().to_string()) + .collect(), + "McpServer" => definition["anyOf"] + .as_array() + .unwrap() + .iter() + .map(|branch| { + branch["properties"]["type"]["const"] + .as_str() + .or_else(|| branch["title"].as_str()) + .unwrap() + .to_string() + }) + .collect(), + "TerminalExitStatus" => strings(["exit_code", "signal"]), + "PermissionOptionKind" + | "ToolCallStatus" + | "ToolKind" + | "PlanEntryPriority" + | "PlanEntryStatus" + | "Role" + | "StopReason" => const_names(definition), + other => panic!("variant inventory extractor missing {other}"), + } +} + +fn validator_for_definition(schema: &Value, definition: &str) -> jsonschema::Validator { + let subschema = json!({ + "$schema": schema["$schema"].clone(), + "$ref": format!("#/$defs/{definition}"), + "$defs": schema["$defs"].clone(), + }); + jsonschema::validator_for(&subschema).unwrap() +} + +fn validate_definition(schema: &Value, definition: &str, value: &Value) { + if let Err(error) = validator_for_definition(schema, definition).validate(value) { + panic!("{definition} fixture is not schema-valid: {error}; value={value}"); + } +} + +#[test] +fn variant_manifest_exactly_matches_and_validates_against_every_stable_union() { + let schema: Value = serde_json::from_str(SCHEMA).unwrap(); + let cases: Vec = serde_json::from_str(VARIANT_CASES).unwrap(); + let required_unions = strings([ + "ContentBlock", + "EmbeddedResourceResource", + "McpServer", + "PermissionOptionKind", + "RequestPermissionOutcome", + "SessionConfigOption", + "SessionUpdate", + "TerminalExitStatus", + "ToolCallContent", + "ToolCallStatus", + "ToolKind", + "PlanEntryPriority", + "PlanEntryStatus", + "Role", + "StopReason", + ]); + let fixture_unions = cases + .iter() + .map(|case| case.union.clone()) + .collect::>(); + assert_eq!(fixture_unions, required_unions); + + for union in required_unions { + let union_cases = cases + .iter() + .filter(|case| case.union == union) + .collect::>(); + let names = union_cases + .iter() + .map(|case| case.name.clone()) + .collect::>(); + assert_eq!( + names, + pinned_variant_names(&schema, &union), + "variant drift in {union}" + ); + assert_eq!( + names.len(), + union_cases.len(), + "duplicate fixture in {union}" + ); + + let validator = validator_for_definition(&schema, &union); + for case in union_cases { + if let Err(error) = validator.validate(&case.value) { + panic!("{}.{} is not schema-valid: {error}", union, case.name); + } + eprintln!("ACP v1 variant: {}.{}", union, case.name); + } + } +} + +fn boolean_combinations(fields: &[&str]) -> Vec { + (0..(1usize << fields.len())) + .map(|bits| { + Value::Object( + fields + .iter() + .enumerate() + .map(|(index, field)| ((*field).to_string(), json!(bits & (1 << index) != 0))) + .collect(), + ) + }) + .collect() +} + +fn session_capability_combinations() -> Vec { + let fields = ["list", "delete", "additionalDirectories", "resume", "close"]; + (0..3usize.pow(fields.len() as u32)) + .map(|mut state| { + let mut object = Map::new(); + for field in fields { + match state % 3 { + 0 => {} + 1 => { + object.insert(field.to_string(), Value::Null); + } + 2 => { + object.insert(field.to_string(), json!({})); + } + _ => unreachable!(), + } + state /= 3; + } + Value::Object(object) + }) + .collect() +} + +#[test] +fn every_capability_shape_round_trips_through_the_real_relay_unchanged() { + let temp = acp_harness::workspace(); + let agent = acp_harness::fixture_agent_command( + temp.path(), + "capability-matrix", + r#"#!/usr/bin/env python3 +import json +import sys + +for line in sys.stdin: + request = json.loads(line) + meta = request["params"]["_meta"] + response = { + "jsonrpc": "2.0", + "id": request["id"], + "result": { + "protocolVersion": 1, + "agentCapabilities": meta["agentCapabilities"], + "_meta": {"clientCapabilities": request["params"]["clientCapabilities"]}, + }, + } + print(json.dumps(response, separators=(",", ":")), flush=True) +"#, + ); + let mut child = acp_harness::spawn_relay(temp.path(), &agent); + let (mut stdin, mut stdout) = acp_harness::relay_stdio(&mut child); + + let mut cases = Vec::new(); + for prompt in boolean_combinations(&["image", "audio", "embeddedContext"]) { + cases.push((json!({}), json!({"promptCapabilities": prompt}))); + } + for mcp in boolean_combinations(&["http", "sse"]) { + cases.push((json!({}), json!({"mcpCapabilities": mcp}))); + } + for client in boolean_combinations(&["readTextFile", "writeTextFile", "terminal"]) { + cases.push(( + json!({ + "fs": { + "readTextFile": client["readTextFile"], + "writeTextFile": client["writeTextFile"], + }, + "terminal": client["terminal"], + }), + json!({}), + )); + } + cases.extend([ + (json!({}), json!({})), + (json!({"session": null}), json!({})), + (json!({"session": {}}), json!({})), + ]); + for session in session_capability_combinations() { + cases.push((json!({}), json!({"sessionCapabilities": session}))); + } + assert_eq!(cases.len(), 8 + 4 + 8 + 3 + 243); + + let wire_validator = + jsonschema::validator_for(&serde_json::from_str::(SCHEMA).unwrap()).unwrap(); + for (index, (client_capabilities, agent_capabilities)) in cases.iter().enumerate() { + let request = json!({ + "jsonrpc": "2.0", + "id": index as u64, + "method": "initialize", + "params": { + "protocolVersion": 1, + "clientCapabilities": client_capabilities, + "_meta": {"agentCapabilities": agent_capabilities, "case": index}, + }, + }); + assert!( + wire_validator.is_valid(&request), + "invalid generated case {index}: {request}" + ); + acp_harness::write_json(&mut stdin, &request); + let response = acp_harness::read_json(&mut stdout); + assert!( + wire_validator.is_valid(&response), + "invalid response case {index}: {response}" + ); + assert_eq!(response["result"]["agentCapabilities"], *agent_capabilities); + assert_eq!( + response["result"]["_meta"]["clientCapabilities"], + *client_capabilities + ); + eprintln!("ACP v1 capability combination: {index}"); + } + drop(stdin); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "relay failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[derive(Clone)] +struct RequestCase { + method: &'static str, + request_definition: &'static str, + response_definition: Option<&'static str>, + params: Value, + result: Option, +} + +fn request_cases(cwd: &str) -> Vec { + vec![ + RequestCase { + method: "initialize", + request_definition: "InitializeRequest", + response_definition: Some("InitializeResponse"), + params: json!({"protocolVersion":1,"clientCapabilities":{"fs":{"readTextFile":true,"writeTextFile":true},"terminal":true},"_meta":{"fixture":"keep"}}), + result: Some( + json!({"protocolVersion":1,"agentCapabilities":{"loadSession":true,"sessionCapabilities":{"list":{},"delete":{},"resume":{},"close":{}}}}), + ), + }, + RequestCase { + method: "authenticate", + request_definition: "AuthenticateRequest", + response_definition: Some("AuthenticateResponse"), + params: json!({"methodId":"fixture-token"}), + result: Some(json!({})), + }, + RequestCase { + method: "session/new", + request_definition: "NewSessionRequest", + response_definition: Some("NewSessionResponse"), + params: json!({"cwd":cwd,"mcpServers":[]}), + result: Some(json!({"sessionId":"matrix-session"})), + }, + RequestCase { + method: "session/load", + request_definition: "LoadSessionRequest", + response_definition: Some("LoadSessionResponse"), + params: json!({"cwd":cwd,"mcpServers":[],"sessionId":"matrix-session"}), + result: Some(json!({})), + }, + RequestCase { + method: "session/resume", + request_definition: "ResumeSessionRequest", + response_definition: Some("ResumeSessionResponse"), + params: json!({"cwd":cwd,"mcpServers":[],"sessionId":"matrix-session"}), + result: Some(json!({})), + }, + RequestCase { + method: "session/list", + request_definition: "ListSessionsRequest", + response_definition: Some("ListSessionsResponse"), + params: json!({}), + result: Some(json!({"sessions":[]})), + }, + RequestCase { + method: "session/prompt", + request_definition: "PromptRequest", + response_definition: Some("PromptResponse"), + params: json!({"sessionId":"matrix-session","prompt":[{"type":"text","text":"method matrix"}]}), + result: Some(json!({"stopReason":"end_turn"})), + }, + RequestCase { + method: "session/set_mode", + request_definition: "SetSessionModeRequest", + response_definition: Some("SetSessionModeResponse"), + params: json!({"sessionId":"matrix-session","modeId":"code"}), + result: Some(json!({})), + }, + RequestCase { + method: "session/set_config_option", + request_definition: "SetSessionConfigOptionRequest", + response_definition: Some("SetSessionConfigOptionResponse"), + params: json!({"sessionId":"matrix-session","configId":"model","value":"fast"}), + result: Some( + json!({"configOptions":[{"id":"model","name":"Model","type":"select","currentValue":"fast","options":[{"value":"fast","name":"Fast"}]}]}), + ), + }, + RequestCase { + method: "session/cancel", + request_definition: "CancelNotification", + response_definition: None, + params: json!({"sessionId":"matrix-session"}), + result: None, + }, + RequestCase { + method: "session/close", + request_definition: "CloseSessionRequest", + response_definition: Some("CloseSessionResponse"), + params: json!({"sessionId":"matrix-session"}), + result: Some(json!({})), + }, + RequestCase { + method: "session/delete", + request_definition: "DeleteSessionRequest", + response_definition: Some("DeleteSessionResponse"), + params: json!({"sessionId":"matrix-session"}), + result: Some(json!({})), + }, + RequestCase { + method: "logout", + request_definition: "LogoutRequest", + response_definition: Some("LogoutResponse"), + params: json!({}), + result: Some(json!({})), + }, + RequestCase { + method: "session/update", + request_definition: "SessionNotification", + response_definition: None, + params: json!({"sessionId":"matrix-session","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"update"}}}), + result: None, + }, + RequestCase { + method: "session/request_permission", + request_definition: "RequestPermissionRequest", + response_definition: Some("RequestPermissionResponse"), + params: json!({"sessionId":"matrix-session","toolCall":{"toolCallId":"tool-1","title":"Permission"},"options":[{"optionId":"allow","name":"Allow","kind":"allow_once"}]}), + result: Some(json!({"outcome":{"outcome":"selected","optionId":"allow"}})), + }, + RequestCase { + method: "fs/read_text_file", + request_definition: "ReadTextFileRequest", + response_definition: Some("ReadTextFileResponse"), + params: json!({"sessionId":"matrix-session","path":format!("{cwd}/README.md"),"line":1,"limit":10}), + result: Some(json!({"content":"ACP conformance fixture\n"})), + }, + RequestCase { + method: "fs/write_text_file", + request_definition: "WriteTextFileRequest", + response_definition: Some("WriteTextFileResponse"), + params: json!({"sessionId":"matrix-session","path":format!("{cwd}/matrix.txt"),"content":"matrix"}), + result: Some(json!({})), + }, + RequestCase { + method: "terminal/create", + request_definition: "CreateTerminalRequest", + response_definition: Some("CreateTerminalResponse"), + params: json!({"sessionId":"matrix-session","command":"printf","args":["matrix"],"cwd":cwd}), + result: Some(json!({"terminalId":"terminal-matrix"})), + }, + RequestCase { + method: "terminal/output", + request_definition: "TerminalOutputRequest", + response_definition: Some("TerminalOutputResponse"), + params: json!({"sessionId":"matrix-session","terminalId":"terminal-matrix"}), + result: Some( + json!({"output":"matrix","truncated":false,"exitStatus":{"exitCode":0,"signal":null}}), + ), + }, + RequestCase { + method: "terminal/wait_for_exit", + request_definition: "WaitForTerminalExitRequest", + response_definition: Some("WaitForTerminalExitResponse"), + params: json!({"sessionId":"matrix-session","terminalId":"terminal-matrix"}), + result: Some(json!({"exitCode":0,"signal":null})), + }, + RequestCase { + method: "terminal/kill", + request_definition: "KillTerminalRequest", + response_definition: Some("KillTerminalResponse"), + params: json!({"sessionId":"matrix-session","terminalId":"terminal-matrix"}), + result: Some(json!({})), + }, + RequestCase { + method: "terminal/release", + request_definition: "ReleaseTerminalRequest", + response_definition: Some("ReleaseTerminalResponse"), + params: json!({"sessionId":"matrix-session","terminalId":"terminal-matrix"}), + result: Some(json!({})), + }, + RequestCase { + method: "$/cancel_request", + request_definition: "CancelRequestNotification", + response_definition: None, + params: json!({"requestId":"request-in-flight"}), + result: None, + }, + ] +} + +fn add_extension_evidence(params: &Value, force_error: bool) -> Value { + let mut params = params.as_object().unwrap().clone(); + let mut metadata = params + .remove("_meta") + .and_then(|value| value.as_object().cloned()) + .unwrap_or_default(); + metadata.insert("extensionToken".to_string(), json!("preserve-me")); + if force_error { + metadata.insert("forceError".to_string(), json!(true)); + } + params.insert("_meta".to_string(), Value::Object(metadata)); + Value::Object(params) +} + +fn response_error(id: Value) -> Value { + json!({"jsonrpc":"2.0","id":id,"error":{"code":-32001,"message":"fixture rejection","data":{"_meta":{"extensionToken":"preserve-me"}}}}) +} + +#[test] +fn all_23_methods_route_success_error_ids_extensions_and_semantic_capture() { + let schema: Value = serde_json::from_str(SCHEMA).unwrap(); + let wire_validator = jsonschema::validator_for(&schema).unwrap(); + let temp = acp_harness::workspace(); + let cwd = temp.path().to_string_lossy().into_owned(); + let cases = request_cases(&cwd); + assert_eq!(cases.len(), 23); + + for case in &cases { + validate_definition(&schema, case.request_definition, &case.params); + if let (Some(definition), Some(result)) = (case.response_definition, &case.result) { + validate_definition(&schema, definition, result); + } + } + + let agent_results = cases + .iter() + .filter(|case| case.method != "initialize") + .filter_map(|case| case.result.as_ref().map(|result| (case.method, result))) + .collect::>(); + let received_path = temp.path().join(".trail/matrix-received.jsonl"); + let source = format!( + r#"#!/usr/bin/env python3 +import json +import sys + +results = json.loads(r'''{results}''') +received_path = {received_path:?} +pending = {{}} + +def send(value): + print(json.dumps(value, separators=(",", ":")), flush=True) + +for line in sys.stdin: + message = json.loads(line) + with open(received_path, "a", encoding="utf-8") as received: + received.write(json.dumps(message, separators=(",", ":")) + "\n") + method = message.get("method") + request_id = message.get("id") + if method == "ext/emit": + frame = message["params"]["frame"] + send(frame) + if "id" in frame: + pending[str(frame["id"])] = request_id + else: + send({{"jsonrpc":"2.0","id":request_id,"result":{{"emitted":True}}}}) + elif method is None: + control_id = pending.pop(str(request_id)) + send({{"jsonrpc":"2.0","id":control_id,"result":{{"observed":message}}}}) + elif "id" not in message: + continue + elif message.get("params", {{}}).get("_meta", {{}}).get("forceError"): + send({{"jsonrpc":"2.0","id":request_id,"error":{{"code":-32001,"message":"fixture rejection","data":{{"_meta":{{"extensionToken":"preserve-me"}}}}}}}}) + else: + if method == "initialize": + result = {{"protocolVersion":1,"agentCapabilities":{{"loadSession":True,"sessionCapabilities":{{"list":{{}},"delete":{{}},"resume":{{}},"close":{{}}}}}}}} + else: + result = json.loads(json.dumps(results[method])) + result["_meta"] = {{"observed":message,"extensionToken":"preserve-me"}} + send({{"jsonrpc":"2.0","id":request_id,"result":result}}) +"#, + results = serde_json::to_string(&agent_results).unwrap(), + received_path = received_path.to_string_lossy(), + ); + let agent = acp_harness::fixture_agent_command(temp.path(), "method-matrix", &source); + let mut child = acp_harness::spawn_relay(temp.path(), &agent); + let (mut stdin, mut stdout) = acp_harness::relay_stdio(&mut child); + + let agent_side_order = [ + "initialize", + "authenticate", + "session/new", + "session/load", + "session/resume", + "session/list", + "session/prompt", + "session/set_mode", + "session/set_config_option", + ]; + let mut numeric_id = 1_000u64; + for method in agent_side_order { + let case = cases.iter().find(|case| case.method == method).unwrap(); + let success_id = json!(format!("success:{method}")); + let request = json!({ + "jsonrpc":"2.0", "id":success_id, "method":method, + "params":add_extension_evidence(&case.params, false), + "unknownTopLevel":{"extension":true} + }); + assert!( + wire_validator.is_valid(&request), + "invalid {method} request: {request}" + ); + acp_harness::write_json(&mut stdin, &request); + let response = acp_harness::read_json(&mut stdout); + assert!( + wire_validator.is_valid(&response), + "invalid {method} response: {response}" + ); + assert_eq!(response["id"], success_id); + validate_definition( + &schema, + case.response_definition.unwrap(), + &response["result"], + ); + assert_eq!( + response["result"]["_meta"]["observed"]["unknownTopLevel"]["extension"], + true + ); + assert_eq!( + response["result"]["_meta"]["observed"]["params"]["_meta"]["extensionToken"], + "preserve-me" + ); + + numeric_id += 1; + let error_request = json!({ + "jsonrpc":"2.0", "id":numeric_id, "method":method, + "params":add_extension_evidence(&case.params, true) + }); + assert!(wire_validator.is_valid(&error_request)); + acp_harness::write_json(&mut stdin, &error_request); + let error = acp_harness::read_json(&mut stdout); + assert!( + wire_validator.is_valid(&error), + "invalid {method} error: {error}" + ); + assert_eq!(error["id"], numeric_id); + assert_eq!(error["error"]["code"], -32001); + assert_eq!( + error["error"]["data"]["_meta"]["extensionToken"], + "preserve-me" + ); + } + + for method in [ + "session/request_permission", + "fs/read_text_file", + "fs/write_text_file", + "terminal/create", + "terminal/output", + "terminal/wait_for_exit", + "terminal/kill", + "terminal/release", + ] { + let case = cases.iter().find(|case| case.method == method).unwrap(); + for force_error in [false, true] { + numeric_id += 1; + let callback_id = if force_error { + json!(numeric_id) + } else { + json!(format!("callback:{method}")) + }; + let callback = json!({ + "jsonrpc":"2.0", "id":callback_id, "method":method, + "params":add_extension_evidence(&case.params, false), + "unknownTopLevel":{"extension":true} + }); + assert!( + wire_validator.is_valid(&callback), + "invalid callback {method}: {callback}" + ); + let control_id = format!("emit:{method}:{force_error}"); + acp_harness::write_json( + &mut stdin, + &json!({"jsonrpc":"2.0","id":control_id,"method":"ext/emit","params":{"frame":callback}}), + ); + let forwarded = acp_harness::read_json(&mut stdout); + assert_eq!(forwarded["method"], method); + assert_eq!(forwarded["id"], callback_id); + assert_eq!(forwarded["unknownTopLevel"]["extension"], true); + assert_eq!( + forwarded["params"]["_meta"]["extensionToken"], + "preserve-me" + ); + + let response = if force_error { + response_error(callback_id.clone()) + } else { + json!({"jsonrpc":"2.0","id":callback_id,"result":case.result.clone().unwrap()}) + }; + assert!( + wire_validator.is_valid(&response), + "invalid callback response {method}: {response}" + ); + acp_harness::write_json(&mut stdin, &response); + let control_response = acp_harness::read_json(&mut stdout); + assert_eq!(control_response["id"], control_id); + assert_eq!(control_response["result"]["observed"], response); + } + } + + for method in ["session/update", "$/cancel_request"] { + let case = cases.iter().find(|case| case.method == method).unwrap(); + let notification = json!({"jsonrpc":"2.0","method":method,"params":add_extension_evidence(&case.params, false),"unknownTopLevel":{"extension":true}}); + assert!(wire_validator.is_valid(¬ification)); + let control_id = format!("emit:{method}"); + acp_harness::write_json( + &mut stdin, + &json!({"jsonrpc":"2.0","id":control_id,"method":"ext/emit","params":{"frame":notification}}), + ); + let forwarded = acp_harness::read_json(&mut stdout); + assert_eq!(forwarded["method"], method); + assert_eq!(forwarded["unknownTopLevel"]["extension"], true); + assert_eq!(acp_harness::read_json(&mut stdout)["id"], control_id); + } + + let read_case = cases + .iter() + .find(|case| case.method == "fs/read_text_file") + .unwrap(); + for (control_id, callback_id) in [("interleave-a", 77), ("interleave-b", 78)] { + let callback = json!({"jsonrpc":"2.0","id":callback_id,"method":"fs/read_text_file","params":read_case.params}); + acp_harness::write_json( + &mut stdin, + &json!({"jsonrpc":"2.0","id":control_id,"method":"ext/emit","params":{"frame":callback}}), + ); + } + let first_callback = acp_harness::read_json(&mut stdout); + let second_callback = acp_harness::read_json(&mut stdout); + assert_eq!( + ( + first_callback["id"].as_u64(), + second_callback["id"].as_u64() + ), + (Some(77), Some(78)) + ); + acp_harness::write_json( + &mut stdin, + &json!({"jsonrpc":"2.0","id":78,"result":{"content":"second"}}), + ); + assert_eq!(acp_harness::read_json(&mut stdout)["id"], "interleave-b"); + acp_harness::write_json( + &mut stdin, + &json!({"jsonrpc":"2.0","id":77,"result":{"content":"first"}}), + ); + assert_eq!(acp_harness::read_json(&mut stdout)["id"], "interleave-a"); + + for method in ["session/cancel", "$/cancel_request"] { + let case = cases.iter().find(|case| case.method == method).unwrap(); + let notification = json!({"jsonrpc":"2.0","method":method,"params":add_extension_evidence(&case.params, false),"unknownTopLevel":{"extension":true}}); + assert!(wire_validator.is_valid(¬ification)); + acp_harness::write_json(&mut stdin, ¬ification); + } + + for method in ["session/close", "session/delete", "logout"] { + let case = cases.iter().find(|case| case.method == method).unwrap(); + for force_error in [false, true] { + numeric_id += 1; + let request = json!({"jsonrpc":"2.0","id":numeric_id,"method":method,"params":add_extension_evidence(&case.params, force_error)}); + acp_harness::write_json(&mut stdin, &request); + let response = acp_harness::read_json(&mut stdout); + assert_eq!(response["id"], numeric_id); + assert!(wire_validator.is_valid(&response)); + if force_error { + assert_eq!(response["error"]["code"], -32001); + } else { + validate_definition( + &schema, + case.response_definition.unwrap(), + &response["result"], + ); + } + } + } + + drop(stdin); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "relay failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let received = fs::read_to_string(received_path).unwrap(); + let received = received + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .collect::>(); + for method in ["session/cancel", "$/cancel_request"] { + let notification = received + .iter() + .find(|message| message["method"] == method) + .unwrap_or_else(|| panic!("agent did not receive {method}")); + assert_eq!(notification["unknownTopLevel"]["extension"], true); + assert_eq!( + notification["params"]["_meta"]["extensionToken"], + "preserve-me" + ); + } + + let db = trail::Trail::open(temp.path()).unwrap(); + let receipts = db + .list_agent_hook_receipts(Some("trail-acp"), None, 1_000) + .unwrap(); + assert!( + receipts.len() >= 23, + "expected durable evidence for method matrix, got {}", + receipts.len() + ); + assert!(receipts + .iter() + .all(|receipt| receipt.connection_sequence.is_some())); + assert!(receipts + .iter() + .any(|receipt| receipt.direction.as_deref() == Some("client_to_agent"))); + assert!(receipts + .iter() + .any(|receipt| receipt.direction.as_deref() == Some("agent_to_client"))); +} + +#[test] +fn conformance_harness_shutdown_is_bounded_with_a_callback_in_flight() { + let temp = acp_harness::workspace(); + let agent = acp_harness::fixture_agent_command( + temp.path(), + "in-flight-shutdown", + r#"#!/usr/bin/env python3 +import json +import sys +import time + +for line in sys.stdin: + message = json.loads(line) + if message.get("method") == "ext/hang": + print(json.dumps({"jsonrpc":"2.0","id":"pending","method":"fs/read_text_file","params":{"sessionId":"session","path":"/tmp/pending"}}, separators=(",", ":")), flush=True) + time.sleep(30) +"#, + ); + let mut child = acp_harness::spawn_relay(temp.path(), &agent); + let (mut stdin, mut stdout) = acp_harness::relay_stdio(&mut child); + acp_harness::write_json( + &mut stdin, + &json!({"jsonrpc":"2.0","id":1,"method":"ext/hang","params":{}}), + ); + assert_eq!(acp_harness::read_json(&mut stdout)["id"], "pending"); + drop(stdin); + let started = Instant::now(); + let status = child.wait().unwrap(); + assert!(started.elapsed() < Duration::from_secs(3)); + assert!(status.success()); +} diff --git a/trail/tests/acp_faults.rs b/trail/tests/acp_faults.rs new file mode 100644 index 0000000..70c9a38 --- /dev/null +++ b/trail/tests/acp_faults.rs @@ -0,0 +1,326 @@ +#![cfg(any(unix, windows))] + +#[path = "support/acp_harness.rs"] +mod acp_harness; + +use std::io::{Read, Write}; +use std::thread; +use std::time::{Duration, Instant}; + +use serde_json::json; + +const ACP_MAX_FRAME_BYTES: usize = 16 * 1024 * 1024; + +fn echo_agent(workspace: &std::path::Path, scenario: &str) -> Vec { + acp_harness::fixture_agent_command( + workspace, + scenario, + r#"#!/usr/bin/env python3 +import sys +for line in sys.stdin.buffer: + sys.stdout.buffer.write(line) + sys.stdout.buffer.flush() +"#, + ) +} + +fn run_with_input(agent_source: &str, input: &[u8]) -> std::process::Output { + let temp = acp_harness::workspace(); + let agent = acp_harness::fixture_agent_command(temp.path(), "fault", agent_source); + let mut child = acp_harness::spawn_relay(temp.path(), &agent); + child.stdin.take().unwrap().write_all(input).unwrap(); + child.wait_with_output().unwrap() +} + +#[test] +fn malformed_utf8_and_invalid_json_rpc_envelopes_fail_without_invented_output() { + let malformed_utf8 = run_with_input( + r#"#!/usr/bin/env python3 +import sys +sys.stdout.buffer.write(b"\xff\n") +sys.stdout.buffer.flush() +"#, + b"", + ); + assert!(!malformed_utf8.status.success()); + assert!(malformed_utf8.stdout.is_empty()); + assert!(String::from_utf8_lossy(&malformed_utf8.stderr).contains("I/O error")); + + let invalid = [ + br#"null +"# + .as_slice(), + br#"{"jsonrpc":"1.0","method":"ext/invalid"} +"#, + br#"{"jsonrpc":"2.0","id":{},"method":"ext/invalid"} +"#, + br#"{"jsonrpc":"2.0","id":1} +"#, + br#"{"jsonrpc":"2.0","id":1,"result":{},"error":{"code":1,"message":"both"}} +"#, + br#"{"jsonrpc":"2.0","id":1,"error":{"code":"bad","message":7}} +"#, + ]; + for (index, frame) in invalid.into_iter().enumerate() { + let temp = acp_harness::workspace(); + let agent = echo_agent(temp.path(), &format!("invalid-{index}")); + let mut child = acp_harness::spawn_relay(temp.path(), &agent); + child.stdin.take().unwrap().write_all(frame).unwrap(); + let output = child.wait_with_output().unwrap(); + assert!( + !output.status.success(), + "invalid envelope {index} was accepted" + ); + assert!( + output.stdout.is_empty(), + "relay invented output for case {index}" + ); + } +} + +fn frame_of_size(size: usize) -> Vec { + let prefix = br#"{"jsonrpc":"2.0","method":"ext/limit","params":{"data":""#; + let suffix = b"\"}}\n"; + assert!(size >= prefix.len() + suffix.len()); + let mut frame = Vec::with_capacity(size); + frame.extend_from_slice(prefix); + frame.resize(size - suffix.len(), b'x'); + frame.extend_from_slice(suffix); + assert_eq!(frame.len(), size); + frame +} + +#[test] +fn transport_accepts_the_frame_limit_and_rejects_one_byte_above_it() { + let temp = acp_harness::workspace(); + let agent = echo_agent(temp.path(), "frame-limit"); + let mut child = acp_harness::spawn_relay(temp.path(), &agent); + let mut stdin = child.stdin.take().unwrap(); + let writer = thread::spawn(move || { + stdin + .write_all(&frame_of_size(ACP_MAX_FRAME_BYTES)) + .unwrap(); + }); + let mut stdout = child.stdout.take().unwrap(); + let mut received = Vec::new(); + let mut chunk = [0_u8; 64 * 1024]; + loop { + let bytes = stdout.read(&mut chunk).unwrap(); + if bytes == 0 { + break; + } + received.extend_from_slice(&chunk[..bytes]); + thread::sleep(Duration::from_millis(2)); + } + writer.join().unwrap(); + let status = child.wait().unwrap(); + assert!(status.success()); + assert_eq!(received.len(), ACP_MAX_FRAME_BYTES); + assert_eq!(received, frame_of_size(ACP_MAX_FRAME_BYTES)); + + let temp = acp_harness::workspace(); + let agent = echo_agent(temp.path(), "frame-too-large"); + let mut child = acp_harness::spawn_relay(temp.path(), &agent); + child + .stdin + .take() + .unwrap() + .write_all(&frame_of_size(ACP_MAX_FRAME_BYTES + 1)) + .unwrap(); + let output = child.wait_with_output().unwrap(); + assert!(!output.status.success()); + assert!(output.stdout.is_empty()); + assert!(String::from_utf8_lossy(&output.stderr).contains("transport limit")); +} + +#[test] +fn one_hundred_requests_per_direction_keep_same_ids_and_reverse_order_correlated() { + const COUNT: usize = 100; + let temp = acp_harness::workspace(); + let agent = acp_harness::fixture_agent_command( + temp.path(), + "concurrency", + &format!( + r#"#!/usr/bin/env python3 +import json +import sys + +requests = [json.loads(sys.stdin.readline()) for _ in range({COUNT})] +for request in requests: + callback = {{"jsonrpc":"2.0","id":request["id"],"method":"fs/read_text_file","params":{{"sessionId":"s","path":"/tmp/" + str(request["id"])}}}} + print(json.dumps(callback, separators=(",", ":")), flush=True) +for request in reversed(requests): + print(json.dumps({{"jsonrpc":"2.0","id":request["id"],"result":{{"sequence":request["id"]}}}}, separators=(",", ":")), flush=True) +responses = [json.loads(sys.stdin.readline()) for _ in range({COUNT})] +assert [response["id"] for response in responses] == list(reversed(range({COUNT}))) +"# + ), + ); + let mut child = acp_harness::spawn_relay(temp.path(), &agent); + let (mut stdin, mut stdout) = acp_harness::relay_stdio(&mut child); + for id in 0..COUNT { + acp_harness::write_json( + &mut stdin, + &json!({"jsonrpc":"2.0","id":id,"method":"ext/concurrent","params":{"sequence":id}}), + ); + } + + let callbacks = (0..COUNT) + .map(|_| acp_harness::read_json(&mut stdout)) + .collect::>(); + assert_eq!( + callbacks + .iter() + .map(|frame| frame["id"].as_u64().unwrap()) + .collect::>(), + (0..COUNT as u64).collect::>() + ); + let responses = (0..COUNT) + .map(|_| acp_harness::read_json(&mut stdout)) + .collect::>(); + assert_eq!( + responses + .iter() + .map(|frame| frame["id"].as_u64().unwrap()) + .collect::>(), + (0..COUNT as u64).rev().collect::>() + ); + + for callback in callbacks.into_iter().rev() { + acp_harness::write_json( + &mut stdin, + &json!({"jsonrpc":"2.0","id":callback["id"],"result":{"content":"ok"}}), + ); + } + drop(stdin); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn fragmented_writer_and_slow_reader_apply_backpressure_without_loss() { + let temp = acp_harness::workspace(); + let agent = echo_agent(temp.path(), "backpressure"); + let mut child = acp_harness::spawn_relay(temp.path(), &agent); + let mut stdin = child.stdin.take().unwrap(); + let frame = json!({"jsonrpc":"2.0","id":"fragmented","method":"ext/backpressure","params":{"data":"x".repeat(1024 * 1024)}}); + let mut raw = serde_json::to_vec(&frame).unwrap(); + raw.push(b'\n'); + let expected = raw.clone(); + let writer = thread::spawn(move || { + for chunk in raw.chunks(4_093) { + stdin.write_all(chunk).unwrap(); + thread::yield_now(); + } + }); + thread::sleep(Duration::from_millis(100)); + let mut received = Vec::new(); + child + .stdout + .take() + .unwrap() + .read_to_end(&mut received) + .unwrap(); + writer.join().unwrap(); + assert!(child.wait().unwrap().success()); + assert_eq!(received, expected); +} + +#[test] +fn child_crash_and_editor_eof_have_deterministic_distinct_outcomes() { + let crashed = run_with_input( + r#"#!/usr/bin/env python3 +import sys +sys.stdin.readline() +sys.exit(17) +"#, + br#"{"jsonrpc":"2.0","id":1,"method":"ext/crash"} +"#, + ); + assert!(!crashed.status.success()); + assert!(crashed.stdout.is_empty()); + assert!(String::from_utf8_lossy(&crashed.stderr).contains("status")); + + let temp = acp_harness::workspace(); + let agent = acp_harness::fixture_agent_command( + temp.path(), + "editor-eof", + r#"#!/usr/bin/env python3 +import time +time.sleep(30) +"#, + ); + let mut child = acp_harness::spawn_relay(temp.path(), &agent); + drop(child.stdin.take().unwrap()); + let started = Instant::now(); + let status = child.wait().unwrap(); + assert!(status.success()); + assert!(started.elapsed() < Duration::from_secs(3)); +} + +#[test] +fn multiple_sessions_and_opposite_direction_cancellation_races_do_not_cross_correlate() { + const SESSION_COUNT: usize = 12; + let temp = acp_harness::workspace(); + let agent = acp_harness::fixture_agent_command( + temp.path(), + "sessions-cancellation", + &format!( + r#"#!/usr/bin/env python3 +import json +import sys + +for line in sys.stdin: + message = json.loads(line) + method = message.get("method") + if method == "initialize": + print(json.dumps({{"jsonrpc":"2.0","id":message["id"],"result":{{"protocolVersion":1,"agentCapabilities":{{}}}}}}, separators=(",", ":")), flush=True) + elif method == "session/new": + print(json.dumps({{"jsonrpc":"2.0","id":message["id"],"result":{{"sessionId":"session-" + str(message["id"])}}}}, separators=(",", ":")), flush=True) + elif method == "session/prompt": + request_id = message["id"] + print(json.dumps({{"jsonrpc":"2.0","method":"$/cancel_request","params":{{"requestId":request_id}}}}, separators=(",", ":")), flush=True) + print(json.dumps({{"jsonrpc":"2.0","id":request_id,"result":{{"stopReason":"cancelled"}}}}, separators=(",", ":")), flush=True) +"# + ), + ); + let mut child = acp_harness::spawn_relay(temp.path(), &agent); + let (mut stdin, mut stdout) = acp_harness::relay_stdio(&mut child); + acp_harness::write_json( + &mut stdin, + &json!({"jsonrpc":"2.0","id":"init","method":"initialize","params":{"protocolVersion":1,"clientCapabilities":{}}}), + ); + assert_eq!(acp_harness::read_json(&mut stdout)["id"], "init"); + for id in 0..SESSION_COUNT { + acp_harness::write_json( + &mut stdin, + &json!({"jsonrpc":"2.0","id":id,"method":"session/new","params":{"cwd":temp.path(),"mcpServers":[]}}), + ); + assert_eq!( + acp_harness::read_json(&mut stdout)["result"]["sessionId"], + format!("session-{id}") + ); + } + for id in 0..SESSION_COUNT { + let request_id = format!("turn-{id}"); + acp_harness::write_json( + &mut stdin, + &json!({"jsonrpc":"2.0","id":request_id,"method":"session/prompt","params":{"sessionId":format!("session-{id}"),"prompt":[{"type":"text","text":"cancel"}]}}), + ); + acp_harness::write_json( + &mut stdin, + &json!({"jsonrpc":"2.0","method":"$/cancel_request","params":{"requestId":request_id}}), + ); + let cancellation = acp_harness::read_json(&mut stdout); + let response = acp_harness::read_json(&mut stdout); + assert_eq!(cancellation["params"]["requestId"], request_id); + assert_eq!(response["id"], request_id); + assert_eq!(response["result"]["stopReason"], "cancelled"); + } + drop(stdin); + assert!(child.wait().unwrap().success()); +} diff --git a/trail/tests/acp_interop.rs b/trail/tests/acp_interop.rs new file mode 100644 index 0000000..75ba2ff --- /dev/null +++ b/trail/tests/acp_interop.rs @@ -0,0 +1,103 @@ +#![cfg(any(unix, windows))] + +#[path = "support/acp_harness.rs"] +mod acp_harness; + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use trail::Trail; + +fn reference_peer() -> Option { + std::env::var_os("TRAIL_ACP_REFERENCE_PEER").map(PathBuf::from) +} + +fn run_client(peer: &Path, mode: &str, workspace: &Path, agent: &Path) -> std::process::Output { + Command::new(peer) + .arg(mode) + .arg(workspace) + .arg(acp_harness::trail_bin()) + .arg(agent) + .output() + .unwrap() +} + +#[test] +fn official_v1_client_through_trail_to_official_v1_agent() { + let Some(peer) = reference_peer() else { + eprintln!("skipped: TRAIL_ACP_REFERENCE_PEER is unavailable"); + return; + }; + let temp = acp_harness::workspace(); + let output = run_client(&peer, "client", temp.path(), &peer); + assert!( + output.status.success(), + "official peer failed: stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let db = Trail::open(temp.path()).unwrap(); + let mapping = db.lane_acp_session("official-session").unwrap(); + let session = db.show_lane_session(&mapping.trail_session_id).unwrap(); + assert!(session + .events + .iter() + .any(|event| event.event_type == "acp_prompt_started")); + assert!(session + .events + .iter() + .any(|event| event.event_type == "acp_prompt_finished")); + let callback_methods = session + .events + .iter() + .filter(|event| event.event_type == "acp_client_callback_requested") + .filter_map(|event| event.payload.as_ref()) + .filter_map(|payload| payload["method"].as_str()) + .collect::>(); + assert_eq!(callback_methods.len(), 8); +} + +#[test] +#[cfg(unix)] +fn official_v1_client_through_trail_to_independent_fixture_agent() { + let Some(peer) = reference_peer() else { + eprintln!("skipped: TRAIL_ACP_REFERENCE_PEER is unavailable"); + return; + }; + let temp = acp_harness::workspace(); + let agent = acp_harness::fixture_agent_command( + temp.path(), + "interop-fixture", + r#"#!/usr/bin/env python3 +import json +import sys + +for line in sys.stdin: + message = json.loads(line) + method = message.get("method") + if method == "initialize": + result = {"protocolVersion":1,"agentCapabilities":{}} + elif method == "session/new": + result = {"sessionId":"fixture-session"} + elif method == "session/prompt": + print(json.dumps({"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"fixture-session","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"fixture update"}}}}, separators=(",", ":")), flush=True) + result = {"stopReason":"end_turn"} + elif method == "session/close": + result = {} + else: + raise RuntimeError("unexpected method " + str(method)) + print(json.dumps({"jsonrpc":"2.0","id":message["id"],"result":result}, separators=(",", ":")), flush=True) +"#, + ); + assert_eq!(agent.len(), 1, "interop fixture expects a direct launcher"); + let output = run_client(&peer, "client-basic", temp.path(), Path::new(&agent[0])); + assert!( + output.status.success(), + "official client / fixture agent failed: stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(fs::metadata(temp.path().join(".trail")).is_ok()); +} diff --git a/trail/tests/acp_session_semantics.rs b/trail/tests/acp_session_semantics.rs new file mode 100644 index 0000000..470356e --- /dev/null +++ b/trail/tests/acp_session_semantics.rs @@ -0,0 +1,398 @@ +#![cfg(unix)] + +use std::fs; +use std::io::{BufRead, BufReader, Read, Write}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use trail::{InitImportMode, Trail}; + +fn trail_bin() -> PathBuf { + std::env::var_os("TRAIL_TEST_BIN") + .map(PathBuf::from) + .or_else(|| option_env!("CARGO_BIN_EXE_trail").map(PathBuf::from)) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/debug/trail")) +} + +fn write_lifecycle_agent(workspace: &Path) -> PathBuf { + let agent = workspace.join("lifecycle-agent.py"); + fs::write( + &agent, + r#"#!/usr/bin/env python3 +import json +import sys + +for line in sys.stdin: + request = json.loads(line) + request_id = request.get("id") + method = request.get("method") + if method == "initialize": + result = { + "protocolVersion": 1, + "agentCapabilities": { + "loadSession": True, + "sessionCapabilities": { + "list": {}, "delete": {}, "resume": {}, "close": {} + } + }, + "authMethods": [{"id": "fixture", "name": "Fixture"}] + } + response = {"jsonrpc": "2.0", "id": request_id, "result": result} + elif str(request_id).endswith("-error"): + response = { + "jsonrpc": "2.0", + "id": request_id, + "error": {"code": -32001, "message": "fixture failure"} + } + elif method == "session/new": + response = {"jsonrpc": "2.0", "id": request_id, "result": {"sessionId": "life-session"}} + elif method == "session/list": + response = { + "jsonrpc": "2.0", + "id": request_id, + "result": {"sessions": [], "nextCursor": "next-page"} + } + else: + response = {"jsonrpc": "2.0", "id": request_id, "result": {}} + print(json.dumps(response, separators=(",", ":")), flush=True) +"#, + ) + .unwrap(); + let mut permissions = fs::metadata(&agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&agent, permissions).unwrap(); + agent +} + +fn write_load_replay_agent(workspace: &Path) -> PathBuf { + let agent = workspace.join("load-replay-agent.py"); + fs::write( + &agent, + r#"#!/usr/bin/env python3 +import json +import sys + +initialize = json.loads(sys.stdin.readline()) +print(json.dumps({"jsonrpc":"2.0","id":initialize["id"],"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":True}}}, separators=(",", ":")), flush=True) +load = json.loads(sys.stdin.readline()) +session_id = load["params"]["sessionId"] +print(json.dumps({"jsonrpc":"2.0","id":load["id"],"result":{"modes":{"currentModeId":"code","availableModes":[]},"configOptions":[]}}, separators=(",", ":")), flush=True) +updates = [ + {"sessionUpdate":"user_message_chunk","messageId":"user-1","content":{"type":"text","text":"Question one"}}, + {"sessionUpdate":"agent_message_chunk","messageId":"agent-1","content":{"type":"text","text":"Answer one"}}, + {"sessionUpdate":"tool_call","toolCallId":"tool-1","title":"Inspect","kind":"read","status":"pending","content":[],"locations":[]}, + {"sessionUpdate":"tool_call_update","toolCallId":"tool-1","status":"completed"}, + {"sessionUpdate":"plan","entries":[{"content":"Inspect files","priority":"high","status":"completed"}]}, + {"sessionUpdate":"usage_update","used":10,"size":100}, + {"sessionUpdate":"user_message_chunk","messageId":"user-2","content":{"type":"text","text":"Question two"}}, + {"sessionUpdate":"agent_message_chunk","messageId":"agent-2","content":{"type":"text","text":"Answer two"}} +] +for update in updates: + print(json.dumps({"jsonrpc":"2.0","method":"session/update","params":{"sessionId":session_id,"update":update}}, separators=(",", ":")), flush=True) +sys.stdin.read() +"#, + ) + .unwrap(); + let mut permissions = fs::metadata(&agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&agent, permissions).unwrap(); + agent +} + +fn exchange( + stdin: &mut impl Write, + stdout: &mut impl BufRead, + request: serde_json::Value, +) -> serde_json::Value { + serde_json::to_writer(&mut *stdin, &request).unwrap(); + stdin.write_all(b"\n").unwrap(); + stdin.flush().unwrap(); + let mut response = String::new(); + stdout.read_line(&mut response).unwrap(); + assert!( + !response.is_empty(), + "agent closed before responding to {request}" + ); + serde_json::from_str(response.trim()).unwrap() +} + +#[test] +fn lifecycle_outcomes_preserve_mapping_state_and_redact_auth_metadata() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "lifecycle fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let agent = write_lifecycle_agent(temp.path()); + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["acp", "relay", "--"]) + .arg(agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + + let initialize = exchange( + &mut stdin, + &mut stdout, + serde_json::json!({ + "jsonrpc":"2.0", "id":"initialize", "method":"initialize", + "params":{"protocolVersion":1,"clientCapabilities":{}} + }), + ); + assert_eq!(initialize["result"]["protocolVersion"], 1); + + for (id, method, params) in [ + ( + "authenticate-error", + "authenticate", + serde_json::json!({"methodId":"fixture","_meta":{"authorization":"Bearer never-store-this"}}), + ), + ( + "authenticate-success", + "authenticate", + serde_json::json!({"methodId":"fixture"}), + ), + ("logout-error", "logout", serde_json::json!({})), + ("logout-success", "logout", serde_json::json!({})), + ( + "new-error", + "session/new", + serde_json::json!({"cwd":temp.path(),"mcpServers":[]}), + ), + ( + "new-success", + "session/new", + serde_json::json!({"cwd":temp.path(),"mcpServers":[]}), + ), + ( + "load-error", + "session/load", + serde_json::json!({"sessionId":"life-session","cwd":temp.path(),"mcpServers":[]}), + ), + ( + "load-success", + "session/load", + serde_json::json!({"sessionId":"life-session","cwd":temp.path(),"mcpServers":[]}), + ), + ( + "resume-error", + "session/resume", + serde_json::json!({"sessionId":"life-session","cwd":temp.path(),"mcpServers":[]}), + ), + ( + "resume-success", + "session/resume", + serde_json::json!({"sessionId":"life-session","cwd":temp.path(),"mcpServers":[]}), + ), + ( + "list-error", + "session/list", + serde_json::json!({"cursor":"first-page"}), + ), + ( + "list-success", + "session/list", + serde_json::json!({"cursor":"first-page"}), + ), + ( + "close-error", + "session/close", + serde_json::json!({"sessionId":"life-session"}), + ), + ( + "close-success", + "session/close", + serde_json::json!({"sessionId":"life-session"}), + ), + ( + "delete-error", + "session/delete", + serde_json::json!({"sessionId":"life-session"}), + ), + ( + "delete-success", + "session/delete", + serde_json::json!({"sessionId":"life-session"}), + ), + ] { + let response = exchange( + &mut stdin, + &mut stdout, + serde_json::json!({"jsonrpc":"2.0","id":id,"method":method,"params":params}), + ); + if id.ends_with("-error") { + assert_eq!(response["error"]["code"], -32001, "{method} error response"); + } else { + assert!( + response.get("result").is_some(), + "{method} success response" + ); + } + } + + drop(stdin); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "relay failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let db = Trail::open(temp.path()).unwrap(); + let mapping = db + .try_lane_acp_session("life-session") + .unwrap() + .expect("successful session/new must create a durable mapping"); + assert_eq!( + db.list_lane_acp_sessions(None).unwrap().sessions.len(), + 1, + "failed lifecycle requests must not create mappings" + ); + assert_eq!(mapping.status, "deleted"); + let session = db.show_lane_session(&mapping.trail_session_id).unwrap(); + let failed_methods = session + .events + .iter() + .filter(|event| event.event_type == "acp_session_request_failed") + .filter_map(|event| event.payload.as_ref()) + .filter_map(|payload| payload.get("method")) + .filter_map(serde_json::Value::as_str) + .collect::>(); + assert!(failed_methods.contains("session/load")); + assert!(failed_methods.contains("session/resume")); + assert!(failed_methods.contains("session/delete")); + assert!(session + .events + .iter() + .any(|event| event.event_type == "acp_session_close_failed")); + assert!(session + .events + .iter() + .any(|event| event.event_type == "acp_session_deleted")); + let connection_events = session + .events + .iter() + .filter(|event| event.event_type == "acp_connection_lifecycle") + .filter_map(|event| event.payload.as_ref()) + .collect::>(); + let connection_methods = connection_events + .iter() + .filter_map(|payload| payload.get("method")) + .filter_map(serde_json::Value::as_str) + .collect::>(); + for method in ["initialize", "authenticate", "logout", "session/list"] { + assert!( + connection_methods.contains(method), + "missing normalized {method} lifecycle evidence: {connection_events:?}" + ); + } + assert!(connection_events.iter().any(|payload| { + payload["method"] == "session/list" + && payload["outcome"] == "success" + && payload["context"]["request_cursor"] == "first-page" + && payload["result"]["nextCursor"] == "next-page" + })); + + let mut persisted = Vec::new(); + for entry in walkdir::WalkDir::new(temp.path().join(".trail")) { + let entry = entry.unwrap(); + if entry.file_type().is_file() { + let mut file = fs::File::open(entry.path()).unwrap(); + file.read_to_end(&mut persisted).unwrap(); + } + } + assert!( + !persisted + .windows(b"never-store-this".len()) + .any(|window| window == b"never-store-this"), + "authentication metadata leaked into Trail storage" + ); +} + +#[test] +fn load_replay_reconstructs_ordered_transcript_turns() { + const UPDATE_COUNT: usize = 8; + + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "load replay fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let agent = write_load_replay_agent(temp.path()); + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["acp", "relay", "--"]) + .arg(agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + exchange( + &mut stdin, + &mut stdout, + serde_json::json!({ + "jsonrpc":"2.0", "id":1, "method":"initialize", + "params":{"protocolVersion":1,"clientCapabilities":{}} + }), + ); + let response = exchange( + &mut stdin, + &mut stdout, + serde_json::json!({ + "jsonrpc":"2.0", "id":2, "method":"session/load", + "params":{"sessionId":"replay-session","cwd":temp.path(),"mcpServers":[]} + }), + ); + assert_eq!(response["result"]["modes"]["currentModeId"], "code"); + for _ in 0..UPDATE_COUNT { + let mut update = String::new(); + stdout.read_line(&mut update).unwrap(); + assert!(update.contains(r#""method":"session/update"#)); + } + drop(stdin); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "relay failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let db = Trail::open(temp.path()).unwrap(); + let mapping = db.try_lane_acp_session("replay-session").unwrap().unwrap(); + let session = db.show_lane_session(&mapping.trail_session_id).unwrap(); + assert_eq!( + session.turns.len(), + 2, + "load replay must create two history turns" + ); + let messages = session + .messages + .iter() + .map(|message| (message.role.as_str(), message.body.as_str())) + .collect::>(); + assert_eq!( + messages, + vec![ + ("user", "Question one"), + ("assistant", "Answer one"), + ("user", "Question two"), + ("assistant", "Answer two") + ] + ); + let event_types = session + .events + .iter() + .map(|event| event.event_type.as_str()) + .collect::>(); + assert!(event_types.contains("tool_call")); + assert!(event_types.contains("tool_call_update")); + assert!(event_types.contains("plan_update")); + assert!(event_types.contains("acp_usage_update")); +} diff --git a/trail/tests/acp_transform.rs b/trail/tests/acp_transform.rs new file mode 100644 index 0000000..47915c6 --- /dev/null +++ b/trail/tests/acp_transform.rs @@ -0,0 +1,347 @@ +#![cfg(unix)] + +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; + +use serde_json::Value; +use trail::{InitImportMode, Trail}; + +fn trail_bin() -> PathBuf { + std::env::var_os("TRAIL_TEST_BIN") + .map(PathBuf::from) + .or_else(|| option_env!("CARGO_BIN_EXE_trail").map(PathBuf::from)) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/debug/trail")) +} + +fn workspace() -> tempfile::TempDir { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "transform fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + temp +} + +fn write_agent(workspace: &Path, name: &str, body: &str) -> PathBuf { + let agent = workspace.join(name); + fs::write(&agent, format!("#!/bin/sh\nset -eu\n{body}")).unwrap(); + let mut permissions = fs::metadata(&agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&agent, permissions).unwrap(); + agent +} + +fn spawn_relay(workspace: &Path, agent: &Path) -> Child { + Command::new(trail_bin()) + .arg("--workspace") + .arg(workspace) + .args(["acp", "relay", "--"]) + .arg(agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap() +} + +fn read_json(reader: &mut impl BufRead) -> Value { + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + assert!(!line.is_empty(), "relay closed before returning a frame"); + serde_json::from_str(line.trim_end()).unwrap() +} + +#[test] +fn selects_v1_without_rewriting_the_offer_and_then_enables_transformations() { + let temp = workspace(); + let received = temp.path().join(".trail/received.jsonl"); + let agent = write_agent( + temp.path(), + "v1-agent.sh", + &format!( + r#"IFS= read -r initialize +printf '%s\n' "$initialize" > '{}' +printf '%s\n' '{{"jsonrpc":"2.0","id":"init","result":{{"protocolVersion":1,"agentCapabilities":{{}},"_meta":{{"agent":"keep"}}}}}}' +IFS= read -r session +printf '%s\n' "$session" >> '{}' +printf '%s\n' '{{"jsonrpc":"2.0","id":2,"result":{{"sessionId":"s"}}}}' +"#, + received.display(), + received.display() + ), + ); + let mut child = spawn_relay(temp.path(), &agent); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + let initialize = r#"{"jsonrpc":"2.0","id":"init","method":"initialize","params":{"protocolVersion":2,"clientCapabilities":{},"_meta":{"offeredVersions":[1,2]}}}"#; + writeln!(stdin, "{initialize}").unwrap(); + let response = read_json(&mut stdout); + assert_eq!(response["result"]["protocolVersion"], 1); + assert_eq!(response["result"]["_meta"]["agent"], "keep"); + assert_eq!(response["result"]["_meta"]["trail"]["relay"], true); + + let session = format!( + r#"{{"jsonrpc":"2.0","id":2,"method":"session/new","params":{{"cwd":"{}","mcpServers":[]}}}}"#, + temp.path().display() + ); + writeln!(stdin, "{session}").unwrap(); + let _ = read_json(&mut stdout); + drop(stdin); + assert!(child.wait().unwrap().success()); + + let lines = fs::read_to_string(received).unwrap(); + let mut lines = lines.lines(); + assert_eq!(lines.next(), Some(initialize)); + let forwarded_session: Value = serde_json::from_str(lines.next().unwrap()).unwrap(); + let servers = forwarded_session["params"]["mcpServers"] + .as_array() + .unwrap(); + assert_eq!(servers.len(), 1); + assert_eq!(servers[0]["name"], "trail"); +} + +#[test] +fn later_version_selection_is_forwarded_without_v1_transformations() { + let temp = workspace(); + let received = temp.path().join(".trail/received.jsonl"); + let response = r#"{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":2,"agentCapabilities":{},"_meta":{"agent":"keep"}}}"#; + let agent = write_agent( + temp.path(), + "v2-agent.sh", + &format!( + r#"IFS= read -r initialize +printf '%s\n' '{}' +IFS= read -r session +printf '%s\n' "$session" > '{}' +printf '%s\n' '{{"jsonrpc":"2.0","id":2,"result":{{"sessionId":"s"}}}}' +"#, + response, + received.display() + ), + ); + let mut child = spawn_relay(temp.path(), &agent); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + writeln!(stdin, r#"{{"jsonrpc":"2.0","id":1,"method":"initialize","params":{{"protocolVersion":2,"clientCapabilities":{{}}}}}}"#).unwrap(); + let mut raw_response = String::new(); + stdout.read_line(&mut raw_response).unwrap(); + assert_eq!(raw_response, format!("{response}\n")); + + let session = format!( + r#"{{"jsonrpc":"2.0","id":2,"method":"session/new","params":{{"cwd":"{}","mcpServers":[]}}}}"#, + temp.path().display() + ); + writeln!(stdin, "{session}").unwrap(); + let _ = read_json(&mut stdout); + drop(stdin); + assert!(child.wait().unwrap().success()); + assert_eq!( + fs::read_to_string(received).unwrap(), + format!("{session}\n") + ); +} + +#[test] +fn initialize_error_disables_session_transformations() { + let temp = workspace(); + let received = temp.path().join(".trail/received.jsonl"); + let agent = write_agent( + temp.path(), + "error-agent.sh", + &format!( + r#"IFS= read -r initialize +printf '%s\n' '{{"jsonrpc":"2.0","id":1,"error":{{"code":-32000,"message":"unsupported"}}}}' +IFS= read -r session +printf '%s\n' "$session" > '{}' +printf '%s\n' '{{"jsonrpc":"2.0","id":2,"result":{{"sessionId":"s"}}}}' +"#, + received.display() + ), + ); + let mut child = spawn_relay(temp.path(), &agent); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + writeln!(stdin, r#"{{"jsonrpc":"2.0","id":1,"method":"initialize","params":{{"protocolVersion":1,"clientCapabilities":{{}}}}}}"#).unwrap(); + let _ = read_json(&mut stdout); + let session = format!( + r#"{{"jsonrpc":"2.0","id":2,"method":"session/new","params":{{"cwd":"{}","mcpServers":[]}}}}"#, + temp.path().display() + ); + writeln!(stdin, "{session}").unwrap(); + let _ = read_json(&mut stdout); + drop(stdin); + assert!(child.wait().unwrap().success()); + assert_eq!( + fs::read_to_string(received).unwrap(), + format!("{session}\n") + ); +} + +#[test] +fn session_before_initialize_is_forwarded_without_creating_a_transformation() { + let temp = workspace(); + let received = temp.path().join(".trail/received.jsonl"); + let agent = write_agent( + temp.path(), + "early-session-agent.sh", + &format!( + r#"IFS= read -r session +printf '%s\n' "$session" > '{}' +printf '%s\n' '{{"jsonrpc":"2.0","id":9,"result":{{"sessionId":"early"}}}}' +"#, + received.display() + ), + ); + let mut child = spawn_relay(temp.path(), &agent); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + let session = format!( + r#"{{"jsonrpc":"2.0","id":9,"method":"session/new","params":{{"cwd":"{}","mcpServers":[]}}}}"#, + temp.path().display() + ); + writeln!(stdin, "{session}").unwrap(); + let _ = read_json(&mut stdout); + drop(stdin); + assert!(child.wait().unwrap().success()); + assert_eq!( + fs::read_to_string(received).unwrap(), + format!("{session}\n") + ); +} + +#[test] +fn duplicate_initialize_is_not_treated_as_a_second_negotiation() { + let temp = workspace(); + let second_response = r#"{"jsonrpc":"2.0","id":2,"result":{"protocolVersion":1,"agentCapabilities":{},"_meta":{"agent":"second"}}}"#; + let agent = write_agent( + temp.path(), + "duplicate-agent.sh", + &format!( + r#"IFS= read -r first +printf '%s\n' '{{"jsonrpc":"2.0","id":1,"result":{{"protocolVersion":1,"agentCapabilities":{{}}}}}}' +IFS= read -r second +printf '%s\n' '{}' +"#, + second_response + ), + ); + let mut child = spawn_relay(temp.path(), &agent); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + writeln!(stdin, r#"{{"jsonrpc":"2.0","id":1,"method":"initialize","params":{{"protocolVersion":1,"clientCapabilities":{{}}}}}}"#).unwrap(); + assert_eq!( + read_json(&mut stdout)["result"]["_meta"]["trail"]["relay"], + true + ); + writeln!(stdin, r#"{{"jsonrpc":"2.0","id":2,"method":"initialize","params":{{"protocolVersion":1,"clientCapabilities":{{}}}}}}"#).unwrap(); + let mut raw = String::new(); + stdout.read_line(&mut raw).unwrap(); + assert_eq!(raw, format!("{second_response}\n")); + drop(stdin); + assert!(child.wait().unwrap().success()); +} + +#[test] +fn invalid_initialize_metadata_candidate_rolls_back_to_raw_bytes() { + let temp = workspace(); + let response = r#"{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{},"_meta":"opaque"}}"#; + let agent = write_agent( + temp.path(), + "invalid-meta-agent.sh", + &format!( + r#"IFS= read -r initialize +printf '%s\n' '{}' +"#, + response + ), + ); + let mut child = spawn_relay(temp.path(), &agent); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + writeln!(stdin, r#"{{"jsonrpc":"2.0","id":1,"method":"initialize","params":{{"protocolVersion":1,"clientCapabilities":{{}}}}}}"#).unwrap(); + let mut raw = String::new(); + stdout.read_line(&mut raw).unwrap(); + assert_eq!(raw, format!("{response}\n")); + drop(stdin); + assert!(child.wait().unwrap().success()); +} + +#[test] +fn mcp_injection_preserves_same_name_servers_and_appends_trail_identity() { + let temp = workspace(); + let received = temp.path().join(".trail/received.jsonl"); + let agent = write_agent( + temp.path(), + "mcp-dedup-agent.sh", + &format!( + r#"IFS= read -r initialize +printf '%s\n' '{{"jsonrpc":"2.0","id":1,"result":{{"protocolVersion":1,"agentCapabilities":{{}}}}}}' +IFS= read -r session +printf '%s\n' "$session" > '{}' +printf '%s\n' '{{"jsonrpc":"2.0","id":2,"result":{{"sessionId":"s"}}}}' +"#, + received.display() + ), + ); + let mut child = spawn_relay(temp.path(), &agent); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + writeln!(stdin, r#"{{"jsonrpc":"2.0","id":1,"method":"initialize","params":{{"protocolVersion":1,"clientCapabilities":{{}}}}}}"#).unwrap(); + let _ = read_json(&mut stdout); + writeln!( + stdin, + r#"{{"jsonrpc":"2.0","id":2,"method":"session/new","params":{{"cwd":"{}","mcpServers":[{{"name":"first","command":"/bin/first","args":[],"env":[]}},{{"name":"trail","command":"/custom/trail","args":["mcp"],"env":[]}}]}}}}"#, + temp.path().display() + ) + .unwrap(); + let _ = read_json(&mut stdout); + drop(stdin); + assert!(child.wait().unwrap().success()); + + let forwarded: Value = + serde_json::from_str(fs::read_to_string(received).unwrap().trim()).unwrap(); + let servers = forwarded["params"]["mcpServers"].as_array().unwrap(); + assert_eq!(servers.len(), 3); + assert_eq!(servers[0]["name"], "first"); + assert_eq!(servers[1]["name"], "trail"); + assert_eq!(servers[1]["command"], "/custom/trail"); + assert_eq!(servers[2]["name"], "trail"); + assert_eq!(servers[2]["args"], serde_json::json!(["mcp"])); +} + +#[test] +fn invalid_mcp_candidate_rolls_back_the_entire_session_transform() { + let temp = workspace(); + let received = temp.path().join(".trail/received.jsonl"); + let agent = write_agent( + temp.path(), + "invalid-mcp-agent.sh", + &format!( + r#"IFS= read -r initialize +printf '%s\n' '{{"jsonrpc":"2.0","id":1,"result":{{"protocolVersion":1,"agentCapabilities":{{}}}}}}' +IFS= read -r session +printf '%s\n' "$session" > '{}' +printf '%s\n' '{{"jsonrpc":"2.0","id":2,"result":{{"sessionId":"s"}}}}' +"#, + received.display() + ), + ); + let mut child = spawn_relay(temp.path(), &agent); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + writeln!(stdin, r#"{{"jsonrpc":"2.0","id":1,"method":"initialize","params":{{"protocolVersion":1,"clientCapabilities":{{}}}}}}"#).unwrap(); + let _ = read_json(&mut stdout); + let session = format!( + r#"{{"jsonrpc":"2.0","id":2,"method":"session/new","params":{{"cwd":"{}","mcpServers":{{"invalid":true}}}}}}"#, + temp.path().display() + ); + writeln!(stdin, "{session}").unwrap(); + let _ = read_json(&mut stdout); + drop(stdin); + assert!(child.wait().unwrap().success()); + assert_eq!( + fs::read_to_string(received).unwrap(), + format!("{session}\n") + ); +} diff --git a/trail/tests/acp_transport.rs b/trail/tests/acp_transport.rs new file mode 100644 index 0000000..0aef1a7 --- /dev/null +++ b/trail/tests/acp_transport.rs @@ -0,0 +1,176 @@ +#![cfg(unix)] + +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use trail::{InitImportMode, Trail}; + +fn trail_bin() -> PathBuf { + option_env!("CARGO_BIN_EXE_trail") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/debug/trail")) +} + +fn workspace() -> tempfile::TempDir { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "transport fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + temp +} + +fn write_agent(workspace: &Path, name: &str, body: &str) -> PathBuf { + let agent = workspace.join(name); + fs::write(&agent, format!("#!/bin/sh\nset -eu\n{body}")).unwrap(); + let mut permissions = fs::metadata(&agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&agent, permissions).unwrap(); + agent +} + +fn spawn_relay(workspace: &Path, agent: &Path) -> Child { + let mut command = Command::new(trail_bin()); + command + .arg("--workspace") + .arg(workspace) + .args(["acp", "relay", "--"]) + .arg(agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + command.spawn().unwrap() +} + +#[test] +fn forwards_unmodified_frames_byte_exactly() { + let temp = workspace(); + let agent = write_agent( + temp.path(), + "echo-acp-agent.sh", + "IFS= read -r frame\nprintf '%s\\n' \"$frame\"\n", + ); + let mut child = spawn_relay(temp.path(), &agent); + let raw = b" { \"method\" : \"ext/byte_exact\", \"params\" : { \"z\" : 1, \"a\" : true }, \"id\" : \"same\", \"jsonrpc\" : \"2.0\" } \n"; + child.stdin.take().unwrap().write_all(raw).unwrap(); + + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "relay failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(output.stdout, raw); +} + +#[test] +fn preserves_crlf_and_keeps_upstream_stderr_off_protocol_stdout() { + let temp = workspace(); + let agent = write_agent( + temp.path(), + "stderr-cat-acp-agent.sh", + "printf '%s\\n' 'agent diagnostic' >&2\nexec /bin/cat\n", + ); + let mut child = spawn_relay(temp.path(), &agent); + let raw = b"{\"jsonrpc\":\"2.0\",\"method\":\"ext/crlf\"}\r\n"; + child.stdin.take().unwrap().write_all(raw).unwrap(); + + let output = child.wait_with_output().unwrap(); + assert!(output.status.success()); + assert_eq!(output.stdout, raw); + assert!(String::from_utf8_lossy(&output.stderr).contains("agent diagnostic")); +} + +#[test] +fn preserves_bidirectional_same_id_traffic_and_out_of_order_responses() { + let temp = workspace(); + let agent = write_agent( + temp.path(), + "concurrent-acp-agent.sh", + r#"IFS= read -r first +IFS= read -r second +printf '%s\n' '{"jsonrpc":"2.0","id":7,"method":"fs/read_text_file","params":{"sessionId":"s","path":"README.md"}}' +printf '%s\n' '{"jsonrpc":"2.0","id":2,"result":{"sessionId":"second"}}' +printf '%s\n' '{"jsonrpc":"2.0","id":1,"result":{"sessionId":"first"}}' +IFS= read -r callback_response +test "$callback_response" = '{"jsonrpc":"2.0","id":7,"result":{"content":"fixture"}}' +"#, + ); + let mut child = spawn_relay(temp.path(), &agent); + let mut stdin = child.stdin.take().unwrap(); + stdin + .write_all( + b"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"ext/first\"}\n{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"ext/second\"}\n", + ) + .unwrap(); + let stdout = child.stdout.take().unwrap(); + let mut stdout = BufReader::new(stdout); + let mut lines = Vec::new(); + for _ in 0..3 { + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + lines.push(serde_json::from_str::(line.trim()).unwrap()); + } + assert_eq!(lines[0]["method"], "fs/read_text_file"); + assert_eq!(lines[0]["id"], 7); + assert_eq!(lines[1]["id"], 2); + assert_eq!(lines[2]["id"], 1); + stdin + .write_all(b"{\"jsonrpc\":\"2.0\",\"id\":7,\"result\":{\"content\":\"fixture\"}}\n") + .unwrap(); + drop(stdin); + + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "relay failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn malformed_agent_frame_fails_without_polluting_stdout() { + let temp = workspace(); + let agent = write_agent( + temp.path(), + "malformed-acp-agent.sh", + "printf '%s\\n' '{not-json}'\n", + ); + let child = spawn_relay(temp.path(), &agent); + + let output = child.wait_with_output().unwrap(); + assert!(!output.status.success()); + assert!(output.stdout.is_empty()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("I/O error") && stderr.contains("line 1 column"), + "unexpected malformed-frame diagnostic: {stderr}" + ); +} + +#[test] +fn editor_eof_bounds_an_unresponsive_agent_shutdown() { + let temp = workspace(); + let agent = write_agent( + temp.path(), + "sleeping-acp-agent.sh", + "printf '%s\\n' '{\"jsonrpc\":\"2.0\",\"method\":\"ext/ready\"}'\nIFS= read -r frame || true\nsleep 10\n", + ); + let mut child = spawn_relay(temp.path(), &agent); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + let mut ready = String::new(); + stdout.read_line(&mut ready).unwrap(); + assert_eq!(ready, "{\"jsonrpc\":\"2.0\",\"method\":\"ext/ready\"}\n"); + drop(child.stdin.take().unwrap()); + let started = Instant::now(); + let status = child.wait().unwrap(); + + let elapsed = started.elapsed(); + assert!( + elapsed < Duration::from_secs(3), + "relay process shutdown took {elapsed:?}" + ); + assert!(status.success()); +} diff --git a/trail/tests/acp_turn_semantics.rs b/trail/tests/acp_turn_semantics.rs new file mode 100644 index 0000000..0a6d958 --- /dev/null +++ b/trail/tests/acp_turn_semantics.rs @@ -0,0 +1,277 @@ +#![cfg(unix)] + +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use trail::{InitImportMode, Trail}; + +fn trail_bin() -> PathBuf { + std::env::var_os("TRAIL_TEST_BIN") + .map(PathBuf::from) + .or_else(|| option_env!("CARGO_BIN_EXE_trail").map(PathBuf::from)) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/debug/trail")) +} + +fn write_turn_agent(workspace: &Path) -> PathBuf { + let agent = workspace.join("turn-agent.py"); + fs::write( + &agent, + r#"#!/usr/bin/env python3 +import json +import sys + +pending_prompt = None +config_values = {} +for line in sys.stdin: + message = json.loads(line) + method = message.get("method") + request_id = message.get("id") + if method == "initialize": + result = {"protocolVersion":1,"agentCapabilities":{"promptCapabilities":{"image":True,"audio":True,"embeddedContext":True},"sessionCapabilities":{}}} + print(json.dumps({"jsonrpc":"2.0","id":request_id,"result":result}, separators=(",", ":")), flush=True) + elif method == "session/new": + print(json.dumps({"jsonrpc":"2.0","id":request_id,"result":{"sessionId":"turn-session"}}, separators=(",", ":")), flush=True) + print(json.dumps({"jsonrpc":"2.0","id":"agent-callback","method":"session/request_permission","params":{"sessionId":"turn-session","toolCall":{"toolCallId":"permission-tool","title":"Permission fixture"},"options":[{"optionId":"allow","name":"Allow","kind":"allow_once"}]}}, separators=(",", ":")), flush=True) + print(json.dumps({"jsonrpc":"2.0","method":"$/cancel_request","params":{"requestId":"agent-callback"}}, separators=(",", ":")), flush=True) + elif method in ("session/set_mode", "session/set_config_option"): + if str(request_id).endswith("-error"): + response = {"jsonrpc":"2.0","id":request_id,"error":{"code":-32002,"message":"configuration rejected"}} + else: + result = {} + if method == "session/set_config_option": + config_id = message["params"]["configId"] + value = message["params"]["value"] + config_values[config_id] = "deep" if config_id == "model" else value + result["configOptions"] = [ + {"id":key,"name":key,"type":"boolean","currentValue":current} + if isinstance(current, bool) + else {"id":key,"name":key,"type":"select","currentValue":current,"options":[{"value":current,"name":current}]} + for key, current in config_values.items() + ] + response = {"jsonrpc":"2.0","id":request_id,"result":result} + print(json.dumps(response, separators=(",", ":")), flush=True) + elif method == "session/prompt" and request_id == "prompt-cancel": + pending_prompt = request_id + print(json.dumps({"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"turn-session","update":{"sessionUpdate":"agent_message_chunk","messageId":"cancelled-answer","content":{"type":"text","text":"partial answer"}}}}, separators=(",", ":")), flush=True) + elif method == "session/cancel": + pass + elif method == "$/cancel_request": + if pending_prompt is not None: + print(json.dumps({"jsonrpc":"2.0","id":pending_prompt,"error":{"code":-32800,"message":"request cancelled"}}, separators=(",", ":")), flush=True) + pending_prompt = None + elif method == "session/prompt": + print(json.dumps({"jsonrpc":"2.0","id":request_id,"result":{"stopReason":"end_turn"}}, separators=(",", ":")), flush=True) + elif method is None: + pass +"#, + ) + .unwrap(); + let mut permissions = fs::metadata(&agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&agent, permissions).unwrap(); + agent +} + +fn send(stdin: &mut impl Write, message: &serde_json::Value) { + serde_json::to_writer(&mut *stdin, message).unwrap(); + stdin.write_all(b"\n").unwrap(); + stdin.flush().unwrap(); +} + +fn receive(stdout: &mut impl BufRead) -> serde_json::Value { + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + assert!(!line.is_empty()); + serde_json::from_str(line.trim()).unwrap() +} + +fn exchange( + stdin: &mut impl Write, + stdout: &mut impl BufRead, + message: serde_json::Value, +) -> serde_json::Value { + send(stdin, &message); + receive(stdout) +} + +#[test] +fn prompt_configuration_and_both_cancellation_directions_are_projected() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "turn fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let agent = write_turn_agent(temp.path()); + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["acp", "relay", "--"]) + .arg(agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + + exchange( + &mut stdin, + &mut stdout, + serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":1,"clientCapabilities":{}}}), + ); + exchange( + &mut stdin, + &mut stdout, + serde_json::json!({"jsonrpc":"2.0","id":2,"method":"session/new","params":{"cwd":temp.path(),"mcpServers":[]}}), + ); + let permission = receive(&mut stdout); + assert_eq!(permission["method"], "session/request_permission"); + let agent_cancel = receive(&mut stdout); + assert_eq!(agent_cancel["method"], "$/cancel_request"); + send( + &mut stdin, + &serde_json::json!({"jsonrpc":"2.0","id":"agent-callback","result":{"outcome":{"outcome":"cancelled"}}}), + ); + + for (id, method, params, succeeds) in [ + ( + "mode-error", + "session/set_mode", + serde_json::json!({"sessionId":"turn-session","modeId":"ask"}), + false, + ), + ( + "mode-success", + "session/set_mode", + serde_json::json!({"sessionId":"turn-session","modeId":"code"}), + true, + ), + ( + "config-bool-success", + "session/set_config_option", + serde_json::json!({"sessionId":"turn-session","configId":"autoFix","type":"boolean","value":true}), + true, + ), + ( + "config-select-success", + "session/set_config_option", + serde_json::json!({"sessionId":"turn-session","configId":"model","value":"fast"}), + true, + ), + ] { + let response = exchange( + &mut stdin, + &mut stdout, + serde_json::json!({"jsonrpc":"2.0","id":id,"method":method,"params":params}), + ); + assert_eq!(response.get("result").is_some(), succeeds); + } + + let prompt = serde_json::json!([ + {"type":"text","text":"Explain the attached context","_meta":{"extension":"text-meta"}}, + {"type":"image","data":"aW1hZ2U=","mimeType":"image/png","_meta":{"extension":"image-meta"}}, + {"type":"audio","data":"YXVkaW8=","mimeType":"audio/wav"}, + {"type":"resource_link","name":"README","uri":"file:///README.md","_meta":{"extension":"link-meta"}}, + {"type":"resource","resource":{"uri":"file:///context.txt","mimeType":"text/plain","text":"embedded context"},"_meta":{"extension":"resource-meta"}} + ]); + send( + &mut stdin, + &serde_json::json!({"jsonrpc":"2.0","id":"prompt-cancel","method":"session/prompt","params":{"sessionId":"turn-session","prompt":prompt}}), + ); + let update = receive(&mut stdout); + assert_eq!(update["method"], "session/update"); + send( + &mut stdin, + &serde_json::json!({"jsonrpc":"2.0","method":"session/cancel","params":{"sessionId":"turn-session"}}), + ); + send( + &mut stdin, + &serde_json::json!({"jsonrpc":"2.0","method":"$/cancel_request","params":{"requestId":"prompt-cancel"}}), + ); + let cancelled = receive(&mut stdout); + assert_eq!(cancelled["error"]["code"], -32800); + + let completed = exchange( + &mut stdin, + &mut stdout, + serde_json::json!({"jsonrpc":"2.0","id":"prompt-complete","method":"session/prompt","params":{"sessionId":"turn-session","prompt":[{"type":"text","text":"Second prompt"}]}}), + ); + assert_eq!(completed["result"]["stopReason"], "end_turn"); + send( + &mut stdin, + &serde_json::json!({"jsonrpc":"2.0","method":"$/cancel_request","params":{"requestId":"prompt-complete"}}), + ); + let reused_id = exchange( + &mut stdin, + &mut stdout, + serde_json::json!({"jsonrpc":"2.0","id":"prompt-complete","method":"session/set_config_option","params":{"sessionId":"turn-session","configId":"afterCancel","value":true}}), + ); + assert!(reused_id.get("result").is_some()); + + drop(stdin); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "relay failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let db = Trail::open(temp.path()).unwrap(); + let mapping = db.lane_acp_session("turn-session").unwrap(); + assert_eq!(mapping.current_mode_id.as_deref(), Some("code")); + assert_eq!(mapping.config_options["autoFix"], true); + assert_eq!(mapping.config_options["model"], "deep"); + assert_eq!(mapping.config_options["afterCancel"], true); + let session = db.show_lane_session(&mapping.trail_session_id).unwrap(); + assert_eq!(session.turns.len(), 2); + let statuses = session + .turns + .iter() + .map(|turn| turn.status.as_str()) + .collect::>(); + assert!(statuses.contains("cancelled")); + assert!(statuses.contains("completed")); + let content_event = session + .events + .iter() + .find(|event| event.event_type == "acp_prompt_content") + .and_then(|event| event.payload.as_ref()) + .unwrap(); + assert_eq!(content_event["blocks"].as_array().unwrap().len(), 5); + assert_eq!( + content_event["blocks"][0]["_meta"]["extension"], + "text-meta" + ); + assert_eq!( + content_event["blocks"][3]["_meta"]["extension"], + "link-meta" + ); + assert_eq!( + content_event["blocks"][4]["resource"]["text"], + "embedded context" + ); + assert_eq!(content_event["blocks"][1]["data"]["encoding"], "base64"); + assert_eq!(content_event["blocks"][2]["data"]["encoding"], "base64"); + assert!(!serde_json::to_string(content_event) + .unwrap() + .contains("aW1hZ2U=")); + let cancel_events = session + .events + .iter() + .filter(|event| event.event_type == "acp_request_cancelled") + .count(); + assert!( + cancel_events >= 3, + "missing cancellation direction/race evidence" + ); + let reused_id_event = session + .events + .iter() + .filter(|event| event.event_type == "acp_session_config_changed") + .filter_map(|event| event.payload.as_ref()) + .find(|payload| payload["config_id"] == "afterCancel") + .unwrap(); + assert_eq!(reused_id_event["cancel_requested"], false); +} diff --git a/trail/tests/acp_update_semantics.rs b/trail/tests/acp_update_semantics.rs new file mode 100644 index 0000000..93f6d12 --- /dev/null +++ b/trail/tests/acp_update_semantics.rs @@ -0,0 +1,318 @@ +#![cfg(unix)] + +use std::collections::BTreeSet; +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use serde_json::Value; +use trail::{InitImportMode, Trail}; + +const FIXTURES: &str = include_str!("fixtures/acp/v1/session_updates.jsonl"); +const SCHEMA: &str = include_str!("fixtures/acp/v1/schema.json"); + +fn trail_bin() -> PathBuf { + std::env::var_os("TRAIL_TEST_BIN") + .map(PathBuf::from) + .or_else(|| option_env!("CARGO_BIN_EXE_trail").map(PathBuf::from)) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/debug/trail")) +} + +fn fixture_messages() -> Vec { + FIXTURES + .lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} + +fn update_kind(message: &Value) -> &str { + message["params"]["update"]["sessionUpdate"] + .as_str() + .unwrap() +} + +fn values_at(messages: &[Value], field: &str) -> BTreeSet { + messages + .iter() + .filter_map(|message| message["params"]["update"].get(field)) + .flat_map(|value| { + value + .as_array() + .cloned() + .unwrap_or_else(|| vec![value.clone()]) + }) + .filter_map(|value| value.as_str().map(str::to_string)) + .collect() +} + +#[test] +fn fixture_inventory_exactly_covers_the_pinned_session_update_union() { + let schema: Value = serde_json::from_str(SCHEMA).unwrap(); + let validator = jsonschema::validator_for(&schema).unwrap(); + let messages = fixture_messages(); + for message in &messages { + if let Err(error) = validator.validate(message) { + panic!( + "{} fixture is not schema-valid: {error}", + update_kind(message) + ); + } + } + + let pinned = schema["$defs"]["SessionUpdate"]["oneOf"] + .as_array() + .unwrap() + .iter() + .map(|branch| { + branch["properties"]["sessionUpdate"]["const"] + .as_str() + .unwrap() + .to_string() + }) + .collect::>(); + let covered = messages + .iter() + .map(update_kind) + .map(str::to_string) + .collect::>(); + assert_eq!(covered, pinned); + + assert_eq!( + values_at(&messages, "kind"), + [ + "delete", + "edit", + "execute", + "fetch", + "move", + "other", + "read", + "search", + "switch_mode", + "think", + ] + .into_iter() + .map(str::to_string) + .collect() + ); + assert_eq!( + values_at(&messages, "status"), + ["completed", "failed", "in_progress", "pending"] + .into_iter() + .map(str::to_string) + .collect() + ); + + let tool_content = messages + .iter() + .flat_map(|message| { + message["params"]["update"]["content"] + .as_array() + .cloned() + .unwrap_or_default() + }) + .filter_map(|content| content["type"].as_str().map(str::to_string)) + .collect::>(); + assert_eq!( + tool_content, + ["content", "diff", "terminal"] + .into_iter() + .map(str::to_string) + .collect() + ); + + let plan_entries = messages + .iter() + .find(|message| update_kind(message) == "plan") + .unwrap()["params"]["update"]["entries"] + .as_array() + .unwrap(); + assert_eq!( + plan_entries + .iter() + .filter_map(|entry| entry["priority"].as_str()) + .collect::>(), + ["high", "low", "medium"].into_iter().collect() + ); + assert_eq!( + plan_entries + .iter() + .filter_map(|entry| entry["status"].as_str()) + .collect::>(), + ["completed", "in_progress", "pending"] + .into_iter() + .collect() + ); +} + +fn write_update_agent(workspace: &Path) -> PathBuf { + let fixture = workspace.join("session_updates.jsonl"); + fs::write(&fixture, FIXTURES).unwrap(); + let agent = workspace.join("update-agent.py"); + let source = format!( + r#"#!/usr/bin/env python3 +import json +import sys + +fixture = {fixture:?} +for line in sys.stdin: + message = json.loads(line) + method = message.get("method") + request_id = message.get("id") + if method == "initialize": + print(json.dumps({{"jsonrpc":"2.0","id":request_id,"result":{{"protocolVersion":1,"agentCapabilities":{{}}}}}}, separators=(",", ":")), flush=True) + elif method == "session/new": + result = {{"sessionId":"update-session","modes":{{"currentModeId":"ask","availableModes":[{{"id":"ask","name":"Ask"}},{{"id":"code","name":"Code"}}]}},"configOptions":[{{"id":"stale","name":"Stale","type":"boolean","currentValue":True}}]}} + print(json.dumps({{"jsonrpc":"2.0","id":request_id,"result":result}}, separators=(",", ":")), flush=True) + elif method == "session/prompt": + with open(fixture, "r", encoding="utf-8") as updates: + for update in updates: + print(update.rstrip("\\n"), flush=True) + print(json.dumps({{"jsonrpc":"2.0","method":"session/update","params":{{"sessionId":"update-session","update":{{"sessionUpdate":"vendor_progress","phase":"indexing","token":"must-redact"}}}}}}, separators=(",", ":")), flush=True) + print(json.dumps({{"jsonrpc":"2.0","id":request_id,"result":{{"stopReason":"end_turn"}}}}, separators=(",", ":")), flush=True) +"#, + fixture = fixture.to_string_lossy() + ); + fs::write(&agent, source).unwrap(); + let mut permissions = fs::metadata(&agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&agent, permissions).unwrap(); + agent +} + +fn send(stdin: &mut impl Write, message: &Value) { + serde_json::to_writer(&mut *stdin, message).unwrap(); + stdin.write_all(b"\n").unwrap(); + stdin.flush().unwrap(); +} + +fn receive(stdout: &mut impl BufRead) -> Value { + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + assert!(!line.is_empty()); + serde_json::from_str(line.trim()).unwrap() +} + +fn exchange(stdin: &mut impl Write, stdout: &mut impl BufRead, message: Value) -> Value { + send(stdin, &message); + receive(stdout) +} + +#[test] +fn every_session_update_has_ordered_redacted_semantic_projection() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "update fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let agent = write_update_agent(temp.path()); + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["acp", "relay", "--"]) + .arg(agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + + exchange( + &mut stdin, + &mut stdout, + serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":1,"clientCapabilities":{}}}), + ); + exchange( + &mut stdin, + &mut stdout, + serde_json::json!({"jsonrpc":"2.0","id":2,"method":"session/new","params":{"cwd":temp.path(),"mcpServers":[]}}), + ); + send( + &mut stdin, + &serde_json::json!({"jsonrpc":"2.0","id":3,"method":"session/prompt","params":{"sessionId":"update-session","prompt":[{"type":"text","text":"Emit update fixtures"}]}}), + ); + loop { + let message = receive(&mut stdout); + if message.get("id") == Some(&Value::from(3)) { + assert_eq!(message["result"]["stopReason"], "end_turn"); + break; + } + } + drop(stdin); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "relay failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let db = Trail::open(temp.path()).unwrap(); + let mapping = db.lane_acp_session("update-session").unwrap(); + assert_eq!(mapping.current_mode_id.as_deref(), Some("code")); + assert_eq!(mapping.config_options["model"], "fast"); + assert_eq!(mapping.config_options["autoFix"], true); + assert_eq!(mapping.config_options.as_object().unwrap().len(), 2); + let session = db.show_lane_session(&mapping.trail_session_id).unwrap(); + assert_eq!( + session.session.title.as_deref(), + Some("Schema fixture session") + ); + + let raw_updates = session + .events + .iter() + .filter(|event| event.event_type == "acp_session_update") + .filter_map(|event| event.payload.as_ref()) + .collect::>(); + assert!(session + .events + .iter() + .any(|event| event.event_type == "acp_session_configuration")); + let stable_order = raw_updates + .iter() + .filter(|payload| payload["stable"] == true) + .filter_map(|payload| payload["acpVariant"].as_str()) + .collect::>(); + let expected_order = fixture_messages() + .iter() + .map(update_kind) + .map(str::to_string) + .collect::>(); + assert_eq!(stable_order, expected_order); + let thought = raw_updates + .iter() + .find(|payload| payload["acpVariant"] == "agent_thought_chunk") + .unwrap(); + assert_eq!(thought["thoughtContentExcluded"], true); + assert!(thought["update"].get("content").is_none()); + assert!(!thought.to_string().contains("private chain of thought")); + let extension = raw_updates + .iter() + .find(|payload| payload["acpVariant"] == "vendor_progress") + .unwrap(); + assert_eq!(extension["stable"], false); + assert!(!extension.to_string().contains("must-redact")); + + assert!(session + .messages + .iter() + .any(|message| message.role == "assistant" && message.body.contains("streamed agent"))); + for event_type in [ + "tool_call", + "tool_call_update", + "plan_update", + "acp_available_commands_update", + "acp_usage_update", + ] { + assert!( + session + .events + .iter() + .any(|event| event.event_type == event_type), + "missing {event_type} projection" + ); + } +} diff --git a/trail/tests/acp_workspace_mapping.rs b/trail/tests/acp_workspace_mapping.rs new file mode 100644 index 0000000..cf81549 --- /dev/null +++ b/trail/tests/acp_workspace_mapping.rs @@ -0,0 +1,252 @@ +#![cfg(unix)] + +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use serde_json::Value; +use trail::{InitImportMode, Trail}; + +fn trail_bin() -> PathBuf { + std::env::var_os("TRAIL_TEST_BIN") + .map(PathBuf::from) + .or_else(|| option_env!("CARGO_BIN_EXE_trail").map(PathBuf::from)) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/debug/trail")) +} + +fn write_agent(workspace: &Path, received: &Path) -> PathBuf { + let agent = workspace.join("mapping-agent.sh"); + fs::write( + &agent, + format!( + r#"#!/bin/sh +set -eu +IFS= read -r initialize +printf '%s\n' '{{"jsonrpc":"2.0","id":1,"result":{{"protocolVersion":1,"agentCapabilities":{{}}}}}}' +IFS= read -r session +printf '%s\n' "$session" > '{}' +printf '%s\n' '{{"jsonrpc":"2.0","id":2,"result":{{"sessionId":"mapped"}}}}' +"#, + received.display() + ), + ) + .unwrap(); + let mut permissions = fs::metadata(&agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&agent, permissions).unwrap(); + agent +} + +#[test] +fn maps_nested_cwd_without_collapsing_to_lane_root() { + let temp = tempfile::tempdir().unwrap(); + fs::create_dir_all(temp.path().join("packages/app")).unwrap(); + fs::write(temp.path().join("README.md"), "mapping fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let received = temp.path().join(".trail/received.json"); + let agent = write_agent(temp.path(), &received); + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args([ + "acp", + "relay", + "--lane", + "mapping-test", + "--materialize", + "--", + ]) + .arg(agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + writeln!(stdin, r#"{{"jsonrpc":"2.0","id":1,"method":"initialize","params":{{"protocolVersion":1,"clientCapabilities":{{}}}}}}"#).unwrap(); + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + assert!(!line.is_empty()); + writeln!( + stdin, + r#"{{"jsonrpc":"2.0","id":2,"method":"session/new","params":{{"cwd":"{}","mcpServers":[]}}}}"#, + temp.path().join("packages/app").display() + ) + .unwrap(); + line.clear(); + stdout.read_line(&mut line).unwrap(); + assert!(!line.is_empty()); + drop(stdin); + assert!(child.wait().unwrap().success()); + + let db = Trail::open(temp.path()).unwrap(); + let lane = db.lane_details("mapping-test").unwrap(); + let lane_root = PathBuf::from(lane.branch.workdir.unwrap()); + let forwarded: Value = serde_json::from_slice(&fs::read(received).unwrap()).unwrap(); + assert_eq!( + PathBuf::from(forwarded["params"]["cwd"].as_str().unwrap()), + lane_root.join("packages/app") + ); +} + +#[test] +fn maps_and_persists_normalized_additional_roots_without_reordering() { + let temp = tempfile::tempdir().unwrap(); + let external = tempfile::tempdir().unwrap(); + fs::create_dir_all(temp.path().join("packages/app")).unwrap(); + fs::create_dir_all(temp.path().join("shared")).unwrap(); + fs::write(temp.path().join("README.md"), "mapping fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let received = temp.path().join(".trail/received.json"); + let agent = write_agent(temp.path(), &received); + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args([ + "acp", + "relay", + "--lane", + "mapping-matrix", + "--materialize", + "--", + ]) + .arg(agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + writeln!(stdin, r#"{{"jsonrpc":"2.0","id":1,"method":"initialize","params":{{"protocolVersion":1,"clientCapabilities":{{}}}}}}"#).unwrap(); + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + assert!(!line.is_empty()); + let cwd = temp.path().join("packages/./app/../app"); + let shared = temp.path().join("shared"); + let request = serde_json::json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "session/new", + "params": { + "cwd": cwd, + "mcpServers": [], + "additionalDirectories": [ + shared, + temp.path().join("shared/."), + temp.path().join("packages"), + external.path(), + external.path(), + "C:\\external\\repo", + "\\\\server\\share" + ] + } + }); + serde_json::to_writer(&mut stdin, &request).unwrap(); + stdin.write_all(b"\n").unwrap(); + line.clear(); + stdout.read_line(&mut line).unwrap(); + assert!(!line.is_empty()); + drop(stdin); + assert!(child.wait().unwrap().success()); + + let db = Trail::open(temp.path()).unwrap(); + let lane = db.lane_details("mapping-matrix").unwrap(); + let lane_root = PathBuf::from(lane.branch.workdir.unwrap()); + let forwarded: Value = serde_json::from_slice(&fs::read(received).unwrap()).unwrap(); + assert_eq!( + forwarded["params"]["cwd"], + lane_root.join("packages/app").to_string_lossy().as_ref() + ); + assert_eq!( + forwarded["params"]["additionalDirectories"], + serde_json::json!([ + lane_root.join("shared").to_string_lossy(), + lane_root.join("packages").to_string_lossy(), + external.path().to_string_lossy(), + "C:\\external\\repo", + "\\\\server\\share" + ]) + ); + + let mapping = db.lane_acp_session("mapped").unwrap(); + assert_eq!(mapping.path_mappings.len(), 6); + assert_eq!(mapping.path_mappings[0].original, cwd.to_string_lossy()); + assert!(mapping.path_mappings[0].isolated); + assert!(mapping.path_mappings[1].isolated); + assert!(mapping.path_mappings[2].isolated); + assert!(!mapping.path_mappings[3].isolated); + assert!(!mapping.path_mappings[4].isolated); + assert!(!mapping.path_mappings[5].isolated); +} + +#[test] +fn symlink_escape_is_preserved_as_an_external_root() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let external = tempfile::tempdir().unwrap(); + fs::create_dir_all(external.path().join("nested")).unwrap(); + symlink(external.path(), temp.path().join("escape")).unwrap(); + fs::write(temp.path().join("README.md"), "mapping fixture\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let received = temp.path().join(".trail/received.json"); + let agent = write_agent(temp.path(), &received); + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args([ + "acp", + "relay", + "--lane", + "mapping-symlink", + "--materialize", + "--", + ]) + .arg(agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + writeln!(stdin, r#"{{"jsonrpc":"2.0","id":1,"method":"initialize","params":{{"protocolVersion":1,"clientCapabilities":{{}}}}}}"#).unwrap(); + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + let escaped = temp.path().join("escape/nested"); + let request = serde_json::json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "session/new", + "params": { + "cwd": temp.path(), + "mcpServers": [], + "additionalDirectories": [escaped] + } + }); + serde_json::to_writer(&mut stdin, &request).unwrap(); + stdin.write_all(b"\n").unwrap(); + line.clear(); + stdout.read_line(&mut line).unwrap(); + drop(stdin); + assert!(child.wait().unwrap().success()); + + let forwarded: Value = serde_json::from_slice(&fs::read(received).unwrap()).unwrap(); + assert_eq!( + forwarded["params"]["additionalDirectories"][0], + escaped.to_string_lossy().as_ref() + ); + let db = Trail::open(temp.path()).unwrap(); + let mapping = db.lane_acp_session("mapped").unwrap(); + let escaped_mapping = mapping + .path_mappings + .iter() + .find(|mapping| mapping.original == escaped.to_string_lossy()) + .unwrap(); + assert!(!escaped_mapping.isolated); + assert_eq!(escaped_mapping.effective, escaped.to_string_lossy()); +} diff --git a/trail/tests/changed_path_ledger_activation.rs b/trail/tests/changed_path_ledger_activation.rs new file mode 100644 index 0000000..62cc8bb --- /dev/null +++ b/trail/tests/changed_path_ledger_activation.rs @@ -0,0 +1,242 @@ +#![cfg(debug_assertions)] + +use std::fs; +use std::path::Path; +use std::process::Command; +use std::sync::{Mutex, MutexGuard, OnceLock}; +use trail::Actor; +use trail::{InitImportMode, Trail}; + +static ACTIVATION_STATE: OnceLock> = OnceLock::new(); + +fn serial() -> MutexGuard<'static, ()> { + ACTIVATION_STATE + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) +} + +fn git(root: &Path, args: &[&str]) { + let output = Command::new("git") + .arg("-C") + .arg(root) + .args(args) + .env("GIT_OPTIONAL_LOCKS", "0") + .output() + .unwrap(); + assert!( + output.status.success(), + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn authority_requires_every_checked_gate_and_supported_platform() { + let complete = trail::test_support::changed_path_activation_evidence().unwrap(); + for gate in [ + "schema_hard_cutover", + "producer_inventory_complete", + "linux_native_suite", + "macos_native_suite", + "crash_matrix", + "corruption_matrix", + "scale_gates", + "metrics_jsonl", + "exact_sha_tag_gate", + "exact_sha_publish_gate", + ] { + assert_eq!(complete[gate], true, "activation gate `{gate}` is absent"); + } + assert_eq!( + complete["producer_inventory_sha256"], + "67027c4bcfba0f3105833978637fe5e81c9cbdb43ba51cbd1a58026a2e067185" + ); + assert_eq!( + complete["raw_mutation_inventory_sha256"], + "668a39497a58335e57b02a0c6fff3e0a0c127b06e0aa63d4cf93255f3942c943" + ); + assert_eq!( + complete["activation_audit_sha256"], + "f538c5750f234ed5b164536ce0603dc92df17324429c7487e225f789a0e27c70" + ); + assert!(!trail::test_support::changed_path_authority_enabled_for("windows").unwrap()); + assert!(!trail::test_support::changed_path_authority_enabled_for("freebsd").unwrap()); + assert_eq!( + trail::test_support::changed_path_production_authority_default(), + cfg!(any(target_os = "linux", target_os = "macos")) + ); +} + +#[test] +fn recovery_corruption_and_native_fault_matrix_remains_fail_closed() { + trail::test_support::changed_path_intent_crash_matrix().unwrap(); + trail::test_support::changed_path_qualified_proof_revalidation().unwrap(); + trail::test_support::changed_path_missing_sidecar_rejection().unwrap(); + trail::test_support::changed_path_ambiguous_recovery_gate().unwrap(); + trail::test_support::changed_path_backup_restore_rotation().unwrap(); + + #[cfg(target_os = "linux")] + { + trail::test_support::changed_path_linux_fault_revocation_matrix().unwrap(); + trail::test_support::changed_path_linux_raw_decoder_faults().unwrap(); + trail::test_support::changed_path_linux_owner_death_and_root_replacement().unwrap(); + trail::test_support::changed_path_linux_unsupported_filesystem_rejection().unwrap(); + } + #[cfg(target_os = "macos")] + { + trail::test_support::changed_path_macos_continuity_fault_matrix().unwrap(); + trail::test_support::changed_path_macos_gap_flag_matrix().unwrap(); + trail::test_support::changed_path_macos_malformed_callbacks().unwrap(); + trail::test_support::changed_path_macos_root_revalidation_failures().unwrap(); + trail::test_support::changed_path_macos_unsupported_filesystem_rejection().unwrap(); + } +} + +#[cfg(target_os = "linux")] +#[test] +fn linux_observer_process_owner_child() { + let Ok(root) = std::env::var("TRAIL_LINUX_OBSERVER_CHILD_ROOT") else { + return; + }; + trail::test_support::changed_path_linux_process_owner_child(&root).unwrap(); +} + +#[cfg(target_os = "macos")] +#[test] +fn fsevents_restart_root_cursor_overflow_and_worker_death_fail_closed() { + if std::env::var_os("TRAIL_MACOS_OBSERVER_OWNER_CHILD_ROOT").is_some() { + trail::test_support::changed_path_macos_continuity_fault_matrix().unwrap(); + } +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn first_authoritative_status_starts_and_reconciles_the_workspace_daemon() { + let _guard = serial(); + let temp = tempfile::tempdir().unwrap(); + git(temp.path(), &["init", "--quiet"]); + git(temp.path(), &["config", "user.name", "Trail Activation"]); + git( + temp.path(), + &["config", "user.email", "trail-activation@example.invalid"], + ); + fs::write(temp.path().join("tracked.txt"), b"base\n").unwrap(); + fs::write(temp.path().join(".gitignore"), b".trail/\n").unwrap(); + git(temp.path(), &["add", "."]); + git(temp.path(), &["commit", "--quiet", "-m", "base"]); + Trail::init(temp.path(), "main", InitImportMode::GitTracked, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + fs::write(temp.path().join("tracked.txt"), b"changed\n").unwrap(); + + trail::test_support::set_changed_path_authority_override(true); + let result = db.status(None); + trail::test_support::set_changed_path_authority_override(false); + let report = result.unwrap(); + assert!( + report + .changed_paths + .iter() + .any(|change| change.path == "tracked.txt"), + "automatic reconciliation omitted the pre-start change" + ); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn activated_non_git_workspace_uses_ledger_without_git_qualification() { + let _guard = serial(); + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("tracked.txt"), b"base\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + fs::write(temp.path().join("tracked.txt"), b"changed\n").unwrap(); + + trail::test_support::set_changed_path_authority_override(true); + let result = (|| { + let status = db.status(None)?; + let diff = db.diff_dirty(false, false)?; + let record = db.record( + Some("main"), + Some("activated non-git record".into()), + Actor::human(), + false, + )?; + Ok::<_, trail::Error>((status, diff, record)) + })(); + trail::test_support::set_changed_path_authority_override(false); + let (status, diff, record) = result.unwrap(); + assert!(status + .changed_paths + .iter() + .any(|path| path.path == "tracked.txt")); + assert!(diff.files.iter().any(|path| path.path == "tracked.txt")); + assert!(record + .changed_paths + .iter() + .any(|path| path.path == "tracked.txt")); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn second_direct_handle_cannot_evict_a_live_workspace_observer() { + let _guard = serial(); + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("tracked.txt"), b"base\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let first = Trail::open(temp.path()).unwrap(); + + trail::test_support::set_changed_path_authority_override(true); + first.status(None).unwrap(); + let second = Trail::open(temp.path()).unwrap(); + let error = second.status(None).unwrap_err(); + assert!( + error + .to_string() + .contains("observer owner is still live; refusing unverified authority replacement"), + "second handle failed for the wrong reason: {error}" + ); + fs::write(temp.path().join("tracked.txt"), b"changed\n").unwrap(); + let report = first.status(None); + trail::test_support::set_changed_path_authority_override(false); + assert!(report + .unwrap() + .changed_paths + .iter() + .any(|path| path.path == "tracked.txt")); +} + +#[test] +fn performance_metrics_file_emits_complete_append_only_jsonl() { + let _guard = serial(); + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("tracked.txt"), b"base\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let metrics = temp.path().join("operation-metrics.jsonl"); + // This test owns the process-global activation lock for the full lifetime + // of the environment mutation and opened Trail handle. + unsafe { std::env::set_var("TRAIL_PERFORMANCE_METRICS_FILE", &metrics) }; + let db = Trail::open(temp.path()).unwrap(); + db.status(None).unwrap(); + let _ = db.diff_range("invalid", false); + unsafe { std::env::remove_var("TRAIL_PERFORMANCE_METRICS_FILE") }; + + let lines = fs::read_to_string(metrics).unwrap(); + let reports = lines + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .collect::>(); + assert_eq!( + reports.len(), + 2, + "one JSON object is required per operation" + ); + assert_eq!(reports[0]["operation"], "status"); + assert_eq!(reports[0]["outcome"], "success"); + assert_eq!(reports[1]["operation"], "diff"); + assert_eq!(reports[1]["outcome"], "error"); + assert_eq!(reports[0]["generation"], 1); + assert_eq!(reports[1]["generation"], 2); + assert!(reports.iter().all(|report| report["wall_time_ns"].is_u64())); +} diff --git a/trail/tests/changed_path_ledger_api.rs b/trail/tests/changed_path_ledger_api.rs new file mode 100644 index 0000000..7212183 --- /dev/null +++ b/trail/tests/changed_path_ledger_api.rs @@ -0,0 +1,463 @@ +use std::fs; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use std::io::{Read, Write}; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use std::net::TcpListener; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use std::path::PathBuf; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use std::process::Command; +#[cfg(any(target_os = "linux", target_os = "macos"))] +use std::thread; + +use trail::{Error, InitImportMode, StructuredErrorEnvelope, Trail}; + +#[cfg(any(target_os = "linux", target_os = "macos"))] +struct AutoDaemonGuard(PathBuf); + +#[cfg(any(target_os = "linux", target_os = "macos"))] +impl Drop for AutoDaemonGuard { + fn drop(&mut self) { + let endpoint = self.0.join(".trail/index/change-ledger/daemon.json"); + if let Ok(bytes) = fs::read(endpoint) { + if let Ok(value) = serde_json::from_slice::(&bytes) { + if let Some(pid) = value["pid"].as_i64() { + unsafe { + libc::kill(pid as i32, libc::SIGTERM); + } + } + } + } + } +} + +fn api_request(method: &str, path: &str, body: serde_json::Value) -> Vec { + api_request_with_headers(method, path, &[], body) +} + +fn api_request_with_headers( + method: &str, + path: &str, + headers: &[(&str, &str)], + body: serde_json::Value, +) -> Vec { + let body = if body.is_null() { + Vec::new() + } else { + serde_json::to_vec(&body).unwrap() + }; + let mut request = format!( + "{method} {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Type: application/json\r\nContent-Length: {}\r\n", + body.len() + ); + for (name, value) in headers { + request.push_str(name); + request.push_str(": "); + request.push_str(value); + request.push_str("\r\n"); + } + request.push_str("\r\n"); + request.into_bytes().into_iter().chain(body).collect() +} + +#[test] +fn reconciliation_report_is_shared_by_rest_and_mcp() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + + let rest = trail::server::handle_http_request( + &mut db, + &api_request("POST", "/v1/index/reconcile", serde_json::json!({})), + ); + assert_eq!(rest.status, 200); + let rest: serde_json::Value = rest.body_json().unwrap(); + assert_eq!(rest["scope_kind"], "workspace"); + assert_eq!(rest["resulting_state"], "trusted"); + assert!(rest["observed_paths"].as_u64().unwrap() >= 1); + + let mcp_temp = tempfile::tempdir().unwrap(); + fs::write(mcp_temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(mcp_temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut mcp_db = Trail::open(mcp_temp.path()).unwrap(); + let mcp = trail::mcp::handle_json_rpc( + &mut mcp_db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "trail.index_reconcile", + "arguments": {} + } + }), + ) + .unwrap(); + assert_eq!(mcp["result"]["isError"], false, "MCP response: {mcp}"); + let mcp = &mcp["result"]["structuredContent"]; + assert!(mcp["scope_id"] + .as_str() + .is_some_and(|value| !value.is_empty())); + assert_eq!(mcp["scope_kind"], "workspace"); + assert_eq!(mcp["resulting_state"], "trusted"); +} + +#[test] +fn reconciliation_supports_materialized_lane_scope() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("reconcile-bot", Some("main"), true, None, None) + .unwrap(); + + let lane = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/index/reconcile", + serde_json::json!({ "lane": "reconcile-bot" }), + ), + ); + let lane_status = lane.status; + let lane: serde_json::Value = lane.body_json().unwrap(); + assert_eq!(lane_status, 200, "lane reconcile response: {lane}"); + assert_eq!(lane["scope_kind"], "materialized_lane"); + assert_eq!(lane["resulting_state"], "trusted"); +} + +#[test] +fn reconcile_required_has_stable_structured_recovery_fields() { + let error = Error::ChangeLedgerReconcileRequired { + scope: "workspace-scope".into(), + state: "untrusted_gap".into(), + reason: "observer overflow".into(), + command: "trail index reconcile".into(), + }; + let value = serde_json::to_value(StructuredErrorEnvelope::from_error(&error)).unwrap(); + assert_eq!( + value.pointer("/error/code").unwrap(), + "CHANGE_LEDGER_RECONCILE_REQUIRED" + ); + assert_eq!(value.pointer("/error/status").unwrap(), 409); + assert_eq!(value.pointer("/error/exit").unwrap(), 16); + assert_eq!(value.pointer("/error/scope").unwrap(), "workspace-scope"); + assert_eq!(value.pointer("/error/state").unwrap(), "untrusted_gap"); + assert_eq!(value.pointer("/error/reason").unwrap(), "observer overflow"); + assert_eq!( + value.pointer("/error/recovery/command").unwrap(), + "trail index reconcile" + ); +} + +#[test] +fn reconcile_required_preserves_lane_recovery_command() { + let error = Error::ChangeLedgerReconcileRequired { + scope: "lane-scope".into(), + state: "untrusted_gap".into(), + reason: "observer overflow".into(), + command: "trail index reconcile --lane reconcile-bot".into(), + }; + let value = serde_json::to_value(StructuredErrorEnvelope::from_error(&error)).unwrap(); + assert_eq!( + value.pointer("/error/recovery/command").unwrap(), + "trail index reconcile --lane reconcile-bot" + ); +} + +#[test] +fn rest_and_mcp_return_identical_lane_reconcile_failure_fields() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("reconcile-bot", Some("main"), true, None, None) + .unwrap(); + + let initial = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/index/reconcile", + serde_json::json!({ "lane": "reconcile-bot" }), + ), + ); + assert_eq!(initial.status, 200); + + let conn = rusqlite::Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); + assert_eq!( + conn.execute( + "UPDATE changed_path_scopes SET epoch=epoch+1 WHERE scope_kind='materialized_lane'", + [], + ) + .unwrap(), + 1 + ); + + let rest = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/index/reconcile", + serde_json::json!({ "lane": "reconcile-bot" }), + ), + ); + assert_eq!(rest.status, 409); + let rest: serde_json::Value = rest.body_json().unwrap(); + + let mcp = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "trail.index_reconcile", + "arguments": { "lane": "reconcile-bot" } + } + }), + ) + .unwrap(); + assert_eq!(mcp["result"]["isError"], true); + let mcp = &mcp["result"]["structuredContent"]; + for pointer in [ + "/error/code", + "/error/status", + "/error/exit", + "/error/scope", + "/error/state", + "/error/reason", + "/error/recovery/command", + ] { + assert_eq!( + rest.pointer(pointer), + mcp.pointer(pointer), + "field {pointer}" + ); + } + assert_eq!( + rest.pointer("/error/recovery/command").unwrap(), + "trail index reconcile --lane reconcile-bot" + ); +} + +#[test] +fn reconcile_route_requires_auth_and_accepts_an_empty_body() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let auth = trail::server::ServerAuth::bearer("secret-token").unwrap(); + + let request = api_request("POST", "/v1/index/reconcile", serde_json::Value::Null); + let missing = trail::server::handle_http_request_with_auth(&mut db, &request, &auth); + assert_eq!(missing.status, 401); + + let authorized = api_request_with_headers( + "POST", + "/v1/index/reconcile", + &[("Authorization", "Bearer secret-token")], + serde_json::Value::Null, + ); + let response = trail::server::handle_http_request_with_auth(&mut db, &authorized, &auth); + assert_eq!(response.status, 200); + let report: serde_json::Value = response.body_json().unwrap(); + assert_eq!(report["scope_kind"], "workspace"); + assert_eq!(report["resulting_state"], "trusted"); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn reconcile_cli_renders_human_json_and_ndjson() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let _daemon = AutoDaemonGuard(temp.path().to_path_buf()); + let canonical = temp.path().canonicalize().unwrap(); + let run = |format: &str| { + Command::new(env!("CARGO_BIN_EXE_trail")) + .arg("--workspace") + .arg(temp.path()) + .args(["--format", format, "index", "reconcile"]) + .env("HOME", &canonical) + .env("XDG_CONFIG_HOME", canonical.join(".config")) + .env("GIT_CONFIG_GLOBAL", "") + .env("GIT_CONFIG_NOSYSTEM", "1") + .output() + .unwrap() + }; + + let human = run("human"); + assert!( + human.status.success(), + "human reconcile failed: {}", + String::from_utf8_lossy(&human.stderr) + ); + assert!(String::from_utf8_lossy(&human.stdout).contains("Reconciled changed-path ledger")); + + let json = run("json"); + assert!( + json.status.success(), + "JSON reconcile failed: {}", + String::from_utf8_lossy(&json.stderr) + ); + let json: serde_json::Value = serde_json::from_slice(&json.stdout).unwrap(); + assert_eq!(json["scope_kind"], "workspace"); + assert_eq!(json["resulting_state"], "trusted"); + + let ndjson = run("ndjson"); + assert!( + ndjson.status.success(), + "NDJSON reconcile failed: {}", + String::from_utf8_lossy(&ndjson.stderr) + ); + let lines = ndjson + .stdout + .split(|byte| *byte == b'\n') + .filter(|line| !line.is_empty()) + .collect::>(); + assert_eq!(lines.len(), 1); + let ndjson: serde_json::Value = serde_json::from_slice(lines[0]).unwrap(); + assert_eq!(ndjson["scope_id"], json["scope_id"]); + assert_eq!(ndjson["resulting_state"], "trusted"); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn reconcile_cli_failure_preserves_lane_recovery_in_every_format() { + let temp = tempfile::tempdir().unwrap(); + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let address = listener.local_addr().unwrap(); + let body = serde_json::json!({ + "error": { + "code": "CHANGE_LEDGER_RECONCILE_REQUIRED", + "status": 409, + "exit": 16, + "message": "changed-path ledger reconciliation required", + "scope": "lane-scope", + "state": "untrusted_gap", + "reason": "observer overflow", + "recovery": { + "command": "trail index reconcile --lane reconcile-bot" + } + } + }) + .to_string(); + let server = thread::spawn(move || { + for _ in 0..3 { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = Vec::new(); + loop { + let mut chunk = [0_u8; 4096]; + let read = stream.read(&mut chunk).unwrap(); + assert!(read > 0, "client closed before completing request"); + request.extend_from_slice(&chunk[..read]); + let Some(header_end) = request.windows(4).position(|window| window == b"\r\n\r\n") + else { + continue; + }; + let headers = std::str::from_utf8(&request[..header_end]).unwrap(); + let content_length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + .unwrap_or(0); + if request.len() >= header_end + 4 + content_length { + break; + } + } + write!( + stream, + "HTTP/1.1 409 Conflict\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ) + .unwrap(); + stream.flush().unwrap(); + } + }); + let daemon_url = format!("http://{address}"); + let run = |format: &str| { + Command::new(env!("CARGO_BIN_EXE_trail")) + .arg("--workspace") + .arg(temp.path()) + .arg("--daemon-url") + .arg(&daemon_url) + .args([ + "--format", + format, + "index", + "reconcile", + "--lane", + "reconcile-bot", + ]) + .output() + .unwrap() + }; + + let human = run("human"); + assert!(!human.status.success()); + assert!( + String::from_utf8_lossy(&human.stderr) + .contains("trail index reconcile --lane reconcile-bot"), + "human stderr: {}", + String::from_utf8_lossy(&human.stderr) + ); + + for format in ["json", "ndjson"] { + let output = run(format); + assert!(!output.status.success(), "{format} unexpectedly succeeded"); + let error: serde_json::Value = serde_json::from_slice(&output.stderr).unwrap(); + assert_eq!(error["error"]["code"], "CHANGE_LEDGER_RECONCILE_REQUIRED"); + assert_eq!(error["error"]["status"], 409); + assert_eq!(error["error"]["exit"], 16); + assert_eq!( + error["error"]["recovery"]["command"], + "trail index reconcile --lane reconcile-bot" + ); + } + server.join().unwrap(); +} + +#[test] +fn schema_reinitialize_guidance_requires_backup_and_force_init() { + let error = Error::SchemaReinitializeRequired { + found: "schema 17".into(), + guidance: "back up this workspace, then run `trail init --force` to create schema v18" + .into(), + }; + let value = serde_json::to_value(StructuredErrorEnvelope::from_error(&error)).unwrap(); + assert_eq!( + value.pointer("/error/code").unwrap(), + "SCHEMA_REINITIALIZE_REQUIRED" + ); + assert_eq!( + value.pointer("/error/recovery/command").unwrap(), + "trail init --force" + ); + let rendered = serde_json::to_string(&value).unwrap().to_ascii_lowercase(); + assert!(rendered.contains("back up")); + assert!(!rendered.contains("migration")); + assert!(!rendered.contains("migrate")); +} + +#[test] +fn openapi_documents_reconcile_request_report_and_structured_error() { + let spec = trail::server::openapi_spec(); + assert!(spec["paths"].get("/v1/index/reconcile").is_some()); + assert_eq!( + spec["components"]["schemas"]["ErrorBody"]["properties"]["error"]["properties"]["recovery"] + ["oneOf"][0]["$ref"], + "#/components/schemas/StructuredRecovery" + ); + let operation = &spec["paths"]["/v1/index/reconcile"]["post"]; + assert_eq!(operation["requestBody"]["required"], false); + assert_eq!( + operation["responses"]["409"]["$ref"], + "#/components/responses/Error" + ); +} diff --git a/trail/tests/changed_path_ledger_command_long_lock.rs b/trail/tests/changed_path_ledger_command_long_lock.rs new file mode 100644 index 0000000..0bb0951 --- /dev/null +++ b/trail/tests/changed_path_ledger_command_long_lock.rs @@ -0,0 +1,5 @@ +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn observed_record_lock_longer_than_old_timeout_preserves_observer_authority() { + trail::test_support::changed_path_command_long_lock_flow().unwrap(); +} diff --git a/trail/tests/changed_path_ledger_commands.rs b/trail/tests/changed_path_ledger_commands.rs new file mode 100644 index 0000000..4498f84 --- /dev/null +++ b/trail/tests/changed_path_ledger_commands.rs @@ -0,0 +1,5 @@ +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn status_and_record_use_one_continuous_fenced_candidate_flow() { + trail::test_support::changed_path_command_flow().unwrap(); +} diff --git a/trail/tests/changed_path_ledger_daemon.rs b/trail/tests/changed_path_ledger_daemon.rs new file mode 100644 index 0000000..e7efefc --- /dev/null +++ b/trail/tests/changed_path_ledger_daemon.rs @@ -0,0 +1,2321 @@ +#![cfg(unix)] + +use std::fs::{self, File}; +use std::os::fd::{AsRawFd, FromRawFd}; +use std::os::unix::fs::{symlink, MetadataExt, PermissionsExt}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::{Arc, Barrier}; +use std::thread; +use std::time::{Duration, Instant}; + +use rusqlite::params; +use serde::Deserialize; +use tempfile::TempDir; +use trail::{InitImportMode, Trail}; + +const DAEMON_PROTOCOL_VERSION: u16 = 2; + +#[derive(Clone, Debug, Deserialize)] +struct Endpoint { + protocol_version: u16, + pid: u32, + process_start_identity: String, + executable_identity: String, + workspace_identity: String, + owner_nonce: String, + auth_token: String, + socket_path: PathBuf, + observer_ready: bool, + recovery_complete: bool, + reconciliation_complete: bool, + live_fence_sequence: u64, + epoch: u64, + daemon_launch_nonce: String, +} + +struct Fixture { + temp: TempDir, +} + +impl Fixture { + fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("tracked.txt"), b"baseline\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + Self { temp } + } + + fn root(&self) -> &Path { + self.temp.path() + } + + fn endpoint_path(&self) -> PathBuf { + self.root().join(".trail/index/change-ledger/daemon.json") + } + + fn authority(&self) -> PathBuf { + self.root().join(".trail/index/change-ledger") + } + + fn token_path(&self) -> PathBuf { + self.authority().join("daemon.token") + } + + fn socket_path(&self) -> PathBuf { + self.root() + .canonicalize() + .unwrap() + .join(".trail/changed-path.sock") + } + + fn create_authority(&self) { + fs::create_dir_all(self.authority()).unwrap(); + fs::set_permissions( + self.root().join(".trail"), + fs::Permissions::from_mode(0o700), + ) + .unwrap(); + fs::set_permissions( + self.root().join(".trail/index"), + fs::Permissions::from_mode(0o700), + ) + .unwrap(); + fs::set_permissions(self.authority(), fs::Permissions::from_mode(0o700)).unwrap(); + } + + fn status(&self) -> std::process::Output { + self.status_with_env(&[]) + } + + fn status_with_env(&self, env: &[(&str, &str)]) -> std::process::Output { + self.run_with_env(&["status"], env) + } + + fn run(&self, args: &[&str]) -> std::process::Output { + self.run_with_env(args, &[]) + } + + fn run_with_env(&self, args: &[&str], env: &[(&str, &str)]) -> std::process::Output { + let canonical_root = self.root().canonicalize().unwrap(); + let mut command = Command::new(env!("CARGO_BIN_EXE_trail")); + command + .arg("--workspace") + .arg(self.root()) + .arg("--json") + .args(args) + .env("HOME", &canonical_root) + .env("XDG_CONFIG_HOME", canonical_root.join(".config")) + .env("GIT_CONFIG_GLOBAL", "") + .env("GIT_CONFIG_NOSYSTEM", "1"); + for (name, value) in env { + command.env(name, value); + } + command.output().unwrap() + } + + fn endpoint(&self) -> Endpoint { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + if let Ok(bytes) = fs::read(self.endpoint_path()) { + if let Ok(endpoint) = serde_json::from_slice(&bytes) { + return endpoint; + } + } + assert!( + Instant::now() < deadline, + "daemon endpoint was not published" + ); + thread::sleep(Duration::from_millis(10)); + } + } +} + +fn assert_status_failed(output: &std::process::Output) { + assert_status_failed_for(output, "status"); +} + +fn assert_status_failed_for(output: &std::process::Output, context: &str) { + assert!( + !output.status.success(), + "{context} unexpectedly succeeded: {}", + String::from_utf8_lossy(&output.stdout) + ); + assert!( + String::from_utf8_lossy(&output.stderr).contains("\"error\""), + "unexpected {context} diagnostic: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +fn write_owner_file(path: &Path, bytes: &[u8]) { + fs::write(path, bytes).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap(); +} + +fn authenticated_post(endpoint: &Endpoint, path: &str, body: serde_json::Value) -> String { + let body = serde_json::to_vec(&body).unwrap(); + let request = format!( + "POST {path} HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n", + endpoint.auth_token, + body.len() + ); + let mut stream = UnixStream::connect(&endpoint.socket_path).unwrap(); + use std::io::{Read as _, Write as _}; + stream.write_all(request.as_bytes()).unwrap(); + stream.write_all(&body).unwrap(); + stream.shutdown(std::net::Shutdown::Write).unwrap(); + let mut response = String::new(); + stream.read_to_string(&mut response).unwrap(); + response +} + +fn spawn_status_waiting_after_daemon_authority_load( + fixture: &Fixture, + barrier: &Path, +) -> std::process::Child { + fs::create_dir(barrier).unwrap(); + let canonical_root = fixture.root().canonicalize().unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_trail")) + .arg("--workspace") + .arg(fixture.root()) + .arg("--json") + .arg("status") + .env("HOME", &canonical_root) + .env("XDG_CONFIG_HOME", canonical_root.join(".config")) + .env("GIT_CONFIG_GLOBAL", "") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("TRAIL_TEST_DAEMON_TRANSITION_AFTER_LOAD_BARRIER", barrier) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(60); + while !barrier.join("loaded").exists() { + if let Some(status) = child.try_wait().unwrap() { + let output = child.wait_with_output().unwrap(); + panic!( + "daemon transition exited before authority load barrier: status={status}\nstdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + if Instant::now() >= deadline { + child.kill().unwrap(); + let output = child.wait_with_output().unwrap(); + panic!( + "daemon transition did not reach authority load barrier\nstdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + thread::sleep(Duration::from_millis(10)); + } + child +} + +fn spawn_status_waiting_after_stale_verification( + fixture: &Fixture, + barrier: &Path, +) -> std::process::Child { + fs::create_dir(barrier).unwrap(); + let canonical_root = fixture.root().canonicalize().unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_trail")) + .arg("--workspace") + .arg(fixture.root()) + .arg("--json") + .arg("status") + .env("HOME", &canonical_root) + .env("XDG_CONFIG_HOME", canonical_root.join(".config")) + .env("GIT_CONFIG_GLOBAL", "") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env( + "TRAIL_TEST_WORKSPACE_DAEMON_AFTER_STALE_VERIFICATION_BARRIER", + barrier, + ) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(60); + while !barrier.join("verified").exists() { + if let Some(status) = child.try_wait().unwrap() { + let output = child.wait_with_output().unwrap(); + panic!( + "daemon launcher exited before stale-verification barrier: status={status}\nstdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + if Instant::now() >= deadline { + child.kill().unwrap(); + let output = child.wait_with_output().unwrap(); + panic!( + "daemon launcher did not reach stale-verification barrier\nstdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + thread::sleep(Duration::from_millis(10)); + } + child +} + +#[derive(Debug, PartialEq)] +struct TransitionAuthoritySnapshot { + scope: ( + i64, + i64, + String, + String, + i64, + String, + Option, + Option, + Option, + String, + i64, + ), + owner: ( + i64, + String, + String, + String, + String, + Option>, + i64, + i64, + i64, + Option, + Option, + ), + limits: (i64, i64, i64, i64, i64, i64, i64), +} + +fn transition_authority_snapshot(database: &Path) -> TransitionAuthoritySnapshot { + let conn = rusqlite::Connection::open(database).unwrap(); + let scope = conn + .query_row( + "SELECT epoch,ref_generation,baseline_root_id,policy_fingerprint, + policy_dependency_generation,filesystem_identity,provider_id, + provider_identity,observer_owner_token,trust_state,continuity_generation + FROM changed_path_scopes", + [], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + row.get(6)?, + row.get(7)?, + row.get(8)?, + row.get(9)?, + row.get(10)?, + )) + }, + ) + .unwrap(); + let owner = conn + .query_row( + "SELECT epoch,owner_token,provider_id,provider_identity,lease_state, + fence_nonce,acquired_at,heartbeat_at,expires_at,error_state,error_at + FROM changed_path_observer_owners", + [], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + row.get(6)?, + row.get(7)?, + row.get(8)?, + row.get(9)?, + row.get(10)?, + )) + }, + ) + .unwrap(); + let limits = conn + .query_row( + "SELECT schema_version,max_candidate_rows,max_prefix_rows, + max_observer_log_bytes,max_segment_bytes,max_unfolded_tail_records, + case_sensitive + FROM changed_path_scopes", + [], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + row.get(6)?, + )) + }, + ) + .unwrap(); + TransitionAuthoritySnapshot { + scope, + owner, + limits, + } +} + +fn kill_and_wait(pid: u32) { + unsafe { libc::kill(pid as i32, libc::SIGKILL) }; + let deadline = Instant::now() + Duration::from_secs(5); + while unsafe { libc::kill(pid as i32, 0) } == 0 { + assert!(Instant::now() < deadline, "daemon did not exit"); + thread::sleep(Duration::from_millis(10)); + } +} + +fn process_command_line(pid: u32) -> String { + #[cfg(target_os = "linux")] + { + return String::from_utf8_lossy( + &fs::read(format!("/proc/{pid}/cmdline")).unwrap_or_default(), + ) + .replace('\0', " "); + } + #[cfg(target_os = "macos")] + { + let output = Command::new("ps") + .args(["-o", "command=", "-p", &pid.to_string()]) + .output() + .unwrap(); + return String::from_utf8_lossy(&output.stdout).into_owned(); + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + String::new() +} + +impl Drop for Fixture { + fn drop(&mut self) { + if let Ok(bytes) = fs::read(self.endpoint_path()) { + if let Ok(endpoint) = serde_json::from_slice::(&bytes) { + unsafe { + libc::kill(endpoint.pid as i32, libc::SIGTERM); + } + } + } + } +} + +#[test] +fn first_status_publishes_a_ready_workspace_daemon() { + let fixture = Fixture::new(); + let output = fixture.status(); + assert!( + output.status.success(), + "status failed:\nstdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let endpoint = fixture.endpoint(); + assert_eq!(endpoint.protocol_version, DAEMON_PROTOCOL_VERSION); + assert!(endpoint.observer_ready); + assert!(endpoint.recovery_complete); + assert!(endpoint.reconciliation_complete); + assert!(endpoint.live_fence_sequence > 0); + let command_line = process_command_line(endpoint.pid); + assert!(!command_line.contains(&endpoint.auth_token)); + assert!(!command_line.contains(&endpoint.owner_nonce)); +} + +#[cfg(target_os = "macos")] +#[test] +fn external_volume_repo_direct_fences_home_git_config_and_reconciles_on_drift() { + let policy_home = tempfile::tempdir().unwrap(); + let policy_device = fs::metadata(policy_home.path()).unwrap().dev(); + let workspace = fs::read_dir("/Volumes") + .ok() + .into_iter() + .flatten() + .flatten() + .filter_map(|entry| { + let path = entry.path(); + let metadata = fs::metadata(&path).ok()?; + (metadata.is_dir() && metadata.dev() != policy_device) + .then(|| { + tempfile::Builder::new() + .prefix("trail-external-policy-status-") + .tempdir_in(path) + .ok() + }) + .flatten() + }) + .next(); + let Some(workspace) = workspace else { + // The native qualification runner may expose only one writable + // volume. The planner/fingerprint matrix still covers the partition; + // this end-to-end case runs whenever a real second volume is present. + return; + }; + + fs::write(workspace.path().join("tracked.txt"), b"baseline\n").unwrap(); + for args in [ + vec!["init", "--quiet"], + vec!["config", "user.email", "trail@example.invalid"], + vec!["config", "user.name", "Trail Test"], + vec!["add", "tracked.txt"], + vec!["commit", "--quiet", "-m", "baseline"], + ] { + let output = Command::new("git") + .args(args) + .current_dir(workspace.path()) + .env("GIT_CONFIG_GLOBAL", "") + .env("GIT_CONFIG_NOSYSTEM", "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "git setup failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + Trail::init(workspace.path(), "main", InitImportMode::GitTracked, false).unwrap(); + let fixture = Fixture { temp: workspace }; + let global = policy_home + .path() + .canonicalize() + .unwrap() + .join("global.gitconfig"); + fs::write(&global, b"[core]\n").unwrap(); + let global_text = global.to_str().unwrap(); + // Override Fixture's hermetic default so this also qualifies the normal + // macOS `/etc -> /private/etc` system-config alias across devices. + let daemon_env = [ + ("GIT_CONFIG_GLOBAL", global_text), + ("GIT_CONFIG_NOSYSTEM", "0"), + ]; + + let first = fixture.status_with_env(&daemon_env); + assert!( + first.status.success(), + "external-volume first status failed:\nstdout={}\nstderr={}", + String::from_utf8_lossy(&first.stdout), + String::from_utf8_lossy(&first.stderr) + ); + let first_endpoint = fixture.endpoint(); + + let excludes = global.parent().unwrap().join("global-ignore"); + fs::write( + &global, + format!("[core]\n\texcludesFile = {}\n", excludes.display()), + ) + .unwrap(); + let second = fixture.run_with_env(&["diff", "--dirty"], &daemon_env); + assert!( + second.status.success(), + "policy-drift diff did not reconcile automatically:\nstdout={}\nstderr={}", + String::from_utf8_lossy(&second.stdout), + String::from_utf8_lossy(&second.stderr) + ); + let second_endpoint = fixture.endpoint(); + assert!( + second_endpoint.epoch > first_endpoint.epoch, + "policy drift did not establish a fresh reconciled authority epoch" + ); + assert!(second_endpoint.reconciliation_complete); +} + +#[test] +fn concurrent_first_status_calls_converge_on_one_ready_owner() { + let fixture = Arc::new(Fixture::new()); + assert!(!fixture.authority().exists()); + let barrier = Arc::new(Barrier::new(16)); + let callers = (0..16) + .map(|_| { + let fixture = Arc::clone(&fixture); + let barrier = Arc::clone(&barrier); + thread::spawn(move || { + barrier.wait(); + fixture.status() + }) + }) + .collect::>(); + + for output in callers.into_iter().map(|caller| caller.join().unwrap()) { + assert!( + output.status.success(), + "status failed:\nstdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + let endpoint = fixture.endpoint(); + let connection = + rusqlite::Connection::open(fixture.root().join(".trail/index/trail.sqlite")).unwrap(); + let active_owners: i64 = connection + .query_row( + "SELECT COUNT(*) FROM changed_path_observer_owners WHERE lease_state='active'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(active_owners, 1); + assert_eq!(endpoint.protocol_version, DAEMON_PROTOCOL_VERSION); + assert!(endpoint.pid > 0); + assert!(!endpoint.process_start_identity.is_empty()); + assert_eq!(endpoint.executable_identity.len(), 64); + assert_eq!(endpoint.workspace_identity.len(), 64); + assert_eq!(endpoint.owner_nonce.len(), 64); + assert_eq!(endpoint.auth_token.len(), 64); + assert_eq!( + endpoint.socket_path, + fixture + .root() + .canonicalize() + .unwrap() + .join(".trail/changed-path.sock") + ); + assert!(endpoint.observer_ready); + assert!(endpoint.recovery_complete); + assert!(endpoint.reconciliation_complete); + assert!(endpoint.live_fence_sequence > 0); +} + +#[test] +fn auto_started_daemon_does_not_retain_unrelated_inherited_file_descriptors() { + const ISOLATED: &str = "TRAIL_TEST_ISOLATED_INHERITED_FD"; + if std::env::var_os(ISOLATED).is_none() { + let output = Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg("auto_started_daemon_does_not_retain_unrelated_inherited_file_descriptors") + .arg("--nocapture") + .env(ISOLATED, "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "isolated inherited-fd check failed: {}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + let fixture = Fixture::new(); + let mut pipe = [0_i32; 2]; + assert_eq!(unsafe { libc::pipe(pipe.as_mut_ptr()) }, 0); + let read = unsafe { File::from_raw_fd(pipe[0]) }; + let write = unsafe { File::from_raw_fd(pipe[1]) }; + let read_flags = unsafe { libc::fcntl(read.as_raw_fd(), libc::F_GETFD) }; + let write_flags = unsafe { libc::fcntl(write.as_raw_fd(), libc::F_GETFD) }; + assert!(read_flags >= 0 && write_flags >= 0); + assert_eq!( + unsafe { + libc::fcntl( + read.as_raw_fd(), + libc::F_SETFD, + read_flags | libc::FD_CLOEXEC, + ) + }, + 0 + ); + assert_eq!( + unsafe { + libc::fcntl( + write.as_raw_fd(), + libc::F_SETFD, + write_flags & !libc::FD_CLOEXEC, + ) + }, + 0 + ); + + let status = fixture.status(); + assert!( + status.status.success(), + "status failed: {}", + String::from_utf8_lossy(&status.stderr) + ); + drop(write); + + let mut poll_fd = libc::pollfd { + fd: read.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + let polled = unsafe { libc::poll(&mut poll_fd, 1, 1_000) }; + assert_eq!( + polled, 1, + "daemon retained an unrelated inherited pipe writer" + ); + assert_ne!(poll_fd.revents & libc::POLLHUP, 0); +} + +#[test] +fn first_diff_and_record_share_the_automatically_started_workspace_daemon() { + let fixture = Fixture::new(); + let diff = fixture.run(&["diff", "--dirty", "--name-only"]); + assert!( + diff.status.success(), + "{}", + String::from_utf8_lossy(&diff.stderr) + ); + let first = fixture.endpoint(); + fs::write(fixture.root().join("tracked.txt"), b"record me\n").unwrap(); + let record = fixture.run(&["record", "-m", "daemon record"]); + assert!( + record.status.success(), + "record failed: {}", + String::from_utf8_lossy(&record.stderr) + ); + let second = fixture.endpoint(); + assert_eq!(second.pid, first.pid); + assert_eq!(second.owner_nonce, first.owner_nonce); + kill_and_wait(second.pid); + let after_record_restart = fixture.status(); + assert!( + after_record_restart.status.success(), + "post-record restart failed: {}", + String::from_utf8_lossy(&after_record_restart.stderr) + ); + assert!(fixture.endpoint().epoch > second.epoch); +} + +#[test] +fn endpoint_and_socket_symlinks_are_rejected_without_touching_targets() { + let fixture = Fixture::new(); + fixture.create_authority(); + let endpoint_target = fixture.root().join("endpoint-target"); + fs::write(&endpoint_target, b"sentinel").unwrap(); + symlink(&endpoint_target, fixture.endpoint_path()).unwrap(); + assert_status_failed(&fixture.status()); + assert_eq!(fs::read(&endpoint_target).unwrap(), b"sentinel"); + fs::remove_file(fixture.endpoint_path()).unwrap(); + + let socket_target = fixture.root().join("socket-target"); + fs::write(&socket_target, b"socket-sentinel").unwrap(); + symlink(&socket_target, fixture.socket_path()).unwrap(); + assert_status_failed(&fixture.status()); + assert_eq!(fs::read(&socket_target).unwrap(), b"socket-sentinel"); +} + +#[test] +fn unsafe_authority_and_endpoint_modes_fail_closed() { + let fixture = Fixture::new(); + fixture.create_authority(); + fs::set_permissions(fixture.authority(), fs::Permissions::from_mode(0o755)).unwrap(); + assert_status_failed(&fixture.status()); + + fs::set_permissions(fixture.authority(), fs::Permissions::from_mode(0o700)).unwrap(); + fs::write(fixture.endpoint_path(), b"{}").unwrap(); + fs::set_permissions(fixture.endpoint_path(), fs::Permissions::from_mode(0o644)).unwrap(); + assert_status_failed(&fixture.status()); +} + +#[test] +fn live_daemon_rejects_tampered_endpoint_and_token_identity() { + let fixture = Fixture::new(); + let output = fixture.status(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let endpoint_bytes = fs::read(fixture.endpoint_path()).unwrap(); + let endpoint: serde_json::Value = serde_json::from_slice(&endpoint_bytes).unwrap(); + + for (field, value) in [ + ("protocol_version", serde_json::json!(99)), + ("owner_nonce", serde_json::json!("0".repeat(64))), + ("workspace_identity", serde_json::json!("1".repeat(64))), + ("executable_identity", serde_json::json!("2".repeat(64))), + ("scope_id", serde_json::json!("3".repeat(64))), + ("epoch", serde_json::json!(u64::MAX)), + ] { + let mut tampered = endpoint.clone(); + tampered[field] = value; + write_owner_file( + &fixture.endpoint_path(), + &serde_json::to_vec_pretty(&tampered).unwrap(), + ); + assert_status_failed_for( + &fixture.status(), + &format!("tampered endpoint field `{field}`"), + ); + write_owner_file(&fixture.endpoint_path(), &endpoint_bytes); + let keepalive = fixture.status(); + assert!( + keepalive.status.success(), + "daemon did not remain live after rejecting tampered endpoint field `{field}`: {}", + String::from_utf8_lossy(&keepalive.stderr) + ); + } + + let mut unrelated_pid = endpoint.clone(); + unrelated_pid["pid"] = serde_json::json!(std::process::id()); + unrelated_pid["process_start_identity"] = serde_json::json!("synthetic-reused-pid-token"); + write_owner_file( + &fixture.endpoint_path(), + &serde_json::to_vec_pretty(&unrelated_pid).unwrap(), + ); + assert_status_failed_for(&fixture.status(), "endpoint with unrelated PID identity"); + write_owner_file(&fixture.endpoint_path(), &endpoint_bytes); + + assert_status_failed_for( + &fixture.status_with_env(&[( + "TRAIL_TEST_WORKSPACE_DAEMON_POST_CHALLENGE_START_IDENTITY", + "synthetic-reuse", + )]), + "post-challenge PID identity drift", + ); + + let token_bytes = fs::read(fixture.token_path()).unwrap(); + let token_target = fixture.root().join("token-target"); + fs::write(&token_target, b"sentinel").unwrap(); + fs::remove_file(fixture.token_path()).unwrap(); + symlink(&token_target, fixture.token_path()).unwrap(); + assert_status_failed_for(&fixture.status(), "symlinked token publication"); + assert_eq!(fs::read(&token_target).unwrap(), b"sentinel"); + fs::remove_file(fixture.token_path()).unwrap(); + write_owner_file(&fixture.token_path(), &token_bytes); + + fs::set_permissions(&fixture.socket_path(), fs::Permissions::from_mode(0o666)).unwrap(); + assert_status_failed_for(&fixture.status(), "unsafe socket mode"); + fs::set_permissions(&fixture.socket_path(), fs::Permissions::from_mode(0o600)).unwrap(); + + let starting = serde_json::json!({ + "protocol_version": endpoint["protocol_version"], + "pid": endpoint["pid"], + "process_start_identity": endpoint["process_start_identity"], + "executable_identity": endpoint["executable_identity"], + "workspace_identity": endpoint["workspace_identity"], + "owner_nonce": endpoint["owner_nonce"], + "socket_path": endpoint["socket_path"], + "socket_device": endpoint["socket_device"], + "socket_inode": endpoint["socket_inode"], + }); + fs::remove_file(fixture.endpoint_path()).unwrap(); + let starting_path = fixture.authority().join("daemon.starting.json"); + write_owner_file( + &starting_path, + &serde_json::to_vec_pretty(&starting).unwrap(), + ); + assert_status_failed_for( + &fixture.status_with_env(&[( + "TRAIL_TEST_WORKSPACE_DAEMON_UNVERIFIABLE_PID", + &endpoint["pid"].as_u64().unwrap().to_string(), + )]), + "unverifiable live startup PID", + ); + assert!(starting_path.exists()); + assert!(fixture.socket_path().exists()); + fs::remove_file(starting_path).unwrap(); + write_owner_file(&fixture.endpoint_path(), &endpoint_bytes); +} + +#[test] +fn dead_daemon_does_not_replace_a_statically_invalid_endpoint() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let endpoint = fixture.endpoint(); + kill_and_wait(endpoint.pid); + + let mut tampered: serde_json::Value = + serde_json::from_slice(&fs::read(fixture.endpoint_path()).unwrap()).unwrap(); + tampered["protocol_version"] = serde_json::json!(99); + write_owner_file( + &fixture.endpoint_path(), + &serde_json::to_vec_pretty(&tampered).unwrap(), + ); + + assert_status_failed_for( + &fixture.status(), + "dead daemon with a statically invalid endpoint", + ); + assert!(fixture.endpoint_path().exists()); +} + +#[test] +fn killed_daemon_is_replaced_and_full_reconciliation_captures_offline_change() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + kill_and_wait(first.pid); + fs::write(fixture.root().join("tracked.txt"), b"changed while down\n").unwrap(); + fs::write(fixture.root().join(".trailignore"), b"new-ignore-rule\n").unwrap(); + + let restarted = fixture.status(); + assert!( + restarted.status.success(), + "restart failed: {}", + String::from_utf8_lossy(&restarted.stderr) + ); + let second = fixture.endpoint(); + assert_ne!(second.pid, first.pid); + assert_ne!(second.owner_nonce, first.owner_nonce); + assert!(second.epoch > first.epoch); + + let connection = + rusqlite::Connection::open(fixture.root().join(".trail/index/trail.sqlite")).unwrap(); + let captured: i64 = connection + .query_row( + "SELECT COUNT(*) FROM changed_path_entries WHERE normalized_path='tracked.txt'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(captured, 1); +} + +#[test] +fn external_materialized_spawn_retires_daemon_and_reconcile_starts_one_fresh_owner() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let daemon = fixture.endpoint(); + + let spawned = fixture.run(&[ + "lane", + "spawn", + "daemon-materialized", + "--from", + "main", + "--materialize", + ]); + assert!( + spawned.status.success(), + "daemon-routed lane spawn failed: {}", + String::from_utf8_lossy(&spawned.stderr) + ); + let reconciled = fixture.run(&["index", "reconcile", "--lane", "daemon-materialized"]); + assert!( + reconciled.status.success(), + "daemon-routed lane reconcile failed: {}", + String::from_utf8_lossy(&reconciled.stderr) + ); + let replacement = fixture.endpoint(); + assert_ne!(replacement.pid, daemon.pid); + assert!(replacement.epoch > daemon.epoch); + + let conn = + rusqlite::Connection::open(fixture.root().join(".trail/index/trail.sqlite")).unwrap(); + let active_lane_owners: i64 = conn + .query_row( + "SELECT COUNT(*) + FROM changed_path_observer_owners owner + JOIN changed_path_scopes scope ON scope.scope_id=owner.scope_id + WHERE scope.scope_kind='materialized_lane' + AND owner.lease_state='active' AND owner.error_state IS NULL", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(active_lane_owners, 1); +} + +#[test] +fn external_lane_spawn_ignores_daemon_response_delay_without_duplicate_fallback() { + let fixture = Fixture::new(); + let started = fixture.status_with_env(&[ + ("TRAIL_TEST_DAEMON_RESPONSE_DELAY_PATH", "/v1/lanes"), + ("TRAIL_TEST_DAEMON_RESPONSE_DELAY_MS", "31000"), + ]); + assert!( + started.status.success(), + "daemon start failed: {}", + String::from_utf8_lossy(&started.stderr) + ); + let daemon = fixture.endpoint(); + + let request_started = Instant::now(); + let spawned = fixture.run(&[ + "lane", + "spawn", + "slow-materialized", + "--from", + "main", + "--materialize", + ]); + assert!( + spawned.status.success(), + "delayed daemon-routed lane spawn failed: {}", + String::from_utf8_lossy(&spawned.stderr) + ); + assert!(request_started.elapsed() < Duration::from_secs(30)); + assert_ne!(unsafe { libc::kill(daemon.pid as i32, 0) }, 0); + + let conn = + rusqlite::Connection::open(fixture.root().join(".trail/index/trail.sqlite")).unwrap(); + let (lanes, spawn_events): (i64, i64) = conn + .query_row( + "SELECT + (SELECT COUNT(*) FROM lanes WHERE name='slow-materialized'), + (SELECT COUNT(*) FROM lane_events event + JOIN lanes lane USING(lane_id) + WHERE lane.name='slow-materialized' AND event.event_type='lane_spawned')", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(lanes, 1); + assert_eq!(spawn_events, 1, "delayed spawn committed more than once"); +} + +#[cfg(target_os = "macos")] +#[test] +fn daemon_routed_lane_spawn_preserves_explicit_nfs_cow_mode() { + let fixture = Fixture::new(); + let direct = fixture.run(&[ + "lane", + "spawn", + "direct-nfs-cow", + "--from", + "main", + "--workdir-mode", + "nfs-cow", + ]); + assert!( + direct.status.success(), + "direct nfs-cow spawn failed: {}", + String::from_utf8_lossy(&direct.stderr) + ); + let direct: serde_json::Value = serde_json::from_slice(&direct.stdout).unwrap(); + + assert!(fixture.status().status.success()); + let daemon = fixture.endpoint(); + let routed = fixture.run(&[ + "lane", + "spawn", + "routed-nfs-cow", + "--from", + "main", + "--workdir-mode", + "nfs-cow", + ]); + assert!( + routed.status.success(), + "daemon-routed nfs-cow spawn failed: {}", + String::from_utf8_lossy(&routed.stderr) + ); + let routed: serde_json::Value = serde_json::from_slice(&routed.stdout).unwrap(); + assert_eq!(fixture.endpoint().pid, daemon.pid); + + for report in [&direct, &routed] { + assert_eq!(report["requested_workdir_mode"], "nfs-cow"); + assert_eq!(report["workdir_mode"], "nfs-cow"); + assert_eq!(report["workdir_backend"], "nfs"); + assert_eq!(report["transparent_cow_available"], true); + assert!(report["workdir"] + .as_str() + .is_some_and(|path| !path.is_empty())); + } + assert_eq!( + direct["requested_workdir_mode"], + routed["requested_workdir_mode"] + ); + assert_eq!(direct["workdir_mode"], routed["workdir_mode"]); + assert_eq!(direct["workdir_backend"], routed["workdir_backend"]); + assert_eq!( + direct["transparent_cow_available"], + routed["transparent_cow_available"] + ); +} + +#[test] +fn rejected_patch_audit_does_not_retire_daemon_and_explicit_empty_patch_is_routed() { + let fixture = Fixture::new(); + let status = fixture.status_with_env(&[("TRAIL_TEST_EXTERNAL_AUDIT_HOLD_MS", "1500")]); + assert!( + status.status.success(), + "status failed: {}", + String::from_utf8_lossy(&status.stderr) + ); + let daemon = fixture.endpoint(); + let spawned = fixture.run(&[ + "lane", + "spawn", + "daemon-empty-patch", + "--from", + "main", + "--materialize", + ]); + assert!( + spawned.status.success(), + "lane spawn failed: {}", + String::from_utf8_lossy(&spawned.stderr) + ); + let spawn_report: serde_json::Value = serde_json::from_slice(&spawned.stdout).unwrap(); + let workdir = PathBuf::from(spawn_report["workdir"].as_str().unwrap()); + + for k in [0_usize, 1, 100] { + for index in 0..k { + let parent = workdir.join(format!("record-{:03}", index / 10)); + fs::create_dir_all(&parent).unwrap(); + fs::write( + parent.join(format!("path-{:03}.txt", index)), + format!("record {k}:{index}\n"), + ) + .unwrap(); + } + let recorded = fixture.run(&[ + "lane", + "record", + "daemon-empty-patch", + "-m", + &format!("multi-runtime record k={k}"), + ]); + assert!( + recorded.status.success(), + "record k={k} failed: {}", + String::from_utf8_lossy(&recorded.stderr) + ); + if k == 0 { + let replacement = fixture.endpoint(); + assert_ne!(replacement.pid, daemon.pid, "record k={k}"); + } + } + + let daemon = fixture.endpoint(); + + let rejected = authenticated_post( + &daemon, + "/v1/lanes/daemon-empty-patch/patches", + serde_json::json!({"message": "missing explicit patch source"}), + ); + assert!(rejected.starts_with("HTTP/1.1 400 "), "{rejected}"); + assert_eq!(fixture.endpoint().pid, daemon.pid); + + for k in [0_usize, 1, 100] { + let edits = (0..k) + .map(|index| { + serde_json::json!({ + "op": "write", + "path": format!("patch/path-{index:03}.txt"), + "content": format!("patch {k}:{index}\n") + }) + }) + .collect::>(); + let patch_path = fixture.root().join(format!("patch-{k}.json")); + fs::write( + &patch_path, + serde_json::to_vec(&serde_json::json!({ + "allow_stale": true, + "message": format!("genuine patch k={k}"), + "edits": edits + })) + .unwrap(), + ) + .unwrap(); + let applied = fixture.run(&[ + "lane", + "apply-patch", + "daemon-empty-patch", + "--patch", + patch_path.to_str().unwrap(), + ]); + assert!( + applied.status.success(), + "patch k={k} failed: {}", + String::from_utf8_lossy(&applied.stderr) + ); + let report: serde_json::Value = serde_json::from_slice(&applied.stdout).unwrap(); + assert_eq!( + report["changed_paths"].as_array().unwrap().len(), + k, + "patch k={k}" + ); + assert_eq!(fixture.endpoint().pid, daemon.pid, "patch k={k}"); + } + + let cow = fixture.run(&[ + "lane", + "spawn", + "daemon-cow-after-patches", + "--from", + "main", + "--workdir-mode", + "native-cow", + ]); + assert!( + cow.status.success(), + "COW spawn failed: {}", + String::from_utf8_lossy(&cow.stderr) + ); + assert_eq!(fixture.endpoint().pid, daemon.pid, "COW spawn"); +} + +#[test] +fn dead_endpoint_with_missing_socket_restarts_safely() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + kill_and_wait(first.pid); + fs::remove_file(&first.socket_path).unwrap(); + let restarted = fixture.status(); + assert!( + restarted.status.success(), + "restart failed: {}", + String::from_utf8_lossy(&restarted.stderr) + ); +} + +#[test] +fn stale_cleanup_refuses_to_unlink_a_substituted_same_user_socket() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + kill_and_wait(first.pid); + fs::remove_file(&first.socket_path).unwrap(); + let unrelated = UnixListener::bind(&first.socket_path).unwrap(); + fs::set_permissions(&first.socket_path, fs::Permissions::from_mode(0o600)).unwrap(); + let substituted_inode = fs::symlink_metadata(&first.socket_path).unwrap().ino(); + + assert_status_failed(&fixture.status()); + assert_eq!( + fs::symlink_metadata(&first.socket_path).unwrap().ino(), + substituted_inode + ); + drop(unrelated); +} + +#[test] +fn stale_cleanup_does_not_unlink_socket_substituted_after_identity_verification() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + kill_and_wait(first.pid); + assert!( + first.socket_path.exists(), + "killed daemon removed the stale socket before cleanup could verify it" + ); + + let barrier = fixture.root().join("socket-unlink-race"); + fs::create_dir(&barrier).unwrap(); + let canonical_root = fixture.root().canonicalize().unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_trail")) + .arg("--workspace") + .arg(fixture.root()) + .arg("--json") + .arg("status") + .env("HOME", &canonical_root) + .env("XDG_CONFIG_HOME", canonical_root.join(".config")) + .env("GIT_CONFIG_GLOBAL", "") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env( + "TRAIL_TEST_WORKSPACE_DAEMON_SOCKET_UNLINK_BARRIER", + &barrier, + ) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + + let deadline = Instant::now() + Duration::from_secs(60); + while !barrier.join("verified").exists() { + if let Some(status) = child.try_wait().unwrap() { + let output = child.wait_with_output().unwrap(); + panic!( + "stale cleanup exited before the verified socket boundary: status={status}\nstdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + if Instant::now() >= deadline { + child.kill().unwrap(); + let output = child.wait_with_output().unwrap(); + panic!( + "stale cleanup did not reach the verified socket boundary before timeout\nstdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + thread::sleep(Duration::from_millis(10)); + } + + fs::remove_file(&first.socket_path).unwrap(); + let unrelated = UnixListener::bind(&first.socket_path).unwrap(); + fs::set_permissions(&first.socket_path, fs::Permissions::from_mode(0o600)).unwrap(); + let substituted_inode = fs::symlink_metadata(&first.socket_path).unwrap().ino(); + fs::write(barrier.join("continue"), b"go").unwrap(); + + let output = child.wait_with_output().unwrap(); + assert_status_failed(&output); + assert_eq!( + fs::symlink_metadata(&first.socket_path).unwrap().ino(), + substituted_inode, + "stale cleanup unlinked the socket substituted after verification" + ); + drop(unrelated); +} + +#[test] +fn stale_cleanup_never_unlinks_socket_substituted_after_quarantine_verification() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + kill_and_wait(first.pid); + assert!(first.socket_path.exists()); + + let barrier = fixture.root().join("socket-quarantine-race"); + fs::create_dir(&barrier).unwrap(); + let canonical_root = fixture.root().canonicalize().unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_trail")) + .arg("--workspace") + .arg(fixture.root()) + .arg("--json") + .arg("status") + .env("HOME", &canonical_root) + .env("XDG_CONFIG_HOME", canonical_root.join(".config")) + .env("GIT_CONFIG_GLOBAL", "") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env( + "TRAIL_TEST_WORKSPACE_DAEMON_SOCKET_QUARANTINE_BARRIER", + &barrier, + ) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + + let verified = barrier.join("verified"); + let deadline = Instant::now() + Duration::from_secs(60); + while !verified.exists() { + if let Some(status) = child.try_wait().unwrap() { + let output = child.wait_with_output().unwrap(); + panic!( + "stale cleanup exited before quarantine verification: status={status}\nstdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + if Instant::now() >= deadline { + child.kill().unwrap(); + let output = child.wait_with_output().unwrap(); + panic!( + "stale cleanup did not reach quarantine verification\nstdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + thread::sleep(Duration::from_millis(10)); + } + + let quarantine_leaf = fs::read_to_string(&verified).unwrap(); + assert!( + quarantine_leaf.starts_with(".changed-path-socket-tombstone."), + "unexpected tombstone namespace: {quarantine_leaf}" + ); + let quarantine_path = fixture.root().join(".trail").join(&quarantine_leaf); + assert!(quarantine_path.exists()); + assert!(!first.socket_path.exists()); + let unrelated = UnixListener::bind(&first.socket_path).unwrap(); + fs::set_permissions(&first.socket_path, fs::Permissions::from_mode(0o600)).unwrap(); + fs::remove_file(&quarantine_path).unwrap(); + fs::rename(&first.socket_path, &quarantine_path).unwrap(); + let substituted_inode = fs::symlink_metadata(&quarantine_path).unwrap().ino(); + fs::write(barrier.join("continue"), b"go").unwrap(); + + let output = child.wait_with_output().unwrap(); + assert!( + quarantine_path.exists(), + "cleanup pathname-unlinked a socket substituted after quarantine verification; status={} stderr={}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + fs::symlink_metadata(&quarantine_path).unwrap().ino(), + substituted_inode + ); + drop(unrelated); +} + +fn populate_socket_tombstones(fixture: &Fixture, count: usize) { + for index in 0..count { + fs::write( + fixture.root().join(".trail").join(format!( + ".changed-path-socket-tombstone.{index:024x}.removing" + )), + b"retained", + ) + .unwrap(); + } +} + +fn is_exact_private_socket_leaf(name: &str) -> bool { + name.len() == 14 + && name.starts_with(".s") + && name[2..] + .as_bytes() + .iter() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) +} + +fn is_exact_socket_tombstone(name: &str) -> bool { + let Some(hex) = name + .strip_prefix(".changed-path-socket-tombstone.") + .and_then(|name| name.strip_suffix(".removing")) + else { + return false; + }; + hex.len() == 24 + && hex + .as_bytes() + .iter() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) +} + +fn socket_cleanup_artifacts(fixture: &Fixture) -> Vec { + let mut artifacts = fs::read_dir(fixture.root().join(".trail")) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.file_name().into_string().ok()) + .filter(|name| is_exact_socket_tombstone(name) || is_exact_private_socket_leaf(name)) + .collect::>(); + artifacts.sort(); + artifacts +} + +fn spawn_status_waiting_at_private_socket_bind( + fixture: &Fixture, + barrier: &Path, +) -> std::process::Child { + fs::create_dir(barrier).unwrap(); + let canonical_root = fixture.root().canonicalize().unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_trail")) + .arg("--workspace") + .arg(fixture.root()) + .arg("--json") + .arg("status") + .env("HOME", &canonical_root) + .env("XDG_CONFIG_HOME", canonical_root.join(".config")) + .env("GIT_CONFIG_GLOBAL", "") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("TRAIL_TEST_WORKSPACE_DAEMON_SOCKET_BOUND_BARRIER", barrier) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(60); + while !barrier.join("bound").exists() || !barrier.join("pid").exists() { + if let Some(status) = child.try_wait().unwrap() { + let output = child.wait_with_output().unwrap(); + panic!( + "daemon exited before socket bind boundary: status={status} stderr={}", + String::from_utf8_lossy(&output.stderr) + ); + } + assert!(Instant::now() < deadline, "socket bind barrier timed out"); + thread::sleep(Duration::from_millis(10)); + } + child +} + +fn kill_bound_daemon_and_wait_for_status_failure( + child: std::process::Child, + barrier: &Path, +) -> std::process::Output { + let daemon_pid = fs::read_to_string(barrier.join("pid")) + .unwrap() + .parse::() + .unwrap(); + assert_eq!(unsafe { libc::kill(daemon_pid, libc::SIGKILL) }, 0); + let output = child.wait_with_output().unwrap(); + assert_status_failed(&output); + output +} + +#[test] +fn sigkill_before_starting_intent_leaves_counted_orphan_and_next_status_starts() { + let fixture = Fixture::new(); + let barrier = fixture.root().join("socket-pre-intent-sigkill"); + let child = spawn_status_waiting_at_private_socket_bind(&fixture, &barrier); + let orphan_leaf = fs::read_to_string(barrier.join("bound")).unwrap(); + assert!(is_exact_private_socket_leaf(&orphan_leaf)); + let orphan_path = fixture.root().join(".trail").join(&orphan_leaf); + assert!(orphan_path.exists()); + kill_bound_daemon_and_wait_for_status_failure(child, &barrier); + assert!(orphan_path.exists()); + assert_eq!( + socket_cleanup_artifacts(&fixture), + vec![orphan_leaf.clone()] + ); + + let restarted = fixture.status(); + assert!( + restarted.status.success(), + "below-cap restart failed: {}", + String::from_utf8_lossy(&restarted.stderr) + ); + assert!(orphan_path.exists()); + assert!(socket_cleanup_artifacts(&fixture).contains(&orphan_leaf)); +} + +#[test] +fn sigkill_private_leaf_reaches_total_cap_and_next_status_refuses_before_bind() { + let fixture = Fixture::new(); + populate_socket_tombstones(&fixture, 1023); + fs::write(fixture.root().join(".trail/.s00000000000G"), b"near-match").unwrap(); + fs::write(fixture.root().join(".trail/.s-short"), b"near-match").unwrap(); + assert_eq!(socket_cleanup_artifacts(&fixture).len(), 1023); + + let barrier = fixture.root().join("socket-cap-pre-intent-sigkill"); + let child = spawn_status_waiting_at_private_socket_bind(&fixture, &barrier); + let orphan_leaf = fs::read_to_string(barrier.join("bound")).unwrap(); + let orphan_path = fixture.root().join(".trail").join(&orphan_leaf); + assert_eq!(socket_cleanup_artifacts(&fixture).len(), 1024); + kill_bound_daemon_and_wait_for_status_failure(child, &barrier); + assert!(orphan_path.exists()); + + let refused = fixture.status(); + assert_status_failed(&refused); + assert!( + String::from_utf8_lossy(&refused.stderr).contains("reinitialize this workspace"), + "missing reinitialize guidance: {}", + String::from_utf8_lossy(&refused.stderr) + ); + assert_eq!(socket_cleanup_artifacts(&fixture).len(), 1024); + assert!(orphan_path.exists()); +} + +#[test] +fn socket_tombstone_cap_minus_one_permits_exactly_the_final_slot() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + kill_and_wait(first.pid); + populate_socket_tombstones(&fixture, 1023); + fs::write( + fixture + .root() + .join(".trail/.changed-path-socket-tombstone.not-hex.removing"), + b"near-match", + ) + .unwrap(); + + let output = fixture.status(); + assert_status_failed(&output); + assert!( + String::from_utf8_lossy(&output.stderr).contains("reinitialize this workspace"), + "missing reinitialize guidance after consuming final slot: {}", + String::from_utf8_lossy(&output.stderr) + ); + let exact = fs::read_dir(fixture.root().join(".trail")) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| { + let name = entry.file_name(); + let name = name.to_string_lossy(); + let Some(hex) = name + .strip_prefix(".changed-path-socket-tombstone.") + .and_then(|name| name.strip_suffix(".removing")) + else { + return false; + }; + hex.len() == 24 + && hex + .as_bytes() + .iter() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) + }) + .count(); + assert_eq!(exact, 1024); + assert!( + !first.socket_path.exists(), + "cap-minus-one did not move the verified stale socket into the final tombstone slot" + ); + let private_bind_leaves = fs::read_dir(fixture.root().join(".trail")) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| { + let name = entry.file_name(); + let name = name.to_string_lossy(); + name.len() == 14 + && name.starts_with(".s") + && name[2..] + .as_bytes() + .iter() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) + }) + .count(); + assert_eq!(private_bind_leaves, 0); +} + +#[test] +fn socket_tombstone_cap_refuses_before_rename_with_reinitialize_guidance() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + kill_and_wait(first.pid); + let original_inode = fs::symlink_metadata(&first.socket_path).unwrap().ino(); + populate_socket_tombstones(&fixture, 1024); + + let output = fixture.status(); + assert_status_failed(&output); + assert!( + String::from_utf8_lossy(&output.stderr).contains("reinitialize this workspace"), + "missing reinitialize guidance: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + fs::symlink_metadata(&first.socket_path).unwrap().ino(), + original_inode, + "cap refusal moved the original socket" + ); +} + +#[test] +fn socket_tombstone_cap_refuses_before_creating_private_bind_leaf() { + let fixture = Fixture::new(); + populate_socket_tombstones(&fixture, 1024); + let output = fixture.status(); + assert_status_failed(&output); + assert!( + String::from_utf8_lossy(&output.stderr).contains("reinitialize this workspace"), + "missing reinitialize guidance: {}", + String::from_utf8_lossy(&output.stderr) + ); + let private_bind_leaves = fs::read_dir(fixture.root().join(".trail")) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| { + let name = entry.file_name(); + let name = name.to_string_lossy(); + name.len() == 14 + && name.starts_with(".s") + && name[2..] + .as_bytes() + .iter() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) + }) + .count(); + assert_eq!(private_bind_leaves, 0); +} + +#[test] +fn private_bind_socket_is_created_with_owner_only_mode_atomically() { + let fixture = Fixture::new(); + let barrier = fixture.root().join("socket-bound-mode"); + fs::create_dir(&barrier).unwrap(); + let canonical_root = fixture.root().canonicalize().unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_trail")) + .arg("--workspace") + .arg(fixture.root()) + .arg("--json") + .arg("status") + .env("HOME", &canonical_root) + .env("XDG_CONFIG_HOME", canonical_root.join(".config")) + .env("GIT_CONFIG_GLOBAL", "") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("TRAIL_TEST_WORKSPACE_DAEMON_SOCKET_BOUND_BARRIER", &barrier) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let bound = barrier.join("bound"); + let deadline = Instant::now() + Duration::from_secs(60); + while !bound.exists() { + if let Some(status) = child.try_wait().unwrap() { + let output = child.wait_with_output().unwrap(); + panic!( + "daemon exited before socket bind boundary: status={status} stderr={}", + String::from_utf8_lossy(&output.stderr) + ); + } + assert!(Instant::now() < deadline, "socket bind barrier timed out"); + thread::sleep(Duration::from_millis(10)); + } + let leaf = fs::read_to_string(&bound).unwrap(); + let socket = fixture.root().join(".trail").join(leaf); + let initial_mode = fs::symlink_metadata(&socket).unwrap().permissions().mode() & 0o777; + fs::write(barrier.join("continue"), b"go").unwrap(); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "daemon startup failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(initial_mode, 0o600); +} + +#[test] +fn socket_tombstone_noreplace_collision_retains_original_socket() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + kill_and_wait(first.pid); + let original_inode = fs::symlink_metadata(&first.socket_path).unwrap().ino(); + let barrier = fixture.root().join("socket-quarantine-prerename-race"); + fs::create_dir(&barrier).unwrap(); + let canonical_root = fixture.root().canonicalize().unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_trail")) + .arg("--workspace") + .arg(fixture.root()) + .arg("--json") + .arg("status") + .env("HOME", &canonical_root) + .env("XDG_CONFIG_HOME", canonical_root.join(".config")) + .env("GIT_CONFIG_GLOBAL", "") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env( + "TRAIL_TEST_WORKSPACE_DAEMON_SOCKET_QUARANTINE_PRE_RENAME_BARRIER", + &barrier, + ) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let prepared = barrier.join("prepared"); + let deadline = Instant::now() + Duration::from_secs(60); + while !prepared.exists() { + if let Some(status) = child.try_wait().unwrap() { + let output = child.wait_with_output().unwrap(); + panic!( + "cleanup exited before pre-rename boundary: status={status} stderr={}", + String::from_utf8_lossy(&output.stderr) + ); + } + assert!(Instant::now() < deadline, "pre-rename barrier timed out"); + thread::sleep(Duration::from_millis(10)); + } + let tombstone_leaf = fs::read_to_string(&prepared).unwrap(); + let collision = fixture.root().join(".trail").join(tombstone_leaf); + fs::write(&collision, b"collision").unwrap(); + fs::write(barrier.join("continue"), b"go").unwrap(); + let output = child.wait_with_output().unwrap(); + assert_status_failed(&output); + assert_eq!(fs::read(&collision).unwrap(), b"collision"); + assert_eq!( + fs::symlink_metadata(&first.socket_path).unwrap().ino(), + original_inode + ); +} + +#[test] +fn crash_after_persisting_ledger_owner_is_automatically_recovered() { + let fixture = Fixture::new(); + let crashed = + fixture.status_with_env(&[("TRAIL_TEST_WORKSPACE_DAEMON_EXIT_AFTER_PREPARE", "1")]); + assert_status_failed(&crashed); + let recovered = fixture.status(); + assert!( + recovered.status.success(), + "recovery failed: {}", + String::from_utf8_lossy(&recovered.stderr) + ); +} + +#[test] +fn crash_after_owner_acquisition_before_bound_starting_publication_recovers() { + let fixture = Fixture::new(); + let crashed = fixture.status_with_env(&[( + "TRAIL_TEST_WORKSPACE_DAEMON_EXIT_AFTER_OWNER_ACQUIRE_BEFORE_BOUND_PUBLICATION", + "1", + )]); + assert_status_failed(&crashed); + + let starting: serde_json::Value = serde_json::from_slice( + &fs::read(fixture.authority().join("daemon.starting.json")).unwrap(), + ) + .unwrap(); + assert_eq!(starting["daemon_launch_nonce"].as_str().unwrap().len(), 64); + assert!(starting["scope_id"].is_null()); + assert!(starting["epoch"].is_null()); + + let conn = + rusqlite::Connection::open(fixture.root().join(".trail/index/trail.sqlite")).unwrap(); + let persisted: (String, i64, String, i64, Option) = conn + .query_row( + "SELECT owner.daemon_launch_nonce,owner.daemon_pid, + owner.daemon_process_start_identity,scope.epoch, + scope.observer_owner_token + FROM changed_path_observer_owners owner + JOIN changed_path_scopes scope ON scope.scope_id=owner.scope_id", + [], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, + ) + .unwrap(); + assert_eq!( + persisted.0, + starting["daemon_launch_nonce"].as_str().unwrap() + ); + assert_eq!(persisted.1, starting["pid"].as_i64().unwrap()); + assert_eq!( + persisted.2, + starting["process_start_identity"].as_str().unwrap() + ); + let owner_epoch_and_token: (i64, String) = conn + .query_row( + "SELECT epoch,owner_token FROM changed_path_observer_owners", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!( + (persisted.3, persisted.4.as_deref()), + ( + owner_epoch_and_token.0, + Some(owner_epoch_and_token.1.as_str()) + ), + "crashed startup must leave scope authority exactly bound to its persisted owner" + ); + drop(conn); + + let recovered = fixture.status(); + assert!( + recovered.status.success(), + "pre-publication crash recovery failed: {}", + String::from_utf8_lossy(&recovered.stderr) + ); +} + +#[test] +fn forged_dead_process_identity_cannot_replace_a_live_observer_owner() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let database = fixture.root().join(".trail/index/trail.sqlite"); + let conn = rusqlite::Connection::open(&database).unwrap(); + let before: (String, i64, String) = conn + .query_row( + "SELECT owner_token,epoch,lease_state FROM changed_path_observer_owners", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + drop(conn); + + let mut forged: serde_json::Value = + serde_json::from_slice(&fs::read(fixture.endpoint_path()).unwrap()).unwrap(); + forged["pid"] = serde_json::json!(999_999_999_u32); + forged["process_start_identity"] = serde_json::json!("forged-dead-process"); + write_owner_file( + &fixture.endpoint_path(), + &serde_json::to_vec_pretty(&forged).unwrap(), + ); + assert_status_failed_for(&fixture.status(), "forged dead process stale-owner handoff"); + + let conn = rusqlite::Connection::open(database).unwrap(); + let after: (String, i64, String) = conn + .query_row( + "SELECT owner_token,epoch,lease_state FROM changed_path_observer_owners", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(after, before); + assert_eq!(after.2, "active"); +} + +#[test] +fn ordinary_error_after_persisting_owner_preserves_recovery_identity() { + let fixture = Fixture::new(); + let failed = + fixture.status_with_env(&[("TRAIL_TEST_WORKSPACE_DAEMON_ERROR_AFTER_PREPARE", "1")]); + assert_status_failed(&failed); + assert!(fixture.authority().join("daemon.starting.json").exists()); + let recovered = fixture.status(); + assert!( + recovered.status.success(), + "ordinary-error recovery failed: {}", + String::from_utf8_lossy(&recovered.stderr) + ); +} + +#[test] +fn readiness_timeout_kills_startup_owner_and_next_status_recovers() { + let fixture = Fixture::new(); + let timed_out = fixture.status_with_env(&[ + ("TRAIL_TEST_WORKSPACE_DAEMON_READY_TIMEOUT_MS", "50"), + ("TRAIL_TEST_WORKSPACE_DAEMON_DELAY_AFTER_INTENT_MS", "500"), + ]); + assert_status_failed(&timed_out); + assert!( + String::from_utf8_lossy(&timed_out.stderr).contains("readiness timed out"), + "unexpected timeout diagnostic: {}", + String::from_utf8_lossy(&timed_out.stderr) + ); + let recovered = fixture.status(); + assert!( + recovered.status.success(), + "recovery failed: {}", + String::from_utf8_lossy(&recovered.stderr) + ); +} + +#[test] +fn live_policy_invalidation_self_restarts_and_reconciles() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + fs::write(fixture.root().join(".trailignore"), b"generated/**\n").unwrap(); + let deadline = Instant::now() + Duration::from_secs(15); + let second = loop { + thread::sleep(Duration::from_millis(100)); + let recovered = fixture.status(); + assert!( + recovered.status.success(), + "live invalidation recovery failed: {}", + String::from_utf8_lossy(&recovered.stderr) + ); + let endpoint = fixture.endpoint(); + if endpoint.epoch > first.epoch { + break endpoint; + } + assert!( + Instant::now() < deadline, + "policy invalidation was not observed" + ); + }; + assert_ne!(second.owner_nonce, first.owner_nonce); + assert!(second.epoch > first.epoch); +} + +#[test] +fn real_git_external_global_config_is_observed_and_live_creation_recovers() { + let temp = tempfile::tempdir().unwrap(); + let workspace = temp.path().join("workspace"); + let home = temp.path().join("home"); + fs::create_dir_all(&workspace).unwrap(); + fs::create_dir_all(&home).unwrap(); + fs::write(workspace.join("tracked.txt"), b"baseline\n").unwrap(); + Trail::init(&workspace, "main", InitImportMode::WorkingTree, false).unwrap(); + let git = Command::new("git") + .args(["-C", workspace.to_str().unwrap(), "init", "--quiet"]) + .output() + .unwrap(); + assert!(git.status.success(), "git init failed"); + let global_config = home.join("missing-global.gitconfig"); + let run_status = || { + Command::new(env!("CARGO_BIN_EXE_trail")) + .args([ + "--workspace", + workspace.to_str().unwrap(), + "--json", + "status", + ]) + .env("HOME", &home) + .env("XDG_CONFIG_HOME", home.join(".config")) + .env("GIT_CONFIG_GLOBAL", &global_config) + .env("GIT_CONFIG_NOSYSTEM", "1") + .output() + .unwrap() + }; + let first_status = run_status(); + assert!( + first_status.status.success(), + "real-Git first status failed: {}", + String::from_utf8_lossy(&first_status.stderr) + ); + let endpoint_path = workspace.join(".trail/index/change-ledger/daemon.json"); + let first: Endpoint = serde_json::from_slice(&fs::read(&endpoint_path).unwrap()).unwrap(); + + fs::write(&global_config, b"[core]\n\tignorecase = false\n").unwrap(); + let deadline = Instant::now() + Duration::from_secs(20); + let second = loop { + thread::sleep(Duration::from_millis(100)); + let status = run_status(); + assert!( + status.status.success(), + "external-policy recovery failed: {}", + String::from_utf8_lossy(&status.stderr) + ); + let endpoint: Endpoint = + serde_json::from_slice(&fs::read(&endpoint_path).unwrap()).unwrap(); + if endpoint.epoch > first.epoch { + break endpoint; + } + assert!( + Instant::now() < deadline, + "global Git config creation was not observed" + ); + }; + assert_ne!(second.owner_nonce, first.owner_nonce); + unsafe { libc::kill(second.pid as i32, libc::SIGTERM) }; +} + +#[test] +fn verified_stale_persisted_identity_drift_rotates_epoch_and_reconciles() { + for column in ["filesystem_identity", "provider_identity"] { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + kill_and_wait(first.pid); + + let conn = + rusqlite::Connection::open(fixture.root().join(".trail/index/trail.sqlite")).unwrap(); + let original_authority: (String, String, String, String, [i64; 7]) = conn + .query_row( + "SELECT filesystem_identity,scope_root_identity, + provider_identity,provider_id, + durable_cursor,linearizable_fence,rename_pairing, + overflow_scope,filesystem_supported,clean_proof_allowed, + power_loss_durability + FROM changed_path_scopes LIMIT 1", + [], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + [ + row.get(4)?, + row.get(5)?, + row.get(6)?, + row.get(7)?, + row.get(8)?, + row.get(9)?, + row.get(10)?, + ], + )) + }, + ) + .unwrap(); + assert_eq!(original_authority.0, original_authority.1); + assert_eq!(original_authority.2, original_authority.3); + conn.execute(&format!("UPDATE changed_path_scopes SET {column}='00'"), []) + .unwrap(); + drop(conn); + + let recovered = fixture.status(); + assert!( + recovered.status.success(), + "{column} drift did not automatically recover: {}", + String::from_utf8_lossy(&recovered.stderr) + ); + let second = fixture.endpoint(); + assert!(second.epoch > first.epoch, "{column} drift kept old epoch"); + assert_ne!(second.owner_nonce, first.owner_nonce); + + let conn = + rusqlite::Connection::open(fixture.root().join(".trail/index/trail.sqlite")).unwrap(); + let (stored_epoch, trust_state, recovered_authority): ( + i64, + String, + (String, String, String, String, [i64; 7]), + ) = conn + .query_row( + "SELECT epoch,trust_state, + filesystem_identity,scope_root_identity, + provider_identity,provider_id, + durable_cursor,linearizable_fence,rename_pairing, + overflow_scope,filesystem_supported,clean_proof_allowed, + power_loss_durability + FROM changed_path_scopes LIMIT 1", + [], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + ( + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + [ + row.get(6)?, + row.get(7)?, + row.get(8)?, + row.get(9)?, + row.get(10)?, + row.get(11)?, + row.get(12)?, + ], + ), + )) + }, + ) + .unwrap(); + assert_eq!(u64::try_from(stored_epoch).unwrap(), second.epoch); + assert_eq!(trust_state, "trusted"); + assert_eq!(recovered_authority, original_authority); + assert_eq!(recovered_authority.0, recovered_authority.1); + assert_eq!(recovered_authority.2, recovered_authority.3); + kill_and_wait(second.pid); + } +} + +#[test] +fn owner_acquired_after_stale_process_verification_is_not_replaced() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + assert_eq!(first.daemon_launch_nonce.len(), 64); + kill_and_wait(first.pid); + + let barrier = fixture.root().join("stale-process-owner-race"); + let child = spawn_status_waiting_after_stale_verification(&fixture, &barrier); + let replacement_token = "cd".repeat(32); + let replacement_launch_nonce = "ef".repeat(32); + let database = fixture.root().join(".trail/index/trail.sqlite"); + let conn = rusqlite::Connection::open(&database).unwrap(); + assert_eq!( + conn.execute( + "UPDATE changed_path_observer_owners + SET owner_token=?1,lease_state='active',error_state=NULL,error_at=NULL, + daemon_launch_nonce=?2,daemon_pid=?3, + daemon_process_start_identity='replacement-owner-start', + heartbeat_at=strftime('%s','now'),expires_at=strftime('%s','now')+30, + updated_at=strftime('%s','now') + WHERE scope_id=(SELECT scope_id FROM changed_path_scopes)", + params![ + &replacement_token, + &replacement_launch_nonce, + i64::from(std::process::id()) + ], + ) + .unwrap(), + 1 + ); + assert_eq!( + conn.execute( + "UPDATE changed_path_scopes SET observer_owner_token=?1", + [&replacement_token], + ) + .unwrap(), + 1 + ); + drop(conn); + + fs::write(barrier.join("continue"), b"go").unwrap(); + let output = child.wait_with_output().unwrap(); + assert_status_failed(&output); + + let conn = rusqlite::Connection::open(database).unwrap(); + let owner: (String, String, String) = conn + .query_row( + "SELECT owner.owner_token,owner.lease_state,owner.daemon_launch_nonce + FROM changed_path_observer_owners owner + JOIN changed_path_scopes scope + ON scope.scope_id=owner.scope_id AND scope.epoch=owner.epoch", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!( + owner, + ( + replacement_token.clone(), + "active".into(), + replacement_launch_nonce + ) + ); + let scope_owner: String = conn + .query_row( + "SELECT observer_owner_token FROM changed_path_scopes", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(scope_owner, replacement_token); +} + +fn assert_loaded_scope_authority_race_is_rejected( + name: &str, + mutation: &str, + retained_column: &str, + expected_value: &str, +) { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + kill_and_wait(first.pid); + let database = fixture.root().join(".trail/index/trail.sqlite"); + let conn = rusqlite::Connection::open(&database).unwrap(); + conn.execute( + "UPDATE changed_path_scopes SET filesystem_identity='00'", + [], + ) + .unwrap(); + drop(conn); + + let barrier = fixture.root().join(format!("{name}-authority-race")); + let child = spawn_status_waiting_after_daemon_authority_load(&fixture, &barrier); + let conn = rusqlite::Connection::open(&database).unwrap(); + assert_eq!(conn.execute(mutation, []).unwrap(), 1); + drop(conn); + let concurrent_authority = transition_authority_snapshot(&database); + fs::write(barrier.join("continue"), b"go").unwrap(); + let output = child.wait_with_output().unwrap(); + assert_status_failed(&output); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("daemon authority transition lost exact loaded authority"), + "{name} race did not fail at the full authority CAS boundary: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + transition_authority_snapshot(&database), + concurrent_authority, + "{name} race partially transitioned authority before failing closed" + ); + let conn = rusqlite::Connection::open(&database).unwrap(); + let retained: String = conn + .query_row( + &format!("SELECT {retained_column} FROM changed_path_scopes"), + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(retained, expected_value, "{name} authority was overwritten"); +} + +#[test] +fn verified_stale_transition_rejects_baseline_change_after_authority_load() { + assert_loaded_scope_authority_race_is_rejected( + "baseline", + "UPDATE changed_path_scopes SET ref_generation=ref_generation+1, baseline_root_id='concurrent-baseline-root'", + "baseline_root_id", + "concurrent-baseline-root", + ); +} + +#[test] +fn verified_stale_transition_rejects_policy_change_after_authority_load() { + assert_loaded_scope_authority_race_is_rejected( + "policy", + "UPDATE changed_path_scopes SET policy_fingerprint='1111111111111111111111111111111111111111111111111111111111111111', policy_dependency_generation=policy_dependency_generation+1", + "policy_fingerprint", + "1111111111111111111111111111111111111111111111111111111111111111", + ); +} + +#[test] +fn verified_stale_transition_rejects_limit_change_after_authority_load() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + kill_and_wait(first.pid); + let database = fixture.root().join(".trail/index/trail.sqlite"); + let conn = rusqlite::Connection::open(&database).unwrap(); + conn.execute( + "UPDATE changed_path_scopes SET filesystem_identity='00'", + [], + ) + .unwrap(); + drop(conn); + + let barrier = fixture.root().join("limit-authority-race"); + let child = spawn_status_waiting_after_daemon_authority_load(&fixture, &barrier); + let conn = rusqlite::Connection::open(&database).unwrap(); + assert_eq!( + conn.execute( + "UPDATE changed_path_scopes SET max_candidate_rows=max_candidate_rows+1", + [], + ) + .unwrap(), + 1 + ); + drop(conn); + let concurrent_authority = transition_authority_snapshot(&database); + let concurrent_max_candidate_rows = concurrent_authority.limits.1; + fs::write(barrier.join("continue"), b"go").unwrap(); + let output = child.wait_with_output().unwrap(); + assert_status_failed(&output); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("daemon authority transition lost exact loaded authority"), + "limit race did not fail at the full authority CAS boundary: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + transition_authority_snapshot(&database), + concurrent_authority, + "limit race partially transitioned authority before failing closed" + ); + let conn = rusqlite::Connection::open(&database).unwrap(); + let retained_limit: i64 = conn + .query_row( + "SELECT max_candidate_rows FROM changed_path_scopes", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(retained_limit, concurrent_max_candidate_rows); +} + +#[test] +fn verified_stale_transition_does_not_revoke_owner_acquired_after_authority_load() { + let fixture = Fixture::new(); + assert!(fixture.status().status.success()); + let first = fixture.endpoint(); + kill_and_wait(first.pid); + let database = fixture.root().join(".trail/index/trail.sqlite"); + let conn = rusqlite::Connection::open(&database).unwrap(); + conn.execute( + "UPDATE changed_path_scopes SET filesystem_identity='00'", + [], + ) + .unwrap(); + drop(conn); + + let barrier = fixture.root().join("owner-authority-race"); + let child = spawn_status_waiting_after_daemon_authority_load(&fixture, &barrier); + let replacement_token = "ab".repeat(32); + let conn = rusqlite::Connection::open(&database).unwrap(); + assert_eq!( + conn.execute( + "UPDATE changed_path_observer_owners + SET owner_token=?1, provider_id='concurrent-provider', + provider_identity='concurrent-provider-identity', lease_state='active', + fence_nonce=x'01020304050607080910111213141516', + error_state=NULL,error_at=NULL,updated_at=strftime('%s','now') + WHERE scope_id=(SELECT scope_id FROM changed_path_scopes)", + [&replacement_token], + ) + .unwrap(), + 1 + ); + assert_eq!( + conn.execute( + "UPDATE changed_path_scopes SET observer_owner_token=?1", + [&replacement_token], + ) + .unwrap(), + 1 + ); + drop(conn); + fs::write(barrier.join("continue"), b"go").unwrap(); + let output = child.wait_with_output().unwrap(); + assert_status_failed(&output); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("daemon authority transition lost exact loaded authority"), + "owner race did not fail at the full authority CAS boundary: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let conn = rusqlite::Connection::open(&database).unwrap(); + let owner: (String, String, String) = conn + .query_row( + "SELECT owner_token,lease_state,provider_id FROM changed_path_observer_owners", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!( + owner, + ( + replacement_token.clone(), + "active".into(), + "concurrent-provider".into() + ) + ); + let scope_owner: String = conn + .query_row( + "SELECT observer_owner_token FROM changed_path_scopes", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(scope_owner, replacement_token); +} diff --git a/trail/tests/changed_path_ledger_git.rs b/trail/tests/changed_path_ledger_git.rs new file mode 100644 index 0000000..f9f0b84 --- /dev/null +++ b/trail/tests/changed_path_ledger_git.rs @@ -0,0 +1,704 @@ +#![cfg(debug_assertions)] + +use serde_json::Value; +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::sync::{Mutex, MutexGuard, OnceLock}; +use trail::{Actor, InitImportMode, Trail}; + +static GIT_QUALIFICATION_TESTS: OnceLock> = OnceLock::new(); + +struct Fixture { + db: Trail, + temp: tempfile::TempDir, +} + +struct EnvironmentGuard { + key: &'static str, + previous: Option, +} + +impl EnvironmentGuard { + fn set(key: &'static str, value: &Path) -> Self { + let previous = std::env::var_os(key); + // Every test in this binary holds `GIT_QUALIFICATION_TESTS`, so no + // other thread in the process reads or writes this selector. + unsafe { std::env::set_var(key, value) }; + Self { key, previous } + } +} + +impl Drop for EnvironmentGuard { + fn drop(&mut self) { + match self.previous.take() { + Some(value) => unsafe { std::env::set_var(self.key, value) }, + None => unsafe { std::env::remove_var(self.key) }, + } + } +} + +impl Fixture { + fn new() -> Self { + let temp = tempfile::tempdir().unwrap(); + git(temp.path(), &["init", "--quiet"]); + git(temp.path(), &["config", "user.name", "Trail Test"]); + git( + temp.path(), + &["config", "user.email", "trail@example.invalid"], + ); + git(temp.path(), &["config", "core.filemode", "true"]); + git(temp.path(), &["config", "core.symlinks", "true"]); + git( + temp.path(), + &[ + "config", + "core.ignorecase", + if filesystem_is_case_insensitive(temp.path()) { + "true" + } else { + "false" + }, + ], + ); + fs::write(temp.path().join("tracked.txt"), b"base\n").unwrap(); + fs::write(temp.path().join("rename-me.txt"), b"rename\n").unwrap(); + fs::write(temp.path().join(".trailignore"), b"").unwrap(); + fs::write(temp.path().join(".gitignore"), b".trail/\n").unwrap(); + git(temp.path(), &["add", "."]); + git(temp.path(), &["commit", "--quiet", "-m", "base"]); + Trail::init(temp.path(), "main", InitImportMode::GitTracked, false).unwrap(); + let db = Trail::open(temp.path()).unwrap(); + trail::test_support::set_changed_path_authority_override(false); + Self { db, temp } + } + + fn root(&self) -> &Path { + self.temp.path() + } + + fn qualify(&self) -> Value { + self.qualify_with_policy_mismatch(false) + } + + fn qualify_with_policy_mismatch(&self, mismatch: bool) -> Value { + trail::test_support::changed_path_git_qualification(&self.db, mismatch).unwrap() + } +} + +fn serial() -> MutexGuard<'static, ()> { + GIT_QUALIFICATION_TESTS + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|poison| poison.into_inner()) +} + +fn git(root: &Path, args: &[&str]) -> Output { + let output = Command::new("git") + .arg("-C") + .arg(root) + .args(args) + .env("GIT_OPTIONAL_LOCKS", "0") + .output() + .unwrap(); + assert!( + output.status.success(), + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr) + ); + output +} + +fn git_text(root: &Path, args: &[&str]) -> String { + String::from_utf8(git(root, args).stdout) + .unwrap() + .trim() + .to_string() +} + +fn qualification(value: &Value) -> &Value { + &value["qualification"] +} + +fn exact_paths(value: &Value) -> BTreeSet { + value["exact_paths"] + .as_array() + .unwrap() + .iter() + .map(|path| path.as_str().unwrap().to_string()) + .collect() +} + +fn reasons(value: &Value) -> BTreeSet { + qualification(value)["advisory_reasons"] + .as_array() + .unwrap() + .iter() + .map(|reason| reason.as_str().unwrap().to_string()) + .collect() +} + +fn absolute_git_path(root: &Path, path: String) -> PathBuf { + let path = PathBuf::from(path); + if path.is_absolute() { + path + } else { + root.join(path) + } +} + +fn replace_regular_file(path: &Path) -> Result<(), String> { + let bytes = fs::read(path).map_err(|error| error.to_string())?; + let replacement = path.with_extension("trail-qualification-replacement"); + fs::write(&replacement, bytes).map_err(|error| error.to_string())?; + fs::rename(replacement, path).map_err(|error| error.to_string()) +} + +fn filesystem_is_case_insensitive(root: &Path) -> bool { + let lower = root.join(".trail-git-case-probe-a"); + let upper = root.join(".TRAIL-GIT-CASE-PROBE-A"); + fs::write(&lower, b"probe").unwrap(); + let insensitive = upper.exists(); + fs::remove_file(lower).unwrap(); + insensitive +} + +#[test] +fn exact_equivalence_allows_clean_proof_and_reports_hidden_git_work() { + let _guard = serial(); + let fixture = Fixture::new(); + let qualified = fixture.qualify(); + let qualification = qualification(&qualified); + + assert_eq!(qualification["clean_proof_allowed"], true); + assert_eq!( + qualification["mapped_trail_root"], + qualification["ledger_baseline_root"] + ); + assert!(exact_paths(&qualified).is_empty()); + assert!(qualified["metrics"]["subprocess_count"].as_u64().unwrap() > 0); + assert!(qualified["metrics"]["trace2_bytes"].as_u64().unwrap() > 0); + assert!( + qualified["metrics"]["external_adapter_global_work"] + .as_u64() + .unwrap() + > 0 + ); + // Includes before/after qualification, post-c2 path capture, and the + // descriptor-held verification read returned to the caller. + assert!(qualified["metrics"]["index_read_count"].as_u64().unwrap() >= 5); + assert!(qualified["metrics"]["index_bytes"].as_u64().unwrap() > 0); + assert_eq!(qualification["worktree_equivalent"], true); + assert!(!qualification["worktree_top_level"] + .as_str() + .unwrap() + .is_empty()); + + // Qualification is testable before activation but must not flip command + // authority on as a side effect. + trail::test_support::set_changed_path_authority_override(false); +} + +#[test] +fn trail_full_scan_policy_oracle_matches_nested_untracked_and_ignored_candidates() { + let _guard = serial(); + let fixture = Fixture::new(); + fs::write(fixture.root().join("tracked.txt"), b"changed\n").unwrap(); + fs::create_dir(fixture.root().join("nested")).unwrap(); + fs::write(fixture.root().join("nested/untracked.txt"), b"new\n").unwrap(); + fs::write(fixture.root().join(".trailignore"), b"nested/ignored.log\n").unwrap(); + fs::write(fixture.root().join("nested/ignored.log"), b"ignored\n").unwrap(); + + let qualified = fixture.qualify(); + let oracle = trail::test_support::changed_path_git_full_scan_oracle(&fixture.db) + .unwrap() + .into_iter() + .collect::>(); + + assert_eq!(exact_paths(&qualified), oracle); + assert!(oracle.contains("nested/untracked.txt")); + assert!(!oracle.contains("nested/ignored.log")); + assert_eq!(qualification(&qualified)["clean_proof_allowed"], true); +} + +#[test] +fn trail_ahead_reversion_and_policy_mismatch_are_advisory_only() { + let _guard = serial(); + let mut fixture = Fixture::new(); + fs::write(fixture.root().join("tracked.txt"), b"trail ahead\n").unwrap(); + fixture + .db + .record( + Some("main"), + Some("advance Trail baseline".to_string()), + Actor::human(), + false, + ) + .unwrap(); + git(fixture.root(), &["checkout", "--", "tracked.txt"]); + + let ahead = fixture.qualify(); + assert_eq!(qualification(&ahead)["clean_proof_allowed"], false); + assert!(reasons(&ahead).contains("head_or_mapping")); + + let mismatch = fixture.qualify_with_policy_mismatch(true); + assert_eq!(qualification(&mismatch)["clean_proof_allowed"], false); + assert!(reasons(&mismatch).contains("policy_fingerprint")); +} + +#[test] +fn porcelain_v2_retains_both_rename_endpoints() { + let _guard = serial(); + let fixture = Fixture::new(); + git(fixture.root(), &["mv", "rename-me.txt", "renamed.txt"]); + + let qualified = fixture.qualify(); + assert!(exact_paths(&qualified).is_superset(&BTreeSet::from([ + "rename-me.txt".to_string(), + "renamed.txt".to_string(), + ]))); + assert!(qualified["rename_pairs"] + .as_array() + .unwrap() + .contains(&serde_json::json!(["rename-me.txt", "renamed.txt"]))); +} + +#[cfg(unix)] +#[test] +fn mode_and_symlink_semantics_are_qualified_conservatively() { + use std::os::unix::fs::PermissionsExt; + + let _guard = serial(); + let fixture = Fixture::new(); + let tracked = fixture.root().join("tracked.txt"); + let mut permissions = fs::metadata(&tracked).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&tracked, permissions).unwrap(); + let mode = fixture.qualify(); + assert!(exact_paths(&mode).contains("tracked.txt")); + assert_eq!(qualification(&mode)["mode_equivalent"], true); + + let fixture = Fixture::new(); + let oid = git_text(fixture.root(), &["rev-parse", "HEAD:tracked.txt"]); + git( + fixture.root(), + &[ + "update-index", + "--add", + "--cacheinfo", + &format!("120000,{oid},tracked-link"), + ], + ); + let symlink = fixture.qualify(); + assert_eq!(qualification(&symlink)["clean_proof_allowed"], false); + assert!(reasons(&symlink).contains("symlink")); +} +#[test] +fn index_flags_sparse_submodules_and_caches_are_advisory_only() { + let _guard = serial(); + + let fixture = Fixture::new(); + git( + fixture.root(), + &["update-index", "--assume-unchanged", "tracked.txt"], + ); + let assume = fixture.qualify(); + assert_eq!(qualification(&assume)["clean_proof_allowed"], false); + assert!(reasons(&assume).contains("index_identity_or_flags")); + + let fixture = Fixture::new(); + git( + fixture.root(), + &["update-index", "--skip-worktree", "tracked.txt"], + ); + let skip = fixture.qualify(); + assert_eq!(qualification(&skip)["clean_proof_allowed"], false); + assert!(reasons(&skip).contains("sparse_or_skip_worktree")); + + let fixture = Fixture::new(); + git(fixture.root(), &["config", "core.sparseCheckout", "true"]); + let sparse = fixture.qualify(); + assert_eq!(qualification(&sparse)["clean_proof_allowed"], false); + assert!(reasons(&sparse).contains("sparse_or_skip_worktree")); + + let fixture = Fixture::new(); + let commit = git_text(fixture.root(), &["rev-parse", "HEAD"]); + git( + fixture.root(), + &[ + "update-index", + "--add", + "--cacheinfo", + &format!("160000,{commit},nested-repository"), + ], + ); + let submodule = fixture.qualify(); + assert_eq!(qualification(&submodule)["clean_proof_allowed"], false); + assert!(reasons(&submodule).contains("submodule")); + + let fixture = Fixture::new(); + git(fixture.root(), &["config", "core.fsmonitor", "/bin/true"]); + let fsmonitor = fixture.qualify(); + assert_eq!(qualification(&fsmonitor)["clean_proof_allowed"], false); + assert!(reasons(&fsmonitor).contains("fsmonitor")); + + let fixture = Fixture::new(); + git(fixture.root(), &["config", "core.untrackedCache", "true"]); + let cache = fixture.qualify(); + assert_eq!(qualification(&cache)["clean_proof_allowed"], false); + assert!(reasons(&cache).contains("untracked_cache")); +} + +#[test] +fn main_and_shared_index_replacement_races_revoke_clean_proof() { + let _guard = serial(); + + let fixture = Fixture::new(); + let index = absolute_git_path( + fixture.root(), + git_text(fixture.root(), &["rev-parse", "--git-path", "index"]), + ); + trail::test_support::install_git_qualification_after_porcelain_hook(move || { + replace_regular_file(&index) + }); + let replaced = fixture.qualify(); + assert_eq!(qualification(&replaced)["clean_proof_allowed"], false); + assert!(reasons(&replaced).contains("index_identity_or_flags")); + + let fixture = Fixture::new(); + git(fixture.root(), &["update-index", "--split-index"]); + let stable_split = fixture.qualify(); + assert!( + stable_split["metrics"]["shared_index_read_count"] + .as_u64() + .unwrap() + >= 2 + ); + assert!( + stable_split["metrics"]["shared_index_bytes"] + .as_u64() + .unwrap() + > 0 + ); + let shared = absolute_git_path( + fixture.root(), + git_text(fixture.root(), &["rev-parse", "--shared-index-path"]), + ); + assert!(shared.is_file()); + trail::test_support::install_git_qualification_after_porcelain_hook(move || { + replace_regular_file(&shared) + }); + match trail::test_support::changed_path_git_qualification(&fixture.db, false) { + Ok(replaced) => { + assert_eq!(qualification(&replaced)["clean_proof_allowed"], false); + assert!(reasons(&replaced).contains("index_identity_or_flags")); + } + Err(error) => assert!(error.contains("shared_index"), "{error}"), + } + + let fixture = Fixture::new(); + git(fixture.root(), &["update-index", "--split-index"]); + let shared = absolute_git_path( + fixture.root(), + git_text(fixture.root(), &["rev-parse", "--shared-index-path"]), + ); + trail::test_support::install_git_qualification_after_porcelain_hook(move || { + fs::remove_file(shared).map_err(|error| error.to_string()) + }); + match trail::test_support::changed_path_git_qualification(&fixture.db, false) { + Ok(missing) => { + assert_eq!(qualification(&missing)["clean_proof_allowed"], false); + assert!(reasons(&missing).contains("index_identity_or_flags")); + } + Err(error) => assert!(error.contains("shared_index"), "{error}"), + } +} + +#[test] +fn post_c2_index_replacement_fails_closed_before_consumption() { + let _guard = serial(); + let fixture = Fixture::new(); + let index = absolute_git_path( + fixture.root(), + git_text(fixture.root(), &["rev-parse", "--git-path", "index"]), + ); + trail::test_support::install_git_qualification_after_c2_hook(move || { + replace_regular_file(&index) + }); + let error = + trail::test_support::changed_path_git_qualification(&fixture.db, false).unwrap_err(); + assert!(error.contains("changed across ledger c2"), "{error}"); +} + +#[test] +fn public_status_does_not_fall_back_when_git_index_disappears() { + let _guard = serial(); + let fixture = Fixture::new(); + fixture.qualify(); + let index = absolute_git_path( + fixture.root(), + git_text(fixture.root(), &["rev-parse", "--git-path", "index"]), + ); + trail::test_support::install_git_qualification_after_c2_hook(move || { + fs::remove_file(index).map_err(|error| error.to_string()) + }); + trail::test_support::set_changed_path_authority_override(true); + let result = fixture.db.status(None); + trail::test_support::set_changed_path_authority_override(false); + let error = result.unwrap_err().to_string(); + assert!( + error.contains("index") || error.contains("changed across ledger c2"), + "{error}" + ); +} + +#[test] +fn public_status_fails_closed_on_git_index_replacement_aba() { + let _guard = serial(); + let fixture = Fixture::new(); + fixture.qualify(); + let index = absolute_git_path( + fixture.root(), + git_text(fixture.root(), &["rev-parse", "--git-path", "index"]), + ); + trail::test_support::install_git_qualification_after_c2_hook(move || { + replace_regular_file(&index) + }); + trail::test_support::set_changed_path_authority_override(true); + let result = fixture.db.status(None); + trail::test_support::set_changed_path_authority_override(false); + let error = result.unwrap_err().to_string(); + assert!(error.contains("changed across ledger c2"), "{error}"); +} + +#[test] +fn post_c2_head_symbolic_ref_and_packed_refs_replacements_fail_closed() { + let _guard = serial(); + + for structural_path in ["HEAD", "symbolic-ref"] { + let fixture = Fixture::new(); + let path = if structural_path == "HEAD" { + absolute_git_path( + fixture.root(), + git_text(fixture.root(), &["rev-parse", "--git-path", "HEAD"]), + ) + } else { + let reference = git_text(fixture.root(), &["symbolic-ref", "HEAD"]); + absolute_git_path( + fixture.root(), + git_text( + fixture.root(), + &["rev-parse", "--git-path", reference.as_str()], + ), + ) + }; + trail::test_support::install_git_qualification_after_c2_hook(move || { + replace_regular_file(&path) + }); + let error = + trail::test_support::changed_path_git_qualification(&fixture.db, false).unwrap_err(); + assert!(error.contains("changed across ledger c2"), "{error}"); + } + + let fixture = Fixture::new(); + git(fixture.root(), &["pack-refs", "--all"]); + let packed_refs = absolute_git_path( + fixture.root(), + git_text(fixture.root(), &["rev-parse", "--git-path", "packed-refs"]), + ); + trail::test_support::install_git_qualification_after_c2_hook(move || { + replace_regular_file(&packed_refs) + }); + let error = + trail::test_support::changed_path_git_qualification(&fixture.db, false).unwrap_err(); + assert!(error.contains("changed across ledger c2"), "{error}"); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn post_c2_worktree_root_replacement_fails_closed() { + let _guard = serial(); + let fixture = Fixture::new(); + let root = fixture.root().to_path_buf(); + let backup = root.with_extension("trail-original-worktree"); + let hook_root = root.clone(); + let hook_backup = backup.clone(); + trail::test_support::install_git_qualification_after_c2_hook(move || { + fs::rename(&hook_root, &hook_backup).map_err(|error| error.to_string())?; + let output = Command::new("cp") + .args(["-R"]) + .arg(&hook_backup) + .arg(&hook_root) + .output() + .map_err(|error| error.to_string())?; + if output.status.success() { + Ok(()) + } else { + Err(String::from_utf8_lossy(&output.stderr).into_owned()) + } + }); + let result = trail::test_support::changed_path_git_qualification(&fixture.db, false); + fs::remove_dir_all(&root).unwrap(); + fs::rename(&backup, &root).unwrap(); + let error = result.unwrap_err(); + assert!(error.contains("changed across ledger c2"), "{error}"); +} + +#[test] +fn ambient_git_worktree_selector_cannot_redirect_qualification() { + let _guard = serial(); + let fixture = Fixture::new(); + let hostile = tempfile::tempdir().unwrap(); + let _environment = EnvironmentGuard::set("GIT_WORK_TREE", hostile.path()); + + let qualified = fixture.qualify(); + assert_eq!(qualification(&qualified)["worktree_equivalent"], true); + assert_eq!(qualification(&qualified)["clean_proof_allowed"], true); +} + +#[test] +fn status_diff_and_record_consume_ledger_paths_under_bounded_git_fence() { + let _guard = serial(); + let mut fixture = Fixture::new(); + fs::create_dir(fixture.root().join("nested-command")).unwrap(); + fs::write( + fixture.root().join("nested-command/untracked.txt"), + b"command flow\n", + ) + .unwrap(); + + let flow = trail::test_support::changed_path_git_command_flow(&mut fixture.db).unwrap(); + for surface in ["status", "diff", "record"] { + assert!(flow[surface] + .as_array() + .unwrap() + .iter() + .any(|path| path == "nested-command/untracked.txt")); + } + trail::test_support::set_changed_path_authority_override(false); +} + +#[test] +fn git_backed_public_commands_emit_zero_global_adapter_work() { + let _guard = serial(); + let metrics_temp = tempfile::tempdir().unwrap(); + let metrics_path = metrics_temp.path().join("git-command-metrics.jsonl"); + let _metrics_environment = + EnvironmentGuard::set("TRAIL_PERFORMANCE_METRICS_FILE", &metrics_path); + let mut fixture = Fixture::new(); + fs::create_dir(fixture.root().join("metrics-command")).unwrap(); + fs::write( + fixture.root().join("metrics-command/untracked.txt"), + b"bounded command flow\n", + ) + .unwrap(); + + trail::test_support::changed_path_git_command_flow(&mut fixture.db).unwrap(); + + let reports = fs::read_to_string(&metrics_path) + .unwrap() + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|report| { + matches!( + report["operation"].as_str(), + Some("status" | "diff" | "record") + ) + }) + .collect::>(); + assert_eq!( + reports.len(), + 3, + "expected one public report for status, diff, and record: {reports:?}" + ); + for report in reports { + assert_eq!(report["outcome"], "success", "{report}"); + assert_eq!(report["external_adapter_global_work"], 0, "{report}"); + assert_eq!(report["git_global_work_count"], 0, "{report}"); + assert_eq!(report["git_index_read_count"], 0, "{report}"); + assert_eq!(report["git_index_bytes"], 0, "{report}"); + assert_eq!(report["git_shared_index_read_count"], 0, "{report}"); + assert_eq!(report["git_shared_index_bytes"], 0, "{report}"); + } + trail::test_support::set_changed_path_authority_override(false); +} + +#[test] +fn authoritative_dirty_diff_materializes_patches_and_line_changes() { + let _guard = serial(); + let mut fixture = Fixture::new(); + fs::write( + fixture.root().join("tracked.txt"), + b"base changed\nsecond line\n", + ) + .unwrap(); + // Establish observer ownership and reconcile the pre-existing mutation. + fixture.qualify(); + trail::test_support::set_changed_path_authority_override(true); + let result = fixture.db.diff_dirty(true, true); + trail::test_support::set_changed_path_authority_override(false); + let diff = result.unwrap(); + let tracked = diff + .files + .iter() + .find(|file| file.path == "tracked.txt") + .expect("authoritative dirty diff omitted tracked.txt"); + let patch = tracked + .patch + .as_deref() + .expect("authoritative dirty diff omitted its patch"); + assert!(patch.contains("-base"), "{patch}"); + assert!(patch.contains("+base changed"), "{patch}"); + assert!( + !tracked.line_changes.is_empty(), + "authoritative dirty diff omitted line changes" + ); +} + +#[test] +fn mcp_status_and_dirty_diff_use_authoritative_public_dispatch() { + let _guard = serial(); + let mut fixture = Fixture::new(); + fs::create_dir(fixture.root().join("mcp-nested")).unwrap(); + fs::write(fixture.root().join("mcp-nested/new.txt"), b"mcp\n").unwrap(); + // Starts the workspace daemon and establishes the trusted ledger baseline. + fixture.qualify(); + trail::test_support::set_changed_path_authority_override(true); + + let status = trail::mcp::handle_json_rpc( + &mut fixture.db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "trail.status", "arguments": {"branch": "main"}} + }), + ) + .unwrap(); + let diff = trail::mcp::handle_json_rpc( + &mut fixture.db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "trail.diff", "arguments": {"dirty": true}} + }), + ) + .unwrap(); + trail::test_support::set_changed_path_authority_override(false); + + assert_eq!(status["result"]["isError"], false, "{status}"); + assert!(status["result"]["structuredContent"]["changed_paths"] + .as_array() + .unwrap() + .iter() + .any(|change| change["path"] == "mcp-nested/new.txt")); + assert_eq!(diff["result"]["isError"], false, "{diff}"); + assert!(diff["result"]["structuredContent"]["files"] + .as_array() + .unwrap() + .iter() + .any(|change| change["path"] == "mcp-nested/new.txt")); +} diff --git a/trail/tests/changed_path_ledger_linux.rs b/trail/tests/changed_path_ledger_linux.rs new file mode 100644 index 0000000..c31913b --- /dev/null +++ b/trail/tests/changed_path_ledger_linux.rs @@ -0,0 +1,84 @@ +#![cfg(target_os = "linux")] + +#[test] +fn recursive_directory_creation_is_covered_before_children_can_be_clean() { + trail::test_support::changed_path_linux_recursive_coverage().unwrap(); +} + +#[test] +fn reconciliation_can_qualify_only_through_a_durable_end_fence() { + trail::test_support::changed_path_linux_reconciliation_interval_qualification().unwrap(); +} + +#[test] +fn content_mode_create_and_delete_are_durably_observed() { + trail::test_support::changed_path_linux_content_mode_create_delete().unwrap(); +} + +#[test] +fn file_directory_and_case_renames_retain_both_endpoints() { + trail::test_support::changed_path_linux_rename_matrix().unwrap(); +} + +#[test] +fn rename_storms_and_expired_cookies_remain_conservative() { + trail::test_support::changed_path_linux_rename_storm_and_cookie_expiry().unwrap(); +} + +#[test] +fn delayed_backlog_is_drained_before_the_fence_returns() { + trail::test_support::changed_path_linux_delayed_backlog().unwrap(); +} + +#[test] +fn controlled_fence_seals_before_a_continuous_later_queue() { + trail::test_support::changed_path_linux_controlled_fence_queue_ordering().unwrap(); +} + +#[test] +fn nonce_fence_orders_durable_create_before_durable_delete() { + trail::test_support::changed_path_linux_fence_ordering().unwrap(); +} + +#[test] +fn overflow_ignored_unknown_decode_watch_add_and_durability_fail_closed() { + trail::test_support::changed_path_linux_fault_revocation_matrix().unwrap(); +} + +#[test] +fn owner_death_and_root_replacement_cannot_prove_clean() { + trail::test_support::changed_path_linux_owner_death_and_root_replacement().unwrap(); +} + +#[test] +fn linux_observer_process_owner_child() { + let Ok(root) = std::env::var("TRAIL_LINUX_OBSERVER_CHILD_ROOT") else { + return; + }; + trail::test_support::changed_path_linux_process_owner_child(&root).unwrap(); +} + +#[test] +fn complete_prefix_races_publish_every_descendant_without_false_clean() { + trail::test_support::changed_path_linux_complete_prefix_publication_races().unwrap(); +} + +#[test] +fn issued_fences_reject_forgery_cross_scope_replay_and_owner_replacement() { + trail::test_support::changed_path_linux_authenticated_fence_rejections().unwrap(); +} + +#[test] +fn native_segment_writer_reconcile_full_publishes_real_sqlite_state() { + trail::test_support::changed_path_linux_segment_writer_reconcile_publication().unwrap(); +} + +#[test] +fn raw_overflow_and_unknown_watch_events_use_the_decoder_fail_closed_path() { + trail::test_support::changed_path_linux_raw_decoder_faults().unwrap(); +} + +#[test] +fn declared_internal_policy_dependencies_invalidate_without_storage_self_feedback() { + trail::test_support::changed_path_linux_policy_dependency_observation().unwrap(); +} diff --git a/trail/tests/changed_path_ledger_macos.rs b/trail/tests/changed_path_ledger_macos.rs new file mode 100644 index 0000000..5f12629 --- /dev/null +++ b/trail/tests/changed_path_ledger_macos.rs @@ -0,0 +1,56 @@ +#![cfg(target_os = "macos")] + +#[test] +fn real_apfs_file_events_are_durable_and_fenced() { + trail::test_support::changed_path_macos_real_apfs_file_events().unwrap(); +} + +#[test] +fn all_fsevents_gap_flags_revoke_cursor_resume() { + trail::test_support::changed_path_macos_gap_flag_matrix().unwrap(); +} + +#[test] +fn fsevents_restart_root_cursor_overflow_and_worker_death_fail_closed() { + trail::test_support::changed_path_macos_continuity_fault_matrix().unwrap(); +} + +#[test] +fn synchronous_flush_orders_callbacks_through_durable_event_id() { + trail::test_support::changed_path_macos_fence_ordering().unwrap(); +} + +#[test] +fn fence_cursor_never_overclaims_a_paused_callback_batch() { + trail::test_support::changed_path_macos_paused_callback_fence().unwrap(); +} + +#[test] +fn resume_is_bound_to_actual_device_history_and_relative_root() { + trail::test_support::changed_path_macos_history_authority().unwrap(); +} + +#[test] +fn late_native_start_is_cancelled_without_blocking_timeout_return() { + trail::test_support::changed_path_macos_startup_cancellation().unwrap(); +} + +#[test] +fn malformed_callback_arrays_revoke_but_zero_count_is_safe() { + trail::test_support::changed_path_macos_malformed_callbacks().unwrap(); +} + +#[test] +fn every_root_revalidation_failure_revokes_globally() { + trail::test_support::changed_path_macos_root_revalidation_failures().unwrap(); +} + +#[test] +fn null_context_generation_revokes_every_live_proof_path() { + trail::test_support::changed_path_macos_null_context_generation().unwrap(); +} + +#[test] +fn database_uuid_is_revalidated_after_start_and_before_proof() { + trail::test_support::changed_path_macos_uuid_revalidation().unwrap(); +} diff --git a/trail/tests/changed_path_ledger_materialized_lane.rs b/trail/tests/changed_path_ledger_materialized_lane.rs new file mode 100644 index 0000000..a3d1c40 --- /dev/null +++ b/trail/tests/changed_path_ledger_materialized_lane.rs @@ -0,0 +1,11 @@ +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn materialized_lane_snapshot_pins_lane_root_and_reconciles_missing_marker() { + trail::test_support::changed_path_materialized_lane_snapshot_flow().unwrap(); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn materialized_candidate_lifecycle_is_exact_and_does_not_accumulate_internal_paths() { + trail::test_support::changed_path_materialized_candidate_lifecycle_flow().unwrap(); +} diff --git a/trail/tests/changed_path_ledger_producers.rs b/trail/tests/changed_path_ledger_producers.rs new file mode 100644 index 0000000..8d40392 --- /dev/null +++ b/trail/tests/changed_path_ledger_producers.rs @@ -0,0 +1,2175 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard, OnceLock}; + +use rusqlite::{params, Connection}; +use sha2::{Digest, Sha256}; +use trail::{Error, InitImportMode, PatchDocument, Trail, WorktreeState}; + +const INVENTORY: &str = include_str!("fixtures/changed_path_producers.v1"); +const RAW_MUTATION_INVENTORY: &str = include_str!("fixtures/changed_path_raw_mutations.v1"); +const PRODUCER_TAG: &str = "TRAIL_FS_PRODUCER:"; + +// These are producer-facing filesystem mutation boundaries. Unlike producer +// annotations, calls to these sinks are discovered across the source tree, so +// adding a new caller requires an explicit review instead of merely omitting a +// tag and escaping the inventory comparison. +const DISCOVERED_MUTATION_SINKS: &[&str] = &[ + "remove_visible_files_absent_from_target", + "materialize_files", + "prepare_checkout_workdir", + "materialize_root_files_at_streaming", + "materialize_lane_workdir_at_paths_with_neighbors", + "materialize_lane_root_staged", + "materialize_sparse_lane_workdir_paths", + "materialize_full_lane_workdir_staged", + "rescue_replaced_lane_workdir_path", + "rescue_dirty_lane_workdir", + "apply_lane_patch_workdir_projection", + "apply_rewind_workdir_projection", + "materialize_files_at", + "sync_lane_workdir", + "complete_workspace_checkpoint", + "ensure_upper_file_under_barrier", + "write_all_file_at", +]; + +const RAW_MUTATION_SINKS: &[&str] = &[ + "fs::write", + "fs::rename", + "fs::remove_file", + "fs::remove_dir", + "fs::remove_dir_all", + "fs::create_dir", + "fs::create_dir_all", + "fs::copy", + "fs::set_permissions", + "File::create", + "symlink_file", +]; + +const OPEN_OPTIONS_MUTATION_SINKS: &[(&str, &str)] = &[ + (".create", "OpenOptions::create"), + (".write", "OpenOptions::write"), + (".truncate", "OpenOptions::truncate"), + (".create_new", "OpenOptions::create_new"), + (".append", "OpenOptions::append"), +]; + +// Implementation calls which are subordinate to an inventoried producer and +// are not independently callable command producers. New entries here are a +// deliberate review boundary and must name an exact function and sink. +const REVIEWED_INTERNAL_MUTATION_CALLERS: &[(&str, &str, &str)] = &[ + ( + "db/lane/lifecycle.rs", + "materialize_lane_workdir_at_paths_with_neighbors", + "materialize_lane_root_staged", + ), + ( + "db/lane/workdir/sync.rs", + "materialize_full_lane_workdir_staged", + "materialize_lane_root_staged", + ), + ( + "db/lane/workdir/sync.rs", + "sync_nfs_cow_lane_workdir", + "rescue_dirty_lane_workdir", + ), + ( + "db/lane/rewind.rs", + "apply_rewind_workdir_projection", + "materialize_files_at", + ), + ( + "db/lane/patching.rs", + "apply_lane_patch_workdir_projection", + "materialize_files_at", + ), + ( + "db/lane/workdir/sync.rs", + "materialize_full_lane_workdir_staged", + "materialize_files_at", + ), + ( + "db/lane/workdir/sync.rs", + "materialize_sparse_lane_workdir_paths", + "materialize_files_at", + ), + ( + "db/storage/content.rs", + "materialize_files", + "materialize_files_at", + ), + ( + "db/record/checkout.rs", + "remove_visible_files_absent_from_target", + "fs::remove_file", + ), + ( + "db/lane/lifecycle.rs", + "write_sparse_workdir_manifest", + "fs::create_dir_all", + ), + ( + "db/lane/workdir/record.rs", + "run_lane_record_after_c2_write", + "fs::write", + ), + ( + "db/lane/workdir/sync.rs", + "materialize_full_lane_workdir_staged", + "fs::remove_file", + ), + ( + "db/lane/workdir/sync.rs", + "materialize_full_lane_workdir_staged", + "fs::create_dir_all", + ), + ( + "db/lane/workdir/sync.rs", + "materialize_full_lane_workdir_staged", + "fs::remove_dir_all", + ), + ( + "db/lane/workdir/sync.rs", + "rescue_dirty_lane_workdir", + "fs::create_dir_all", + ), + ( + "db/lane/workdir/sync.rs", + "rescue_dirty_lane_workdir", + "fs::write", + ), + ( + "db/lane/workdir/sync.rs", + "rescue_replaced_lane_workdir_path", + "fs::create_dir_all", + ), + ( + "db/lane/workdir/sync.rs", + "rescue_replaced_lane_workdir_path", + "fs::write", + ), + ( + "db/lane/workdir/sync.rs", + "create_unique_lane_workdir_rescue_dir", + "fs::create_dir", + ), + ( + "db/lane/workdir/sync.rs", + "create_unique_lane_workdir_sync_stage_dir", + "fs::create_dir", + ), + ( + "db/lane/workdir/sync.rs", + "replace_lane_workdir_with_stage", + "fs::rename", + ), + ( + "db/lane/workdir/sync.rs", + "move_existing_lane_workdir_to_backup", + "fs::rename", + ), + ( + "db/lane/workdir/sync.rs", + "remove_existing_lane_workdir_path", + "fs::remove_dir_all", + ), + ( + "db/lane/workdir/sync.rs", + "remove_existing_lane_workdir_path", + "fs::remove_file", + ), + ("db/lane/workdir/sync.rs", "begin", "fs::create_dir_all"), + ("db/lane/workdir/sync.rs", "begin", "fs::remove_dir_all"), + ("db/lane/workdir/sync.rs", "commit", "fs::remove_dir_all"), + ("db/lane/workdir/sync.rs", "rollback", "fs::remove_dir_all"), + ( + "db/lane/workdir/sync.rs", + "restore_sparse_hydration_snapshot", + "fs::create_dir_all", + ), + ( + "db/lane/workdir/sync.rs", + "remove_sparse_hydration_target", + "fs::remove_dir_all", + ), + ( + "db/lane/workdir/sync.rs", + "remove_sparse_hydration_target", + "fs::remove_file", + ), + ( + "db/lane/workdir/sync.rs", + "rescue_dirty_lane_workdir", + "fs::copy", + ), + ( + "db/lane/workdir/sync.rs", + "rescue_replaced_lane_workdir_path", + "fs::copy", + ), + ( + "db/lane/workdir/sync.rs", + "snapshot_sparse_hydration_target", + "fs::copy", + ), + ( + "db/lane/workdir/sync.rs", + "restore_sparse_hydration_snapshot", + "fs::copy", + ), + ( + "db/lane/workdir/sync.rs", + "restore_sparse_hydration_snapshot", + "fs::set_permissions", + ), + ( + "db/lane/workdir/sync.rs", + "restore_sparse_hydration_snapshot", + "symlink_file", + ), + ( + "db/lane/workdir/sync.rs", + "sparse_hydration_file_matches_snapshot", + "fs::set_permissions", + ), +]; + +static RUNTIME_TEST_LOCK: OnceLock> = OnceLock::new(); + +struct AuthorityGuard { + _lock: MutexGuard<'static, ()>, +} + +impl AuthorityGuard { + fn enabled() -> Self { + let lock = RUNTIME_TEST_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + trail::test_support::set_changed_path_authority_override(true); + Self { _lock: lock } + } + + fn disabled() -> Self { + let lock = RUNTIME_TEST_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + trail::test_support::set_changed_path_authority_override(false); + Self { _lock: lock } + } +} + +impl Drop for AuthorityGuard { + fn drop(&mut self) { + trail::test_support::set_changed_path_authority_override(false); + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +struct ProducerSite { + status: String, + source: String, + function: String, + site: String, + producer: String, + protocol: String, + sinks: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +struct ReviewedRawMutation { + class: String, + source: String, + function: String, + sink: String, + count: usize, +} + +fn manifest_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) +} + +fn producer_inventory() -> Vec { + INVENTORY + .lines() + .filter(|line| !line.trim().is_empty() && !line.trim_start().starts_with('#')) + .map(|line| { + let fields = line.split('|').collect::>(); + assert_eq!(fields.len(), 7, "invalid producer inventory row: {line}"); + ProducerSite { + status: fields[0].to_string(), + source: fields[1].to_string(), + function: fields[2].to_string(), + site: fields[3].to_string(), + producer: fields[4].to_string(), + protocol: fields[5].to_string(), + sinks: fields[6].split(',').map(str::to_string).collect(), + } + }) + .collect() +} + +fn raw_mutation_inventory() -> Vec { + RAW_MUTATION_INVENTORY + .lines() + .filter(|line| !line.trim().is_empty() && !line.trim_start().starts_with('#')) + .map(|line| { + let fields = line.split('|').collect::>(); + assert_eq!( + fields.len(), + 5, + "invalid raw mutation inventory row: {line}" + ); + assert!( + matches!(fields[0], "reviewed" | "pending_task12"), + "invalid raw mutation review class in row: {line}" + ); + ReviewedRawMutation { + class: fields[0].to_string(), + source: fields[1].to_string(), + function: fields[2].to_string(), + sink: fields[3].to_string(), + count: fields[4] + .parse() + .unwrap_or_else(|_| panic!("invalid raw mutation count in row: {line}")), + } + }) + .collect() +} + +/// Return the exact Rust function body containing `fn name`, ignoring braces +/// in comments, strings, chars, and raw strings. This keeps the source audit +/// tied to a concrete entry point instead of accepting a protocol call placed +/// elsewhere in the file. +fn rust_function_body<'a>(source: &'a str, name: &str) -> &'a str { + let needle = format!("fn {name}"); + let start = source + .find(&needle) + .unwrap_or_else(|| panic!("missing inventoried function `{name}`")); + let open = source[start..] + .find('{') + .map(|offset| start + offset) + .unwrap_or_else(|| panic!("function `{name}` has no body")); + let bytes = source.as_bytes(); + let mut index = open; + let mut depth = 0usize; + let mut state = LexState::Code; + while index < bytes.len() { + match state { + LexState::Code => match bytes[index] { + b'/' if bytes.get(index + 1) == Some(&b'/') => { + state = LexState::LineComment; + index += 2; + continue; + } + b'/' if bytes.get(index + 1) == Some(&b'*') => { + state = LexState::BlockComment(1); + index += 2; + continue; + } + b'"' => state = LexState::String, + b'\'' => state = LexState::Char, + b'r' if matches!(bytes.get(index + 1), Some(b'"' | b'#')) => { + let mut cursor = index + 1; + let mut hashes = 0usize; + while bytes.get(cursor) == Some(&b'#') { + hashes += 1; + cursor += 1; + } + if bytes.get(cursor) == Some(&b'"') { + state = LexState::RawString(hashes); + index = cursor; + } + } + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + return &source[start..=index]; + } + } + _ => {} + }, + LexState::LineComment => { + if bytes[index] == b'\n' { + state = LexState::Code; + } + } + LexState::BlockComment(level) => { + if bytes[index] == b'/' && bytes.get(index + 1) == Some(&b'*') { + state = LexState::BlockComment(level + 1); + index += 2; + continue; + } + if bytes[index] == b'*' && bytes.get(index + 1) == Some(&b'/') { + state = if level == 1 { + LexState::Code + } else { + LexState::BlockComment(level - 1) + }; + index += 2; + continue; + } + } + LexState::String => { + if bytes[index] == b'\\' { + index += 2; + continue; + } + if bytes[index] == b'"' { + state = LexState::Code; + } + } + LexState::Char => { + if bytes[index] == b'\\' { + index += 2; + continue; + } + if bytes[index] == b'\'' { + state = LexState::Code; + } + } + LexState::RawString(hashes) => { + if bytes[index] == b'"' + && bytes.get(index + 1..index + 1 + hashes) + == Some(vec![b'#'; hashes].as_slice()) + { + state = LexState::Code; + index += hashes; + } + } + } + index += 1; + } + panic!("unterminated inventoried function `{name}`") +} + +#[derive(Clone, Copy)] +enum LexState { + Code, + LineComment, + BlockComment(usize), + String, + Char, + RawString(usize), +} + +/// Remove comments and literal contents while preserving executable Rust +/// tokens. Protocol and sink checks run against this representation, so a +/// comment or diagnostic string cannot impersonate a reviewed call. +fn rust_code_only(source: &str) -> String { + let bytes = source.as_bytes(); + let mut output = vec![b' '; bytes.len()]; + let mut index = 0usize; + let mut state = LexState::Code; + while index < bytes.len() { + match state { + LexState::Code => match bytes[index] { + b'/' if bytes.get(index + 1) == Some(&b'/') => { + state = LexState::LineComment; + index += 2; + continue; + } + b'/' if bytes.get(index + 1) == Some(&b'*') => { + state = LexState::BlockComment(1); + index += 2; + continue; + } + b'"' => state = LexState::String, + b'\'' + if bytes.get(index + 2) == Some(&b'\'') + || bytes.get(index + 1) == Some(&b'\\') => + { + state = LexState::Char; + } + b'r' if matches!(bytes.get(index + 1), Some(b'"' | b'#')) => { + let mut cursor = index + 1; + let mut hashes = 0usize; + while bytes.get(cursor) == Some(&b'#') { + hashes += 1; + cursor += 1; + } + if bytes.get(cursor) == Some(&b'"') { + state = LexState::RawString(hashes); + index = cursor; + } else { + output[index] = bytes[index]; + } + } + _ => output[index] = bytes[index], + }, + LexState::LineComment => { + if bytes[index] == b'\n' { + output[index] = b'\n'; + state = LexState::Code; + } + } + LexState::BlockComment(level) => { + if bytes[index] == b'/' && bytes.get(index + 1) == Some(&b'*') { + state = LexState::BlockComment(level + 1); + index += 2; + continue; + } + if bytes[index] == b'*' && bytes.get(index + 1) == Some(&b'/') { + state = if level == 1 { + LexState::Code + } else { + LexState::BlockComment(level - 1) + }; + index += 2; + continue; + } + } + LexState::String => { + if bytes[index] == b'\\' { + index += 2; + continue; + } + if bytes[index] == b'"' { + state = LexState::Code; + } + } + LexState::Char => { + if bytes[index] == b'\\' { + index += 2; + continue; + } + if bytes[index] == b'\'' { + state = LexState::Code; + } + } + LexState::RawString(hashes) => { + if bytes[index] == b'"' + && bytes.get(index + 1..index + 1 + hashes) + == Some(vec![b'#'; hashes].as_slice()) + { + state = LexState::Code; + index += hashes; + } + } + } + index += 1; + } + String::from_utf8(output).unwrap() +} + +fn rust_call_offsets(code: &str, callable: &str) -> Vec { + code.match_indices(callable) + .filter_map(|(start, _)| { + let before = code[..start].chars().next_back(); + if !callable.starts_with('.') + && before.is_some_and(|ch| ch == '_' || ch.is_ascii_alphanumeric()) + { + return None; + } + if !code[start + callable.len()..].trim_start().starts_with('(') { + return None; + } + // `fn sink(` declares a function; it is not a concrete call to + // the mutation boundary. + if code[..start].trim_end().ends_with("fn") { + return None; + } + Some(start) + }) + .collect() +} + +fn contains_rust_call(code: &str, callable: &str) -> bool { + !rust_call_offsets(code, callable).is_empty() +} + +fn rust_call_argument(code_after_callable: &str) -> Option<&str> { + let call = code_after_callable.trim_start(); + let call = call.strip_prefix('(')?; + let mut depth = 1usize; + for (index, byte) in call.as_bytes().iter().enumerate() { + match byte { + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return Some(&call[..index]); + } + } + _ => {} + } + } + None +} + +fn mutating_open_options_call_offsets(code: &str) -> Vec<(usize, &'static str)> { + let mut calls = Vec::new(); + for start in rust_call_offsets(code, "OpenOptions::new") { + let tail = &code[start..]; + let statement_end = tail.find(';').unwrap_or(tail.len()); + let statement = &tail[..statement_end]; + let (chain, base) = + if let Some(open) = rust_call_offsets(statement, ".open").into_iter().next() { + (&statement[..open], start) + } else { + let (_, function_range) = enclosing_rust_function_range(code, start).unwrap(); + let declaration_start = [b';', b'{', b'}'] + .into_iter() + .filter_map(|delimiter| { + code.as_bytes()[function_range.start..start] + .iter() + .rposition(|byte| *byte == delimiter) + }) + .max() + .map(|relative| function_range.start + relative + 1) + .unwrap_or(function_range.start); + let declaration = &code[declaration_start..start]; + let Some(equals) = declaration.rfind('=') else { + continue; + }; + let mut binding = declaration[..equals].trim(); + let Some(rest) = binding.strip_prefix("let ") else { + continue; + }; + binding = rest.trim_start(); + if let Some(rest) = binding.strip_prefix("mut ") { + binding = rest.trim_start(); + } + let name_len = binding + .chars() + .take_while(|ch| *ch == '_' || ch.is_ascii_alphanumeric()) + .map(char::len_utf8) + .sum::(); + if name_len == 0 { + continue; + } + let binding = &binding[..name_len]; + let configured = &code[start + statement_end + 1..function_range.end]; + let open_callable = format!("{binding}.open"); + let Some(open) = rust_call_offsets(configured, &open_callable) + .into_iter() + .next() + else { + continue; + }; + (&configured[..open], start + statement_end + 1) + }; + for (callable, sink) in OPEN_OPTIONS_MUTATION_SINKS { + for relative in rust_call_offsets(chain, callable) { + let argument = rust_call_argument(&chain[relative + callable.len()..]).unwrap(); + if argument.trim() == "false" { + continue; + } + calls.push((base + relative, *sink)); + } + } + } + calls +} + +fn rust_function_range(source: &str, name: &str) -> std::ops::Range { + let body = rust_function_body(source, name); + let start = body.as_ptr() as usize - source.as_ptr() as usize; + start..start + body.len() +} + +fn enclosing_rust_function_range( + code: &str, + offset: usize, +) -> Option<(&str, std::ops::Range)> { + code[..offset] + .match_indices("fn ") + .map(|(start, _)| start) + .collect::>() + .into_iter() + .rev() + .find_map(|start| { + let before = code[..start].chars().next_back(); + if before.is_some_and(|ch| ch == '_' || ch.is_ascii_alphanumeric()) { + return None; + } + let name_start = start + 3; + let name_len = code[name_start..] + .chars() + .take_while(|ch| *ch == '_' || ch.is_ascii_alphanumeric()) + .map(char::len_utf8) + .sum::(); + if name_len == 0 { + return None; + } + let open = code[start..].find('{').map(|relative| start + relative)?; + let mut depth = 0usize; + for (relative, byte) in code.as_bytes()[open..].iter().enumerate() { + match byte { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + return (offset <= open + relative).then_some(( + &code[name_start..name_start + name_len], + start..open + relative + 1, + )); + } + } + _ => {} + } + } + None + }) +} + +fn enclosing_rust_function(code: &str, offset: usize) -> Option<&str> { + enclosing_rust_function_range(code, offset).map(|(name, _)| name) +} + +fn rust_source_files(source_root: &Path) -> Vec { + let mut files = Vec::new(); + let mut stack = vec![source_root.to_path_buf()]; + while let Some(path) = stack.pop() { + for entry in fs::read_dir(path).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else if path.extension().and_then(|value| value.to_str()) == Some("rs") { + files.push(path); + } + } + } + files.sort(); + files +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ProductionCfgValue { + False, + True, + Unknown, +} + +impl ProductionCfgValue { + fn not(self) -> Self { + match self { + Self::False => Self::True, + Self::True => Self::False, + Self::Unknown => Self::Unknown, + } + } +} + +struct ProductionCfgParser<'a> { + input: &'a [u8], + index: usize, +} + +impl ProductionCfgParser<'_> { + fn skip_whitespace(&mut self) { + while self + .input + .get(self.index) + .is_some_and(u8::is_ascii_whitespace) + { + self.index += 1; + } + } + + fn parse_identifier(&mut self) -> String { + self.skip_whitespace(); + let start = self.index; + while self + .input + .get(self.index) + .is_some_and(|byte| *byte == b'_' || byte.is_ascii_alphanumeric()) + { + self.index += 1; + } + std::str::from_utf8(&self.input[start..self.index]) + .unwrap() + .to_string() + } + + fn parse_expression(&mut self) -> ProductionCfgValue { + let identifier = self.parse_identifier(); + self.skip_whitespace(); + if self.input.get(self.index) != Some(&b'(') { + let bare_predicate = matches!(self.input.get(self.index), None | Some(b',' | b')')); + while !matches!(self.input.get(self.index), None | Some(b',' | b')')) { + self.index += 1; + } + return if identifier == "test" && bare_predicate { + ProductionCfgValue::False + } else { + ProductionCfgValue::Unknown + }; + } + self.index += 1; + let mut arguments = Vec::new(); + loop { + self.skip_whitespace(); + if self.input.get(self.index) == Some(&b')') { + self.index += 1; + break; + } + arguments.push(self.parse_expression()); + self.skip_whitespace(); + match self.input.get(self.index) { + Some(b',') => self.index += 1, + Some(b')') => { + self.index += 1; + break; + } + _ => break, + } + } + match identifier.as_str() { + "not" => arguments + .into_iter() + .next() + .unwrap_or(ProductionCfgValue::Unknown) + .not(), + "all" => { + if arguments.contains(&ProductionCfgValue::False) { + ProductionCfgValue::False + } else if arguments + .iter() + .all(|value| *value == ProductionCfgValue::True) + { + ProductionCfgValue::True + } else { + ProductionCfgValue::Unknown + } + } + "any" => { + if arguments.contains(&ProductionCfgValue::True) { + ProductionCfgValue::True + } else if arguments + .iter() + .all(|value| *value == ProductionCfgValue::False) + { + ProductionCfgValue::False + } else { + ProductionCfgValue::Unknown + } + } + _ => ProductionCfgValue::Unknown, + } + } +} + +fn cfg_item_is_disabled_in_production(attribute: &str) -> bool { + let Some(cfg) = attribute.strip_prefix("#[cfg") else { + return false; + }; + let cfg = cfg.trim_start(); + let Some(expression) = cfg.strip_prefix('(') else { + return false; + }; + ProductionCfgParser { + input: expression.as_bytes(), + index: 0, + } + .parse_expression() + == ProductionCfgValue::False +} + +fn nonproduction_cfg_ranges(code: &str) -> Vec> { + let mut ranges = Vec::new(); + for (attribute_start, _) in code.match_indices("#[cfg") { + let Some(attribute_end) = code[attribute_start..] + .find(']') + .map(|relative| attribute_start + relative) + else { + continue; + }; + let attribute = &code[attribute_start..=attribute_end]; + if !cfg_item_is_disabled_in_production(attribute) { + continue; + } + let mut item_start = attribute_end + 1; + loop { + let trimmed = code[item_start..].trim_start(); + item_start = code.len() - trimmed.len(); + if !trimmed.starts_with("#[") { + break; + } + let Some(end) = trimmed.find(']') else { break }; + item_start += end + 1; + } + let mut item = code[item_start..].trim_start(); + for prefix in [ + "pub(crate) ", + "pub(super) ", + "pub ", + "async ", + "unsafe ", + "const ", + ] { + if let Some(rest) = item.strip_prefix(prefix) { + item = rest.trim_start(); + } + } + if ![ + "fn ", "impl ", "struct ", "enum ", "mod ", "trait ", "union ", + ] + .iter() + .any(|prefix| item.starts_with(prefix)) + { + continue; + } + let Some(open) = code[item_start..] + .find('{') + .map(|relative| item_start + relative) + else { + continue; + }; + let mut depth = 0usize; + for (relative, byte) in code.as_bytes()[open..].iter().enumerate() { + match byte { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + ranges.push(attribute_start..open + relative + 1); + break; + } + } + _ => {} + } + } + } + ranges +} + +fn unreviewed_mutation_calls( + source_root: &Path, + inventory: &[ProducerSite], + sinks: &[&str], + source_filter: Option<&BTreeSet>, +) -> Vec { + let mut unreviewed = Vec::new(); + for path in rust_source_files(source_root) { + let source = fs::read_to_string(&path).unwrap(); + let code = rust_code_only(&source); + let nonproduction_ranges = nonproduction_cfg_ranges(&code); + let relative = path + .strip_prefix(source_root) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + if source_filter.is_some_and(|filter| !filter.contains(&relative)) { + continue; + } + for sink in sinks { + let mut reviewed_ranges = inventory + .iter() + .filter(|site| { + site.source == relative && site.sinks.iter().any(|item| item == sink) + }) + .map(|site| rust_function_range(&source, &site.function)) + .collect::>(); + reviewed_ranges.extend( + REVIEWED_INTERNAL_MUTATION_CALLERS + .iter() + .filter(|(reviewed_source, _, reviewed_sink)| { + *reviewed_source == relative && *reviewed_sink == *sink + }) + .map(|(_, function, _)| rust_function_range(&source, function)), + ); + for offset in rust_call_offsets(&code, sink) { + if nonproduction_ranges + .iter() + .any(|range| range.contains(&offset)) + { + continue; + } + if reviewed_ranges.iter().any(|range| range.contains(&offset)) { + continue; + } + let line = source[..offset] + .bytes() + .filter(|byte| *byte == b'\n') + .count() + + 1; + let function = enclosing_rust_function(&code, offset).unwrap_or(""); + unreviewed.push(format!("{relative}:{line} in `{function}` calls `{sink}`")); + } + } + } + unreviewed.sort(); + unreviewed +} + +fn raw_mutation_call_counts(source_root: &Path) -> BTreeMap<(String, String, String), usize> { + let mut calls = BTreeMap::new(); + for path in rust_source_files(&source_root.join("db")) { + let source = fs::read_to_string(&path).unwrap(); + let code = rust_code_only(&source); + let nonproduction_ranges = nonproduction_cfg_ranges(&code); + let relative = path + .strip_prefix(source_root) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + for sink in RAW_MUTATION_SINKS { + for offset in rust_call_offsets(&code, sink) { + if nonproduction_ranges + .iter() + .any(|range| range.contains(&offset)) + { + continue; + } + let function = enclosing_rust_function(&code, offset).unwrap_or(""); + *calls + .entry((relative.clone(), function.to_string(), (*sink).to_string())) + .or_insert(0) += 1; + } + } + for (offset, sink) in mutating_open_options_call_offsets(&code) { + if nonproduction_ranges + .iter() + .any(|range| range.contains(&offset)) + { + continue; + } + let function = enclosing_rust_function(&code, offset).unwrap_or(""); + *calls + .entry((relative.clone(), function.to_string(), sink.to_string())) + .or_insert(0) += 1; + } + } + calls +} + +fn unreviewed_raw_mutation_calls( + source_root: &Path, + reviewed: &[ReviewedRawMutation], +) -> Vec { + let actual = raw_mutation_call_counts(source_root); + let mut expected = BTreeMap::new(); + for row in reviewed { + assert!( + row.count > 0, + "raw mutation count must be positive: {row:?}" + ); + assert!( + RAW_MUTATION_SINKS.contains(&row.sink.as_str()) + || OPEN_OPTIONS_MUTATION_SINKS + .iter() + .any(|(_, sink)| *sink == row.sink.as_str()), + "raw mutation inventory uses unknown sink: {row:?}" + ); + assert!( + expected + .insert( + (row.source.clone(), row.function.clone(), row.sink.clone()), + row.count, + ) + .is_none(), + "duplicate raw mutation inventory row: {row:?}" + ); + } + let keys = actual + .keys() + .chain(expected.keys()) + .cloned() + .collect::>(); + keys.into_iter() + .filter_map(|(source, function, sink)| { + let actual_count = actual + .get(&(source.clone(), function.clone(), sink.clone())) + .copied() + .unwrap_or(0); + let expected_count = expected + .get(&(source.clone(), function.clone(), sink.clone())) + .copied() + .unwrap_or(0); + (actual_count != expected_count).then(|| { + format!( + "{source}|{function}|{sink}: expected {expected_count}, found {actual_count}" + ) + }) + }) + .collect() +} + +fn annotated_sites(source_root: &Path) -> BTreeMap { + let mut discovered = BTreeMap::new(); + let mut stack = vec![source_root.to_path_buf()]; + while let Some(path) = stack.pop() { + for entry in fs::read_dir(&path).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_dir() { + stack.push(path); + continue; + } + if path.extension().and_then(|value| value.to_str()) != Some("rs") { + continue; + } + let source = fs::read_to_string(&path).unwrap(); + for line in source.lines().filter(|line| line.contains(PRODUCER_TAG)) { + let annotation = line.split_once(PRODUCER_TAG).unwrap().1.trim(); + let fields = annotation.split_whitespace().collect::>(); + assert_eq!( + fields.len(), + 3, + "producer annotation must be ` `: {}:{}", + path.display(), + line + ); + let relative = path + .strip_prefix(source_root) + .unwrap() + .to_string_lossy() + .replace('\\', "/"); + assert!( + discovered + .insert( + fields[0].to_string(), + (relative, fields[1].to_string(), fields[2].to_string()) + ) + .is_none(), + "duplicate producer site id `{}`", + fields[0] + ); + } + } + } + discovered +} + +#[test] +fn every_filesystem_producer_entry_calls_its_reviewed_protocol() { + let source_root = manifest_dir().join("src"); + let inventory = producer_inventory(); + let mut ids = BTreeSet::new(); + let mut uncovered = Vec::new(); + for site in &inventory { + assert!( + ids.insert(site.site.clone()), + "duplicate inventory site `{}`", + site.site + ); + let source = fs::read_to_string(source_root.join(&site.source)).unwrap(); + let body = rust_function_body(&source, &site.function); + let code = rust_code_only(body); + for sink in &site.sinks { + assert!( + contains_rust_call(&code, sink), + "inventoried mutation sink `{sink}` disappeared from {}::{}; review and update the concrete producer inventory", + site.source, + site.function + ); + } + match site.status.as_str() { + "controlled" => { + if !contains_rust_call(&code, &site.protocol) { + uncovered.push(format!( + "controlled producer `{}` reaches {:?} but does not call `{}` inside {}::{}", + site.site, site.sinks, site.protocol, site.source, site.function + )); + } + let tag = format!("{PRODUCER_TAG} {} {} controlled", site.site, site.producer); + assert!( + body.contains(&tag), + "missing exact producer annotation `{tag}`" + ); + } + "exempt" => { + assert!( + matches!( + site.producer.as_str(), + "exempt_rescue_output" | "exempt_alternate_output" + ), + "unreviewed exemption class `{}`", + site.producer + ); + let tag = format!("{PRODUCER_TAG} {} {} exempt", site.site, site.producer); + assert!( + body.contains(&tag), + "missing exact exemption annotation `{tag}`" + ); + } + "pending_task12" | "pending_task12_boundary" => { + assert_eq!(site.producer, "CowPublication"); + assert_eq!(site.protocol, "task12_qualified_view_journal"); + assert!( + !code.contains("run_projection_alignment") + && !code.contains("run_ref_advancing_projection"), + "Task 12 mounted callback `{}` must not be accidentally treated as Task 11 clean authority", + site.site + ); + if site.status == "pending_task12_boundary" { + let tag = format!( + "{PRODUCER_TAG} {} {} task12_callback_boundary", + site.site, site.producer + ); + assert!( + body.contains(&tag), + "missing exact Task 12 boundary `{tag}`" + ); + } + } + other => panic!("unknown producer inventory status `{other}`"), + } + } + let annotations = annotated_sites(&source_root.join("db")); + let inventoried_annotations = inventory + .iter() + .filter(|site| site.status != "pending_task12") + .map(|site| site.site.clone()) + .collect::>(); + assert_eq!( + annotations.keys().cloned().collect::>(), + inventoried_annotations, + "producer annotations and the concrete reviewed inventory diverged" + ); + assert!( + uncovered.is_empty(), + "uncovered changed-path producers:\n{}", + uncovered.join("\n") + ); + let unreviewed_calls = + unreviewed_mutation_calls(&source_root, &inventory, DISCOVERED_MUTATION_SINKS, None); + assert!( + unreviewed_calls.is_empty(), + "unreviewed direct filesystem mutation producers:\n{}", + unreviewed_calls.join("\n") + ); + let raw_inventory = raw_mutation_inventory(); + assert!( + raw_inventory + .iter() + .filter(|row| row.class == "pending_task12") + .all(|row| { + row.source.starts_with("db/lane/workdir/view_") + || matches!( + row.source.as_str(), + "db/lane/workdir/dokan.rs" + | "db/lane/workdir/fuse.rs" + | "db/lane/workdir/nfs_overlay.rs" + ) + }), + "Task 12 raw mutation boundaries must remain isolated to mounted-view helpers" + ); + let unreviewed_raw_calls = unreviewed_raw_mutation_calls(&source_root, &raw_inventory); + assert!( + unreviewed_raw_calls.is_empty(), + "unreviewed raw filesystem mutation producers:\n{}", + unreviewed_raw_calls.join("\n") + ); +} + +#[test] +fn producer_inventory_ignores_protocol_names_in_comments_and_literals() { + let fake = r##" + fn fake_producer() { + // run_projection_alignment(fake_sink()); + let _comment = "run_ref_advancing_projection(other_sink())"; + let _raw = r#"materialize_lane_root_staged()"#; + reviewed_sink(); + } + "##; + let body = rust_function_body(fake, "fake_producer"); + let code = rust_code_only(body); + assert!(!code.contains("run_projection_alignment")); + assert!(!code.contains("run_ref_advancing_projection")); + assert!(!code.contains("materialize_lane_root_staged")); + assert!(contains_rust_call(&code, "reviewed_sink")); +} + +#[test] +fn producer_discovery_rejects_an_untagged_direct_mutation_sink() { + let source_root = tempfile::tempdir().unwrap(); + fs::create_dir(source_root.path().join("db")).unwrap(); + fs::write( + source_root.path().join("db/fake.rs"), + r#" + fn reviewed() { + run_projection_alignment(|| materialize_files()); + } + + fn newly_added_untagged_producer() { + materialize_files(); + } + "#, + ) + .unwrap(); + let inventory = vec![ProducerSite { + status: "controlled".into(), + source: "db/fake.rs".into(), + function: "reviewed".into(), + site: "reviewed".into(), + producer: "Checkout".into(), + protocol: "run_projection_alignment".into(), + sinks: vec!["materialize_files".into()], + }]; + + let unreviewed = + unreviewed_mutation_calls(source_root.path(), &inventory, &["materialize_files"], None); + assert_eq!(unreviewed.len(), 1, "unexpected discovery: {unreviewed:?}"); + assert!(unreviewed[0].contains("db/fake.rs")); + assert!(unreviewed[0].contains("calls `materialize_files`")); +} + +#[test] +fn producer_discovery_rejects_new_rewind_materialization_and_raw_sinks() { + let source_root = tempfile::tempdir().unwrap(); + fs::create_dir(source_root.path().join("db")).unwrap(); + fs::write( + source_root.path().join("db/fake.rs"), + r#" + fn reviewed() { + run_ref_advancing_projection(|| apply_rewind_workdir_projection()); + } + + fn unreviewed_projection() { + apply_rewind_workdir_projection(); + materialize_files_at(); + fs::rename("before", "after"); + } + "#, + ) + .unwrap(); + let inventory = vec![ProducerSite { + status: "controlled".into(), + source: "db/fake.rs".into(), + function: "reviewed".into(), + site: "reviewed-rewind".into(), + producer: "RestoreProjection".into(), + protocol: "run_ref_advancing_projection".into(), + sinks: vec!["apply_rewind_workdir_projection".into()], + }]; + let unreviewed = unreviewed_mutation_calls( + source_root.path(), + &inventory, + &[ + "apply_rewind_workdir_projection", + "materialize_files_at", + "fs::rename", + ], + None, + ); + assert_eq!(unreviewed.len(), 3, "unexpected discovery: {unreviewed:?}"); + assert!(unreviewed + .iter() + .any(|call| call.contains("apply_rewind_workdir_projection"))); + assert!(unreviewed + .iter() + .any(|call| call.contains("materialize_files_at"))); + assert!(unreviewed.iter().any(|call| call.contains("fs::rename"))); +} + +#[test] +fn raw_producer_discovery_rejects_a_new_uninventoried_db_source_file() { + let source_root = tempfile::tempdir().unwrap(); + fs::create_dir(source_root.path().join("db")).unwrap(); + fs::write( + source_root.path().join("db/reviewed.rs"), + r#" + fn reviewed() { + fs::write("reviewed", b"reviewed").unwrap(); + } + "#, + ) + .unwrap(); + fs::write( + source_root.path().join("db/new_producer.rs"), + r#" + fn newly_added_uninventoried_producer() { + fs::write("unreviewed", b"unreviewed").unwrap(); + } + "#, + ) + .unwrap(); + let inventory = vec![ReviewedRawMutation { + class: "reviewed".into(), + source: "db/reviewed.rs".into(), + function: "reviewed".into(), + sink: "fs::write".into(), + count: 1, + }]; + + let unreviewed = unreviewed_raw_mutation_calls(source_root.path(), &inventory); + assert_eq!(unreviewed.len(), 1, "unexpected discovery: {unreviewed:?}"); + assert!(unreviewed[0].contains("db/new_producer.rs")); + assert!(unreviewed[0].contains("fs::write")); + assert!(unreviewed[0].contains("expected 0, found 1")); +} + +#[test] +fn raw_producer_discovery_rejects_an_extra_call_inside_a_reviewed_function() { + let source_root = tempfile::tempdir().unwrap(); + fs::create_dir(source_root.path().join("db")).unwrap(); + fs::write( + source_root.path().join("db/reviewed.rs"), + r#" + fn reviewed() { + fs::write("first", b"first").unwrap(); + fs::write("second", b"second").unwrap(); + } + "#, + ) + .unwrap(); + let inventory = vec![ReviewedRawMutation { + class: "reviewed".into(), + source: "db/reviewed.rs".into(), + function: "reviewed".into(), + sink: "fs::write".into(), + count: 1, + }]; + + let unreviewed = unreviewed_raw_mutation_calls(source_root.path(), &inventory); + assert_eq!(unreviewed.len(), 1, "unexpected discovery: {unreviewed:?}"); + assert!(unreviewed[0].contains("db/reviewed.rs|reviewed|fs::write")); + assert!(unreviewed[0].contains("expected 1, found 2")); +} + +#[test] +fn raw_producer_discovery_covers_remove_dir_file_create_and_open_options_chains() { + let source_root = tempfile::tempdir().unwrap(); + fs::create_dir(source_root.path().join("db")).unwrap(); + fs::write( + source_root.path().join("db/raw_apis.rs"), + r#" + fn remove_one_directory() { fs::remove_dir("dir").unwrap(); } + fn create_one_file() { File::create("file").unwrap(); } + fn open_with_create() { OpenOptions::new().create(true).open("file").unwrap(); } + fn open_with_write() { OpenOptions::new().write(true).open("file").unwrap(); } + fn open_with_truncate() { OpenOptions::new().truncate(true).open("file").unwrap(); } + fn open_with_create_new() { OpenOptions::new().create_new(true).open("file").unwrap(); } + fn open_with_append() { OpenOptions::new().append(true).open("file").unwrap(); } + fn open_with_conditional_write(enabled: bool) { + OpenOptions::new().write(false || enabled).open("file").unwrap(); + } + fn open_with_named_builder() { + let mut options = OpenOptions::new(); + options.write(true).truncate(true); + options.open("file").unwrap(); + } + fn read_only() { OpenOptions::new().read(true).write(false).open("file").unwrap(); } + "#, + ) + .unwrap(); + + let unreviewed = unreviewed_raw_mutation_calls(source_root.path(), &[]); + for expected in [ + "remove_one_directory|fs::remove_dir", + "create_one_file|File::create", + "open_with_create|OpenOptions::create", + "open_with_write|OpenOptions::write", + "open_with_truncate|OpenOptions::truncate", + "open_with_create_new|OpenOptions::create_new", + "open_with_append|OpenOptions::append", + "open_with_conditional_write|OpenOptions::write", + "open_with_named_builder|OpenOptions::write", + "open_with_named_builder|OpenOptions::truncate", + ] { + assert!( + unreviewed.iter().any(|call| call.contains(expected)), + "missing `{expected}` from {unreviewed:?}" + ); + } + assert_eq!(unreviewed.len(), 10, "unexpected discovery: {unreviewed:?}"); +} + +#[test] +fn raw_producer_discovery_scans_production_capable_cfg_items() { + let source_root = tempfile::tempdir().unwrap(); + fs::create_dir(source_root.path().join("db")).unwrap(); + fs::write( + source_root.path().join("db/cfg.rs"), + r#" + #[cfg(not(test))] + fn enabled_outside_tests() { fs::write("production", b"production").unwrap(); } + + #[cfg(any(test, unix))] + fn enabled_in_some_production_builds() { fs::write("unix", b"unix").unwrap(); } + + #[cfg(debug_assertions)] + fn enabled_in_debug_production_builds() { fs::write("debug", b"debug").unwrap(); } + + #[cfg(test = "custom")] + fn custom_test_key_value_is_not_the_test_harness() { + fs::write("custom-test", b"custom-test").unwrap(); + } + + #[cfg(test)] + fn test_only() { fs::write("test", b"test").unwrap(); } + + #[cfg(all(test, unix))] + fn still_test_only() { fs::write("test-unix", b"test-unix").unwrap(); } + "#, + ) + .unwrap(); + + let unreviewed = unreviewed_raw_mutation_calls(source_root.path(), &[]); + assert_eq!(unreviewed.len(), 4, "unexpected discovery: {unreviewed:?}"); + assert!(unreviewed + .iter() + .any(|call| call.contains("enabled_outside_tests|fs::write"))); + assert!(unreviewed + .iter() + .any(|call| call.contains("enabled_in_some_production_builds|fs::write"))); + assert!(unreviewed + .iter() + .any(|call| call.contains("enabled_in_debug_production_builds|fs::write"))); + assert!(unreviewed + .iter() + .any(|call| call.contains("custom_test_key_value_is_not_the_test_harness|fs::write"))); + assert!(!unreviewed.iter().any(|call| call.contains("test_only"))); + assert!(!unreviewed + .iter() + .any(|call| call.contains("still_test_only"))); +} + +#[test] +fn raw_producer_discovery_does_not_skip_a_tests_rs_source_file() { + let source_root = tempfile::tempdir().unwrap(); + fs::create_dir(source_root.path().join("db")).unwrap(); + fs::write( + source_root.path().join("db/tests.rs"), + r#" + fn uninventoried_tests_rs_producer() { + fs::write("unreviewed", b"unreviewed").unwrap(); + } + "#, + ) + .unwrap(); + + let unreviewed = unreviewed_raw_mutation_calls(source_root.path(), &[]); + assert_eq!(unreviewed.len(), 1, "unexpected discovery: {unreviewed:?}"); + assert!(unreviewed[0].contains("db/tests.rs|uninventoried_tests_rs_producer|fs::write")); +} + +#[test] +fn controlled_rewind_sink_is_inside_the_ref_advancing_projection() { + let rewind = fs::read_to_string(manifest_dir().join("src/db/lane/rewind.rs")).unwrap(); + let entry = rust_code_only(rust_function_body(&rewind, "rewind_lane")); + let protocol = entry.find("run_ref_advancing_projection").unwrap(); + let controlled_interval = entry + .find("with_materialized_lane_controlled_interval") + .unwrap(); + let controlled_sink = entry.find("apply_rewind_workdir_projection").unwrap(); + let fallback_commit = entry.find("commit_lane_operation_atomic").unwrap(); + assert_eq!( + rust_call_offsets(&entry, "apply_rewind_workdir_projection").len(), + 1 + ); + assert!( + protocol < controlled_interval + && controlled_interval < controlled_sink + && controlled_sink < fallback_commit + ); + + let helper = rust_code_only(rust_function_body( + &rewind, + "apply_rewind_workdir_projection", + )); + assert!(contains_rust_call(&helper, "materialize_files_at")); +} + +#[test] +fn every_materialized_lane_clean_consumer_uses_the_exact_snapshot_gateway() { + let source_root = manifest_dir().join("src/db"); + let merge = fs::read_to_string(source_root.join("merge/lane.rs")).unwrap(); + let record = fs::read_to_string(source_root.join("lane/workdir/record.rs")).unwrap(); + let patch = fs::read_to_string(source_root.join("lane/patching.rs")).unwrap(); + let identity = fs::read_to_string(source_root.join("lane/identity.rs")).unwrap(); + let readiness = fs::read_to_string(source_root.join("lane/readiness.rs")).unwrap(); + + let changed_paths = rust_code_only(rust_function_body(&merge, "lane_workdir_changed_paths")); + assert!( + contains_rust_call(&changed_paths, "compare_materialized_lane_candidates") + || contains_rust_call( + &changed_paths, + "with_materialized_lane_authoritative_snapshot", + ), + "lane status/merge cleanliness still bypasses the exact materialized-lane snapshot" + ); + let record_candidates = rust_code_only(rust_function_body( + &record, + "lane_workdir_record_changed_paths_with_case_fold", + )); + assert!( + contains_rust_call(&record_candidates, "compare_materialized_lane_candidates") + || contains_rust_call( + &record_candidates, + "with_materialized_lane_authoritative_snapshot", + ), + "lane record and normal preview still bypass the exact materialized-lane snapshot" + ); + let patch_entry = rust_code_only(rust_function_body(&patch, "apply_lane_patch_locked")); + assert!( + contains_rust_call(&patch_entry, "with_materialized_lane_controlled_interval") + && contains_rust_call(&patch_entry, "compare_controlled_projection_target"), + "structured-patch authority no longer verifies its Prepared projection through the shared controlled interval" + ); + let patch_apply = rust_code_only(rust_function_body( + &patch, + "apply_lane_patch_workdir_projection", + )); + assert!( + patch_apply.contains("allow_legacy_manifest_shortcut") + && contains_rust_call( + &patch_apply, + "refresh_clean_materialized_workdir_for_lane_patch", + ), + "legacy manifest refresh is no longer explicitly isolated behind the authority-off apply decision" + ); + + assert!( + rust_function_body(&identity, "lane_status").contains("lane_workdir_changed_paths"), + "lane status no longer uses the reviewed lane clean gateway" + ); + assert!( + rust_function_body(&record, "preview_lane_workdir_record") + .contains("lane_workdir_record_changed_paths_with_case_fold"), + "normal lane preview no longer uses the reviewed lane candidate gateway" + ); + let record_entry = rust_code_only(rust_function_body(&record, "record_lane_workdir_locked")); + assert!( + contains_rust_call( + &record_entry, + "lane_workdir_record_changed_paths_with_case_fold", + ) || contains_rust_call(&record_entry, "compare_materialized_lane_candidates") + || contains_rust_call( + &record_entry, + "with_materialized_lane_authoritative_snapshot", + ), + "lane record bypasses the reviewed lane candidate gateway" + ); + assert!( + rust_function_body(&readiness, "lane_readiness").contains("lane_status"), + "lane readiness no longer inherits the exact status snapshot" + ); + assert!( + rust_function_body(&merge, "ensure_lane_workdir_clean") + .contains("lane_workdir_changed_paths"), + "merge cleanliness no longer inherits the exact lane snapshot" + ); +} + +fn init_lane_fixture(lane: &str, sparse: bool) -> (tempfile::TempDir, Trail, PathBuf) { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "base\n").unwrap(); + fs::create_dir(temp.path().join("src")).unwrap(); + fs::write(temp.path().join("src/lib.rs"), "pub fn base() {}\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let spawn_result = if sparse { + db.spawn_lane_with_workdir_paths( + lane, + Some("main"), + true, + None, + None, + None, + &["README.md".to_string()], + ) + } else { + db.spawn_lane(lane, Some("main"), true, None, None) + }; + let lane_root = match spawn_result { + Ok(spawned) => PathBuf::from(spawned.workdir.unwrap()), + Err(Error::CommittedRepairRequired { + operation, + repair, + reason, + }) if matches!( + repair.as_str(), + "initial materialized lane ledger reconciliation" + | "initial materialized lane ledger alignment" + ) => + { + // Association and materialization committed before this bounded + // observer-boundary repair failure. Resolve that committed lane + // through the public authoritative reconciliation path; never + // retry creation or accept unrelated committed failures. + let details = db.lane_details(lane).unwrap_or_else(|recovery| { + panic!( + "lane spawn operation {operation} committed but `{repair}` could not resolve its association: {reason}; recovery: {recovery}" + ) + }); + let workdir = PathBuf::from(details.branch.workdir.unwrap_or_else(|| { + panic!( + "lane spawn operation {operation} committed but `{repair}` left no materialized workdir: {reason}" + ) + })); + let status = db.lane_status(lane).unwrap_or_else(|recovery| { + panic!( + "lane spawn operation {operation} committed but `{repair}` reconciliation failed: {reason}; recovery: {recovery}" + ) + }); + assert_eq!( + status.workdir_state, + Some(WorktreeState::Clean), + "committed lane spawn reconciliation did not restore clean authority" + ); + assert!(status.workdir_changed_paths.is_empty()); + let repaired = read_marker(&workdir); + let resolved = db.lane_details(lane).unwrap().branch; + assert_eq!(repaired["version"], 2); + assert_eq!(repaired["root_id"], resolved.head_root.0); + workdir + } + Err(error) => panic!("failed to initialize lane fixture `{lane}`: {error}"), + }; + (temp, db, lane_root) +} + +fn read_marker(workdir: &Path) -> serde_json::Value { + serde_json::from_slice(&fs::read(workdir.join(".trail/workdir-manifest.json")).unwrap()) + .unwrap() +} + +#[test] +fn materialized_lane_marker_is_compact_v2_and_sparse_selection_is_separate() { + let _authority = AuthorityGuard::enabled(); + let (_temp, db, workdir) = init_lane_fixture("marker-sparse", true); + let marker = read_marker(&workdir); + assert_eq!(marker["version"], 2); + for field in [ + "scope_id", + "filesystem_identity", + "ref_name", + "ref_generation", + "root_id", + "policy_fingerprint", + "epoch", + "provider_cut", + "provider_segment_id", + "sparse_selection_fingerprint", + ] { + assert!(!marker[field].is_null(), "v2 marker is missing `{field}`"); + } + assert!( + marker.get("files").is_none(), + "v2 marker regressed to an N-entry manifest" + ); + + let sparse: serde_json::Value = + serde_json::from_slice(&fs::read(workdir.join(".trail/sparse-selection.json")).unwrap()) + .unwrap(); + assert_eq!(sparse["materialized_paths"][0], "README.md"); + let status = db.lane_status("marker-sparse").unwrap(); + assert_eq!( + status.workdir_state, + Some(WorktreeState::Clean), + "fresh sparse authoritative status reported {:?}", + status.workdir_changed_paths + ); + + fs::write(workdir.join("new.txt"), "visible sparse addition\n").unwrap(); + let dirty = db.lane_status("marker-sparse").unwrap(); + assert_eq!(dirty.workdir_state, Some(WorktreeState::DirtyUntracked)); + assert_eq!(dirty.workdir_changed_paths.len(), 1); + assert_eq!(dirty.workdir_changed_paths[0].path, "new.txt"); + assert!( + dirty + .workdir_changed_paths + .iter() + .all(|change| change.path != "src/lib.rs"), + "unmaterialized baseline path was reported as deleted" + ); +} + +#[test] +fn malformed_sparse_selection_cannot_reconcile_or_publish_clean_authority() { + let _authority = AuthorityGuard::enabled(); + for (suffix, malformed, expected_reason) in [ + ( + "missing-paths", + serde_json::json!({}), + "required `materialized_paths` field is missing", + ), + ( + "non-array", + serde_json::json!({"version": 1, "materialized_paths": "README.md"}), + "`materialized_paths` must be an array", + ), + ( + "mixed-types", + serde_json::json!({"version": 1, "materialized_paths": ["README.md", 7]}), + "`materialized_paths[1]` must be a string", + ), + ] { + let lane = format!("sparse-malformed-{suffix}"); + let (_temp, db, workdir) = init_lane_fixture(&lane, true); + let marker_path = workdir.join(".trail/workdir-manifest.json"); + let marker_before = fs::read(&marker_path).unwrap(); + fs::remove_file(workdir.join("README.md")).unwrap(); + fs::write( + workdir.join(".trail/sparse-selection.json"), + serde_json::to_vec(&malformed).unwrap(), + ) + .unwrap(); + + let error = db.lane_status(&lane).unwrap_err(); + assert_eq!(error.code(), "DATABASE_CORRUPT"); + assert!(error.to_string().contains(expected_reason)); + assert_eq!( + fs::read(marker_path).unwrap(), + marker_before, + "malformed sparse selection published a replacement clean marker" + ); + } +} + +#[test] +fn explicit_empty_sparse_selection_excludes_baseline_absence_but_not_visible_additions() { + let _authority = AuthorityGuard::enabled(); + let (_temp, db, workdir) = init_lane_fixture("sparse-explicit-empty", true); + let marker_before = read_marker(&workdir); + fs::remove_file(workdir.join("README.md")).unwrap(); + fs::write( + workdir.join(".trail/sparse-selection.json"), + serde_json::to_vec(&serde_json::json!({ + "version": 1, + "materialized_paths": [] + })) + .unwrap(), + ) + .unwrap(); + + let clean = db.lane_status("sparse-explicit-empty").unwrap(); + assert_eq!(clean.workdir_state, Some(WorktreeState::Clean)); + assert!(clean.workdir_changed_paths.is_empty()); + let marker_after = read_marker(&workdir); + assert_eq!(marker_after["version"], 2); + assert_ne!( + marker_after["sparse_selection_fingerprint"], marker_before["sparse_selection_fingerprint"], + "explicit empty selection was confused with the prior selected path set" + ); + + fs::write(workdir.join("visible.txt"), "visible addition\n").unwrap(); + let dirty = db.lane_status("sparse-explicit-empty").unwrap(); + assert_eq!(dirty.workdir_state, Some(WorktreeState::DirtyUntracked)); + assert_eq!(dirty.workdir_changed_paths.len(), 1); + assert_eq!(dirty.workdir_changed_paths[0].path, "visible.txt"); +} + +#[test] +fn authority_off_uses_direct_lane_observation_and_creates_no_trusted_scope() { + let _authority = AuthorityGuard::disabled(); + let (temp, db, workdir) = init_lane_fixture("authority-off", false); + let before = read_marker(&workdir); + assert_ne!( + before["version"], 2, + "authority-off spawn wrote a V2 marker" + ); + let status = db.lane_status("authority-off").unwrap(); + assert_eq!(status.workdir_state, Some(WorktreeState::Clean)); + let after = read_marker(&workdir); + assert_ne!( + after["version"], 2, + "authority-off status wrote a V2 marker" + ); + assert!( + after.get("files").is_some(), + "authority-off status did not preserve the usable V1 manifest cache" + ); + + let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); + let trusted: i64 = conn + .query_row( + "SELECT COUNT(*) FROM changed_path_scopes + WHERE scope_kind='materialized_lane' AND trust_state='trusted' AND retired_at IS NULL", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + trusted, 0, + "authority-off lane status persisted trusted scope state" + ); +} + +#[test] +fn sparse_hydration_failure_restores_bytes_selection_and_v2_marker() { + struct SparseSelectionFailure; + impl SparseSelectionFailure { + fn enable() -> Self { + trail::test_support::set_sparse_selection_write_failure_for_current_thread(true); + Self + } + } + impl Drop for SparseSelectionFailure { + fn drop(&mut self) { + trail::test_support::set_sparse_selection_write_failure_for_current_thread(false); + } + } + + let _authority = AuthorityGuard::enabled(); + let (_temp, mut db, workdir) = init_lane_fixture("sparse-rollback-ledger", true); + let initial_status = db.lane_status("sparse-rollback-ledger").unwrap(); + assert_eq!( + initial_status.workdir_state, + Some(WorktreeState::Clean), + "fresh sparse authoritative status reported {:?}", + initial_status.workdir_changed_paths + ); + let metadata_dir = workdir.join(".trail"); + let selection = metadata_dir.join("sparse-selection.json"); + let marker = metadata_dir.join("workdir-manifest.json"); + let selection_before = fs::read(&selection).unwrap(); + let marker_before = fs::read(&marker).unwrap(); + assert!(!workdir.join("src/lib.rs").exists()); + + let result = { + let _failure = SparseSelectionFailure::enable(); + db.sync_lane_workdir_with_paths( + "sparse-rollback-ledger", + false, + &["src/lib.rs".to_string()], + ) + }; + + assert!( + result.is_err(), + "injected sparse-selection failure unexpectedly committed" + ); + assert!(!workdir.join("src/lib.rs").exists()); + assert_eq!(fs::read(&selection).unwrap(), selection_before); + assert_eq!(fs::read(&marker).unwrap(), marker_before); + assert_eq!( + db.lane_status("sparse-rollback-ledger") + .unwrap() + .workdir_state, + Some(WorktreeState::Clean) + ); +} + +#[test] +fn legacy_future_and_mismatched_lane_markers_reconcile_before_clean() { + let _authority = AuthorityGuard::enabled(); + let (_temp, db, workdir) = init_lane_fixture("marker-reconcile", false); + let marker_path = workdir.join(".trail/workdir-manifest.json"); + let clean_head = db + .lane_details("marker-reconcile") + .unwrap() + .branch + .head_root; + + let invalid_markers = [ + serde_json::json!({"version": 1, "root_id": clean_head.0, "files": {}}), + serde_json::json!({"version": 99}), + { + let mut marker = read_marker(&workdir); + marker["root_id"] = serde_json::Value::String("sha256:wrong-root".into()); + marker + }, + { + let mut marker = read_marker(&workdir); + marker["filesystem_identity"] = serde_json::json!([0, 1, 2]); + marker + }, + ]; + + for invalid in invalid_markers { + fs::write(&marker_path, serde_json::to_vec(&invalid).unwrap()).unwrap(); + let status = db.lane_status("marker-reconcile").unwrap(); + assert_eq!(status.workdir_state, Some(WorktreeState::Clean)); + let repaired = read_marker(&workdir); + assert_eq!(repaired["version"], 2); + assert_eq!(repaired["root_id"], clean_head.0); + assert!(repaired.get("files").is_none()); + } +} + +#[test] +fn observer_owner_loss_reconciles_lane_and_preserves_dirty_evidence() { + let _authority = AuthorityGuard::enabled(); + let (temp, db, workdir) = init_lane_fixture("owner-loss", false); + assert_eq!( + db.lane_status("owner-loss").unwrap().workdir_state, + Some(WorktreeState::Clean) + ); + let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); + let lane_id = db.lane_details("owner-loss").unwrap().branch.lane_id; + let scope_id: String = conn + .query_row( + "SELECT scope_id FROM changed_path_scopes + WHERE scope_kind='materialized_lane' AND owner_id=?1 AND retired_at IS NULL", + params![lane_id], + |row| row.get(0), + ) + .unwrap(); + conn.execute( + "DELETE FROM changed_path_observer_owners WHERE scope_id=?1", + params![scope_id], + ) + .unwrap(); + fs::write(workdir.join("README.md"), "external after owner loss\n").unwrap(); + + let status = db.lane_status("owner-loss").unwrap(); + assert_eq!(status.workdir_state, Some(WorktreeState::DirtyTracked)); + assert_eq!(status.workdir_changed_paths.len(), 1); + assert_eq!(status.workdir_changed_paths[0].path, "README.md"); +} + +#[test] +fn public_lane_preview_record_and_status_share_one_authoritative_lane_scope() { + let _authority = AuthorityGuard::enabled(); + let (_temp, mut db, workdir) = init_lane_fixture("public-record", false); + fs::write(workdir.join("README.md"), "recorded through lane ledger\n").unwrap(); + + let preview = db.preview_lane_workdir_record("public-record").unwrap(); + assert!(!preview.clean); + assert_eq!(preview.changed_paths.len(), 1); + assert_eq!(preview.changed_paths[0].path, "README.md"); + + let recorded = db + .record_lane_workdir("public-record", Some("authoritative lane record".into())) + .unwrap(); + assert!(recorded.operation.is_some()); + assert_eq!(recorded.changed_paths.len(), 1); + assert_eq!(recorded.changed_paths[0].path, "README.md"); + let status = db.lane_status("public-record").unwrap(); + assert_eq!(status.workdir_state, Some(WorktreeState::Clean)); + assert!(status.workdir_changed_paths.is_empty()); + assert_eq!(read_marker(&workdir)["root_id"], recorded.root_id.0); +} + +#[test] +fn observed_lane_record_builds_from_fenced_bytes_and_retains_same_path_after_c2() { + let _authority = AuthorityGuard::enabled(); + let (_temp, mut db, workdir) = init_lane_fixture("record-fenced-bytes", false); + let captured = b"captured before c2\n"; + let after_c2 = b"written after c2\n"; + fs::write(workdir.join("README.md"), captured).unwrap(); + trail::test_support::install_lane_record_after_c2_write_for_current_thread( + workdir.join("README.md"), + after_c2.to_vec(), + ); + + let recorded = db + .record_lane_workdir("record-fenced-bytes", Some("fenced bytes".into())) + .unwrap(); + let recorded_file = db + .inspect_root(&recorded.root_id.0) + .unwrap() + .files + .into_iter() + .find(|file| file.path == "README.md") + .unwrap(); + assert_eq!( + recorded_file.content_hash, + hex::encode(Sha256::digest(captured)), + "lane record reread the same path after c2 instead of using fenced bytes" + ); + assert_eq!(fs::read(workdir.join("README.md")).unwrap(), after_c2); + + let status = db.lane_status("record-fenced-bytes").unwrap(); + assert_eq!(status.workdir_state, Some(WorktreeState::DirtyTracked)); + assert_eq!(status.workdir_changed_paths.len(), 1); + assert_eq!(status.workdir_changed_paths[0].path, "README.md"); +} + +#[test] +fn observed_lane_record_postcommit_failures_require_committed_repair() { + let _authority = AuthorityGuard::enabled(); + for boundary in ["manifest", "event", "turn", "marker"] { + let (temp, mut db, workdir) = + init_lane_fixture(&format!("record-postcommit-{boundary}"), false); + let before = db + .lane_details(&format!("record-postcommit-{boundary}")) + .unwrap() + .branch; + fs::write( + workdir.join("README.md"), + format!("changed before {boundary}\n"), + ) + .unwrap(); + trail::test_support::set_lane_record_postcommit_failure_for_current_thread(Some(boundary)); + let error = db + .record_lane_workdir( + &format!("record-postcommit-{boundary}"), + Some(format!("postcommit {boundary}")), + ) + .unwrap_err(); + trail::test_support::set_lane_record_postcommit_failure_for_current_thread(None); + + assert_eq!(error.code(), "COMMITTED_REPAIR_REQUIRED"); + let Error::CommittedRepairRequired { + operation, + repair, + reason, + } = error + else { + unreachable!() + }; + assert!(!operation.is_empty()); + assert!(repair.contains(boundary)); + assert!(reason.contains(boundary)); + + let after = db + .lane_details(&format!("record-postcommit-{boundary}")) + .unwrap() + .branch; + assert_ne!(after.head_change, before.head_change); + assert_ne!(after.head_root, before.head_root); + let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); + let ledger_root: String = conn + .query_row( + "SELECT baseline_root_id FROM changed_path_scopes + WHERE scope_kind='materialized_lane' AND retired_at IS NULL", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(ledger_root, after.head_root.0); + } +} + +#[test] +fn patch_and_rewind_publish_ref_lane_and_compact_marker_at_one_visible_boundary() { + let _authority = AuthorityGuard::enabled(); + let (_temp, mut db, workdir) = init_lane_fixture("projection-boundary", false); + let base = db + .lane_details("projection-boundary") + .unwrap() + .branch + .head_change; + let mut patch: PatchDocument = serde_json::from_value(serde_json::json!({ + "message": "projection boundary", + "edits": [{"op": "write", "path": "README.md", "content": "patched\n"}] + })) + .unwrap(); + patch.base_change = Some(base.0.clone()); + let applied = db.apply_lane_patch("projection-boundary", patch).unwrap(); + let after_patch = db.lane_details("projection-boundary").unwrap(); + assert_eq!(after_patch.branch.head_change, applied.operation); + assert_eq!(after_patch.branch.head_root, applied.root_id); + assert_eq!(read_marker(&workdir)["root_id"], applied.root_id.0); + assert_eq!( + fs::read_to_string(workdir.join("README.md")).unwrap(), + "patched\n" + ); + + let rewound = db + .rewind_lane("projection-boundary", &base.0, false, true) + .unwrap(); + let after_rewind = db.lane_details("projection-boundary").unwrap(); + assert_eq!(after_rewind.branch.head_change, rewound.operation); + assert_eq!(after_rewind.branch.head_root, rewound.root_id); + assert_eq!(read_marker(&workdir)["root_id"], rewound.root_id.0); + assert_eq!( + fs::read_to_string(workdir.join("README.md")).unwrap(), + "base\n" + ); +} + +#[test] +fn mutable_materialized_lane_authority_pins_the_lane_root_and_reconciles_markers() { + let _authority = AuthorityGuard::enabled(); + trail::test_support::changed_path_materialized_lane_snapshot_flow().unwrap(); +} + +#[test] +fn producer_publication_retains_later_same_path_events_and_matches_full_scan_oracle() { + let _authority = AuthorityGuard::enabled(); + trail::test_support::changed_path_intent_acknowledgement_race().unwrap(); + trail::test_support::changed_path_reconciliation_races().unwrap(); + trail::test_support::changed_path_reconciliation_oracle().unwrap(); +} diff --git a/trail/tests/changed_path_ledger_reconcile.rs b/trail/tests/changed_path_ledger_reconcile.rs new file mode 100644 index 0000000..e942bee --- /dev/null +++ b/trail/tests/changed_path_ledger_reconcile.rs @@ -0,0 +1,55 @@ +use rusqlite::{params, Connection}; +use trail::{InitImportMode, Trail}; + +#[test] +fn reconciliation_staging_has_separate_binary_guard_namespace() { + let workspace = tempfile::tempdir().unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::Empty, false).unwrap(); + let conn = Connection::open(workspace.path().join(".trail/index/trail.sqlite")).unwrap(); + + let columns = conn + .prepare("PRAGMA table_info(changed_path_reconciliation_guards)") + .unwrap() + .query_map([], |row| { + Ok((row.get::<_, String>(1)?, row.get::<_, String>(2)?)) + }) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(columns.contains(&("relative_path".into(), "BLOB".into()))); + assert!(columns.contains(&("directory_identity".into(), "BLOB".into()))); + + conn.pragma_update(None, "foreign_keys", false).unwrap(); + let arbitrary_path = b"\xff\0#directory-guard/61"; + conn.execute( + "INSERT INTO changed_path_reconciliation_guards( + attempt_id,relative_path,directory_identity,staged_at + ) VALUES('independent-harness',?1,?2,0)", + params![arbitrary_path.as_slice(), b"identity".as_slice()], + ) + .unwrap(); + let stored: Vec = conn + .query_row( + "SELECT relative_path FROM changed_path_reconciliation_guards + WHERE attempt_id='independent-harness'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(stored, arbitrary_path); +} + +#[test] +fn compiled_harness_exercises_reconciliation_oracle() { + trail::test_support::changed_path_reconciliation_oracle().unwrap(); +} + +#[test] +fn compiled_harness_exercises_fail_closed_publication_races() { + trail::test_support::changed_path_reconciliation_races().unwrap(); +} + +#[test] +fn compiled_harness_exercises_callback_spooling() { + trail::test_support::changed_path_reconciliation_callback_spool().unwrap(); +} diff --git a/trail/tests/changed_path_ledger_recovery.rs b/trail/tests/changed_path_ledger_recovery.rs new file mode 100644 index 0000000..58afbef --- /dev/null +++ b/trail/tests/changed_path_ledger_recovery.rs @@ -0,0 +1,159 @@ +#[test] +fn concurrent_same_path_event_survives_intent_acknowledgement() { + trail::test_support::changed_path_intent_acknowledgement_race().unwrap(); +} + +#[test] +fn prepared_target_is_a_gc_root_until_terminal_recovery() { + trail::test_support::changed_path_intent_gc_root_lifecycle().unwrap(); +} + +#[test] +fn deterministic_crash_boundaries_recover_without_false_clean_results() { + trail::test_support::changed_path_intent_crash_matrix().unwrap(); +} + +#[test] +fn backup_and_restore_never_transfer_trusted_filesystem_identity() { + trail::test_support::changed_path_backup_restore_rotation().unwrap(); +} + +#[test] +fn unqualified_or_stale_filesystem_proof_cannot_publish_a_baseline() { + trail::test_support::changed_path_qualified_proof_revalidation().unwrap(); +} + +#[test] +fn ambiguous_recovery_requires_reconciliation_before_another_intent() { + trail::test_support::changed_path_ambiguous_recovery_gate().unwrap(); +} + +#[test] +fn failed_backup_overwrite_retains_the_previous_valid_tree() { + trail::test_support::changed_path_backup_overwrite_rollback().unwrap(); +} + +#[test] +fn retirement_validates_paths_and_waits_for_preexisting_readers() { + trail::test_support::changed_path_retirement_barrier().unwrap(); +} + +#[test] +fn lane_deletion_retires_changed_path_scope_before_filesystem_removal() { + trail::test_support::changed_path_lane_deletion_retirement().unwrap(); +} + +#[test] +fn metadata_only_intent_proof_without_a_real_sidecar_is_rejected() { + trail::test_support::changed_path_missing_sidecar_rejection().unwrap(); +} + +#[test] +fn authenticated_intent_cut_remains_a_prefix_after_later_observer_advance() { + trail::test_support::changed_path_advanced_prefix_recovery().unwrap(); +} + +#[test] +fn pre_and_post_events_cannot_bridge_an_empty_exact_path_interval() { + trail::test_support::changed_path_exact_interval_bridge_rejection().unwrap(); +} + +#[test] +fn pre_and_post_events_cannot_bridge_an_empty_complete_prefix_interval() { + trail::test_support::changed_path_prefix_interval_bridge_rejection().unwrap(); +} + +#[test] +fn authenticated_complete_prefix_interval_preserves_later_suffix() { + trail::test_support::changed_path_valid_prefix_interval_recovery().unwrap(); +} + +#[cfg(unix)] +#[test] +fn ancestor_directory_substitution_is_rejected_at_marking() { + trail::test_support::changed_path_mark_ancestor_substitution_rejection().unwrap(); +} + +#[cfg(unix)] +#[test] +fn ancestor_directory_substitution_is_rejected_at_recovery() { + trail::test_support::changed_path_recovery_ancestor_substitution_rejection().unwrap(); +} + +#[test] +fn retired_segment_deletion_keeps_retained_parent_authority() { + trail::test_support::changed_path_deletion_parent_substitution_rejection().unwrap(); +} + +#[cfg(unix)] +#[test] +fn retired_segment_deletion_rejects_leaf_substitution() { + trail::test_support::changed_path_deletion_leaf_substitution_rejection().unwrap(); +} + +#[test] +fn retired_segment_deletion_rejects_name_substitution_after_verification() { + trail::test_support::changed_path_deletion_post_verification_substitution_rejection().unwrap(); +} + +#[test] +fn retired_segment_deletion_rejects_substitution_after_quarantine_verification() { + trail::test_support::changed_path_deletion_post_quarantine_verification_substitution_rejection( + ) + .unwrap(); +} + +#[test] +fn retired_segment_deletion_retry_rejects_hostile_quarantine_replacement() { + trail::test_support::changed_path_deletion_retry_hostile_quarantine_replacement_rejection() + .unwrap(); +} + +#[test] +fn retired_segment_deletion_normal_retry_is_durably_idempotent() { + trail::test_support::changed_path_deletion_normal_retry_idempotence().unwrap(); +} + +#[test] +fn retirement_waits_for_retained_segment_writer_close_acknowledgement() { + trail::test_support::changed_path_retained_writer_quiescence().unwrap(); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn orphan_quarantine_substitution_after_verification_is_rejected() { + trail::test_support::changed_path_orphan_quarantine_substitution_rejection().unwrap(); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn empty_orphan_quarantine_is_retained_and_rejected() { + trail::test_support::changed_path_empty_orphan_quarantine_rejection().unwrap(); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn normal_retirement_allocates_fresh_quarantine_without_orphan() { + trail::test_support::changed_path_no_orphan_quarantine_allocation().unwrap(); +} + +#[test] +fn retired_segment_deletion_quiesced_retry_rejects_missing_quarantine() { + trail::test_support::changed_path_deletion_quiesced_missing_quarantine_rejection().unwrap(); +} + +#[test] +fn retired_segment_deletion_quiesced_retry_rejects_reappeared_original() { + trail::test_support::changed_path_deletion_quiesced_reappeared_original_rejection().unwrap(); +} + +#[test] +fn restored_nullable_provider_lane_and_view_scopes_can_be_deleted() { + trail::test_support::changed_path_restored_nullable_provider_lane_deletion().unwrap(); +} + +#[cfg(unix)] +#[test] +fn lossless_database_path_supports_mark_recovery_and_retirement() { + trail::test_support::changed_path_non_utf_database_path_mark_recover_and_retire().unwrap(); +} diff --git a/trail/tests/changed_path_ledger_views.rs b/trail/tests/changed_path_ledger_views.rs new file mode 100644 index 0000000..6ede358 --- /dev/null +++ b/trail/tests/changed_path_ledger_views.rs @@ -0,0 +1,5 @@ +#[cfg(unix)] +#[test] +fn qualified_view_intents_recover_without_false_clean_results() { + trail::test_support::changed_path_view_flow().unwrap(); +} diff --git a/trail/tests/e2e.rs b/trail/tests/e2e.rs index afbf956..78239a6 100644 --- a/trail/tests/e2e.rs +++ b/trail/tests/e2e.rs @@ -1,7 +1,11 @@ use std::collections::{BTreeMap, BTreeSet}; +#[cfg(target_os = "linux")] +use std::ffi::OsString; use std::fs; use std::io::{BufRead, Read, Write}; use std::net::{TcpListener, TcpStream}; +#[cfg(target_os = "linux")] +use std::os::unix::ffi::OsStringExt; #[cfg(unix)] use std::os::unix::fs::{MetadataExt, PermissionsExt}; use std::path::{Path, PathBuf}; @@ -13,8 +17,9 @@ use rusqlite::Connection; use trail::{ Actor, ConflictManualFile, ConflictManualResolution, Error, InitImportMode, LaneGateOptions, LaneMessageReport, LanePatchReport, LaneRewindReport, LaneTurnDetails, LaneTurnEndReport, - LaneTurnEventReport, LaneTurnStartReport, LaneWorkdirMode, OperationKind, PatchDocument, - ShowResult, TextContent, TextRepresentation, Trail, WorktreeState, + LaneTurnEventReport, LaneTurnStartReport, LaneWorkdirMode, MaterializationFallbackReason, + ObjectId, OperationKind, PatchDocument, ShowResult, TextContent, TextRepresentation, Trail, + WorkdirBackend, WorktreeRoot, WorktreeState, WORKTREE_ROOT_KIND, }; fn git_available() -> bool { @@ -58,6 +63,34 @@ fn git_output(cwd: &Path, args: &[&str]) -> String { String::from_utf8_lossy(&output.stdout).trim().to_string() } +fn git_output_raw(cwd: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .arg("-C") + .arg(cwd) + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "git {:?} failed\nstdout:\n{}\nstderr:\n{}", + args, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() +} + +fn git_object_count(cwd: &Path) -> u64 { + let output = git_output(cwd, &["count-objects", "-v"]); + let counts = output + .lines() + .filter_map(|line| line.split_once(' ')) + .collect::>(); + let loose = counts.get("count:").unwrap().parse::().unwrap(); + let packed = counts.get("in-pack:").unwrap().parse::().unwrap(); + loose + packed +} + fn trail_bin() -> PathBuf { std::env::var_os("CARGO_BIN_EXE_trail") .map(PathBuf::from) @@ -78,1183 +111,2773 @@ fn cli_reports_package_version() { ); } -#[cfg(unix)] -struct StubAcpAgentOptions<'a> { - session_id: &'a str, - lane_workdir: Option<&'a Path>, - assistant_text_before_tool: Option, - assistant_text: String, - write_text: Option<&'a str>, - crash_after_update: bool, - malformed_after_update: bool, - request_permission: bool, - sleep_before_result_ms: Option, -} +#[test] +fn agent_default_provider_is_a_typed_workspace_config_value() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); -#[cfg(unix)] -impl<'a> StubAcpAgentOptions<'a> { - fn new(session_id: &'a str) -> Self { - Self { - session_id, - lane_workdir: None, - assistant_text_before_tool: None, - assistant_text: "diagnostic complete".to_string(), - write_text: Some("diagnostic complete"), - crash_after_update: false, - malformed_after_update: false, - request_permission: false, - sleep_before_result_ms: None, - } - } -} + let set = run_trail_json( + temp.path(), + &["config", "set", "agent.default_provider", "codex"], + ); + assert_eq!(set["key"], "agent.default_provider"); + assert_eq!(set["new_value"], "codex"); -#[cfg(unix)] -fn shell_quote(value: &str) -> String { - format!("'{}'", value.replace('\'', "'\\''")) + let get = run_trail_json(temp.path(), &["config", "get", "agent.default_provider"]); + assert_eq!(get["value"], "codex"); } #[cfg(unix)] -fn write_stub_acp_agent( - workspace: &Path, - filename: &str, - options: StubAcpAgentOptions<'_>, -) -> PathBuf { - let agent = workspace.join(filename); - let write_dir = options - .lane_workdir - .map(|path| path.to_string_lossy().to_string()) - .unwrap_or_default(); - let assistant_before_tool = options - .assistant_text_before_tool - .as_ref() - .map(|text| { - let update = serde_json::json!({ - "jsonrpc": "2.0", - "method": "session/update", - "params": { - "sessionId": options.session_id, - "update": { - "sessionUpdate": "agent_message_chunk", - "messageId": "msg_before_tool", - "content": { - "type": "text", - "text": text - } - } - } - }); - format!("printf '%s\\n' {}\n", shell_quote(&update.to_string())) - }) - .unwrap_or_default(); - let permission_request = if options.request_permission { - format!( - "printf '%s\\n' '{{\"jsonrpc\":\"2.0\",\"id\":50,\"method\":\"session/request_permission\",\"params\":{{\"sessionId\":\"{}\",\"toolCall\":{{\"title\":\"approve diagnostic write\"}},\"options\":[{{\"optionId\":\"allow\",\"kind\":\"allow_once\",\"name\":\"Allow\"}}]}}}}'\n", - options.session_id - ) - } else { - String::new() - }; - let malformed = if options.malformed_after_update { - "printf '%s\\n' '{not-json'\nexit 0\n" - } else { - "" - }; - let crash = if options.crash_after_update { - "exit 42\n" - } else { - "" - }; - let sleep = options - .sleep_before_result_ms - .map(|ms| format!("sleep {}\n", (ms as f64) / 1000.0)) - .unwrap_or_default(); - let write_file = options - .write_text - .map(|text| { - format!( - "if [ -n \"$WRITE_DIR\" ]; then\n mkdir -p \"$WRITE_DIR\"\n printf '%s\\n' {} > \"$WRITE_DIR/README.md\"\nfi\n", - shell_quote(text) - ) - }) - .unwrap_or_default(); +#[test] +fn terminal_agent_uses_configured_provider_when_argument_is_absent() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + run_trail_json( + temp.path(), + &["config", "set", "agent.default_provider", "custom"], + ); - fs::write( - &agent, - format!( - r#"#!/bin/sh -set -eu -SESSION_ID={} -WRITE_DIR={} -IFS= read -r init -printf '%s\n' '{{"jsonrpc":"2.0","id":0,"result":{{"protocolVersion":1,"agentCapabilities":{{}}}}}}' -IFS= read -r session_new -if [ -z "$WRITE_DIR" ]; then - WRITE_DIR=$(printf '%s\n' "$session_new" | sed -n 's/.*"cwd":"\([^"]*\)".*/\1/p') -fi -printf '%s\n' '{{"jsonrpc":"2.0","id":1,"result":{{"sessionId":"{}"}}}}' -IFS= read -r prompt -printf '%s\n' '{{"jsonrpc":"2.0","method":"session/update","params":{{"sessionId":"{}","update":{{"sessionUpdate":"available_commands_update","commands":[{{"name":"write_file","description":"diagnostic command"}}]}}}}}}' -{} -printf '%s\n' '{{"jsonrpc":"2.0","method":"session/update","params":{{"sessionId":"{}","update":{{"sessionUpdate":"tool_call","toolCallId":"tool_stub","title":"write README","kind":"edit","status":"pending"}}}}}}' -{} -printf '%s\n' '{{"jsonrpc":"2.0","method":"session/update","params":{{"sessionId":"{}","update":{{"sessionUpdate":"tool_call_update","toolCallId":"tool_stub","status":"completed"}}}}}}' -printf '%s\n' '{{"jsonrpc":"2.0","method":"session/update","params":{{"sessionId":"{}","update":{{"sessionUpdate":"agent_message_chunk","messageId":"msg_stub","content":{{"type":"text","text":{}}}}}}}}}' -{}{}{}{} -printf '%s\n' '{{"jsonrpc":"2.0","id":2,"result":{{"stopReason":"end_turn"}}}}' -"#, - shell_quote(options.session_id), - shell_quote(&write_dir), - options.session_id, - options.session_id, - assistant_before_tool, - options.session_id, - permission_request, - options.session_id, - options.session_id, - serde_json::to_string(&options.assistant_text).unwrap(), - malformed, - crash, - sleep, - write_file - ), - ) - .unwrap(); - let mut permissions = fs::metadata(&agent).unwrap().permissions(); - permissions.set_mode(0o755); - fs::set_permissions(&agent, permissions).unwrap(); - agent + let report = run_trail_json(temp.path(), &["agent", "start", "--", "/usr/bin/true"]); + assert_eq!(report["provider"], "custom"); } -fn patch_with_lane_head(db: &Trail, lane: &str, mut patch: PatchDocument) -> PatchDocument { - if patch.base_change.is_none() { - patch.base_change = Some(db.lane_details(lane).unwrap().branch.head_change.0); - } - patch -} +#[test] +fn agent_acp_setup_plan_uses_the_hidden_run_command() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); -fn apply_lane_patch_at_head( - db: &mut Trail, - lane: &str, - patch: PatchDocument, -) -> Result { - let patch = patch_with_lane_head(db, lane, patch); - db.apply_lane_patch(lane, patch) + let report = run_trail_json( + temp.path(), + &[ + "agent", "acp", "setup", "codex", "--editor", "generic", "--print", + ], + ); + assert_eq!(report["transport"], "acp"); + assert_eq!(report["provider"], "codex"); + assert_eq!(report["editor"], "generic"); + assert_eq!(report["applied"], false); + assert_eq!( + report["command"], + serde_json::json!([ + trail_bin().canonicalize().unwrap(), + "--workspace", + temp.path().canonicalize().unwrap(), + "agent", + "acp", + "run", + "codex" + ]) + ); } -fn only_conflict_path_class(db: &Trail) -> (String, String) { - let conflicts = db.list_conflicts().unwrap(); - assert_eq!(conflicts.len(), 1); - let shown = db.show_conflict(&conflicts[0].conflict_set_id).unwrap(); - let explanation = shown.explanation.as_ref().unwrap(); - assert_eq!(explanation.paths.len(), 1); - ( - explanation.paths[0].path.clone(), - explanation.paths[0].conflict_class.clone(), - ) +#[test] +fn agent_acp_setup_falls_back_to_a_generic_entry_for_unknown_editors() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let report = run_trail_json( + temp.path(), + &[ + "agent", "acp", "setup", "codex", "--editor", "neovim", "--print", + ], + ); + assert_eq!(report["editor"], "neovim"); + assert_eq!(report["action"], "print"); + assert!(report["snippet"].as_str().unwrap().contains("ACP command:")); + assert!(report["warnings"] + .as_array() + .unwrap() + .iter() + .any(|warning| warning.as_str().unwrap().contains("generic entry"))); } -fn run_trail_json(workspace: &Path, args: &[&str]) -> serde_json::Value { +#[test] +fn agent_acp_setup_merges_the_owned_zed_entry() { + let workspace = tempfile::tempdir().unwrap(); + let home = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "hello\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let settings = test_zed_settings_path(home.path()); + fs::create_dir_all(settings.parent().unwrap()).unwrap(); + fs::write(&settings, "{\"theme\":\"One Dark\"}\n").unwrap(); + let output = Command::new(trail_bin()) .arg("--workspace") - .arg(workspace) + .arg(workspace.path()) .arg("--json") - .args(args) + .args(["agent", "acp", "setup", "codex", "--editor", "zed", "--yes"]) + .env("HOME", home.path()) + .env("XDG_CONFIG_HOME", home.path().join(".config")) .output() .unwrap(); assert!( output.status.success(), - "trail {:?} failed\nstdout:\n{}\nstderr:\n{}", - args, + "ACP setup failed\nstdout:\n{}\nstderr:\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); - serde_json::from_slice(&output.stdout).unwrap() -} + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(report["applied"], true); + assert_eq!(report["action"], "update"); + let configured: serde_json::Value = + serde_json::from_slice(&fs::read(&settings).unwrap()).unwrap(); + assert_eq!(configured["theme"], "One Dark"); + assert_eq!( + configured["agent_servers"]["trail-codex"]["args"] + .as_array() + .unwrap() + .last() + .unwrap(), + "codex" + ); -fn run_trail_json_daemon(workspace: &Path, daemon_url: &str, args: &[&str]) -> serde_json::Value { - let output = Command::new(trail_bin()) + let repeated = Command::new(trail_bin()) .arg("--workspace") - .arg(workspace) - .arg("--daemon-url") - .arg(daemon_url) + .arg(workspace.path()) .arg("--json") - .args(args) + .args(["agent", "acp", "setup", "codex", "--editor", "zed", "--yes"]) + .env("HOME", home.path()) + .env("XDG_CONFIG_HOME", home.path().join(".config")) .output() .unwrap(); - assert!( - output.status.success(), - "trail --daemon-url {:?} failed\nstdout:\n{}\nstderr:\n{}", - args, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - serde_json::from_slice(&output.stdout).unwrap() + assert!(repeated.status.success()); + let repeated: serde_json::Value = serde_json::from_slice(&repeated.stdout).unwrap(); + assert_eq!(repeated["action"], "noop"); + assert_eq!(repeated["applied"], true); } -fn wait_for_child_exit(child: &mut Child) { - for _ in 0..100 { - if child.try_wait().unwrap().is_some() { - return; - } - thread::sleep(Duration::from_millis(25)); - } - panic!("daemon did not exit"); -} - -fn wait_for_daemon_endpoint(path: &Path) -> serde_json::Value { - for _ in 0..100 { - if let Ok(bytes) = fs::read(path) { - if let Ok(value) = serde_json::from_slice(&bytes) { - return value; - } - } - thread::sleep(Duration::from_millis(25)); - } - panic!("daemon endpoint was not published at {}", path.display()); -} - -struct DaemonGuard { - child: Child, -} - -impl Drop for DaemonGuard { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} - -fn free_loopback_port() -> u16 { - TcpListener::bind(("127.0.0.1", 0)) - .unwrap() - .local_addr() - .unwrap() - .port() -} - -fn wait_for_daemon_health(port: u16) { - for _ in 0..100 { - if daemon_health_ok(port) { - return; - } - thread::sleep(Duration::from_millis(25)); +fn test_zed_settings_path(home: &Path) -> PathBuf { + #[cfg(target_os = "macos")] + { + home.join("Library/Application Support/Zed/settings.json") } - panic!("daemon did not become healthy on port {port}"); -} - -fn daemon_health_ok(port: u16) -> bool { - let Ok(mut stream) = TcpStream::connect(("127.0.0.1", port)) else { - return false; - }; - let _ = stream.set_read_timeout(Some(Duration::from_millis(500))); - if stream - .write_all(b"GET /v1/health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") - .is_err() + #[cfg(not(target_os = "macos"))] { - return false; + home.join(".config/zed/settings.json") } - let mut response = String::new(); - stream.read_to_string(&mut response).is_ok() && response.contains(" 200 ") } -fn raw_http_request(port: u16, request: &[u8]) -> String { - let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap(); - let _ = stream.set_read_timeout(Some(Duration::from_millis(1000))); - stream.write_all(request).unwrap(); - let mut response = String::new(); - stream.read_to_string(&mut response).unwrap(); - response -} +#[cfg(unix)] +#[test] +fn terminal_agent_start_aligns_process_context_with_the_lane_workdir() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); -fn api_request(method: &str, path: &str, body: serde_json::Value) -> Vec { - api_request_with_headers(method, path, &[], body) + let output = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .args([ + "agent", + "start", + "--provider", + "claude-code", + "--name", + "context", + "--workdir-mode", + "native-cow", + "--", + "/usr/bin/env", + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "agent start failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + let workdir = report["workdir"].as_str().unwrap(); + let workspace = temp.path().canonicalize().unwrap(); + let environment = String::from_utf8_lossy(&output.stderr); + assert!(environment + .lines() + .any(|line| line == format!("PWD={workdir}"))); + assert!(environment + .lines() + .any(|line| line == format!("TRAIL_WORKSPACE={}", workspace.display()))); + assert!(environment + .lines() + .any(|line| line.starts_with("TRAIL_LANE=lane_"))); + assert!(environment + .lines() + .any(|line| line.starts_with("TRAIL_SOURCE_ROOT=object_"))); + assert!(environment.lines().any(|line| { + line.strip_prefix("GIT_CEILING_DIRECTORIES=") + .is_some_and(|value| { + value + .split(':') + .any(|path| path == workspace.to_string_lossy()) + }) + })); } -fn conflicted_readme_workspace( - lane_content: &str, - human_content: &str, -) -> (tempfile::TempDir, Trail, String) { +#[cfg(unix)] +#[test] +fn terminal_agent_start_loads_project_hook_settings_in_the_isolated_provider() { let temp = tempfile::tempdir().unwrap(); - fs::write(temp.path().join("README.md"), "hello\nworld\n").unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let install = run_trail_json( + temp.path(), + &[ + "agent", + "hooks", + "setup", + "claude-code", + "--scope", + "project", + "--yes", + ], + ); + let settings = install["config_path"].as_str().unwrap().to_string(); - let mut db = Trail::open(temp.path()).unwrap(); - db.spawn_lane("manual-bot", Some("main"), false, None, None) - .unwrap(); - let patch: PatchDocument = serde_json::from_value(serde_json::json!({ - "message": "lane edits readme", - "edits": [ - {"op": "write", "path": "README.md", "content": lane_content} - ] - })) - .unwrap(); - apply_lane_patch_at_head(&mut db, "manual-bot", patch).unwrap(); - - fs::write(temp.path().join("README.md"), human_content).unwrap(); - db.record( - Some("main"), - Some("human edit".to_string()), - Actor::human(), - false, + let bin = tempfile::tempdir().unwrap(); + let fake_claude = bin.path().join("claude"); + fs::write( + &fake_claude, + "#!/bin/sh\nprintf '%s\\n' \"$@\" > CLAUDE_ARGS.txt\n", ) .unwrap(); - db.enqueue_merge("manual-bot", "main", 0).unwrap(); - let run = db.run_merge_queue(None).unwrap(); - assert!(run.stopped_on_conflict); - let conflict_id = db.list_conflicts().unwrap()[0].conflict_set_id.clone(); - (temp, db, conflict_id) -} + let mut permissions = fs::metadata(&fake_claude).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&fake_claude, permissions).unwrap(); + let path = std::env::join_paths(std::iter::once(bin.path().to_path_buf()).chain( + std::env::split_paths(&std::env::var_os("PATH").unwrap_or_default()), + )) + .unwrap(); -fn api_request_with_headers( - method: &str, - path: &str, - headers: &[(&str, &str)], - body: serde_json::Value, -) -> Vec { - let body = if body.is_null() { - Vec::new() - } else { - serde_json::to_vec(&body).unwrap() - }; - let mut head = format!("{method} {path} HTTP/1.1\r\nHost: localhost\r\n"); - for (name, value) in headers { - head.push_str(name); - head.push_str(": "); - head.push_str(value); - head.push_str("\r\n"); - } - head.push_str(&format!( - "Content-Type: application/json\r\nContent-Length: {}\r\n\r\n", - body.len() - )); - [head.into_bytes(), body].concat() + let output = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .env("PATH", path) + .args([ + "agent", + "start", + "--provider", + "claude-code", + "--name", + "hook-settings", + "--workdir-mode", + "auto", + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "agent start failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + let workdir = PathBuf::from(report["workdir"].as_str().unwrap()); + assert_eq!( + fs::read_to_string(workdir.join("CLAUDE_ARGS.txt")).unwrap(), + format!("--settings\n{settings}\n") + ); } +#[cfg(unix)] #[test] -fn cli_json_errors_are_machine_readable() { +fn terminal_agent_native_cow_does_not_discover_or_write_the_parent_git_checkout() { + if !git_available() { + return; + } let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "root baseline\n").unwrap(); + run_git(temp.path(), &["init", "-q"]); + run_git(temp.path(), &["config", "user.email", "trail@example.com"]); + run_git(temp.path(), &["config", "user.name", "Trail Test"]); + run_git(temp.path(), &["add", "README.md"]); + run_git(temp.path(), &["commit", "-qm", "baseline"]); + Trail::init(temp.path(), "main", InitImportMode::GitTracked, false).unwrap(); - let parse_output = Command::new(trail_bin()) - .arg("--json") - .arg("definitely-not-a-command") - .output() - .unwrap(); - assert!(!parse_output.status.success()); - assert_eq!(parse_output.status.code(), Some(2)); - assert!(parse_output.stdout.is_empty()); - let parse_stderr: serde_json::Value = serde_json::from_slice(&parse_output.stderr).unwrap(); - assert_eq!(parse_stderr["error"]["code"], "INVALID_INPUT"); - assert_eq!(parse_stderr["error"]["exit_code"], 2); - assert!(parse_stderr["error"]["message"] - .as_str() - .unwrap() - .contains("definitely-not-a-command")); + let provider = tempfile::NamedTempFile::new().unwrap(); + fs::write( + provider.path(), + "#!/bin/sh\nset -eu\nprintf 'lane change\\n' > LANE_ONLY.md\nif root=$(git rev-parse --show-toplevel 2>/dev/null); then\n printf 'escaped\\n' > \"$root/ESCAPED.md\"\nfi\n", + ) + .unwrap(); + let mut permissions = fs::metadata(provider.path()).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(provider.path(), permissions).unwrap(); let output = Command::new(trail_bin()) .arg("--workspace") .arg(temp.path()) .arg("--json") - .arg("status") + .args([ + "agent", + "start", + "--provider", + "claude-code", + "--name", + "containment", + "--workdir-mode", + "native-cow", + "--", + ]) + .arg(provider.path()) .output() .unwrap(); - assert!(!output.status.success()); - assert_eq!(output.status.code(), Some(3)); - assert!(output.stdout.is_empty()); - let stderr: serde_json::Value = serde_json::from_slice(&output.stderr).unwrap(); - assert_eq!(stderr["error"]["code"], "WORKSPACE_NOT_FOUND"); - assert_eq!(stderr["error"]["exit_code"], 3); - assert!(stderr["error"]["message"] - .as_str() + assert!( + output.status.success(), + "agent start failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + let workdir = PathBuf::from(report["workdir"].as_str().unwrap()); + assert_eq!( + fs::read_to_string(temp.path().join("README.md")).unwrap(), + "root baseline\n" + ); + assert!(!temp.path().join("ESCAPED.md").exists()); + assert_eq!( + fs::read_to_string(workdir.join("LANE_ONLY.md")).unwrap(), + "lane change\n" + ); + assert!(report["recorded"]["changed_paths"] + .as_array() .unwrap() - .contains("workspace not found")); + .iter() + .any(|path| path["path"] == "LANE_ONLY.md")); +} - let format_output = Command::new(trail_bin()) +#[cfg(target_os = "macos")] +#[test] +fn terminal_agent_native_cow_denies_explicit_writes_to_the_original_workspace() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "root baseline\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let provider = tempfile::NamedTempFile::new().unwrap(); + fs::write( + provider.path(), + "#!/bin/sh\nprintf 'lane change\\n' > LANE_ONLY.md\nif printf 'escaped\\n' > \"$1/ESCAPED.md\"; then\n printf 'workspace write was allowed\\n' > CONTAINMENT.txt\nelse\n printf 'workspace write was blocked\\n' > CONTAINMENT.txt\nfi\n", + ) + .unwrap(); + let mut permissions = fs::metadata(provider.path()).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(provider.path(), permissions).unwrap(); + + let output = Command::new(trail_bin()) .arg("--workspace") .arg(temp.path()) - .arg("--format") - .arg("json") - .arg("status") - .output() - .unwrap(); - assert_eq!(format_output.status.code(), Some(3)); - let format_stderr: serde_json::Value = serde_json::from_slice(&format_output.stderr).unwrap(); - assert_eq!(format_stderr["error"]["code"], "WORKSPACE_NOT_FOUND"); - - let env_parse_output = Command::new(trail_bin()) - .env("TRAIL_FORMAT", "json") - .arg("still-not-a-command") + .arg("--json") + .args([ + "agent", + "start", + "--provider", + "claude-code", + "--name", + "explicit-containment", + "--workdir-mode", + "native-cow", + "--", + ]) + .arg(provider.path()) + .arg(temp.path()) .output() .unwrap(); - assert!(!env_parse_output.status.success()); - assert_eq!(env_parse_output.status.code(), Some(2)); - let env_parse_stderr: serde_json::Value = - serde_json::from_slice(&env_parse_output.stderr).unwrap(); - assert_eq!(env_parse_stderr["error"]["code"], "INVALID_INPUT"); - assert!(env_parse_stderr["error"]["message"] - .as_str() - .unwrap() - .contains("still-not-a-command")); + assert!( + output.status.success(), + "agent start failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + let workdir = PathBuf::from(report["workdir"].as_str().unwrap()); + assert!(!temp.path().join("ESCAPED.md").exists()); + assert_eq!( + fs::read_to_string(workdir.join("LANE_ONLY.md")).unwrap(), + "lane change\n" + ); + assert_eq!( + fs::read_to_string(workdir.join("CONTAINMENT.txt")).unwrap(), + "workspace write was blocked\n" + ); } +#[cfg(unix)] #[test] -fn cli_env_defaults_select_workspace_db_branch_and_format() { +fn terminal_agent_native_hooks_enrich_the_existing_task_without_duplication() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); - let mut db = Trail::open(temp.path()).unwrap(); - db.create_branch("scratch", Some("main")).unwrap(); - drop(db); - - let workspace_output = Command::new(trail_bin()) - .env("TRAIL_WORKSPACE", temp.path()) - .env_remove("TRAIL_DIR") - .env("TRAIL_FORMAT", "json") - .env("TRAIL_BRANCH", "scratch") - .arg("status") - .output() - .unwrap(); - assert!( - workspace_output.status.success(), - "status with env workspace failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&workspace_output.stdout), - String::from_utf8_lossy(&workspace_output.stderr) - ); - let workspace_status: serde_json::Value = - serde_json::from_slice(&workspace_output.stdout).unwrap(); - assert_eq!(workspace_status["branch"], "scratch"); - let local_branch_status = run_trail_json(temp.path(), &["status", "--branch", "scratch"]); - assert_eq!(local_branch_status["branch"], "scratch"); + let provider = tempfile::NamedTempFile::new().unwrap(); + fs::write( + provider.path(), + "#!/bin/sh\nset -eu\nsend() {\n event=$1\n payload=$2\n printf '%s' \"$payload\" | \"$TRAIL_TEST_BIN\" --workspace \"$TRAIL_WORKSPACE\" agent hook receive claude-code \"$event\" >/dev/null\n}\ncwd=$(pwd)\nsend SessionStart \"{\\\"session_id\\\":\\\"native-terminal-1\\\",\\\"cwd\\\":\\\"$cwd\\\"}\"\nsend UserPromptSubmit \"{\\\"session_id\\\":\\\"native-terminal-1\\\",\\\"cwd\\\":\\\"$cwd\\\",\\\"prompt\\\":\\\"edit lane\\\"}\"\nprintf 'hooked change\\n' > HOOKED.md\nsend Stop \"{\\\"session_id\\\":\\\"native-terminal-1\\\",\\\"cwd\\\":\\\"$cwd\\\",\\\"last_assistant_message\\\":\\\"done\\\"}\"\nsend SessionEnd \"{\\\"session_id\\\":\\\"native-terminal-1\\\",\\\"cwd\\\":\\\"$cwd\\\"}\"\n", + ) + .unwrap(); + let mut permissions = fs::metadata(provider.path()).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(provider.path(), permissions).unwrap(); - let db_dir_output = Command::new(trail_bin()) - .env_remove("TRAIL_WORKSPACE") - .env_remove("TRAIL_BRANCH") - .env("TRAIL_DIR", temp.path().join(".trail")) - .env("TRAIL_FORMAT", "json") - .arg("status") + let output = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .env("TRAIL_TEST_BIN", trail_bin()) + .args([ + "agent", + "start", + "--provider", + "claude-code", + "--name", + "hooked-terminal", + "--workdir-mode", + "native-cow", + "--", + ]) + .arg(provider.path()) .output() .unwrap(); assert!( - db_dir_output.status.success(), - "status with env db dir failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&db_dir_output.stdout), - String::from_utf8_lossy(&db_dir_output.stderr) + output.status.success(), + "agent start failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) ); - let db_dir_status: serde_json::Value = serde_json::from_slice(&db_dir_output.stdout).unwrap(); - assert_eq!(db_dir_status["branch"], "main"); - - let invalid_format = Command::new(trail_bin()) - .env("TRAIL_WORKSPACE", temp.path()) - .env_remove("TRAIL_DIR") - .env_remove("TRAIL_BRANCH") - .env("TRAIL_FORMAT", "xml") - .arg("status") - .output() - .unwrap(); - assert!(!invalid_format.status.success()); - assert!(String::from_utf8_lossy(&invalid_format.stderr) - .contains("TRAIL_FORMAT must be human, json, or ndjson")); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + let task_id = report["task"]["task_id"].as_str().unwrap(); + let list = run_trail_json(temp.path(), &["agent", "list", "--all"]); + assert_eq!(list["tasks"].as_array().unwrap().len(), 1); + assert_eq!(list["tasks"][0]["task_id"], task_id); + assert_eq!(list["tasks"][0]["turns"], 1); + assert!(list["tasks"][0]["changed_paths"] + .as_array() + .unwrap() + .iter() + .any(|path| path["path"] == "HOOKED.md")); + let view = run_trail_json(temp.path(), &["agent", "view", task_id]); + assert_eq!( + view["review"]["recent_sessions"].as_array().unwrap().len(), + 1 + ); + assert_eq!(view["transcript"]["turns"].as_array().unwrap().len(), 1); } #[test] -fn agent_help_is_curated_and_hidden_commands_still_work() { - let help = Command::new(trail_bin()) - .args(["agent", "--help"]) - .output() +fn native_hook_cli_journals_codex_stop_and_returns_required_success_json() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let payload = serde_json::json!({ + "session_id": "codex-session-1", + "turn_id": "codex-turn-1", + "hook_event_name": "Stop", + "last_assistant_message": "done" + }) + .to_string(); + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["agent", "hook", "receive", "codex", "Stop"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() .unwrap(); - assert!( - help.status.success(), - "agent --help failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&help.stdout), - String::from_utf8_lossy(&help.stderr) - ); - let stdout = String::from_utf8_lossy(&help.stdout); - assert!(stdout.contains("Daily path:")); - assert!(stdout.contains("trail agent ask what should I do")); - assert!(stdout.contains(" action")); - assert!(stdout.contains(" changes")); - assert!(!stdout.contains("review-data")); - assert!(!stdout.contains("turn-diff")); - - let hidden_help = Command::new(trail_bin()) - .args(["agent", "review-data", "--help"]) - .output() + child + .stdin + .take() + .unwrap() + .write_all(payload.as_bytes()) .unwrap(); + let output = child.wait_with_output().unwrap(); assert!( - hidden_help.status.success(), - "agent review-data --help failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&hidden_help.stdout), - String::from_utf8_lossy(&hidden_help.stderr) + output.status.success(), + "hook ingress failed: {}", + String::from_utf8_lossy(&output.stderr) ); - assert!(String::from_utf8_lossy(&hidden_help.stdout).contains("review-data")); -} - -#[test] -fn init_record_why_and_fsck_work() { - let temp = tempfile::tempdir().unwrap(); - fs::write(temp.path().join("README.md"), "hello\nworld\n").unwrap(); - - let init = Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); - assert_eq!(init.imported.files, 1); - - fs::write(temp.path().join("README.md"), "hello\nTrail\n").unwrap(); - let mut db = Trail::open(temp.path()).unwrap(); - let record = db - .record( - Some("main"), - Some("edit readme".to_string()), - Actor::human(), - false, - ) + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + "{\"continue\":true}" + ); + let db = Trail::open(temp.path()).unwrap(); + let receipts = db + .list_agent_hook_receipts(Some("codex"), Some("processed"), 10) .unwrap(); - assert!(record.operation.is_some()); - assert_eq!(record.changed_paths.len(), 1); - - let why = db.why("README.md:2", Some("main")).unwrap(); - assert_eq!(why.current_text, "Trail"); - assert_eq!(why.history.len(), 1); - - let fsck = db.fsck().unwrap(); - assert!(fsck.errors.is_empty(), "{:?}", fsck.errors); + assert_eq!(receipts.len(), 1); + assert_eq!( + receipts[0].native_session_id.as_deref(), + Some("codex-session-1") + ); + assert_eq!(receipts[0].native_turn_id.as_deref(), Some("codex-turn-1")); } #[test] -fn doctor_reports_operational_health_across_cli_api_and_mcp() { +fn agent_hook_http_openapi_and_mcp_surfaces_share_durable_receipts() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let payload = serde_json::json!({ + "session_id": "codex-http-session", + "turn_id": "codex-http-turn", + "hook_event_name": "Stop" + }); - { - let db = Trail::open(temp.path()).unwrap(); - let clean = db.doctor().unwrap(); - assert_eq!(clean.status, "ok"); - assert!(clean - .checks - .iter() - .any(|check| check.name == "fsck" && check.status == "ok")); - assert!(clean.checks.iter().any(|check| { - check.name == "schema_version" - && check.status == "ok" - && check.details.as_ref().unwrap()["sqlite_user_version"] == 4 - })); - } + let unauthenticated = trail::server::handle_http_request( + &mut db, + &api_request("POST", "/v1/agent-hooks/codex/Stop", payload.clone()), + ); + assert_eq!(unauthenticated.status, 401); - let cli = run_trail_json(temp.path(), &["doctor"]); - assert_eq!(cli["status"], "ok"); - assert!(cli["checks"] - .as_array() - .unwrap() - .iter() - .any(|check| check["name"] == "current_branch" && check["status"] == "ok")); + let auth = trail::server::ServerAuth::bearer("agent-hook-test-token").unwrap(); + let ingested = trail::server::handle_http_request_with_auth( + &mut db, + &api_request_with_headers( + "POST", + "/v1/agent-hooks/codex/Stop", + &[("Authorization", "Bearer agent-hook-test-token")], + payload, + ), + &auth, + ); + assert_eq!(ingested.status, 200); + let ingested: serde_json::Value = ingested.body_json().unwrap(); + assert_eq!(ingested["continue"], true); - let mut db = Trail::open(temp.path()).unwrap(); - let api = trail::server::handle_http_request( + let receipts = trail::server::handle_http_request_with_auth( &mut db, - &api_request("GET", "/v1/doctor", serde_json::Value::Null), + &api_request_with_headers( + "GET", + "/v1/agent-hooks/receipts?provider=codex", + &[("Authorization", "Bearer agent-hook-test-token")], + serde_json::Value::Null, + ), + &auth, ); - assert_eq!(api.status, 200); - let api: serde_json::Value = api.body_json().unwrap(); - assert_eq!(api["status"], "ok"); + assert_eq!(receipts.status, 200); + let receipts: serde_json::Value = receipts.body_json().unwrap(); + assert_eq!(receipts.as_array().unwrap().len(), 1); + let receipt_id = receipts[0]["receipt_id"].as_str().unwrap().to_string(); + let cli_receipts = run_trail_json( + temp.path(), + &["agent", "hooks", "events", "codex", "--last", "10"], + ); + assert_eq!(cli_receipts[0]["receipt_id"], receipt_id); + let cli_receipts_after_first = run_trail_json( + temp.path(), + &[ + "agent", "hooks", "events", "codex", "--offset", "1", "--last", "1", + ], + ); + assert_eq!(cli_receipts_after_first, serde_json::json!([])); - let tools = trail::mcp::handle_json_rpc( + let receipts_after_first = trail::server::handle_http_request_with_auth( + &mut db, + &api_request_with_headers( + "GET", + "/v1/agent-hooks/receipts?provider=codex&offset=1&limit=1", + &[("Authorization", "Bearer agent-hook-test-token")], + serde_json::Value::Null, + ), + &auth, + ); + assert_eq!(receipts_after_first.status, 200); + let receipts_after_first: serde_json::Value = receipts_after_first.body_json().unwrap(); + assert_eq!(receipts_after_first, serde_json::json!([])); + + let capabilities = trail::mcp::handle_json_rpc( &mut db, serde_json::json!({ "jsonrpc": "2.0", "id": 1, - "method": "tools/list", - "params": {} + "method": "tools/call", + "params": { + "name": "trail.agent_integrations", + "arguments": {"provider": "codex"} + } }), ) .unwrap(); - let tool_list = tools["result"]["tools"].as_array().unwrap(); - assert!(tool_list.iter().any(|tool| tool["name"] == "trail.doctor")); + assert_eq!(capabilities["result"]["isError"], false); + assert_eq!( + capabilities["result"]["structuredContent"]["provider"], + "codex" + ); - let mcp = trail::mcp::handle_json_rpc( + let mcp_receipts = trail::mcp::handle_json_rpc( &mut db, serde_json::json!({ "jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": { - "name": "trail.doctor", - "arguments": {} + "name": "trail.agent_hook_receipts", + "arguments": {"provider": "codex"} } }), ) .unwrap(); - assert_eq!(mcp["result"]["isError"], false); - assert_eq!(mcp["result"]["structuredContent"]["status"], "ok"); - - db.spawn_lane("doctor-bot", Some("main"), false, None, None) - .unwrap(); - db.request_lane_approval( - "doctor-bot", - "shell.exec", - "Run release smoke tests", - None, - None, - None, + assert_eq!(mcp_receipts["result"]["isError"], false); + assert_eq!( + mcp_receipts["result"]["structuredContent"] + .as_array() + .unwrap() + .len(), + 1 + ); + assert_eq!( + mcp_receipts["result"]["structuredContent"][0]["receipt_id"], + receipt_id + ); + let mcp_receipts_after_first = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": { + "name": "trail.agent_hook_receipts", + "arguments": {"provider": "codex", "offset": 1, "limit": 1} + } + }), ) .unwrap(); - let warning = db.doctor().unwrap(); - assert_eq!(warning.status, "warning"); - let pending = warning - .checks - .iter() - .find(|check| check.name == "pending_approvals") - .unwrap(); - assert_eq!(pending.status, "warning"); - assert_eq!(pending.details.as_ref().unwrap()["count"], 1); -} - -#[test] -fn trail_refuses_workspaces_with_newer_schema_versions() { - let temp = tempfile::tempdir().unwrap(); - fs::write(temp.path().join("README.md"), "hello\n").unwrap(); - Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); - - let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); - conn.execute_batch("PRAGMA user_version = 999;").unwrap(); - drop(conn); - - let err = match Trail::open(temp.path()) { - Ok(_) => panic!("opening a future schema should fail"), - Err(err) => err, - }; - assert!(err - .to_string() - .contains("schema version 999 is newer than supported version")); -} + assert_eq!(mcp_receipts_after_first["result"]["isError"], false); + assert_eq!( + mcp_receipts_after_first["result"]["structuredContent"], + serde_json::json!([]) + ); -#[test] -fn init_creates_lane_observability_indexes() { - let temp = tempfile::tempdir().unwrap(); - fs::write(temp.path().join("README.md"), "hello\n").unwrap(); - Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let resource = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 3, + "method": "resources/read", + "params": {"uri": "trail://workspace/agent-hooks/receipts"} + }), + ) + .unwrap(); + assert!(resource["result"]["contents"][0]["text"] + .as_str() + .unwrap() + .contains("codex-http-session")); - let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); - let indexes = { - let mut stmt = conn - .prepare("SELECT name FROM sqlite_master WHERE type = 'index'") - .unwrap(); - stmt.query_map([], |row| row.get::<_, String>(0)) - .unwrap() - .collect::, _>>() - .unwrap() - }; - for expected in [ - "lane_turns_session_started_idx", - "lane_turns_lane_started_idx", - "lane_events_lane_created_idx", - "lane_events_session_created_idx", - "lane_events_turn_created_idx", - "lane_events_type_created_idx", - "lane_events_lane_type_created_idx", - "lane_events_session_type_created_idx", - "lane_events_turn_type_created_idx", - "lane_trace_span_events_span_created_idx", - "lane_trace_span_events_trace_created_idx", - "lane_acp_sessions_lane_idx", - "lane_acp_sessions_trail_session_idx", - "external_mutation_audit_created_idx", - "external_mutation_audit_surface_created_idx", - "external_mutation_audit_lane_created_idx", - "http_idempotency_keys_updated_idx", - "conflict_resolution_suggestions_signature_idx", - ] { - assert!( - indexes.iter().any(|index| index == expected), - "missing index {expected}" - ); - } + let spec = trail::server::openapi_spec(); + assert!(spec["paths"]["/v1/agent-hooks/{provider}/{event}"]["post"].is_object()); + assert!(spec["paths"]["/v1/agent-sessions/{id}/provenance"]["get"].is_object()); + assert_eq!( + spec["components"]["schemas"]["AgentCaptureRunRequest"]["additionalProperties"], + false + ); } #[test] -fn acp_setup_commands_report_profiles_install_and_doctor() { +fn native_hook_management_cli_installs_reports_and_removes_owned_config() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); - let profiles = run_trail_json(temp.path(), &["acp", "list"]); - assert!(profiles - .as_array() - .unwrap() - .iter() - .any(|profile| profile["agent"] == "claude-code")); - assert!(profiles - .as_array() - .unwrap() - .iter() - .any(|profile| profile["agent"] == "codex")); - assert!(profiles - .as_array() - .unwrap() - .iter() - .any(|profile| profile["agent"] == "cursor" - && profile["supports_acp"] == true - && profile["supports_terminal"] == true)); - assert!(!profiles - .as_array() - .unwrap() - .iter() - .any(|profile| profile["agent"] == "fake")); - - let install = run_trail_json( - temp.path(), - &[ - "acp", - "install", - "--agent", - "claude-code", - "--editor", - "generic", - "--dry-run", - ], - ); - assert_eq!(install["agent"], "claude-code"); - assert_eq!( - install["relay_command"], - serde_json::json!(["trail", "acp", "relay", "claude-code"]) - ); - assert!(install["snippet"] - .as_str() - .unwrap() - .contains("trail acp relay claude-code")); - - let zed_install = run_trail_json( - temp.path(), - &[ - "acp", - "install", - "--agent", - "claude-code", - "--editor", - "zed", - "--dry-run", - ], + let setup = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .args(["agent", "hooks", "setup", "codex", "--yes"]) + .output() + .unwrap(); + assert!( + setup.status.success(), + "hook install failed: {}", + String::from_utf8_lossy(&setup.stderr) ); - let zed_snippet: serde_json::Value = - serde_json::from_str(zed_install["snippet"].as_str().unwrap()).unwrap(); - let zed_agent = &zed_snippet["agent_servers"]["trail-claude-code"]; - assert_eq!(zed_agent["type"], "custom"); - assert_eq!(zed_agent["command"], "trail"); - assert!(zed_agent["args"] - .as_array() - .unwrap() - .iter() - .any(|part| part == "relay")); - - let doctor = run_trail_json(temp.path(), &["acp", "doctor", "--agent", "claude-code"]); - let checks = doctor["checks"].as_array().unwrap(); - assert!(checks - .iter() - .any(|check| check["name"] == "workspace" && check["status"] == "ok")); - assert!(checks - .iter() - .any(|check| check["name"] == "capture" && check["status"] == "skipped")); - assert!(doctor["lane"].is_null()); - assert!(doctor["session_id"].is_null()); + let setup: serde_json::Value = serde_json::from_slice(&setup.stdout).unwrap(); + assert_eq!(setup["provider"], "codex"); + let hooks_path = temp.path().join(".codex/hooks.json"); + let hooks = fs::read_to_string(&hooks_path).unwrap(); + assert!(hooks.contains(" agent hook receive codex Stop --installation hook_")); - let custom_doctor = run_trail_json( - temp.path(), - &[ - "acp", - "doctor", - "--agent", - "custom-acp", - "--relay-command", - "trail", - "acp", - "relay", - "--", - "", - ], - ); - assert_eq!(custom_doctor["provider"], "custom-acp"); - assert!(custom_doctor["checks"] - .as_array() - .unwrap() - .iter() - .any(|check| check["name"] == "relay" && check["status"] == "ok")); + let db = Trail::open(temp.path()).unwrap(); + let records = db.list_agent_hook_installations(Some("codex")).unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].status, "installed"); + drop(db); - let codex_install = run_trail_json( - temp.path(), - &[ - "acp", - "install", - "--agent", - "codex", - "--editor", - "zed", - "--dry-run", - ], - ); - assert_eq!(codex_install["agent"], "codex"); - assert_eq!( - codex_install["relay_command"], - serde_json::json!(["trail", "acp", "relay", "codex"]) - ); - let codex_zed_snippet: serde_json::Value = - serde_json::from_str(codex_install["snippet"].as_str().unwrap()).unwrap(); - assert_eq!( - codex_zed_snippet["agent_servers"]["trail-codex"]["command"], - "trail" - ); + let status = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .args(["agent", "hooks", "status", "codex"]) + .output() + .unwrap(); + assert!(status.status.success()); + let status: serde_json::Value = serde_json::from_slice(&status.stdout).unwrap(); + assert_eq!(status["installations"][0]["filesystem_status"], "installed"); - let cursor_install = run_trail_json( - temp.path(), - &[ - "acp", - "install", - "--agent", - "cursor", - "--editor", - "generic", - "--dry-run", - ], + let remove = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["agent", "hooks", "remove", "codex"]) + .output() + .unwrap(); + assert!( + remove.status.success(), + "hook removal failed: {}", + String::from_utf8_lossy(&remove.stderr) ); - assert_eq!(cursor_install["agent"], "cursor"); + let remaining: serde_json::Value = + serde_json::from_slice(&fs::read(&hooks_path).unwrap()).unwrap(); + assert!(remaining.get("hooks").is_none()); + let db = Trail::open(temp.path()).unwrap(); assert_eq!( - cursor_install["relay_command"], - serde_json::json!(["trail", "acp", "relay", "cursor"]) + db.list_agent_hook_installations(Some("codex")).unwrap()[0].status, + "removed" ); } -#[cfg(unix)] #[test] -fn acp_relay_accepts_a_built_in_agent_shortcut() { +fn native_hook_cli_spools_during_database_failure_and_replays_after_recovery() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let index = temp.path().join(".trail/index/trail.sqlite"); + let backup = temp.path().join(".trail/index/trail.sqlite.saved"); + fs::rename(&index, &backup).unwrap(); + fs::create_dir(&index).unwrap(); + + let payload = serde_json::json!({ + "session_id": "spooled-session-1", + "turn_id": "spooled-turn-1", + "hook_event_name": "Stop" + }) + .to_string(); + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["agent", "hook", "receive", "codex", "Stop"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(payload.as_bytes()) + .unwrap(); + let output = child.wait_with_output().unwrap(); + assert!(output.status.success()); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + "{\"continue\":true}" + ); + let spool = temp.path().join(".trail/runtime/agent-hooks-spool"); + assert_eq!(fs::read_dir(&spool).unwrap().count(), 1); - let output = Command::new(trail_bin()) + fs::remove_dir(&index).unwrap(); + fs::rename(&backup, &index).unwrap(); + let replay = Command::new(trail_bin()) .arg("--workspace") .arg(temp.path()) - .args(["acp", "relay", "codex", "--", "/usr/bin/true"]) + .arg("--json") + .args(["agent", "hooks", "replay", "--pending"]) .output() .unwrap(); assert!( - output.status.success(), - "trail acp relay codex failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); + replay.status.success(), + "spool replay failed: {}", + String::from_utf8_lossy(&replay.stderr) + ); + let replay: serde_json::Value = serde_json::from_slice(&replay.stdout).unwrap(); + assert_eq!(replay["spool"]["imported"], 1); + assert_eq!(replay["replayed"].as_array().unwrap().len(), 1); + assert_eq!(fs::read_dir(&spool).unwrap().count(), 0); } -#[cfg(unix)] #[test] -fn agent_setup_and_stub_acp_use_fresh_lanes() { +fn agent_capture_and_portable_evidence_cli_surface_round_trips() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); - let default_setup = run_trail_json(temp.path(), &["agent", "setup"]); - assert_eq!(default_setup["provider"], "claude-code"); - assert_eq!(default_setup["editor"], "vscode"); - assert_eq!(default_setup["mode"], "acp"); - assert_eq!(default_setup["supports_acp"], true); - assert_eq!(default_setup["supports_terminal"], true); - assert!(default_setup["snippet"] - .as_str() - .unwrap() - .contains("Trail Claude Code")); - assert!(default_setup["suggestions"] - .as_array() - .unwrap() - .iter() - .any(|suggestion| suggestion["command"] == "trail agent")); - assert!(default_setup["suggestions"] - .as_array() - .unwrap() - .iter() - .any(|suggestion| suggestion["command"] - .as_str() - .unwrap() - .contains("agent doctor"))); - - let generic_setup = run_trail_json(temp.path(), &["agent", "setup", "--editor", "generic"]); - assert_eq!(generic_setup["editor"], "generic"); - assert!(generic_setup["snippet"] - .as_str() - .unwrap() - .contains("ACP command:")); - assert!(generic_setup["suggestions"] - .as_array() - .unwrap() - .iter() - .any(|suggestion| suggestion["command"] == "trail agent action")); - - let codex_setup = run_trail_json( - temp.path(), - &[ + let begin = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .args([ "agent", - "setup", - "--provider", + "capture", + "begin", + "--owner", "codex", - "--editor", - "vscode", - ], - ); - assert_eq!(codex_setup["provider"], "codex"); - assert!(codex_setup["snippet"] - .as_str() - .unwrap() - .contains("Trail Codex")); - - let cursor_setup = run_trail_json( - temp.path(), - &[ - "agent", - "setup", - "--provider", - "cursor", - "--editor", - "vscode", - ], + "--session", + "owner-1", + "--workdir", + ]) + .arg(temp.path()) + .output() + .unwrap(); + assert!( + begin.status.success(), + "{}", + String::from_utf8_lossy(&begin.stderr) ); - assert_eq!(cursor_setup["provider"], "cursor"); - assert_eq!(cursor_setup["mode"], "acp"); - assert!(cursor_setup["snippet"] - .as_str() - .unwrap() - .contains("Trail Cursor")); - - let gemini_setup = run_trail_json( - temp.path(), - &[ + let begin: serde_json::Value = serde_json::from_slice(&begin.stdout).unwrap(); + let run_id = begin["capture_run_id"].as_str().unwrap(); + let status = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .args(["agent", "capture", "status"]) + .output() + .unwrap(); + let status: serde_json::Value = serde_json::from_slice(&status.stdout).unwrap(); + assert_eq!(status[0]["capture_run_id"], run_id); + let end = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .args([ "agent", - "setup", - "--provider", - "gemini", - "--editor", - "generic", - ], + "capture", + "end", + run_id, + "--owner", + "codex", + "--session", + "owner-1", + ]) + .output() + .unwrap(); + assert!( + end.status.success(), + "{}", + String::from_utf8_lossy(&end.stderr) ); - assert_eq!(gemini_setup["provider"], "gemini"); - assert_eq!(gemini_setup["mode"], "terminal"); - assert_eq!(gemini_setup["supports_acp"], false); - assert_eq!(gemini_setup["supports_mcp"], true); - assert!(gemini_setup["command"] - .as_array() - .unwrap() - .iter() - .any(|part| part == "start")); - assert!(gemini_setup["snippet"] - .as_str() - .unwrap() - .contains("Terminal command:")); - assert!(gemini_setup["snippet"] - .as_str() - .unwrap() - .contains("Trail MCP server command:")); - let gemini_doctor = run_trail_json(temp.path(), &["agent", "doctor", "--provider", "gemini"]); - assert_eq!(gemini_doctor["provider"], "gemini"); - assert_eq!(gemini_doctor["capabilities"]["terminal"], true); - assert_eq!(gemini_doctor["capabilities"]["mcp"], true); - assert!(gemini_doctor["checks"] - .as_array() - .unwrap() - .iter() - .any(|check| check["name"] == "acp" && check["status"] == "skipped")); - let setup = run_trail_json( - temp.path(), - &[ - "agent", - "setup", - "--provider", - "claude-code", - "--editor", - "vscode", - ], - ); - assert_eq!(setup["provider"], "claude-code"); - let command = setup["command"].as_array().unwrap(); - assert!(command.iter().any(|part| part == "agent")); - assert!(command.iter().any(|part| part == "acp")); - assert!(!command.iter().any(|part| part == "--lane")); - let snippet: serde_json::Value = - serde_json::from_str(setup["snippet"].as_str().unwrap()).unwrap(); - let vscode_entry = &snippet["Trail Claude Code"]; - assert_eq!(vscode_entry["command"], "trail"); - assert!(vscode_entry["args"] - .as_array() + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("portable", None, false, Some("codex".to_string()), None) + .unwrap(); + let session = db + .start_lane_session("portable", Some("portable".to_string()), None) .unwrap() - .iter() - .any(|part| part == "agent")); - assert!(!vscode_entry["args"] - .as_array() + .session; + let turn = db + .begin_lane_session_turn("portable", &session.session_id, None) .unwrap() - .iter() - .any(|part| part == "--lane")); + .turn; + db.add_lane_turn_message(&turn.turn_id, "user", "portable trace") + .unwrap(); + db.end_lane_turn(&turn.turn_id, "completed").unwrap(); + db.create_turn_evidence_manifest(&turn.turn_id).unwrap(); + let attestation = db + .create_session_attestation(&session.session_id, "test", None) + .unwrap(); + drop(db); - { - let mut db = Trail::open(temp.path()).unwrap(); - db.spawn_lane("manual-low-level", Some("main"), false, None, None) - .unwrap(); - } - let empty_list = run_trail_json(temp.path(), &["agent", "list"]); - assert_eq!(empty_list["tasks"].as_array().unwrap().len(), 0); - let empty_status = run_trail_json(temp.path(), &["agent", "status"]); - assert_eq!(empty_status["status"], "empty"); - let empty_next = run_trail_json(temp.path(), &["agent", "next"]); - assert_eq!(empty_next["focus"], "setup"); - assert_eq!(empty_next["primary"]["command"], "trail agent setup"); - let empty_actions = run_trail_json(temp.path(), &["agent", "action"]); - assert_eq!(empty_actions["status"], "empty"); - assert!(empty_actions["task"].is_null()); - assert_eq!(empty_actions["next"]["command"], "trail agent setup"); - assert!(empty_actions["actions"] - .as_array() - .unwrap() - .iter() - .any(|action| action["id"] == "setup_vscode" && action["command"] == "trail agent setup")); - assert!(empty_actions["actions"] - .as_array() - .unwrap() - .iter() - .any(|action| action["id"] == "setup_codex_vscode" - && action["command"] == "trail agent setup --provider codex")); - assert!(empty_actions["actions"] - .as_array() - .unwrap() - .iter() - .any(|action| action["id"] == "doctor_codex" - && action["command"] == "trail agent doctor --provider codex")); - assert!(empty_actions["actions"] - .as_array() - .unwrap() - .iter() - .any(|action| action["id"] == "setup_cursor_vscode" - && action["command"] == "trail agent setup --provider cursor")); - assert!(empty_actions["actions"] - .as_array() - .unwrap() - .iter() - .any(|action| action["id"] == "start_terminal_task" - && action["requires_confirmation"] == true)); - assert!(empty_actions["actions"] - .as_array() - .unwrap() - .iter() - .any(|action| action["id"] == "start_gemini_task" - && action["command"] == "trail agent start --provider gemini")); - let empty_ask_actions = run_trail_json(temp.path(), &["agent", "ask", "show", "actions"]); - assert_eq!(empty_ask_actions["status"], "empty"); - assert!(empty_ask_actions["task"].is_null()); - assert!(empty_ask_actions["actions"] - .as_array() - .unwrap() - .iter() - .any(|action| action["id"] == "setup_vscode")); - let empty_setup_action = run_trail_json(temp.path(), &["agent", "action", "setup_vscode"]); - assert_eq!(empty_setup_action["provider"], "claude-code"); - assert_eq!(empty_setup_action["editor"], "vscode"); - assert!(empty_setup_action["command"] - .as_array() - .unwrap() - .iter() - .any(|part| part == "agent" || part == "acp")); - let empty_doctor_action = - run_trail_json(temp.path(), &["agent", "action", "doctor_claude_code"]); - assert_eq!(empty_doctor_action["provider"], "claude-code"); - assert!(empty_doctor_action["checks"] - .as_array() - .unwrap() - .iter() - .any(|check| check["name"] == "workspace" && check["status"] == "ok")); - let empty_start_print = run_trail_json( - temp.path(), - &["agent", "action", "start_terminal_task", "--print"], - ); - assert_eq!(empty_start_print["status"], "empty"); - assert!(empty_start_print["task"].is_null()); - assert_eq!(empty_start_print["action"]["id"], "start_terminal_task"); - assert!(empty_start_print["action"]["command"] - .as_str() - .unwrap() - .contains("agent start")); - let empty_start_guard = Command::new(trail_bin()) + let verify = Command::new(trail_bin()) .arg("--workspace") .arg(temp.path()) .arg("--json") - .args(["agent", "action", "start_terminal_task"]) + .args(["agent", "attest", "verify", &attestation.attestation_id]) .output() .unwrap(); - assert!(!empty_start_guard.status.success()); - let empty_start_stderr: serde_json::Value = - serde_json::from_slice(&empty_start_guard.stderr).unwrap(); - assert!(empty_start_stderr["error"]["message"] - .as_str() - .unwrap() - .contains("requires --confirm")); - for (command, requested) in [ - (vec!["agent", "view", "latest"], "view"), - (vec!["agent", "changes", "latest"], "changes"), - (vec!["agent", "apply", "latest", "--dry-run"], "apply"), - ] { - let empty_hint = run_trail_json(temp.path(), &command); - assert_eq!(empty_hint["status"], "empty"); - assert!(empty_hint["task"].is_null()); - assert_eq!(empty_hint["requested"], requested); - assert!(empty_hint["summary"] - .as_str() - .unwrap() - .contains("No agent task is recorded yet")); - assert!(!empty_hint["summary"] - .as_str() - .unwrap() - .contains("to changes")); - assert_eq!(empty_hint["next"]["command"], "trail agent setup"); - assert!(empty_hint["actions"] - .as_array() - .unwrap() - .iter() - .any(|action| action["id"] == "setup_vscode")); - } - let empty_dashboard = run_trail_json(temp.path(), &["agent", "dashboard"]); - assert_eq!(empty_dashboard["status"], "empty"); - assert!(empty_dashboard["task"].is_null()); - assert_eq!(empty_dashboard["next"]["command"], "trail agent setup"); - let empty_inbox = run_trail_json(temp.path(), &["agent", "inbox"]); - assert_eq!(empty_inbox["total"], 0); - assert_eq!(empty_inbox["attention_count"], 0); - assert_eq!(empty_inbox["items"].as_array().unwrap().len(), 0); - assert_eq!(empty_inbox["next"]["command"], "trail agent setup"); - let empty_bare_agent = run_trail_json(temp.path(), &["agent"]); - assert_eq!(empty_bare_agent["total"], 0); - assert_eq!(empty_bare_agent["next"]["command"], "trail agent setup"); + assert!(verify.status.success()); + let verify: serde_json::Value = serde_json::from_slice(&verify.stdout).unwrap(); + assert_eq!(verify["valid"], true); - let doctor = run_trail_json( - temp.path(), - &["agent", "doctor", "--provider", "claude-code"], + let export = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["agent", "export", &session.session_id]) + .output() + .unwrap(); + assert!( + export.status.success(), + "{}", + String::from_utf8_lossy(&export.stderr) ); - assert_eq!(doctor["provider"], "claude-code"); - assert!(doctor["checks"] - .as_array() - .unwrap() - .iter() - .any(|check| { check["name"] == "workspace" && check["status"] == "ok" })); - assert_eq!(doctor["suggestions"][0]["command"], "trail agent setup"); + let trace = trail::PortableAgentTrace::from_json(&export.stdout).unwrap(); + assert_eq!(trace.session_id, session.session_id); + assert!(trace.verify().valid); +} - let first_agent = write_stub_acp_agent( - temp.path(), - "agent-acp-stub-a.sh", - StubAcpAgentOptions::new("sess_agent_stub_a"), - ); - let second_agent = write_stub_acp_agent( - temp.path(), - "agent-acp-stub-b.sh", - StubAcpAgentOptions::new("sess_agent_stub_b"), - ); - run_agent_acp_stub_session(temp.path(), &first_agent); - let one_task_home = run_trail_json(temp.path(), &["agent"]); - assert!(one_task_home["task"]["name"] - .as_str() - .unwrap() - .starts_with("agent-claude-code-")); - assert_eq!(one_task_home["focus"]["path"], "README.md"); - run_agent_acp_stub_session(temp.path(), &second_agent); +#[cfg(unix)] +struct StubAcpAgentOptions<'a> { + session_id: &'a str, + lane_workdir: Option<&'a Path>, + assistant_text_before_tool: Option, + assistant_text: String, + write_text: Option<&'a str>, + crash_after_update: bool, + malformed_after_update: bool, + request_permission: bool, + sleep_before_result_ms: Option, +} + +#[cfg(unix)] +impl<'a> StubAcpAgentOptions<'a> { + fn new(session_id: &'a str) -> Self { + Self { + session_id, + lane_workdir: None, + assistant_text_before_tool: None, + assistant_text: "diagnostic complete".to_string(), + write_text: Some("diagnostic complete"), + crash_after_update: false, + malformed_after_update: false, + request_permission: false, + sleep_before_result_ms: None, + } + } +} + +#[cfg(unix)] +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\\''")) +} + +#[cfg(unix)] +fn write_stub_acp_agent( + workspace: &Path, + filename: &str, + options: StubAcpAgentOptions<'_>, +) -> PathBuf { + let agent = workspace.join(filename); + let write_dir = options + .lane_workdir + .map(|path| path.to_string_lossy().to_string()) + .unwrap_or_default(); + let assistant_before_tool = options + .assistant_text_before_tool + .as_ref() + .map(|text| { + let update = serde_json::json!({ + "jsonrpc": "2.0", + "method": "session/update", + "params": { + "sessionId": options.session_id, + "update": { + "sessionUpdate": "agent_message_chunk", + "messageId": "msg_before_tool", + "content": { + "type": "text", + "text": text + } + } + } + }); + format!("printf '%s\\n' {}\n", shell_quote(&update.to_string())) + }) + .unwrap_or_default(); + let permission_request = if options.request_permission { + format!( + "printf '%s\\n' '{{\"jsonrpc\":\"2.0\",\"id\":50,\"method\":\"session/request_permission\",\"params\":{{\"sessionId\":\"{}\",\"toolCall\":{{\"title\":\"approve diagnostic write\"}},\"options\":[{{\"optionId\":\"allow\",\"kind\":\"allow_once\",\"name\":\"Allow\"}}]}}}}'\n", + options.session_id + ) + } else { + String::new() + }; + let malformed = if options.malformed_after_update { + "printf '%s\\n' '{not-json'\nexit 0\n" + } else { + "" + }; + let crash = if options.crash_after_update { + "exit 42\n" + } else { + "" + }; + let sleep = options + .sleep_before_result_ms + .map(|ms| format!("sleep {}\n", (ms as f64) / 1000.0)) + .unwrap_or_default(); + let write_file = options + .write_text + .map(|text| { + format!( + "if [ -n \"$WRITE_DIR\" ]; then\n mkdir -p \"$WRITE_DIR\"\n printf '%s\\n' {} > \"$WRITE_DIR/README.md\"\nfi\n", + shell_quote(text) + ) + }) + .unwrap_or_default(); + + fs::write( + &agent, + format!( + r#"#!/bin/sh +set -eu +SESSION_ID={} +WRITE_DIR={} +IFS= read -r init +printf '%s\n' '{{"jsonrpc":"2.0","id":0,"result":{{"protocolVersion":1,"agentCapabilities":{{}}}}}}' +IFS= read -r session_new +if [ -z "$WRITE_DIR" ]; then + WRITE_DIR=$(printf '%s\n' "$session_new" | sed -n 's/.*"cwd":"\([^"]*\)".*/\1/p') +fi +printf '%s\n' '{{"jsonrpc":"2.0","id":1,"result":{{"sessionId":"{}"}}}}' +IFS= read -r prompt +printf '%s\n' '{{"jsonrpc":"2.0","method":"session/update","params":{{"sessionId":"{}","update":{{"sessionUpdate":"available_commands_update","commands":[{{"name":"write_file","description":"diagnostic command"}}]}}}}}}' +{} +printf '%s\n' '{{"jsonrpc":"2.0","method":"session/update","params":{{"sessionId":"{}","update":{{"sessionUpdate":"tool_call","toolCallId":"tool_stub","title":"write README","kind":"edit","status":"pending"}}}}}}' +{} +printf '%s\n' '{{"jsonrpc":"2.0","method":"session/update","params":{{"sessionId":"{}","update":{{"sessionUpdate":"tool_call_update","toolCallId":"tool_stub","status":"completed"}}}}}}' +printf '%s\n' '{{"jsonrpc":"2.0","method":"session/update","params":{{"sessionId":"{}","update":{{"sessionUpdate":"agent_message_chunk","messageId":"msg_stub","content":{{"type":"text","text":{}}}}}}}}}' +{}{}{}{} +printf '%s\n' '{{"jsonrpc":"2.0","id":2,"result":{{"stopReason":"end_turn"}}}}' +"#, + shell_quote(options.session_id), + shell_quote(&write_dir), + options.session_id, + options.session_id, + assistant_before_tool, + options.session_id, + permission_request, + options.session_id, + options.session_id, + serde_json::to_string(&options.assistant_text).unwrap(), + malformed, + crash, + sleep, + write_file + ), + ) + .unwrap(); + let mut permissions = fs::metadata(&agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&agent, permissions).unwrap(); + agent +} + +fn patch_with_lane_head(db: &Trail, lane: &str, mut patch: PatchDocument) -> PatchDocument { + if patch.base_change.is_none() { + patch.base_change = Some(db.lane_details(lane).unwrap().branch.head_change.0); + } + patch +} + +fn apply_lane_patch_at_head( + db: &mut Trail, + lane: &str, + patch: PatchDocument, +) -> Result { + let patch = patch_with_lane_head(db, lane, patch); + db.apply_lane_patch(lane, patch) +} + +fn ready_agent_lane_with_mode(mode: InitImportMode) -> (tempfile::TempDir, Trail) { + let temp = tempfile::tempdir().unwrap(); + run_git(temp.path(), &["init"]); + run_git(temp.path(), &["config", "user.email", "trail@example.test"]); + run_git(temp.path(), &["config", "user.name", "Trail Test"]); + fs::write(temp.path().join("README.md"), "base\n").unwrap(); + run_git(temp.path(), &["add", "README.md"]); + run_git(temp.path(), &["commit", "-m", "initial"]); + Trail::init(temp.path(), "main", mode, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("apply-bot", Some("main"), false, None, None) + .unwrap(); + let patch: PatchDocument = serde_json::from_value(serde_json::json!({ + "message": "one changed path", + "edits": [{"op": "write", "path": "AGENT.md", "content": "agent change\n"}] + })) + .unwrap(); + apply_lane_patch_at_head(&mut db, "apply-bot", patch).unwrap(); + db.agent_mark_reviewed("apply-bot", None).unwrap(); + (temp, db) +} + +fn ready_agent_lane_with_changed_paths(changed_path_count: usize) -> (tempfile::TempDir, Trail) { + let temp = tempfile::tempdir().unwrap(); + run_git(temp.path(), &["init"]); + run_git(temp.path(), &["config", "user.email", "trail@example.test"]); + run_git(temp.path(), &["config", "user.name", "Trail Test"]); + fs::write(temp.path().join("README.md"), "base\n").unwrap(); + run_git(temp.path(), &["add", "README.md"]); + run_git(temp.path(), &["commit", "-m", "initial"]); + Trail::init(temp.path(), "main", InitImportMode::GitTracked, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("apply-bot", Some("main"), false, None, None) + .unwrap(); + let edits = (0..changed_path_count) + .map(|index| { + serde_json::json!({ + "op": "write", + "path": format!("agent-{index:03}.md"), + "content": format!("agent change {index}\n"), + }) + }) + .collect::>(); + let patch: PatchDocument = serde_json::from_value(serde_json::json!({ + "message": "many changed paths", + "edits": edits, + })) + .unwrap(); + apply_lane_patch_at_head(&mut db, "apply-bot", patch).unwrap(); + db.agent_mark_reviewed("apply-bot", None).unwrap(); + (temp, db) +} + +#[test] +fn agent_apply_reports_one_tracked_status_query() { + if !git_available() { + return; + } + + let (_temp, mut db) = ready_agent_lane_with_mode(InitImportMode::GitTracked); + let dry_run = db.agent_apply("apply-bot", true, None).unwrap(); + assert_eq!(dry_run.performance.tracked_status_count, 1); + assert_eq!(dry_run.performance.full_root_file_count, 0); + assert_eq!(dry_run.performance.export_mode, "mapped_delta"); +} + +#[test] +fn agent_apply_actual_reports_mapped_delta_metrics() { + if !git_available() { + return; + } + + let (_temp, mut db) = ready_agent_lane_with_mode(InitImportMode::GitTracked); + let applied = db.agent_apply("apply-bot", false, None).unwrap(); + assert_eq!(applied.performance.tracked_status_count, 1); + assert_eq!(applied.performance.full_root_file_count, 0); + assert_eq!(applied.performance.export_mode, "mapped_delta"); + assert_eq!(applied.performance.changed_path_count, 1); + assert_eq!(applied.performance.blob_write_count, 1); +} + +#[test] +fn agent_apply_batches_git_plumbing_for_many_paths() { + if !git_available() { + return; + } + + let (temp, mut db) = ready_agent_lane_with_changed_paths(100); + let applied = db.agent_apply("apply-bot", false, None).unwrap(); + assert_eq!(applied.performance.changed_path_count, 100); + assert_eq!(applied.performance.blob_write_count, 100); + assert_eq!(applied.performance.git_plumbing_command_count, 5); + assert_eq!( + applied + .git_export + .as_ref() + .unwrap() + .performance + .git_plumbing_command_count, + 5 + ); + let tmp = temp.path().join(".trail/tmp"); + if tmp.is_dir() { + assert!(!fs::read_dir(tmp).unwrap().any(|entry| { + entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("git-delta-") + })); + } +} + +#[test] +fn agent_apply_batch_preserves_modes_deletions_and_safe_special_paths() { + if !git_available() { + return; + } + + let temp = tempfile::tempdir().unwrap(); + run_git(temp.path(), &["init"]); + run_git(temp.path(), &["config", "user.email", "trail@example.test"]); + run_git(temp.path(), &["config", "user.name", "Trail Test"]); + fs::write(temp.path().join("delete-me.txt"), "remove me\n").unwrap(); + fs::write(temp.path().join("mode-change.sh"), "echo old\n").unwrap(); + run_git(temp.path(), &["add", "delete-me.txt", "mode-change.sh"]); + run_git(temp.path(), &["commit", "-m", "initial"]); + Trail::init(temp.path(), "main", InitImportMode::GitTracked, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("apply-bot", Some("main"), false, None, None) + .unwrap(); + let special_path = "dir with space/-leading-ünicode.sh"; + let patch: PatchDocument = serde_json::from_value(serde_json::json!({ + "message": "mixed Git index batch", + "edits": [ + {"op": "delete", "path": "delete-me.txt"}, + { + "op": "write", + "path": "mode-change.sh", + "content": "echo changed\n", + "executable": true + }, + { + "op": "write", + "path": special_path, + "content": "echo special\n", + "executable": true + } + ] + })) + .unwrap(); + apply_lane_patch_at_head(&mut db, "apply-bot", patch).unwrap(); + db.agent_mark_reviewed("apply-bot", None).unwrap(); + + let applied = db.agent_apply("apply-bot", false, None).unwrap(); + + assert_eq!(applied.performance.git_plumbing_command_count, 5); + assert_eq!( + git_output(temp.path(), &["ls-tree", "HEAD", "--", "delete-me.txt"]), + "" + ); + for path in ["mode-change.sh", special_path] { + let entry = git_output(temp.path(), &["ls-tree", "HEAD", "--", path]); + assert!( + entry.starts_with("100755 blob "), + "expected executable Git entry for {path:?}, got {entry:?}" + ); + } + assert_eq!( + git_output_raw(temp.path(), &["show", "HEAD:mode-change.sh"]), + "echo changed\n" + ); + assert_eq!( + git_output_raw(temp.path(), &["show", &format!("HEAD:{special_path}")]), + "echo special\n" + ); +} + +#[test] +fn agent_apply_batch_hashes_exact_trail_bytes_without_git_filters() { + if !git_available() { + return; + } + + let temp = tempfile::tempdir().unwrap(); + run_git(temp.path(), &["init"]); + run_git(temp.path(), &["config", "user.email", "trail@example.test"]); + run_git(temp.path(), &["config", "user.name", "Trail Test"]); + run_git( + temp.path(), + &["config", "filter.trail-uppercase.clean", "tr a-z A-Z"], + ); + run_git( + temp.path(), + &["config", "filter.trail-uppercase.smudge", "cat"], + ); + run_git( + temp.path(), + &["config", "filter.trail-uppercase.required", "true"], + ); + fs::write( + temp.path().join(".gitattributes"), + "* filter=trail-uppercase\n.gitattributes -filter\n", + ) + .unwrap(); + run_git(temp.path(), &["add", ".gitattributes"]); + run_git(temp.path(), &["commit", "-m", "initial"]); + Trail::init(temp.path(), "main", InitImportMode::GitTracked, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("apply-bot", Some("main"), false, None, None) + .unwrap(); + let patch: PatchDocument = serde_json::from_value(serde_json::json!({ + "message": "preserve exact bytes", + "edits": [{ + "op": "write", + "path": "payload.txt", + "content": "Exact Trail bytes\n" + }] + })) + .unwrap(); + apply_lane_patch_at_head(&mut db, "apply-bot", patch).unwrap(); + db.agent_mark_reviewed("apply-bot", None).unwrap(); + + db.agent_apply("apply-bot", false, None).unwrap(); + + assert_eq!( + git_output_raw(temp.path(), &["show", "HEAD:payload.txt"]), + "Exact Trail bytes\n" + ); +} + +#[test] +fn agent_apply_batches_git_plumbing_with_external_db_dir() { + if !git_available() { + return; + } + + let temp = tempfile::tempdir().unwrap(); + let workspace = temp.path().join("workspace"); + #[cfg(target_os = "linux")] + let external_db = temp + .path() + .join(OsString::from_vec(b"external-trail-db-\xff".to_vec())); + #[cfg(not(target_os = "linux"))] + let external_db = temp.path().join("external-trail-db"); + fs::create_dir(&workspace).unwrap(); + run_git(&workspace, &["init"]); + run_git(&workspace, &["config", "user.email", "trail@example.test"]); + run_git(&workspace, &["config", "user.name", "Trail Test"]); + fs::write(workspace.join("README.md"), "base\n").unwrap(); + run_git(&workspace, &["add", "README.md"]); + run_git(&workspace, &["commit", "-m", "initial"]); + Trail::init(&workspace, "main", InitImportMode::GitTracked, false).unwrap(); + fs::rename(workspace.join(".trail"), &external_db).unwrap(); + + let mut db = Trail::open_with_db_dir(&workspace, &external_db).unwrap(); + db.spawn_lane("apply-bot", Some("main"), false, None, None) + .unwrap(); + let patch: PatchDocument = serde_json::from_value(serde_json::json!({ + "message": "apply with external database", + "edits": [{ + "op": "write", + "path": "external-db.txt", + "content": "external database path\n" + }] + })) + .unwrap(); + apply_lane_patch_at_head(&mut db, "apply-bot", patch).unwrap(); + db.agent_mark_reviewed("apply-bot", None).unwrap(); + + let applied = db.agent_apply("apply-bot", false, None).unwrap(); + + assert_eq!(applied.performance.git_plumbing_command_count, 5); + assert_eq!( + git_output_raw(&workspace, &["show", "HEAD:external-db.txt"]), + "external database path\n" + ); + let tmp = external_db.join("tmp"); + if tmp.is_dir() { + assert!(!fs::read_dir(tmp).unwrap().any(|entry| { + entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("git-delta-") + })); + } +} + +#[test] +fn agent_apply_requires_mapping_before_git_or_trail_mutation() { + if !git_available() { + return; + } + + let (temp, mut db) = ready_agent_lane_with_mode(InitImportMode::WorkingTree); + let git_head = git_output(temp.path(), &["rev-parse", "HEAD"]); + assert!(db.git_mappings(10).unwrap().is_empty()); + + let err = db.agent_apply("apply-bot", true, None).unwrap_err(); + assert!(matches!(err, Error::GitMappingRequired(_))); + assert!(db.git_mappings(10).unwrap().is_empty()); + assert_eq!(git_output(temp.path(), &["rev-parse", "HEAD"]), git_head); +} + +#[cfg(unix)] +#[test] +fn agent_apply_preserves_git_only_symlinks() { + if !git_available() { + return; + } + + let temp = tempfile::tempdir().unwrap(); + run_git(temp.path(), &["init"]); + run_git(temp.path(), &["config", "user.email", "trail@example.test"]); + run_git(temp.path(), &["config", "user.name", "Trail Test"]); + fs::write(temp.path().join("target.md"), "target\n").unwrap(); + std::os::unix::fs::symlink("target.md", temp.path().join("link.md")).unwrap(); + run_git(temp.path(), &["add", "target.md", "link.md"]); + run_git(temp.path(), &["commit", "-m", "initial"]); + let link_tree_entry_before = git_output(temp.path(), &["ls-tree", "HEAD", "--", "link.md"]); + assert!( + link_tree_entry_before.starts_with("120000 blob "), + "expected a Git symlink entry, got {link_tree_entry_before:?}" + ); + + let init = Trail::init(temp.path(), "main", InitImportMode::GitTracked, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let root = db.inspect_root(&init.root_id.0).unwrap(); + assert!(!root.files.iter().any(|file| file.path == "link.md")); + db.spawn_lane("apply-bot", Some("main"), false, None, None) + .unwrap(); + let patch: PatchDocument = serde_json::from_value(serde_json::json!({ + "message": "add readme", + "edits": [{"op": "write", "path": "README.md", "content": "agent change"}] + })) + .unwrap(); + apply_lane_patch_at_head(&mut db, "apply-bot", patch).unwrap(); + db.agent_mark_reviewed("apply-bot", None).unwrap(); + + db.agent_apply("apply-bot", false, None).unwrap(); + + assert_eq!( + git_output(temp.path(), &["ls-tree", "HEAD", "--", "link.md"]), + link_tree_entry_before + ); + assert_eq!( + git_output_raw(temp.path(), &["show", "HEAD:link.md"]), + "target.md" + ); + assert_eq!( + git_output_raw(temp.path(), &["show", "HEAD:target.md"]), + "target\n" + ); + assert_eq!( + git_output_raw(temp.path(), &["show", "HEAD:README.md"]), + "agent change" + ); +} + +#[test] +fn agent_apply_dry_run_writes_no_git_or_mapping_state() { + if !git_available() { + return; + } + + let (temp, mut db) = ready_agent_lane_with_mode(InitImportMode::GitTracked); + let head_before = git_output(temp.path(), &["rev-parse", "HEAD"]); + let index_before = fs::read(temp.path().join(".git/index")).unwrap(); + let mappings_before = db.git_mappings(100).unwrap().len(); + let objects_before = git_object_count(temp.path()); + + let report = db.agent_apply("apply-bot", true, None).unwrap(); + assert!(report.dry_run); + + assert_eq!(git_output(temp.path(), &["rev-parse", "HEAD"]), head_before); + assert_eq!( + fs::read(temp.path().join(".git/index")).unwrap(), + index_before + ); + assert_eq!(db.git_mappings(100).unwrap().len(), mappings_before); + assert_eq!(git_object_count(temp.path()), objects_before); +} + +fn only_conflict_path_class(db: &Trail) -> (String, String) { + let conflicts = db.list_conflicts().unwrap(); + assert_eq!(conflicts.len(), 1); + let shown = db.show_conflict(&conflicts[0].conflict_set_id).unwrap(); + let explanation = shown.explanation.as_ref().unwrap(); + assert_eq!(explanation.paths.len(), 1); + ( + explanation.paths[0].path.clone(), + explanation.paths[0].conflict_class.clone(), + ) +} + +fn run_trail_json(workspace: &Path, args: &[&str]) -> serde_json::Value { + let output = Command::new(trail_bin()) + .arg("--workspace") + .arg(workspace) + .arg("--json") + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "trail {:?} failed\nstdout:\n{}\nstderr:\n{}", + args, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).unwrap() +} + +fn make_current_branch_root_legacy(workspace: &Path) -> ObjectId { + let sqlite_path = workspace.join(".trail/index/trail.sqlite"); + let conn = Connection::open(sqlite_path).unwrap(); + let root_id: String = conn + .query_row( + "SELECT root_id FROM refs WHERE name = 'refs/branches/main'", + [], + |row| row.get(0), + ) + .unwrap(); + let (version, bytes): (i64, Vec) = conn + .query_row( + "SELECT version, bytes FROM objects WHERE object_id = ?1 AND kind = ?2", + rusqlite::params![root_id, WORKTREE_ROOT_KIND], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + let mut root: WorktreeRoot = serde_cbor::from_slice(&bytes).unwrap(); + assert!(root.file_count > 0); + assert!(root.case_fold_map_root.take().is_some()); + let legacy_bytes = serde_cbor::to_vec(&root).unwrap(); + let legacy_root = ObjectId::for_bytes( + WORKTREE_ROOT_KIND, + u16::try_from(version).unwrap(), + &legacy_bytes, + ); + conn.execute( + "INSERT INTO objects (object_id, kind, version, codec, hash_alg, size_bytes, bytes, created_at) \ + VALUES (?1, ?2, ?3, 'cbor', 'sha256', ?4, ?5, 0)", + rusqlite::params![ + legacy_root.0.as_str(), + WORKTREE_ROOT_KIND, + version, + legacy_bytes.len() as i64, + legacy_bytes + ], + ) + .unwrap(); + conn.execute( + "UPDATE refs SET root_id = ?1 WHERE name = 'refs/branches/main'", + rusqlite::params![legacy_root.0.as_str()], + ) + .unwrap(); + legacy_root +} + +fn run_trail_json_daemon(workspace: &Path, daemon_url: &str, args: &[&str]) -> serde_json::Value { + let output = Command::new(trail_bin()) + .arg("--workspace") + .arg(workspace) + .arg("--daemon-url") + .arg(daemon_url) + .arg("--json") + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "trail --daemon-url {:?} failed\nstdout:\n{}\nstderr:\n{}", + args, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).unwrap() +} + +fn wait_for_child_exit(child: &mut Child) { + for _ in 0..100 { + if child.try_wait().unwrap().is_some() { + return; + } + thread::sleep(Duration::from_millis(25)); + } + panic!("daemon did not exit"); +} + +fn wait_for_daemon_endpoint(path: &Path) -> serde_json::Value { + for _ in 0..100 { + if let Ok(bytes) = fs::read(path) { + if let Ok(value) = serde_json::from_slice(&bytes) { + return value; + } + } + thread::sleep(Duration::from_millis(25)); + } + panic!("daemon endpoint was not published at {}", path.display()); +} + +struct DaemonGuard { + child: Child, +} + +impl Drop for DaemonGuard { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn free_loopback_port() -> u16 { + TcpListener::bind(("127.0.0.1", 0)) + .unwrap() + .local_addr() + .unwrap() + .port() +} + +fn wait_for_daemon_health(port: u16) { + for _ in 0..400 { + if daemon_health_ok(port) { + return; + } + thread::sleep(Duration::from_millis(25)); + } + panic!("daemon did not become healthy on port {port}"); +} + +fn daemon_health_ok(port: u16) -> bool { + let Ok(mut stream) = TcpStream::connect(("127.0.0.1", port)) else { + return false; + }; + let _ = stream.set_read_timeout(Some(Duration::from_millis(500))); + if stream + .write_all(b"GET /v1/health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .is_err() + { + return false; + } + let mut response = String::new(); + stream.read_to_string(&mut response).is_ok() && response.contains(" 200 ") +} + +fn raw_http_request(port: u16, request: &[u8]) -> String { + let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap(); + let _ = stream.set_read_timeout(Some(Duration::from_millis(1000))); + stream.write_all(request).unwrap(); + let mut response = String::new(); + stream.read_to_string(&mut response).unwrap(); + response +} + +fn api_request(method: &str, path: &str, body: serde_json::Value) -> Vec { + api_request_with_headers(method, path, &[], body) +} + +fn conflicted_readme_workspace( + lane_content: &str, + human_content: &str, +) -> (tempfile::TempDir, Trail, String) { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\nworld\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("manual-bot", Some("main"), false, None, None) + .unwrap(); + let patch: PatchDocument = serde_json::from_value(serde_json::json!({ + "message": "lane edits readme", + "edits": [ + {"op": "write", "path": "README.md", "content": lane_content} + ] + })) + .unwrap(); + apply_lane_patch_at_head(&mut db, "manual-bot", patch).unwrap(); + + fs::write(temp.path().join("README.md"), human_content).unwrap(); + db.record( + Some("main"), + Some("human edit".to_string()), + Actor::human(), + false, + ) + .unwrap(); + db.enqueue_lane_merge("manual-bot", "main", 0).unwrap(); + let run = db.run_lane_merge_queue(None).unwrap(); + assert!(run.stopped_on_conflict); + let conflict_id = db.list_conflicts().unwrap()[0].conflict_set_id.clone(); + (temp, db, conflict_id) +} + +fn api_request_with_headers( + method: &str, + path: &str, + headers: &[(&str, &str)], + body: serde_json::Value, +) -> Vec { + let body = if body.is_null() { + Vec::new() + } else { + serde_json::to_vec(&body).unwrap() + }; + let mut head = format!("{method} {path} HTTP/1.1\r\nHost: localhost\r\n"); + for (name, value) in headers { + head.push_str(name); + head.push_str(": "); + head.push_str(value); + head.push_str("\r\n"); + } + head.push_str(&format!( + "Content-Type: application/json\r\nContent-Length: {}\r\n\r\n", + body.len() + )); + [head.into_bytes(), body].concat() +} + +#[test] +fn cli_json_errors_are_machine_readable() { + let temp = tempfile::tempdir().unwrap(); + + let parse_output = Command::new(trail_bin()) + .arg("--json") + .arg("definitely-not-a-command") + .output() + .unwrap(); + assert!(!parse_output.status.success()); + assert_eq!(parse_output.status.code(), Some(2)); + assert!(parse_output.stdout.is_empty()); + let parse_stderr: serde_json::Value = serde_json::from_slice(&parse_output.stderr).unwrap(); + assert_eq!(parse_stderr["error"]["code"], "INVALID_INPUT"); + assert_eq!(parse_stderr["error"]["exit"], 2); + assert!(parse_stderr["error"]["message"] + .as_str() + .unwrap() + .contains("definitely-not-a-command")); + + let output = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .arg("status") + .output() + .unwrap(); + assert!(!output.status.success()); + assert_eq!(output.status.code(), Some(3)); + assert!(output.stdout.is_empty()); + let stderr: serde_json::Value = serde_json::from_slice(&output.stderr).unwrap(); + assert_eq!(stderr["error"]["code"], "WORKSPACE_NOT_FOUND"); + assert_eq!(stderr["error"]["exit"], 3); + assert!(stderr["error"]["message"] + .as_str() + .unwrap() + .contains("workspace not found")); + + let format_output = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--format") + .arg("json") + .arg("status") + .output() + .unwrap(); + assert_eq!(format_output.status.code(), Some(3)); + let format_stderr: serde_json::Value = serde_json::from_slice(&format_output.stderr).unwrap(); + assert_eq!(format_stderr["error"]["code"], "WORKSPACE_NOT_FOUND"); + + let env_parse_output = Command::new(trail_bin()) + .env("TRAIL_FORMAT", "json") + .arg("still-not-a-command") + .output() + .unwrap(); + assert!(!env_parse_output.status.success()); + assert_eq!(env_parse_output.status.code(), Some(2)); + let env_parse_stderr: serde_json::Value = + serde_json::from_slice(&env_parse_output.stderr).unwrap(); + assert_eq!(env_parse_stderr["error"]["code"], "INVALID_INPUT"); + assert!(env_parse_stderr["error"]["message"] + .as_str() + .unwrap() + .contains("still-not-a-command")); +} + +#[test] +fn cli_path_index_required_human_json_rebuild_and_retry_lifecycle() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "base\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let legacy_root = make_current_branch_root_legacy(temp.path()); + + let status = run_trail_json(temp.path(), &["status"]); + assert_eq!(status["head"]["root_id"], legacy_root.0); + assert_eq!(status["worktree_state"], "Clean"); + fs::write(temp.path().join("README.md"), "changed\n").unwrap(); + + let human = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["record", "-m", "blocked before upgrade"]) + .output() + .unwrap(); + assert_eq!(human.status.code(), Some(9)); + let human_stderr = String::from_utf8(human.stderr).unwrap(); + assert!( + human_stderr.contains("Trail workspace upgrade is required"), + "{human_stderr}" + ); + assert!( + human_stderr.contains("trail index rebuild"), + "{human_stderr}" + ); + assert!(!human_stderr.contains("trail --help"), "{human_stderr}"); + + let json = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .args(["record", "-m", "blocked before upgrade"]) + .output() + .unwrap(); + assert_eq!(json.status.code(), Some(9)); + assert!(json.stdout.is_empty()); + let json_error: serde_json::Value = serde_json::from_slice(&json.stderr).unwrap(); + assert_eq!(json_error["error"]["code"], "PATH_INDEX_REQUIRED"); + assert_eq!(json_error["error"]["exit"], 9); + assert!(json_error["error"]["message"] + .as_str() + .unwrap() + .contains("trail index rebuild")); + + let rebuilt = run_trail_json(temp.path(), &["index", "rebuild"]); + assert_eq!( + rebuilt["path_index_repaired_roots"] + .as_array() + .unwrap() + .len(), + 1 + ); + assert_eq!( + rebuilt["path_index_repaired_refs"] + .as_array() + .unwrap() + .len(), + 1 + ); + assert_eq!( + rebuilt["path_index_repaired_refs"][0]["old_root"], + legacy_root.0 + ); + let repaired_root = rebuilt["path_index_repaired_refs"][0]["new_root"] + .as_str() + .unwrap() + .to_string(); + let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); + let load_root = |root_id: &str| -> WorktreeRoot { + let bytes: Vec = conn + .query_row( + "SELECT bytes FROM objects WHERE object_id = ?1", + rusqlite::params![root_id], + |row| row.get(0), + ) + .unwrap(); + serde_cbor::from_slice(&bytes).unwrap() + }; + let legacy = load_root(&legacy_root.0); + let repaired = load_root(&repaired_root); + assert_eq!(repaired.path_map_root, legacy.path_map_root); + assert_eq!(repaired.file_index_map_root, legacy.file_index_map_root); + assert_eq!(repaired.file_count, legacy.file_count); + assert_eq!(repaired.total_text_bytes, legacy.total_text_bytes); + assert_eq!(repaired.created_by, legacy.created_by); + assert!(repaired.case_fold_map_root.is_some()); + drop(conn); + + let recorded = run_trail_json(temp.path(), &["record", "-m", "after upgrade"]); + assert_eq!(recorded["changed_paths"].as_array().unwrap().len(), 1); + let second = run_trail_json(temp.path(), &["index", "rebuild"]); + assert!(second["path_index_repaired_roots"] + .as_array() + .unwrap() + .is_empty()); + assert!(second["path_index_repaired_refs"] + .as_array() + .unwrap() + .is_empty()); +} + +#[cfg(unix)] +#[test] +fn acp_doctor_and_relay_preflight_report_legacy_path_index() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "base\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + make_current_branch_root_legacy(temp.path()); + + let doctor = run_trail_json(temp.path(), &["agent", "acp", "doctor", "claude-code"]); + assert_eq!(doctor["status"], "failed"); + assert!(doctor["checks"].as_array().unwrap().iter().any(|check| { + check["name"] == "path_invariant_index" + && check["status"] == "failed" + && check["message"] + .as_str() + .is_some_and(|message| message.contains("trail index rebuild")) + })); + + let relay = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .args(["acp", "relay", "--", "/bin/true"]) + .output() + .unwrap(); + assert_eq!(relay.status.code(), Some(9)); + assert!(relay.stdout.is_empty()); + let error: serde_json::Value = serde_json::from_slice(&relay.stderr).unwrap(); + assert_eq!(error["error"]["code"], "PATH_INDEX_REQUIRED"); + assert!(error["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("trail index rebuild"))); +} + +#[test] +fn cli_env_defaults_select_workspace_db_branch_and_format() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.create_branch("scratch", Some("main")).unwrap(); + drop(db); + + let workspace_output = Command::new(trail_bin()) + .env("TRAIL_WORKSPACE", temp.path()) + .env_remove("TRAIL_DIR") + .env("TRAIL_FORMAT", "json") + .env("TRAIL_BRANCH", "scratch") + .arg("status") + .output() + .unwrap(); + assert!( + workspace_output.status.success(), + "status with env workspace failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&workspace_output.stdout), + String::from_utf8_lossy(&workspace_output.stderr) + ); + let workspace_status: serde_json::Value = + serde_json::from_slice(&workspace_output.stdout).unwrap(); + assert_eq!(workspace_status["branch"], "scratch"); + + let local_branch_status = run_trail_json(temp.path(), &["status", "--branch", "scratch"]); + assert_eq!(local_branch_status["branch"], "scratch"); + + let db_dir_output = Command::new(trail_bin()) + .env_remove("TRAIL_WORKSPACE") + .env_remove("TRAIL_BRANCH") + .env("TRAIL_DIR", temp.path().join(".trail")) + .env("TRAIL_FORMAT", "json") + .arg("status") + .output() + .unwrap(); + assert!( + db_dir_output.status.success(), + "status with env db dir failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&db_dir_output.stdout), + String::from_utf8_lossy(&db_dir_output.stderr) + ); + let db_dir_status: serde_json::Value = serde_json::from_slice(&db_dir_output.stdout).unwrap(); + assert_eq!(db_dir_status["branch"], "main"); + + let invalid_format = Command::new(trail_bin()) + .env("TRAIL_WORKSPACE", temp.path()) + .env_remove("TRAIL_DIR") + .env_remove("TRAIL_BRANCH") + .env("TRAIL_FORMAT", "xml") + .arg("status") + .output() + .unwrap(); + assert!(!invalid_format.status.success()); + assert!(String::from_utf8_lossy(&invalid_format.stderr) + .contains("TRAIL_FORMAT must be human, plain, json, or ndjson")); +} + +#[test] +fn ndjson_rejects_single_report_commands_with_a_structured_diagnostic() { + let output = Command::new(trail_bin()) + .args(["--format", "ndjson", "status"]) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2)); + let error: serde_json::Value = serde_json::from_slice(&output.stderr).unwrap(); + assert_eq!(error["error"]["code"], "INVALID_INPUT"); + assert!(error["error"]["message"] + .as_str() + .unwrap() + .contains("streaming watch commands")); +} + +#[test] +fn plain_redirected_and_quiet_output_follow_terminal_policy() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let plain = Command::new(trail_bin()) + .args([ + "--workspace", + temp.path().to_str().unwrap(), + "--format", + "plain", + "--color", + "always", + "status", + ]) + .output() + .unwrap(); + assert!(plain.status.success()); + let plain_stdout = String::from_utf8(plain.stdout).unwrap(); + assert!(plain_stdout.contains("Worktree clean")); + assert!(!plain_stdout.contains("\u{1b}[")); + + let quiet = Command::new(trail_bin()) + .args([ + "--workspace", + temp.path().to_str().unwrap(), + "--quiet", + "status", + ]) + .output() + .unwrap(); + assert!(quiet.status.success()); + assert!(quiet.stdout.is_empty()); + assert!(quiet.stderr.is_empty()); +} + +#[test] +fn clap_diagnostics_keep_usage_on_separate_lines() { + let output = Command::new(trail_bin()) + .args(["agent", "hook", "receive"]) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2)); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains("the following required arguments were not provided:\n ")); + assert!(stderr.contains("\nUsage: trail agent hook receive \n")); + assert!(!stderr.contains(r"\nUsage:")); +} + +#[test] +fn agent_help_is_curated_and_hidden_commands_still_work() { + let help = Command::new(trail_bin()) + .args(["agent", "--help"]) + .output() + .unwrap(); + assert!( + help.status.success(), + "agent --help failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&help.stdout), + String::from_utf8_lossy(&help.stderr) + ); + let stdout = String::from_utf8_lossy(&help.stdout); + assert!(stdout.contains("Daily path:")); + assert!(stdout.contains("trail agent ask what should I do")); + assert!(stdout.contains(" action")); + assert!(stdout.contains(" changes")); + assert!(stdout.contains(" acp")); + assert!(stdout.contains(" hooks")); + assert!(!stdout.contains(" setup ")); + assert!(!stdout.contains("review-data")); + assert!(!stdout.contains("turn-diff")); + + let acp_help = Command::new(trail_bin()) + .args(["agent", "acp", "--help"]) + .output() + .unwrap(); + assert!(acp_help.status.success()); + let acp_stdout = String::from_utf8_lossy(&acp_help.stdout); + assert!(acp_stdout.contains(" setup")); + assert!(acp_stdout.contains(" status")); + assert!(!acp_stdout.contains(" run")); + + let hidden_help = Command::new(trail_bin()) + .args(["agent", "review-data", "--help"]) + .output() + .unwrap(); + assert!( + hidden_help.status.success(), + "agent review-data --help failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&hidden_help.stdout), + String::from_utf8_lossy(&hidden_help.stderr) + ); + assert!(String::from_utf8_lossy(&hidden_help.stdout).contains("review-data")); +} + +#[test] +fn init_record_why_and_fsck_work() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\nworld\n").unwrap(); + + let init = Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + assert_eq!(init.imported.files, 1); + assert!(init.workspace_id.0.starts_with("workspace_")); + assert!(init.operation.0.starts_with("change_")); + assert!(init.root_id.0.starts_with("object_")); + + fs::write(temp.path().join("README.md"), "hello\nTrail\n").unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let record = db + .record( + Some("main"), + Some("edit readme".to_string()), + Actor::human(), + false, + ) + .unwrap(); + assert!(record.operation.is_some()); + assert!(record.operation.as_ref().unwrap().0.starts_with("change_")); + assert!(record.root_id.0.starts_with("object_")); + assert_eq!(record.changed_paths.len(), 1); + + let why = db.why("README.md:2", Some("main")).unwrap(); + assert_eq!(why.current_text, "Trail"); + assert_eq!(why.history.len(), 1); + + let fsck = db.fsck().unwrap(); + assert!(fsck.errors.is_empty(), "{:?}", fsck.errors); +} + +#[test] +fn doctor_reports_operational_health_across_cli_api_and_mcp() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + { + let db = Trail::open(temp.path()).unwrap(); + let clean = db.doctor().unwrap(); + assert_eq!(clean.status, "ok"); + assert!(clean + .checks + .iter() + .any(|check| check.name == "fsck" && check.status == "ok")); + assert!(clean.checks.iter().any(|check| { + check.name == "schema_version" + && check.status == "ok" + && check.details.as_ref().unwrap()["sqlite_user_version"] == 18 + })); + } + + let cli = run_trail_json(temp.path(), &["doctor"]); + assert_eq!(cli["status"], "ok"); + assert!(cli["checks"] + .as_array() + .unwrap() + .iter() + .any(|check| check["name"] == "current_branch" && check["status"] == "ok")); + + let mut db = Trail::open(temp.path()).unwrap(); + let api = trail::server::handle_http_request( + &mut db, + &api_request("GET", "/v1/doctor", serde_json::Value::Null), + ); + assert_eq!(api.status, 200); + let api: serde_json::Value = api.body_json().unwrap(); + assert_eq!(api["status"], "ok"); + + let tools = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/list", + "params": {} + }), + ) + .unwrap(); + let tool_list = tools["result"]["tools"].as_array().unwrap(); + assert!(tool_list.iter().any(|tool| tool["name"] == "trail.doctor")); + + let mcp = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "trail.doctor", + "arguments": {} + } + }), + ) + .unwrap(); + assert_eq!(mcp["result"]["isError"], false); + assert_eq!(mcp["result"]["structuredContent"]["status"], "ok"); + + db.spawn_lane("doctor-bot", Some("main"), false, None, None) + .unwrap(); + db.request_lane_approval( + "doctor-bot", + "shell.exec", + "Run release smoke tests", + None, + None, + None, + ) + .unwrap(); + let warning = db.doctor().unwrap(); + assert_eq!(warning.status, "warning"); + let pending = warning + .checks + .iter() + .find(|check| check.name == "pending_approvals") + .unwrap(); + assert_eq!(pending.status, "warning"); + assert_eq!(pending.details.as_ref().unwrap()["count"], 1); +} + +#[test] +fn environment_component_report_keeps_component_and_adapter_identities_separate_json() { + let report = trail::EnvironmentComponentStateReport { + view_id: "view-1".to_string(), + component: trail::EnvironmentComponentIdentityReport { + component_id: "web.dependencies".to_string(), + kind: "dependency".to_string(), + }, + adapter: trail::EnvironmentAdapterIdentityReport { + namespace: "trail".to_string(), + name: "node".to_string(), + contract_major: 1, + implementation_version: "0.5.0".to_string(), + distribution_digest: Some("builtin".to_string()), + }, + expected_key: "expected".to_string(), + attached_key: Some("attached".to_string()), + status: "stale".to_string(), + reason: Some("lock changed".to_string()), + updated_at: 7, + }; + + let json = serde_json::to_value(&report).unwrap(); + assert_eq!(json["component"]["component_id"], "web.dependencies"); + assert_eq!(json["adapter"]["namespace"], "trail"); + assert_eq!(json["adapter"]["contract_major"], 1); + assert!(json["component"].get("adapter").is_none()); +} + +#[test] +fn environment_component_report_keeps_component_and_adapter_identities_separate() { + let report = trail::EnvironmentComponentStateReport { + view_id: "view-1".to_string(), + component: trail::EnvironmentComponentIdentityReport { + component_id: "web.dependencies".to_string(), + kind: "dependency".to_string(), + }, + adapter: trail::EnvironmentAdapterIdentityReport { + namespace: "trail".to_string(), + name: "node".to_string(), + contract_major: 1, + implementation_version: "0.5.0".to_string(), + distribution_digest: Some("builtin".to_string()), + }, + expected_key: "expected".to_string(), + attached_key: Some("attached".to_string()), + status: "stale".to_string(), + reason: Some("lock changed".to_string()), + updated_at: 7, + }; + + let json = serde_json::to_value(&report).unwrap(); + assert_eq!(json["component"]["component_id"], "web.dependencies"); + assert_eq!(json["adapter"]["namespace"], "trail"); + assert_eq!(json["adapter"]["contract_major"], 1); + assert!(json["component"].get("adapter").is_none()); +} + +#[test] +fn environment_plugin_publisher_trust_cli_is_append_only_and_catalogued() { + let workspace = tempfile::tempdir().unwrap(); + fs::write(workspace.path().join("README.md"), "root\n").unwrap(); + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let key = workspace.path().join("publisher-key.toml"); + fs::write( + &key, + r#"schema = "trail.environment-adapter-publisher-key/v1" +publisher = "rfc8032-test" +public_key = "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a" +"#, + ) + .unwrap(); + let key_path = key.to_string_lossy().into_owned(); + + let added = run_trail_json( + workspace.path(), + &["env", "plugin", "trust", "add", &key_path], + ); + assert_eq!(added["action"], "trust"); + assert_eq!(added["publisher"], "rfc8032-test"); + let key_id = added["key_id"].as_str().unwrap().to_string(); + assert!(key_id.starts_with("sha256:")); + + let listed = run_trail_json(workspace.path(), &["env", "plugin", "trust", "list"]); + assert_eq!(listed["keys"].as_array().unwrap().len(), 1); + assert_eq!(listed["keys"][0]["key_id"], key_id); + + let catalog = run_trail_json(workspace.path(), &["env", "adapters"]); + assert!(catalog["adapters"].as_array().unwrap().iter().all(|entry| { + entry["trust"] == "builtin" + && entry["publisher"] == "trail" + && entry["certification_tier"].as_str().is_some() + })); + + let removed = run_trail_json( + workspace.path(), + &["env", "plugin", "trust", "remove", &key_id], + ); + assert_eq!(removed["action"], "remove"); + assert_eq!(removed["key_id"], key_id); + let listed = run_trail_json(workspace.path(), &["env", "plugin", "trust", "list"]); + assert!(listed["keys"].as_array().unwrap().is_empty()); +} + +#[test] +fn trail_refuses_workspaces_with_newer_schema_versions() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); + conn.execute_batch("PRAGMA user_version = 999;").unwrap(); + drop(conn); + + let err = match Trail::open(temp.path()) { + Ok(_) => panic!("opening a future schema should fail"), + Err(err) => err, + }; + assert!(err + .to_string() + .contains("schema version 999 is newer than supported version")); +} + +#[test] +fn init_creates_lane_observability_indexes() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); + let indexes = { + let mut stmt = conn + .prepare("SELECT name FROM sqlite_master WHERE type = 'index'") + .unwrap(); + stmt.query_map([], |row| row.get::<_, String>(0)) + .unwrap() + .collect::, _>>() + .unwrap() + }; + for expected in [ + "lane_turns_session_started_idx", + "lane_turns_lane_started_idx", + "lane_events_lane_created_idx", + "lane_events_session_created_idx", + "lane_events_turn_created_idx", + "lane_events_type_created_idx", + "lane_events_lane_type_created_idx", + "lane_events_session_type_created_idx", + "lane_events_turn_type_created_idx", + "lane_trace_span_events_span_created_idx", + "lane_trace_span_events_trace_created_idx", + "lane_acp_sessions_lane_idx", + "lane_acp_sessions_trail_session_idx", + "external_mutation_audit_created_idx", + "external_mutation_audit_surface_created_idx", + "external_mutation_audit_lane_created_idx", + "http_idempotency_keys_updated_idx", + "conflict_resolution_suggestions_signature_idx", + ] { + assert!( + indexes.iter().any(|index| index == expected), + "missing index {expected}" + ); + } +} + +#[test] +fn agent_acp_doctor_reports_status_setup_and_verifiable_evidence() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let profiles = run_trail_json(temp.path(), &["agent", "acp", "status"]); + assert!(profiles + .as_array() + .unwrap() + .iter() + .any(|profile| profile["agent"] == "claude-code")); + assert!(profiles + .as_array() + .unwrap() + .iter() + .any(|profile| profile["agent"] == "codex")); + assert!(profiles + .as_array() + .unwrap() + .iter() + .any(|profile| profile["agent"] == "cursor" + && profile["supports_acp"] == true + && profile["supports_terminal"] == true)); + assert!(!profiles + .as_array() + .unwrap() + .iter() + .any(|profile| profile["agent"] == "fake")); + + let setup = run_trail_json( + temp.path(), + &[ + "agent", + "acp", + "setup", + "claude-code", + "--editor", + "generic", + "--print", + ], + ); + assert_eq!(setup["transport"], "acp"); + assert_eq!(setup["provider"], "claude-code"); + assert_eq!(setup["editor"], "generic"); + assert!(setup["snippet"] + .as_str() + .unwrap() + .contains("agent acp run claude-code")); + assert!(setup["command"] + .as_array() + .unwrap() + .iter() + .any(|part| part == "run")); + + let doctor = run_trail_json(temp.path(), &["agent", "acp", "doctor", "claude-code"]); + let checks = doctor["checks"].as_array().unwrap(); + assert!(checks + .iter() + .any(|check| check["name"] == "workspace" && check["status"] == "ok")); + assert!(checks + .iter() + .any(|check| { check["name"] == "capture_journal" && check["status"] == "ok" })); + assert!(checks + .iter() + .any(|check| check["name"] == "path_mapping" && check["status"] == "ok")); + assert_eq!(doctor["conformance"]["wire_version"], 1); + assert_eq!( + doctor["conformance"]["schema_commit"], + "64cbd71ae520b89aac54164d8c1d364333c8ee5f" + ); + assert_eq!( + doctor["conformance"]["schema_sha256"], + "92c1dfcda10dd47e99127500a3763da2b471f9ac61e12b9bf0430c32cf953796" + ); + assert_eq!( + doctor["conformance"]["meta_sha256"], + "e0bf36f8123b2544b499174197fdc371ec49a1b4572a35114513d56492741599" + ); + assert_eq!(doctor["conformance"]["transport"], "stdio"); + assert_eq!(doctor["conformance"]["method_count"], 23); + let source_revision = option_env!("TRAIL_SOURCE_REVISION") + .filter(|revision| !revision.is_empty()) + .unwrap_or("unverified"); + let expected_evidence = if source_revision != "unverified" + && option_env!("TRAIL_ACP_V1_CONFORMANCE_VERIFIED") == Some(source_revision) + { + "verified" + } else { + "unverified" + }; + assert_eq!(doctor["conformance"]["evidence_status"], expected_evidence); + assert!(doctor["conformance"]["build_identifier"] + .as_str() + .unwrap() + .starts_with(concat!(env!("CARGO_PKG_VERSION"), "+"))); + assert_eq!( + doctor["conformance"]["exclusions"], + serde_json::json!(["ACP v2", "draft remote HTTP transport"]) + ); + assert!(doctor["lane"].is_null()); + assert!(doctor["session_id"].is_null()); + + let human = Command::new(trail_bin()) + .current_dir(temp.path()) + .args(["agent", "acp", "doctor", "claude-code"]) + .output() + .unwrap(); + assert!(human.status.success()); + let human = String::from_utf8_lossy(&human.stdout); + for expected in [ + "Wire version", + "Schema commit", + "Schema SHA-256", + "Metadata SHA-256", + "Transport", + "Evidence", + "ACP v2", + "draft remote HTTP transport", + "capture_journal", + "path_mapping", + ] { + assert!( + human.contains(expected), + "missing `{expected}` in:\n{human}" + ); + } + if expected_evidence == "unverified" { + assert!(!human.contains("ACP v1 conformant")); + } else { + assert!(human.contains("ACP v1 conformant")); + } + + let custom_doctor = run_trail_json( + temp.path(), + &[ + "agent", + "acp", + "doctor", + "custom-acp", + "--relay-command", + "trail", + "acp", + "relay", + "--", + "", + ], + ); + assert_eq!(custom_doctor["provider"], "custom-acp"); + assert!(custom_doctor["checks"] + .as_array() + .unwrap() + .iter() + .any(|check| check["name"] == "relay" && check["status"] == "ok")); + + let codex = run_trail_json(temp.path(), &["agent", "acp", "status", "codex"]); + assert_eq!(codex.as_array().unwrap().len(), 1); + assert_eq!(codex[0]["agent"], "codex"); +} + +#[test] +fn agent_hooks_setup_is_read_only_until_yes_is_passed() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let config_path = temp.path().join(".codex/hooks.json"); + let preview = run_trail_json( + temp.path(), + &["agent", "hooks", "setup", "codex", "--print"], + ); + assert_eq!(preview["provider"], "codex"); + assert_eq!(preview["dry_run"], true); + assert!(!config_path.exists()); + + let applied = run_trail_json(temp.path(), &["agent", "hooks", "setup", "codex", "--yes"]); + assert_eq!(applied["provider"], "codex"); + assert_eq!(applied["dry_run"], false); + assert!(config_path.is_file()); +} + +#[cfg(unix)] +#[test] +fn acp_relay_accepts_a_built_in_agent_shortcut() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let output = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["acp", "relay", "codex", "--", "/usr/bin/true"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "trail acp relay codex failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[cfg(unix)] +#[test] +fn agent_acp_setup_and_hidden_runner_use_fresh_lanes() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let setup = run_trail_json( + temp.path(), + &[ + "agent", + "acp", + "setup", + "claude-code", + "--editor", + "vscode", + "--print", + ], + ); + assert_eq!(setup["transport"], "acp"); + assert_eq!(setup["provider"], "claude-code"); + assert_eq!(setup["editor"], "vscode"); + assert_eq!(setup["applied"], false); + assert!(setup["command"] + .as_array() + .unwrap() + .windows(3) + .any(|parts| parts == ["acp", "run", "claude-code"])); + + let gemini_doctor = run_trail_json(temp.path(), &["agent", "doctor", "gemini"]); + assert_eq!(gemini_doctor["provider"], "gemini"); + assert_eq!(gemini_doctor["capabilities"]["terminal"], true); + assert_eq!(gemini_doctor["capabilities"]["mcp"], true); + + { + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("manual-low-level", Some("main"), false, None, None) + .unwrap(); + } + let empty_list = run_trail_json(temp.path(), &["agent", "list"]); + assert_eq!(empty_list["tasks"].as_array().unwrap().len(), 0); + let empty_status = run_trail_json(temp.path(), &["agent", "status"]); + assert_eq!(empty_status["status"], "empty"); + let empty_next = run_trail_json(temp.path(), &["agent", "next"]); + assert_eq!(empty_next["focus"], "setup"); + assert_eq!( + empty_next["primary"]["command"], + "trail agent acp setup claude-code --editor vscode" + ); + let empty_actions = run_trail_json(temp.path(), &["agent", "action"]); + assert_eq!(empty_actions["status"], "empty", "{empty_actions}"); + assert!(empty_actions["task"].is_null()); + assert_eq!( + empty_actions["next"]["command"], + "trail agent acp setup claude-code --editor vscode" + ); + assert!(empty_actions["actions"] + .as_array() + .unwrap() + .iter() + .any(|action| action["id"] == "acp_setup_vscode" + && action["command"] == "trail agent acp setup claude-code --editor vscode")); + assert!(empty_actions["actions"] + .as_array() + .unwrap() + .iter() + .any(|action| action["id"] == "acp_setup_codex_vscode" + && action["command"] == "trail agent acp setup codex --editor vscode")); + assert!(empty_actions["actions"] + .as_array() + .unwrap() + .iter() + .any(|action| action["id"] == "doctor_codex" + && action["command"] == "trail agent doctor codex")); + assert!(empty_actions["actions"] + .as_array() + .unwrap() + .iter() + .any(|action| action["id"] == "acp_setup_cursor_vscode" + && action["command"] == "trail agent acp setup cursor --editor vscode")); + assert!(empty_actions["actions"] + .as_array() + .unwrap() + .iter() + .any(|action| action["id"] == "start_terminal_task" + && action["requires_confirmation"] == true)); + assert!(empty_actions["actions"] + .as_array() + .unwrap() + .iter() + .any(|action| action["id"] == "start_gemini_task" + && action["command"] == "trail agent start gemini")); + let empty_ask_actions = run_trail_json(temp.path(), &["agent", "ask", "show", "actions"]); + assert_eq!(empty_ask_actions["status"], "empty"); + assert!(empty_ask_actions["task"].is_null()); + assert!(empty_ask_actions["actions"] + .as_array() + .unwrap() + .iter() + .any(|action| action["id"] == "acp_setup_vscode")); + let empty_setup_action = run_trail_json(temp.path(), &["agent", "action", "acp_setup_vscode"]); + assert_eq!(empty_setup_action["provider"], "claude-code"); + assert_eq!(empty_setup_action["editor"], "vscode"); + assert!(empty_setup_action["command"] + .as_array() + .unwrap() + .iter() + .any(|part| part == "agent" || part == "acp")); + let empty_doctor_action = + run_trail_json(temp.path(), &["agent", "action", "doctor_claude_code"]); + assert_eq!(empty_doctor_action["provider"], "claude-code"); + assert!(empty_doctor_action["checks"] + .as_array() + .unwrap() + .iter() + .any(|check| check["name"] == "workspace" && check["status"] == "ok")); + let empty_start_print = run_trail_json( + temp.path(), + &["agent", "action", "start_terminal_task", "--print"], + ); + assert_eq!(empty_start_print["status"], "empty"); + assert!(empty_start_print["task"].is_null()); + assert_eq!(empty_start_print["action"]["id"], "start_terminal_task"); + assert!(empty_start_print["action"]["command"] + .as_str() + .unwrap() + .contains("agent start")); + let empty_start_guard = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("--json") + .args(["agent", "action", "start_terminal_task"]) + .output() + .unwrap(); + assert!(!empty_start_guard.status.success()); + let empty_start_stderr: serde_json::Value = + serde_json::from_slice(&empty_start_guard.stderr).unwrap(); + assert!(empty_start_stderr["error"]["message"] + .as_str() + .unwrap() + .contains("requires --confirm")); + for (command, requested) in [ + (vec!["agent", "view", "latest"], "view"), + (vec!["agent", "changes", "latest"], "changes"), + (vec!["agent", "apply", "latest", "--dry-run"], "apply"), + ] { + let empty_hint = run_trail_json(temp.path(), &command); + assert_eq!(empty_hint["status"], "empty"); + assert!(empty_hint["task"].is_null()); + assert_eq!(empty_hint["requested"], requested); + assert!(empty_hint["summary"] + .as_str() + .unwrap() + .contains("No agent task is recorded yet")); + assert!(!empty_hint["summary"] + .as_str() + .unwrap() + .contains("to changes")); + assert_eq!( + empty_hint["next"]["command"], + "trail agent acp setup claude-code --editor vscode" + ); + assert!(empty_hint["actions"] + .as_array() + .unwrap() + .iter() + .any(|action| action["id"] == "acp_setup_vscode")); + } + let empty_dashboard = run_trail_json(temp.path(), &["agent", "dashboard"]); + assert_eq!(empty_dashboard["status"], "empty"); + assert!(empty_dashboard["task"].is_null()); + assert_eq!( + empty_dashboard["next"]["command"], + "trail agent acp setup claude-code --editor vscode" + ); + let empty_inbox = run_trail_json(temp.path(), &["agent", "inbox"]); + assert_eq!(empty_inbox["total"], 0); + assert_eq!(empty_inbox["attention_count"], 0); + assert_eq!(empty_inbox["items"].as_array().unwrap().len(), 0); + assert_eq!( + empty_inbox["next"]["command"], + "trail agent acp setup claude-code --editor vscode" + ); + let empty_bare_agent = run_trail_json(temp.path(), &["agent"]); + assert_eq!(empty_bare_agent["total"], 0); + assert_eq!( + empty_bare_agent["next"]["command"], + "trail agent acp setup claude-code --editor vscode" + ); + + let doctor = run_trail_json( + temp.path(), + &["agent", "doctor", "--provider", "claude-code"], + ); + assert_eq!(doctor["provider"], "claude-code"); + assert!(doctor["checks"] + .as_array() + .unwrap() + .iter() + .any(|check| { check["name"] == "workspace" && check["status"] == "ok" })); + assert_eq!( + doctor["suggestions"][0]["command"], + "trail agent acp setup claude-code --editor vscode" + ); + + let first_agent = write_stub_acp_agent( + temp.path(), + "agent-acp-stub-a.sh", + StubAcpAgentOptions::new("sess_agent_stub_a"), + ); + let second_agent = write_stub_acp_agent( + temp.path(), + "agent-acp-stub-b.sh", + StubAcpAgentOptions::new("sess_agent_stub_b"), + ); + run_agent_acp_stub_session(temp.path(), &first_agent); + let one_task_home = run_trail_json(temp.path(), &["agent"]); + assert!(one_task_home["task"]["name"] + .as_str() + .unwrap() + .starts_with("agent-claude-code-")); + assert_eq!(one_task_home["focus"]["path"], "README.md"); + run_agent_acp_stub_session(temp.path(), &second_agent); let list = run_trail_json(temp.path(), &["agent", "list"]); let tasks = list["tasks"].as_array().unwrap(); @@ -1388,16 +3011,14 @@ fn agent_setup_and_stub_acp_use_fresh_lanes() { .unwrap() .iter() .any(|command| command == "write_file")); - assert!(agent_tools["tools"] - .as_array() - .unwrap() - .iter() - .any(|tool| tool["name"] == "write README" + assert!(agent_tools["tools"].as_array().unwrap().iter().any(|tool| { + tool["name"] == "write README" && tool["turns"][0]["changed_paths"] .as_array() .unwrap() .iter() - .any(|path| path["path"] == "README.md"))); + .any(|path| path["path"] == "README.md") + })); let impact = run_trail_json(temp.path(), &["agent", "impact", "latest"]); assert_eq!(impact["task"]["lane"], view["task"]["lane"]); assert_eq!(impact["highest_impact"], "low"); @@ -1859,14 +3480,11 @@ fn agent_setup_and_stub_acp_use_fresh_lanes() { .unwrap() .iter() .any(|item| item.as_str().unwrap().contains("Git preflight failed"))); - assert!(diagnose["evidence"] - .as_array() - .unwrap() - .iter() - .any(|item| item - .as_str() + assert!(diagnose["evidence"].as_array().unwrap().iter().any(|item| { + item.as_str() .unwrap() - .contains("friendly checkpoint target"))); + .contains("friendly checkpoint target") + })); assert!(diagnose["recovery_options"] .as_array() .unwrap() @@ -1918,9 +3536,12 @@ fn agent_setup_and_stub_acp_use_fresh_lanes() { String::from_utf8_lossy(&diagnose_text.stderr) ); let diagnose_stdout = String::from_utf8_lossy(&diagnose_text.stdout); - assert!(diagnose_stdout.contains("Agent diagnose:")); - assert!(diagnose_stdout.contains("Likely issue:")); - assert!(diagnose_stdout.contains("Recovery options:")); + assert!( + diagnose_stdout.contains("Agent diagnosis: git blocked"), + "{diagnose_stdout}" + ); + assert!(diagnose_stdout.contains("Likely Issue")); + assert!(diagnose_stdout.contains("Recovery Options")); assert!(diagnose_stdout.contains("requires a Git working tree")); let report = run_trail_json(temp.path(), &["agent", "report", "latest"]); assert_eq!(report["task"]["lane"], lane); @@ -2141,7 +3762,11 @@ fn agent_setup_and_stub_acp_use_fresh_lanes() { String::from_utf8_lossy(&open_launch.stdout), String::from_utf8_lossy(&open_launch.stderr) ); - assert!(String::from_utf8_lossy(&open_launch.stdout).contains("Opened:")); + assert!( + String::from_utf8_lossy(&open_launch.stdout).contains("Opened agent focus"), + "{}", + String::from_utf8_lossy(&open_launch.stdout) + ); let ask_dashboard = run_trail_json( temp.path(), &["agent", "ask", "show", "me", "the", "dashboard"], @@ -2298,7 +3923,7 @@ fn agent_setup_and_stub_acp_use_fresh_lanes() { ); let summary_stdout = String::from_utf8_lossy(&summary_text.stdout); assert!(summary_stdout.contains("Agent summary:")); - assert!(summary_stdout.contains("PR title:")); + assert!(summary_stdout.contains("Pr Title"), "{summary_stdout}"); let compare = run_trail_json(temp.path(), &["agent", "compare", first, second]); assert_eq!(compare["left"]["lane"], first); assert_eq!(compare["right"]["lane"], second); @@ -2414,9 +4039,12 @@ fn agent_setup_and_stub_acp_use_fresh_lanes() { String::from_utf8_lossy(&change_text.stderr) ); let change_stdout = String::from_utf8_lossy(&change_text.stdout); - assert!(change_stdout.contains("Agent change set:")); + assert!( + change_stdout.contains("Agent change set"), + "{change_stdout}" + ); assert!(change_stdout.contains("Docs and getting-started")); - assert!(change_stdout.contains("Files:")); + assert!(change_stdout.contains("Files")); let timeline = run_trail_json(temp.path(), &["agent", "timeline", "latest"]); assert_eq!(timeline["task"]["lane"], lane); @@ -2470,9 +4098,9 @@ fn agent_setup_and_stub_acp_use_fresh_lanes() { String::from_utf8_lossy(&timeline_text.stderr) ); let timeline_stdout = String::from_utf8_lossy(&timeline_text.stdout); - assert!(timeline_stdout.contains("Agent timeline:")); - assert!(timeline_stdout.contains("Timeline:")); - assert!(timeline_stdout.contains("Rewind before:")); + assert!(timeline_stdout.contains("Agent timeline")); + assert!(timeline_stdout.contains("Items"), "{timeline_stdout}"); + assert!(timeline_stdout.contains("Before Change")); let delta = run_trail_json(temp.path(), &["agent", "delta", "latest"]); assert_eq!(delta["task"]["lane"], lane); @@ -2535,8 +4163,11 @@ fn agent_setup_and_stub_acp_use_fresh_lanes() { String::from_utf8_lossy(&delta_text.stderr) ); let delta_stdout = String::from_utf8_lossy(&delta_text.stdout); - assert!(delta_stdout.contains("Agent delta:")); - assert!(delta_stdout.contains("Newest delta:")); + assert!(delta_stdout.contains("Agent delta")); + assert!( + delta_stdout.contains("Latest turn 1 changed 1 file(s)"), + "{delta_stdout}" + ); let last_alias = run_trail_json(temp.path(), &["agent", "last", "latest"]); assert_eq!(last_alias["task"]["lane"], lane); assert_eq!(last_alias["mode"], delta["mode"]); @@ -2654,8 +4285,12 @@ fn agent_setup_and_stub_acp_use_fresh_lanes() { String::from_utf8_lossy(&new_text.stderr) ); let new_stdout = String::from_utf8_lossy(&new_text.stdout); - assert!(new_stdout.contains("Agent new:")); - assert!(new_stdout.contains("Status: up_to_date")); + assert!( + new_stdout.contains("Created agent task: up to date"), + "{new_stdout}" + ); + assert!(new_stdout.contains("Status")); + assert!(new_stdout.contains("up to date")); let next_after_reviewed = run_trail_json(temp.path(), &["agent", "next", "latest"]); assert_eq!(next_after_reviewed["focus"], "preview_apply"); assert_eq!( @@ -2777,9 +4412,10 @@ fn agent_setup_and_stub_acp_use_fresh_lanes() { String::from_utf8_lossy(&file_text.stderr) ); let file_stdout = String::from_utf8_lossy(&file_text.stdout); - assert!(file_stdout.contains("Agent file: README.md")); - assert!(file_stdout.contains("Change sets:")); - assert!(file_stdout.contains("Changed in:")); + assert!(file_stdout.contains("Agent file"), "{file_stdout}"); + assert!(file_stdout.contains("Path : README.md")); + assert!(file_stdout.contains("Change Cards")); + assert!(file_stdout.contains("Touched By")); let checkpoints = run_trail_json(temp.path(), &["agent", "checkpoints", "latest"]); assert_eq!(checkpoints["task"]["lane"], lane); @@ -2865,7 +4501,7 @@ fn agent_setup_and_stub_acp_use_fresh_lanes() { turn_latest["turn_envelope"]["schema"], "trail.turn_envelope" ); - assert_eq!(turn_latest["turn_envelope"]["version"], 1); + assert_eq!(turn_latest["turn_envelope"]["version"], 2); assert_eq!(turn_latest["turn_envelope"]["provider"], "claude-code"); assert_eq!(turn_latest["turn_envelope"]["kind"], "acp_prompt"); assert_eq!(turn_latest["turn_envelope"]["protocol"], "acp"); @@ -3008,10 +4644,11 @@ fn agent_setup_and_stub_acp_use_fresh_lanes() { .contains("agent why"))); let checkpoint = changes["groups"][0]["checkpoint"].as_str().unwrap(); + let checkpoint_alias = checkpoint.replacen("change_", "checkpoint_", 1); let before_turn = changes["groups"][0]["before_change"].as_str().unwrap(); let checkpoint_diff = run_trail_json( temp.path(), - &["agent", "diff", "latest", "--checkpoint", checkpoint], + &["agent", "diff", "latest", "--checkpoint", &checkpoint_alias], ); assert_eq!(checkpoint_diff["after_change"], checkpoint); @@ -3041,7 +4678,7 @@ fn run_agent_acp_stub_session(workspace: &Path, agent: &Path) { .arg(workspace) .arg("agent") .arg("acp") - .arg("--provider") + .arg("run") .arg("claude-code") .arg("--no-mcp") .arg("--") @@ -3130,7 +4767,7 @@ fn agent_start_custom_command_applies_task_to_git_with_guided_flow() { .as_array() .unwrap() .iter() - .any(|step| step["command"] == "trail agent setup")); + .any(|step| step["command"] == "trail agent acp setup claude-code --editor vscode")); let edit_script = temp.path().join("edit-readme.sh"); fs::write( &edit_script, @@ -3187,14 +4824,12 @@ fn agent_start_custom_command_applies_task_to_git_with_guided_flow() { .unwrap() .iter() .any(|step| step["command"] == format!("trail agent action {task_name}"))); - assert!(task_guide["steps"] - .as_array() - .unwrap() - .iter() - .any(|step| step["command"] + assert!(task_guide["steps"].as_array().unwrap().iter().any(|step| { + step["command"] .as_str() .unwrap() - .contains("trail agent ask --selector"))); + .contains("trail agent ask --selector") + })); assert!(task_guide["concepts"] .as_array() .unwrap() @@ -3717,7 +5352,7 @@ fn acp_relay_captures_session_prompt_mcp_and_workdir_edits() { Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let stub_agent = temp.path().join("stub-acp-agent.sh"); - let session_request_log = temp.path().join("session-new.jsonl"); + let session_request_log = temp.path().join(".trail/session-new.jsonl"); let lane_workdir = temp .path() .canonicalize() @@ -3886,45 +5521,290 @@ printf '%s\n' '{{"jsonrpc":"2.0","id":2,"result":{{"stopReason":"end_turn"}}}}' .operations .iter() .any(|operation| operation.kind == OperationKind::LaneRecord)); - let status = db.lane_status("acp-test").unwrap(); - assert!(status - .changed_paths - .iter() - .any(|path| path.path == "README.md")); + let envelope = turn + .turn_envelope + .as_ref() + .expect("ACP prompt must have a typed turn envelope"); + assert_eq!(envelope.outcome.status.as_deref(), Some("completed")); + assert!(!envelope.outcome.no_changes); + assert!(envelope.outcome.checkpoint.is_some()); + assert_eq!(envelope.outcome.checkpoint, turn.turn.after_change); + assert!(turn + .events + .iter() + .any(|event| event.event_type == "workdir_recorded")); + let status = db.lane_status("acp-test").unwrap(); + assert_eq!(status.workdir_state, Some(WorktreeState::Clean)); + assert!(status + .changed_paths + .iter() + .any(|path| path.path == "README.md")); + + let acp_sessions = run_trail_json( + temp.path(), + &["agent", "acp", "sessions", "--lane", "acp-test"], + ); + assert_eq!(acp_sessions["sessions"][0]["acp_session_id"], "sess_stub"); + + let transcript = run_trail_json(temp.path(), &["transcript", "acp-test"]); + assert_eq!(transcript["resolved_kind"], "lane"); + assert_eq!(transcript["acp_session"]["acp_session_id"], "sess_stub"); + assert!(transcript["turns"][0]["messages"] + .as_array() + .unwrap() + .iter() + .any(|message| message["role"] == "assistant" + && message["body"].as_str().unwrap().contains("done"))); + assert!(transcript["turns"][0]["tool_summaries"] + .as_array() + .unwrap() + .iter() + .any(|summary| summary.as_str().unwrap().contains("write README"))); + assert_eq!( + transcript["turns"][0]["checkpoint"], + transcript["turns"][0]["turn_envelope"]["outcome"]["checkpoint"] + ); + + let turn_alias = run_trail_json( + temp.path(), + &["turn", "show", session.turns[0].turn_id.as_str()], + ); + assert_eq!(turn_alias["turn"]["turn_id"], session.turns[0].turn_id); + + let workspace_status = run_trail_json(temp.path(), &["status"]); + assert!(workspace_status["suggestions"] + .as_array() + .unwrap() + .iter() + .any(|suggestion| suggestion["command"] + .as_str() + .unwrap() + .contains("trail transcript acp-test"))); +} + +#[cfg(unix)] +#[test] +fn acp_relay_remaps_workspace_file_resources_into_lane() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let stub_agent = temp.path().join("resource-acp-agent.sh"); + let forwarded_prompt_log = temp.path().join(".trail/forwarded-resource-prompt.jsonl"); + let lane_workdir = temp + .path() + .canonicalize() + .unwrap() + .join(".trail/worktrees/acp-resource-test"); + fs::write( + &stub_agent, + format!( + r#"#!/bin/sh +set -eu +IFS= read -r init +printf '%s\n' '{{"jsonrpc":"2.0","id":0,"result":{{"protocolVersion":1,"agentCapabilities":{{}}}}}}' +IFS= read -r session_new +printf '%s\n' '{{"jsonrpc":"2.0","id":1,"result":{{"sessionId":"sess_resource_stub"}}}}' +IFS= read -r prompt +printf '%s\n' "$prompt" > "{}" +resource_uri=${{prompt#*\"uri\":\"}} +resource_uri=${{resource_uri%%\"*}} +case "$resource_uri" in + file://*) resource_path=${{resource_uri#file://}} ;; + *) exit 41 ;; +esac +printf '%s\n' 'changed through forwarded resource' > "$resource_path" +printf '%s\n' '{{"jsonrpc":"2.0","id":2,"result":{{"stopReason":"end_turn"}}}}' +"#, + forwarded_prompt_log.display(), + ), + ) + .unwrap(); + let mut permissions = fs::metadata(&stub_agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&stub_agent, permissions).unwrap(); + + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("acp") + .arg("relay") + .arg("--lane") + .arg("acp-resource-test") + .arg("--materialize") + .arg("--provider") + .arg("test-stub") + .arg("--") + .arg(&stub_agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + + let mut stdin = child.stdin.take().unwrap(); + let stdout = child.stdout.take().unwrap(); + let mut stdout = std::io::BufReader::new(stdout); + writeln!( + stdin, + r#"{{"jsonrpc":"2.0","id":0,"method":"initialize","params":{{"protocolVersion":1}}}}"# + ) + .unwrap(); + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + writeln!( + stdin, + r#"{{"jsonrpc":"2.0","id":1,"method":"session/new","params":{{"cwd":"{}","mcpServers":[]}}}}"#, + temp.path().display() + ) + .unwrap(); + line.clear(); + stdout.read_line(&mut line).unwrap(); + writeln!( + stdin, + r#"{{"jsonrpc":"2.0","id":2,"method":"session/prompt","params":{{"sessionId":"sess_resource_stub","prompt":[{{"type":"resource","resource":{{"uri":"file://{}/README.md","text":"hello"}}}},{{"type":"resource_link","name":"source","uri":"file://{}/src/lib.rs"}},{{"type":"resource_link","name":"external","uri":"file:///outside/shared.md"}},{{"type":"resource_link","name":"remote","uri":"https://example.com/context"}}]}}}}"#, + temp.path().display(), + temp.path().display(), + ) + .unwrap(); + line.clear(); + stdout.read_line(&mut line).unwrap(); + assert!(line.contains(r#""id":2"#), "unexpected relay frame: {line}"); + drop(stdin); + + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "relay failed\nstderr:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + + let forwarded_prompt: serde_json::Value = + serde_json::from_slice(&fs::read(&forwarded_prompt_log).unwrap()).unwrap(); + assert_eq!( + forwarded_prompt["params"]["prompt"][0]["resource"]["uri"], + format!("file://{}/README.md", lane_workdir.display()) + ); + assert_eq!( + forwarded_prompt["params"]["prompt"][1]["uri"], + format!("file://{}/src/lib.rs", lane_workdir.display()) + ); + assert_eq!( + forwarded_prompt["params"]["prompt"][2]["uri"], + "file:///outside/shared.md" + ); + assert_eq!( + forwarded_prompt["params"]["prompt"][3]["uri"], + "https://example.com/context" + ); + assert_eq!( + fs::read_to_string(temp.path().join("README.md")).unwrap(), + "hello\n", + "the ACP agent must not edit the main workspace" + ); + assert_eq!( + fs::read_to_string(lane_workdir.join("README.md")).unwrap(), + "changed through forwarded resource\n" + ); + + let db = Trail::open(temp.path()).unwrap(); + let status = db.lane_status("acp-resource-test").unwrap(); + assert!(status + .changed_paths + .iter() + .any(|path| path.path == "README.md")); + let mapping = db + .try_lane_acp_session("sess_resource_stub") + .unwrap() + .unwrap(); + let session = db.show_lane_session(&mapping.trail_session_id).unwrap(); + let turn = db.show_lane_turn(&session.turns[0].turn_id).unwrap(); + assert!(turn + .operations + .iter() + .any(|operation| operation.kind == OperationKind::LaneRecord)); +} - let acp_sessions = run_trail_json(temp.path(), &["acp", "sessions", "--lane", "acp-test"]); - assert_eq!(acp_sessions["sessions"][0]["acp_session_id"], "sess_stub"); +#[cfg(target_os = "macos")] +#[test] +fn acp_relay_blocks_materialized_agent_writes_to_main_workspace() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); - let transcript = run_trail_json(temp.path(), &["transcript", "acp-test"]); - assert_eq!(transcript["resolved_kind"], "lane"); - assert_eq!(transcript["acp_session"]["acp_session_id"], "sess_stub"); - assert!(transcript["turns"][0]["messages"] - .as_array() - .unwrap() - .iter() - .any(|message| message["role"] == "assistant" - && message["body"].as_str().unwrap().contains("done"))); - assert!(transcript["turns"][0]["tool_summaries"] - .as_array() - .unwrap() - .iter() - .any(|summary| summary.as_str().unwrap().contains("write README"))); + let escaped_path = temp.path().join("ESCAPED.md"); + let stub_agent = temp.path().join("escaping-acp-agent.sh"); + fs::write( + &stub_agent, + format!( + r#"#!/bin/sh +set -eu +IFS= read -r init +printf '%s\n' '{{"jsonrpc":"2.0","id":0,"result":{{"protocolVersion":1,"agentCapabilities":{{}}}}}}' +IFS= read -r session_new +printf '%s\n' '{{"jsonrpc":"2.0","id":1,"result":{{"sessionId":"sess_escape_stub"}}}}' +IFS= read -r prompt +printf '%s\n' 'escaped' > "{}" +printf '%s\n' '{{"jsonrpc":"2.0","id":2,"result":{{"stopReason":"end_turn"}}}}' +"#, + escaped_path.display(), + ), + ) + .unwrap(); + let mut permissions = fs::metadata(&stub_agent).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&stub_agent, permissions).unwrap(); - let turn_alias = run_trail_json( - temp.path(), - &["turn", "show", session.turns[0].turn_id.as_str()], - ); - assert_eq!(turn_alias["turn"]["turn_id"], session.turns[0].turn_id); + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .arg("acp") + .arg("relay") + .arg("--lane") + .arg("acp-escape-test") + .arg("--materialize") + .arg("--provider") + .arg("test-stub") + .arg("--") + .arg(&stub_agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + writeln!( + stdin, + r#"{{"jsonrpc":"2.0","id":0,"method":"initialize","params":{{"protocolVersion":1}}}}"# + ) + .unwrap(); + writeln!( + stdin, + r#"{{"jsonrpc":"2.0","id":1,"method":"session/new","params":{{"cwd":"{}","mcpServers":[]}}}}"#, + temp.path().display() + ) + .unwrap(); + writeln!( + stdin, + r#"{{"jsonrpc":"2.0","id":2,"method":"session/prompt","params":{{"sessionId":"sess_escape_stub","prompt":[{{"type":"text","text":"escape"}}]}}}}"# + ) + .unwrap(); + drop(stdin); - let workspace_status = run_trail_json(temp.path(), &["status"]); - assert!(workspace_status["suggestions"] - .as_array() - .unwrap() - .iter() - .any(|suggestion| suggestion["command"] - .as_str() - .unwrap() - .contains("trail transcript acp-test"))); + let output = child.wait_with_output().unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !output.status.success(), + "relay unexpectedly allowed an ACP agent to write the main workspace" + ); + assert!( + stderr.contains("Operation not permitted") || stderr.contains("Permission denied"), + "relay failed for an unexpected reason:\n{stderr}" + ); + assert!( + !escaped_path.exists(), + "materialized ACP agent escaped its Trail lane" + ); } #[cfg(unix)] @@ -3990,14 +5870,13 @@ fn acp_relay_closes_failed_turn_on_malformed_upstream_json() { .unwrap(); let session = db.show_lane_session(&mapping.trail_session_id).unwrap(); assert_eq!(session.turns[0].status, "failed"); - assert!(session - .events - .iter() - .any(|event| event.event_type == "acp_relay_turn_closed" + assert!(session.events.iter().any(|event| { + event.event_type == "acp_relay_turn_closed" && event .payload .as_ref() - .is_some_and(|payload| payload.to_string().contains("malformed JSON")))); + .is_some_and(|payload| payload.to_string().contains("malformed JSON")) + })); } #[cfg(unix)] @@ -4363,14 +6242,20 @@ fn run_stub_acp_relay_scenario_with_session_id( #[cfg(unix)] #[test] -fn acp_relay_runs_two_concurrent_relays_on_distinct_lanes() { +fn acp_relay_runs_three_synchronized_relays_on_distinct_lanes() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(3)); let workspace_a = temp.path().to_path_buf(); + let barrier_a = std::sync::Arc::clone(&barrier); let workspace_b = temp.path().to_path_buf(); + let barrier_b = std::sync::Arc::clone(&barrier); + let workspace_c = temp.path().to_path_buf(); + let barrier_c = std::sync::Arc::clone(&barrier); let relay_a = thread::spawn(move || { + barrier_a.wait(); run_stub_acp_relay_scenario_with_session_id( &workspace_a, "acp-parallel-a", @@ -4380,6 +6265,7 @@ fn acp_relay_runs_two_concurrent_relays_on_distinct_lanes() { ) }); let relay_b = thread::spawn(move || { + barrier_b.wait(); run_stub_acp_relay_scenario_with_session_id( &workspace_b, "acp-parallel-b", @@ -4388,9 +6274,20 @@ fn acp_relay_runs_two_concurrent_relays_on_distinct_lanes() { true, ) }); + let relay_c = thread::spawn(move || { + barrier_c.wait(); + run_stub_acp_relay_scenario_with_session_id( + &workspace_c, + "acp-parallel-c", + "sess_parallel_c", + &["--sleep-before-result-ms", "100"], + true, + ) + }); let output_a = relay_a.join().unwrap(); let output_b = relay_b.join().unwrap(); + let output_c = relay_c.join().unwrap(); assert!( output_a.status.success(), "relay a failed\nstdout:\n{}\nstderr:\n{}", @@ -4403,11 +6300,18 @@ fn acp_relay_runs_two_concurrent_relays_on_distinct_lanes() { String::from_utf8_lossy(&output_b.stdout), String::from_utf8_lossy(&output_b.stderr) ); + assert!( + output_c.status.success(), + "relay c failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output_c.stdout), + String::from_utf8_lossy(&output_c.stderr) + ); let db = Trail::open(temp.path()).unwrap(); for (lane, session_id) in [ ("acp-parallel-a", "sess_parallel_a"), ("acp-parallel-b", "sess_parallel_b"), + ("acp-parallel-c", "sess_parallel_c"), ] { let mapping = db.try_lane_acp_session(session_id).unwrap().unwrap(); let lane_details = db.lane_details(lane).unwrap(); @@ -4424,6 +6328,115 @@ fn acp_relay_runs_two_concurrent_relays_on_distinct_lanes() { } } +#[cfg(unix)] +#[test] +fn acp_relay_drains_delayed_terminal_frames_and_spill_before_finalizing() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let lane = "acp-finalization-order"; + let session_id = "sess_finalization_order"; + let lane_workdir = temp + .path() + .canonicalize() + .unwrap() + .join(format!(".trail/worktrees/{lane}")); + let mut options = StubAcpAgentOptions::new(session_id); + options.lane_workdir = Some(&lane_workdir); + options.sleep_before_result_ms = Some(100); + let stub_agent = write_stub_acp_agent(temp.path(), "finalization-order-agent.sh", options); + + let mut child = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args([ + "acp", + "relay", + "--lane", + lane, + "--materialize", + "--provider", + "test-stub", + "--", + ]) + .arg(stub_agent) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = std::io::BufReader::new(child.stdout.take().unwrap()); + writeln!( + stdin, + r#"{{"jsonrpc":"2.0","id":0,"method":"initialize","params":{{"protocolVersion":1}}}}"# + ) + .unwrap(); + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + assert!(line.contains(r#""id":0"#)); + writeln!( + stdin, + r#"{{"jsonrpc":"2.0","id":1,"method":"session/new","params":{{"cwd":"{}","mcpServers":[]}}}}"#, + temp.path().display() + ) + .unwrap(); + line.clear(); + stdout.read_line(&mut line).unwrap(); + assert!(line.contains(session_id)); + + let lock_path = temp.path().join(".trail/lock"); + fs::write( + &lock_path, + format!("pid={} created_at=0", std::process::id()), + ) + .unwrap(); + let lock_remover = { + let lock_path = lock_path.clone(); + thread::spawn(move || { + thread::sleep(Duration::from_millis(350)); + match fs::remove_file(lock_path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => panic!("failed to release capture writer lock: {error}"), + } + }) + }; + writeln!( + stdin, + r#"{{"jsonrpc":"2.0","id":2,"method":"session/prompt","params":{{"sessionId":"{session_id}","prompt":[{{"type":"text","text":"finish after drain"}}]}}}}"# + ) + .unwrap(); + drop(stdin); + + let mut remaining_stdout = String::new(); + stdout.read_to_string(&mut remaining_stdout).unwrap(); + let mut stderr = String::new(); + child + .stderr + .take() + .unwrap() + .read_to_string(&mut stderr) + .unwrap(); + let status = child.wait().unwrap(); + lock_remover.join().unwrap(); + assert!( + status.success(), + "relay failed\nstdout:\n{remaining_stdout}\nstderr:\n{stderr}" + ); + assert!(remaining_stdout.contains(r#""id":2"#)); + + let db = Trail::open(temp.path()).unwrap(); + let mapping = db.try_lane_acp_session(session_id).unwrap().unwrap(); + let session = db.show_lane_session(&mapping.trail_session_id).unwrap(); + assert_eq!(session.turns.len(), 1); + assert_eq!(session.turns[0].status, "completed"); + assert!(session.messages.iter().any(|message| { + message.role == "assistant" && message.body.contains("diagnostic complete") + })); +} + #[cfg(unix)] #[test] fn acp_relay_waits_for_transient_workspace_writer_lock() { @@ -4492,7 +6505,11 @@ printf '%s\n' '{"jsonrpc":"2.0","id":2,"result":{"stopReason":"end_turn"}}' let lock_path = lock_path.clone(); thread::spawn(move || { thread::sleep(Duration::from_millis(100)); - fs::remove_file(lock_path).unwrap(); + match fs::remove_file(lock_path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => panic!("failed to release transient writer lock: {error}"), + } }) }; @@ -4584,19 +6601,119 @@ fn backup_create_verify_and_restore_roundtrip() { false, ) .unwrap(); - db.spawn_lane("backup-bot", Some("main"), true, None, None) + let backup_parent = tempfile::tempdir().unwrap(); + let backup_path = backup_parent.path().join("trail-backup"); + let external_workdirs = tempfile::tempdir().unwrap(); + let external_workdir = external_workdirs.path().join("backup-bot"); + let spawned = db + .spawn_lane_with_workdir( + "backup-bot", + Some("main"), + true, + None, + None, + Some(external_workdir.clone()), + ) + .unwrap(); + let lane_head = db.lane_details("backup-bot").unwrap().branch; + let manifest_path = PathBuf::from(spawned.workdir.unwrap()) + .join(".trail") + .join("workdir-manifest.json"); + let mut legacy_manifest: serde_json::Value = + serde_json::from_slice(&fs::read(&manifest_path).unwrap()).unwrap(); + legacy_manifest["root_id"] = serde_json::Value::String("object_backup_pending_old".into()); + let legacy_manifest_bytes = serde_json::to_vec_pretty(&legacy_manifest).unwrap(); + fs::write(&manifest_path, &legacy_manifest_bytes).unwrap(); + let source_view_dir = temp.path().join(".trail/views/source-backup-view"); + let source_view_meta = source_view_dir.join("meta"); + fs::create_dir_all(&source_view_meta).unwrap(); + let source_checkpoint_path = source_view_meta.join("clean-checkpoint.json"); + let source_checkpoint_bytes = serde_json::to_vec_pretty(&serde_json::json!({ + "view_id": "source-backup-view", + "operation": lane_head.head_change.0, + "root_id": "object_backup_checkpoint_old", + "journal_sequence": 0, + })) + .unwrap(); + fs::write(&source_checkpoint_path, &source_checkpoint_bytes).unwrap(); + let pending_conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); + pending_conn + .execute( + "INSERT INTO workspace_views \ + (view_id, lane_id, base_change, base_root, backend, mountpoint, \ + source_upper, generated_upper, scratch_upper, meta_dir, journal_path, \ + generation, checkpoint_seq, checkpoint_root, status, created_at, updated_at) \ + VALUES ('source-backup-view', ?1, ?2, ?3, 'test-cow', ?4, ?5, ?6, ?7, ?8, ?9, \ + 1, 0, ?3, 'unmounted', 1, 1)", + rusqlite::params![ + lane_head.lane_id, + lane_head.head_change.0, + lane_head.head_root.0, + external_workdir.to_string_lossy(), + source_view_dir.join("source-upper").to_string_lossy(), + source_view_dir.join("generated-upper").to_string_lossy(), + source_view_dir.join("scratch-upper").to_string_lossy(), + source_view_meta.to_string_lossy(), + source_view_meta + .join("mutation-journal.jsonl") + .to_string_lossy(), + ], + ) + .unwrap(); + pending_conn + .execute( + "INSERT INTO pending_path_index_derived_repairs \ + (ref_name, repair_kind, old_root, new_root, new_change, created_at) \ + VALUES (?1, 'lane_manifest', ?2, ?3, ?4, 1)", + rusqlite::params![ + lane_head.ref_name, + "object_backup_pending_old", + lane_head.head_root.0, + lane_head.head_change.0, + ], + ) .unwrap(); + pending_conn + .execute( + "INSERT INTO pending_path_index_derived_repairs \ + (ref_name, repair_kind, old_root, new_root, new_change, created_at) \ + VALUES (?1, 'workspace_checkpoint', ?2, ?3, ?4, 1)", + rusqlite::params![ + lane_head.ref_name, + "object_backup_checkpoint_old", + lane_head.head_root.0, + lane_head.head_change.0, + ], + ) + .unwrap(); + drop(pending_conn); + let created = db.create_backup(&backup_path, false).unwrap(); + assert_eq!(created.branch, "main"); + assert!(created.sqlite_bytes > 0); + let backed_up_workdir = backup_path.join("worktrees/backup-bot"); + fs::create_dir_all(backed_up_workdir.join(".trail")).unwrap(); + fs::copy( + external_workdir.join("README.md"), + backed_up_workdir.join("README.md"), + ) + .unwrap(); + fs::copy( + &manifest_path, + backed_up_workdir.join(".trail/workdir-manifest.json"), + ) + .unwrap(); drop(db); - let backup_parent = tempfile::tempdir().unwrap(); - let backup_path = backup_parent.path().join("trail-backup"); - let created = run_trail_json( - temp.path(), - &["backup", "create", backup_path.to_str().unwrap()], - ); - assert_eq!(created["branch"], "main"); - assert!(created["sqlite_bytes"].as_u64().unwrap() > 0); - assert!(created["worktree_bytes"].as_u64().unwrap() > 0); + let backup_conn = Connection::open(backup_path.join("index/trail.sqlite")).unwrap(); + let backed_up_pending: i64 = backup_conn + .query_row( + "SELECT COUNT(*) FROM pending_path_index_derived_repairs", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(backed_up_pending, 2); + drop(backup_conn); let verified = run_trail_json( temp.path(), @@ -4620,16 +6737,43 @@ fn backup_create_verify_and_restore_roundtrip() { assert_eq!(restored_report["rewritten_workdirs"], 1); let restored_db = Trail::open(restored.path()).unwrap(); + let restored_conn = + Connection::open(restored.path().join(".trail/index/trail.sqlite")).unwrap(); + assert_eq!( + restored_conn + .query_row( + "SELECT COUNT(*) FROM pending_path_index_derived_repairs", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + drop(restored_conn); + assert_eq!(fs::read(&manifest_path).unwrap(), legacy_manifest_bytes); + assert_eq!( + fs::read(&source_checkpoint_path).unwrap(), + source_checkpoint_bytes + ); let why = restored_db.why("README.md:2", Some("main")).unwrap(); assert_eq!(why.current_text, "backup"); let fsck = restored_db.fsck().unwrap(); assert!(fsck.errors.is_empty(), "{:?}", fsck.errors); let lane = restored_db.lane_details("backup-bot").unwrap(); + assert!(restored_db + .lane_workspace_view("backup-bot") + .unwrap() + .is_none()); let workdir = lane.branch.workdir.as_ref().unwrap(); let restored_db_dir = restored.path().canonicalize().unwrap().join(".trail"); assert!(workdir.starts_with(&restored_db_dir.to_string_lossy().to_string())); assert!(PathBuf::from(workdir).is_dir()); + let restored_manifest: serde_json::Value = serde_json::from_slice( + &fs::read(PathBuf::from(workdir).join(".trail/workdir-manifest.json")).unwrap(), + ) + .unwrap(); + assert_eq!(restored_manifest["root_id"], lane_head.head_root.0); let status = restored_db.lane_status("backup-bot").unwrap(); assert_eq!(status.workdir_state, Some(WorktreeState::Clean)); } @@ -5266,7 +7410,7 @@ fn lane_patch_requires_base_change_unless_allow_stale() { assert!(err.to_string().contains("allow_stale=true")); let stale_base: PatchDocument = serde_json::from_value(serde_json::json!({ - "base_change": "ch_stale", + "base_change": "change_stale", "edits": [ {"op": "write", "path": "README.md", "content": "stale base\n"} ] @@ -5334,6 +7478,13 @@ fn lane_patch_rejects_hardened_paths_and_quota_violations() { .branch .head_change .clone(); + let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); + let objects_before: i64 = conn + .query_row("SELECT COUNT(*) FROM objects", [], |row| row.get(0)) + .unwrap(); + let prolly_nodes_before: i64 = conn + .query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row.get(0)) + .unwrap(); let patch: PatchDocument = serde_json::from_value(serde_json::json!({ "edits": [ {"op": "write", "path": colliding_path, "content": "case collision\n"} @@ -5349,6 +7500,18 @@ fn lane_patch_rejects_hardened_paths_and_quota_violations() { db.lane_details("policy-bot").unwrap().branch.head_change, before ); + assert_eq!( + conn.query_row("SELECT COUNT(*) FROM objects", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + objects_before + ); + assert_eq!( + conn.query_row("SELECT COUNT(*) FROM prolly_nodes", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + prolly_nodes_before + ); } db.config_set("lane.max_patch_file_bytes", "4").unwrap(); @@ -5388,6 +7551,38 @@ fn lane_patch_rejects_hardened_paths_and_quota_violations() { assert!(matches!(err, Error::PatchRejected(message) if message.contains("max_patch_bytes"))); } +#[test] +fn lane_patch_allows_case_only_rename_when_final_root_is_safe() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("case-rename-bot", Some("main"), false, None, None) + .unwrap(); + let patch: PatchDocument = serde_json::from_value(serde_json::json!({ + "edits": [ + {"op": "rename", "from": "README.md", "to": "readme.md"} + ] + })) + .unwrap(); + + let report = apply_lane_patch_at_head(&mut db, "case-rename-bot", patch).unwrap(); + assert_eq!(report.changed_paths.len(), 1); + assert_eq!(report.changed_paths[0].kind, trail::FileChangeKind::Renamed); + assert_eq!( + report.changed_paths[0].old_path.as_deref(), + Some("README.md") + ); + assert_eq!(report.changed_paths[0].path, "readme.md"); + assert_eq!( + db.why("readme.md:1", Some("refs/lanes/case-rename-bot")) + .unwrap() + .current_text, + "hello" + ); +} + #[test] fn local_api_and_mcp_expose_ignore_controls() { let temp = tempfile::tempdir().unwrap(); @@ -6048,6 +8243,9 @@ fn local_api_and_mcp_manage_human_approval_gates() { ] { assert!(tool_list.iter().any(|tool| tool["name"] == name), "{name}"); } + assert!(!tool_list + .iter() + .any(|tool| tool["name"] == "trail.merge_queue_add")); let mcp_run_show = trail::mcp::handle_json_rpc( &mut db, @@ -7160,1762 +9358,2640 @@ fn local_lane_http_api_records_turn_messages_and_patches() { } #[test] -fn mutation_json_payloads_reject_unknown_fields() { +fn mutation_json_payloads_reject_unknown_fields() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + + let bad_turn = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/lane/turns", + serde_json::json!({ + "lane": "strict-api", + "branch": "main", + "surprise": true + }), + ), + ); + assert_eq!(bad_turn.status, 400); + let bad_turn_body: serde_json::Value = bad_turn.body_json().unwrap(); + assert!(bad_turn_body["error"]["message"] + .as_str() + .unwrap() + .contains("unknown field")); + + let turn_response = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/lane/turns", + serde_json::json!({ + "lane": "strict-api", + "branch": "main" + }), + ), + ); + assert_eq!(turn_response.status, 201); + let turn: LaneTurnStartReport = turn_response.body_json().unwrap(); + + let bad_message = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + &format!("/v1/lane/turns/{}/messages", turn.turn.turn_id), + serde_json::json!({ + "role": "user", + "content": "hello", + "surprise": true + }), + ), + ); + assert_eq!(bad_message.status, 400); + let bad_message_body: serde_json::Value = bad_message.body_json().unwrap(); + assert!(bad_message_body["error"]["message"] + .as_str() + .unwrap() + .contains("unknown field")); + + let bad_patch = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + &format!("/v1/lane/turns/{}/patches", turn.turn.turn_id), + serde_json::json!({ + "message": "bad patch", + "files": [ + { + "type": "add_text", + "path": "src/strict.rs", + "content": "pub fn strict() -> bool { true }\n", + "surprise": true + } + ] + }), + ), + ); + assert_eq!(bad_patch.status, 400); + let bad_patch_body: serde_json::Value = bad_patch.body_json().unwrap(); + assert!(bad_patch_body["error"]["message"] + .as_str() + .unwrap() + .contains("unknown field")); + + let direct_patch_error = serde_json::from_value::(serde_json::json!({ + "message": "bad direct patch", + "edits": [ + { + "op": "write", + "path": "src/direct.rs", + "content": "pub fn direct() -> bool { true }\n", + "surprise": true + } + ] + })) + .unwrap_err(); + assert!(direct_patch_error.to_string().contains("unknown field")); + + let mcp_bad_begin = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "trail.begin_turn", + "arguments": { + "lane": "strict-mcp", + "branch": "main", + "surprise": true + } + } + }), + ) + .unwrap(); + assert_eq!(mcp_bad_begin["result"]["isError"], true); + assert!(mcp_bad_begin["result"]["content"][0]["text"] + .as_str() + .unwrap() + .contains("unknown field")); + + let mcp_bad_status = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 11, + "method": "tools/call", + "params": { + "name": "trail.status", + "arguments": { + "branch": "main", + "surprise": true + } + } + }), + ) + .unwrap(); + assert_eq!(mcp_bad_status["result"]["isError"], true); + assert!(mcp_bad_status["result"]["content"][0]["text"] + .as_str() + .unwrap() + .contains("unknown field")); + + let mcp_bad_event_list = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 12, + "method": "tools/call", + "params": { + "name": "trail.event_list", + "arguments": { + "lane": "strict-mcp", + "limit": 10, + "surprise": true + } + } + }), + ) + .unwrap(); + assert_eq!(mcp_bad_event_list["result"]["isError"], true); + assert!(mcp_bad_event_list["result"]["content"][0]["text"] + .as_str() + .unwrap() + .contains("unknown field")); + + let mcp_bad_conflict_show = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 13, + "method": "tools/call", + "params": { + "name": "trail.conflict_show", + "arguments": { + "conflict_set_id": "conflict_missing", + "limit": 10, + "surprise": true + } + } + }), + ) + .unwrap(); + assert_eq!(mcp_bad_conflict_show["result"]["isError"], true); + assert!(mcp_bad_conflict_show["result"]["content"][0]["text"] + .as_str() + .unwrap() + .contains("unknown field")); + + let mcp_begin = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "trail.begin_turn", + "arguments": { + "lane": "strict-mcp", + "branch": "main" + } + } + }), + ) + .unwrap(); + assert_eq!(mcp_begin["result"]["isError"], false); + let turn_id = mcp_begin["result"]["structuredContent"]["turn"]["turn_id"] + .as_str() + .unwrap() + .to_string(); + + let mcp_bad_patch = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": "trail.apply_patch", + "arguments": { + "turn_id": turn_id, + "message": "bad mcp patch", + "files": [ + { + "type": "add_text", + "path": "src/mcp_strict.rs", + "content": "pub fn mcp_strict() -> bool { true }\n", + "surprise": true + } + ] + } + } + }), + ) + .unwrap(); + assert_eq!(mcp_bad_patch["result"]["isError"], true); + assert!(mcp_bad_patch["result"]["content"][0]["text"] + .as_str() + .unwrap() + .contains("unknown field")); +} + +#[test] +fn external_patch_payloads_accept_explicit_empty_and_reject_missing_or_ambiguous_sources() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(temp.path()).unwrap(); - - let bad_turn = trail::server::handle_http_request( + let http_turn = trail::server::handle_http_request( &mut db, &api_request( "POST", "/v1/lane/turns", serde_json::json!({ - "lane": "strict-api", - "branch": "main", - "surprise": true + "lane": "strict-http-patch", + "branch": "main" }), ), ); - assert_eq!(bad_turn.status, 400); - let bad_turn_body: serde_json::Value = bad_turn.body_json().unwrap(); - assert!(bad_turn_body["error"]["message"] + assert_eq!(http_turn.status, 201); + let http_turn: LaneTurnStartReport = http_turn.body_json().unwrap(); + + let http_missing_patch = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + &format!("/v1/lane/turns/{}/patches", http_turn.turn.turn_id), + serde_json::json!({ + "message": "missing patch source" + }), + ), + ); + assert_eq!(http_missing_patch.status, 400); + let body: serde_json::Value = http_missing_patch.body_json().unwrap(); + assert!(body["error"]["message"] .as_str() .unwrap() - .contains("unknown field")); + .contains("requires exactly one explicit edit source")); - let turn_response = trail::server::handle_http_request( + let http_empty_patch = trail::server::handle_http_request( &mut db, &api_request( "POST", - "/v1/lane/turns", + &format!("/v1/lane/turns/{}/patches", http_turn.turn.turn_id), serde_json::json!({ - "lane": "strict-api", - "branch": "main" + "message": "empty patch", + "edits": [] }), ), ); - assert_eq!(turn_response.status, 201); - let turn: LaneTurnStartReport = turn_response.body_json().unwrap(); + assert_eq!(http_empty_patch.status, 200); + let http_empty_report: LanePatchReport = http_empty_patch.body_json().unwrap(); + assert!(http_empty_report.changed_paths.is_empty()); - let bad_message = trail::server::handle_http_request( + let http_ambiguous_patch = trail::server::handle_http_request( &mut db, &api_request( "POST", - &format!("/v1/lane/turns/{}/messages", turn.turn.turn_id), + &format!("/v1/lane/turns/{}/patches", http_turn.turn.turn_id), serde_json::json!({ - "role": "user", - "content": "hello", - "surprise": true + "message": "ambiguous patch", + "edits": [ + {"op": "delete", "path": "old-http.md"} + ], + "files": [ + {"type": "delete", "path": "new-http.md"} + ] }), ), ); - assert_eq!(bad_message.status, 400); - let bad_message_body: serde_json::Value = bad_message.body_json().unwrap(); - assert!(bad_message_body["error"]["message"] + assert_eq!(http_ambiguous_patch.status, 400); + let body: serde_json::Value = http_ambiguous_patch.body_json().unwrap(); + assert!(body["error"]["message"] .as_str() .unwrap() - .contains("unknown field")); + .contains("must use either `edits` or `files`")); - let bad_patch = trail::server::handle_http_request( + let http_missing_base_patch = trail::server::handle_http_request( &mut db, &api_request( "POST", - &format!("/v1/lane/turns/{}/patches", turn.turn.turn_id), + "/v1/lanes/strict-http-patch/patches", serde_json::json!({ - "message": "bad patch", + "message": "direct patch without fresh base", "files": [ { "type": "add_text", - "path": "src/strict.rs", - "content": "pub fn strict() -> bool { true }\n", - "surprise": true + "path": "src/http_direct.rs", + "content": "pub fn http_direct() -> bool { true }\n" } ] }), ), ); - assert_eq!(bad_patch.status, 400); - let bad_patch_body: serde_json::Value = bad_patch.body_json().unwrap(); - assert!(bad_patch_body["error"]["message"] + assert_eq!(http_missing_base_patch.status, 409); + let body: serde_json::Value = http_missing_base_patch.body_json().unwrap(); + assert!(body["error"]["message"] .as_str() .unwrap() - .contains("unknown field")); + .contains("base_change")); - let direct_patch_error = serde_json::from_value::(serde_json::json!({ - "message": "bad direct patch", - "edits": [ - { - "op": "write", - "path": "src/direct.rs", - "content": "pub fn direct() -> bool { true }\n", - "surprise": true + let mcp_begin = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "trail.begin_turn", + "arguments": { + "lane": "strict-mcp-patch", + "branch": "main" + } } - ] - })) - .unwrap_err(); - assert!(direct_patch_error.to_string().contains("unknown field")); + }), + ) + .unwrap(); + assert_eq!(mcp_begin["result"]["isError"], false); + let mcp_lane_id = mcp_begin["result"]["structuredContent"]["turn"]["lane_id"] + .as_str() + .unwrap() + .to_string(); + let mcp_turn_id = mcp_begin["result"]["structuredContent"]["turn"]["turn_id"] + .as_str() + .unwrap() + .to_string(); + + let mcp_missing_patch = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "trail.apply_patch", + "arguments": { + "turn_id": mcp_turn_id.clone(), + "message": "missing patch source" + } + } + }), + ) + .unwrap(); + assert_eq!(mcp_missing_patch["result"]["isError"], true); + assert!(mcp_missing_patch["result"]["content"][0]["text"] + .as_str() + .unwrap() + .contains("requires exactly one explicit edit source")); + + let mcp_empty_patch = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 20, + "method": "tools/call", + "params": { + "name": "trail.apply_patch", + "arguments": { + "turn_id": mcp_turn_id.clone(), + "message": "empty patch", + "edits": [] + } + } + }), + ) + .unwrap(); + assert_eq!(mcp_empty_patch["result"]["isError"], false); + assert_eq!( + mcp_empty_patch["result"]["structuredContent"]["changed_paths"], + serde_json::json!([]) + ); + + let mcp_ambiguous_patch = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": "trail.apply_patch", + "arguments": { + "turn_id": mcp_turn_id, + "message": "ambiguous patch", + "edits": [ + {"op": "delete", "path": "old-mcp.md"} + ], + "files": [ + {"type": "delete", "path": "new-mcp.md"} + ] + } + } + }), + ) + .unwrap(); + assert_eq!(mcp_ambiguous_patch["result"]["isError"], true); + assert!(mcp_ambiguous_patch["result"]["content"][0]["text"] + .as_str() + .unwrap() + .contains("must use either `edits` or `files`")); + + let audits = db.list_external_mutation_audit(20).unwrap(); + assert!(audits.iter().any(|audit| { + audit.surface == "http" + && audit.command.contains("/patches") + && audit.status == "error" + && audit.status_code == Some(400) + && audit.lane_id.as_deref() == Some(http_turn.turn.lane_id.as_str()) + && audit.target_ref.as_deref() == Some("refs/lanes/strict-http-patch") + && audit + .summary + .as_ref() + .and_then(|summary| summary["error"].as_str()) + .is_some_and(|message| { + message.contains("requires exactly one explicit edit source") + }) + })); + assert!(audits.iter().any(|audit| { + audit.surface == "http" + && audit.command == "POST /v1/lanes/strict-http-patch/patches" + && audit.status == "error" + && audit.status_code == Some(409) + && audit.lane_id.as_deref() == Some(http_turn.turn.lane_id.as_str()) + && audit.target_ref.as_deref() == Some("refs/lanes/strict-http-patch") + && audit + .summary + .as_ref() + .and_then(|summary| summary["error"].as_str()) + .is_some_and(|message| message.contains("base_change")) + })); + assert!(audits.iter().any(|audit| { + audit.surface == "mcp" + && audit.command == "trail.apply_patch" + && audit.status == "error" + && audit.lane_id.as_deref() == Some(mcp_lane_id.as_str()) + && audit.target_ref.as_deref() == Some("refs/lanes/strict-mcp-patch") + && audit + .summary + .as_ref() + .and_then(|summary| summary["error"].as_str()) + .is_some_and(|message| message.contains("must use either `edits` or `files`")) + })); +} + +#[test] +fn external_http_and_mcp_mutations_emit_audit_events() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + assert!(db.list_external_mutation_audit(10).unwrap().is_empty()); + + let http_turn = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/lane/turns", + serde_json::json!({ + "lane": "audit-http", + "branch": "main" + }), + ), + ); + assert_eq!(http_turn.status, 201); + let http_turn: LaneTurnStartReport = http_turn.body_json().unwrap(); + + let http_bad_turn = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/lane/turns", + serde_json::json!({ + "lane": "audit-http", + "branch": "main", + "unexpected": true + }), + ), + ); + assert_eq!(http_bad_turn.status, 400); - let mcp_bad_begin = trail::mcp::handle_json_rpc( + let mcp_read_only = trail::mcp::handle_json_rpc( &mut db, serde_json::json!({ "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": { - "name": "trail.begin_turn", + "name": "trail.status", + "arguments": {} + } + }), + ) + .unwrap(); + assert_eq!(mcp_read_only["result"]["isError"], false); + + let mcp_set = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "trail.config_set", "arguments": { - "lane": "strict-mcp", - "branch": "main", - "surprise": true + "key": "lane.default_materialize", + "value": "false" } } }), ) .unwrap(); - assert_eq!(mcp_bad_begin["result"]["isError"], true); - assert!(mcp_bad_begin["result"]["content"][0]["text"] - .as_str() - .unwrap() - .contains("unknown field")); + assert_eq!(mcp_set["result"]["isError"], false); - let mcp_bad_status = trail::mcp::handle_json_rpc( + let mcp_bad_set = trail::mcp::handle_json_rpc( &mut db, serde_json::json!({ "jsonrpc": "2.0", - "id": 11, + "id": 3, "method": "tools/call", "params": { - "name": "trail.status", + "name": "trail.config_set", "arguments": { - "branch": "main", - "surprise": true + "key": "lane.default_materialize", + "value": "true", + "unexpected": true } } }), ) .unwrap(); - assert_eq!(mcp_bad_status["result"]["isError"], true); - assert!(mcp_bad_status["result"]["content"][0]["text"] - .as_str() - .unwrap() - .contains("unknown field")); + assert_eq!(mcp_bad_set["result"]["isError"], true); - let mcp_bad_event_list = trail::mcp::handle_json_rpc( + let mcp_begin = trail::mcp::handle_json_rpc( &mut db, serde_json::json!({ "jsonrpc": "2.0", - "id": 12, + "id": 4, "method": "tools/call", "params": { - "name": "trail.event_list", + "name": "trail.begin_turn", "arguments": { - "lane": "strict-mcp", - "limit": 10, - "surprise": true + "lane": "audit-mcp", + "branch": "main" } } }), ) .unwrap(); - assert_eq!(mcp_bad_event_list["result"]["isError"], true); - assert!(mcp_bad_event_list["result"]["content"][0]["text"] + assert_eq!(mcp_begin["result"]["isError"], false); + let mcp_lane_id = mcp_begin["result"]["structuredContent"]["turn"]["lane_id"] .as_str() .unwrap() - .contains("unknown field")); + .to_string(); + let mcp_turn_id = mcp_begin["result"]["structuredContent"]["turn"]["turn_id"] + .as_str() + .unwrap() + .to_string(); - let mcp_bad_conflict_show = trail::mcp::handle_json_rpc( + let mcp_patch = trail::mcp::handle_json_rpc( &mut db, serde_json::json!({ "jsonrpc": "2.0", - "id": 13, + "id": 5, "method": "tools/call", "params": { - "name": "trail.conflict_show", + "name": "trail.apply_patch", "arguments": { - "conflict_set_id": "conflict_missing", - "limit": 10, - "surprise": true + "turn_id": mcp_turn_id, + "message": "mcp audit patch", + "files": [ + { + "type": "add_text", + "path": "src/mcp_audit.rs", + "content": "pub fn mcp_audit() -> bool { true }\n" + } + ] } } }), ) .unwrap(); - assert_eq!(mcp_bad_conflict_show["result"]["isError"], true); - assert!(mcp_bad_conflict_show["result"]["content"][0]["text"] - .as_str() - .unwrap() - .contains("unknown field")); + assert_eq!(mcp_patch["result"]["isError"], false); - let mcp_begin = trail::mcp::handle_json_rpc( + let mcp_bad_begin = trail::mcp::handle_json_rpc( &mut db, serde_json::json!({ "jsonrpc": "2.0", - "id": 2, + "id": 6, "method": "tools/call", "params": { "name": "trail.begin_turn", "arguments": { - "lane": "strict-mcp", - "branch": "main" + "lane": "audit-mcp", + "branch": "main", + "unexpected": true } } }), ) .unwrap(); - assert_eq!(mcp_begin["result"]["isError"], false); - let turn_id = mcp_begin["result"]["structuredContent"]["turn"]["turn_id"] - .as_str() - .unwrap() - .to_string(); + assert_eq!(mcp_bad_begin["result"]["isError"], true); - let mcp_bad_patch = trail::mcp::handle_json_rpc( + let mcp_bad_queue_add = trail::mcp::handle_json_rpc( &mut db, serde_json::json!({ "jsonrpc": "2.0", - "id": 3, + "id": 7, "method": "tools/call", "params": { - "name": "trail.apply_patch", + "name": "trail.lane_merge_queue_add", "arguments": { - "turn_id": turn_id, - "message": "bad mcp patch", - "files": [ - { - "type": "add_text", - "path": "src/mcp_strict.rs", - "content": "pub fn mcp_strict() -> bool { true }\n", - "surprise": true - } - ] + "lane": "audit-mcp", + "target": "main", + "unexpected": true } } }), ) .unwrap(); - assert_eq!(mcp_bad_patch["result"]["isError"], true); - assert!(mcp_bad_patch["result"]["content"][0]["text"] - .as_str() - .unwrap() - .contains("unknown field")); + assert_eq!(mcp_bad_queue_add["result"]["isError"], true); + + let audits = db.list_external_mutation_audit(20).unwrap(); + assert!(audits.iter().any(|audit| { + audit.actor == "http:no-auth" + && audit.surface == "http" + && audit.command == "POST /v1/lane/turns" + && audit.status == "ok" + && audit.status_code == Some(201) + && audit.target_ref.as_deref() == Some("refs/lanes/audit-http") + })); + assert!(audits.iter().any(|audit| { + audit.actor == "http:no-auth" + && audit.surface == "http" + && audit.command == "POST /v1/lane/turns" + && audit.status == "error" + && audit.status_code == Some(400) + && audit.lane_id.as_deref() == Some(http_turn.turn.lane_id.as_str()) + && audit.target_ref.as_deref() == Some("refs/lanes/audit-http") + && audit + .summary + .as_ref() + .and_then(|summary| summary["error"].as_str()) + .is_some_and(|message| message.contains("unknown field")) + })); + assert!(audits.iter().any(|audit| { + audit.actor == "mcp:stdio" + && audit.surface == "mcp" + && audit.command == "trail.config_set" + && audit.status == "ok" + })); + assert!(audits.iter().any(|audit| { + audit.actor == "mcp:stdio" + && audit.surface == "mcp" + && audit.command == "trail.config_set" + && audit.status == "error" + && audit + .summary + .as_ref() + .and_then(|summary| summary["error"].as_str()) + .is_some_and(|message| message.contains("unknown field")) + })); + assert!(audits.iter().any(|audit| { + audit.actor == "mcp:stdio" + && audit.surface == "mcp" + && audit.command == "trail.begin_turn" + && audit.status == "ok" + && audit.lane_id.as_deref() == Some(mcp_lane_id.as_str()) + && audit.target_ref.as_deref() == Some("refs/lanes/audit-mcp") + })); + assert!(audits.iter().any(|audit| { + audit.actor == "mcp:stdio" + && audit.surface == "mcp" + && audit.command == "trail.begin_turn" + && audit.status == "error" + && audit.lane_id.as_deref() == Some(mcp_lane_id.as_str()) + && audit.target_ref.as_deref() == Some("refs/lanes/audit-mcp") + && audit + .summary + .as_ref() + .and_then(|summary| summary["error"].as_str()) + .is_some_and(|message| message.contains("unknown field")) + })); + assert!(!audits.iter().any(|audit| { + audit.actor == "mcp:stdio" + && audit.command == "trail.begin_turn" + && audit.status == "error" + && audit.target_ref.as_deref() == Some("main") + })); + assert!(audits.iter().any(|audit| { + audit.actor == "mcp:stdio" + && audit.surface == "mcp" + && audit.command == "trail.lane_merge_queue_add" + && audit.status == "error" + && audit.lane_id.as_deref() == Some(mcp_lane_id.as_str()) + && audit.target_ref.as_deref() == Some("refs/heads/main") + && audit + .summary + .as_ref() + .and_then(|summary| summary["error"].as_str()) + .is_some_and(|message| message.contains("unknown field")) + })); + assert!(!audits.iter().any(|audit| { + audit.actor == "mcp:stdio" + && audit.command == "trail.lane_merge_queue_add" + && audit.status == "error" + && audit.target_ref.as_deref() == Some("main") + })); + assert!(audits.iter().any(|audit| { + audit.actor == "mcp:stdio" + && audit.surface == "mcp" + && audit.command == "trail.apply_patch" + && audit.status == "ok" + && audit.lane_id.as_deref() == Some(mcp_lane_id.as_str()) + && audit.target_ref.as_deref() == Some("refs/lanes/audit-mcp") + && audit.change_id.is_some() + && audit + .summary + .as_ref() + .and_then(|summary| summary["change_id"].as_str()) + .is_some() + })); + assert!(!audits.iter().any(|audit| audit.command == "trail.status")); } #[test] -fn external_patch_payloads_reject_empty_or_ambiguous_edit_sources() { +fn local_lane_http_api_replays_idempotent_mutation_requests() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); - let mut db = Trail::open(temp.path()).unwrap(); - let http_turn = trail::server::handle_http_request( - &mut db, - &api_request( - "POST", - "/v1/lane/turns", - serde_json::json!({ - "lane": "strict-http-patch", - "branch": "main" - }), - ), + let request_body = serde_json::json!({ + "lane": "idempotent-api", + "branch": "main", + "session_title": "First request" + }); + let request = api_request_with_headers( + "POST", + "/v1/lane/turns", + &[("Idempotency-Key", "turn-key-1")], + request_body.clone(), ); - assert_eq!(http_turn.status, 201); - let http_turn: LaneTurnStartReport = http_turn.body_json().unwrap(); - let http_empty_patch = trail::server::handle_http_request( - &mut db, - &api_request( - "POST", - &format!("/v1/lane/turns/{}/patches", http_turn.turn.turn_id), - serde_json::json!({ - "message": "empty patch" - }), - ), - ); - assert_eq!(http_empty_patch.status, 400); - let body: serde_json::Value = http_empty_patch.body_json().unwrap(); - assert!(body["error"]["message"] - .as_str() - .unwrap() - .contains("requires at least one edit")); + let mut db = Trail::open(temp.path()).unwrap(); + let first = trail::server::handle_http_request(&mut db, &request); + assert_eq!(first.status, 201); + let first_turn: LaneTurnStartReport = first.body_json().unwrap(); + drop(db); - let http_ambiguous_patch = trail::server::handle_http_request( - &mut db, - &api_request( - "POST", - &format!("/v1/lane/turns/{}/patches", http_turn.turn.turn_id), - serde_json::json!({ - "message": "ambiguous patch", - "edits": [ - {"op": "delete", "path": "old-http.md"} - ], - "files": [ - {"type": "delete", "path": "new-http.md"} - ] - }), - ), + let mut reopened = Trail::open(temp.path()).unwrap(); + let replayed = trail::server::handle_http_request(&mut reopened, &request); + assert_eq!(replayed.status, 201); + let replayed_turn: LaneTurnStartReport = replayed.body_json().unwrap(); + assert_eq!(replayed_turn.turn.turn_id, first_turn.turn.turn_id); + assert_eq!( + replayed_turn.session.session_id, + first_turn.session.session_id ); - assert_eq!(http_ambiguous_patch.status, 400); - let body: serde_json::Value = http_ambiguous_patch.body_json().unwrap(); - assert!(body["error"]["message"] - .as_str() - .unwrap() - .contains("must use either `edits` or `files`")); + drop(reopened); - let http_missing_base_patch = trail::server::handle_http_request( + let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); + let turns: i64 = conn + .query_row("SELECT COUNT(*) FROM lane_turns", [], |row| row.get(0)) + .unwrap(); + assert_eq!(turns, 1); + drop(conn); + + let mut db = Trail::open(temp.path()).unwrap(); + let conflicting = trail::server::handle_http_request( &mut db, - &api_request( + &api_request_with_headers( "POST", - "/v1/lanes/strict-http-patch/patches", + "/v1/lane/turns", + &[("Idempotency-Key", "turn-key-1")], serde_json::json!({ - "message": "direct patch without fresh base", - "files": [ - { - "type": "add_text", - "path": "src/http_direct.rs", - "content": "pub fn http_direct() -> bool { true }\n" - } - ] + "lane": "idempotent-api", + "branch": "main", + "session_title": "Different request" }), ), ); - assert_eq!(http_missing_base_patch.status, 409); - let body: serde_json::Value = http_missing_base_patch.body_json().unwrap(); - assert!(body["error"]["message"] + assert_eq!(conflicting.status, 400); + let error: serde_json::Value = conflicting.body_json().unwrap(); + assert!(error["error"]["message"] .as_str() .unwrap() - .contains("base_change")); + .contains("already used for a different request")); - let mcp_begin = trail::mcp::handle_json_rpc( - &mut db, - serde_json::json!({ - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": { - "name": "trail.begin_turn", - "arguments": { - "lane": "strict-mcp-patch", - "branch": "main" - } - } - }), - ) - .unwrap(); - assert_eq!(mcp_begin["result"]["isError"], false); - let mcp_lane_id = mcp_begin["result"]["structuredContent"]["turn"]["lane_id"] - .as_str() - .unwrap() - .to_string(); - let mcp_turn_id = mcp_begin["result"]["structuredContent"]["turn"]["turn_id"] - .as_str() - .unwrap() - .to_string(); + let invalid_idempotency_body = serde_json::json!({ + "lane": "idempotent-api", + "branch": "main", + "session_title": "Invalid idempotency key" + }); + let overlong_idempotency_key = "x".repeat(201); + for key in ["", overlong_idempotency_key.as_str(), "key\twith-tab"] { + let invalid_key = trail::server::handle_http_request( + &mut db, + &api_request_with_headers( + "POST", + "/v1/lane/turns", + &[("Idempotency-Key", key)], + invalid_idempotency_body.clone(), + ), + ); + assert_eq!(invalid_key.status, 400); + let error: serde_json::Value = invalid_key.body_json().unwrap(); + assert!(error["error"]["message"] + .as_str() + .unwrap() + .contains("Idempotency-Key")); + } - let mcp_empty_patch = trail::mcp::handle_json_rpc( - &mut db, + let auth = trail::server::ServerAuth::bearer("secret-token").unwrap(); + let auth_request = api_request_with_headers( + "POST", + "/v1/lane/turns", + &[("Idempotency-Key", "auth-key-1")], serde_json::json!({ - "jsonrpc": "2.0", - "id": 2, - "method": "tools/call", - "params": { - "name": "trail.apply_patch", - "arguments": { - "turn_id": mcp_turn_id.clone(), - "message": "empty patch" - } - } + "lane": "auth-idempotent-api", + "branch": "main" }), - ) - .unwrap(); - assert_eq!(mcp_empty_patch["result"]["isError"], true); - assert!(mcp_empty_patch["result"]["content"][0]["text"] - .as_str() - .unwrap() - .contains("requires at least one edit")); - - let mcp_ambiguous_patch = trail::mcp::handle_json_rpc( - &mut db, + ); + let missing_auth = trail::server::handle_http_request_with_auth(&mut db, &auth_request, &auth); + assert_eq!(missing_auth.status, 401); + let authorized_request = api_request_with_headers( + "POST", + "/v1/lane/turns", + &[ + ("Idempotency-Key", "auth-key-1"), + ("Authorization", "Bearer secret-token"), + ], serde_json::json!({ - "jsonrpc": "2.0", - "id": 3, - "method": "tools/call", - "params": { - "name": "trail.apply_patch", - "arguments": { - "turn_id": mcp_turn_id, - "message": "ambiguous patch", - "edits": [ - {"op": "delete", "path": "old-mcp.md"} - ], - "files": [ - {"type": "delete", "path": "new-mcp.md"} - ] - } - } + "lane": "auth-idempotent-api", + "branch": "main" }), - ) - .unwrap(); - assert_eq!(mcp_ambiguous_patch["result"]["isError"], true); - assert!(mcp_ambiguous_patch["result"]["content"][0]["text"] - .as_str() - .unwrap() - .contains("must use either `edits` or `files`")); + ); + let authorized = + trail::server::handle_http_request_with_auth(&mut db, &authorized_request, &auth); + assert_eq!(authorized.status, 201); + let unauthorized_replay = + trail::server::handle_http_request_with_auth(&mut db, &auth_request, &auth); + assert_eq!(unauthorized_replay.status, 401); - let audits = db.list_external_mutation_audit(20).unwrap(); + let audits = db.list_external_mutation_audit(30).unwrap(); + let idempotent_turn_audits = audits + .iter() + .filter(|audit| { + audit.actor == "http:no-auth" + && audit.surface == "http" + && audit.command == "POST /v1/lane/turns" + && audit.status == "ok" + && audit.status_code == Some(201) + && audit.target_ref.as_deref() == Some("refs/lanes/idempotent-api") + }) + .collect::>(); + assert_eq!(idempotent_turn_audits.len(), 2); + assert_eq!( + idempotent_turn_audits + .iter() + .filter(|audit| { + audit + .summary + .as_ref() + .and_then(|summary| summary["idempotency_replay"].as_bool()) + == Some(true) + }) + .count(), + 1 + ); assert!(audits.iter().any(|audit| { - audit.surface == "http" - && audit.command.contains("/patches") + audit.actor == "http:no-auth" + && audit.surface == "http" + && audit.command == "POST /v1/lane/turns" && audit.status == "error" && audit.status_code == Some(400) - && audit.lane_id.as_deref() == Some(http_turn.turn.lane_id.as_str()) - && audit.target_ref.as_deref() == Some("refs/lanes/strict-http-patch") - && audit - .summary - .as_ref() - .and_then(|summary| summary["error"].as_str()) - .is_some_and(|message| message.contains("requires at least one edit")) - })); - assert!(audits.iter().any(|audit| { - audit.surface == "http" - && audit.command == "POST /v1/lanes/strict-http-patch/patches" - && audit.status == "error" - && audit.status_code == Some(409) - && audit.lane_id.as_deref() == Some(http_turn.turn.lane_id.as_str()) - && audit.target_ref.as_deref() == Some("refs/lanes/strict-http-patch") && audit .summary .as_ref() .and_then(|summary| summary["error"].as_str()) - .is_some_and(|message| message.contains("base_change")) + .is_some_and(|message| message.contains("already used for a different request")) })); + assert_eq!( + audits + .iter() + .filter(|audit| { + audit.actor == "http:no-auth" + && audit.surface == "http" + && audit.command == "POST /v1/lane/turns" + && audit.status == "error" + && audit.status_code == Some(400) + && audit.target_ref.as_deref() == Some("refs/lanes/idempotent-api") + && audit + .summary + .as_ref() + .and_then(|summary| summary["error"].as_str()) + .is_some_and(|message| { + message.contains("Idempotency-Key") + && !message.contains("already used for a different request") + }) + }) + .count(), + 3 + ); assert!(audits.iter().any(|audit| { - audit.surface == "mcp" - && audit.command == "trail.apply_patch" - && audit.status == "error" - && audit.lane_id.as_deref() == Some(mcp_lane_id.as_str()) - && audit.target_ref.as_deref() == Some("refs/lanes/strict-mcp-patch") - && audit - .summary - .as_ref() - .and_then(|summary| summary["error"].as_str()) - .is_some_and(|message| message.contains("must use either `edits` or `files`")) + audit.actor == "http:bearer" + && audit.surface == "http" + && audit.command == "POST /v1/lane/turns" + && audit.status == "ok" + && audit.status_code == Some(201) + && audit.target_ref.as_deref() == Some("refs/lanes/auth-idempotent-api") })); + assert_eq!( + audits + .iter() + .filter(|audit| { + audit.actor == "http:no-auth" + && audit.surface == "http" + && audit.command == "POST /v1/lane/turns" + && audit.status == "error" + && audit.status_code == Some(401) + }) + .count(), + 2 + ); } #[test] -fn external_http_and_mcp_mutations_emit_audit_events() { +fn local_api_and_mcp_patch_payloads_respect_ignore_policy() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(temp.path()).unwrap(); - assert!(db.list_external_mutation_audit(10).unwrap().is_empty()); + db.ignore_add("host-secret.txt").unwrap(); - let http_turn = trail::server::handle_http_request( + let turn_response = trail::server::handle_http_request( &mut db, &api_request( "POST", "/v1/lane/turns", serde_json::json!({ - "lane": "audit-http", - "branch": "main" + "lane": "privacy-api", + "branch": "main", + "session_title": "Privacy policy" }), ), ); - assert_eq!(http_turn.status, 201); - let http_turn: LaneTurnStartReport = http_turn.body_json().unwrap(); + assert_eq!(turn_response.status, 201); + let turn: LaneTurnStartReport = turn_response.body_json().unwrap(); - let http_bad_turn = trail::server::handle_http_request( + let blocked = trail::server::handle_http_request( &mut db, &api_request( "POST", - "/v1/lane/turns", + &format!("/v1/lane/turns/{}/patches", turn.turn.turn_id), serde_json::json!({ - "lane": "audit-http", - "branch": "main", - "unexpected": true + "message": "blocked ignored write", + "files": [ + { + "type": "add_text", + "path": "host-secret.txt", + "content": "blocked\n" + } + ] }), ), ); - assert_eq!(http_bad_turn.status, 400); + assert_eq!(blocked.status, 400); + let error: serde_json::Value = blocked.body_json().unwrap(); + assert!(error["error"]["message"] + .as_str() + .unwrap() + .contains("ignored path `host-secret.txt`")); - let mcp_read_only = trail::mcp::handle_json_rpc( + let tools = trail::mcp::handle_json_rpc( &mut db, serde_json::json!({ "jsonrpc": "2.0", "id": 1, - "method": "tools/call", - "params": { - "name": "trail.status", - "arguments": {} - } - }), - ) - .unwrap(); - assert_eq!(mcp_read_only["result"]["isError"], false); - - let mcp_set = trail::mcp::handle_json_rpc( - &mut db, - serde_json::json!({ - "jsonrpc": "2.0", - "id": 2, - "method": "tools/call", - "params": { - "name": "trail.config_set", - "arguments": { - "key": "lane.default_materialize", - "value": "false" - } - } - }), - ) - .unwrap(); - assert_eq!(mcp_set["result"]["isError"], false); - - let mcp_bad_set = trail::mcp::handle_json_rpc( - &mut db, - serde_json::json!({ - "jsonrpc": "2.0", - "id": 3, - "method": "tools/call", - "params": { - "name": "trail.config_set", - "arguments": { - "key": "lane.default_materialize", - "value": "true", - "unexpected": true - } - } - }), - ) - .unwrap(); - assert_eq!(mcp_bad_set["result"]["isError"], true); - - let mcp_begin = trail::mcp::handle_json_rpc( - &mut db, - serde_json::json!({ - "jsonrpc": "2.0", - "id": 4, - "method": "tools/call", - "params": { - "name": "trail.begin_turn", - "arguments": { - "lane": "audit-mcp", - "branch": "main" - } - } + "method": "tools/list", + "params": {} }), ) .unwrap(); - assert_eq!(mcp_begin["result"]["isError"], false); - let mcp_lane_id = mcp_begin["result"]["structuredContent"]["turn"]["lane_id"] - .as_str() + let apply_patch_schema = tools["result"]["tools"] + .as_array() .unwrap() - .to_string(); - let mcp_turn_id = mcp_begin["result"]["structuredContent"]["turn"]["turn_id"] - .as_str() + .iter() + .find(|tool| tool["name"] == "trail.apply_patch") + .unwrap(); + let apply_patch_schema = &apply_patch_schema["inputSchema"]; + assert_eq!( + apply_patch_schema["properties"]["allow_ignored"]["type"], + "boolean" + ); + assert_eq!( + apply_patch_schema["properties"]["allow_stale"]["type"], + "boolean" + ); + let edit_source_modes = apply_patch_schema["oneOf"].as_array().unwrap(); + assert_eq!(edit_source_modes.len(), 2); + assert_eq!( + edit_source_modes[0]["required"], + serde_json::json!(["edits"]) + ); + assert_eq!( + edit_source_modes[0]["not"]["required"], + serde_json::json!(["files"]) + ); + assert_eq!( + edit_source_modes[1]["required"], + serde_json::json!(["files"]) + ); + assert_eq!( + edit_source_modes[1]["not"]["required"], + serde_json::json!(["edits"]) + ); + assert!(apply_patch_schema["properties"]["edits"] + .get("minItems") + .is_none()); + assert!(apply_patch_schema["properties"]["files"] + .get("minItems") + .is_none()); + let edit_variants = apply_patch_schema["properties"]["edits"]["items"]["oneOf"] + .as_array() + .unwrap(); + assert_eq!(edit_variants.len(), 5); + for variant in edit_variants { + assert_eq!(variant["additionalProperties"], false); + } + assert!(edit_variants[2]["required"] + .as_array() .unwrap() - .to_string(); + .iter() + .any(|field| field == "expected_text")); + let file_variants = apply_patch_schema["properties"]["files"]["items"]["oneOf"] + .as_array() + .unwrap(); + assert_eq!(file_variants.len(), 5); + for variant in file_variants { + assert_eq!(variant["additionalProperties"], false); + } + let nested_line_edit = &file_variants[1]["properties"]["edits"]["items"]["oneOf"][0]; + assert_eq!(nested_line_edit["additionalProperties"], false); + assert!(nested_line_edit["required"] + .as_array() + .unwrap() + .iter() + .any(|field| field == "expected_text")); - let mcp_patch = trail::mcp::handle_json_rpc( + let allowed = trail::mcp::handle_json_rpc( &mut db, serde_json::json!({ "jsonrpc": "2.0", - "id": 5, + "id": 2, "method": "tools/call", "params": { "name": "trail.apply_patch", "arguments": { - "turn_id": mcp_turn_id, - "message": "mcp audit patch", - "files": [ - { - "type": "add_text", - "path": "src/mcp_audit.rs", - "content": "pub fn mcp_audit() -> bool { true }\n" - } - ] - } - } - }), - ) - .unwrap(); - assert_eq!(mcp_patch["result"]["isError"], false); - - let mcp_bad_begin = trail::mcp::handle_json_rpc( - &mut db, - serde_json::json!({ - "jsonrpc": "2.0", - "id": 6, - "method": "tools/call", - "params": { - "name": "trail.begin_turn", - "arguments": { - "lane": "audit-mcp", - "branch": "main", - "unexpected": true - } - } - }), - ) - .unwrap(); - assert_eq!(mcp_bad_begin["result"]["isError"], true); - - let mcp_bad_queue_add = trail::mcp::handle_json_rpc( - &mut db, - serde_json::json!({ - "jsonrpc": "2.0", - "id": 7, - "method": "tools/call", - "params": { - "name": "trail.merge_queue_add", - "arguments": { - "source": "refs/lanes/audit-mcp", - "target": "main", - "unexpected": true + "turn_id": turn.turn.turn_id, + "message": "explicit ignored write", + "allow_ignored": true, + "files": [ + { + "type": "add_text", + "path": "host-secret.txt", + "content": "allowed fixture\n" + } + ] } } }), ) .unwrap(); - assert_eq!(mcp_bad_queue_add["result"]["isError"], true); + assert_eq!(allowed["result"]["isError"], false); + assert_eq!( + allowed["result"]["structuredContent"]["changed_paths"][0]["path"], + "host-secret.txt" + ); - let audits = db.list_external_mutation_audit(20).unwrap(); - assert!(audits.iter().any(|audit| { - audit.actor == "http:no-auth" - && audit.surface == "http" - && audit.command == "POST /v1/lane/turns" - && audit.status == "ok" - && audit.status_code == Some(201) - && audit.target_ref.as_deref() == Some("refs/lanes/audit-http") - })); - assert!(audits.iter().any(|audit| { - audit.actor == "http:no-auth" - && audit.surface == "http" - && audit.command == "POST /v1/lane/turns" - && audit.status == "error" - && audit.status_code == Some(400) - && audit.lane_id.as_deref() == Some(http_turn.turn.lane_id.as_str()) - && audit.target_ref.as_deref() == Some("refs/lanes/audit-http") - && audit - .summary - .as_ref() - .and_then(|summary| summary["error"].as_str()) - .is_some_and(|message| message.contains("unknown field")) - })); - assert!(audits.iter().any(|audit| { - audit.actor == "mcp:stdio" - && audit.surface == "mcp" - && audit.command == "trail.config_set" - && audit.status == "ok" - })); - assert!(audits.iter().any(|audit| { - audit.actor == "mcp:stdio" - && audit.surface == "mcp" - && audit.command == "trail.config_set" - && audit.status == "error" - && audit - .summary - .as_ref() - .and_then(|summary| summary["error"].as_str()) - .is_some_and(|message| message.contains("unknown field")) - })); - assert!(audits.iter().any(|audit| { - audit.actor == "mcp:stdio" - && audit.surface == "mcp" - && audit.command == "trail.begin_turn" - && audit.status == "ok" - && audit.lane_id.as_deref() == Some(mcp_lane_id.as_str()) - && audit.target_ref.as_deref() == Some("refs/lanes/audit-mcp") - })); - assert!(audits.iter().any(|audit| { - audit.actor == "mcp:stdio" - && audit.surface == "mcp" - && audit.command == "trail.begin_turn" - && audit.status == "error" - && audit.lane_id.as_deref() == Some(mcp_lane_id.as_str()) - && audit.target_ref.as_deref() == Some("refs/lanes/audit-mcp") - && audit - .summary - .as_ref() - .and_then(|summary| summary["error"].as_str()) - .is_some_and(|message| message.contains("unknown field")) - })); - assert!(!audits.iter().any(|audit| { - audit.actor == "mcp:stdio" - && audit.command == "trail.begin_turn" - && audit.status == "error" - && audit.target_ref.as_deref() == Some("main") - })); - assert!(audits.iter().any(|audit| { - audit.actor == "mcp:stdio" - && audit.surface == "mcp" - && audit.command == "trail.merge_queue_add" - && audit.status == "error" - && audit.lane_id.as_deref() == Some(mcp_lane_id.as_str()) - && audit.target_ref.as_deref() == Some("refs/heads/main") - && audit - .summary - .as_ref() - .and_then(|summary| summary["error"].as_str()) - .is_some_and(|message| message.contains("unknown field")) - })); - assert!(!audits.iter().any(|audit| { - audit.actor == "mcp:stdio" - && audit.command == "trail.merge_queue_add" - && audit.status == "error" - && audit.target_ref.as_deref() == Some("main") - })); - assert!(audits.iter().any(|audit| { - audit.actor == "mcp:stdio" - && audit.surface == "mcp" - && audit.command == "trail.apply_patch" - && audit.status == "ok" - && audit.lane_id.as_deref() == Some(mcp_lane_id.as_str()) - && audit.target_ref.as_deref() == Some("refs/lanes/audit-mcp") - && audit.change_id.is_some() - && audit - .summary - .as_ref() - .and_then(|summary| summary["change_id"].as_str()) - .is_some() - })); - assert!(!audits.iter().any(|audit| audit.command == "trail.status")); + let details = db.show_lane_turn(&turn.turn.turn_id).unwrap(); + let patch_event = details + .events + .iter() + .find(|event| event.event_type == "patch_applied") + .unwrap(); + assert_eq!( + patch_event.payload.as_ref().unwrap()["allow_ignored"], + serde_json::Value::Bool(true) + ); } #[test] -fn local_lane_http_api_replays_idempotent_mutation_requests() { +fn local_lane_http_api_manages_lane_branch_lifecycle() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); - let request_body = serde_json::json!({ - "lane": "idempotent-api", - "branch": "main", - "session_title": "First request" - }); - let request = api_request_with_headers( - "POST", - "/v1/lane/turns", - &[("Idempotency-Key", "turn-key-1")], - request_body.clone(), + let mut db = Trail::open(temp.path()).unwrap(); + let status_response = trail::server::handle_http_request( + &mut db, + &api_request("GET", "/v1/status", serde_json::Value::Null), + ); + assert_eq!(status_response.status, 200); + let status: serde_json::Value = status_response.body_json().unwrap(); + assert_eq!(status["branch"], "main"); + + let spawn_response = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/lanes", + serde_json::json!({ + "name": "api-branch-lane", + "from_ref": "main", + "materialize": true + }), + ), + ); + assert_eq!(spawn_response.status, 201); + let spawned: serde_json::Value = spawn_response.body_json().unwrap(); + let lane_id = spawned["lane_id"].as_str().unwrap().to_string(); + let lane_base_change = spawned["base_change"].as_str().unwrap().to_string(); + assert_eq!(spawned["ref_name"], "refs/lanes/api-branch-lane"); + let workdir = spawned["workdir"].as_str().unwrap().to_string(); + + let lanes_response = trail::server::handle_http_request( + &mut db, + &api_request("GET", "/v1/lanes", serde_json::Value::Null), + ); + assert_eq!(lanes_response.status, 200); + let lanes: serde_json::Value = lanes_response.body_json().unwrap(); + assert_eq!(lanes.as_array().unwrap().len(), 1); + assert_eq!(lanes[0]["record"]["name"], "api-branch-lane"); + assert_eq!(lanes[0]["branch"]["ref_name"], "refs/lanes/api-branch-lane"); + + let lane_status_response = trail::server::handle_http_request( + &mut db, + &api_request( + "GET", + &format!("/v1/lanes/{lane_id}/status"), + serde_json::Value::Null, + ), + ); + assert_eq!(lane_status_response.status, 200); + let lane_status: serde_json::Value = lane_status_response.body_json().unwrap(); + assert_eq!(lane_status["lane"]["record"]["name"], "api-branch-lane"); + + let patch_response = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + &format!("/v1/lanes/{lane_id}/patches"), + serde_json::json!({ + "base_change": lane_base_change, + "message": "add API file", + "files": [ + { + "type": "add_text", + "path": "src/api.rs", + "content": "pub fn api() -> bool { true }\n" + } + ] + }), + ), + ); + assert_eq!(patch_response.status, 200); + let patch: LanePatchReport = patch_response.body_json().unwrap(); + assert_eq!(patch.lane_id, lane_id); + assert_eq!(patch.changed_paths[0].path, "src/api.rs"); + assert_eq!( + fs::read_to_string(std::path::Path::new(&workdir).join("src/api.rs")).unwrap(), + "pub fn api() -> bool { true }\n" + ); + + fs::write( + std::path::Path::new(&workdir).join("README.md"), + "hello\napi dirty\n", + ) + .unwrap(); + let record_preview_response = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + &format!("/v1/lanes/{lane_id}/record"), + serde_json::json!({ "preview": true }), + ), + ); + assert_eq!(record_preview_response.status, 200); + let record_preview: serde_json::Value = record_preview_response.body_json().unwrap(); + assert_eq!(record_preview["clean"], false); + assert!(record_preview["changed_paths"] + .as_array() + .unwrap() + .iter() + .any(|path| path["path"] == "README.md")); + assert!(record_preview["operation"].is_null()); + + let sync_conflict = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + &format!("/v1/lanes/{lane_id}/sync-workdir"), + serde_json::json!({}), + ), + ); + assert_eq!(sync_conflict.status, 409); + + let sync_response = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + &format!("/v1/lanes/{lane_id}/sync-workdir"), + serde_json::json!({ "force": true }), + ), ); - - let mut db = Trail::open(temp.path()).unwrap(); - let first = trail::server::handle_http_request(&mut db, &request); - assert_eq!(first.status, 201); - let first_turn: LaneTurnStartReport = first.body_json().unwrap(); - drop(db); - - let mut reopened = Trail::open(temp.path()).unwrap(); - let replayed = trail::server::handle_http_request(&mut reopened, &request); - assert_eq!(replayed.status, 201); - let replayed_turn: LaneTurnStartReport = replayed.body_json().unwrap(); - assert_eq!(replayed_turn.turn.turn_id, first_turn.turn.turn_id); + assert_eq!(sync_response.status, 200); + let synced: serde_json::Value = sync_response.body_json().unwrap(); + assert_eq!(synced["forced"], true); assert_eq!( - replayed_turn.session.session_id, - first_turn.session.session_id + fs::read_to_string(std::path::Path::new(&workdir).join("README.md")).unwrap(), + "hello\n" ); - drop(reopened); - - let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); - let turns: i64 = conn - .query_row("SELECT COUNT(*) FROM lane_turns", [], |row| row.get(0)) - .unwrap(); - assert_eq!(turns, 1); - drop(conn); - let mut db = Trail::open(temp.path()).unwrap(); - let conflicting = trail::server::handle_http_request( + let test_response = trail::server::handle_http_request( &mut db, - &api_request_with_headers( + &api_request( "POST", - "/v1/lane/turns", - &[("Idempotency-Key", "turn-key-1")], + &format!("/v1/lanes/{lane_id}/tests"), serde_json::json!({ - "lane": "idempotent-api", - "branch": "main", - "session_title": "Different request" + "command": ["sh", "-c", "printf api-test"], + "timeout_secs": 5 }), ), ); - assert_eq!(conflicting.status, 400); - let error: serde_json::Value = conflicting.body_json().unwrap(); - assert!(error["error"]["message"] + assert_eq!(test_response.status, 200); + let test: serde_json::Value = test_response.body_json().unwrap(); + assert_eq!(test["success"], true); + assert_eq!(test["stdout_preview"], "api-test"); + + let diff_response = trail::server::handle_http_request( + &mut db, + &api_request( + "GET", + &format!("/v1/lanes/{lane_id}/diff?patch=true"), + serde_json::Value::Null, + ), + ); + assert_eq!(diff_response.status, 200); + let diff: serde_json::Value = diff_response.body_json().unwrap(); + assert_eq!(diff["files"][0]["path"], "src/api.rs"); + assert!(diff["files"][0]["patch"] .as_str() .unwrap() - .contains("already used for a different request")); - - let invalid_idempotency_body = serde_json::json!({ - "lane": "idempotent-api", - "branch": "main", - "session_title": "Invalid idempotency key" - }); - let overlong_idempotency_key = "x".repeat(201); - for key in ["", overlong_idempotency_key.as_str(), "key\twith-tab"] { - let invalid_key = trail::server::handle_http_request( - &mut db, - &api_request_with_headers( - "POST", - "/v1/lane/turns", - &[("Idempotency-Key", key)], - invalid_idempotency_body.clone(), - ), - ); - assert_eq!(invalid_key.status, 400); - let error: serde_json::Value = invalid_key.body_json().unwrap(); - assert!(error["error"]["message"] - .as_str() - .unwrap() - .contains("Idempotency-Key")); - } + .contains("api()")); - let auth = trail::server::ServerAuth::bearer("secret-token").unwrap(); - let auth_request = api_request_with_headers( - "POST", - "/v1/lane/turns", - &[("Idempotency-Key", "auth-key-1")], - serde_json::json!({ - "lane": "auth-idempotent-api", - "branch": "main" - }), + let contribution_response = trail::server::handle_http_request( + &mut db, + &api_request( + "GET", + &format!("/v1/lanes/{lane_id}/contribution?limit=5"), + serde_json::Value::Null, + ), ); - let missing_auth = trail::server::handle_http_request_with_auth(&mut db, &auth_request, &auth); - assert_eq!(missing_auth.status, 401); - let authorized_request = api_request_with_headers( - "POST", - "/v1/lane/turns", - &[ - ("Idempotency-Key", "auth-key-1"), - ("Authorization", "Bearer secret-token"), - ], - serde_json::json!({ - "lane": "auth-idempotent-api", - "branch": "main" - }), + assert_eq!(contribution_response.status, 200); + let contribution: serde_json::Value = contribution_response.body_json().unwrap(); + assert_eq!( + contribution["status"]["lane"]["record"]["name"], + "api-branch-lane" ); - let authorized = - trail::server::handle_http_request_with_auth(&mut db, &authorized_request, &auth); - assert_eq!(authorized.status, 201); - let unauthorized_replay = - trail::server::handle_http_request_with_auth(&mut db, &auth_request, &auth); - assert_eq!(unauthorized_replay.status, 401); - - let audits = db.list_external_mutation_audit(30).unwrap(); - let idempotent_turn_audits = audits + assert!(contribution["status"]["changed_paths"] + .as_array() + .unwrap() .iter() - .filter(|audit| { - audit.actor == "http:no-auth" - && audit.surface == "http" - && audit.command == "POST /v1/lane/turns" - && audit.status == "ok" - && audit.status_code == Some(201) - && audit.target_ref.as_deref() == Some("refs/lanes/idempotent-api") - }) - .collect::>(); - assert_eq!(idempotent_turn_audits.len(), 2); - assert_eq!( - idempotent_turn_audits - .iter() - .filter(|audit| { - audit - .summary - .as_ref() - .and_then(|summary| summary["idempotency_replay"].as_bool()) - == Some(true) - }) - .count(), - 1 + .any(|path| path["path"] == "src/api.rs")); + assert!(contribution["operations"] + .as_array() + .unwrap() + .iter() + .any(|operation| operation["message"] == "add API file")); + assert_eq!(contribution["status"]["latest_test"]["success"], true); + assert!(contribution["recent_events"] + .as_array() + .unwrap() + .iter() + .any(|event| event["event_type"] == "test_finished")); + + let cli_contribution = run_trail_json( + temp.path(), + &["lane", "contribution", "api-branch-lane", "--limit", "5"], ); - assert!(audits.iter().any(|audit| { - audit.actor == "http:no-auth" - && audit.surface == "http" - && audit.command == "POST /v1/lane/turns" - && audit.status == "error" - && audit.status_code == Some(400) - && audit - .summary - .as_ref() - .and_then(|summary| summary["error"].as_str()) - .is_some_and(|message| message.contains("already used for a different request")) - })); assert_eq!( - audits - .iter() - .filter(|audit| { - audit.actor == "http:no-auth" - && audit.surface == "http" - && audit.command == "POST /v1/lane/turns" - && audit.status == "error" - && audit.status_code == Some(400) - && audit.target_ref.as_deref() == Some("refs/lanes/idempotent-api") - && audit - .summary - .as_ref() - .and_then(|summary| summary["error"].as_str()) - .is_some_and(|message| { - message.contains("Idempotency-Key") - && !message.contains("already used for a different request") - }) - }) - .count(), - 3 + cli_contribution["status"]["lane"]["record"]["lane_id"], + lane_id ); - assert!(audits.iter().any(|audit| { - audit.actor == "http:bearer" - && audit.surface == "http" - && audit.command == "POST /v1/lane/turns" - && audit.status == "ok" - && audit.status_code == Some(201) - && audit.target_ref.as_deref() == Some("refs/lanes/auth-idempotent-api") - })); - assert_eq!( - audits - .iter() - .filter(|audit| { - audit.actor == "http:no-auth" - && audit.surface == "http" - && audit.command == "POST /v1/lane/turns" - && audit.status == "error" - && audit.status_code == Some(401) - }) - .count(), - 2 + + let review_response = trail::server::handle_http_request( + &mut db, + &api_request( + "GET", + &format!("/v1/lanes/{lane_id}/review?limit=5"), + serde_json::Value::Null, + ), ); -} + assert_eq!(review_response.status, 200); + let review: serde_json::Value = review_response.body_json().unwrap(); + assert_eq!(review["lane"]["record"]["name"], "api-branch-lane"); + assert_eq!(review["readiness"]["ready"], true); + assert!(review["changed_paths"] + .as_array() + .unwrap() + .iter() + .any(|path| path["path"] == "src/api.rs")); + assert_eq!(review["latest_test"]["success"], true); + assert!(review["recent_gates"] + .as_array() + .unwrap() + .iter() + .any(|gate| gate["kind"] == "test")); + assert!(review["recent_operations"] + .as_array() + .unwrap() + .iter() + .any(|operation| operation["message"] == "add API file")); + assert!(review["evidence_summary"]["operations"].as_u64().unwrap() >= 1); + assert!(review["next_steps"] + .as_array() + .unwrap() + .iter() + .any(|step| step.as_str().unwrap().contains("Start a new session"))); -#[test] -fn local_api_and_mcp_patch_payloads_respect_ignore_policy() { - let temp = tempfile::tempdir().unwrap(); - fs::write(temp.path().join("README.md"), "hello\n").unwrap(); - Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let cli_review = run_trail_json( + temp.path(), + &["lane", "review", "api-branch-lane", "--limit", "5"], + ); + assert_eq!(cli_review["lane"]["record"]["lane_id"], lane_id); + assert_eq!(cli_review["readiness"]["ready"], true); - let mut db = Trail::open(temp.path()).unwrap(); - db.ignore_add("host-secret.txt").unwrap(); + let cli_review_text = Command::new(trail_bin()) + .arg("--workspace") + .arg(temp.path()) + .args(["lane", "review", "api-branch-lane", "--limit", "5"]) + .output() + .unwrap(); + assert!(cli_review_text.status.success()); + let cli_review_stdout = String::from_utf8_lossy(&cli_review_text.stdout); + assert!( + cli_review_stdout.contains("Lane api-branch-lane is ready for review"), + "{cli_review_stdout}" + ); + assert!(cli_review_stdout.contains("Operations: 1")); + assert!(cli_review_stdout.contains("Next:")); - let turn_response = trail::server::handle_http_request( + let readiness_response = trail::server::handle_http_request( &mut db, &api_request( - "POST", - "/v1/lane/turns", - serde_json::json!({ - "lane": "privacy-api", - "branch": "main", - "session_title": "Privacy policy" - }), + "GET", + &format!("/v1/lanes/{lane_id}/readiness"), + serde_json::Value::Null, ), ); - assert_eq!(turn_response.status, 201); - let turn: LaneTurnStartReport = turn_response.body_json().unwrap(); + assert_eq!(readiness_response.status, 200); + let readiness: serde_json::Value = readiness_response.body_json().unwrap(); + assert_eq!(readiness["lane"]["record"]["name"], "api-branch-lane"); + assert_eq!(readiness["ready"], true); + assert!(readiness["blockers"].as_array().unwrap().is_empty()); + assert_eq!(readiness["latest_test"]["success"], true); + assert!(readiness["warnings"] + .as_array() + .unwrap() + .iter() + .any(|issue| issue["code"] == "missing_latest_eval")); - let blocked = trail::server::handle_http_request( + let cli_readiness = run_trail_json(temp.path(), &["lane", "readiness", "api-branch-lane"]); + assert_eq!(cli_readiness["lane"]["record"]["lane_id"], lane_id); + assert_eq!(cli_readiness["ready"], true); + + let handoff_response = trail::server::handle_http_request( &mut db, &api_request( - "POST", - &format!("/v1/lane/turns/{}/patches", turn.turn.turn_id), - serde_json::json!({ - "message": "blocked ignored write", - "files": [ - { - "type": "add_text", - "path": "host-secret.txt", - "content": "blocked\n" - } - ] - }), + "GET", + &format!("/v1/lanes/{lane_id}/handoff?limit=5"), + serde_json::Value::Null, ), ); - assert_eq!(blocked.status, 400); - let error: serde_json::Value = blocked.body_json().unwrap(); - assert!(error["error"]["message"] - .as_str() - .unwrap() - .contains("ignored path `host-secret.txt`")); - - let tools = trail::mcp::handle_json_rpc( - &mut db, - serde_json::json!({ - "jsonrpc": "2.0", - "id": 1, - "method": "tools/list", - "params": {} - }), - ) - .unwrap(); - let apply_patch_schema = tools["result"]["tools"] + assert_eq!(handoff_response.status, 200); + let handoff: serde_json::Value = handoff_response.body_json().unwrap(); + assert_eq!(handoff["lane"]["record"]["name"], "api-branch-lane"); + assert_eq!(handoff["readiness"]["ready"], true); + assert!(handoff["current_session"].is_null()); + assert!(handoff["recent_operations"] .as_array() .unwrap() .iter() - .find(|tool| tool["name"] == "trail.apply_patch") - .unwrap(); - let apply_patch_schema = &apply_patch_schema["inputSchema"]; - assert_eq!( - apply_patch_schema["properties"]["allow_ignored"]["type"], - "boolean" - ); - assert_eq!( - apply_patch_schema["properties"]["allow_stale"]["type"], - "boolean" - ); - let edit_source_modes = apply_patch_schema["oneOf"].as_array().unwrap(); - assert_eq!(edit_source_modes.len(), 2); - assert_eq!( - edit_source_modes[0]["required"], - serde_json::json!(["edits"]) - ); - assert_eq!( - edit_source_modes[0]["not"]["required"], - serde_json::json!(["files"]) - ); - assert_eq!( - edit_source_modes[1]["required"], - serde_json::json!(["files"]) - ); - assert_eq!( - edit_source_modes[1]["not"]["required"], - serde_json::json!(["edits"]) - ); - assert_eq!( - apply_patch_schema["properties"]["edits"]["minItems"], - serde_json::json!(1) - ); - assert_eq!( - apply_patch_schema["properties"]["files"]["minItems"], - serde_json::json!(1) - ); - let edit_variants = apply_patch_schema["properties"]["edits"]["items"]["oneOf"] - .as_array() - .unwrap(); - assert_eq!(edit_variants.len(), 5); - for variant in edit_variants { - assert_eq!(variant["additionalProperties"], false); - } - assert!(edit_variants[2]["required"] + .any(|operation| operation["message"] == "add API file")); + assert!(handoff["recent_events"] .as_array() .unwrap() .iter() - .any(|field| field == "expected_text")); - let file_variants = apply_patch_schema["properties"]["files"]["items"]["oneOf"] + .any(|event| event["event_type"] == "test_finished")); + assert!(handoff["next_steps"] .as_array() - .unwrap(); - assert_eq!(file_variants.len(), 5); - for variant in file_variants { - assert_eq!(variant["additionalProperties"], false); - } - let nested_line_edit = &file_variants[1]["properties"]["edits"]["items"]["oneOf"][0]; - assert_eq!(nested_line_edit["additionalProperties"], false); - assert!(nested_line_edit["required"] + .unwrap() + .iter() + .any(|step| step.as_str().unwrap().contains("Start a new session"))); + + let cli_handoff = run_trail_json( + temp.path(), + &["lane", "handoff", "api-branch-lane", "--limit", "5"], + ); + assert_eq!(cli_handoff["lane"]["record"]["lane_id"], lane_id); + assert_eq!(cli_handoff["readiness"]["ready"], true); + + let remove_dirty_response = trail::server::handle_http_request( + &mut db, + &api_request( + "DELETE", + &format!("/v1/lanes/{lane_id}"), + serde_json::Value::Null, + ), + ); + assert_eq!(remove_dirty_response.status, 400); + + let merge_response = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + &format!("/v1/lanes/{lane_id}/merge"), + serde_json::json!({ + "into": "main", + "strategy": "line_id_aware", + "direct": true + }), + ), + ); + assert_eq!(merge_response.status, 200); + let merge: serde_json::Value = merge_response.body_json().unwrap(); + assert_eq!(merge["source_ref"], "refs/lanes/api-branch-lane"); + assert_eq!(merge["target_ref"], "refs/branches/main"); + assert!(merge["changed_paths"] .as_array() .unwrap() .iter() - .any(|field| field == "expected_text")); + .any(|path| path["path"] == "src/api.rs")); - let allowed = trail::mcp::handle_json_rpc( + let why = db.why("src/api.rs:1", Some("main")).unwrap(); + assert_eq!(why.current_text, "pub fn api() -> bool { true }"); + + let remove_response = trail::server::handle_http_request( &mut db, - serde_json::json!({ - "jsonrpc": "2.0", - "id": 2, - "method": "tools/call", - "params": { - "name": "trail.apply_patch", - "arguments": { - "turn_id": turn.turn.turn_id, - "message": "explicit ignored write", - "allow_ignored": true, - "files": [ - { - "type": "add_text", - "path": "host-secret.txt", - "content": "allowed fixture\n" - } - ] - } - } - }), + &api_request( + "DELETE", + &format!("/v1/lanes/{lane_id}"), + serde_json::Value::Null, + ), + ); + assert_eq!(remove_response.status, 200); + let removed: serde_json::Value = remove_response.body_json().unwrap(); + assert_eq!(removed["lane_id"], lane_id); + assert_eq!(removed["forced"], false); + assert_eq!(removed["removed_workdir"], workdir); + assert!(!std::path::Path::new(&workdir).exists()); + assert_eq!(db.lane_details(&lane_id).unwrap().branch.status, "removed"); +} + +#[test] +fn layered_workspace_reports_have_http_mcp_and_openapi_parity() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "baseline\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "surface", + Some("main"), + mode, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let view = db.lane_workspace_view("surface").unwrap().unwrap(); + fs::write( + Path::new(&view.source_upper).join("surface.txt"), + "surface parity\n", ) .unwrap(); - assert_eq!(allowed["result"]["isError"], false); - assert_eq!( - allowed["result"]["structuredContent"]["changed_paths"][0]["path"], - "host-secret.txt" - ); - let details = db.show_lane_turn(&turn.turn.turn_id).unwrap(); - let patch_event = details - .events - .iter() - .find(|event| event.event_type == "patch_applied") - .unwrap(); - assert_eq!( - patch_event.payload.as_ref().unwrap()["allow_ignored"], - serde_json::Value::Bool(true) + let http_workspace = trail::server::handle_http_request( + &mut db, + &api_request( + "GET", + "/v1/lanes/surface/workspace", + serde_json::Value::Null, + ), ); -} - -#[test] -fn local_lane_http_api_manages_lane_branch_lifecycle() { - let temp = tempfile::tempdir().unwrap(); - fs::write(temp.path().join("README.md"), "hello\n").unwrap(); - Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + assert_eq!(http_workspace.status, 200); + let http_workspace: serde_json::Value = http_workspace.body_json().unwrap(); + assert_eq!(http_workspace["view_id"], view.view_id); - let mut db = Trail::open(temp.path()).unwrap(); - let status_response = trail::server::handle_http_request( + let http_space = trail::server::handle_http_request( &mut db, - &api_request("GET", "/v1/status", serde_json::Value::Null), + &api_request("GET", "/v1/lanes/surface/space", serde_json::Value::Null), ); - assert_eq!(status_response.status, 200); - let status: serde_json::Value = status_response.body_json().unwrap(); - assert_eq!(status["branch"], "main"); + assert_eq!(http_space.status, 200); + let http_space: serde_json::Value = http_space.body_json().unwrap(); + assert_eq!(http_space["view_id"], view.view_id); - let spawn_response = trail::server::handle_http_request( + let http_unmount = trail::server::handle_http_request( &mut db, - &api_request( - "POST", - "/v1/lanes", - serde_json::json!({ - "name": "api-branch-lane", - "from_ref": "main", - "materialize": true - }), - ), + &api_request("POST", "/v1/lanes/surface/unmount", serde_json::Value::Null), + ); + assert_eq!(http_unmount.status, 200); + assert_eq!( + http_unmount.body_json::().unwrap()["view_id"], + view.view_id ); - assert_eq!(spawn_response.status, 201); - let spawned: serde_json::Value = spawn_response.body_json().unwrap(); - let lane_id = spawned["lane_id"].as_str().unwrap().to_string(); - let lane_base_change = spawned["base_change"].as_str().unwrap().to_string(); - assert_eq!(spawned["ref_name"], "refs/lanes/api-branch-lane"); - let workdir = spawned["workdir"].as_str().unwrap().to_string(); - let lanes_response = trail::server::handle_http_request( + let http_dependencies = trail::server::handle_http_request( &mut db, - &api_request("GET", "/v1/lanes", serde_json::Value::Null), + &api_request( + "GET", + "/v1/lanes/surface/dependencies", + serde_json::Value::Null, + ), ); - assert_eq!(lanes_response.status, 200); - let lanes: serde_json::Value = lanes_response.body_json().unwrap(); - assert_eq!(lanes.as_array().unwrap().len(), 1); - assert_eq!(lanes[0]["record"]["name"], "api-branch-lane"); - assert_eq!(lanes[0]["branch"]["ref_name"], "refs/lanes/api-branch-lane"); + assert_eq!(http_dependencies.status, 200); + assert!(http_dependencies + .body_json::() + .unwrap() + .as_array() + .unwrap() + .is_empty()); - let lane_status_response = trail::server::handle_http_request( + let http_environment = trail::server::handle_http_request( &mut db, &api_request( "GET", - &format!("/v1/lanes/{lane_id}/status"), + "/v1/lanes/surface/environment", serde_json::Value::Null, ), ); - assert_eq!(lane_status_response.status, 200); - let lane_status: serde_json::Value = lane_status_response.body_json().unwrap(); - assert_eq!(lane_status["lane"]["record"]["name"], "api-branch-lane"); + assert_eq!(http_environment.status, 200); + assert!(http_environment + .body_json::() + .unwrap() + .as_array() + .unwrap() + .is_empty()); - let patch_response = trail::server::handle_http_request( + let http_checkpoint = trail::server::handle_http_request( &mut db, &api_request( "POST", - &format!("/v1/lanes/{lane_id}/patches"), - serde_json::json!({ - "base_change": lane_base_change, - "message": "add API file", - "files": [ - { - "type": "add_text", - "path": "src/api.rs", - "content": "pub fn api() -> bool { true }\n" - } - ] - }), + "/v1/lanes/surface/checkpoint", + serde_json::json!({"message": "HTTP checkpoint"}), ), ); - assert_eq!(patch_response.status, 200); - let patch: LanePatchReport = patch_response.body_json().unwrap(); - assert_eq!(patch.lane_id, lane_id); - assert_eq!(patch.changed_paths[0].path, "src/api.rs"); + assert_eq!(http_checkpoint.status, 200); + let http_checkpoint: serde_json::Value = http_checkpoint.body_json().unwrap(); assert_eq!( - fs::read_to_string(std::path::Path::new(&workdir).join("src/api.rs")).unwrap(), - "pub fn api() -> bool { true }\n" + http_checkpoint["source_paths"], + serde_json::json!(["surface.txt"]) ); - fs::write( - std::path::Path::new(&workdir).join("README.md"), - "hello\napi dirty\n", - ) - .unwrap(); - let record_preview_response = trail::server::handle_http_request( + let http_update = trail::server::handle_http_request( &mut db, &api_request( "POST", - &format!("/v1/lanes/{lane_id}/record"), - serde_json::json!({ "preview": true }), + "/v1/lanes/surface/update", + serde_json::json!({"from": "main"}), ), ); - assert_eq!(record_preview_response.status, 200); - let record_preview: serde_json::Value = record_preview_response.body_json().unwrap(); - assert_eq!(record_preview["clean"], false); - assert!(record_preview["changed_paths"] - .as_array() - .unwrap() - .iter() - .any(|path| path["path"] == "README.md")); - assert!(record_preview["operation"].is_null()); + assert_eq!(http_update.status, 200); + let http_update: serde_json::Value = http_update.body_json().unwrap(); + assert_eq!(http_update["source_ref"], "refs/branches/main"); + assert_eq!(http_update["target_ref"], "refs/lanes/surface"); - let sync_conflict = trail::server::handle_http_request( + let http_exec_error = trail::server::handle_http_request( &mut db, &api_request( "POST", - &format!("/v1/lanes/{lane_id}/sync-workdir"), - serde_json::json!({}), + "/v1/lanes/surface/exec", + serde_json::json!({"command": []}), ), ); - assert_eq!(sync_conflict.status, 409); + assert_eq!(http_exec_error.status, 400); + assert!( + http_exec_error.body_json::().unwrap()["error"]["message"] + .as_str() + .unwrap() + .contains("requires a command") + ); - let sync_response = trail::server::handle_http_request( + let http_sync_error = trail::server::handle_http_request( &mut db, &api_request( "POST", - &format!("/v1/lanes/{lane_id}/sync-workdir"), - serde_json::json!({ "force": true }), + "/v1/lanes/surface/dependencies/sync", + serde_json::json!({}), ), ); - assert_eq!(sync_response.status, 200); - let synced: serde_json::Value = sync_response.body_json().unwrap(); - assert_eq!(synced["forced"], true); - assert_eq!( - fs::read_to_string(std::path::Path::new(&workdir).join("README.md")).unwrap(), - "hello\n" - ); + assert_eq!(http_sync_error.status, 400); - let test_response = trail::server::handle_http_request( + let http_environment_sync_error = trail::server::handle_http_request( &mut db, &api_request( "POST", - &format!("/v1/lanes/{lane_id}/tests"), - serde_json::json!({ - "command": ["sh", "-c", "printf api-test"], - "timeout_secs": 5 - }), + "/v1/lanes/surface/environment/sync", + serde_json::json!({"adapter": "trail/node@1"}), ), ); - assert_eq!(test_response.status, 200); - let test: serde_json::Value = test_response.body_json().unwrap(); - assert_eq!(test["success"], true); - assert_eq!(test["stdout_preview"], "api-test"); + assert_eq!(http_environment_sync_error.status, 400); - let diff_response = trail::server::handle_http_request( + let http_cache = trail::server::handle_http_request( &mut db, - &api_request( - "GET", - &format!("/v1/lanes/{lane_id}/diff?patch=true"), - serde_json::Value::Null, - ), + &api_request("GET", "/v1/cache/layers", serde_json::Value::Null), ); - assert_eq!(diff_response.status, 200); - let diff: serde_json::Value = diff_response.body_json().unwrap(); - assert_eq!(diff["files"][0]["path"], "src/api.rs"); - assert!(diff["files"][0]["patch"] - .as_str() + assert_eq!(http_cache.status, 200); + assert!(http_cache + .body_json::() .unwrap() - .contains("api()")); - - let contribution_response = trail::server::handle_http_request( + .as_array() + .unwrap() + .is_empty()); + let http_gc = trail::server::handle_http_request( &mut db, &api_request( - "GET", - &format!("/v1/lanes/{lane_id}/contribution?limit=5"), - serde_json::Value::Null, + "POST", + "/v1/cache/gc", + serde_json::json!({"dry_run": true, "retention_secs": 0}), ), ); - assert_eq!(contribution_response.status, 200); - let contribution: serde_json::Value = contribution_response.body_json().unwrap(); + assert_eq!(http_gc.status, 200); assert_eq!( - contribution["status"]["lane"]["record"]["name"], - "api-branch-lane" + http_gc.body_json::().unwrap()["dry_run"], + true + ); + + let mcp_call = |db: &mut Trail, id: u64, name: &str, arguments: serde_json::Value| { + trail::mcp::handle_json_rpc( + db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": id, + "method": "tools/call", + "params": {"name": name, "arguments": arguments} + }), + ) + .unwrap() + }; + let mcp_workspace = mcp_call( + &mut db, + 1, + "trail.lane_workspace", + serde_json::json!({"lane": "surface"}), + ); + assert_eq!(mcp_workspace["result"]["isError"], false); + assert_eq!( + mcp_workspace["result"]["structuredContent"]["view_id"], + view.view_id + ); + let mcp_space = mcp_call( + &mut db, + 2, + "trail.lane_space", + serde_json::json!({"lane": "surface"}), + ); + assert_eq!( + mcp_space["result"]["structuredContent"]["view_id"], + view.view_id + ); + let mcp_dependencies = mcp_call( + &mut db, + 3, + "trail.deps_status", + serde_json::json!({"lane": "surface"}), ); - assert!(contribution["status"]["changed_paths"] + assert!(mcp_dependencies["result"]["structuredContent"] .as_array() .unwrap() - .iter() - .any(|path| path["path"] == "src/api.rs")); - assert!(contribution["operations"] + .is_empty()); + let mcp_environment = mcp_call( + &mut db, + 32, + "trail.env_status", + serde_json::json!({"lane": "surface"}), + ); + assert!(mcp_environment["result"]["structuredContent"] .as_array() .unwrap() - .iter() - .any(|operation| operation["message"] == "add API file")); - assert_eq!(contribution["status"]["latest_test"]["success"], true); - assert!(contribution["recent_events"] + .is_empty()); + let mcp_environment = mcp_call( + &mut db, + 32, + "trail.env_status", + serde_json::json!({"lane": "surface"}), + ); + assert!(mcp_environment["result"]["structuredContent"] .as_array() .unwrap() - .iter() - .any(|event| event["event_type"] == "test_finished")); - - let cli_contribution = run_trail_json( - temp.path(), - &["lane", "contribution", "api-branch-lane", "--limit", "5"], + .is_empty()); + let mcp_unmount = mcp_call( + &mut db, + 31, + "trail.lane_unmount", + serde_json::json!({"lane": "surface"}), ); assert_eq!( - cli_contribution["status"]["lane"]["record"]["lane_id"], - lane_id + mcp_unmount["result"]["structuredContent"]["view_id"], + view.view_id ); - - let review_response = trail::server::handle_http_request( + let mcp_checkpoint = mcp_call( &mut db, - &api_request( - "GET", - &format!("/v1/lanes/{lane_id}/review?limit=5"), - serde_json::Value::Null, - ), + 4, + "trail.lane_checkpoint", + serde_json::json!({"lane": "surface", "message": "MCP checkpoint"}), ); - assert_eq!(review_response.status, 200); - let review: serde_json::Value = review_response.body_json().unwrap(); - assert_eq!(review["lane"]["record"]["name"], "api-branch-lane"); - assert_eq!(review["readiness"]["ready"], true); - assert!(review["changed_paths"] + assert_eq!(mcp_checkpoint["result"]["isError"], false); + assert_eq!( + mcp_checkpoint["result"]["structuredContent"]["root_id"], + http_checkpoint["root_id"] + ); + let mcp_cache = mcp_call(&mut db, 5, "trail.cache_list", serde_json::json!({})); + assert!(mcp_cache["result"]["structuredContent"] .as_array() .unwrap() - .iter() - .any(|path| path["path"] == "src/api.rs")); - assert_eq!(review["latest_test"]["success"], true); - assert!(review["recent_gates"] + .is_empty()); + let mcp_gc = mcp_call( + &mut db, + 6, + "trail.cache_gc", + serde_json::json!({"dry_run": true, "retention_secs": 0}), + ); + assert_eq!(mcp_gc["result"]["structuredContent"]["dry_run"], true); + let mcp_update = mcp_call( + &mut db, + 7, + "trail.lane_update", + serde_json::json!({"lane": "surface", "source": "main"}), + ); + assert_eq!(mcp_update["result"]["isError"], false); + assert_eq!( + mcp_update["result"]["structuredContent"]["source_ref"], + "refs/branches/main" + ); + + let http_adapters = trail::server::handle_http_request( + &mut db, + &api_request("GET", "/v1/environment/adapters", serde_json::Value::Null), + ); + assert_eq!(http_adapters.status, 200); + let http_adapters: serde_json::Value = http_adapters.body_json().unwrap(); + assert_eq!(http_adapters["contract_major"], 1); + let adapter_identities = http_adapters["adapters"] .as_array() .unwrap() .iter() - .any(|gate| gate["kind"] == "test")); - assert!(review["recent_operations"] + .map(|adapter| adapter["canonical_identity"].as_str().unwrap()) + .collect::>(); + assert_eq!( + adapter_identities, + BTreeSet::from([ + "trail/cargo-target-seed@1", + "trail/cmake-build@1", + "trail/command@1", + "trail/go-vendor@1", + "trail/node@1", + "trail/oci-image@1", + "trail/python-venv@1", + ]) + ); + assert!(http_adapters["adapters"] .as_array() .unwrap() .iter() - .any(|operation| operation["message"] == "add API file")); - assert!(review["evidence_summary"]["operations"].as_u64().unwrap() >= 1); - assert!(review["next_steps"] + .any(|adapter| { + adapter["canonical_identity"] == "trail/node@1" + && adapter["discovery_markers"][0] == "package.json" + })); + assert!(http_adapters["adapters"] .as_array() .unwrap() .iter() - .any(|step| step.as_str().unwrap().contains("Start a new session"))); - - let cli_review = run_trail_json( - temp.path(), - &["lane", "review", "api-branch-lane", "--limit", "5"], - ); - assert_eq!(cli_review["lane"]["record"]["lane_id"], lane_id); - assert_eq!(cli_review["readiness"]["ready"], true); + .any(|adapter| { + adapter["canonical_identity"] == "trail/cmake-build@1" + && adapter["kind"] == "build" + && adapter["discovery_markers"][0] == "CMakeLists.txt" + })); + let mcp_adapters = mcp_call(&mut db, 8, "trail.env_adapters", serde_json::json!({})); + assert_eq!(mcp_adapters["result"]["isError"], false); + assert_eq!(mcp_adapters["result"]["structuredContent"], http_adapters); - let cli_review_text = Command::new(trail_bin()) - .arg("--workspace") - .arg(temp.path()) - .args(["lane", "review", "api-branch-lane", "--limit", "5"]) - .output() - .unwrap(); - assert!(cli_review_text.status.success()); - let cli_review_stdout = String::from_utf8_lossy(&cli_review_text.stdout); - assert!(cli_review_stdout.contains("Lane review: api-branch-lane")); - assert!(cli_review_stdout.contains("Evidence:")); - assert!(cli_review_stdout.contains("Next steps:")); + let tools = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 9, + "method": "tools/list", + "params": {} + }), + ) + .unwrap(); + let tools = tools["result"]["tools"].as_array().unwrap(); + for (name, read_only, destructive, open_world) in [ + ("trail.lane_workspace", true, false, false), + ("trail.lane_mount", false, false, false), + ("trail.lane_unmount", false, false, false), + ("trail.lane_checkpoint", false, false, false), + ("trail.lane_update", false, false, false), + ("trail.lane_exec", false, false, true), + ("trail.deps_sync", false, false, true), + ("trail.env_adapters", true, false, false), + ("trail.env_status", true, false, false), + ("trail.env_discover", true, false, false), + ("trail.env_graph", true, false, false), + ("trail.env_generation", true, false, false), + ("trail.env_explain", true, false, false), + ("trail.env_plan", true, false, false), + ("trail.env_sync", false, false, true), + ("trail.env_sync_all", false, false, true), + ("trail.env_status", true, false, false), + ("trail.env_sync", false, false, true), + ("trail.cache_gc", false, true, false), + ] { + let tool = tools.iter().find(|tool| tool["name"] == name).unwrap(); + assert_eq!(tool["annotations"]["readOnlyHint"], read_only); + assert_eq!(tool["annotations"]["destructiveHint"], destructive); + assert_eq!(tool["annotations"]["openWorldHint"], open_world); + } - let readiness_response = trail::server::handle_http_request( + let openapi = trail::server::handle_http_request( &mut db, - &api_request( - "GET", - &format!("/v1/lanes/{lane_id}/readiness"), - serde_json::Value::Null, - ), + &api_request("GET", "/v1/openapi.json", serde_json::Value::Null), ); - assert_eq!(readiness_response.status, 200); - let readiness: serde_json::Value = readiness_response.body_json().unwrap(); - assert_eq!(readiness["lane"]["record"]["name"], "api-branch-lane"); - assert_eq!(readiness["ready"], true); - assert!(readiness["blockers"].as_array().unwrap().is_empty()); - assert_eq!(readiness["latest_test"]["success"], true); - assert!(readiness["warnings"] - .as_array() - .unwrap() - .iter() - .any(|issue| issue["code"] == "missing_latest_eval")); + let openapi: serde_json::Value = openapi.body_json().unwrap(); + for path in [ + "/v1/environment/adapters", + "/v1/lanes/{lane_or_id}/workspace", + "/v1/lanes/{lane_or_id}/mount", + "/v1/lanes/{lane_or_id}/unmount", + "/v1/lanes/{lane_or_id}/checkpoint", + "/v1/lanes/{lane_or_id}/update", + "/v1/lanes/{lane_or_id}/space", + "/v1/lanes/{lane_or_id}/exec", + "/v1/lanes/{lane_or_id}/dependencies", + "/v1/lanes/{lane_or_id}/dependencies/sync", + "/v1/lanes/{lane_or_id}/environment", + "/v1/lanes/{lane_or_id}/environment/discover", + "/v1/lanes/{lane_or_id}/environment/graph", + "/v1/lanes/{lane_or_id}/environment/generation", + "/v1/lanes/{lane_or_id}/environment/explain", + "/v1/lanes/{lane_or_id}/environment/plan", + "/v1/lanes/{lane_or_id}/environment/sync", + "/v1/lanes/{lane_or_id}/environment/sync-all", + "/v1/cache/layers", + "/v1/cache/gc", + ] { + assert!( + openapi["paths"].get(path).is_some(), + "missing OpenAPI path {path}" + ); + } +} - let cli_readiness = run_trail_json(temp.path(), &["lane", "readiness", "api-branch-lane"]); - assert_eq!(cli_readiness["lane"]["record"]["lane_id"], lane_id); - assert_eq!(cli_readiness["ready"], true); +#[test] +fn environment_graph_has_cli_http_mcp_and_openapi_parity() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("input.txt"), "graph\n").unwrap(); + fs::write( + temp.path().join("trail.environment.toml"), + r#"schema = "trail.environment/v1" + +[[component]] +id = "graph.a" +adapter = "trail/command@1" +kind = "generated" +inputs = [{ path = "input.txt" }] +outputs = [{ source = "generated-a", target = ".trail-generated/a" }] +[component.build] +command = ["git", "--version"] + +[[component]] +id = "graph.b" +adapter = "trail/command@1" +kind = "generated" +depends_on = ["graph.a"] +inputs = [{ path = "input.txt" }] +outputs = [{ source = "generated-b", target = ".trail-generated/b" }] +[component.build] +command = ["git", "--version"] +"#, + ) + .unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "graph", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); - let handoff_response = trail::server::handle_http_request( + let rust = db.workspace_environment_graph("graph", None).unwrap(); + let expected = serde_json::to_value(&rust).unwrap(); + assert_eq!(rust.nodes.len(), 2); + assert_eq!(rust.edges.len(), 1); + assert_eq!(rust.nodes[0].component_id, "graph.a"); + assert_eq!(rust.nodes[1].component_id, "graph.b"); + assert_eq!( + rust.edges[0].source_component_key, + rust.nodes[0].component_key + ); + + let http = trail::server::handle_http_request( &mut db, &api_request( "GET", - &format!("/v1/lanes/{lane_id}/handoff?limit=5"), + "/v1/lanes/graph/environment/graph", serde_json::Value::Null, ), ); - assert_eq!(handoff_response.status, 200); - let handoff: serde_json::Value = handoff_response.body_json().unwrap(); - assert_eq!(handoff["lane"]["record"]["name"], "api-branch-lane"); - assert_eq!(handoff["readiness"]["ready"], true); - assert!(handoff["current_session"].is_null()); - assert!(handoff["recent_operations"] - .as_array() - .unwrap() - .iter() - .any(|operation| operation["message"] == "add API file")); - assert!(handoff["recent_events"] - .as_array() - .unwrap() - .iter() - .any(|event| event["event_type"] == "test_finished")); - assert!(handoff["next_steps"] - .as_array() - .unwrap() - .iter() - .any(|step| step.as_str().unwrap().contains("Start a new session"))); + assert_eq!(http.status, 200); + assert_eq!(http.body_json::().unwrap(), expected); - let cli_handoff = run_trail_json( - temp.path(), - &["lane", "handoff", "api-branch-lane", "--limit", "5"], + let mcp = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 41, + "method": "tools/call", + "params": { + "name": "trail.env_graph", + "arguments": {"lane": "graph"} + } + }), + ) + .unwrap(); + assert_eq!(mcp["result"]["isError"], false); + assert_eq!(mcp["result"]["structuredContent"], expected); + + let cli = run_trail_json(temp.path(), &["env", "graph", "graph"]); + assert_eq!(cli, expected); + let openapi = trail::server::openapi_spec(); + assert_eq!( + openapi["paths"]["/v1/lanes/{lane_or_id}/environment/graph"]["get"]["responses"]["200"] + ["content"]["application/json"]["schema"]["$ref"], + "#/components/schemas/EnvironmentGraphReport" ); - assert_eq!(cli_handoff["lane"]["record"]["lane_id"], lane_id); - assert_eq!(cli_handoff["readiness"]["ready"], true); + assert!(db.list_workspace_layers().unwrap().is_empty()); +} + +#[test] +fn pinned_oci_metadata_has_cli_http_mcp_openapi_and_gc_parity() { + let temp = tempfile::tempdir().unwrap(); + let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + fs::write( + temp.path().join("trail.oci.toml"), + format!( + "schema = \"trail.oci-images/v1\"\n\n[[image]]\nname = \"web\"\nreference = \"ghcr.io/example/web@{digest}\"\nplatform = \"linux/amd64\"\n" + ), + ) + .unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "oci-surfaces", + Some("main"), + if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }, + None, + None, + None, + &[], + false, + ) + .unwrap(); - let remove_dirty_response = trail::server::handle_http_request( + let http_plan = trail::server::handle_http_request( &mut db, &api_request( - "DELETE", - &format!("/v1/lanes/{lane_id}"), + "GET", + "/v1/lanes/oci-surfaces/environment/plan?adapter=oci-image", serde_json::Value::Null, ), ); - assert_eq!(remove_dirty_response.status, 400); + assert_eq!(http_plan.status, 200); + let plan: serde_json::Value = http_plan.body_json().unwrap(); + assert_eq!(plan["component_id"], "oci-images"); + assert_eq!(plan["kind"], "external"); + assert!(plan["outputs"].as_array().unwrap().is_empty()); + assert!(plan["commands"].as_array().unwrap().is_empty()); + assert_eq!(plan["external_artifacts"][0]["digest"], digest); + assert_eq!( + plan["capabilities"]["sandbox"], + "not-applicable-metadata-only" + ); + assert_eq!( + run_trail_json( + temp.path(), + &[ + "env", + "plan", + "oci-surfaces", + "--adapter", + "trail/oci-image@1", + ], + ), + plan + ); - let merge_response = trail::server::handle_http_request( + let sync = trail::server::handle_http_request( &mut db, &api_request( "POST", - "/v1/branches/main/merge-lane", - serde_json::json!({ - "lane_id": lane_id, - "strategy": "line_id_aware", - "direct": true - }), + "/v1/lanes/oci-surfaces/environment/sync", + serde_json::json!({"adapter": "trail/oci-image@1"}), ), ); - assert_eq!(merge_response.status, 200); - let merge: serde_json::Value = merge_response.body_json().unwrap(); - assert_eq!(merge["source_ref"], "refs/lanes/api-branch-lane"); - assert_eq!(merge["target_ref"], "refs/branches/main"); - assert!(merge["changed_paths"] + assert_eq!(sync.status, 200); + let sync: serde_json::Value = sync.body_json().unwrap(); + assert!(sync["layers"].as_array().unwrap().is_empty()); + assert!(sync["generation"]["components"][0]["outputs"] .as_array() .unwrap() - .iter() - .any(|path| path["path"] == "src/api.rs")); - - let why = db.why("src/api.rs:1", Some("main")).unwrap(); - assert_eq!(why.current_text, "pub fn api() -> bool { true }"); + .is_empty()); + assert!(sync["generation"]["components"][0]["layer_id"].is_null()); + assert!(sync["generation"]["components"][0]["mount_path"].is_null()); + assert_eq!( + sync["generation"]["components"][0]["external_artifacts"][0]["reference"], + format!("ghcr.io/example/web@{digest}") + ); + assert_eq!( + sync["generation"]["components"][0]["external_artifacts"][0]["cleanup_owner"], + "external" + ); - let remove_response = trail::server::handle_http_request( + let mcp = trail::mcp::handle_json_rpc( &mut db, - &api_request( - "DELETE", - &format!("/v1/lanes/{lane_id}"), - serde_json::Value::Null, - ), + serde_json::json!({ + "jsonrpc": "2.0", + "id": 42, + "method": "tools/call", + "params": { + "name": "trail.env_generation", + "arguments": {"lane": "oci-surfaces"} + } + }), + ) + .unwrap(); + assert_eq!(mcp["result"]["isError"], false); + assert_eq!(mcp["result"]["structuredContent"], sync["generation"]); + + let gc = db.workspace_cache_gc(false, Some(0)).unwrap(); + assert!(gc + .deleted + .iter() + .all(|entry| entry.kind != "external_artifact")); + let active = db + .active_environment_generation("oci-surfaces") + .unwrap() + .unwrap(); + assert_eq!(active.components[0].external_artifacts[0].digest, digest); + assert!(db.list_workspace_layers().unwrap().is_empty()); + + let openapi = trail::server::openapi_spec(); + assert!(openapi["components"]["schemas"] + .get("EnvironmentExternalArtifactReport") + .is_some()); + assert!( + openapi["components"]["schemas"]["EnvironmentPlanReport"]["required"] + .as_array() + .unwrap() + .iter() + .any(|field| field == "external_artifacts") ); - assert_eq!(remove_response.status, 200); - let removed: serde_json::Value = remove_response.body_json().unwrap(); - assert_eq!(removed["lane_id"], lane_id); - assert_eq!(removed["forced"], false); - assert_eq!(removed["removed_workdir"], workdir); - assert!(!std::path::Path::new(&workdir).exists()); - assert_eq!(db.lane_details(&lane_id).unwrap().branch.status, "removed"); } #[test] -fn layered_workspace_reports_have_http_mcp_and_openapi_parity() { +fn environment_sync_reuses_one_node_layer_across_http_and_mcp_parity() { + if Command::new("npm").arg("--version").output().is_err() + || Command::new("node").arg("--version").output().is_err() + { + return; + } let temp = tempfile::tempdir().unwrap(); - fs::write(temp.path().join("README.md"), "baseline\n").unwrap(); + fs::write( + temp.path().join("package.json"), + r#"{"name":"env-surface","version":"1.0.0","private":true}"#, + ) + .unwrap(); + fs::write( + temp.path().join("package-lock.json"), + r#"{"name":"env-surface","version":"1.0.0","lockfileVersion":3,"requires":true,"packages":{"":{"name":"env-surface","version":"1.0.0"}}}"#, + ) + .unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); let mut db = Trail::open(temp.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; - db.spawn_lane_with_workdir_mode_paths_and_neighbors( - "surface", - Some("main"), - mode, - None, - None, - None, - &[], - false, - ) - .unwrap(); - let view = db.lane_workspace_view("surface").unwrap().unwrap(); - fs::write( - Path::new(&view.source_upper).join("surface.txt"), - "surface parity\n", - ) - .unwrap(); + for lane in ["env-http", "env-mcp", "env-all-http", "env-all-mcp"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + mode.clone(), + None, + None, + None, + &[], + false, + ) + .unwrap(); + } - let http_workspace = trail::server::handle_http_request( + let http_plan = trail::server::handle_http_request( &mut db, &api_request( "GET", - "/v1/lanes/surface/workspace", + "/v1/lanes/env-http/environment/plan", serde_json::Value::Null, ), ); - assert_eq!(http_workspace.status, 200); - let http_workspace: serde_json::Value = http_workspace.body_json().unwrap(); - assert_eq!(http_workspace["view_id"], view.view_id); - - let http_space = trail::server::handle_http_request( - &mut db, - &api_request("GET", "/v1/lanes/surface/space", serde_json::Value::Null), - ); - assert_eq!(http_space.status, 200); - let http_space: serde_json::Value = http_space.body_json().unwrap(); - assert_eq!(http_space["view_id"], view.view_id); - - let http_unmount = trail::server::handle_http_request( + assert_eq!(http_plan.status, 200); + let http_plan: serde_json::Value = http_plan.body_json().unwrap(); + assert_eq!(http_plan["component_id"], "node"); + assert_eq!(http_plan["capabilities"]["sandbox"], "trusted-builtin"); + assert!(http_plan["dependencies"].as_array().unwrap().is_empty()); + let mcp_plan = trail::mcp::handle_json_rpc( &mut db, - &api_request("POST", "/v1/lanes/surface/unmount", serde_json::Value::Null), - ); - assert_eq!(http_unmount.status, 200); + serde_json::json!({ + "jsonrpc": "2.0", + "id": 20, + "method": "tools/call", + "params": { + "name": "trail.env_plan", + "arguments": {"lane": "env-mcp"} + } + }), + ) + .unwrap(); + assert_eq!(mcp_plan["result"]["isError"], false); assert_eq!( - http_unmount.body_json::().unwrap()["view_id"], - view.view_id + mcp_plan["result"]["structuredContent"]["component_key"], + http_plan["component_key"] ); - let http_dependencies = trail::server::handle_http_request( + let discovery = trail::server::handle_http_request( &mut db, &api_request( "GET", - "/v1/lanes/surface/dependencies", + "/v1/lanes/env-http/environment/discover", serde_json::Value::Null, ), ); - assert_eq!(http_dependencies.status, 200); - assert!(http_dependencies - .body_json::() - .unwrap() - .as_array() - .unwrap() - .is_empty()); + assert_eq!(discovery.status, 200); + let discovery: serde_json::Value = discovery.body_json().unwrap(); + assert_eq!(discovery["components"][0]["component_id"], "node"); + assert!(discovery["conflicts"].as_array().unwrap().is_empty()); + let mcp_discovery = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 0, + "method": "tools/call", + "params": { + "name": "trail.env_discover", + "arguments": {"lane": "env-mcp"} + } + }), + ) + .unwrap(); + assert_eq!(mcp_discovery["result"]["isError"], false); + assert_eq!( + mcp_discovery["result"]["structuredContent"]["source_root"], + discovery["source_root"] + ); - let http_checkpoint = trail::server::handle_http_request( + let http = trail::server::handle_http_request( &mut db, &api_request( "POST", - "/v1/lanes/surface/checkpoint", - serde_json::json!({"message": "HTTP checkpoint"}), + "/v1/lanes/env-http/environment/sync", + serde_json::json!({"adapter": "trail/node@1"}), ), ); - assert_eq!(http_checkpoint.status, 200); - let http_checkpoint: serde_json::Value = http_checkpoint.body_json().unwrap(); + assert_eq!(http.status, 200); + let http_report: serde_json::Value = http.body_json().unwrap(); + let http_layer = &http_report["layers"][0]; + + let mcp = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "trail.env_sync", + "arguments": {"lane": "env-mcp", "adapter": "auto"} + } + }), + ) + .unwrap(); + assert_eq!(mcp["result"]["isError"], false); assert_eq!( - http_checkpoint["source_paths"], - serde_json::json!(["surface.txt"]) + mcp["result"]["structuredContent"]["layers"][0]["layer_id"], + http_layer["layer_id"] ); - let http_update = trail::server::handle_http_request( + let status = trail::server::handle_http_request( &mut db, &api_request( - "POST", - "/v1/lanes/surface/update", - serde_json::json!({"from": "main"}), + "GET", + "/v1/lanes/env-http/environment", + serde_json::Value::Null, ), ); - assert_eq!(http_update.status, 200); - let http_update: serde_json::Value = http_update.body_json().unwrap(); - assert_eq!(http_update["source_ref"], "refs/branches/main"); - assert_eq!(http_update["target_ref"], "refs/lanes/surface"); + let status: serde_json::Value = status.body_json().unwrap(); + assert_eq!(status[0]["component"]["component_id"], "node"); + assert_eq!(status[0]["adapter"]["name"], "node"); + assert_eq!(status[0]["status"], "ready"); + assert_eq!( + status[0]["adapter"]["distribution_digest"], + "builtin:node-plan-v1" + ); - let http_exec_error = trail::server::handle_http_request( + let generation = trail::server::handle_http_request( &mut db, &api_request( - "POST", - "/v1/lanes/surface/exec", - serde_json::json!({"command": []}), + "GET", + "/v1/lanes/env-http/environment/generation", + serde_json::Value::Null, ), ); - assert_eq!(http_exec_error.status, 400); - assert!( - http_exec_error.body_json::().unwrap()["error"]["message"] - .as_str() - .unwrap() - .contains("requires a command") + assert_eq!(generation.status, 200); + let generation: serde_json::Value = generation.body_json().unwrap(); + assert_eq!(generation["generation_sequence"], 1); + assert_eq!(generation["components"][0]["component_id"], "node"); + assert!(generation["components"][0]["dependencies"] + .as_array() + .unwrap() + .is_empty()); + assert_eq!( + generation["components"][0]["layer_id"], + http_layer["layer_id"] ); - - let http_sync_error = trail::server::handle_http_request( + let mcp_generation = trail::mcp::handle_json_rpc( &mut db, - &api_request( - "POST", - "/v1/lanes/surface/dependencies/sync", - serde_json::json!({}), - ), + serde_json::json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "trail.env_generation", + "arguments": {"lane": "env-http"} + } + }), + ) + .unwrap(); + assert_eq!(mcp_generation["result"]["isError"], false); + assert_eq!( + mcp_generation["result"]["structuredContent"]["generation_id"], + generation["generation_id"] ); - assert_eq!(http_sync_error.status, 400); - let http_cache = trail::server::handle_http_request( + let view = db.lane_workspace_view("env-http").unwrap().unwrap(); + fs::write( + Path::new(&view.source_upper).join("package-lock.json"), + r#"{"name":"env-surface","version":"1.0.1","lockfileVersion":3,"requires":true,"packages":{"":{"name":"env-surface","version":"1.0.1"}}}"#, + ) + .unwrap(); + db.checkpoint_lane_workspace("env-http", Some("change lock".to_string())) + .unwrap(); + db.lane_readiness("env-http").unwrap(); + let explained = trail::server::handle_http_request( &mut db, - &api_request("GET", "/v1/cache/layers", serde_json::Value::Null), + &api_request( + "GET", + "/v1/lanes/env-http/environment/explain?component=node", + serde_json::Value::Null, + ), ); - assert_eq!(http_cache.status, 200); - assert!(http_cache - .body_json::() - .unwrap() + assert_eq!(explained.status, 200); + let explained: serde_json::Value = explained.body_json().unwrap(); + assert_eq!(explained["status"], "stale"); + assert_eq!(explained["complete"], true); + assert!(explained["changes"] .as_array() .unwrap() - .is_empty()); - let http_gc = trail::server::handle_http_request( + .iter() + .any(|change| { + change["dimension"] == "input" + && change["name"] == "package-lock.json" + && change["change"] == "modified" + })); + let mcp_explained = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 21, + "method": "tools/call", + "params": { + "name": "trail.env_explain", + "arguments": {"lane": "env-http", "component": "node"} + } + }), + ) + .unwrap(); + assert_eq!(mcp_explained["result"]["isError"], false); + assert_eq!(mcp_explained["result"]["structuredContent"], explained); + let cli_explained = run_trail_json( + temp.path(), + &["env", "explain", "env-http", "--component", "node"], + ); + assert_eq!(cli_explained, explained); + + let all_http = trail::server::handle_http_request( &mut db, &api_request( "POST", - "/v1/cache/gc", - serde_json::json!({"dry_run": true, "retention_secs": 0}), + "/v1/lanes/env-all-http/environment/sync-all", + serde_json::json!({}), ), ); - assert_eq!(http_gc.status, 200); + assert_eq!(all_http.status, 200); + let all_http: serde_json::Value = all_http.body_json().unwrap(); + assert_eq!(all_http["generation"]["generation_sequence"], 1); + assert_eq!(all_http["layers"][0]["layer_id"], http_layer["layer_id"]); + let all_mcp = trail::mcp::handle_json_rpc( + &mut db, + serde_json::json!({ + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": "trail.env_sync_all", + "arguments": {"lane": "env-all-mcp"} + } + }), + ) + .unwrap(); + assert_eq!(all_mcp["result"]["isError"], false); assert_eq!( - http_gc.body_json::().unwrap()["dry_run"], - true + all_mcp["result"]["structuredContent"]["layers"][0]["layer_id"], + http_layer["layer_id"] ); +} - let mcp_call = |db: &mut Trail, id: u64, name: &str, arguments: serde_json::Value| { - trail::mcp::handle_json_rpc( - db, - serde_json::json!({ - "jsonrpc": "2.0", - "id": id, - "method": "tools/call", - "params": {"name": name, "arguments": arguments} - }), +#[cfg(target_os = "macos")] +#[test] +fn writable_private_environment_sync_has_cli_http_mcp_and_openapi_parity() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("input.txt"), "private input\n").unwrap(); + fs::write( + temp.path().join("trail.environment.toml"), + r#"schema = "trail.environment/v1" + +[environment] +default_network = "deny" +default_scripts = "deny" + +[[component]] +id = "generated.private" +adapter = "trail/command@1" +root = "." +kind = "generated" + +[[component.input]] +path = "input.txt" +role = "identity" +format = "bytes" + +[component.build] +command = ["cp", "input.txt", "generated/copied.txt"] +cwd = "." +network = "deny" +scripts = "deny" + +[[component.output]] +name = "private" +source = "generated" +target = ".trail-generated/private" +policy = "writable_private" +portability = "host" +"#, + ) + .unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + for lane in ["private-http", "private-mcp", "private-cli"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + LaneWorkdirMode::NfsCow, + None, + None, + None, + &[], + false, ) - .unwrap() - }; - let mcp_workspace = mcp_call( + .unwrap(); + } + + let http = trail::server::handle_http_request( &mut db, - 1, - "trail.lane_workspace", - serde_json::json!({"lane": "surface"}), - ); - assert_eq!(mcp_workspace["result"]["isError"], false); - assert_eq!( - mcp_workspace["result"]["structuredContent"]["view_id"], - view.view_id + &api_request( + "POST", + "/v1/lanes/private-http/environment/sync", + serde_json::json!({"adapter": "trail/command@1"}), + ), ); - let mcp_space = mcp_call( + assert_eq!(http.status, 200); + let http: serde_json::Value = http.body_json().unwrap(); + assert!(http["layers"].as_array().unwrap().is_empty()); + let http_output = &http["generation"]["components"][0]["outputs"][0]; + assert_eq!(http_output["policy"], "writable_private"); + assert!(http_output["layer_id"].is_null()); + assert!(http_output["storage_identity"] + .as_str() + .unwrap() + .starts_with("private_")); + + let mcp = trail::mcp::handle_json_rpc( &mut db, - 2, - "trail.lane_space", - serde_json::json!({"lane": "surface"}), - ); + serde_json::json!({ + "jsonrpc": "2.0", + "id": 31, + "method": "tools/call", + "params": { + "name": "trail.env_sync", + "arguments": {"lane": "private-mcp", "adapter": "trail/command@1"} + } + }), + ) + .unwrap(); + assert_eq!(mcp["result"]["isError"], false); + let mcp = &mcp["result"]["structuredContent"]; + assert!(mcp["layers"].as_array().unwrap().is_empty()); assert_eq!( - mcp_space["result"]["structuredContent"]["view_id"], - view.view_id + mcp["generation"]["components"][0]["outputs"][0]["policy"], + http_output["policy"] ); - let mcp_dependencies = mcp_call( - &mut db, - 3, - "trail.deps_status", - serde_json::json!({"lane": "surface"}), + assert!(mcp["generation"]["components"][0]["outputs"][0]["layer_id"].is_null()); + + let cli = run_trail_json( + temp.path(), + &["env", "sync", "private-cli", "--adapter", "trail/command@1"], ); - assert!(mcp_dependencies["result"]["structuredContent"] - .as_array() - .unwrap() - .is_empty()); - let mcp_unmount = mcp_call( - &mut db, - 31, - "trail.lane_unmount", - serde_json::json!({"lane": "surface"}), + assert!(cli["layers"].as_array().unwrap().is_empty()); + assert_eq!( + cli["generation"]["components"][0]["outputs"][0]["policy"], + "writable_private" ); + assert!(cli["generation"]["components"][0]["outputs"][0]["layer_id"].is_null()); + + let openapi = trail::server::openapi_spec(); assert_eq!( - mcp_unmount["result"]["structuredContent"]["view_id"], - view.view_id + openapi["paths"]["/v1/lanes/{lane_or_id}/environment/sync"]["post"]["responses"]["200"] + ["content"]["application/json"]["schema"]["$ref"], + "#/components/schemas/EnvironmentSyncReport" ); - let mcp_checkpoint = mcp_call( - &mut db, - 4, - "trail.lane_checkpoint", - serde_json::json!({"lane": "surface", "message": "MCP checkpoint"}), + assert_eq!( + openapi["components"]["schemas"]["EnvironmentGenerationOutputReport"]["properties"] + ["policy"]["enum"][1], + "writable_private" ); - assert_eq!(mcp_checkpoint["result"]["isError"], false); assert_eq!( - mcp_checkpoint["result"]["structuredContent"]["root_id"], - http_checkpoint["root_id"] + openapi["components"]["schemas"]["EnvironmentPlanReport"]["properties"]["dependencies"] + ["items"]["type"], + "string" ); - let mcp_cache = mcp_call(&mut db, 5, "trail.cache_list", serde_json::json!({})); - assert!(mcp_cache["result"]["structuredContent"] - .as_array() - .unwrap() - .is_empty()); - let mcp_gc = mcp_call( - &mut db, - 6, - "trail.cache_gc", - serde_json::json!({"dry_run": true, "retention_secs": 0}), + assert_eq!( + openapi["components"]["schemas"]["EnvironmentGenerationComponentReport"]["properties"] + ["dependencies"]["items"]["$ref"], + "#/components/schemas/EnvironmentGenerationDependencyReport" ); - assert_eq!(mcp_gc["result"]["structuredContent"]["dry_run"], true); - let mcp_update = mcp_call( +} + +#[test] +fn environment_sync_reuses_one_node_layer_across_http_and_mcp() { + if Command::new("npm").arg("--version").output().is_err() + || Command::new("node").arg("--version").output().is_err() + { + return; + } + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join("package.json"), + r#"{"name":"env-surface","version":"1.0.0","private":true}"#, + ) + .unwrap(); + fs::write( + temp.path().join("package-lock.json"), + r#"{"name":"env-surface","version":"1.0.0","lockfileVersion":3,"requires":true,"packages":{"":{"name":"env-surface","version":"1.0.0"}}}"#, + ) + .unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + let mode = if cfg!(target_os = "macos") { + LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow + } else { + LaneWorkdirMode::FuseCow + }; + for lane in ["env-http", "env-mcp"] { + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + lane, + Some("main"), + mode.clone(), + None, + None, + None, + &[], + false, + ) + .unwrap(); + } + + let http = trail::server::handle_http_request( &mut db, - 7, - "trail.lane_update", - serde_json::json!({"lane": "surface", "source": "main"}), - ); - assert_eq!(mcp_update["result"]["isError"], false); - assert_eq!( - mcp_update["result"]["structuredContent"]["source_ref"], - "refs/branches/main" + &api_request( + "POST", + "/v1/lanes/env-http/environment/sync", + serde_json::json!({"adapter": "trail/node@1"}), + ), ); + assert_eq!(http.status, 200); + let http_report: serde_json::Value = http.body_json().unwrap(); + let http_layer = &http_report["layers"][0]; - let tools = trail::mcp::handle_json_rpc( + let mcp = trail::mcp::handle_json_rpc( &mut db, serde_json::json!({ "jsonrpc": "2.0", - "id": 8, - "method": "tools/list", - "params": {} + "id": 1, + "method": "tools/call", + "params": { + "name": "trail.env_sync", + "arguments": {"lane": "env-mcp", "adapter": "auto"} + } }), ) .unwrap(); - let tools = tools["result"]["tools"].as_array().unwrap(); - for (name, read_only, destructive, open_world) in [ - ("trail.lane_workspace", true, false, false), - ("trail.lane_mount", false, false, false), - ("trail.lane_unmount", false, false, false), - ("trail.lane_checkpoint", false, false, false), - ("trail.lane_update", false, false, false), - ("trail.lane_exec", false, false, true), - ("trail.deps_sync", false, false, true), - ("trail.cache_gc", false, true, false), - ] { - let tool = tools.iter().find(|tool| tool["name"] == name).unwrap(); - assert_eq!(tool["annotations"]["readOnlyHint"], read_only); - assert_eq!(tool["annotations"]["destructiveHint"], destructive); - assert_eq!(tool["annotations"]["openWorldHint"], open_world); - } + assert_eq!(mcp["result"]["isError"], false); + assert_eq!( + mcp["result"]["structuredContent"]["layers"][0]["layer_id"], + http_layer["layer_id"] + ); - let openapi = trail::server::handle_http_request( + let status = trail::server::handle_http_request( &mut db, - &api_request("GET", "/v1/openapi.json", serde_json::Value::Null), + &api_request( + "GET", + "/v1/lanes/env-http/environment", + serde_json::Value::Null, + ), + ); + let status: serde_json::Value = status.body_json().unwrap(); + assert_eq!(status[0]["component"]["component_id"], "node"); + assert_eq!(status[0]["adapter"]["name"], "node"); + assert_eq!(status[0]["status"], "ready"); + assert_eq!( + status[0]["adapter"]["distribution_digest"], + "builtin:node-plan-v1" ); - let openapi: serde_json::Value = openapi.body_json().unwrap(); - for path in [ - "/v1/lanes/{lane_or_id}/workspace", - "/v1/lanes/{lane_or_id}/mount", - "/v1/lanes/{lane_or_id}/unmount", - "/v1/lanes/{lane_or_id}/checkpoint", - "/v1/lanes/{lane_or_id}/update", - "/v1/lanes/{lane_or_id}/space", - "/v1/lanes/{lane_or_id}/exec", - "/v1/lanes/{lane_or_id}/dependencies", - "/v1/lanes/{lane_or_id}/dependencies/sync", - "/v1/cache/layers", - "/v1/cache/gc", - ] { - assert!( - openapi["paths"].get(path).is_some(), - "missing OpenAPI path {path}" - ); - } } #[test] @@ -9419,11 +12495,15 @@ fn daemon_no_auth_prints_loud_stderr_warning() { let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); let daemon_url = format!("http://127.0.0.1:{port}"); - assert!(stdout.contains(&format!("Trail API listening on {daemon_url}"))); + assert!(stdout.contains("Trail API daemon listening"), "{stdout}"); + assert!( + stdout.contains(&format!("Endpoint: {daemon_url}")), + "{stdout}" + ); assert!(stderr.contains("WARNING"), "{stderr}"); assert!(stderr.contains("daemon auth is disabled"), "{stderr}"); assert!( - stderr.contains("any local process can mutate this workspace"), + stderr.contains("Any local process can mutate this workspace"), "{stderr}" ); assert!(stderr.contains(&daemon_url), "{stderr}"); @@ -10022,7 +13102,7 @@ fn cli_daemon_url_routes_hot_lane_commands() { let merge = run_trail_json_daemon( temp.path(), &daemon_url, - &["merge-lane", "rpc-bot", "--into", "main", "--dry-run"], + &["lane", "merge", "rpc-bot", "--into", "main", "--dry-run"], ); assert_eq!(merge["dry_run"], true); assert!(merge["changed_paths"] @@ -10062,8 +13142,8 @@ fn cli_auto_discovers_daemon_for_hot_commands_and_falls_back_on_stale_endpoint() assert_eq!(endpoint["url"], format!("http://127.0.0.1:{port}")); assert_eq!(endpoint["auth"], true); - let status = run_trail_json(temp.path(), &["status"]); - assert_eq!(status["branch"], "main"); + let lanes = run_trail_json(temp.path(), &["lane", "list"]); + assert!(lanes.as_array().unwrap().is_empty()); wait_for_child_exit(&mut daemon.child); assert!(!endpoint_path.exists()); @@ -10078,8 +13158,8 @@ fn cli_auto_discovers_daemon_for_hot_commands_and_falls_back_on_stale_endpoint() .unwrap(), ) .unwrap(); - let fallback = run_trail_json(temp.path(), &["status"]); - assert_eq!(fallback["branch"], "main"); + let fallback = run_trail_json(temp.path(), &["lane", "list"]); + assert!(fallback.as_array().unwrap().is_empty()); } #[test] @@ -10091,12 +13171,42 @@ fn local_api_and_cli_export_openapi_contract() { let cli = run_trail_json(temp.path(), &["api", "openapi"]); assert_eq!(cli["openapi"], "3.1.0"); assert!(cli["paths"].get("/v1/openapi.json").is_some()); + assert!(cli["paths"].get("/v1/index/reconcile").is_some()); + assert_eq!( + cli["paths"]["/v1/index/reconcile"]["post"]["responses"]["200"]["content"] + ["application/json"]["schema"]["$ref"], + "#/components/schemas/ChangeLedgerReconcileReport" + ); assert!(cli["paths"]["/v1/lanes"]["get"].is_object()); assert!(cli["paths"]["/v1/lanes/{lane_or_id}"]["delete"].is_object()); assert!(cli["paths"] .get("/v1/lanes/{lane_or_id}/read-file") .is_some()); + assert_eq!( + cli["paths"]["/v1/lanes/{lane}/merge"]["post"]["operationId"], + "laneMerge" + ); + assert!(cli["paths"] + .get("/v1/branches/{branch}/merge-lane") + .is_none()); + assert_eq!( + cli["components"]["schemas"]["LaneMergeRequest"]["required"], + serde_json::json!(["into"]) + ); + assert!(cli["components"]["schemas"] + .get("MergeLaneRequest") + .is_none()); assert!(cli["components"]["schemas"]["LaneReadFileRequest"].is_object()); + let workdir_modes = cli["components"]["schemas"]["SpawnLaneRequest"]["properties"] + ["workdir_mode"]["enum"] + .as_array() + .unwrap(); + assert!(workdir_modes.iter().any(|mode| mode == "native-cow")); + assert!(workdir_modes.iter().any(|mode| mode == "portable-copy")); + assert!(workdir_modes.iter().any(|mode| mode == "fuse-cow")); + assert!(workdir_modes.iter().any(|mode| mode == "dokan-cow")); + assert!(!workdir_modes.iter().any(|mode| mode == "full-cow")); + assert!(!workdir_modes.iter().any(|mode| mode == "overlay-cow")); assert!(cli["paths"].get("/v1/lane/events").is_some()); assert!(cli["paths"].get("/v1/lane/spans").is_some()); assert!(cli["paths"].get("/v1/lane/turns/{turn_id}/spans").is_some()); @@ -10178,14 +13288,12 @@ fn local_api_and_cli_export_openapi_contract() { patch_request_modes[1]["not"]["required"], serde_json::json!(["edits"]) ); - assert_eq!( - patch_request["properties"]["edits"]["minItems"], - serde_json::json!(1) - ); - assert_eq!( - patch_request["properties"]["files"]["minItems"], - serde_json::json!(1) - ); + assert!(patch_request["properties"]["edits"] + .get("minItems") + .is_none()); + assert!(patch_request["properties"]["files"] + .get("minItems") + .is_none()); assert_eq!( patch_request["properties"]["edits"]["items"]["$ref"], "#/components/schemas/PatchEdit" @@ -10567,6 +13675,12 @@ fn mcp_stdio_tools_drive_lane_turn_workflow() { assert!(resources_list .iter() .any(|resource| resource["uri"] == "trail://workspace/status")); + assert!(resources_list + .iter() + .any(|resource| resource["uri"] == "trail://workspace/lane-merge-queue")); + assert!(!resources_list + .iter() + .any(|resource| resource["uri"] == "trail://workspace/merge-queue")); assert!(resources_list .iter() .any(|resource| resource["uri"] == "trail://workspace/agent-tasks")); @@ -11167,12 +14281,16 @@ fn mcp_stdio_tools_drive_lane_turn_workflow() { .as_bool() .unwrap() }; - assert!(tool_annotation("trail.status", "readOnlyHint")); + assert!(!tool_annotation("trail.status", "readOnlyHint")); assert!(!tool_annotation("trail.status", "destructiveHint")); + assert!(tool_annotation("trail.status", "idempotentHint")); assert!(!tool_annotation("trail.apply_patch", "readOnlyHint")); assert!(tool_annotation("trail.apply_patch", "destructiveHint")); assert!(tool_annotation("trail.lane_rewind", "destructiveHint")); - assert!(tool_annotation("trail.merge_queue_run", "destructiveHint")); + assert!(tool_annotation( + "trail.lane_merge_queue_run", + "destructiveHint" + )); assert!(tool_annotation("trail.run_test", "openWorldHint")); assert!(tool_annotation("trail.guardrail_check", "readOnlyHint")); assert!(tool_annotation("trail.lane_review", "readOnlyHint")); @@ -14858,7 +17976,7 @@ fn mcp_stdio_tools_drive_lane_turn_workflow() { }), ) .unwrap(); - assert_eq!(sync["result"]["isError"], false); + assert_eq!(sync["result"]["isError"], false, "{sync:#}"); assert_eq!(sync["result"]["structuredContent"]["forced"], true); let test = trail::mcp::handle_json_rpc( @@ -14982,7 +18100,7 @@ fn mcp_stdio_tools_drive_lane_turn_workflow() { } #[test] -fn mcp_status_read_only_does_not_refresh_worktree_index() { +fn mcp_status_refreshes_index_while_status_resource_remains_read_only() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("a.txt"), "a1\n").unwrap(); fs::write(temp.path().join("b.txt"), "b1\n").unwrap(); @@ -15024,7 +18142,7 @@ fn mcp_status_read_only_does_not_refresh_worktree_index() { assert!(changed_paths.iter().any(|path| path["path"] == "a.txt")); assert!(changed_paths.iter().any(|path| path["path"] == "c.txt")); - assert_eq!(count_rows("worktree_file_index"), index_before); + assert_eq!(count_rows("worktree_file_index"), index_before + 1); assert_eq!(count_rows("objects"), objects_before); assert_eq!(count_rows("prolly_nodes"), prolly_nodes_before); @@ -15048,11 +18166,26 @@ fn mcp_status_read_only_does_not_refresh_worktree_index() { assert!(resource_paths.iter().any(|path| path["path"] == "a.txt")); assert!(resource_paths.iter().any(|path| path["path"] == "c.txt")); - assert_eq!(count_rows("worktree_file_index"), index_before); + assert_eq!(count_rows("worktree_file_index"), index_before + 1); assert_eq!(count_rows("objects"), objects_before); assert_eq!(count_rows("prolly_nodes"), prolly_nodes_before); } +#[test] +fn changed_path_activation_is_checked_and_linux_macos_only() { + let evidence = trail::test_support::changed_path_activation_evidence().unwrap(); + assert!(evidence["schema_hard_cutover"].as_bool().unwrap()); + assert!(evidence["producer_inventory_complete"].as_bool().unwrap()); + assert!(evidence["crash_matrix"].as_bool().unwrap()); + assert!(evidence["corruption_matrix"].as_bool().unwrap()); + assert!(evidence["scale_gates"].as_bool().unwrap()); + assert_eq!( + trail::test_support::changed_path_production_authority_default(), + cfg!(any(target_os = "linux", target_os = "macos")) + ); + assert!(!trail::test_support::changed_path_authority_enabled_for("windows").unwrap()); +} + #[test] fn mcp_stdio_reports_parse_errors_and_continues() { let temp = tempfile::tempdir().unwrap(); @@ -15753,10 +18886,7 @@ fn same_position_rewrite_preserves_line_identity() { .iter() .any(|entry| entry.kind == trail::LineChangeKind::Modified)); - let line_id = format!( - "{}:{}", - before.line_id.origin_change.0, before.line_id.local_seq - ); + let line_id = before.line_id.alias(); let cli_by_line = run_trail_json(temp.path(), &["why", "--line-id", &line_id, "--at", "main"]); assert_eq!(cli_by_line["current_text"], "lane rewrote this line"); let cli_at_root = run_trail_json( @@ -16085,6 +19215,7 @@ fn anchors_follow_stable_line_identity() { let created = db .create_anchor("note.txt:2", "important line", Some("main")) .unwrap(); + assert!(created.anchor.id.0.starts_with("anchor_")); assert_eq!(db.list_anchors().unwrap().len(), 1); let resolved = db .resolve_anchor(&created.anchor.id.0, Some("main")) @@ -16146,8 +19277,12 @@ fn local_api_and_mcp_expose_review_provenance_and_anchors() { let why: serde_json::Value = why.body_json().unwrap(); assert_eq!(why["current_text"], "lane"); let line_id = format!( - "{}:{}", - why["line_id"]["origin_change"].as_str().unwrap(), + "line_{}:{}", + why["line_id"]["origin_change"] + .as_str() + .unwrap() + .strip_prefix("change_") + .unwrap(), why["line_id"]["local_seq"].as_u64().unwrap() ); @@ -16406,6 +19541,14 @@ fn refish_aliases_accept_branch_lane_and_root_selectors() { assert_eq!(root_preview.change_id, branch.from); assert_eq!(root_preview.root_id, branch.root_id); + let checkpoint_selector = branch.from.checkpoint_alias(); + let checkpoint_preview = db + .checkout_with_options(&checkpoint_selector, true, true, None, false) + .unwrap(); + assert!(checkpoint_preview.dry_run); + assert_eq!(checkpoint_preview.change_id, branch.from); + assert_eq!(checkpoint_preview.root_id, branch.root_id); + let raw_root = branch.root_id.0.clone(); drop(db); @@ -16756,7 +19899,8 @@ fn merge_dry_run_reports_without_mutating_refs() { let cli = run_trail_json( temp.path(), &[ - "merge-lane", + "lane", + "merge", "doc-bot", "--strategy", "line-id-aware", @@ -16770,9 +19914,9 @@ fn merge_dry_run_reports_without_mutating_refs() { &mut db, &api_request( "POST", - "/v1/branches/main/merge-lane", + "/v1/lanes/doc-bot/merge", serde_json::json!({ - "lane": "doc-bot", + "into": "main", "dry_run": true }), ), @@ -16781,6 +19925,19 @@ fn merge_dry_run_reports_without_mutating_refs() { let api: serde_json::Value = api_response.body_json().unwrap(); assert_eq!(api["dry_run"], true); + let legacy_response = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/branches/main/merge-lane", + serde_json::json!({ + "lane": "doc-bot", + "dry_run": true + }), + ), + ); + assert_eq!(legacy_response.status, 400); + let dry_run = db.merge_lane_with_options("doc-bot", "main", true).unwrap(); assert!(dry_run.dry_run); assert_eq!(dry_run.changed_paths.len(), 1); @@ -16809,7 +19966,7 @@ fn merge_dry_run_reports_without_mutating_refs() { } #[test] -fn user_facing_merge_lane_prefers_queue_for_default_branch() { +fn user_facing_lane_merge_prefers_queue_for_default_branch() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); @@ -16829,7 +19986,7 @@ fn user_facing_merge_lane_prefers_queue_for_default_branch() { .arg("--workspace") .arg(temp.path()) .arg("--json") - .args(["merge-lane", "cli-bot", "--into", "main"]) + .args(["lane", "merge", "cli-bot", "--into", "main"]) .output() .unwrap(); assert!(!blocked.status.success()); @@ -16837,12 +19994,12 @@ fn user_facing_merge_lane_prefers_queue_for_default_branch() { assert!(blocked_stderr["error"]["message"] .as_str() .unwrap() - .contains("merge-queue add cli-bot --into main")); + .contains("lane merge-queue add cli-bot --into main")); assert_eq!(db.status(Some("main")).unwrap().changed_paths.len(), 0); let direct = run_trail_json( temp.path(), - &["merge-lane", "cli-bot", "--into", "main", "--direct"], + &["lane", "merge", "cli-bot", "--into", "main", "--direct"], ); assert_eq!(direct["dry_run"], false); assert_eq!(direct["changed_paths"][0]["path"], "docs/cli.md"); @@ -16860,8 +20017,8 @@ fn user_facing_merge_lane_prefers_queue_for_default_branch() { &mut db, &api_request( "POST", - "/v1/branches/main/merge-lane", - serde_json::json!({ "lane": "api-bot" }), + "/v1/lanes/api-bot/merge", + serde_json::json!({ "into": "main" }), ), ); assert_eq!(blocked_api.status, 400); @@ -16869,14 +20026,14 @@ fn user_facing_merge_lane_prefers_queue_for_default_branch() { assert!(blocked_api_body["error"]["message"] .as_str() .unwrap() - .contains("merge-queue add api-bot --into main")); + .contains("lane merge-queue add api-bot --into main")); let direct_api = trail::server::handle_http_request( &mut db, &api_request( "POST", - "/v1/branches/main/merge-lane", - serde_json::json!({ "lane": "api-bot", "direct": true }), + "/v1/lanes/api-bot/merge", + serde_json::json!({ "into": "main", "direct": true }), ), ); assert_eq!(direct_api.status, 200); @@ -16918,12 +20075,9 @@ fn lane_diff_cli_renders_scannable_overview() { ); let stdout = String::from_utf8_lossy(&output.stdout); assert!(stdout.contains("Lane diff: doc-bot")); - assert!(stdout.contains("from:")); - assert!(stdout.contains("to:")); - assert!(stdout.contains("2 file(s) changed, +2 -0")); - assert!(stdout.contains("M modified README.md (+1 -0) +")); - assert!(stdout.contains("A added docs/guide.md (+1 -0) +")); - assert!(stdout.contains("2 files changed, 2 additions, 0 deletions")); + assert!(stdout.contains("M README.md +1 -0"), "{stdout}"); + assert!(stdout.contains("A docs/guide.md +1 -0"), "{stdout}"); + assert!(stdout.contains("2 files changed, 2 insertions, 0 deletions")); } #[test] @@ -16934,8 +20088,10 @@ fn layered_lane_update_preserves_lane_changes_and_advances_its_pinned_base() { let mut db = Trail::open(temp.path()).unwrap(); let mode = if cfg!(target_os = "macos") { LaneWorkdirMode::NfsCow + } else if cfg!(target_os = "windows") { + LaneWorkdirMode::DokanCow } else { - LaneWorkdirMode::OverlayCow + LaneWorkdirMode::FuseCow }; db.spawn_lane_with_workdir_mode_paths_and_neighbors( "updatable", @@ -17205,7 +20361,7 @@ fn lane_status_surfaces_stale_base_lag() { assert!(cli_text.status.success()); let stdout = String::from_utf8_lossy(&cli_text.stdout); assert!( - stdout.contains("Lane started 2 operations behind main"), + stdout.contains("Base: 2 operation(s) behind main"), "{stdout}" ); } @@ -17245,7 +20401,7 @@ fn merge_dry_run_reports_conflicts_without_opening_conflict_state() { ); assert_eq!(db.lane_details("doc-bot").unwrap().branch.status, "active"); - let cli = run_trail_json(temp.path(), &["merge-lane", "doc-bot", "--dry-run"]); + let cli = run_trail_json(temp.path(), &["lane", "merge", "doc-bot", "--dry-run"]); assert_eq!(cli["dry_run"], true); assert_eq!(cli["conflicts"][0], "both changed `README.md` differently"); assert!(db.list_conflicts().unwrap().is_empty()); @@ -17319,13 +20475,14 @@ fn local_api_direct_merge_lane_conflict_records_conflict_set() { &mut db, &api_request( "POST", - "/v1/branches/main/merge-lane", - serde_json::json!({ "lane_id": "api-bot", "direct": true }), + "/v1/lanes/api-bot/merge", + serde_json::json!({ "into": "main", "direct": true }), ), ); assert_eq!(response.status, 409); let body: serde_json::Value = response.body_json().unwrap(); - assert_eq!(body["error"]["code"], 6); + assert_eq!(body["error"]["code"], "MERGE_CONFLICT"); + assert_eq!(body["error"]["exit"], 6); assert!(body["error"]["message"] .as_str() .unwrap() @@ -17352,7 +20509,7 @@ fn lane_patch_can_replace_stable_line_with_expected_text() { db.spawn_lane("doc-bot", Some("main"), false, None, None) .unwrap(); let why = db.why("README.md:2", Some("refs/lanes/doc-bot")).unwrap(); - let line_id = format!("{}:{}", why.line_id.origin_change.0, why.line_id.local_seq); + let line_id = why.line_id.alias(); let missing_expected: PatchDocument = serde_json::from_value(serde_json::json!({ "message": "line-id patch without guard", "edits": [{ @@ -17403,7 +20560,7 @@ fn lane_patch_can_replace_stable_line_with_expected_text() { "edits": [{ "op": "replace_line", "path": "README.md", - "line_id": format!("{}:{}", why.line_id.origin_change.0, why.line_id.local_seq), + "line_id": why.line_id.alias(), "expected_text": "two", "new_text": "stale" }] @@ -17411,7 +20568,7 @@ fn lane_patch_can_replace_stable_line_with_expected_text() { .unwrap(); let err = apply_lane_patch_at_head(&mut db, "doc-bot", stale_patch).unwrap_err(); assert!(matches!(err, Error::PatchRejected(_))); - assert!(err.to_string().contains("expected text mismatch")); + assert!(err.to_string().contains("expected text mismatch"), "{err}"); } #[test] @@ -17429,14 +20586,8 @@ fn lane_patch_rejected_line_id_batch_rolls_back_candidate_objects() { let second = db .why("README.md:2", Some("refs/lanes/atomic-line-bot")) .unwrap(); - let first_line_id = format!( - "{}:{}", - first.line_id.origin_change.0, first.line_id.local_seq - ); - let second_line_id = format!( - "{}:{}", - second.line_id.origin_change.0, second.line_id.local_seq - ); + let first_line_id = first.line_id.alias(); + let second_line_id = second.line_id.alias(); let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); let count_rows = |table: &str| -> i64 { @@ -17470,7 +20621,7 @@ fn lane_patch_rejected_line_id_batch_rolls_back_candidate_objects() { .unwrap(); let err = apply_lane_patch_at_head(&mut db, "atomic-line-bot", patch).unwrap_err(); assert!(matches!(err, Error::PatchRejected(_))); - assert!(err.to_string().contains("expected text mismatch")); + assert!(err.to_string().contains("expected text mismatch"), "{err}"); assert_eq!(count_rows("objects"), objects_before); assert_eq!(count_rows("prolly_nodes"), prolly_nodes_before); @@ -17557,7 +20708,7 @@ fn lane_patch_replace_line_fuzzes_batch_expected_text_edits() { Some("refs/lanes/line-fuzz-bot"), ) .unwrap(); - let line_id = format!("{}:{}", why.line_id.origin_change.0, why.line_id.local_seq); + let line_id = why.line_id.alias(); expected_ids.push(why.line_id); edits.push(serde_json::json!({ "op": "replace_line", @@ -18747,6 +21898,42 @@ fn materialized_lane_status_and_record_handle_workdir_renames() { ); } +#[test] +fn materialized_lane_record_allows_case_only_rename_when_final_root_is_safe() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + let spawned = db + .spawn_lane("record-case-rename", Some("main"), true, None, None) + .unwrap(); + let workdir = PathBuf::from(spawned.workdir.unwrap()); + fs::rename(workdir.join("README.md"), workdir.join("rename-staging")).unwrap(); + fs::rename(workdir.join("rename-staging"), workdir.join("readme.md")).unwrap(); + + let preview = db + .preview_lane_workdir_record("record-case-rename") + .unwrap(); + assert!(preview.policy.allowed, "{:?}", preview.policy.error); + let recorded = db + .record_lane_workdir( + "record-case-rename", + Some("record case-only rename".to_string()), + ) + .unwrap(); + assert_eq!(recorded.changed_paths.len(), 1); + assert_eq!( + recorded.changed_paths[0].kind, + trail::FileChangeKind::Renamed + ); + assert_eq!( + recorded.changed_paths[0].old_path.as_deref(), + Some("README.md") + ); + assert_eq!(recorded.changed_paths[0].path, "readme.md"); +} + #[test] fn lane_workdir_record_preview_reports_risks_and_oversized_files() { let temp = tempfile::tempdir().unwrap(); @@ -18964,10 +22151,11 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { &["lane", "spawn", "default-bot", "--from", "main"], ); assert!(default_spawn["workdir"].is_null()); + assert_eq!(default_spawn["requested_workdir_mode"], "virtual"); assert_eq!(default_spawn["workdir_mode"], "virtual"); - assert!(default_spawn["cow_backend"].is_null()); + assert_eq!(default_spawn["workdir_backend"], "virtual"); assert_eq!(default_spawn["sparse_paths"].as_array().unwrap().len(), 0); - assert_eq!(default_spawn["overlay_available"], false); + assert_eq!(default_spawn["transparent_cow_available"], false); let cli_workdir = workdir_parent.path().join("cli-bot"); let cli_spawn = run_trail_json( @@ -18982,8 +22170,16 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { cli_workdir.to_str().unwrap(), ], ); - assert_eq!(cli_spawn["workdir_mode"], "full-cow"); - assert_eq!(cli_spawn["cow_backend"], "filesystem-clone"); + assert_eq!(cli_spawn["requested_workdir_mode"], "auto"); + assert_eq!(cli_spawn["workdir_mode"], "native-cow"); + assert_eq!(cli_spawn["workdir_backend"], "clone"); + assert_eq!(cli_spawn["materialization"]["copied_files"], 0); + assert!( + cli_spawn["materialization"]["cloned_files"] + .as_u64() + .unwrap() + > 0 + ); assert_eq!( PathBuf::from(cli_spawn["workdir"].as_str().unwrap()) .canonicalize() @@ -18995,6 +22191,46 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { "hello\n" ); + let native_workdir = workdir_parent.path().join("native-bot"); + let native_spawn = run_trail_json( + temp.path(), + &[ + "lane", + "spawn", + "native-bot", + "--from", + "main", + "--workdir-mode", + "native-cow", + "--workdir", + native_workdir.to_str().unwrap(), + ], + ); + assert_eq!(native_spawn["requested_workdir_mode"], "native-cow"); + assert_eq!(native_spawn["workdir_mode"], "native-cow"); + assert_eq!(native_spawn["workdir_backend"], "clone"); + assert_eq!(native_spawn["materialization"]["copied_files"], 0); + + let portable_workdir = workdir_parent.path().join("portable-bot"); + let portable_spawn = run_trail_json( + temp.path(), + &[ + "lane", + "spawn", + "portable-bot", + "--from", + "main", + "--workdir-mode", + "portable-copy", + "--workdir", + portable_workdir.to_str().unwrap(), + ], + ); + assert_eq!(portable_spawn["requested_workdir_mode"], "portable-copy"); + assert_eq!(portable_spawn["workdir_mode"], "portable-copy"); + assert_eq!(portable_spawn["workdir_backend"], "clone"); + assert_eq!(portable_spawn["materialization"]["copied_files"], 0); + let headless_spawn = run_trail_json( temp.path(), &[ @@ -19023,38 +22259,53 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { assert!(no_materialize_spawn["workdir"].is_null()); assert_eq!(no_materialize_spawn["workdir_mode"], "virtual"); - #[cfg(any( - target_os = "linux", - target_os = "windows", - all(target_os = "macos", feature = "macfuse") - ))] + #[cfg(any(target_os = "linux", all(target_os = "macos", feature = "macfuse")))] { - let overlay_spawn = run_trail_json( + let fuse_spawn = run_trail_json( temp.path(), &[ "lane", "spawn", - "overlay-bot", + "fuse-bot", "--from", "main", "--workdir-mode", - "overlay-cow", + "fuse-cow", ], ); - assert_eq!(overlay_spawn["workdir_mode"], "overlay-cow"); - assert_eq!(overlay_spawn["cow_backend"], "overlay"); - assert_eq!(overlay_spawn["overlay_available"], true); - assert_eq!(overlay_spawn["sparse_paths"].as_array().unwrap().len(), 0); - let overlay_workdir = PathBuf::from(overlay_spawn["workdir"].as_str().unwrap()); - assert!(overlay_workdir.is_dir()); - assert!(fs::read_dir(&overlay_workdir).unwrap().next().is_none()); - assert!(!overlay_workdir.join("README.md").exists()); + assert_eq!(fuse_spawn["workdir_mode"], "fuse-cow"); + assert_eq!(fuse_spawn["workdir_backend"], "fuse"); + assert_eq!(fuse_spawn["transparent_cow_available"], true); + assert_eq!(fuse_spawn["sparse_paths"].as_array().unwrap().len(), 0); + let fuse_workdir = PathBuf::from(fuse_spawn["workdir"].as_str().unwrap()); + assert!(fuse_workdir.is_dir()); + assert!(fs::read_dir(&fuse_workdir).unwrap().next().is_none()); + assert!(!fuse_workdir.join("README.md").exists()); let db = Trail::open(temp.path()).unwrap(); - let view = db.lane_workspace_view("overlay-bot").unwrap().unwrap(); + let view = db.lane_workspace_view("fuse-bot").unwrap().unwrap(); assert!(Path::new(&view.source_upper).is_dir()); assert!(Path::new(&view.meta_dir).is_dir()); } + #[cfg(target_os = "windows")] + { + let dokan_spawn = run_trail_json( + temp.path(), + &[ + "lane", + "spawn", + "dokan-bot", + "--from", + "main", + "--workdir-mode", + "dokan-cow", + ], + ); + assert_eq!(dokan_spawn["workdir_mode"], "dokan-cow"); + assert_eq!(dokan_spawn["workdir_backend"], "dokan"); + assert_eq!(dokan_spawn["transparent_cow_available"], true); + } + #[cfg(target_os = "macos")] { let nfs_spawn = run_trail_json( @@ -19070,8 +22321,8 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { ], ); assert_eq!(nfs_spawn["workdir_mode"], "nfs-cow"); - assert_eq!(nfs_spawn["cow_backend"], "nfs-overlay"); - assert_eq!(nfs_spawn["overlay_available"], true); + assert_eq!(nfs_spawn["workdir_backend"], "nfs"); + assert_eq!(nfs_spawn["transparent_cow_available"], true); let db = Trail::open(temp.path()).unwrap(); let view = db.lane_workspace_view("nfs-bot").unwrap().unwrap(); assert!(Path::new(&view.source_upper).is_dir()); @@ -19083,6 +22334,17 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { db.lane_status("cli-bot").unwrap().workdir_state, Some(WorktreeState::Clean) ); + let persisted_workdir = db.lane_workdir("cli-bot").unwrap(); + assert_eq!( + persisted_workdir.requested_workdir_mode, + LaneWorkdirMode::Auto + ); + assert_eq!(persisted_workdir.workdir_mode, LaneWorkdirMode::NativeCow); + assert_eq!( + persisted_workdir.workdir_backend, + Some(WorkdirBackend::Clone) + ); + assert!(persisted_workdir.materialization.is_some()); let sparse_spawn = run_trail_json( temp.path(), @@ -19097,7 +22359,9 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { ], ); assert_eq!(sparse_spawn["workdir_mode"], "sparse"); - assert_eq!(sparse_spawn["cow_backend"], "filesystem-clone"); + assert_eq!(sparse_spawn["workdir_backend"], "clone"); + assert_eq!(sparse_spawn["materialization"]["cloned_files"], 1); + assert_eq!(sparse_spawn["materialization"]["copied_files"], 0); assert_eq!( sparse_spawn["sparse_paths"] .as_array() @@ -19295,7 +22559,7 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { assert_eq!(sparse_hydrated.workdir_state, Some(WorktreeState::Clean)); assert!(sparse_hydrated.workdir_changed_paths.is_empty()); let sparse_manifest: serde_json::Value = serde_json::from_str( - &fs::read_to_string(sparse_workdir.join(".trail/sparse-workdir.json")).unwrap(), + &fs::read_to_string(sparse_workdir.join(".trail/sparse-selection.json")).unwrap(), ) .unwrap(); let sparse_manifest_paths = sparse_manifest["materialized_paths"] @@ -19394,14 +22658,14 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { serde_json::json!({ "name": "api-bot", "from_ref": "main", - "workdir_mode": "full-cow", + "workdir_mode": "native-cow", "workdir": api_workdir }), ), ); assert_eq!(api_response.status, 201); let api_spawn: serde_json::Value = api_response.body_json().unwrap(); - assert_eq!(api_spawn["workdir_mode"], "full-cow"); + assert_eq!(api_spawn["workdir_mode"], "native-cow"); assert_eq!( PathBuf::from(api_spawn["workdir"].as_str().unwrap()) .canonicalize() @@ -19513,7 +22777,7 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { "arguments": { "name": "mcp-bot", "from_ref": "main", - "workdir_mode": "full-cow", + "workdir_mode": "native-cow", "workdir": mcp_workdir } } @@ -19523,7 +22787,7 @@ fn lane_spawn_supports_custom_and_configured_workdirs() { assert_eq!(mcp_spawn["result"]["isError"], false); assert_eq!( mcp_spawn["result"]["structuredContent"]["workdir_mode"], - "full-cow" + "native-cow" ); assert_eq!( PathBuf::from( @@ -19646,19 +22910,15 @@ fn sparse_lane_path_sync_rolls_back_hydrated_files_when_manifest_update_fails() assert!(sparse_workdir.join("README.md").is_file()); assert!(!sparse_workdir.join("src/lib.rs").exists()); - let sparse_manifest_path = sparse_workdir.join(".trail/sparse-workdir.json"); + let sparse_manifest_path = sparse_workdir.join(".trail/sparse-selection.json"); let clean_manifest_path = sparse_workdir.join(".trail/workdir-manifest.json"); let sparse_manifest_before = fs::read(&sparse_manifest_path).unwrap(); let clean_manifest_before = fs::read(&clean_manifest_path).unwrap(); - let metadata_dir = sparse_workdir.join(".trail"); - let original_permissions = fs::metadata(&metadata_dir).unwrap().permissions(); - let mut readonly_permissions = original_permissions.clone(); - readonly_permissions.set_mode(0o555); - fs::set_permissions(&metadata_dir, readonly_permissions).unwrap(); + trail::test_support::set_sparse_selection_write_failure_for_current_thread(true); let sync_result = db.sync_lane_workdir_with_paths("sparse-rollback", false, &["src/lib.rs".to_string()]); - fs::set_permissions(&metadata_dir, original_permissions).unwrap(); + trail::test_support::set_sparse_selection_write_failure_for_current_thread(false); let err = sync_result.unwrap_err(); assert!(matches!(err, Error::Io(_))); @@ -19704,7 +22964,7 @@ fn sparse_lane_path_sync_recovers_when_sparse_manifest_is_missing() { assert!(sparse_workdir.join("README.md").is_file()); assert!(!sparse_workdir.join("src/lib.rs").exists()); - let sparse_manifest_path = sparse_workdir.join(".trail/sparse-workdir.json"); + let sparse_manifest_path = sparse_workdir.join(".trail/sparse-selection.json"); fs::remove_file(&sparse_manifest_path).unwrap(); let report = db .sync_lane_workdir_with_paths( @@ -19812,6 +23072,16 @@ fn lane_spawn_materialization_ignores_dirty_workspace_for_recorded_root() { let spawned = db .spawn_lane("doc-bot", Some("main"), true, None, None) .unwrap(); + assert_eq!(spawned.requested_workdir_mode, LaneWorkdirMode::Auto); + assert_eq!(spawned.workdir_mode, LaneWorkdirMode::PortableCopy); + assert_eq!(spawned.workdir_backend, Some(WorkdirBackend::Copy)); + assert_eq!( + spawned + .materialization + .as_ref() + .and_then(|report| report.fallback_reason), + Some(MaterializationFallbackReason::NativeSourceUnavailable) + ); let workdir = PathBuf::from(spawned.workdir.unwrap()); assert_eq!( @@ -19824,6 +23094,183 @@ fn lane_spawn_materialization_ignores_dirty_workspace_for_recorded_root() { ); } +#[test] +fn auto_reports_mixed_when_portable_restart_can_clone_only_clean_files() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("clean.txt"), "clean\n").unwrap(); + fs::write(temp.path().join("dirty.txt"), "recorded\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + fs::write(temp.path().join("dirty.txt"), "changed\n").unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + let spawned = db + .spawn_lane("mixed-bot", Some("main"), true, None, None) + .unwrap(); + let report = spawned.materialization.unwrap(); + assert_eq!(spawned.workdir_mode, LaneWorkdirMode::PortableCopy); + assert_eq!(spawned.workdir_backend, Some(WorkdirBackend::Mixed)); + assert_eq!(report.cloned_files, 1); + assert_eq!(report.copied_files, 1); + assert_eq!( + report.fallback_reason, + Some(MaterializationFallbackReason::NativeSourceUnavailable) + ); + let workdir = PathBuf::from(spawned.workdir.unwrap()); + assert_eq!( + fs::read_to_string(workdir.join("clean.txt")).unwrap(), + "clean\n" + ); + assert_eq!( + fs::read_to_string(workdir.join("dirty.txt")).unwrap(), + "recorded\n" + ); +} + +#[test] +fn strict_native_cow_refuses_an_unvalidated_source_without_copying() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + fs::write(temp.path().join("README.md"), "hello\ndirty\n").unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + let workdir_parent = tempfile::tempdir().unwrap(); + let destination = workdir_parent.path().join("strict-workdir"); + let error = db + .spawn_lane_with_workdir_mode_paths_and_neighbors( + "strict-bot", + Some("main"), + LaneWorkdirMode::NativeCow, + None, + None, + Some(destination.clone()), + &[], + false, + ) + .unwrap_err(); + + assert!(matches!(error, Error::NativeCowSourceUnavailable)); + assert!(!destination.exists()); + assert!(db.lane_details("strict-bot").is_err()); +} + +#[test] +fn strict_native_cow_reuses_a_complete_clean_lane_source() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("source-bot", Some("main"), true, None, None) + .unwrap(); + fs::write(temp.path().join("README.md"), "hello\ndirty\n").unwrap(); + assert_eq!( + db.lane_status("source-bot").unwrap().workdir_state, + Some(WorktreeState::Clean) + ); + + let spawned = db + .spawn_lane_with_workdir_mode_paths_and_neighbors( + "strict-from-lane", + Some("main"), + LaneWorkdirMode::NativeCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + assert_eq!(spawned.workdir_mode, LaneWorkdirMode::NativeCow); + assert_eq!(spawned.workdir_backend, Some(WorkdirBackend::Clone)); + assert_eq!( + fs::read_to_string(PathBuf::from(spawned.workdir.unwrap()).join("README.md")).unwrap(), + "hello\n" + ); +} + +#[cfg(unix)] +#[test] +fn strict_native_cow_does_not_preserve_source_hardlink_aliasing() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("a.txt"), "shared\n").unwrap(); + fs::hard_link(temp.path().join("a.txt"), temp.path().join("b.txt")).unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + let spawned = db + .spawn_lane_with_workdir_mode_paths_and_neighbors( + "hardlink-bot", + Some("main"), + LaneWorkdirMode::NativeCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let workdir = PathBuf::from(spawned.workdir.unwrap()); + let a = fs::metadata(workdir.join("a.txt")).unwrap(); + let b = fs::metadata(workdir.join("b.txt")).unwrap(); + assert_ne!(a.ino(), b.ino()); + fs::write(workdir.join("a.txt"), "changed\n").unwrap(); + assert_eq!( + fs::read_to_string(workdir.join("b.txt")).unwrap(), + "shared\n" + ); +} + +#[test] +fn strict_native_cow_probes_an_empty_root() { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + let spawned = db + .spawn_lane_with_workdir_mode_paths_and_neighbors( + "empty-bot", + Some("main"), + LaneWorkdirMode::NativeCow, + None, + None, + None, + &[], + false, + ) + .unwrap(); + assert_eq!(spawned.workdir_mode, LaneWorkdirMode::NativeCow); + assert_eq!(spawned.workdir_backend, Some(WorkdirBackend::Clone)); + assert_eq!(spawned.materialization.unwrap().cloned_files, 0); +} + +#[test] +fn legacy_cow_backend_is_not_treated_as_strict_clone_evidence() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("legacy-bot", Some("main"), true, None, None) + .unwrap(); + drop(db); + + let connection = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); + connection + .execute( + "UPDATE lanes SET metadata_json = ?1 WHERE name = 'legacy-bot'", + [r#"{"workdir_mode":"native-cow","cow_backend":"clone"}"#], + ) + .unwrap(); + drop(connection); + + let db = Trail::open(temp.path()).unwrap(); + let report = db.lane_workdir("legacy-bot").unwrap(); + assert_eq!(report.workdir_mode, LaneWorkdirMode::NativeCow); + assert_eq!(report.requested_workdir_mode, LaneWorkdirMode::NativeCow); + assert_eq!(report.workdir_backend, None); + assert_eq!(report.materialization, None); +} + #[test] fn lane_workdir_sync_refuses_dirty_and_force_refreshes() { let temp = tempfile::tempdir().unwrap(); @@ -20083,8 +23530,8 @@ fn dirty_lane_workdir_must_be_recorded_before_merge() { assert!(matches!(err, Error::DirtyWorktreeWithMessage(_))); assert!(err.to_string().contains("lane record doc-bot")); - db.enqueue_merge("doc-bot", "main", 0).unwrap(); - let run = db.run_merge_queue(None).unwrap(); + db.enqueue_lane_merge("doc-bot", "main", 0).unwrap(); + let run = db.run_lane_merge_queue(None).unwrap(); assert_eq!(run.processed.len(), 1); assert_eq!(run.processed[0].status, "failed"); assert!(run.stopped_on_failure); @@ -20364,7 +23811,7 @@ fn sparse_lane_path_enforcement_blocks_patch_and_record_outside_selected_paths() let err = apply_lane_patch_at_head(&mut db, "sparse-bot", blocked_rename).unwrap_err(); assert!(matches!(err, Error::PatchRejected(message) if message.contains("src/renamed.md"))); - let sparse_manifest = workdir.join(".trail/sparse-workdir.json"); + let sparse_manifest = workdir.join(".trail/sparse-selection.json"); fs::remove_file(&sparse_manifest).unwrap(); let blocked_without_manifest: PatchDocument = serde_json::from_value(serde_json::json!({ "edits": [ @@ -21049,7 +24496,7 @@ fn local_http_bodyless_mutations_reject_request_bodies() { .create_anchor("README.md:1", "stable readme", Some("main")) .unwrap() .anchor; - let queued = db.enqueue_merge("doc-bot", "main", 0).unwrap().entry; + let queued = db.enqueue_lane_merge("doc-bot", "main", 0).unwrap().entry; let assert_rejected = |response: &trail::server::HttpResponse, endpoint: &str| { assert_eq!(response.status, 400); @@ -21096,13 +24543,16 @@ fn local_http_bodyless_mutations_reject_request_bodies() { &mut db, &api_request( "DELETE", - &format!("/v1/merge-queue/{}", queued.queue_id), + &format!("/v1/lanes/merges/queue/{}", queued.queue_id), serde_json::json!({ "unexpected": true }), ), ); - assert_rejected(&bad_queue_remove, "DELETE /v1/merge-queue/{queue_id}"); + assert_rejected( + &bad_queue_remove, + "DELETE /v1/lanes/merges/queue/{selector}", + ); assert!(db - .list_merge_queue() + .list_lane_merge_queue() .unwrap() .iter() .any(|item| item.queue_id == queued.queue_id && item.status == "queued")); @@ -21156,8 +24606,8 @@ fn merge_lane_and_queue_enforce_readiness_blockers() { assert!(direct_err.to_string().contains("not merge-ready")); assert!(direct_err.to_string().contains("pending_approvals")); - db.enqueue_merge("approval-bot", "main", 0).unwrap(); - let explain = db.explain_merge_queue("approval-bot").unwrap(); + db.enqueue_lane_merge("approval-bot", "main", 0).unwrap(); + let explain = db.explain_lane_merge_queue("approval-bot").unwrap(); assert!(explain .blockers .iter() @@ -21170,7 +24620,7 @@ fn merge_lane_and_queue_enforce_readiness_blockers() { )) })); - let run = db.run_merge_queue(None).unwrap(); + let run = db.run_lane_merge_queue(None).unwrap(); assert_eq!(run.processed.len(), 1); assert_eq!(run.processed[0].status, "failed"); assert!(run.stopped_on_failure); @@ -21191,7 +24641,9 @@ fn merge_lane_and_queue_enforce_readiness_blockers() { })) .unwrap(); apply_lane_patch_at_head(&mut db, "late-approval-bot", late_patch).unwrap(); - let late_queue = db.enqueue_merge("late-approval-bot", "main", 0).unwrap(); + let late_queue = db + .enqueue_lane_merge("late-approval-bot", "main", 0) + .unwrap(); assert!(db.lane_readiness("late-approval-bot").unwrap().ready); db.request_lane_approval( "late-approval-bot", @@ -21202,7 +24654,7 @@ fn merge_lane_and_queue_enforce_readiness_blockers() { None, ) .unwrap(); - let late_run = db.run_merge_queue(None).unwrap(); + let late_run = db.run_lane_merge_queue(None).unwrap(); assert_eq!(late_run.processed.len(), 1); assert_eq!(late_run.processed[0].queue_id, late_queue.entry.queue_id); assert_eq!(late_run.processed[0].status, "failed"); @@ -21406,7 +24858,7 @@ fn required_gate_config_blocks_merge_until_test_and_eval_pass() { } #[test] -fn merge_queue_runs_lane_branch_into_main() { +fn lane_merge_queue_runs_lane_branch_into_main() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); @@ -21425,11 +24877,12 @@ fn merge_queue_runs_lane_branch_into_main() { .unwrap(); apply_lane_patch_at_head(&mut db, "doc-bot", patch).unwrap(); - let queued = db.enqueue_merge("doc-bot", "main", 10).unwrap(); + let queued = db.enqueue_lane_merge("doc-bot", "main", 10).unwrap(); assert_eq!(queued.entry.status, "queued"); - assert_eq!(db.list_merge_queue().unwrap().len(), 1); + assert_eq!(queued.entry.lane, "doc-bot"); + assert_eq!(db.list_lane_merge_queue().unwrap().len(), 1); - let run = db.run_merge_queue(None).unwrap(); + let run = db.run_lane_merge_queue(None).unwrap(); assert_eq!(run.processed.len(), 1); assert_eq!(run.processed[0].status, "merged"); assert!(!run.stopped_on_conflict); @@ -21444,7 +24897,7 @@ fn merge_queue_runs_lane_branch_into_main() { let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); let queue_status: String = conn .query_row( - "SELECT status FROM merge_queue WHERE queue_id = ?1", + "SELECT status FROM lane_merge_queue WHERE queue_id = ?1", [&queued.entry.queue_id], |row| row.get(0), ) @@ -21457,7 +24910,87 @@ fn merge_queue_runs_lane_branch_into_main() { } #[test] -fn merge_queue_explain_reports_dry_run_conflicts_without_recording_conflict_state() { +fn lane_merge_queue_rejects_branch_sources() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + let error = db.enqueue_lane_merge("main", "main", 0).unwrap_err(); + assert!(matches!(error, Error::InvalidInput(_))); + assert!(error.to_string().contains("lane")); +} + +#[test] +fn local_api_drives_lane_merge_queue() { + let temp = tempfile::tempdir().unwrap(); + fs::write(temp.path().join("README.md"), "hello\n").unwrap(); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + + let mut db = Trail::open(temp.path()).unwrap(); + db.spawn_lane("api-bot", Some("main"), false, None, None) + .unwrap(); + let add = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/lanes/merges/queue", + serde_json::json!({ + "lane": "api-bot", + "into": "main", + "priority": 10 + }), + ), + ); + assert_eq!(add.status, 201); + let add: serde_json::Value = add.body_json().unwrap(); + assert_eq!(add["entry"]["lane"], "api-bot"); + assert!(add["entry"]["queue_id"] + .as_str() + .unwrap() + .starts_with("lmq_")); + + let list = trail::server::handle_http_request( + &mut db, + &api_request("GET", "/v1/lanes/merges/queue", serde_json::Value::Null), + ); + assert_eq!(list.status, 200); + assert_eq!( + list.body_json::().unwrap()[0]["lane"], + "api-bot" + ); + + let aliased_body = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/lanes/merges/queue", + serde_json::json!({"source": "api-bot", "target": "main"}), + ), + ); + assert_eq!(aliased_body.status, 400); + + let legacy = trail::server::handle_http_request( + &mut db, + &api_request( + "POST", + "/v1/merge-queue", + serde_json::json!({"source": "api-bot", "target": "main"}), + ), + ); + assert_eq!(legacy.status, 400); + + let openapi = trail::server::openapi_spec(); + assert!(openapi["paths"].get("/v1/lanes/merges/queue").is_some()); + assert!(openapi["paths"].get("/v1/merge-queue").is_none()); + assert_eq!( + openapi["components"]["schemas"]["LaneMergeQueueAddRequest"]["required"], + serde_json::json!(["lane", "into"]) + ); +} + +#[test] +fn lane_merge_queue_explain_reports_dry_run_conflicts_without_recording_conflict_state() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\nworld\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); @@ -21480,10 +25013,10 @@ fn merge_queue_explain_reports_dry_run_conflicts_without_recording_conflict_stat false, ) .unwrap(); - let queued = db.enqueue_merge("explain-bot", "main", 0).unwrap(); + let queued = db.enqueue_lane_merge("explain-bot", "main", 0).unwrap(); let queue_id = queued.entry.queue_id.clone(); - let explain = db.explain_merge_queue("explain-bot").unwrap(); + let explain = db.explain_lane_merge_queue("explain-bot").unwrap(); assert!(explain .blockers .iter() @@ -21498,7 +25031,7 @@ fn merge_queue_explain_reports_dry_run_conflicts_without_recording_conflict_stat &mut db, &api_request( "GET", - &format!("/v1/merge-queue/{queue_id}/explain"), + &format!("/v1/lanes/merges/queue/{queue_id}/explain"), serde_json::Value::Null, ), ); @@ -21515,16 +25048,13 @@ fn merge_queue_explain_reports_dry_run_conflicts_without_recording_conflict_stat &mut db, &api_request( "GET", - "/v1/merge-queue/explain?selector=refs/lanes/explain-bot", + "/v1/lanes/merges/queue/explain-bot/explain", serde_json::Value::Null, ), ); assert_eq!(api_ref_explain.status, 200); let api_ref_explain: serde_json::Value = api_ref_explain.body_json().unwrap(); - assert_eq!( - api_ref_explain["entry"]["source_ref"], - "refs/lanes/explain-bot" - ); + assert_eq!(api_ref_explain["entry"]["lane"], "explain-bot"); let tools = trail::mcp::handle_json_rpc( &mut db, @@ -21540,7 +25070,7 @@ fn merge_queue_explain_reports_dry_run_conflicts_without_recording_conflict_stat .as_array() .unwrap() .iter() - .any(|tool| tool["name"] == "trail.merge_queue_explain")); + .any(|tool| tool["name"] == "trail.lane_merge_queue_explain")); let mcp_explain = trail::mcp::handle_json_rpc( &mut db, @@ -21549,7 +25079,7 @@ fn merge_queue_explain_reports_dry_run_conflicts_without_recording_conflict_stat "id": 2, "method": "tools/call", "params": { - "name": "trail.merge_queue_explain", + "name": "trail.lane_merge_queue_explain", "arguments": { "selector": "explain-bot" } @@ -21565,8 +25095,11 @@ fn merge_queue_explain_reports_dry_run_conflicts_without_recording_conflict_stat assert!(db.list_conflicts().unwrap().is_empty()); drop(db); - let cli = run_trail_json(temp.path(), &["merge-queue", "explain", "explain-bot"]); - assert_eq!(cli["entry"]["source_ref"], "refs/lanes/explain-bot"); + let cli = run_trail_json( + temp.path(), + &["lane", "merge-queue", "explain", "explain-bot"], + ); + assert_eq!(cli["entry"]["lane"], "explain-bot"); assert!(cli["blockers"] .as_array() .unwrap() @@ -21600,7 +25133,7 @@ fn merge_queue_explain_reports_dry_run_conflicts_without_recording_conflict_stat let daemon_cli = run_trail_json_daemon( temp.path(), &daemon_url, - &["merge-queue", "explain", "refs/lanes/explain-bot"], + &["lane", "merge-queue", "explain", "explain-bot"], ); assert_eq!(daemon_cli["entry"]["queue_id"], queue_id); assert!(daemon_cli["blockers"] @@ -21615,7 +25148,7 @@ fn merge_queue_explain_reports_dry_run_conflicts_without_recording_conflict_stat } #[test] -fn merge_queue_pauses_on_conflict() { +fn lane_merge_queue_pauses_on_conflict() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\nworld\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); @@ -21642,9 +25175,9 @@ fn merge_queue_pauses_on_conflict() { false, ) .unwrap(); - let queued = db.enqueue_merge("doc-bot", "main", 0).unwrap(); + let queued = db.enqueue_lane_merge("doc-bot", "main", 0).unwrap(); - let run = db.run_merge_queue(None).unwrap(); + let run = db.run_lane_merge_queue(None).unwrap(); assert_eq!(run.processed.len(), 1); assert_eq!(run.processed[0].status, "conflicted"); assert!(run.stopped_on_conflict); @@ -21653,7 +25186,7 @@ fn merge_queue_pauses_on_conflict() { let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); let queue_status: String = conn .query_row( - "SELECT status FROM merge_queue WHERE queue_id = ?1", + "SELECT status FROM lane_merge_queue WHERE queue_id = ?1", [&queued.entry.queue_id], |row| row.get(0), ) @@ -21715,7 +25248,7 @@ fn merge_queue_pauses_on_conflict() { let queue_status: String = conn .query_row( - "SELECT status FROM merge_queue WHERE queue_id = ?1", + "SELECT status FROM lane_merge_queue WHERE queue_id = ?1", [&queued.entry.queue_id], |row| row.get(0), ) @@ -22251,7 +25784,7 @@ fn manual_conflict_resolution_works_through_db_cli_http_and_mcp() { } #[test] -fn local_api_and_mcp_drive_merge_queue_and_conflicts() { +fn local_api_and_mcp_drive_lane_merge_queue_and_conflicts() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("README.md"), "hello\nworld\n").unwrap(); Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false).unwrap(); @@ -22283,10 +25816,10 @@ fn local_api_and_mcp_drive_merge_queue_and_conflicts() { &mut db, &api_request( "POST", - "/v1/merge-queue", + "/v1/lanes/merges/queue", serde_json::json!({ - "source": "api-queue-bot", - "target": "main", + "lane": "api-queue-bot", + "into": "main", "priority": 5 }), ), @@ -22298,7 +25831,7 @@ fn local_api_and_mcp_drive_merge_queue_and_conflicts() { let listed = trail::server::handle_http_request( &mut db, - &api_request("GET", "/v1/merge-queue", serde_json::Value::Null), + &api_request("GET", "/v1/lanes/merges/queue", serde_json::Value::Null), ); assert_eq!(listed.status, 200); let listed: serde_json::Value = listed.body_json().unwrap(); @@ -22320,11 +25853,11 @@ fn local_api_and_mcp_drive_merge_queue_and_conflicts() { .unwrap(); let tool_list = tools["result"]["tools"].as_array().unwrap(); for name in [ - "trail.merge_queue_add", - "trail.merge_queue_list", - "trail.merge_queue_run", - "trail.merge_queue_explain", - "trail.merge_queue_remove", + "trail.lane_merge_queue_add", + "trail.lane_merge_queue_list", + "trail.lane_merge_queue_run", + "trail.lane_merge_queue_explain", + "trail.lane_merge_queue_remove", "trail.conflict_list", "trail.conflict_show", "trail.conflict_resolve", @@ -22365,7 +25898,7 @@ fn local_api_and_mcp_drive_merge_queue_and_conflicts() { let run = trail::server::handle_http_request( &mut db, - &api_request("POST", "/v1/merge-queue/run", serde_json::json!({})), + &api_request("POST", "/v1/lanes/merges/queue/run", serde_json::json!({})), ); assert_eq!(run.status, 200); let run: serde_json::Value = run.body_json().unwrap(); @@ -22467,7 +26000,7 @@ fn local_api_and_mcp_drive_merge_queue_and_conflicts() { "id": 4, "method": "tools/call", "params": { - "name": "trail.merge_queue_list", + "name": "trail.lane_merge_queue_list", "arguments": {} } }), @@ -22489,9 +26022,9 @@ fn local_api_and_mcp_drive_merge_queue_and_conflicts() { "id": 5, "method": "tools/call", "params": { - "name": "trail.merge_queue_add", + "name": "trail.lane_merge_queue_add", "arguments": { - "source": "cancel-queue-bot", + "lane": "cancel-queue-bot", "target": "main", "priority": 1 } @@ -22509,7 +26042,7 @@ fn local_api_and_mcp_drive_merge_queue_and_conflicts() { &mut db, &api_request( "DELETE", - &format!("/v1/merge-queue/{cancel_queue_id}"), + &format!("/v1/lanes/merges/queue/{cancel_queue_id}"), serde_json::Value::Null, ), ); @@ -22987,6 +26520,10 @@ fn show_history_and_code_from_use_recorded_indexes() { } other => panic!("expected operation show result, got {other:?}"), } + match db.show(&applied.operation.checkpoint_alias()).unwrap() { + ShowResult::Operation { value } => assert_eq!(value.operation.change_id, applied.operation), + other => panic!("expected checkpoint alias to show operation, got {other:?}"), + } let conn = Connection::open(temp.path().join(".trail/index/trail.sqlite")).unwrap(); let message_id: String = conn @@ -22994,6 +26531,7 @@ fn show_history_and_code_from_use_recorded_indexes() { row.get(0) }) .unwrap(); + assert!(message_id.starts_with("message_")); match db.show(&message_id).unwrap() { ShowResult::Message { value } => assert_eq!(value.body, "lane adds line"), other => panic!("expected message show result, got {other:?}"), @@ -23003,7 +26541,8 @@ fn show_history_and_code_from_use_recorded_indexes() { assert!(file_history.file_history.len() >= 2); let why = db.why("README.md:2", Some("refs/lanes/doc-bot")).unwrap(); - let line_id = format!("{}:{}", why.line_id.origin_change.0, why.line_id.local_seq); + let line_id = why.line_id.alias(); + assert!(line_id.starts_with("line_")); let line_history = db.history_for_line_id(&line_id).unwrap(); assert!(!line_history.line_history.is_empty()); @@ -23101,7 +26640,7 @@ fn gc_prunes_unreachable_known_objects_and_preserves_reachable_roots() { conn.execute( "INSERT INTO objects \ (object_id, kind, version, codec, hash_alg, size_bytes, bytes, created_at) \ - VALUES ('obj_unreachable_test', 'Blob', 1, 'cbor', 'sha256', 0, x'', 0)", + VALUES ('object_unreachable_test', 'Blob', 1, 'cbor', 'sha256', 0, x'', 0)", [], ) .unwrap(); @@ -23115,7 +26654,7 @@ fn gc_prunes_unreachable_known_objects_and_preserves_reachable_roots() { assert!(report.pruned_objects >= 1); let still_exists: i64 = conn .query_row( - "SELECT COUNT(*) FROM objects WHERE object_id = 'obj_unreachable_test'", + "SELECT COUNT(*) FROM objects WHERE object_id = 'object_unreachable_test'", [], |row| row.get(0), ) diff --git a/trail/tests/fixtures/acp/v1/messages.jsonl b/trail/tests/fixtures/acp/v1/messages.jsonl new file mode 100644 index 0000000..cdae078 --- /dev/null +++ b/trail/tests/fixtures/acp/v1/messages.jsonl @@ -0,0 +1,4 @@ +{"jsonrpc":"2.0","id":"request-1","method":"session/list","params":{},"_meta":{"extension":true}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"session-1","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hello"}}}} +{"jsonrpc":"2.0","id":7,"result":{"sessions":[]},"unknown":"preserved"} +{"jsonrpc":"2.0","id":8,"error":{"code":-32602,"message":"invalid params","data":{"extension":true}}} diff --git a/trail/tests/fixtures/acp/v1/meta.json b/trail/tests/fixtures/acp/v1/meta.json new file mode 100644 index 0000000..670d278 --- /dev/null +++ b/trail/tests/fixtures/acp/v1/meta.json @@ -0,0 +1,32 @@ +{ + "version": 1, + "agentMethods": { + "initialize": "initialize", + "authenticate": "authenticate", + "session_new": "session/new", + "session_load": "session/load", + "session_set_mode": "session/set_mode", + "session_set_config_option": "session/set_config_option", + "session_prompt": "session/prompt", + "session_cancel": "session/cancel", + "session_list": "session/list", + "session_delete": "session/delete", + "session_resume": "session/resume", + "session_close": "session/close", + "logout": "logout" + }, + "clientMethods": { + "session_request_permission": "session/request_permission", + "session_update": "session/update", + "fs_write_text_file": "fs/write_text_file", + "fs_read_text_file": "fs/read_text_file", + "terminal_create": "terminal/create", + "terminal_output": "terminal/output", + "terminal_release": "terminal/release", + "terminal_wait_for_exit": "terminal/wait_for_exit", + "terminal_kill": "terminal/kill" + }, + "protocolMethods": { + "cancel_request": "$/cancel_request" + } +} diff --git a/trail/tests/fixtures/acp/v1/method_cases.json b/trail/tests/fixtures/acp/v1/method_cases.json new file mode 100644 index 0000000..d1c49f9 --- /dev/null +++ b/trail/tests/fixtures/acp/v1/method_cases.json @@ -0,0 +1,25 @@ +[ + {"method":"initialize","side":"agent","envelope":"request"}, + {"method":"authenticate","side":"agent","envelope":"request"}, + {"method":"logout","side":"agent","envelope":"request"}, + {"method":"session/new","side":"agent","envelope":"request"}, + {"method":"session/load","side":"agent","envelope":"request"}, + {"method":"session/resume","side":"agent","envelope":"request"}, + {"method":"session/close","side":"agent","envelope":"request"}, + {"method":"session/list","side":"agent","envelope":"request"}, + {"method":"session/delete","side":"agent","envelope":"request"}, + {"method":"session/prompt","side":"agent","envelope":"request"}, + {"method":"session/cancel","side":"agent","envelope":"notification"}, + {"method":"session/set_mode","side":"agent","envelope":"request"}, + {"method":"session/set_config_option","side":"agent","envelope":"request"}, + {"method":"session/update","side":"client","envelope":"notification"}, + {"method":"session/request_permission","side":"client","envelope":"request"}, + {"method":"fs/read_text_file","side":"client","envelope":"request"}, + {"method":"fs/write_text_file","side":"client","envelope":"request"}, + {"method":"terminal/create","side":"client","envelope":"request"}, + {"method":"terminal/output","side":"client","envelope":"request"}, + {"method":"terminal/wait_for_exit","side":"client","envelope":"request"}, + {"method":"terminal/kill","side":"client","envelope":"request"}, + {"method":"terminal/release","side":"client","envelope":"request"}, + {"method":"$/cancel_request","side":"protocol","envelope":"notification"} +] diff --git a/trail/tests/fixtures/acp/v1/schema.json b/trail/tests/fixtures/acp/v1/schema.json new file mode 100644 index 0000000..0a83014 --- /dev/null +++ b/trail/tests/fixtures/acp/v1/schema.json @@ -0,0 +1,4667 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Agent Client Protocol", + "anyOf": [ + { + "title": "Agent", + "description": "A message (request, response, or notification) with `\"jsonrpc\": \"2.0\"` specified as\n[required by JSON-RPC 2.0 Specification][1].\n\n[1]: https://www.jsonrpc.org/specification#compatibility", + "type": "object", + "properties": { + "jsonrpc": { + "type": "string", + "enum": ["2.0"] + } + }, + "required": ["jsonrpc"], + "anyOf": [ + { + "title": "Request", + "allOf": [ + { + "$ref": "#/$defs/AgentRequest" + } + ] + }, + { + "title": "Response", + "allOf": [ + { + "$ref": "#/$defs/AgentResponse" + } + ] + }, + { + "title": "Notification", + "allOf": [ + { + "$ref": "#/$defs/AgentNotification" + } + ] + } + ] + }, + { + "title": "Client", + "description": "A message (request, response, or notification) with `\"jsonrpc\": \"2.0\"` specified as\n[required by JSON-RPC 2.0 Specification][1].\n\n[1]: https://www.jsonrpc.org/specification#compatibility", + "type": "object", + "properties": { + "jsonrpc": { + "type": "string", + "enum": ["2.0"] + } + }, + "required": ["jsonrpc"], + "anyOf": [ + { + "title": "Request", + "allOf": [ + { + "$ref": "#/$defs/ClientRequest" + } + ] + }, + { + "title": "Response", + "allOf": [ + { + "$ref": "#/$defs/ClientResponse" + } + ] + }, + { + "title": "Notification", + "allOf": [ + { + "$ref": "#/$defs/ClientNotification" + } + ] + } + ] + }, + { + "title": "ProtocolLevel", + "description": "A message (request, response, or notification) with `\"jsonrpc\": \"2.0\"` specified as\n[required by JSON-RPC 2.0 Specification][1].\n\n[1]: https://www.jsonrpc.org/specification#compatibility", + "type": "object", + "properties": { + "jsonrpc": { + "type": "string", + "enum": ["2.0"] + }, + "method": { + "description": "The notification method name.", + "type": "string" + }, + "params": { + "description": "Method-specific notification parameters.", + "anyOf": [ + { + "description": "General protocol-level notifications that all sides are expected to\nimplement.\n\nNotifications whose methods start with '$/' are messages which\nare protocol implementation dependent and might not be implementable in all\nclients or agents. For example if the implementation uses a single threaded\nsynchronous programming language then there is little it can do to react to\na `$/cancel_request` notification. If an agent or client receives\nnotifications starting with '$/' it is free to ignore the notification.\n\nNotifications do not expect a response.", + "anyOf": [ + { + "title": "CancelRequestNotification", + "description": "Cancels an ongoing request.\n\nThis is a notification sent by the side that sent a request to cancel that request.\n\nUpon receiving this notification, the receiver:\n\n1. MAY cancel the corresponding request activity and all nested activities\n2. MAY send any pending notifications.\n3. MUST send one of these responses for the original request:\n - Valid response with appropriate data (partial results or cancellation marker)\n - Error response with code `-32800` (Cancelled)\n\nSee protocol docs: [Cancellation](https://agentclientprotocol.com/protocol/cancellation)", + "allOf": [ + { + "$ref": "#/$defs/CancelRequestNotification" + } + ] + } + ] + }, + { + "type": "null" + } + ] + } + }, + "required": ["jsonrpc", "method"], + "x-docs-ignore": true + } + ], + "$defs": { + "AgentRequest": { + "description": "A JSON-RPC request object.", + "type": "object", + "properties": { + "id": { + "description": "The request id used to correlate the matching response.", + "allOf": [ + { + "$ref": "#/$defs/RequestId" + } + ] + }, + "method": { + "description": "The method name to invoke.", + "type": "string" + }, + "params": { + "description": "Method-specific request parameters.", + "anyOf": [ + { + "description": "All possible requests that an agent can send to a client.\n\nThis enum is used internally for routing RPC requests. You typically won't need\nto use this directly.\n\nThis enum encompasses all method calls from agent to client.", + "anyOf": [ + { + "title": "WriteTextFileRequest", + "description": "Writes content to a text file in the client's file system.\n\nOnly available if the client advertises the `fs.writeTextFile` capability.\nAllows the agent to create or modify files within the client's environment.\n\nSee protocol docs: [Client](https://agentclientprotocol.com/protocol/overview#client)", + "allOf": [ + { + "$ref": "#/$defs/WriteTextFileRequest" + } + ] + }, + { + "title": "ReadTextFileRequest", + "description": "Reads content from a text file in the client's file system.\n\nOnly available if the client advertises the `fs.readTextFile` capability.\nAllows the agent to access file contents within the client's environment.\n\nSee protocol docs: [Client](https://agentclientprotocol.com/protocol/overview#client)", + "allOf": [ + { + "$ref": "#/$defs/ReadTextFileRequest" + } + ] + }, + { + "title": "RequestPermissionRequest", + "description": "Requests permission from the user for a tool call operation.\n\nCalled by the agent when it needs user authorization before executing\na potentially sensitive operation. The client should present the options\nto the user and return their decision.\n\nIf the client cancels the prompt turn via `session/cancel`, it MUST\nrespond to this request with `RequestPermissionOutcome::Cancelled`.\n\nSee protocol docs: [Requesting Permission](https://agentclientprotocol.com/protocol/tool-calls#requesting-permission)", + "allOf": [ + { + "$ref": "#/$defs/RequestPermissionRequest" + } + ] + }, + { + "title": "CreateTerminalRequest", + "description": "Executes a command in a new terminal\n\nOnly available if the `terminal` Client capability is set to `true`.\n\nReturns a `TerminalId` that can be used with other terminal methods\nto get the current output, wait for exit, and kill the command.\n\nThe `TerminalId` can also be used to embed the terminal in a tool call\nby using the `ToolCallContent::Terminal` variant.\n\nThe Agent is responsible for releasing the terminal by using the `terminal/release`\nmethod.\n\nSee protocol docs: [Terminals](https://agentclientprotocol.com/protocol/terminals)", + "allOf": [ + { + "$ref": "#/$defs/CreateTerminalRequest" + } + ] + }, + { + "title": "TerminalOutputRequest", + "description": "Gets the terminal output and exit status\n\nReturns the current content in the terminal without waiting for the command to exit.\nIf the command has already exited, the exit status is included.\n\nSee protocol docs: [Terminals](https://agentclientprotocol.com/protocol/terminals)", + "allOf": [ + { + "$ref": "#/$defs/TerminalOutputRequest" + } + ] + }, + { + "title": "ReleaseTerminalRequest", + "description": "Releases a terminal\n\nThe command is killed if it hasn't exited yet. Use `terminal/wait_for_exit`\nto wait for the command to exit before releasing the terminal.\n\nAfter release, the `TerminalId` can no longer be used with other `terminal/*` methods,\nbut tool calls that already contain it, continue to display its output.\n\nThe `terminal/kill` method can be used to terminate the command without releasing\nthe terminal, allowing the Agent to call `terminal/output` and other methods.\n\nSee protocol docs: [Terminals](https://agentclientprotocol.com/protocol/terminals)", + "allOf": [ + { + "$ref": "#/$defs/ReleaseTerminalRequest" + } + ] + }, + { + "title": "WaitForTerminalExitRequest", + "description": "Waits for the terminal command to exit and return its exit status\n\nSee protocol docs: [Terminals](https://agentclientprotocol.com/protocol/terminals)", + "allOf": [ + { + "$ref": "#/$defs/WaitForTerminalExitRequest" + } + ] + }, + { + "title": "KillTerminalRequest", + "description": "Kills the terminal command without releasing the terminal\n\nWhile `terminal/release` will also kill the command, this method will keep\nthe `TerminalId` valid so it can be used with other methods.\n\nThis method can be helpful when implementing command timeouts which terminate\nthe command as soon as elapsed, and then get the final output so it can be sent\nto the model.\n\nNote: Call `terminal/release` when `TerminalId` is no longer needed.\n\nSee protocol docs: [Terminals](https://agentclientprotocol.com/protocol/terminals)", + "allOf": [ + { + "$ref": "#/$defs/KillTerminalRequest" + } + ] + }, + { + "title": "ExtMethodRequest", + "description": "Handles extension method requests from the agent.\n\nAllows the Agent to send an arbitrary request that is not part of the ACP spec.\nExtension methods provide a way to add custom functionality while maintaining\nprotocol compatibility.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "allOf": [ + { + "$ref": "#/$defs/ExtRequest" + } + ] + } + ] + }, + { + "type": "null" + } + ] + } + }, + "required": ["id", "method"], + "x-docs-ignore": true + }, + "RequestId": { + "description": "JSON RPC Request Id\n\nAn identifier established by the Client that MUST contain a String, Number, or NULL value if included. If it is not included it is assumed to be a notification. The value SHOULD normally not be Null \\[1\\] and Numbers SHOULD NOT contain fractional parts \\[2\\]\n\nThe Server MUST reply with the same value in the Response object if included. This member is used to correlate the context between the two objects.\n\n\\[1\\] The use of Null as a value for the id member in a Request object is discouraged, because this specification uses a value of Null for Responses with an unknown id. Also, because JSON-RPC 1.0 uses an id value of Null for Notifications this could cause confusion in handling.\n\n\\[2\\] Fractional parts may be problematic, since many decimal fractions cannot be represented exactly as binary fractions.", + "anyOf": [ + { + "title": "Null", + "description": "The JSON-RPC `null` request id.", + "type": "null" + }, + { + "title": "Number", + "description": "A numeric JSON-RPC request id.", + "type": "integer", + "format": "int64" + }, + { + "title": "Str", + "description": "A string JSON-RPC request id.", + "type": "string" + } + ] + }, + "WriteTextFileRequest": { + "description": "Request to write content to a text file.\n\nOnly available if the client supports the `fs.writeTextFile` capability.", + "type": "object", + "properties": { + "sessionId": { + "description": "The session ID for this request.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "path": { + "description": "Absolute path to the file to write.", + "type": "string" + }, + "content": { + "description": "The text content to write to the file.", + "type": "string" + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "path", "content"], + "x-side": "client", + "x-method": "fs/write_text_file" + }, + "SessionId": { + "description": "A unique identifier for a conversation session between a client and agent.\n\nSessions maintain their own context, conversation history, and state,\nallowing multiple independent interactions with the same agent.\n\nSee protocol docs: [Session ID](https://agentclientprotocol.com/protocol/session-setup#session-id)", + "type": "string" + }, + "ReadTextFileRequest": { + "description": "Request to read content from a text file.\n\nOnly available if the client supports the `fs.readTextFile` capability.", + "type": "object", + "properties": { + "sessionId": { + "description": "The session ID for this request.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "path": { + "description": "Absolute path to the file to read.", + "type": "string" + }, + "line": { + "description": "Line number to start reading from (1-based).", + "type": ["integer", "null"], + "format": "uint32", + "minimum": 0, + "x-deserialize-default-on-error": true + }, + "limit": { + "description": "Maximum number of lines to read.", + "type": ["integer", "null"], + "format": "uint32", + "minimum": 0, + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "path"], + "x-side": "client", + "x-method": "fs/read_text_file" + }, + "RequestPermissionRequest": { + "description": "Request for user permission to execute a tool call.\n\nSent when the agent needs authorization before performing a sensitive operation.\n\nSee protocol docs: [Requesting Permission](https://agentclientprotocol.com/protocol/tool-calls#requesting-permission)", + "type": "object", + "properties": { + "sessionId": { + "description": "The session ID for this request.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "toolCall": { + "description": "Details about the tool call requiring permission.", + "allOf": [ + { + "$ref": "#/$defs/ToolCallUpdate" + } + ] + }, + "options": { + "description": "Available permission options for the user to choose from.", + "type": "array", + "items": { + "$ref": "#/$defs/PermissionOption" + } + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "toolCall", "options"], + "x-side": "client", + "x-method": "session/request_permission" + }, + "ToolCallUpdate": { + "description": "An update to an existing tool call.\n\nUsed to report progress and results as tools execute. All fields except\nthe tool call ID are optional - only changed fields need to be included.\n\nSee protocol docs: [Updating](https://agentclientprotocol.com/protocol/tool-calls#updating)", + "type": "object", + "properties": { + "toolCallId": { + "description": "The ID of the tool call being updated.", + "allOf": [ + { + "$ref": "#/$defs/ToolCallId" + } + ] + }, + "kind": { + "description": "Update the tool kind.", + "anyOf": [ + { + "$ref": "#/$defs/ToolKind" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "status": { + "description": "Update the execution status.", + "anyOf": [ + { + "$ref": "#/$defs/ToolCallStatus" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "title": { + "description": "Update the human-readable title.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "content": { + "description": "Replace the content collection.", + "type": ["array", "null"], + "items": { + "$ref": "#/$defs/ToolCallContent" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "locations": { + "description": "Replace the locations collection.", + "type": ["array", "null"], + "items": { + "$ref": "#/$defs/ToolCallLocation" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "rawInput": { + "description": "Update the raw input.", + "x-deserialize-default-on-error": true + }, + "rawOutput": { + "description": "Update the raw output.", + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["toolCallId"] + }, + "ToolCallId": { + "description": "Unique identifier for a tool call within a session.", + "type": "string" + }, + "ToolKind": { + "description": "Categories of tools that can be invoked.\n\nTool kinds help clients choose appropriate icons and optimize how they\ndisplay tool execution progress.\n\nSee protocol docs: [Creating](https://agentclientprotocol.com/protocol/tool-calls#creating)", + "oneOf": [ + { + "description": "Reading files or data.", + "type": "string", + "const": "read" + }, + { + "description": "Modifying files or content.", + "type": "string", + "const": "edit" + }, + { + "description": "Removing files or data.", + "type": "string", + "const": "delete" + }, + { + "description": "Moving or renaming files.", + "type": "string", + "const": "move" + }, + { + "description": "Searching for information.", + "type": "string", + "const": "search" + }, + { + "description": "Running commands or code.", + "type": "string", + "const": "execute" + }, + { + "description": "Internal reasoning or planning.", + "type": "string", + "const": "think" + }, + { + "description": "Retrieving external data.", + "type": "string", + "const": "fetch" + }, + { + "description": "Switching the current session mode.", + "type": "string", + "const": "switch_mode" + }, + { + "description": "Other tool types (default).", + "type": "string", + "const": "other" + } + ] + }, + "ToolCallStatus": { + "description": "Execution status of a tool call.\n\nTool calls progress through different statuses during their lifecycle.\n\nSee protocol docs: [Status](https://agentclientprotocol.com/protocol/tool-calls#status)", + "oneOf": [ + { + "description": "The tool call hasn't started running yet because the input is either\nstreaming or we're awaiting approval.", + "type": "string", + "const": "pending" + }, + { + "description": "The tool call is currently running.", + "type": "string", + "const": "in_progress" + }, + { + "description": "The tool call completed successfully.", + "type": "string", + "const": "completed" + }, + { + "description": "The tool call failed with an error.", + "type": "string", + "const": "failed" + } + ] + }, + "ToolCallContent": { + "description": "Content produced by a tool call.\n\nTool calls can produce different types of content including\nstandard content blocks (text, images) or file diffs.\n\nSee protocol docs: [Content](https://agentclientprotocol.com/protocol/tool-calls#content)", + "oneOf": [ + { + "description": "Standard content block (text, images, resources).", + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "content" + } + }, + "required": ["type"], + "allOf": [ + { + "$ref": "#/$defs/Content" + } + ] + }, + { + "description": "File modification shown as a diff.", + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "diff" + } + }, + "required": ["type"], + "allOf": [ + { + "$ref": "#/$defs/Diff" + } + ] + }, + { + "description": "Embed a terminal created with `terminal/create` by its id.\n\nThe terminal must be added before calling `terminal/release`.\n\nSee protocol docs: [Terminal](https://agentclientprotocol.com/protocol/terminals)", + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "terminal" + } + }, + "required": ["type"], + "allOf": [ + { + "$ref": "#/$defs/Terminal" + } + ] + } + ], + "discriminator": { + "propertyName": "type" + } + }, + "ContentBlock": { + "description": "Content blocks represent displayable information in the Agent Client Protocol.\n\nThey provide a structured way to handle various types of user-facing content—whether\nit's text from language models, images for analysis, or embedded resources for context.\n\nContent blocks appear in:\n- User prompts sent via `session/prompt`\n- Language model output streamed through `session/update` notifications\n- Progress updates and results from tool calls\n\nThis structure is compatible with the Model Context Protocol (MCP), enabling\nagents to seamlessly forward content from MCP tool outputs without transformation.\n\nSee protocol docs: [Content](https://agentclientprotocol.com/protocol/content)", + "oneOf": [ + { + "description": "Text content. May be plain text or formatted with Markdown.\n\nAll agents MUST support text content blocks in prompts.\nClients SHOULD render this text as Markdown.", + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "text" + } + }, + "required": ["type"], + "allOf": [ + { + "$ref": "#/$defs/TextContent" + } + ] + }, + { + "description": "Images for visual context or analysis.\n\nRequires the `image` prompt capability when included in prompts.", + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "image" + } + }, + "required": ["type"], + "allOf": [ + { + "$ref": "#/$defs/ImageContent" + } + ] + }, + { + "description": "Audio data for transcription or analysis.\n\nRequires the `audio` prompt capability when included in prompts.", + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "audio" + } + }, + "required": ["type"], + "allOf": [ + { + "$ref": "#/$defs/AudioContent" + } + ] + }, + { + "description": "References to resources that the agent can access.\n\nAll agents MUST support resource links in prompts.", + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "resource_link" + } + }, + "required": ["type"], + "allOf": [ + { + "$ref": "#/$defs/ResourceLink" + } + ] + }, + { + "description": "Complete resource contents embedded directly in the message.\n\nPreferred for including context as it avoids extra round-trips.\n\nRequires the `embeddedContext` prompt capability when included in prompts.", + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "resource" + } + }, + "required": ["type"], + "allOf": [ + { + "$ref": "#/$defs/EmbeddedResource" + } + ] + } + ], + "discriminator": { + "propertyName": "type" + } + }, + "Annotations": { + "description": "Optional annotations for the client. The client can use annotations to inform how objects are used or displayed", + "type": "object", + "properties": { + "audience": { + "description": "Intended recipients for this content, such as the user or assistant.", + "type": ["array", "null"], + "items": { + "$ref": "#/$defs/Role" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "lastModified": { + "description": "Timestamp indicating when the underlying resource was last modified.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "priority": { + "description": "Relative importance of this content when clients choose what to surface.", + "type": ["number", "null"], + "format": "double", + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "Role": { + "description": "The sender or recipient of messages and data in a conversation.", + "oneOf": [ + { + "description": "The assistant side of a conversation.", + "type": "string", + "const": "assistant" + }, + { + "description": "The user side of a conversation.", + "type": "string", + "const": "user" + } + ] + }, + "TextContent": { + "description": "Text provided to or from an LLM.", + "type": "object", + "properties": { + "annotations": { + "description": "Optional annotations that help clients decide how to display or route this content.", + "anyOf": [ + { + "$ref": "#/$defs/Annotations" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "text": { + "description": "Text payload carried by this content block.", + "type": "string" + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["text"] + }, + "ImageContent": { + "description": "An image provided to or from an LLM.", + "type": "object", + "properties": { + "annotations": { + "description": "Optional annotations that help clients decide how to display or route this content.", + "anyOf": [ + { + "$ref": "#/$defs/Annotations" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "data": { + "description": "Base64-encoded media payload.", + "type": "string" + }, + "mimeType": { + "description": "MIME type describing the encoded media payload.", + "type": "string" + }, + "uri": { + "description": "URI associated with this resource or media payload.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["data", "mimeType"] + }, + "AudioContent": { + "description": "Audio provided to or from an LLM.", + "type": "object", + "properties": { + "annotations": { + "description": "Optional annotations that help clients decide how to display or route this content.", + "anyOf": [ + { + "$ref": "#/$defs/Annotations" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "data": { + "description": "Base64-encoded media payload.", + "type": "string" + }, + "mimeType": { + "description": "MIME type describing the encoded media payload.", + "type": "string" + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["data", "mimeType"] + }, + "ResourceLink": { + "description": "A resource that the server is capable of reading, included in a prompt or tool call result.", + "type": "object", + "properties": { + "annotations": { + "description": "Optional annotations that help clients decide how to display or route this content.", + "anyOf": [ + { + "$ref": "#/$defs/Annotations" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "description": { + "description": "Optional human-readable details shown with this protocol object.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "mimeType": { + "description": "MIME type describing the encoded media payload.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "name": { + "description": "Human-readable name shown for this protocol object.", + "type": "string" + }, + "size": { + "description": "Optional size of the linked resource in bytes, if known.", + "type": ["integer", "null"], + "format": "int64", + "x-deserialize-default-on-error": true + }, + "title": { + "description": "Optional display title for end-user UI.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "uri": { + "description": "URI associated with this resource or media payload.", + "type": "string" + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["name", "uri"] + }, + "EmbeddedResourceResource": { + "description": "Resource content that can be embedded in a message.", + "anyOf": [ + { + "title": "TextResourceContents", + "description": "Text resource contents embedded directly in the message.", + "allOf": [ + { + "$ref": "#/$defs/TextResourceContents" + } + ] + }, + { + "title": "BlobResourceContents", + "description": "Binary resource contents embedded directly in the message.", + "allOf": [ + { + "$ref": "#/$defs/BlobResourceContents" + } + ] + } + ] + }, + "TextResourceContents": { + "description": "Text-based resource contents.", + "type": "object", + "properties": { + "mimeType": { + "description": "MIME type describing the encoded media payload.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "text": { + "description": "Text payload carried by this content block.", + "type": "string" + }, + "uri": { + "description": "URI associated with this resource or media payload.", + "type": "string" + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["text", "uri"] + }, + "BlobResourceContents": { + "description": "Binary resource contents.", + "type": "object", + "properties": { + "blob": { + "description": "Base64-encoded bytes for a binary resource payload.", + "type": "string" + }, + "mimeType": { + "description": "MIME type describing the encoded media payload.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "uri": { + "description": "URI associated with this resource or media payload.", + "type": "string" + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["blob", "uri"] + }, + "EmbeddedResource": { + "description": "The contents of a resource, embedded into a prompt or tool call result.", + "type": "object", + "properties": { + "annotations": { + "description": "Optional annotations that help clients decide how to display or route this content.", + "anyOf": [ + { + "$ref": "#/$defs/Annotations" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "resource": { + "description": "Embedded resource payload, either text or binary data.", + "allOf": [ + { + "$ref": "#/$defs/EmbeddedResourceResource" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["resource"] + }, + "Content": { + "description": "Standard content block (text, images, resources).", + "type": "object", + "properties": { + "content": { + "description": "The actual content block.", + "allOf": [ + { + "$ref": "#/$defs/ContentBlock" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["content"] + }, + "Diff": { + "description": "A diff representing file modifications.\n\nShows changes to files in a format suitable for display in the client UI.\n\nSee protocol docs: [Content](https://agentclientprotocol.com/protocol/tool-calls#content)", + "type": "object", + "properties": { + "path": { + "description": "The absolute file path being modified.", + "type": "string" + }, + "oldText": { + "description": "The original content (None for new files).", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "newText": { + "description": "The new content after modification.", + "type": "string" + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["path", "newText"] + }, + "TerminalId": { + "description": "Typed identifier used for terminal values on the wire.", + "type": "string" + }, + "Terminal": { + "description": "Embed a terminal created with `terminal/create` by its id.\n\nThe terminal must be added before calling `terminal/release`.\n\nSee protocol docs: [Terminal](https://agentclientprotocol.com/protocol/terminals)", + "type": "object", + "properties": { + "terminalId": { + "description": "Identifier of the terminal instance to embed in the content stream.", + "allOf": [ + { + "$ref": "#/$defs/TerminalId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["terminalId"] + }, + "ToolCallLocation": { + "description": "A file location being accessed or modified by a tool.\n\nEnables clients to implement \"follow-along\" features that track\nwhich files the agent is working with in real-time.\n\nSee protocol docs: [Following the Agent](https://agentclientprotocol.com/protocol/tool-calls#following-the-agent)", + "type": "object", + "properties": { + "path": { + "description": "The absolute file path being accessed or modified.", + "type": "string" + }, + "line": { + "description": "Optional line number within the file.", + "type": ["integer", "null"], + "format": "uint32", + "minimum": 0, + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["path"] + }, + "PermissionOption": { + "description": "An option presented to the user when requesting permission.", + "type": "object", + "properties": { + "optionId": { + "description": "Unique identifier for this permission option.", + "allOf": [ + { + "$ref": "#/$defs/PermissionOptionId" + } + ] + }, + "name": { + "description": "Human-readable label to display to the user.", + "type": "string" + }, + "kind": { + "description": "Hint about the nature of this permission option.", + "allOf": [ + { + "$ref": "#/$defs/PermissionOptionKind" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["optionId", "name", "kind"] + }, + "PermissionOptionId": { + "description": "Unique identifier for a permission option.", + "type": "string" + }, + "PermissionOptionKind": { + "description": "The type of permission option being presented to the user.\n\nHelps clients choose appropriate icons and UI treatment.", + "oneOf": [ + { + "description": "Allow this operation only this time.", + "type": "string", + "const": "allow_once" + }, + { + "description": "Allow this operation and remember the choice.", + "type": "string", + "const": "allow_always" + }, + { + "description": "Reject this operation only this time.", + "type": "string", + "const": "reject_once" + }, + { + "description": "Reject this operation and remember the choice.", + "type": "string", + "const": "reject_always" + } + ] + }, + "CreateTerminalRequest": { + "description": "Request to create a new terminal and execute a command.", + "type": "object", + "properties": { + "sessionId": { + "description": "The session ID for this request.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "command": { + "description": "The command to execute.", + "type": "string" + }, + "args": { + "description": "Array of command arguments.", + "type": "array", + "items": { + "type": "string" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "env": { + "description": "Environment variables for the command.", + "type": "array", + "items": { + "$ref": "#/$defs/EnvVariable" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "cwd": { + "description": "Working directory for the command. Must be an absolute path.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "outputByteLimit": { + "description": "Maximum number of output bytes to retain.\n\nWhen the limit is exceeded, the Client truncates from the beginning of the output\nto stay within the limit.\n\nThe Client MUST ensure truncation happens at a character boundary to maintain valid\nstring output, even if this means the retained output is slightly less than the\nspecified limit.", + "type": ["integer", "null"], + "format": "uint64", + "minimum": 0, + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "command"], + "x-side": "client", + "x-method": "terminal/create" + }, + "EnvVariable": { + "description": "An environment variable to set when launching an MCP server.", + "type": "object", + "properties": { + "name": { + "description": "The name of the environment variable.", + "type": "string" + }, + "value": { + "description": "The value to set for the environment variable.", + "type": "string" + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["name", "value"] + }, + "TerminalOutputRequest": { + "description": "Request to get the current output and status of a terminal.", + "type": "object", + "properties": { + "sessionId": { + "description": "The session ID for this request.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "terminalId": { + "description": "The ID of the terminal to get output from.", + "allOf": [ + { + "$ref": "#/$defs/TerminalId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "terminalId"], + "x-side": "client", + "x-method": "terminal/output" + }, + "ReleaseTerminalRequest": { + "description": "Request to release a terminal and free its resources.", + "type": "object", + "properties": { + "sessionId": { + "description": "The session ID for this request.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "terminalId": { + "description": "The ID of the terminal to release.", + "allOf": [ + { + "$ref": "#/$defs/TerminalId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "terminalId"], + "x-side": "client", + "x-method": "terminal/release" + }, + "WaitForTerminalExitRequest": { + "description": "Request to wait for a terminal command to exit.", + "type": "object", + "properties": { + "sessionId": { + "description": "The session ID for this request.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "terminalId": { + "description": "The ID of the terminal to wait for.", + "allOf": [ + { + "$ref": "#/$defs/TerminalId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "terminalId"], + "x-side": "client", + "x-method": "terminal/wait_for_exit" + }, + "KillTerminalRequest": { + "description": "Request to kill a terminal without releasing it.", + "type": "object", + "properties": { + "sessionId": { + "description": "The session ID for this request.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "terminalId": { + "description": "The ID of the terminal to kill.", + "allOf": [ + { + "$ref": "#/$defs/TerminalId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "terminalId"], + "x-side": "client", + "x-method": "terminal/kill" + }, + "ExtRequest": { + "description": "Allows for sending an arbitrary request that is not part of the ACP spec.\nExtension methods provide a way to add custom functionality while maintaining\nprotocol compatibility.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)" + }, + "AgentResponse": { + "description": "A JSON-RPC response object.", + "anyOf": [ + { + "title": "Result", + "description": "A successful JSON-RPC response.", + "type": "object", + "properties": { + "id": { + "description": "The id of the request this response answers.", + "allOf": [ + { + "$ref": "#/$defs/RequestId" + } + ] + }, + "result": { + "description": "Method-specific response data.", + "anyOf": [ + { + "title": "InitializeResponse", + "description": "Successful result returned for a `initialize` request.", + "allOf": [ + { + "$ref": "#/$defs/InitializeResponse" + } + ] + }, + { + "title": "AuthenticateResponse", + "description": "Successful result returned for a `authenticate` request.", + "allOf": [ + { + "$ref": "#/$defs/AuthenticateResponse" + } + ] + }, + { + "title": "LogoutResponse", + "description": "Successful result returned for a `logout` request.", + "allOf": [ + { + "$ref": "#/$defs/LogoutResponse" + } + ] + }, + { + "title": "NewSessionResponse", + "description": "Successful result returned for a `session/new` request.", + "allOf": [ + { + "$ref": "#/$defs/NewSessionResponse" + } + ] + }, + { + "title": "LoadSessionResponse", + "description": "Successful result returned for a `session/load` request.", + "allOf": [ + { + "$ref": "#/$defs/LoadSessionResponse" + } + ] + }, + { + "title": "ListSessionsResponse", + "description": "Successful result returned for a `session/list` request.", + "allOf": [ + { + "$ref": "#/$defs/ListSessionsResponse" + } + ] + }, + { + "title": "DeleteSessionResponse", + "description": "Successful result returned for a `session/delete` request.", + "allOf": [ + { + "$ref": "#/$defs/DeleteSessionResponse" + } + ] + }, + { + "title": "ResumeSessionResponse", + "description": "Successful result returned for a `session/resume` request.", + "allOf": [ + { + "$ref": "#/$defs/ResumeSessionResponse" + } + ] + }, + { + "title": "CloseSessionResponse", + "description": "Successful result returned for a `session/close` request.", + "allOf": [ + { + "$ref": "#/$defs/CloseSessionResponse" + } + ] + }, + { + "title": "SetSessionModeResponse", + "description": "Successful result returned for a `session/set_mode` request.", + "allOf": [ + { + "$ref": "#/$defs/SetSessionModeResponse" + } + ] + }, + { + "title": "SetSessionConfigOptionResponse", + "description": "Successful result returned for a `session/set_config_option` request.", + "allOf": [ + { + "$ref": "#/$defs/SetSessionConfigOptionResponse" + } + ] + }, + { + "title": "PromptResponse", + "description": "Successful result returned for a `session/prompt` request.", + "allOf": [ + { + "$ref": "#/$defs/PromptResponse" + } + ] + }, + { + "title": "ExtMethodResponse", + "description": "Successful result returned by an extension method outside the core ACP method set.", + "allOf": [ + { + "$ref": "#/$defs/ExtResponse" + } + ] + } + ] + } + }, + "required": ["id", "result"] + }, + { + "title": "Error", + "description": "A failed JSON-RPC response.", + "type": "object", + "properties": { + "id": { + "description": "The id of the request this response answers.", + "allOf": [ + { + "$ref": "#/$defs/RequestId" + } + ] + }, + "error": { + "description": "Method-specific error data.", + "allOf": [ + { + "$ref": "#/$defs/Error" + } + ] + } + }, + "required": ["id", "error"] + } + ], + "x-docs-ignore": true + }, + "InitializeResponse": { + "description": "Response to the `initialize` method.\n\nContains the negotiated protocol version and agent capabilities.\n\nSee protocol docs: [Initialization](https://agentclientprotocol.com/protocol/initialization)", + "type": "object", + "properties": { + "protocolVersion": { + "description": "The protocol version the client specified if supported by the agent,\nor the latest protocol version supported by the agent.\n\nThe client should disconnect, if it doesn't support this version.", + "allOf": [ + { + "$ref": "#/$defs/ProtocolVersion" + } + ] + }, + "agentCapabilities": { + "description": "Capabilities supported by the agent.", + "x-deserialize-default-on-error": true, + "default": { + "loadSession": false, + "promptCapabilities": { + "image": false, + "audio": false, + "embeddedContext": false + }, + "mcpCapabilities": { + "http": false, + "sse": false + }, + "sessionCapabilities": {}, + "auth": {} + }, + "allOf": [ + { + "$ref": "#/$defs/AgentCapabilities" + } + ] + }, + "authMethods": { + "description": "Authentication methods supported by the agent.", + "type": "array", + "items": { + "$ref": "#/$defs/AuthMethod" + }, + "default": [], + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "agentInfo": { + "description": "Information about the Agent name and version sent to the Client.\n\nNote: in future versions of the protocol, this will be required.", + "anyOf": [ + { + "$ref": "#/$defs/Implementation" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["protocolVersion"], + "x-side": "agent", + "x-method": "initialize" + }, + "ProtocolVersion": { + "description": "Protocol version identifier.\n\nThis version is only bumped for breaking changes.\nNon-breaking changes should be introduced via capabilities.", + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + }, + "AgentCapabilities": { + "description": "Capabilities supported by the agent.\n\nAdvertised during initialization to inform the client about\navailable features and content types.\n\nSee protocol docs: [Agent Capabilities](https://agentclientprotocol.com/protocol/initialization#agent-capabilities)", + "type": "object", + "properties": { + "loadSession": { + "description": "Whether the agent supports `session/load`.", + "type": "boolean", + "default": false, + "x-deserialize-default-on-error": true + }, + "promptCapabilities": { + "description": "Prompt capabilities supported by the agent.", + "x-deserialize-default-on-error": true, + "default": { + "image": false, + "audio": false, + "embeddedContext": false + }, + "allOf": [ + { + "$ref": "#/$defs/PromptCapabilities" + } + ] + }, + "mcpCapabilities": { + "description": "MCP capabilities supported by the agent.", + "x-deserialize-default-on-error": true, + "default": { + "http": false, + "sse": false + }, + "allOf": [ + { + "$ref": "#/$defs/McpCapabilities" + } + ] + }, + "sessionCapabilities": { + "description": "Session lifecycle and prompt capabilities advertised by the agent.", + "x-deserialize-default-on-error": true, + "default": {}, + "allOf": [ + { + "$ref": "#/$defs/SessionCapabilities" + } + ] + }, + "auth": { + "description": "Authentication-related capabilities supported by the agent.", + "x-deserialize-default-on-error": true, + "default": {}, + "allOf": [ + { + "$ref": "#/$defs/AgentAuthCapabilities" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "PromptCapabilities": { + "description": "Prompt capabilities supported by the agent in `session/prompt` requests.\n\nBaseline agent functionality requires support for [`ContentBlock::Text`]\nand [`ContentBlock::ResourceLink`] in prompt requests.\n\nOther variants must be explicitly opted in to.\nCapabilities for different types of content in prompt requests.\n\nIndicates which content types beyond the baseline (text and resource links)\nthe agent can process.\n\nSee protocol docs: [Prompt Capabilities](https://agentclientprotocol.com/protocol/initialization#prompt-capabilities)", + "type": "object", + "properties": { + "image": { + "description": "Agent supports [`ContentBlock::Image`].", + "type": "boolean", + "default": false, + "x-deserialize-default-on-error": true + }, + "audio": { + "description": "Agent supports [`ContentBlock::Audio`].", + "type": "boolean", + "default": false, + "x-deserialize-default-on-error": true + }, + "embeddedContext": { + "description": "Agent supports embedded context in `session/prompt` requests.\n\nWhen enabled, the Client is allowed to include [`ContentBlock::Resource`]\nin prompt requests for pieces of context that are referenced in the message.", + "type": "boolean", + "default": false, + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "McpCapabilities": { + "description": "MCP capabilities supported by the agent", + "type": "object", + "properties": { + "http": { + "description": "Agent supports [`McpServer::Http`].", + "type": "boolean", + "default": false, + "x-deserialize-default-on-error": true + }, + "sse": { + "description": "Agent supports [`McpServer::Sse`].", + "type": "boolean", + "default": false, + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "SessionCapabilities": { + "description": "Session capabilities supported by the agent.\n\nAs a baseline, all Agents **MUST** support `session/new`, `session/prompt`, `session/cancel`, and `session/update`.\n\nOptionally, they **MAY** support other session methods and notifications by specifying additional capabilities.\n\nNote: `session/load` is still handled by the top-level `load_session` capability. This will be unified in future versions of the protocol.\n\nSee protocol docs: [Session Capabilities](https://agentclientprotocol.com/protocol/initialization#session-capabilities)", + "type": "object", + "properties": { + "list": { + "description": "Whether the agent supports `session/list`.\n\nOptional. Omitted or `null` both mean the agent does not advertise support.\nSupplying `{}` means the agent supports listing sessions.", + "anyOf": [ + { + "$ref": "#/$defs/SessionListCapabilities" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "delete": { + "description": "Whether the agent supports `session/delete`.\n\nOptional. Omitted or `null` both mean the agent does not advertise support.\nSupplying `{}` means the agent supports deleting sessions from `session/list`.", + "anyOf": [ + { + "$ref": "#/$defs/SessionDeleteCapabilities" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "additionalDirectories": { + "description": "Whether the agent supports `additionalDirectories` on supported session lifecycle requests.\n\nOptional. Omitted or `null` both mean the agent does not advertise support.\nSupplying `{}` means the agent supports `additionalDirectories` on\nsupported session lifecycle requests.\n\nAgents that also support `session/list` may return\n`SessionInfo.additionalDirectories` to report the complete ordered\nadditional-root list associated with a listed session.", + "anyOf": [ + { + "$ref": "#/$defs/SessionAdditionalDirectoriesCapabilities" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "resume": { + "description": "Whether the agent supports `session/resume`.\n\nOptional. Omitted or `null` both mean the agent does not advertise support.\nSupplying `{}` means the agent supports resuming sessions.", + "anyOf": [ + { + "$ref": "#/$defs/SessionResumeCapabilities" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "close": { + "description": "Whether the agent supports `session/close`.\n\nOptional. Omitted or `null` both mean the agent does not advertise support.\nSupplying `{}` means the agent supports closing sessions.", + "anyOf": [ + { + "$ref": "#/$defs/SessionCloseCapabilities" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "SessionListCapabilities": { + "description": "Capabilities for the `session/list` method.\n\nSupplying `{}` means the agent supports listing sessions.", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "SessionDeleteCapabilities": { + "description": "Capabilities for the `session/delete` method.\n\nSupplying `{}` means the agent supports deleting sessions from `session/list`.", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "SessionAdditionalDirectoriesCapabilities": { + "description": "Capabilities for additional session directories support.\n\nSupplying `{}` means the agent supports the `additionalDirectories` field on\nsupported session lifecycle requests. Agents that also support\n`session/list` may return `SessionInfo.additionalDirectories` to report the\ncomplete ordered additional-root list associated with a listed session.", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "SessionResumeCapabilities": { + "description": "Capabilities for the `session/resume` method.\n\nSupplying `{}` means the agent supports resuming sessions.", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "SessionCloseCapabilities": { + "description": "Capabilities for the `session/close` method.\n\nSupplying `{}` means the agent supports closing sessions.", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "AgentAuthCapabilities": { + "description": "Authentication-related capabilities supported by the agent.", + "type": "object", + "properties": { + "logout": { + "description": "Whether the agent supports the logout method.\n\nOptional. Omitted or `null` both mean the agent does not advertise support.\nSupplying `{}` means the agent supports the logout method.", + "anyOf": [ + { + "$ref": "#/$defs/LogoutCapabilities" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "LogoutCapabilities": { + "description": "Logout capabilities supported by the agent.\n\nSupplying `{}` means the agent supports the logout method.", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "AuthMethod": { + "description": "Describes an available authentication method.\n\nThe `type` field acts as the discriminator in the serialized JSON form.\nWhen no `type` is present, the method is treated as `agent`.", + "anyOf": [ + { + "title": "agent", + "description": "Agent handles authentication itself.\n\nThis is the default when no `type` is specified.", + "allOf": [ + { + "$ref": "#/$defs/AuthMethodAgent" + } + ] + } + ] + }, + "AuthMethodAgent": { + "description": "Agent handles authentication itself.\n\nThis is the default authentication method type.", + "type": "object", + "properties": { + "id": { + "description": "Unique identifier for this authentication method.", + "allOf": [ + { + "$ref": "#/$defs/AuthMethodId" + } + ] + }, + "name": { + "description": "Human-readable name of the authentication method.", + "type": "string" + }, + "description": { + "description": "Optional description providing more details about this authentication method.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["id", "name"] + }, + "AuthMethodId": { + "description": "Typed identifier used for auth method values on the wire.", + "type": "string" + }, + "Implementation": { + "description": "Metadata about the implementation of the client or agent.\nDescribes the name and version of an ACP implementation, with an optional\ntitle for UI representation.", + "type": "object", + "properties": { + "name": { + "description": "Intended for programmatic or logical use, but can be used as a display\nname fallback if title isn’t present.", + "type": "string" + }, + "title": { + "description": "Intended for UI and end-user contexts — optimized to be human-readable\nand easily understood.\n\nIf not provided, the name should be used for display.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "version": { + "description": "Version of the implementation. Can be displayed to the user or used\nfor debugging or metrics purposes. (e.g. \"1.0.0\").", + "type": "string" + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["name", "version"] + }, + "AuthenticateResponse": { + "description": "Response to the `authenticate` method.", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "x-side": "agent", + "x-method": "authenticate" + }, + "LogoutResponse": { + "description": "Response to the `logout` method.", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "x-side": "agent", + "x-method": "logout" + }, + "NewSessionResponse": { + "description": "Response from creating a new session.\n\nSee protocol docs: [Creating a Session](https://agentclientprotocol.com/protocol/session-setup#creating-a-session)", + "type": "object", + "properties": { + "sessionId": { + "description": "Unique identifier for the created session.\n\nUsed in all subsequent requests for this conversation.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "modes": { + "description": "Initial mode state if supported by the Agent\n\nSee protocol docs: [Session Modes](https://agentclientprotocol.com/protocol/session-modes)", + "anyOf": [ + { + "$ref": "#/$defs/SessionModeState" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "configOptions": { + "description": "Initial session configuration options if supported by the Agent.", + "type": ["array", "null"], + "items": { + "$ref": "#/$defs/SessionConfigOption" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId"], + "x-side": "agent", + "x-method": "session/new" + }, + "SessionModeState": { + "description": "The set of modes and the one currently active.", + "type": "object", + "properties": { + "currentModeId": { + "description": "The current mode the Agent is in.", + "allOf": [ + { + "$ref": "#/$defs/SessionModeId" + } + ] + }, + "availableModes": { + "description": "The set of modes that the Agent can operate in", + "type": "array", + "items": { + "$ref": "#/$defs/SessionMode" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["currentModeId", "availableModes"] + }, + "SessionModeId": { + "description": "Unique identifier for a Session Mode.", + "type": "string" + }, + "SessionMode": { + "description": "A mode the agent can operate in.\n\nSee protocol docs: [Session Modes](https://agentclientprotocol.com/protocol/session-modes)", + "type": "object", + "properties": { + "id": { + "description": "Stable identifier used to refer to this protocol object in later messages.", + "allOf": [ + { + "$ref": "#/$defs/SessionModeId" + } + ] + }, + "name": { + "description": "Human-readable name shown for this protocol object.", + "type": "string" + }, + "description": { + "description": "Optional human-readable details shown with this protocol object.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["id", "name"] + }, + "SessionConfigOption": { + "description": "A session configuration option selector and its current state.", + "type": "object", + "properties": { + "id": { + "description": "Unique identifier for the configuration option.", + "allOf": [ + { + "$ref": "#/$defs/SessionConfigId" + } + ] + }, + "name": { + "description": "Human-readable label for the option.", + "type": "string" + }, + "description": { + "description": "Optional description for the Client to display to the user.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "category": { + "description": "Optional semantic category for this option (UX only).", + "anyOf": [ + { + "$ref": "#/$defs/SessionConfigOptionCategory" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["id", "name"], + "oneOf": [ + { + "description": "Single-value selector (dropdown).", + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "select" + } + }, + "required": ["type"], + "allOf": [ + { + "$ref": "#/$defs/SessionConfigSelect" + } + ] + }, + { + "description": "Boolean on/off toggle.", + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "boolean" + } + }, + "required": ["type"], + "allOf": [ + { + "$ref": "#/$defs/SessionConfigBoolean" + } + ] + } + ], + "discriminator": { + "propertyName": "type" + } + }, + "SessionConfigId": { + "description": "Unique identifier for a session configuration option.", + "type": "string" + }, + "SessionConfigOptionCategory": { + "description": "Semantic category for a session configuration option.\n\nThis is intended to help Clients distinguish broadly common selectors (e.g. model selector vs\nsession mode selector vs thought/reasoning level) for UX purposes (keyboard shortcuts, icons,\nplacement). It MUST NOT be required for correctness. Clients MUST handle missing or unknown\ncategories gracefully.\n\nCategory names beginning with `_` are free for custom use, like other ACP extension methods.\nCategory names that do not begin with `_` are reserved for the ACP spec.", + "anyOf": [ + { + "description": "Session mode selector.", + "type": "string", + "const": "mode" + }, + { + "description": "Model selector.", + "type": "string", + "const": "model" + }, + { + "description": "Model-related configuration parameter.", + "type": "string", + "const": "model_config" + }, + { + "description": "Thought/reasoning level selector.", + "type": "string", + "const": "thought_level" + }, + { + "title": "other", + "description": "Unknown / uncategorized selector.", + "type": "string" + } + ] + }, + "SessionConfigValueId": { + "description": "Unique identifier for a session configuration option value.", + "type": "string" + }, + "SessionConfigSelectOptions": { + "description": "Possible values for a session configuration option.", + "anyOf": [ + { + "title": "Ungrouped", + "description": "A flat list of options with no grouping.", + "type": "array", + "items": { + "$ref": "#/$defs/SessionConfigSelectOption" + } + }, + { + "title": "Grouped", + "description": "A list of options grouped under headers.", + "type": "array", + "items": { + "$ref": "#/$defs/SessionConfigSelectGroup" + } + } + ] + }, + "SessionConfigSelectOption": { + "description": "A possible value for a session configuration option.", + "type": "object", + "properties": { + "value": { + "description": "Unique identifier for this option value.", + "allOf": [ + { + "$ref": "#/$defs/SessionConfigValueId" + } + ] + }, + "name": { + "description": "Human-readable label for this option value.", + "type": "string" + }, + "description": { + "description": "Optional description for this option value.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["value", "name"] + }, + "SessionConfigSelectGroup": { + "description": "A group of possible values for a session configuration option.", + "type": "object", + "properties": { + "group": { + "description": "Unique identifier for this group.", + "allOf": [ + { + "$ref": "#/$defs/SessionConfigGroupId" + } + ] + }, + "name": { + "description": "Human-readable label for this group.", + "type": "string" + }, + "options": { + "description": "The set of option values in this group.", + "type": "array", + "items": { + "$ref": "#/$defs/SessionConfigSelectOption" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["group", "name", "options"] + }, + "SessionConfigGroupId": { + "description": "Unique identifier for a session configuration option value group.", + "type": "string" + }, + "SessionConfigSelect": { + "description": "A single-value selector (dropdown) session configuration option payload.", + "type": "object", + "properties": { + "currentValue": { + "description": "The currently selected value.", + "allOf": [ + { + "$ref": "#/$defs/SessionConfigValueId" + } + ] + }, + "options": { + "description": "The set of selectable options.", + "allOf": [ + { + "$ref": "#/$defs/SessionConfigSelectOptions" + } + ] + } + }, + "required": ["currentValue", "options"] + }, + "SessionConfigBoolean": { + "description": "A boolean on/off toggle session configuration option payload.", + "type": "object", + "properties": { + "currentValue": { + "description": "The current value of the boolean option.", + "type": "boolean" + } + }, + "required": ["currentValue"] + }, + "LoadSessionResponse": { + "description": "Response from loading an existing session.", + "type": "object", + "properties": { + "modes": { + "description": "Initial mode state if supported by the Agent\n\nSee protocol docs: [Session Modes](https://agentclientprotocol.com/protocol/session-modes)", + "anyOf": [ + { + "$ref": "#/$defs/SessionModeState" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "configOptions": { + "description": "Initial session configuration options if supported by the Agent.", + "type": ["array", "null"], + "items": { + "$ref": "#/$defs/SessionConfigOption" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "x-side": "agent", + "x-method": "session/load" + }, + "ListSessionsResponse": { + "description": "Response from listing sessions.", + "type": "object", + "properties": { + "sessions": { + "description": "Array of session information objects", + "type": "array", + "items": { + "$ref": "#/$defs/SessionInfo" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "nextCursor": { + "description": "Opaque cursor token. If present, pass this in the next request's cursor parameter\nto fetch the next page. If absent, there are no more results.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessions"], + "x-side": "agent", + "x-method": "session/list" + }, + "SessionInfo": { + "description": "Information about a session returned by session/list", + "type": "object", + "properties": { + "sessionId": { + "description": "Unique identifier for the session", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "cwd": { + "description": "The working directory for this session. Must be an absolute path.", + "type": "string" + }, + "additionalDirectories": { + "description": "Additional workspace roots reported for this session. Each path must be absolute.\n\nWhen present, this is the complete ordered additional-root list reported\nby the Agent. Omitted and empty values are equivalent: the response\nreports no additional roots.", + "type": "array", + "items": { + "type": "string" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "title": { + "description": "Human-readable title for the session", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "updatedAt": { + "description": "ISO 8601 timestamp of last activity", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "cwd"] + }, + "DeleteSessionResponse": { + "description": "Response from deleting a session.", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "x-side": "agent", + "x-method": "session/delete" + }, + "ResumeSessionResponse": { + "description": "Response from resuming an existing session.", + "type": "object", + "properties": { + "modes": { + "description": "Initial mode state if supported by the Agent\n\nSee protocol docs: [Session Modes](https://agentclientprotocol.com/protocol/session-modes)", + "anyOf": [ + { + "$ref": "#/$defs/SessionModeState" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "configOptions": { + "description": "Initial session configuration options if supported by the Agent.", + "type": ["array", "null"], + "items": { + "$ref": "#/$defs/SessionConfigOption" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "x-side": "agent", + "x-method": "session/resume" + }, + "CloseSessionResponse": { + "description": "Response from closing a session.", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "x-side": "agent", + "x-method": "session/close" + }, + "SetSessionModeResponse": { + "description": "Response to `session/set_mode` method.", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "x-side": "agent", + "x-method": "session/set_mode" + }, + "SetSessionConfigOptionResponse": { + "description": "Response to `session/set_config_option` method.", + "type": "object", + "properties": { + "configOptions": { + "description": "The full set of configuration options and their current values.", + "type": "array", + "items": { + "$ref": "#/$defs/SessionConfigOption" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["configOptions"], + "x-side": "agent", + "x-method": "session/set_config_option" + }, + "PromptResponse": { + "description": "Response from processing a user prompt.\n\nSee protocol docs: [Check for Completion](https://agentclientprotocol.com/protocol/prompt-turn#4-check-for-completion)", + "type": "object", + "properties": { + "stopReason": { + "description": "Indicates why the agent stopped processing the turn.", + "allOf": [ + { + "$ref": "#/$defs/StopReason" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["stopReason"], + "x-side": "agent", + "x-method": "session/prompt" + }, + "StopReason": { + "description": "Reasons why an agent stops processing a prompt turn.\n\nSee protocol docs: [Stop Reasons](https://agentclientprotocol.com/protocol/prompt-turn#stop-reasons)", + "oneOf": [ + { + "description": "The turn ended successfully.", + "type": "string", + "const": "end_turn" + }, + { + "description": "The turn ended because the agent reached the maximum number of tokens.", + "type": "string", + "const": "max_tokens" + }, + { + "description": "The turn ended because the agent reached the maximum number of allowed\nagent requests between user turns.", + "type": "string", + "const": "max_turn_requests" + }, + { + "description": "The turn ended because the agent refused to continue. The user prompt\nand everything that comes after it won't be included in the next\nprompt, so this should be reflected in the UI.", + "type": "string", + "const": "refusal" + }, + { + "description": "The turn was cancelled by the client via `session/cancel`.\n\nThis stop reason MUST be returned when the client sends a `session/cancel`\nnotification, even if the cancellation causes exceptions in underlying operations.\nAgents should catch these exceptions and return this semantically meaningful\nresponse to confirm successful cancellation.", + "type": "string", + "const": "cancelled" + } + ] + }, + "ExtResponse": { + "description": "Allows for sending an arbitrary response to an [`ExtRequest`] that is not part of the ACP spec.\nExtension methods provide a way to add custom functionality while maintaining\nprotocol compatibility.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)" + }, + "Error": { + "description": "JSON-RPC error object.\n\nRepresents an error that occurred during method execution, following the\nJSON-RPC 2.0 error object specification with optional additional data.\n\nSee protocol docs: [JSON-RPC Error Object](https://www.jsonrpc.org/specification#error_object)", + "type": "object", + "properties": { + "code": { + "description": "A number indicating the error type that occurred.\nThis must be an integer as defined in the JSON-RPC specification.", + "allOf": [ + { + "$ref": "#/$defs/ErrorCode" + } + ] + }, + "message": { + "description": "A string providing a short description of the error.\nThe message should be limited to a concise single sentence.", + "type": "string" + }, + "data": { + "description": "Optional primitive or structured value that contains additional information about the error.\nThis may include debugging information or context-specific details.", + "x-deserialize-default-on-error": true + } + }, + "required": ["code", "message"] + }, + "ErrorCode": { + "description": "Predefined error codes for common JSON-RPC and ACP-specific errors.\n\nThese codes follow the JSON-RPC 2.0 specification for standard errors\nand use the reserved range (-32000 to -32099) for protocol-specific errors.", + "anyOf": [ + { + "title": "Parse error", + "description": "**Parse error**: Invalid JSON was received by the server.\nAn error occurred on the server while parsing the JSON text.", + "type": "integer", + "format": "int32", + "const": -32700 + }, + { + "title": "Invalid request", + "description": "**Invalid request**: The JSON sent is not a valid Request object.", + "type": "integer", + "format": "int32", + "const": -32600 + }, + { + "title": "Method not found", + "description": "**Method not found**: The method does not exist or is not available.", + "type": "integer", + "format": "int32", + "const": -32601 + }, + { + "title": "Invalid params", + "description": "**Invalid params**: Invalid method parameter(s).", + "type": "integer", + "format": "int32", + "const": -32602 + }, + { + "title": "Internal error", + "description": "**Internal error**: Internal JSON-RPC error.\nReserved for implementation-defined server errors.", + "type": "integer", + "format": "int32", + "const": -32603 + }, + { + "title": "Request cancelled", + "description": "**Request cancelled**: Execution of the method was aborted either due to a cancellation request from the caller or\nbecause of resource constraints or shutdown.", + "type": "integer", + "format": "int32", + "const": -32800 + }, + { + "title": "Authentication required", + "description": "**Authentication required**: Authentication is required before this operation can be performed.", + "type": "integer", + "format": "int32", + "const": -32000 + }, + { + "title": "Resource not found", + "description": "**Resource not found**: A given resource, such as a file, was not found.", + "type": "integer", + "format": "int32", + "const": -32002 + }, + { + "title": "Other", + "description": "Other undefined error code.", + "type": "integer", + "format": "int32" + } + ] + }, + "AgentNotification": { + "description": "A JSON-RPC notification object.", + "type": "object", + "properties": { + "method": { + "description": "The notification method name.", + "type": "string" + }, + "params": { + "description": "Method-specific notification parameters.", + "anyOf": [ + { + "description": "All possible notifications that an agent can send to a client.\n\nThis enum is used internally for routing RPC notifications. You typically won't need\nto use this directly.\n\nNotifications do not expect a response.", + "anyOf": [ + { + "title": "SessionNotification", + "description": "Handles session update notifications from the agent.\n\nThis is a notification endpoint (no response expected) that receives\nreal-time updates about session progress, including message chunks,\ntool calls, and execution plans.\n\nNote: Clients SHOULD continue accepting tool call updates even after\nsending a `session/cancel` notification, as the agent may send final\nupdates before responding with the cancelled stop reason.\n\nSee protocol docs: [Agent Reports Output](https://agentclientprotocol.com/protocol/prompt-turn#3-agent-reports-output)", + "allOf": [ + { + "$ref": "#/$defs/SessionNotification" + } + ] + }, + { + "title": "ExtNotification", + "description": "Handles extension notifications from the agent.\n\nAllows the Agent to send an arbitrary notification that is not part of the ACP spec.\nExtension notifications provide a way to send one-way messages for custom functionality\nwhile maintaining protocol compatibility.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "allOf": [ + { + "$ref": "#/$defs/ExtNotification" + } + ] + } + ] + }, + { + "type": "null" + } + ] + } + }, + "required": ["method"], + "x-docs-ignore": true + }, + "SessionNotification": { + "description": "Notification containing a session update from the agent.\n\nUsed to stream real-time progress and results during prompt processing.\n\nSee protocol docs: [Agent Reports Output](https://agentclientprotocol.com/protocol/prompt-turn#3-agent-reports-output)", + "type": "object", + "properties": { + "sessionId": { + "description": "The ID of the session this update pertains to.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "update": { + "description": "The actual update content.", + "allOf": [ + { + "$ref": "#/$defs/SessionUpdate" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "update"], + "x-side": "client", + "x-method": "session/update" + }, + "SessionUpdate": { + "description": "Different types of updates that can be sent during session processing.\n\nThese updates provide real-time feedback about the agent's progress.\n\nSee protocol docs: [Agent Reports Output](https://agentclientprotocol.com/protocol/prompt-turn#3-agent-reports-output)", + "oneOf": [ + { + "description": "A chunk of the user's message being streamed.", + "type": "object", + "properties": { + "sessionUpdate": { + "type": "string", + "const": "user_message_chunk" + } + }, + "required": ["sessionUpdate"], + "allOf": [ + { + "$ref": "#/$defs/ContentChunk" + } + ] + }, + { + "description": "A chunk of the agent's response being streamed.", + "type": "object", + "properties": { + "sessionUpdate": { + "type": "string", + "const": "agent_message_chunk" + } + }, + "required": ["sessionUpdate"], + "allOf": [ + { + "$ref": "#/$defs/ContentChunk" + } + ] + }, + { + "description": "A chunk of the agent's internal reasoning being streamed.", + "type": "object", + "properties": { + "sessionUpdate": { + "type": "string", + "const": "agent_thought_chunk" + } + }, + "required": ["sessionUpdate"], + "allOf": [ + { + "$ref": "#/$defs/ContentChunk" + } + ] + }, + { + "description": "Notification that a new tool call has been initiated.", + "type": "object", + "properties": { + "sessionUpdate": { + "type": "string", + "const": "tool_call" + } + }, + "required": ["sessionUpdate"], + "allOf": [ + { + "$ref": "#/$defs/ToolCall" + } + ] + }, + { + "description": "Update on the status or results of a tool call.", + "type": "object", + "properties": { + "sessionUpdate": { + "type": "string", + "const": "tool_call_update" + } + }, + "required": ["sessionUpdate"], + "allOf": [ + { + "$ref": "#/$defs/ToolCallUpdate" + } + ] + }, + { + "description": "The agent's execution plan for complex tasks.\nSee protocol docs: [Agent Plan](https://agentclientprotocol.com/protocol/agent-plan)", + "type": "object", + "properties": { + "sessionUpdate": { + "type": "string", + "const": "plan" + } + }, + "required": ["sessionUpdate"], + "allOf": [ + { + "$ref": "#/$defs/Plan" + } + ] + }, + { + "description": "Available commands are ready or have changed", + "type": "object", + "properties": { + "sessionUpdate": { + "type": "string", + "const": "available_commands_update" + } + }, + "required": ["sessionUpdate"], + "allOf": [ + { + "$ref": "#/$defs/AvailableCommandsUpdate" + } + ] + }, + { + "description": "The current mode of the session has changed\n\nSee protocol docs: [Session Modes](https://agentclientprotocol.com/protocol/session-modes)", + "type": "object", + "properties": { + "sessionUpdate": { + "type": "string", + "const": "current_mode_update" + } + }, + "required": ["sessionUpdate"], + "allOf": [ + { + "$ref": "#/$defs/CurrentModeUpdate" + } + ] + }, + { + "description": "Session configuration options have been updated.", + "type": "object", + "properties": { + "sessionUpdate": { + "type": "string", + "const": "config_option_update" + } + }, + "required": ["sessionUpdate"], + "allOf": [ + { + "$ref": "#/$defs/ConfigOptionUpdate" + } + ] + }, + { + "description": "Session metadata has been updated (title, timestamps, custom metadata)", + "type": "object", + "properties": { + "sessionUpdate": { + "type": "string", + "const": "session_info_update" + } + }, + "required": ["sessionUpdate"], + "allOf": [ + { + "$ref": "#/$defs/SessionInfoUpdate" + } + ] + }, + { + "description": "Context window and cost update for the session.", + "type": "object", + "properties": { + "sessionUpdate": { + "type": "string", + "const": "usage_update" + } + }, + "required": ["sessionUpdate"], + "allOf": [ + { + "$ref": "#/$defs/UsageUpdate" + } + ] + } + ], + "discriminator": { + "propertyName": "sessionUpdate" + } + }, + "MessageId": { + "description": "Unique identifier for a message within a session.", + "type": "string" + }, + "ContentChunk": { + "description": "A streamed item of content", + "type": "object", + "properties": { + "content": { + "description": "A single item of content", + "allOf": [ + { + "$ref": "#/$defs/ContentBlock" + } + ] + }, + "messageId": { + "description": "A unique identifier for the message this chunk belongs to.\n\nAll chunks belonging to the same message share the same `messageId`.\nA change in `messageId` indicates a new message has started.", + "anyOf": [ + { + "$ref": "#/$defs/MessageId" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["content"] + }, + "ToolCall": { + "description": "Represents a tool call that the language model has requested.\n\nTool calls are actions that the agent executes on behalf of the language model,\nsuch as reading files, executing code, or fetching data from external sources.\n\nSee protocol docs: [Tool Calls](https://agentclientprotocol.com/protocol/tool-calls)", + "type": "object", + "properties": { + "toolCallId": { + "description": "Unique identifier for this tool call within the session.", + "allOf": [ + { + "$ref": "#/$defs/ToolCallId" + } + ] + }, + "title": { + "description": "Human-readable title describing what the tool is doing.", + "type": "string" + }, + "kind": { + "description": "The category of tool being invoked.\nHelps clients choose appropriate icons and UI treatment.", + "x-deserialize-default-on-error": true, + "allOf": [ + { + "$ref": "#/$defs/ToolKind" + } + ] + }, + "status": { + "description": "Current execution status of the tool call.", + "x-deserialize-default-on-error": true, + "allOf": [ + { + "$ref": "#/$defs/ToolCallStatus" + } + ] + }, + "content": { + "description": "Content produced by the tool call.", + "type": "array", + "items": { + "$ref": "#/$defs/ToolCallContent" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "locations": { + "description": "File locations affected by this tool call.\nEnables \"follow-along\" features in clients.", + "type": "array", + "items": { + "$ref": "#/$defs/ToolCallLocation" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "rawInput": { + "description": "Raw input parameters sent to the tool.", + "x-deserialize-default-on-error": true + }, + "rawOutput": { + "description": "Raw output returned by the tool.", + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["toolCallId", "title"] + }, + "PlanEntry": { + "description": "A single entry in the execution plan.\n\nRepresents a task or goal that the assistant intends to accomplish\nas part of fulfilling the user's request.\nSee protocol docs: [Plan Entries](https://agentclientprotocol.com/protocol/agent-plan#plan-entries)", + "type": "object", + "properties": { + "content": { + "description": "Human-readable description of what this task aims to accomplish.", + "type": "string" + }, + "priority": { + "description": "The relative importance of this task.\nUsed to indicate which tasks are most critical to the overall goal.", + "allOf": [ + { + "$ref": "#/$defs/PlanEntryPriority" + } + ] + }, + "status": { + "description": "Current execution status of this task.", + "allOf": [ + { + "$ref": "#/$defs/PlanEntryStatus" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["content", "priority", "status"] + }, + "PlanEntryPriority": { + "description": "Priority levels for plan entries.\n\nUsed to indicate the relative importance or urgency of different\ntasks in the execution plan.\nSee protocol docs: [Plan Entries](https://agentclientprotocol.com/protocol/agent-plan#plan-entries)", + "oneOf": [ + { + "description": "High priority task - critical to the overall goal.", + "type": "string", + "const": "high" + }, + { + "description": "Medium priority task - important but not critical.", + "type": "string", + "const": "medium" + }, + { + "description": "Low priority task - nice to have but not essential.", + "type": "string", + "const": "low" + } + ] + }, + "PlanEntryStatus": { + "description": "Status of a plan entry in the execution flow.\n\nTracks the lifecycle of each task from planning through completion.\nSee protocol docs: [Plan Entries](https://agentclientprotocol.com/protocol/agent-plan#plan-entries)", + "oneOf": [ + { + "description": "The task has not started yet.", + "type": "string", + "const": "pending" + }, + { + "description": "The task is currently being worked on.", + "type": "string", + "const": "in_progress" + }, + { + "description": "The task has been successfully completed.", + "type": "string", + "const": "completed" + } + ] + }, + "Plan": { + "description": "An execution plan for accomplishing complex tasks.\n\nPlans consist of multiple entries representing individual tasks or goals.\nAgents report plans to clients to provide visibility into their execution strategy.\nPlans can evolve during execution as the agent discovers new requirements or completes tasks.\n\nSee protocol docs: [Agent Plan](https://agentclientprotocol.com/protocol/agent-plan)", + "type": "object", + "properties": { + "entries": { + "description": "The list of tasks to be accomplished.\n\nWhen updating a plan, the agent must send a complete list of all entries\nwith their current status. The client replaces the entire plan with each update.", + "type": "array", + "items": { + "$ref": "#/$defs/PlanEntry" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["entries"] + }, + "AvailableCommand": { + "description": "Information about a command.", + "type": "object", + "properties": { + "name": { + "description": "Command name (e.g., `create_plan`, `research_codebase`).", + "type": "string" + }, + "description": { + "description": "Human-readable description of what the command does.", + "type": "string" + }, + "input": { + "description": "Input for the command if required", + "anyOf": [ + { + "$ref": "#/$defs/AvailableCommandInput" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["name", "description"] + }, + "AvailableCommandInput": { + "description": "The input specification for a command.", + "anyOf": [ + { + "title": "unstructured", + "description": "All text that was typed after the command name is provided as input.", + "allOf": [ + { + "$ref": "#/$defs/UnstructuredCommandInput" + } + ] + } + ] + }, + "UnstructuredCommandInput": { + "description": "All text that was typed after the command name is provided as input.", + "type": "object", + "properties": { + "hint": { + "description": "A hint to display when the input hasn't been provided yet", + "type": "string" + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["hint"] + }, + "AvailableCommandsUpdate": { + "description": "Available commands are ready or have changed", + "type": "object", + "properties": { + "availableCommands": { + "description": "Commands the agent can execute", + "type": "array", + "items": { + "$ref": "#/$defs/AvailableCommand" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["availableCommands"] + }, + "CurrentModeUpdate": { + "description": "The current mode of the session has changed\n\nSee protocol docs: [Session Modes](https://agentclientprotocol.com/protocol/session-modes)", + "type": "object", + "properties": { + "currentModeId": { + "description": "The ID of the current mode", + "allOf": [ + { + "$ref": "#/$defs/SessionModeId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["currentModeId"] + }, + "ConfigOptionUpdate": { + "description": "Session configuration options have been updated.", + "type": "object", + "properties": { + "configOptions": { + "description": "The full set of configuration options and their current values.", + "type": "array", + "items": { + "$ref": "#/$defs/SessionConfigOption" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["configOptions"] + }, + "SessionInfoUpdate": { + "description": "Update to session metadata. All fields are optional to support partial updates.\n\nAgents send this notification to update session information like title or custom metadata.\nThis allows clients to display dynamic session names and track session state changes.", + "type": "object", + "properties": { + "title": { + "description": "Human-readable title for the session. Set to null to clear.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "updatedAt": { + "description": "ISO 8601 timestamp of last activity. Set to null to clear.", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "Cost": { + "description": "Cost information for a session.", + "type": "object", + "properties": { + "amount": { + "description": "Total cumulative cost for session.", + "type": "number", + "format": "double" + }, + "currency": { + "description": "ISO 4217 currency code (e.g., \"USD\", \"EUR\").", + "type": "string" + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["amount", "currency"] + }, + "UsageUpdate": { + "description": "Context window and cost update for a session.", + "type": "object", + "properties": { + "used": { + "description": "Tokens currently in context.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "size": { + "description": "Total context window size in tokens.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "cost": { + "description": "Cumulative session cost (optional).", + "anyOf": [ + { + "$ref": "#/$defs/Cost" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["used", "size"] + }, + "ExtNotification": { + "description": "Allows the Agent to send an arbitrary notification that is not part of the ACP spec.\nExtension notifications provide a way to send one-way messages for custom functionality\nwhile maintaining protocol compatibility.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)" + }, + "ClientRequest": { + "description": "A JSON-RPC request object.", + "type": "object", + "properties": { + "id": { + "description": "The request id used to correlate the matching response.", + "allOf": [ + { + "$ref": "#/$defs/RequestId" + } + ] + }, + "method": { + "description": "The method name to invoke.", + "type": "string" + }, + "params": { + "description": "Method-specific request parameters.", + "anyOf": [ + { + "description": "All possible requests that a client can send to an agent.\n\nThis enum is used internally for routing RPC requests. You typically won't need\nto use this directly.\n\nThis enum encompasses all method calls from client to agent.", + "anyOf": [ + { + "title": "InitializeRequest", + "description": "Establishes the connection with a client and negotiates protocol capabilities.\n\nThis method is called once at the beginning of the connection to:\n- Negotiate the protocol version to use\n- Exchange capability information between client and agent\n- Determine available authentication methods\n\nThe agent should respond with its supported protocol version and capabilities.\n\nSee protocol docs: [Initialization](https://agentclientprotocol.com/protocol/initialization)", + "allOf": [ + { + "$ref": "#/$defs/InitializeRequest" + } + ] + }, + { + "title": "AuthenticateRequest", + "description": "Authenticates the client using the specified authentication method.\n\nCalled when the agent requires authentication before allowing session creation.\nThe client provides the authentication method ID that was advertised during initialization.\n\nAfter successful authentication, the client can proceed to create sessions with\n`new_session` without receiving an `auth_required` error.\n\nSee protocol docs: [Initialization](https://agentclientprotocol.com/protocol/initialization)", + "allOf": [ + { + "$ref": "#/$defs/AuthenticateRequest" + } + ] + }, + { + "title": "LogoutRequest", + "description": "Logs out of the current authenticated state.\n\nAfter a successful logout, all new sessions will require authentication.\nThere is no guarantee about the behavior of already running sessions.", + "allOf": [ + { + "$ref": "#/$defs/LogoutRequest" + } + ] + }, + { + "title": "NewSessionRequest", + "description": "Creates a new conversation session with the agent.\n\nSessions represent independent conversation contexts with their own history and state.\n\nThe agent should:\n- Create a new session context\n- Connect to any specified MCP servers\n- Return a unique session ID for future requests\n\nMay return an `auth_required` error if the agent requires authentication.\n\nSee protocol docs: [Session Setup](https://agentclientprotocol.com/protocol/session-setup)", + "allOf": [ + { + "$ref": "#/$defs/NewSessionRequest" + } + ] + }, + { + "title": "LoadSessionRequest", + "description": "Loads an existing session to resume a previous conversation.\n\nThis method is only available if the agent advertises the `loadSession` capability.\n\nThe agent should:\n- Restore the session context and conversation history\n- Connect to the specified MCP servers\n- Stream the entire conversation history back to the client via notifications\n\nSee protocol docs: [Loading Sessions](https://agentclientprotocol.com/protocol/session-setup#loading-sessions)", + "allOf": [ + { + "$ref": "#/$defs/LoadSessionRequest" + } + ] + }, + { + "title": "ListSessionsRequest", + "description": "Lists existing sessions known to the agent.\n\nThis method is only available if the agent advertises the `sessionCapabilities.list` capability.\n\nThe agent should return metadata about sessions with optional filtering and pagination support.", + "allOf": [ + { + "$ref": "#/$defs/ListSessionsRequest" + } + ] + }, + { + "title": "DeleteSessionRequest", + "description": "Deletes an existing session from `session/list`.\n\nThis method is only available if the agent advertises the `sessionCapabilities.delete` capability.", + "allOf": [ + { + "$ref": "#/$defs/DeleteSessionRequest" + } + ] + }, + { + "title": "ResumeSessionRequest", + "description": "Resumes an existing session without returning previous messages.\n\nThis method is only available if the agent advertises the `sessionCapabilities.resume` capability.\n\nThe agent should resume the session context, allowing the conversation to continue\nwithout replaying the message history (unlike `session/load`).", + "allOf": [ + { + "$ref": "#/$defs/ResumeSessionRequest" + } + ] + }, + { + "title": "CloseSessionRequest", + "description": "Closes an active session and frees up any resources associated with it.\n\nThis method is only available if the agent advertises the `sessionCapabilities.close` capability.\n\nThe agent must cancel any ongoing work (as if `session/cancel` was called)\nand then free up any resources associated with the session.", + "allOf": [ + { + "$ref": "#/$defs/CloseSessionRequest" + } + ] + }, + { + "title": "SetSessionModeRequest", + "description": "Sets the current mode for a session.\n\nAllows switching between different agent modes (e.g., \"ask\", \"architect\", \"code\")\nthat affect system prompts, tool availability, and permission behaviors.\n\nThe mode must be one of the modes advertised in `availableModes` during session\ncreation or loading. Agents may also change modes autonomously and notify the\nclient via `current_mode_update` notifications.\n\nThis method can be called at any time during a session, whether the Agent is\nidle or actively generating a response.\n\nSee protocol docs: [Session Modes](https://agentclientprotocol.com/protocol/session-modes)", + "allOf": [ + { + "$ref": "#/$defs/SetSessionModeRequest" + } + ] + }, + { + "title": "SetSessionConfigOptionRequest", + "description": "Sets the current value for a session configuration option.", + "allOf": [ + { + "$ref": "#/$defs/SetSessionConfigOptionRequest" + } + ] + }, + { + "title": "PromptRequest", + "description": "Processes a user prompt within a session.\n\nThis method handles the whole lifecycle of a prompt:\n- Receives user messages with optional context (files, images, etc.)\n- Processes the prompt using language models\n- Reports language model content and tool calls to the Clients\n- Requests permission to run tools\n- Executes any requested tool calls\n- Returns when the turn is complete with a stop reason\n\nSee protocol docs: [Prompt Turn](https://agentclientprotocol.com/protocol/prompt-turn)", + "allOf": [ + { + "$ref": "#/$defs/PromptRequest" + } + ] + }, + { + "title": "ExtMethodRequest", + "description": "Handles extension method requests from the client.\n\nExtension methods provide a way to add custom functionality while maintaining\nprotocol compatibility.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "allOf": [ + { + "$ref": "#/$defs/ExtRequest" + } + ] + } + ] + }, + { + "type": "null" + } + ] + } + }, + "required": ["id", "method"], + "x-docs-ignore": true + }, + "InitializeRequest": { + "description": "Request parameters for the initialize method.\n\nSent by the client to establish connection and negotiate capabilities.\n\nSee protocol docs: [Initialization](https://agentclientprotocol.com/protocol/initialization)", + "type": "object", + "properties": { + "protocolVersion": { + "description": "The latest protocol version supported by the client.", + "allOf": [ + { + "$ref": "#/$defs/ProtocolVersion" + } + ] + }, + "clientCapabilities": { + "description": "Capabilities supported by the client.", + "x-deserialize-default-on-error": true, + "default": { + "fs": { + "readTextFile": false, + "writeTextFile": false + }, + "terminal": false + }, + "allOf": [ + { + "$ref": "#/$defs/ClientCapabilities" + } + ] + }, + "clientInfo": { + "description": "Information about the Client name and version sent to the Agent.\n\nNote: in future versions of the protocol, this will be required.", + "anyOf": [ + { + "$ref": "#/$defs/Implementation" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["protocolVersion"], + "x-side": "agent", + "x-method": "initialize" + }, + "ClientCapabilities": { + "description": "Capabilities supported by the client.\n\nAdvertised during initialization to inform the agent about\navailable features and methods.\n\nSee protocol docs: [Client Capabilities](https://agentclientprotocol.com/protocol/initialization#client-capabilities)", + "type": "object", + "properties": { + "fs": { + "description": "File system capabilities supported by the client.\nDetermines which file operations the agent can request.", + "x-deserialize-default-on-error": true, + "default": { + "readTextFile": false, + "writeTextFile": false + }, + "allOf": [ + { + "$ref": "#/$defs/FileSystemCapabilities" + } + ] + }, + "terminal": { + "description": "Whether the Client support all `terminal/*` methods.", + "type": "boolean", + "default": false, + "x-deserialize-default-on-error": true + }, + "session": { + "description": "Session-related capabilities supported by the client.\n\nOptional. Omitted or `null` both mean the client does not advertise any\nsession-related extensions.", + "anyOf": [ + { + "$ref": "#/$defs/ClientSessionCapabilities" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "FileSystemCapabilities": { + "description": "File system capabilities that a client may support.\n\nSee protocol docs: [FileSystem](https://agentclientprotocol.com/protocol/initialization#filesystem)", + "type": "object", + "properties": { + "readTextFile": { + "description": "Whether the Client supports `fs/read_text_file` requests.", + "type": "boolean", + "default": false, + "x-deserialize-default-on-error": true + }, + "writeTextFile": { + "description": "Whether the Client supports `fs/write_text_file` requests.", + "type": "boolean", + "default": false, + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "ClientSessionCapabilities": { + "description": "Session-related capabilities supported by the client.", + "type": "object", + "properties": { + "configOptions": { + "description": "Config option capabilities supported by the client.\n\nOmitted or `null` both mean the client does not advertise support for any\nconfig option extensions.", + "anyOf": [ + { + "$ref": "#/$defs/SessionConfigOptionsCapabilities" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "SessionConfigOptionsCapabilities": { + "description": "Session configuration option capabilities supported by the client.", + "type": "object", + "properties": { + "boolean": { + "description": "Whether the client supports boolean session configuration options.\n\nOptional. Omitted or `null` both mean the client does not advertise support.\nSupplying `{}` means agents may include `type: \"boolean\"` entries in\n`configOptions`, and the client may send `session/set_config_option`\nrequests with `type: \"boolean\"` and a boolean `value`.", + "anyOf": [ + { + "$ref": "#/$defs/BooleanConfigOptionCapabilities" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "BooleanConfigOptionCapabilities": { + "description": "Capabilities for boolean session configuration options.\n\nSupplying `{}` means the client supports boolean session configuration options.", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "AuthenticateRequest": { + "description": "Request parameters for the authenticate method.\n\nSpecifies which authentication method to use.", + "type": "object", + "properties": { + "methodId": { + "description": "The ID of the authentication method to use.\nMust be one of the methods advertised in the initialize response.", + "allOf": [ + { + "$ref": "#/$defs/AuthMethodId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["methodId"], + "x-side": "agent", + "x-method": "authenticate" + }, + "LogoutRequest": { + "description": "Request parameters for the logout method.\n\nTerminates the current authenticated session.", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "x-side": "agent", + "x-method": "logout" + }, + "NewSessionRequest": { + "description": "Request parameters for creating a new session.\n\nSee protocol docs: [Creating a Session](https://agentclientprotocol.com/protocol/session-setup#creating-a-session)", + "type": "object", + "properties": { + "cwd": { + "description": "The working directory for this session. Must be an absolute path.", + "type": "string" + }, + "additionalDirectories": { + "description": "Additional workspace roots for this session. Each path must be absolute.\n\nThese expand the session's filesystem scope without changing `cwd`, which\nremains the base for relative paths. When omitted or empty, no\nadditional roots are activated for the new session.", + "type": "array", + "items": { + "type": "string" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "mcpServers": { + "description": "List of MCP (Model Context Protocol) servers the agent should connect to.", + "type": "array", + "items": { + "$ref": "#/$defs/McpServer" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["cwd", "mcpServers"], + "x-side": "agent", + "x-method": "session/new" + }, + "McpServer": { + "description": "Configuration for connecting to an MCP (Model Context Protocol) server.\n\nMCP servers provide tools and context that the agent can use when\nprocessing prompts.\n\nSee protocol docs: [MCP Servers](https://agentclientprotocol.com/protocol/session-setup#mcp-servers)", + "anyOf": [ + { + "description": "HTTP transport configuration\n\nOnly available when the Agent capabilities indicate `mcp_capabilities.http` is `true`.", + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "http" + } + }, + "required": ["type"], + "allOf": [ + { + "$ref": "#/$defs/McpServerHttp" + } + ] + }, + { + "description": "SSE transport configuration\n\nOnly available when the Agent capabilities indicate `mcp_capabilities.sse` is `true`.", + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "sse" + } + }, + "required": ["type"], + "allOf": [ + { + "$ref": "#/$defs/McpServerSse" + } + ] + }, + { + "title": "stdio", + "description": "Stdio transport configuration\n\nAll Agents MUST support this transport.", + "allOf": [ + { + "$ref": "#/$defs/McpServerStdio" + } + ] + } + ] + }, + "HttpHeader": { + "description": "An HTTP header to set when making requests to the MCP server.", + "type": "object", + "properties": { + "name": { + "description": "The name of the HTTP header.", + "type": "string" + }, + "value": { + "description": "The value to set for the HTTP header.", + "type": "string" + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["name", "value"] + }, + "McpServerHttp": { + "description": "HTTP transport configuration for MCP.", + "type": "object", + "properties": { + "name": { + "description": "Human-readable name identifying this MCP server.", + "type": "string" + }, + "url": { + "description": "URL to the MCP server.", + "type": "string" + }, + "headers": { + "description": "HTTP headers to set when making requests to the MCP server.", + "type": "array", + "items": { + "$ref": "#/$defs/HttpHeader" + } + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["name", "url", "headers"] + }, + "McpServerSse": { + "description": "SSE transport configuration for MCP.", + "type": "object", + "properties": { + "name": { + "description": "Human-readable name identifying this MCP server.", + "type": "string" + }, + "url": { + "description": "URL to the MCP server.", + "type": "string" + }, + "headers": { + "description": "HTTP headers to set when making requests to the MCP server.", + "type": "array", + "items": { + "$ref": "#/$defs/HttpHeader" + } + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["name", "url", "headers"] + }, + "McpServerStdio": { + "description": "Stdio transport configuration for MCP.", + "type": "object", + "properties": { + "name": { + "description": "Human-readable name identifying this MCP server.", + "type": "string" + }, + "command": { + "description": "Absolute path to the MCP server executable.", + "type": "string" + }, + "args": { + "description": "Command-line arguments to pass to the MCP server.", + "type": "array", + "items": { + "type": "string" + } + }, + "env": { + "description": "Environment variables to set when launching the MCP server.", + "type": "array", + "items": { + "$ref": "#/$defs/EnvVariable" + } + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["name", "command", "args", "env"] + }, + "LoadSessionRequest": { + "description": "Request parameters for loading an existing session.\n\nOnly available if the Agent supports the `loadSession` capability.\n\nSee protocol docs: [Loading Sessions](https://agentclientprotocol.com/protocol/session-setup#loading-sessions)", + "type": "object", + "properties": { + "mcpServers": { + "description": "List of MCP servers to connect to for this session.", + "type": "array", + "items": { + "$ref": "#/$defs/McpServer" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "cwd": { + "description": "The working directory for this session. Must be an absolute path.", + "type": "string" + }, + "additionalDirectories": { + "description": "Additional workspace roots to activate for this session. Each path must be absolute.\n\nWhen omitted or empty, no additional roots are activated. When non-empty,\nthis is the complete resulting additional-root list for the loaded\nsession. It may differ from any previously used or reported list as long as\nthe request `cwd` matches the session's `cwd`.", + "type": "array", + "items": { + "type": "string" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "sessionId": { + "description": "The ID of the session to load.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["mcpServers", "cwd", "sessionId"], + "x-side": "agent", + "x-method": "session/load" + }, + "ListSessionsRequest": { + "description": "Request parameters for listing existing sessions.\n\nOnly available if the Agent supports the `sessionCapabilities.list` capability.", + "type": "object", + "properties": { + "cwd": { + "description": "Filter sessions by working directory. Must be an absolute path.", + "type": ["string", "null"] + }, + "cursor": { + "description": "Opaque cursor token from a previous response's nextCursor field for cursor-based pagination", + "type": ["string", "null"] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "x-side": "agent", + "x-method": "session/list" + }, + "DeleteSessionRequest": { + "description": "Request parameters for deleting an existing session from `session/list`.\n\nOnly available if the Agent supports the `sessionCapabilities.delete` capability.", + "type": "object", + "properties": { + "sessionId": { + "description": "The ID of the session to delete.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId"], + "x-side": "agent", + "x-method": "session/delete" + }, + "ResumeSessionRequest": { + "description": "Request parameters for resuming an existing session.\n\nResumes an existing session without returning previous messages (unlike `session/load`).\nThis is useful for agents that can resume sessions but don't implement full session loading.\n\nOnly available if the Agent supports the `sessionCapabilities.resume` capability.", + "type": "object", + "properties": { + "sessionId": { + "description": "The ID of the session to resume.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "cwd": { + "description": "The working directory for this session. Must be an absolute path.", + "type": "string" + }, + "additionalDirectories": { + "description": "Additional workspace roots to activate for this session. Each path must be absolute.\n\nWhen omitted or empty, no additional roots are activated. When non-empty,\nthis is the complete resulting additional-root list for the resumed\nsession. It may differ from any previously used or reported list as long as\nthe request `cwd` matches the session's `cwd`.", + "type": "array", + "items": { + "type": "string" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "mcpServers": { + "description": "List of MCP servers to connect to for this session.", + "type": "array", + "items": { + "$ref": "#/$defs/McpServer" + }, + "x-deserialize-default-on-error": true, + "x-deserialize-skip-invalid-items": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "cwd"], + "x-side": "agent", + "x-method": "session/resume" + }, + "CloseSessionRequest": { + "description": "Request parameters for closing an active session.\n\nIf supported, the agent **must** cancel any ongoing work related to the session\n(treat it as if `session/cancel` was called) and then free up any resources\nassociated with the session.\n\nOnly available if the Agent supports the `sessionCapabilities.close` capability.", + "type": "object", + "properties": { + "sessionId": { + "description": "The ID of the session to close.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId"], + "x-side": "agent", + "x-method": "session/close" + }, + "SetSessionModeRequest": { + "description": "Request parameters for setting a session mode.", + "type": "object", + "properties": { + "sessionId": { + "description": "The ID of the session to set the mode for.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "modeId": { + "description": "The ID of the mode to set.", + "allOf": [ + { + "$ref": "#/$defs/SessionModeId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "modeId"], + "x-side": "agent", + "x-method": "session/set_mode" + }, + "SetSessionConfigOptionRequest": { + "description": "Request parameters for setting a session configuration option.", + "type": "object", + "properties": { + "sessionId": { + "description": "The ID of the session to set the configuration option for.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "configId": { + "description": "The ID of the configuration option to set.", + "allOf": [ + { + "$ref": "#/$defs/SessionConfigId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "configId"], + "anyOf": [ + { + "description": "A boolean value (`type: \"boolean\"`).", + "type": "object", + "properties": { + "value": { + "description": "The boolean value.", + "type": "boolean" + }, + "type": { + "type": "string", + "const": "boolean" + } + }, + "required": ["type", "value"] + }, + { + "title": "value_id", + "description": "A [`SessionConfigValueId`] string value.\n\nThis is the default when `type` is absent on the wire. Unknown `type`\nvalues with string payloads also gracefully deserialize into this\nvariant.", + "type": "object", + "properties": { + "value": { + "description": "The value ID.", + "allOf": [ + { + "$ref": "#/$defs/SessionConfigValueId" + } + ] + } + }, + "required": ["value"] + } + ], + "x-side": "agent", + "x-method": "session/set_config_option" + }, + "PromptRequest": { + "description": "Request parameters for sending a user prompt to the agent.\n\nContains the user's message and any additional context.\n\nSee protocol docs: [User Message](https://agentclientprotocol.com/protocol/prompt-turn#1-user-message)", + "type": "object", + "properties": { + "sessionId": { + "description": "The ID of the session to send this user message to", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "prompt": { + "description": "The blocks of content that compose the user's message.\n\nAs a baseline, the Agent MUST support [`ContentBlock::Text`] and [`ContentBlock::ResourceLink`],\nwhile other variants are optionally enabled via [`PromptCapabilities`].\n\nThe Client MUST adapt its interface according to [`PromptCapabilities`].\n\nThe client MAY include referenced pieces of context as either\n[`ContentBlock::Resource`] or [`ContentBlock::ResourceLink`].\n\nWhen available, [`ContentBlock::Resource`] is preferred\nas it avoids extra round-trips and allows the message to include\npieces of context from sources the agent may not have access to.", + "type": "array", + "items": { + "$ref": "#/$defs/ContentBlock" + } + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId", "prompt"], + "x-side": "agent", + "x-method": "session/prompt" + }, + "ClientResponse": { + "description": "A JSON-RPC response object.", + "anyOf": [ + { + "title": "Result", + "description": "A successful JSON-RPC response.", + "type": "object", + "properties": { + "id": { + "description": "The id of the request this response answers.", + "allOf": [ + { + "$ref": "#/$defs/RequestId" + } + ] + }, + "result": { + "description": "Method-specific response data.", + "anyOf": [ + { + "title": "WriteTextFileResponse", + "description": "Successful result returned for a `fs/write_text_file` request.", + "allOf": [ + { + "$ref": "#/$defs/WriteTextFileResponse" + } + ] + }, + { + "title": "ReadTextFileResponse", + "description": "Successful result returned for a `fs/read_text_file` request.", + "allOf": [ + { + "$ref": "#/$defs/ReadTextFileResponse" + } + ] + }, + { + "title": "RequestPermissionResponse", + "description": "Successful result returned for a `session/request_permission` request.", + "allOf": [ + { + "$ref": "#/$defs/RequestPermissionResponse" + } + ] + }, + { + "title": "CreateTerminalResponse", + "description": "Successful result returned for a `terminal/create` request.", + "allOf": [ + { + "$ref": "#/$defs/CreateTerminalResponse" + } + ] + }, + { + "title": "TerminalOutputResponse", + "description": "Successful result returned for a `terminal/output` request.", + "allOf": [ + { + "$ref": "#/$defs/TerminalOutputResponse" + } + ] + }, + { + "title": "ReleaseTerminalResponse", + "description": "Successful result returned for a `terminal/release` request.", + "allOf": [ + { + "$ref": "#/$defs/ReleaseTerminalResponse" + } + ] + }, + { + "title": "WaitForTerminalExitResponse", + "description": "Successful result returned for a `terminal/wait_for_exit` request.", + "allOf": [ + { + "$ref": "#/$defs/WaitForTerminalExitResponse" + } + ] + }, + { + "title": "KillTerminalResponse", + "description": "Successful result returned for a `terminal/kill` request.", + "allOf": [ + { + "$ref": "#/$defs/KillTerminalResponse" + } + ] + }, + { + "title": "ExtMethodResponse", + "description": "Successful result returned by an extension method outside the core ACP method set.", + "allOf": [ + { + "$ref": "#/$defs/ExtResponse" + } + ] + } + ] + } + }, + "required": ["id", "result"] + }, + { + "title": "Error", + "description": "A failed JSON-RPC response.", + "type": "object", + "properties": { + "id": { + "description": "The id of the request this response answers.", + "allOf": [ + { + "$ref": "#/$defs/RequestId" + } + ] + }, + "error": { + "description": "Method-specific error data.", + "allOf": [ + { + "$ref": "#/$defs/Error" + } + ] + } + }, + "required": ["id", "error"] + } + ], + "x-docs-ignore": true + }, + "WriteTextFileResponse": { + "description": "Response to `fs/write_text_file`", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "x-side": "client", + "x-method": "fs/write_text_file" + }, + "ReadTextFileResponse": { + "description": "Response containing the contents of a text file.", + "type": "object", + "properties": { + "content": { + "description": "Content payload returned by this response.", + "type": "string" + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["content"], + "x-side": "client", + "x-method": "fs/read_text_file" + }, + "RequestPermissionResponse": { + "description": "Response to a permission request.", + "type": "object", + "properties": { + "outcome": { + "description": "The user's decision on the permission request.", + "allOf": [ + { + "$ref": "#/$defs/RequestPermissionOutcome" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["outcome"], + "x-side": "client", + "x-method": "session/request_permission" + }, + "RequestPermissionOutcome": { + "description": "The outcome of a permission request.", + "oneOf": [ + { + "description": "The prompt turn was cancelled before the user responded.\n\nWhen a client sends a `session/cancel` notification to cancel an ongoing\nprompt turn, it MUST respond to all pending `session/request_permission`\nrequests with this `Cancelled` outcome.\n\nSee protocol docs: [Cancellation](https://agentclientprotocol.com/protocol/prompt-turn#cancellation)", + "type": "object", + "properties": { + "outcome": { + "type": "string", + "const": "cancelled" + } + }, + "required": ["outcome"] + }, + { + "description": "The user selected one of the provided options.", + "type": "object", + "properties": { + "outcome": { + "type": "string", + "const": "selected" + } + }, + "required": ["outcome"], + "allOf": [ + { + "$ref": "#/$defs/SelectedPermissionOutcome" + } + ] + } + ], + "discriminator": { + "propertyName": "outcome" + } + }, + "SelectedPermissionOutcome": { + "description": "The user selected one of the provided options.", + "type": "object", + "properties": { + "optionId": { + "description": "The ID of the option the user selected.", + "allOf": [ + { + "$ref": "#/$defs/PermissionOptionId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["optionId"] + }, + "CreateTerminalResponse": { + "description": "Response containing the ID of the created terminal.", + "type": "object", + "properties": { + "terminalId": { + "description": "The unique identifier for the created terminal.", + "allOf": [ + { + "$ref": "#/$defs/TerminalId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["terminalId"], + "x-side": "client", + "x-method": "terminal/create" + }, + "TerminalOutputResponse": { + "description": "Response containing the terminal output and exit status.", + "type": "object", + "properties": { + "output": { + "description": "The terminal output captured so far.", + "type": "string" + }, + "truncated": { + "description": "Whether the output was truncated due to byte limits.", + "type": "boolean" + }, + "exitStatus": { + "description": "Exit status if the command has completed.", + "anyOf": [ + { + "$ref": "#/$defs/TerminalExitStatus" + }, + { + "type": "null" + } + ], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["output", "truncated"], + "x-side": "client", + "x-method": "terminal/output" + }, + "TerminalExitStatus": { + "description": "Exit status of a terminal command.", + "type": "object", + "properties": { + "exitCode": { + "description": "The process exit code (may be null if terminated by signal).", + "type": ["integer", "null"], + "format": "uint32", + "minimum": 0, + "x-deserialize-default-on-error": true + }, + "signal": { + "description": "The signal that terminated the process (may be null if exited normally).", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + } + }, + "ReleaseTerminalResponse": { + "description": "Response to terminal/release method", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "x-side": "client", + "x-method": "terminal/release" + }, + "WaitForTerminalExitResponse": { + "description": "Response containing the exit status of a terminal command.", + "type": "object", + "properties": { + "exitCode": { + "description": "The process exit code (may be null if terminated by signal).", + "type": ["integer", "null"], + "format": "uint32", + "minimum": 0, + "x-deserialize-default-on-error": true + }, + "signal": { + "description": "The signal that terminated the process (may be null if exited normally).", + "type": ["string", "null"], + "x-deserialize-default-on-error": true + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "x-side": "client", + "x-method": "terminal/wait_for_exit" + }, + "KillTerminalResponse": { + "description": "Response to `terminal/kill` method", + "type": "object", + "properties": { + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "x-side": "client", + "x-method": "terminal/kill" + }, + "ClientNotification": { + "description": "A JSON-RPC notification object.", + "type": "object", + "properties": { + "method": { + "description": "The notification method name.", + "type": "string" + }, + "params": { + "description": "Method-specific notification parameters.", + "anyOf": [ + { + "description": "All possible notifications that a client can send to an agent.\n\nThis enum is used internally for routing RPC notifications. You typically won't need\nto use this directly.\n\nNotifications do not expect a response.", + "anyOf": [ + { + "title": "CancelNotification", + "description": "Cancels ongoing operations for a session.\n\nThis is a notification sent by the client to cancel an ongoing prompt turn.\n\nUpon receiving this notification, the Agent SHOULD:\n- Stop all language model requests as soon as possible\n- Abort all tool call invocations in progress\n- Send any pending `session/update` notifications\n- Respond to the original `session/prompt` request with `StopReason::Cancelled`\n\nSee protocol docs: [Cancellation](https://agentclientprotocol.com/protocol/prompt-turn#cancellation)", + "allOf": [ + { + "$ref": "#/$defs/CancelNotification" + } + ] + }, + { + "title": "ExtNotification", + "description": "Handles extension notifications from the client.\n\nExtension notifications provide a way to send one-way messages for custom functionality\nwhile maintaining protocol compatibility.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "allOf": [ + { + "$ref": "#/$defs/ExtNotification" + } + ] + } + ] + }, + { + "type": "null" + } + ] + } + }, + "required": ["method"], + "x-docs-ignore": true + }, + "CancelNotification": { + "description": "Notification to cancel ongoing operations for a session.\n\nSee protocol docs: [Cancellation](https://agentclientprotocol.com/protocol/prompt-turn#cancellation)", + "type": "object", + "properties": { + "sessionId": { + "description": "The ID of the session to cancel operations for.", + "allOf": [ + { + "$ref": "#/$defs/SessionId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["sessionId"], + "x-side": "agent", + "x-method": "session/cancel" + }, + "CancelRequestNotification": { + "description": "Notification to cancel an ongoing request.\n\nSee protocol docs: [Cancellation](https://agentclientprotocol.com/protocol/cancellation)", + "type": "object", + "properties": { + "requestId": { + "description": "The ID of the request to cancel.", + "allOf": [ + { + "$ref": "#/$defs/RequestId" + } + ] + }, + "_meta": { + "description": "The _meta property is reserved by ACP to allow clients and agents to attach additional\nmetadata to their interactions. Implementations MUST NOT make assumptions about values at\nthese keys.\n\nSee protocol docs: [Extensibility](https://agentclientprotocol.com/protocol/extensibility)", + "type": ["object", "null"], + "x-deserialize-default-on-error": true, + "additionalProperties": true + } + }, + "required": ["requestId"], + "x-side": "protocol", + "x-method": "$/cancel_request" + } + } +} diff --git a/trail/tests/fixtures/acp/v1/session_updates.jsonl b/trail/tests/fixtures/acp/v1/session_updates.jsonl new file mode 100644 index 0000000..d8bec93 --- /dev/null +++ b/trail/tests/fixtures/acp/v1/session_updates.jsonl @@ -0,0 +1,20 @@ +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"user_message_chunk","messageId":"user-1","content":{"type":"text","text":"streamed user message"},"_meta":{"fixture":"user"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"agent_message_chunk","messageId":"agent-1","content":{"type":"text","text":"streamed agent message"},"_meta":{"fixture":"agent"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"agent_thought_chunk","messageId":"thought-1","content":{"type":"text","text":"private chain of thought must not be stored"},"_meta":{"fixture":"thought"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"tool_call","toolCallId":"tool-read","title":"Read","kind":"read","status":"pending","content":[{"type":"content","content":{"type":"text","text":"tool output"}},{"type":"diff","path":"/tmp/acp-fixture.txt","oldText":"old","newText":"new"},{"type":"terminal","terminalId":"terminal-1"}],"_meta":{"fixture":"tool-content-shapes"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"tool_call","toolCallId":"tool-edit","title":"Edit","kind":"edit","status":"in_progress"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"tool_call","toolCallId":"tool-delete","title":"Delete","kind":"delete","status":"completed"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"tool_call","toolCallId":"tool-move","title":"Move","kind":"move","status":"failed"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"tool_call","toolCallId":"tool-search","title":"Search","kind":"search"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"tool_call","toolCallId":"tool-execute","title":"Execute","kind":"execute"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"tool_call","toolCallId":"tool-think","title":"Think","kind":"think"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"tool_call","toolCallId":"tool-fetch","title":"Fetch","kind":"fetch"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"tool_call","toolCallId":"tool-switch","title":"Switch mode","kind":"switch_mode"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"tool_call","toolCallId":"tool-other","title":"Other","kind":"other"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"tool_call_update","toolCallId":"tool-read","kind":"read","status":"completed","title":"Read complete","rawInput":{"path":"README.md"},"rawOutput":{"token":"must-redact"},"_meta":{"fixture":"tool-update"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"plan","entries":[{"content":"High pending","priority":"high","status":"pending"},{"content":"Medium active","priority":"medium","status":"in_progress"},{"content":"Low complete","priority":"low","status":"completed"}],"_meta":{"fixture":"plan-values"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"available_commands_update","availableCommands":[{"name":"review","description":"Review the workspace","input":{"hint":"optional scope"},"_meta":{"fixture":"command"}}],"_meta":{"fixture":"commands"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"current_mode_update","currentModeId":"code","_meta":{"fixture":"mode"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"config_option_update","configOptions":[{"id":"model","name":"Model","type":"select","currentValue":"fast","options":[{"value":"fast","name":"Fast"},{"value":"deep","name":"Deep"}],"category":"model"},{"id":"autoFix","name":"Auto fix","type":"boolean","currentValue":true,"category":"model_config"}],"_meta":{"fixture":"config"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"session_info_update","title":"Schema fixture session","updatedAt":"2026-07-12T12:00:00Z","_meta":{"fixture":"session-info"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"update-session","update":{"sessionUpdate":"usage_update","used":2048,"size":128000,"cost":{"amount":0.125,"currency":"USD"},"_meta":{"fixture":"usage"}}}} diff --git a/trail/tests/fixtures/acp/v1/source.json b/trail/tests/fixtures/acp/v1/source.json new file mode 100644 index 0000000..6488069 --- /dev/null +++ b/trail/tests/fixtures/acp/v1/source.json @@ -0,0 +1,7 @@ +{ + "repository": "https://github.com/agentclientprotocol/agent-client-protocol", + "commit": "64cbd71ae520b89aac54164d8c1d364333c8ee5f", + "wireVersion": 1, + "schemaSha256": "92c1dfcda10dd47e99127500a3763da2b471f9ac61e12b9bf0430c32cf953796", + "metaSha256": "e0bf36f8123b2544b499174197fdc371ec49a1b4572a35114513d56492741599" +} diff --git a/trail/tests/fixtures/acp/v1/variant_cases.json b/trail/tests/fixtures/acp/v1/variant_cases.json new file mode 100644 index 0000000..dee751d --- /dev/null +++ b/trail/tests/fixtures/acp/v1/variant_cases.json @@ -0,0 +1,77 @@ +[ + {"union":"ContentBlock","name":"text","value":{"type":"text","text":"hello"}}, + {"union":"ContentBlock","name":"image","value":{"type":"image","data":"aW1hZ2U=","mimeType":"image/png"}}, + {"union":"ContentBlock","name":"audio","value":{"type":"audio","data":"YXVkaW8=","mimeType":"audio/wav"}}, + {"union":"ContentBlock","name":"resource_link","value":{"type":"resource_link","name":"README","uri":"file:///README.md"}}, + {"union":"ContentBlock","name":"resource","value":{"type":"resource","resource":{"uri":"file:///context.txt","text":"context"}}}, + + {"union":"EmbeddedResourceResource","name":"TextResourceContents","value":{"uri":"file:///text.txt","text":"text"}}, + {"union":"EmbeddedResourceResource","name":"BlobResourceContents","value":{"uri":"file:///blob.bin","blob":"YmxvYg=="}}, + + {"union":"McpServer","name":"http","value":{"type":"http","name":"http","url":"https://example.test/mcp","headers":[]}}, + {"union":"McpServer","name":"sse","value":{"type":"sse","name":"sse","url":"https://example.test/sse","headers":[]}}, + {"union":"McpServer","name":"stdio","value":{"name":"stdio","command":"/usr/bin/env","args":["true"],"env":[]}}, + + {"union":"PermissionOptionKind","name":"allow_once","value":"allow_once"}, + {"union":"PermissionOptionKind","name":"allow_always","value":"allow_always"}, + {"union":"PermissionOptionKind","name":"reject_once","value":"reject_once"}, + {"union":"PermissionOptionKind","name":"reject_always","value":"reject_always"}, + + {"union":"RequestPermissionOutcome","name":"cancelled","value":{"outcome":"cancelled"}}, + {"union":"RequestPermissionOutcome","name":"selected","value":{"outcome":"selected","optionId":"allow"}}, + + {"union":"SessionConfigOption","name":"select","value":{"id":"model","name":"Model","type":"select","currentValue":"fast","options":[{"value":"fast","name":"Fast"}]}}, + {"union":"SessionConfigOption","name":"boolean","value":{"id":"autoFix","name":"Auto fix","type":"boolean","currentValue":true}}, + + {"union":"SessionUpdate","name":"user_message_chunk","value":{"sessionUpdate":"user_message_chunk","content":{"type":"text","text":"user"}}}, + {"union":"SessionUpdate","name":"agent_message_chunk","value":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"agent"}}}, + {"union":"SessionUpdate","name":"agent_thought_chunk","value":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"thought"}}}, + {"union":"SessionUpdate","name":"tool_call","value":{"sessionUpdate":"tool_call","toolCallId":"tool","title":"Tool"}}, + {"union":"SessionUpdate","name":"tool_call_update","value":{"sessionUpdate":"tool_call_update","toolCallId":"tool","status":"completed"}}, + {"union":"SessionUpdate","name":"plan","value":{"sessionUpdate":"plan","entries":[]}}, + {"union":"SessionUpdate","name":"available_commands_update","value":{"sessionUpdate":"available_commands_update","availableCommands":[]}}, + {"union":"SessionUpdate","name":"current_mode_update","value":{"sessionUpdate":"current_mode_update","currentModeId":"code"}}, + {"union":"SessionUpdate","name":"config_option_update","value":{"sessionUpdate":"config_option_update","configOptions":[]}}, + {"union":"SessionUpdate","name":"session_info_update","value":{"sessionUpdate":"session_info_update","title":"Title"}}, + {"union":"SessionUpdate","name":"usage_update","value":{"sessionUpdate":"usage_update","used":1,"size":100}}, + + {"union":"TerminalExitStatus","name":"exit_code","value":{"exitCode":0,"signal":null}}, + {"union":"TerminalExitStatus","name":"signal","value":{"exitCode":null,"signal":"SIGTERM"}}, + + {"union":"ToolCallContent","name":"content","value":{"type":"content","content":{"type":"text","text":"output"}}}, + {"union":"ToolCallContent","name":"diff","value":{"type":"diff","path":"/tmp/file","newText":"new"}}, + {"union":"ToolCallContent","name":"terminal","value":{"type":"terminal","terminalId":"terminal-1"}}, + + {"union":"ToolCallStatus","name":"pending","value":"pending"}, + {"union":"ToolCallStatus","name":"in_progress","value":"in_progress"}, + {"union":"ToolCallStatus","name":"completed","value":"completed"}, + {"union":"ToolCallStatus","name":"failed","value":"failed"}, + + {"union":"ToolKind","name":"read","value":"read"}, + {"union":"ToolKind","name":"edit","value":"edit"}, + {"union":"ToolKind","name":"delete","value":"delete"}, + {"union":"ToolKind","name":"move","value":"move"}, + {"union":"ToolKind","name":"search","value":"search"}, + {"union":"ToolKind","name":"execute","value":"execute"}, + {"union":"ToolKind","name":"think","value":"think"}, + {"union":"ToolKind","name":"fetch","value":"fetch"}, + {"union":"ToolKind","name":"switch_mode","value":"switch_mode"}, + {"union":"ToolKind","name":"other","value":"other"}, + + {"union":"PlanEntryPriority","name":"high","value":"high"}, + {"union":"PlanEntryPriority","name":"medium","value":"medium"}, + {"union":"PlanEntryPriority","name":"low","value":"low"}, + + {"union":"PlanEntryStatus","name":"pending","value":"pending"}, + {"union":"PlanEntryStatus","name":"in_progress","value":"in_progress"}, + {"union":"PlanEntryStatus","name":"completed","value":"completed"}, + + {"union":"Role","name":"assistant","value":"assistant"}, + {"union":"Role","name":"user","value":"user"}, + + {"union":"StopReason","name":"end_turn","value":"end_turn"}, + {"union":"StopReason","name":"max_tokens","value":"max_tokens"}, + {"union":"StopReason","name":"max_turn_requests","value":"max_turn_requests"}, + {"union":"StopReason","name":"refusal","value":"refusal"}, + {"union":"StopReason","name":"cancelled","value":"cancelled"} +] diff --git a/trail/tests/fixtures/agent-hooks/claude-code/contracts.json b/trail/tests/fixtures/agent-hooks/claude-code/contracts.json new file mode 100644 index 0000000..be2864c --- /dev/null +++ b/trail/tests/fixtures/agent-hooks/claude-code/contracts.json @@ -0,0 +1,5 @@ +[ + {"event":"UserPromptSubmit","payload":{"session_id":"claude-s1","turn_id":"claude-t1","prompt":"fix it"},"expected":["turn.started","message.user"]}, + {"event":"PostToolUseFailure","payload":{"session_id":"claude-s1","tool_use_id":"tool-1","error":"failed"},"expected":["tool.failed"]}, + {"event":"StopFailure","payload":{"session_id":"claude-s1","turn_id":"claude-t1","error":"model failed"},"expected":["turn.failed"]} +] diff --git a/trail/tests/fixtures/agent-hooks/codex/contracts.json b/trail/tests/fixtures/agent-hooks/codex/contracts.json new file mode 100644 index 0000000..3e3dd7e --- /dev/null +++ b/trail/tests/fixtures/agent-hooks/codex/contracts.json @@ -0,0 +1,5 @@ +[ + {"event":"UserPromptSubmit","payload":{"session_id":"codex-s1","turn_id":"codex-t1","prompt":"fix it"},"expected":["turn.started","message.user"]}, + {"event":"PostToolUse","payload":{"session_id":"codex-s1","turn_id":"codex-t1","tool_use_id":"tool-1","error":"exit 1"},"expected":["tool.failed"]}, + {"event":"Stop","payload":{"session_id":"codex-s1","turn_id":"codex-t1","status":"cancelled"},"expected":["turn.cancelled"]} +] diff --git a/trail/tests/fixtures/agent-hooks/copilot/contracts.json b/trail/tests/fixtures/agent-hooks/copilot/contracts.json new file mode 100644 index 0000000..52961e4 --- /dev/null +++ b/trail/tests/fixtures/agent-hooks/copilot/contracts.json @@ -0,0 +1,5 @@ +[ + {"event":"userPromptSubmitted","payload":{"sessionId":"copilot-s1","turnId":"copilot-t1","prompt":"fix it"},"expected":["turn.started","message.user"]}, + {"event":"postToolUseFailure","payload":{"sessionId":"copilot-s1","toolCallId":"tool-1","error":"failed"},"expected":["tool.failed"]}, + {"event":"agentStop","payload":{"sessionId":"copilot-s1","turnId":"copilot-t1","status":"cancelled"},"expected":["turn.cancelled"]} +] diff --git a/trail/tests/fixtures/agent-hooks/cursor/contracts.json b/trail/tests/fixtures/agent-hooks/cursor/contracts.json new file mode 100644 index 0000000..e048219 --- /dev/null +++ b/trail/tests/fixtures/agent-hooks/cursor/contracts.json @@ -0,0 +1,5 @@ +[ + {"event":"beforeSubmitPrompt","payload":{"conversation_id":"cursor-s1","turn_id":"cursor-t1","prompt":"fix it"},"expected":["turn.started","message.user"]}, + {"event":"postToolUse","payload":{"conversation_id":"cursor-s1","tool_id":"tool-1","error":"failed"},"expected":["tool.failed"]}, + {"event":"stop","payload":{"conversation_id":"cursor-s1","turn_id":"cursor-t1","status":"cancelled"},"expected":["turn.cancelled"]} +] diff --git a/trail/tests/fixtures/agent-hooks/gemini/contracts.json b/trail/tests/fixtures/agent-hooks/gemini/contracts.json new file mode 100644 index 0000000..0eed30e --- /dev/null +++ b/trail/tests/fixtures/agent-hooks/gemini/contracts.json @@ -0,0 +1,5 @@ +[ + {"event":"BeforeAgent","payload":{"sessionId":"gemini-s1","turnId":"gemini-t1","prompt":"fix it"},"expected":["turn.started","message.user"]}, + {"event":"AfterTool","payload":{"sessionId":"gemini-s1","toolCallId":"tool-1","error":"failed"},"expected":["tool.failed"]}, + {"event":"AfterAgent","payload":{"sessionId":"gemini-s1","turnId":"gemini-t1","status":"cancelled"},"expected":["turn.cancelled"]} +] diff --git a/trail/tests/fixtures/agent-hooks/grok/contracts.json b/trail/tests/fixtures/agent-hooks/grok/contracts.json new file mode 100644 index 0000000..ff6b8b5 --- /dev/null +++ b/trail/tests/fixtures/agent-hooks/grok/contracts.json @@ -0,0 +1,5 @@ +[ + {"event":"UserPromptSubmit","payload":{"session_id":"grok-s1","turn_id":"grok-t1","prompt":"fix it"},"expected":["turn.started","message.user"]}, + {"event":"PostToolUseFailure","payload":{"session_id":"grok-s1","tool_use_id":"tool-1","error":"failed"},"expected":["tool.failed"]}, + {"event":"StopFailure","payload":{"session_id":"grok-s1","turn_id":"grok-t1","error":"model failed"},"expected":["turn.failed"]} +] diff --git a/trail/tests/fixtures/agent-hooks/kiro/contracts.json b/trail/tests/fixtures/agent-hooks/kiro/contracts.json new file mode 100644 index 0000000..2cd0616 --- /dev/null +++ b/trail/tests/fixtures/agent-hooks/kiro/contracts.json @@ -0,0 +1,5 @@ +[ + {"event":"UserPromptSubmit","payload":{"hook_event_name":"UserPromptSubmit","session_id":"kiro-s1","prompt":"fix it"},"expected":["turn.started","message.user"]}, + {"event":"PostToolUse","payload":{"hook_event_name":"PostToolUse","session_id":"kiro-s1","tool_name":"shell","tool_response":{"success":false,"error":"failed"}},"expected":["tool.failed"]}, + {"event":"Stop","payload":{"hook_event_name":"Stop","session_id":"kiro-s1","assistant_response":"Implemented and tested."},"expected":["message.assistant.completed","turn.completed"]} +] diff --git a/trail/tests/fixtures/agent-hooks/opencode/contracts.json b/trail/tests/fixtures/agent-hooks/opencode/contracts.json new file mode 100644 index 0000000..89de43a --- /dev/null +++ b/trail/tests/fixtures/agent-hooks/opencode/contracts.json @@ -0,0 +1,5 @@ +[ + {"event":"chat.message","payload":{"properties":{"sessionID":"opencode-s1","messageID":"opencode-t1","message":"fix it"}},"expected":["turn.started","message.user"]}, + {"event":"tool.execute.after","payload":{"sessionID":"opencode-s1","callID":"tool-1","error":"failed"},"expected":["tool.failed"]}, + {"event":"session.idle","payload":{"sessionID":"opencode-s1"},"expected":["turn.completed"]} +] diff --git a/trail/tests/fixtures/agent-hooks/pi/contracts.json b/trail/tests/fixtures/agent-hooks/pi/contracts.json new file mode 100644 index 0000000..2da182d --- /dev/null +++ b/trail/tests/fixtures/agent-hooks/pi/contracts.json @@ -0,0 +1,5 @@ +[ + {"event":"before_agent_start","payload":{"session":{"sessionId":"pi-s1"},"input":{"turnId":"pi-t1","message":"fix it"}},"expected":["turn.started","message.user"]}, + {"event":"tool_execution_end","payload":{"sessionId":"pi-s1","toolCallId":"tool-1","isError":true},"expected":["tool.failed"]}, + {"event":"agent_end","payload":{"sessionId":"pi-s1","turnId":"pi-t1","status":"cancelled"},"expected":["turn.cancelled"]} +] diff --git a/trail/tests/fixtures/changed_path_producers.v1 b/trail/tests/fixtures/changed_path_producers.v1 new file mode 100644 index 0000000..bf0dff1 --- /dev/null +++ b/trail/tests/fixtures/changed_path_producers.v1 @@ -0,0 +1,28 @@ +# status|source|entry function|site id|producer/exemption|reviewed protocol|mutation sinks +# +# This is an inventory of concrete mutation entry points, not of +# IntentProducer enum variants. `changed_path_ledger_producers.rs` verifies +# that each controlled entry still contains both the listed mutation sinks and +# a call to the reviewed orchestration protocol. Merely adding a +# TRAIL_FS_PRODUCER comment therefore cannot satisfy the audit. +controlled|db/record/checkout.rs|checkout_with_options|primary_workspace_checkout|Checkout|run_projection_alignment|remove_visible_files_absent_from_target,materialize_files +exempt|db/record/checkout.rs|checkout_with_options|alternate_checkout_destination|exempt_alternate_output|none|prepare_checkout_workdir,materialize_root_files_at_streaming,materialize_files_at +controlled|db/lane/lifecycle.rs|spawn_lane_with_workdir_mode_paths_and_neighbors_inner|lane_spawn_materialize|Materialize|run_projection_alignment|materialize_lane_workdir_at_paths_with_neighbors,materialize_lane_root_staged +controlled|db/lane/lifecycle.rs|ensure_lane_workdir_materialized|lane_ensure_materialized|Materialize|run_projection_alignment|materialize_lane_root_staged +controlled|db/lane/control/turn_setup.rs|spawn_lane_branch_for_turn|turn_lane_spawn|Materialize|run_projection_alignment|materialize_lane_root_staged +controlled|db/lane/workdir/sync.rs|sync_lane_workdir_with_paths_and_neighbors|lane_sync|LaneSync|run_projection_alignment|materialize_sparse_lane_workdir_paths,materialize_full_lane_workdir_staged,materialize_files_at,fs::create_dir_all +exempt|db/lane/workdir/sync.rs|sync_lane_workdir_with_paths_and_neighbors|lane_sync_rescue|exempt_rescue_output|none|rescue_replaced_lane_workdir_path,rescue_dirty_lane_workdir +controlled|db/lane/workdir/sync.rs|hydrate_sparse_lane_workdir_paths_unlocked|sparse_hydration|LaneSync|run_projection_alignment|materialize_sparse_lane_workdir_paths +controlled|db/lane/patching.rs|apply_lane_patch_locked|structured_patch_projection|StructuredPatchProjection|run_ref_advancing_projection|apply_lane_patch_workdir_projection +controlled|db/lane/rewind.rs|rewind_lane|lane_rewind_projection|RestoreProjection|run_ref_advancing_projection|apply_rewind_workdir_projection,sync_lane_workdir +controlled|db/lane/workdir/record.rs|record_lane_workdir_locked|observed_lane_checkpoint|ObservedCheckpoint|run_ref_advancing_projection|build_root_for_selected_disk_files_incremental,complete_workspace_checkpoint +controlled|db/merge/lane.rs|update_layered_lane_from|layered_lane_update|RestoreProjection|run_ref_advancing_projection|complete_workspace_checkpoint +controlled|db/lane/workspace_view.rs|complete_workspace_checkpoint|cow_checkpoint|CowPublication|checkpoint_view|write_workspace_checkpoint_mirror +controlled|db/lane/workdir/view_core.rs|ensure_upper_file|mounted_cow_copy_up|CowPublication|begin_qualified_view_mutation|ensure_upper_file_under_barrier +controlled|db/lane/workdir/view_core.rs|write|mounted_cow_write|CowPublication|begin_qualified_view_mutation|ensure_upper_file_under_barrier,write_all_file_at +controlled|db/lane/workdir/view_core.rs|create|mounted_cow_create|CowPublication|begin_qualified_view_mutation|OpenOptions::new +controlled|db/lane/workdir/view_core.rs|mkdir|mounted_cow_mkdir|CowPublication|begin_qualified_view_mutation|fs::create_dir_all +controlled|db/lane/workdir/view_core.rs|symlink|mounted_cow_symlink|CowPublication|begin_qualified_view_mutation|symlink +controlled|db/lane/workdir/view_core.rs|setattr|mounted_cow_setattr|CowPublication|begin_qualified_view_mutation|ensure_upper_file_under_barrier,set_len +controlled|db/lane/workdir/view_core.rs|remove|mounted_cow_remove|CowPublication|begin_qualified_view_mutation|fs::remove_file +controlled|db/lane/workdir/view_core.rs|rename|mounted_cow_rename|CowPublication|begin_qualified_view_mutation|fs::rename diff --git a/trail/tests/fixtures/changed_path_raw_mutations.v1 b/trail/tests/fixtures/changed_path_raw_mutations.v1 new file mode 100644 index 0000000..fbc7dc9 --- /dev/null +++ b/trail/tests/fixtures/changed_path_raw_mutations.v1 @@ -0,0 +1,377 @@ +# class|source|function|sink|expected_count +reviewed|db/change_ledger/daemon.rs|start|fs::create_dir_all|1 +reviewed|db/change_ledger/daemon.rs|test_daemon_transition_after_load_boundary|fs::write|1 +reviewed|db/change_ledger/log/tests.rs|coordinated_segment_id_path_and_file_rename_cannot_defeat_derivation|fs::rename|1 +reviewed|db/change_ledger/log/tests.rs|directory_sync_runs_on_the_current_host|fs::write|1 +reviewed|db/change_ledger/log/tests.rs|flush_rejects_a_synchronized_file_length_different_from_claimed_offset|OpenOptions::append|1 +reviewed|db/change_ledger/log/tests.rs|authenticated_orphan_from_superseded_epoch_does_not_poison_current_epoch|fs::write|1 +reviewed|db/change_ledger/log/tests.rs|invalid_other_epoch_filename_metadata_cannot_suppress_an_orphan|fs::write|1 +reviewed|db/change_ledger/log/tests.rs|orphan_next_header_requires_reconciliation|fs::write|1 +reviewed|db/change_ledger/log/tests.rs|read_only_recovery_does_not_create_wal_or_shm_sidecars_to_inspect|fs::remove_file|2 +reviewed|db/change_ledger/log/tests.rs|recovery_rejects_symlink_segment_final_component_with_no_follow|fs::copy|1 +reviewed|db/change_ledger/log/tests.rs|recovery_rejects_symlink_segment_final_component_with_no_follow|fs::remove_file|1 +reviewed|db/change_ledger/log/tests.rs|trailing_bytes_in_a_sealed_middle_segment_fail_closed|OpenOptions::append|1 +reviewed|db/change_ledger/log/writer.rs|acquire_inner|fs::create_dir_all|1 +reviewed|db/change_ledger/log/writer.rs|acquire_inner|OpenOptions::create_new|1 +reviewed|db/change_ledger/log/writer.rs|acquire_inner|OpenOptions::write|1 +reviewed|db/change_ledger/log/writer.rs|rotate_inner|OpenOptions::create_new|1 +reviewed|db/change_ledger/log/writer.rs|rotate_inner|OpenOptions::write|1 +reviewed|db/change_ledger/observer/linux.rs|new_with_scope|fs::create_dir_all|1 +reviewed|db/change_ledger/observer/linux.rs|new_with_scope|fs::write|1 +reviewed|db/change_ledger/observer/linux.rs|run_content_mode_create_delete|fs::remove_file|1 +reviewed|db/change_ledger/observer/linux.rs|run_content_mode_create_delete|fs::set_permissions|1 +reviewed|db/change_ledger/observer/linux.rs|run_content_mode_create_delete|fs::write|2 +reviewed|db/change_ledger/observer/linux.rs|run_delayed_backlog|fs::write|1 +reviewed|db/change_ledger/observer/linux.rs|run_fault_revocation_matrix|fs::create_dir|2 +reviewed|db/change_ledger/observer/linux.rs|run_fault_revocation_matrix|fs::remove_dir|1 +reviewed|db/change_ledger/observer/linux.rs|run_fault_revocation_matrix|fs::write|3 +reviewed|db/change_ledger/observer/linux.rs|run_fence_ordering|fs::write|1 +reviewed|db/change_ledger/observer/linux.rs|run_owner_death_and_root_replacement|fs::create_dir|1 +reviewed|db/change_ledger/observer/linux.rs|run_owner_death_and_root_replacement|fs::remove_dir_all|1 +reviewed|db/change_ledger/observer/linux.rs|run_owner_death_and_root_replacement|fs::rename|2 +reviewed|db/change_ledger/observer/linux.rs|run_policy_dependency_observation|fs::create_dir_all|3 +reviewed|db/change_ledger/observer/linux.rs|run_policy_dependency_observation|fs::create_dir|1 +reviewed|db/change_ledger/observer/linux.rs|run_policy_dependency_observation|fs::remove_file|1 +reviewed|db/change_ledger/observer/linux.rs|run_policy_dependency_observation|fs::rename|1 +reviewed|db/change_ledger/observer/linux.rs|run_policy_dependency_observation|fs::write|3 +reviewed|db/change_ledger/observer/linux.rs|run_policy_dependency_observation|OpenOptions::append|1 +reviewed|db/change_ledger/observer/linux.rs|run_policy_dependency_observation|OpenOptions::create|1 +reviewed|db/change_ledger/observer/linux.rs|run_reconciliation_interval_qualification|fs::write|1 +reviewed|db/change_ledger/observer/linux.rs|run_recursive_coverage|fs::create_dir|2 +reviewed|db/change_ledger/observer/linux.rs|run_recursive_coverage|fs::write|1 +reviewed|db/change_ledger/observer/linux.rs|run_rename_matrix|fs::create_dir|1 +reviewed|db/change_ledger/observer/linux.rs|run_rename_matrix|fs::rename|3 +reviewed|db/change_ledger/observer/linux.rs|run_rename_matrix|fs::write|4 +reviewed|db/change_ledger/observer/linux.rs|run_rename_storm_and_cookie_expiry|fs::rename|2 +reviewed|db/change_ledger/observer/linux.rs|run_rename_storm_and_cookie_expiry|fs::write|2 +reviewed|db/change_ledger/observer/linux.rs|run|fs::create_dir_all|2 +reviewed|db/change_ledger/observer/linux.rs|run|fs::rename|2 +reviewed|db/change_ledger/observer/linux.rs|run|fs::write|10 +reviewed|db/change_ledger/observer/macos.rs|new|fs::create_dir|1 +reviewed|db/change_ledger/observer/macos.rs|run|fs::create_dir_all|3 +reviewed|db/change_ledger/observer/macos.rs|run|fs::create_dir|13 +reviewed|db/change_ledger/observer/macos.rs|run|fs::remove_dir|3 +reviewed|db/change_ledger/observer/macos.rs|run|fs::remove_file|2 +reviewed|db/change_ledger/observer/macos.rs|run|fs::rename|13 +reviewed|db/change_ledger/observer/macos.rs|run|fs::set_permissions|4 +reviewed|db/change_ledger/observer/macos.rs|run|fs::write|24 +reviewed|db/change_ledger/reconcile.rs|drain_through|fs::write|2 +reviewed|db/change_ledger/reconcile.rs|new|fs::write|2 +reviewed|db/change_ledger/reconcile.rs|oracle|fs::remove_file|1 +reviewed|db/change_ledger/reconcile.rs|oracle|fs::write|2 +reviewed|db/change_ledger/reconcile.rs|races|fs::write|1 +reviewed|db/change_ledger/recovery.rs|backup_overwrite_failure_preserves_previous|fs::remove_file|1 +reviewed|db/change_ledger/recovery.rs|deletion_leaf_substitution_fails_closed|fs::write|1 +reviewed|db/change_ledger/recovery.rs|deletion_name_substitution_after_verification_fails_closed|fs::write|1 +reviewed|db/change_ledger/recovery.rs|deletion_parent_substitution_uses_retained_authority|fs::create_dir|1 +reviewed|db/change_ledger/recovery.rs|deletion_parent_substitution_uses_retained_authority|fs::rename|1 +reviewed|db/change_ledger/recovery.rs|deletion_parent_substitution_uses_retained_authority|fs::write|1 +reviewed|db/change_ledger/recovery.rs|deletion_quiesced_retry_rejects_missing_quarantine|fs::rename|1 +reviewed|db/change_ledger/recovery.rs|deletion_quiesced_retry_rejects_reappeared_original|fs::write|1 +reviewed|db/change_ledger/recovery.rs|deletion_retry_rejects_hostile_quarantine_replacement|fs::rename|1 +reviewed|db/change_ledger/recovery.rs|deletion_retry_rejects_hostile_quarantine_replacement|fs::write|1 +reviewed|db/change_ledger/recovery.rs|deletion_substitution_after_quarantine_verification_fails_closed|fs::rename|1 +reviewed|db/change_ledger/recovery.rs|deletion_substitution_after_quarantine_verification_fails_closed|fs::write|1 +reviewed|db/change_ledger/recovery.rs|direct_quarantine_rejects_source_substitution_before_atomic_rename|fs::rename|1 +reviewed|db/change_ledger/recovery.rs|direct_quarantine_rejects_source_substitution_before_atomic_rename|fs::write|1 +reviewed|db/change_ledger/recovery.rs|direct_quarantine_rejects_target_substitution_after_rename_before_verify|fs::rename|1 +reviewed|db/change_ledger/recovery.rs|direct_quarantine_rejects_target_substitution_after_rename_before_verify|fs::write|1 +reviewed|db/change_ledger/recovery.rs|direct_target_collision_is_retained|fs::write|1 +reviewed|db/change_ledger/recovery.rs|gc_root_lifecycle|fs::write|1 +reviewed|db/change_ledger/recovery.rs|install_scope_directory_symlink_substitution|fs::rename|1 +reviewed|db/change_ledger/recovery.rs|lane_deletion_retires_scope_first|fs::write|1 +reviewed|db/change_ledger/recovery.rs|new|fs::create_dir_all|1 +reviewed|db/change_ledger/recovery.rs|non_utf_database_path_mark_recover_and_retire|fs::rename|1 +reviewed|db/change_ledger/recovery.rs|rejects_metadata_only_proof_without_sidecar|fs::remove_file|1 +reviewed|db/change_ledger/recovery.rs|restored_nullable_provider_lane_deletion|fs::write|1 +reviewed|db/change_ledger/recovery.rs|retirement_never_uses_separable_mkdir_open_authority|fs::create_dir|1 +reviewed|db/change_ledger/recovery.rs|retirement_never_uses_separable_mkdir_open_authority|fs::rename|1 +reviewed|db/change_ledger/recovery.rs|retirement_never_uses_separable_mkdir_open_authority|fs::set_permissions|1 +reviewed|db/change_ledger/recovery.rs|run_real_crash_scenario|fs::remove_file|1 +reviewed|db/change_ledger/recovery.rs|run_real_crash_scenario|fs::write|2 +reviewed|db/change_ledger/recovery.rs|subprocess_kill_and_reopen_covers_intent_durability_boundaries|fs::remove_file|2 +reviewed|db/change_ledger/recovery.rs|subprocess_kill_and_reopen_covers_intent_durability_boundaries|fs::write|1 +reviewed|db/change_ledger/recovery.rs|subprocess_kill_and_reopen_covers_intent_durability_boundaries|OpenOptions::write|2 +reviewed|db/change_ledger/recovery.rs|subprocess_kill_and_retry_covers_quarantine_deletion_boundaries|fs::write|2 +reviewed|db/change_ledger/recovery.rs|subprocess_kill_preserves_old_or_new_backup_and_restore_tree|fs::create_dir_all|1 +reviewed|db/change_ledger/recovery.rs|subprocess_kill_preserves_old_or_new_backup_and_restore_tree|fs::write|7 +reviewed|db/change_ledger/snapshot.rs|run_command_flow_inner|fs::create_dir|1 +reviewed|db/change_ledger/snapshot.rs|run_command_flow_inner|fs::write|5 +reviewed|db/change_ledger/snapshot.rs|run_materialized_lane_snapshot_flow|fs::create_dir|2 +reviewed|db/change_ledger/snapshot.rs|run_materialized_lane_snapshot_flow|fs::remove_file|1 +reviewed|db/change_ledger/snapshot.rs|run_materialized_lane_snapshot_flow|fs::rename|1 +reviewed|db/change_ledger/snapshot.rs|run_materialized_lane_snapshot_flow|fs::set_permissions|1 +reviewed|db/change_ledger/snapshot.rs|run_materialized_lane_snapshot_flow|fs::write|4 +reviewed|db/change_ledger/snapshot.rs|run_materialized_candidate_lifecycle_flow|fs::create_dir|1 +reviewed|db/change_ledger/snapshot.rs|run_materialized_candidate_lifecycle_flow|fs::create_dir_all|1 +reviewed|db/change_ledger/snapshot.rs|run_materialized_candidate_lifecycle_flow|fs::remove_file|1 +reviewed|db/change_ledger/snapshot.rs|run_materialized_candidate_lifecycle_flow|fs::rename|2 +reviewed|db/change_ledger/snapshot.rs|run_materialized_candidate_lifecycle_flow|fs::write|3 +reviewed|db/change_ledger/snapshot.rs|run_materialized_candidate_lifecycle_flow|OpenOptions::append|1 +reviewed|db/core/backup/create.rs|create_backup_inner|fs::copy|3 +reviewed|db/core/backup/create.rs|create_backup_inner|fs::create_dir_all|3 +reviewed|db/core/backup/create.rs|create_backup_inner|fs::write|3 +reviewed|db/core/backup/create.rs|create_backup|fs::create_dir_all|1 +reviewed|db/core/backup/publication.rs|publish_staged_tree_with_exchange|fs::rename|1 +reviewed|db/core/backup/publication.rs|remove_any|fs::remove_dir_all|1 +reviewed|db/core/backup/publication.rs|remove_any|fs::remove_file|1 +reviewed|db/core/backup/publication.rs|sibling_stage|fs::create_dir|1 +reviewed|db/core/backup/restore_transaction.rs|allocate_policy_stage|OpenOptions::create_new|1 +reviewed|db/core/backup/restore_transaction.rs|allocate_policy_stage|OpenOptions::write|1 +reviewed|db/core/backup/restore_transaction.rs|prepare|fs::copy|2 +reviewed|db/core/backup/restore_transaction.rs|prepare|fs::write|1 +reviewed|db/core/backup/restore_transaction.rs|publish_policy_entry|fs::rename|1 +reviewed|db/core/backup/restore_transaction.rs|restore_old_entry|fs::rename|1 +reviewed|db/core/backup/restore_transaction.rs|write_marker|fs::rename|1 +reviewed|db/core/backup/restore_transaction.rs|write_marker|fs::write|1 +reviewed|db/core/backup/restore.rs|restore_backup|fs::copy|3 +reviewed|db/core/backup/restore.rs|restore_backup|fs::create_dir_all|4 +reviewed|db/core/backup/restore.rs|restore_backup|fs::write|2 +reviewed|db/core/backup/verify.rs|verify_backup|fs::copy|3 +reviewed|db/core/backup/verify.rs|verify_backup|fs::create_dir_all|1 +reviewed|db/core/backup/verify.rs|verify_backup|fs::remove_dir_all|1 +reviewed|db/core/backup/verify.rs|verify_backup|fs::write|2 +reviewed|db/core/init.rs|init_with_options|fs::create_dir_all|4 +reviewed|db/core/init.rs|init_with_options|fs::remove_dir_all|1 +reviewed|db/core/init.rs|init_with_options|fs::set_permissions|2 +reviewed|db/core/init.rs|init_with_options|fs::write|4 +reviewed|db/core/init.rs|open_at_without_recovery|fs::create_dir_all|1 +reviewed|db/core/workspace/ignore.rs|ignore_add|fs::write|1 +reviewed|db/core/workspace/ignore.rs|ignore_remove|fs::write|1 +reviewed|db/lane/lifecycle.rs|write_sparse_workdir_manifest|fs::create_dir_all|1 +pending_task12|db/lane/workdir/dokan.rs|flush_file_buffers|OpenOptions::write|1 +pending_task12|db/lane/workdir/dokan.rs|mount_dokan_cow_for_lane_with_view|fs::create_dir_all|1 +pending_task12|db/lane/workdir/dokan.rs|prepare_dokan_cow_workdir|fs::create_dir_all|1 +pending_task12|db/lane/workdir/dokan.rs|prepare_overlay_mountpoint|fs::create_dir_all|1 +pending_task12|db/lane/workdir/fuse.rs|mount_fuse_cow_for_lane_with_view|fs::create_dir_all|2 +pending_task12|db/lane/workdir/fuse.rs|prepare_fuse_cow_workdir|fs::create_dir_all|1 +pending_task12|db/lane/workdir/fuse.rs|prepare_overlay_mountpoint|fs::create_dir_all|1 +reviewed|db/lane/workdir/lifecycle.rs|remove_lane|fs::remove_dir_all|2 +reviewed|db/lane/workdir/manifest.rs|remove_clean_workdir_manifest_path|fs::remove_file|1 +reviewed|db/lane/workdir/manifest.rs|write_clean_workdir_manifest_entries_to_path|fs::create_dir_all|1 +reviewed|db/lane/workdir/materialize.rs|prepare_staged_destination|fs::remove_dir|1 +reviewed|db/lane/workdir/materialize.rs|probe_native_clone|fs::remove_file|2 +reviewed|db/lane/workdir/materialize.rs|probe_native_clone|fs::write|1 +reviewed|db/lane/workdir/nfs_overlay.rs|drop|fs::remove_file|2 +reviewed|db/lane/workdir/nfs_overlay.rs|mount_nfs_cow_for_lane_with_view|fs::create_dir_all|2 +reviewed|db/lane/workdir/nfs_overlay.rs|mount_nfs_cow_for_lane_with_view|OpenOptions::create_new|1 +reviewed|db/lane/workdir/nfs_overlay.rs|mount_nfs_cow_for_lane_with_view|OpenOptions::write|1 +reviewed|db/lane/workdir/nfs_overlay.rs|prepare_nfs_cow_workdir|fs::create_dir_all|1 +reviewed|db/lane/workdir/nfs_overlay.rs|recover_stale_mount|fs::remove_file|1 +reviewed|db/lane/workdir/record.rs|run_lane_record_after_c2_write|fs::write|1 +reviewed|db/lane/workdir/sync.rs|begin|fs::create_dir_all|1 +reviewed|db/lane/workdir/sync.rs|begin|fs::remove_dir_all|1 +reviewed|db/lane/workdir/sync.rs|commit|fs::remove_dir_all|1 +reviewed|db/lane/workdir/sync.rs|create_unique_lane_workdir_rescue_dir|fs::create_dir|1 +reviewed|db/lane/workdir/sync.rs|create_unique_lane_workdir_sync_stage_dir|fs::create_dir|1 +reviewed|db/lane/workdir/sync.rs|materialize_full_lane_workdir_staged|fs::create_dir_all|1 +reviewed|db/lane/workdir/sync.rs|materialize_full_lane_workdir_staged|fs::remove_dir_all|2 +reviewed|db/lane/workdir/sync.rs|materialize_full_lane_workdir_staged|fs::remove_file|1 +reviewed|db/lane/workdir/sync.rs|move_existing_lane_workdir_to_backup|fs::rename|1 +reviewed|db/lane/workdir/sync.rs|remove_existing_lane_workdir_path|fs::remove_dir_all|1 +reviewed|db/lane/workdir/sync.rs|remove_existing_lane_workdir_path|fs::remove_file|1 +reviewed|db/lane/workdir/sync.rs|remove_sparse_hydration_target|fs::remove_dir_all|1 +reviewed|db/lane/workdir/sync.rs|remove_sparse_hydration_target|fs::remove_file|1 +reviewed|db/lane/workdir/sync.rs|replace_lane_workdir_with_stage|fs::rename|2 +reviewed|db/lane/workdir/sync.rs|rescue_dirty_lane_workdir|fs::copy|1 +reviewed|db/lane/workdir/sync.rs|rescue_dirty_lane_workdir|fs::create_dir_all|3 +reviewed|db/lane/workdir/sync.rs|rescue_dirty_lane_workdir|fs::write|1 +reviewed|db/lane/workdir/sync.rs|rescue_replaced_lane_workdir_path|fs::copy|1 +reviewed|db/lane/workdir/sync.rs|rescue_replaced_lane_workdir_path|fs::create_dir_all|2 +reviewed|db/lane/workdir/sync.rs|rescue_replaced_lane_workdir_path|fs::write|1 +reviewed|db/lane/workdir/sync.rs|restore_sparse_hydration_snapshot|fs::copy|1 +reviewed|db/lane/workdir/sync.rs|restore_sparse_hydration_snapshot|fs::create_dir_all|2 +reviewed|db/lane/workdir/sync.rs|restore_sparse_hydration_snapshot|fs::set_permissions|1 +reviewed|db/lane/workdir/sync.rs|restore_sparse_hydration_snapshot|symlink_file|1 +reviewed|db/lane/workdir/sync.rs|rollback|fs::remove_dir_all|1 +reviewed|db/lane/workdir/sync.rs|snapshot_sparse_hydration_target|fs::copy|1 +reviewed|db/lane/workdir/sync.rs|sparse_hydration_file_matches_snapshot|fs::set_permissions|1 +reviewed|db/lane/workdir/sync.rs|sync_lane_workdir_with_paths_and_neighbors|fs::create_dir_all|1 +reviewed|db/lane/workdir/view_barrier.rs|acquire|fs::create_dir_all|1 +reviewed|db/lane/workdir/view_barrier.rs|open_barrier_no_follow|OpenOptions::create|1 +reviewed|db/lane/workdir/view_barrier.rs|open_barrier_no_follow|OpenOptions::write|1 +reviewed|db/lane/workdir/view_conformance.rs|run_mounted_view_conformance|fs::create_dir_all|1 +reviewed|db/lane/workdir/view_conformance.rs|run_mounted_view_conformance|fs::remove_file|2 +reviewed|db/lane/workdir/view_conformance.rs|run_mounted_view_conformance|fs::rename|1 +reviewed|db/lane/workdir/view_conformance.rs|run_mounted_view_conformance|fs::set_permissions|1 +reviewed|db/lane/workdir/view_conformance.rs|run_mounted_view_conformance|fs::write|2 +reviewed|db/lane/workdir/view_conformance.rs|run_mounted_view_conformance|OpenOptions::write|1 +reviewed|db/lane/workdir/view_core.rs|commit|fs::remove_file|1 +reviewed|db/lane/workdir/view_core.rs|create_layer_mount_reset_paths|fs::create_dir_all|2 +reviewed|db/lane/workdir/view_core.rs|create|OpenOptions::create_new|1 +reviewed|db/lane/workdir/view_core.rs|create|OpenOptions::create|1 +reviewed|db/lane/workdir/view_core.rs|create|OpenOptions::truncate|1 +reviewed|db/lane/workdir/view_core.rs|create|OpenOptions::write|1 +reviewed|db/lane/workdir/view_core.rs|ensure_declared_private_mount_path|fs::create_dir_all|1 +reviewed|db/lane/workdir/view_core.rs|ensure_upper_file_under_barrier|File::create|2 +reviewed|db/lane/workdir/view_core.rs|ensure_upper_file_under_barrier|OpenOptions::truncate|1 +reviewed|db/lane/workdir/view_core.rs|ensure_upper_file_under_barrier|OpenOptions::write|1 +reviewed|db/lane/workdir/view_core.rs|ensure_upper_parent|fs::create_dir_all|1 +reviewed|db/lane/workdir/view_core.rs|merge_lower_subtree_into_upper|fs::create_dir_all|2 +reviewed|db/lane/workdir/view_core.rs|mkdir|fs::create_dir_all|1 +reviewed|db/lane/workdir/view_core.rs|move_upper_subtree|fs::create_dir_all|2 +reviewed|db/lane/workdir/view_core.rs|move_upper_subtree|fs::remove_dir_all|2 +reviewed|db/lane/workdir/view_core.rs|move_upper_subtree|fs::remove_file|1 +reviewed|db/lane/workdir/view_core.rs|move_upper_subtree|fs::rename|1 +reviewed|db/lane/workdir/view_core.rs|prepare_validated_layer_mount_path|fs::rename|1 +reviewed|db/lane/workdir/view_core.rs|recover_layer_mount_resets|fs::create_dir_all|1 +reviewed|db/lane/workdir/view_core.rs|recover_layer_mount_resets|fs::rename|1 +reviewed|db/lane/workdir/view_core.rs|remove_layer_mount_reset_path|fs::remove_dir_all|1 +reviewed|db/lane/workdir/view_core.rs|remove_layer_mount_reset_path|fs::remove_file|1 +reviewed|db/lane/workdir/view_core.rs|remove|fs::remove_dir|1 +reviewed|db/lane/workdir/view_core.rs|remove|fs::remove_file|1 +reviewed|db/lane/workdir/view_core.rs|rename|fs::remove_dir|1 +reviewed|db/lane/workdir/view_core.rs|rename|fs::remove_file|1 +reviewed|db/lane/workdir/view_core.rs|rename|fs::rename|1 +reviewed|db/lane/workdir/view_core.rs|rollback|fs::create_dir_all|1 +reviewed|db/lane/workdir/view_core.rs|rollback|fs::rename|1 +reviewed|db/lane/workdir/view_core.rs|set_file_mode|fs::set_permissions|2 +reviewed|db/lane/workdir/view_core.rs|run_changed_path_view_flow|fs::write|1 +reviewed|db/lane/workdir/view_core.rs|run_changed_path_view_flow|OpenOptions::append|1 +reviewed|db/lane/workdir/view_journal.rs|acquire|fs::create_dir_all|1 +reviewed|db/lane/workdir/view_journal.rs|append_authenticated_record|fs::create_dir_all|1 +reviewed|db/lane/workdir/view_journal.rs|open_regular_no_follow|OpenOptions::append|1 +reviewed|db/lane/workdir/view_journal.rs|open_regular_no_follow|OpenOptions::create|1 +reviewed|db/lane/workdir/view_journal.rs|open_regular_no_follow|OpenOptions::write|1 +reviewed|db/lane/workdir/view_journal.rs|compact_inactive_generations|fs::remove_file|2 +reviewed|db/lane/workdir/view_journal.rs|drop|fs::remove_file|1 +reviewed|db/lane/workdir/view_journal.rs|initialize_storage|OpenOptions::create_new|1 +reviewed|db/lane/workdir/view_journal.rs|initialize_storage|OpenOptions::write|1 +reviewed|db/lane/workdir/view_journal.rs|rotate_after_checkpoint|OpenOptions::create_new|1 +reviewed|db/lane/workdir/view_journal.rs|rotate_after_checkpoint|OpenOptions::write|1 +reviewed|db/lane/workdir/view_layout.rs|ensure|fs::create_dir_all|1 +reviewed|db/lane/workspace_environment.rs|acquire_workspace_environment_cache_uses|fs::create_dir_all|4 +reviewed|db/lane/workspace_environment.rs|acquire_workspace_environment_cache_uses|fs::remove_file|2 +reviewed|db/lane/workspace_environment.rs|acquire_workspace_environment_cache_uses|fs::rename|1 +reviewed|db/lane/workspace_environment.rs|acquire_workspace_environment_cache_uses|OpenOptions::create_new|2 +reviewed|db/lane/workspace_environment.rs|acquire_workspace_environment_cache_uses|OpenOptions::write|2 +reviewed|db/lane/workspace_environment.rs|cleanup_mounted_environment_candidates|fs::remove_dir_all|1 +reviewed|db/lane/workspace_environment.rs|drop|fs::remove_file|2 +reviewed|db/lane/workspace_environment.rs|environment_cache_namespace_has_live_leases|fs::remove_dir|1 +reviewed|db/lane/workspace_environment.rs|environment_cache_namespace_has_live_leases|fs::remove_file|1 +reviewed|db/lane/workspace_environment.rs|execute_workspace_environment_plan_in_directory|fs::create_dir_all|2 +reviewed|db/lane/workspace_environment.rs|initialize_mounted_workspace_environment_plans|fs::create_dir_all|1 +reviewed|db/lane/workspace_environment.rs|initialize_mounted_workspace_environment_plans|fs::remove_dir_all|1 +reviewed|db/lane/workspace_environment.rs|materialize_workspace_environment_input|fs::copy|1 +reviewed|db/lane/workspace_environment.rs|materialize_workspace_environment_input|fs::create_dir_all|1 +reviewed|db/lane/workspace_environment.rs|materialize_workspace_environment_input|fs::set_permissions|1 +reviewed|db/lane/workspace_environment.rs|materialize_workspace_environment_input|OpenOptions::write|1 +reviewed|db/lane/workspace_environment.rs|run_mounted_workspace_environment_command|fs::copy|1 +reviewed|db/lane/workspace_environment.rs|run_mounted_workspace_environment_command|fs::create_dir_all|3 +reviewed|db/lane/workspace_environment.rs|run_mounted_workspace_environment_command|fs::set_permissions|1 +reviewed|db/lane/workspace_environment.rs|run_workspace_environment_command|fs::create_dir_all|3 +reviewed|db/lane/workspace_git.rs|ensure_workspace_git_shadow|fs::create_dir_all|3 +reviewed|db/lane/workspace_layer.rs|acquire_environment_cache_maintenance|fs::create_dir_all|1 +reviewed|db/lane/workspace_layer.rs|acquire_environment_cache_maintenance|fs::rename|1 +reviewed|db/lane/workspace_layer.rs|acquire_environment_cache_maintenance|OpenOptions::create_new|1 +reviewed|db/lane/workspace_layer.rs|acquire_environment_cache_maintenance|OpenOptions::write|1 +reviewed|db/lane/workspace_layer.rs|active_cache_builder_count|fs::remove_file|1 +reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight|fs::create_dir_all|2 +reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight|fs::remove_dir_all|1 +reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight|fs::rename|2 +reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight|OpenOptions::create_new|1 +reviewed|db/lane/workspace_layer.rs|build_workspace_layer_singleflight|OpenOptions::write|1 +reviewed|db/lane/workspace_layer.rs|copy_layer_tree|fs::copy|1 +reviewed|db/lane/workspace_layer.rs|copy_layer_tree|fs::create_dir_all|4 +reviewed|db/lane/workspace_layer.rs|create_layer_symlink|symlink_file|1 +reviewed|db/lane/workspace_layer.rs|drop|fs::remove_file|1 +reviewed|db/lane/workspace_layer.rs|make_layer_root_writable|fs::set_permissions|1 +reviewed|db/lane/workspace_layer.rs|make_tree_writable|fs::set_permissions|1 +reviewed|db/lane/workspace_layer.rs|preserve_layer_mode|fs::set_permissions|1 +reviewed|db/lane/workspace_layer.rs|publish_workspace_layer_from_directory|fs::create_dir_all|2 +reviewed|db/lane/workspace_layer.rs|publish_workspace_layer_from_directory|fs::rename|2 +reviewed|db/lane/workspace_layer.rs|remove_workspace_layer_trash_entries|fs::remove_dir_all|1 +reviewed|db/lane/workspace_layer.rs|remove_workspace_layer_trash_entries|fs::remove_file|1 +reviewed|db/lane/workspace_layer.rs|set_layer_read_only|fs::set_permissions|2 +reviewed|db/lane/workspace_layer.rs|workspace_cache_gc|fs::create_dir_all|1 +reviewed|db/lane/workspace_layer.rs|workspace_cache_gc|fs::remove_dir_all|3 +reviewed|db/lane/workspace_layer.rs|workspace_cache_gc|fs::remove_file|3 +reviewed|db/lane/workspace_layer.rs|workspace_cache_gc|fs::rename|5 +reviewed|db/lane/workspace_plugin.rs|append_environment_plugin_registry_record|fs::create_dir_all|1 +reviewed|db/lane/workspace_plugin.rs|append_publisher_trust_record|fs::create_dir_all|1 +reviewed|db/lane/workspace_plugin.rs|install_environment_adapter_plugin|fs::create_dir_all|2 +reviewed|db/lane/workspace_plugin.rs|install_environment_adapter_plugin|fs::create_dir|1 +reviewed|db/lane/workspace_plugin.rs|install_environment_adapter_plugin|fs::remove_dir_all|2 +reviewed|db/lane/workspace_plugin.rs|install_environment_adapter_plugin|fs::rename|2 +reviewed|db/lane/workspace_plugin.rs|install_environment_adapter_plugin|fs::set_permissions|1 +reviewed|db/lane/workspace_plugin.rs|install_environment_adapter_plugin|fs::write|1 +reviewed|db/lane/workspace_plugin.rs|invoke_environment_plugin|fs::copy|1 +reviewed|db/lane/workspace_plugin.rs|invoke_environment_plugin|fs::create_dir|2 +reviewed|db/lane/workspace_plugin.rs|invoke_environment_plugin|fs::set_permissions|1 +reviewed|db/lane/workspace_view.rs|create_workspace_view|fs::create_dir_all|1 +reviewed|db/lane/workspace_view.rs|mount_lane_workspace_until_requested|fs::remove_file|2 +reviewed|db/lane/workspace_view.rs|prepare_workspace_view_storage_for_lane_name|fs::create_dir_all|1 +reviewed|db/lane/workspace_view.rs|recover_workspace_views|fs::remove_file|1 +reviewed|db/lane/workspace_view.rs|release|fs::remove_file|1 +reviewed|db/lane/workspace_view.rs|workspace_command_environment|fs::create_dir_all|1 +reviewed|db/merge/git_export.rs|write_patch_to|fs::create_dir_all|1 +reviewed|db/merge/git_export.rs|write_patch_to|fs::write|1 +reviewed|db/mod.rs|acquire_workspace_lock_for_database|OpenOptions::create|1 +reviewed|db/mod.rs|acquire_workspace_lock_for_database|OpenOptions::write|1 +reviewed|db/mod.rs|cleanup_abandoned_workspace_lock_candidates|fs::remove_file|1 +reviewed|db/mod.rs|cleanup_abandoned_workspace_lock_candidates|OpenOptions::write|1 +reviewed|db/mod.rs|create_workspace_lock_candidate|fs::remove_file|1 +reviewed|db/mod.rs|create_workspace_lock_candidate|OpenOptions::create_new|1 +reviewed|db/mod.rs|create_workspace_lock_candidate|OpenOptions::write|1 +reviewed|db/mod.rs|inspect_existing_workspace_lock|fs::remove_file|1 +reviewed|db/mod.rs|inspect_existing_workspace_lock|OpenOptions::write|1 +reviewed|db/mod.rs|publish_workspace_lock|fs::remove_file|1 +reviewed|db/mod.rs|remove_owned_workspace_lock|fs::remove_file|1 +reviewed|db/mod.rs|remove_workspace_lock_candidate_if_matches|fs::remove_file|1 +reviewed|db/mod.rs|workspace_lock_candidate_unlink_result|fs::remove_file|1 +reviewed|db/mod.rs|workspace_lock_rollback_unlink_result|fs::remove_file|1 +reviewed|db/mod.rs|coordinate_schema_snapshot_validation|OpenOptions::create_new|1 +reviewed|db/mod.rs|coordinate_schema_snapshot_validation|OpenOptions::write|2 +reviewed|db/mod.rs|drop|fs::remove_file|1 +reviewed|db/mod.rs|secure_schema_validation_socket|fs::set_permissions|1 +reviewed|db/mod.rs|validate_schema_snapshot|fs::copy|1 +reviewed|db/performance.rs|emit_operation_metrics_report|OpenOptions::append|1 +reviewed|db/performance.rs|emit_operation_metrics_report|OpenOptions::create|1 +reviewed|db/record/branches.rs|rename_branch|fs::write|1 +reviewed|db/record/checkout.rs|remove_visible_files_absent_from_target|fs::remove_file|1 +reviewed|db/storage/content.rs|make_projection_writable|fs::set_permissions|1 +reviewed|db/storage/content.rs|project_entry_file|fs::remove_file|1 +reviewed|db/storage/content.rs|project_entry_file|fs::set_permissions|1 +reviewed|db/storage/git.rs|git_write_tree_from_head_delta|fs::create_dir_all|1 +reviewed|db/storage/git.rs|git_write_tree_from_head_delta|fs::create_dir|1 +reviewed|db/storage/git.rs|git_write_tree_from_head_delta|fs::write|1 +reviewed|db/storage/lifecycle/rebuild.rs|remove_path_index_derived_mirror|fs::remove_file|1 +reviewed|db/util/config_io.rs|write_config|fs::remove_file|1 +reviewed|db/util/config_io.rs|write_config|fs::rename|1 +reviewed|db/util/config_io.rs|write_config|fs::write|1 +reviewed|db/util/executable.rs|set_executable|fs::set_permissions|1 +reviewed|db/util/fs_cow.rs|clone_file_cow_clean|fs::remove_file|1 +reviewed|db/util/fs_cow.rs|clone_file_native|fs::remove_file|2 +reviewed|db/util/fs_cow.rs|clone_file_native|OpenOptions::create_new|1 +reviewed|db/util/fs_cow.rs|clone_file_native|OpenOptions::write|1 +reviewed|db/util/fs_cow.rs|clone_or_copy_projected_file|fs::copy|1 +reviewed|db/util/fs_cow.rs|clone_or_copy_projected_file|fs::create_dir_all|1 +reviewed|db/util/fs_cow.rs|materialize_workspace_file_cow_status_if_matching_with_durability|fs::create_dir_all|1 +reviewed|db/util/fs_cow.rs|materialize_workspace_file_cow_status_if_stamp_matches|fs::create_dir_all|1 +reviewed|db/util/fs_cow.rs|remove_cow_attempt_files|fs::remove_file|1 +reviewed|db/util/ignore.rs|write_default_trailignore|fs::write|1 +reviewed|db/util/materialize.rs|copy_dir_recursive|fs::copy|1 +reviewed|db/util/materialize.rs|copy_dir_recursive|fs::create_dir_all|1 +reviewed|db/util/materialize.rs|copy_dir_recursive|symlink_file|1 +reviewed|db/util/materialize.rs|create_dir_all_durable|fs::create_dir|1 +reviewed|db/util/materialize.rs|create_materialize_temp_file|OpenOptions::create_new|1 +reviewed|db/util/materialize.rs|create_materialize_temp_file|OpenOptions::write|1 +reviewed|db/util/materialize.rs|is_case_insensitive_filesystem|fs::remove_file|1 +reviewed|db/util/materialize.rs|is_case_insensitive_filesystem|OpenOptions::create_new|1 +reviewed|db/util/materialize.rs|is_case_insensitive_filesystem|OpenOptions::write|1 +reviewed|db/util/materialize.rs|materialize_batch|fs::create_dir_all|1 +reviewed|db/util/materialize.rs|materialize_into_batched_with_durability|fs::remove_file|1 +reviewed|db/util/materialize.rs|write_file_atomic|fs::create_dir_all|1 +reviewed|db/util/materialize.rs|write_file_atomic|fs::remove_file|1 +reviewed|db/util/materialize.rs|write_file_atomic|fs::rename|1 +reviewed|db/util/materialize.rs|write_materialized_file_with_durability|fs::create_dir_all|1 +reviewed|db/util/materialize.rs|write_materialized_file_with_durability|fs::remove_file|1 +reviewed|db/util/materialize.rs|write_materialized_file_with_durability|fs::rename|1 +reviewed|db/util/path.rs|prepare_checkout_workdir|fs::create_dir_all|1 +reviewed|db/util/path.rs|prepare_lane_workdir|fs::create_dir_all|1 +reviewed|db/util/path.rs|prepare_lane_workdir|fs::remove_dir_all|1 +reviewed|db/util/process_liveness.rs|test_crash_point|File::create|1 +reviewed|db/util/process_liveness.rs|test_crash_point|fs::create_dir_all|1 +reviewed|db/util/ref_files.rs|remove_ref_file|fs::remove_file|1 +reviewed|db/util/ref_files.rs|write_ref_file|fs::create_dir_all|1 +reviewed|db/util/ref_files.rs|write_ref_file|fs::write|1 diff --git a/trail/tests/schema_v18_hard_cutover.rs b/trail/tests/schema_v18_hard_cutover.rs new file mode 100644 index 0000000..0ed8e64 --- /dev/null +++ b/trail/tests/schema_v18_hard_cutover.rs @@ -0,0 +1,1024 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +#[cfg(target_os = "linux")] +use std::ffi::OsString; +#[cfg(target_os = "linux")] +use std::os::unix::ffi::OsStringExt; + +use rusqlite::Connection; +use trail::{InitImportMode, Trail}; + +struct SchemaFixture { + temp: tempfile::TempDir, + wal_writer: Option, +} + +impl SchemaFixture { + fn versioned(version: i64) -> Self { + let fixture = Self::fresh_v18(); + let conn = Connection::open(fixture.sqlite_path()).unwrap(); + conn.pragma_update(None, "user_version", version).unwrap(); + conn.execute( + "UPDATE schema_meta SET value = ?1 WHERE key = 'schema.version'", + [version.to_string()], + ) + .unwrap(); + drop(conn); + fixture + } + + fn partial_v18(missing_table: &str) -> Self { + let fixture = Self::fresh_v18(); + let conn = Connection::open(fixture.sqlite_path()).unwrap(); + conn.execute_batch("PRAGMA foreign_keys = OFF;").unwrap(); + conn.execute_batch(&format!("DROP TABLE {missing_table};")) + .unwrap(); + drop(conn); + fixture + } + + fn with_sql(mutator: impl FnOnce(&Connection)) -> Self { + let fixture = Self::fresh_v18(); + let conn = Connection::open(fixture.sqlite_path()).unwrap(); + mutator(&conn); + drop(conn); + fixture + } + + fn with_persistent_wal(mutator: impl FnOnce(&Connection)) -> Self { + let mut fixture = Self::fresh_v18(); + let conn = Connection::open(fixture.sqlite_path()).unwrap(); + assert_eq!( + conn.pragma_update_and_check(None, "journal_mode", "WAL", |row| { + row.get::<_, String>(0) + }) + .unwrap() + .to_ascii_lowercase(), + "wal" + ); + conn.pragma_update(None, "wal_autocheckpoint", 0).unwrap(); + mutator(&conn); + assert!(fixture.sqlite_path().with_extension("sqlite-wal").exists()); + fixture.wal_writer = Some(conn); + fixture + } + + fn mutated_master_sql(object: &str, from: &str, to: &str) -> Self { + Self::with_sql(|conn| { + conn.execute_batch("PRAGMA writable_schema = ON;").unwrap(); + let changed = conn + .execute( + "UPDATE sqlite_master SET sql = replace(sql, ?2, ?3) WHERE name = ?1", + (object, from, to), + ) + .unwrap(); + assert_eq!(changed, 1); + conn.execute_batch("PRAGMA writable_schema = OFF;").unwrap(); + let schema_version: i64 = conn + .query_row("PRAGMA schema_version", [], |row| row.get(0)) + .unwrap(); + conn.pragma_update(None, "schema_version", schema_version + 1) + .unwrap(); + }) + } + + fn fresh_v18() -> Self { + let temp = tempfile::tempdir().unwrap(); + Trail::init(temp.path(), "main", InitImportMode::Empty, false).unwrap(); + Self { + temp, + wal_writer: None, + } + } + + fn root(&self) -> &Path { + self.temp.path() + } + + fn sqlite_path(&self) -> PathBuf { + self.root().join(".trail/index/trail.sqlite") + } + + fn snapshot_tree_bytes(&self) -> BTreeMap> { + fn visit(root: &Path, path: &Path, files: &mut BTreeMap>) { + let mut entries = fs::read_dir(path) + .unwrap() + .collect::, _>>() + .unwrap(); + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + let path = entry.path(); + let file_type = entry.file_type().unwrap(); + if file_type.is_dir() { + visit(root, &path, files); + } else if file_type.is_file() { + files.insert( + path.strip_prefix(root).unwrap().to_path_buf(), + fs::read(path).unwrap(), + ); + } + } + } + + let trail_dir = self.root().join(".trail"); + let mut files = BTreeMap::new(); + visit(&trail_dir, &trail_dir, &mut files); + files + } + + fn use_slatedb_backend(&self) { + let config_path = self.root().join(".trail/config.toml"); + let config = fs::read_to_string(&config_path).unwrap(); + let config = config.replace( + "prolly_backend = \"sqlite\"", + "prolly_backend = \"slatedb\"", + ); + assert!(config.contains("prolly_backend = \"slatedb\"")); + fs::write(config_path, config).unwrap(); + } +} + +fn open_error(root: &Path) -> trail::Error { + match Trail::open(root) { + Ok(_) => panic!("existing incompatible schema was opened"), + Err(err) => err, + } +} + +fn assert_tree_unchanged(fixture: &SchemaFixture, before: &BTreeMap>) { + let after = fixture.snapshot_tree_bytes(); + let changes = before + .keys() + .chain(after.keys()) + .collect::>() + .into_iter() + .filter_map(|path| { + let old = before.get(path); + let new = after.get(path); + (old != new).then(|| { + format!( + "{}: {:?} -> {:?} bytes", + path.display(), + old.map(Vec::len), + new.map(Vec::len) + ) + }) + }) + .collect::>(); + assert!(changes.is_empty(), "workspace bytes changed: {changes:?}"); +} + +fn assert_persistent_wal_rejected_byte_invariantly(fixture: &SchemaFixture) { + let before = fixture.snapshot_tree_bytes(); + assert!( + before.keys().any(|path| path.ends_with("trail.sqlite-wal")), + "fixture did not retain a WAL generation" + ); + let err = open_error(fixture.root()); + assert_eq!(err.code(), "SCHEMA_REINITIALIZE_REQUIRED"); + assert_tree_unchanged(fixture, &before); +} + +fn insert_malformed_retirement_graph(conn: &Connection, kind: &str) { + conn.execute_batch( + "INSERT INTO changed_path_scopes( + scope_id,scope_kind,owner_id,scope_root,scope_root_identity, + filesystem_identity,filesystem_kind,case_sensitive,ref_name,ref_generation, + change_id,baseline_root_id,policy_fingerprint,policy_dependency_generation, + provider_id,provider_identity, + trust_state,trust_reason,retired_at,created_at,updated_at) + VALUES('scope-a','workspace','owner-a','','root-id','fs-id','native',1, + 'refs/heads/main',1,'change-a','root-a','policy-a',1, + 'provider','', + 'untrusted_gap','scope_retired',1,1,1); + INSERT INTO changed_path_observer_owners( + scope_id,epoch,owner_token,provider_id,provider_identity,lease_state, + fence_nonce,acquired_at,heartbeat_at,expires_at,updated_at) + VALUES('scope-a',1, + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'provider','','revoked', + X'0000000000000000000000000000000000000000000000000000000000000000', + 1,1,1,1); + INSERT INTO changed_path_observer_segments( + scope_id,epoch,segment_id,owner_token,provider_id,first_sequence, + durable_end_offset,folded_end_offset,segment_path,state,retirement_source_state, + retirement_file_length,retirement_file_hash,retirement_durable_hash, + retirement_source_device,retirement_source_inode,created_at,updated_at) + VALUES + ('scope-a',1,'segment-a', + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'provider',1,0,0,'segment-a.cpl','retired','open',0, + '0000000000000000000000000000000000000000000000000000000000000000', + '0000000000000000000000000000000000000000000000000000000000000000', + '7','8',1,1), + ('scope-a',1,'segment-b', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'provider',2,0,0,'segment-b.cpl','retired','open',0, + '0000000000000000000000000000000000000000000000000000000000000000', + '0000000000000000000000000000000000000000000000000000000000000000', + '9','10',1,1);", + ) + .unwrap(); + let allocation_state = if kind == "state_invalid" { + "allocated" + } else { + "bound" + }; + let bound_at = if allocation_state == "bound" { + "1" + } else { + "NULL" + }; + conn.execute_batch(&format!( + "INSERT INTO changed_path_segment_quarantine_allocations( + attempt_nonce,scope_id,epoch,segment_id,quarantine_leaf, + scope_directory_device,scope_directory_inode,identity_policy, + source_segment_device,source_segment_inode,quarantine_device,quarantine_inode,state, + created_at,updated_at,allocated_at,bound_at) + VALUES( + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'scope-a',1,'segment-a','.trail-delete-a.cplq','1','2', + 'direct_noreplace_same_directory_v1','7','8','7','8','{allocation_state}',1,1,1,{bound_at});" + )) + .unwrap(); + if kind == "cross_wired" { + conn.execute_batch( + "INSERT INTO changed_path_segment_quarantine_allocations( + attempt_nonce,scope_id,epoch,segment_id,quarantine_leaf, + scope_directory_device,scope_directory_inode,identity_policy, + source_segment_device,source_segment_inode,quarantine_device,quarantine_inode,state, + created_at,updated_at,allocated_at,bound_at) + VALUES( + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'scope-a',1,'segment-b','.trail-delete-b.cplq','1','2', + 'direct_noreplace_same_directory_v1','9','10','9','10','bound',1,1,1,1);", + ) + .unwrap(); + } + if kind != "orphan" { + let allocation_nonce = if kind == "cross_wired" { + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } else { + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }; + conn.execute_batch(&format!( + "INSERT INTO changed_path_segment_deletions( + scope_id,epoch,segment_id,original_leaf,quarantine_leaf,allocation_nonce, + log_format_version,provider_id,folded_end_offset, + retirement_continuity_generation,retirement_fence_nonce, + scope_directory_device,scope_directory_inode,quarantine_device,quarantine_inode, + segment_device,segment_inode,file_length,file_hash,durable_end_offset, + durable_hash,max_observer_log_bytes,max_segment_bytes,max_unfolded_tail_records, + owner_token,first_sequence,last_sequence,previous_segment_id, + previous_segment_hash,source_state,state,created_at,updated_at,completed_at) + VALUES('scope-a',1,'segment-a','segment-a.cpl','.trail-delete-a.cplq', + '{allocation_nonce}',1,'provider',0,1, + X'0000000000000000000000000000000000000000000000000000000000000000', + '1','2','7','8','7','8',0, + '0000000000000000000000000000000000000000000000000000000000000000',0, + '0000000000000000000000000000000000000000000000000000000000000000', + 1024,512,16, + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 1,NULL,NULL, + '0000000000000000000000000000000000000000000000000000000000000000', + 'open','quiesced',1,1,1);" + )) + .unwrap(); + } +} + +fn insert_source_quarantine_cross_wire(conn: &Connection) { + insert_malformed_retirement_graph(conn, "valid"); + conn.execute( + "DELETE FROM changed_path_observer_segments + WHERE scope_id='scope-a' AND segment_id='segment-b'", + [], + ) + .unwrap(); + conn.execute_batch( + "PRAGMA ignore_check_constraints=ON; + UPDATE changed_path_segment_quarantine_allocations + SET quarantine_device='17',quarantine_inode='18' + WHERE scope_id='scope-a' AND segment_id='segment-a'; + UPDATE changed_path_segment_deletions + SET quarantine_device='17',quarantine_inode='18' + WHERE scope_id='scope-a' AND segment_id='segment-a'; + PRAGMA ignore_check_constraints=OFF;", + ) + .unwrap(); +} + +fn insert_policy_scope(conn: &Connection) { + conn.execute_batch( + "INSERT INTO changed_path_scopes( + scope_id,scope_kind,owner_id,scope_root,scope_root_identity, + filesystem_identity,filesystem_kind,case_sensitive,ref_name,ref_generation, + change_id,baseline_root_id,policy_fingerprint,policy_dependency_generation, + trust_state,trust_reason,created_at,updated_at) + VALUES('policy-scope','workspace','policy-owner','','root-id','fs-id','native',1, + 'refs/heads/main',1,'change-a','root-a','policy-a',1, + 'stale_baseline','policy_fixture',1,1);", + ) + .unwrap(); +} + +fn insert_policy_dependency( + conn: &Connection, + identity: &str, + kind: &str, + content_identity: &[u8], + generation: i64, +) { + conn.execute_batch("PRAGMA ignore_check_constraints=ON;") + .unwrap(); + conn.execute( + "INSERT INTO changed_path_policy_dependencies( + scope_id,dependency_identity,dependency_kind,content_identity, + metadata_identity,observable,generation,last_source_sequence,created_at,updated_at) + VALUES('policy-scope',?1,?2,?3,X'73796e7468657469632d7631',1,?4,0,1,1)", + (identity, kind, content_identity, generation), + ) + .unwrap(); + conn.execute_batch("PRAGMA ignore_check_constraints=OFF;") + .unwrap(); +} + +#[test] +fn existing_v17_is_rejected_without_mutating_any_trail_byte() { + let fixture = SchemaFixture::versioned(17); + let before = fixture.snapshot_tree_bytes(); + let err = open_error(fixture.root()); + assert_eq!(err.code(), "SCHEMA_REINITIALIZE_REQUIRED"); + match &err { + trail::Error::SchemaReinitializeRequired { found, guidance } => { + assert_eq!( + found, + "database corrupt: found version 17; expected version 18" + ); + assert_eq!( + guidance, + "back up this workspace, then run `trail init --force` to create schema v18" + ); + } + other => panic!("unexpected error: {other}"), + } + assert_eq!( + err.to_string(), + "workspace schema database corrupt: found version 17; expected version 18 cannot be opened; back up this workspace, then run `trail init --force` to create schema v18" + ); + assert_tree_unchanged(&fixture, &before); +} + +#[test] +fn missing_or_changed_prolly_schema_is_rejected_without_mutation() { + let fixtures = [ + SchemaFixture::with_sql(|conn| { + conn.execute_batch("DROP TABLE prolly_hints;").unwrap(); + }), + SchemaFixture::mutated_master_sql("prolly_nodes", "node BLOB NOT NULL", "node BLOB"), + ]; + + for fixture in fixtures { + let before = fixture.snapshot_tree_bytes(); + let err = open_error(fixture.root()); + assert_eq!(err.code(), "SCHEMA_REINITIALIZE_REQUIRED"); + assert_tree_unchanged(&fixture, &before); + } +} + +#[test] +fn extra_views_and_triggers_are_rejected_without_mutation() { + let fixtures = [ + SchemaFixture::with_sql(|conn| { + conn.execute_batch( + "CREATE VIEW changed_paths AS SELECT normalized_path FROM changed_path_entries;", + ) + .unwrap(); + }), + SchemaFixture::with_sql(|conn| { + conn.execute_batch( + "CREATE TRIGGER trail_extra_trigger AFTER INSERT ON schema_meta + BEGIN SELECT NEW.key; END;", + ) + .unwrap(); + }), + ]; + + for fixture in fixtures { + let before = fixture.snapshot_tree_bytes(); + let err = open_error(fixture.root()); + assert_eq!(err.code(), "SCHEMA_REINITIALIZE_REQUIRED"); + assert_tree_unchanged(&fixture, &before); + } +} + +#[test] +fn slatedb_backend_runs_sql_schema_preflight_before_mutable_backend_open() { + let fixture = SchemaFixture::versioned(17); + fixture.use_slatedb_backend(); + let err = open_error(fixture.root()); + assert!(matches!( + err, + trail::Error::SchemaReinitializeRequired { .. } + )); +} + +#[cfg(target_os = "linux")] +#[test] +fn existing_schema_opens_from_a_non_utf8_workspace_path() { + let temp = tempfile::tempdir().unwrap(); + let workspace = temp + .path() + .join(OsString::from_vec(b"workspace-\xff".to_vec())); + fs::create_dir(&workspace).unwrap(); + Trail::init(&workspace, "main", InitImportMode::Empty, false).unwrap(); + + Trail::open(&workspace).unwrap(); +} + +#[test] +fn partial_v18_is_rejected_without_repair() { + let fixture = SchemaFixture::partial_v18("changed_path_scopes"); + let before = fixture.snapshot_tree_bytes(); + let err = open_error(fixture.root()); + assert_eq!(err.code(), "SCHEMA_REINITIALIZE_REQUIRED"); + assert_tree_unchanged(&fixture, &before); +} + +#[test] +fn existing_v0_v17_and_v19_are_rejected_without_mutation() { + for version in [0, 17, 19] { + let fixture = SchemaFixture::versioned(version); + let before = fixture.snapshot_tree_bytes(); + let err = open_error(fixture.root()); + assert_eq!(err.code(), "SCHEMA_REINITIALIZE_REQUIRED"); + assert!(err.to_string().contains("trail init --force")); + assert_tree_unchanged(&fixture, &before); + } +} + +#[test] +fn fresh_init_creates_the_exact_v18_ledger_shape() { + let fixture = SchemaFixture::fresh_v18(); + let conn = Connection::open(fixture.sqlite_path()).unwrap(); + assert_eq!( + conn.query_row("PRAGMA user_version", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 18 + ); + + let metadata = [ + ("schema.version", "18"), + ("changed_path.observer_log_format_min", "1"), + ("changed_path.observer_log_format_max", "1"), + ]; + for (key, expected) in metadata { + let value: String = conn + .query_row( + "SELECT value FROM schema_meta WHERE key = ?1", + [key], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(value, expected); + } + + for table in [ + "changed_path_scopes", + "changed_path_entries", + "changed_path_prefixes", + "changed_path_policy_dependencies", + "changed_path_intents", + "changed_path_intent_paths", + "changed_path_intent_prefixes", + "changed_path_reconciliations", + "changed_path_observer_segments", + "changed_path_observer_owners", + "changed_path_segment_quarantine_allocations", + "changed_path_segment_deletions", + ] { + let sql: String = conn + .query_row( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?1", + [table], + |row| row.get(0), + ) + .unwrap(); + assert!(!sql.contains("IF NOT EXISTS")); + assert!(!sql.contains("legacy_reconcile_required")); + } + assert_eq!( + conn.query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE name = 'changed_path_observer_leases'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(), + 0 + ); + + let entry_sql: String = conn + .query_row( + "SELECT sql FROM sqlite_master WHERE name = 'changed_path_entries'", + [], + |row| row.get(0), + ) + .unwrap(); + assert!(entry_sql.contains("normalized_path TEXT COLLATE BINARY NOT NULL")); + + let continuity_column: (String, String, i64, String) = conn + .query_row( + "SELECT name,type,\"notnull\",dflt_value + FROM pragma_table_info('changed_path_scopes') + WHERE name='continuity_generation'", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .unwrap(); + assert_eq!( + continuity_column, + ( + "continuity_generation".into(), + "INTEGER".into(), + 1, + "1".into() + ) + ); + + let scope_columns = conn + .prepare( + "SELECT name, type, [notnull], dflt_value FROM pragma_table_xinfo(\ + 'changed_path_scopes'\ + ) WHERE name LIKE 'max_%' ORDER BY cid", + ) + .unwrap() + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, String>(3)?, + )) + }) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!( + scope_columns, + vec![ + ( + "max_candidate_rows".into(), + "INTEGER".into(), + 1, + "250000".into() + ), + ( + "max_prefix_rows".into(), + "INTEGER".into(), + 1, + "16384".into() + ), + ( + "max_observer_log_bytes".into(), + "INTEGER".into(), + 1, + "268435456".into(), + ), + ( + "max_segment_bytes".into(), + "INTEGER".into(), + 1, + "16777216".into(), + ), + ( + "max_unfolded_tail_records".into(), + "INTEGER".into(), + 1, + "65536".into(), + ), + ] + ); + let scope_sql: String = conn + .query_row( + "SELECT sql FROM sqlite_master WHERE name = 'changed_path_scopes'", + [], + |row| row.get(0), + ) + .unwrap(); + for check in [ + "CHECK(max_candidate_rows>0)", + "CHECK(max_prefix_rows>0)", + "CHECK(max_observer_log_bytes>0)", + "CHECK(max_segment_bytes>0 AND max_segment_bytes<=max_observer_log_bytes)", + "CHECK(max_unfolded_tail_records>0)", + ] { + assert!(scope_sql.contains(check), "missing {check}"); + } + + let policy_sql: String = conn + .query_row( + "SELECT sql FROM sqlite_master WHERE name = 'changed_path_policy_dependencies'", + [], + |row| row.get(0), + ) + .unwrap(); + assert!(policy_sql.contains("dependency_identity TEXT COLLATE BINARY NOT NULL")); + assert!(policy_sql.contains("WITHOUT ROWID")); + assert!(policy_sql.contains("content_identity BLOB NOT NULL")); + assert!(policy_sql.contains("metadata_identity BLOB NOT NULL")); + assert!( + policy_sql.contains("'ignore'"), + "the walker-native .ignore source requires a distinct durable dependency role" + ); + + let policy_index_columns = conn + .prepare( + "SELECT name, coll FROM pragma_index_xinfo(\ + 'changed_path_policy_dependencies_generation_idx'\ + ) WHERE key = 1 ORDER BY seqno", + ) + .unwrap() + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + }) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!( + policy_index_columns, + vec![ + ("scope_id".to_string(), "BINARY".to_string()), + ("generation".to_string(), "BINARY".to_string()), + ("last_source_sequence".to_string(), "BINARY".to_string()), + ] + ); + + let delete_action: String = conn + .query_row( + "SELECT on_delete FROM pragma_foreign_key_list('changed_path_entries') \ + WHERE [from] = 'intent_id'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(delete_action, "SET NULL"); + + let allocation_columns = conn + .prepare( + "SELECT name FROM pragma_table_info('changed_path_segment_quarantine_allocations') + ORDER BY cid", + ) + .unwrap() + .query_map([], |row| row.get::<_, String>(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!( + allocation_columns, + vec![ + "attempt_nonce", + "scope_id", + "epoch", + "segment_id", + "quarantine_leaf", + "scope_directory_device", + "scope_directory_inode", + "identity_policy", + "source_segment_device", + "source_segment_inode", + "quarantine_device", + "quarantine_inode", + "observed_conflict_device", + "observed_conflict_inode", + "retained_reason", + "state", + "created_at", + "updated_at", + "allocated_at", + "bound_at", + "abandoned_at", + ] + ); + let deletion_allocation_column: (String, String, i64) = conn + .query_row( + "SELECT name,type,[notnull] + FROM pragma_table_info('changed_path_segment_deletions') + WHERE name='allocation_nonce'", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!( + deletion_allocation_column, + ("allocation_nonce".into(), "TEXT".into(), 1) + ); + + let index_columns = conn + .prepare( + "SELECT name, coll FROM pragma_index_xinfo('changed_path_entries_sequence_idx') \ + WHERE key = 1 ORDER BY seqno", + ) + .unwrap() + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + }) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!( + index_columns, + vec![ + ("scope_id".to_string(), "BINARY".to_string()), + ("last_sequence".to_string(), "BINARY".to_string()), + ] + ); +} + +#[test] +fn malformed_v18_attributes_are_rejected_read_only() { + let fixtures = [ + SchemaFixture::mutated_master_sql( + "changed_path_scopes", + "DEFAULT 'reconciling'", + "DEFAULT 'trusted'", + ), + SchemaFixture::mutated_master_sql( + "changed_path_scopes", + "DEFAULT 250000 CHECK(max_candidate_rows>0)", + "DEFAULT 250000 CHECK(max_candidate_rows>=0)", + ), + SchemaFixture::mutated_master_sql( + "changed_path_entries", + "normalized_path TEXT COLLATE BINARY", + "normalized_path TEXT COLLATE NOCASE", + ), + SchemaFixture::mutated_master_sql( + "changed_path_entries", + "ON DELETE SET NULL", + "ON DELETE CASCADE", + ), + SchemaFixture::mutated_master_sql( + "changed_path_entries_sequence_idx", + "scope_id, last_sequence", + "last_sequence, scope_id", + ), + SchemaFixture::mutated_master_sql( + "changed_path_observer_segments", + "CHECK (log_format_version = 1)", + "CHECK (log_format_version BETWEEN 1 AND 2)", + ), + SchemaFixture::with_sql(|conn| { + conn.execute( + "UPDATE schema_meta SET value = '2' \ + WHERE key = 'changed_path.observer_log_format_max'", + [], + ) + .unwrap(); + }), + ]; + + for fixture in fixtures { + let before = fixture.snapshot_tree_bytes(); + let err = open_error(fixture.root()); + assert_eq!(err.code(), "SCHEMA_REINITIALIZE_REQUIRED"); + assert_tree_unchanged(&fixture, &before); + } +} + +#[test] +fn malformed_retirement_graph_is_rejected_read_only() { + for kind in ["orphan", "cross_wired", "state_invalid"] { + let fixture = SchemaFixture::with_sql(|conn| { + insert_malformed_retirement_graph(conn, kind); + }); + let before = fixture.snapshot_tree_bytes(); + let err = open_error(fixture.root()); + assert_eq!(err.code(), "SCHEMA_REINITIALIZE_REQUIRED", "kind={kind}"); + assert_tree_unchanged(&fixture, &before); + } +} + +#[test] +fn source_segment_and_quarantine_identity_cross_wire_is_rejected_read_only() { + let fixture = SchemaFixture::with_sql(insert_source_quarantine_cross_wire); + let before = fixture.snapshot_tree_bytes(); + let err = open_error(fixture.root()); + assert_eq!(err.code(), "SCHEMA_REINITIALIZE_REQUIRED"); + assert_tree_unchanged(&fixture, &before); +} + +#[test] +fn malformed_policy_dependency_rows_are_rejected_read_only() { + let content_identity = [7_u8; 32]; + let fixtures = [ + SchemaFixture::with_sql(|conn| { + insert_policy_scope(conn); + insert_policy_dependency(conn, "builtin:recording-policy", "builtin", &[7], 1); + }), + SchemaFixture::with_sql(|conn| { + insert_policy_scope(conn); + insert_policy_dependency( + conn, + "builtin:recording-policy", + "builtin", + &content_identity, + 2, + ); + }), + SchemaFixture::with_sql(|conn| { + insert_policy_scope(conn); + insert_policy_dependency( + conn, + "synthetic:not-a-path", + "gitignore", + &content_identity, + 1, + ); + }), + SchemaFixture::with_sql(|conn| { + insert_policy_scope(conn); + insert_policy_dependency( + conn, + "path:2f746d702f612f2e2e2f62", + "gitignore", + &content_identity, + 1, + ); + }), + SchemaFixture::with_sql(|conn| { + insert_policy_scope(conn); + insert_policy_dependency( + conn, + "path:2f746d702f706f6c696379", + "builtin", + &content_identity, + 1, + ); + }), + ]; + for fixture in fixtures { + let before = fixture.snapshot_tree_bytes(); + let err = open_error(fixture.root()); + assert_eq!(err.code(), "SCHEMA_REINITIALIZE_REQUIRED"); + assert_tree_unchanged(&fixture, &before); + } +} + +#[test] +fn malformed_scope_segment_allocation_deletion_graph_is_rejected_read_only() { + let fixtures = [ + SchemaFixture::with_sql(|conn| { + insert_malformed_retirement_graph(conn, "orphan"); + }), + SchemaFixture::with_sql(|conn| { + insert_malformed_retirement_graph(conn, "state_invalid"); + conn.execute_batch( + "UPDATE changed_path_scopes + SET retired_at=NULL,trust_reason='active_scope' + WHERE scope_id='scope-a'; + UPDATE changed_path_observer_segments + SET state='open' WHERE scope_id='scope-a';", + ) + .unwrap(); + }), + SchemaFixture::with_sql(|conn| { + insert_malformed_retirement_graph(conn, "valid"); + conn.execute( + "DELETE FROM changed_path_observer_segments + WHERE scope_id='scope-a' AND segment_id='segment-b'", + [], + ) + .unwrap(); + conn.execute( + "UPDATE changed_path_segment_deletions + SET original_leaf='different.cpl' WHERE scope_id='scope-a'", + [], + ) + .unwrap(); + }), + SchemaFixture::with_sql(|conn| { + insert_malformed_retirement_graph(conn, "valid"); + conn.execute( + "DELETE FROM changed_path_observer_segments + WHERE scope_id='scope-a' AND segment_id='segment-b'", + [], + ) + .unwrap(); + conn.execute( + "UPDATE changed_path_segment_deletions + SET owner_token='cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc' + WHERE scope_id='scope-a'", + [], + ) + .unwrap(); + }), + SchemaFixture::with_sql(|conn| { + insert_malformed_retirement_graph(conn, "cross_wired"); + }), + ]; + for fixture in fixtures { + let before = fixture.snapshot_tree_bytes(); + let err = open_error(fixture.root()); + assert_eq!(err.code(), "SCHEMA_REINITIALIZE_REQUIRED"); + assert_tree_unchanged(&fixture, &before); + } +} + +#[test] +fn persistent_wal_malformed_retirement_graph_is_rejected_byte_invariantly() { + let fixture = SchemaFixture::with_persistent_wal(|conn| { + insert_malformed_retirement_graph(conn, "state_invalid"); + }); + assert_persistent_wal_rejected_byte_invariantly(&fixture); +} + +#[test] +fn persistent_wal_cross_wired_retirement_identity_is_rejected_byte_invariantly() { + let fixture = SchemaFixture::with_persistent_wal(insert_source_quarantine_cross_wire); + assert_persistent_wal_rejected_byte_invariantly(&fixture); +} + +#[test] +fn persistent_wal_malformed_policy_dependencies_are_rejected_byte_invariantly() { + let malformed_generation = SchemaFixture::with_persistent_wal(|conn| { + insert_policy_scope(conn); + insert_policy_dependency(conn, "builtin:recording-policy", "builtin", &[7_u8; 32], 2); + }); + assert_persistent_wal_rejected_byte_invariantly(&malformed_generation); + + let malformed_path = SchemaFixture::with_persistent_wal(|conn| { + insert_policy_scope(conn); + insert_policy_dependency( + conn, + "path:2f746d702f612f2e2e2f62", + "gitignore", + &[7_u8; 32], + 1, + ); + }); + assert_persistent_wal_rejected_byte_invariantly(&malformed_path); +} + +#[test] +fn persistent_wal_schema_fk_and_partial_generations_are_rejected_byte_invariantly() { + let malformed_schema = SchemaFixture::with_persistent_wal(|conn| { + conn.execute_batch("PRAGMA writable_schema=ON;").unwrap(); + conn.execute( + "UPDATE sqlite_master SET sql=replace(sql,'CHECK (state = ''quiesced'')', + 'CHECK (state IN (''quiesced'',''deleted''))') + WHERE name='changed_path_segment_deletions'", + [], + ) + .unwrap(); + conn.execute_batch("PRAGMA writable_schema=OFF;").unwrap(); + let version: i64 = conn + .query_row("PRAGMA schema_version", [], |row| row.get(0)) + .unwrap(); + conn.pragma_update(None, "schema_version", version + 1) + .unwrap(); + }); + assert_persistent_wal_rejected_byte_invariantly(&malformed_schema); + + let orphan_policy = SchemaFixture::with_persistent_wal(|conn| { + conn.pragma_update(None, "foreign_keys", false).unwrap(); + conn.execute( + "INSERT INTO changed_path_policy_dependencies( + scope_id,dependency_identity,dependency_kind,content_identity, + metadata_identity,observable,generation,last_source_sequence,created_at,updated_at) + VALUES('missing-scope','builtin:recording-policy','builtin',zeroblob(32),X'02',1,1,0,1,1)", + [], + ) + .unwrap(); + }); + assert_persistent_wal_rejected_byte_invariantly(&orphan_policy); + + let partial = SchemaFixture::with_persistent_wal(|conn| { + conn.execute_batch("PRAGMA foreign_keys=OFF; DROP TABLE changed_path_policy_dependencies;") + .unwrap(); + }); + assert_persistent_wal_rejected_byte_invariantly(&partial); +} + +#[test] +fn valid_persistent_wal_generation_survives_snapshot_preflight_and_mutable_handoff() { + let fixture = SchemaFixture::with_persistent_wal(|conn| { + conn.execute( + "UPDATE schema_meta SET value='wal-visible' WHERE key='app.version'", + [], + ) + .unwrap(); + }); + let opened = Trail::open(fixture.root()).unwrap(); + drop(opened); + let conn = Connection::open(fixture.sqlite_path()).unwrap(); + let value: String = conn + .query_row( + "SELECT value FROM schema_meta WHERE key='app.version'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(value, "wal-visible"); +} diff --git a/trail/tests/support/acp_harness.rs b/trail/tests/support/acp_harness.rs new file mode 100644 index 0000000..3fe812f --- /dev/null +++ b/trail/tests/support/acp_harness.rs @@ -0,0 +1,106 @@ +#![allow(dead_code)] + +use std::fs; +use std::io::{BufRead, BufReader, Write}; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; + +use serde_json::Value; +use trail::{InitImportMode, Trail}; + +pub fn trail_bin() -> PathBuf { + std::env::var_os("TRAIL_TEST_BIN") + .map(PathBuf::from) + .or_else(|| option_env!("CARGO_BIN_EXE_trail").map(PathBuf::from)) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/debug/trail")) +} + +pub fn workspace() -> tempfile::TempDir { + let temp = tempfile::tempdir().expect("create ACP harness workspace"); + fs::write(temp.path().join("README.md"), "ACP conformance fixture\n") + .expect("write workspace fixture"); + Trail::init(temp.path(), "main", InitImportMode::WorkingTree, false) + .expect("initialize Trail workspace"); + temp +} + +/// Writes the platform launcher for a named reference-agent scenario. +/// +/// Scenario implementations are deliberately data-driven Python programs. The +/// launcher itself is the native command required by the relay contract: a +/// POSIX shell script on Unix and a PowerShell script on Windows. +pub fn fixture_agent_command(workspace: &Path, scenario: &str, source: &str) -> Vec { + let program = workspace.join(format!("{scenario}-agent.py")); + fs::write(&program, source).expect("write reference agent program"); + + #[cfg(unix)] + { + let launcher = workspace.join(format!("{scenario}-agent.sh")); + fs::write( + &launcher, + format!("#!/bin/sh\nset -eu\nexec python3 '{}'\n", program.display()), + ) + .expect("write reference agent launcher"); + let mut permissions = fs::metadata(&launcher).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&launcher, permissions).unwrap(); + return vec![launcher.to_string_lossy().into_owned()]; + } + + #[cfg(windows)] + { + let launcher = workspace.join(format!("{scenario}-agent.ps1")); + fs::write( + &launcher, + format!( + "$ErrorActionPreference = 'Stop'\npython '{}'\nexit $LASTEXITCODE\n", + program.display() + ), + ) + .expect("write reference agent launcher"); + return vec![ + "powershell.exe".to_string(), + "-NoProfile".to_string(), + "-File".to_string(), + launcher.to_string_lossy().into_owned(), + ]; + } +} + +pub fn spawn_relay(workspace: &Path, agent_command: &[String]) -> Child { + assert!(!agent_command.is_empty(), "agent command must not be empty"); + Command::new(trail_bin()) + .arg("--workspace") + .arg(workspace) + .args(["acp", "relay", "--"]) + .args(agent_command) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn Trail ACP relay") +} + +pub fn write_json(writer: &mut impl Write, value: &Value) { + serde_json::to_writer(&mut *writer, value).expect("serialize ACP frame"); + writer.write_all(b"\n").expect("write ACP frame delimiter"); + writer.flush().expect("flush ACP frame"); +} + +pub fn read_json(reader: &mut impl BufRead) -> Value { + let mut line = String::new(); + reader.read_line(&mut line).expect("read ACP frame"); + assert!( + !line.is_empty(), + "relay closed before returning an ACP frame" + ); + serde_json::from_str(line.trim_end()).expect("parse ACP frame") +} + +pub fn relay_stdio(child: &mut Child) -> (impl Write + use<>, impl BufRead + use<>) { + let stdin = child.stdin.take().expect("relay stdin"); + let stdout = BufReader::new(child.stdout.take().expect("relay stdout")); + (stdin, stdout) +} diff --git a/trail/tests/terminal_output_guard.rs b/trail/tests/terminal_output_guard.rs new file mode 100644 index 0000000..6b7b2be --- /dev/null +++ b/trail/tests/terminal_output_guard.rs @@ -0,0 +1,40 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +/// Command adapters describe terminal semantics. The only low-level terminal +/// writer is `render/ui.rs`, which owns buffering, ANSI, pager, and progress. +#[test] +fn command_adapters_do_not_write_directly_to_the_terminal() { + let command_root = Path::new(env!("CARGO_MANIFEST_DIR")).join("src/cli/command"); + let mut files = Vec::new(); + collect_rust_files(&command_root.join("handler"), &mut files); + collect_rust_files(&command_root.join("render"), &mut files); + files.push(command_root.join("handler.rs")); + files.push(command_root.join("render.rs")); + + for file in files { + if file.ends_with("render/ui.rs") { + continue; + } + let source = fs::read_to_string(&file).expect("read command source"); + for forbidden in ["println!", "eprintln!", "print!", "\\x1b[", "\\x1B["] { + assert!( + !source.contains(forbidden), + "{} bypasses the terminal renderer with {forbidden}", + file.display() + ); + } + } +} + +fn collect_rust_files(root: &Path, files: &mut Vec) { + for entry in fs::read_dir(root).expect("read command source directory") { + let entry = entry.expect("read command source entry"); + let path = entry.path(); + if path.is_dir() { + collect_rust_files(&path, files); + } else if path.extension().is_some_and(|extension| extension == "rs") { + files.push(path); + } + } +}