ci: auto GitHub Releases + parallel test scenarios (#11) #61
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Test" | |
| on: | |
| pull_request: | |
| push: | |
| branches: [main, develop] | |
| schedule: | |
| - cron: "0 4 * * 1" | |
| concurrency: | |
| group: "${{ github.workflow }}-${{ github.ref }}" | |
| cancel-in-progress: true | |
| permissions: {} | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: ShellCheck | |
| uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # 2.0.0 | |
| with: | |
| severity: warning | |
| - name: Validate JSON | |
| run: | | |
| FAIL=0 | |
| while IFS= read -r -d '' f; do | |
| if ! python3 -m json.tool "$f" > /dev/null 2>&1; then | |
| echo "ERROR: Invalid JSON: $f" | |
| FAIL=1 | |
| fi | |
| done < <(find . -name '*.json' -not -path './.git/*' -print0) | |
| exit "$FAIL" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 | |
| - name: Validate YAML | |
| run: | | |
| uvx [email protected] -d relaxed .github/workflows/ | |
| - name: Prettier check | |
| run: | | |
| npx [email protected] --check "**/*.{json,yml,yaml,md}" --ignore-path .gitignore | |
| - name: Markdownlint | |
| run: | | |
| npx [email protected] "**/*.md" --ignore node_modules | |
| - name: shfmt format check | |
| run: | | |
| curl -fsSL https://github.com/mvdan/sh/releases/download/v3.13.0/shfmt_v3.13.0_linux_amd64 \ | |
| -o /usr/local/bin/shfmt | |
| echo "70aa99784703a8d6569bbf0b1e43e1a91906a4166bf1a79de42050a6d0de7551 /usr/local/bin/shfmt" | sha256sum -c - | |
| chmod +x /usr/local/bin/shfmt | |
| shfmt -ln bash -d -i 4 -ci src/ test/ | |
| - name: Check .sh files are executable | |
| run: | | |
| FAIL=0 | |
| while IFS= read -r -d '' f; do | |
| if [[ ! -x "$f" ]]; then | |
| echo "ERROR: $f is not executable" | |
| FAIL=1 | |
| fi | |
| done < <(find . -name '*.sh' -not -path './.git/*' -print0) | |
| exit "$FAIL" | |
| # Run per-scenario tests in parallel groups for faster CI. | |
| # The 17 scenarios are split into 3 groups by estimated build cost. | |
| # Each group dynamically filters scenarios.json to run only its subset. | |
| test-scenarios: | |
| name: test-scenarios (${{ matrix.group.name }}) | |
| needs: lint | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| group: | |
| - name: fast | |
| scenarios: >- | |
| default_options | |
| completions_pipeline | |
| mcp_enabled | |
| negative_validation | |
| security_permissions | |
| completions_disabled | |
| - name: medium | |
| scenarios: >- | |
| custom_version | |
| custom_install_path | |
| custom_node_version | |
| install_path_with_completions | |
| mount_host_config | |
| node_preinstalled | |
| - name: slow | |
| scenarios: >- | |
| idempotency | |
| upgrade_version | |
| multi_feature_combo | |
| alpine_specific | |
| fedora_default | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Install devcontainer CLI | |
| run: npm install -g @devcontainers/[email protected] | |
| - name: Generate group scenarios.json | |
| env: | |
| SCENARIOS: ${{ matrix.group.scenarios }} | |
| GROUP_NAME: ${{ matrix.group.name }} | |
| run: | | |
| FULL="test/claude-code/scenarios.json" | |
| # Build a jq filter that selects only the named scenarios | |
| JQ_FILTER=$(echo "${SCENARIOS}" | tr -s ' ' '\n' | sed 's/.*/"&"/' | paste -sd, | sed 's/^/[/;s/$/]/') | |
| jq --argjson keys "${JQ_FILTER}" 'with_entries(select(.key as $k | $keys | index($k)))' "${FULL}" > /tmp/group-scenarios.json | |
| echo "::group::scenarios.json for group '${GROUP_NAME}'" | |
| cat /tmp/group-scenarios.json | |
| echo "::endgroup::" | |
| cp /tmp/group-scenarios.json "${FULL}" | |
| - name: Run scenarios (${{ matrix.group.name }}) | |
| run: | | |
| devcontainer features test --project-folder . 2>&1 | tee /tmp/scenario-test-output.log | |
| # Workaround: devcontainers/[email protected] exits 0 even when feature install fails. | |
| if grep -qE "Exit code [1-9][0-9]*|failed to install|Failed to launch|Failed:| FAIL:" /tmp/scenario-test-output.log; then | |
| echo "ERROR: Test output contains failures." | |
| exit 1 | |
| fi | |
| # Positive assertion: verify at least one test passed | |
| if ! grep -qE "[0-9]+ passed" /tmp/scenario-test-output.log; then | |
| echo "ERROR: No pass markers found — test may not have run." | |
| exit 1 | |
| fi | |
| - name: Annotate install warnings | |
| if: always() | |
| run: | | |
| mapfile -t warnings < <(grep -oP '(?<=claude-code feature\] WARNING: ).*' /tmp/scenario-test-output.log 2>/dev/null | tr -d '\r' | sort -u || true) | |
| if [[ ${#warnings[@]} -gt 0 ]]; then | |
| echo "## :warning: Install Warnings" >> "$GITHUB_STEP_SUMMARY" | |
| for msg in "${warnings[@]}"; do | |
| echo "- ${msg}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "::warning title=Install Warning::${msg}" | |
| done | |
| fi | |
| - name: Upload logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: logs-scenarios-${{ matrix.group.name }} | |
| path: /tmp/scenario-test-output.log | |
| retention-days: 7 | |
| # Run core assertions across all supported base images | |
| test-image-matrix: | |
| needs: lint | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 8 | |
| matrix: | |
| image: | |
| - "mcr.microsoft.com/devcontainers/base:ubuntu" | |
| - "mcr.microsoft.com/devcontainers/base:debian" | |
| - "mcr.microsoft.com/devcontainers/base:alpine" | |
| - "mcr.microsoft.com/devcontainers/javascript-node" | |
| - "ubuntu:24.04" | |
| - "alpine:3.21" | |
| - "archlinux:latest" | |
| - "fedora:40" | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Install devcontainer CLI | |
| run: npm install -g @devcontainers/[email protected] | |
| - name: Test on ${{ matrix.image }} | |
| run: | | |
| devcontainer features test \ | |
| --features claude-code \ | |
| --skip-scenarios \ | |
| --base-image "${{ matrix.image }}" \ | |
| --project-folder . 2>&1 | tee /tmp/test-output.log | |
| # Workaround: devcontainers/[email protected] exits 0 even when feature install fails. | |
| # Grep for known failure strings and fail explicitly. Revisit on CLI upgrade. | |
| if grep -qE "Exit code [1-9][0-9]*|failed to install|Failed to launch|Failed:| FAIL:" /tmp/test-output.log; then | |
| echo "ERROR: Test output contains failures." | |
| exit 1 | |
| fi | |
| # Positive assertion: verify at least one test passed | |
| if ! grep -qE "[0-9]+ passed" /tmp/test-output.log; then | |
| echo "ERROR: No pass markers found — test may not have run." | |
| exit 1 | |
| fi | |
| - name: Annotate install warnings | |
| if: always() | |
| run: | | |
| mapfile -t warnings < <(grep -oP '(?<=claude-code feature\] WARNING: ).*' /tmp/test-output.log 2>/dev/null | tr -d '\r' | sort -u || true) | |
| if [[ ${#warnings[@]} -gt 0 ]]; then | |
| echo "## :warning: Install Warnings" >> "$GITHUB_STEP_SUMMARY" | |
| for msg in "${warnings[@]}"; do | |
| echo "- ${msg}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "::warning title=Install Warning::${msg}" | |
| done | |
| fi | |
| - name: Upload logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: logs-amd64-${{ strategy.job-index }} | |
| path: /tmp/test-output.log | |
| retention-days: 7 | |
| # Nightly-only extended image matrix for broader coverage | |
| test-image-matrix-extended: | |
| needs: lint | |
| if: github.event_name == 'schedule' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 5 | |
| matrix: | |
| image: | |
| - "rockylinux:9" | |
| - "amazonlinux:2023" | |
| - "debian:bookworm" | |
| - "ubuntu:22.04" | |
| - "alpine:3.20" | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Install devcontainer CLI | |
| run: npm install -g @devcontainers/[email protected] | |
| - name: Test on ${{ matrix.image }} | |
| run: | | |
| devcontainer features test \ | |
| --features claude-code \ | |
| --skip-scenarios \ | |
| --base-image "${{ matrix.image }}" \ | |
| --project-folder . 2>&1 | tee /tmp/test-output.log | |
| # Workaround: devcontainers/[email protected] exits 0 even when feature install fails. | |
| # Grep for known failure strings and fail explicitly. Revisit on CLI upgrade. | |
| if grep -qE "Exit code [1-9][0-9]*|failed to install|Failed to launch|Failed:| FAIL:" /tmp/test-output.log; then | |
| echo "ERROR: Test output contains failures." | |
| exit 1 | |
| fi | |
| # Positive assertion: verify at least one test passed | |
| if ! grep -qE "[0-9]+ passed" /tmp/test-output.log; then | |
| echo "ERROR: No pass markers found — test may not have run." | |
| exit 1 | |
| fi | |
| - name: Annotate install warnings | |
| if: always() | |
| run: | | |
| mapfile -t warnings < <(grep -oP '(?<=claude-code feature\] WARNING: ).*' /tmp/test-output.log 2>/dev/null | tr -d '\r' | sort -u || true) | |
| if [[ ${#warnings[@]} -gt 0 ]]; then | |
| echo "## :warning: Install Warnings" >> "$GITHUB_STEP_SUMMARY" | |
| for msg in "${warnings[@]}"; do | |
| echo "- ${msg}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "::warning title=Install Warning::${msg}" | |
| done | |
| fi | |
| - name: Upload logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: logs-extended-${{ strategy.job-index }} | |
| path: /tmp/test-output.log | |
| retention-days: 7 | |
| # arm64 tests via QEMU emulation on standard ubuntu-latest runners | |
| test-arm64: | |
| needs: lint | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 # QEMU emulation is significantly slower than native | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 2 | |
| matrix: | |
| image: | |
| - "ubuntu:24.04" | |
| - "alpine:3.21" | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Set up QEMU for arm64 emulation | |
| uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Install devcontainer CLI | |
| run: npm install -g @devcontainers/[email protected] | |
| - name: Test on ${{ matrix.image }} (arm64) | |
| env: | |
| DOCKER_DEFAULT_PLATFORM: linux/arm64 | |
| run: | | |
| devcontainer features test \ | |
| --features claude-code \ | |
| --skip-scenarios \ | |
| --base-image "${{ matrix.image }}" \ | |
| --project-folder . 2>&1 | tee /tmp/test-output.log | |
| # Workaround: devcontainers/[email protected] exits 0 even when feature install fails. | |
| # Grep for known failure strings and fail explicitly. Revisit on CLI upgrade. | |
| if grep -qE "Exit code [1-9][0-9]*|failed to install|Failed to launch|Failed:| FAIL:" /tmp/test-output.log; then | |
| echo "ERROR: Test output contains failures." | |
| exit 1 | |
| fi | |
| # Positive assertion: verify at least one test passed | |
| if ! grep -qE "[0-9]+ passed" /tmp/test-output.log; then | |
| echo "ERROR: No pass markers found — test may not have run." | |
| exit 1 | |
| fi | |
| - name: Annotate install warnings | |
| if: always() | |
| run: | | |
| mapfile -t warnings < <(grep -oP '(?<=claude-code feature\] WARNING: ).*' /tmp/test-output.log 2>/dev/null | tr -d '\r' | sort -u || true) | |
| if [[ ${#warnings[@]} -gt 0 ]]; then | |
| echo "## :warning: Install Warnings" >> "$GITHUB_STEP_SUMMARY" | |
| for msg in "${warnings[@]}"; do | |
| echo "- ${msg}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "::warning title=Install Warning::${msg}" | |
| done | |
| fi | |
| - name: Upload logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: logs-arm64-${{ strategy.job-index }} | |
| path: /tmp/test-output.log | |
| retention-days: 7 |