From b6101a103fc614fd04b60439b77d2a4f36df87c2 Mon Sep 17 00:00:00 2001 From: Glenn Harper Date: Mon, 13 Jul 2026 10:52:18 -0400 Subject: [PATCH 1/2] test(ext-agents): guard tier selectors against false-green Centralize the Tier 0 and Tier 1 go test patterns, then validate that each selector matches tests, no test belongs to both tiers, and every Test_AIAgent_ test is assigned exactly once. This makes renamed or newly added tests fail CI instead of being silently skipped by go test -run. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6eccd2f8-19dc-4da9-abf7-12eb690cf50c --- .../workflows/test-ext-azure-ai-agents.yml | 70 ++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-ext-azure-ai-agents.yml b/.github/workflows/test-ext-azure-ai-agents.yml index b2d924d763e..0b597ac3476 100644 --- a/.github/workflows/test-ext-azure-ai-agents.yml +++ b/.github/workflows/test-ext-azure-ai-agents.yml @@ -20,6 +20,9 @@ permissions: jobs: test: runs-on: ubuntu-latest + env: + TIER_0_TEST_PATTERN: '^Test_AIAgent_(Version|Help|Init_NoPrompt_MissingFlags|Doctor_Help)$' + TIER_1_TEST_PATTERN: '^Test_AIAgent_(Init_(NoPrompt_(Defer|WithProject)|NegativeControl_BadCassette)|SampleList_(Recorded|JSON_Recorded))$' steps: - uses: actions/checkout@v6 @@ -73,12 +76,75 @@ jobs: azd x publish azd extension install azure.ai.agents --source local --force --no-prompt + - name: Validate tier test selectors + working-directory: cli/azd + shell: bash + run: | + set -euo pipefail + + list_tests() { + go test -tags=record -list "$1" ./test/functional/ai_agents/ | + awk '/^Test_AIAgent_/ { print }' | + tr -d '\r' + } + + all_tests="$(list_tests '^Test_AIAgent_')" + tier_0_tests="$(list_tests "$TIER_0_TEST_PATTERN")" + tier_1_tests="$(list_tests "$TIER_1_TEST_PATTERN")" + + if [[ -z "$all_tests" ]]; then + echo "::error::No Test_AIAgent_ tests were discovered." + exit 1 + fi + if [[ -z "$tier_0_tests" ]]; then + echo "::error::The Tier 0 test selector matched no tests." + exit 1 + fi + if [[ -z "$tier_1_tests" ]]; then + echo "::error::The Tier 1 test selector matched no tests." + exit 1 + fi + + duplicate_tests="$( + printf '%s\n%s\n' "$tier_0_tests" "$tier_1_tests" | + sort | + uniq -d | + tr -d '\r' + )" + if [[ -n "$duplicate_tests" ]]; then + echo "::error::Tests are assigned to multiple tiers:" + echo "$duplicate_tests" + exit 1 + fi + + all_tests_sorted="$( + printf '%s\n' "$all_tests" | + sort | + tr -d '\r' + )" + selected_tests_sorted="$( + printf '%s\n%s\n' "$tier_0_tests" "$tier_1_tests" | + sort | + tr -d '\r' + )" + if ! diff -u \ + <(printf '%s\n' "$all_tests_sorted") \ + <(printf '%s\n' "$selected_tests_sorted"); then + echo "::error::Tier selectors must cover every Test_AIAgent_ test exactly once." + exit 1 + fi + + echo "Tier 0 tests:" + echo "$tier_0_tests" + echo "Tier 1 tests:" + echo "$tier_1_tests" + - name: Run Tier 0 tests (offline, production extension) working-directory: cli/azd env: CLI_TEST_AZD_PATH: ${{ github.workspace }}/cli/azd/azd-record run: | - go test -tags=record -run "Test_AIAgent_(Version|Help|Init_NoPrompt_MissingFlags|Doctor_Help)" \ + go test -tags=record -run "$TIER_0_TEST_PATTERN" \ ./test/functional/ai_agents/ -v -timeout 5m - name: Swap extension to record binary for Tier 1 @@ -94,5 +160,5 @@ jobs: AZURE_RECORD_MODE: playback CLI_TEST_AZD_PATH: ${{ github.workspace }}/cli/azd/azd-record run: | - go test -tags=record -run "Test_AIAgent_(Init_(NoPrompt_(Defer|WithProject)|NegativeControl_BadCassette)|SampleList_(Recorded|JSON_Recorded))" \ + go test -tags=record -run "$TIER_1_TEST_PATTERN" \ ./test/functional/ai_agents/ -v -timeout 5m From a1f99f374a79d14ce3a855b6285cd6e07321eab2 Mon Sep 17 00:00:00 2001 From: Glenn Harper Date: Mon, 13 Jul 2026 19:50:25 -0400 Subject: [PATCH 2/2] test(ext-agents): validate tier selectors before builds Move the selector guard directly after setup-go so regex drift fails before building azd, the inspector dependency, or the agents extension. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6eccd2f8-19dc-4da9-abf7-12eb690cf50c --- .../workflows/test-ext-azure-ai-agents.yml | 92 +++++++++---------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/.github/workflows/test-ext-azure-ai-agents.yml b/.github/workflows/test-ext-azure-ai-agents.yml index 0b597ac3476..4954deb6f14 100644 --- a/.github/workflows/test-ext-azure-ai-agents.yml +++ b/.github/workflows/test-ext-azure-ai-agents.yml @@ -30,52 +30,6 @@ jobs: with: go-version-file: cli/azd/go.mod - - name: Build azd - working-directory: cli/azd - run: | - go build -tags=record -o azd-record ./ - # Make the record-tagged binary available as "azd" on PATH. - # Required by extension sub-processes (azd x publish) and by - # azidentity.AzureDeveloperCLICredential (exec.LookPath("azd")) in playback. - sudo ln -sf "$GITHUB_WORKSPACE/cli/azd/azd-record" /usr/local/bin/azd - - - name: Install extension toolchain - run: | - # Install the developer extension (provides 'azd x' commands). - # The built-in 'azd' source is auto-registered, so no source-add needed. - azd extension install microsoft.azd.extensions --no-prompt - - - name: Build & install inspector dependency from source - working-directory: cli/azd/extensions/azure.ai.inspector - run: | - # Build azure.ai.inspector from source to ensure PR CI always has a - # compatible version — dependency releases are post-merge, but PRs may - # include inspector changes from this same repo. - pwsh -File ci-build.ps1 -OutputFileName azure-ai-inspector-linux-amd64 -Version "$(cat version.txt | tr -d '\r\n')" - mkdir -p bin - mv azure-ai-inspector-linux-amd64 bin/ - azd x pack - azd x publish - azd extension install azure.ai.inspector --source local --force --no-prompt - - - name: Build extension (ci-build.ps1) - working-directory: cli/azd/extensions/azure.ai.agents - run: | - # Use the canonical ci-build.ps1 which includes security flags - # (trimpath, pie, cfi, cfg, osusergo) matching release builds. - # -BuildRecordMode produces both a production and a record-tagged binary. - pwsh -File ci-build.ps1 -BuildRecordMode -OutputFileName azure-ai-agents-linux-amd64 -Version "$(cat version.txt | tr -d '\r\n')" - # Move production binary to bin/ where azd x pack expects it. - mkdir -p bin - mv azure-ai-agents-linux-amd64 bin/ - - - name: Pack, publish & install extension (production) - working-directory: cli/azd/extensions/azure.ai.agents - run: | - azd x pack - azd x publish - azd extension install azure.ai.agents --source local --force --no-prompt - - name: Validate tier test selectors working-directory: cli/azd shell: bash @@ -139,6 +93,52 @@ jobs: echo "Tier 1 tests:" echo "$tier_1_tests" + - name: Build azd + working-directory: cli/azd + run: | + go build -tags=record -o azd-record ./ + # Make the record-tagged binary available as "azd" on PATH. + # Required by extension sub-processes (azd x publish) and by + # azidentity.AzureDeveloperCLICredential (exec.LookPath("azd")) in playback. + sudo ln -sf "$GITHUB_WORKSPACE/cli/azd/azd-record" /usr/local/bin/azd + + - name: Install extension toolchain + run: | + # Install the developer extension (provides 'azd x' commands). + # The built-in 'azd' source is auto-registered, so no source-add needed. + azd extension install microsoft.azd.extensions --no-prompt + + - name: Build & install inspector dependency from source + working-directory: cli/azd/extensions/azure.ai.inspector + run: | + # Build azure.ai.inspector from source to ensure PR CI always has a + # compatible version — dependency releases are post-merge, but PRs may + # include inspector changes from this same repo. + pwsh -File ci-build.ps1 -OutputFileName azure-ai-inspector-linux-amd64 -Version "$(cat version.txt | tr -d '\r\n')" + mkdir -p bin + mv azure-ai-inspector-linux-amd64 bin/ + azd x pack + azd x publish + azd extension install azure.ai.inspector --source local --force --no-prompt + + - name: Build extension (ci-build.ps1) + working-directory: cli/azd/extensions/azure.ai.agents + run: | + # Use the canonical ci-build.ps1 which includes security flags + # (trimpath, pie, cfi, cfg, osusergo) matching release builds. + # -BuildRecordMode produces both a production and a record-tagged binary. + pwsh -File ci-build.ps1 -BuildRecordMode -OutputFileName azure-ai-agents-linux-amd64 -Version "$(cat version.txt | tr -d '\r\n')" + # Move production binary to bin/ where azd x pack expects it. + mkdir -p bin + mv azure-ai-agents-linux-amd64 bin/ + + - name: Pack, publish & install extension (production) + working-directory: cli/azd/extensions/azure.ai.agents + run: | + azd x pack + azd x publish + azd extension install azure.ai.agents --source local --force --no-prompt + - name: Run Tier 0 tests (offline, production extension) working-directory: cli/azd env: