From f609bfa6e671b54a0d5c643f62d5feb59f11a82b Mon Sep 17 00:00:00 2001 From: hemarina Date: Fri, 10 Jul 2026 14:44:42 -0700 Subject: [PATCH 01/12] feat(telemetry): emit infra.provider on provision, up, and down Record the resolved IaC provider (bicep/terraform/...) as a usage attribute in Manager.newProvider so it flows onto the provision/up/down command spans via the existing usage-attribute mechanism. Reuses the existing InfraProviderKey (no new field); updates the metrics-audit docs and telemetry_test comments accordingly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/telemetry_test.go | 6 +++--- cli/azd/pkg/infra/provisioning/manager.go | 6 ++++++ docs/reference/telemetry-data.md | 2 +- docs/specs/metrics-audit/feature-telemetry-matrix.md | 6 +++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/cli/azd/cmd/telemetry_test.go b/cli/azd/cmd/telemetry_test.go index 188b7cc2450..0c98d970136 100644 --- a/cli/azd/cmd/telemetry_test.go +++ b/cli/azd/cmd/telemetry_test.go @@ -199,7 +199,7 @@ func TestCommandTelemetryCoverage(t *testing.T) { "auth login", // auth.method "build", // (via hooks middleware) "deploy", // infra.provider, service attributes (via hooks middleware) - "down", // infra.provider (via hooks middleware) + "down", // infra.provider (resolved provider, via provisioning manager) "env list", // env.count "extension install", // extension.source.kind "extension list", // extension.source.kind @@ -210,14 +210,14 @@ func TestCommandTelemetryCoverage(t *testing.T) { "init", // init.method, appinit.* fields "package", // (via hooks middleware) "pipeline config", // pipeline.provider, pipeline.auth - "provision", // infra.provider (via hooks middleware) + "provision", // infra.provider (resolved provider, via provisioning manager) "restore", // (via hooks middleware) "tool check", // tool.check.updates_available "tool install", // tool.id(s), tool.dry_run, tool.install.* aggregate + per-tool fields "tool show", // tool.id "tool uninstall", // tool.id(s), tool.dry_run, tool.install.* aggregate + per-tool fields "tool upgrade", // tool.id(s), tool.dry_run, tool.install.* aggregate + tool.upgrade.* versions - "up", // infra.provider (via hooks middleware, composes provision+deploy) + "up", // infra.provider (via provisioning manager; composes provision+deploy) "update", // update.* fields } diff --git a/cli/azd/pkg/infra/provisioning/manager.go b/cli/azd/pkg/infra/provisioning/manager.go index 5d9b81caf0c..cdf13aa09e5 100644 --- a/cli/azd/pkg/infra/provisioning/manager.go +++ b/cli/azd/pkg/infra/provisioning/manager.go @@ -11,6 +11,8 @@ import ( "os" "path/filepath" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/alpha" "github.com/azure/azure-dev/cli/azd/pkg/azapi" "github.com/azure/azure-dev/cli/azd/pkg/azsdk/storage" @@ -530,6 +532,10 @@ func (m *Manager) newProvider(ctx context.Context) (Provider, error) { providerKey = defaultProvider } + // Record the resolved IaC provider (bicep / terraform / ...) as a usage attribute so it flows + // onto the host command span (provision / up / down), letting telemetry split runs by provider. + tracing.SetUsageAttributes(fields.InfraProviderKey.String(string(providerKey))) + var provider Provider err = m.serviceLocator.ResolveNamed(string(providerKey), &provider) if err != nil { diff --git a/docs/reference/telemetry-data.md b/docs/reference/telemetry-data.md index 3ce70fc5f4c..e3654654fff 100644 --- a/docs/reference/telemetry-data.md +++ b/docs/reference/telemetry-data.md @@ -678,7 +678,7 @@ How to find telemetry for a given feature area. Start here if you know the featu | **Container Apps (Aspire)** | `cmd.deploy`, `cmd.provision` | `project.service.targets` = `containerapp-dotnet`, `platform.type` = `aca` | Aspire-specific adoption and success | | **Language Support** | `cmd.deploy`, `cmd.package`, `cmd.restore` | `project.service.languages`, `project.service.language` | Usage by language | | **Templates** | `cmd.init`, `cmd.up` | `project.template.id` (hashed — join with template lookup to resolve) | Template adoption, success by template | -| **Provisioning (IaC)** | `cmd.provision`, `arm.deploy.*`, `arm.validate.*` | `infra.provider` (`bicep`, `terraform`) | Provision success, ARM errors, duration | +| **Provisioning (IaC)** | `cmd.provision`, `cmd.up`, `cmd.down`, `arm.deploy.*`, `arm.validate.*` | `infra.provider` (`bicep`, `terraform`) | Provision success, ARM errors, duration | | **Authentication** | `cmd.auth.login` | `auth.method` | Auth method usage, failure rates | | **CI/CD Pipelines** | `cmd.pipeline.config` | `pipeline.provider` | Pipeline setup adoption | | **Extensions** | `ext.run`, `ext.install`, `ext.upgrade` | `extension.id`, `extension.version`, `extension.installed` | Extension adoption, errors | diff --git a/docs/specs/metrics-audit/feature-telemetry-matrix.md b/docs/specs/metrics-audit/feature-telemetry-matrix.md index cf5930e3423..a07e8840c73 100644 --- a/docs/specs/metrics-audit/feature-telemetry-matrix.md +++ b/docs/specs/metrics-audit/feature-telemetry-matrix.md @@ -72,12 +72,12 @@ These commands emit attributes or events beyond the global middleware span. | **Core Lifecycle** | | | | | | | `restore` | — | ✅ | ❌ | ❌ | Via hooks middleware | | `build` | — | ✅ | ❌ | ❌ | Via hooks middleware | -| `provision` | — | ✅ | ✅ | ✅ | Emits `validation.preflight`, 8 `arm.*` events, `aks.postprovision.skip`, and per-layer `provision.layer.*` counts (`count`, `max_parallel`, `safe_fallback_count`, `explicit_dependson_count`) for multi-layer infra | +| `provision` | — | ✅ | ✅ | ✅ | Emits `validation.preflight`, 8 `arm.*` events, `aks.postprovision.skip`, and per-layer `provision.layer.*` counts (`count`, `max_parallel`, `safe_fallback_count`, `explicit_dependson_count`) for multi-layer infra; sets `infra.provider` (resolved IaC provider) as a usage attribute | | `package` | — | ✅ | ✅ | ✅ | Via hooks middleware; container service targets emit `container.credentials`, `container.publish`, `container.remotebuild` events | | `deploy` | — | ✅ | ✅ | ✅ | App Service zip-deploy emits `deploy.appservice.zip` (`deploy.appservice.linux`, `deploy.appservice.attempt`); container service targets emit `container.*` events | | `publish` | — | ✅ | ✅ | ✅ | Same as `deploy` (alias behavior) | | `up` | — | ✅ | ✅ | ✅ | Composes provision+deploy and inherits all their events; the up-graph runner emits `perf.provision_duration_ms`, `perf.deploy_duration_ms`, `perf.total_duration_ms` (`internal/cmd/up_graph.go`) | -| `down` | — | ✅ | ❌ | ❌ | Teardown flow; pre/postdown lifecycle hooks emit `hooks.exec` via the hooks middleware | +| `down` | — | ✅ | ❌ | ❌ | Teardown flow; pre/postdown lifecycle hooks emit `hooks.exec` via the hooks middleware; sets `infra.provider` (resolved IaC provider) as a usage attribute | | **Add** | | | | | | | `add` | — | ✅ | ❌ | ❌ | Low priority | | **Completion** | | | | | | @@ -122,7 +122,7 @@ command-specific telemetry fields provide analytical value beyond the command na | Hooks kind | `hooks.kind` | `hooks run` | Distinguishes the script runtime used to execute the hook (`sh`, `pwsh`, `js`, `ts`, `python`, `dotnet`) | | Pipeline provider | `pipeline.provider` | `pipeline config` | Distinguishes GitHub vs Azure DevOps | | Pipeline auth | `pipeline.auth` | `pipeline config` | Distinguishes federated vs client-credentials | -| Infra provider | `infra.provider` | `infra generate`, `infra synth` | Distinguishes Bicep vs Terraform | +| Infra provider | `infra.provider` | `infra generate`, `infra synth`, `provision`, `up`, `down` | Distinguishes Bicep vs Terraform | | Tool ID | `tool.id` / `tool.ids` | `tool *` | Identifies which managed tool (e.g., bicep, gh, kubectl) the command acted on | | Tool install metrics | `tool.install.*` | `tool install`, `tool upgrade`, `tool uninstall`, first-run middleware | Success count, failure count, duration, strategy — quantitative install health | | Tool upgrade versions | `tool.upgrade.from_version`, `tool.upgrade.to_version` | `tool upgrade` | Tracks adoption of new tool versions | From c915e6e43217ac219191a2b686746e0b34ca7f80 Mon Sep 17 00:00:00 2001 From: hemarina Date: Fri, 10 Jul 2026 16:08:36 -0700 Subject: [PATCH 02/12] address feedback --- cli/azd/cmd/down.go | 6 + cli/azd/cmd/down_test.go | 2 +- cli/azd/internal/cmd/provision_graph.go | 4 + cli/azd/internal/cmd/up_graph.go | 3 + cli/azd/pkg/infra/provisioning/manager.go | 55 ++++++++- .../provisioning/manager_telemetry_test.go | 110 ++++++++++++++++++ docs/reference/telemetry-data.md | 4 +- .../metrics-audit/feature-telemetry-matrix.md | 2 +- docs/specs/metrics-audit/telemetry-schema.md | 2 +- 9 files changed, 179 insertions(+), 9 deletions(-) create mode 100644 cli/azd/pkg/infra/provisioning/manager_telemetry_test.go diff --git a/cli/azd/cmd/down.go b/cli/azd/cmd/down.go index 27c854fca80..94274a11de7 100644 --- a/cli/azd/cmd/down.go +++ b/cli/azd/cmd/down.go @@ -67,6 +67,7 @@ type downAction struct { flags *downFlags args []string provisionManager *provisioning.Manager + defaultProvider provisioning.DefaultProviderResolver importManager *project.ImportManager env *environment.Environment envManager environment.Manager @@ -79,6 +80,7 @@ func newDownAction( args []string, flags *downFlags, provisionManager *provisioning.Manager, + defaultProvider provisioning.DefaultProviderResolver, env *environment.Environment, envManager environment.Manager, projectConfig *project.ProjectConfig, @@ -89,6 +91,7 @@ func newDownAction( return &downAction{ flags: flags, provisionManager: provisionManager, + defaultProvider: defaultProvider, env: env, envManager: envManager, console: console, @@ -154,6 +157,9 @@ func (a *downAction) Run(ctx context.Context) (*actions.ActionResult, error) { } } + // Record the resolved IaC provider(s) now that teardown succeeded (no-op when there are no layers). + provisioning.RecordInfraProviderUsage(layers, a.defaultProvider) + // Invalidate cache after successful down so azd show will refresh if err := a.envManager.InvalidateEnvCache(ctx, a.env.Name()); err != nil { log.Printf("warning: failed to invalidate state cache: %v", err) diff --git a/cli/azd/cmd/down_test.go b/cli/azd/cmd/down_test.go index 986b4d51246..36eff8d66e0 100644 --- a/cli/azd/cmd/down_test.go +++ b/cli/azd/cmd/down_test.go @@ -18,7 +18,7 @@ func Test_NewDownAction_Constructor(t *testing.T) { t.Parallel() flags := &downFlags{} console := mockinput.NewMockConsole() - a := newDownAction(nil, flags, nil, nil, nil, nil, console, nil, nil) + a := newDownAction(nil, flags, nil, nil, nil, nil, nil, console, nil, nil) da := a.(*downAction) require.Same(t, flags, da.flags) } diff --git a/cli/azd/internal/cmd/provision_graph.go b/cli/azd/internal/cmd/provision_graph.go index 7d0c998d46a..da8c98fb098 100644 --- a/cli/azd/internal/cmd/provision_graph.go +++ b/cli/azd/internal/cmd/provision_graph.go @@ -306,6 +306,10 @@ func (p *ProvisionAction) provisionLayersGraph( return nil, p.wrapProvisionError(ctx, unwrapStepErrors(result)) } + // Record the resolved IaC provider (bicep / terraform / ... or "mixed") on the command + // span now that provisioning succeeded, so it reflects only providers that resolved. + provisioning.RecordInfraProviderUsage(layers, p.defaultProvider) + // ── shared finalization ────────────────────────────────────────────── if allSkipped { return &actions.ActionResult{ diff --git a/cli/azd/internal/cmd/up_graph.go b/cli/azd/internal/cmd/up_graph.go index 76f5c6946d4..72b61d67205 100644 --- a/cli/azd/internal/cmd/up_graph.go +++ b/cli/azd/internal/cmd/up_graph.go @@ -637,6 +637,9 @@ func (u *UpGraphAction) Run( tracing.SetUsageAttributes(fields.PerfDeployDurationMs.Int64(deployDur.Milliseconds())) } + // Record the resolved IaC provider(s) now that up succeeded (no-op when there are no layers). + provisioning.RecordInfraProviderUsage(layers, u.defaultProvider) + return &actions.ActionResult{ Message: &actions.ResultMessage{ Header: fmt.Sprintf( diff --git a/cli/azd/pkg/infra/provisioning/manager.go b/cli/azd/pkg/infra/provisioning/manager.go index cdf13aa09e5..0b63f879877 100644 --- a/cli/azd/pkg/infra/provisioning/manager.go +++ b/cli/azd/pkg/infra/provisioning/manager.go @@ -532,10 +532,6 @@ func (m *Manager) newProvider(ctx context.Context) (Provider, error) { providerKey = defaultProvider } - // Record the resolved IaC provider (bicep / terraform / ...) as a usage attribute so it flows - // onto the host command span (provision / up / down), letting telemetry split runs by provider. - tracing.SetUsageAttributes(fields.InfraProviderKey.String(string(providerKey))) - var provider Provider err = m.serviceLocator.ResolveNamed(string(providerKey), &provider) if err != nil { @@ -544,3 +540,54 @@ func (m *Manager) newProvider(ctx context.Context) (Provider, error) { return provider, nil } + +// InfraProviderMixed is the infra.provider telemetry value recorded when a project's +// provisioning layers do not all resolve to the same IaC provider. +const InfraProviderMixed = "mixed" + +// RecordInfraProviderUsage records the resolved IaC provider for a provisioning command +// (provision / up / down) as the infra.provider usage attribute on the ambient command span. +// When every layer resolves to the same provider it records that provider (for example +// "bicep", "terraform", "arm"); when layers resolve to different providers it records +// InfraProviderMixed ("mixed"). Layers that leave the provider unspecified resolve through +// defaultProvider. It is a no-op when no provider can be resolved. +// +// Callers must invoke this once per command, after provisioning has succeeded, so that only +// providers that actually resolved are recorded and the value is computed deterministically +// (rather than racing concurrent per-layer resolution). +func RecordInfraProviderUsage(layers []Options, defaultProvider DefaultProviderResolver) { + seen := map[ProviderKind]struct{}{} + for _, layer := range layers { + kind := layer.Provider + if kind == NotSpecified { + if defaultProvider == nil { + continue + } + + resolved, err := defaultProvider() + if err != nil { + continue + } + + kind = resolved + } + + parsed, err := ParseProvider(kind) + if err != nil { + continue + } + + seen[parsed] = struct{}{} + } + + switch len(seen) { + case 0: + return + case 1: + for kind := range seen { + tracing.SetUsageAttributes(fields.InfraProviderKey.String(string(kind))) + } + default: + tracing.SetUsageAttributes(fields.InfraProviderKey.String(InfraProviderMixed)) + } +} diff --git a/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go b/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go new file mode 100644 index 00000000000..89ff9386a25 --- /dev/null +++ b/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package provisioning_test + +import ( + "errors" + "testing" + + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" + "github.com/azure/azure-dev/cli/azd/pkg/infra/provisioning" + "github.com/stretchr/testify/require" +) + +func TestRecordInfraProviderUsage(t *testing.T) { + bicepDefault := func() (provisioning.ProviderKind, error) { return provisioning.Bicep, nil } + failingResolver := func() (provisioning.ProviderKind, error) { + return provisioning.NotSpecified, errors.New("no default provider") + } + + tests := []struct { + name string + layers []provisioning.Options + defaultProvider provisioning.DefaultProviderResolver + expected string // "" means no infra.provider attribute is recorded + }{ + { + name: "single explicit bicep", + layers: []provisioning.Options{{Provider: provisioning.Bicep}}, + defaultProvider: bicepDefault, + expected: "bicep", + }, + { + name: "single explicit terraform", + layers: []provisioning.Options{{Provider: provisioning.Terraform}}, + defaultProvider: bicepDefault, + expected: "terraform", + }, + { + name: "unspecified resolves through default", + layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, + defaultProvider: bicepDefault, + expected: "bicep", + }, + { + name: "uniform provider across layers", + layers: []provisioning.Options{ + {Provider: provisioning.Bicep}, + {Provider: provisioning.Bicep}, + }, + defaultProvider: bicepDefault, + expected: "bicep", + }, + { + name: "different providers across layers records mixed", + layers: []provisioning.Options{ + {Provider: provisioning.Bicep}, + {Provider: provisioning.Terraform}, + }, + defaultProvider: bicepDefault, + expected: provisioning.InfraProviderMixed, + }, + { + name: "no layers records nothing", + layers: nil, + defaultProvider: bicepDefault, + expected: "", + }, + { + name: "unspecified with nil resolver records nothing", + layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, + defaultProvider: nil, + expected: "", + }, + { + name: "unspecified with failing resolver records nothing", + layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, + defaultProvider: failingResolver, + expected: "", + }, + } + + // Usage attributes are process-global, so these subtests must not run in parallel. + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tracing.ResetUsageAttributesForTest() + t.Cleanup(tracing.ResetUsageAttributesForTest) + + provisioning.RecordInfraProviderUsage(tt.layers, tt.defaultProvider) + + var got string + var found bool + for _, attr := range tracing.GetUsageAttributes() { + if attr.Key == fields.InfraProviderKey.Key { + got = attr.Value.AsString() + found = true + } + } + + if tt.expected == "" { + require.False(t, found, "expected no infra.provider attribute, got %q", got) + return + } + + require.True(t, found, "expected infra.provider attribute to be recorded") + require.Equal(t, tt.expected, got) + }) + } +} diff --git a/docs/reference/telemetry-data.md b/docs/reference/telemetry-data.md index e3654654fff..b3638d5da74 100644 --- a/docs/reference/telemetry-data.md +++ b/docs/reference/telemetry-data.md @@ -345,7 +345,7 @@ Set **only when an external command-line tool invocation fails**, during error c | Field Key | Type | Description | |-----------|------|-------------| -| `infra.provider` | string | `bicep`, `terraform`, `auto` | +| `infra.provider` | string | `bicep`, `terraform`, `arm`, `pulumi`, custom provider name, `mixed` (layers differ), or `auto` (generate/synth) |
@@ -678,7 +678,7 @@ How to find telemetry for a given feature area. Start here if you know the featu | **Container Apps (Aspire)** | `cmd.deploy`, `cmd.provision` | `project.service.targets` = `containerapp-dotnet`, `platform.type` = `aca` | Aspire-specific adoption and success | | **Language Support** | `cmd.deploy`, `cmd.package`, `cmd.restore` | `project.service.languages`, `project.service.language` | Usage by language | | **Templates** | `cmd.init`, `cmd.up` | `project.template.id` (hashed — join with template lookup to resolve) | Template adoption, success by template | -| **Provisioning (IaC)** | `cmd.provision`, `cmd.up`, `cmd.down`, `arm.deploy.*`, `arm.validate.*` | `infra.provider` (`bicep`, `terraform`) | Provision success, ARM errors, duration | +| **Provisioning (IaC)** | `cmd.provision`, `cmd.up`, `cmd.down`, `arm.deploy.*`, `arm.validate.*` | `infra.provider` (`bicep`/`terraform`/`arm`/`pulumi`/custom, or `mixed`) | Provision success, ARM errors, duration | | **Authentication** | `cmd.auth.login` | `auth.method` | Auth method usage, failure rates | | **CI/CD Pipelines** | `cmd.pipeline.config` | `pipeline.provider` | Pipeline setup adoption | | **Extensions** | `ext.run`, `ext.install`, `ext.upgrade` | `extension.id`, `extension.version`, `extension.installed` | Extension adoption, errors | diff --git a/docs/specs/metrics-audit/feature-telemetry-matrix.md b/docs/specs/metrics-audit/feature-telemetry-matrix.md index a07e8840c73..10346349a41 100644 --- a/docs/specs/metrics-audit/feature-telemetry-matrix.md +++ b/docs/specs/metrics-audit/feature-telemetry-matrix.md @@ -122,7 +122,7 @@ command-specific telemetry fields provide analytical value beyond the command na | Hooks kind | `hooks.kind` | `hooks run` | Distinguishes the script runtime used to execute the hook (`sh`, `pwsh`, `js`, `ts`, `python`, `dotnet`) | | Pipeline provider | `pipeline.provider` | `pipeline config` | Distinguishes GitHub vs Azure DevOps | | Pipeline auth | `pipeline.auth` | `pipeline config` | Distinguishes federated vs client-credentials | -| Infra provider | `infra.provider` | `infra generate`, `infra synth`, `provision`, `up`, `down` | Distinguishes Bicep vs Terraform | +| Infra provider | `infra.provider` | `infra generate`, `infra synth`, `provision`, `up`, `down` | Resolved IaC provider (`bicep`/`terraform`/`arm`/`pulumi`/custom); `mixed` when a project's layers differ; `auto` on generate/synth before resolution | | Tool ID | `tool.id` / `tool.ids` | `tool *` | Identifies which managed tool (e.g., bicep, gh, kubectl) the command acted on | | Tool install metrics | `tool.install.*` | `tool install`, `tool upgrade`, `tool uninstall`, first-run middleware | Success count, failure count, duration, strategy — quantitative install health | | Tool upgrade versions | `tool.upgrade.from_version`, `tool.upgrade.to_version` | `tool upgrade` | Tracks adoption of new tool versions | diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md index 13226b0e419..c20cb76b9d6 100644 --- a/docs/specs/metrics-audit/telemetry-schema.md +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -274,7 +274,7 @@ The following fields are defined in `fields.go`. | Hooks kind | `hooks.kind` | SystemMetadata | FeatureInsight | Executor used to run the hook. Values: `sh`, `pwsh`, `python`, `js`, `ts`, `dotnet` | | Pipeline provider | `pipeline.provider` | SystemMetadata | FeatureInsight | Resolved provider display name after auto-detection: `GitHub`, `Azure DevOps` | | Pipeline auth | `pipeline.auth` | SystemMetadata | FeatureInsight | Emitted only when `--auth-type` is set on `pipeline config`: `federated`, `client-credentials` | -| Infra provider | `infra.provider` | SystemMetadata | FeatureInsight | `bicep`, `terraform`, `auto` (auto-detected from files) | +| Infra provider | `infra.provider` | SystemMetadata | FeatureInsight | Resolved IaC provider `bicep`, `terraform`, `arm`, `pulumi`, or a custom/extension provider name; `mixed` when a project's layers differ (provision/up/down); `auto` on `infra generate`/`synth` before resolution | ### App Service Deploy From 4c183f3306c671800073dd2bd90e34984dfb4fad Mon Sep 17 00:00:00 2001 From: hemarina Date: Fri, 10 Jul 2026 16:16:53 -0700 Subject: [PATCH 03/12] chore(spelling): add 'pulumi' to cspell dictionary for telemetry docs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .vscode/cspell.misc.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.vscode/cspell.misc.yaml b/.vscode/cspell.misc.yaml index 4517fcbdac0..81e6e4f27aa 100644 --- a/.vscode/cspell.misc.yaml +++ b/.vscode/cspell.misc.yaml @@ -44,6 +44,7 @@ overrides: - dependson - exegraph - pseudonymizing + - pulumi - filename: ./README.md words: - VSIX @@ -133,6 +134,7 @@ overrides: - preprovision - postdeploy - pseudonymized + - pulumi - remotebuild - resourcegroup - springapp From 36fe1580fbc5a388f2fb19660c6cfef08e4c0fac Mon Sep 17 00:00:00 2001 From: hemarina Date: Fri, 10 Jul 2026 16:41:11 -0700 Subject: [PATCH 04/12] address review: record infra.provider early to cover failure/preview and keep it off cmd.package span Emit infra.provider from the provision/up/down actions before provider work (instead of only after success), so failed and preview runs are segmentable by provider. Filter the key off up's synthetic cmd.package span, and add a unit test for that filter. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/down.go | 7 ++++--- cli/azd/internal/cmd/provision_graph.go | 8 ++++---- cli/azd/internal/cmd/up_graph.go | 25 +++++++++++++++++++---- cli/azd/internal/cmd/up_graph_test.go | 20 ++++++++++++++++++ cli/azd/pkg/infra/provisioning/manager.go | 6 +++--- 5 files changed, 52 insertions(+), 14 deletions(-) diff --git a/cli/azd/cmd/down.go b/cli/azd/cmd/down.go index 94274a11de7..685e1af387d 100644 --- a/cli/azd/cmd/down.go +++ b/cli/azd/cmd/down.go @@ -136,6 +136,10 @@ func (a *downAction) Run(ctx context.Context) (*actions.ActionResult, error) { } slices.Reverse(layers) + // Record the resolved IaC provider (single, or "mixed") on the command span up front, so it is + // present on success and failure alike (no-op when there are no layers). + provisioning.RecordInfraProviderUsage(layers, a.defaultProvider) + for _, layer := range layers { if downLayer != "" || len(layers) > 1 { a.console.EnsureBlankLine(ctx) @@ -157,9 +161,6 @@ func (a *downAction) Run(ctx context.Context) (*actions.ActionResult, error) { } } - // Record the resolved IaC provider(s) now that teardown succeeded (no-op when there are no layers). - provisioning.RecordInfraProviderUsage(layers, a.defaultProvider) - // Invalidate cache after successful down so azd show will refresh if err := a.envManager.InvalidateEnvCache(ctx, a.env.Name()); err != nil { log.Printf("warning: failed to invalidate state cache: %v", err) diff --git a/cli/azd/internal/cmd/provision_graph.go b/cli/azd/internal/cmd/provision_graph.go index da8c98fb098..7596b324392 100644 --- a/cli/azd/internal/cmd/provision_graph.go +++ b/cli/azd/internal/cmd/provision_graph.go @@ -84,6 +84,10 @@ func (p *ProvisionAction) provisionLayersGraph( }, nil } + // Record the resolved IaC provider (single, or "mixed") on the command span up front, so it is + // present on success, failure, and preview runs alike — scoped to the provisioning lifecycle. + provisioning.RecordInfraProviderUsage(layers, p.defaultProvider) + // ── provider-agnostic provision validation (once per command) ───────── // Dispatch extension-registered "provision" checks a single time, before // the preview / single-layer / multi-layer paths run. This deliberately @@ -306,10 +310,6 @@ func (p *ProvisionAction) provisionLayersGraph( return nil, p.wrapProvisionError(ctx, unwrapStepErrors(result)) } - // Record the resolved IaC provider (bicep / terraform / ... or "mixed") on the command - // span now that provisioning succeeded, so it reflects only providers that resolved. - provisioning.RecordInfraProviderUsage(layers, p.defaultProvider) - // ── shared finalization ────────────────────────────────────────────── if allSkipped { return &actions.ActionResult{ diff --git a/cli/azd/internal/cmd/up_graph.go b/cli/azd/internal/cmd/up_graph.go index 72b61d67205..32d4842ece0 100644 --- a/cli/azd/internal/cmd/up_graph.go +++ b/cli/azd/internal/cmd/up_graph.go @@ -34,6 +34,7 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/output/ux" "github.com/azure/azure-dev/cli/azd/pkg/project" "github.com/spf13/pflag" + "go.opentelemetry.io/otel/attribute" ) // UpGraphAction is the single implementation of `azd up`'s default path: it @@ -168,12 +169,18 @@ func (u *UpGraphAction) Run( // any values set during Run. Globals (e.g. SubscriptionIdKey) are // applied automatically by wrapperSpan.End(). usageAttrs := tracing.GetUsageAttributes() - packageSpan.SetAttributes(usageAttrs...) + // infra.provider is scoped to the provisioning lifecycle; keep it off the synthetic + // cmd.package span (it belongs on cmd.provision and the parent cmd.up span only). + packageSpan.SetAttributes(usageAttributesExcluding(usageAttrs, fields.InfraProviderKey.Key)...) packageSpan.End() provisionSpan.SetAttributes(usageAttrs...) provisionSpan.End() }() + // Record the resolved IaC provider (single, or "mixed") on the command span up front, so it is + // present on success and failure alike (no-op when there are no provision layers). + provisioning.RecordInfraProviderUsage(layers, u.defaultProvider) + // 1. Analyze provision layer dependencies. Empty layers → empty graph. var layerDeps *bicep.LayerDependencies if len(layers) > 0 { @@ -637,9 +644,6 @@ func (u *UpGraphAction) Run( tracing.SetUsageAttributes(fields.PerfDeployDurationMs.Int64(deployDur.Milliseconds())) } - // Record the resolved IaC provider(s) now that up succeeded (no-op when there are no layers). - provisioning.RecordInfraProviderUsage(layers, u.defaultProvider) - return &actions.ActionResult{ Message: &actions.ResultMessage{ Header: fmt.Sprintf( @@ -674,6 +678,19 @@ func phaseTimingBreakdown(steps []exegraph.StepTiming) string { // phaseDurations computes the wall-clock duration for provisioning and deploying phases. // Returns zero durations for phases that were not executed. +// usageAttributesExcluding returns usageAttrs without the attribute matching exclude. It keeps +// provisioning-scoped attributes (for example infra.provider) off the synthetic cmd.package span. +func usageAttributesExcluding(usageAttrs []attribute.KeyValue, exclude attribute.Key) []attribute.KeyValue { + filtered := make([]attribute.KeyValue, 0, len(usageAttrs)) + for _, attr := range usageAttrs { + if attr.Key != exclude { + filtered = append(filtered, attr) + } + } + + return filtered +} + func phaseDurations(steps []exegraph.StepTiming) (provision, deploy time.Duration) { var provStart, deployStart time.Time var provEnd, deployEnd time.Time diff --git a/cli/azd/internal/cmd/up_graph_test.go b/cli/azd/internal/cmd/up_graph_test.go index e39cea392fa..cf80db9ceb4 100644 --- a/cli/azd/internal/cmd/up_graph_test.go +++ b/cli/azd/internal/cmd/up_graph_test.go @@ -7,9 +7,29 @@ import ( "testing" "time" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/exegraph" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" ) +func TestUsageAttributesExcluding(t *testing.T) { + t.Parallel() + + attrs := []attribute.KeyValue{ + fields.InfraProviderKey.String("bicep"), + fields.PerfProvisionDurationMs.Int64(42), + } + + got := usageAttributesExcluding(attrs, fields.InfraProviderKey.Key) + + require.Len(t, got, 1) + require.Equal(t, fields.PerfProvisionDurationMs.Key, got[0].Key) + for _, a := range got { + require.NotEqual(t, fields.InfraProviderKey.Key, a.Key, "infra.provider must be excluded from cmd.package span") + } +} + func TestPhaseTimingBreakdown(t *testing.T) { t.Parallel() base := time.Now() diff --git a/cli/azd/pkg/infra/provisioning/manager.go b/cli/azd/pkg/infra/provisioning/manager.go index 0b63f879877..f1e7c7e48f4 100644 --- a/cli/azd/pkg/infra/provisioning/manager.go +++ b/cli/azd/pkg/infra/provisioning/manager.go @@ -552,9 +552,9 @@ const InfraProviderMixed = "mixed" // InfraProviderMixed ("mixed"). Layers that leave the provider unspecified resolve through // defaultProvider. It is a no-op when no provider can be resolved. // -// Callers must invoke this once per command, after provisioning has succeeded, so that only -// providers that actually resolved are recorded and the value is computed deterministically -// (rather than racing concurrent per-layer resolution). +// Callers must invoke this once per command, before provider work begins, so the attribute is +// present on success, failure, and preview spans alike. The value is computed deterministically +// from configuration (rather than racing concurrent per-layer resolution). func RecordInfraProviderUsage(layers []Options, defaultProvider DefaultProviderResolver) { seen := map[ProviderKind]struct{}{} for _, layer := range layers { From 433d5ea2f46159b9b73ea8dd72572a48ad2207bb Mon Sep 17 00:00:00 2001 From: hemarina Date: Fri, 10 Jul 2026 17:01:23 -0700 Subject: [PATCH 05/12] address review: emit infra.provider on common up path, bucket custom providers, fix doc comment - up: record on upAction.Run before the custom-workflow branch and early init, so cmd.up carries the field on all paths (not just the unified graph). - privacy: emit built-in provider kinds raw, bucket any other (extension) provider to 'custom' so user-chosen names are never sent. - up_graph: remove now-duplicate emit; restore phaseDurations doc comment split by the helper. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/cmd/up.go | 8 ++++ cli/azd/cmd/up_test.go | 2 +- cli/azd/internal/cmd/up_graph.go | 8 +--- cli/azd/pkg/infra/provisioning/manager.go | 41 ++++++++++++++----- .../provisioning/manager_telemetry_test.go | 21 ++++++++++ docs/reference/telemetry-data.md | 2 +- .../metrics-audit/feature-telemetry-matrix.md | 2 +- docs/specs/metrics-audit/telemetry-schema.md | 2 +- 8 files changed, 66 insertions(+), 20 deletions(-) diff --git a/cli/azd/cmd/up.go b/cli/azd/cmd/up.go index ab3345ef02e..a3883798169 100644 --- a/cli/azd/cmd/up.go +++ b/cli/azd/cmd/up.go @@ -70,6 +70,7 @@ type upAction struct { env *environment.Environment projectConfig *project.ProjectConfig provisioningManager *provisioning.Manager + defaultProvider provisioning.DefaultProviderResolver envManager environment.Manager prompters prompt.Prompter importManager *project.ImportManager @@ -83,6 +84,7 @@ func newUpAction( env *environment.Environment, projectConfig *project.ProjectConfig, provisioningManager *provisioning.Manager, + defaultProvider provisioning.DefaultProviderResolver, envManager environment.Manager, prompters prompt.Prompter, importManager *project.ImportManager, @@ -95,6 +97,7 @@ func newUpAction( env: env, projectConfig: projectConfig, provisioningManager: provisioningManager, + defaultProvider: defaultProvider, envManager: envManager, prompters: prompters, importManager: importManager, @@ -142,6 +145,11 @@ func (u *upAction) Run(ctx context.Context) (*actions.ActionResult, error) { } defer func() { _ = infra.Cleanup() }() + // Record the resolved IaC provider (single, or "mixed") on the cmd.up span up front — before + // provider init and the custom-workflow branch — so every up path (including custom workflows + // and early init failures) carries it. + provisioning.RecordInfraProviderUsage(infra.Options.GetLayers(), u.defaultProvider) + // TODO(weilim): remove this once we have decided if it's okay to not set AZURE_SUBSCRIPTION_ID and AZURE_LOCATION // early in the up workflow in #3745 err = u.provisioningManager.Initialize(ctx, u.projectConfig.Path, infra.Options) diff --git a/cli/azd/cmd/up_test.go b/cli/azd/cmd/up_test.go index 065f579f742..bb69cebc2d4 100644 --- a/cli/azd/cmd/up_test.go +++ b/cli/azd/cmd/up_test.go @@ -18,7 +18,7 @@ func Test_NewUpAction_Constructor(t *testing.T) { t.Parallel() flags := &upFlags{} console := mockinput.NewMockConsole() - a := newUpAction(flags, console, nil, nil, nil, nil, nil, nil, nil, nil) + a := newUpAction(flags, console, nil, nil, nil, nil, nil, nil, nil, nil, nil) ua := a.(*upAction) require.Same(t, flags, ua.flags) } diff --git a/cli/azd/internal/cmd/up_graph.go b/cli/azd/internal/cmd/up_graph.go index 32d4842ece0..b4b28a6293e 100644 --- a/cli/azd/internal/cmd/up_graph.go +++ b/cli/azd/internal/cmd/up_graph.go @@ -177,10 +177,6 @@ func (u *UpGraphAction) Run( provisionSpan.End() }() - // Record the resolved IaC provider (single, or "mixed") on the command span up front, so it is - // present on success and failure alike (no-op when there are no provision layers). - provisioning.RecordInfraProviderUsage(layers, u.defaultProvider) - // 1. Analyze provision layer dependencies. Empty layers → empty graph. var layerDeps *bicep.LayerDependencies if len(layers) > 0 { @@ -676,8 +672,6 @@ func phaseTimingBreakdown(steps []exegraph.StepTiming) string { return strings.Join(lines, "\n") } -// phaseDurations computes the wall-clock duration for provisioning and deploying phases. -// Returns zero durations for phases that were not executed. // usageAttributesExcluding returns usageAttrs without the attribute matching exclude. It keeps // provisioning-scoped attributes (for example infra.provider) off the synthetic cmd.package span. func usageAttributesExcluding(usageAttrs []attribute.KeyValue, exclude attribute.Key) []attribute.KeyValue { @@ -691,6 +685,8 @@ func usageAttributesExcluding(usageAttrs []attribute.KeyValue, exclude attribute return filtered } +// phaseDurations computes the wall-clock duration for provisioning and deploying phases. +// Returns zero durations for phases that were not executed. func phaseDurations(steps []exegraph.StepTiming) (provision, deploy time.Duration) { var provStart, deployStart time.Time var provEnd, deployEnd time.Time diff --git a/cli/azd/pkg/infra/provisioning/manager.go b/cli/azd/pkg/infra/provisioning/manager.go index f1e7c7e48f4..4fe45c65b3f 100644 --- a/cli/azd/pkg/infra/provisioning/manager.go +++ b/cli/azd/pkg/infra/provisioning/manager.go @@ -541,22 +541,31 @@ func (m *Manager) newProvider(ctx context.Context) (Provider, error) { return provider, nil } -// InfraProviderMixed is the infra.provider telemetry value recorded when a project's -// provisioning layers do not all resolve to the same IaC provider. -const InfraProviderMixed = "mixed" +const ( + // InfraProviderMixed is the infra.provider telemetry value recorded when a project's + // provisioning layers do not all resolve to the same IaC provider. + InfraProviderMixed = "mixed" + + // InfraProviderCustom is the infra.provider telemetry value recorded for a provider that is + // not one of the built-in kinds (for example an extension-registered provider). The raw name + // is never emitted because it may embed user-chosen project or organization identifiers. + InfraProviderCustom = "custom" +) // RecordInfraProviderUsage records the resolved IaC provider for a provisioning command // (provision / up / down) as the infra.provider usage attribute on the ambient command span. // When every layer resolves to the same provider it records that provider (for example -// "bicep", "terraform", "arm"); when layers resolve to different providers it records -// InfraProviderMixed ("mixed"). Layers that leave the provider unspecified resolve through -// defaultProvider. It is a no-op when no provider can be resolved. +// "bicep", "terraform", "arm"); non-built-in (extension) providers are bucketed to +// InfraProviderCustom ("custom") so raw user-chosen names are never emitted. When layers +// resolve to different providers it records InfraProviderMixed ("mixed"). Layers that leave +// the provider unspecified resolve through defaultProvider. It is a no-op when no provider +// can be resolved. // // Callers must invoke this once per command, before provider work begins, so the attribute is // present on success, failure, and preview spans alike. The value is computed deterministically // from configuration (rather than racing concurrent per-layer resolution). func RecordInfraProviderUsage(layers []Options, defaultProvider DefaultProviderResolver) { - seen := map[ProviderKind]struct{}{} + seen := map[string]struct{}{} for _, layer := range layers { kind := layer.Provider if kind == NotSpecified { @@ -577,17 +586,29 @@ func RecordInfraProviderUsage(layers []Options, defaultProvider DefaultProviderR continue } - seen[parsed] = struct{}{} + seen[infraProviderTelemetryValue(parsed)] = struct{}{} } switch len(seen) { case 0: return case 1: - for kind := range seen { - tracing.SetUsageAttributes(fields.InfraProviderKey.String(string(kind))) + for value := range seen { + tracing.SetUsageAttributes(fields.InfraProviderKey.String(value)) } default: tracing.SetUsageAttributes(fields.InfraProviderKey.String(InfraProviderMixed)) } } + +// infraProviderTelemetryValue maps a provider kind to a value that is safe to emit raw. Built-in +// kinds are returned verbatim; any other (custom / extension-registered) provider is bucketed to +// InfraProviderCustom so a user-chosen provider name is never sent as telemetry. +func infraProviderTelemetryValue(kind ProviderKind) string { + switch kind { + case Bicep, Terraform, Arm, Pulumi, Test: + return string(kind) + default: + return InfraProviderCustom + } +} diff --git a/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go b/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go index 89ff9386a25..a4d7a032dd5 100644 --- a/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go +++ b/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go @@ -61,6 +61,27 @@ func TestRecordInfraProviderUsage(t *testing.T) { defaultProvider: bicepDefault, expected: provisioning.InfraProviderMixed, }, + { + name: "single explicit arm", + layers: []provisioning.Options{{Provider: provisioning.Arm}}, + defaultProvider: bicepDefault, + expected: "arm", + }, + { + name: "custom provider is bucketed", + layers: []provisioning.Options{{Provider: provisioning.ProviderKind("my-extension-provider")}}, + defaultProvider: bicepDefault, + expected: provisioning.InfraProviderCustom, + }, + { + name: "built-in plus custom records mixed", + layers: []provisioning.Options{ + {Provider: provisioning.Bicep}, + {Provider: provisioning.ProviderKind("my-extension-provider")}, + }, + defaultProvider: bicepDefault, + expected: provisioning.InfraProviderMixed, + }, { name: "no layers records nothing", layers: nil, diff --git a/docs/reference/telemetry-data.md b/docs/reference/telemetry-data.md index b3638d5da74..efbadbc9ad6 100644 --- a/docs/reference/telemetry-data.md +++ b/docs/reference/telemetry-data.md @@ -345,7 +345,7 @@ Set **only when an external command-line tool invocation fails**, during error c | Field Key | Type | Description | |-----------|------|-------------| -| `infra.provider` | string | `bicep`, `terraform`, `arm`, `pulumi`, custom provider name, `mixed` (layers differ), or `auto` (generate/synth) | +| `infra.provider` | string | `bicep`, `terraform`, `arm`, `pulumi`, `custom` (any other/extension provider; raw name not emitted), `mixed` (layers differ), or `auto` (generate/synth) |
diff --git a/docs/specs/metrics-audit/feature-telemetry-matrix.md b/docs/specs/metrics-audit/feature-telemetry-matrix.md index 10346349a41..8c9e1be704f 100644 --- a/docs/specs/metrics-audit/feature-telemetry-matrix.md +++ b/docs/specs/metrics-audit/feature-telemetry-matrix.md @@ -122,7 +122,7 @@ command-specific telemetry fields provide analytical value beyond the command na | Hooks kind | `hooks.kind` | `hooks run` | Distinguishes the script runtime used to execute the hook (`sh`, `pwsh`, `js`, `ts`, `python`, `dotnet`) | | Pipeline provider | `pipeline.provider` | `pipeline config` | Distinguishes GitHub vs Azure DevOps | | Pipeline auth | `pipeline.auth` | `pipeline config` | Distinguishes federated vs client-credentials | -| Infra provider | `infra.provider` | `infra generate`, `infra synth`, `provision`, `up`, `down` | Resolved IaC provider (`bicep`/`terraform`/`arm`/`pulumi`/custom); `mixed` when a project's layers differ; `auto` on generate/synth before resolution | +| Infra provider | `infra.provider` | `infra generate`, `infra synth`, `provision`, `up`, `down` | Resolved IaC provider (`bicep`/`terraform`/`arm`/`pulumi`, or `custom` for extension providers — raw name not emitted); `mixed` when layers differ; `auto` on generate/synth before resolution | | Tool ID | `tool.id` / `tool.ids` | `tool *` | Identifies which managed tool (e.g., bicep, gh, kubectl) the command acted on | | Tool install metrics | `tool.install.*` | `tool install`, `tool upgrade`, `tool uninstall`, first-run middleware | Success count, failure count, duration, strategy — quantitative install health | | Tool upgrade versions | `tool.upgrade.from_version`, `tool.upgrade.to_version` | `tool upgrade` | Tracks adoption of new tool versions | diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md index c20cb76b9d6..da81456d43a 100644 --- a/docs/specs/metrics-audit/telemetry-schema.md +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -274,7 +274,7 @@ The following fields are defined in `fields.go`. | Hooks kind | `hooks.kind` | SystemMetadata | FeatureInsight | Executor used to run the hook. Values: `sh`, `pwsh`, `python`, `js`, `ts`, `dotnet` | | Pipeline provider | `pipeline.provider` | SystemMetadata | FeatureInsight | Resolved provider display name after auto-detection: `GitHub`, `Azure DevOps` | | Pipeline auth | `pipeline.auth` | SystemMetadata | FeatureInsight | Emitted only when `--auth-type` is set on `pipeline config`: `federated`, `client-credentials` | -| Infra provider | `infra.provider` | SystemMetadata | FeatureInsight | Resolved IaC provider `bicep`, `terraform`, `arm`, `pulumi`, or a custom/extension provider name; `mixed` when a project's layers differ (provision/up/down); `auto` on `infra generate`/`synth` before resolution | +| Infra provider | `infra.provider` | SystemMetadata | FeatureInsight | Resolved IaC provider `bicep`, `terraform`, `arm`, `pulumi`, or `custom` for any other (extension) provider — the raw name is never emitted; `mixed` when a project's layers differ (provision/up/down); `auto` on `infra generate`/`synth` before resolution | ### App Service Deploy From 55c4ec2764e0c42937cd78cf5c5f32df52a68e50 Mon Sep 17 00:00:00 2001 From: hemarina Date: Fri, 10 Jul 2026 17:34:27 -0700 Subject: [PATCH 06/12] address review: count distinct providers before bucketing, exclude Test, scope docs, refresh field doc - Dedup resolved ProviderKind before bucketing so two different custom providers record 'mixed'. - Drop Test from the raw allowlist (buckets to 'custom') to match the documented enum. - Scope the 'raw name not emitted' guarantee to provision/up/down; note infra generate/synth emit the configured provider directly. - Refresh the InfraProviderKey doc comment in fields.go. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cli/azd/internal/tracing/fields/fields.go | 6 ++++-- cli/azd/pkg/infra/provisioning/manager.go | 12 +++++++----- .../pkg/infra/provisioning/manager_telemetry_test.go | 9 +++++++++ docs/reference/telemetry-data.md | 2 +- docs/specs/metrics-audit/feature-telemetry-matrix.md | 2 +- docs/specs/metrics-audit/telemetry-schema.md | 2 +- 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/cli/azd/internal/tracing/fields/fields.go b/cli/azd/internal/tracing/fields/fields.go index 61f985560bb..4da1b5547fe 100644 --- a/cli/azd/internal/tracing/fields/fields.go +++ b/cli/azd/internal/tracing/fields/fields.go @@ -397,9 +397,11 @@ var ( // Infrastructure command related fields var ( - // The IaC provider used for infrastructure generation. + // The IaC provider. Emitted by `infra generate` / `synth` (the configured `--provider` value, + // for example "bicep", "terraform", "auto") and by provision / up / down (the resolved + // provider, "mixed" when layers differ, or "custom" for a non-built-in extension provider). // - // Example: "bicep", "terraform" + // Example: "bicep", "terraform", "arm", "pulumi", "mixed", "custom" InfraProviderKey = AttributeKey{ Key: attribute.Key("infra.provider"), Classification: SystemMetadata, diff --git a/cli/azd/pkg/infra/provisioning/manager.go b/cli/azd/pkg/infra/provisioning/manager.go index 4fe45c65b3f..216d217a885 100644 --- a/cli/azd/pkg/infra/provisioning/manager.go +++ b/cli/azd/pkg/infra/provisioning/manager.go @@ -565,7 +565,7 @@ const ( // present on success, failure, and preview spans alike. The value is computed deterministically // from configuration (rather than racing concurrent per-layer resolution). func RecordInfraProviderUsage(layers []Options, defaultProvider DefaultProviderResolver) { - seen := map[string]struct{}{} + seen := map[ProviderKind]struct{}{} for _, layer := range layers { kind := layer.Provider if kind == NotSpecified { @@ -586,15 +586,17 @@ func RecordInfraProviderUsage(layers []Options, defaultProvider DefaultProviderR continue } - seen[infraProviderTelemetryValue(parsed)] = struct{}{} + seen[parsed] = struct{}{} } + // Distinct resolved kinds are collected first so that two different custom providers still + // record "mixed" (bucketing only the single-provider result, not before counting). switch len(seen) { case 0: return case 1: - for value := range seen { - tracing.SetUsageAttributes(fields.InfraProviderKey.String(value)) + for kind := range seen { + tracing.SetUsageAttributes(fields.InfraProviderKey.String(infraProviderTelemetryValue(kind))) } default: tracing.SetUsageAttributes(fields.InfraProviderKey.String(InfraProviderMixed)) @@ -606,7 +608,7 @@ func RecordInfraProviderUsage(layers []Options, defaultProvider DefaultProviderR // InfraProviderCustom so a user-chosen provider name is never sent as telemetry. func infraProviderTelemetryValue(kind ProviderKind) string { switch kind { - case Bicep, Terraform, Arm, Pulumi, Test: + case Bicep, Terraform, Arm, Pulumi: return string(kind) default: return InfraProviderCustom diff --git a/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go b/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go index a4d7a032dd5..99176d44633 100644 --- a/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go +++ b/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go @@ -82,6 +82,15 @@ func TestRecordInfraProviderUsage(t *testing.T) { defaultProvider: bicepDefault, expected: provisioning.InfraProviderMixed, }, + { + name: "two distinct custom providers record mixed", + layers: []provisioning.Options{ + {Provider: provisioning.ProviderKind("vendor.one")}, + {Provider: provisioning.ProviderKind("vendor.two")}, + }, + defaultProvider: bicepDefault, + expected: provisioning.InfraProviderMixed, + }, { name: "no layers records nothing", layers: nil, diff --git a/docs/reference/telemetry-data.md b/docs/reference/telemetry-data.md index efbadbc9ad6..22e79444228 100644 --- a/docs/reference/telemetry-data.md +++ b/docs/reference/telemetry-data.md @@ -345,7 +345,7 @@ Set **only when an external command-line tool invocation fails**, during error c | Field Key | Type | Description | |-----------|------|-------------| -| `infra.provider` | string | `bicep`, `terraform`, `arm`, `pulumi`, `custom` (any other/extension provider; raw name not emitted), `mixed` (layers differ), or `auto` (generate/synth) | +| `infra.provider` | string | provision/up/down: `bicep`/`terraform`/`arm`/`pulumi`, `custom` (extension providers; raw name not emitted), or `mixed`. generate/synth: configured `--provider` value directly (`bicep`/`terraform`/`auto`, or a raw custom name) |
diff --git a/docs/specs/metrics-audit/feature-telemetry-matrix.md b/docs/specs/metrics-audit/feature-telemetry-matrix.md index 8c9e1be704f..bb45a8c169d 100644 --- a/docs/specs/metrics-audit/feature-telemetry-matrix.md +++ b/docs/specs/metrics-audit/feature-telemetry-matrix.md @@ -122,7 +122,7 @@ command-specific telemetry fields provide analytical value beyond the command na | Hooks kind | `hooks.kind` | `hooks run` | Distinguishes the script runtime used to execute the hook (`sh`, `pwsh`, `js`, `ts`, `python`, `dotnet`) | | Pipeline provider | `pipeline.provider` | `pipeline config` | Distinguishes GitHub vs Azure DevOps | | Pipeline auth | `pipeline.auth` | `pipeline config` | Distinguishes federated vs client-credentials | -| Infra provider | `infra.provider` | `infra generate`, `infra synth`, `provision`, `up`, `down` | Resolved IaC provider (`bicep`/`terraform`/`arm`/`pulumi`, or `custom` for extension providers — raw name not emitted); `mixed` when layers differ; `auto` on generate/synth before resolution | +| Infra provider | `infra.provider` | `infra generate`, `infra synth`, `provision`, `up`, `down` | provision/up/down: resolved `bicep`/`terraform`/`arm`/`pulumi`, `custom` for extension providers (raw name not emitted), or `mixed` when layers differ. `infra generate`/`synth`: the configured `--provider` value emitted directly (`bicep`/`terraform`/`auto`, or a raw custom name) | | Tool ID | `tool.id` / `tool.ids` | `tool *` | Identifies which managed tool (e.g., bicep, gh, kubectl) the command acted on | | Tool install metrics | `tool.install.*` | `tool install`, `tool upgrade`, `tool uninstall`, first-run middleware | Success count, failure count, duration, strategy — quantitative install health | | Tool upgrade versions | `tool.upgrade.from_version`, `tool.upgrade.to_version` | `tool upgrade` | Tracks adoption of new tool versions | diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md index da81456d43a..c0112de7027 100644 --- a/docs/specs/metrics-audit/telemetry-schema.md +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -274,7 +274,7 @@ The following fields are defined in `fields.go`. | Hooks kind | `hooks.kind` | SystemMetadata | FeatureInsight | Executor used to run the hook. Values: `sh`, `pwsh`, `python`, `js`, `ts`, `dotnet` | | Pipeline provider | `pipeline.provider` | SystemMetadata | FeatureInsight | Resolved provider display name after auto-detection: `GitHub`, `Azure DevOps` | | Pipeline auth | `pipeline.auth` | SystemMetadata | FeatureInsight | Emitted only when `--auth-type` is set on `pipeline config`: `federated`, `client-credentials` | -| Infra provider | `infra.provider` | SystemMetadata | FeatureInsight | Resolved IaC provider `bicep`, `terraform`, `arm`, `pulumi`, or `custom` for any other (extension) provider — the raw name is never emitted; `mixed` when a project's layers differ (provision/up/down); `auto` on `infra generate`/`synth` before resolution | +| Infra provider | `infra.provider` | SystemMetadata | FeatureInsight | provision/up/down: resolved `bicep`/`terraform`/`arm`/`pulumi`, `custom` for any other (extension) provider (raw name not emitted), or `mixed` when layers differ. `infra generate`/`synth`: the configured `--provider` value directly (`bicep`/`terraform`/`auto`, or a raw custom name) | ### App Service Deploy From 37f7272c65e41582d979f526e6775411cfc24923 Mon Sep 17 00:00:00 2001 From: hemarina Date: Tue, 14 Jul 2026 14:09:55 -0700 Subject: [PATCH 07/12] address feedback --- cli/azd/pkg/infra/provisioning/manager.go | 5 ++--- cli/azd/pkg/infra/provisioning/manager_telemetry_test.go | 9 +++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/cli/azd/pkg/infra/provisioning/manager.go b/cli/azd/pkg/infra/provisioning/manager.go index 216d217a885..8ec0b270ed9 100644 --- a/cli/azd/pkg/infra/provisioning/manager.go +++ b/cli/azd/pkg/infra/provisioning/manager.go @@ -581,12 +581,11 @@ func RecordInfraProviderUsage(layers []Options, defaultProvider DefaultProviderR kind = resolved } - parsed, err := ParseProvider(kind) - if err != nil { + if kind == NotSpecified { continue } - seen[parsed] = struct{}{} + seen[kind] = struct{}{} } // Distinct resolved kinds are collected first so that two different custom providers still diff --git a/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go b/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go index 99176d44633..0e14186b6ec 100644 --- a/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go +++ b/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go @@ -18,6 +18,9 @@ func TestRecordInfraProviderUsage(t *testing.T) { failingResolver := func() (provisioning.ProviderKind, error) { return provisioning.NotSpecified, errors.New("no default provider") } + unspecifiedResolver := func() (provisioning.ProviderKind, error) { + return provisioning.NotSpecified, nil + } tests := []struct { name string @@ -109,6 +112,12 @@ func TestRecordInfraProviderUsage(t *testing.T) { defaultProvider: failingResolver, expected: "", }, + { + name: "unspecified resolving to NotSpecified records nothing", + layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, + defaultProvider: unspecifiedResolver, + expected: "", + }, } // Usage attributes are process-global, so these subtests must not run in parallel. From 23854136b48ceaaf0d21ed390da031568090b551 Mon Sep 17 00:00:00 2001 From: hemarina Date: Tue, 14 Jul 2026 14:23:11 -0700 Subject: [PATCH 08/12] address copilot feedback --- cli/azd/pkg/infra/provisioning/manager.go | 23 +++++++++++++++---- .../metrics-audit/feature-telemetry-matrix.md | 4 ++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/cli/azd/pkg/infra/provisioning/manager.go b/cli/azd/pkg/infra/provisioning/manager.go index 8ec0b270ed9..6bcf8ba6c7e 100644 --- a/cli/azd/pkg/infra/provisioning/manager.go +++ b/cli/azd/pkg/infra/provisioning/manager.go @@ -565,20 +565,33 @@ const ( // present on success, failure, and preview spans alike. The value is computed deterministically // from configuration (rather than racing concurrent per-layer resolution). func RecordInfraProviderUsage(layers []Options, defaultProvider DefaultProviderResolver) { + // The default provider is resolved lazily and at most once per call: every unspecified layer + // resolves to the same default, so caching keeps the value deterministic and avoids repeating + // resolver work (which may do I/O) per layer. + var cachedDefault ProviderKind + defaultResolved := false + defaultFailed := false + seen := map[ProviderKind]struct{}{} for _, layer := range layers { kind := layer.Provider if kind == NotSpecified { - if defaultProvider == nil { + if defaultProvider == nil || defaultFailed { continue } - resolved, err := defaultProvider() - if err != nil { - continue + if !defaultResolved { + resolved, err := defaultProvider() + if err != nil { + defaultFailed = true + continue + } + + cachedDefault = resolved + defaultResolved = true } - kind = resolved + kind = cachedDefault } if kind == NotSpecified { diff --git a/docs/specs/metrics-audit/feature-telemetry-matrix.md b/docs/specs/metrics-audit/feature-telemetry-matrix.md index bb45a8c169d..9b03d2d95bb 100644 --- a/docs/specs/metrics-audit/feature-telemetry-matrix.md +++ b/docs/specs/metrics-audit/feature-telemetry-matrix.md @@ -65,8 +65,8 @@ These commands emit attributes or events beyond the global middleware span. | **Show** | | | | | | | `show` | — | ✅ | ❌ | ❌ | Redundant — output format not analytically useful | | **Infrastructure** | | | | | | -| `infra generate` | — | ✅ | ✅ | ❌ | `infra.provider` (bicep/terraform) | -| `infra synth` | — | ✅ | ✅ | ❌ | `infra.provider` (bicep/terraform) | +| `infra generate` | — | ✅ | ✅ | ❌ | `infra.provider` (configured `--provider`: `bicep`/`terraform`/`auto`, or a raw custom name — see cross-cutting row) | +| `infra synth` | — | ✅ | ✅ | ❌ | `infra.provider` (configured `--provider`: `bicep`/`terraform`/`auto`, or a raw custom name — see cross-cutting row) | | `infra create` | — (hidden, deprecated) | ✅ | ❌ | ❌ | Wraps `provision`; inherits its telemetry | | `infra delete` | — (hidden, deprecated) | ✅ | ❌ | ❌ | Wraps `down`; inherits its telemetry | | **Core Lifecycle** | | | | | | From ca72fc449303cdd86dbd98e5bed9390ce83f075d Mon Sep 17 00:00:00 2001 From: hemarina Date: Tue, 14 Jul 2026 15:21:13 -0700 Subject: [PATCH 09/12] address feedback --- cli/azd/cmd/down.go | 5 +- cli/azd/cmd/down_test.go | 2 +- cli/azd/cmd/up.go | 5 +- cli/azd/cmd/up_test.go | 2 +- cli/azd/internal/cmd/provision_graph.go | 2 +- cli/azd/pkg/infra/provisioning/manager.go | 10 +-- .../provisioning/manager_telemetry_test.go | 67 +++++++++---------- 7 files changed, 43 insertions(+), 50 deletions(-) diff --git a/cli/azd/cmd/down.go b/cli/azd/cmd/down.go index 685e1af387d..3b6dfa7a6b9 100644 --- a/cli/azd/cmd/down.go +++ b/cli/azd/cmd/down.go @@ -67,7 +67,6 @@ type downAction struct { flags *downFlags args []string provisionManager *provisioning.Manager - defaultProvider provisioning.DefaultProviderResolver importManager *project.ImportManager env *environment.Environment envManager environment.Manager @@ -80,7 +79,6 @@ func newDownAction( args []string, flags *downFlags, provisionManager *provisioning.Manager, - defaultProvider provisioning.DefaultProviderResolver, env *environment.Environment, envManager environment.Manager, projectConfig *project.ProjectConfig, @@ -91,7 +89,6 @@ func newDownAction( return &downAction{ flags: flags, provisionManager: provisionManager, - defaultProvider: defaultProvider, env: env, envManager: envManager, console: console, @@ -138,7 +135,7 @@ func (a *downAction) Run(ctx context.Context) (*actions.ActionResult, error) { // Record the resolved IaC provider (single, or "mixed") on the command span up front, so it is // present on success and failure alike (no-op when there are no layers). - provisioning.RecordInfraProviderUsage(layers, a.defaultProvider) + a.provisionManager.RecordInfraProviderUsage(layers) for _, layer := range layers { if downLayer != "" || len(layers) > 1 { diff --git a/cli/azd/cmd/down_test.go b/cli/azd/cmd/down_test.go index 36eff8d66e0..986b4d51246 100644 --- a/cli/azd/cmd/down_test.go +++ b/cli/azd/cmd/down_test.go @@ -18,7 +18,7 @@ func Test_NewDownAction_Constructor(t *testing.T) { t.Parallel() flags := &downFlags{} console := mockinput.NewMockConsole() - a := newDownAction(nil, flags, nil, nil, nil, nil, nil, console, nil, nil) + a := newDownAction(nil, flags, nil, nil, nil, nil, console, nil, nil) da := a.(*downAction) require.Same(t, flags, da.flags) } diff --git a/cli/azd/cmd/up.go b/cli/azd/cmd/up.go index a3883798169..77672af819f 100644 --- a/cli/azd/cmd/up.go +++ b/cli/azd/cmd/up.go @@ -70,7 +70,6 @@ type upAction struct { env *environment.Environment projectConfig *project.ProjectConfig provisioningManager *provisioning.Manager - defaultProvider provisioning.DefaultProviderResolver envManager environment.Manager prompters prompt.Prompter importManager *project.ImportManager @@ -84,7 +83,6 @@ func newUpAction( env *environment.Environment, projectConfig *project.ProjectConfig, provisioningManager *provisioning.Manager, - defaultProvider provisioning.DefaultProviderResolver, envManager environment.Manager, prompters prompt.Prompter, importManager *project.ImportManager, @@ -97,7 +95,6 @@ func newUpAction( env: env, projectConfig: projectConfig, provisioningManager: provisioningManager, - defaultProvider: defaultProvider, envManager: envManager, prompters: prompters, importManager: importManager, @@ -148,7 +145,7 @@ func (u *upAction) Run(ctx context.Context) (*actions.ActionResult, error) { // Record the resolved IaC provider (single, or "mixed") on the cmd.up span up front — before // provider init and the custom-workflow branch — so every up path (including custom workflows // and early init failures) carries it. - provisioning.RecordInfraProviderUsage(infra.Options.GetLayers(), u.defaultProvider) + u.provisioningManager.RecordInfraProviderUsage(infra.Options.GetLayers()) // TODO(weilim): remove this once we have decided if it's okay to not set AZURE_SUBSCRIPTION_ID and AZURE_LOCATION // early in the up workflow in #3745 diff --git a/cli/azd/cmd/up_test.go b/cli/azd/cmd/up_test.go index bb69cebc2d4..065f579f742 100644 --- a/cli/azd/cmd/up_test.go +++ b/cli/azd/cmd/up_test.go @@ -18,7 +18,7 @@ func Test_NewUpAction_Constructor(t *testing.T) { t.Parallel() flags := &upFlags{} console := mockinput.NewMockConsole() - a := newUpAction(flags, console, nil, nil, nil, nil, nil, nil, nil, nil, nil) + a := newUpAction(flags, console, nil, nil, nil, nil, nil, nil, nil, nil) ua := a.(*upAction) require.Same(t, flags, ua.flags) } diff --git a/cli/azd/internal/cmd/provision_graph.go b/cli/azd/internal/cmd/provision_graph.go index 7596b324392..d0d3471cfc5 100644 --- a/cli/azd/internal/cmd/provision_graph.go +++ b/cli/azd/internal/cmd/provision_graph.go @@ -86,7 +86,7 @@ func (p *ProvisionAction) provisionLayersGraph( // Record the resolved IaC provider (single, or "mixed") on the command span up front, so it is // present on success, failure, and preview runs alike — scoped to the provisioning lifecycle. - provisioning.RecordInfraProviderUsage(layers, p.defaultProvider) + p.provisionManager.RecordInfraProviderUsage(layers) // ── provider-agnostic provision validation (once per command) ───────── // Dispatch extension-registered "provision" checks a single time, before diff --git a/cli/azd/pkg/infra/provisioning/manager.go b/cli/azd/pkg/infra/provisioning/manager.go index 6bcf8ba6c7e..67f44e2b65c 100644 --- a/cli/azd/pkg/infra/provisioning/manager.go +++ b/cli/azd/pkg/infra/provisioning/manager.go @@ -558,13 +558,13 @@ const ( // "bicep", "terraform", "arm"); non-built-in (extension) providers are bucketed to // InfraProviderCustom ("custom") so raw user-chosen names are never emitted. When layers // resolve to different providers it records InfraProviderMixed ("mixed"). Layers that leave -// the provider unspecified resolve through defaultProvider. It is a no-op when no provider -// can be resolved. +// the provider unspecified resolve through the manager's default provider. It is a no-op when +// no provider can be resolved. // // Callers must invoke this once per command, before provider work begins, so the attribute is // present on success, failure, and preview spans alike. The value is computed deterministically // from configuration (rather than racing concurrent per-layer resolution). -func RecordInfraProviderUsage(layers []Options, defaultProvider DefaultProviderResolver) { +func (m *Manager) RecordInfraProviderUsage(layers []Options) { // The default provider is resolved lazily and at most once per call: every unspecified layer // resolves to the same default, so caching keeps the value deterministic and avoids repeating // resolver work (which may do I/O) per layer. @@ -576,12 +576,12 @@ func RecordInfraProviderUsage(layers []Options, defaultProvider DefaultProviderR for _, layer := range layers { kind := layer.Provider if kind == NotSpecified { - if defaultProvider == nil || defaultFailed { + if m.defaultProvider == nil || defaultFailed { continue } if !defaultResolved { - resolved, err := defaultProvider() + resolved, err := m.defaultProvider() if err != nil { defaultFailed = true continue diff --git a/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go b/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go index 0e14186b6ec..3f2fe42b82e 100644 --- a/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go +++ b/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package provisioning_test +package provisioning import ( "errors" @@ -9,90 +9,89 @@ import ( "github.com/azure/azure-dev/cli/azd/internal/tracing" "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" - "github.com/azure/azure-dev/cli/azd/pkg/infra/provisioning" "github.com/stretchr/testify/require" ) func TestRecordInfraProviderUsage(t *testing.T) { - bicepDefault := func() (provisioning.ProviderKind, error) { return provisioning.Bicep, nil } - failingResolver := func() (provisioning.ProviderKind, error) { - return provisioning.NotSpecified, errors.New("no default provider") + bicepDefault := func() (ProviderKind, error) { return Bicep, nil } + failingResolver := func() (ProviderKind, error) { + return NotSpecified, errors.New("no default provider") } - unspecifiedResolver := func() (provisioning.ProviderKind, error) { - return provisioning.NotSpecified, nil + unspecifiedResolver := func() (ProviderKind, error) { + return NotSpecified, nil } tests := []struct { name string - layers []provisioning.Options - defaultProvider provisioning.DefaultProviderResolver + layers []Options + defaultProvider DefaultProviderResolver expected string // "" means no infra.provider attribute is recorded }{ { name: "single explicit bicep", - layers: []provisioning.Options{{Provider: provisioning.Bicep}}, + layers: []Options{{Provider: Bicep}}, defaultProvider: bicepDefault, expected: "bicep", }, { name: "single explicit terraform", - layers: []provisioning.Options{{Provider: provisioning.Terraform}}, + layers: []Options{{Provider: Terraform}}, defaultProvider: bicepDefault, expected: "terraform", }, { name: "unspecified resolves through default", - layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, + layers: []Options{{Provider: NotSpecified}}, defaultProvider: bicepDefault, expected: "bicep", }, { name: "uniform provider across layers", - layers: []provisioning.Options{ - {Provider: provisioning.Bicep}, - {Provider: provisioning.Bicep}, + layers: []Options{ + {Provider: Bicep}, + {Provider: Bicep}, }, defaultProvider: bicepDefault, expected: "bicep", }, { name: "different providers across layers records mixed", - layers: []provisioning.Options{ - {Provider: provisioning.Bicep}, - {Provider: provisioning.Terraform}, + layers: []Options{ + {Provider: Bicep}, + {Provider: Terraform}, }, defaultProvider: bicepDefault, - expected: provisioning.InfraProviderMixed, + expected: InfraProviderMixed, }, { name: "single explicit arm", - layers: []provisioning.Options{{Provider: provisioning.Arm}}, + layers: []Options{{Provider: Arm}}, defaultProvider: bicepDefault, expected: "arm", }, { name: "custom provider is bucketed", - layers: []provisioning.Options{{Provider: provisioning.ProviderKind("my-extension-provider")}}, + layers: []Options{{Provider: ProviderKind("my-extension-provider")}}, defaultProvider: bicepDefault, - expected: provisioning.InfraProviderCustom, + expected: InfraProviderCustom, }, { name: "built-in plus custom records mixed", - layers: []provisioning.Options{ - {Provider: provisioning.Bicep}, - {Provider: provisioning.ProviderKind("my-extension-provider")}, + layers: []Options{ + {Provider: Bicep}, + {Provider: ProviderKind("my-extension-provider")}, }, defaultProvider: bicepDefault, - expected: provisioning.InfraProviderMixed, + expected: InfraProviderMixed, }, { name: "two distinct custom providers record mixed", - layers: []provisioning.Options{ - {Provider: provisioning.ProviderKind("vendor.one")}, - {Provider: provisioning.ProviderKind("vendor.two")}, + layers: []Options{ + {Provider: ProviderKind("vendor.one")}, + {Provider: ProviderKind("vendor.two")}, }, defaultProvider: bicepDefault, - expected: provisioning.InfraProviderMixed, + expected: InfraProviderMixed, }, { name: "no layers records nothing", @@ -102,19 +101,19 @@ func TestRecordInfraProviderUsage(t *testing.T) { }, { name: "unspecified with nil resolver records nothing", - layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, + layers: []Options{{Provider: NotSpecified}}, defaultProvider: nil, expected: "", }, { name: "unspecified with failing resolver records nothing", - layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, + layers: []Options{{Provider: NotSpecified}}, defaultProvider: failingResolver, expected: "", }, { name: "unspecified resolving to NotSpecified records nothing", - layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, + layers: []Options{{Provider: NotSpecified}}, defaultProvider: unspecifiedResolver, expected: "", }, @@ -126,7 +125,7 @@ func TestRecordInfraProviderUsage(t *testing.T) { tracing.ResetUsageAttributesForTest() t.Cleanup(tracing.ResetUsageAttributesForTest) - provisioning.RecordInfraProviderUsage(tt.layers, tt.defaultProvider) + (&Manager{defaultProvider: tt.defaultProvider}).RecordInfraProviderUsage(tt.layers) var got string var found bool From e11764ce45df0092da6790cb47db920d9a052b6c Mon Sep 17 00:00:00 2001 From: hemarina Date: Tue, 14 Jul 2026 15:27:51 -0700 Subject: [PATCH 10/12] combine the test --- .../provisioning/manager_telemetry_test.go | 148 ------------------ .../pkg/infra/provisioning/manager_test.go | 139 ++++++++++++++++ 2 files changed, 139 insertions(+), 148 deletions(-) delete mode 100644 cli/azd/pkg/infra/provisioning/manager_telemetry_test.go diff --git a/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go b/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go deleted file mode 100644 index 3f2fe42b82e..00000000000 --- a/cli/azd/pkg/infra/provisioning/manager_telemetry_test.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package provisioning - -import ( - "errors" - "testing" - - "github.com/azure/azure-dev/cli/azd/internal/tracing" - "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" - "github.com/stretchr/testify/require" -) - -func TestRecordInfraProviderUsage(t *testing.T) { - bicepDefault := func() (ProviderKind, error) { return Bicep, nil } - failingResolver := func() (ProviderKind, error) { - return NotSpecified, errors.New("no default provider") - } - unspecifiedResolver := func() (ProviderKind, error) { - return NotSpecified, nil - } - - tests := []struct { - name string - layers []Options - defaultProvider DefaultProviderResolver - expected string // "" means no infra.provider attribute is recorded - }{ - { - name: "single explicit bicep", - layers: []Options{{Provider: Bicep}}, - defaultProvider: bicepDefault, - expected: "bicep", - }, - { - name: "single explicit terraform", - layers: []Options{{Provider: Terraform}}, - defaultProvider: bicepDefault, - expected: "terraform", - }, - { - name: "unspecified resolves through default", - layers: []Options{{Provider: NotSpecified}}, - defaultProvider: bicepDefault, - expected: "bicep", - }, - { - name: "uniform provider across layers", - layers: []Options{ - {Provider: Bicep}, - {Provider: Bicep}, - }, - defaultProvider: bicepDefault, - expected: "bicep", - }, - { - name: "different providers across layers records mixed", - layers: []Options{ - {Provider: Bicep}, - {Provider: Terraform}, - }, - defaultProvider: bicepDefault, - expected: InfraProviderMixed, - }, - { - name: "single explicit arm", - layers: []Options{{Provider: Arm}}, - defaultProvider: bicepDefault, - expected: "arm", - }, - { - name: "custom provider is bucketed", - layers: []Options{{Provider: ProviderKind("my-extension-provider")}}, - defaultProvider: bicepDefault, - expected: InfraProviderCustom, - }, - { - name: "built-in plus custom records mixed", - layers: []Options{ - {Provider: Bicep}, - {Provider: ProviderKind("my-extension-provider")}, - }, - defaultProvider: bicepDefault, - expected: InfraProviderMixed, - }, - { - name: "two distinct custom providers record mixed", - layers: []Options{ - {Provider: ProviderKind("vendor.one")}, - {Provider: ProviderKind("vendor.two")}, - }, - defaultProvider: bicepDefault, - expected: InfraProviderMixed, - }, - { - name: "no layers records nothing", - layers: nil, - defaultProvider: bicepDefault, - expected: "", - }, - { - name: "unspecified with nil resolver records nothing", - layers: []Options{{Provider: NotSpecified}}, - defaultProvider: nil, - expected: "", - }, - { - name: "unspecified with failing resolver records nothing", - layers: []Options{{Provider: NotSpecified}}, - defaultProvider: failingResolver, - expected: "", - }, - { - name: "unspecified resolving to NotSpecified records nothing", - layers: []Options{{Provider: NotSpecified}}, - defaultProvider: unspecifiedResolver, - expected: "", - }, - } - - // Usage attributes are process-global, so these subtests must not run in parallel. - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tracing.ResetUsageAttributesForTest() - t.Cleanup(tracing.ResetUsageAttributesForTest) - - (&Manager{defaultProvider: tt.defaultProvider}).RecordInfraProviderUsage(tt.layers) - - var got string - var found bool - for _, attr := range tracing.GetUsageAttributes() { - if attr.Key == fields.InfraProviderKey.Key { - got = attr.Value.AsString() - found = true - } - } - - if tt.expected == "" { - require.False(t, found, "expected no infra.provider attribute, got %q", got) - return - } - - require.True(t, found, "expected infra.provider attribute to be recorded") - require.Equal(t, tt.expected, got) - }) - } -} diff --git a/cli/azd/pkg/infra/provisioning/manager_test.go b/cli/azd/pkg/infra/provisioning/manager_test.go index bcb5f0e0e36..5920507d679 100644 --- a/cli/azd/pkg/infra/provisioning/manager_test.go +++ b/cli/azd/pkg/infra/provisioning/manager_test.go @@ -10,6 +10,8 @@ import ( "testing" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm/policy" + "github.com/azure/azure-dev/cli/azd/internal/tracing" + "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" "github.com/azure/azure-dev/cli/azd/pkg/account" "github.com/azure/azure-dev/cli/azd/pkg/azapi" "github.com/azure/azure-dev/cli/azd/pkg/cloud" @@ -395,3 +397,140 @@ func registerContainerDependencies(mockContext *mocks.MockContext, env *environm func defaultProvider() (provisioning.ProviderKind, error) { return provisioning.Bicep, nil } + +func TestRecordInfraProviderUsage(t *testing.T) { + failingResolver := func() (provisioning.ProviderKind, error) { + return provisioning.NotSpecified, errors.New("no default provider") + } + unspecifiedResolver := func() (provisioning.ProviderKind, error) { + return provisioning.NotSpecified, nil + } + + tests := []struct { + name string + layers []provisioning.Options + defaultProvider provisioning.DefaultProviderResolver + expected string // "" means no infra.provider attribute is recorded + }{ + { + name: "single explicit bicep", + layers: []provisioning.Options{{Provider: provisioning.Bicep}}, + defaultProvider: defaultProvider, + expected: "bicep", + }, + { + name: "single explicit terraform", + layers: []provisioning.Options{{Provider: provisioning.Terraform}}, + defaultProvider: defaultProvider, + expected: "terraform", + }, + { + name: "unspecified resolves through default", + layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, + defaultProvider: defaultProvider, + expected: "bicep", + }, + { + name: "uniform provider across layers", + layers: []provisioning.Options{ + {Provider: provisioning.Bicep}, + {Provider: provisioning.Bicep}, + }, + defaultProvider: defaultProvider, + expected: "bicep", + }, + { + name: "different providers across layers records mixed", + layers: []provisioning.Options{ + {Provider: provisioning.Bicep}, + {Provider: provisioning.Terraform}, + }, + defaultProvider: defaultProvider, + expected: provisioning.InfraProviderMixed, + }, + { + name: "single explicit arm", + layers: []provisioning.Options{{Provider: provisioning.Arm}}, + defaultProvider: defaultProvider, + expected: "arm", + }, + { + name: "custom provider is bucketed", + layers: []provisioning.Options{{Provider: provisioning.ProviderKind("my-extension-provider")}}, + defaultProvider: defaultProvider, + expected: provisioning.InfraProviderCustom, + }, + { + name: "built-in plus custom records mixed", + layers: []provisioning.Options{ + {Provider: provisioning.Bicep}, + {Provider: provisioning.ProviderKind("my-extension-provider")}, + }, + defaultProvider: defaultProvider, + expected: provisioning.InfraProviderMixed, + }, + { + name: "two distinct custom providers record mixed", + layers: []provisioning.Options{ + {Provider: provisioning.ProviderKind("vendor.one")}, + {Provider: provisioning.ProviderKind("vendor.two")}, + }, + defaultProvider: defaultProvider, + expected: provisioning.InfraProviderMixed, + }, + { + name: "no layers records nothing", + layers: nil, + defaultProvider: defaultProvider, + expected: "", + }, + { + name: "unspecified with nil resolver records nothing", + layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, + defaultProvider: nil, + expected: "", + }, + { + name: "unspecified with failing resolver records nothing", + layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, + defaultProvider: failingResolver, + expected: "", + }, + { + name: "unspecified resolving to NotSpecified records nothing", + layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, + defaultProvider: unspecifiedResolver, + expected: "", + }, + } + + // Usage attributes are process-global, so these subtests must not run in parallel. + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tracing.ResetUsageAttributesForTest() + t.Cleanup(tracing.ResetUsageAttributesForTest) + + // RecordInfraProviderUsage only reads the manager's default provider, so the remaining + // dependencies are intentionally nil. + mgr := provisioning.NewManager(nil, tt.defaultProvider, nil, nil, nil, nil, nil, nil) + mgr.RecordInfraProviderUsage(tt.layers) + + var got string + var found bool + for _, attr := range tracing.GetUsageAttributes() { + if attr.Key == fields.InfraProviderKey.Key { + got = attr.Value.AsString() + found = true + } + } + + if tt.expected == "" { + require.False(t, found, "expected no infra.provider attribute, got %q", got) + return + } + + require.True(t, found, "expected infra.provider attribute to be recorded") + require.Equal(t, tt.expected, got) + }) + } +} From 358a02d16efcf9c03cc5b07ab9b19d94a629f38e Mon Sep 17 00:00:00 2001 From: hemarina Date: Tue, 14 Jul 2026 15:53:00 -0700 Subject: [PATCH 11/12] remove mix and track all infra, address comment --- cli/azd/internal/cmd/provision.go | 5 +++ cli/azd/internal/cmd/provision_graph.go | 4 -- cli/azd/internal/tracing/fields/fields.go | 11 +++-- cli/azd/pkg/infra/provisioning/manager.go | 45 ++++++++++--------- .../pkg/infra/provisioning/manager_test.go | 42 ++++++++--------- cli/azd/test/functional/telemetry_test.go | 14 ++++++ docs/reference/telemetry-data.md | 4 +- .../metrics-audit/feature-telemetry-matrix.md | 6 +-- docs/specs/metrics-audit/telemetry-schema.md | 2 +- 9 files changed, 77 insertions(+), 56 deletions(-) diff --git a/cli/azd/internal/cmd/provision.go b/cli/azd/internal/cmd/provision.go index 6a4340f7fdd..eb8c558a41e 100644 --- a/cli/azd/internal/cmd/provision.go +++ b/cli/azd/internal/cmd/provision.go @@ -291,6 +291,11 @@ func (p *ProvisionAction) Run(ctx context.Context) (*actions.ActionResult, error layers = []provisioning.Options{layerOption} } + // Record the resolved IaC provider(s) on the cmd.provision span up front — before the + // preview/multi-layer validation and provider work — so the attribute is present on success, + // failure, and preview runs alike, scoped to the provisioning lifecycle. + p.provisionManager.RecordInfraProviderUsage(layers) + if previewMode && len(layers) > 1 { return nil, &internal.ErrorWithSuggestion{ Err: internal.ErrPreviewMultipleLayers, diff --git a/cli/azd/internal/cmd/provision_graph.go b/cli/azd/internal/cmd/provision_graph.go index d0d3471cfc5..7d0c998d46a 100644 --- a/cli/azd/internal/cmd/provision_graph.go +++ b/cli/azd/internal/cmd/provision_graph.go @@ -84,10 +84,6 @@ func (p *ProvisionAction) provisionLayersGraph( }, nil } - // Record the resolved IaC provider (single, or "mixed") on the command span up front, so it is - // present on success, failure, and preview runs alike — scoped to the provisioning lifecycle. - p.provisionManager.RecordInfraProviderUsage(layers) - // ── provider-agnostic provision validation (once per command) ───────── // Dispatch extension-registered "provision" checks a single time, before // the preview / single-layer / multi-layer paths run. This deliberately diff --git a/cli/azd/internal/tracing/fields/fields.go b/cli/azd/internal/tracing/fields/fields.go index 4da1b5547fe..b7908fb4a9e 100644 --- a/cli/azd/internal/tracing/fields/fields.go +++ b/cli/azd/internal/tracing/fields/fields.go @@ -397,11 +397,14 @@ var ( // Infrastructure command related fields var ( - // The IaC provider. Emitted by `infra generate` / `synth` (the configured `--provider` value, - // for example "bicep", "terraform", "auto") and by provision / up / down (the resolved - // provider, "mixed" when layers differ, or "custom" for a non-built-in extension provider). + // The IaC provider. Emitted by `infra generate` / `synth` as a single string (the value read + // from azure.yaml's `infra.provider`, for example "bicep", "terraform", or "auto" when unset) + // and by provision / up / down as a sorted, de-duplicated string slice of the resolved + // provider(s) — built-in kinds verbatim, or "custom" for a non-built-in extension provider. + // Multi-layer projects that combine providers record each distinct value (for example + // ["bicep", "terraform"]). // - // Example: "bicep", "terraform", "arm", "pulumi", "mixed", "custom" + // Example: "bicep", "terraform", "arm", "pulumi", "custom", ["bicep", "terraform"] InfraProviderKey = AttributeKey{ Key: attribute.Key("infra.provider"), Classification: SystemMetadata, diff --git a/cli/azd/pkg/infra/provisioning/manager.go b/cli/azd/pkg/infra/provisioning/manager.go index 67f44e2b65c..c2346d03c5a 100644 --- a/cli/azd/pkg/infra/provisioning/manager.go +++ b/cli/azd/pkg/infra/provisioning/manager.go @@ -8,8 +8,10 @@ import ( "encoding/json" "errors" "fmt" + "maps" "os" "path/filepath" + "slices" "github.com/azure/azure-dev/cli/azd/internal/tracing" "github.com/azure/azure-dev/cli/azd/internal/tracing/fields" @@ -542,24 +544,23 @@ func (m *Manager) newProvider(ctx context.Context) (Provider, error) { } const ( - // InfraProviderMixed is the infra.provider telemetry value recorded when a project's - // provisioning layers do not all resolve to the same IaC provider. - InfraProviderMixed = "mixed" - // InfraProviderCustom is the infra.provider telemetry value recorded for a provider that is // not one of the built-in kinds (for example an extension-registered provider). The raw name // is never emitted because it may embed user-chosen project or organization identifiers. InfraProviderCustom = "custom" ) -// RecordInfraProviderUsage records the resolved IaC provider for a provisioning command +// RecordInfraProviderUsage records the resolved IaC provider(s) for a provisioning command // (provision / up / down) as the infra.provider usage attribute on the ambient command span. -// When every layer resolves to the same provider it records that provider (for example -// "bicep", "terraform", "arm"); non-built-in (extension) providers are bucketed to -// InfraProviderCustom ("custom") so raw user-chosen names are never emitted. When layers -// resolve to different providers it records InfraProviderMixed ("mixed"). Layers that leave -// the provider unspecified resolve through the manager's default provider. It is a no-op when -// no provider can be resolved. +// The attribute is a sorted, de-duplicated string slice of the distinct providers the command's +// layers resolve to: built-in kinds are recorded verbatim (for example "bicep", "terraform", +// "arm"), while non-built-in (extension) providers are bucketed to InfraProviderCustom ("custom") +// so raw user-chosen names are never emitted. A single-provider project records a one-element +// slice (for example ["bicep"]); a multi-layer project that mixes providers records each distinct +// value (for example ["bicep", "terraform"]) rather than a lossy "mixed" marker, preserving which +// providers were combined while staying low-cardinality. Layers that leave the provider +// unspecified resolve through the manager's default provider. It is a no-op when no provider can +// be resolved. // // Callers must invoke this once per command, before provider work begins, so the attribute is // present on success, failure, and preview spans alike. The value is computed deterministically @@ -601,18 +602,20 @@ func (m *Manager) RecordInfraProviderUsage(layers []Options) { seen[kind] = struct{}{} } - // Distinct resolved kinds are collected first so that two different custom providers still - // record "mixed" (bucketing only the single-provider result, not before counting). - switch len(seen) { - case 0: + if len(seen) == 0 { return - case 1: - for kind := range seen { - tracing.SetUsageAttributes(fields.InfraProviderKey.String(infraProviderTelemetryValue(kind))) - } - default: - tracing.SetUsageAttributes(fields.InfraProviderKey.String(InfraProviderMixed)) } + + // Map the distinct resolved kinds to telemetry-safe values (built-ins verbatim, extensions + // bucketed to InfraProviderCustom), then de-duplicate and sort so the recorded slice is + // deterministic. De-duplicating after bucketing means two different extension providers both + // collapse to a single "custom" entry. + values := map[string]struct{}{} + for kind := range seen { + values[infraProviderTelemetryValue(kind)] = struct{}{} + } + + tracing.SetUsageAttributes(fields.InfraProviderKey.StringSlice(slices.Sorted(maps.Keys(values)))) } // infraProviderTelemetryValue maps a provider kind to a value that is safe to emit raw. Built-in diff --git a/cli/azd/pkg/infra/provisioning/manager_test.go b/cli/azd/pkg/infra/provisioning/manager_test.go index 5920507d679..18af9131879 100644 --- a/cli/azd/pkg/infra/provisioning/manager_test.go +++ b/cli/azd/pkg/infra/provisioning/manager_test.go @@ -410,25 +410,25 @@ func TestRecordInfraProviderUsage(t *testing.T) { name string layers []provisioning.Options defaultProvider provisioning.DefaultProviderResolver - expected string // "" means no infra.provider attribute is recorded + expected []string // nil means no infra.provider attribute is recorded }{ { name: "single explicit bicep", layers: []provisioning.Options{{Provider: provisioning.Bicep}}, defaultProvider: defaultProvider, - expected: "bicep", + expected: []string{"bicep"}, }, { name: "single explicit terraform", layers: []provisioning.Options{{Provider: provisioning.Terraform}}, defaultProvider: defaultProvider, - expected: "terraform", + expected: []string{"terraform"}, }, { name: "unspecified resolves through default", layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, defaultProvider: defaultProvider, - expected: "bicep", + expected: []string{"bicep"}, }, { name: "uniform provider across layers", @@ -437,70 +437,70 @@ func TestRecordInfraProviderUsage(t *testing.T) { {Provider: provisioning.Bicep}, }, defaultProvider: defaultProvider, - expected: "bicep", + expected: []string{"bicep"}, }, { - name: "different providers across layers records mixed", + name: "different providers across layers record each provider", layers: []provisioning.Options{ {Provider: provisioning.Bicep}, {Provider: provisioning.Terraform}, }, defaultProvider: defaultProvider, - expected: provisioning.InfraProviderMixed, + expected: []string{"bicep", "terraform"}, }, { name: "single explicit arm", layers: []provisioning.Options{{Provider: provisioning.Arm}}, defaultProvider: defaultProvider, - expected: "arm", + expected: []string{"arm"}, }, { name: "custom provider is bucketed", layers: []provisioning.Options{{Provider: provisioning.ProviderKind("my-extension-provider")}}, defaultProvider: defaultProvider, - expected: provisioning.InfraProviderCustom, + expected: []string{provisioning.InfraProviderCustom}, }, { - name: "built-in plus custom records mixed", + name: "built-in plus custom records both", layers: []provisioning.Options{ {Provider: provisioning.Bicep}, {Provider: provisioning.ProviderKind("my-extension-provider")}, }, defaultProvider: defaultProvider, - expected: provisioning.InfraProviderMixed, + expected: []string{"bicep", provisioning.InfraProviderCustom}, }, { - name: "two distinct custom providers record mixed", + name: "two distinct custom providers collapse to custom", layers: []provisioning.Options{ {Provider: provisioning.ProviderKind("vendor.one")}, {Provider: provisioning.ProviderKind("vendor.two")}, }, defaultProvider: defaultProvider, - expected: provisioning.InfraProviderMixed, + expected: []string{provisioning.InfraProviderCustom}, }, { name: "no layers records nothing", layers: nil, defaultProvider: defaultProvider, - expected: "", + expected: nil, }, { name: "unspecified with nil resolver records nothing", layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, defaultProvider: nil, - expected: "", + expected: nil, }, { name: "unspecified with failing resolver records nothing", layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, defaultProvider: failingResolver, - expected: "", + expected: nil, }, { name: "unspecified resolving to NotSpecified records nothing", layers: []provisioning.Options{{Provider: provisioning.NotSpecified}}, defaultProvider: unspecifiedResolver, - expected: "", + expected: nil, }, } @@ -515,17 +515,17 @@ func TestRecordInfraProviderUsage(t *testing.T) { mgr := provisioning.NewManager(nil, tt.defaultProvider, nil, nil, nil, nil, nil, nil) mgr.RecordInfraProviderUsage(tt.layers) - var got string + var got []string var found bool for _, attr := range tracing.GetUsageAttributes() { if attr.Key == fields.InfraProviderKey.Key { - got = attr.Value.AsString() + got = attr.Value.AsStringSlice() found = true } } - if tt.expected == "" { - require.False(t, found, "expected no infra.provider attribute, got %q", got) + if tt.expected == nil { + require.False(t, found, "expected no infra.provider attribute, got %v", got) return } diff --git a/cli/azd/test/functional/telemetry_test.go b/cli/azd/test/functional/telemetry_test.go index f91fa1475a2..edba6923200 100644 --- a/cli/azd/test/functional/telemetry_test.go +++ b/cli/azd/test/functional/telemetry_test.go @@ -331,6 +331,20 @@ func Test_CLI_Telemetry_NestedCommands(t *testing.T) { upAttrs := attributesMap(cmdSpans["cmd.up"].Attributes) require.Contains(t, upAttrs, fields.CmdArgsCount.Key) require.Equal(t, float64(0), upAttrs[fields.CmdArgsCount.Key]) + + // infra.provider is scoped to the provisioning lifecycle. The minimal project has no explicit + // provider, so it resolves to the default ("bicep"). It is recorded as a slice of the resolved + // providers and must be present with that value on cmd.provision and the parent cmd.up span, + // and absent from the synthetic cmd.package span (packaging is unrelated to the IaC provider). + // Recorded up front on the common `up` path, so it is present even though provisioning + // subsequently fails. + require.NotContains(t, attributesMap(cmdSpans["cmd.package"].Attributes), fields.InfraProviderKey.Key, + "infra.provider must not be recorded on the synthetic cmd.package span") + provisionAttrs := attributesMap(cmdSpans["cmd.provision"].Attributes) + require.ElementsMatch(t, []string{"bicep"}, provisionAttrs[fields.InfraProviderKey.Key], + "cmd.provision should carry the resolved infra.provider") + require.ElementsMatch(t, []string{"bicep"}, upAttrs[fields.InfraProviderKey.Key], + "cmd.up should carry the resolved infra.provider") } func Test_Telemetry_AlphaFeatures_Enabled(t *testing.T) { diff --git a/docs/reference/telemetry-data.md b/docs/reference/telemetry-data.md index 22e79444228..0590ad85ce6 100644 --- a/docs/reference/telemetry-data.md +++ b/docs/reference/telemetry-data.md @@ -345,7 +345,7 @@ Set **only when an external command-line tool invocation fails**, during error c | Field Key | Type | Description | |-----------|------|-------------| -| `infra.provider` | string | provision/up/down: `bicep`/`terraform`/`arm`/`pulumi`, `custom` (extension providers; raw name not emitted), or `mixed`. generate/synth: configured `--provider` value directly (`bicep`/`terraform`/`auto`, or a raw custom name) | +| `infra.provider` | string or string[] | provision/up/down: sorted, de-duplicated string slice of resolved providers — `bicep`/`terraform`/`arm`/`pulumi`, or `custom` (extension providers; raw name not emitted); multi-layer projects record each distinct value (e.g. `["bicep","terraform"]`). generate/synth: the value read from azure.yaml's `infra.provider` as a single string (`bicep`/`terraform`, `auto` when unset, or a raw custom name) |
@@ -678,7 +678,7 @@ How to find telemetry for a given feature area. Start here if you know the featu | **Container Apps (Aspire)** | `cmd.deploy`, `cmd.provision` | `project.service.targets` = `containerapp-dotnet`, `platform.type` = `aca` | Aspire-specific adoption and success | | **Language Support** | `cmd.deploy`, `cmd.package`, `cmd.restore` | `project.service.languages`, `project.service.language` | Usage by language | | **Templates** | `cmd.init`, `cmd.up` | `project.template.id` (hashed — join with template lookup to resolve) | Template adoption, success by template | -| **Provisioning (IaC)** | `cmd.provision`, `cmd.up`, `cmd.down`, `arm.deploy.*`, `arm.validate.*` | `infra.provider` (`bicep`/`terraform`/`arm`/`pulumi`/custom, or `mixed`) | Provision success, ARM errors, duration | +| **Provisioning (IaC)** | `cmd.provision`, `cmd.up`, `cmd.down`, `arm.deploy.*`, `arm.validate.*` | `infra.provider` (`bicep`/`terraform`/`arm`/`pulumi`/custom; slice of each distinct provider for multi-layer projects) | Provision success, ARM errors, duration | | **Authentication** | `cmd.auth.login` | `auth.method` | Auth method usage, failure rates | | **CI/CD Pipelines** | `cmd.pipeline.config` | `pipeline.provider` | Pipeline setup adoption | | **Extensions** | `ext.run`, `ext.install`, `ext.upgrade` | `extension.id`, `extension.version`, `extension.installed` | Extension adoption, errors | diff --git a/docs/specs/metrics-audit/feature-telemetry-matrix.md b/docs/specs/metrics-audit/feature-telemetry-matrix.md index 9b03d2d95bb..57471ea5e78 100644 --- a/docs/specs/metrics-audit/feature-telemetry-matrix.md +++ b/docs/specs/metrics-audit/feature-telemetry-matrix.md @@ -65,8 +65,8 @@ These commands emit attributes or events beyond the global middleware span. | **Show** | | | | | | | `show` | — | ✅ | ❌ | ❌ | Redundant — output format not analytically useful | | **Infrastructure** | | | | | | -| `infra generate` | — | ✅ | ✅ | ❌ | `infra.provider` (configured `--provider`: `bicep`/`terraform`/`auto`, or a raw custom name — see cross-cutting row) | -| `infra synth` | — | ✅ | ✅ | ❌ | `infra.provider` (configured `--provider`: `bicep`/`terraform`/`auto`, or a raw custom name — see cross-cutting row) | +| `infra generate` | — | ✅ | ✅ | ❌ | `infra.provider` (from azure.yaml's `infra.provider`: `bicep`/`terraform`, `auto` when unset, or a raw custom name — see cross-cutting row) | +| `infra synth` | — | ✅ | ✅ | ❌ | `infra.provider` (from azure.yaml's `infra.provider`: `bicep`/`terraform`, `auto` when unset, or a raw custom name — see cross-cutting row) | | `infra create` | — (hidden, deprecated) | ✅ | ❌ | ❌ | Wraps `provision`; inherits its telemetry | | `infra delete` | — (hidden, deprecated) | ✅ | ❌ | ❌ | Wraps `down`; inherits its telemetry | | **Core Lifecycle** | | | | | | @@ -122,7 +122,7 @@ command-specific telemetry fields provide analytical value beyond the command na | Hooks kind | `hooks.kind` | `hooks run` | Distinguishes the script runtime used to execute the hook (`sh`, `pwsh`, `js`, `ts`, `python`, `dotnet`) | | Pipeline provider | `pipeline.provider` | `pipeline config` | Distinguishes GitHub vs Azure DevOps | | Pipeline auth | `pipeline.auth` | `pipeline config` | Distinguishes federated vs client-credentials | -| Infra provider | `infra.provider` | `infra generate`, `infra synth`, `provision`, `up`, `down` | provision/up/down: resolved `bicep`/`terraform`/`arm`/`pulumi`, `custom` for extension providers (raw name not emitted), or `mixed` when layers differ. `infra generate`/`synth`: the configured `--provider` value emitted directly (`bicep`/`terraform`/`auto`, or a raw custom name) | +| Infra provider | `infra.provider` | `infra generate`, `infra synth`, `provision`, `up`, `down` | provision/up/down: sorted, de-duplicated string slice of resolved providers — `bicep`/`terraform`/`arm`/`pulumi` verbatim, `custom` for extension providers (raw name not emitted); multi-layer projects that combine providers record each distinct value (e.g. `["bicep","terraform"]`). `infra generate`/`synth`: the value read from azure.yaml's `infra.provider` emitted directly as a single string (`bicep`/`terraform`, `auto` when unset, or a raw custom name) | | Tool ID | `tool.id` / `tool.ids` | `tool *` | Identifies which managed tool (e.g., bicep, gh, kubectl) the command acted on | | Tool install metrics | `tool.install.*` | `tool install`, `tool upgrade`, `tool uninstall`, first-run middleware | Success count, failure count, duration, strategy — quantitative install health | | Tool upgrade versions | `tool.upgrade.from_version`, `tool.upgrade.to_version` | `tool upgrade` | Tracks adoption of new tool versions | diff --git a/docs/specs/metrics-audit/telemetry-schema.md b/docs/specs/metrics-audit/telemetry-schema.md index c0112de7027..6b622e78e67 100644 --- a/docs/specs/metrics-audit/telemetry-schema.md +++ b/docs/specs/metrics-audit/telemetry-schema.md @@ -274,7 +274,7 @@ The following fields are defined in `fields.go`. | Hooks kind | `hooks.kind` | SystemMetadata | FeatureInsight | Executor used to run the hook. Values: `sh`, `pwsh`, `python`, `js`, `ts`, `dotnet` | | Pipeline provider | `pipeline.provider` | SystemMetadata | FeatureInsight | Resolved provider display name after auto-detection: `GitHub`, `Azure DevOps` | | Pipeline auth | `pipeline.auth` | SystemMetadata | FeatureInsight | Emitted only when `--auth-type` is set on `pipeline config`: `federated`, `client-credentials` | -| Infra provider | `infra.provider` | SystemMetadata | FeatureInsight | provision/up/down: resolved `bicep`/`terraform`/`arm`/`pulumi`, `custom` for any other (extension) provider (raw name not emitted), or `mixed` when layers differ. `infra generate`/`synth`: the configured `--provider` value directly (`bicep`/`terraform`/`auto`, or a raw custom name) | +| Infra provider | `infra.provider` | SystemMetadata | FeatureInsight | provision/up/down: sorted, de-duplicated string slice of resolved providers — `bicep`/`terraform`/`arm`/`pulumi` verbatim, `custom` for any other (extension) provider (raw name not emitted); multi-layer projects that combine providers record each distinct value (e.g. `["bicep","terraform"]`). `infra generate`/`synth`: the value read from azure.yaml's `infra.provider` directly as a single string (`bicep`/`terraform`, `auto` when unset, or a raw custom name) | ### App Service Deploy From e4403b31c9de3441a70e70925eaf06f07f99c77d Mon Sep 17 00:00:00 2001 From: hemarina Date: Tue, 14 Jul 2026 16:28:13 -0700 Subject: [PATCH 12/12] address comment --- cli/azd/cmd/down.go | 2 +- cli/azd/cmd/up.go | 14 +++++++++----- docs/reference/telemetry-data.md | 12 ++++++++++++ .../metrics-audit/feature-telemetry-matrix.md | 2 +- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/cli/azd/cmd/down.go b/cli/azd/cmd/down.go index 3b6dfa7a6b9..b01ac4f2619 100644 --- a/cli/azd/cmd/down.go +++ b/cli/azd/cmd/down.go @@ -133,7 +133,7 @@ func (a *downAction) Run(ctx context.Context) (*actions.ActionResult, error) { } slices.Reverse(layers) - // Record the resolved IaC provider (single, or "mixed") on the command span up front, so it is + // Record the resolved IaC provider(s) on the command span up front, so it is // present on success and failure alike (no-op when there are no layers). a.provisionManager.RecordInfraProviderUsage(layers) diff --git a/cli/azd/cmd/up.go b/cli/azd/cmd/up.go index 77672af819f..8574c4f1429 100644 --- a/cli/azd/cmd/up.go +++ b/cli/azd/cmd/up.go @@ -142,11 +142,6 @@ func (u *upAction) Run(ctx context.Context) (*actions.ActionResult, error) { } defer func() { _ = infra.Cleanup() }() - // Record the resolved IaC provider (single, or "mixed") on the cmd.up span up front — before - // provider init and the custom-workflow branch — so every up path (including custom workflows - // and early init failures) carries it. - u.provisioningManager.RecordInfraProviderUsage(infra.Options.GetLayers()) - // TODO(weilim): remove this once we have decided if it's okay to not set AZURE_SUBSCRIPTION_ID and AZURE_LOCATION // early in the up workflow in #3745 err = u.provisioningManager.Initialize(ctx, u.projectConfig.Path, infra.Options) @@ -192,6 +187,15 @@ func (u *upAction) Run(ctx context.Context) (*actions.ActionResult, error) { } layers := infra.Options.GetLayers() + + // Record the resolved IaC provider(s) as a usage attribute on the cmd.up span. This is scoped + // to the unified-graph path rather than set before the custom-workflow branch on purpose: + // custom `workflows.up` runs `azd package`, `azd provision`, and `azd deploy` as in-process + // child commands that each attach the ambient usage attributes to their own spans, so recording + // it earlier would leak infra.provider onto the package/deploy spans and let a nested + // single-layer `provision` overwrite the aggregated value. Custom workflows instead carry + // infra.provider on their own child `azd provision` span. + u.provisioningManager.RecordInfraProviderUsage(layers) return u.upGraph.Run(ctx, layers, &u.flags.DeployFlags, u.flags.flagSet, startTime) } diff --git a/docs/reference/telemetry-data.md b/docs/reference/telemetry-data.md index 0590ad85ce6..3d6f4775044 100644 --- a/docs/reference/telemetry-data.md +++ b/docs/reference/telemetry-data.md @@ -593,6 +593,18 @@ The `execution.environment` field identifies where azd is running. Format: `