Unify internal CI matrix and refactor CG mode to SDL mode

.vsts-ci.yml

@@ -1,4 +1,10 @@
-# Pipeline: https://dev.azure.com/dnceng/internal/_build?definitionId=286
+# This yml is used by these pipelines:
+# - dotnet-sdk-official-ci
+#   https://dev.azure.com/dnceng/internal/_build?definitionId=286
+#   Triggers: CI batch and weekly schedule (complete CG scan)
+# - dotnet-sdk-unofficial-ci
+#   https://dev.azure.com/dnceng/internal/_build?definitionId=1472
+#   Triggers: PR
 
 trigger:
   batch: true
@@ -25,9 +31,14 @@ pr:
     - internal/release/*
 
 parameters:
-# When true, runs the pipeline in the same way as the PR pipeline.
-- name: runTestBuild
-  displayName: Run A Test Build
+# When true, forces test mode on non-PR runs.
+- name: forceTestBuild
+  displayName: Force test build
+  type: boolean
+  default: false
+# When true, forces a complete CG scan including test template acquisition for non-scheduled runs.
+- name: forceCompleteCg
+  displayName: Force complete CG scan
   type: boolean
   default: false
 
@@ -39,23 +50,19 @@ variables:
 # Variables used: HelixApiAccessToken
 - group: DotNet-HelixApi-Access
 - group: AzureDevOps-Artifact-Feeds-Pats
+- ${{ if or(eq(parameters.forceTestBuild, true), eq(variables['Build.Reason'], 'PullRequest')) }}:
+  - name: runInTestMode
+    value: true
+- ${{ if or(eq(parameters.forceCompleteCg, true), eq(variables['Build.Reason'], 'Schedule')) }}:
+  - name: runInCompleteCgMode
+    value: true
 # Allows Arcade to run a signed build by disabling post-build signing for release-branch builds or manual builds that are not running tests.
-- ${{ if and(eq(parameters.runTestBuild, false), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual'))) }}:
+- ${{ if and(eq(parameters.forceTestBuild, false), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual'))) }}:
   - name: PostBuildSign
     value: false
 # Provides TSA variables for automatic bug reporting.
 - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
   - group: DotNet-CLI-SDLValidation-Params
-### LOCAL ONLY ###
-- name: _publishArgument
-  value: -publish
-- name: _signArgument
-  value: -sign
-- name: _officialBuildProperties
-  # The OfficialBuilder property is set to Microsoft for the official build only.
-  # This property is checked in Directory.Build.props and adds the MICROSOFT_ENABLE_TELEMETRY constant.
-  # This constant is used in CompileOptions.cs to set both TelemetryOptOutDefault and TelemetryOptOutDefaultString.
-  value: /p:DotNetPublishUsingPipelines=true /p:OfficialBuilder=Microsoft /p:OfficialBuildId=$(Build.BuildNumber)
 
 resources:
   repositories:
@@ -93,18 +100,17 @@ extends:
         enabled: true
       binskim:
         enabled: true
-      ${{ if or(eq(parameters.runTestBuild, true), eq(variables['Build.Reason'], 'PullRequest')) }}:
-        componentgovernance:
-          # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/
-          ignoreDirectories: artifacts, .packages
+      componentgovernance:
+        # Refdoc: https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/
+        ignoreDirectories: artifacts, .packages
 
     stages:
     ############### BUILD STAGE ###############
     - stage: build
       displayName: Build
       jobs:
       ############### HELIX JOB MONITOR ###############
-      - ${{ if or(eq(parameters.runTestBuild, true), eq(variables['Build.Reason'], 'PullRequest')) }}:
+      - ${{ if eq(variables['runInTestMode'], 'true') }}:
         - template: /eng/common/core-templates/job/helix-job-monitor.yml@self
           parameters:
             helixAccessToken: $(HelixApiAccessToken)
@@ -120,54 +126,24 @@ extends:
           oneESCompat:
             templateFolderName: templates-official
             publishTaskPrefix: 1ES.
+          runTests: ${{ eq(variables['runInTestMode'], 'true') }}
           populateInternalRuntimeVariables: true
           runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64)
           locBranch: release/10.0.3xx
           # WORKAROUND: BinSkim requires the folder exist prior to scanning.
           preSteps:
           - powershell: New-Item -ItemType Directory -Path $(Build.SourcesDirectory)/artifacts/bin -Force
             displayName: Create artifacts/bin directory
-          ${{ if and(eq(parameters.runTestBuild, false), ne(variables['Build.Reason'], 'PullRequest'), ne(variables['Build.Reason'], 'BatchedCI')) }}:
-            timeoutInMinutes: 180
-            windowsJobParameterSets:
-            ### OFFICIAL ###
-            - categoryName: Official
-              publishArgument: $(_publishArgument)
-              signArgument: $(_signArgument)
-              officialBuildProperties: $(_officialBuildProperties) /p:BuildWorkloads=true
-              enableDefaultArtifacts: true
-              runTests: false
-              publishRetryConfig: true
-              variables:
-                _SignType: real
-            - categoryName: Official
-              targetArchitecture: x86
-              publishArgument: $(_publishArgument)
-              signArgument: $(_signArgument)
-              officialBuildProperties: $(_officialBuildProperties)
-              runTests: false
-              variables:
-                _SignType: real
-              dependsOn: Windows_x64_Official
-              downloadManifestMsiPackages: true
-            - categoryName: Official
-              targetArchitecture: arm64
-              publishArgument: $(_publishArgument)
-              signArgument: $(_signArgument)
-              officialBuildProperties: $(_officialBuildProperties)
-              runTests: false
-              variables:
-                _SignType: real
-              dependsOn: Windows_x64_Official
-              downloadManifestMsiPackages: true
-            ### TestTemplatesCG ###
+          ${{ if and(eq(parameters.forceTestBuild, false), eq(variables['runInCompleteCgMode'], 'true')) }}:
+            additionalWindowsJobParameterSets:
             # Note: This job is only used to allow the test templates to be built locally on the agent as opposed to Helix.
             # The tests acquire the templates' PackageReferences from NuGet, which allows them to be scanned by CG (component governance).
-            # CG is only ran internally, so this job makes sense to only run alongside of the official jobs.
+            # CG is only run internally, so this job runs only in CG mode.
             - categoryName: TestTemplatesCG
+              runTests: true
+              testExecutionMode: local
               testProjects: $(Build.SourcesDirectory)/test/dotnet-new.IntegrationTests/dotnet-new.IntegrationTests.csproj
-              testRunnerAdditionalArguments: -class Microsoft.DotNet.Cli.New.IntegrationTests.DotnetNewTestTemplatesTests
-              publishXunitResults: true
+              testFilter: FullyQualifiedName~Microsoft.DotNet.Cli.New.IntegrationTests.DotnetNewTestTemplatesTests
 
       ############### LINUX ###############
       - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self
@@ -180,77 +156,9 @@ extends:
           oneESCompat:
             templateFolderName: templates-official
             publishTaskPrefix: 1ES.
+          runTests: ${{ eq(variables['runInTestMode'], 'true') }}
           populateInternalRuntimeVariables: true
           runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64)
-          ${{ if and(eq(parameters.runTestBuild, false), ne(variables['Build.Reason'], 'PullRequest'), ne(variables['Build.Reason'], 'BatchedCI')) }}:
-            timeoutInMinutes: 90
-            linuxJobParameterSets:
-            ### OFFICIAL ###
-            # Note: These builds are also glibc like the glibc category, but that category uses containers, and doesn't publish zips and tarballs.
-            - categoryName: Official
-              publishArgument: $(_publishArgument)
-              officialBuildProperties: $(_officialBuildProperties)
-              osProperties: $(linuxOsglibcProperties)
-              runTests: false
-            - categoryName: Official
-              targetArchitecture: arm
-              runtimeIdentifier: linux-arm
-              publishArgument: $(_publishArgument)
-              officialBuildProperties: $(_officialBuildProperties)
-              osProperties: $(linuxOsglibcProperties)
-              runTests: false
-            - categoryName: Official
-              targetArchitecture: arm64
-              runtimeIdentifier: linux-arm64
-              publishArgument: $(_publishArgument)
-              officialBuildProperties: $(_officialBuildProperties)
-              osProperties: $(linuxOsglibcProperties)
-              runTests: false
-            ### glibc ###
-            - categoryName: glibc
-              # Do not publish zips and tarballs. The linux-x64 binaries are already published by Official.
-              publishArgument: $(_publishArgument) /p:PublishBinariesAndBadge=false
-              officialBuildProperties: $(_officialBuildProperties)
-              osProperties: $(linuxOsglibcProperties) /p:BuildSdkDeb=true /p:BuildSdkRpm=true
-              runTests: false
-            - categoryName: glibc
-              targetArchitecture: arm64
-              runtimeIdentifier: linux-arm64
-              # Do not publish zips and tarballs. The linux-arm64 binaries are already published by Official.
-              publishArgument: $(_publishArgument) /p:PublishBinariesAndBadge=false
-              officialBuildProperties: $(_officialBuildProperties)
-              osProperties: $(linuxOsglibcProperties) /p:BuildSdkDeb=true /p:BuildSdkRpm=true
-              runTests: false
-            ### musl ###
-            - categoryName: musl
-              container: azureLinux30Amd64
-              runtimeIdentifier: linux-musl-x64
-              publishArgument: $(_publishArgument)
-              officialBuildProperties: $(_officialBuildProperties)
-              # Use HostOS when running on alpine.
-              osProperties: /p:HostOS=linux-musl
-              # SBOM generation is not supported for alpine.
-              enableSbom: false
-              runTests: false
-              # Helix is hanging on this job using the container. See: https://github.com/dotnet/dnceng/issues/6000
-              disableJob: true
-            - categoryName: musl
-              container: azureLinux30Amd64
-              targetArchitecture: arm
-              runtimeIdentifier: linux-musl-arm
-              publishArgument: $(_publishArgument)
-              officialBuildProperties: $(_officialBuildProperties)
-              osProperties: /p:OSName=linux-musl
-              runTests: false
-              # Helix is hanging on this job using the container. See: https://github.com/dotnet/dnceng/issues/6000
-              disableJob: true
-            - categoryName: musl
-              targetArchitecture: arm64
-              runtimeIdentifier: linux-musl-arm64
-              publishArgument: $(_publishArgument)
-              officialBuildProperties: $(_officialBuildProperties)
-              osProperties: /p:OSName=linux-musl
-              runTests: false
 
       ############### MACOS ###############
       - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self
@@ -263,28 +171,16 @@ extends:
           oneESCompat:
             templateFolderName: templates-official
             publishTaskPrefix: 1ES.
+          runTests: ${{ eq(variables['runInTestMode'], 'true') }}
           populateInternalRuntimeVariables: true
           runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64)
-          ${{ if and(eq(parameters.runTestBuild, false), ne(variables['Build.Reason'], 'PullRequest'), ne(variables['Build.Reason'], 'BatchedCI')) }}:
-            macOSJobParameterSets:
-            ### OFFICIAL ###
-            - categoryName: Official
-              runtimeIdentifier: osx-x64
-              publishArgument: $(_publishArgument)
-              officialBuildProperties: $(_officialBuildProperties)
-              runTests: false
-            - categoryName: Official
-              targetArchitecture: arm64
-              runtimeIdentifier: osx-arm64
-              publishArgument: $(_publishArgument)
-              officialBuildProperties: $(_officialBuildProperties)
-              runTests: false
 
     ############### PUBLISH STAGE ###############
     - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
       - stage: publish
         displayName: Publish
         dependsOn: []
+        condition: ne(variables['runInCompleteCgMode'], 'true')
         jobs:
         - template: /eng/common/templates-official/job/publish-build-assets.yml@self
           parameters:

.vsts-pr.yml

@@ -1,4 +1,7 @@
-# Pipeline: https://dev.azure.com/dnceng-public/public/_build?definitionId=101
+# This yml is used by these pipelines:
+# - dotnet-sdk-public-ci
+#   https://dev.azure.com/dnceng-public/public/_build?definitionId=101
+#   Triggers: CI batch and PR
 
 trigger:
   batch: true

eng/common/core-templates/job/helix-job-monitor.yml

@@ -96,9 +96,11 @@ jobs:
   pool:
     ${{ if eq(variables['System.TeamProject'], 'public') }}:
       name: $(DncEngPublicBuildPool)
+      os: linux
       demands: ImageOverride -equals build.azurelinux.3.amd64.open
     ${{ else }}:
       name: $(DncEngInternalBuildPool)
+      os: linux
       demands: ImageOverride -equals build.azurelinux.3.amd64
   steps:
   - checkout: self