[7.0.x] Use 1ES OutputParentDirectory to reduce scanning tasks (#7027)

Author: zivkan

build/symbols.proj

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="15.0" DefaultTargets="GetSymbolsAndAssembliesToIndex" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), 'README.md'))\build\common.props" />
-  
+
   <!-- Configuration/global properties -->
   <PropertyGroup>
     <CommonMSBuildProperties>
@@ -13,6 +13,7 @@
   </PropertyGroup>
 
   <Target Name="GetSymbolsAndAssembliesToIndex">
+    <Error Condition="'$(SymbolsOutputDirectory)' == ''" Text="SymbolsOutputDirectory property must be set."/>
     <MSBuild
       Projects="@(SolutionProjectsWithoutVSIX)"
       Targets="GetSymbolsToIndex"
@@ -25,10 +26,10 @@
     <ItemGroup>
       <FilteredSymbolsToIndex Include="@(SymbolsToIndex)" Condition="'%(SymbolsToIndex.Extension)' == '.dll' OR '%(SymbolsToIndex.Extension)' == '.exe'
                                                           OR   '%(SymbolsToIndex.Extension)' == '.pdb'">
-          <DestinationDir>$(ArtifactsDirectory)symbolstoindex\$([MSBuild]::MakeRelative($(ArtifactsDirectory), %(SymbolsToIndex.Identity)))</DestinationDir>
+          <DestinationDir>$(SymbolsOutputDirectory)\$([MSBuild]::MakeRelative($(ArtifactsDirectory), %(SymbolsToIndex.Identity)))</DestinationDir>
       </FilteredSymbolsToIndex>
     </ItemGroup>
     <Copy SourceFiles="@(FilteredSymbolsToIndex->'%(Identity)')" DestinationFiles="@(FilteredSymbolsToIndex->'%(DestinationDir)')"/>
     <Message Text="SymbolsToIndex: @(FilteredSymbolsToIndex, '%0a')" Importance="High"/>
   </Target>
-</Project>
+</Project>

eng/pipelines/templates/Build.yml

@@ -15,7 +15,13 @@ steps:
   - task: PowerShell@1
     inputs:
       scriptName: "$(Build.Repository.LocalPath)\\scripts\\cibuild\\ConfigureVstsBuild.ps1"
-      arguments: "-BuildRTM $(BuildRTM) -RepositoryPath $(Build.Repository.LocalPath) -BranchName $(SourceBranch) -CommitHash $(Build.SourceVersion) -BuildNumber $(Build.BuildNumber)"
+      arguments: >-
+        -BuildRTM $(BuildRTM)
+        -RepositoryPath $(Build.Repository.LocalPath)
+        -BranchName $(SourceBranch)
+        -CommitHash $(Build.SourceVersion)
+        -BuildNumber $(Build.BuildNumber)
+        -BuildInfoDirectory $(Build.StagingDirectory)\BuildInfo
     displayName: "Configure VSTS CI Environment"
 
   - task: PowerShell@1
@@ -225,7 +231,7 @@ steps:
     displayName: "Generate .runsettings files"
     inputs:
       solution: 'build\runsettings.proj'
-      msbuildArguments: '/restore:false /property:OutputPath="$(Build.Repository.LocalPath)\artifacts\RunSettings" /property:TestDrop="$(RunSettingsDropName)" /property:ProfilingInputsDrop="$(ProfilingInputsDropName)" /binarylogger:$(Build.StagingDirectory)\\binlog\\17.GenerateRunSettings.binlog'
+      msbuildArguments: '/restore:false /property:OutputPath="$(Build.StagingDirectory)\RunSettings" /property:TestDrop="$(RunSettingsDropName)" /property:ProfilingInputsDrop="$(ProfilingInputsDropName)" /binarylogger:$(Build.StagingDirectory)\\binlog\\17.GenerateRunSettings.binlog'
     condition: "and(succeeded(), eq(variables['BuildRTM'], 'false'))"
 
   - task: PowerShell@1
@@ -251,26 +257,45 @@ steps:
     inputs:
       solution: "build\\symbols.proj"
       configuration: "$(BuildConfiguration)"
-      msbuildArguments: "/restore:false /property:BuildProjectReferences=false /property:IsSymbolBuild=true /property:BuildRTM=$(BuildRTM) /binarylogger:$(Build.StagingDirectory)\\binlog\\18.CollectBuildSymbols.binlog"
+      msbuildArguments: >-
+        /restore:false
+        /property:BuildProjectReferences=false
+        /property:IsSymbolBuild=true
+        /property:BuildRTM=$(BuildRTM)
+        /property:SymbolsOutputDirectory=$(Build.StagingDirectory)\\symbolstoindex
+        /binarylogger:$(Build.StagingDirectory)\\binlog\\18.CollectBuildSymbols.binlog
       maximumCpuCount: true
-    condition: "and(succeeded(), eq(variables['IsOfficialBuild'], 'true'))"
 
   - task: MSBuild@1
     displayName: "LocValidation: Verify VSIX"
     inputs:
       solution: "build\\BuildValidator.proj"
       configuration: "$(BuildConfiguration)"
-      msbuildArguments: "/target:ValidateVsix /property:BuildRTM=$(BuildRTM) /property:LogsBasePath=$(Build.Repository.LocalPath)\\logs\\BuildValidatorLogs /property:TempDirectory=$(Agent.TempDirectory) /binarylogger:$(Build.StagingDirectory)\\binlog\\19.ValidateVsixLocalization.binlog"
+      msbuildArguments: "/target:ValidateVsix /property:BuildRTM=$(BuildRTM) /property:LogsBasePath=$(Build.StagingDirectory)\\BuildValidatorLogs /property:TempDirectory=$(Agent.TempDirectory) /binarylogger:$(Build.StagingDirectory)\\binlog\\19.ValidateVsixLocalization.binlog"
     condition: "and(succeeded(), eq(variables['BuildRTM'], 'false'))"
 
   - task: MSBuild@1
     displayName: "LocValidation: Verify Artifacts"
     inputs:
       solution: "build\\BuildValidator.proj"
       configuration: "$(BuildConfiguration)"
-      msbuildArguments: "/target:ValidateArtifacts /property:BuildRTM=$(BuildRTM) /property:LogsBasePath=$(Build.Repository.LocalPath)\\logs\\BuildValidatorLogs /property:TempDirectory=$(Agent.TempDirectory) /binarylogger:$(Build.StagingDirectory)\\binlog\\20.ValidateArtifactsLocalization.binlog"
+      msbuildArguments: "/target:ValidateArtifacts /property:BuildRTM=$(BuildRTM) /property:LogsBasePath=$(Build.StagingDirectory)\\BuildValidatorLogs /property:TempDirectory=$(Agent.TempDirectory) /binarylogger:$(Build.StagingDirectory)\\binlog\\20.ValidateArtifactsLocalization.binlog"
     condition: "and(succeeded(), eq(variables['BuildRTM'], 'false'))"
 
+  - task: CopyFiles@2
+    displayName: Copy product to staging directory
+    inputs:
+      SourceFolder: '$(Build.Repository.LocalPath)\\artifacts\\VS15'
+      Contents: '**'
+      TargetFolder: '$(Build.StagingDirectory)\\VS15'
+
+  - task: CopyFiles@2
+    displayName: Copy nupkgs to staging directory
+    inputs:
+      SourceFolder: '$(Build.Repository.LocalPath)\\artifacts\\$(NupkgOutputDir)'
+      Contents: '**'
+      TargetFolder: '$(Build.StagingDirectory)\\nupkgs'
+
     # Use dotnet msbuild instead of MSBuild CLI.
     # Using MSBuild CLI results in an assembly binding failure for NuGet.Common.dll 4.4.0.3 because Microsoft.DotNet.Build.Tasks.Feed.dll references SleetLib.dll which references NuGet.Common 4.4.0.3.
     # The Microsoft.DotNet.Build.Tasks.Feed package includes NuGet.Common 4.9.0.6 and a binding redirection in Microsoft.DotNet.Build.Tasks.Feed.dll.config but the binding redirection is not processed.

eng/pipelines/templates/pipeline.yml

@@ -94,18 +94,18 @@ stages:
           DropNamePrefix: OptimizationInputs/$(System.TeamProject)/$(Build.Repository.Name)
           ShouldSkipOptimize: $(ShouldSkipOptimize)
           AccessToken: $(System.AccessToken)
+      outputParentDirectory: '$(Build.StagingDirectory)'
       outputs:
       - output: pipelineArtifact
         displayName: 'Publish buildinfo.json as an artifact'
         condition: "succeeded()"
-        targetPath: '$(Build.Repository.LocalPath)\artifacts\buildinfo.json'
+        targetPath: '$(Build.StagingDirectory)\BuildInfo'
         artifactName: 'BuildInfo'
         sbomBuildDropPath: $(Build.SourcesDirectory)/artifacts
 
       - output: pipelineArtifact
         displayName: 'Publish nupkgs'
-        condition: "and(succeeded(), eq(variables['IsOfficialBuild'], 'true'))"
-        targetPath: "$(Build.Repository.LocalPath)\\artifacts\\$(NupkgOutputDir)"
+        targetPath: "$(Build.StagingDirectory)\\nupkgs"
         artifactName: "nupkgs - $(RtmLabel)"
         sbomBuildDropPath: $(Build.SourcesDirectory)/artifacts
 
@@ -114,7 +114,7 @@ stages:
         condition: "succeeded()"
         dropServiceURI: 'https://devdiv.artifacts.visualstudio.com'
         buildNumber: '$(RunSettingsDropName)'
-        sourcePath: 'artifacts\RunSettings'
+        sourcePath: '$(Build.StagingDirectory)\RunSettings'
         toLowerCase: false
         usePat: true
         dropMetadataContainerName: 'DropMetadata-RunSettings'
@@ -126,21 +126,20 @@ stages:
         condition: "and(succeeded(), eq(variables['IsOfficialBuild'], 'true'))"
         dropServiceURI: 'https://devdiv.artifacts.visualstudio.com'
         buildNumber: '$(ProfilingInputsDropName)'
-        sourcePath: '$(Build.ArtifactStagingDirectory)\OptProf\ProfilingInputs'
+        sourcePath: '$(Build.StagingDirectory)\OptProf\ProfilingInputs'
         toLowerCase: false
         usePat: true
         dropMetadataContainerName: 'DropMetadata-ProfilingInputs'
 
       - output: pipelineArtifact
         displayName: 'Publish NuGet.exe and VSIX as artifact'
-        targetPath: "$(Build.Repository.LocalPath)\\artifacts\\VS15"
+        targetPath: "$(Build.StagingDirectory)\\VS15"
         artifactName: "$(VsixPublishDir)"
         sbomBuildDropPath: $(Build.SourcesDirectory)/artifacts
 
       - output: pipelineArtifact
         displayName: 'Publish symbols as pipeline artifacts'
-        condition: "and(succeeded(), eq(variables['IsOfficialBuild'], 'true'))"
-        targetPath: "$(Build.Repository.LocalPath)\\artifacts\\symbolstoindex"
+        targetPath: "$(Build.StagingDirectory)\\symbolstoindex"
         artifactName: "symbols - $(RtmLabel)"
         sbomBuildDropPath: $(Build.SourcesDirectory)/artifacts
 
@@ -149,7 +148,7 @@ stages:
         condition: "succeeded()"
         dropServiceURI: 'https://devdiv.artifacts.visualstudio.com'
         buildNumber: '$(MicroBuild.ManifestDropName)'
-        sourcePath: "$(Build.Repository.LocalPath)\\artifacts\\VS15"
+        sourcePath: "$(Build.StagingDirectory)\\VS15"
         toLowerCase: false
         usePat: true
         dropMetadataContainerName: "DropMetadata-Product"
@@ -158,7 +157,7 @@ stages:
         displayName: 'LocValidation: Publish Logs as an artifact'
         condition: "succeededOrFailed()"
         artifactName: LocValidationLogs - Attempt $(System.JobAttempt)
-        targetPath: "$(Build.Repository.LocalPath)\\logs\\BuildValidatorLogs"
+        targetPath: "$(Build.StagingDirectory)\\BuildValidatorLogs"
         sbomEnabled: true
 
       - output: pipelineArtifact
@@ -171,7 +170,7 @@ stages:
       - output: pipelineArtifact
         displayName: Publish SBOM manifest
         artifactName: $(ARTIFACT_NAME)
-        targetPath: "$(Build.SourcesDirectory)/artifacts/_manifest"
+        targetPath: "$(Build.StagingDirectory)/sbom"
         sbomEnabled: false
 
     steps:
@@ -209,24 +208,24 @@ stages:
           enabled: true
         optprof:
           enabled: false
+      outputParentDirectory: '$(Build.StagingDirectory)'
       outputs:
       - output: pipelineArtifact
         displayName: 'Publish nupkgs'
         condition: "succeeded()"
-        targetPath: "$(Build.Repository.LocalPath)\\artifacts\\$(NupkgOutputDir)"
+        targetPath: "$(Build.StagingDirectory)\\$(NupkgOutputDir)"
         artifactName: "nupkgs - $(RtmLabel)"
         sbomBuildDropPath: $(Build.SourcesDirectory)/artifacts
 
       - output: pipelineArtifact
         displayName: 'Publish NuGet.exe and VSIX as artifact'
-        targetPath: "$(Build.Repository.LocalPath)\\artifacts\\VS15"
+        targetPath: "$(Build.StagingDirectory)\\VS15"
         artifactName: "$(VsixPublishDir)"
         sbomBuildDropPath: $(Build.SourcesDirectory)/artifacts
 
       - output: pipelineArtifact
         displayName: 'Publish symbols as pipeline artifacts'
-        condition: "and(succeeded(), eq(variables['IsOfficialBuild'], 'true'))"
-        targetPath: "$(Build.Repository.LocalPath)\\artifacts\\symbolstoindex"
+        targetPath: "$(Build.StagingDirectory)\\symbolstoindex"
         artifactName: "symbols - $(RtmLabel)"
         sbomBuildDropPath: $(Build.SourcesDirectory)/artifacts
 

scripts/cibuild/ConfigureVstsBuild.ps1

@@ -21,6 +21,9 @@ The commit hash being built
 
 .PARAMETER BuildNumber
 The build number of the current build
+
+.PARAMETER BuildInfoDirectory
+Optional directory path to write buildinfo.json to. When not provided, defaults to the repository's artifacts directory.
 #>
 
 param
@@ -34,7 +37,9 @@ param
     [Parameter(Mandatory=$true)]
     [string]$CommitHash,
     [Parameter(Mandatory=$true)]
-    [string]$BuildNumber
+    [string]$BuildNumber,
+    [Parameter(Mandatory=$false)]
+    [string]$BuildInfoDirectory
 )
 
 Function Get-Version {
@@ -141,11 +146,21 @@ else
         NuGetVsVersion = $GetNuGetVsVersion
     }
 
-    # First create the file locally so that we can laster publish it as a build artifact from a local source file instead of a remote source file.
-    $localBuildInfoJsonFilePath = [System.IO.Path]::Combine("$RepositoryPath\artifacts", 'buildinfo.json')
+    if (-not [string]::IsNullOrWhiteSpace($BuildInfoDirectory))
+    {
+        $buildInfoDirectoryPath = $BuildInfoDirectory
+    }
+    else
+    {
+        $buildInfoDirectoryPath = Join-Path $RepositoryPath 'artifacts'
+    }
+
+    New-Item -Path $buildInfoDirectoryPath -ItemType Directory -Force | Out-Null
+    $localBuildInfoJsonFilePath = Join-Path $buildInfoDirectoryPath 'buildinfo.json'
 
     New-Item $localBuildInfoJsonFilePath -Force | Out-Null
     $jsonRepresentation | ConvertTo-Json | Set-Content $localBuildInfoJsonFilePath
+    Write-Host "Created $localBuildInfoJsonFilePath"
 
     Update-VsixVersion -manifestName source.extension.vsixmanifest -buildNumber $BuildNumber -RepositoryPath $RepositoryPath
 }