Skip to content

Commit 58f39ab

Browse files
zivkanzivkan
authored
Improve CI drop management (#6869) (#6887)
1 parent dbe646b commit 58f39ab

3 files changed

Lines changed: 34 additions & 88 deletions

File tree

‎eng/pipelines/optprof.yml‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,16 @@ stages:
9090
- download: ComponentBuildUnderTest
9191
artifact: BuildInfo
9292
displayName: 'Download buildinfo.json'
93+
- task: DownloadPipelineArtifact@2
94+
displayName: 'Download RunSettings drop metadata'
95+
inputs:
96+
buildType: 'specific'
97+
artifactName: 'DropMetadata-RunSettings'
98+
targetPath: $(Pipeline.Workspace)\ComponentBuildUnderTest\DropMetadata-RunSettings
99+
project: '$(System.TeamProjectId)'
100+
definition: $(System.DefinitionId)
101+
buildVersionToDownload: 'specific'
102+
pipelineId: $(resources.pipeline.ComponentBuildUnderTest.runId)
93103
- powershell: |
94104
try {
95105
Write-Host "Set VSBranch, VsTargetChannel & VsTargetMajorVersion variables"
@@ -132,15 +142,23 @@ stages:
132142
displayName: 'Set VSBranch variable'
133143
- powershell : |
134144
try {
135-
$branchName = "$(resources.pipeline.ComponentBuildUnderTest.sourceBranch)"
136-
$branchName = $branchName.Replace('refs/heads/', '')
137-
138-
$RunSettingsURI = "https://vsdrop.microsoft.com/file/v1/RunSettings/$(System.TeamProject)/$(Build.Repository.Name)/$branchName/$(resources.pipeline.ComponentBuildUnderTest.runID);NuGet.OptProfV2.runsettings"
139-
Write-Host "RunSettingsURI: $RunSettingsURI"
140-
# non-output variable for VS config in the configure machine job
141-
# output variable for test execution job
142-
Set-AzurePipelinesVariable 'RunSettingsURI' $RunSettingsURI
143-
Set-AzurePipelinesVariable -IsOutput 'RunSettingsURI' $RunSettingsURI
145+
$dropMetadata = ConvertFrom-Json (Get-Content "$(Pipeline.Workspace)\ComponentBuildUnderTest\DropMetadata-RunSettings\VSTSDrop.json" -Raw)
146+
$dropUri = $dropMetadata.VstsDropBuildArtifact.VstsDropUrl
147+
if ($dropUri)
148+
{
149+
$dropUri = $dropUri -replace '^(.*?)/RunSettings/', 'https://vsdrop.microsoft.com/file/v1/RunSettings/'
150+
$RunSettingsURI = "$dropUri;NuGet.OptProfV2.runsettings"
151+
Write-Host "RunSettingsURI: $RunSettingsURI"
152+
# non-output variable for VS config in the configure machine job
153+
# output variable for test execution job
154+
Set-AzurePipelinesVariable 'RunSettingsURI' $RunSettingsURI
155+
Set-AzurePipelinesVariable -IsOutput 'RunSettingsURI' $RunSettingsURI
156+
}
157+
else
158+
{
159+
Write-Error "Drop metadata: $dropMetadata"
160+
throw "VSTSDropUrl not found in VSTSDrop.json"
161+
}
144162
}
145163
catch {
146164
Write-Host $_

‎eng/pipelines/templates/Build_and_UnitTest.yml‎

Lines changed: 2 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ steps:
209209
inputs:
210210
solution: "build\\sign.proj"
211211
configuration: "$(BuildConfiguration)"
212-
msbuildArguments: "/restore /p:DryRunSigning=false /p:DotNetSignType=$(SigningType) /p:SigningLogDirectory=$(Build.StagingDirectory)\\binlog /binarylogger:$(Build.StagingDirectory)\\binlog\\15.Sign.binlog"
212+
msbuildArguments: "/restore /p:DryRunSigning=false /p:DotNetSignType=${{parameters.SigningType}} /p:SigningLogDirectory=$(Build.StagingDirectory)\\binlog /binarylogger:$(Build.StagingDirectory)\\binlog\\15.Sign.binlog"
213213

214214
- task: NuGetToolInstaller@1
215215
displayName: Use NuGet 6.x
@@ -264,76 +264,11 @@ steps:
264264
arguments: 'install Microsoft.DevDiv.Validation.TestPlatform.Settings.Tasks -Version 1.0.655 -Source $(VsPackageFeedUrl) -ConfigFile $(System.DefaultWorkingDirectory)\NuGet.config -OutputDirectory $(System.DefaultWorkingDirectory)\packages'
265265
condition: "and(succeeded(), eq(variables['BuildRTM'], 'false'))"
266266

267-
- task: MicroBuildBuildVSBootstrapper@3
268-
displayName: "Build a Visual Studio bootstrapper for tests"
269-
inputs:
270-
azureSubscription: "VSEng-VSDrop-MI"
271-
channelName: "$(VsTargetChannelForTests)"
272-
vsMajorVersion: "$(VsTargetMajorVersion)"
273-
manifests: '$(Build.Repository.LocalPath)\artifacts\VS15\Microsoft.VisualStudio.NuGet.Core.vsman'
274-
outputFolder: '$(Build.Repository.LocalPath)\artifacts\VS15'
275-
condition: "and(succeeded(), eq(variables['BuildRTM'], 'false'))"
276-
env:
277-
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
278-
- task: PowerShell@1
279-
displayName: "Set Bootstrapper URL variable for tests"
280-
name: "vsbootstrapper"
281-
inputs:
282-
scriptType: "inlineScript"
283-
# MicroBuildOutputFolderOverride defined by template and redirects MicroBuildBuildVSBootstrapper output https://dev.azure.com/devdiv/1ESPipelineTemplates/_git/MicroBuildTemplate?path=/azure-pipelines/Jobs/Job.yml&version=GBrelease&line=21&lineEnd=22&lineStartColumn=1&lineEndColumn=36&lineStyle=plain&_a=contents
284-
inlineScript: |
285-
try {
286-
$json = Get-Content "${env:MicroBuildOutputFolderOverride}\MicroBuild\Output\Bootstrapperinfo.json" | ConvertFrom-Json
287-
$bootstrapperUrl = $json[0].bootstrapperUrl;
288-
Write-Host "Bootstrapper URL: $bootstrapperUrl"
289-
Write-Host "##vso[task.setvariable variable=bootstrapperUrl;isOutput=true]$bootstrapperUrl"
290-
} catch {
291-
Write-Host "##vso[task.LogIssue type=error;]Unable to set bootstrapperUrl: $_"
292-
exit 1
293-
}
294-
condition: "and(succeeded(), eq(variables['BuildRTM'], 'false'))"
295-
296-
- task: PowerShell@1
297-
displayName: "Set CloudBuild Session ID variable for tests"
298-
name: "setcloudbuildsessionid"
299-
continueOnError: true
300-
inputs:
301-
scriptType: "inlineScript"
302-
# MicroBuildOutputFolderOverride defined by template and redirects MicroBuildBuildVSBootstrapper output https://dev.azure.com/devdiv/1ESPipelineTemplates/_git/MicroBuildTemplate?path=/azure-pipelines/Jobs/Job.yml&version=GBrelease&line=21&lineEnd=22&lineStartColumn=1&lineEndColumn=36&lineStyle=plain&_a=contents
303-
inlineScript: |
304-
try {
305-
$json = Get-Content "${env:MicroBuildOutputFolderOverride}\MicroBuild\Output\Bootstrapperinfo.json" | ConvertFrom-Json
306-
$qBuildSessionId = $json[0].QBuildSessionId;
307-
Write-Host "CloudBuild Session ID: $qBuildSessionId"
308-
Write-Host "##vso[task.setvariable variable=QBuildSessionId;isOutput=true]$qBuildSessionId"
309-
} catch {
310-
Write-Host "##vso[task.LogIssue type=error;]Unable to set CloudBuild Session ID: $_"
311-
}
312-
condition: "and(succeeded(), eq(variables['BuildRTM'], 'false'))"
313-
314-
- task: PowerShell@1
315-
displayName: "Set Base Build Drop variable for tests"
316-
name: "setbasebuilddrop"
317-
continueOnError: true
318-
inputs:
319-
scriptType: "inlineScript"
320-
# MicroBuildOutputFolderOverride defined by template and redirects MicroBuildBuildVSBootstrapper output https://dev.azure.com/devdiv/1ESPipelineTemplates/_git/MicroBuildTemplate?path=/azure-pipelines/Jobs/Job.yml&version=GBrelease&line=21&lineEnd=22&lineStartColumn=1&lineEndColumn=36&lineStyle=plain&_a=contents
321-
inlineScript: |
322-
try {
323-
$json = Get-Content "${env:MicroBuildOutputFolderOverride}\MicroBuild\Output\Bootstrapperinfo.json" | ConvertFrom-Json
324-
$buildDrop = $json[0].BuildDrop;
325-
Write-Host "Base Build Drop: $buildDrop"
326-
Write-Host "##vso[task.setvariable variable=BaseBuildDrop;isOutput=true]$buildDrop"
327-
} catch {
328-
Write-Host "##vso[task.LogIssue type=error;]Unable to set Base Build Drop: $_"
329-
}
330-
condition: "and(succeeded(), eq(variables['BuildRTM'], 'false'))"
331-
332267
- task: MSBuild@1
333268
displayName: "Generate .runsettings files"
334269
inputs:
335270
solution: 'build\runsettings.proj'
336-
msbuildArguments: '/restore:false /property:OutputPath="$(Build.Repository.LocalPath)\artifacts\RunSettings" /property:TestDrop="RunSettings/$(System.TeamProject)/$(Build.Repository.Name)/$(SourceBranch)/$(Build.BuildId)" /property:ProfilingInputsDrop="ProfilingInputs/$(System.TeamProject)/$(Build.Repository.Name)/$(SourceBranch)/$(Build.BuildId)" /binarylogger:$(Build.StagingDirectory)\\binlog\\17.GenerateRunSettings.binlog'
271+
msbuildArguments: '/restore:false /property:OutputPath="$(Build.Repository.LocalPath)\artifacts\RunSettings" /property:TestDrop="$(RunSettingsDropName)" /property:ProfilingInputsDrop="$(ProfilingInputsDropName)" /binarylogger:$(Build.StagingDirectory)\\binlog\\17.GenerateRunSettings.binlog'
337272
condition: "and(succeeded(), eq(variables['BuildRTM'], 'false'))"
338273

339274
- task: PowerShell@1

‎eng/pipelines/templates/pipeline.yml‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ stages:
119119
isOfficialBuild: ${{ parameters.isOfficialBuild }}
120120
BuildRTM: "false"
121121
SemanticVersion: $[stageDependencies.Initialize.GetSemanticVersion.outputs['setsemanticversion.SemanticVersion']]
122+
RunSettingsDropName: 'RunSettings/$(System.TeamProject)/$(Build.Repository.Name)/$(SourceBranch)/$(Build.BuildId)'
123+
ProfilingInputsDropName: 'ProfilingInputs/$(System.TeamProject)/$(Build.Repository.Name)/$(SourceBranch)/$(Build.BuildId)'
122124
pool:
123125
name: VSEngSS-MicroBuild2022-1ES
124126
templateContext:
@@ -160,20 +162,11 @@ stages:
160162
artifactName: "nupkgs - $(RtmLabel)"
161163
sbomBuildDropPath: $(Build.SourcesDirectory)/artifacts
162164

163-
- output: pipelineArtifact
164-
displayName: 'Publish BootstrapperInfo.json as a build artifact'
165-
condition: "succeeded()"
166-
targetPath: $(MicroBuildOutputFolderOverride)\MicroBuild\Output
167-
artifactName: MicroBuildOutputs
168-
# need to make daily Apex test pipeline build its own bootstrapper, and remove this from official pipeline
169-
codeSignValidationEnabled: false
170-
sbomEnabled: false
171-
172165
- output: artifactsDrop
173166
displayName: 'Publish the .runsettings files to artifact services'
174167
condition: "succeeded()"
175168
dropServiceURI: 'https://devdiv.artifacts.visualstudio.com'
176-
buildNumber: 'RunSettings/$(System.TeamProject)/$(Build.Repository.Name)/$(SourceBranch)/$(Build.BuildId)'
169+
buildNumber: '$(RunSettingsDropName)'
177170
sourcePath: 'artifacts\RunSettings'
178171
toLowerCase: false
179172
usePat: true
@@ -185,7 +178,7 @@ stages:
185178
displayName: 'OptProfV2: publish profiling inputs to artifact services'
186179
condition: "and(succeeded(), eq(variables['IsOfficialBuild'], 'true'))"
187180
dropServiceURI: 'https://devdiv.artifacts.visualstudio.com'
188-
buildNumber: 'ProfilingInputs/$(System.TeamProject)/$(Build.Repository.Name)/$(SourceBranch)/$(Build.BuildId)'
181+
buildNumber: '$(ProfilingInputsDropName)'
189182
sourcePath: '$(Build.ArtifactStagingDirectory)\OptProf\ProfilingInputs'
190183
toLowerCase: false
191184
usePat: true
@@ -208,7 +201,7 @@ stages:
208201
displayName: 'Upload VSTS Drop'
209202
condition: "succeeded()"
210203
dropServiceURI: 'https://devdiv.artifacts.visualstudio.com'
211-
buildNumber: 'Products/$(System.TeamProject)/$(Build.Repository.Name)/$(SourceBranch)/$(Build.BuildNumber)'
204+
buildNumber: '$(MicroBuild.ManifestDropName)'
212205
sourcePath: "$(Build.Repository.LocalPath)\\artifacts\\VS15"
213206
toLowerCase: false
214207
usePat: true

0 commit comments

Comments
 (0)