Skip to content

Commit fb12995

Browse files
donnie-msftdonnie-msft
authored
[release-5.11.x] Use 1ES Templates in CI Pipelines & Configure SBOM (#6698)
1 parent 5681733 commit fb12995

16 files changed

Lines changed: 1352 additions & 992 deletions

‎build/OptProfV2.props‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
<Technology>IBC</Technology>
2424
<InstallationPath>Common7\IDE\CommonExtensions\Microsoft\NuGet\$(AssemblyName).dll</InstallationPath>
2525
<InstrumentationArguments>/ExeConfig:"%VisualStudio.InstallationUnderTest.Path%\Common7\IDE\vsn.exe"</InstrumentationArguments>
26-
<OptimizationArguments>-PartialNGEN</OptimizationArguments>
2726
<Scenarios>
2827
<TestContainer Name="NuGet.OptProf">
2928
<TestCase Weight="100" FullyQualifiedName="NuGet.OptProfV2Tests.IVsPackageSourceProvider_GetSources" />
@@ -41,12 +40,11 @@
4140
<Technology>IBC</Technology>
4241
<InstallationPath>Common7\IDE\CommonExtensions\Microsoft\NuGet\$(AssemblyName).dll</InstallationPath>
4342
<InstrumentationArguments>/ExeConfig:"%VisualStudio.InstallationUnderTest.Path%\Common7\IDE\vsn.exe"</InstrumentationArguments>
44-
<OptimizationArguments>-PartialNGEN</OptimizationArguments>
4543
<Scenarios>
4644
<TestContainer Name="NuGet.OptProf">
4745
<TestCase Weight="100" FullyQualifiedName="NuGet.OptProfV2Tests.IVsPackageSourceProvider_GetSources" />
4846
</TestContainer>
4947
</Scenarios>
5048
</OptProf>
5149
</ItemGroup>
52-
</Project>
50+
</Project>

‎eng/common/README.md‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Don't touch this folder
2+
3+
uuuuuuuuuuuuuuuuuuuu
4+
u" uuuuuuuuuuuuuuuuuu "u
5+
u" u$$$$$$$$$$$$$$$$$$$$u "u
6+
u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u
7+
u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
8+
u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
9+
u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
10+
$ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
11+
$ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
12+
$ $$$" ... "$... ...$" ... "$$$ ... "$$$ $
13+
$ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $
14+
$ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $
15+
$ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $
16+
$ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $
17+
$ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
18+
"u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
19+
"u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
20+
"u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
21+
"u "$$$$$$$$$$$$$$$$$$$$$$$$" u"
22+
"u "$$$$$$$$$$$$$$$$$$$$" u"
23+
"u """""""""""""""""" u"
24+
""""""""""""""""""""
25+
26+
!!! Changes made in this directory are subject to being overwritten by automation !!!
27+
28+
The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first.

‎eng/common/generate-sbom-prep.ps1‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
Param(
2+
[Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed
3+
)
4+
5+
# create directory for sbom manifest to be placed
6+
if (!(Test-Path -path $ManifestDirPath))
7+
{
8+
Write-Host "Creating dir $ManifestDirPath"
9+
New-Item -ItemType Directory -path $ManifestDirPath
10+
Write-Host "Successfully created directory $ManifestDirPath"
11+
}
12+
13+
Write-Host "Updating artifact name"
14+
$artifact_name = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -replace '["/:<>\\|?@*"() ]', '_'
15+
Write-Host "Artifact name $artifact_name"
16+
Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$artifact_name"

‎eng/common/generate-sbom-prep.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env bash
2+
3+
source="${BASH_SOURCE[0]}"
4+
5+
manifest_dir=$1
6+
7+
if [ ! -d "$manifest_dir" ] ; then
8+
mkdir -p "$manifest_dir"
9+
echo "Sbom directory created." $manifest_dir
10+
else
11+
Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder."
12+
fi
13+
14+
artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
15+
echo "Artifact name before : "$artifact_name
16+
# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
17+
safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
18+
echo "Artifact name after : "$safe_artifact_name
19+
export ARTIFACT_NAME=$safe_artifact_name
20+
echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name"
21+
22+
exit 0

‎eng/common/templates/steps/generate-sbom.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated.
2+
# PackageName - The name of the package this SBOM represents.
3+
# PackageVersion - The version of the package this SBOM represents.
4+
# ManifestDirPath - The path of the directory where the generated manifest files will be placed
5+
6+
parameters:
7+
PackageVersion: 7.0.0
8+
BuildDropPath: '$(Build.SourcesDirectory)\artifacts'
9+
PackageName: '.NET'
10+
ManifestDirPath: '$(Build.SourcesDirectory)\artifacts'
11+
sbomContinueOnError: true
12+
13+
steps:
14+
- task: PowerShell@2
15+
displayName: Prep for SBOM generation in (Non-linux)
16+
condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin'))
17+
inputs:
18+
filePath: ./eng/common/generate-sbom-prep.ps1
19+
arguments: ${{parameters.manifestDirPath}}
20+
21+
# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461
22+
- script: |
23+
chmod +x ./eng/common/generate-sbom-prep.sh
24+
./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}}
25+
displayName: Prep for SBOM generation in (Linux)
26+
condition: eq(variables['Agent.Os'], 'Linux')
27+
continueOnError: ${{ parameters.sbomContinueOnError }}
28+
29+
- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
30+
displayName: 'Generate SBOM manifest'
31+
continueOnError: ${{ parameters.sbomContinueOnError }}
32+
inputs:
33+
PackageName: ${{ parameters.packageName }}
34+
BuildDropPath: ${{ parameters.buildDropPath }}
35+
PackageVersion: ${{ parameters.packageVersion }}
36+
ManifestDirPath: ${{ parameters.manifestDirPath }}

‎eng/pipelines/official.yml‎

Lines changed: 87 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,93 @@
11
parameters:
2-
- name: DartLabEnvironment
3-
displayName: DartLab Environment
2+
- name: RunBuildForPublishing
3+
displayName: Build bits for publishing
4+
type: boolean
5+
default: true
6+
# Disable on this branch.
7+
# - name: RunCrossFrameworkTestsOnWindows
8+
# displayName: Run cross framweork tests on Windows
9+
# type: boolean
10+
# default: true
11+
- name: RunFunctionalTestsOnWindows
12+
displayName: Run functional tests on Windows
13+
type: boolean
14+
default: true
15+
- name: RunSourceBuild
16+
displayName: Run source build
17+
type: boolean
18+
default: true
19+
- name: RunTestsOnLinux
20+
displayName: Run tests on Linux
21+
type: boolean
22+
default: true
23+
- name: RunTestsOnMac
24+
displayName: Run tests on Mac
25+
type: boolean
26+
default: true
27+
- name: RunMonoTestsOnMac
28+
displayName: Run Mono tests on Mac
29+
type: boolean
30+
default: true
31+
- name: SigningType
32+
displayName: Type of signing to use
433
type: string
5-
default: Production
34+
default: Real
635
values:
7-
- Production
8-
- Staging
9-
- name: E2EPart1AgentCleanup
10-
displayName: Delete or keep E2E Part 1 machine for debugging
11-
type: string
12-
default: delete
13-
values:
14-
- delete
15-
- stop
16-
- name: E2EPart2AgentCleanup
17-
displayName: Delete or keep E2E Part 2 machine for debugging
18-
type: string
19-
default: delete
20-
values:
21-
- delete
22-
- stop
23-
- name: ApexAgentCleanup
24-
displayName: Delete or keep VS Apex test machine for debugging
25-
type: string
26-
default: delete
27-
values:
28-
- delete
29-
- stop
36+
- Real
37+
- Test
38+
39+
resources:
40+
repositories:
41+
- repository: MicroBuildTemplate
42+
type: git
43+
name: 1ESPipelineTemplates/MicroBuildTemplate
44+
ref: refs/tags/release
45+
46+
variables:
47+
BINLOG_DIRECTORY: $(Build.StagingDirectory)/binlog
48+
DOTNET_NOLOGO: 1
49+
NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY: 3,1000
50+
Codeql.Enabled: false
51+
Codeql.TSAEnabled: false
52+
RunBuildForPublishing: ${{ parameters.RunBuildForPublishing }}
53+
# Disable on this branch. RunCrossFrameworkTestsOnWindows: ${{ parameters.RunCrossFrameworkTestsOnWindows }}
54+
RunFunctionalTestsOnWindows: ${{ parameters.RunFunctionalTestsOnWindows }}
55+
RunSourceBuild: ${{ parameters.RunSourceBuild }}
56+
RunTestsOnLinux: ${{ parameters.RunTestsOnLinux }}
57+
RunTestsOnMac: ${{ parameters.RunTestsOnMac }}
58+
RunMonoTestsOnMac: ${{ parameters.RunMonoTestsOnMac }}
3059

3160
extends:
32-
template: templates/pipeline.yml
61+
template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate
3362
parameters:
34-
isOfficialBuild: true
35-
DartLabEnvironment: ${{parameters.DartLabEnvironment}}
36-
E2EPart1AgentCleanup: ${{parameters.E2EPart1AgentCleanup}}
37-
E2EPart2AgentCleanup: ${{parameters.E2EPart2AgentCleanup}}
38-
ApexAgentCleanup: ${{parameters.ApexAgentCleanup}}
63+
sdl:
64+
sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES
65+
binskim:
66+
enabled: true
67+
scanOutputDirectoryOnly: true
68+
sbom:
69+
enabled: true
70+
credscan:
71+
enabled: false
72+
pool:
73+
name: VSEngSS-MicroBuild2019-1ES
74+
os: windows
75+
demands:
76+
- ImageOverride -equals server2022-microbuildVS2019-1es
77+
featureFlags:
78+
enablePrepareFilesForSbom: true
79+
customBuildTags:
80+
- ES365AIMigrationTooling
81+
stages:
82+
- template: /eng/pipelines/templates/pipeline.yml@self
83+
parameters:
84+
isOfficialBuild: true
85+
NuGetLocalizationType: Full
86+
RunBuildForPublishing: ${{parameters.RunBuildForPublishing}}
87+
# Disable on this branch. RunCrossFrameworkTestsOnWindows: ${{parameters.RunCrossFrameworkTestsOnWindows}}
88+
RunFunctionalTestsOnWindows: ${{parameters.RunFunctionalTestsOnWindows}}
89+
RunSourceBuild: ${{parameters.RunSourceBuild}}
90+
RunTestsOnLinux: ${{parameters.RunTestsOnLinux}}
91+
RunTestsOnMac: ${{parameters.RunTestsOnMac}}
92+
RunMonoTestsOnMac: ${{parameters.RunMonoTestsOnMac}}
93+
SigningType: ${{ parameters.SigningType }}

‎eng/pipelines/pull_request.yml‎

Lines changed: 90 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,93 @@
11
parameters:
2-
- name: DartLabEnvironment
3-
displayName: DartLab Environment
4-
type: string
5-
default: Production
6-
values:
7-
- Production
8-
- Staging
9-
- name: E2EPart1AgentCleanup
10-
displayName: Delete or keep E2E Part 1 machine for debugging
11-
type: string
12-
default: delete
13-
values:
14-
- delete
15-
- stop
16-
- name: E2EPart2AgentCleanup
17-
displayName: Delete or keep E2E Part 2 machine for debugging
18-
type: string
19-
default: delete
20-
values:
21-
- delete
22-
- stop
23-
- name: ApexAgentCleanup
24-
displayName: Delete or keep VS Apex test machine for debugging
25-
type: string
26-
default: delete
27-
values:
28-
- delete
29-
- stop
30-
2+
- name: NuGetLocalizationType
3+
displayName: Whether to do production-ready localization (Full), or pseudo-localization, aka PLOC, (Pseudo) for testing.
4+
type: string
5+
default: Full
6+
values:
7+
- Full
8+
- Pseudo
9+
- name: RunBuildForPublishing
10+
displayName: Build bits for publishing
11+
type: boolean
12+
default: true
13+
# Disable on this branch.
14+
# - name: RunCrossFrameworkTestsOnWindows
15+
# displayName: Run cross framweork tests on Windows
16+
# type: boolean
17+
# default: true
18+
- name: RunFunctionalTestsOnWindows
19+
displayName: Run functional tests on Windows
20+
type: boolean
21+
default: true
22+
- name: RunSourceBuild
23+
displayName: Run source build
24+
type: boolean
25+
default: true
26+
- name: RunTestsOnLinux
27+
displayName: Run tests on Linux
28+
type: boolean
29+
default: true
30+
- name: RunTestsOnMac
31+
displayName: Run tests on Mac
32+
type: boolean
33+
default: true
34+
- name: RunMonoTestsOnMac
35+
displayName: Run Mono tests on Mac
36+
type: boolean
37+
default: true
38+
- name: SigningType
39+
displayName: Type of signing to use
40+
type: string
41+
default: Test
42+
resources:
43+
repositories:
44+
- repository: MicroBuildTemplate
45+
type: git
46+
name: 1ESPipelineTemplates/MicroBuildTemplate
47+
ref: refs/tags/release
48+
variables:
49+
BINLOG_DIRECTORY: $(Build.StagingDirectory)/binlog
50+
DOTNET_NOLOGO: 1
51+
NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY: 3,1000
52+
Codeql.Enabled: false
53+
Codeql.TSAEnabled: false
54+
RunBuildForPublishing: ${{ parameters.RunBuildForPublishing }}
55+
# Disable on this branch. RunCrossFrameworkTestsOnWindows: ${{ parameters.RunCrossFrameworkTestsOnWindows }}
56+
RunFunctionalTestsOnWindows: ${{ parameters.RunFunctionalTestsOnWindows }}
57+
RunSourceBuild: ${{ parameters.RunSourceBuild }}
58+
RunTestsOnLinux: ${{ parameters.RunTestsOnLinux }}
59+
RunTestsOnMac: ${{ parameters.RunTestsOnMac }}
60+
RunMonoTestsOnMac: ${{ parameters.RunMonoTestsOnMac }}
3161
extends:
32-
template: templates/pipeline.yml
62+
template: azure-pipelines/MicroBuild.1ES.Unofficial.yml@MicroBuildTemplate
3363
parameters:
34-
DartLabEnvironment: ${{parameters.DartLabEnvironment}}
35-
E2EPart1AgentCleanup: ${{parameters.E2EPart1AgentCleanup}}
36-
E2EPart2AgentCleanup: ${{parameters.E2EPart2AgentCleanup}}
37-
ApexAgentCleanup: ${{parameters.ApexAgentCleanup}}
64+
sdl:
65+
sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES
66+
binskim:
67+
enabled: true
68+
scanOutputDirectoryOnly: true
69+
sbom:
70+
enabled: true
71+
credscan:
72+
enabled: false
73+
pool:
74+
name: VSEngSS-MicroBuild2019-1ES
75+
os: windows
76+
demands:
77+
- ImageOverride -equals server2022-microbuildVS2019-1es
78+
featureFlags:
79+
enablePrepareFilesForSbom: true
80+
customBuildTags:
81+
- ES365AIMigrationTooling
82+
stages:
83+
- template: /eng/pipelines/templates/pipeline.yml@self
84+
parameters:
85+
NuGetLocalizationType: ${{parameters.NuGetLocalizationType}}
86+
RunBuildForPublishing: ${{parameters.RunBuildForPublishing}}
87+
# Disable on this branch. RunCrossFrameworkTestsOnWindows: ${{parameters.RunCrossFrameworkTestsOnWindows}}
88+
RunFunctionalTestsOnWindows: ${{parameters.RunFunctionalTestsOnWindows}}
89+
RunSourceBuild: ${{parameters.RunSourceBuild}}
90+
RunTestsOnLinux: ${{parameters.RunTestsOnLinux}}
91+
RunTestsOnMac: ${{parameters.RunTestsOnMac}}
92+
RunMonoTestsOnMac: ${{parameters.RunMonoTestsOnMac}}
93+
SigningType: ${{ parameters.SigningType }}

‎eng/pipelines/templates/Apex_Tests_On_Windows.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,8 @@ stages:
5656
inlineScript: |
5757
Get-ChildItem Env: | Sort-Object Name | Format-Table -Wrap -AutoSize
5858
59-
- task: DownloadBuildArtifacts@0
60-
displayName: "Download Build artifacts"
59+
- task: DownloadPipelineArtifact@0
60+
displayName: "Download Pipeline artifacts"
6161
inputs:
6262
artifactName: "VS15"
6363
downloadPath: "$(Build.Repository.LocalPath)/artifacts"

0 commit comments

Comments
 (0)