Skip to content

Commit 3050c87

Browse files
zivkanzivkan
authored
Improve Apex test reliability (#6973)
1 parent 663b89f commit 3050c87

9 files changed

Lines changed: 51 additions & 45 deletions

File tree

‎eng/pipelines/dailytests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,6 @@ stages:
5959
value: $[stageDependencies.Build.Build.outputs['dartlab_variables.VsBootstrapperBranch']]
6060
bootstrapperUrl: $(bootstrapperUrl)
6161
runSettingsURI: https://vsdrop.corp.microsoft.com/file/v1/$(RunSettingsDrop);NuGet.Tests.Apex.Daily.runsettings
62-
testExecutionJobTimeoutInMinutes: 150
62+
testExecutionJobTimeoutInMinutes: 380 # cloud test minimum
6363
testDeploymentCleanupMode: ${{parameters.ApexAgentCleanup}}
6464
VsBootstrapperBranch: $(VsBootstrapperBranch)

‎eng/pipelines/vs-tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,6 @@ stages:
143143
value: $[stageDependencies.Build.Build.outputs['dartlab_variables.VsBootstrapperBranch']]
144144
bootstrapperUrl: $(bootstrapperUrl)
145145
runSettingsURI: https://vsdrop.corp.microsoft.com/file/v1/$(RunSettingsDrop);NuGet.Tests.Apex.runsettings
146-
testExecutionJobTimeoutInMinutes: 150
146+
testExecutionJobTimeoutInMinutes: 380 # cloud test minimum
147147
testDeploymentCleanupMode: ${{parameters.ApexAgentCleanup}}
148148
VsBootstrapperBranch: $(VsBootstrapperBranch)

‎test/NuGet.Tests.Apex/NuGet.Tests.Apex.Daily/NuGetConsoleTestCase.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ public async Task UpdateAllPackagesInPMC(ProjectTemplate projectTemplate, string
101101
}
102102
}
103103

104+
[Ignore("MAUI projects cause UAC prompt for the remainder of the test run, blocking all tests video recordings")]
104105
[DataTestMethod]
105106
[DynamicData(nameof(GetMauiTemplates), DynamicDataSourceType.Method)]
106107
[Timeout(DefaultTimeout)]
@@ -135,6 +136,7 @@ public async Task InstallPackageForPRInPMC(ProjectTemplate projectTemplate)
135136
}
136137
}
137138

139+
[Ignore("MAUI projects cause UAC prompt for the remainder of the test run, blocking all tests video recordings")]
138140
[DataTestMethod]
139141
[DynamicData(nameof(GetMauiTemplates), DynamicDataSourceType.Method)]
140142
[Timeout(DefaultTimeout)]
@@ -176,6 +178,7 @@ public async Task UpdatePackageForPRInPMC(ProjectTemplate projectTemplate)
176178
}
177179
}
178180

181+
[Ignore("MAUI projects cause UAC prompt for the remainder of the test run, blocking all tests video recordings")]
179182
[DataTestMethod]
180183
[DynamicData(nameof(GetMauiTemplates), DynamicDataSourceType.Method)]
181184
[Timeout(DefaultTimeout)]

‎test/NuGet.Tests.Apex/NuGet.Tests.Apex.Daily/NuGetUITestCase.cs‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,16 @@ public class NuGetUITestCase : SharedVisualStudioHostTestClass
1919
private const string TestPackageVersionV1 = "1.0.0";
2020
private const string TestPackageVersionV2 = "2.0.0";
2121

22+
private const ProjectTargetFramework DefaultTargetFramework = ProjectTargetFramework.V48;
23+
2224
private readonly SimpleTestPathContext _pathContext = new SimpleTestPathContext();
2325

2426
[TestMethod]
25-
[DataRow(ProjectTemplate.WebSiteEmpty, ProjectTargetFramework.V48)]
26-
[DataRow(ProjectTemplate.WebSite, ProjectTargetFramework.V48)]
27-
[DataRow(ProjectTemplate.WebSiteRazorV3, ProjectTargetFramework.V48)]
28-
[DataRow(ProjectTemplate.WebSiteDynamicDataEntityFramework, ProjectTargetFramework.V48)]
29-
[DataRow(ProjectTemplate.WebSiteDynamicDataLinqToSql, ProjectTargetFramework.V452)]
27+
[DataRow(ProjectTemplate.WebSiteEmpty, DefaultTargetFramework)]
28+
[DataRow(ProjectTemplate.WebSite, DefaultTargetFramework)]
29+
[DataRow(ProjectTemplate.WebSiteRazorV3, DefaultTargetFramework)]
30+
[DataRow(ProjectTemplate.WebSiteDynamicDataEntityFramework, DefaultTargetFramework)]
31+
[DataRow(ProjectTemplate.WebSiteDynamicDataLinqToSql, DefaultTargetFramework)]
3032
[Timeout(DefaultTimeout)]
3133
public async Task InstallPackageToWebSiteProjectFromUI(ProjectTemplate projectTemplate, ProjectTargetFramework projectTargetFramework)
3234
{
@@ -52,11 +54,11 @@ public async Task InstallPackageToWebSiteProjectFromUI(ProjectTemplate projectTe
5254
}
5355

5456
[TestMethod]
55-
[DataRow(ProjectTemplate.WebSiteEmpty, ProjectTargetFramework.V48)]
56-
[DataRow(ProjectTemplate.WebSite, ProjectTargetFramework.V48)]
57-
[DataRow(ProjectTemplate.WebSiteRazorV3, ProjectTargetFramework.V48)]
58-
[DataRow(ProjectTemplate.WebSiteDynamicDataEntityFramework, ProjectTargetFramework.V48)]
59-
[DataRow(ProjectTemplate.WebSiteDynamicDataLinqToSql, ProjectTargetFramework.V452)]
57+
[DataRow(ProjectTemplate.WebSiteEmpty, DefaultTargetFramework)]
58+
[DataRow(ProjectTemplate.WebSite, DefaultTargetFramework)]
59+
[DataRow(ProjectTemplate.WebSiteRazorV3, DefaultTargetFramework)]
60+
[DataRow(ProjectTemplate.WebSiteDynamicDataEntityFramework, DefaultTargetFramework)]
61+
[DataRow(ProjectTemplate.WebSiteDynamicDataLinqToSql, DefaultTargetFramework)]
6062
[Timeout(DefaultTimeout)]
6163
public async Task UpdateWebSitePackageFromUI(ProjectTemplate projectTemplate, ProjectTargetFramework projectTargetFramework)
6264
{
@@ -85,11 +87,11 @@ public async Task UpdateWebSitePackageFromUI(ProjectTemplate projectTemplate, Pr
8587
}
8688

8789
[TestMethod]
88-
[DataRow(ProjectTemplate.WebSiteEmpty, ProjectTargetFramework.V48)]
89-
[DataRow(ProjectTemplate.WebSite, ProjectTargetFramework.V48)]
90-
[DataRow(ProjectTemplate.WebSiteRazorV3, ProjectTargetFramework.V48)]
91-
[DataRow(ProjectTemplate.WebSiteDynamicDataEntityFramework, ProjectTargetFramework.V48)]
92-
[DataRow(ProjectTemplate.WebSiteDynamicDataLinqToSql, ProjectTargetFramework.V452)]
90+
[DataRow(ProjectTemplate.WebSiteEmpty, DefaultTargetFramework)]
91+
[DataRow(ProjectTemplate.WebSite, DefaultTargetFramework)]
92+
[DataRow(ProjectTemplate.WebSiteRazorV3, DefaultTargetFramework)]
93+
[DataRow(ProjectTemplate.WebSiteDynamicDataEntityFramework, DefaultTargetFramework)]
94+
[DataRow(ProjectTemplate.WebSiteDynamicDataLinqToSql, DefaultTargetFramework)]
9395
[Timeout(DefaultTimeout)]
9496
public async Task UninstallWebSitePackageFromUI(ProjectTemplate projectTemplate, ProjectTargetFramework projectTargetFramework)
9597
{
@@ -128,7 +130,7 @@ public async Task SearchPackageInBrowseTabFromUI()
128130

129131
SolutionService solutionService = VisualStudio.Get<SolutionService>();
130132
solutionService.CreateEmptySolution("TestSolution", _pathContext.SolutionRoot);
131-
ProjectTestExtension project = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, ProjectTargetFramework.V48, "TestProject");
133+
ProjectTestExtension project = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, DefaultTargetFramework, "TestProject");
132134
VisualStudio.ClearOutputWindow();
133135
solutionService.SaveAll();
134136

@@ -159,7 +161,7 @@ public async Task SearchPackageInInstalledTabFromUI()
159161

160162
SolutionService solutionService = VisualStudio.Get<SolutionService>();
161163
solutionService.CreateEmptySolution("TestSolution", _pathContext.SolutionRoot);
162-
ProjectTestExtension project = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, ProjectTargetFramework.V48, "TestProject");
164+
ProjectTestExtension project = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, DefaultTargetFramework, "TestProject");
163165
VisualStudio.ClearOutputWindow();
164166
solutionService.SaveAll();
165167

@@ -189,7 +191,7 @@ public async Task SearchPackageInUpdatesTabFromUI()
189191

190192
SolutionService solutionService = VisualStudio.Get<SolutionService>();
191193
solutionService.CreateEmptySolution("TestSolution", _pathContext.SolutionRoot);
192-
ProjectTestExtension project = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, ProjectTargetFramework.V48, "TestProject");
194+
ProjectTestExtension project = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, DefaultTargetFramework, "TestProject");
193195
VisualStudio.ClearOutputWindow();
194196
solutionService.SaveAll();
195197

@@ -731,7 +733,7 @@ public void VerifyPackageNotRestoredAfterDisablingPackageRestore()
731733
// Arrange
732734
SolutionService solutionService = VisualStudio.Get<SolutionService>();
733735
solutionService.CreateEmptySolution("TestSolution", _pathContext.SolutionRoot);
734-
solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.MauiClassLibrary, "TestProject");
736+
solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.NetCoreClassLib, "TestProject");
735737
VisualStudio.ClearOutputWindow();
736738
solutionService.SaveAll();
737739

‎test/NuGet.Tests.Apex/NuGet.Tests.Apex/NuGetEndToEndTests/IVsServicesTestCase.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ public void SimpleInstallFromIVsInstaller()
3232

3333
SolutionService solutionService = VisualStudio.Get<SolutionService>();
3434
solutionService.CreateEmptySolution();
35-
ProjectTestExtension projExt = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, ProjectTargetFramework.V46, "TestProject");
35+
ProjectTestExtension projExt = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, CommonUtility.DefaultTargetFramework, "TestProject");
3636
EnvDTE.Project project = VisualStudio.Dte.Solution.Projects.Item(1);
3737

3838
// Act
@@ -51,7 +51,7 @@ public void SimpleUninstallFromIVsInstaller()
5151

5252
SolutionService solutionService = VisualStudio.Get<SolutionService>();
5353
solutionService.CreateEmptySolution();
54-
ProjectTestExtension projExt = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, ProjectTargetFramework.V46, "TestProject");
54+
ProjectTestExtension projExt = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, CommonUtility.DefaultTargetFramework, "TestProject");
5555
string projectUniqueName = VisualStudio.Dte.Solution.Projects.Item(1).UniqueName;
5656

5757
// Act
@@ -93,7 +93,7 @@ public async Task SimpleInstallFromIVsInstaller_PackageSourceMapping_WithSingleF
9393

9494
SolutionService solutionService = VisualStudio.Get<SolutionService>();
9595
solutionService.CreateEmptySolution("TestSolution", _pathContext.SolutionRoot);
96-
ProjectTestExtension projExt = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, ProjectTargetFramework.V46, "TestProject");
96+
ProjectTestExtension projExt = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, CommonUtility.DefaultTargetFramework, "TestProject");
9797
solutionService.SaveAll();
9898
EnvDTE.Project project = VisualStudio.Dte.Solution.Projects.Item(1);
9999

@@ -117,7 +117,7 @@ public async Task SimpleUpdateFromIVsInstaller_PackageSourceMapping_WithSingleFe
117117

118118
SolutionService solutionService = VisualStudio.Get<SolutionService>();
119119
solutionService.CreateEmptySolution("TestSolution", _pathContext.SolutionRoot);
120-
ProjectTestExtension projExt = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, ProjectTargetFramework.V46, "TestProject");
120+
ProjectTestExtension projExt = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, CommonUtility.DefaultTargetFramework, "TestProject");
121121
solutionService.SaveAll();
122122
EnvDTE.Project project = VisualStudio.Dte.Solution.Projects.Item(1);
123123

@@ -152,7 +152,7 @@ public async Task SimpleInstallFromIVsInstaller_PackageSourceMapping_WithMultipl
152152

153153
SolutionService solutionService = VisualStudio.Get<SolutionService>();
154154
solutionService.CreateEmptySolution("TestSolution", _pathContext.SolutionRoot);
155-
ProjectTestExtension projExt = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, ProjectTargetFramework.V46, "TestProject");
155+
ProjectTestExtension projExt = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, CommonUtility.DefaultTargetFramework, "TestProject");
156156
solutionService.SaveAll();
157157
EnvDTE.Project project = VisualStudio.Dte.Solution.Projects.Item(1);
158158

@@ -186,7 +186,7 @@ public async Task SimpleUpdateFromIVsInstaller_PackageSourceMapping_WithMultiple
186186

187187
SolutionService solutionService = VisualStudio.Get<SolutionService>();
188188
solutionService.CreateEmptySolution("TestSolution", _pathContext.SolutionRoot);
189-
ProjectTestExtension projExt = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, ProjectTargetFramework.V46, "TestProject");
189+
ProjectTestExtension projExt = solutionService.AddProject(ProjectLanguage.CSharp, ProjectTemplate.ClassLibrary, CommonUtility.DefaultTargetFramework, "TestProject");
190190
solutionService.SaveAll();
191191
EnvDTE.Project project = VisualStudio.Dte.Solution.Projects.Item(1);
192192

‎test/NuGet.Tests.Apex/NuGet.Tests.Apex/NuGetEndToEndTests/NetCoreProjectTestCase.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public void CreateNetCoreProject_AddProjectReference(ProjectTemplate projectTemp
2929
{
3030
using (var testContext = new ApexTestContext(VisualStudio, projectTemplate, Logger, addNetStandardFeeds: true))
3131
{
32-
var project2 = testContext.SolutionService.AddProject(ProjectLanguage.CSharp, projectTemplate, ProjectTargetFramework.V46, "TestProject2");
32+
var project2 = testContext.SolutionService.AddProject(ProjectLanguage.CSharp, projectTemplate, CommonUtility.DefaultTargetFramework, "TestProject2");
3333
project2.Build();
3434

3535
testContext.Project.References.Dte.AddProjectReference(project2);

‎test/NuGet.Tests.Apex/NuGet.Tests.Apex/NuGetEndToEndTests/NuGetConsoleTestCase.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -187,11 +187,11 @@ public async Task NetCoreTransitivePackageReferenceLimitAsync(ProjectTemplate pr
187187
{
188188
using (var testContext = new ApexTestContext(VisualStudio, projectTemplate, Logger, addNetStandardFeeds: true))
189189
{
190-
var project2 = testContext.SolutionService.AddProject(ProjectLanguage.CSharp, projectTemplate, ProjectTargetFramework.V46, "TestProject2");
190+
var project2 = testContext.SolutionService.AddProject(ProjectLanguage.CSharp, projectTemplate, CommonUtility.DefaultTargetFramework, "TestProject2");
191191
project2.Build();
192-
var project3 = testContext.SolutionService.AddProject(ProjectLanguage.CSharp, projectTemplate, ProjectTargetFramework.V46, "TestProject3");
192+
var project3 = testContext.SolutionService.AddProject(ProjectLanguage.CSharp, projectTemplate, CommonUtility.DefaultTargetFramework, "TestProject3");
193193
project3.Build();
194-
var projectX = testContext.SolutionService.AddProject(ProjectLanguage.CSharp, projectTemplate, ProjectTargetFramework.V46, "TestProjectX");
194+
var projectX = testContext.SolutionService.AddProject(ProjectLanguage.CSharp, projectTemplate, CommonUtility.DefaultTargetFramework, "TestProjectX");
195195
projectX.Build();
196196
testContext.SolutionService.Build();
197197

0 commit comments

Comments
 (0)