Skip to content

Commit 61fed82

Browse files
authored
[Database Migration] Fix CI and Enhance protection. (#7150)
* fix and generate the artifacts for different target databases * delete csproj * Fix CI * Exclude scripts * Wait for the logger to set the telemetry * nit changes * throw exception during deployment * Enhance protection * Add some tests * nit change * Clean one case
1 parent 7b30ebf commit 61fed82

10 files changed

Lines changed: 287 additions & 23 deletions

File tree

build.ps1

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,12 +94,14 @@ Invoke-BuildStep 'Building solution' {
9494
Build-Solution $Configuration $BuildNumber -MSBuildVersion "15" $SolutionPath -SkipRestore:$SkipRestore -MSBuildProperties "/p:MvcBuildViews=true" `
9595
} `
9696
-ev +BuildErrors
97-
97+
9898
Invoke-BuildStep 'Creating artifacts' { `
9999
New-ProjectPackage (Join-Path $PSScriptRoot "src\NuGetGallery.Core\NuGetGallery.Core.csproj") -Configuration $Configuration -Symbols -BuildNumber $BuildNumber -Version $SemanticVersion -PackageId "NuGetGallery.Core$PackageSuffix"
100100
New-ProjectPackage (Join-Path $PSScriptRoot "src\NuGet.Services.Entities\NuGet.Services.Entities.csproj") -Configuration $Configuration -Symbols -BuildNumber $BuildNumber -Version $SemanticVersion
101101
New-ProjectPackage (Join-Path $PSScriptRoot "src\NuGet.Services.DatabaseMigration\NuGet.Services.DatabaseMigration.csproj") -Configuration $Configuration -Symbols -BuildNumber $BuildNumber -Version $SemanticVersion
102-
New-Package (Join-Path $PSScriptRoot "src\DatabaseMigrationTools\DatabaseMigrationTools.csproj") -Configuration $Configuration -BuildNumber $BuildNumber -Version $SemanticVersion -Branch $Branch -MSBuildVersion "15"
102+
New-Package (Join-Path $PSScriptRoot "src\DatabaseMigrationTools\DatabaseMigration.Gallery.nuspec") -Configuration $Configuration -BuildNumber $BuildNumber -Version $SemanticVersion -Branch $Branch -MSBuildVersion "15"
103+
New-Package (Join-Path $PSScriptRoot "src\DatabaseMigrationTools\DatabaseMigration.SupportRequest.nuspec") -Configuration $Configuration -BuildNumber $BuildNumber -Version $SemanticVersion -Branch $Branch -MSBuildVersion "15"
104+
New-Package (Join-Path $PSScriptRoot "src\DatabaseMigrationTools\DatabaseMigration.Validation.nuspec") -Configuration $Configuration -BuildNumber $BuildNumber -Version $SemanticVersion -Branch $Branch -MSBuildVersion "15"
103105
} `
104106
-ev +BuildErrors
105107

sign.thirdparty.targets

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
2+
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<ItemGroup>
4+
<SignFilesDependsOn Include="EnumerateThirdPartyBinariesToSign" />
5+
</ItemGroup>
6+
<Target Name="EnumerateThirdPartyBinariesToSign" AfterTargets="AfterBuild" Condition="'$(SignType)' != 'none'">
7+
<ItemGroup>
8+
<ThirdPartyBinaries Include="AnglicanGeek.MarkdownMailer.dll" />
9+
<ThirdPartyBinaries Include="Autofac.dll" />
10+
<ThirdPartyBinaries Include="Autofac.Extensions.DependencyInjection.dll" />
11+
<ThirdPartyBinaries Include="Autofac.Integration.Mvc.dll" />
12+
<ThirdPartyBinaries Include="Autofac.Integration.Mvc.Owin.dll" />
13+
<ThirdPartyBinaries Include="Autofac.Integration.Owin.dll" />
14+
<ThirdPartyBinaries Include="Autofac.Integration.WebApi.dll" />
15+
<ThirdPartyBinaries Include="CommonMark.dll" />
16+
<ThirdPartyBinaries Include="Dapper.StrongName.dll" />
17+
<ThirdPartyBinaries Include="DynamicData.EFCodeFirstProvider.dll" />
18+
<ThirdPartyBinaries Include="Elmah.dll" />
19+
<ThirdPartyBinaries Include="ICSharpCode.SharpZipLib.dll" />
20+
<ThirdPartyBinaries Include="Lucene.Net.Contrib.Analyzers.dll" />
21+
<ThirdPartyBinaries Include="Lucene.Net.Contrib.Core.dll" />
22+
<ThirdPartyBinaries Include="Lucene.Net.Contrib.FastVectorHighlighter.dll" />
23+
<ThirdPartyBinaries Include="Lucene.Net.Contrib.Highlighter.dll" />
24+
<ThirdPartyBinaries Include="Lucene.Net.Contrib.Memory.dll" />
25+
<ThirdPartyBinaries Include="Lucene.Net.Contrib.Queries.dll" />
26+
<ThirdPartyBinaries Include="Lucene.Net.Contrib.Regex.dll" />
27+
<ThirdPartyBinaries Include="Lucene.Net.Contrib.SimpleFacetedSearch.dll" />
28+
<ThirdPartyBinaries Include="Lucene.Net.Contrib.Snowball.dll" />
29+
<ThirdPartyBinaries Include="Lucene.Net.Contrib.SpellChecker.dll" />
30+
<ThirdPartyBinaries Include="Lucene.Net.dll" />
31+
<ThirdPartyBinaries Include="Markdig.dll" />
32+
<ThirdPartyBinaries Include="MarkdownSharp.dll" />
33+
<ThirdPartyBinaries Include="Microsoft.ApplicationServer.Caching.Client.dll" />
34+
<ThirdPartyBinaries Include="Microsoft.ApplicationServer.Caching.Core.dll" />
35+
<ThirdPartyBinaries Include="Microsoft.AspNet.WebApi.MessageHandlers.Compression.dll" />
36+
<ThirdPartyBinaries Include="Microsoft.CookieCompliance.dll" />
37+
<ThirdPartyBinaries Include="Microsoft.CookieCompliance.NetStd.dll" />
38+
<ThirdPartyBinaries Include="Newtonsoft.Json.dll" />
39+
<ThirdPartyBinaries Include="Polly.dll" />
40+
<ThirdPartyBinaries Include="Polly.Extensions.Http.dll" />
41+
<ThirdPartyBinaries Include="QueryInterceptor.dll" />
42+
<ThirdPartyBinaries Include="RouteMagic.dll" />
43+
<ThirdPartyBinaries Include="Serilog.dll" />
44+
<ThirdPartyBinaries Include="Serilog.Enrichers.Environment.dll" />
45+
<ThirdPartyBinaries Include="Serilog.Enrichers.Process.dll" />
46+
<ThirdPartyBinaries Include="Serilog.Extensions.Logging.dll" />
47+
<ThirdPartyBinaries Include="Serilog.Sinks.ApplicationInsights.dll" />
48+
<ThirdPartyBinaries Include="Serilog.Sinks.ColoredConsole.dll" />
49+
<ThirdPartyBinaries Include="SerilogTraceListener.dll" />
50+
<ThirdPartyBinaries Include="WebActivator.dll" />
51+
<ThirdPartyBinaries Include="WebActivatorEx.dll" />
52+
<ThirdPartyBinaries Include="WebApi.OutputCache.Core.dll" />
53+
<ThirdPartyBinaries Include="WebApi.OutputCache.V2.dll" />
54+
<ThirdPartyBinaries Include="WebBackgrounder.dll" />
55+
<ThirdPartyBinaries Include="WebBackgrounder.EntityFramework.dll" />
56+
</ItemGroup>
57+
<ItemGroup>
58+
<FilesToSign Include="$(OutDir)%(ThirdPartyBinaries.Identity)" Condition="Exists('$(OutDir)%(ThirdPartyBinaries.Identity)')">
59+
<Authenticode>3PartySHA2</Authenticode>
60+
</FilesToSign>
61+
</ItemGroup>
62+
<Message Text="Files to sign:%0A@(FilesToSign, '%0A')" Importance="High" />
63+
</Target>
64+
</Project>

src/DatabaseMigrationTools/DatabaseMigrationTools.csproj

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
<Reference Include="System.Xml" />
4444
</ItemGroup>
4545
<ItemGroup>
46-
<Compile Include="MigrationTargetDatabaseArgumentNames.cs" />
4746
<Compile Include="SupportRequestDbMigrationContext.cs" />
4847
<Compile Include="GalleryDbMigrationContext.cs" />
4948
<Compile Include="MigrationContextFactory.cs" />
@@ -126,4 +125,11 @@
126125
</PropertyGroup>
127126
<Import Project="$(SignPath)\sign.targets" Condition="Exists('$(SignPath)\sign.targets')" />
128127
<Import Project="$(SignPath)\sign.microbuild.targets" Condition="Exists('$(SignPath)\sign.microbuild.targets')" />
128+
<ItemGroup>
129+
<PowerShellScriptsToNotSign Include="PreDeploy.ps1" Visible="false" />
130+
<PowerShellScriptsToNotSign Include="Functions.ps1" Visible="false" />
131+
<ExecutablesToNotSign Include="nssm.exe" Visible="false" />
132+
</ItemGroup>
133+
<Import Project="$(SignPath)\sign.scripts.targets" Condition="Exists('$(SignPath)\sign.scripts.targets')" />
134+
<Import Project="..\..\sign.thirdparty.targets" />
129135
</Project>

src/DatabaseMigrationTools/Program.cs

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,24 @@
33

44
using NuGet.Jobs;
55
using NuGet.Services.DatabaseMigration;
6+
using System.Threading;
67

78
namespace NuGetGallery.DatabaseMigrationTools
89
{
910
class Program
1011
{
11-
static void Main(string[] args)
12+
static int Main(string[] args)
1213
{
1314
var migrationContextFactory = new MigrationContextFactory();
1415
var job = new Job(migrationContextFactory);
1516
JobRunner.RunOnce(job, args).GetAwaiter().GetResult();
17+
18+
// Have to use Thread.Sleep() and wait for the logger.
19+
// "TelemetryConfiguration.Active.TelemetryChannel.Flush()" is not reliable.
20+
// Hit the issue: https://github.com/Microsoft/ApplicationInsights-dotnet/issues/407
21+
Thread.Sleep(30000);
22+
23+
return job.ExitCode;
1624
}
1725
}
1826
}
Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,4 @@
1-
.\RunJob.cmd
1+
.\RunJob.cmd
2+
if ($LastExitCode -ne 0) {
3+
throw "Database Migration Job Failed"
4+
}

src/NuGet.Services.DatabaseMigration/Job.cs

Lines changed: 101 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,32 +20,100 @@ namespace NuGet.Services.DatabaseMigration
2020
{
2121
public class Job : JsonConfigurationJob
2222
{
23-
private const string MigrationTargetDatabaseArgument = "MigrationTargetDatabase";
23+
public int ExitCode { get; set; }
2424

2525
private string _migrationTargetDatabase;
2626
private IMigrationContextFactory _migrationContextFactory;
2727

28+
private const string MigrationTargetDatabaseArgument = "MigrationTargetDatabase";
29+
// There is a Gallery migration file which doens't exist in the local migration folder;
30+
// Need to skip this migration file for the validation check.
31+
private const string SkipGalleryDatabaseMigrationFile = "201304262247205_CuratedPackagesUniqueIndex";
32+
2833
public Job(IMigrationContextFactory migrationContextFactory)
2934
{
3035
_migrationContextFactory = migrationContextFactory;
3136
}
3237

3338
public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
3439
{
35-
base.Init(serviceContainer, jobArgsDictionary);
36-
_migrationTargetDatabase = JobConfigurationManager.GetArgument(jobArgsDictionary, MigrationTargetDatabaseArgument);
40+
try
41+
{
42+
base.Init(serviceContainer, jobArgsDictionary);
43+
_migrationTargetDatabase = JobConfigurationManager.GetArgument(jobArgsDictionary, MigrationTargetDatabaseArgument);
44+
}
45+
catch (Exception)
46+
{
47+
ExitCode = 1;
48+
throw;
49+
}
3750
}
3851

3952
public override async Task Run()
4053
{
4154
Logger.LogInformation("Initializing database migration context...");
42-
var migrationContext = await _migrationContextFactory.CreateMigrationContextAsync(_migrationTargetDatabase, _serviceProvider);
4355

44-
ExecuteDatabaseMigration(migrationContext.GetDbMigrator,
45-
migrationContext.SqlConnection,
46-
migrationContext.SqlConnectionAccessToken);
56+
IMigrationContext migrationContext = null;
57+
try
58+
{
59+
migrationContext = await _migrationContextFactory.CreateMigrationContextAsync(_migrationTargetDatabase, _serviceProvider);
4760

48-
migrationContext.SqlConnection.Dispose();
61+
ExecuteDatabaseMigration(migrationContext.GetDbMigrator,
62+
migrationContext.SqlConnection,
63+
migrationContext.SqlConnectionAccessToken);
64+
}
65+
catch (Exception)
66+
{
67+
ExitCode = 1;
68+
throw;
69+
}
70+
finally
71+
{
72+
migrationContext?.SqlConnection?.Dispose();
73+
}
74+
}
75+
76+
public void CheckIsValidMigration(List<string> databaseMigrations, List<string> localMigrations)
77+
{
78+
if (databaseMigrations == null)
79+
{
80+
throw new ArgumentNullException(nameof(databaseMigrations));
81+
}
82+
if (localMigrations == null)
83+
{
84+
throw new ArgumentNullException(nameof(localMigrations));
85+
}
86+
if (databaseMigrations.Count == 0)
87+
{
88+
throw new InvalidOperationException("Migration validation failed: Unexpected empty history of database migrations.");
89+
}
90+
if (localMigrations.Count == 0)
91+
{
92+
throw new InvalidOperationException("Migration validation failed: Unexpected empty history of local migrations.");
93+
}
94+
95+
var databaseMigrationsCursor = 0;
96+
var localMigrationsCursor = 0;
97+
while (databaseMigrationsCursor < databaseMigrations.Count &&
98+
localMigrationsCursor < localMigrations.Count)
99+
{
100+
if (_migrationTargetDatabase != null &&
101+
_migrationTargetDatabase.Equals(MigrationTargetDatabaseArgumentNames.GalleryDatabase) &&
102+
databaseMigrations[databaseMigrationsCursor].Equals(SkipGalleryDatabaseMigrationFile))
103+
{
104+
databaseMigrationsCursor++;
105+
}
106+
else
107+
{
108+
if (!databaseMigrations[databaseMigrationsCursor].Equals(localMigrations[localMigrationsCursor]))
109+
{
110+
throw new InvalidOperationException($"Migration validation failed: Mismatch local migration file: {localMigrations[localMigrationsCursor]}.");
111+
}
112+
113+
localMigrationsCursor++;
114+
databaseMigrationsCursor++;
115+
}
116+
}
49117
}
50118

51119
private void ExecuteDatabaseMigration(Func<DbMigrator> getMigrator, SqlConnection sqlConnection, string accessToken)
@@ -56,32 +124,48 @@ private void ExecuteDatabaseMigration(Func<DbMigrator> getMigrator, SqlConnectio
56124
OverwriteSqlConnection(migrator, sqlConnection, accessToken);
57125
OverwriteSqlConnection(migratorForScripting, sqlConnection, accessToken);
58126

59-
Logger.LogInformation("Target database is: {DataSource}/{Database}", sqlConnection.DataSource, sqlConnection.Database);
127+
var sqlConnectionDataSource = sqlConnection.DataSource;
128+
var sqlConnectionDatabase = sqlConnection.Database;
129+
130+
Logger.LogInformation("Target database is: {DataSource}/{Database}.", sqlConnectionDataSource, sqlConnectionDatabase);
60131
var pendingMigrations = migrator.GetPendingMigrations();
61132
if (pendingMigrations.Count() > 0)
62133
{
63-
Logger.LogInformation("Applying pending migrations: \n {PendingMigrations}", String.Join("\n", pendingMigrations));
134+
var databaseMigrations = migrator.GetDatabaseMigrations().ToList();
135+
databaseMigrations.Reverse();
136+
var localMigrations = migrator.GetLocalMigrations().ToList();
137+
try
138+
{
139+
CheckIsValidMigration(databaseMigrations, localMigrations);
140+
}
141+
catch (Exception e)
142+
{
143+
Logger.LogError(0, e, "Validation check of database migrations failed.");
144+
throw;
145+
}
146+
147+
Logger.LogInformation("Applying pending migrations: \n {PendingMigrations}.", String.Join("\n", pendingMigrations));
64148

65149
var migratorScripter = new MigratorScriptingDecorator(migratorForScripting);
66150
var migrationScripts = migratorScripter.ScriptUpdate(sourceMigration: null, targetMigration: null);
67-
Logger.LogInformation("Applying explicit migration SQL scripts: \n {migrationScripts}", migrationScripts);
151+
Logger.LogInformation("Applying explicit migration SQL scripts: \n {migrationScripts}.", migrationScripts);
68152

69153
try
70154
{
71155
Logger.LogInformation("Executing migrations...");
72156

73157
migrator.Update();
74158

75-
Logger.LogInformation("Finished executing {pendingMigrationsCount} migrations successfully on the target database {DataSource}/{Database}",
159+
Logger.LogInformation("Finished executing {pendingMigrationsCount} migrations successfully on the target database {DataSource}/{Database}.",
76160
pendingMigrations.Count(),
77-
sqlConnection.DataSource,
78-
sqlConnection.Database);
161+
sqlConnectionDataSource,
162+
sqlConnectionDatabase);
79163
}
80164
catch (Exception e)
81165
{
82-
Logger.LogError(0, e, "Failed to execute migrations on the target database {DataSource}/{Database}",
83-
sqlConnection.DataSource,
84-
sqlConnection.Database);
166+
Logger.LogError(0, e, "Failed to execute migrations on the target database {DataSource}/{Database}.",
167+
sqlConnectionDataSource,
168+
sqlConnectionDatabase);
85169
throw;
86170
}
87171
}

src/DatabaseMigrationTools/MigrationTargetDatabaseArgumentNames.cs renamed to src/NuGet.Services.DatabaseMigration/MigrationTargetDatabaseArgumentNames.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

4-
namespace NuGetGallery.DatabaseMigrationTools
4+
namespace NuGet.Services.DatabaseMigration
55
{
66
public static class MigrationTargetDatabaseArgumentNames
77
{

src/NuGet.Services.DatabaseMigration/NuGet.Services.DatabaseMigration.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
<Compile Include="IMigrationContext.cs" />
5757
<Compile Include="IMigrationContextFactory.cs" />
5858
<Compile Include="Job.cs" />
59+
<Compile Include="MigrationTargetDatabaseArgumentNames.cs" />
5960
<Compile Include="Properties\AssemblyInfo.cs" />
6061
<Compile Include="Properties\AssemblyInfo.*.cs" />
6162
</ItemGroup>

tests/NuGet.Services.DatabaseMigration.Facts/NuGet.Services.DatabaseMigration.Facts.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
<Compile Include="DatabaseMigrationFacts.cs" />
4747
<Compile Include="PendingMigrationsFacts.cs" />
4848
<Compile Include="Properties\AssemblyInfo.cs" />
49+
<Compile Include="ValidateMigrationsFacts.cs" />
4950
</ItemGroup>
5051
<ItemGroup>
5152
<PackageReference Include="Moq">

0 commit comments

Comments
 (0)