Skip to content

Commit 4b504f5

Browse files
authored
The new DB migration job (#7095)
* new database migration job * Address feedback and refactor for flexibility and extensibility. * Update * Support logging the SQL scripts * Extend to other databases * Create artifact for database migration tools * Add scripts and nuspec * Roll back wrong build files * Async method * Finalize the structure * Address some feedbacks * Fix some issues and verify EF version * Add signing * new database migration job * Address feedback and refactor for flexibility and extensibility. * Update * Support logging the SQL scripts * Extend to other databases * Create artifact for database migration tools * Add scripts and nuspec * Roll back wrong build files * Async method * Finalize the structure * Address some feedbacks * Fix some issues and verify EF version * Add signing * Rebase and better message * Generate assemblyinfo * Create NuGet.Services.DatabaseMigration package * Add package info for NuGet.Services.DatabaseMigration * Create built artifacts for DatabaseMigrationTools * Address feedback * Keep same with other jobs * Add micro-build * nit change * Move base class to shared package * Extend to Validation database * Update * check in master branch of validation package * nit change
1 parent 00fc0f0 commit 4b504f5

27 files changed

Lines changed: 959 additions & 21 deletions

NuGetGallery.sln

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11

22
Microsoft Visual Studio Solution File, Format Version 12.00
3-
# Visual Studio 15
4-
VisualStudioVersion = 15.0.27323.2
3+
# Visual Studio Version 16
4+
VisualStudioVersion = 16.0.28516.95
55
MinimumVisualStudioVersion = 10.0.40219.1
66
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{96E4AFF8-D3A1-4102-ADCF-05F186F916A9}"
77
ProjectSection(SolutionItems) = preProject
@@ -32,6 +32,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Entities", "
3232
EndProject
3333
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Entities.Tests", "tests\NuGet.Services.Entities.Tests\NuGet.Services.Entities.Tests.csproj", "{79C831E9-7C88-4B98-B084-4DE940C73FC7}"
3434
EndProject
35+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DatabaseMigrationTools", "src\DatabaseMigrationTools\DatabaseMigrationTools.csproj", "{DCED7162-A24C-4579-96C8-544BFAF4C305}"
36+
EndProject
37+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.DatabaseMigration", "src\NuGet.Services.DatabaseMigration\NuGet.Services.DatabaseMigration.csproj", "{F4C8C34F-72A9-4773-A315-8FA3F2D5CE4E}"
38+
EndProject
39+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.DatabaseMigration.Facts", "tests\NuGet.Services.DatabaseMigration.Facts\NuGet.Services.DatabaseMigration.Facts.csproj", "{082357A1-682E-4CCC-8FCD-FA250204CDB6}"
40+
EndProject
3541
Global
3642
GlobalSection(SolutionConfigurationPlatforms) = preSolution
3743
Debug|Any CPU = Debug|Any CPU
@@ -70,6 +76,18 @@ Global
7076
{79C831E9-7C88-4B98-B084-4DE940C73FC7}.Debug|Any CPU.Build.0 = Debug|Any CPU
7177
{79C831E9-7C88-4B98-B084-4DE940C73FC7}.Release|Any CPU.ActiveCfg = Release|Any CPU
7278
{79C831E9-7C88-4B98-B084-4DE940C73FC7}.Release|Any CPU.Build.0 = Release|Any CPU
79+
{DCED7162-A24C-4579-96C8-544BFAF4C305}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
80+
{DCED7162-A24C-4579-96C8-544BFAF4C305}.Debug|Any CPU.Build.0 = Debug|Any CPU
81+
{DCED7162-A24C-4579-96C8-544BFAF4C305}.Release|Any CPU.ActiveCfg = Release|Any CPU
82+
{DCED7162-A24C-4579-96C8-544BFAF4C305}.Release|Any CPU.Build.0 = Release|Any CPU
83+
{F4C8C34F-72A9-4773-A315-8FA3F2D5CE4E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
84+
{F4C8C34F-72A9-4773-A315-8FA3F2D5CE4E}.Debug|Any CPU.Build.0 = Debug|Any CPU
85+
{F4C8C34F-72A9-4773-A315-8FA3F2D5CE4E}.Release|Any CPU.ActiveCfg = Release|Any CPU
86+
{F4C8C34F-72A9-4773-A315-8FA3F2D5CE4E}.Release|Any CPU.Build.0 = Release|Any CPU
87+
{082357A1-682E-4CCC-8FCD-FA250204CDB6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
88+
{082357A1-682E-4CCC-8FCD-FA250204CDB6}.Debug|Any CPU.Build.0 = Debug|Any CPU
89+
{082357A1-682E-4CCC-8FCD-FA250204CDB6}.Release|Any CPU.ActiveCfg = Release|Any CPU
90+
{082357A1-682E-4CCC-8FCD-FA250204CDB6}.Release|Any CPU.Build.0 = Release|Any CPU
7391
EndGlobalSection
7492
GlobalSection(SolutionProperties) = preSolution
7593
HideSolutionNode = FALSE
@@ -83,6 +101,9 @@ Global
83101
{C5849063-8CDC-4561-BA5C-7D97BD905DC3} = {2204C510-A559-4ED7-9590-FDC09093575B}
84102
{6262F4FC-29BE-4226-B676-DB391C89D396} = {155100FF-524B-4CAF-93C6-A57478B3DBAD}
85103
{79C831E9-7C88-4B98-B084-4DE940C73FC7} = {39E54EC3-CBAA-453A-BE64-748FE1559A58}
104+
{DCED7162-A24C-4579-96C8-544BFAF4C305} = {2204C510-A559-4ED7-9590-FDC09093575B}
105+
{F4C8C34F-72A9-4773-A315-8FA3F2D5CE4E} = {155100FF-524B-4CAF-93C6-A57478B3DBAD}
106+
{082357A1-682E-4CCC-8FCD-FA250204CDB6} = {39E54EC3-CBAA-453A-BE64-748FE1559A58}
86107
EndGlobalSection
87108
GlobalSection(ExtensibilityGlobals) = postSolution
88109
SolutionGuid = {064A3BDE-5203-4AD6-A6C9-5CF08301EC8F}

build.ps1

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,9 @@ Invoke-BuildStep 'Set version metadata in AssemblyInfo.cs' {
7979
$Paths = `
8080
(Join-Path $PSScriptRoot "src\NuGetGallery\Properties\AssemblyInfo.g.cs"), `
8181
(Join-Path $PSScriptRoot "src\NuGetGallery.Core\Properties\AssemblyInfo.g.cs"), `
82-
(Join-Path $PSScriptRoot "src\NuGet.Services.Entities\Properties\AssemblyInfo.g.cs")
82+
(Join-Path $PSScriptRoot "src\NuGet.Services.Entities\Properties\AssemblyInfo.g.cs"), `
83+
(Join-Path $PSScriptRoot "src\NuGet.Services.DatabaseMigration\Properties\AssemblyInfo.g.cs"), `
84+
(Join-Path $PSScriptRoot "src\DatabaseMigrationTools\Properties\AssemblyInfo.g.cs")
8385

8486
Foreach ($Path in $Paths) {
8587
Set-VersionInfo -Path $Path -Version $SimpleVersion -Branch $Branch -Commit $CommitSHA
@@ -96,6 +98,8 @@ Invoke-BuildStep 'Building solution' {
9698
Invoke-BuildStep 'Creating artifacts' { `
9799
New-ProjectPackage (Join-Path $PSScriptRoot "src\NuGetGallery.Core\NuGetGallery.Core.csproj") -Configuration $Configuration -Symbols -BuildNumber $BuildNumber -Version $SemanticVersion -PackageId "NuGetGallery.Core$PackageSuffix"
98100
New-ProjectPackage (Join-Path $PSScriptRoot "src\NuGet.Services.Entities\NuGet.Services.Entities.csproj") -Configuration $Configuration -Symbols -BuildNumber $BuildNumber -Version $SemanticVersion
101+
New-ProjectPackage (Join-Path $PSScriptRoot "src\NuGet.Services.DatabaseMigration\NuGet.Services.DatabaseMigration.csproj") -Configuration $Configuration -Symbols -BuildNumber $BuildNumber -Version $SemanticVersion
102+
New-Package (Join-Path $PSScriptRoot "src\DatabaseMigrationTools\DatabaseMigrationTools.csproj") -Configuration $Configuration -BuildNumber $BuildNumber -Version $SemanticVersion -Branch $Branch -MSBuildVersion "15"
99103
} `
100104
-ev +BuildErrors
101105

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<configuration>
3+
<startup>
4+
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2" />
5+
</startup>
6+
</configuration>
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<?xml version="1.0"?>
2+
<package >
3+
<metadata>
4+
<id>DatabaseMigration.Gallery</id>
5+
<version>$version$</version>
6+
<title>DatabaseMigration.Gallery</title>
7+
<authors>.NET Foundation</authors>
8+
<owners>.NET Foundation</owners>
9+
<description>DatabaseMigration.Gallery</description>
10+
<copyright>Copyright .NET Foundation</copyright>
11+
</metadata>
12+
<files>
13+
<file src="bin\$configuration$\*.*" target="bin"/>
14+
<file src="Scripts\PostDeploy.ps1" />
15+
</files>
16+
</package>
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<?xml version="1.0"?>
2+
<package >
3+
<metadata>
4+
<id>DatabaseMigration.SupportRequest</id>
5+
<version>$version$</version>
6+
<title>DatabaseMigration.SupportRequest</title>
7+
<authors>.NET Foundation</authors>
8+
<owners>.NET Foundation</owners>
9+
<description>DatabaseMigration.SupportRequest</description>
10+
<copyright>Copyright .NET Foundation</copyright>
11+
</metadata>
12+
<files>
13+
<file src="bin\$configuration$\*.*" target="bin"/>
14+
<file src="Scripts\PostDeploy.ps1" />
15+
</files>
16+
</package>
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<?xml version="1.0"?>
2+
<package >
3+
<metadata>
4+
<id>DatabaseMigration.Validation</id>
5+
<version>$version$</version>
6+
<title>DatabaseMigration.Validation</title>
7+
<authors>.NET Foundation</authors>
8+
<owners>.NET Foundation</owners>
9+
<description>DatabaseMigration.Validation</description>
10+
<copyright>Copyright .NET Foundation</copyright>
11+
</metadata>
12+
<files>
13+
<file src="bin\$configuration$\*.*" target="bin"/>
14+
<file src="Scripts\PostDeploy.ps1" />
15+
</files>
16+
</package>
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
4+
<PropertyGroup>
5+
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
6+
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
7+
<ProjectGuid>{DCED7162-A24C-4579-96C8-544BFAF4C305}</ProjectGuid>
8+
<OutputType>Exe</OutputType>
9+
<RootNamespace>DatabaseMigrationTools</RootNamespace>
10+
<AssemblyName>DatabaseMigrationTools</AssemblyName>
11+
<TargetFrameworkVersion>v4.6.2</TargetFrameworkVersion>
12+
<FileAlignment>512</FileAlignment>
13+
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
14+
<Deterministic>true</Deterministic>
15+
</PropertyGroup>
16+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
17+
<PlatformTarget>AnyCPU</PlatformTarget>
18+
<DebugSymbols>true</DebugSymbols>
19+
<DebugType>full</DebugType>
20+
<Optimize>false</Optimize>
21+
<OutputPath>bin\Debug\</OutputPath>
22+
<DefineConstants>DEBUG;TRACE</DefineConstants>
23+
<ErrorReport>prompt</ErrorReport>
24+
<WarningLevel>4</WarningLevel>
25+
</PropertyGroup>
26+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
27+
<PlatformTarget>AnyCPU</PlatformTarget>
28+
<DebugType>pdbonly</DebugType>
29+
<Optimize>true</Optimize>
30+
<OutputPath>bin\Release\</OutputPath>
31+
<DefineConstants>TRACE</DefineConstants>
32+
<ErrorReport>prompt</ErrorReport>
33+
<WarningLevel>4</WarningLevel>
34+
</PropertyGroup>
35+
<ItemGroup>
36+
<Reference Include="System" />
37+
<Reference Include="System.Core" />
38+
<Reference Include="System.Xml.Linq" />
39+
<Reference Include="System.Data.DataSetExtensions" />
40+
<Reference Include="Microsoft.CSharp" />
41+
<Reference Include="System.Data" />
42+
<Reference Include="System.Net.Http" />
43+
<Reference Include="System.Xml" />
44+
</ItemGroup>
45+
<ItemGroup>
46+
<Compile Include="MigrationTargetDatabaseArgumentNames.cs" />
47+
<Compile Include="SupportRequestDbMigrationContext.cs" />
48+
<Compile Include="GalleryDbMigrationContext.cs" />
49+
<Compile Include="MigrationContextFactory.cs" />
50+
<Compile Include="Program.cs" />
51+
<Compile Include="Properties\AssemblyInfo.cs" />
52+
<Compile Include="ValidationDbMigrationContext.cs" />
53+
</ItemGroup>
54+
<ItemGroup>
55+
<None Include="App.config" />
56+
</ItemGroup>
57+
<ItemGroup>
58+
<PackageReference Include="Autofac.Extensions.DependencyInjection">
59+
<Version>4.2.0</Version>
60+
</PackageReference>
61+
<PackageReference Include="EntityFramework">
62+
<Version>6.2.0</Version>
63+
</PackageReference>
64+
<PackageReference Include="MicroBuild.Core">
65+
<Version>0.3.0</Version>
66+
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
67+
<PrivateAssets>all</PrivateAssets>
68+
</PackageReference>
69+
<PackageReference Include="Microsoft.Extensions.Configuration">
70+
<Version>1.1.2</Version>
71+
</PackageReference>
72+
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions">
73+
<Version>1.1.2</Version>
74+
</PackageReference>
75+
<PackageReference Include="Microsoft.Extensions.Configuration.Binder">
76+
<Version>1.1.2</Version>
77+
</PackageReference>
78+
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables">
79+
<Version>1.1.2</Version>
80+
</PackageReference>
81+
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions">
82+
<Version>1.1.2</Version>
83+
</PackageReference>
84+
<PackageReference Include="Microsoft.Extensions.Configuration.Json">
85+
<Version>1.1.2</Version>
86+
</PackageReference>
87+
<PackageReference Include="Microsoft.Extensions.Logging">
88+
<Version>1.0.0</Version>
89+
</PackageReference>
90+
<PackageReference Include="Microsoft.Extensions.Options">
91+
<Version>1.1.2</Version>
92+
</PackageReference>
93+
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions">
94+
<Version>1.1.2</Version>
95+
</PackageReference>
96+
<PackageReference Include="NuGet.Jobs.Common">
97+
<Version>4.1.0-master-2602271</Version>
98+
</PackageReference>
99+
<PackageReference Include="NuGet.Services.Configuration">
100+
<Version>2.46.0</Version>
101+
</PackageReference>
102+
<PackageReference Include="NuGet.Services.Validation">
103+
<Version>2.47.0-master-2652814</Version>
104+
</PackageReference>
105+
</ItemGroup>
106+
<ItemGroup>
107+
<ProjectReference Include="..\NuGet.Services.DatabaseMigration\NuGet.Services.DatabaseMigration.csproj">
108+
<Project>{f4c8c34f-72a9-4773-a315-8fa3f2d5ce4e}</Project>
109+
<Name>NuGet.Services.DatabaseMigration</Name>
110+
</ProjectReference>
111+
<ProjectReference Include="..\NuGetGallery.Core\NuGetGallery.Core.csproj">
112+
<Project>{097b2cdd-9623-4c34-93c2-d373d51f5b4e}</Project>
113+
<Name>NuGetGallery.Core</Name>
114+
</ProjectReference>
115+
<ProjectReference Include="..\NuGetGallery\NuGetGallery.csproj">
116+
<Project>{1dacf781-5cd0-4123-8bac-cd385d864be5}</Project>
117+
<Name>NuGetGallery</Name>
118+
</ProjectReference>
119+
</ItemGroup>
120+
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
121+
<PropertyGroup>
122+
<SignPath>..\..\build</SignPath>
123+
<SignPath Condition="'$(BUILD_SOURCESDIRECTORY)' != ''">$(BUILD_SOURCESDIRECTORY)\build</SignPath>
124+
<SignPath Condition="'$(NuGetBuildPath)' != ''">$(NuGetBuildPath)</SignPath>
125+
<SignType Condition="'$(SignType)' == ''">none</SignType>
126+
</PropertyGroup>
127+
<Import Project="$(SignPath)\sign.targets" Condition="Exists('$(SignPath)\sign.targets')" />
128+
<Import Project="$(SignPath)\sign.microbuild.targets" Condition="Exists('$(SignPath)\sign.microbuild.targets')" />
129+
</Project>
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Data.Entity.Migrations;
6+
using System.Data.SqlClient;
7+
using NuGet.Services.DatabaseMigration;
8+
using NuGetGallery.Migrations;
9+
10+
namespace NuGetGallery.DatabaseMigrationTools
11+
{
12+
public class GalleryDbMigrationContext : BaseDbMigrationContext
13+
{
14+
public GalleryDbMigrationContext(SqlConnection sqlConnection)
15+
{
16+
SqlConnection = sqlConnection ?? throw new ArgumentNullException(nameof(sqlConnection));
17+
SqlConnectionAccessToken = sqlConnection.AccessToken;
18+
19+
GalleryDbContextFactory.GalleryEntitiesContextFactory = () =>
20+
{
21+
SetSqlConnectionAccessToken();
22+
return new EntitiesContext(SqlConnection, readOnly: false);
23+
};
24+
25+
var migrationsConfiguration = new MigrationsConfiguration();
26+
GetDbMigrator = () => new DbMigrator(migrationsConfiguration);
27+
}
28+
}
29+
}
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Threading.Tasks;
7+
using Microsoft.Extensions.DependencyInjection;
8+
using NuGet.Jobs;
9+
using NuGet.Jobs.Configuration;
10+
using NuGet.Services.DatabaseMigration;
11+
12+
namespace NuGetGallery.DatabaseMigrationTools
13+
{
14+
public class MigrationContextFactory : IMigrationContextFactory
15+
{
16+
private static IReadOnlyDictionary<string, Func<IServiceProvider, Task<IMigrationContext>>> _dictionary = new Dictionary<string, Func<IServiceProvider, Task<IMigrationContext>>>
17+
{
18+
{
19+
MigrationTargetDatabaseArgumentNames.GalleryDatabase, async(IServiceProvider serviceProvider) =>
20+
{
21+
var sqlConnection = await serviceProvider.GetRequiredService<ISqlConnectionFactory<GalleryDbConfiguration>>().CreateAsync();
22+
return new GalleryDbMigrationContext(sqlConnection);
23+
}
24+
},
25+
{
26+
MigrationTargetDatabaseArgumentNames.SupportRequestDatabase, async(IServiceProvider serviceProvider) =>
27+
{
28+
var sqlConnection = await serviceProvider.GetRequiredService<ISqlConnectionFactory<SupportRequestDbConfiguration>>().CreateAsync();
29+
return new SupportRequestDbMigrationContext(sqlConnection);
30+
}
31+
},
32+
{
33+
MigrationTargetDatabaseArgumentNames.ValidationDatabase, async(IServiceProvider serviceProvider) =>
34+
{
35+
var sqlConnection = await serviceProvider.GetRequiredService<ISqlConnectionFactory<ValidationDbConfiguration>>().CreateAsync();
36+
return new ValidationDbMigrationContext(sqlConnection);
37+
}
38+
}
39+
};
40+
41+
public async Task<IMigrationContext> CreateMigrationContextAsync(string migrationTargetDatabase, IServiceProvider serviceProvider)
42+
{
43+
Func<IServiceProvider, Task<IMigrationContext>> migrationContext;
44+
if (_dictionary.TryGetValue(migrationTargetDatabase, out migrationContext))
45+
{
46+
return await migrationContext(serviceProvider);
47+
}
48+
else
49+
{
50+
throw new ArgumentException($"Invalid migration target database: {migrationTargetDatabase}", nameof(migrationTargetDatabase));
51+
}
52+
}
53+
}
54+
}
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
namespace NuGetGallery.DatabaseMigrationTools
5+
{
6+
internal static class MigrationTargetDatabaseArgumentNames
7+
{
8+
public const string GalleryDatabase = "GalleryDatabase";
9+
public const string SupportRequestDatabase = "SupportRequestDatabase";
10+
public const string ValidationDatabase = "ValidationDatabase";
11+
}
12+
}

0 commit comments

Comments
 (0)