Merge pull request #101 from NuGet/dev

Merging dev->master

Author: Christy Henriksson

.gitignore

@@ -1,3 +1,5 @@
+nuget.exe
+
 ## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.
 
@@ -13,11 +15,15 @@
 [Rr]eleases/
 x64/
 x86/
-build/
 bld/
 [Bb]in/
 [Oo]bj/
 
+# Files created and used by our build scripts
+build/ 
+tools/
+.nuget/CredentialProviderBundle.zip
+
 # Roslyn cache directories
 *.ide/
 
@@ -189,5 +195,5 @@ FakesAssemblies/
 *.webtestresult
 
 # Vs2015
-
-.vs/config/applicationhost.config
+.vs/config/applicationhost.config
+*.jfm

.nuget/packages.config

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="xunit.runner.console" version="2.1.0" />
+  <package id="xunit.runner.visualstudio" version="2.1.0" />
+</packages>

.vs/config/applicationhost.config

@@ -163,7 +163,7 @@
             </site>
             <site name="Validation.Callback.Vcs" id="2">
                 <application path="/" applicationPool="Clr4IntegratedAppPool">
-                    <virtualDirectory path="/" physicalPath="C:\Users\scottbom\NuGet.Jobs\src\Validation.Callback.Vcs" />
+                    <virtualDirectory path="/" physicalPath="C:\src\NuGet.Jobs\src\Validation.Callback.Vcs" />
                 </application>
                 <bindings>
                     <binding protocol="http" bindingInformation="*:8955:localhost" />

build.cmd

@@ -1,27 +1 @@
-@echo Off
-set config=%1
-if "%config%" == "" (
-   set config=Release
-)
-set msbuild="%ProgramFiles(x86)%\MSBuild\14.0\bin\amd64\msbuild.exe"
-
-REM Package restore
-Powershell.exe -NoProfile -ExecutionPolicy ByPass -Command "& '%cd%\restoreNuGetExe.ps1'"
-tools\nuget.exe restore NuGet.Jobs.sln -OutputDirectory %cd%\packages -NonInteractive -source "https://api.nuget.org/v3/index.json;https://www.myget.org/F/nugetbuild/api/v3/index.json"
-if not "%errorlevel%"=="0" goto failure
-
-REM Build
-%msbuild% NuGet.Jobs.sln /p:Configuration="%config%" /m /v:M /fl /flp:LogFile=msbuild.log;Verbosity=Normal /nr:false
-if not "%errorlevel%"=="0" goto failure
-
-REM Test
-tools\nuget.exe install xunit.runner.console -Version 2.0.0 -OutputDirectory packages
-packages\xunit.runner.console.2.0.0\tools\xunit.console.exe tests\Tests.Stats.CollectAzureCdnLogs\bin\%config%\Tests.Stats.CollectAzureCdnLogs.dll
-packages\xunit.runner.console.2.0.0\tools\xunit.console.exe tests\Tests.Stats.ImportAzureCdnStatistics\bin\%config%\Tests.Stats.ImportAzureCdnStatistics.dll
-if not "%errorlevel%"=="0" goto failure
-
-:success
-exit 0
-
-:failure
-exit -1
+PowerShell.exe -NoProfile -ExecutionPolicy ByPass ".\buildandtest.ps1 -Configuration 'Release' -Verbose"

build.ps1

@@ -0,0 +1,129 @@
+[CmdletBinding(DefaultParameterSetName='RegularBuild')]
+param (
+    [ValidateSet("debug", "release")]
+    [string]$Configuration = 'debug',
+    [ValidateSet("Release","rtm", "rc", "beta", "beta2", "final", "xprivate", "zlocal")]
+    [string]$ReleaseLabel = 'zlocal',
+    [int]$BuildNumber,
+    [switch]$SkipRestore,
+    [switch]$CleanCache,
+    [string]$SimpleVersion = '1.0.0',
+    [string]$SemanticVersion = '1.0.0-zlocal',
+    [string]$Branch,
+    [string]$CommitSHA,
+    [string]$BuildBranch = 'a8c27302d8720d094c2dce87e00324d84b4697b2'
+)
+
+# For TeamCity - If any issue occurs, this script fails the build. - By default, TeamCity returns an exit code of 0 for all powershell scripts, even if they fail
+trap {
+    Write-Host "BUILD FAILED: $_" -ForegroundColor Red
+    Write-Host "ERROR DETAILS:" -ForegroundColor Red
+    Write-Host $_.Exception -ForegroundColor Red
+    Write-Host ("`r`n" * 3)
+    exit 1
+}
+
+if (-not (Test-Path "$PSScriptRoot/build")) {
+    New-Item -Path "$PSScriptRoot/build" -ItemType "directory"
+}
+wget -UseBasicParsing -Uri "https://raw.githubusercontent.com/NuGet/ServerCommon/$BuildBranch/build/init.ps1" -OutFile "$PSScriptRoot/build/init.ps1"
+. "$PSScriptRoot/build/init.ps1" -BuildBranch "$BuildBranch"
+
+Function Clean-Tests {
+    [CmdletBinding()]
+    param()
+    
+    Trace-Log 'Cleaning test results'
+    
+    Remove-Item (Join-Path $PSScriptRoot "Results.*.xml")
+}
+
+Function Prepare-Vcs-Callback {
+    [CmdletBinding()]
+    param()
+    
+    Trace-Log 'Preparing Validation.Callback.Vcs Package'
+    
+    $ZipPackagePath = "src\Validation.Callback.Vcs\obj\Validation.Callback.Vcs.zip"
+    
+    if (Test-Path $ZipPackagePath) {
+        Remove-Item $ZipPackagePath
+    }
+    
+    Build-Solution $Configuration $BuildNumber -MSBuildVersion "14" "src\Validation.Callback.Vcs\Validation.Callback.Vcs.csproj" -Target "Package" -MSBuildProperties "/P:PackageLocation=obj\Validation.Callback.Vcs.zip" -SkipRestore
+}
+
+Write-Host ("`r`n" * 3)
+Trace-Log ('=' * 60)
+
+$startTime = [DateTime]::UtcNow
+if (-not $BuildNumber) {
+    $BuildNumber = Get-BuildNumber
+}
+Trace-Log "Build #$BuildNumber started at $startTime"
+
+$BuildErrors = @()
+    
+Invoke-BuildStep 'Cleaning test results' { Clean-Tests } `
+    -ev +BuildErrors
+
+Invoke-BuildStep 'Installing NuGet.exe' { Install-NuGet } `
+    -ev +BuildErrors
+    
+Invoke-BuildStep 'Clearing package cache' { Clear-PackageCache } `
+    -skip:(-not $CleanCache) `
+    -ev +BuildErrors
+    
+Invoke-BuildStep 'Clearing artifacts' { Clear-Artifacts } `
+    -ev +BuildErrors
+    
+Invoke-BuildStep 'Restoring solution packages' { `
+    Install-SolutionPackages -path (Join-Path $PSScriptRoot ".nuget\packages.config") -output (Join-Path $PSScriptRoot "packages") -ExcludeVersion } `
+    -skip:$SkipRestore `
+    -ev +BuildErrors
+
+Invoke-BuildStep 'Building solution' { 
+    param($Configuration, $BuildNumber, $SolutionPath, $SkipRestore)
+    Build-Solution $Configuration $BuildNumber -MSBuildVersion "14" $SolutionPath -SkipRestore:$SkipRestore `
+    } `
+    -args $Configuration, $BuildNumber, (Join-Path $PSScriptRoot "NuGet.Jobs.sln"), $SkipRestore `
+    -ev +BuildErrors
+    
+Invoke-BuildStep 'Prepare Validation.Callback.Vcs Package' { Prepare-Vcs-Callback } `
+    -ev +BuildErrors
+    
+Invoke-BuildStep 'Creating artifacts' {
+        $Projects = "src/Stats.CollectAzureCdnLogs/Stats.CollectAzureCdnLogs.csproj", `
+            "src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj", `
+            "src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj", `
+            "src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj", `
+            "src/UpdateLicenseReports/UpdateLicenseReports.csproj", `
+            "src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj", `
+            "src/ArchivePackages/ArchivePackages.csproj", `
+            "src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj", `
+            "src/HandlePackageEdits/HandlePackageEdits.csproj", `
+            "src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.csproj", `
+            "src/Validation.Callback.Vcs/Validation.Callback.Vcs.csproj", `
+            "src/Validation.Runner/Validation.Runner.csproj"
+        
+        Foreach ($Project in $Projects) {
+            New-Package (Join-Path $PSScriptRoot "$Project") -Configuration $Configuration -BuildNumber $BuildNumber -ReleaseLabel $ReleaseLabel -Version $SemanticVersion -Branch $Branch
+        }
+    } `
+    -ev +BuildErrors
+
+Trace-Log ('-' * 60)
+
+## Calculating Build time
+$endTime = [DateTime]::UtcNow
+Trace-Log "Build #$BuildNumber ended at $endTime"
+Trace-Log "Time elapsed $(Format-ElapsedTime ($endTime - $startTime))"
+
+Trace-Log ('=' * 60)
+
+if ($BuildErrors) {
+    $ErrorLines = $BuildErrors | %{ ">>> $($_.Exception.Message)" }
+    Error-Log "Builds completed with $($BuildErrors.Count) error(s):`r`n$($ErrorLines -join "`r`n")" -Fatal
+}
+
+Write-Host ("`r`n" * 3)

buildandtest.ps1

@@ -0,0 +1,19 @@
+[CmdletBinding(DefaultParameterSetName='RegularBuild')]
+param (
+    [ValidateSet("debug", "release")]
+    [string]$Configuration = 'debug',
+    [ValidateSet("Release","rtm", "rc", "beta", "beta2", "final", "xprivate", "zlocal")]
+    [string]$ReleaseLabel = 'zlocal',
+    [int]$BuildNumber,
+    [switch]$SkipRestore,
+    [switch]$CleanCache,
+    [string]$SimpleVersion = '1.0.0',
+    [string]$SemanticVersion = '1.0.0-zlocal',
+    [string]$Branch,
+    [string]$CommitSHA
+)
+
+$ScriptPath = Split-Path $MyInvocation.InvocationName
+
+& "$ScriptPath\build.ps1" -Configuration $Configuration -ReleaseLabel $ReleaseLabel -BuildNumber $BuildNumber -SkipRestore:$SkipRestore -CleanCache:$CleanCache -SimpleVersion "$SimpleVersion" -SemanticVersion "$SemanticVersion" -Branch "$Branch" -CommitSHA "$CommitSHA"
+& "$ScriptPath\test.ps1" -Configuration $Configuration -BuildNumber $BuildNumber

src/HandlePackageEdits/HandlePackageEdits.Job.cs

@@ -33,6 +33,7 @@ FROM PackageEdits e
 
         public const string DefaultSourceContainerName = "packages";
         public const string DefaultBackupContainerName = "package-backups";
+        public const int DefaultMaxRetryCount = 10;
 
         /// <summary>
         /// Gets or sets an Azure Storage Uri referring to a container to use as the source for package blobs
@@ -91,6 +92,8 @@ public override bool Init(IDictionary<string, string> jobArgsDictionary)
 
                 SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(SourceContainerName);
                 BackupsContainer = Backups.CreateCloudBlobClient().GetContainerReference(BackupsContainerName);
+
+                MaxTryCount = DefaultMaxRetryCount;
             }
             catch (Exception exception)
             {
@@ -147,7 +150,7 @@ public override async Task<bool> Run()
                 }
                 catch (Exception exception)
                 {
-                    Trace.TraceError($"Error editing package {edit.Id} {edit.Version}! {exception}");
+                    Trace.TraceError($"Error editing package {edit.Id} {edit.Version} (try {edit.TriedCount + 1} / {MaxTryCount})! {exception}");
                     await UpdatePackageEditDbWithError(exception, edit.Key);
                 }
             }
@@ -257,6 +260,13 @@ private async Task ApplyEdit(PackageEdit edit)
 
         private async Task UpdateDatabaseWithEdit(PackageEdit edit, string hash, long size)
         {
+            // insert missing authors as empty in authors table for consistency with gallery
+            // scenario is metadata edit during verification of package uploaded without authors
+            if (string.IsNullOrWhiteSpace(edit.Authors))
+            {
+                edit.Authors = string.Empty;
+            }
+
             using (var connection = await PackageDatabase.ConnectTo())
             {
                 var parameters = new DynamicParameters(new

src/Stats.AggregateCdnDownloadsInGallery/Job.cs

@@ -125,6 +125,7 @@ await statisticsDatabase.QueryWithRetryAsync<DownloadCountData>(
                         var reader = await command.ExecuteReaderAsync();
                         aggregateCdnDownloadsInGalleryTable.Load(reader);
                         aggregateCdnDownloadsInGalleryTable.Rows.Clear();
+                        aggregateCdnDownloadsInGalleryTable.TableName = $"dbo.{_tempTableName}";
                         _logger.LogInformation("Created temporary table.");
 
                         // Populate temporary table in memory

src/Stats.AzureCdnLogs.Common/LogEvents.cs

@@ -16,5 +16,10 @@ public class LogEvents
         public static EventId FailedBlobLease = new EventId(506, "Failed to lease blob");
         public static EventId FailedDimensionRetrieval = new EventId(507, "Failed to retrieve dimension");
         public static EventId FailedToParseLogFileEntry = new EventId(508, "Failed to parse log file entry");
+        public static EventId FailedToProcessLogStream = new EventId(509, "Error processing log stream");
+        public static EventId UnknownAzureCdnPlatform = new EventId(510, "Unknown Azure CDN platform");
+        public static EventId InvalidRawLogFileName = new EventId(511, "Invalid raw log filename");
+        public static EventId FailedToGetFtpResponse = new EventId(512, "Failed to get FTP response");
+        public static EventId JobRunFailed = new EventId(550, "Job run failed");
     }
 }

src/Stats.CollectAzureCdnLogs/Blob/CloudBlobRawLogClient.cs

@@ -64,16 +64,16 @@ public async Task<CloudBlobStream> OpenBlobForWriteAsync(CloudBlobContainer targ
             return await blob.OpenWriteAsync();
         }
 
-        public async Task<bool> CheckIfBlobExistsAsync(CloudBlobContainer targetContainer, RawLogFileInfo logFile)
+        public async Task<bool> CheckIfBlobExistsAsync(CloudBlobContainer targetContainer, string fileName)
         {
-            using (_logger.BeginScope("Checking if file '{FileName}' exists.", logFile.FileName))
+            using (_logger.BeginScope("Checking if file '{FileName}' exists.", fileName))
             {
                 try
                 {
-                    var blob = targetContainer.GetBlockBlobReference(logFile.FileName);
+                    var blob = targetContainer.GetBlockBlobReference(fileName);
                     var exists = await blob.ExistsAsync();
 
-                    _logger.LogInformation("Finished checking if file '{FileName}' exists (exists = {FileExists}).", logFile.FileName, exists);
+                    _logger.LogInformation("Finished checking if file '{FileName}' exists (exists = {FileExists}).", fileName, exists);
 
                     return exists;
                 }