Merge pull request #502 from NuGet/dev

[ReleasePrep][2018.07.31]RI of dev into master

Author: loic-sharma

.nuget/NuGet.config

@@ -6,6 +6,7 @@
   <packageSources>
     <add key="nugetbuild" value="https://dotnet.myget.org/F/nuget-build/api/v3/index.json" />
     <add key="nuget.org" value="https://www.nuget.org/api/v2" />
+    <add key="dotnet.core" value="https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" />
   </packageSources>
   <disabledPackageSources>
     <clear />

NuGet.Jobs.sln

@@ -107,6 +107,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Common.Job", "sr
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PackageHash", "src\PackageHash\PackageHash.csproj", "{40843020-6F0A-48F0-AC28-42FFE3A5C21E}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StatusAggregator", "src\StatusAggregator\StatusAggregator.csproj", "{D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Common.Job.Tests", "tests\Validation.Common.Job.Tests\Validation.Common.Job.Tests.csproj", "{430F63C7-20C2-4872-AC3E-DDE846E50AA4}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.PackageSigning.ProcessSignature", "src\Validation.PackageSigning.ProcessSignature\Validation.PackageSigning.ProcessSignature.csproj", "{DD043977-6BCD-475A-BEE2-8C34309EC622}"
@@ -135,12 +137,18 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Revalidate",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Revalidate.Tests", "tests\NuGet.Services.Revalidate.Tests\NuGet.Services.Revalidate.Tests.csproj", "{19780DCB-B307-4254-B10C-4335FC784DEA}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StatusAggregator.Tests", "tests\StatusAggregator.Tests\StatusAggregator.Tests.csproj", "{784F938D-4142-4C1C-B654-0978FEAD1731}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols.Core", "src\Validation.Symbols.Core\Validation.Symbols.Core.csproj", "{17510A22-176F-4E96-A867-E79F1B54F54F}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Monitoring.RebootSearchInstance", "src\Monitoring.RebootSearchInstance\Monitoring.RebootSearchInstance.csproj", "{ECD8DFCE-8E3C-4510-AFE3-D7EC168E8D66}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Monitoring.RebootSearchInstance.Tests", "tests\Monitoring.RebootSearchInstance.Tests\Monitoring.RebootSearchInstance.Tests.csproj", "{21C0A0EE-8696-4013-950F-D6495D0C6E40}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols", "src\Validation.Symbols\Validation.Symbols.csproj", "{2DD07A73-8C88-4429-BB24-C2813586EF92}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Validation.Symbols.Tests", "tests\Validation.Symbols.Tests\Validation.Symbols.Tests.csproj", "{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -305,6 +313,10 @@ Global
 		{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{40843020-6F0A-48F0-AC28-42FFE3A5C21E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB}.Release|Any CPU.Build.0 = Release|Any CPU
 		{430F63C7-20C2-4872-AC3E-DDE846E50AA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{430F63C7-20C2-4872-AC3E-DDE846E50AA4}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{430F63C7-20C2-4872-AC3E-DDE846E50AA4}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -357,6 +369,10 @@ Global
 		{19780DCB-B307-4254-B10C-4335FC784DEA}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{19780DCB-B307-4254-B10C-4335FC784DEA}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{19780DCB-B307-4254-B10C-4335FC784DEA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{784F938D-4142-4C1C-B654-0978FEAD1731}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{784F938D-4142-4C1C-B654-0978FEAD1731}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{784F938D-4142-4C1C-B654-0978FEAD1731}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{784F938D-4142-4C1C-B654-0978FEAD1731}.Release|Any CPU.Build.0 = Release|Any CPU
 		{17510A22-176F-4E96-A867-E79F1B54F54F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{17510A22-176F-4E96-A867-E79F1B54F54F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{17510A22-176F-4E96-A867-E79F1B54F54F}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -369,6 +385,14 @@ Global
 		{21C0A0EE-8696-4013-950F-D6495D0C6E40}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{21C0A0EE-8696-4013-950F-D6495D0C6E40}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{21C0A0EE-8696-4013-950F-D6495D0C6E40}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2DD07A73-8C88-4429-BB24-C2813586EF92}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2DD07A73-8C88-4429-BB24-C2813586EF92}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2DD07A73-8C88-4429-BB24-C2813586EF92}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2DD07A73-8C88-4429-BB24-C2813586EF92}.Release|Any CPU.Build.0 = Release|Any CPU
+		{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -413,6 +437,7 @@ Global
 		{B4B7564A-965B-447B-927F-6749E2C08880} = {6A776396-02B1-475D-A104-26940ADB04AB}
 		{FA87D075-A934-4443-8D0B-5DB32640B6D7} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
 		{40843020-6F0A-48F0-AC28-42FFE3A5C21E} = {FA5644B5-4F08-43F6-86B3-039374312A47}
+		{D357FDB5-BF19-41A5-82B0-14C8CEC2A5EB} = {FA5644B5-4F08-43F6-86B3-039374312A47}
 		{430F63C7-20C2-4872-AC3E-DDE846E50AA4} = {6A776396-02B1-475D-A104-26940ADB04AB}
 		{DD043977-6BCD-475A-BEE2-8C34309EC622} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
 		{ED2D370C-D921-433A-A0B9-A601F936EDD3} = {FA5644B5-4F08-43F6-86B3-039374312A47}
@@ -426,9 +451,12 @@ Global
 		{60152AB1-2EB4-4D44-B6D6-EEE24209A1F7} = {6A776396-02B1-475D-A104-26940ADB04AB}
 		{1963909D-8BE3-4CB8-B57E-AB6A8CB22FED} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
 		{19780DCB-B307-4254-B10C-4335FC784DEA} = {6A776396-02B1-475D-A104-26940ADB04AB}
+		{784F938D-4142-4C1C-B654-0978FEAD1731} = {6A776396-02B1-475D-A104-26940ADB04AB}
 		{17510A22-176F-4E96-A867-E79F1B54F54F} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
 		{ECD8DFCE-8E3C-4510-AFE3-D7EC168E8D66} = {814F9B31-4AF3-46CC-AD61-CEB40F47083A}
 		{21C0A0EE-8696-4013-950F-D6495D0C6E40} = {6A776396-02B1-475D-A104-26940ADB04AB}
+		{2DD07A73-8C88-4429-BB24-C2813586EF92} = {678D7B14-F8BC-4193-99AF-2EE8AA390A02}
+		{640D29AB-4D1B-4FC7-AE67-AD12EE5AC503} = {6A776396-02B1-475D-A104-26940ADB04AB}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B}

build.ps1

@@ -115,6 +115,7 @@ Invoke-BuildStep 'Set version metadata in AssemblyInfo.cs' { `
             "$PSScriptRoot\src\Validation.Common.Job\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\Validation.ScanAndSign.Core\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\PackageLagMonitor\Properties\AssemblyInfo.g.cs",
+            "$PSScriptRoot\src\StatusAggregator\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\Validation.Symbols.Core\Properties\AssemblyInfo.g.cs",
             "$PSScriptRoot\src\Monitoring.RebootSearchInstance\Properties\AssemblyInfo.g.cs"
 
@@ -149,7 +150,8 @@ Invoke-BuildStep 'Creating artifacts' {
         $ProjectsWithSymbols =
             "src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj",
             "src/Validation.Common.Job/Validation.Common.Job.csproj",
-            "src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj"
+            "src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj",
+            "src/Validation.Symbols.Core/Validation.Symbols.Core.csproj"
 
         $Projects = `
             "src/Stats.CollectAzureCdnLogs/Stats.CollectAzureCdnLogs.csproj", `
@@ -176,8 +178,9 @@ Invoke-BuildStep 'Creating artifacts' {
             "src/Validation.PackageSigning.ValidateCertificate/Validation.PackageSigning.ValidateCertificate.csproj", `
             "src/Validation.PackageSigning.RevalidateCertificate/Validation.PackageSigning.RevalidateCertificate.csproj", `
             "src/PackageLagMonitor/Monitoring.PackageLag.csproj", `
+            "src/StatusAggregator/StatusAggregator.csproj", `
             "src/Validation.Symbols.Core/Validation.Symbols.Core.csproj", `
-            "src/Monitoring.RebootSearchInstance/Monitoring.RebootSearchInstance.csproj" `
+            "src/Validation.Symbols/Validation.Symbols.csproj" `
             + $ProjectsWithSymbols
 
         Foreach ($Project in $Projects) {

src/NuGet.Jobs.Common/Configuration/JobArgumentNames.cs

@@ -118,6 +118,19 @@ public static class JobArgumentNames
         public const string MailFrom = "MailFrom";
         public const string SmtpUri = "SmtpUri";
 
+        // Arguments specific to StatusAggregator
+        public const string StatusStorageAccount = "StatusStorageAccount";
+        public const string StatusContainerName = "StatusContainerName";
+        public const string StatusTableName = "StatusTableName";
+        public const string StatusEnvironment = "StatusEnvironment";
+        public const string StatusMaximumSeverity = "StatusMaximumSeverity";
+        public const string StatusIncidentApiBaseUri = "StatusIncidentApiBaseUri";
+        public const string StatusIncidentApiCertificate = "StatusIncidentApiCertificate";
+        public const string StatusIncidentApiTeamId = "StatusIncidentApiTeamId";
+        public const string StatusEventStartMessageDelayMinutes = "StatusEventStartMessageDelayMinutes";
+        public const string StatusEventEndDelayMinutes = "StatusEventEndDelayMinutes";
+        public const string StatusEventVisibilityPeriodDays = "StatusEventVisibilityPeriodDays";
+
         // Arguments specific to Stats.AggregateCdnDownloadsInGallery
         public static string BatchSleepSeconds = "BatchSleepSeconds";
     }

src/NuGet.Jobs.Common/Configuration/JobConfigurationManager.cs

@@ -5,7 +5,9 @@
 using System.Collections.Generic;
 using System.ComponentModel.Design;
 using System.Linq;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using NuGet.Jobs.Extensions;
 using NuGet.Services.Configuration;
 using NuGet.Services.KeyVault;
 
@@ -190,7 +192,7 @@ private static Dictionary<string, string> ReadCommandLineArguments(ILogger logge
 
         private static IDictionary<string, string> InjectSecrets(IServiceContainer serviceContainer, Dictionary<string, string> argsDictionary)
         {
-            var secretReaderFactory = (ISecretReaderFactory)serviceContainer.GetService(typeof(ISecretReaderFactory));
+            var secretReaderFactory = serviceContainer.GetRequiredService<ISecretReaderFactory>();
 
             var secretReader = secretReaderFactory.CreateSecretReader(argsDictionary);
             if (secretReader == null)

src/NuGet.Jobs.Common/Extensions/LoggerExtensions.cs

@@ -0,0 +1,54 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.Extensions.Logging;
+using System;
+
+namespace NuGet.Jobs.Extensions
+{
+    public static class LoggerExtensions
+    {
+        /// <summary>
+        /// Calls <see cref="ILogger.BeginScope{TState}(TState)"/> and logs a message when entering and leaving the scope.
+        /// </summary>
+        public static IDisposable Scope(
+            this ILogger logger, 
+            string message, 
+            params object[] args)
+        {
+            return new LoggerScopeHelper(logger, message, args);
+        }
+
+        private class LoggerScopeHelper : IDisposable
+        {
+            private readonly ILogger _logger;
+            private readonly IDisposable _scope;
+
+            private readonly string _message;
+            private readonly object[] _args;
+
+            private bool _isDisposed = false;
+
+            public LoggerScopeHelper(
+                ILogger logger, string message, object[] args)
+            {
+                _logger = logger;
+                _message = message;
+                _args = args;
+
+                _scope = logger.BeginScope(_message, _args);
+                _logger.LogInformation("Entering scope: " + _message, _args);
+            }
+
+            public void Dispose()
+            {
+                if (!_isDisposed)
+                {
+                    _logger.LogInformation("Leaving scope: " + _message, _args);
+                    _scope?.Dispose(); // ILogger can return a null scope (most notably during testing with a Mock<ILogger>)
+                    _isDisposed = true;
+                }
+            }
+        }
+    }
+}

src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj

@@ -49,6 +49,7 @@
     <Compile Include="Configuration\ServiceBusConfiguration.cs" />
     <Compile Include="Configuration\ValidationDbConfiguration.cs" />
     <Compile Include="Configuration\ValidationStorageConfiguration.cs" />
+    <Compile Include="Extensions\LoggerExtensions.cs" />
     <Compile Include="Extensions\SqlConnectionStringBuilderExtensions.cs" />
     <Compile Include="Extensions\XElementExtensions.cs" />
     <Compile Include="SecretReader\ISecretReaderFactory.cs" />

src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj

@@ -127,7 +127,7 @@
       <Version>2.26.0-master-34394</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Validation.Issues">
-      <Version>2.26.0-master-34394</Version>
+      <Version>2.27.0-master-35351</Version>
     </PackageReference>
   </ItemGroup>
   <ItemGroup>

src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs

@@ -5,12 +5,13 @@
 using System.Collections.Generic;
 using System.Data.Entity;
 using System.Linq;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using NuGet.Jobs.Validation;
-using NuGet.Jobs.Validation.Storage;
 using NuGet.Jobs.Validation.ScanAndSign;
+using NuGet.Jobs.Validation.Storage;
 using NuGet.Services.Validation.Vcs;
 using NuGetGallery;
 
@@ -19,6 +20,8 @@ namespace NuGet.Services.Validation.Orchestrator.PackageSigning.ScanAndSign
     [ValidatorName(ValidatorName.ScanAndSign)]
     public class ScanAndSignProcessor : IProcessor
     {
+        private const string UsernameRegex = @"^[A-Za-z0-9][A-Za-z0-9_\.-]+[A-Za-z0-9]$";
+
         private readonly IValidationEntitiesContext _validationContext;
         private readonly IValidatorStateService _validatorStateService;
         private readonly ICorePackageService _packageService;
@@ -131,10 +134,10 @@ public async Task<IValidationResult> StartAsync(IValidationRequest request)
                 return processorStatus.ToValidationResult();
             }
 
-            if (await ShouldRepositorySignAsync(request))
-            {
-                var owners = FindPackageOwners(request);
+            var owners = FindPackageOwners(request);
 
+            if (await ShouldRepositorySignAsync(request, owners))
+            {
                 _logger.LogInformation(
                     "Repository signing {PackageId} {PackageVersion} with {ServiceIndex} and {Owners}",
                     request.PackageId,
@@ -197,7 +200,7 @@ private bool ShouldSkipScan(IValidationRequest request)
             return false;
         }
 
-        private async Task<bool> ShouldRepositorySignAsync(IValidationRequest request)
+        private async Task<bool> ShouldRepositorySignAsync(IValidationRequest request, List<string> owners)
         {
             var hasRepositorySignature = await _validationContext
                 .PackageSignatures
@@ -215,7 +218,18 @@ private async Task<bool> ShouldRepositorySignAsync(IValidationRequest request)
                 return false;
             }
 
-           return true;
+            if (owners.Any(IsInvalidUsername))
+            {
+                _logger.LogWarning(
+                    "Package {PackageId} {PackageVersion} has an owner with an invalid username. Scanning instead of signing. {Owners}",
+                    request.PackageId,
+                    request.PackageVersion,
+                    owners);
+
+                return false;
+            }
+
+            return true;
         }
 
         private List<string> FindPackageOwners(IValidationRequest request)
@@ -234,5 +248,10 @@ private List<string> FindPackageOwners(IValidationRequest request)
                 .Select(o => o.Username)
                 .ToList();
         }
+
+        private bool IsInvalidUsername(string username)
+        {
+            return !Regex.IsMatch(username, UsernameRegex, RegexOptions.None, TimeSpan.FromSeconds(5));
+        }
     }
 }

src/StatusAggregator/Cursor.cs

@@ -0,0 +1,60 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using NuGet.Jobs.Extensions;
+using NuGet.Services.Status.Table;
+using StatusAggregator.Table;
+
+namespace StatusAggregator
+{
+    public class Cursor : ICursor
+    {
+        public Cursor(
+            ITableWrapper table,
+            ILogger<Cursor> logger)
+        {
+            _table = table ?? throw new ArgumentNullException(nameof(table));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        private readonly ITableWrapper _table;
+
+        private readonly ILogger<Cursor> _logger;
+
+        public async Task<DateTime> Get()
+        {
+            using (_logger.Scope("Fetching cursor."))
+            {
+                var cursor = await _table.Retrieve<CursorEntity>(
+                    CursorEntity.DefaultPartitionKey, CursorEntity.DefaultRowKey);
+
+                DateTime value;
+                if (cursor == null)
+                {
+                    // If we can't find a cursor, the job is likely uninitialized, so start at the beginning of time.
+                    value = DateTime.MinValue;
+                    _logger.LogInformation("Could not fetch cursor, reinitializing cursor at {Cursor}.", value);
+                }
+                else
+                {
+                    value = cursor.Value;
+                    _logger.LogInformation("Fetched cursor with value {Cursor}.", value);
+                }
+
+                return value;
+            }
+        }
+
+        public Task Set(DateTime value)
+        {
+            using (_logger.Scope("Updating cursor to {Cursor}.", value))
+            {
+                var cursorEntity = new CursorEntity(value);
+                return _table.InsertOrReplaceAsync(cursorEntity);
+            }
+        }
+    }
+}