Merge pull request #586 from NuGet/dev

[ReleasePrep][2018.10.15]RI of dev into master

Author: Christy Henriksson

build.ps1

@@ -9,7 +9,7 @@ param (
     [string]$SemanticVersion = '1.0.0-zlocal',
     [string]$Branch = 'zlocal',
     [string]$CommitSHA,
-    [string]$BuildBranch = 'c35bbc228717720bdbc610f3285259391635e90e'
+    [string]$BuildBranch = '80b8f1b4a1cfe57367881e59fca063866dcaaa42'
 )
 
 $msBuildVersion = 15;

src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs

@@ -61,19 +61,21 @@ public override async Task RunAsync(Job job)
             if (expectedRowCount > 0)
             {
                 using (var connection = await job.OpenSqlConnectionAsync<GalleryDbConfiguration>())
+                using (var transaction = connection.BeginTransaction())
+                using (var command = connection.CreateCommand())
                 {
-                    using (var transaction = connection.BeginTransaction())
-                    {
-                        var numKeys = 0;
-                        var parameters = credentialKeys.Select(c => new SqlParameter("@Key" + numKeys++, SqlDbType.Int) { Value = c }).ToArray();
-
-                        rowCount = await connection.ExecuteAsync(
-                            string.Format(DeleteQuery, string.Join(",", parameters.Select(p => p.ParameterName))),
-                            parameters,
-                            transaction, _commandTimeout);
-
-                        transaction.Commit();
-                    }
+                    var numKeys = 0;
+                    var parameters = credentialKeys.Select(c => new SqlParameter("@Key" + numKeys++, SqlDbType.Int) { Value = c }).ToArray();
+                    command.Parameters.AddRange(parameters);
+
+                    command.CommandText = string.Format(DeleteQuery, string.Join(",", parameters.Select(p => p.ParameterName)));
+                    command.CommandType = CommandType.Text;
+                    command.CommandTimeout = (int)_commandTimeout.TotalSeconds;
+                    command.Transaction = transaction;
+
+                    rowCount = await command.ExecuteNonQueryAsync();
+
+                    transaction.Commit();
                 }
             }
 

src/NuGet.Jobs.Common/Extensions/DapperExtensions.cs

@@ -10,30 +10,6 @@ namespace System.Data.SqlClient
 {
     public static class DapperExtensions
     {
-        public static Task<int> ExecuteAsync(this SqlConnection connection, string sql, SqlParameter[] parameters = null, SqlTransaction transaction = null, TimeSpan? commandTimeout = null)
-        {
-            SqlCommand cmd = connection.CreateCommand();
-            cmd.CommandText = sql;
-            cmd.CommandType = CommandType.Text;
-
-            if (parameters != null)
-            {
-                cmd.Parameters.AddRange(parameters);
-            }
-
-            if (commandTimeout.HasValue)
-            {
-                cmd.CommandTimeout = (int)commandTimeout.Value.TotalSeconds;
-            }
-
-            if (transaction != null)
-            {
-                cmd.Transaction = transaction;
-            }
-
-            return cmd.ExecuteNonQueryAsync();
-        }
-
         public static async Task<IEnumerable<T>> QueryWithRetryAsync<T>(
             this SqlConnection connection,
             string sql,

src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj

@@ -94,13 +94,13 @@
       <Version>2.27.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.KeyVault">
-      <Version>2.28.0</Version>
+      <Version>2.29.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Logging">
       <Version>2.27.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Sql">
-      <Version>2.28.0</Version>
+      <Version>2.29.0</Version>
     </PackageReference>
     <PackageReference Include="System.Net.Http">
       <Version>4.3.3</Version>

src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj

@@ -117,10 +117,10 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="NuGet.Services.Status">
-      <Version>2.27.0</Version>
+      <Version>2.29.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Storage">
-      <Version>2.27.0</Version>
+      <Version>2.29.0</Version>
     </PackageReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

src/NuGet.Services.Validation.Orchestrator/Job.cs

@@ -599,6 +599,7 @@ private static void ConfigureOrchestratorSymbolTypes(IServiceCollection services
             services.AddTransient<IValidationSetProvider<SymbolPackage>, ValidationSetProvider<SymbolPackage>>();
             services.AddTransient<IMessageService<SymbolPackage>, SymbolsPackageMessageService>();
             services.AddTransient<IBrokeredMessageSerializer<SymbolsValidatorMessage>, SymbolsValidatorMessageSerializer>();
+            services.AddTransient<IBrokeredMessageSerializer<SymbolsIngesterMessage>, SymbolsIngesterMessageSerializer>();
             services.AddTransient<ISymbolsValidationEntitiesService, SymbolsValidationEntitiesService>();
         }
 
@@ -653,25 +654,24 @@ private static void ConfigureSymbolsIngester(ContainerBuilder builder)
                 .Keyed<ITopicClient>(SymbolsIngesterBindingKey);
 
             builder
-                .RegisterType<SymbolsMessageEnqueuer>()
+                .RegisterType<SymbolsIngesterMessageEnqueuer>()
                 .WithKeyedParameter(typeof(ITopicClient), SymbolsIngesterBindingKey)
                 .WithParameter(
                     (pi, ctx) => pi.ParameterType == typeof(TimeSpan?),
                     (pi, ctx) => ctx.Resolve<IOptionsSnapshot<SymbolsIngesterConfiguration>>().Value.MessageDelay)
-                .Keyed<ISymbolsMessageEnqueuer>(SymbolsIngesterBindingKey)
-                .As<ISymbolsMessageEnqueuer>();
+                .Keyed<ISymbolsIngesterMessageEnqueuer>(SymbolsIngesterBindingKey)
+                .As<ISymbolsIngesterMessageEnqueuer>();
 
             builder
                 .RegisterType<SymbolsIngester>()
                 .WithKeyedParameter(typeof(IValidatorStateService), SymbolsIngesterBindingKey)
-                .WithKeyedParameter(typeof(ISymbolsMessageEnqueuer), SymbolsIngesterBindingKey)
+                .WithKeyedParameter(typeof(ISymbolsIngesterMessageEnqueuer), SymbolsIngesterBindingKey)
                 .AsSelf();
         }
 
         private T GetRequiredService<T>()
         {
             return _serviceProvider.GetRequiredService<T>();
         }
-
     }
 }

src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj

@@ -53,6 +53,7 @@
     <Compile Include="Error.cs" />
     <Compile Include="Services\MessageServiceConfiguration.cs" />
     <Compile Include="Services\SymbolsMessageService.cs" />
+    <Compile Include="Symbols\ISymbolsIngesterMessageEnqueuer.cs" />
     <Compile Include="Symbols\SymbolScanOnlyConfiguration.cs" />
     <Compile Include="Symbols\SymbolCriteriaEvaluator.cs" />
     <Compile Include="Symbols\SymbolScanValidator.cs" />
@@ -62,6 +63,7 @@
     <Compile Include="IPackageStatusProcessor.cs" />
     <Compile Include="Services\SymbolEntityService.cs" />
     <Compile Include="Symbols\ISymbolsMessageEnqueuer.cs" />
+    <Compile Include="Symbols\SymbolsIngesterMessageEnqueuer.cs" />
     <Compile Include="Symbols\SymbolsMessageEnqueuer.cs" />
     <Compile Include="Symbols\SymbolsIngesterConfiguration.cs" />
     <Compile Include="Symbols\SymbolsIngester.cs" />

src/NuGet.Services.Validation.Orchestrator/Symbols/ISymbolsIngesterMessageEnqueuer.cs

@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+using NuGet.Jobs.Validation.Symbols.Core;
+
+namespace NuGet.Services.Validation.Symbols
+{
+    public interface ISymbolsIngesterMessageEnqueuer
+    {
+        /// <summary>
+        /// Enqueues a message to one of the topics used by the Symbol Ingester.
+        /// </summary>
+        /// <param name="request">The validation request.</param>
+        /// <returns>A <see cref="Task"/> that will be completed when the execution is completed.</returns>
+        Task<SymbolsIngesterMessage> EnqueueSymbolsIngestionMessageAsync(IValidationRequest request);
+    }
+}

src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsIngester.cs

@@ -15,13 +15,13 @@ namespace NuGet.Services.Validation.Symbols
     public class SymbolsIngester : BaseValidator, IValidator
     {
         private readonly ISymbolsValidationEntitiesService _symbolsValidationEntitiesService;
-        private readonly ISymbolsMessageEnqueuer _symbolMessageEnqueuer;
+        private readonly ISymbolsIngesterMessageEnqueuer _symbolMessageEnqueuer;
         private readonly ITelemetryService _telemetryService;
         private readonly ILogger<SymbolsIngester> _logger;
 
         public SymbolsIngester(
             ISymbolsValidationEntitiesService symbolsValidationEntitiesService,
-            ISymbolsMessageEnqueuer symbolMessageEnqueuer,
+            ISymbolsIngesterMessageEnqueuer symbolMessageEnqueuer,
             ITelemetryService telemetryService,
             ILogger<SymbolsIngester> logger)
         {
@@ -40,11 +40,12 @@ public async Task<IValidationResult> GetResultAsync(IValidationRequest request)
 
             var result = SymbolsValidationEntitiesService.ConvertToIValidationResult(await _symbolsValidationEntitiesService.GetSymbolsServerRequestAsync(request));
             _logger.LogInformation(
-                    "Symbols status {Status} for PackageId: {PackageId}, PackageNormalizedVersion {PackageNormalizedVersion}, SymbolsPackageKey {SymbolsPackageKey} ",
+                    "Symbols status {Status} for PackageId: {PackageId}, PackageNormalizedVersion {PackageNormalizedVersion}, SymbolsPackageKey {SymbolsPackageKey} ValidationId {ValidationId}",
                     result.Status,
                     request.PackageId,
                     request.PackageVersion,
-                    request.PackageKey);
+                    request.PackageKey,
+                    request.ValidationId);
 
             return result;
         }
@@ -78,9 +79,9 @@ public async Task<IValidationResult> StartAsync(IValidationRequest request)
             }
 
             _telemetryService.TrackSymbolsMessageEnqueued(ValidatorName.SymbolsIngester, request.ValidationId);
-            await _symbolMessageEnqueuer.EnqueueSymbolsValidationMessageAsync(request);
+            var message = await _symbolMessageEnqueuer.EnqueueSymbolsIngestionMessageAsync(request);
 
-            var newSymbolsRequest = SymbolsValidationEntitiesService.CreateFromValidationRequest(request, SymbolsPackageIngestRequestStatus.Ingesting);
+            var newSymbolsRequest = SymbolsValidationEntitiesService.CreateFromValidationRequest(request, SymbolsPackageIngestRequestStatus.Ingesting, message.RequestName);
             var savedSymbolRequest = await _symbolsValidationEntitiesService.AddSymbolsServerRequestAsync(newSymbolsRequest);
 
             if(savedSymbolRequest.RequestStatusKey != SymbolsPackageIngestRequestStatus.Ingesting)

src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsIngesterMessageEnqueuer.cs

@@ -0,0 +1,45 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Options;
+using NuGet.Jobs.Validation.Symbols.Core;
+using NuGet.Services.ServiceBus;
+
+namespace NuGet.Services.Validation.Symbols
+{
+    public class SymbolsIngesterMessageEnqueuer : ISymbolsIngesterMessageEnqueuer
+    {
+        private readonly ITopicClient _topicClient;
+        private readonly TimeSpan? _messageDelay;
+        private readonly IBrokeredMessageSerializer<SymbolsIngesterMessage> _serializer;
+
+        public SymbolsIngesterMessageEnqueuer(
+            ITopicClient topicClient,
+            IBrokeredMessageSerializer<SymbolsIngesterMessage> serializer,
+            TimeSpan? messageDelay)
+        {
+            _topicClient = topicClient ?? throw new ArgumentNullException(nameof(topicClient));
+            _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer));
+            _messageDelay = messageDelay;
+        }
+
+        public async Task<SymbolsIngesterMessage> EnqueueSymbolsIngestionMessageAsync(IValidationRequest request)
+        {
+            var message = new SymbolsIngesterMessage(validationId: request.ValidationId, 
+                symbolPackageKey: request.PackageKey,
+                packageId: request.PackageId,
+                packageNormalizedVersion: request.PackageVersion,
+                snupkgUrl: request.NupkgUrl,
+                requestName : SymbolsValidationEntitiesService.CreateSymbolServerRequestNameFromValidationRequest(request));
+            var brokeredMessage = _serializer.Serialize(message);
+
+            var visibleAt = DateTimeOffset.UtcNow + (_messageDelay ?? TimeSpan.Zero);
+            brokeredMessage.ScheduledEnqueueTimeUtc = visibleAt;
+
+            await _topicClient.SendAsync(brokeredMessage);
+            return message;
+        }
+    }
+}