Merge pull request #8416 from NuGet/master

Merge branch 'master' into dev

Author: joelverhagen

.github/CODEOWNERS

@@ -2,4 +2,4 @@
 # the repo. Unless a later match takes precedence,
 # review when someone opens a pull request.
 # For more on how to customize the CODEOWNERS file - https://help.github.com/en/articles/about-code-owners
-*       @agr @cristinamanum @dannyjdev @drewgillies @joelverhagen @loic-sharma @lyndaidaii @ryuyu @skofman1 @shishirx34 @xavierdecoster @zhhyu 
+*       @agr @dannyjdev @drewgillies @joelverhagen @loic-sharma @lyndaidaii @ryuyu @skofman1 @shishirx34 @xavierdecoster @zhhyu

src/GalleryTools/Commands/ReserveNamespacesCommand.cs

@@ -0,0 +1,265 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Autofac;
+using Microsoft.Extensions.CommandLineUtils;
+using NuGet.Packaging;
+using NuGet.Packaging.Core;
+using NuGet.Services.Entities;
+using NuGet.Versioning;
+using NuGetGallery;
+
+namespace GalleryTools.Commands
+{
+    public static class ReserveNamespacesCommand
+    {
+        private const int DefaultSleepSeconds = 0;
+        private const string PathOption = "--path";
+
+        public static void Configure(CommandLineApplication config)
+        {
+            config.Description = "Bulk reserve namespaces";
+            config.HelpOption("-? | -h | --help");
+
+            var pathOptions = config.Option(
+                $"-p | {PathOption}",
+                "A path to a simple text file, which is the list of namespaces to reserve. One namespace per line. Prefix namespaces should end with '*', otherwise they will be exact match reservations.",
+                CommandOptionType.SingleValue);
+
+            var sleepDurationOption = config.Option(
+                "-s | --sleep",
+                $"The duration in seconds to sleep between each reservation (default: {DefaultSleepSeconds}).",
+                CommandOptionType.SingleValue);
+
+            var unreserveOption = config.Option(
+                "-u | --unreserve",
+                $"Unreserve namespaces, instead of reserving them. Meant for a rollback of bulk reserving. Note that you must clear the progress file to reuse the same input file.",
+                CommandOptionType.NoValue);
+
+            config.OnExecute(() =>
+            {
+                return ExecuteAsync(pathOptions, sleepDurationOption, unreserveOption).GetAwaiter().GetResult();
+            });
+        }
+
+        private static async Task<int> ExecuteAsync(
+            CommandOption pathOption,
+            CommandOption sleepDurationOption,
+            CommandOption unreserveOption)
+        {
+            if (!pathOption.HasValue())
+            {
+                Console.WriteLine($"The '{PathOption}' parameter is required.");
+                return 1;
+            }
+
+            var sleepDuration = TimeSpan.FromSeconds(DefaultSleepSeconds);
+            if (sleepDurationOption.HasValue())
+            {
+                sleepDuration = TimeSpan.FromSeconds(int.Parse(sleepDurationOption.Value()));
+
+                if (sleepDuration < TimeSpan.Zero)
+                {
+                    Console.WriteLine("The sleep duration must be zero or more seconds.");
+                    return 1;
+                }
+            }
+
+            var unreserve = unreserveOption.HasValue();
+
+            var path = pathOption.Value();
+            var completedPath = path + ".progress";
+            var remainingList = GetRemainingList(path, completedPath);
+            Console.WriteLine($"{remainingList.Count} reserved namespace(s) to {(unreserve ? "remove" : "add")}.");
+            if (!remainingList.Any())
+            {
+                Console.WriteLine("No namespaces were found to reserve.");
+                return 1;
+            }
+
+            var builder = new ContainerBuilder();
+            builder.RegisterType<ReservedNamespaceService>().AsSelf();
+            builder.RegisterAssemblyModules(typeof(DefaultDependenciesModule).Assembly);
+            var container = builder.Build();
+            var service = container.Resolve<ReservedNamespaceService>();
+
+            var totalCounter = 0;
+            foreach (var reservedNamespace in remainingList)
+            {
+                Console.Write($"{(unreserve ? "Removing" : "Adding")} '{reservedNamespace.Value}' IsPrefix = {reservedNamespace.IsPrefix}...");
+                try
+                {
+                    var matching = service
+                        .FindReservedNamespacesForPrefixList(new[] { reservedNamespace.Value })
+                        .SingleOrDefault(x => ReservedNamespaceComparer.Instance.Equals(x, reservedNamespace));
+
+                    if (unreserve)
+                    {
+                        if (matching == null)
+                        {
+                            Console.WriteLine(" does not exist.");
+                            AppendReservedNamespace(completedPath, reservedNamespace);
+                            continue;
+                        }
+
+                        await service.DeleteReservedNamespaceAsync(reservedNamespace.Value);
+                    }
+                    else
+                    {
+                        if (matching != null)
+                        {
+                            Console.WriteLine(" already exists.");
+                            AppendReservedNamespace(completedPath, reservedNamespace);
+                            continue;
+                        }
+
+                        await service.AddReservedNamespaceAsync(reservedNamespace);
+                    }
+
+                    AppendReservedNamespace(completedPath, reservedNamespace);
+                    totalCounter++;
+                    Console.WriteLine(" done.");
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(" error!");
+                    Console.WriteLine(e);
+                }
+
+                if (sleepDuration > TimeSpan.Zero)
+                {
+                    Console.WriteLine($"Sleeping for {sleepDuration}.");
+                    await Task.Delay(sleepDuration);
+                }
+            }
+
+            Console.WriteLine($"All done. {(unreserve ? "Removed" : "Added")} {totalCounter} reserved namespace(s).");
+
+            return 0;
+        }
+
+        private static List<ReservedNamespace> GetRemainingList(string path, string completedPath)
+        {
+            Console.WriteLine($"Reading reserved namespaces from {path}...");
+            var all = ReadReservedNamespaces(path);
+
+            var completed = new List<ReservedNamespace>();
+            if (File.Exists(completedPath))
+            {
+                Console.WriteLine($"Reading completed reserved namespaces from {completedPath}...");
+                completed.AddRange(ReadReservedNamespaces(completedPath));
+            }
+
+            var remaining = all.Except(completed, ReservedNamespaceComparer.Instance).ToList();
+            if (remaining.Count != all.Count)
+            {
+                Console.WriteLine($"{all.Count - remaining.Count} reserved namespaces(s) are already done.");
+            }
+
+            return remaining;
+        }
+
+        private static void AppendReservedNamespace(string completedListPath, ReservedNamespace reservedNamespace)
+        {
+            using (var fileStream = new FileStream(completedListPath, FileMode.Append, FileAccess.Write))
+            using (var writer = new StreamWriter(fileStream))
+            {
+                writer.WriteLine($"{reservedNamespace.Value}{(reservedNamespace.IsPrefix ? "*" : string.Empty)}");
+            }
+        }
+
+        private static List<ReservedNamespace> ReadReservedNamespaces(string path)
+        {
+            var uniqueReservedNamespaces = new HashSet<ReservedNamespace>(new ReservedNamespaceComparer());
+            var output = new List<ReservedNamespace>();
+            int lineNumber = 0;
+            string line;
+            using (var fileStream = new FileStream(path, FileMode.Open, FileAccess.Read))
+            using (var reader = new StreamReader(fileStream))
+            {
+                while ((line = reader.ReadLine()) != null)
+                {
+                    lineNumber++;
+
+                    if (string.IsNullOrWhiteSpace(line))
+                    {
+                        continue;
+                    }
+
+                    var value = line.Trim();
+                    var isPrefix = false;
+                    if (line.EndsWith("*"))
+                    {
+                        value = value.Substring(0, value.Length - 1);
+                        isPrefix = true;
+                    }
+
+                    // Ensure the reserved namespace is actually a valid package ID.
+                    var validatedPrefix = value;
+                    if (isPrefix)
+                    {
+                        // Prefix reserved namespaces can end with '-' and '.'. Package IDs cannot.
+                        if (value.EndsWith("-") || value.EndsWith("."))
+                        {
+                            validatedPrefix = validatedPrefix.Substring(0, validatedPrefix.Length - 1);
+                        }
+                    }
+
+                    if (!PackageIdValidator.IsValidPackageId(validatedPrefix))
+                    {
+                        Console.WriteLine($"Line {lineNumber}: Ignoring invalid reserved namespace (validated: '{validatedPrefix}', original: '{line}').");
+                        continue;
+                    }
+
+                    var reservedNamespace = new ReservedNamespace
+                    {
+                        Value = value,
+                        IsPrefix = isPrefix,
+                        IsSharedNamespace = false,
+                    };
+
+                    if (!uniqueReservedNamespaces.Add(reservedNamespace))
+                    {
+                        Console.WriteLine($"Line {lineNumber}: Ignoring duplicate reserved namespace.");
+                        continue;
+                    }
+
+                    output.Add(reservedNamespace);
+                }
+            }
+
+            return output;
+        }
+
+        private class ReservedNamespaceComparer : IEqualityComparer<ReservedNamespace>
+        {
+            public static ReservedNamespaceComparer Instance { get; } = new ReservedNamespaceComparer();
+
+            public bool Equals(ReservedNamespace x, ReservedNamespace y)
+            {
+                if (ReferenceEquals(x, y))
+                {
+                    return true;
+                }
+
+                if (x is null || y is null)
+                {
+                    return false;
+                }
+
+                return StringComparer.OrdinalIgnoreCase.Equals(x.Value, y.Value)
+                    && x.IsPrefix == y.IsPrefix;
+            }
+
+            public int GetHashCode(ReservedNamespace obj)
+            {
+                return StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Value) ^ obj.IsPrefix.GetHashCode();
+            }
+        }
+    }
+}

src/GalleryTools/GalleryTools.csproj

@@ -49,6 +49,7 @@
     <Compile Include="Commands\BackfillRepositoryMetadataCommand.cs" />
     <Compile Include="Commands\HashCommand.cs" />
     <Compile Include="Commands\ApplyTenantPolicyCommand.cs" />
+    <Compile Include="Commands\ReserveNamespacesCommand.cs" />
     <Compile Include="Commands\ReflowCommand.cs" />
     <Compile Include="Commands\UpdateIsLatestCommand.cs" />
     <Compile Include="Commands\VerifyApiKeyCommand.cs" />

src/GalleryTools/Program.cs

@@ -21,6 +21,7 @@ public static int Main(params string[] args)
             commandLineApplication.Command("filldevdeps", BackfillDevelopmentDependencyCommand.Configure);
             commandLineApplication.Command("verifyapikey", VerifyApiKeyCommand.Configure);
             commandLineApplication.Command("updateIsLatest", UpdateIsLatestCommand.Configure);
+            commandLineApplication.Command("reservenamespaces", ReserveNamespacesCommand.Configure);
 
             try
             {

src/GitHubVulnerabilities2Db/Gallery/ThrowingTelemetryService.cs

@@ -256,6 +256,11 @@ public void TrackPackagePushNamespaceConflictEvent(string packageId, string pack
             throw new NotImplementedException();
         }
 
+        public void TrackPackagePushOwnerlessNamespaceConflictEvent(string packageId, string packageVersion, User user, IIdentity identity)
+        {
+            throw new NotImplementedException();
+        }
+
         public void TrackPackageReadMeChangeEvent(Package package, string readMeSourceType, PackageEditReadMeState readMeState)
         {
             throw new NotImplementedException();

src/NuGetGallery.Services/Permissions/ActionRequiringReservedNamespacePermissions.cs

@@ -57,9 +57,21 @@ protected override PermissionsCheckResult CheckPermissionsForEntity(User account
                 return PermissionsCheckResult.Allowed;
             }
 
+            var hasAnyOwners = reservedNamespaces.Any(rn => rn.Owners.Any());
+
             // Permissions on only a single namespace are required to perform the action.
-            return reservedNamespaces.Any(rn => PermissionsHelpers.IsRequirementSatisfied(ReservedNamespacePermissionsRequirement, account, rn)) ?
-                PermissionsCheckResult.Allowed : PermissionsCheckResult.ReservedNamespaceFailure;
+            if (reservedNamespaces.Any(rn => PermissionsHelpers.IsRequirementSatisfied(ReservedNamespacePermissionsRequirement, account, rn)))
+            {
+                return PermissionsCheckResult.Allowed;
+            }
+            else if (hasAnyOwners)
+            {
+                return PermissionsCheckResult.ReservedNamespaceFailure;
+            }
+            else
+            {
+                return PermissionsCheckResult.OwnerlessReservedNamespaceFailure;
+            }
         }
 
         public PermissionsCheckResult CheckPermissionsOnBehalfOfAnyAccount(User currentUser, ActionOnNewPackageContext newPackageContext)

src/NuGetGallery.Services/Permissions/PermissionsCheckResult.cs

@@ -31,6 +31,12 @@ public enum PermissionsCheckResult
         /// <summary>
         /// The current user does not have permissions to perform the action on the <see cref="ReservedNamespace"/> on behalf of another <see cref="User"/>.
         /// </summary>
-        ReservedNamespaceFailure
+        ReservedNamespaceFailure,
+
+        /// <summary>
+        /// The current user does not have permissions to perform the action on the <see cref="ReservedNamespace"/> on behalf of another <see cref="User"/>
+        /// but none of the namespaces currently have owners.
+        /// </summary>
+        OwnerlessReservedNamespaceFailure,
     }
 }

src/NuGetGallery.Services/Telemetry/ITelemetryService.cs

@@ -61,6 +61,8 @@ void TrackPackageDeprecate(
 
         void TrackPackagePushNamespaceConflictEvent(string packageId, string packageVersion, User user, IIdentity identity);
 
+        void TrackPackagePushOwnerlessNamespaceConflictEvent(string packageId, string packageVersion, User user, IIdentity identity);
+
         void TrackVerifyPackageKeyEvent(string packageId, string packageVersion, User user, IIdentity identity, int statusCode);
 
         void TrackNewUserRegistrationEvent(User user, Credential identity);

src/NuGetGallery.Services/Telemetry/TelemetryService.cs

@@ -28,6 +28,7 @@ public class Events
             public const string VerifyPackageKey = "VerifyPackageKey";
             public const string PackageReadMeChanged = "PackageReadMeChanged";
             public const string PackagePushNamespaceConflict = "PackagePushNamespaceConflict";
+            public const string PackagePushOwnerlessNamespaceConflict = "PackagePushOwnerlessNamespaceConflict";
             public const string NewUserRegistration = "NewUserRegistration";
             public const string CredentialAdded = "CredentialAdded";
             public const string CredentialUsed = "CredentialUsed";
@@ -364,6 +365,11 @@ public void TrackPackagePushNamespaceConflictEvent(string packageId, string pack
             TrackMetricForPackage(Events.PackagePushNamespaceConflict, packageId, packageVersion, user, identity);
         }
 
+        public void TrackPackagePushOwnerlessNamespaceConflictEvent(string packageId, string packageVersion, User user, IIdentity identity)
+        {
+            TrackMetricForPackage(Events.PackagePushOwnerlessNamespaceConflict, packageId, packageVersion, user, identity);
+        }
+
         public void TrackCreatePackageVerificationKeyEvent(string packageId, string packageVersion, User user, IIdentity identity)
         {
             TrackMetricForPackage(Events.CreatePackageVerificationKey, packageId, packageVersion, user, identity);

src/NuGetGallery/Controllers/ApiController.cs

@@ -1195,6 +1195,11 @@ private HttpStatusCodeWithBodyResult GetHttpResultFromFailedApiScopeEvaluationHe
                 TelemetryService.TrackPackagePushNamespaceConflictEvent(id, versionString, GetCurrentUser(), User.Identity);
                 return new HttpStatusCodeWithBodyResult(HttpStatusCode.Conflict, Strings.UploadPackage_IdNamespaceConflict);
             }
+            else if (result.PermissionsCheckResult == PermissionsCheckResult.OwnerlessReservedNamespaceFailure)
+            {
+                TelemetryService.TrackPackagePushOwnerlessNamespaceConflictEvent(id, versionString, GetCurrentUser(), User.Identity);
+                return new HttpStatusCodeWithBodyResult(HttpStatusCode.Conflict, Strings.UploadPackage_OwnerlessIdNamespaceConflict);
+            }
 
             var message = result.PermissionsCheckResult == PermissionsCheckResult.Allowed && !result.IsOwnerConfirmed ?
                 Strings.ApiKeyOwnerUnconfirmed : Strings.ApiKeyNotAuthorized;