Merge pull request #6412 from NuGet/dev

* Update bootstrap dependencies (#6325)
Addresses https://github.com/NuGet/Engineering/issues/1634
* Simplify the security policy service to have 1 handler list for all accounts (#6330)
* Set-StrictMode in build script (#6338)
* Add member/add owner/transform to org fails with specific message if email is input instead of username (#6329)
* Move admin pages to redesigned layout and remove unused scripts (#6323)
* Use strong name version of third party dependencies (#6337)
Progress on https://github.com/NuGet/Engineering/issues/1577
Address #3849
Copy in ServiceResponse from NuGet.Services.Platform to eliminate a dependency
* Typosquatting: add typosquatting check service codes  (#6315)
Add the codes of typosquatting algorithms and retrieve the latest owner list info. 
1. Typosquatting check service with distance calculation and comparison; 
2. Normalize the string before checking;
3. Call `public bool IsDistanceLessThanThreshold(string str1, string str2, int threshold)` to compare two strings; (changed to private)
4. Call `public bool IsUploadedPackageIdTyposquatting(string uploadedPackageId)` to check typosquatting in the checlist.
Fixes: https://github.com/NuGet/Engineering/issues/1593
* Validation warnings for repository URLs (#6335)
* works
* improve log
* PR comments
* Fix comments
* bug fix
* repo url warnings
* add UT
* align
* Update _VerifyMetadata.cshtml
Fixed: issue**s**(s) to issue(s)
* PR feedback
* support explicit 443 port for github
* fix build
* Add MicroBuild dependency and signing of output DLLs (#6344)
Progress on https://github.com/NuGet/Engineering/issues/1577
* Gallery:  enable blocking packages with too many package entries (#6358)
Progress on #6357.
* Fix PackageRef dependencies
* PR feedback
* Add max-age for the symbol packages (#6343)
* Fix repo url validation message UI (#6366)
* tablist is not under a menubar (#6345)
* Log search service exceptions encountered during retry (#6232)
* Typo squatting checklist retrieval (#6361)
* Typo squatting checklist retrieval
* refactor and delete PackagesCheckListService
* update codes based on the review
* Change linq and Delete concurrent dictionary for normalized Ids
* Update index migration files and Fix issues in reviews
* include ID in the index
* Delete unnecessary codes in Facts
* Fix threshold
* Add tool to verify the clear-text of an API key (#6383)
* Additional copyright values for auto-add MSFT feature (#6376)
* Additional copyright values for auto-add MSFT feature #1664
* Removing support for a specific copyright value in MicrosoftTeamSubscription
* improve test readability
* Fix inconsistent messaging around expired legacy API key (#6384)
Address #6318
* Add BlockSearchEngineIndexing to allow site-wide search engine blocking (#6385)
Progress on #6381
* Fix incorrect pending revalidation count (#6382)
Fixes #6362
Depends on NuGet/ServerCommon#190
* Telemetry metrics for symbol push. (#6372)
* Add gallery telemetry metrics for the symbol push.
* ApiController actions should not throw exceptions when called with an unparseable version (#6089)
* Typosquatting: finish configuration file and feature flags (#6390)
* Typosquatting: finish configuration file and feature flags
* Update unit tests
* Update
* update unit tests
* Transform Project Url scheme to HTTPs for known domains (#6398)
* Transform Project Url scheme to HTTPs for known domains
* PR feedback
* Add telemetry for auto-add package owner feature (#6374)
* Fix msa login account selection (#6400)
* [NuGet Symbol Server]Symbols validation status view (#6407)
* Obfuscate secret tokens in logging (#6399)
Issue 1670 fix - obfuscate secret tokens.
* Covert to https URLs in known domains in release notes, description (#6408)
* Covert to https URLs in known domains in release notes, description
* another UT
* PR comments

Author: ryuyu

build.ps1

@@ -10,9 +10,11 @@ param (
     [string]$PackageSuffix,
     [string]$Branch,
     [string]$CommitSHA,
-    [string]$BuildBranch = '37ff6e758c38b3f513af39f881399ce85f4ff20b'
+    [string]$BuildBranch = 'cb604c2cd1b2f7f71fb574cdda4c83ddb1464cc7'
 )
 
+Set-StrictMode -Version 1.0
+
 # This script should fail the build if any issue occurs.
 trap {
     Write-Host "BUILD FAILED: $_" -ForegroundColor Red
@@ -91,11 +93,11 @@ Invoke-BuildStep 'Building solution' {
     -ev +BuildErrors
 
 Invoke-BuildStep 'Creating artifacts' {
-        $packageId = 'NuGetGallery.Core'+$PackageSuffix 
-		New-Package (Join-Path $PSScriptRoot "src\NuGetGallery.Core\NuGetGallery.Core.csproj") -Configuration $Configuration -Symbols -BuildNumber $BuildNumber -Version $SemanticVersion -PackageId $packageId `
-		-ev +BuildErrors
-	}
-	
+    $packageId = 'NuGetGallery.Core'+$PackageSuffix 
+    New-ProjectPackage (Join-Path $PSScriptRoot "src\NuGetGallery.Core\NuGetGallery.Core.csproj") -Configuration $Configuration -Symbols -BuildNumber $BuildNumber -Version $SemanticVersion -PackageId $packageId `
+    -ev +BuildErrors
+}
+
 Trace-Log ('-' * 60)
 
 ## Calculating Build time
@@ -110,4 +112,4 @@ if ($BuildErrors) {
     Error-Log "Builds completed with $($BuildErrors.Count) error(s):`r`n$($ErrorLines -join "`r`n")" -Fatal
 }
 
-Write-Host ("`r`n" * 3)
+Write-Host ("`r`n" * 3)

src/Bootstrap/package-lock.json

@@ -87,7 +87,9 @@
     ]
   },
   "dependencies": {
+    "debug": "^2.6.9",
     "fresh": "^0.5.2",
+    "hoek": "^4.2.1",
     "mime": "^1.4.1",
     "no-case": "^2.3.2",
     "tough-cookie": "^2.3.3"

src/Bootstrap/package.json

@@ -0,0 +1,95 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Extensions.CommandLineUtils;
+using NuGetGallery;
+using NuGetGallery.Infrastructure.Authentication;
+
+namespace GalleryTools.Commands
+{
+    public static class VerifyApiKeyCommand
+    {
+        public static void Configure(CommandLineApplication config)
+        {
+            config.Description = "Verify a clear text API key matches a hashed API key.";
+            config.HelpOption("-? | -h | --help");
+
+            var plainTextOption = config.Option(
+                "--clear-text",
+                "The clear text value of the API key.",
+                CommandOptionType.SingleValue);
+
+            var credentialTypesOption = config.Option(
+                "--type",
+                "One or more credential types.",
+                CommandOptionType.MultipleValue);
+
+            var credentialValuesOption = config.Option(
+                "--value",
+                "One or more credential values.",
+                CommandOptionType.MultipleValue);
+
+            config.OnExecute(() => Execute(
+                plainTextOption,
+                credentialTypesOption,
+                credentialValuesOption));
+        }
+
+        private static int Execute(
+            CommandOption clearTextOption,
+            CommandOption credentialTypesOption,
+            CommandOption credentialValuesOption)
+        {
+            if (!clearTextOption.HasValue())
+            {
+                Console.Error.WriteLine($"The {clearTextOption.Template} option is required.");
+                return 1;
+            }
+
+            if (!credentialTypesOption.HasValue())
+            {
+                Console.Error.WriteLine($"The {credentialTypesOption.Template} option is required.");
+                return 1;
+            }
+
+            if (!credentialValuesOption.HasValue())
+            {
+                Console.Error.WriteLine($"The {credentialValuesOption.Template} option is required.");
+                return 1;
+            }
+
+            if (credentialTypesOption.Values.Count != credentialValuesOption.Values.Count)
+            {
+                Console.Error.WriteLine($"The should be the same number of {credentialTypesOption.Template} options as {credentialValuesOption.Template} options.");
+                return 1;
+            }
+
+            var credentialValidator = new CredentialValidator();
+
+            Console.WriteLine($"Testing {credentialTypesOption.Values.Count} API key(s).");
+
+            for (var i = 0; i < credentialTypesOption.Values.Count; i++)
+            {
+                var credential = new Credential
+                {
+                    Type = credentialTypesOption.Values[i],
+                    Value = credentialValuesOption.Values[i],
+                };
+
+                var validCredentials = credentialValidator.GetValidCredentialsForApiKey(
+                    new[] { credential }.AsQueryable(),
+                    clearTextOption.Value());
+
+                Console.WriteLine();
+                Console.WriteLine($"API key {i + 1}:");
+                Console.WriteLine($"  Type:    {credential.Type}");
+                Console.WriteLine($"  Value:   {credential.Value}");
+                Console.WriteLine($"  Matches: {validCredentials.Any().ToString().ToUpperInvariant()}");
+            }
+
+            return 0;
+        }
+    }
+}

src/GalleryTools/Commands/VerifyApiKeyCommand.cs

@@ -47,6 +47,7 @@
     <Compile Include="Commands\BackfillRepositoryMetadataCommand.cs" />
     <Compile Include="Commands\HashCommand.cs" />
     <Compile Include="Commands\ReflowCommand.cs" />
+    <Compile Include="Commands\VerifyApiKeyCommand.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>

src/GalleryTools/GalleryTools.csproj

@@ -17,6 +17,7 @@ public static int Main(params string[] args)
             commandLineApplication.Command("hash", HashCommand.Configure);
             commandLineApplication.Command("reflow", ReflowCommand.Configure);
             commandLineApplication.Command("fillrepodata", BackfillRepositoryMetadataCommand.Configure);
+            commandLineApplication.Command("verifyapikey", VerifyApiKeyCommand.Configure);
 
             try
             {

src/GalleryTools/Program.cs

@@ -15,6 +15,7 @@ public sealed class RetryingHttpClientWrapper
     {
         private readonly HttpClient _httpClient;
         private readonly IEndpointHealthIndicatorStore _endpointHealthIndicatorStore;
+        private readonly Action<Exception> _onException;
 
         private static readonly int PeriodToDelayAlternateRequest = 3000;
         private static readonly IComparer<int> HealthComparer;
@@ -24,16 +25,16 @@ static RetryingHttpClientWrapper()
             HealthComparer = new WeightedRandomComparer();
         }
 
-        public RetryingHttpClientWrapper(HttpClient httpClient)
-            : this (httpClient, new BaseUrlHealthIndicatorStore(new NullHealthIndicatorLogger()))
+        public RetryingHttpClientWrapper(HttpClient httpClient, Action<Exception> onException)
+            : this (httpClient, new BaseUrlHealthIndicatorStore(new NullHealthIndicatorLogger()), onException)
         {
-            _httpClient = httpClient;
         }
 
-        public RetryingHttpClientWrapper(HttpClient httpClient, IEndpointHealthIndicatorStore endpointHealthIndicatorStore)
+        public RetryingHttpClientWrapper(HttpClient httpClient, IEndpointHealthIndicatorStore endpointHealthIndicatorStore, Action<Exception> onException)
         {
-            _httpClient = httpClient;
-            _endpointHealthIndicatorStore = endpointHealthIndicatorStore;
+            _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
+            _endpointHealthIndicatorStore = endpointHealthIndicatorStore ?? throw new ArgumentNullException(nameof(endpointHealthIndicatorStore));
+            _onException = onException ?? throw new ArgumentNullException(nameof(onException));
         }
 
         public async Task<string> GetStringAsync(IEnumerable<Uri> endpoints)
@@ -91,6 +92,11 @@ private async Task<HttpResponseMessage> GetWithRetry(IEnumerable<Uri> endpoints,
                 cancellationTokenSource.Cancel(false);
             }
 
+            foreach (var exception in exceptions)
+            {
+                _onException(exception);
+            }
+
             if (completedTask.IsFaulted || completedTask.IsCanceled)
             {
                 var exceptionsToThrow = exceptions.Where(e => !(e is TaskCanceledException || e.InnerException is TaskCanceledException)).ToList();

src/NuGet.Services.Search.Client/Client/RetryingHttpClientWrapper.cs

@@ -8,7 +8,6 @@
 using System.Net.Http;
 using System.Threading.Tasks;
 using Newtonsoft.Json.Linq;
-using NuGet.Services.Client;
 using NuGet.Services.Search.Models;
 
 namespace NuGet.Services.Search.Client
@@ -25,8 +24,8 @@ public class SearchClient
         /// </summary>
         /// <param name="baseUri">The URL to the root of the service</param>
         /// <param name="handlers">Handlers to apply to the request in order from first to last</param>
-        public SearchClient(Uri baseUri, params DelegatingHandler[] handlers)
-            : this(baseUri, "SearchGalleryQueryService/3.0.0-rc", null, new BaseUrlHealthIndicatorStore(new NullHealthIndicatorLogger()), handlers)
+        public SearchClient(Uri baseUri, Action<Exception> onException, params DelegatingHandler[] handlers)
+            : this(baseUri, "SearchGalleryQueryService/3.0.0-rc", null, new BaseUrlHealthIndicatorStore(new NullHealthIndicatorLogger()), onException, handlers)
         {
         }
 
@@ -38,7 +37,7 @@ public SearchClient(Uri baseUri, params DelegatingHandler[] handlers)
         /// <param name="credentials">The credentials to connect to the service with</param>
         /// <param name="healthIndicatorStore">Health indicator store</param>
         /// <param name="handlers">Handlers to apply to the request in order from first to last</param>
-        public SearchClient(Uri baseUri, string resourceType, ICredentials credentials, IEndpointHealthIndicatorStore healthIndicatorStore, params DelegatingHandler[] handlers)
+        public SearchClient(Uri baseUri, string resourceType, ICredentials credentials, IEndpointHealthIndicatorStore healthIndicatorStore, Action<Exception> onException, params DelegatingHandler[] handlers)
         {
             _resourceType = resourceType;
 
@@ -58,7 +57,7 @@ public SearchClient(Uri baseUri, string resourceType, ICredentials credentials,
 
             _httpClient = new HttpClient(handler, disposeHandler: true);
 
-            _retryingHttpClientWrapper = new RetryingHttpClientWrapper(_httpClient, healthIndicatorStore);
+            _retryingHttpClientWrapper = new RetryingHttpClientWrapper(_httpClient, healthIndicatorStore, onException);
             _discoveryClient = new ServiceDiscoveryClient(_httpClient, baseUri);
         }