Typosquatting: add typosquatting check service codes (#6315)

Add the codes of typosquatting algorithms and retrieve the latest owner list info. 

1. Typosquatting check service with distance calculation and comparison; 
2. Normalize the string before checking;
3. Call `public bool IsDistanceLessThanThreshold(string str1, string str2, int threshold)` to compare two strings; (changed to private)
4. Call `public bool IsUploadedPackageIdTyposquatting(string uploadedPackageId)` to check typosquatting in the checlist.

Fixes: https://github.com/NuGet/Engineering/issues/1593

Author: zhhyu

src/NuGetGallery/Services/ITyposquattingCheckService.cs

@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGetGallery
+{
+    /// <summary>
+    /// This interface is used to check typo-squatting of uploaded package ID with the owner.  
+    /// </summary>
+    public interface ITyposquattingCheckService
+    {
+        /// <summary>
+        /// The function is used to check whether the uploaded package is a typo-squatting package.
+        /// </summary>
+        /// <param name="uploadedPackageId"> The package ID of the uploaded package. We check the pacakge ID with the packages in the gallery for typo-squatting issue</param>
+        /// <param name="uploadedPackageOwner"> The package owner of the uploaded package.</param>
+        bool IsUploadedPackageIdTyposquatting(string uploadedPackageId, User uploadedPackageOwner);
+    }
+}

src/NuGetGallery/Services/ITyposquattingUserService.cs

@@ -0,0 +1,19 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGetGallery
+{
+    /// <summary>
+    /// The interface and method are used to check the latest info like owners' list from the DB for typo-squatting. 
+    /// </summary>
+    public interface ITyposquattingUserService
+    {
+        /// <summary>
+        /// The function is used to check the latest info of owners from the DB to confirm that the uploaded package and the conflict package are not shared by the same user. 
+        /// </summary>
+        /// <param name="packageId"> The package ID of the potential conflict package in the gallery. 
+        ///                          We'd like to double check that the conflict package and uploaded package don't share the same user</param>
+        /// <param name="userName"> The package owner of the uploaded package.</param>
+        bool CanUserTyposquat(string packageId, string userName);
+    }
+}

src/NuGetGallery/Services/TyposquattingCheckService.cs

@@ -0,0 +1,100 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace NuGetGallery
+{
+    public class TyposquattingCheckService : ITyposquattingCheckService
+    {
+        // TODO: Threshold parameters will be saved in the configuration file.
+        // https://github.com/NuGet/Engineering/issues/1645
+        private static List<ThresholdInfo> _thresholdsList = new List<ThresholdInfo>
+        {
+            new ThresholdInfo { LowerBound = 0, UpperBound = 30, Threshold = 0 },
+            new ThresholdInfo { LowerBound = 30, UpperBound = 50, Threshold = 1 },
+            new ThresholdInfo { LowerBound = 50, UpperBound = 120, Threshold = 2 }
+        };
+        
+        // TODO: popular packages checklist will be implemented
+        // https://github.com/NuGet/Engineering/issues/1624
+        public static List<PackageInfo> PackagesCheckList { get; set; }
+
+        private readonly ITyposquattingUserService _userTyposquattingService;
+
+        public TyposquattingCheckService(ITyposquattingUserService typosquattingUserService)
+        {
+            _userTyposquattingService = typosquattingUserService ?? throw new ArgumentNullException(nameof(typosquattingUserService));
+        }
+
+        public bool IsUploadedPackageIdTyposquatting(string uploadedPackageId, User uploadedPackageOwner)
+        {
+            if (uploadedPackageId == null)
+            {
+                throw new ArgumentNullException(nameof(uploadedPackageId));
+            }
+
+            if (uploadedPackageOwner == null)
+            {
+                throw new ArgumentNullException(nameof(uploadedPackageOwner));
+            }
+
+            var threshold = GetThreshold(uploadedPackageId);
+            uploadedPackageId = TyposquattingStringNormalization.NormalizeString(uploadedPackageId);
+
+            var countCollision = 0;
+            Parallel.ForEach(PackagesCheckList, (package, loopState) =>
+            {
+                // TODO: handle the package which is owned by an organization. 
+                // https://github.com/NuGet/Engineering/issues/1656
+                if (package.Owners.Contains(uploadedPackageOwner.Username))
+                {
+                    return;
+                }
+
+                if (TyposquattingDistanceCalculation.IsDistanceLessThanThreshold(uploadedPackageId, package.Id, threshold))
+                {
+                    // Double check the owners list in the latest DB. 
+                    if (_userTyposquattingService.CanUserTyposquat(package.Id, uploadedPackageOwner.Username))
+                    {
+                        return;
+                    }
+
+                    Interlocked.Increment(ref countCollision);
+                    loopState.Stop();
+                }
+            });
+
+            return countCollision != 0;
+        }
+
+        private static int GetThreshold(string packageId)
+        {
+            foreach (var thresholdInfo in _thresholdsList)
+            {
+                if (packageId.Length >= thresholdInfo.LowerBound && packageId.Length < thresholdInfo.UpperBound)
+                {
+                    return thresholdInfo.Threshold;
+                }
+            }
+
+            throw new ArgumentException("There is no predefined typo-squatting threshold for this package Id: " + packageId);
+        }
+    }
+
+    public class PackageInfo
+    {
+        public string Id { get; set; }
+        public HashSet<string> Owners { get; set; }
+    }
+
+    public class ThresholdInfo
+    {
+        public int LowerBound { get; set; }
+        public int UpperBound { get; set; }
+        public int Threshold { get; set; }
+    }
+}

src/NuGetGallery/Services/TyposquattingDistanceCalculation.cs

@@ -0,0 +1,212 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace NuGetGallery
+{
+    public static class TyposquattingDistanceCalculation
+    {
+        private const char PlaceholderForAlignment = '*';  // This const place holder variable is used for strings alignment
+
+        private static readonly HashSet<char> SpecialCharacters = new HashSet<char> { '.', '_', '-' };
+        private static readonly string SpecialCharactersToString = "[" + new string(SpecialCharacters.ToArray()) + "]";
+
+        private class BasicEditDistanceInfo
+        {
+            public int Distance { get; set; }
+            public PathInfo[,] Path { get; set; }
+        }
+
+        private enum PathInfo
+        {
+            Match,
+            Delete,
+            Substitute,
+            Insert,
+        }
+
+        public static bool IsDistanceLessThanThreshold(string str1, string str2, int threshold)
+        {
+            if (str1 == null)
+            {
+                throw new ArgumentNullException(nameof(str1));
+            }
+            if (str2 == null)
+            {
+                throw new ArgumentNullException(nameof(str2));
+            }
+
+            var newStr1 = Regex.Replace(str1, SpecialCharactersToString, string.Empty);
+            var newStr2 = Regex.Replace(str2, SpecialCharactersToString, string.Empty);
+            if (Math.Abs(newStr1.Length - newStr2.Length) > threshold)
+            {
+                return false;
+            }
+
+            return GetDistance(str1, str2, threshold) <= threshold;
+        }
+        private static int GetDistance(string str1, string str2, int threshold)
+        {
+            var basicEditDistanceInfo = GetBasicEditDistanceWithPath(str1, str2);
+            if (basicEditDistanceInfo.Distance <= threshold)
+            {
+                return basicEditDistanceInfo.Distance;
+            }
+            var alignedStrings = TraceBackAndAlignStrings(basicEditDistanceInfo.Path, str1, str2);
+            var refreshedEditDistance = RefreshEditDistance(alignedStrings[0], alignedStrings[1], basicEditDistanceInfo.Distance);
+
+            return refreshedEditDistance;
+        }
+
+        /// <summary>
+        /// The following function is used to calculate the classical edit distance and construct the path in dynamic programming way.
+        /// </summary>
+        private static BasicEditDistanceInfo GetBasicEditDistanceWithPath(string str1, string str2)
+        {
+            var distances = new int[str1.Length + 1, str2.Length + 1];
+            var path = new PathInfo[str1.Length + 1, str2.Length + 1];
+            distances[0, 0] = 0;
+            path[0, 0] = PathInfo.Match;
+            for (var i = 1; i <= str1.Length; i++)
+            {
+                distances[i, 0] = i;
+                path[i, 0] = PathInfo.Delete;
+            }
+
+            for (var j = 1; j <= str2.Length; j++)
+            {
+                distances[0, j] = j;
+                path[0, j] = PathInfo.Insert;
+            }
+
+            for (var i = 1; i <= str1.Length; i++)
+            {
+                for (var j = 1; j <= str2.Length; j++)
+                {
+                    if (str1[i - 1] == str2[j - 1])
+                    {
+                        distances[i, j] = distances[i - 1, j - 1];
+                        path[i, j] = PathInfo.Match;
+                    }
+                    else
+                    {
+                        distances[i, j] = distances[i - 1, j - 1] + 1;
+                        path[i, j] = PathInfo.Substitute;
+
+                        if (distances[i - 1, j] + 1 < distances[i, j])
+                        {
+                            distances[i, j] = distances[i - 1, j] + 1;
+                            path[i, j] = PathInfo.Delete;
+                        }
+
+                        if (distances[i, j - 1] + 1 < distances[i, j])
+                        {
+                            distances[i, j] = distances[i, j - 1] + 1;
+                            path[i, j] = PathInfo.Insert;
+                        }
+                    }
+                }
+            }
+
+            return new BasicEditDistanceInfo
+            {
+                Distance = distances[str1.Length, str2.Length],
+                Path = path
+            };
+        }
+
+        /// <summary>
+        /// The following function is used to traceback based on the construction path and align two strings.
+        /// Example:  For two strings: "asp.net" "aspnet". After traceback and alignment, we will have aligned strings as "asp.net" "asp*net" ('*' is the placeholder).
+        /// The returned strings contain the two inputted strings after alignment. 
+        /// </summary>
+        private static string[] TraceBackAndAlignStrings(PathInfo[,] path, string str1, string str2)
+        {
+            var newStr1 = new StringBuilder(str1);
+            var newStr2 = new StringBuilder(str2);
+            var alignedStrs = new string[2];
+
+            var i = str1.Length;
+            var j = str2.Length;
+            while (i > 0 && j > 0)
+            {
+                switch (path[i, j])
+                {
+                    case PathInfo.Match:
+                        i--;
+                        j--;
+                        break;
+                    case PathInfo.Substitute:
+                        i--;
+                        j--;
+                        break;
+                    case PathInfo.Delete:
+                        newStr2.Insert(j, PlaceholderForAlignment);
+                        i--;
+                        break;
+                    case PathInfo.Insert:
+                        newStr1.Insert(i, PlaceholderForAlignment);
+                        j--;
+                        break;
+                    default:
+                        throw new ArgumentException("Invalidate operation for edit distance trace back: " + path[i, j]);
+                }
+            }
+
+            for (var k = 0; k < i; k++)
+            {
+                newStr2.Insert(k, PlaceholderForAlignment);
+            }
+
+            for (var k = 0; k < j; k++)
+            {
+                newStr1.Insert(k, PlaceholderForAlignment);
+            }
+
+            alignedStrs[0] = newStr1.ToString();
+            alignedStrs[1] = newStr2.ToString();
+
+            return alignedStrs;
+        }
+
+        /// <summary>
+        /// The following function is used to refresh the edit distance based on predefined rules. (Insert/Delete special characters will not account for distance)
+        /// Example:  For two aligned strings: "asp.net" "asp*net" ('*' is the placeholder), we will scan the two strings again and the mapping from '.' to '*' will not account for the distance.
+        ///           So the final distance will be 0 for these two strings "asp.net" "aspnet".
+        /// </summary>
+        private static int RefreshEditDistance(string alignedStr1, string alignedStr2, int basicEditDistance)
+        {
+            if (alignedStr1.Length != alignedStr2.Length)
+            {
+                throw new ArgumentException("The lengths of two aligned strings are not same!");
+            }
+
+            var sameSubstitution = 0;
+            for (var i = 0; i < alignedStr2.Length; i++)
+            {
+                if (alignedStr1[i] != alignedStr2[i])
+                {
+                    if (alignedStr1[i] == PlaceholderForAlignment && SpecialCharacters.Contains(alignedStr2[i]))
+                    {
+                        sameSubstitution += 1;
+                    }
+                    else if (alignedStr2[i] == PlaceholderForAlignment && SpecialCharacters.Contains(alignedStr1[i]))
+                    {
+                        sameSubstitution += 1;
+                    }
+                    else
+                    {
+                        continue;
+                    }
+                }
+            }
+
+            return basicEditDistance - sameSubstitution;
+        }        
+    }
+}