Enrich typo dictionary (#8741)

* Add

* Use StringInfo

* nit

Author: zhhyu

src/NuGetGallery/Services/TyposquattingDistanceCalculation.cs

@@ -30,7 +30,7 @@ private enum PathInfo
             Insert,
         }
 
-        public static bool IsDistanceLessThanThreshold(string str1, string str2, int threshold)
+        public static bool IsDistanceLessThanOrEqualToThreshold(string str1, string str2, int threshold)
         {
             if (str1 == null)
             {
@@ -50,6 +50,7 @@ public static bool IsDistanceLessThanThreshold(string str1, string str2, int thr
 
             return GetDistance(str1, str2, threshold) <= threshold;
         }
+
         private static int GetDistance(string str1, string str2, int threshold)
         {
             var basicEditDistanceInfo = GetBasicEditDistanceWithPath(str1, str2);

src/NuGetGallery/Services/TyposquattingService.cs

@@ -76,7 +76,7 @@ public bool IsUploadedPackageIdTyposquatting(string uploadedPackageId, User uplo
             Parallel.ForEach(packageIdsCheckList, (packageId, loopState) =>
             {
                 string normalizedPackageId = TyposquattingStringNormalization.NormalizeString(packageId);
-                if (TyposquattingDistanceCalculation.IsDistanceLessThanThreshold(normalizedUploadedPackageId, normalizedPackageId, threshold))
+                if (TyposquattingDistanceCalculation.IsDistanceLessThanOrEqualToThreshold(normalizedUploadedPackageId, normalizedPackageId, threshold))
                 {
                     collisionIds.Add(packageId);
                 }

src/NuGetGallery/Services/TyposquattingStringNormalization.cs

@@ -1,8 +1,9 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using System.Collections.Generic;
 using System.Text;
+using System.Globalization;
+using System.Collections.Generic;
 
 namespace NuGetGallery
 {
@@ -12,63 +13,70 @@ public static class TyposquattingStringNormalization
         /// The following dictionary is built through picking up similar characters manually from wiki unicode page.
         /// https://en.wikipedia.org/wiki/List_of_Unicode_characters
         /// </summary>
-        private static readonly IReadOnlyDictionary<char, string> SimilarCharacterDictionary = new Dictionary<char, string>()
+        private static readonly IReadOnlyDictionary<string, string> SimilarCharacterDictionary = new Dictionary<string, string>()
         {
-            {'a', "AÀÁÂÃÄÅàáâãäåĀāĂ㥹ǍǎǞǟǠǡǺǻȀȁȂȃȦȧȺΆΑάαἀἁἂἃἄἅἆἇἈἉἊἋἌἍἎἏӐӑӒӓὰάᾀᾁᾂᾃᾄᾅᾆᾇᾈᾊᾋᾌᾍᾎᾏᾰᾱᾲᾳᾴᾶᾷᾸᾹᾺΆᾼАДад"},
-            {'b', "BƀƁƂƃƄƅɃḂḃΒϦЂБВЪЬвъьѢѣҌҍႦႪხҔҕӃӄ" },
-            {'c', "CÇçĆćĈĉĊċČčƇƈȻȼϲϹСсҪҫ𐒨"},
-            {'d', "DÐĎďĐđƉƊƋƌǷḊḋԀԁԂԃ"},
-            {'e', "EÈÉÊËèéêëĒēĔĕĖėĘęĚěȄȅȆȇȨȩɆɇΈΕЀЁЄѐёҼҽҾҿӖӗἘἙἚἛἜἝῈΈЕе"},
-            {'f', "FƑƒḞḟϜϝҒғӺӻ"},
-            {'g', "GĜĝĞğĠġĢģƓǤǥǦǧǴǵԌԍ"},
-            {'h', "HĤĥħǶȞȟΉΗἨἩἪἫἬἭἮἯᾘᾙᾚᾛᾜᾝᾞᾟῊΉῌЋНнћҢңҤҥҺһӇӈӉӊԊԋԦԧԨԩհႬႹ𐒅𐒌𐒎𐒣"},
-            {'i', "I¡ìíîïǐȉȋΐίιϊіїὶίῐῑῒΐῖῗΊΙΪȊȈἰἱἲἳἴἵἶἷἸἹἺἻἼἽἾἿῘῙῚΊІЇӀӏÌÍÎÏĨĩĪīĬĭĮįİǏ"},
-            {'j', "JĴĵǰȷͿϳЈ"},
-            {'k', "KĶķĸƘƙǨǩΚκϏЌКкќҚқҜҝҞҟҠҡԞԟ"},
-            {'l', "LĹĺĻļĽľĿŀŁłſƖƪȴẛ"},
-            {'m', "MṀṁΜϺϻМмӍӎ𐒄"},
-            {'n', "NÑñŃńŅņŇňʼnƝǸǹΝᾐᾑᾒᾓᾔᾕᾖᾗῂῃῄῆῇпԤԥԦԧԮԯ𐒐"},
-            {'o', "OÒÓÔÕÖðòóôõöøŌōŎŏŐőƠơǑǒǪǫǬǭȌȍȎȏȪȫȬȭȮȯȰȱΌΟδοόϘϙὀὁὂὃὄὅὈὉὊὋὌὍὸόῸΌОоӦӧՕჿჾ𐒆𐒠0"},
-            {'p', "PÞþƤƥƿṖṗΡρϷϸῤῥῬРрҎҏႲႼ"},
-            {'q', "QȡɊɋԚԛգႭႳ"},
-            {'r', "RŔŕŖŗŘřƦȐȑȒȓɌɼгѓ"},
-            {'s', "SŚśŜŝŞşŠšȘșȿṠṡЅѕՏႽჽ𐒖𐒡"},
-            {'t', "TŢţŤťŦŧƬƭƮȚțȾṪṫͲͳΤτТтҬҭէ"},
-            {'u', "UÙÚÛÜùúûüŨũŪūŬŭŮůŰűŲųƯưǓǔǕǖǗǘǙǚǛǜȔȕȖȗμυϋύὐὑὒὓὔὕὖὗὺύῠῡῢΰῦῧՍႮ𐒩"},
-            {'v', "VƔƲνѴѵѶѷ"},
-            {'w', "WŴŵƜẀẁẂẃẄẅωώШЩшщѡѿὠὡὢὣὤὥὦὧὼώᾠᾡᾢᾣᾤᾥᾦᾧῲῳῴῶῷԜԝ"},
-            {'x', "X×ΧχХхҲҳӼӽӾӿჯ"},
-            {'y', "YÝýÿŶŷŸƳƴȲȳɎɏỲỳΎΥΫγϒϓϔЎУЧуўҮүҶҷҸҹӋӌӮӯӰӱӲӳӴӵὙὛὝὟῨῩῪΎႯႸ𐒋𐒦"},
-            {'z', "ZŹźŻżŽžƵƶȤȥΖჍ"},
-            {'3', "ƷǮǯȜȝʒЗзэӞӟӠӡჳ"},
-            {'8', "Ȣȣ"},
-            {'_', ".-" }
+            { "a", "AΑАαаÀÁÂÃÄÅàáâãäåĀāĂ㥹ǍǎǞǟǠǡǺǻȀȁȂȃȦȧȺΆάἀἁἂἃἄἅἆἇἈἉἊἋἌΆἍἎἏӐӑӒӓὰάᾀᾁᾂᾃᾄᾅᾆᾇᾈᾊᾋᾌᾍᾎᾏᾰᾱᾲᾳᾴᾶᾷᾸᾹᾺᾼДд"},
+            { "b", "BΒВЪЬƀƁƂƃƄƅɃḂḃϦЂБвъьѢѣҌҍႦႪხҔҕӃӄ"},
+            { "c", "CСсϹϲÇçĆćĈĉĊċČčƇƈȻȼҪҫ𐒨"},
+            { "d", "DƊԁÐĎďĐđƉƋƌǷḊḋԀԂԃ"},
+            { "e", "EΕЕеÈÉÊËèéêëĒēĔĕĖėĘęĚěȄȅȆȇȨȩɆɇΈЀЁЄѐёҼҽҾҿӖӗἘἙἚἛἜἝῈΈ"},
+            { "f", "FϜƑƒḞḟϝҒғӺӻ"},
+            { "g", "GǤԌĜĝĞğĠġĢģƓǥǦǧǴǵԍ"},
+            { "h", "HΗНһհҺĤĥħǶȞȟΉἨἩἪἫἬἭἮἯᾘᾙᾚᾛᾜᾝᾞᾟῊΉῌЋнћҢңҤҥӇӈӉӊԊԋԦԧԨԩႬႹ𐒅𐒌𐒎𐒣"},
+            { "i", "IΙІӀ¡ìíîïǐȉȋΐίιϊіїὶίῐῑῒΐῖῗΊΪȊȈἰἱἲἳἴἵἶἷἸἹἺἻἼἽἾἿῘῙῚΊЇӏÌÍÎÏĨĩĪīĬĭĮįİǏ"},
+            { "j", "JЈͿϳĴĵǰȷ"},
+            { "k", "KΚКKĶķĸƘƙǨǩκϏЌкќҚқҜҝҞҟҠҡԞԟ"},
+            { "l", "LĹĺĻļĽľĿŀŁłſƖƪȴẛ"},
+            { "m", "MΜМṀṁϺϻмӍӎ𐒄"},
+            { "n", "NΝпÑñŃńŅņŇňʼnƝǸǹᾐᾑᾒᾓᾔᾕᾖᾗῂῃῄῆῇԤԥԮԯ𐒐"},
+            { "o", "OΟОՕჿоοÒÓÔÕÖðòóôõöøŌōŎŏŐőƠơǑǒǪǫǬǭȌȍȎȏȪȫȬȭȮȯȰȱΌδόϘϙὀὁὂὃὄὅὈὉὊὋὌὍὸόῸΌӦӧჾ𐒆𐒠0"},
+            { "p", "PΡРрρÞþƤƥƿṖṗϷϸῤῥῬҎҏႲႼ"},
+            { "q", "QգԛȡɊɋԚႭႳ"},
+            { "r", "RгŔŕŖŗŘřƦȐȑȒȓɌɼѓ"},
+            { "s", "SЅѕՏႽჽŚśŜŝŞşŠšȘșȿṠṡ𐒖𐒡"},
+            { "t", "TΤТͲͳŢţŤťŦŧƬƭƮȚțȾṪṫτтҬҭէ"},
+            { "u", "UՍႮÙÚÛÜùúûüŨũŪūŬŭŮůŰűŲųƯưǓǔǕǖǗǘǙǚǛǜȔȕȖȗμυϋύὐὑὒὓὔὕὖὗὺύῠῡῢΰῦῧ𐒩"},
+            { "v", "VνѴѵƔƲѶѷ"},
+            { "w", "WωшԜԝŴŵƜẀẁẂẃẄẅώШЩщѡѿὠὡὢὣὤὥὦὧὼώᾠᾡᾢᾣᾤᾥᾦᾧῲῳῴῶῷ"},
+            { "x", "XХΧх×χҲҳӼӽӾӿჯ"},
+            { "y", "YΥҮƳуУÝýÿŶŷŸƴȲȳɎɏỲỳΎΫγϒϓϔЎЧўүҶҷҸҹӋӌӮӯӰӱӲӳӴӵὙὛὝὟῨῩῪΎႯႸ𐒋𐒦"},
+            { "z", "ZΖჍŹźŻżŽžƵƶȤȥ"},
+            { "3", "ƷЗʒӡჳǮǯȜȝзэӞӟӠ"},
+            { "8", "Ȣȣ"},
+            { "_", ".-" }
         };
-        
-        private static readonly IReadOnlyDictionary<char, char> NormalizedMappingDictionary = GetNormalizedMappingDictionary(SimilarCharacterDictionary);
+
+        private static readonly IReadOnlyDictionary<string, string> NormalizedMappingDictionary = GetNormalizedMappingDictionary(SimilarCharacterDictionary);
 
         public static string NormalizeString(string str)
         {
-            var normalizedStr = new StringBuilder(str);
-            for (var i = 0; i < normalizedStr.Length; i++)
+            var normalizedString = new StringBuilder();
+            var textElementEnumerator = StringInfo.GetTextElementEnumerator(str);
+            while (textElementEnumerator.MoveNext())
             {
-                if (NormalizedMappingDictionary.TryGetValue(normalizedStr[i], out var normalizedCharacter))
+                var textElement = textElementEnumerator.GetTextElement();
+                if (NormalizedMappingDictionary.TryGetValue(textElement, out var normalizedTextElement))
+                {
+                    normalizedString.Append(normalizedTextElement);
+                }
+                else
                 {
-                    normalizedStr[i] = normalizedCharacter;
+                    normalizedString.Append(textElement);
                 }
             }
 
-            return normalizedStr.ToString();
+            return normalizedString.ToString();
         }
 
-        private static Dictionary<char, char> GetNormalizedMappingDictionary(IReadOnlyDictionary<char, string> similarCharacterDictionary)
+        private static Dictionary<string, string> GetNormalizedMappingDictionary(IReadOnlyDictionary<string, string> similarCharacterDictionary)
         {
-            var normalizedMappingDictionary = new Dictionary<char, char>();
+            var normalizedMappingDictionary = new Dictionary<string, string>();
             foreach (var item in similarCharacterDictionary)
             {
-                foreach (var c in item.Value)
+                var textElementEnumerator = StringInfo.GetTextElementEnumerator(item.Value);
+                while (textElementEnumerator.MoveNext())
                 {
-                    normalizedMappingDictionary[c] = item.Key;
+                    normalizedMappingDictionary[textElementEnumerator.GetTextElement()] = item.Key;
                 }
             }
 

tests/NuGetGallery.Facts/Services/TyposquattingServiceFacts.cs

@@ -2,11 +2,12 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.Collections.Generic;
 using System.Linq;
+using System.Globalization;
+using System.Collections.Generic;
 using Moq;
-using NuGet.Services.Entities;
 using Xunit;
+using NuGet.Services.Entities;
 
 namespace NuGetGallery
 {
@@ -451,8 +452,8 @@ public void CheckTyposquattingDistance(string str1, string str2, int threshold)
             str2 = TyposquattingStringNormalization.NormalizeString(str2);
 
             // Act
-            var checkResult = TyposquattingDistanceCalculation.IsDistanceLessThanThreshold(str1, str2, threshold);
-            
+            var checkResult = TyposquattingDistanceCalculation.IsDistanceLessThanOrEqualToThreshold(str1, str2, threshold);
+
             // Assert
             Assert.True(checkResult);
         }
@@ -470,13 +471,26 @@ public void CheckNotTyposquattingDistance(string str1, string str2, int threshol
             str2 = TyposquattingStringNormalization.NormalizeString(str2);
 
             // Act
-            var checkResult = TyposquattingDistanceCalculation.IsDistanceLessThanThreshold(str1, str2, threshold);
-            
+            var checkResult = TyposquattingDistanceCalculation.IsDistanceLessThanOrEqualToThreshold(str1, str2, threshold);
+
             // Assert
             Assert.False(checkResult);
         }
-        
+
         [Theory]
+        [InlineData("ă", "a")]
+        [InlineData("aă", "aa")]
+        [InlineData("aăăa", "aaaa")]
+        [InlineData("𐒎", "h")]
+        [InlineData("h𐒎", "hh")]
+        [InlineData("h𐒎𐒎h", "hhhh")]
+        [InlineData("aă𐒎a", "aaha")]
+        [InlineData("a𐒎ăa", "ahaa")]
+        [InlineData("aă𐒎ăa", "aahaa")]
+        [InlineData("a𐒎ă𐒎a", "ahaha")]
+        [InlineData("aă𐒎ă𐒎a", "aahaha")]
+        [InlineData("aă𐒎𐒎ăă𐒎a", "aahhaaha")]
+        [InlineData("aă𐒎𐒎a𐒎ăă𐒎ă𐒎a", "aahhahaahaha")]
         [InlineData("Microsoft_NetFramework_v1", "microsoft_netframework_v1")]
         [InlineData("Microsoft.netframework-v1", "microsoft_netframework_v1")]
         [InlineData("mícr0s0ft.nёtFrǎmȇwὀrk.v1", "microsoft_netframework_v1")]
@@ -488,5 +502,60 @@ public void CheckNormalization(string str1, string str2)
             // Assert
             Assert.Equal(str1, str2);
         }
+
+        [Fact]
+        public void CheckNormalizationDictionary()
+        {
+            // Arrange
+            var similarCharacterDictionary = new Dictionary<string, string>()
+            {
+                { "a", "AΑАαаÀÁÂÃÄÅàáâãäåĀāĂ㥹ǍǎǞǟǠǡǺǻȀȁȂȃȦȧȺΆάἀἁἂἃἄἅἆἇἈἉἊἋἌΆἍἎἏӐӑӒӓὰάᾀᾁᾂᾃᾄᾅᾆᾇᾈᾊᾋᾌᾍᾎᾏᾰᾱᾲᾳᾴᾶᾷᾸᾹᾺᾼДд"},
+                { "b", "BΒВЪЬƀƁƂƃƄƅɃḂḃϦЂБвъьѢѣҌҍႦႪხҔҕӃӄ"},
+                { "c", "CСсϹϲÇçĆćĈĉĊċČčƇƈȻȼҪҫ𐒨"},
+                { "d", "DƊԁÐĎďĐđƉƋƌǷḊḋԀԂԃ"},
+                { "e", "EΕЕеÈÉÊËèéêëĒēĔĕĖėĘęĚěȄȅȆȇȨȩɆɇΈЀЁЄѐёҼҽҾҿӖӗἘἙἚἛἜἝῈΈ"},
+                { "f", "FϜƑƒḞḟϝҒғӺӻ"},
+                { "g", "GǤԌĜĝĞğĠġĢģƓǥǦǧǴǵԍ"},
+                { "h", "HΗНһհҺĤĥħǶȞȟΉἨἩἪἫἬἭἮἯᾘᾙᾚᾛᾜᾝᾞᾟῊΉῌЋнћҢңҤҥӇӈӉӊԊԋԦԧԨԩႬႹ𐒅𐒌𐒎𐒣"},
+                { "i", "IΙІӀ¡ìíîïǐȉȋΐίιϊіїὶίῐῑῒΐῖῗΊΪȊȈἰἱἲἳἴἵἶἷἸἹἺἻἼἽἾἿῘῙῚΊЇӏÌÍÎÏĨĩĪīĬĭĮįİǏ"},
+                { "j", "JЈͿϳĴĵǰȷ"},
+                { "k", "KΚКKĶķĸƘƙǨǩκϏЌкќҚқҜҝҞҟҠҡԞԟ"},
+                { "l", "LĹĺĻļĽľĿŀŁłſƖƪȴẛ"},
+                { "m", "MΜМṀṁϺϻмӍӎ𐒄"},
+                { "n", "NΝпÑñŃńŅņŇňʼnƝǸǹᾐᾑᾒᾓᾔᾕᾖᾗῂῃῄῆῇԤԥԮԯ𐒐"},
+                { "o", "OΟОՕჿоοÒÓÔÕÖðòóôõöøŌōŎŏŐőƠơǑǒǪǫǬǭȌȍȎȏȪȫȬȭȮȯȰȱΌδόϘϙὀὁὂὃὄὅὈὉὊὋὌὍὸόῸΌӦӧჾ𐒆𐒠0"},
+                { "p", "PΡРрρÞþƤƥƿṖṗϷϸῤῥῬҎҏႲႼ"},
+                { "q", "QգԛȡɊɋԚႭႳ"},
+                { "r", "RгŔŕŖŗŘřƦȐȑȒȓɌɼѓ"},
+                { "s", "SЅѕՏႽჽŚśŜŝŞşŠšȘșȿṠṡ𐒖𐒡"},
+                { "t", "TΤТͲͳŢţŤťŦŧƬƭƮȚțȾṪṫτтҬҭէ"},
+                { "u", "UՍႮÙÚÛÜùúûüŨũŪūŬŭŮůŰűŲųƯưǓǔǕǖǗǘǙǚǛǜȔȕȖȗμυϋύὐὑὒὓὔὕὖὗὺύῠῡῢΰῦῧ𐒩"},
+                { "v", "VνѴѵƔƲѶѷ"},
+                { "w", "WωшԜԝŴŵƜẀẁẂẃẄẅώШЩщѡѿὠὡὢὣὤὥὦὧὼώᾠᾡᾢᾣᾤᾥᾦᾧῲῳῴῶῷ"},
+                { "x", "XХΧх×χҲҳӼӽӾӿჯ"},
+                { "y", "YΥҮƳуУÝýÿŶŷŸƴȲȳɎɏỲỳΎΫγϒϓϔЎЧўүҶҷҸҹӋӌӮӯӰӱӲӳӴӵὙὛὝὟῨῩῪΎႯႸ𐒋𐒦"},
+                { "z", "ZΖჍŹźŻżŽžƵƶȤȥ"},
+                { "3", "ƷЗʒӡჳǮǯȜȝзэӞӟӠ"},
+                { "8", "Ȣȣ"},
+                { "_", ".-" }
+            };
+
+            var testPackageName = "testpackage";
+            foreach (var item in similarCharacterDictionary)
+            {
+                var textElementEnumerator = StringInfo.GetTextElementEnumerator(item.Value);
+                while (textElementEnumerator.MoveNext())
+                {
+                    var typoString = testPackageName + textElementEnumerator.GetTextElement();
+                    var baseString = testPackageName + item.Key;
+
+                    // Act
+                    var normalizedString = TyposquattingStringNormalization.NormalizeString(typoString);
+
+                    // Assert
+                    Assert.Equal(baseString, normalizedString);
+                }
+            }
+        }
     }
 }