Well-known user agents from China get parsed as (unknown) #1764 (#660)

Author: xavierdecoster

src/Stats.ImportAzureCdnStatistics/UserAgentParser.cs

@@ -4,21 +4,47 @@
 using System;
 using System.IO;
 using System.Reflection;
+using System.Text.RegularExpressions;
 using UAParser;
 
 namespace Stats.ImportAzureCdnStatistics
 {
     public class UserAgentParser
     {
         private static readonly Parser _defaultParser;
-        private static readonly Parser _customParser;
+        private static readonly Parser _knownClientsParser;
+        private static readonly Parser _knownClientsInChinaParser;
 
         static UserAgentParser()
         {
             _defaultParser = Parser.GetDefault();
 
             var yaml = ReadKnownClientsYaml();
-            _customParser = Parser.FromYaml(yaml);
+            _knownClientsParser = Parser.FromYaml(yaml);
+
+            var patchedYaml = AddSupportForChinaCdn(yaml);
+            _knownClientsInChinaParser = Parser.FromYaml(patchedYaml);
+        }
+
+        private static string AddSupportForChinaCdn(string yaml)
+        {
+            // Seems like user agent headers from requests hitting the China CDN endpoints 
+            // are using '+' characters instead of whitespace characters
+
+            var patchedYaml = Regex.Replace(
+                yaml,
+                @"(?:[:]\s'\()+([\w-.\s]+)(?:\))+", // Look for any matches of : '(user agent)
+                ReplaceWhitespaceWithPlusSign, // Replace whitespace ' ' character with '+' character in the user agent matches
+                RegexOptions.Compiled);
+
+            return patchedYaml;
+        }
+
+        private static string ReplaceWhitespaceWithPlusSign(Match match)
+        {
+            // The + sign needs to be escaped by a \ 
+            // as it is output to another regex in YAML.
+            return ": '(" + match.Groups[1].Value.Replace(" ", @"\+") + ")";
         }
 
         private static string ReadKnownClientsYaml()
@@ -35,7 +61,14 @@ private static string ReadKnownClientsYaml()
         public UserAgent ParseUserAgent(string userAgent)
         {
             // try custom parser first
-            var parsedResult = _customParser.ParseUserAgent(userAgent);
+            var parsedResult = _knownClientsParser.ParseUserAgent(userAgent);
+
+            if (string.Equals(parsedResult.Family, "other", StringComparison.InvariantCultureIgnoreCase))
+            {
+                // fallback to China parser
+                parsedResult = _knownClientsInChinaParser.ParseUserAgent(userAgent);
+            }
+
             if (string.Equals(parsedResult.Family, "other", StringComparison.InvariantCultureIgnoreCase))
             {
                 // fallback to default parser
@@ -47,7 +80,14 @@ public UserAgent ParseUserAgent(string userAgent)
         public OS ParseOS(string userAgent)
         {
             // try custom parser first
-            var parsedResult = _customParser.ParseOS(userAgent);
+            var parsedResult = _knownClientsParser.ParseOS(userAgent);
+
+            if (string.Equals(parsedResult.Family, "other", StringComparison.InvariantCultureIgnoreCase))
+            {
+                // fallback to China parser
+                parsedResult = _knownClientsInChinaParser.ParseOS(userAgent);
+            }
+
             if (string.Equals(parsedResult.Family, "other", StringComparison.InvariantCultureIgnoreCase))
             {
                 // fallback to default parser
@@ -60,7 +100,14 @@ public OS ParseOS(string userAgent)
         public Device ParseDevice(string userAgent)
         {
             // try custom parser first
-            var parsedResult = _customParser.ParseDevice(userAgent);
+            var parsedResult = _knownClientsParser.ParseDevice(userAgent);
+
+            if (string.Equals(parsedResult.Family, "other", StringComparison.InvariantCultureIgnoreCase))
+            {
+                // fallback to China parser
+                parsedResult = _knownClientsInChinaParser.ParseDevice(userAgent);
+            }
+
             if (string.Equals(parsedResult.Family, "other", StringComparison.InvariantCultureIgnoreCase))
             {
                 // fallback to default parser

tests/Tests.Stats.ImportAzureCdnStatistics/UserAgentParserFacts.cs

@@ -67,6 +67,8 @@ public TheFromPackageStatisticMethod(ITestOutputHelper testOutputHelper)
             [InlineData("NuGet Desktop MSBuild Task/4.4.0 (Microsoft Windows 10.0.15063)", "NuGet Desktop MSBuild Task", "4", "4", "0")]
             [InlineData("Cake NuGet Client/4.3.0 (Microsoft Windows 10.0.15063)", "Cake NuGet Client", "4", "3", "0")]
             [InlineData("NuGet VS VSIX/4.3.0 (Microsoft Windows 10.0.15063)", "NuGet VS VSIX", "4", "3", "0")]
+            [InlineData("NuGet+VS+VSIX/4.8.1+(Microsoft+Windows+NT+10.0.17134.0,+VS+Enterprise/15.0)", "NuGet VS VSIX", "4", "8", "1")]
+            [InlineData("NuGet+Command+Line/4.3.0+(Microsoft+Windows+NT+6.2.9200.0)", "NuGet Command Line", "4", "3", "0")]
             public void RecognizesCustomClients(string userAgent, string expectedClient, string expectedMajor, string expectedMinor, string expectedPatch)
             {
                 var parsed = _parser.ParseUserAgent(userAgent);