Change parsing algorithm from download URL to package id+version (#343)

* Change parsing algorithm from download URL to package id+version

* PR comments

* bug fix

* check failing cases

Author: skofman1

src/Stats.AzureCdnLogs.Common/LogEvents.cs

@@ -22,6 +22,8 @@ public class LogEvents
         public static EventId FailedToGetFtpResponse = new EventId(512, "Failed to get FTP response");
         public static EventId FailedToCheckAlreadyProcessedLogFilePackageStatistics = new EventId(513, "Failed to check already processed package statistics for log file");
         public static EventId FailedToCheckAlreadyProcessedLogFileToolStatistics = new EventId(514, "Failed to check already processed tool statistics for log file");
+        public static EventId MultiplePackageIDVersionParseOptions = new EventId(515, "Multiple package id/version parse options");
+        public static EventId TranslatedPackageIdVersion = new EventId(516, "Translated package id and version");
         public static EventId JobRunFailed = new EventId(550, "Job run failed");
         public static EventId JobInitFailed = new EventId(551, "Job initialization failed");
     }

src/Stats.AzureCdnLogs.Common/PackageDefinition.cs

@@ -92,7 +92,6 @@
     <Compile Include="CdnLogCustomFieldParser.cs" />
     <Compile Include="CdnLogEntryParser.cs" />
     <Compile Include="NuGetCustomHeaders.cs" />
-    <Compile Include="PackageDefinition.cs" />
     <Compile Include="PackageStatistics.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="ToolStatistics.cs" />

src/Stats.AzureCdnLogs.Common/Stats.AzureCdnLogs.Common.csproj

@@ -86,7 +86,7 @@ public override async Task Run()
             foreach (var leasedLogFile in leasedLogFiles)
             {
                 var packageTranslator = new PackageTranslator("packagetranslations.json");
-                var packageStatisticsParser = new PackageStatisticsParser(packageTranslator);
+                var packageStatisticsParser = new PackageStatisticsParser(packageTranslator, LoggerFactory);
                 await logProcessor.ProcessLogFileAsync(leasedLogFile, packageStatisticsParser, _aggregatesOnly);
 
                 if (_aggregatesOnly)

src/Stats.ImportAzureCdnStatistics/Job.cs

@@ -0,0 +1,90 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Web;
+using NuGet.Versioning;
+
+namespace Stats.ImportAzureCdnStatistics
+{
+    public class PackageDefinition
+    {
+        private const string _nupkgExtension = ".nupkg";
+        private const string _dotSeparator = ".";
+
+        public string PackageId { get; set; }
+        public string PackageVersion { get; set; }
+
+        public PackageDefinition()
+        {
+        }
+
+        private PackageDefinition(string packageId, string packageVersion)
+        {
+            PackageId = packageId.Trim();
+            PackageVersion = packageVersion.Trim();
+        }
+
+        public static IList<PackageDefinition> FromRequestUrl(string requestUrl)
+        {
+            if (string.IsNullOrWhiteSpace(requestUrl) || !requestUrl.EndsWith(_nupkgExtension, StringComparison.InvariantCultureIgnoreCase))
+            {
+                return null;
+            }
+
+            List<PackageDefinition> resolutionOptions = new List<PackageDefinition>();
+
+            requestUrl = HttpUtility.UrlDecode(requestUrl);
+
+            var urlSegments = requestUrl.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
+          
+            var fileName = urlSegments.Last();
+           
+            fileName = fileName.Remove(fileName.Length - _nupkgExtension.Length, _nupkgExtension.Length);
+
+            // Special handling for flat container
+            if (urlSegments.Length > 3)
+            {
+                var packageIdContainer = urlSegments[urlSegments.Length - 3];
+                var packageVersionContainer = urlSegments[urlSegments.Length - 2];
+
+                if (string.Equals(fileName, $"{packageIdContainer}.{packageVersionContainer}", StringComparison.InvariantCultureIgnoreCase))
+                {
+                    resolutionOptions.Add(new PackageDefinition(packageIdContainer, packageVersionContainer));
+                }
+            }
+
+            if (!resolutionOptions.Any())
+            {
+                var nextDotIndex = fileName.IndexOf('.');
+
+                while (nextDotIndex != -1)
+                {
+                    string packagePart = fileName.Substring(0, nextDotIndex);
+                    string versionPart = fileName.Substring(nextDotIndex + 1);
+
+                    if (NuGetVersion.TryParse(versionPart, out var parsedVersion))
+                    {
+                        var normalizedVersion = parsedVersion.ToNormalizedString();
+
+                        if (string.Equals(normalizedVersion, versionPart, StringComparison.InvariantCultureIgnoreCase))
+                        {
+                            resolutionOptions.Add(new PackageDefinition(packagePart, versionPart));
+                        }
+                    }
+
+                    nextDotIndex = fileName.IndexOf('.', nextDotIndex + 1);
+                }
+            }
+
+            return resolutionOptions;
+        }
+
+        public override string ToString()
+        {
+            return $"[{PackageId}, {PackageVersion}]";
+        }
+    }
+}

src/Stats.ImportAzureCdnStatistics/PackageDefinition.cs

@@ -1,7 +1,9 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using NuGet;
+using System;
+using System.Linq;
+using Microsoft.Extensions.Logging;
 using NuGet.Versioning;
 using Stats.AzureCdnLogs.Common;
 
@@ -10,25 +12,50 @@ namespace Stats.ImportAzureCdnStatistics
     public class PackageStatisticsParser
         : StatisticsParser, IPackageStatisticsParser
     {
+        private readonly ILogger<PackageStatisticsParser> _logger;
         private readonly PackageTranslator _packageTranslator;
 
-        public PackageStatisticsParser(PackageTranslator packageTranslator)
+        public PackageStatisticsParser(PackageTranslator packageTranslator, ILoggerFactory loggerFactory)
         {
+            if (loggerFactory == null)
+            {
+                throw new ArgumentNullException(nameof(loggerFactory));
+            }
+
             _packageTranslator = packageTranslator;
+            _logger = loggerFactory.CreateLogger<PackageStatisticsParser>();
         }
 
         public PackageStatistics FromCdnLogEntry(CdnLogEntry cdnLogEntry)
         {
-            var packageDefinition = PackageDefinition.FromRequestUrl(cdnLogEntry.RequestUrl);
+            var packageDefinitions = PackageDefinition.FromRequestUrl(cdnLogEntry.RequestUrl);
 
-            if (packageDefinition == null)
+            if (packageDefinitions == null || !packageDefinitions.Any())
             {
                 return null;
             }
 
+            if (packageDefinitions.Count > 1)
+            {
+                _logger.LogWarning(LogEvents.MultiplePackageIDVersionParseOptions, 
+                                   "Found multiple parse options for URL {RequestUrl}: {PackageDefinitions}",
+                                   cdnLogEntry.RequestUrl,
+                                   string.Join(", ", packageDefinitions));
+            }
+
+            var packageDefinition = packageDefinitions.First();
+
             if (_packageTranslator != null)
             {
-                packageDefinition = _packageTranslator.TranslatePackageDefinition(packageDefinition);
+                bool translateOccured = _packageTranslator.TryTranslatePackageDefinition(packageDefinition);
+
+                if (translateOccured)
+                {
+                    _logger.LogInformation(LogEvents.TranslatedPackageIdVersion,
+                                           "Translated package. Url: {RequestUrl}, New definition: {PackageDefinition}",
+                                           cdnLogEntry.RequestUrl,
+                                           packageDefinition);
+                }
             }
 
             var statistic = new PackageStatistics();
@@ -42,7 +69,7 @@ public PackageStatistics FromCdnLogEntry(CdnLogEntry cdnLogEntry)
             statistic.UserAgent = GetUserAgentValue(cdnLogEntry);
             statistic.EdgeServerIpAddress = cdnLogEntry.EdgeServerIpAddress;
 
-            // ignore blacklisted user agents
+            // Ignore blacklisted user agents
             if (!IsBlackListed(statistic.UserAgent))
             {
                 return statistic;

src/Stats.ImportAzureCdnStatistics/PackageStatisticsParser.cs

@@ -6,7 +6,6 @@
 using System.IO;
 using System.Text.RegularExpressions;
 using Newtonsoft.Json.Linq;
-using Stats.AzureCdnLogs.Common;
 
 namespace Stats.ImportAzureCdnStatistics
 {
@@ -45,8 +44,10 @@ public PackageTranslator(string packageTranslationsJsonPath)
             }
         }
 
-        public PackageDefinition TranslatePackageDefinition(PackageDefinition packageDefinition)
+        public bool TryTranslatePackageDefinition(PackageDefinition packageDefinition)
         {
+            bool translateOccurred = false;
+
             if (packageDefinition != null
                 && !string.IsNullOrEmpty(packageDefinition.PackageId)
                 && !string.IsNullOrEmpty(packageDefinition.PackageVersion)
@@ -63,12 +64,13 @@ public PackageDefinition TranslatePackageDefinition(PackageDefinition packageDef
                     {
                         packageDefinition.PackageId = potentialTranslation.CorrectedPackageId;
                         packageDefinition.PackageVersion = correctedPackageVersion;
+                        translateOccurred = true;
                         break;
                     }
                 }
             }
 
-            return packageDefinition;
+            return translateOccurred;
         }
     }
 }

src/Stats.ImportAzureCdnStatistics/PackageTranslator.cs

@@ -124,6 +124,7 @@
     <Reference Include="System.Spatial, Version=5.7.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\..\packages\System.Spatial.5.7.0\lib\net40\System.Spatial.dll</HintPath>
     </Reference>
+    <Reference Include="System.Web" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
     <Reference Include="Microsoft.CSharp" />
@@ -138,6 +139,7 @@
     <Compile Include="ApplicationInsightsHelper.cs" />
     <Compile Include="IPackageStatisticsParser.cs" />
     <Compile Include="IStatisticsBlobContainerUtility.cs" />
+    <Compile Include="PackageDefinition.cs" />
     <Compile Include="StatisticsBlobContainerUtility.cs" />
     <Compile Include="Dimensions\IpAddressFact.cs" />
     <Compile Include="IStatisticsWarehouse.cs" />

src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj

@@ -24,7 +24,7 @@ public class LogFileProcessorFacts
             new PackageTranslator("packageTranslations.json");
 
         private static readonly IPackageStatisticsParser _packageStatisticsParser =
-            new PackageStatisticsParser(_packageTranslator);
+            new PackageStatisticsParser(_packageTranslator, new LoggerFactory());
 
         public class WhenOnlyPackageStatisticsInLogFile
         {