CollectAzureCdnLogs should properly handle files ending in .tmp (#673)

Author: Scott Bommarito

src/Stats.CollectAzureCdnLogs/Ftp/FtpRawLogClient.cs

@@ -97,7 +97,7 @@ public async Task DeleteAsync(Uri uri)
             }
         }
 
-        public async Task<IEnumerable<RawLogFileInfo>> GetRawLogFiles(Uri uri)
+        public async Task<IEnumerable<Uri>> GetRawLogFileUris(Uri uri)
         {
             if (uri == null)
             {
@@ -124,15 +124,13 @@ public async Task<IEnumerable<RawLogFileInfo>> GetRawLogFiles(Uri uri)
 
                     var fileNames = directoryList.Split(Environment.NewLine.ToCharArray(),
                         StringSplitOptions.RemoveEmptyEntries);
-                    var rawLogFiles = fileNames.Select(fn => new RawLogFileInfo(new Uri(uri.EnsureTrailingSlash(), fn)));
-
-                    return rawLogFiles;
+                    return fileNames.Select(fn => new Uri(uri.EnsureTrailingSlash(), fn));
                 }
                 catch (Exception e)
                 {
                     Logger.LogError(LogEvents.FailedBlobListing, e, "Failed to get raw log files.");
 
-                    return Enumerable.Empty<RawLogFileInfo>();
+                    return Enumerable.Empty<Uri>();
                 }
             }
         }

src/Stats.CollectAzureCdnLogs/IRawLogClient.cs

@@ -10,7 +10,7 @@ namespace Stats.CollectAzureCdnLogs
 {
     internal interface IRawLogClient
     {
-        Task<IEnumerable<RawLogFileInfo>> GetRawLogFiles(Uri uri);
+        Task<IEnumerable<Uri>> GetRawLogFileUris(Uri uri);
         Task<Stream> OpenReadAsync(Uri uri);
         Task<bool> RenameAsync(Uri uri, string newFileName);
         Task DeleteAsync(Uri uri);

src/Stats.CollectAzureCdnLogs/Job.cs

@@ -114,15 +114,17 @@ public override async Task Run()
             var azureClient = new CloudBlobRawLogClient(LoggerFactory, _cloudStorageAccount);
 
             // Collect directory listing.
-            var rawLogFiles = await ftpClient.GetRawLogFiles(_ftpServerUri);
+            var rawLogFileUris = await ftpClient.GetRawLogFileUris(_ftpServerUri);
 
             // Prepare cloud storage blob container.
             var cloudBlobContainer = await azureClient.CreateContainerIfNotExistsAsync(_cloudStorageContainerName);
 
-            foreach (var rawLogFile in rawLogFiles)
+            foreach (var rawLogFileUri in rawLogFileUris)
             {
                 try
                 {
+                    var rawLogFile = new RawLogFileInfo(rawLogFileUri);
+
                     if (_azureCdnPlatform != rawLogFile.AzureCdnPlatform
                         || !_azureCdnAccountNumber.Equals(rawLogFile.AzureCdnAccountNumber, StringComparison.InvariantCultureIgnoreCase))
                     {

src/Stats.CollectAzureCdnLogs/RawLogFileInfo.cs

@@ -70,10 +70,8 @@ private void TryParseFileName()
                     if (Extension.EndsWith(FileExtensions.Gzip, StringComparison.InvariantCultureIgnoreCase))
                     {
                         ContentType = _contentTypeGzip;
-                    }
-                    else
-                    {
-                        throw new InvalidRawLogFileNameException(FileName);
+
+                        return;
                     }
                 }
                 else if (lastPart.Count() == 4)
@@ -85,17 +83,13 @@ private void TryParseFileName()
                     {
                         IsPendingDownload = true;
                         ContentType = _contentTypeGzip;
+
+                        return;
                     }
                 }
-                else
-                {
-                    throw new InvalidRawLogFileNameException(FileName);
-                }
-            }
-            else
-            {
-                throw new InvalidRawLogFileNameException(FileName);
             }
+
+            throw new InvalidRawLogFileNameException(FileName);
         }
 
         private int TryParseRollingFileNumber(string rollingFileNumberString)

tests/Tests.Stats.CollectAzureCdnLogs/RawLogFileInfoTests.cs

@@ -60,6 +60,7 @@ public void ThrowsWhenInvalidUri()
         [InlineData("ftp://someserver/logs/wpc_A000_20150603_0058.log")]
         [InlineData("ftp://someserver/logs/wpc_A000_20151342_0058.log.gz")]
         [InlineData("ftp://someserver/logs/wpc_A000_20150603_0058.log.download")]
+        [InlineData("ftp://someserver/logs/wpc_A000_20150603_0058.log.gz.tmp")]
         public void ThrowsWhenInvalidRawLogFileName(string uriString)
         {
             Assert.Throws<InvalidRawLogFileNameException>(() => new RawLogFileInfo(new Uri(uriString)));