Re-injecting the dead-letter files through Stats.ImportAzureCdnStatistics could lead to duplicate data (#107)

Author: xavierdecoster

src/Stats.AzureCdnLogs.Common/LogEvents.cs

@@ -20,6 +20,8 @@ public class LogEvents
         public static EventId UnknownAzureCdnPlatform = new EventId(510, "Unknown Azure CDN platform");
         public static EventId InvalidRawLogFileName = new EventId(511, "Invalid raw log filename");
         public static EventId FailedToGetFtpResponse = new EventId(512, "Failed to get FTP response");
+        public static EventId FailedToCheckAlreadyProcessedLogFilePackageStatistics = new EventId(513, "Failed to check already processed package statistics for log file");
+        public static EventId FailedToCheckAlreadyProcessedLogFileToolStatistics = new EventId(514, "Failed to check already processed tool statistics for log file");
         public static EventId JobRunFailed = new EventId(550, "Job run failed");
         public static EventId JobInitFailed = new EventId(551, "Job initialization failed");
     }

src/Stats.ImportAzureCdnStatistics/ILeasedLogFile.cs

@@ -7,13 +7,15 @@
 
 namespace Stats.ImportAzureCdnStatistics
 {
-    internal interface ILeasedLogFile
+    public interface ILeasedLogFile
         : IDisposable
     {
         string LeaseId { get; }
 
         string Uri { get; }
 
+        string BlobName { get; }
+
         CloudBlockBlob Blob { get; }
 
         Task AcquireInfiniteLeaseAsync();

src/Stats.ImportAzureCdnStatistics/IPackageStatisticsParser.cs

@@ -0,0 +1,12 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Stats.AzureCdnLogs.Common;
+
+namespace Stats.ImportAzureCdnStatistics
+{
+    public interface IPackageStatisticsParser
+    {
+        PackageStatistics FromCdnLogEntry(CdnLogEntry cdnLogEntry);
+    }
+}

src/Stats.ImportAzureCdnStatistics/IStatisticsBlobContainerUtility.cs

@@ -0,0 +1,17 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace Stats.ImportAzureCdnStatistics
+{
+    public interface IStatisticsBlobContainerUtility
+    {
+        Task<Stream> OpenCompressedBlobAsync(ILeasedLogFile logFile);
+        Task CopyToDeadLetterContainerAsync(ILeasedLogFile logFile, Exception e);
+        Task DeleteSourceBlobAsync(ILeasedLogFile logFile);
+        Task ArchiveBlobAsync(ILeasedLogFile logFile);
+    }
+}

src/Stats.ImportAzureCdnStatistics/IStatisticsWarehouse.cs

@@ -0,0 +1,20 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Data;
+using System.Threading.Tasks;
+using Stats.AzureCdnLogs.Common;
+
+namespace Stats.ImportAzureCdnStatistics
+{
+    public interface IStatisticsWarehouse
+    {
+        Task<List<DataTable>> CreateAsync(IReadOnlyCollection<PackageStatistics> sourceData, string logFileName);
+        Task<List<DataTable>> CreateAsync(IReadOnlyCollection<ToolStatistics> sourceData, string logFileName);
+        Task<bool> HasImportedToolStatisticsAsync(string logFileName);
+        Task<bool> HasImportedPackageStatisticsAsync(string logFileName);
+        Task InsertDownloadFactsAsync(List<DataTable> downloadFactsDataTables, string logFileName);
+        Task StoreLogFileAggregatesAsync(LogFileAggregates logFileAggregates);
+    }
+}

src/Stats.ImportAzureCdnStatistics/Job.cs

@@ -86,7 +86,12 @@ public override async Task<bool> Run()
 
                 // Create a parser
                 var warehouse = new Warehouse(_loggerFactory, _targetDatabase);
-                var logProcessor = new LogFileProcessor(targetBlobContainer, deadLetterBlobContainer, _loggerFactory, warehouse);
+                var statisticsBlobContainerUtility = new StatisticsBlobContainerUtility(
+                    targetBlobContainer,
+                    deadLetterBlobContainer,
+                    _loggerFactory);
+
+                var logProcessor = new LogFileProcessor(statisticsBlobContainerUtility, _loggerFactory, warehouse);
 
                 // Get the next to-be-processed raw log file using the cdn raw log file name prefix
                 var prefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_", _azureCdnPlatform.GetRawLogFilePrefix(), _azureCdnAccountNumber);

src/Stats.ImportAzureCdnStatistics/LogFileAggregates.cs

@@ -5,7 +5,7 @@
 
 namespace Stats.ImportAzureCdnStatistics
 {
-    internal class LogFileAggregates
+    public class LogFileAggregates
     {
         public LogFileAggregates(string logFileName)
         {