[ST] 1. Catalog2Dnx: Apply caching control to package version index (#10685)

* add

* update

Author: zhhyu

src/Catalog/Dnx/DnxCatalogCollector.cs

@@ -79,13 +79,15 @@ protected override Task<IEnumerable<CatalogCommitItemBatch>> CreateBatchesAsync(
             return Task.FromResult(batches);
         }
 
-        protected override Task<bool> FetchAsync(
+        protected override async Task<bool> FetchAsync(
             CollectorHttpClient client,
             ReadWriteCursor front,
             ReadCursor back,
             CancellationToken cancellationToken)
         {
-            return CatalogCommitUtilities.ProcessCatalogCommitsAsync(
+            await DnxPackageVersionIndexCacheControl.LoadPackageIdsToIncludeAsync(_storageFactory.Create(), _logger, cancellationToken);
+
+            return await CatalogCommitUtilities.ProcessCatalogCommitsAsync(
                 client,
                 front,
                 back,
@@ -156,7 +158,7 @@ await catalogEntries.ForEachAsync(_maxConcurrentCommitItemsWithinBatch, async ca
                             cancellationToken);
                         var areRequiredPropertiesPresent = await AreRequiredPropertiesPresentAsync(destinationStorage, destinationUri);
 
-                        if (isNupkgSynchronized && isPackageInIndex && areRequiredPropertiesPresent)
+                        if (isNupkgSynchronized && areRequiredPropertiesPresent && isPackageInIndex && !DnxPackageVersionIndexCacheControl.PackageIdsToInclude.Contains(packageId))
                         {
                             _logger.LogInformation("No changes detected: {Id}/{Version}", packageId, normalizedPackageVersion);
 

src/Catalog/Dnx/DnxMaker.cs

@@ -219,7 +219,7 @@ public async Task UpdatePackageVersionIndexAsync(string id, Action<HashSet<NuGet
                 // Store versions (sorted)
                 result.Sort();
 
-                await storage.SaveAsync(resourceUri, CreateContent(result.Select(version => version.ToNormalizedString())), cancellationToken);
+                await storage.SaveAsync(resourceUri, CreateContentForPackageVersionIndex(id, result.Select(version => version.ToNormalizedString())), cancellationToken);
             }
             else
             {
@@ -260,10 +260,10 @@ private static HashSet<NuGetVersion> GetVersions(string json)
             return result;
         }
 
-        private StorageContent CreateContent(IEnumerable<string> versions)
+        private StorageContent CreateContentForPackageVersionIndex(string id, IEnumerable<string> versions)
         {
             JObject obj = new JObject { { "versions", new JArray(versions) } };
-            return new StringStorageContent(obj.ToString(), "application/json", Constants.NoStoreCacheControl);
+            return new StringStorageContent(obj.ToString(), "application/json", DnxPackageVersionIndexCacheControl.GetCacheControl(id, _logger));
         }
 
         private async Task<Uri> SaveNupkgAsync(Stream nupkgStream, CatalogStorage storage, string id, string version, CancellationToken cancellationToken)

src/Catalog/Dnx/DnxPackageVersionIndexCacheControl.cs

@@ -0,0 +1,63 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json.Linq;
+using NuGet.Services.Metadata.Catalog.Persistence;
+
+namespace NuGet.Services.Metadata.Catalog.Dnx
+{
+    public static class DnxPackageVersionIndexCacheControl
+    {
+        private const string DefaultCacheControlForPackageVersionIndex = "max-age=10";
+        private const string BlobNameOfPackageIdsToInclude = "PackageIdsToIncludeForCachingPackageVersionIndex.json";
+
+        public static HashSet<string> PackageIdsToInclude = new HashSet<string>();
+
+        public static string GetCacheControl(string id, ILogger logger)
+        {
+            if (PackageIdsToInclude.Contains(id))
+            {
+                logger.LogInformation("Add caching to the package version index of Package Id: {id}.", id);
+
+                return DefaultCacheControlForPackageVersionIndex;
+            }
+            else
+            {
+                return Constants.NoStoreCacheControl;
+            }
+        }
+
+        public static async Task LoadPackageIdsToIncludeAsync(IStorage storage, ILogger logger, CancellationToken cancellationToken)
+        {
+            if (!storage.Exists(BlobNameOfPackageIdsToInclude))
+            {
+                logger.LogInformation("{BlobName} does not exist, at {Address}.", BlobNameOfPackageIdsToInclude, storage.BaseAddress);
+
+                return;
+            }
+
+            logger.LogInformation("Loading the list of Package Ids from {BlobName}, at {Address}.", BlobNameOfPackageIdsToInclude, storage.BaseAddress);
+
+            PackageIdsToInclude = new HashSet<string>();
+            string jsonFile = await storage.LoadStringAsync(storage.ResolveUri(BlobNameOfPackageIdsToInclude), cancellationToken);
+            if (jsonFile != null)
+            {
+                JObject obj = JObject.Parse(jsonFile);
+                JArray ids = obj["ids"] as JArray;
+                if (ids != null)
+                {
+                    foreach (JToken id in ids)
+                    {
+                        PackageIdsToInclude.Add(id.ToString().ToLowerInvariant());
+                    }
+                }
+            }
+
+            logger.LogInformation("Loaded the list of Package Ids (Count: {Count}) from {BlobName}, at {Address}.", PackageIdsToInclude.Count, BlobNameOfPackageIdsToInclude, storage.BaseAddress);
+        }
+    }
+}

src/Catalog/Persistence/IStorage.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -27,6 +27,7 @@ Task<OptimisticConcurrencyControlToken> GetOptimisticConcurrencyControlTokenAsyn
         Task<string> LoadStringAsync(Uri resourceUri, CancellationToken cancellationToken);
         Uri ResolveUri(string relativeUri);
         Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken);
+        bool Exists(string fileName);
 
         /// <summary>
         /// Updates the cache control header on the provided resource URI (blob). This method throws an exception if
@@ -38,4 +39,4 @@ Task<OptimisticConcurrencyControlToken> GetOptimisticConcurrencyControlTokenAsyn
         /// <returns>True if the Cache-Control changes, false if the Cache-Control already matched the provided value.</returns>
         Task<bool> UpdateCacheControlAsync(Uri resourceUri, string cacheControl, CancellationToken cancellationToken);
     }
-}
+}

tests/CatalogTests/Dnx/DnxPackageVersionIndexCacheControlTests.cs

@@ -0,0 +1,58 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Moq;
+using NuGet.Services.Metadata.Catalog;
+using NuGet.Services.Metadata.Catalog.Dnx;
+using NuGet.Services.Metadata.Catalog.Persistence;
+using Xunit;
+
+namespace CatalogTests.Dnx
+{
+    public class DnxPackageVersionIndexCacheControlTests
+    {
+        [Fact]
+        public async Task LoadPackageIdsToIncludeAsync_BlobDoesNotExist()
+        {
+            var storage = new Mock<IStorage>();
+            storage.Setup(s => s.Exists(It.IsAny<string>())).Returns(false);
+
+            DnxPackageVersionIndexCacheControl.PackageIdsToInclude = new HashSet<string>();
+
+            await DnxPackageVersionIndexCacheControl.LoadPackageIdsToIncludeAsync(storage.Object, Mock.Of<ILogger>(), It.IsAny<CancellationToken>());
+
+            Assert.Empty(DnxPackageVersionIndexCacheControl.PackageIdsToInclude);
+        }
+
+        [Theory]
+        [InlineData("{\"ids\":[]}", 0)]
+        [InlineData("{\"ids\":[\"PackageId1\",\"packageid1\"]}", 1)]
+        [InlineData("{\"ids\":[\"PackageId1\",\"PackageId2\"]}", 2)]
+        public async Task LoadPackageIdsToIncludeAsync_BlobExists(string json, int count)
+        {
+            var storage = new Mock<IStorage>();
+            storage.Setup(s => s.Exists(It.IsAny<string>())).Returns(true);
+            storage.Setup(x => x.LoadStringAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>())).ReturnsAsync(json);
+
+            DnxPackageVersionIndexCacheControl.PackageIdsToInclude = new HashSet<string>();
+
+            await DnxPackageVersionIndexCacheControl.LoadPackageIdsToIncludeAsync(storage.Object, Mock.Of<ILogger>(), It.IsAny<CancellationToken>());
+
+            Assert.Equal(count, DnxPackageVersionIndexCacheControl.PackageIdsToInclude.Count);
+        }
+
+        [Fact]
+        public void GetCacheControl()
+        {
+            DnxPackageVersionIndexCacheControl.PackageIdsToInclude = new HashSet<string>() { "packageid1" };
+
+            Assert.Equal("max-age=10", DnxPackageVersionIndexCacheControl.GetCacheControl("packageid1", Mock.Of<ILogger>()));
+            Assert.Equal(Constants.NoStoreCacheControl, DnxPackageVersionIndexCacheControl.GetCacheControl("packageid2", Mock.Of<ILogger>()));
+        }
+    }
+}