diff --git a/src/NuGet.Core/NuGet.Protocol/Model/ServiceIndexModel.cs b/src/NuGet.Core/NuGet.Protocol/Model/ServiceIndexModel.cs new file mode 100644 index 00000000000..e3c2c6a0f0d --- /dev/null +++ b/src/NuGet.Core/NuGet.Protocol/Model/ServiceIndexModel.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace NuGet.Protocol.Model +{ + internal sealed class ServiceIndexModel + { + [JsonPropertyName("version")] + public string? Version { get; set; } + + [JsonPropertyName("resources")] + public IReadOnlyList? Resources { get; set; } + } + + internal sealed class ServiceIndexEntryModel + { + [JsonPropertyName("@id")] + public string? Id { get; set; } + + /// JSON string or array of strings. + [JsonPropertyName("@type")] + public JsonElement Type { get; set; } + + /// Optional JSON string or array of strings. + [JsonPropertyName("clientVersion")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public JsonElement ClientVersion { get; set; } + } +} diff --git a/src/NuGet.Core/NuGet.Protocol/Providers/ServiceIndexResourceV3Provider.cs b/src/NuGet.Core/NuGet.Protocol/Providers/ServiceIndexResourceV3Provider.cs index 06688731eb2..30f878064d7 100644 --- a/src/NuGet.Core/NuGet.Protocol/Providers/ServiceIndexResourceV3Provider.cs +++ b/src/NuGet.Core/NuGet.Protocol/Providers/ServiceIndexResourceV3Provider.cs @@ -7,12 +7,14 @@ using System.Collections.Concurrent; using System.Globalization; using System.IO; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Newtonsoft.Json.Linq; using NuGet.Common; using NuGet.Configuration; using NuGet.Protocol.Core.Types; +using NuGet.Protocol.Model; +using NuGet.Protocol.Utility; using NuGet.Versioning; namespace NuGet.Protocol @@ -190,34 +192,34 @@ ex.InnerException is IOException && private static async Task ConsumeServiceIndexStreamAsync(Stream stream, DateTime utcNow, PackageSource source, CancellationToken token) { - // Parse the JSON - JObject json = await stream.AsJObjectAsync(token); - - // Use SemVer instead of NuGetVersion, the service index should always be - // in strict SemVer format - JToken versionToken; - if (json.TryGetValue("version", out versionToken) && - versionToken.Type == JTokenType.String) + ServiceIndexModel index; + try { - SemanticVersion version; - if (SemanticVersion.TryParse((string)versionToken, out version) && - version.Major == 3) - { - return new ServiceIndexResourceV3(json, utcNow, source); - } - else - { - string errorMessage = string.Format( - CultureInfo.CurrentCulture, - Strings.Protocol_UnsupportedVersion, - (string)versionToken); - throw new InvalidDataException(errorMessage); - } + index = await JsonSerializer.DeserializeAsync(stream, JsonContext.Default.ServiceIndexModel, token); } - else + catch (JsonException ex) + { + throw new InvalidDataException(string.Format( + CultureInfo.CurrentCulture, + Strings.Protocol_InvalidJsonObject, + source.Source), ex); + } + + if (index?.Version is not string versionString) { throw new InvalidDataException(Strings.Protocol_MissingVersion); } + + // Use SemVer instead of NuGetVersion; the service index should always be in strict SemVer format. + if (!SemanticVersion.TryParse(versionString, out SemanticVersion version) || version.Major != 3) + { + throw new InvalidDataException(string.Format( + CultureInfo.CurrentCulture, + Strings.Protocol_UnsupportedVersion, + versionString)); + } + + return new ServiceIndexResourceV3(index, utcNow, source); } protected class ServiceIndexCacheInfo diff --git a/src/NuGet.Core/NuGet.Protocol/Resources/ServiceIndexResourceV3.cs b/src/NuGet.Core/NuGet.Protocol/Resources/ServiceIndexResourceV3.cs index cda1adc1869..9bd8c276346 100644 --- a/src/NuGet.Core/NuGet.Protocol/Resources/ServiceIndexResourceV3.cs +++ b/src/NuGet.Core/NuGet.Protocol/Resources/ServiceIndexResourceV3.cs @@ -6,11 +6,14 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.Json; using Newtonsoft.Json.Linq; using NuGet.Configuration; using NuGet.Packaging; using NuGet.Protocol.Core.Types; using NuGet.Protocol.Events; +using NuGet.Protocol.Model; +using NuGet.Protocol.Utility; using NuGet.Versioning; namespace NuGet.Protocol @@ -21,6 +24,7 @@ namespace NuGet.Protocol public class ServiceIndexResourceV3 : INuGetResource { private readonly string _json; + private readonly ServiceIndexModel _model; private readonly IDictionary> _index; private readonly DateTime _requestTime; private static readonly IReadOnlyList _emptyEntries = new List(); @@ -35,6 +39,13 @@ internal ServiceIndexResourceV3(JObject index, DateTime requestTime, PackageSour public ServiceIndexResourceV3(JObject index, DateTime requestTime) : this(index, requestTime, null) { } + internal ServiceIndexResourceV3(ServiceIndexModel model, DateTime requestTime, PackageSource packageSource) + { + _model = model; + _index = MakeLookup(model, packageSource); + _requestTime = requestTime; + } + /// /// Time the index was requested /// @@ -56,10 +67,7 @@ public virtual IReadOnlyList Entries public virtual string Json { - get - { - return _json; - } + get { return _json ?? JsonSerializer.Serialize(_model, JsonContext.Default.ServiceIndexModel); } } /// @@ -148,6 +156,87 @@ public virtual IReadOnlyList GetServiceEntryUris(NuGetVersion clientVersion return GetServiceEntries(clientVersion, orderedTypes).Select(e => e.Uri).ToList(); } + private static IDictionary> MakeLookup(ServiceIndexModel index, PackageSource packageSource) + { + var result = new Dictionary>(StringComparer.Ordinal); + + if (index?.Resources is null) + { + return result; + } + + foreach (var resource in index.Resources) + { + var id = resource.Id; + if (string.IsNullOrEmpty(id) || !Uri.TryCreate(id, UriKind.Absolute, out Uri uri)) + { + continue; + } + + if (packageSource != null && uri.Scheme == Uri.UriSchemeHttp && packageSource.IsHttps) + { + ProtocolDiagnostics.RaiseEvent(new ProtocolDiagnosticServiceIndexEntryEvent(source: packageSource.Source, httpsSourceHasHttpResource: true)); + } + + var types = GetStringValues(resource.Type).ToArray(); + + var clientVersions = new List(); + if (resource.ClientVersion.ValueKind == JsonValueKind.Undefined) + { + clientVersions.Add(_defaultVersion); + } + else + { + foreach (var versionString in GetStringValues(resource.ClientVersion)) + { + if (SemanticVersion.TryParse(versionString, out SemanticVersion semVer)) + { + clientVersions.Add(semVer); + } + } + } + + foreach (var type in types) + { + foreach (var clientVersion in clientVersions) + { + if (!result.TryGetValue(type, out List entries)) + { + entries = new List(); + result.Add(type, entries); + } + + entries.Add(new ServiceIndexEntry(uri, type, clientVersion)); + } + } + } + + foreach (var type in result.Keys.ToArray()) + { + result[type] = result[type].OrderByDescending(e => e.ClientVersion).ToList(); + } + + return result; + } + + private static IEnumerable GetStringValues(JsonElement element) + { + if (element.ValueKind == JsonValueKind.Array) + { + foreach (var item in element.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.String) + { + yield return item.GetString(); + } + } + } + else if (element.ValueKind == JsonValueKind.String) + { + yield return element.GetString(); + } + } + private static IDictionary> MakeLookup(JObject index, PackageSource packageSource) { var result = new Dictionary>(StringComparer.Ordinal); diff --git a/src/NuGet.Core/NuGet.Protocol/Utility/JsonContext.cs b/src/NuGet.Core/NuGet.Protocol/Utility/JsonContext.cs index 2a181357a86..2786a20be6e 100644 --- a/src/NuGet.Core/NuGet.Protocol/Utility/JsonContext.cs +++ b/src/NuGet.Core/NuGet.Protocol/Utility/JsonContext.cs @@ -11,12 +11,12 @@ namespace NuGet.Protocol.Utility #pragma warning disable CS3016 // Arrays as attribute arguments is not CLS-compliant [JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true, PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, - GenerationMode = JsonSourceGenerationMode.Metadata, Converters = [typeof(VersionRangeStjConverter)])] #pragma warning restore CS3016 // Arrays as attribute arguments is not CLS-compliant [JsonSerializable(typeof(HttpFileSystemBasedFindPackageByIdResource.FlatContainerVersionList))] [JsonSerializable(typeof(IReadOnlyList), TypeInfoPropertyName = "VulnerabilityIndex")] [JsonSerializable(typeof(CaseInsensitiveDictionary>), TypeInfoPropertyName = "VulnerabilityPage")] + [JsonSerializable(typeof(ServiceIndexModel))] internal partial class JsonContext : JsonSerializerContext { } diff --git a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/ServiceIndexResourceV3ProviderTests.cs b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/ServiceIndexResourceV3ProviderTests.cs index d8dc90a285d..409c4592bca 100644 --- a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/ServiceIndexResourceV3ProviderTests.cs +++ b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/ServiceIndexResourceV3ProviderTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. #nullable disable @@ -105,8 +105,8 @@ public async Task TryCreate_Throws_IfSourceLocationDoesNotReturnValidJson(string } [Theory] - [InlineData("{ version: \"not-semver\" } ")] - [InlineData("{ version: \"3.0.0.0\" } ")] // not strict semver + [InlineData("{ \"version\": \"not-semver\" }")] + [InlineData("{ \"version\": \"3.0.0.0\" }")] // not strict semver public async Task TryCreate_Throws_IfInvalidVersionInJson(string content) { // Arrange @@ -127,9 +127,7 @@ public async Task TryCreate_Throws_IfInvalidVersionInJson(string content) } [Theory] - [InlineData("{ json: \"that does not contain version.\" }")] - [InlineData("{ version: 3 } ")] // version is not a string - [InlineData("{ version: { value: 3 } } ")] // version is not a string + [InlineData("{ \"json\": \"that does not contain version.\" }")] public async Task TryCreate_Throws_IfNoVersionInJson(string content) { // Arrange @@ -149,12 +147,36 @@ public async Task TryCreate_Throws_IfNoVersionInJson(string content) Assert.Equal("The source does not have the 'version' property.", exception.InnerException.Message); } + [Theory] + [InlineData("{ \"version\": 3 }")] // version is not a string + [InlineData("{ \"version\": { \"value\": 3 } }")] // version is not a string + public async Task TryCreate_Throws_IfVersionFieldIsWrongType(string content) + { + // Arrange + // STJ throws JsonException for type mismatches during deserialization, which + // maps to Protocol_InvalidJsonObject rather than Protocol_MissingVersion. + var source = "https://contoso.test/index.json"; + var httpProvider = StaticHttpSource.CreateHttpSource(new Dictionary { { source, content } }); + var provider = new ServiceIndexResourceV3Provider(); + var sourceRepository = new SourceRepository(new PackageSource(source), + new INuGetResourceProvider[] { httpProvider, provider }); + + // Act + var exception = await Assert.ThrowsAsync(async () => + { + var result = await provider.TryCreate(sourceRepository, default(CancellationToken)); + }); + + // Assert + Assert.IsType(exception.InnerException); + } + [Fact] public async Task TryCreate_ReturnsTrue_IfSourceLocationReturnsValidJson() { // Arrange var source = $"https://some-site-{new Guid().ToString()}.org/test.json"; - var content = @"{ version: '3.1.0-beta' }"; + var content = @"{ ""version"": ""3.1.0-beta"" }"; var httpProvider = StaticHttpSource.CreateHttpSource(new Dictionary { { source, content } }); var provider = new ServiceIndexResourceV3Provider(); var sourceRepository = new SourceRepository(new PackageSource(source), diff --git a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/ServiceIndexResourceV3Tests.cs b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/ServiceIndexResourceV3Tests.cs index d1085c4cfbe..0ef2f182667 100644 --- a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/ServiceIndexResourceV3Tests.cs +++ b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/ServiceIndexResourceV3Tests.cs @@ -1,11 +1,15 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; using System.Collections.Generic; +using System.Text.Json; using Newtonsoft.Json.Linq; -using Xunit; using NuGet.Protocol.Events; +using NuGet.Protocol.Model; +using NuGet.Protocol.Utility; +using NuGet.Versioning; +using Xunit; namespace NuGet.Protocol.Tests { @@ -115,5 +119,100 @@ private static JObject CreateServiceIndex() } }; } + + // STJ path: mirrors of the JObject-based tests above + + [Fact] + public void Constructor_WithModel_InitializesProperties() + { + var model = DeserializeModel(@"{""version"":""1.2.3"",""resources"":[{""@id"":""http://contoso.test/b"",""@type"":""a""}]}"); + var expectedRequestTime = DateTime.UtcNow; + + var resource = new ServiceIndexResourceV3(model, expectedRequestTime, packageSource: null); + + Assert.Equal(expectedRequestTime, resource.RequestTime); + Assert.Equal(1, resource.Entries.Count); + Assert.Equal("a", resource.Entries[0].Type); + Assert.Equal("http://contoso.test/b", resource.Entries[0].Uri.ToString()); + } + + [Fact] + public void Constructor_WithModel_JsonProperty_RoundTrips() + { + var model = DeserializeModel(@"{""version"":""1.2.3"",""resources"":[{""@id"":""http://contoso.test/b"",""@type"":""a""}]}"); + + var resource = new ServiceIndexResourceV3(model, DateTime.UtcNow, packageSource: null); + + // Json property re-serializes from the model; verify it round-trips to equivalent content. + using var doc = JsonDocument.Parse(resource.Json); + Assert.Equal("1.2.3", doc.RootElement.GetProperty("version").GetString()); + Assert.Equal(1, doc.RootElement.GetProperty("resources").GetArrayLength()); + } + + [Fact] + public void Constructor_WithModel_TypeAsArray_ExpandsToEntryPerType() + { + var model = DeserializeModel(@"{""version"":""3.0.0"",""resources"":[{""@id"":""http://contoso.test/b"",""@type"":[""a"",""b""]}]}"); + + var resource = new ServiceIndexResourceV3(model, DateTime.UtcNow, packageSource: null); + + Assert.Equal(2, resource.Entries.Count); + Assert.Contains(resource.Entries, e => e.Type == "a"); + Assert.Contains(resource.Entries, e => e.Type == "b"); + Assert.All(resource.Entries, e => Assert.Equal("http://contoso.test/b", e.Uri.ToString())); + } + + [Fact] + public void Constructor_WithModel_ClientVersionAsArray_CreatesEntryPerVersion() + { + var model = DeserializeModel(@"{""version"":""3.0.0"",""resources"":[{""@id"":""http://contoso.test/b"",""@type"":""a"",""clientVersion"":[""4.0.0"",""5.0.0""]}]}"); + + var resource = new ServiceIndexResourceV3(model, DateTime.UtcNow, packageSource: null); + + Assert.Equal(2, resource.Entries.Count); + Assert.Contains(resource.Entries, e => e.ClientVersion == new SemanticVersion(4, 0, 0)); + Assert.Contains(resource.Entries, e => e.ClientVersion == new SemanticVersion(5, 0, 0)); + } + + [Fact] + public void GetServiceEntries_WithModel_InvokesDiagnosticEventForHttpResourcesUnderHttpsSource() + { + int eventInvokeCount = 0; + var capturedEvents = new List(); + + ProtocolDiagnostics.ServiceIndexEntryEvent += (pdEvent) => + { + eventInvokeCount++; + capturedEvents.Add(pdEvent); + }; + + var source = "https://contoso.test/index.json"; + var model = DeserializeModel(CreateServiceIndexJsonWithFourResourceTypesTwoHttp()); + + var resource = new ServiceIndexResourceV3(model, DateTime.UtcNow, new Configuration.PackageSource(source)); + resource.GetServiceEntries(ServiceTypes.SearchQueryService); + + int httpResourceCapture = 0; + foreach (var serviceIndexEvent in capturedEvents) + { + Assert.Equal(source, serviceIndexEvent.Source); + httpResourceCapture += serviceIndexEvent.HttpsSourceHasHttpResource ? 1 : 0; + } + + Assert.Equal(2, httpResourceCapture); + Assert.Equal(2, eventInvokeCount); + } + + private static ServiceIndexModel DeserializeModel(string json) + => JsonSerializer.Deserialize(json, JsonContext.Default.ServiceIndexModel)!; + + private static string CreateServiceIndexJsonWithFourResourceTypesTwoHttp() => @"{ + ""version"": ""3.1.0-beta"", + ""resources"": [ + { ""@type"": ""SearchQueryService/Versioned"", ""@id"": ""http://contoso.test/A/5.0.0/2"", ""clientVersion"": ""5.0.0"" }, + { ""@type"": ""SearchQueryService/Versioned"", ""@id"": ""http://contoso.test/A/5.0.0/1"", ""clientVersion"": ""5.0.0"" }, + { ""@type"": ""SearchQueryService/Versioned"", ""@id"": ""https://contoso.test"", ""clientVersion"": ""4.0.0"" }, + { ""@type"": ""SearchQueryService/Versioned"", ""@id"": ""https://contoso.test"", ""clientVersion"": ""5.0.0"" } + ]}"; } }