[Repair Item] Fix unicode character on nuspec. (#10157)

* fix unicode characters
* update test package
* removed non needed file
* gitignore update for testdata nupkg
* added normalized warning and test.
* removed non needed nupkg updated test for valid

Author: dannyjdev

.gitignore

@@ -194,6 +194,7 @@ PublishScripts/
 
 # NuGet Packages
 *.nupkg
+!**/TestData/*.nupkg
 # NuGet Symbol Packages
 *.snupkg
 # The packages folder can be ignored because of Package Restore

src/NuGetGallery/Services/PackageMetadataValidationService.cs

@@ -1,10 +1,11 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Xml.Linq;
@@ -99,7 +100,7 @@ public async Task<PackageValidationResult> ValidateMetadataBeforeUploadAsync(
         {
             var warnings = new List<IValidationMessage>();
 
-            var result = await CheckPackageEntryCountAsync(nuGetPackage, warnings);
+            PackageValidationResult result = await CheckPackageEntryCountAsync(nuGetPackage, warnings);
 
             if (result != null)
             {
@@ -112,7 +113,7 @@ public async Task<PackageValidationResult> ValidateMetadataBeforeUploadAsync(
             {
                 return result;
             }
-
+            
             var nuspecFileEntry = nuGetPackage.GetEntry(nuGetPackage.GetNuspecFile());
             using (var nuspecFileStream = await nuGetPackage.GetNuspecAsync(CancellationToken.None))
             {
@@ -122,6 +123,13 @@ public async Task<PackageValidationResult> ValidateMetadataBeforeUploadAsync(
                 }
             }
 
+            result = CheckNuspecNormalized(nuGetPackage, warnings);
+
+            if (result != null)
+            {
+                return result;
+            }
+
             result = await CheckForUnsignedPushAfterAuthorSignedAsync(
                 nuGetPackage,
                 warnings);
@@ -161,6 +169,26 @@ public async Task<PackageValidationResult> ValidateMetadataBeforeUploadAsync(
             return PackageValidationResult.AcceptedWithWarnings(warnings);
         }
 
+        private PackageValidationResult CheckNuspecNormalized(PackageArchiveReader nuGetPackage, List<IValidationMessage> warnings)
+        {
+            try
+            {
+                UserContentEnabledNuspecReader nuspecReader = GetNuspecReader(nuGetPackage);
+                string nuspec = nuspecReader.Xml.ToStringSafe();
+
+                if (!nuspec.IsNormalized(NormalizationForm.FormC))
+                {
+                    warnings.Add(new PlainTextOnlyValidationMessage(Strings.UploadPackage_NuspecIsNotNormalized));
+                }
+            }
+            catch (Exception)
+            {
+                return PackageValidationResult.Invalid(Strings.UploadPackage_NuspecContainsInvalidUnicodeCharacters);
+            }
+
+            return null;
+        }
+
         private async Task<PackageValidationResult> CheckLicenseMetadataAsync(PackageArchiveReader nuGetPackage, List<IValidationMessage> warnings, User user)
         {
             var nuspecReader = GetNuspecReader(nuGetPackage);
@@ -873,4 +901,4 @@ public async Task<PackageValidationResult> ValidateSignatureFilePresenceAsync(
             return null;
         }
     }
-}
+}

src/NuGetGallery/Strings.Designer.cs

@@ -1263,4 +1263,10 @@ The {1} Team</value>
   <data name="FrameworkFilters_Tooltip" xml:space="preserve">
     <value>Filters packages based on the target frameworks they are compatible with.</value>
   </data>
+  <data name="UploadPackage_NuspecContainsInvalidUnicodeCharacters" xml:space="preserve">
+    <value>The package nuspec file contains invalid unicode characters.</value>
+  </data>
+  <data name="UploadPackage_NuspecIsNotNormalized" xml:space="preserve">
+    <value>The package nuspec file is not compliant with the normalization form C (NFC).</value>
+  </data>
 </root>

src/NuGetGallery/Strings.resx

@@ -340,4 +340,4 @@ public static MemoryStream CreateTestPackageStream(Action<ZipArchive> populatePa
             return packageStream;
         }
     }
-}
+}

tests/NuGetGallery.Core.Facts/TestUtils/TestPackage.cs

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <TargetFramework>net472</TargetFramework>
     <RootNamespace>NuGetGallery</RootNamespace>
@@ -14,6 +14,7 @@
     <EmbeddedResource Include="TestData\PackageWithDoubleForwardSlash.1.0.0.nupkg" />
     <EmbeddedResource Include="TestData\PackageWithVeryLongZipFileEntry.1.0.0.nupkg" />
     <EmbeddedResource Include="TestData\Zip64Package.Corrupted.1.0.0.nupkg" />
+    <EmbeddedResource Include="TestData\PackageWithInvalidUnicodeCharacters.1.0.0.nupkg" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\src\NuGetGallery\NuGetGallery.csproj" />

tests/NuGetGallery.Facts/NuGetGallery.Facts.csproj

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -361,6 +361,64 @@ public async Task WithNotTooManyPackageEntries_WhenRejectPackagesWithTooManyPack
                 Assert.Empty(result.Warnings);
             }
 
+            [Fact]
+            public async Task WithInvalidUnicodeCharactersInNuspec_ReturnsInvalidPackage()
+            {
+                // Arrange
+                var package = TestDataResourceUtility.GetResourceBytes("PackageWithInvalidUnicodeCharacters.1.0.0.nupkg");
+                var fakeFileStream = new MemoryStream(package);
+                var reader = new PackageArchiveReader(fakeFileStream);
+
+                // Act
+                PackageValidationResult result = await _target.ValidateMetadataBeforeUploadAsync(
+                    reader,
+                    PackageMetadata.FromNuspecReader(reader.GetNuspecReader(), strict: true),
+                    _currentUser);
+
+                // Assert
+                Assert.Equal(PackageValidationResultType.Invalid, result.Type);
+                Assert.Equal("The package nuspec file contains invalid unicode characters.", result.Message.PlainTextMessage);
+                Assert.Empty(result.Warnings);
+            }
+
+            [Fact]
+            public async Task WithValidUnicodeCharactersInNuspec_ReturnsAcceptedPackage()
+            {
+                // Arrange
+                var package = GeneratePackageWithUserContent(
+                    licenseUrl: GetLicenseExpressionDeprecationUrl("MIT"),
+                    licenseExpression: "MIT",
+                    releaseNotes: "Release notes");
+
+                // Act
+                PackageValidationResult result = await _target.ValidateMetadataBeforeUploadAsync(
+                    package.Object,
+                    GetPackageMetadata(package),
+                    _currentUser);
+
+                // Assert
+                Assert.Equal(PackageValidationResultType.Accepted, result.Type);
+                Assert.Empty(result.Warnings);
+            }
+
+            [Fact]
+            public async Task WithNonNormalizeNuspec_ReturnsAcceptedPackageWithWarnings()
+            {
+                // Arrange
+                var package = GeneratePackageWithUserContent(releaseNotes: "Non normalized character: e\u0301");
+
+                // Act
+                PackageValidationResult result = await _target.ValidateMetadataBeforeUploadAsync(
+                    package.Object,
+                    GetPackageMetadata(package),
+                    _currentUser);
+
+                // Assert
+                Assert.Equal(PackageValidationResultType.Accepted, result.Type);
+                Assert.NotEmpty(result.Warnings);
+                Assert.Equal("The package nuspec file is not compliant with the normalization form C (NFC).", result.Warnings[0].PlainTextMessage);
+            }
+
             [Theory]
             [InlineData("duplicatedFile.txt", "duplicatedFile.txt")]
             [InlineData("./temp/duplicatedFile.txt", "./temp/duplicatedFile.txt")]
@@ -448,10 +506,10 @@ public async Task HandlesMissingLicenseAccordingToSettings(bool allowLicenseless
             public async Task HandlesMissingLicenseAccordingToSettingsWhenDisplayUploadWarningV2Enabled(bool allowLicenselessPackages, bool expectedSuccess)
             {
                 _nuGetPackage = GeneratePackageWithUserContent(
-                    licenseUrl: null, 
-                    licenseExpression: null, 
-                    licenseFilename: null, 
-                    readmeFilename:"readme.md",
+                    licenseUrl: null,
+                    licenseExpression: null,
+                    licenseFilename: null,
+                    readmeFilename: "readme.md",
                     readmeFileContents: "read me");
 
                 _config
@@ -2136,6 +2194,7 @@ protected static Mock<TestPackageReader> GeneratePackageWithUserContent(
                 string readmeFilename = null,
                 string readmeFileContents = null,
                 byte[] readmeFileBinaryContents = null,
+                string releaseNotes = null,
                 IReadOnlyList<string> entryNames = null)
             {
                 var packageStream = GeneratePackageStream(
@@ -2155,6 +2214,7 @@ protected static Mock<TestPackageReader> GeneratePackageWithUserContent(
                     readmeFilename: readmeFilename,
                     readmeFileContents: readmeFileContents,
                     readmeFileBinaryContents: readmeFileBinaryContents,
+                    releaseNotes: releaseNotes,
                     entryNames: entryNames);
 
                 return PackageServiceUtility.CreateNuGetPackage(packageStream);
@@ -2177,6 +2237,7 @@ protected static MemoryStream GeneratePackageStream(
                 string readmeFilename = null,
                 string readmeFileContents = null,
                 byte[] readmeFileBinaryContents = null,
+                string releaseNotes = null,
                 IReadOnlyList<string> entryNames = null)
             {
                 return PackageServiceUtility.CreateNuGetPackageStream(
@@ -2195,6 +2256,7 @@ protected static MemoryStream GeneratePackageStream(
                     iconFileBinaryContents: iconFileBinaryContents,
                     readmeFilename: readmeFilename,
                     readmeFileContents: GetBinaryLicenseOrReadmeFileContents(readmeFileBinaryContents, readmeFileContents),
+                    releaseNotes: releaseNotes,
                     entryNames: entryNames);
             }
 
@@ -2214,4 +2276,4 @@ private static byte[] GetBinaryLicenseOrReadmeFileContents(byte[] binaryContents
             }
         }
     }
-}
+}