Skip to content

Commit becd47d

Browse files
lyndaidaiilyndaidaii
authored
Gallery store/fetch/delete embedded readmes file from flat container (#8377)
* gallery fetch/delete readmes file from flat container * add deletion for embedded readme
1 parent 61bf3ce commit becd47d

16 files changed

Lines changed: 613 additions & 240 deletions

‎src/NuGetGallery.Core/Services/CoreReadmeFileService.cs‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.IO;
6+
using System.Threading.Tasks;
7+
using NuGet.Packaging;
8+
using NuGet.Services.Entities;
9+
using NuGetGallery.Packaging;
10+
11+
namespace NuGetGallery
12+
{
13+
public class CoreReadmeFileService : ICoreReadmeFileService
14+
{
15+
private const string ReadmeFileName = "readme";
16+
17+
private readonly ICoreFileStorageService _fileStorageService;
18+
private readonly IContentFileMetadataService _metadata;
19+
20+
public CoreReadmeFileService(ICoreFileStorageService fileStorageService, IContentFileMetadataService metadata)
21+
{
22+
_fileStorageService = fileStorageService ?? throw new ArgumentNullException(nameof(fileStorageService));
23+
_metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
24+
}
25+
26+
/// <summary>
27+
/// Saves the package readme.md file to storage. This method should throw if the package
28+
/// does not have an embedded readme file
29+
/// </summary>
30+
/// <param name="package">The package associated with the readme.</param>
31+
/// <param name="readmeFile">The content of readme file.</param>
32+
private Task SaveReadmeFileAsync(Package package, Stream readmeFile)
33+
{
34+
if (package == null)
35+
{
36+
throw new ArgumentNullException(nameof(package));
37+
}
38+
39+
if (readmeFile == null)
40+
{
41+
throw new ArgumentNullException(nameof(readmeFile));
42+
}
43+
44+
if (package.EmbeddedReadmeType == EmbeddedReadmeFileType.Absent)
45+
{
46+
throw new ArgumentException("Package must have an embedded readme", nameof(package));
47+
}
48+
49+
var fileName = BuildReadmeFileName(package);
50+
51+
return _fileStorageService.SaveFileAsync(_metadata.PackageContentFolderName, fileName, readmeFile, overwrite: true);
52+
}
53+
54+
/// <summary>
55+
/// Save the readme file from package stream. This method should throw if the package
56+
/// does not have an embedded readme file
57+
/// </summary>
58+
/// <param name="package">Package information.</param>
59+
/// <param name="packageStream">Package stream with .nupkg contents.</param>
60+
public async Task ExtractAndSaveReadmeFileAsync(Package package, Stream packageStream)
61+
{
62+
if (package == null)
63+
{
64+
throw new ArgumentNullException(nameof(package));
65+
}
66+
67+
if (packageStream == null)
68+
{
69+
throw new ArgumentNullException(nameof(packageStream));
70+
}
71+
72+
packageStream.Seek(0, SeekOrigin.Begin);
73+
using (var packageArchiveReader = new PackageArchiveReader(packageStream, leaveStreamOpen: true))
74+
{
75+
var packageMetadata = PackageMetadata.FromNuspecReader(packageArchiveReader.GetNuspecReader(), strict: true);
76+
if (string.IsNullOrWhiteSpace(packageMetadata.ReadmeFile))
77+
{
78+
throw new InvalidOperationException("No readme file specified in the nuspec");
79+
}
80+
81+
var filename = FileNameHelper.GetZipEntryPath(packageMetadata.ReadmeFile);
82+
var ReadmeFileEntry = packageArchiveReader.GetEntry(filename); // throws on non-existent file
83+
using (var readmeFileStream = ReadmeFileEntry.Open())
84+
{
85+
await SaveReadmeFileAsync(package, readmeFileStream);
86+
}
87+
}
88+
}
89+
90+
public async Task<string> DownloadReadmeFileAsync(Package package)
91+
{
92+
if (package == null)
93+
{
94+
throw new ArgumentNullException(nameof(package));
95+
}
96+
97+
var fileName = BuildReadmeFileName(package);
98+
99+
using (var readmeFileStream = await _fileStorageService.GetFileAsync(_metadata.PackageContentFolderName, fileName))
100+
{
101+
if (readmeFileStream != null)
102+
{
103+
using (var readMeMdReader = new StreamReader(readmeFileStream))
104+
{
105+
return await readMeMdReader.ReadToEndAsync();
106+
}
107+
}
108+
}
109+
return null;
110+
}
111+
112+
public Task DeleteReadmeFileAsync(string id, string version)
113+
{
114+
if (id == null)
115+
{
116+
throw new ArgumentNullException(nameof(id));
117+
}
118+
119+
if (string.IsNullOrWhiteSpace(id))
120+
{
121+
throw new ArgumentException($"{nameof(id)} cannot be empty", nameof(id));
122+
}
123+
124+
if (version == null)
125+
{
126+
throw new ArgumentNullException(nameof(version));
127+
}
128+
129+
if (string.IsNullOrWhiteSpace(version))
130+
{
131+
throw new ArgumentException($"{nameof(version)} cannot be empty", nameof(version));
132+
}
133+
134+
var normalizedVersion = NuGetVersionFormatter.Normalize(version);
135+
var fileName = BuildReadmeFileName(id, normalizedVersion);
136+
137+
return _fileStorageService.DeleteFileAsync(_metadata.PackageContentFolderName, fileName);
138+
}
139+
140+
private string ReadmePathTemplate => $"{_metadata.PackageContentPathTemplate}/{ReadmeFileName}";
141+
142+
private string BuildReadmeFileName(Package package)
143+
=> FileNameHelper.BuildFileName(package, ReadmePathTemplate, string.Empty);
144+
145+
private string BuildReadmeFileName(string id, string version)
146+
=> FileNameHelper.BuildFileName(id, version, ReadmePathTemplate, string.Empty);
147+
}
148+
}

‎src/NuGetGallery.Core/Services/ICoreReadmeFileService.cs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
3+
4+
using System.IO;
5+
using System.Threading.Tasks;
6+
using NuGet.Services.Entities;
7+
8+
namespace NuGetGallery
9+
{
10+
/// <summary>
11+
/// Provides readmes file related operations
12+
/// </summary>
13+
public interface ICoreReadmeFileService
14+
{
15+
/// <summary>
16+
/// Save the readme file from package stream. This method should throw if the package
17+
/// does not have an embedded readme file
18+
/// </summary>
19+
/// <param name="package">Package information.</param>
20+
/// <param name="packageStream">Package stream with .nupkg contents.</param>
21+
Task ExtractAndSaveReadmeFileAsync(Package package, Stream packageStream);
22+
23+
/// <summary>
24+
/// Downloads previously saved readme file for a specified package.
25+
/// </summary>
26+
Task<string> DownloadReadmeFileAsync(Package package);
27+
28+
/// <summary>
29+
/// Deletes the readme file for the package from the publicly available storage for the package content.
30+
/// </summary>
31+
/// <param name="id">The package ID. This value is case-insensitive.</param>
32+
/// <param name="version">The package version. This value is case-insensitive and need not be normalized.</param>
33+
/// <returns></returns>
34+
Task DeleteReadmeFileAsync(string id, string version);
35+
}
36+
}

‎src/NuGetGallery/App_Start/StorageDependent.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ public static IEnumerable<StorageDependent> GetAll(IAppConfiguration configurati
9292
Create<SymbolPackageFileService, ISymbolPackageFileService>(configuration.AzureStorage_Packages_ConnectionString, isSingleInstance: false),
9393
Create<UploadFileService, IUploadFileService>(configuration.AzureStorage_Uploads_ConnectionString, isSingleInstance: false),
9494
Create<CoreLicenseFileService, ICoreLicenseFileService>(configuration.AzureStorage_FlatContainer_ConnectionString, isSingleInstance: false),
95+
Create<CoreReadmeFileService, ICoreReadmeFileService>(configuration.AzureStorage_FlatContainer_ConnectionString, isSingleInstance: false),
9596
Create<RevalidationStateService, IRevalidationStateService>(configuration.AzureStorage_Revalidation_ConnectionString, isSingleInstance: false),
9697
Create<EditableFeatureFlagFileStorageService, IFeatureFlagStorageService>(configuration.AzureStorage_Content_ConnectionString, isSingleInstance: true)
9798
};

‎src/NuGetGallery/Services/IPackageFileService.cs‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -34,22 +34,6 @@ public interface IPackageFileService : ICorePackageFileService
3434
/// <param name="readMeMd">Markdown content.</param>
3535
Task SaveReadMeMdFileAsync(Package package, string readMeMd);
3636

37-
/// <summary>
38-
/// Save the readme file from package stream. This method should throw if the package
39-
/// does not have an embedded readme file
40-
/// </summary>
41-
/// <param name="package">Package information.</param>
42-
/// <param name="packageStream">Package stream with .nupkg contents.</param>
43-
Task ExtractAndSaveReadmeFileAsync(Package package, Stream packageStream);
44-
45-
/// <summary>
46-
/// Saves the package readme.md file to storage. This method should throw if the package
47-
/// does not have an embedded readme file
48-
/// </summary>
49-
/// <param name="package">The package associated with the readme.</param>
50-
/// <param name="readmeFile">The content of readme file.</param>
51-
Task SaveReadmeFileAsync(Package package, Stream readmeFile);
52-
5337
/// <summary>
5438
/// Downloads the readme.md from storage.
5539
/// </summary>

‎src/NuGetGallery/Services/PackageDeleteService.cs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ DELETE pr FROM PackageRegistrations AS pr
5555
private readonly ISymbolPackageService _symbolPackageService;
5656
private readonly IEntityRepository<SymbolPackage> _symbolPackageRepository;
5757
private readonly ICoreLicenseFileService _coreLicenseFileService;
58+
private readonly ICoreReadmeFileService _coreReadmeFileService;
5859

5960
public PackageDeleteService(
6061
IEntityRepository<Package> packageRepository,
@@ -71,7 +72,8 @@ public PackageDeleteService(
7172
ISymbolPackageFileService symbolPackageFileService,
7273
ISymbolPackageService symbolPackageService,
7374
IEntityRepository<SymbolPackage> symbolPackageRepository,
74-
ICoreLicenseFileService coreLicenseFileService)
75+
ICoreLicenseFileService coreLicenseFileService,
76+
ICoreReadmeFileService coreReadmeFileService)
7577
{
7678
_packageRepository = packageRepository ?? throw new ArgumentNullException(nameof(packageRepository));
7779
_packageRegistrationRepository = packageRegistrationRepository ?? throw new ArgumentNullException(nameof(packageRegistrationRepository));
@@ -88,6 +90,7 @@ public PackageDeleteService(
8890
_symbolPackageService = symbolPackageService ?? throw new ArgumentNullException(nameof(symbolPackageService));
8991
_symbolPackageRepository = symbolPackageRepository ?? throw new ArgumentNullException(nameof(symbolPackageRepository));
9092
_coreLicenseFileService = coreLicenseFileService ?? throw new ArgumentNullException(nameof(coreLicenseFileService));
93+
_coreReadmeFileService = coreReadmeFileService ?? throw new ArgumentNullException(nameof(coreReadmeFileService));
9194

9295
if (config.HourLimitWithMaximumDownloads.HasValue
9396
&& config.StatisticsUpdateFrequencyInHours.HasValue
@@ -517,7 +520,14 @@ private async Task TryDeleteReadMeMdFile(Package package)
517520
{
518521
try
519522
{
520-
await _packageFileService.DeleteReadMeMdFileAsync(package);
523+
if (package.HasEmbeddedReadme)
524+
{
525+
await _coreReadmeFileService.DeleteReadmeFileAsync(package.Id, package.Version);
526+
}
527+
else
528+
{
529+
await _packageFileService.DeleteReadMeMdFileAsync(package);
530+
}
521531
}
522532
catch (StorageException) { }
523533
}

‎src/NuGetGallery/Services/PackageFileService.cs‎

Lines changed: 0 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -75,70 +75,6 @@ public async Task SaveReadMeMdFileAsync(Package package, string readMeMd)
7575
}
7676
}
7777

78-
/// <summary>
79-
/// Save the readme file from package stream. This method should throw if the package
80-
/// does not have an embedded readme file
81-
/// </summary>
82-
/// <param name="package">Package information.</param>
83-
/// <param name="packageStream">Package stream with .nupkg contents.</param>
84-
public async Task ExtractAndSaveReadmeFileAsync(Package package, Stream packageStream)
85-
{
86-
if (package == null)
87-
{
88-
throw new ArgumentNullException(nameof(package));
89-
}
90-
91-
if (packageStream == null)
92-
{
93-
throw new ArgumentNullException(nameof(packageStream));
94-
}
95-
96-
packageStream.Seek(0, SeekOrigin.Begin);
97-
using (var packageArchiveReader = new PackageArchiveReader(packageStream, leaveStreamOpen: true))
98-
{
99-
var packageMetadata = PackageMetadata.FromNuspecReader(packageArchiveReader.GetNuspecReader(), strict: true);
100-
if (string.IsNullOrWhiteSpace(packageMetadata.ReadmeFile))
101-
{
102-
throw new InvalidOperationException("No readme file specified in the nuspec");
103-
}
104-
105-
var filename = FileNameHelper.GetZipEntryPath(packageMetadata.ReadmeFile);
106-
var ReadmeFileEntry = packageArchiveReader.GetEntry(filename); // throws on non-existent file
107-
using (var readmeFileStream = ReadmeFileEntry.Open())
108-
{
109-
await SaveReadmeFileAsync(package, readmeFileStream);
110-
}
111-
}
112-
}
113-
114-
/// <summary>
115-
/// Saves the package readme.md file to storage. This method should throw if the package
116-
/// does not have an embedded readme file
117-
/// </summary>
118-
/// <param name="package">The package associated with the readme.</param>
119-
/// <param name="readmeFile">The content of readme file.</param>
120-
public Task SaveReadmeFileAsync(Package package, Stream readmeFile)
121-
{
122-
if (package == null)
123-
{
124-
throw new ArgumentNullException(nameof(package));
125-
}
126-
127-
if (readmeFile == null)
128-
{
129-
throw new ArgumentNullException(nameof(readmeFile));
130-
}
131-
132-
if (package.EmbeddedReadmeType == EmbeddedReadmeFileType.Absent)
133-
{
134-
throw new ArgumentException("Package must have an embedded readme", nameof(package));
135-
}
136-
137-
var fileName = FileNameHelper.BuildFileName(package, ReadMeFilePathTemplateActive, ServicesConstants.MarkdownFileExtension);
138-
139-
return _fileStorageService.SaveFileAsync(CoreConstants.Folders.PackageReadMesFolderName, fileName, readmeFile, overwrite: true);
140-
}
141-
14278
/// <summary>
14379
/// Downloads the readme.md from storage.
14480
/// </summary>

‎src/NuGetGallery/Services/PackageUploadService.cs‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ public class PackageUploadService : IPackageUploadService
2222
private readonly IReservedNamespaceService _reservedNamespaceService;
2323
private readonly IValidationService _validationService;
2424
private readonly ICoreLicenseFileService _coreLicenseFileService;
25+
private readonly ICoreReadmeFileService _coreReadmeFileService;
2526
private readonly IPackageVulnerabilitiesManagementService _vulnerabilityService;
2627
private readonly IPackageMetadataValidationService _metadataValidationService;
2728

@@ -32,6 +33,7 @@ public PackageUploadService(
3233
IReservedNamespaceService reservedNamespaceService,
3334
IValidationService validationService,
3435
ICoreLicenseFileService coreLicenseFileService,
36+
ICoreReadmeFileService coreReadmeFileService,
3537
IDiagnosticsService diagnosticsService,
3638
IPackageVulnerabilitiesManagementService vulnerabilityService,
3739
IPackageMetadataValidationService metadataValidationService)
@@ -42,6 +44,8 @@ public PackageUploadService(
4244
_reservedNamespaceService = reservedNamespaceService ?? throw new ArgumentNullException(nameof(reservedNamespaceService));
4345
_validationService = validationService ?? throw new ArgumentNullException(nameof(validationService));
4446
_coreLicenseFileService = coreLicenseFileService ?? throw new ArgumentNullException(nameof(coreLicenseFileService));
47+
_coreReadmeFileService = coreReadmeFileService ?? throw new ArgumentNullException(nameof(coreReadmeFileService));
48+
4549
if (diagnosticsService == null)
4650
{
4751
throw new ArgumentNullException(nameof(diagnosticsService));
@@ -193,7 +197,7 @@ await _packageFileService.DeleteValidationPackageFileAsync(
193197
var isReadmeFileExtractedAndSaved = false;
194198
if (package.HasReadMe && package.EmbeddedReadmeType != EmbeddedReadmeFileType.Absent)
195199
{
196-
await _packageFileService.ExtractAndSaveReadmeFileAsync(package, packageFile);
200+
await _coreReadmeFileService.ExtractAndSaveReadmeFileAsync(package, packageFile);
197201
isReadmeFileExtractedAndSaved = true;
198202
}
199203

@@ -213,7 +217,9 @@ await _coreLicenseFileService.DeleteLicenseFileAsync(
213217

214218
if (isReadmeFileExtractedAndSaved)
215219
{
216-
await _packageFileService.DeleteReadMeMdFileAsync(package);
220+
await _coreReadmeFileService.DeleteReadmeFileAsync(
221+
package.PackageRegistration.Id,
222+
package.NormalizedVersion);
217223
}
218224
throw;
219225
}
@@ -253,7 +259,9 @@ await _packageFileService.DeletePackageFileAsync(
253259
await _coreLicenseFileService.DeleteLicenseFileAsync(
254260
package.PackageRegistration.Id,
255261
package.NormalizedVersion);
256-
await _packageFileService.DeleteReadMeMdFileAsync(package);
262+
await _coreReadmeFileService.DeleteReadmeFileAsync(
263+
package.PackageRegistration.Id,
264+
package.NormalizedVersion);
257265
}
258266

259267
return ReturnConflictOrThrow(ex);

0 commit comments

Comments
 (0)