From 7b1afec8104473da4e77da4fdba4c0ceca124971 Mon Sep 17 00:00:00 2001 From: Orion Edwards Date: Sun, 10 May 2026 23:08:59 +1200 Subject: [PATCH 1/2] ConsolidatePackages: Use System.IO.Compression and slightly tweak extraction loop (#1913) --- ...alamari.ConsolidateCalamariPackages.csproj | 3 --- .../ConsolidatedPackage.cs | 25 ++++++++----------- .../ConsolidatedPackageIndexLoader.cs | 9 +++---- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/source/Calamari.ConsolidateCalamariPackages/Calamari.ConsolidateCalamariPackages.csproj b/source/Calamari.ConsolidateCalamariPackages/Calamari.ConsolidateCalamariPackages.csproj index 11096e249b..ee3cc2d8a8 100644 --- a/source/Calamari.ConsolidateCalamariPackages/Calamari.ConsolidateCalamariPackages.csproj +++ b/source/Calamari.ConsolidateCalamariPackages/Calamari.ConsolidateCalamariPackages.csproj @@ -19,9 +19,6 @@ - - NU1902 - diff --git a/source/Calamari.ConsolidateCalamariPackages/ConsolidatedPackage.cs b/source/Calamari.ConsolidateCalamariPackages/ConsolidatedPackage.cs index 61b5c105a5..4b1e931562 100644 --- a/source/Calamari.ConsolidateCalamariPackages/ConsolidatedPackage.cs +++ b/source/Calamari.ConsolidateCalamariPackages/ConsolidatedPackage.cs @@ -1,9 +1,9 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Compression; using System.Linq; using Octopus.Calamari.ConsolidatedPackage.Api; -using SharpCompress.Archives.Zip; namespace Octopus.Calamari.ConsolidatedPackage { @@ -28,21 +28,16 @@ public ConsolidatedPackage(IConsolidatedPackageStreamProvider packageStreamProvi throw new Exception($"Could not find platform {platform} for {calamariFlavour}"); } - using (var sourceStream = packageStreamProvider.OpenStream()) + var platformFilesLookup = platformFiles.ToDictionary(f => f.Source); + + using var sourceStream = packageStreamProvider.OpenStream(); + using var source = new ZipArchive(sourceStream, ZipArchiveMode.Read); + foreach (var sourceEntry in source.Entries) { - using (var source = ZipArchive.Open(sourceStream)) - { - foreach (var fileTransfer in platformFiles) - { - var sourceEntry = source.Entries.FirstOrDefault(e => e.Key is not null && e.Key.Equals(fileTransfer.Source)); - if(sourceEntry is null) continue; - - using (var sourceEntryStream = sourceEntry.OpenEntryStream()) - { - yield return (fileTransfer.Destination, sourceEntry.Size, sourceEntryStream); - } - } - } + if (!platformFilesLookup.TryGetValue(sourceEntry.FullName, out var fileTransfer)) continue; + + using var sourceEntryStream = sourceEntry.Open(); + yield return (fileTransfer.Destination, sourceEntry.Length, sourceEntryStream); } } } diff --git a/source/Calamari.ConsolidateCalamariPackages/ConsolidatedPackageIndexLoader.cs b/source/Calamari.ConsolidateCalamariPackages/ConsolidatedPackageIndexLoader.cs index 24e58bef0e..8d325a6166 100644 --- a/source/Calamari.ConsolidateCalamariPackages/ConsolidatedPackageIndexLoader.cs +++ b/source/Calamari.ConsolidateCalamariPackages/ConsolidatedPackageIndexLoader.cs @@ -1,9 +1,8 @@ using System; using System.IO; -using System.Linq; +using System.IO.Compression; using Newtonsoft.Json; using Octopus.Calamari.ConsolidatedPackage.Api; -using SharpCompress.Archives.Zip; namespace Octopus.Calamari.ConsolidatedPackage { @@ -11,14 +10,14 @@ public class ConsolidatedPackageIndexLoader { public IConsolidatedPackageIndex Load(Stream zipStream) { - using var zip = ZipArchive.Open(zipStream); - var entry = zip.Entries.First(e => e.Key == "index.json"); + using var zip = new ZipArchive(zipStream, ZipArchiveMode.Read); + var entry = zip.GetEntry("index.json"); if (entry == null) { throw new Exception($"index.json not found in supplied stream."); } - using var entryStream = entry.OpenEntryStream(); + using var entryStream = entry.Open(); return From(entryStream); } From e94312fc4538db6475c63dd699a085de7d8ea29f Mon Sep 17 00:00:00 2001 From: Orion Edwards Date: Tue, 12 May 2026 11:52:55 +1200 Subject: [PATCH 2/2] Fix handling of duplicate files in the Calamari package, add tests (#1926) --- .../ConsolidatedPackageTests.cs | 127 ++++++++++++++++++ .../ConsolidatedPackage.cs | 15 ++- 2 files changed, 136 insertions(+), 6 deletions(-) create mode 100644 source/Calamari.ConsolidateCalamariPackages.Tests/ConsolidatedPackageTests.cs diff --git a/source/Calamari.ConsolidateCalamariPackages.Tests/ConsolidatedPackageTests.cs b/source/Calamari.ConsolidateCalamariPackages.Tests/ConsolidatedPackageTests.cs new file mode 100644 index 0000000000..1ca32d81fa --- /dev/null +++ b/source/Calamari.ConsolidateCalamariPackages.Tests/ConsolidatedPackageTests.cs @@ -0,0 +1,127 @@ +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using FluentAssertions; +using FluentAssertions.Execution; +using NUnit.Framework; +using Octopus.Calamari.ConsolidatedPackage; +using Octopus.Calamari.ConsolidatedPackage.Api; + +namespace Calamari.ConsolidateCalamariPackages.Tests; + +[TestFixture] +public class ConsolidatedPackageTests +{ + [Test] + public void ExtractCalamariPackage_ExtractsPlatformFiles() + { + // Taken from real Calamari index.json + var index = new ConsolidatedPackageIndex(new Dictionary + { + ["Calamari"] = + new("Calamari", + "2025.4.682", + IsNupkg: true, + new Dictionary + { + ["linux-x64"] = + [ + new("e53319a5e0ad28139f18abff2a3846a2/Octopus.Calamari.linux-x64.nuspec", "Octopus.Calamari.linux-x64.nuspec"), + new("3103c6689c0c54d1951cd48c91fd07d3/Calamari.Shared.dll", "Calamari.Shared.dll"), + new("9530f39bf5be0d28a050a0d536354840/Calamari.dll", "Calamari.dll"), + ], + ["win-x64"] = + [ + new("6f7d961cb2643f9b4676db6c6b2fb050/Octopus.Calamari.win-x64.nuspec", "Octopus.Calamari.win-x64.nuspec"), + new("aa773c55c461d2bc95d3ec23d0c2affc/Calamari.Shared.dll", "Calamari.Shared.dll"), + new("31e6370cd8472603f2082ada35be3cc3/Calamari.dll", "Calamari.dll"), + ], + }) + }); + + var streamProvider = + BuildZipArchive(("e53319a5e0ad28139f18abff2a3846a2/Octopus.Calamari.linux-x64.nuspec", "aaa_Octopus.Calamari.linux-x64"), + ("3103c6689c0c54d1951cd48c91fd07d3/Calamari.Shared.dll", "aab_Calamari.Shared.dll"), + ("9530f39bf5be0d28a050a0d536354840/Calamari.dll", "aac_Calamari.dll"), + ("6f7d961cb2643f9b4676db6c6b2fb050/Octopus.Calamari.win-x64.nuspec", "baa_Octopus.Calamari.linux-x64"), + ("aa773c55c461d2bc95d3ec23d0c2affc/Calamari.Shared.dll", "bab_Calamari.Shared.dll"), + ("31e6370cd8472603f2082ada35be3cc3/Calamari.dll", "bac_Calamari.dll")); + + var p = new ConsolidatedPackage(streamProvider, index); + + var linux64 = p.ExtractCalamariPackage("Calamari", "linux-x64").Select(i => i.entryName).ToArray(); + linux64.Should().Equal("Octopus.Calamari.linux-x64.nuspec", "Calamari.Shared.dll", "Calamari.dll"); + + var win64 = p.ExtractCalamariPackage("Calamari", "win-x64").Select(i => i.entryName).ToArray(); + win64.Should().Equal("Octopus.Calamari.win-x64.nuspec", "Calamari.Shared.dll", "Calamari.dll"); + } + + [Test] + public void ExtractCalamariPackage_ExtractsSameFileContentsToDifferentOutputs() + { + var index = new ConsolidatedPackageIndex(new Dictionary + { + ["Chicken"] = + new("Chicken", + "1", + IsNupkg: true, + new Dictionary + { + ["linux-x64"] = + [ + new("e53319a5e0ad28139f18abff2a3846a2/Chicken.txt", "Chicken.txt"), + new("e53319a5e0ad28139f18abff2a3846a2/Chicken.txt", "Subfolder/Chicken.txt"), + ], + }) + }); + + var customStreamProvider = BuildZipArchive(("e53319a5e0ad28139f18abff2a3846a2/Chicken.txt", "Crazy chickens jab, peck, and flap while quietly dozing foxes get very humid")); + + var p = new ConsolidatedPackage(customStreamProvider, index); + + // ExtractCalamariPackage returns an IEnumerable containing streams which are closed as we move through the IEnumerable + // We can't call ToArray on it or all the streams get closed before we can read them. + using var enumerator = p.ExtractCalamariPackage("Chicken", "linux-x64").GetEnumerator(); + + enumerator.MoveNext().Should().BeTrue(); + { + var (entryName, size, sourceStream) = enumerator.Current; + entryName.Should().Be("Chicken.txt"); + size.Should().Be(76); + new StreamReader(sourceStream).ReadToEnd().Should().Be("Crazy chickens jab, peck, and flap while quietly dozing foxes get very humid"); + } + + enumerator.MoveNext().Should().BeTrue(); + { + var (entryName, size, sourceStream) = enumerator.Current; + entryName.Should().Be("Subfolder/Chicken.txt"); + size.Should().Be(76); + new StreamReader(sourceStream).ReadToEnd().Should().Be("Crazy chickens jab, peck, and flap while quietly dozing foxes get very humid"); + } + + enumerator.MoveNext().Should().BeFalse(because: "No more files in the index"); + } + + class FakeConsolidatedPackageStreamProvider(byte[] zipFileContents) : IConsolidatedPackageStreamProvider + { + public Stream OpenStream() => new MemoryStream(zipFileContents); + } + + static FakeConsolidatedPackageStreamProvider BuildZipArchive(params (string, string)[] filesAndContents) + { + using var stream = new MemoryStream(); + using (var z = new ZipArchive(stream, ZipArchiveMode.Create)) + { + foreach (var (file, content) in filesAndContents) + { + var entry = z.CreateEntry(file, CompressionLevel.NoCompression); + using var entryStream = entry.Open(); + using var writer = new StreamWriter(entryStream); + writer.Write(content); + } + } + + return new(stream.ToArray()); + } +} \ No newline at end of file diff --git a/source/Calamari.ConsolidateCalamariPackages/ConsolidatedPackage.cs b/source/Calamari.ConsolidateCalamariPackages/ConsolidatedPackage.cs index 4b1e931562..184570683b 100644 --- a/source/Calamari.ConsolidateCalamariPackages/ConsolidatedPackage.cs +++ b/source/Calamari.ConsolidateCalamariPackages/ConsolidatedPackage.cs @@ -10,7 +10,7 @@ namespace Octopus.Calamari.ConsolidatedPackage public class ConsolidatedPackage : IConsolidatedPackage { readonly IConsolidatedPackageStreamProvider packageStreamProvider; - + public ConsolidatedPackage(IConsolidatedPackageStreamProvider packageStreamProvider, IConsolidatedPackageIndex index) { this.packageStreamProvider = packageStreamProvider; @@ -28,17 +28,20 @@ public ConsolidatedPackage(IConsolidatedPackageStreamProvider packageStreamProvi throw new Exception($"Could not find platform {platform} for {calamariFlavour}"); } - var platformFilesLookup = platformFiles.ToDictionary(f => f.Source); - using var sourceStream = packageStreamProvider.OpenStream(); using var source = new ZipArchive(sourceStream, ZipArchiveMode.Read); - foreach (var sourceEntry in source.Entries) + + var entriesLookup = source.Entries.ToDictionary(e => e.FullName); + foreach (var fileTransfer in platformFiles) { - if (!platformFilesLookup.TryGetValue(sourceEntry.FullName, out var fileTransfer)) continue; + if (!entriesLookup.TryGetValue(fileTransfer.Source, out var sourceEntry)) + { + continue; + } using var sourceEntryStream = sourceEntry.Open(); yield return (fileTransfer.Destination, sourceEntry.Length, sourceEntryStream); } } } -} +} \ No newline at end of file