From 79c4ac8ce2f3b7bfb5424e33752504a6d93b2354 Mon Sep 17 00:00:00 2001 From: Alex Guidi Date: Mon, 8 Jun 2026 17:07:08 +0200 Subject: [PATCH 01/10] feat: Add sparse manifest platform filtering for multi-arch images Implement InstancePlatformFilter feature with sparse manifest support to allow users to specify OS/Architecture pairs for filtering multi-architecture images across all image types (additional images, operators, helm charts, and releases). This feature brings the platform filtering capability from skopeo (containers/image PR #2874) to oc-mirror, enabling users to mirror only specific platforms instead of all architectures. The implementation uses sparse manifests where the manifest list references all platforms but only the specified platforms have their actual layer blobs downloaded, significantly reducing disk usage and bandwidth. Platform filtering is applied during Mirror-to-Disk and Mirror-to-Mirror workflows. Disk-to-Mirror workflow uses already-filtered images from disk without re-applying platform filters. The implementation includes backward compatibility with the deprecated Platform.Architectures field and follows the same strategy as skopeo for consistency. Fixes: CLID-290 Signed-off-by: Alex Guidi --- .../pkg/additional/local_stored_collector.go | 11 ++- internal/pkg/api/v2alpha1/type_config.go | 26 ++++++ internal/pkg/api/v2alpha1/type_internal.go | 9 ++ internal/pkg/api/v2alpha1/type_platform.go | 30 +++++++ internal/pkg/batch/concurrent_chan_worker.go | 5 ++ internal/pkg/helm/local_stored_collector.go | 38 +++++++-- internal/pkg/mirror/mirror.go | 83 ++++++++++++++----- internal/pkg/mirror/mirror_test.go | 32 +++++-- internal/pkg/mirror/options.go | 1 + internal/pkg/operator/catalog_handler.go | 11 +-- internal/pkg/operator/catalog_handler_test.go | 2 +- internal/pkg/operator/filtered_collector.go | 5 +- .../pkg/operator/filtered_collector_test.go | 2 +- internal/pkg/operator/interface.go | 2 +- .../pkg/release/local_stored_collector.go | 65 ++++++++++++++- 15 files changed, 272 insertions(+), 50 deletions(-) diff --git a/internal/pkg/additional/local_stored_collector.go b/internal/pkg/additional/local_stored_collector.go index 32e52e253..77fb1c129 100644 --- a/internal/pkg/additional/local_stored_collector.go +++ b/internal/pkg/additional/local_stored_collector.go @@ -114,7 +114,16 @@ func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) ([ o.Log.Debug(collectorPrefix+"source %s", src) o.Log.Debug(collectorPrefix+"destination %s", dest) - allImages = append(allImages, v2alpha1.CopyImageSchema{Source: src, Destination: dest, Origin: origin, Type: v2alpha1.TypeGeneric}) + // Convert platform filters to string format for CopyImageSchema + platforms := v2alpha1.ConvertPlatformsToStringSlice(img.Platforms) + + allImages = append(allImages, v2alpha1.CopyImageSchema{ + Source: src, + Destination: dest, + Origin: origin, + Type: v2alpha1.TypeGeneric, + Platforms: platforms, + }) } return allImages, errors.Join(allErrs...) } diff --git a/internal/pkg/api/v2alpha1/type_config.go b/internal/pkg/api/v2alpha1/type_config.go index 12b24c3f3..44d0b7149 100644 --- a/internal/pkg/api/v2alpha1/type_config.go +++ b/internal/pkg/api/v2alpha1/type_config.go @@ -94,7 +94,18 @@ type Platform struct { // Architectures defines one or more architectures // to mirror for the release image. This is defined at the // platform level to enable cross-channel upgrades. + // + // Deprecated: Use Platforms instead for fine-grained OS/Architecture control. + // + // For backward compatibility, if Platforms is empty and Architectures is set, + // it will be interpreted as linux/ for each architecture. Architectures []string `json:"architectures,omitempty"` + // Platforms defines one or more OS/Architecture pairs to mirror + // for multi-architecture release images. If empty and Architectures is empty, + // defaults to linux/amd64. This is defined at the platform level to enable + // cross-channel upgrades. + // Example: [{OS: "linux", Architecture: "amd64"}, {OS: "linux", Architecture: "arm64"}] + Platforms []InstancePlatformFilter `json:"platforms,omitempty"` // This new field will allow the diskToMirror functionality // to copy from a release location on disk Release string `json:"release,omitempty"` @@ -115,6 +126,9 @@ func (p Platform) DeepCopy() Platform { platformCopy.Architectures = make([]string, len(p.Architectures)) copy(platformCopy.Architectures, p.Architectures) + platformCopy.Platforms = make([]InstancePlatformFilter, len(p.Platforms)) + copy(platformCopy.Platforms, p.Platforms) + return platformCopy } @@ -183,6 +197,10 @@ type Operator struct { // path on disk for a template to use to complete catalogSource custom resource // generated by oc-mirror TargetCatalogSourceTemplate string `json:"targetCatalogSourceTemplate,omitempty"` + // Platforms defines one or more OS/Architecture pairs to mirror + // for multi-architecture catalog and operator images. If empty, mirrors all platforms. + // Example: [{OS: "linux", Architecture: "amd64"}, {OS: "linux", Architecture: "arm64"}] + Platforms []InstancePlatformFilter `json:"platforms,omitempty"` } // GetUniqueName determines the catalog name that will @@ -266,6 +284,10 @@ type Chart struct { // ImagePaths are custom JSON paths for images location // in the helm manifest or templates ImagePaths []string `json:"imagePaths,omitempty"` + // Platforms defines one or more OS/Architecture pairs to mirror + // for multi-architecture images referenced by this chart. If empty, mirrors all platforms. + // Example: [{OS: "linux", Architecture: "amd64"}, {OS: "linux", Architecture: "arm64"}] + Platforms []InstancePlatformFilter `json:"platforms,omitempty"` } // AdditionalImage contains image pull information for additional images, @@ -291,6 +313,10 @@ type AdditionalImage struct { // the image will be mirrored with the provided tag in the Name // field or a tag calculated from the partial digest. TargetTag string `json:"targetTag,omitempty"` + // Platforms defines one or more OS/Architecture pairs to mirror + // for multi-architecture images. If empty, mirrors all platforms. + // Example: [{OS: "linux", Architecture: "amd64"}, {OS: "linux", Architecture: "arm64"}] + Platforms []InstancePlatformFilter `json:"platforms,omitempty"` } // GetUniqueName determines the image name that will diff --git a/internal/pkg/api/v2alpha1/type_internal.go b/internal/pkg/api/v2alpha1/type_internal.go index 7374f5edd..f1061337d 100644 --- a/internal/pkg/api/v2alpha1/type_internal.go +++ b/internal/pkg/api/v2alpha1/type_internal.go @@ -182,6 +182,10 @@ type RelatedImage struct { OriginFromOperatorCatalogOnDisk bool // Used to identify if it is a full catalog (full: true && zero packages) FullCatalog bool + // Platforms defines one or more OS/Architecture pairs to mirror + // for this multi-architecture image. Formatted as "os/arch" (e.g., "linux/amd64"). + // Using a pointer to maintain struct comparability. + Platforms *[]string `json:"-"` } type CollectorSchema struct { @@ -211,6 +215,11 @@ type CopyImageSchema struct { // it doesn´t need to be persisted to json Type ImageType `json:"-"` RebuiltTag string `json:"rebuiltTag"` + // Platforms defines one or more OS/Architecture pairs to mirror + // for this multi-architecture image. If nil, defaults to "all" (mirrors all platforms). + // Formatted as "os/arch" (e.g., "linux/amd64", "linux/arm64"). + // Using a pointer to maintain struct comparability for use in maps and slices.Compact. + Platforms *[]string `json:"-"` } // SignatureContentSchema diff --git a/internal/pkg/api/v2alpha1/type_platform.go b/internal/pkg/api/v2alpha1/type_platform.go index 60e70e669..39f06a816 100644 --- a/internal/pkg/api/v2alpha1/type_platform.go +++ b/internal/pkg/api/v2alpha1/type_platform.go @@ -60,3 +60,33 @@ func (pt PlatformType) validate() error { } return nil } + +// InstancePlatformFilter defines OS and Architecture for filtering +// multi-architecture manifest lists. This allows selecting specific +// platforms (e.g., linux/amd64, linux/arm64) when mirroring images. +type InstancePlatformFilter struct { + // OS is the operating system (e.g., "linux", "windows") + OS string `json:"os"` + // Architecture is the CPU architecture (e.g., "amd64", "arm64", "ppc64le", "s390x") + Architecture string `json:"architecture"` +} + +// String returns the platform in "os/architecture" format (e.g., "linux/amd64") +func (p InstancePlatformFilter) String() string { + return p.OS + "/" + p.Architecture +} + +// ConvertPlatformsToStringSlice converts a slice of InstancePlatformFilter to a pointer +// to a slice of platform strings in "os/architecture" format. +// Returns nil if the input slice is empty. +func ConvertPlatformsToStringSlice(platforms []InstancePlatformFilter) *[]string { + if len(platforms) == 0 { + return nil + } + + platformStrs := make([]string, len(platforms)) + for i, p := range platforms { + platformStrs[i] = p.String() + } + return &platformStrs +} diff --git a/internal/pkg/batch/concurrent_chan_worker.go b/internal/pkg/batch/concurrent_chan_worker.go index 61f03dc3f..c08e3aa29 100644 --- a/internal/pkg/batch/concurrent_chan_worker.go +++ b/internal/pkg/batch/concurrent_chan_worker.go @@ -144,6 +144,11 @@ func (o *ChannelConcurrentBatch) Worker(ctx context.Context, collectorSchema v2a options.PreserveDigests = false } + // If this image has specific platform requirements, use them + if img.Platforms != nil && len(*img.Platforms) > 0 { + options.InstancePlatforms = *img.Platforms + } + err = o.Mirror.Run(timeoutCtx, img.Source, img.Destination, mirror.Mode(opts.Function), &options) //nolint:contextcheck switch { diff --git a/internal/pkg/helm/local_stored_collector.go b/internal/pkg/helm/local_stored_collector.go index 4fff91415..e7e9ee69c 100644 --- a/internal/pkg/helm/local_stored_collector.go +++ b/internal/pkg/helm/local_stored_collector.go @@ -130,7 +130,10 @@ func (o *LocalStorageCollector) HelmImageCollector(ctx context.Context) ([]v2alp continue } - imgs, err := getImages(path, chart.ImagePaths...) + // Convert platform filters to string format + platforms := v2alpha1.ConvertPlatformsToStringSlice(chart.Platforms) + + imgs, err := getImages(path, platforms, chart.ImagePaths...) if err != nil { errs = append(errs, err) } @@ -176,7 +179,10 @@ func (o *LocalStorageCollector) HelmImageCollector(ctx context.Context) ([]v2alp continue } - imgs, err := getImages(path, chart.ImagePaths...) + // Convert platform filters to string format + platforms := v2alpha1.ConvertPlatformsToStringSlice(chart.Platforms) + + imgs, err := getImages(path, platforms, chart.ImagePaths...) if err != nil { errs = append(errs, err) } @@ -231,7 +237,10 @@ func getHelmImagesFromLocalChart() ([]v2alpha1.RelatedImage, []error) { var errs []error for _, chart := range lsc.Config.Mirror.Helm.Local { - imgs, err := getImages(chart.Path, chart.ImagePaths...) + // Convert platform filters to string format + platforms := v2alpha1.ConvertPlatformsToStringSlice(chart.Platforms) + + imgs, err := getImages(chart.Path, platforms, chart.ImagePaths...) if err != nil { errs = append(errs, err) } @@ -422,7 +431,7 @@ func buildChartCandidatePath(baseDir, name, ver string) (string, error) { return p, nil } -func getImages(path string, imagePaths ...string) (images []v2alpha1.RelatedImage, err error) { +func getImages(path string, platforms *[]string, imagePaths ...string) (images []v2alpha1.RelatedImage, err error) { lsc.Log.Debug("Reading from path %s", path) p := getImagesPath(imagePaths...) @@ -444,6 +453,11 @@ func getImages(path string, imagePaths ...string) (images []v2alpha1.RelatedImag return nil, err } + // Set platforms for each image + for i := range imgs { + imgs[i].Platforms = platforms + } + images = append(images, imgs...) } @@ -590,7 +604,13 @@ func prepareM2DCopyBatch(images []v2alpha1.RelatedImage) ([]v2alpha1.CopyImageSc lsc.Log.Debug("source %s", src) lsc.Log.Debug("destination %s", dest) - result = append(result, v2alpha1.CopyImageSchema{Origin: img.Image, Source: src, Destination: dest, Type: img.Type}) + result = append(result, v2alpha1.CopyImageSchema{ + Origin: img.Image, + Source: src, + Destination: dest, + Type: img.Type, + Platforms: img.Platforms, + }) } return result, nil } @@ -630,7 +650,13 @@ func prepareD2MCopyBatch(images []v2alpha1.RelatedImage, generateV1TagsFromDiges lsc.Log.Debug("source %s", src) lsc.Log.Debug("destination %s", dest) - result = append(result, v2alpha1.CopyImageSchema{Origin: img.Image, Source: src, Destination: dest, Type: img.Type}) + result = append(result, v2alpha1.CopyImageSchema{ + Origin: img.Image, + Source: src, + Destination: dest, + Type: img.Type, + Platforms: img.Platforms, + }) } return result, nil diff --git a/internal/pkg/mirror/mirror.go b/internal/pkg/mirror/mirror.go index 9f1d04471..329e5f815 100644 --- a/internal/pkg/mirror/mirror.go +++ b/internal/pkg/mirror/mirror.go @@ -135,20 +135,9 @@ func (o *Mirror) copy(ctx context.Context, src, dest string, opts *CopyOptions) } } - imageListSelection := copy.CopySystemImage - if len(opts.MultiArch) > 0 && opts.All { - return fmt.Errorf("cannot use --all and --multi-arch flags together") - } - - if len(opts.MultiArch) > 0 { - imageListSelection, err = parseMultiArch(opts.MultiArch) - if err != nil { - return err - } - } - - if opts.All { - imageListSelection = copy.CopyAllImages + imageListSelection, instancePlatforms, err := determinePlatformSelection(opts) + if err != nil { + return err } if len(opts.EncryptionKeys) > 0 && len(opts.DecryptionKeys) > 0 { @@ -193,6 +182,7 @@ func (o *Mirror) copy(ctx context.Context, src, dest string, opts *CopyOptions) DestinationCtx: destinationCtx, ForceManifestMIMEType: manifestType, ImageListSelection: imageListSelection, + InstancePlatforms: instancePlatforms, PreserveDigests: opts.PreserveDigests, MaxParallelDownloads: opts.ParallelLayerImages, } @@ -323,22 +313,69 @@ func (o *Mirror) delete(ctx context.Context, image string, opts *CopyOptions) er }, opts.RetryOpts) } -// parseMultiArch -func parseMultiArch(multiArch string) (copy.ImageListSelection, error) { +// determinePlatformSelection determines the image list selection and platform filters +// based on the provided copy options. It prioritizes InstancePlatforms (from ImageSetConfig) +// over the MultiArch flag. +func determinePlatformSelection(opts *CopyOptions) (copy.ImageListSelection, []copy.InstancePlatformFilter, error) { + // Priority 1: Use InstancePlatforms if provided (set by batch worker from ImageSetConfig) + if len(opts.InstancePlatforms) > 0 { + platformsStr := strings.Join(opts.InstancePlatforms, ",") + return parseMultiArch(platformsStr) + } + + // Priority 2: Fall back to MultiArch flag (for CLI direct usage or default "all") + if len(opts.MultiArch) > 0 && opts.All { + return copy.CopySystemImage, nil, fmt.Errorf("cannot use --all and --multi-arch flags together") + } + + imageListSelection := copy.CopySystemImage + var instancePlatforms []copy.InstancePlatformFilter + var err error + + if len(opts.MultiArch) > 0 { + imageListSelection, instancePlatforms, err = parseMultiArch(opts.MultiArch) + if err != nil { + return copy.CopySystemImage, nil, err + } + } + + if opts.All { + imageListSelection = copy.CopyAllImages + } + + return imageListSelection, instancePlatforms, nil +} + +// parseMultiArch parses the multi-arch option and returns the image list selection +// and optional platform filters for specific OS/Architecture combinations. +// Supports legacy options ('system', 'all', 'index-only') and new platform +// specifications (e.g., 'linux/amd64,linux/arm64'). +func parseMultiArch(multiArch string) (copy.ImageListSelection, []copy.InstancePlatformFilter, error) { switch multiArch { case "system": - return copy.CopySystemImage, nil + return copy.CopySystemImage, nil, nil case "all": - return copy.CopyAllImages, nil + return copy.CopyAllImages, nil, nil // There is no CopyNoImages value in copy.ImageListSelection, but because we // don't provide an option to select a set of images to copy, we can use // CopySpecificImages. case "index-only": - return copy.CopySpecificImages, nil - // We don't expose CopySpecificImages other than index-only above, because - // we currently don't provide an option to choose the images to copy. That - // could be added in the future. + return copy.CopySpecificImages, nil, nil default: - return copy.CopySystemImage, fmt.Errorf("unknown multi-arch option %q. Choose one of the supported options: 'system', 'all', or 'index-only'", multiArch) + // Try to parse as comma-separated platform list (e.g., "linux/amd64,linux/arm64") + // Parse comma-separated platform list + var platforms []copy.InstancePlatformFilter + for _, platform := range strings.Split(multiArch, ",") { + platform = strings.TrimSpace(platform) + parts := strings.Split(platform, "/") + if len(parts) != 2 { + return copy.CopySystemImage, nil, fmt.Errorf("unknown multi-arch option %q. Choose one of the supported options: 'system', 'all', 'index-only', or a comma-separated platform list like 'linux/amd64,linux/arm64'", multiArch) + } + platforms = append(platforms, copy.InstancePlatformFilter{ + OS: parts[0], + Architecture: parts[1], + }) + } + return copy.CopySpecificImages, platforms, nil } } diff --git a/internal/pkg/mirror/mirror_test.go b/internal/pkg/mirror/mirror_test.go index 58810cf62..1b4cc1f1e 100644 --- a/internal/pkg/mirror/mirror_test.go +++ b/internal/pkg/mirror/mirror_test.go @@ -67,7 +67,7 @@ func TestMirrorCopy(t *testing.T) { opts.MultiArch = "other" t.Run("Testing Mirror : copy should fail", func(t *testing.T) { err := m.Run(context.Background(), consts.DockerProtocol+"localhost.localdomain:5000/tes", "oci:test", "copy", &opts) - assert.Equal(t, "unknown multi-arch option \"other\". Choose one of the supported options: 'system', 'all', or 'index-only'", err.Error()) + assert.Equal(t, "unknown multi-arch option \"other\". Choose one of the supported options: 'system', 'all', 'index-only', or a comma-separated platform list like 'linux/amd64,linux/arm64'", err.Error()) }) opts.All = true @@ -225,17 +225,35 @@ func TestMirrorDelete(t *testing.T) { // TestMirrorParseMultiArch func TestMirrorParseMultiArch(t *testing.T) { - res, _ := parseMultiArch("system") + res, platforms, err := parseMultiArch("system") + assert.NoError(t, err) assert.Equal(t, copy.ImageListSelection(0), res) + assert.Nil(t, platforms) - res, _ = parseMultiArch("all") + res, platforms, err = parseMultiArch("all") + assert.NoError(t, err) assert.Equal(t, copy.ImageListSelection(1), res) + assert.Nil(t, platforms) - res, _ = parseMultiArch("index-only") + res, platforms, err = parseMultiArch("index-only") + assert.NoError(t, err) assert.Equal(t, copy.ImageListSelection(2), res) - - _, err := parseMultiArch("other") - assert.Equal(t, "unknown multi-arch option \"other\". Choose one of the supported options: 'system', 'all', or 'index-only'", err.Error()) + assert.Nil(t, platforms) + + // Test platform specification + res, platforms, err = parseMultiArch("linux/amd64,linux/arm64") + assert.NoError(t, err) + assert.Equal(t, copy.ImageListSelection(2), res) // CopySpecificImages + assert.Len(t, platforms, 2) + assert.Equal(t, "linux", platforms[0].OS) + assert.Equal(t, "amd64", platforms[0].Architecture) + assert.Equal(t, "linux", platforms[1].OS) + assert.Equal(t, "arm64", platforms[1].Architecture) + + // Test invalid option + _, _, err = parseMultiArch("other") + assert.Error(t, err) + assert.Contains(t, err.Error(), "unknown multi-arch option") } type ( diff --git a/internal/pkg/mirror/options.go b/internal/pkg/mirror/options.go index 331b7598d..ad2f2d97d 100644 --- a/internal/pkg/mirror/options.go +++ b/internal/pkg/mirror/options.go @@ -79,6 +79,7 @@ type CopyOptions struct { Format string // Force conversion of the image to a specified format All bool // Copy all of the images if the source is a list MultiArch string // How to handle multi architecture images + InstancePlatforms []string // Platform specifications (e.g., "linux/amd64", "linux/arm64") for multi-arch images PreserveDigests bool // Preserve digests during copy EncryptLayer []int // The list of layers to encrypt EncryptionKeys []string // Keys needed to encrypt the image diff --git a/internal/pkg/operator/catalog_handler.go b/internal/pkg/operator/catalog_handler.go index 2de5dd42c..acef708ca 100644 --- a/internal/pkg/operator/catalog_handler.go +++ b/internal/pkg/operator/catalog_handler.go @@ -109,11 +109,11 @@ func filterCatalog(ctx context.Context, operatorCatalog declcfg.DeclarativeConfi return dc, nil } -func (o CatalogHandler) getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfig, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) (map[string][]v2alpha1.RelatedImage, error) { +func (o CatalogHandler) getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfig, platforms *[]string, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) (map[string][]v2alpha1.RelatedImage, error) { var errs []error relatedImages := make(map[string][]v2alpha1.RelatedImage) for _, bundle := range dc.Bundles { - ris, err := handleRelatedImages(bundle, bundle.Package, copyImageSchemaMap) + ris, err := handleRelatedImages(bundle, bundle.Package, platforms, copyImageSchemaMap) if err != nil { o.Log.Warn("%s SKIPPING bundle %s of operator %s", err.Error(), bundle.Name, bundle.Package) errs = append(errs, err) @@ -129,15 +129,16 @@ func (o CatalogHandler) getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfi return relatedImages, errors.Join(errs...) } -func handleRelatedImages(bundle declcfg.Bundle, operatorName string, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) ([]v2alpha1.RelatedImage, error) { +func handleRelatedImages(bundle declcfg.Bundle, operatorName string, platforms *[]string, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) ([]v2alpha1.RelatedImage, error) { var relatedImages []v2alpha1.RelatedImage for _, ri := range bundle.RelatedImages { if strings.Contains(ri.Image, consts.OciProtocol) { return relatedImages, fmt.Errorf("invalid image: %s 'oci' is not supported in operator catalogs", ri.Image) } relatedImage := v2alpha1.RelatedImage{ - Name: ri.Name, - Image: ri.Image, + Name: ri.Name, + Image: ri.Image, + Platforms: platforms, } if ri.Image == bundle.Image { relatedImage.Type = v2alpha1.TypeOperatorBundle diff --git a/internal/pkg/operator/catalog_handler_test.go b/internal/pkg/operator/catalog_handler_test.go index e806a3c0d..9a15f07aa 100644 --- a/internal/pkg/operator/catalog_handler_test.go +++ b/internal/pkg/operator/catalog_handler_test.go @@ -42,7 +42,7 @@ func TestRelatedImagesFromCatalog(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.caseName, func(t *testing.T) { copyImageSchemaMap := &v2alpha1.CopyImageSchemaMap{OperatorsByImage: make(map[string]map[string]struct{}), BundlesByImage: make(map[string]map[string]string)} - res, err := handler.getRelatedImagesFromCatalog(testCase.cfg, copyImageSchemaMap) + res, err := handler.getRelatedImagesFromCatalog(testCase.cfg, nil, copyImageSchemaMap) if testCase.expectedError != nil { assert.EqualError(t, err, testCase.expectedError.Error()) } else { diff --git a/internal/pkg/operator/filtered_collector.go b/internal/pkg/operator/filtered_collector.go index 5ad831303..e8163820b 100644 --- a/internal/pkg/operator/filtered_collector.go +++ b/internal/pkg/operator/filtered_collector.go @@ -249,7 +249,10 @@ func (o FilterCollector) collectOperator( //nolint:cyclop // TODO: this needs fu return v2alpha1.CatalogFilterResult{}, err } - ri, err := o.ctlgHandler.getRelatedImagesFromCatalog(result.DeclConfig, copyImageSchemaMap) + // Convert platform filters to string format + platforms := v2alpha1.ConvertPlatformsToStringSlice(op.Platforms) + + ri, err := o.ctlgHandler.getRelatedImagesFromCatalog(result.DeclConfig, platforms, copyImageSchemaMap) if err != nil { return v2alpha1.CatalogFilterResult{}, err } diff --git a/internal/pkg/operator/filtered_collector_test.go b/internal/pkg/operator/filtered_collector_test.go index b9847081a..614b06d63 100644 --- a/internal/pkg/operator/filtered_collector_test.go +++ b/internal/pkg/operator/filtered_collector_test.go @@ -1058,7 +1058,7 @@ func (o MockManifest) ImageManifest(ctx context.Context, srcCtx *types.SystemCon return nil, "", errors.New("not implemented") } -func (o MockHandler) getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfig, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) (map[string][]v2alpha1.RelatedImage, error) { +func (o MockHandler) getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfig, platforms *[]string, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) (map[string][]v2alpha1.RelatedImage, error) { relatedImages := make(map[string][]v2alpha1.RelatedImage) relatedImages["abc"] = []v2alpha1.RelatedImage{ {Name: "testA", Image: "sometestimage-a@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea"}, diff --git a/internal/pkg/operator/interface.go b/internal/pkg/operator/interface.go index aca34a586..7ad11520f 100644 --- a/internal/pkg/operator/interface.go +++ b/internal/pkg/operator/interface.go @@ -16,7 +16,7 @@ type CollectorInterface interface { type catalogHandlerInterface interface { GetDeclarativeConfig(ctx context.Context, filePath string) (*declcfg.DeclarativeConfig, error) - getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfig, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) (map[string][]v2alpha1.RelatedImage, error) + getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfig, platforms *[]string, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) (map[string][]v2alpha1.RelatedImage, error) EnsureCatalogInOCIFormat(ctx context.Context, imgSpec image.ImageSpec, catalog, imageIndexDir string, opts mirror.CopyOptions) error ExtractOCIConfigLayers(imgSpec image.ImageSpec, imageIndexDir string) (string, error) } diff --git a/internal/pkg/release/local_stored_collector.go b/internal/pkg/release/local_stored_collector.go index 4cd24261c..7ea6125cc 100644 --- a/internal/pkg/release/local_stored_collector.go +++ b/internal/pkg/release/local_stored_collector.go @@ -50,6 +50,31 @@ func (o LocalStorageCollector) destinationRegistry() string { return o.destReg } +// getPlatformFilters converts Platform configuration to platform filter strings. +// It prioritizes the new Platforms field over the deprecated Architectures field. +// Returns nil if no platforms are specified (meaning mirror all platforms). +func (o LocalStorageCollector) getPlatformFilters() *[]string { + // Use new Platforms field if available + if len(o.Config.Mirror.Platform.Platforms) > 0 { + return v2alpha1.ConvertPlatformsToStringSlice(o.Config.Mirror.Platform.Platforms) + } + + // Fall back to deprecated Architectures field (assume linux) + //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility + if len(o.Config.Mirror.Platform.Architectures) > 0 { + //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility + platformStrs := make([]string, len(o.Config.Mirror.Platform.Architectures)) + //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility + for i, arch := range o.Config.Mirror.Platform.Architectures { + platformStrs[i] = "linux/" + arch + } + return &platformStrs + } + + // No platforms specified - mirror all + return nil +} + func (o *LocalStorageCollector) ReleaseImageCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) { // we just care for 1 platform release, in order to read release images o.Opts.MultiArch = "system" @@ -114,8 +139,21 @@ func (o *LocalStorageCollector) collectImageFromMirror(ctx context.Context) ([]v return []v2alpha1.CopyImageSchema{}, fmt.Errorf("%s%w", collectorPrefix, err) } + // Convert platform configuration to string format for RelatedImages + platforms := o.getPlatformFilters() + + // Set platforms for all collected images + for i := range allRelatedImages { + allRelatedImages[i].Platforms = platforms + } + // add the release image itself - allRelatedImages = append(allRelatedImages, v2alpha1.RelatedImage{Image: value.Source, Name: value.Source, Type: v2alpha1.TypeOCPRelease}) + allRelatedImages = append(allRelatedImages, v2alpha1.RelatedImage{ + Image: value.Source, + Name: value.Source, + Type: v2alpha1.TypeOCPRelease, + Platforms: platforms, + }) tmpAllImages, err := o.prepareM2DCopyBatch(allRelatedImages, releaseTag) if err != nil { logCollectionError(o.Log, spinner, o.Opts.Global.IsTerminal, value.Source, err) @@ -355,7 +393,13 @@ func (o LocalStorageCollector) prepareM2DCopyBatch(images []v2alpha1.RelatedImag o.Log.Debug("source %s", src) o.Log.Debug("destination %s", dest) - result = append(result, v2alpha1.CopyImageSchema{Origin: img.Image, Source: src, Destination: dest, Type: img.Type}) + result = append(result, v2alpha1.CopyImageSchema{ + Origin: img.Image, + Source: src, + Destination: dest, + Type: img.Type, + Platforms: img.Platforms, + }) } return result, nil } @@ -383,7 +427,13 @@ func (o LocalStorageCollector) prepareD2MCopyBatch(images []v2alpha1.RelatedImag o.Log.Debug("source %s", src) o.Log.Debug("destination %s", dest) - result = append(result, v2alpha1.CopyImageSchema{Origin: img.Image, Source: src, Destination: dest, Type: img.Type}) + result = append(result, v2alpha1.CopyImageSchema{ + Origin: img.Image, + Source: src, + Destination: dest, + Type: img.Type, + Platforms: img.Platforms, + }) } return result, nil @@ -417,7 +467,11 @@ func (o LocalStorageCollector) identifyReleases(ctx context.Context) ([]v2alpha1 releasePath = strings.TrimPrefix(releasePath, consts.OciProtocolTrimmed) releaseHoldPath := strings.Replace(releasePath, releaseImageDir, releaseImageExtractDir, 1) releaseFolders = append(releaseFolders, releaseHoldPath) - releaseImages = append(releaseImages, v2alpha1.RelatedImage{Name: copy.Source, Image: copy.Source, Type: v2alpha1.TypeOCPRelease}) + releaseImages = append(releaseImages, v2alpha1.RelatedImage{ + Name: copy.Source, + Image: copy.Source, + Type: v2alpha1.TypeOCPRelease, + }) } return releaseImages, releaseFolders, nil } @@ -545,6 +599,7 @@ func (o LocalStorageCollector) handleGraphImage(ctx context.Context) (v2alpha1.C Destination: graphImgRef, Origin: cachedImageRef, Type: v2alpha1.TypeCincinnatiGraph, + Platforms: o.getPlatformFilters(), } return graphCopy, nil } @@ -559,6 +614,7 @@ func (o LocalStorageCollector) handleGraphImage(ctx context.Context) (v2alpha1.C Destination: graphImgRef, Origin: workingDirGraphImageRef, Type: v2alpha1.TypeCincinnatiGraph, + Platforms: o.getPlatformFilters(), } return graphCopy, nil } @@ -576,6 +632,7 @@ func (o LocalStorageCollector) handleGraphImage(ctx context.Context) (v2alpha1.C Destination: graphImgRef, Origin: graphImgRef, Type: v2alpha1.TypeCincinnatiGraph, + Platforms: o.getPlatformFilters(), } return graphCopy, nil } From 68c9d08fbb2c7f45c253c0fa728f6fe0dd469d0a Mon Sep 17 00:00:00 2001 From: Alex Guidi Date: Wed, 8 Jul 2026 12:09:34 +0200 Subject: [PATCH 02/10] fixup! feat: Add sparse manifest platform filtering for multi-arch images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit refactor: Replace per-image Platforms with CollectorSchema.PlatformFilters map This is a cleaner approach than the previous commit. Instead of storing platform filters on RelatedImage and CopyImageSchema (polluting image structs with filtering concerns), each collector now populates a PlatformFilters map[string][]string on CollectorSchema, keyed by image origin. The batch worker reads from this map at copy time. Note: platform filtering for release content images is not yet functional end-to-end. For it to work, the Cincinnati query must return the multi-arch payload (arch=multi) so that image-references contains manifest list digests rather than single-arch digests. This is tracked as a follow-up. Please review the TBD comments in the code — particularly in executor.go (cross-collector platform merge semantics) and filtered_collector.go (localRelatedImages approach) — as these decisions benefit from team input. --- internal/pkg/additional/interface.go | 2 +- .../pkg/additional/local_stored_collector.go | 16 ++++-- .../additional/local_stored_collector_test.go | 12 ++-- internal/pkg/api/v2alpha1/type_internal.go | 12 +--- internal/pkg/api/v2alpha1/type_platform.go | 9 ++- internal/pkg/batch/concurrent_chan_worker.go | 13 ++++- internal/pkg/cli/executor.go | 34 ++++++----- internal/pkg/cli/executor_test.go | 18 +++--- internal/pkg/helm/interface.go | 2 +- internal/pkg/helm/local_stored_collector.go | 57 ++++++++++++------- .../pkg/helm/local_stored_collector_test.go | 6 +- internal/pkg/operator/catalog_handler.go | 11 ++-- internal/pkg/operator/catalog_handler_test.go | 2 +- internal/pkg/operator/filtered_collector.go | 29 ++++++++-- .../pkg/operator/filtered_collector_test.go | 2 +- internal/pkg/operator/interface.go | 2 +- internal/pkg/release/interface.go | 2 +- .../pkg/release/local_stored_collector.go | 50 +++++++++------- .../release/local_stored_collector_test.go | 12 ++-- 19 files changed, 173 insertions(+), 118 deletions(-) diff --git a/internal/pkg/additional/interface.go b/internal/pkg/additional/interface.go index 057d6b1d1..5ee8d1eaf 100644 --- a/internal/pkg/additional/interface.go +++ b/internal/pkg/additional/interface.go @@ -7,5 +7,5 @@ import ( ) type CollectorInterface interface { - AdditionalImagesCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) + AdditionalImagesCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) } diff --git a/internal/pkg/additional/local_stored_collector.go b/internal/pkg/additional/local_stored_collector.go index 77fb1c129..635257d3a 100644 --- a/internal/pkg/additional/local_stored_collector.go +++ b/internal/pkg/additional/local_stored_collector.go @@ -47,10 +47,11 @@ func (o LocalStorageCollector) destinationRegistry() string { // AdditionalImagesCollector - this looks into the additional images field // taking into account the mode we are in (mirrorToDisk, diskToMirror) // the image is downloaded in oci format -func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) { +func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { var allImages []v2alpha1.CopyImageSchema var allErrs []error + platformFilters := make(map[string][]string) o.Log.Debug(collectorPrefix+"setting copy option o.Opts.MultiArch=%s when collecting releases image", o.Opts.MultiArch) for _, img := range o.Config.ImageSetConfigurationSpec.Mirror.AdditionalImages { @@ -114,18 +115,21 @@ func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) ([ o.Log.Debug(collectorPrefix+"source %s", src) o.Log.Debug(collectorPrefix+"destination %s", dest) - // Convert platform filters to string format for CopyImageSchema - platforms := v2alpha1.ConvertPlatformsToStringSlice(img.Platforms) - allImages = append(allImages, v2alpha1.CopyImageSchema{ Source: src, Destination: dest, Origin: origin, Type: v2alpha1.TypeGeneric, - Platforms: platforms, }) + if platforms := v2alpha1.ConvertPlatformsToStringSlice(img.Platforms); len(platforms) > 0 { + platformFilters[origin] = platforms + } + } + cs := v2alpha1.CollectorSchema{AllImages: allImages} + if len(platformFilters) > 0 { + cs.PlatformFilters = platformFilters } - return allImages, errors.Join(allErrs...) + return cs, errors.Join(allErrs...) } // buildMirrorToDiskPaths constructs source and destination paths for mirror-to-disk and mirror-to-mirror operations diff --git a/internal/pkg/additional/local_stored_collector_test.go b/internal/pkg/additional/local_stored_collector_test.go index 9fa031182..fa5d63594 100644 --- a/internal/pkg/additional/local_stored_collector_test.go +++ b/internal/pkg/additional/local_stored_collector_test.go @@ -107,7 +107,7 @@ func TestAdditionalImageCollector(t *testing.T) { log.Error(" %v ", err) t.Fatalf("should not fail") } - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) }) // update opts @@ -148,7 +148,7 @@ func TestAdditionalImageCollector(t *testing.T) { log.Error(" %v ", err) t.Fatalf("should not fail") } - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) }) t.Run("Testing AdditionalImagesCollector : diskToMirror with generateV1Tags should use latest for images by digest", func(t *testing.T) { @@ -187,7 +187,7 @@ func TestAdditionalImageCollector(t *testing.T) { log.Error(" %v ", err) t.Fatalf("should not fail") } - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) }) // should error mirrorToDisk @@ -219,7 +219,7 @@ func TestAdditionalImageCollector(t *testing.T) { res, err := ex.AdditionalImagesCollector(ctx) // Should return error for images that failed to parse require.Error(t, err) - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) }) t.Run("Testing AdditionalImagesCollector : diskToMirror should collect valid images and return parse errors", func(t *testing.T) { @@ -250,7 +250,7 @@ func TestAdditionalImageCollector(t *testing.T) { res, err := ex.AdditionalImagesCollector(ctx) // Should return error for images that failed to parse require.Error(t, err) - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) }) } @@ -586,7 +586,7 @@ func TestAdditionalImageCollectorWithTargetRepoAndTag(t *testing.T) { } else { require.NoError(t, err) } - assert.ElementsMatch(t, c.expected, res) + assert.ElementsMatch(t, c.expected, res.AllImages) }) } } diff --git a/internal/pkg/api/v2alpha1/type_internal.go b/internal/pkg/api/v2alpha1/type_internal.go index f1061337d..d384b78d0 100644 --- a/internal/pkg/api/v2alpha1/type_internal.go +++ b/internal/pkg/api/v2alpha1/type_internal.go @@ -182,10 +182,6 @@ type RelatedImage struct { OriginFromOperatorCatalogOnDisk bool // Used to identify if it is a full catalog (full: true && zero packages) FullCatalog bool - // Platforms defines one or more OS/Architecture pairs to mirror - // for this multi-architecture image. Formatted as "os/arch" (e.g., "linux/amd64"). - // Using a pointer to maintain struct comparability. - Platforms *[]string `json:"-"` } type CollectorSchema struct { @@ -196,6 +192,9 @@ type CollectorSchema struct { AllImages []CopyImageSchema CopyImageSchemaMap CopyImageSchemaMap CatalogToFBCMap map[string]CatalogFilterResult // key is the mirror.operator.catalog + // PlatformFilters maps image origin to its platform filter list ("os/arch" strings). + // Populated by collectors; consumed by the batch worker to set InstancePlatforms per image. + PlatformFilters map[string][]string } type CopyImageSchemaMap struct { @@ -215,11 +214,6 @@ type CopyImageSchema struct { // it doesn´t need to be persisted to json Type ImageType `json:"-"` RebuiltTag string `json:"rebuiltTag"` - // Platforms defines one or more OS/Architecture pairs to mirror - // for this multi-architecture image. If nil, defaults to "all" (mirrors all platforms). - // Formatted as "os/arch" (e.g., "linux/amd64", "linux/arm64"). - // Using a pointer to maintain struct comparability for use in maps and slices.Compact. - Platforms *[]string `json:"-"` } // SignatureContentSchema diff --git a/internal/pkg/api/v2alpha1/type_platform.go b/internal/pkg/api/v2alpha1/type_platform.go index 39f06a816..188da03d1 100644 --- a/internal/pkg/api/v2alpha1/type_platform.go +++ b/internal/pkg/api/v2alpha1/type_platform.go @@ -76,10 +76,9 @@ func (p InstancePlatformFilter) String() string { return p.OS + "/" + p.Architecture } -// ConvertPlatformsToStringSlice converts a slice of InstancePlatformFilter to a pointer -// to a slice of platform strings in "os/architecture" format. -// Returns nil if the input slice is empty. -func ConvertPlatformsToStringSlice(platforms []InstancePlatformFilter) *[]string { +// ConvertPlatformsToStringSlice converts a slice of InstancePlatformFilter to a slice +// of platform strings in "os/architecture" format. Returns nil if the input is empty. +func ConvertPlatformsToStringSlice(platforms []InstancePlatformFilter) []string { if len(platforms) == 0 { return nil } @@ -88,5 +87,5 @@ func ConvertPlatformsToStringSlice(platforms []InstancePlatformFilter) *[]string for i, p := range platforms { platformStrs[i] = p.String() } - return &platformStrs + return platformStrs } diff --git a/internal/pkg/batch/concurrent_chan_worker.go b/internal/pkg/batch/concurrent_chan_worker.go index c08e3aa29..295525849 100644 --- a/internal/pkg/batch/concurrent_chan_worker.go +++ b/internal/pkg/batch/concurrent_chan_worker.go @@ -145,12 +145,21 @@ func (o *ChannelConcurrentBatch) Worker(ctx context.Context, collectorSchema v2a } // If this image has specific platform requirements, use them - if img.Platforms != nil && len(*img.Platforms) > 0 { - options.InstancePlatforms = *img.Platforms + if platforms, ok := collectorSchema.PlatformFilters[img.Origin]; ok && len(platforms) > 0 { + options.InstancePlatforms = platforms } err = o.Mirror.Run(timeoutCtx, img.Source, img.Destination, mirror.Mode(opts.Function), &options) //nolint:contextcheck + // TODO + // TBD not sure if I like this approach below + // Single-arch images don't support platform filtering; retry without it. + if err != nil && options.InstancePlatforms != nil && strings.Contains(err.Error(), "no instances found for platform") { + o.Log.Debug("Platform filtering not supported for %s, retrying without filter", img.Source) + options.InstancePlatforms = nil + err = o.Mirror.Run(timeoutCtx, img.Source, img.Destination, mirror.Mode(opts.Function), &options) //nolint:contextcheck + } + switch { case err == nil: spinner.Increment() diff --git a/internal/pkg/cli/executor.go b/internal/pkg/cli/executor.go index b3bdd6925..993d5afd6 100644 --- a/internal/pkg/cli/executor.go +++ b/internal/pkg/cli/executor.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "log" + "maps" "net" "net/http" "net/url" @@ -1146,8 +1147,10 @@ func (o *ExecutorSchema) CollectAll(ctx context.Context) (v2alpha1.CollectorSche if len(o.Config.Mirror.Platform.Channels) > 0 || o.Config.Mirror.Platform.Release != "" || o.Config.Mirror.Platform.Graph { o.Log.Info(emoji.LeftPointingMagnifyingGlass + " collecting release images...") } + collectorSchema.PlatformFilters = make(map[string][]string) + // collect releases - releaseImgs, err := o.Release.ReleaseImageCollector(ctx) + releaseCS, err := o.Release.ReleaseImageCollector(ctx) if err != nil { if !o.Opts.IsDelete() { // Release image errors are fatal: we don't want to continue collection when they happen. @@ -1155,13 +1158,12 @@ func (o *ExecutorSchema) CollectAll(ctx context.Context) (v2alpha1.CollectorSche } releaseErr = err } - // exclude blocked images - releaseImgs = excludeImages(releaseImgs, o.Config.Mirror.BlockedImages) + releaseImgs := excludeImages(releaseCS.AllImages, o.Config.Mirror.BlockedImages) releaseImgs = removeDuplicatedImages(releaseImgs, o.Opts.Function) - collectorSchema.TotalReleaseImages = len(releaseImgs) o.Log.Debug(collecAllPrefix+"total release images to %s %d ", o.Opts.Function, collectorSchema.TotalReleaseImages) allRelatedImages = append(allRelatedImages, releaseImgs...) + maps.Copy(collectorSchema.PlatformFilters, releaseCS.PlatformFilters) if len(o.Config.Mirror.Operators) > 0 { o.Log.Info(emoji.LeftPointingMagnifyingGlass + " collecting operator images...") @@ -1171,45 +1173,51 @@ func (o *ExecutorSchema) CollectAll(ctx context.Context) (v2alpha1.CollectorSche if err != nil { operatorErr = err } else { - oImgs := operatorImgs.AllImages - // exclude blocked images - oImgs = excludeImages(oImgs, o.Config.Mirror.BlockedImages) + oImgs := excludeImages(operatorImgs.AllImages, o.Config.Mirror.BlockedImages) oImgs = removeDuplicatedImages(oImgs, o.Opts.Function) collectorSchema.TotalOperatorImages = len(oImgs) o.Log.Debug(collecAllPrefix+"total operator images to %s %d ", o.Opts.Function, collectorSchema.TotalOperatorImages) allRelatedImages = append(allRelatedImages, oImgs...) collectorSchema.CopyImageSchemaMap = operatorImgs.CopyImageSchemaMap collectorSchema.CatalogToFBCMap = operatorImgs.CatalogToFBCMap + + // TODO: if images are shared between types, maps.Copy will override platform filters for + // the same origin. Consider replacing with an append-based merge across all collectors: + // for k, v := range releaseCS.PlatformFilters { + // collectorSchema.PlatformFilters[k] = append(collectorSchema.PlatformFilters[k], v...) + // } + + maps.Copy(collectorSchema.PlatformFilters, operatorImgs.PlatformFilters) } if len(o.Config.Mirror.AdditionalImages) > 0 { o.Log.Info(emoji.LeftPointingMagnifyingGlass + " collecting additional images...") } // collect additionalImages - aImgs, err := o.AdditionalImages.AdditionalImagesCollector(ctx) + additionalCS, err := o.AdditionalImages.AdditionalImagesCollector(ctx) if err != nil { additionalImgErr = err } - // exclude blocked images - aImgs = excludeImages(aImgs, o.Config.Mirror.BlockedImages) + aImgs := excludeImages(additionalCS.AllImages, o.Config.Mirror.BlockedImages) aImgs = removeDuplicatedImages(aImgs, o.Opts.Function) collectorSchema.TotalAdditionalImages = len(aImgs) o.Log.Debug(collecAllPrefix+"total additional images to %s %d ", o.Opts.Function, collectorSchema.TotalAdditionalImages) allRelatedImages = append(allRelatedImages, aImgs...) + maps.Copy(collectorSchema.PlatformFilters, additionalCS.PlatformFilters) if len(o.Config.Mirror.Helm.Repositories) > 0 || len(o.Config.Mirror.Helm.Local) > 0 { o.Log.Info(emoji.LeftPointingMagnifyingGlass + " collecting helm images...") } - hImgs, err := o.HelmCollector.HelmImageCollector(ctx) + helmCS, err := o.HelmCollector.HelmImageCollector(ctx) if err != nil { helmErr = err } else { - // exclude blocked images - hImgs = excludeImages(hImgs, o.Config.Mirror.BlockedImages) + hImgs := excludeImages(helmCS.AllImages, o.Config.Mirror.BlockedImages) hImgs = removeDuplicatedImages(hImgs, o.Opts.Function) collectorSchema.TotalHelmImages = len(hImgs) o.Log.Debug(collecAllPrefix+"total helm images to %s %d ", o.Opts.Function, collectorSchema.TotalHelmImages) allRelatedImages = append(allRelatedImages, hImgs...) + maps.Copy(collectorSchema.PlatformFilters, helmCS.PlatformFilters) } sort.Sort(customsort.ByTypePriority(allRelatedImages)) diff --git a/internal/pkg/cli/executor_test.go b/internal/pkg/cli/executor_test.go index cddbbe837..1f6fe4a91 100644 --- a/internal/pkg/cli/executor_test.go +++ b/internal/pkg/cli/executor_test.go @@ -1245,9 +1245,9 @@ func (o *Collector) OperatorImageCollector(ctx context.Context) (v2alpha1.Collec return v2alpha1.CollectorSchema{AllImages: test}, nil } -func (o *Collector) ReleaseImageCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) { +func (o *Collector) ReleaseImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { if o.Fail { - return []v2alpha1.CopyImageSchema{}, fmt.Errorf("forced error release collector") + return v2alpha1.CollectorSchema{}, fmt.Errorf("forced error release collector") } test := []v2alpha1.CopyImageSchema{ {Source: "docker://registry/name/namespace/sometestimage-a@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea", Destination: "oci:test"}, @@ -1257,7 +1257,7 @@ func (o *Collector) ReleaseImageCollector(ctx context.Context) ([]v2alpha1.CopyI {Source: "docker://registry/name/namespace/sometestimage-e@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea", Destination: "oci:test"}, {Source: "docker://registry/name/namespace/sometestimage-f@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea", Destination: "oci:test"}, } - return test, nil + return v2alpha1.CollectorSchema{AllImages: test}, nil } func (o *Collector) GraphImage() (string, error) { @@ -1268,9 +1268,9 @@ func (o *Collector) ReleaseImage(ctx context.Context) (string, error) { return "quay.io/openshift-release-dev/ocp-release:4.13.10-x86_64", nil } -func (o *Collector) AdditionalImagesCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) { +func (o *Collector) AdditionalImagesCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { if o.Fail { - return []v2alpha1.CopyImageSchema{}, fmt.Errorf("forced error additionalImages collector") + return v2alpha1.CollectorSchema{}, fmt.Errorf("forced error additionalImages collector") } test := []v2alpha1.CopyImageSchema{ {Source: "docker://registry/name/namespace/sometestimage-a@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea", Destination: "oci:test"}, @@ -1280,14 +1280,14 @@ func (o *Collector) AdditionalImagesCollector(ctx context.Context) ([]v2alpha1.C {Source: "docker://registry/name/namespace/sometestimage-e@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea", Destination: "oci:test"}, {Source: "docker://registry/name/namespace/sometestimage-f@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea", Destination: "oci:test"}, } - return test, nil + return v2alpha1.CollectorSchema{AllImages: test}, nil } -func (o *Collector) HelmImageCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) { +func (o *Collector) HelmImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { if o.Fail { - return []v2alpha1.CopyImageSchema{}, fmt.Errorf("forced error helm collector") + return v2alpha1.CollectorSchema{}, fmt.Errorf("forced error helm collector") } - return []v2alpha1.CopyImageSchema{}, nil + return v2alpha1.CollectorSchema{}, nil } func (o MockArchiver) BuildArchive(ctx context.Context, collectedImages []v2alpha1.CopyImageSchema) error { diff --git a/internal/pkg/helm/interface.go b/internal/pkg/helm/interface.go index 8b8211a90..0cc50ee3a 100644 --- a/internal/pkg/helm/interface.go +++ b/internal/pkg/helm/interface.go @@ -8,7 +8,7 @@ import ( ) type CollectorInterface interface { - HelmImageCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) + HelmImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) } type indexDownloader interface { diff --git a/internal/pkg/helm/local_stored_collector.go b/internal/pkg/helm/local_stored_collector.go index e7e9ee69c..b7dfa3109 100644 --- a/internal/pkg/helm/local_stored_collector.go +++ b/internal/pkg/helm/local_stored_collector.go @@ -78,24 +78,28 @@ func WithV1Tags(o CollectorInterface) CollectorInterface { return o } -func (o *LocalStorageCollector) HelmImageCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) { +func (o *LocalStorageCollector) HelmImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { var ( allImages []v2alpha1.CopyImageSchema allHelmImages []v2alpha1.RelatedImage errs []error ) + platformFilters := make(map[string][]string) switch { case lsc.Opts.IsMirrorToDisk() || lsc.Opts.IsMirrorToMirror(): defer lsc.cleanup() var err error - imgs, errors := getHelmImagesFromLocalChart() + imgs, localPlatforms, errors := getHelmImagesFromLocalChart() if len(errors) > 0 { errs = append(errs, errors...) } if len(imgs) > 0 { allHelmImages = append(allHelmImages, imgs...) + for k, v := range localPlatforms { + platformFilters[k] = v + } } for _, repo := range lsc.Config.Mirror.Helm.Repositories { @@ -130,15 +134,19 @@ func (o *LocalStorageCollector) HelmImageCollector(ctx context.Context) ([]v2alp continue } - // Convert platform filters to string format platforms := v2alpha1.ConvertPlatformsToStringSlice(chart.Platforms) - imgs, err := getImages(path, platforms, chart.ImagePaths...) + imgs, err := getImages(path, chart.ImagePaths...) if err != nil { errs = append(errs, err) } allHelmImages = append(allHelmImages, imgs...) + if len(platforms) > 0 { + for _, img := range imgs { + platformFilters[img.Image] = append(platformFilters[img.Image], platforms...) + } + } } } @@ -150,12 +158,15 @@ func (o *LocalStorageCollector) HelmImageCollector(ctx context.Context) ([]v2alp } case lsc.Opts.IsDiskToMirror(): - imgs, errors := getHelmImagesFromLocalChart() + imgs, localPlatforms, errors := getHelmImagesFromLocalChart() if len(errors) > 0 { errs = append(errs, errors...) } if len(imgs) > 0 { allHelmImages = append(allHelmImages, imgs...) + for k, v := range localPlatforms { + platformFilters[k] = v + } } for _, repo := range lsc.Config.Mirror.Helm.Repositories { @@ -179,15 +190,19 @@ func (o *LocalStorageCollector) HelmImageCollector(ctx context.Context) ([]v2alp continue } - // Convert platform filters to string format platforms := v2alpha1.ConvertPlatformsToStringSlice(chart.Platforms) - imgs, err := getImages(path, platforms, chart.ImagePaths...) + imgs, err := getImages(path, chart.ImagePaths...) if err != nil { errs = append(errs, err) } allHelmImages = append(allHelmImages, imgs...) + if len(platforms) > 0 { + for _, img := range imgs { + platformFilters[img.Image] = append(platformFilters[img.Image], platforms...) + } + } } } @@ -200,7 +215,11 @@ func (o *LocalStorageCollector) HelmImageCollector(ctx context.Context) ([]v2alp } } - return allImages, errors.Join(errs...) + cs := v2alpha1.CollectorSchema{AllImages: allImages} + if len(platformFilters) > 0 { + cs.PlatformFilters = platformFilters + } + return cs, errors.Join(errs...) } func createTempFile(dir string) (func(), string, error) { @@ -232,25 +251,30 @@ func GetDefaultChartDownloader() chartDownloader { } } -func getHelmImagesFromLocalChart() ([]v2alpha1.RelatedImage, []error) { +func getHelmImagesFromLocalChart() ([]v2alpha1.RelatedImage, map[string][]string, []error) { var allHelmImages []v2alpha1.RelatedImage var errs []error + platformFilters := make(map[string][]string) for _, chart := range lsc.Config.Mirror.Helm.Local { - // Convert platform filters to string format platforms := v2alpha1.ConvertPlatformsToStringSlice(chart.Platforms) - imgs, err := getImages(chart.Path, platforms, chart.ImagePaths...) + imgs, err := getImages(chart.Path, chart.ImagePaths...) if err != nil { errs = append(errs, err) } if len(imgs) > 0 { allHelmImages = append(allHelmImages, imgs...) + if len(platforms) > 0 { + for _, img := range imgs { + platformFilters[img.Image] = append(platformFilters[img.Image], platforms...) + } + } } } - return allHelmImages, errs + return allHelmImages, platformFilters, errs } func repoAdd(chartRepo v2alpha1.Repository) error { @@ -431,7 +455,7 @@ func buildChartCandidatePath(baseDir, name, ver string) (string, error) { return p, nil } -func getImages(path string, platforms *[]string, imagePaths ...string) (images []v2alpha1.RelatedImage, err error) { +func getImages(path string, imagePaths ...string) (images []v2alpha1.RelatedImage, err error) { lsc.Log.Debug("Reading from path %s", path) p := getImagesPath(imagePaths...) @@ -453,11 +477,6 @@ func getImages(path string, platforms *[]string, imagePaths ...string) (images [ return nil, err } - // Set platforms for each image - for i := range imgs { - imgs[i].Platforms = platforms - } - images = append(images, imgs...) } @@ -609,7 +628,6 @@ func prepareM2DCopyBatch(images []v2alpha1.RelatedImage) ([]v2alpha1.CopyImageSc Source: src, Destination: dest, Type: img.Type, - Platforms: img.Platforms, }) } return result, nil @@ -655,7 +673,6 @@ func prepareD2MCopyBatch(images []v2alpha1.RelatedImage, generateV1TagsFromDiges Source: src, Destination: dest, Type: img.Type, - Platforms: img.Platforms, }) } diff --git a/internal/pkg/helm/local_stored_collector_test.go b/internal/pkg/helm/local_stored_collector_test.go index 5d813432d..ed7eca3f1 100644 --- a/internal/pkg/helm/local_stored_collector_test.go +++ b/internal/pkg/helm/local_stored_collector_test.go @@ -709,8 +709,8 @@ func TestHelmImageCollector(t *testing.T) { } if len(testCase.expectedResult) > 0 { - assert.NotEmpty(t, imgs) - assert.ElementsMatch(t, testCase.expectedResult, imgs) + assert.NotEmpty(t, imgs.AllImages) + assert.ElementsMatch(t, testCase.expectedResult, imgs.AllImages) } }) } @@ -873,7 +873,7 @@ func TestHelmImageCollectorVPrefixDiskToMirror(t *testing.T) { imgs, err := helmCollector.HelmImageCollector(ctx) assert.NoError(t, err) - assert.ElementsMatch(t, tc.expectedImages, imgs) + assert.ElementsMatch(t, tc.expectedImages, imgs.AllImages) }) } } diff --git a/internal/pkg/operator/catalog_handler.go b/internal/pkg/operator/catalog_handler.go index acef708ca..2de5dd42c 100644 --- a/internal/pkg/operator/catalog_handler.go +++ b/internal/pkg/operator/catalog_handler.go @@ -109,11 +109,11 @@ func filterCatalog(ctx context.Context, operatorCatalog declcfg.DeclarativeConfi return dc, nil } -func (o CatalogHandler) getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfig, platforms *[]string, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) (map[string][]v2alpha1.RelatedImage, error) { +func (o CatalogHandler) getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfig, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) (map[string][]v2alpha1.RelatedImage, error) { var errs []error relatedImages := make(map[string][]v2alpha1.RelatedImage) for _, bundle := range dc.Bundles { - ris, err := handleRelatedImages(bundle, bundle.Package, platforms, copyImageSchemaMap) + ris, err := handleRelatedImages(bundle, bundle.Package, copyImageSchemaMap) if err != nil { o.Log.Warn("%s SKIPPING bundle %s of operator %s", err.Error(), bundle.Name, bundle.Package) errs = append(errs, err) @@ -129,16 +129,15 @@ func (o CatalogHandler) getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfi return relatedImages, errors.Join(errs...) } -func handleRelatedImages(bundle declcfg.Bundle, operatorName string, platforms *[]string, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) ([]v2alpha1.RelatedImage, error) { +func handleRelatedImages(bundle declcfg.Bundle, operatorName string, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) ([]v2alpha1.RelatedImage, error) { var relatedImages []v2alpha1.RelatedImage for _, ri := range bundle.RelatedImages { if strings.Contains(ri.Image, consts.OciProtocol) { return relatedImages, fmt.Errorf("invalid image: %s 'oci' is not supported in operator catalogs", ri.Image) } relatedImage := v2alpha1.RelatedImage{ - Name: ri.Name, - Image: ri.Image, - Platforms: platforms, + Name: ri.Name, + Image: ri.Image, } if ri.Image == bundle.Image { relatedImage.Type = v2alpha1.TypeOperatorBundle diff --git a/internal/pkg/operator/catalog_handler_test.go b/internal/pkg/operator/catalog_handler_test.go index 9a15f07aa..e806a3c0d 100644 --- a/internal/pkg/operator/catalog_handler_test.go +++ b/internal/pkg/operator/catalog_handler_test.go @@ -42,7 +42,7 @@ func TestRelatedImagesFromCatalog(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.caseName, func(t *testing.T) { copyImageSchemaMap := &v2alpha1.CopyImageSchemaMap{OperatorsByImage: make(map[string]map[string]struct{}), BundlesByImage: make(map[string]map[string]string)} - res, err := handler.getRelatedImagesFromCatalog(testCase.cfg, nil, copyImageSchemaMap) + res, err := handler.getRelatedImagesFromCatalog(testCase.cfg, copyImageSchemaMap) if testCase.expectedError != nil { assert.EqualError(t, err, testCase.expectedError.Error()) } else { diff --git a/internal/pkg/operator/filtered_collector.go b/internal/pkg/operator/filtered_collector.go index e8163820b..ead86e6a8 100644 --- a/internal/pkg/operator/filtered_collector.go +++ b/internal/pkg/operator/filtered_collector.go @@ -39,6 +39,7 @@ func (o *FilterCollector) OperatorImageCollector(ctx context.Context) (v2alpha1. relatedImages := make(map[string][]v2alpha1.RelatedImage) collectorSchema := v2alpha1.CollectorSchema{ CatalogToFBCMap: make(map[string]v2alpha1.CatalogFilterResult), + PlatformFilters: make(map[string][]string), } copyImageSchemaMap := &v2alpha1.CopyImageSchemaMap{ OperatorsByImage: make(map[string]map[string]struct{}), @@ -70,13 +71,34 @@ func (o *FilterCollector) OperatorImageCollector(ctx context.Context) (v2alpha1. continue } - result, err := o.collectOperator(ctx, op, relatedImages, copyImageSchemaMap) + // Use a fresh local map per catalog so we can read exactly which images this + // catalog contributed — needed to populate platform filters correctly when the + // same bundle appears in multiple catalogs with different platform requirements. + localRelatedImages := make(map[string][]v2alpha1.RelatedImage) + result, err := o.collectOperator(ctx, op, localRelatedImages, copyImageSchemaMap) if err != nil { spinner.Abort(true) spinner.Wait() allErrs = append(allErrs, fmt.Errorf("collect catalog %q: %w", op.Catalog, err)) continue } + + // Append this catalog's platform filters into PlatformFilters using localRelatedImages. + // The same bundle appearing in two catalogs with different platforms gets both appended. + if platforms := v2alpha1.ConvertPlatformsToStringSlice(op.Platforms); len(platforms) > 0 { + for _, imgs := range localRelatedImages { + for _, img := range imgs { + if ref, parseErr := image.ParseRef(img.Image); parseErr == nil { + origin := ref.ReferenceWithTransport + collectorSchema.PlatformFilters[origin] = append(collectorSchema.PlatformFilters[origin], platforms...) + } + } + } + } + + // Merge this catalog's images into the global map. + maps.Copy(relatedImages, localRelatedImages) + // OCPBUGS-81712: In M2D/M2M modes, op.Catalog is already pinned to digest by executor.go // CLID-513: For OCI paths with digest, use a consistent key format (without digest) // This matches how catalogImage is constructed in collectOperator @@ -249,10 +271,7 @@ func (o FilterCollector) collectOperator( //nolint:cyclop // TODO: this needs fu return v2alpha1.CatalogFilterResult{}, err } - // Convert platform filters to string format - platforms := v2alpha1.ConvertPlatformsToStringSlice(op.Platforms) - - ri, err := o.ctlgHandler.getRelatedImagesFromCatalog(result.DeclConfig, platforms, copyImageSchemaMap) + ri, err := o.ctlgHandler.getRelatedImagesFromCatalog(result.DeclConfig, copyImageSchemaMap) if err != nil { return v2alpha1.CatalogFilterResult{}, err } diff --git a/internal/pkg/operator/filtered_collector_test.go b/internal/pkg/operator/filtered_collector_test.go index 614b06d63..b9847081a 100644 --- a/internal/pkg/operator/filtered_collector_test.go +++ b/internal/pkg/operator/filtered_collector_test.go @@ -1058,7 +1058,7 @@ func (o MockManifest) ImageManifest(ctx context.Context, srcCtx *types.SystemCon return nil, "", errors.New("not implemented") } -func (o MockHandler) getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfig, platforms *[]string, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) (map[string][]v2alpha1.RelatedImage, error) { +func (o MockHandler) getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfig, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) (map[string][]v2alpha1.RelatedImage, error) { relatedImages := make(map[string][]v2alpha1.RelatedImage) relatedImages["abc"] = []v2alpha1.RelatedImage{ {Name: "testA", Image: "sometestimage-a@sha256:f30638f60452062aba36a26ee6c036feead2f03b28f2c47f2b0a991e41baebea"}, diff --git a/internal/pkg/operator/interface.go b/internal/pkg/operator/interface.go index 7ad11520f..aca34a586 100644 --- a/internal/pkg/operator/interface.go +++ b/internal/pkg/operator/interface.go @@ -16,7 +16,7 @@ type CollectorInterface interface { type catalogHandlerInterface interface { GetDeclarativeConfig(ctx context.Context, filePath string) (*declcfg.DeclarativeConfig, error) - getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfig, platforms *[]string, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) (map[string][]v2alpha1.RelatedImage, error) + getRelatedImagesFromCatalog(dc *declcfg.DeclarativeConfig, copyImageSchemaMap *v2alpha1.CopyImageSchemaMap) (map[string][]v2alpha1.RelatedImage, error) EnsureCatalogInOCIFormat(ctx context.Context, imgSpec image.ImageSpec, catalog, imageIndexDir string, opts mirror.CopyOptions) error ExtractOCIConfigLayers(imgSpec image.ImageSpec, imageIndexDir string) (string, error) } diff --git a/internal/pkg/release/interface.go b/internal/pkg/release/interface.go index 5fe7b17dd..3ca4c4b48 100644 --- a/internal/pkg/release/interface.go +++ b/internal/pkg/release/interface.go @@ -7,7 +7,7 @@ import ( ) type CollectorInterface interface { - ReleaseImageCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) + ReleaseImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) // Returns the graphImage if generated, especially with the reference to the image // on the destination repository // Returns an error if the graph image was not generated (graph : false) diff --git a/internal/pkg/release/local_stored_collector.go b/internal/pkg/release/local_stored_collector.go index 7ea6125cc..50d8c3d66 100644 --- a/internal/pkg/release/local_stored_collector.go +++ b/internal/pkg/release/local_stored_collector.go @@ -53,7 +53,7 @@ func (o LocalStorageCollector) destinationRegistry() string { // getPlatformFilters converts Platform configuration to platform filter strings. // It prioritizes the new Platforms field over the deprecated Architectures field. // Returns nil if no platforms are specified (meaning mirror all platforms). -func (o LocalStorageCollector) getPlatformFilters() *[]string { +func (o LocalStorageCollector) getPlatformFilters() []string { // Use new Platforms field if available if len(o.Config.Mirror.Platform.Platforms) > 0 { return v2alpha1.ConvertPlatformsToStringSlice(o.Config.Mirror.Platform.Platforms) @@ -66,16 +66,19 @@ func (o LocalStorageCollector) getPlatformFilters() *[]string { platformStrs := make([]string, len(o.Config.Mirror.Platform.Architectures)) //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility for i, arch := range o.Config.Mirror.Platform.Architectures { + if arch == "multi" { + continue + } platformStrs[i] = "linux/" + arch } - return &platformStrs + return platformStrs } // No platforms specified - mirror all return nil } -func (o *LocalStorageCollector) ReleaseImageCollector(ctx context.Context) ([]v2alpha1.CopyImageSchema, error) { +func (o *LocalStorageCollector) ReleaseImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { // we just care for 1 platform release, in order to read release images o.Opts.MultiArch = "system" o.Log.Debug(collectorPrefix+"setting copy option o.Opts.MultiArch=%s when collecting releases image", o.Opts.MultiArch) @@ -93,7 +96,7 @@ func (o *LocalStorageCollector) ReleaseImageCollector(ctx context.Context) ([]v2 err = fmt.Errorf("release collector: invalid mirror mode %s", o.Opts.Mode) } if err != nil { - return []v2alpha1.CopyImageSchema{}, err + return v2alpha1.CollectorSchema{}, err } // OCPBUGS-43275: deduplicating @@ -110,7 +113,22 @@ func (o *LocalStorageCollector) ReleaseImageCollector(ctx context.Context) ([]v2 }) allImages = slices.Compact(allImages) - return allImages, nil + // TODO probably does not need to loop all the images, it would be possible to check the image type, if it is from a release, apply always the same os/arch saved somewhere else. + // TBD with team - Keeping in this way the advantages is that we create a standard and don't treat releases in a different way + // The disadvantage is performance + cs := v2alpha1.CollectorSchema{AllImages: allImages} + if platforms := o.getPlatformFilters(); len(platforms) > 0 { + cs.PlatformFilters = make(map[string][]string, len(allImages)) + for _, img := range allImages { + // Graph images are excluded: in the non-enclave case they are already built for + // the requested architectures by CreateGraphImage; in the enclave case they are + // pre-built external artifacts that must be copied whole. + if img.Type != v2alpha1.TypeCincinnatiGraph { + cs.PlatformFilters[img.Origin] = platforms + } + } + } + return cs, nil } // collects release images from a mirror. @@ -139,20 +157,11 @@ func (o *LocalStorageCollector) collectImageFromMirror(ctx context.Context) ([]v return []v2alpha1.CopyImageSchema{}, fmt.Errorf("%s%w", collectorPrefix, err) } - // Convert platform configuration to string format for RelatedImages - platforms := o.getPlatformFilters() - - // Set platforms for all collected images - for i := range allRelatedImages { - allRelatedImages[i].Platforms = platforms - } - // add the release image itself allRelatedImages = append(allRelatedImages, v2alpha1.RelatedImage{ - Image: value.Source, - Name: value.Source, - Type: v2alpha1.TypeOCPRelease, - Platforms: platforms, + Image: value.Source, + Name: value.Source, + Type: v2alpha1.TypeOCPRelease, }) tmpAllImages, err := o.prepareM2DCopyBatch(allRelatedImages, releaseTag) if err != nil { @@ -398,7 +407,6 @@ func (o LocalStorageCollector) prepareM2DCopyBatch(images []v2alpha1.RelatedImag Source: src, Destination: dest, Type: img.Type, - Platforms: img.Platforms, }) } return result, nil @@ -432,7 +440,6 @@ func (o LocalStorageCollector) prepareD2MCopyBatch(images []v2alpha1.RelatedImag Source: src, Destination: dest, Type: img.Type, - Platforms: img.Platforms, }) } @@ -599,7 +606,6 @@ func (o LocalStorageCollector) handleGraphImage(ctx context.Context) (v2alpha1.C Destination: graphImgRef, Origin: cachedImageRef, Type: v2alpha1.TypeCincinnatiGraph, - Platforms: o.getPlatformFilters(), } return graphCopy, nil } @@ -614,12 +620,13 @@ func (o LocalStorageCollector) handleGraphImage(ctx context.Context) (v2alpha1.C Destination: graphImgRef, Origin: workingDirGraphImageRef, Type: v2alpha1.TypeCincinnatiGraph, - Platforms: o.getPlatformFilters(), } return graphCopy, nil } } else { + // TODO: pass o.getPlatformFilters() to CreateGraphImage so the graph image is built + // only for the requested architectures. Tracked as a separate PR. graphImgRef, err := o.CreateGraphImage(ctx, graphURL) if err != nil { return v2alpha1.CopyImageSchema{}, err @@ -632,7 +639,6 @@ func (o LocalStorageCollector) handleGraphImage(ctx context.Context) (v2alpha1.C Destination: graphImgRef, Origin: graphImgRef, Type: v2alpha1.TypeCincinnatiGraph, - Platforms: o.getPlatformFilters(), } return graphCopy, nil } diff --git a/internal/pkg/release/local_stored_collector_test.go b/internal/pkg/release/local_stored_collector_test.go index f29c2ed6d..351399211 100644 --- a/internal/pkg/release/local_stored_collector_test.go +++ b/internal/pkg/release/local_stored_collector_test.go @@ -111,7 +111,7 @@ func TestReleaseLocalStoredCollector(t *testing.T) { Type: v2alpha1.TypeCincinnatiGraph, }, } - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) log.Debug("completed test related images %v ", res) }) @@ -137,10 +137,10 @@ func TestReleaseLocalStoredCollector(t *testing.T) { if err != nil { t.Fatalf("should not fail: %v", err) } - if len(res) == 0 { + if len(res.AllImages) == 0 { t.Fatalf("should contain at least 1 image") } - if !strings.Contains(res[0].Source, ex.LocalStorageFQDN) { + if !strings.Contains(res.AllImages[0].Source, ex.LocalStorageFQDN) { t.Fatalf("source images should be from local storage") } // must contain 4 release component images @@ -191,7 +191,7 @@ func TestReleaseLocalStoredCollector(t *testing.T) { Type: v2alpha1.TypeCincinnatiGraph, }, } - assert.ElementsMatch(t, expected, res) + assert.ElementsMatch(t, expected, res.AllImages) log.Debug("completed test related images %v ", res) }) @@ -218,10 +218,10 @@ func TestReleaseLocalStoredCollector(t *testing.T) { if err != nil { t.Fatalf("should not fail: %v", err) } - if len(res) == 0 { + if len(res.AllImages) == 0 { t.Fatalf("should contain at least 1 image") } - if !strings.Contains(res[0].Source, ex.LocalStorageFQDN) { + if !strings.Contains(res.AllImages[0].Source, ex.LocalStorageFQDN) { t.Fatalf("source images should be from local storage") } log.Debug("completed test related images %v ", res) From e126cb8dd00fbb515da4fc25797c2bd76666c9fb Mon Sep 17 00:00:00 2001 From: Alex Guidi Date: Wed, 8 Jul 2026 12:11:21 +0200 Subject: [PATCH 03/10] fixup! feat: Add sparse manifest platform filtering for multi-arch images fix: Address code review suggestions from adolfo-ab and coderabbitai - Add Validate() to InstancePlatformFilter to reject entries with empty OS or Architecture, preventing malformed strings like "linux/" or "/amd64" - Update ConvertPlatformsToStringSlice to skip invalid entries rather than producing malformed platform strings - Add deprecation warning log when the Architectures field is used, directing users to switch to the new platform.platforms field --- internal/pkg/api/v2alpha1/type_platform.go | 25 ++++++++++++++++--- .../pkg/release/local_stored_collector.go | 1 + 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/internal/pkg/api/v2alpha1/type_platform.go b/internal/pkg/api/v2alpha1/type_platform.go index 188da03d1..1d8e4159e 100644 --- a/internal/pkg/api/v2alpha1/type_platform.go +++ b/internal/pkg/api/v2alpha1/type_platform.go @@ -3,6 +3,7 @@ package v2alpha1 import ( "encoding/json" "errors" + "fmt" ) // DefaultPlatformArchitecture defines the default @@ -76,16 +77,34 @@ func (p InstancePlatformFilter) String() string { return p.OS + "/" + p.Architecture } +// Validate returns an error if OS or Architecture is empty. +func (p InstancePlatformFilter) Validate() error { + if p.OS == "" { + return fmt.Errorf("platform OS must not be empty (architecture: %q)", p.Architecture) + } + if p.Architecture == "" { + return fmt.Errorf("platform architecture must not be empty (OS: %q)", p.OS) + } + return nil +} + // ConvertPlatformsToStringSlice converts a slice of InstancePlatformFilter to a slice // of platform strings in "os/architecture" format. Returns nil if the input is empty. +// Entries with an empty OS or Architecture are skipped. func ConvertPlatformsToStringSlice(platforms []InstancePlatformFilter) []string { if len(platforms) == 0 { return nil } - platformStrs := make([]string, len(platforms)) - for i, p := range platforms { - platformStrs[i] = p.String() + platformStrs := make([]string, 0, len(platforms)) + for _, p := range platforms { + if p.Validate() != nil { + continue + } + platformStrs = append(platformStrs, p.String()) + } + if len(platformStrs) == 0 { + return nil } return platformStrs } diff --git a/internal/pkg/release/local_stored_collector.go b/internal/pkg/release/local_stored_collector.go index 50d8c3d66..2a1dcb89d 100644 --- a/internal/pkg/release/local_stored_collector.go +++ b/internal/pkg/release/local_stored_collector.go @@ -62,6 +62,7 @@ func (o LocalStorageCollector) getPlatformFilters() []string { // Fall back to deprecated Architectures field (assume linux) //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility if len(o.Config.Mirror.Platform.Architectures) > 0 { + o.Log.Warn("platform.architectures is deprecated; use platform.platforms instead for fine-grained OS/architecture control") //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility platformStrs := make([]string, len(o.Config.Mirror.Platform.Architectures)) //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility From 729ab1a6f76465091b0190502f177390c0e9af23 Mon Sep 17 00:00:00 2001 From: Alex Guidi Date: Wed, 8 Jul 2026 12:44:26 +0200 Subject: [PATCH 04/10] fixup! feat: Add sparse manifest platform filtering for multi-arch images fix: Reduce cyclomatic complexity in collector functions to pass linter Extract helpers to bring three functions within the cyclop threshold (max 10): - ReleaseImageCollector: extract compareByOriginSourceDest sort helper - AdditionalImagesCollector: extract resolveAdditionalImageSrcDest and resolveTargetRepoTag helpers - HelmImageCollector: extract collectHelmImagesM2D and collectHelmImagesD2M helpers, reducing the main function to a simple mode dispatch --- .../pkg/additional/local_stored_collector.go | 116 +++++----- internal/pkg/helm/local_stored_collector.go | 215 ++++++++---------- .../pkg/release/local_stored_collector.go | 25 +- 3 files changed, 175 insertions(+), 181 deletions(-) diff --git a/internal/pkg/additional/local_stored_collector.go b/internal/pkg/additional/local_stored_collector.go index 635257d3a..c905f9c93 100644 --- a/internal/pkg/additional/local_stored_collector.go +++ b/internal/pkg/additional/local_stored_collector.go @@ -48,70 +48,17 @@ func (o LocalStorageCollector) destinationRegistry() string { // taking into account the mode we are in (mirrorToDisk, diskToMirror) // the image is downloaded in oci format func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { - var allImages []v2alpha1.CopyImageSchema var allErrs []error platformFilters := make(map[string][]string) o.Log.Debug(collectorPrefix+"setting copy option o.Opts.MultiArch=%s when collecting releases image", o.Opts.MultiArch) for _, img := range o.Config.ImageSetConfigurationSpec.Mirror.AdditionalImages { - var src, dest, tmpSrc, tmpDest string - - origin := img.Name - - imgSpec, err := image.ParseRef(img.Name) + src, dest, origin, err := o.resolveAdditionalImageSrcDest(img) if err != nil { - // OCPBUGS-33081 - skip if parse error (i.e semver and other) - o.Log.Warn("%v : SKIPPING", err) - allErrs = append(allErrs, fmt.Errorf("parse image %q: %w", img.Name, err)) - continue - } - - if img.TargetRepo != "" && !v2alpha1.IsValidPathComponent(img.TargetRepo) { - o.Log.Warn("invalid targetRepo %s for image %s : SKIPPING", img.TargetRepo, img.Name) - allErrs = append(allErrs, fmt.Errorf("invalid targetRepo %s for image %s", img.TargetRepo, img.Name)) + allErrs = append(allErrs, err) continue } - - targetRepo := imgSpec.PathComponent - if img.TargetRepo != "" { - targetRepo = img.TargetRepo - } - - targetTag := imgSpec.Tag - if img.TargetTag != "" { - targetTag = img.TargetTag - } - - switch { - case o.Opts.IsMirrorToDisk(), o.Opts.IsMirrorToMirror(): - tmpSrc, tmpDest = o.buildMirrorToDiskPaths(img, imgSpec, targetRepo, targetTag) - case o.Opts.IsDiskToMirror(), o.Opts.IsDelete(): - tmpSrc, tmpDest = o.buildDiskToMirrorPaths(img, imgSpec, targetRepo, targetTag) - } - - if tmpSrc == "" || tmpDest == "" { - o.Log.Error(collectorPrefix+"unable to determine src %s or dst %s for %s", tmpSrc, tmpDest, img.Name) - allErrs = append(allErrs, fmt.Errorf("unable to determine src %s or dst %s for %s", tmpSrc, tmpDest, img.Name)) - continue - } - - srcSpec, err := image.ParseRef(tmpSrc) // makes sure this ref is valid, and adds transport if needed - if err != nil { - o.Log.Error(errMsg, err.Error()) - allErrs = append(allErrs, fmt.Errorf("parse source %q: %w", tmpSrc, err)) - continue - } - src = srcSpec.ReferenceWithTransport - - destSpec, err := image.ParseRef(tmpDest) // makes sure this ref is valid, and adds transport if needed - if err != nil { - o.Log.Error(errMsg, err.Error()) - allErrs = append(allErrs, fmt.Errorf("parse destination %q: %w", tmpDest, err)) - continue - } - dest = destSpec.ReferenceWithTransport - o.Log.Debug(collectorPrefix+"source %s", src) o.Log.Debug(collectorPrefix+"destination %s", dest) @@ -132,6 +79,51 @@ func (o LocalStorageCollector) AdditionalImagesCollector(ctx context.Context) (v return cs, errors.Join(allErrs...) } +// resolveAdditionalImageSrcDest validates an additional image entry and returns its +// resolved source, destination and origin references ready for mirroring. +func (o LocalStorageCollector) resolveAdditionalImageSrcDest(img v2alpha1.AdditionalImage) (src, dest, origin string, err error) { + origin = img.Name + + imgSpec, err := image.ParseRef(img.Name) + if err != nil { + // OCPBUGS-33081 - skip if parse error (i.e semver and other) + o.Log.Warn("%v : SKIPPING", err) + return "", "", "", fmt.Errorf("parse image %q: %w", img.Name, err) + } + + if img.TargetRepo != "" && !v2alpha1.IsValidPathComponent(img.TargetRepo) { + o.Log.Warn("invalid targetRepo %s for image %s : SKIPPING", img.TargetRepo, img.Name) + return "", "", "", fmt.Errorf("invalid targetRepo %s for image %s", img.TargetRepo, img.Name) + } + + targetRepo, targetTag := resolveTargetRepoTag(img, imgSpec) + + var tmpSrc, tmpDest string + switch { + case o.Opts.IsMirrorToDisk(), o.Opts.IsMirrorToMirror(): + tmpSrc, tmpDest = o.buildMirrorToDiskPaths(img, imgSpec, targetRepo, targetTag) + case o.Opts.IsDiskToMirror(), o.Opts.IsDelete(): + tmpSrc, tmpDest = o.buildDiskToMirrorPaths(img, imgSpec, targetRepo, targetTag) + } + + if tmpSrc == "" || tmpDest == "" { + o.Log.Error(collectorPrefix+"unable to determine src %s or dst %s for %s", tmpSrc, tmpDest, img.Name) + return "", "", "", fmt.Errorf("unable to determine src %s or dst %s for %s", tmpSrc, tmpDest, img.Name) + } + + srcSpec, err := image.ParseRef(tmpSrc) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return "", "", "", fmt.Errorf("parse source %q: %w", tmpSrc, err) + } + destSpec, err := image.ParseRef(tmpDest) + if err != nil { + o.Log.Error(errMsg, err.Error()) + return "", "", "", fmt.Errorf("parse destination %q: %w", tmpDest, err) + } + return srcSpec.ReferenceWithTransport, destSpec.ReferenceWithTransport, origin, nil +} + // buildMirrorToDiskPaths constructs source and destination paths for mirror-to-disk and mirror-to-mirror operations func (o LocalStorageCollector) buildMirrorToDiskPaths(img v2alpha1.AdditionalImage, imgSpec image.ImageSpec, targetRepo, targetTag string) (string, string) { tmpSrc := imgSpec.ReferenceWithTransport @@ -201,3 +193,17 @@ func (o LocalStorageCollector) buildDiskToMirrorPaths(img v2alpha1.AdditionalIma return tmpSrc, tmpDest } + +// resolveTargetRepoTag returns the effective target repository and tag for an additional image, +// applying any overrides from the image configuration. +func resolveTargetRepoTag(img v2alpha1.AdditionalImage, imgSpec image.ImageSpec) (string, string) { + targetRepo := imgSpec.PathComponent + if img.TargetRepo != "" { + targetRepo = img.TargetRepo + } + targetTag := imgSpec.Tag + if img.TargetTag != "" { + targetTag = img.TargetTag + } + return targetRepo, targetTag +} diff --git a/internal/pkg/helm/local_stored_collector.go b/internal/pkg/helm/local_stored_collector.go index b7dfa3109..cdac8a6fb 100644 --- a/internal/pkg/helm/local_stored_collector.go +++ b/internal/pkg/helm/local_stored_collector.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io/fs" + "maps" "net/http" "os" "path/filepath" @@ -79,147 +80,131 @@ func WithV1Tags(o CollectorInterface) CollectorInterface { } func (o *LocalStorageCollector) HelmImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { - var ( - allImages []v2alpha1.CopyImageSchema - allHelmImages []v2alpha1.RelatedImage - errs []error - ) - platformFilters := make(map[string][]string) + var allImages []v2alpha1.CopyImageSchema + var platformFilters map[string][]string + var errs []error switch { case lsc.Opts.IsMirrorToDisk() || lsc.Opts.IsMirrorToMirror(): defer lsc.cleanup() + allImages, platformFilters, errs = lsc.collectHelmImagesM2D() + case lsc.Opts.IsDiskToMirror(): + allImages, platformFilters, errs = lsc.collectHelmImagesD2M(o.generateV1DestTags) + } - var err error - imgs, localPlatforms, errors := getHelmImagesFromLocalChart() - if len(errors) > 0 { - errs = append(errs, errors...) + cs := v2alpha1.CollectorSchema{AllImages: allImages} + if len(platformFilters) > 0 { + cs.PlatformFilters = platformFilters + } + return cs, errors.Join(errs...) +} + +func (lsc *LocalStorageCollector) collectHelmImagesM2D() ([]v2alpha1.CopyImageSchema, map[string][]string, []error) { //nolint:cyclop + var allHelmImages []v2alpha1.RelatedImage + var errs []error + platformFilters := make(map[string][]string) + + imgs, localPlatforms, errors := getHelmImagesFromLocalChart() + errs = append(errs, errors...) + if len(imgs) > 0 { + allHelmImages = append(allHelmImages, imgs...) + maps.Copy(platformFilters, localPlatforms) + } + + for _, repo := range lsc.Config.Mirror.Helm.Repositories { + charts := repo.Charts + if err := repoAdd(repo); err != nil { + errs = append(errs, err) + continue } - if len(imgs) > 0 { - allHelmImages = append(allHelmImages, imgs...) - for k, v := range localPlatforms { - platformFilters[k] = v + if charts == nil { + var indexFile helmrepo.IndexFile + var err error + if indexFile, err = createIndexFile(repo.URL); err != nil { + errs = append(errs, err) + continue + } + if charts, err = getChartsFromIndex("", indexFile); err != nil && charts == nil { + errs = append(errs, err) + continue } } - - for _, repo := range lsc.Config.Mirror.Helm.Repositories { - charts := repo.Charts - - if err := repoAdd(repo); err != nil { + for _, chart := range charts { + lsc.Log.Debug("Pulling chart %s", chart.Name) + ref := fmt.Sprintf("%s/%s", repo.Name, chart.Name) + dest := filepath.Join(lsc.Opts.Global.WorkingDir, helmDir, helmChartDir) + path, _, err := lsc.Downloaders.chartDownloader.DownloadTo(ref, chart.Version, dest) + if err != nil { errs = append(errs, err) + lsc.Log.Error("error pulling chart %s:%s", ref, err.Error()) continue } - - if charts == nil { - var indexFile helmrepo.IndexFile - if indexFile, err = createIndexFile(repo.URL); err != nil { - errs = append(errs, err) - continue - } - - if charts, err = getChartsFromIndex("", indexFile); err != nil && charts == nil { - errs = append(errs, err) - continue - } + platforms := v2alpha1.ConvertPlatformsToStringSlice(chart.Platforms) + chartImgs, err := getImages(path, chart.ImagePaths...) + if err != nil { + errs = append(errs, err) } - - for _, chart := range charts { - lsc.Log.Debug("Pulling chart %s", chart.Name) - ref := fmt.Sprintf("%s/%s", repo.Name, chart.Name) - dest := filepath.Join(lsc.Opts.Global.WorkingDir, helmDir, helmChartDir) - path, _, err := lsc.Downloaders.chartDownloader.DownloadTo(ref, chart.Version, dest) - if err != nil { - errs = append(errs, err) - lsc.Log.Error("error pulling chart %s:%s", ref, err.Error()) - continue - } - - platforms := v2alpha1.ConvertPlatformsToStringSlice(chart.Platforms) - - imgs, err := getImages(path, chart.ImagePaths...) - if err != nil { - errs = append(errs, err) - } - - allHelmImages = append(allHelmImages, imgs...) - if len(platforms) > 0 { - for _, img := range imgs { - platformFilters[img.Image] = append(platformFilters[img.Image], platforms...) - } - } - + allHelmImages = append(allHelmImages, chartImgs...) + for _, img := range chartImgs { + platformFilters[img.Image] = append(platformFilters[img.Image], platforms...) } } + } - allImages, err = prepareM2DCopyBatch(allHelmImages) - if err != nil { - lsc.Log.Error(errMsg, err.Error()) - errs = append(errs, err) - } - - case lsc.Opts.IsDiskToMirror(): - imgs, localPlatforms, errors := getHelmImagesFromLocalChart() - if len(errors) > 0 { - errs = append(errs, errors...) - } - if len(imgs) > 0 { - allHelmImages = append(allHelmImages, imgs...) - for k, v := range localPlatforms { - platformFilters[k] = v - } - } + allImages, err := prepareM2DCopyBatch(allHelmImages) + if err != nil { + lsc.Log.Error(errMsg, err.Error()) + errs = append(errs, err) + } + return allImages, platformFilters, errs +} - for _, repo := range lsc.Config.Mirror.Helm.Repositories { - charts := repo.Charts +func (lsc *LocalStorageCollector) collectHelmImagesD2M(generateV1Tags bool) ([]v2alpha1.CopyImageSchema, map[string][]string, []error) { + var allHelmImages []v2alpha1.RelatedImage + var errs []error + platformFilters := make(map[string][]string) - if charts == nil { - var err error - if charts, err = getChartsFromIndex(repo.URL, helmrepo.IndexFile{}); err != nil { - errs = append(errs, err) - if charts == nil { - continue - } - } - } + imgs, localPlatforms, errors := getHelmImagesFromLocalChart() + errs = append(errs, errors...) + allHelmImages = append(allHelmImages, imgs...) + maps.Copy(platformFilters, localPlatforms) - for _, chart := range charts { - chartDir := filepath.Join(lsc.Opts.Global.WorkingDir, helmDir, helmChartDir) - path, err := resolveChartPath(chartDir, chart.Name, chart.Version) - if err != nil { - errs = append(errs, err) + for _, repo := range lsc.Config.Mirror.Helm.Repositories { + charts := repo.Charts + if charts == nil { + var err error + if charts, err = getChartsFromIndex(repo.URL, helmrepo.IndexFile{}); err != nil { + errs = append(errs, err) + if charts == nil { continue } - - platforms := v2alpha1.ConvertPlatformsToStringSlice(chart.Platforms) - - imgs, err := getImages(path, chart.ImagePaths...) - if err != nil { - errs = append(errs, err) - } - - allHelmImages = append(allHelmImages, imgs...) - if len(platforms) > 0 { - for _, img := range imgs { - platformFilters[img.Image] = append(platformFilters[img.Image], platforms...) - } - } - } } - - var err error - allImages, err = prepareD2MCopyBatch(allHelmImages, o.generateV1DestTags) - if err != nil { - lsc.Log.Error(errMsg, err.Error()) - errs = append(errs, err) + for _, chart := range charts { + chartDir := filepath.Join(lsc.Opts.Global.WorkingDir, helmDir, helmChartDir) + path, err := resolveChartPath(chartDir, chart.Name, chart.Version) + if err != nil { + errs = append(errs, err) + continue + } + platforms := v2alpha1.ConvertPlatformsToStringSlice(chart.Platforms) + chartImgs, err := getImages(path, chart.ImagePaths...) + if err != nil { + errs = append(errs, err) + } + allHelmImages = append(allHelmImages, chartImgs...) + for _, img := range chartImgs { + platformFilters[img.Image] = append(platformFilters[img.Image], platforms...) + } } } - cs := v2alpha1.CollectorSchema{AllImages: allImages} - if len(platformFilters) > 0 { - cs.PlatformFilters = platformFilters + allImages, err := prepareD2MCopyBatch(allHelmImages, generateV1Tags) + if err != nil { + lsc.Log.Error(errMsg, err.Error()) + errs = append(errs, err) } - return cs, errors.Join(errs...) + return allImages, platformFilters, errs } func createTempFile(dir string) (func(), string, error) { diff --git a/internal/pkg/release/local_stored_collector.go b/internal/pkg/release/local_stored_collector.go index 2a1dcb89d..1725182f1 100644 --- a/internal/pkg/release/local_stored_collector.go +++ b/internal/pkg/release/local_stored_collector.go @@ -101,17 +101,7 @@ func (o *LocalStorageCollector) ReleaseImageCollector(ctx context.Context) (v2al } // OCPBUGS-43275: deduplicating - slices.SortFunc(allImages, func(a, b v2alpha1.CopyImageSchema) int { - cmp := strings.Compare(a.Origin, b.Origin) - if cmp == 0 { - cmp = strings.Compare(a.Source, b.Source) - } - if cmp == 0 { // this comparison is important because the same digest can be used - // several times in image-references for different components - cmp = strings.Compare(a.Destination, b.Destination) - } - return cmp - }) + slices.SortFunc(allImages, compareByOriginSourceDest) allImages = slices.Compact(allImages) // TODO probably does not need to loop all the images, it would be possible to check the image type, if it is from a release, apply always the same os/arch saved somewhere else. @@ -645,6 +635,19 @@ func (o LocalStorageCollector) handleGraphImage(ctx context.Context) (v2alpha1.C } } +// compareByOriginSourceDest sorts CopyImageSchema entries deterministically. +// The same digest can appear several times in image-references for different +// components, so all three fields are compared to produce a stable order. +func compareByOriginSourceDest(a, b v2alpha1.CopyImageSchema) int { + if cmp := strings.Compare(a.Origin, b.Origin); cmp != 0 { + return cmp + } + if cmp := strings.Compare(a.Source, b.Source); cmp != 0 { + return cmp + } + return strings.Compare(a.Destination, b.Destination) +} + func preparePathComponents(imgSpec image.ImageSpec, imgType v2alpha1.ImageType, imgName string) string { pathComponents := "" switch { From 90b178ba7f0f2da54e51bedafc8c1c194a2afe8e Mon Sep 17 00:00:00 2001 From: Alex Guidi Date: Wed, 8 Jul 2026 15:37:05 +0200 Subject: [PATCH 05/10] fixup! feat: Add sparse manifest platform filtering for multi-arch images fix: Enable sparse manifest support in the local cache registry Add validation.manifests.indexes.platforms: none to the local cache registry configuration so it matches the behavior of a target registry configured for sparse manifests. Without this setting, the distribution registry validates that all blobs referenced in a manifest list are present before accepting the push, causing "blob unknown to registry" errors when platform filtering leaves some architecture blobs uncopied. --- internal/pkg/cli/executor.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/pkg/cli/executor.go b/internal/pkg/cli/executor.go index 993d5afd6..acba9a745 100644 --- a/internal/pkg/cli/executor.go +++ b/internal/pkg/cli/executor.go @@ -671,6 +671,10 @@ http: #htpasswd: #realm: basic-realm #path: /etc/registry +validation: + manifests: + indexes: + platforms: none ` var buff bytes.Buffer From 3755f43f3591df0e088d11ca7bc1d40b87a791a4 Mon Sep 17 00:00:00 2001 From: Alex Guidi Date: Thu, 9 Jul 2026 17:31:20 +0200 Subject: [PATCH 06/10] fixup! feat: Add sparse manifest platform filtering for multi-arch images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor release platform handling to preserve backward compatibility while enabling sparse manifest filtering for users who adopt the new Platforms field. Cincinnati GetReleaseReferenceImages now branches on Platform.Platforms vs Platform.Architectures: - Platforms set: always queries Cincinnati with ?arch=multi (the only manifest list payload) and warns that the target registry must support sparse manifest lists (validation.manifests.indexes.platforms: none). - Architectures set (or defaulted to ["amd64"] by Complete()): preserves the original per-arch Cincinnati loop unchanged. The two fields are mutually exclusive — a new validateReleasePlatformFields check in Validate() returns an error if both are specified. defaults.go no longer auto-fills Architectures when Platforms is already set, preventing spurious defaults from being added alongside an explicit Platforms configuration. ReleaseImageCollector sets MultiArch="system" by default (original behavior) and overrides to "all" only when Platforms is configured, so that ensureReleaseInOCIFormat downloads all arch payloads from the multi-arch release payload and can read image-references with manifest-list component digests. RemoveSignatures=true is restored in ensureReleaseInOCIFormat: converting docker to OCI format changes the image digest, which invalidates any attached signatures. Since this copy is for metadata extraction only, signatures are not needed and keeping them would cause downstream failures. --- internal/pkg/config/defaults.go | 11 +- internal/pkg/config/validate.go | 15 +- internal/pkg/release/cincinnati.go | 236 ++++++++++-------- .../pkg/release/local_stored_collector.go | 10 +- 4 files changed, 164 insertions(+), 108 deletions(-) diff --git a/internal/pkg/config/defaults.go b/internal/pkg/config/defaults.go index 7f985c9e5..671d0c0e7 100644 --- a/internal/pkg/config/defaults.go +++ b/internal/pkg/config/defaults.go @@ -4,14 +4,19 @@ import ( "github.com/openshift/oc-mirror/v2/internal/pkg/api/v2alpha1" ) -// Complete set default values in the ImageSetConfiguration -// when applicable +// Complete sets default values in the ImageSetConfiguration when applicable. func Complete(cfg *v2alpha1.ImageSetConfiguration) { completeReleaseArchitectures(cfg) } func completeReleaseArchitectures(cfg *v2alpha1.ImageSetConfiguration) { - if len(cfg.Mirror.Platform.Channels) != 0 && len(cfg.Mirror.Platform.Architectures) == 0 { + // Only auto-fill the default architecture when neither Platforms nor Architectures is set. + // If Platforms is set, the Platforms branch handles everything; do not pollute Architectures. + //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility + if len(cfg.Mirror.Platform.Channels) != 0 && + len(cfg.Mirror.Platform.Architectures) == 0 && + len(cfg.Mirror.Platform.Platforms) == 0 { + //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility cfg.Mirror.Platform.Architectures = []string{v2alpha1.DefaultPlatformArchitecture} } } diff --git a/internal/pkg/config/validate.go b/internal/pkg/config/validate.go index c71e2fce7..d1d08dba1 100644 --- a/internal/pkg/config/validate.go +++ b/internal/pkg/config/validate.go @@ -17,7 +17,7 @@ type ( ) var ( - validationChecks = []validationFunc{validateOperatorOptions, validateReleaseChannels, validateBlockedImages} + validationChecks = []validationFunc{validateOperatorOptions, validateReleaseChannels, validateBlockedImages, validateReleasePlatformFields} validationDeleteChecks = []validationDeleteFunc{validateOperatorOptionsDelete, validateReleaseChannelsDelete} ) @@ -142,6 +142,19 @@ func validatePackageChannel(ctlgName string, pkg *v2alpha1.IncludePackage, ch *v return nil } +// validateReleasePlatformFields ensures that platform.platforms and platform.architectures +// are not used together — they are mutually exclusive. +func validateReleasePlatformFields(cfg *v2alpha1.ImageSetConfiguration) []error { + //nolint:staticcheck // SA1019: Architectures is deprecated but we check it for mutual exclusivity + if len(cfg.Mirror.Platform.Platforms) > 0 && len(cfg.Mirror.Platform.Architectures) > 0 { + return []error{fmt.Errorf( + "platform.platforms and platform.architectures cannot be used together; " + + "use platform.platforms for sparse manifest filtering (requires disabling manifest list blob checks on container registry side), or platform.architectures (deprecated) for OCP multi payload or single arch payloads", + )} + } + return nil +} + func validateReleaseChannels(cfg *v2alpha1.ImageSetConfiguration) []error { channels := sets.New[string]() for _, channel := range cfg.Mirror.Platform.Channels { diff --git a/internal/pkg/release/cincinnati.go b/internal/pkg/release/cincinnati.go index 0b29eef16..d2c9233fb 100644 --- a/internal/pkg/release/cincinnati.go +++ b/internal/pkg/release/cincinnati.go @@ -159,109 +159,28 @@ func (o *CincinnatiSchema) GetReleaseReferenceImages(ctx context.Context) ([]v2a filterCopy := o.Config.Mirror.Platform.DeepCopy() - for _, arch := range filterCopy.Architectures { - cincinnatiParams.Arch = arch + if len(filterCopy.Platforms) > 0 { + // New Platforms field: always use the multi-arch payload. It is the only manifest + // list, enabling sparse manifest platform filtering. Requires the target registry + // to support sparse manifest lists (e.g. validation.manifests.indexes.platforms: none). + o.Log.Warn("Platform filtering with the Platforms field requires a registry with manifest list " + + "blob check disabled (e.g. validation.manifests.indexes.platforms: none). " + + "Ensure your target registry supports sparse manifest lists.") + cincinnatiParams.Arch = "multi" o.CincinnatiParams = cincinnatiParams - versionsByChannel := make(map[string]v2alpha1.ReleaseChannel, len(filterCopy.Channels)) - for _, ch := range filterCopy.Channels { - var err error - switch ch.Type { - case v2alpha1.TypeOCP: - err = o.NewOCPClient() - if err != nil { - errs = append(errs, err) - } - case v2alpha1.TypeOKD: - err = o.NewOKDClient() - if err != nil { - errs = append(errs, err) - } - default: - errs = append(errs, fmt.Errorf("invalid platform type %v", ch.Type)) - continue - } - if err != nil { - errs = append(errs, err) - continue - } - - // CLID-135 - // detect and log as early as possible - if len(ch.MaxVersion) > 0 && len(ch.MinVersion) > 0 { - max := semver.MustParse(ch.MaxVersion) - min := semver.MustParse(ch.MinVersion) - if strings.Contains(ch.Name, "eus") && ((max.Minor - min.Minor) >= 2) && !flagReport { - msg := "Extended Update Support (EUS) channel detected with minor version range >= 2\n" + - "\t\t\t\tPlease refer to the web console https://access.redhat.com/labs/ocpupgradegraph/update_path\n" + - "\t\t\t\tTo correctly determine the upgrade path for EUS releases" - flagReport = true - o.Log.Warn(msg) - } - } - - if len(ch.MaxVersion) == 0 || len(ch.MinVersion) == 0 { - // Find channel maximum value and only set the minimum as well if heads-only is true - if len(ch.MaxVersion) == 0 { - latest, err := GetChannelMinOrMax(ctx, *o, ch.Name, false) - if err != nil { - errs = append(errs, err) - continue - } - - // Update version to release channel - ch.MaxVersion = latest.String() - o.Log.Debug("detected minimum version as %s", ch.MaxVersion) - if len(ch.MinVersion) == 0 && ch.IsHeadsOnly() { - min := latest.String() - ch.MinVersion = min - o.Log.Debug("detected minimum version as %s\n", ch.MinVersion) - } - } - - // Find channel minimum if full is true or just the minimum is not set - // in the config - if len(ch.MinVersion) == 0 { - first, err := GetChannelMinOrMax(ctx, *o, ch.Name, true) - if err != nil { - errs = append(errs, err) - continue - } - ch.MinVersion = first.String() - o.Log.Debug("detected minimum version as %s\n", ch.MinVersion) - } - versionsByChannel[ch.Name] = ch - } else { - // Range is set. Ensure full is true so this - // is skipped when processing release metadata. - o.Log.Debug("processing minimum version %s and maximum version %s", ch.MinVersion, ch.MaxVersion) - ch.Full = true - versionsByChannel[ch.Name] = ch - } - - downloads, err := getChannelDownloads(ctx, *o, nil, ch) - if err != nil { - errs = append(errs, err) - continue - } - allImages = append(allImages, downloads...) - } - - // Update cfg release channels with maximum and minimum versions - // if applicable - for i, ch := range filterCopy.Channels { - ch, found := versionsByChannel[ch.Name] - if found { - filterCopy.Channels[i] = ch - } - } - - if len(filterCopy.Channels) > 1 { - newDownloads, err := getCrossChannelDownloads(ctx, *o, filterCopy.Channels) - if err != nil { - errs = append(errs, fmt.Errorf("[GetReleaseReferenceImages] error calculating cross channel upgrades: %w", err)) - continue - } - allImages = append(allImages, newDownloads...) + imgs, channelErrs := o.collectChannelImages(ctx, filterCopy.Channels, &flagReport) + allImages = append(allImages, imgs...) + errs = append(errs, channelErrs...) + } else { + // Deprecated Architectures field: original per-arch loop. + // Each arch results in a separate Cincinnati call and a separate release payload. + //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility + for _, arch := range filterCopy.Architectures { + cincinnatiParams.Arch = arch + o.CincinnatiParams = cincinnatiParams + imgs, channelErrs := o.collectChannelImages(ctx, filterCopy.Channels, &flagReport) + allImages = append(allImages, imgs...) + errs = append(errs, channelErrs...) } } @@ -285,6 +204,119 @@ func (o *CincinnatiSchema) GetReleaseReferenceImages(ctx context.Context) ([]v2a return imgs, nil } +// collectChannelImages queries the Cincinnati API for all configured channels with +// the arch already set on o.CincinnatiParams, returning collected images and errors. +// The channels slice is updated in place with resolved min/max versions. +// flagReport is shared across calls to ensure the EUS warning is emitted only once. +func (o *CincinnatiSchema) collectChannelImages(ctx context.Context, channels []v2alpha1.ReleaseChannel, flagReport *bool) ([]v2alpha1.CopyImageSchema, []error) { + var allImages []v2alpha1.CopyImageSchema + var errs []error + + versionsByChannel := make(map[string]v2alpha1.ReleaseChannel, len(channels)) + for _, ch := range channels { + if err := o.initCincinnatiClient(ch); err != nil { + errs = append(errs, err) + continue + } + o.warnEUSChannel(ch, flagReport) + + resolved, err := o.resolveChannelVersions(ctx, ch) + if err != nil { + errs = append(errs, err) + continue + } + versionsByChannel[resolved.Name] = resolved + + downloads, err := getChannelDownloads(ctx, *o, nil, resolved) + if err != nil { + errs = append(errs, err) + continue + } + allImages = append(allImages, downloads...) + } + + // Update channels with resolved min/max versions + for i, ch := range channels { + if updated, found := versionsByChannel[ch.Name]; found { + channels[i] = updated + } + } + + if len(channels) > 1 { + newDownloads, err := getCrossChannelDownloads(ctx, *o, channels) + if err != nil { + errs = append(errs, fmt.Errorf("[GetReleaseReferenceImages] error calculating cross channel upgrades: %w", err)) + } else { + allImages = append(allImages, newDownloads...) + } + } + + return allImages, errs +} + +// initCincinnatiClient sets up the OCP or OKD Cincinnati client for the given channel type. +func (o *CincinnatiSchema) initCincinnatiClient(ch v2alpha1.ReleaseChannel) error { + switch ch.Type { + case v2alpha1.TypeOCP: + return o.NewOCPClient() + case v2alpha1.TypeOKD: + return o.NewOKDClient() + default: + return fmt.Errorf("invalid platform type %v", ch.Type) + } +} + +// warnEUSChannel emits a one-time warning when an EUS channel spans two or more minor versions. +func (o *CincinnatiSchema) warnEUSChannel(ch v2alpha1.ReleaseChannel, flagReport *bool) { + if len(ch.MaxVersion) == 0 || len(ch.MinVersion) == 0 || *flagReport { + return + } + // CLID-135: detect and log as early as possible + max := semver.MustParse(ch.MaxVersion) + min := semver.MustParse(ch.MinVersion) + if strings.Contains(ch.Name, "eus") && (max.Minor-min.Minor) >= 2 { + o.Log.Warn("Extended Update Support (EUS) channel detected with minor version range >= 2\n" + + "\t\t\t\tPlease refer to the web console https://access.redhat.com/labs/ocpupgradegraph/update_path\n" + + "\t\t\t\tTo correctly determine the upgrade path for EUS releases") + *flagReport = true + } +} + +// resolveChannelVersions detects and fills in missing min/max version bounds for a channel, +// or marks the channel as full when both are explicitly provided. +func (o *CincinnatiSchema) resolveChannelVersions(ctx context.Context, ch v2alpha1.ReleaseChannel) (v2alpha1.ReleaseChannel, error) { + if len(ch.MaxVersion) > 0 && len(ch.MinVersion) > 0 { + // Range is set. Ensure full is true so this is skipped when processing release metadata. + o.Log.Debug("processing minimum version %s and maximum version %s", ch.MinVersion, ch.MaxVersion) + ch.Full = true + return ch, nil + } + + if len(ch.MaxVersion) == 0 { + latest, err := GetChannelMinOrMax(ctx, *o, ch.Name, false) + if err != nil { + return ch, err + } + ch.MaxVersion = latest.String() + o.Log.Debug("detected maximum version as %s", ch.MaxVersion) + if len(ch.MinVersion) == 0 && ch.IsHeadsOnly() { + ch.MinVersion = latest.String() + o.Log.Debug("detected minimum version as %s\n", ch.MinVersion) + } + } + + if len(ch.MinVersion) == 0 { + first, err := GetChannelMinOrMax(ctx, *o, ch.Name, true) + if err != nil { + return ch, err + } + ch.MinVersion = first.String() + o.Log.Debug("detected minimum version as %s\n", ch.MinVersion) + } + + return ch, nil +} + // getDownloads will prepare the downloads map for mirroring func getChannelDownloads(ctx context.Context, cs CincinnatiSchema, lastChannels []v2alpha1.ReleaseChannel, channel v2alpha1.ReleaseChannel) ([]v2alpha1.CopyImageSchema, error) { var allImages []v2alpha1.CopyImageSchema diff --git a/internal/pkg/release/local_stored_collector.go b/internal/pkg/release/local_stored_collector.go index 1725182f1..067f279b4 100644 --- a/internal/pkg/release/local_stored_collector.go +++ b/internal/pkg/release/local_stored_collector.go @@ -82,6 +82,11 @@ func (o LocalStorageCollector) getPlatformFilters() []string { func (o *LocalStorageCollector) ReleaseImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { // we just care for 1 platform release, in order to read release images o.Opts.MultiArch = "system" + if len(o.Config.Mirror.Platform.Platforms) > 0 { + // New Platforms field: download all arch payloads from the multi-arch release so that + // ensureReleaseInOCIFormat can extract image-references with manifest-list component digests. + o.Opts.MultiArch = "all" + } o.Log.Debug(collectorPrefix+"setting copy option o.Opts.MultiArch=%s when collecting releases image", o.Opts.MultiArch) var err error @@ -238,12 +243,13 @@ func (o *LocalStorageCollector) ensureReleaseInOCIFormat(ctx context.Context, re optsCopy := o.Opts optsCopy.Stdout = io.Discard + // Converting docker→OCI changes the digest, which invalidates existing signatures. + // This copy is for metadata extraction only, so signatures are not needed. + optsCopy.RemoveSignatures = true src := consts.DockerProtocol + release.Source dest := consts.OciProtocolTrimmed + dir - optsCopy.RemoveSignatures = true - if err := o.Mirror.Run(ctx, src, dest, "copy", &optsCopy); err != nil { return fmt.Errorf("copy release index image: %w", err) } From ba4add6f3d25e61d80100c55031eb497fbdb3816 Mon Sep 17 00:00:00 2001 From: Alex Guidi Date: Fri, 10 Jul 2026 15:37:51 +0200 Subject: [PATCH 07/10] fixup! feat: Add sparse manifest platform filtering for multi-arch images Simplify getPlatformFilters(): the Architectures field is only for Cincinnati (?arch=), so the batch worker does not need InstancePlatforms for that path. Returning nil for the Architectures case preserves the original behavior where per-arch single-arch payloads are copied whole without platform filtering. Move the Architectures deprecation warning from getPlatformFilters() to the Cincinnati Architectures branch in GetReleaseReferenceImages, where the field is actually consumed. --- internal/pkg/release/cincinnati.go | 1 + .../pkg/release/local_stored_collector.go | 30 +++---------------- 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/internal/pkg/release/cincinnati.go b/internal/pkg/release/cincinnati.go index d2c9233fb..71f4f0960 100644 --- a/internal/pkg/release/cincinnati.go +++ b/internal/pkg/release/cincinnati.go @@ -174,6 +174,7 @@ func (o *CincinnatiSchema) GetReleaseReferenceImages(ctx context.Context) ([]v2a } else { // Deprecated Architectures field: original per-arch loop. // Each arch results in a separate Cincinnati call and a separate release payload. + o.Log.Warn("platform.architectures is deprecated; use platform.platforms instead for sparse manifest platform filtering") //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility for _, arch := range filterCopy.Architectures { cincinnatiParams.Arch = arch diff --git a/internal/pkg/release/local_stored_collector.go b/internal/pkg/release/local_stored_collector.go index 067f279b4..6582a9f47 100644 --- a/internal/pkg/release/local_stored_collector.go +++ b/internal/pkg/release/local_stored_collector.go @@ -50,33 +50,11 @@ func (o LocalStorageCollector) destinationRegistry() string { return o.destReg } -// getPlatformFilters converts Platform configuration to platform filter strings. -// It prioritizes the new Platforms field over the deprecated Architectures field. -// Returns nil if no platforms are specified (meaning mirror all platforms). +// getPlatformFilters returns the platform filter strings for the new Platforms field. +// Returns nil when Platforms is not set — the Architectures path uses Cincinnati +// directly and does not need InstancePlatforms in the batch worker. func (o LocalStorageCollector) getPlatformFilters() []string { - // Use new Platforms field if available - if len(o.Config.Mirror.Platform.Platforms) > 0 { - return v2alpha1.ConvertPlatformsToStringSlice(o.Config.Mirror.Platform.Platforms) - } - - // Fall back to deprecated Architectures field (assume linux) - //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility - if len(o.Config.Mirror.Platform.Architectures) > 0 { - o.Log.Warn("platform.architectures is deprecated; use platform.platforms instead for fine-grained OS/architecture control") - //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility - platformStrs := make([]string, len(o.Config.Mirror.Platform.Architectures)) - //nolint:staticcheck // SA1019: Architectures is deprecated but we maintain backward compatibility - for i, arch := range o.Config.Mirror.Platform.Architectures { - if arch == "multi" { - continue - } - platformStrs[i] = "linux/" + arch - } - return platformStrs - } - - // No platforms specified - mirror all - return nil + return v2alpha1.ConvertPlatformsToStringSlice(o.Config.Mirror.Platform.Platforms) } func (o *LocalStorageCollector) ReleaseImageCollector(ctx context.Context) (v2alpha1.CollectorSchema, error) { From fd2d08284931d901c7abaa52d662f2c9db30961e Mon Sep 17 00:00:00 2001 From: Alex Guidi Date: Mon, 13 Jul 2026 11:03:10 +0200 Subject: [PATCH 08/10] fixup! feat: Add sparse manifest platform filtering for multi-arch images Fix archive creation failing for M2D when platform filtering is active. When sparse manifest filtering is used, the local cache registry stores a complete manifest list index but only the blobs for the selected platforms. The archive step calls ImageManifest for every digest in the manifest list, including platforms that were never mirrored, which returns "manifest unknown" from the registry and caused the archive build to abort. Fix: skip absent platform manifests in multiArchBlobs with the same "manifest unknown" check already used in mirror.go, logging a debug message and continuing to the next platform. --- internal/pkg/archive/image-blob-gatherer.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/pkg/archive/image-blob-gatherer.go b/internal/pkg/archive/image-blob-gatherer.go index 24bb6c85c..63171d685 100644 --- a/internal/pkg/archive/image-blob-gatherer.go +++ b/internal/pkg/archive/image-blob-gatherer.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" digest "github.com/opencontainers/go-digest" "go.podman.io/image/v5/manifest" @@ -98,6 +99,13 @@ func (o *ImageBlobGatherer) multiArchBlobs(ctx context.Context, in internalImage singleIn.manifestBytes, singleIn.mimeType, err = o.ocmirrormanifest.ImageManifest(ctx, in.sourceCtx, in.imgRef, &digest) if err != nil { + // A sparse manifest list may reference platform digests that were never + // mirrored (e.g. only selected architectures were copied). Skip those + // absent platforms rather than aborting the archive build. + if strings.Contains(err.Error(), "manifest unknown") { + o.log.Debug("Skipping absent platform blob %s for %s (sparse manifest): %s", digest.String(), in.imgRef, err.Error()) + continue + } return nil, err } singleIn.digest = digest From bf107be113c06054484adbfcfa8c99dda9e5e439 Mon Sep 17 00:00:00 2001 From: Alex Guidi Date: Mon, 13 Jul 2026 16:49:46 +0200 Subject: [PATCH 09/10] fixup! feat: Add sparse manifest platform filtering for multi-arch images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix M2D archive creation failing when sparse manifest platform filtering is active. The archive step iterates over every digest in a release manifest list to collect the blobs that need to be packaged. With sparse filtering, the local cache only stores blobs for the configured platforms — other platforms are absent. Two bugs caused the archive build to fail: 1. ImageManifest returned "manifest unknown" for absent platform digests, which was treated as a fatal error instead of an expected condition. 2. The absent platform digest was inserted into the blobs set before the manifest check, so the archive later tried to locate a blob file that was never stored. Fix: skip absent platform digests that are not in the image's allowed platform set (CollectorSchema.PlatformFilters[img.Origin]). The digest is only added to the blobs set after a successful manifest fetch. Platforms that are in the allowed set but return "manifest unknown" still fail — that indicates a genuine error. To make PlatformFilters available to the archive builder without a separate setter or interface change, BuildArchive now accepts a CollectorSchema instead of a plain image slice, and the batch worker propagates PlatformFilters into the returned CollectorSchema. --- internal/pkg/archive/archive.go | 11 +-- internal/pkg/archive/archive_test.go | 12 ++-- internal/pkg/archive/image-blob-gatherer.go | 71 ++++++++++++++----- .../pkg/archive/image-blob-gatherer_test.go | 8 +-- internal/pkg/archive/interface.go | 8 ++- internal/pkg/batch/concurrent_chan_worker.go | 3 +- internal/pkg/cli/executor.go | 2 +- internal/pkg/cli/executor_test.go | 2 +- internal/pkg/delete/delete_images_test.go | 2 +- 9 files changed, 82 insertions(+), 37 deletions(-) diff --git a/internal/pkg/archive/archive.go b/internal/pkg/archive/archive.go index 790e09ba4..801557a2c 100644 --- a/internal/pkg/archive/archive.go +++ b/internal/pkg/archive/archive.go @@ -66,7 +66,7 @@ func NewMirrorArchive(opts *mirror.CopyOptions, destination, iscPath, workingDir // * docker/v2/blobs/sha256 : blobs that haven't been mirrored (diff) // * working-dir // * image set config -func (o *MirrorArchive) BuildArchive(ctx context.Context, collectedImages []v2alpha1.CopyImageSchema) error { +func (o *MirrorArchive) BuildArchive(ctx context.Context, schema v2alpha1.CollectorSchema) error { if err := o.createTarball(); err != nil { return fmt.Errorf("unable to create the mirror archive: %w", err) } @@ -97,7 +97,7 @@ func (o *MirrorArchive) BuildArchive(ctx context.Context, collectedImages []v2al } // ignoring the error otherwise: continuing with an empty map in blobsInHistory - addedBlobs, err := o.addImagesDiff(ctx, collectedImages, blobsInHistory) + addedBlobs, err := o.addImagesDiff(ctx, schema, blobsInHistory) if err != nil { return fmt.Errorf("unable to add image blobs to the archive : %w", err) } @@ -136,10 +136,11 @@ func (o *MirrorArchive) createTarball() error { return nil } -func (o *MirrorArchive) addImagesDiff(ctx context.Context, collectedImages []v2alpha1.CopyImageSchema, historyBlobs sets.Set[string]) (sets.Set[string], error) { +func (o *MirrorArchive) addImagesDiff(ctx context.Context, schema v2alpha1.CollectorSchema, historyBlobs sets.Set[string]) (sets.Set[string], error) { allAddedBlobs := sets.New[string]() - for _, img := range collectedImages { - imgBlobs, err := o.blobGatherer.GatherBlobs(ctx, img.Destination) + for _, img := range schema.AllImages { + allowedPlatforms := schema.PlatformFilters[img.Origin] + imgBlobs, err := o.blobGatherer.GatherBlobs(ctx, img.Destination, allowedPlatforms) var sigErr *SignatureBlobGathererError if err != nil && !errors.As(err, &sigErr) { return nil, fmt.Errorf("unable to find blobs corresponding to %s: %w", img.Destination, err) diff --git a/internal/pkg/archive/archive_test.go b/internal/pkg/archive/archive_test.go index a7adef36b..635a5df42 100644 --- a/internal/pkg/archive/archive_test.go +++ b/internal/pkg/archive/archive_test.go @@ -81,7 +81,7 @@ func TestArchive_BuildArchive(t *testing.T) { Origin: consts.DockerProtocol + "registry.redhat.io/ubi8/ubi:latest", }, } - err = ma.BuildArchive(context.Background(), images) + err = ma.BuildArchive(context.Background(), v2alpha1.CollectorSchema{AllImages: images}) if err != nil { t.Fatal(err) } @@ -104,7 +104,7 @@ func TestArchive_BuildArchive(t *testing.T) { Origin: consts.DockerProtocol + "registry.redhat.io/ubi8/ubi:latest", }, } - err = ma.BuildArchive(context.Background(), images) + err = ma.BuildArchive(context.Background(), v2alpha1.CollectorSchema{AllImages: images}) if err != nil { t.Fatal(err) } @@ -133,7 +133,7 @@ func TestArchive_CacheDirError(t *testing.T) { ma.cacheDir = "none" ma.workingDir = consts.TestFolder + "working-dir-fake" - err = ma.BuildArchive(context.Background(), images) + err = ma.BuildArchive(context.Background(), v2alpha1.CollectorSchema{AllImages: images}) if err == nil { t.Fatal("should fail") } @@ -158,7 +158,7 @@ func TestArchive_WorkingDirError(t *testing.T) { ma.cacheDir = consts.TestFolder + "cache-fake" ma.workingDir = "none" - err = ma.BuildArchive(context.Background(), images) + err = ma.BuildArchive(context.Background(), v2alpha1.CollectorSchema{AllImages: images}) if err == nil { t.Fatal("should fail") } @@ -182,7 +182,7 @@ func TestArchive_FileError(t *testing.T) { // force error for addFile ma.iscPath = "none" - err = ma.BuildArchive(context.Background(), images) + err = ma.BuildArchive(context.Background(), v2alpha1.CollectorSchema{AllImages: images}) if err == nil { t.Fatal("should fail") } @@ -463,7 +463,7 @@ func (ma *MirrorArchive) WithFakes(maxArchiveSize int64) (*MirrorArchive, error) return ma, nil } -func (mbg mockBlobGatherer) GatherBlobs(ctx context.Context, imgRef string) (sets.Set[string], error) { +func (mbg mockBlobGatherer) GatherBlobs(ctx context.Context, imgRef string, allowedPlatforms []string) (sets.Set[string], error) { blobs := sets.New( "sha256:2e39d55595ea56337b5b788e96e6afdec3db09d2759d903cbe120468187c4644", "sha256:94343313ec1512ab02267e4bc3ce09eecb01fda5bf26c56e2f028ecc72e80b18", diff --git a/internal/pkg/archive/image-blob-gatherer.go b/internal/pkg/archive/image-blob-gatherer.go index 63171d685..2646b49c7 100644 --- a/internal/pkg/archive/image-blob-gatherer.go +++ b/internal/pkg/archive/image-blob-gatherer.go @@ -2,6 +2,7 @@ package archive import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -25,12 +26,13 @@ type ImageBlobGatherer struct { } type internalImageBlobGatherer struct { - imgRef string - sourceCtx *types.SystemContext - manifestBytes []byte - mimeType string - digest digest.Digest - copySignatures bool + imgRef string + sourceCtx *types.SystemContext + manifestBytes []byte + mimeType string + digest digest.Digest + copySignatures bool + allowedPlatforms []string // non-empty = only these os/arch pairs were mirrored (sparse) } func NewImageBlobGatherer(opts *mirror.CopyOptions, log clog.PluggableLoggerInterface) *ImageBlobGatherer { @@ -41,8 +43,9 @@ func NewImageBlobGatherer(opts *mirror.CopyOptions, log clog.PluggableLoggerInte } } -// GatherBlobs returns all container image blobs (including signature blobs if they exists). -func (o *ImageBlobGatherer) GatherBlobs(ctx context.Context, imgRef string) (blobs sets.Set[string], retErr error) { +// GatherBlobs returns all container image blobs (including signature blobs if they exist). +// See BlobsGatherer.GatherBlobs for the allowedPlatforms semantics. +func (o *ImageBlobGatherer) GatherBlobs(ctx context.Context, imgRef string, allowedPlatforms []string) (blobs sets.Set[string], retErr error) { // we are always gathering blobs from the local cache registry - skipping tls verification sourceCtx, err := o.opts.SrcImage.NewSystemContext() if err != nil { @@ -60,7 +63,7 @@ func (o *ImageBlobGatherer) GatherBlobs(ctx context.Context, imgRef string) (blo return nil, fmt.Errorf("error to get the digest of the image manifest %w", err) } - inImageBlogGather := internalImageBlobGatherer{imgRef: imgRef, sourceCtx: sourceCtx, manifestBytes: manifestBytes, mimeType: mime, digest: digest, copySignatures: !o.opts.RemoveSignatures} + inImageBlogGather := internalImageBlobGatherer{imgRef: imgRef, sourceCtx: sourceCtx, manifestBytes: manifestBytes, mimeType: mime, digest: digest, copySignatures: !o.opts.RemoveSignatures, allowedPlatforms: allowedPlatforms} if manifest.MIMETypeIsMultiImage(mime) { return o.multiArchBlobs(ctx, inImageBlogGather) @@ -94,20 +97,23 @@ func (o *ImageBlobGatherer) multiArchBlobs(ctx context.Context, in internalImage digests := manifestList.Instances() for _, digest := range digests { - blobs.Insert(digest.String()) singleIn := in singleIn.manifestBytes, singleIn.mimeType, err = o.ocmirrormanifest.ImageManifest(ctx, in.sourceCtx, in.imgRef, &digest) if err != nil { - // A sparse manifest list may reference platform digests that were never - // mirrored (e.g. only selected architectures were copied). Skip those - // absent platforms rather than aborting the archive build. - if strings.Contains(err.Error(), "manifest unknown") { - o.log.Debug("Skipping absent platform blob %s for %s (sparse manifest): %s", digest.String(), in.imgRef, err.Error()) - continue + if strings.Contains(err.Error(), "manifest unknown") && len(in.allowedPlatforms) > 0 { + platform := platformForDigest(in.manifestBytes, digest.String()) + if !platformAllowed(platform, in.allowedPlatforms) { + // Platform was intentionally not mirrored (sparse manifest filtering). + // Do not add its digest to blobs — the blob file does not exist locally. + o.log.Debug("Skipping absent platform %q for %s (not in allowed platforms)", platform, in.imgRef) + continue + } } return nil, err } + // Only insert the platform digest after confirming the manifest is present. + blobs.Insert(digest.String()) singleIn.digest = digest singleArchBlobs, err := o.singleArchBlobs(ctx, singleIn) @@ -196,3 +202,36 @@ func (o *ImageBlobGatherer) imageSignatureBlobs(ctx context.Context, in internal return sigBlobs, nil } + +// platformForDigest returns the "os/arch" string for the manifest list entry +// matching digestStr, or an empty string if not found. +func platformForDigest(manifestBytes []byte, digestStr string) string { + var ml struct { + Manifests []struct { + Digest string `json:"digest"` + Platform struct { + OS string `json:"os"` + Architecture string `json:"architecture"` + } `json:"platform"` + } `json:"manifests"` + } + if err := json.Unmarshal(manifestBytes, &ml); err != nil { + return "" + } + for _, m := range ml.Manifests { + if m.Digest == digestStr { + return m.Platform.OS + "/" + m.Platform.Architecture + } + } + return "" +} + +// platformAllowed reports whether platform is in allowedPlatforms. +func platformAllowed(platform string, allowedPlatforms []string) bool { + for _, allowed := range allowedPlatforms { + if platform == allowed { + return true + } + } + return false +} diff --git a/internal/pkg/archive/image-blob-gatherer_test.go b/internal/pkg/archive/image-blob-gatherer_test.go index 469f3474a..57fba166a 100644 --- a/internal/pkg/archive/image-blob-gatherer_test.go +++ b/internal/pkg/archive/image-blob-gatherer_test.go @@ -221,7 +221,7 @@ func TestImageBlobGatherer_GatherBlobs(t *testing.T) { gatherer := NewImageBlobGatherer(&opts, clog.New("trace")) - blobs, err := gatherer.GatherBlobs(ctx, parentImage.destProtocol+u.Host+parentImage.dest) + blobs, err := gatherer.GatherBlobs(ctx, parentImage.destProtocol+u.Host+parentImage.dest, nil) if tc.expectedErrorType != nil { assert.True(t, reflect.TypeOf(err) == reflect.TypeOf(tc.expectedErrorType)) } else { @@ -257,7 +257,7 @@ func TestImageBlobGatherer_ImgRefError(t *testing.T) { } gatherer := NewImageBlobGatherer(&opts, clog.New("trace")) - _, err := gatherer.GatherBlobs(ctx, "error") + _, err := gatherer.GatherBlobs(ctx, "error", nil) assert.Equal(t, "invalid source name error: Invalid image name \"error\", expected colon-separated transport:reference", err.Error()) } @@ -289,7 +289,7 @@ func TestImageBlobGatherer_SrcContextError(t *testing.T) { } gatherer := NewImageBlobGatherer(&opts, clog.New("trace")) - _, err := gatherer.GatherBlobs(ctx, consts.DockerProtocol+"localhost/test:latest") + _, err := gatherer.GatherBlobs(ctx, consts.DockerProtocol+"localhost/test:latest", nil) assert.Contains(t, err.Error(), "pinging container registry localhost: Get \"http://localhost/v2/\": dial tcp [::1]:80: connect: connection refused") } @@ -328,6 +328,6 @@ func TestImageBlobGatherer_ImageSourceError(t *testing.T) { } gatherer := NewImageBlobGatherer(&opts, clog.New("trace")) - _, err = gatherer.GatherBlobs(ctx, consts.DockerProtocol+u.Host+"/bad-test:latest") + _, err = gatherer.GatherBlobs(ctx, consts.DockerProtocol+u.Host+"/bad-test:latest", nil) assert.Contains(t, err.Error(), "name unknown: Unknown name") } diff --git a/internal/pkg/archive/interface.go b/internal/pkg/archive/interface.go index fcb818e48..6a92d927b 100644 --- a/internal/pkg/archive/interface.go +++ b/internal/pkg/archive/interface.go @@ -9,11 +9,15 @@ import ( ) type BlobsGatherer interface { - GatherBlobs(ctx context.Context, imgRef string) (sets.Set[string], error) + // GatherBlobs returns all blobs for the given image. + // allowedPlatforms is the set of os/arch pairs that were intentionally mirrored + // (from CollectorSchema.PlatformFilters). Missing platforms outside this set are + // skipped; missing platforms inside it are real errors. + GatherBlobs(ctx context.Context, imgRef string, allowedPlatforms []string) (sets.Set[string], error) } type Archiver interface { - BuildArchive(ctx context.Context, collectedImages []v2alpha1.CopyImageSchema) error + BuildArchive(ctx context.Context, schema v2alpha1.CollectorSchema) error } type UnArchiver interface { diff --git a/internal/pkg/batch/concurrent_chan_worker.go b/internal/pkg/batch/concurrent_chan_worker.go index 295525849..a08a93b0b 100644 --- a/internal/pkg/batch/concurrent_chan_worker.go +++ b/internal/pkg/batch/concurrent_chan_worker.go @@ -52,7 +52,8 @@ func (o *ChannelConcurrentBatch) Worker(ctx context.Context, collectorSchema v2a startTime := time.Now() copiedImages := v2alpha1.CollectorSchema{ - AllImages: []v2alpha1.CopyImageSchema{}, + AllImages: []v2alpha1.CopyImageSchema{}, + PlatformFilters: collectorSchema.PlatformFilters, } var errArray []mirrorErrorSchema diff --git a/internal/pkg/cli/executor.go b/internal/pkg/cli/executor.go index acba9a745..49c7fb25d 100644 --- a/internal/pkg/cli/executor.go +++ b/internal/pkg/cli/executor.go @@ -927,7 +927,7 @@ func (o *ExecutorSchema) RunMirrorToDisk(cmd *cobra.Command, args []string) erro } o.Log.Info(emoji.Package + " Preparing the tarball archive...") - return o.MirrorArchiver.BuildArchive(cmd.Context(), copiedSchema.AllImages) + return o.MirrorArchiver.BuildArchive(cmd.Context(), copiedSchema) } // RunMirrorToMirror - execute the mirror to mirror functionality diff --git a/internal/pkg/cli/executor_test.go b/internal/pkg/cli/executor_test.go index 1f6fe4a91..95fb48e68 100644 --- a/internal/pkg/cli/executor_test.go +++ b/internal/pkg/cli/executor_test.go @@ -1290,7 +1290,7 @@ func (o *Collector) HelmImageCollector(ctx context.Context) (v2alpha1.CollectorS return v2alpha1.CollectorSchema{}, nil } -func (o MockArchiver) BuildArchive(ctx context.Context, collectedImages []v2alpha1.CopyImageSchema) error { +func (o MockArchiver) BuildArchive(ctx context.Context, schema v2alpha1.CollectorSchema) error { // return filepath.Join(o.destination, "mirror_000001.tar"), nil return nil } diff --git a/internal/pkg/delete/delete_images_test.go b/internal/pkg/delete/delete_images_test.go index a328c2fe7..807467094 100644 --- a/internal/pkg/delete/delete_images_test.go +++ b/internal/pkg/delete/delete_images_test.go @@ -522,7 +522,7 @@ func (o mockBatch) Worker(ctx context.Context, collectorSchema v2alpha1.Collecto return collectorSchema, nil } -func (o *mockBlobs) GatherBlobs(ctx context.Context, image string) (sets.Set[string], error) { +func (o *mockBlobs) GatherBlobs(ctx context.Context, image string, allowedPlatforms []string) (sets.Set[string], error) { res := sets.New("sha256:95ad8395795ee0460baf05458f669d3b865535f213f015519ef9a221a6a08280") if o.Fail { return nil, fmt.Errorf("forced error") From 9ee5963524ff7fa6244683983c8893dc11439533 Mon Sep 17 00:00:00 2001 From: Alex Guidi Date: Fri, 17 Jul 2026 16:39:25 +0200 Subject: [PATCH 10/10] fixup! feat: Add sparse manifest platform filtering for multi-arch images Add integration tests and multi-arch release image builder infrastructure: - sparse_manifest_test.go: 10 Ginkgo specs covering all 7 manual test scenarios (M2M filtered, M2M backward compat, M2D+D2M filtered, M2D+D2M backward compat, config validation, deprecated architectures + release channel, negative test) - sparse_manifest_helpers_test.go: test helpers for platform blob presence assertions - ISC fixtures: 8 image set config files for the sparse manifest test scenarios - registry-config-sparse.yaml: distribution registry config with platforms:none - create-release-multi.sh: build and push the 4-arch fake OCP release manifest list - generate-release-signature-multi.sh: GPG-sign both single and multi-arch releases with a shared key so a single release-pk.asc validates all signatures - Per-arch OCI image config blobs (arm64, s390x, ppc64le) alongside existing amd64 - config-multi.json: 4-placeholder manifest template for multi-arch builds - release-payload-multi/index.json + oci-layout: committed OCI manifest list - Makefile: add sign-multi cosign target - testdata/keys: updated GPG public key and signatures for both release images - helpers_test.go: copy all signature files in setupWorkDir (not just one) --- tests/integration/helpers_test.go | 10 +- .../image-builders/release/Makefile | 4 + ...e80519b6b431e3c9feff0470aa35a91c87b71feff2 | 48 +++ ...5c4d5a82333ab51937563523acebac831bffdd9956 | 48 +++ .../artifacts/config/config-multi.json | 16 + ...f1c999eef5d058f7ca27e1ae57c16d88c8939df7eb | 48 +++ .../release/create-release-multi.sh | 132 +++++++ .../generate-release-signature-multi.sh | 110 ++++++ .../release/release-payload-multi/index.json | 1 + .../release/release-payload-multi/oci-layout | 1 + tests/integration/keys/release-pk.asc | 29 ++ ...afe3e4cebd16fc590b73962be64e083199e8ea6f83 | Bin 0 -> 865 bytes ...a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b | Bin 0 -> 856 bytes .../sparse_manifest_helpers_test.go | 95 +++++ tests/integration/sparse_manifest_test.go | 324 ++++++++++++++++++ .../sparse_manifest/isc-additional-amd64.yaml | 8 + .../isc-additional-no-platforms.yaml | 5 + .../isc-archs-multi-release.yaml | 13 + .../sparse_manifest/isc-archs-multi.yaml | 9 + .../isc-platforms-and-archs.yaml | 17 + .../sparse_manifest/isc-sparse-full.yaml | 37 ++ .../sparse_manifest/isc-sparse-mixed.yaml | 19 + .../isc-sparse-no-platforms.yaml | 9 + .../integration/testdata/keys/release-pk.asc | 50 +-- ...afe3e4cebd16fc590b73962be64e083199e8ea6f83 | Bin 0 -> 863 bytes ...a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b | Bin 857 -> 855 bytes .../testdata/registry-config-sparse.yaml | 27 ++ 27 files changed, 1031 insertions(+), 29 deletions(-) create mode 100644 tests/integration/image-builders/release/artifacts/config/2adb492d1caf0f239c447de80519b6b431e3c9feff0470aa35a91c87b71feff2 create mode 100644 tests/integration/image-builders/release/artifacts/config/87e0d3b6c43e0b8c022f3a5c4d5a82333ab51937563523acebac831bffdd9956 create mode 100644 tests/integration/image-builders/release/artifacts/config/config-multi.json create mode 100644 tests/integration/image-builders/release/artifacts/config/dbf7b4a726c89654b4d389f1c999eef5d058f7ca27e1ae57c16d88c8939df7eb create mode 100755 tests/integration/image-builders/release/create-release-multi.sh create mode 100755 tests/integration/image-builders/release/generate-release-signature-multi.sh create mode 100644 tests/integration/image-builders/release/release-payload-multi/index.json create mode 100644 tests/integration/image-builders/release/release-payload-multi/oci-layout create mode 100644 tests/integration/keys/release-pk.asc create mode 100644 tests/integration/keys/v0.0.1-sha256-e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83 create mode 100644 tests/integration/keys/v0.0.1-sha256-f81792339c8b5934191d18a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b create mode 100644 tests/integration/sparse_manifest_helpers_test.go create mode 100644 tests/integration/sparse_manifest_test.go create mode 100644 tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-amd64.yaml create mode 100644 tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-no-platforms.yaml create mode 100644 tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi-release.yaml create mode 100644 tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi.yaml create mode 100644 tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-platforms-and-archs.yaml create mode 100644 tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-full.yaml create mode 100644 tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-mixed.yaml create mode 100644 tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-no-platforms.yaml create mode 100644 tests/integration/testdata/keys/v0.0.1-sha256-e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83 create mode 100644 tests/integration/testdata/registry-config-sparse.yaml diff --git a/tests/integration/helpers_test.go b/tests/integration/helpers_test.go index 4b7609e6f..bb2e5fc0c 100644 --- a/tests/integration/helpers_test.go +++ b/tests/integration/helpers_test.go @@ -512,7 +512,7 @@ func setupWorkDir() string { err = os.MkdirAll(sigDir, 0o755) Expect(err).NotTo(HaveOccurred()) - // Find the single signature file in keys/ (any file that isn't the public key) + // Copy all release signature files from keys/ (any file that isn't the public key or cosign key) entries, err := os.ReadDir(keysDir) Expect(err).NotTo(HaveOccurred()) @@ -522,10 +522,12 @@ func setupWorkDir() string { sigFiles = append(sigFiles, entry.Name()) } } - Expect(sigFiles).To(HaveLen(1), "expected exactly one signature file in keys/, found: %v", sigFiles) + Expect(sigFiles).NotTo(BeEmpty(), "no signature files found in keys/") - err = copyFile(filepath.Join(keysDir, sigFiles[0]), filepath.Join(sigDir, sigFiles[0])) - Expect(err).NotTo(HaveOccurred()) + for _, sigFile := range sigFiles { + err = copyFile(filepath.Join(keysDir, sigFile), filepath.Join(sigDir, sigFile)) + Expect(err).NotTo(HaveOccurred()) + } return workDir } diff --git a/tests/integration/image-builders/release/Makefile b/tests/integration/image-builders/release/Makefile index 3460fb238..804bb6457 100755 --- a/tests/integration/image-builders/release/Makefile +++ b/tests/integration/image-builders/release/Makefile @@ -26,3 +26,7 @@ sign: COSIGN_PASSWORD="" cosign sign --key $(COSIGN_KEY) --tlog-upload=false quay.io/oc-mirror/release/test-release-index:v0.0.1 .PHONY: sign +sign-multi: + COSIGN_PASSWORD="" cosign sign --recursive --key $(COSIGN_KEY) --tlog-upload=false quay.io/oc-mirror/release/test-release-index-multi:v0.0.1 +.PHONY: sign-multi + diff --git a/tests/integration/image-builders/release/artifacts/config/2adb492d1caf0f239c447de80519b6b431e3c9feff0470aa35a91c87b71feff2 b/tests/integration/image-builders/release/artifacts/config/2adb492d1caf0f239c447de80519b6b431e3c9feff0470aa35a91c87b71feff2 new file mode 100644 index 000000000..a4413707d --- /dev/null +++ b/tests/integration/image-builders/release/artifacts/config/2adb492d1caf0f239c447de80519b6b431e3c9feff0470aa35a91c87b71feff2 @@ -0,0 +1,48 @@ +{ + "created": "2023-01-24T02:27:38Z", + "architecture": "arm64", + "os": "linux", + "config": { + "User": "0", + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "container=oci", + "__doozer=merge", + "BUILD_RELEASE=202212061458.p0.g7e8a010.assembly.stream", + "BUILD_VERSION=v4.12.0", + "OS_GIT_MAJOR=4", + "OS_GIT_MINOR=12", + "OS_GIT_PATCH=0", + "OS_GIT_TREE_STATE=clean", + "OS_GIT_VERSION=4.12.0-202212061458.p0.g7e8a010.assembly.stream-7e8a010", + "SOURCE_GIT_TREE_STATE=clean", + "OS_GIT_COMMIT=7e8a010", + "SOURCE_DATE_EPOCH=1666997438", + "SOURCE_GIT_COMMIT=7e8a0105eb7369f3f92ad7b2581a2efffab5b28e", + "SOURCE_GIT_TAG=7e8a010", + "SOURCE_GIT_URL=https://github.com/oc-mirror-test/integration", + "GODEBUG=x509ignoreCN=0,madvdontneed=1", + "OPENSHIFT_CI=true", + "BUILD_LOGLEVEL=0", + "OPENSHIFT_BUILD_NAME=test-release-image", + "OPENSHIFT_BUILD_NAMESPACE=ci-op-1c0qqwdy" + ], + "Entrypoint": [ + "/usr/bin/simple-release" + ], + "Labels": { + "io.openshift.release": "4.18.0-0.ocp-2025-03-24-022732", + "io.openshift.release.base-image-digest": "sha256:0ed55ee636f5703b212337ff8a82833c5dfc4553069dc3f6cd52564cc0adcd08" + } + }, + "rootfs": { + "type": "layers", + "diff_ids": [] + }, + "history": [ + { + "created": "2025-03-24T02:27:38Z", + "comment": "Release image for testing oc-mirror v2" + } + ] +} diff --git a/tests/integration/image-builders/release/artifacts/config/87e0d3b6c43e0b8c022f3a5c4d5a82333ab51937563523acebac831bffdd9956 b/tests/integration/image-builders/release/artifacts/config/87e0d3b6c43e0b8c022f3a5c4d5a82333ab51937563523acebac831bffdd9956 new file mode 100644 index 000000000..ec10d66f8 --- /dev/null +++ b/tests/integration/image-builders/release/artifacts/config/87e0d3b6c43e0b8c022f3a5c4d5a82333ab51937563523acebac831bffdd9956 @@ -0,0 +1,48 @@ +{ + "created": "2023-01-24T02:27:38Z", + "architecture": "ppc64le", + "os": "linux", + "config": { + "User": "0", + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "container=oci", + "__doozer=merge", + "BUILD_RELEASE=202212061458.p0.g7e8a010.assembly.stream", + "BUILD_VERSION=v4.12.0", + "OS_GIT_MAJOR=4", + "OS_GIT_MINOR=12", + "OS_GIT_PATCH=0", + "OS_GIT_TREE_STATE=clean", + "OS_GIT_VERSION=4.12.0-202212061458.p0.g7e8a010.assembly.stream-7e8a010", + "SOURCE_GIT_TREE_STATE=clean", + "OS_GIT_COMMIT=7e8a010", + "SOURCE_DATE_EPOCH=1666997438", + "SOURCE_GIT_COMMIT=7e8a0105eb7369f3f92ad7b2581a2efffab5b28e", + "SOURCE_GIT_TAG=7e8a010", + "SOURCE_GIT_URL=https://github.com/oc-mirror-test/integration", + "GODEBUG=x509ignoreCN=0,madvdontneed=1", + "OPENSHIFT_CI=true", + "BUILD_LOGLEVEL=0", + "OPENSHIFT_BUILD_NAME=test-release-image", + "OPENSHIFT_BUILD_NAMESPACE=ci-op-1c0qqwdy" + ], + "Entrypoint": [ + "/usr/bin/simple-release" + ], + "Labels": { + "io.openshift.release": "4.18.0-0.ocp-2025-03-24-022732", + "io.openshift.release.base-image-digest": "sha256:0ed55ee636f5703b212337ff8a82833c5dfc4553069dc3f6cd52564cc0adcd08" + } + }, + "rootfs": { + "type": "layers", + "diff_ids": [] + }, + "history": [ + { + "created": "2025-03-24T02:27:38Z", + "comment": "Release image for testing oc-mirror v2" + } + ] +} diff --git a/tests/integration/image-builders/release/artifacts/config/config-multi.json b/tests/integration/image-builders/release/artifacts/config/config-multi.json new file mode 100644 index 000000000..cd944b386 --- /dev/null +++ b/tests/integration/image-builders/release/artifacts/config/config-multi.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:CONFIG_DIGEST", + "size": CONFIG_SIZE + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:LAYER_DIGEST", + "size": LAYER_SIZE + } + ] +} diff --git a/tests/integration/image-builders/release/artifacts/config/dbf7b4a726c89654b4d389f1c999eef5d058f7ca27e1ae57c16d88c8939df7eb b/tests/integration/image-builders/release/artifacts/config/dbf7b4a726c89654b4d389f1c999eef5d058f7ca27e1ae57c16d88c8939df7eb new file mode 100644 index 000000000..20f912481 --- /dev/null +++ b/tests/integration/image-builders/release/artifacts/config/dbf7b4a726c89654b4d389f1c999eef5d058f7ca27e1ae57c16d88c8939df7eb @@ -0,0 +1,48 @@ +{ + "created": "2023-01-24T02:27:38Z", + "architecture": "s390x", + "os": "linux", + "config": { + "User": "0", + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "container=oci", + "__doozer=merge", + "BUILD_RELEASE=202212061458.p0.g7e8a010.assembly.stream", + "BUILD_VERSION=v4.12.0", + "OS_GIT_MAJOR=4", + "OS_GIT_MINOR=12", + "OS_GIT_PATCH=0", + "OS_GIT_TREE_STATE=clean", + "OS_GIT_VERSION=4.12.0-202212061458.p0.g7e8a010.assembly.stream-7e8a010", + "SOURCE_GIT_TREE_STATE=clean", + "OS_GIT_COMMIT=7e8a010", + "SOURCE_DATE_EPOCH=1666997438", + "SOURCE_GIT_COMMIT=7e8a0105eb7369f3f92ad7b2581a2efffab5b28e", + "SOURCE_GIT_TAG=7e8a010", + "SOURCE_GIT_URL=https://github.com/oc-mirror-test/integration", + "GODEBUG=x509ignoreCN=0,madvdontneed=1", + "OPENSHIFT_CI=true", + "BUILD_LOGLEVEL=0", + "OPENSHIFT_BUILD_NAME=test-release-image", + "OPENSHIFT_BUILD_NAMESPACE=ci-op-1c0qqwdy" + ], + "Entrypoint": [ + "/usr/bin/simple-release" + ], + "Labels": { + "io.openshift.release": "4.18.0-0.ocp-2025-03-24-022732", + "io.openshift.release.base-image-digest": "sha256:0ed55ee636f5703b212337ff8a82833c5dfc4553069dc3f6cd52564cc0adcd08" + } + }, + "rootfs": { + "type": "layers", + "diff_ids": [] + }, + "history": [ + { + "created": "2025-03-24T02:27:38Z", + "comment": "Release image for testing oc-mirror v2" + } + ] +} diff --git a/tests/integration/image-builders/release/create-release-multi.sh b/tests/integration/image-builders/release/create-release-multi.sh new file mode 100755 index 000000000..0435b839e --- /dev/null +++ b/tests/integration/image-builders/release/create-release-multi.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# Builds a multi-arch fake OCP release payload OCI layout and pushes it as +# quay.io/oc-mirror/release/test-release-index-multi:v0.0.1 +# +# Prerequisites: skopeo, jq, sha256sum, stat +# Run from the image-builders/release/ directory. +# +# After a successful run: +# make sign-multi (cosign signature) +# ./generate-release-signature.sh (GPG atomic container signature) +# Commit: release-payload-multi/index.json and the new GPG signature in ../../testdata/keys/ + +set -ex + +REPO="quay.io/oc-mirror/release/test-release-index-multi" +TAG="v0.0.1" + +# Per-arch config blob filenames (sha256 of content, pre-committed) +declare -A ARCH_CONFIG_BLOB=( + ["amd64"]="8366d414d18526e99f4781ed84a93b5b96ebe464d223e587cf7f705829b27a12" + ["arm64"]="2adb492d1caf0f239c447de80519b6b431e3c9feff0470aa35a91c87b71feff2" + ["s390x"]="dbf7b4a726c89654b4d389f1c999eef5d058f7ca27e1ae57c16d88c8939df7eb" + ["ppc64le"]="87e0d3b6c43e0b8c022f3a5c4d5a82333ab51937563523acebac831bffdd9956" +) + +rm -rf release-payload-multi/blobs/sha256 +mkdir -p release-payload-multi/blobs/sha256 artifacts/staging + +# Step 1: Build the shared release-manifests layer blob (same for all arches) +cd artifacts/staging +tar -czvf tmp_layer.tar.gz ../release-manifests/ +LAYER_DIGEST=$(sha256sum tmp_layer.tar.gz | cut -d ' ' -f 1) +LAYER_SIZE=$(stat --printf="%s" tmp_layer.tar.gz) +cp tmp_layer.tar.gz ../../release-payload-multi/blobs/sha256/${LAYER_DIGEST} +cd ../../ + +# Step 2: For each arch, build an OCI image manifest referencing the shared layer +MANIFESTS_JSON="" + +for ARCH in amd64 arm64 s390x ppc64le; do + CONFIG_BLOB_NAME="${ARCH_CONFIG_BLOB[$ARCH]}" + CONFIG_BLOB_PATH="artifacts/config/${CONFIG_BLOB_NAME}" + CONFIG_SIZE=$(stat --printf="%s" "${CONFIG_BLOB_PATH}") + + # Copy config blob to payload blobs directory + cp "${CONFIG_BLOB_PATH}" release-payload-multi/blobs/sha256/${CONFIG_BLOB_NAME} + + # Build per-arch OCI image manifest by substituting all four placeholders + sed -e "s/CONFIG_DIGEST/${CONFIG_BLOB_NAME}/g" \ + -e "s/CONFIG_SIZE/${CONFIG_SIZE}/g" \ + -e "s/LAYER_DIGEST/${LAYER_DIGEST}/g" \ + -e "s/LAYER_SIZE/${LAYER_SIZE}/g" \ + artifacts/config/config-multi.json > /tmp/manifest_${ARCH}.json + + MANIFEST_DIGEST=$(sha256sum /tmp/manifest_${ARCH}.json | cut -d ' ' -f 1) + MANIFEST_SIZE=$(stat --printf="%s" /tmp/manifest_${ARCH}.json) + cp /tmp/manifest_${ARCH}.json release-payload-multi/blobs/sha256/${MANIFEST_DIGEST} + + # Accumulate manifest entry for OCI index + ENTRY="{\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"digest\":\"sha256:${MANIFEST_DIGEST}\",\"size\":${MANIFEST_SIZE},\"platform\":{\"os\":\"linux\",\"architecture\":\"${ARCH}\"}}" + if [ -z "${MANIFESTS_JSON}" ]; then + MANIFESTS_JSON="${ENTRY}" + else + MANIFESTS_JSON="${MANIFESTS_JSON},${ENTRY}" + fi +done + +# Step 3: Push each per-arch image to the registry with an arch-specific tag, +# then assemble a manifest list using podman manifest. +for ARCH in amd64 arm64 s390x ppc64le; do + CONFIG_BLOB_NAME="${ARCH_CONFIG_BLOB[$ARCH]}" + ARCH_MANIFEST_FILE="/tmp/manifest_${ARCH}.json" + ARCH_MANIFEST_DIGEST=$(sha256sum "${ARCH_MANIFEST_FILE}" | cut -d ' ' -f 1) + ARCH_MANIFEST_SIZE=$(stat --printf="%s" "${ARCH_MANIFEST_FILE}") + + # Build a single-arch OCI layout for this arch + ARCH_DIR="/tmp/oci_${ARCH}" + rm -rf "${ARCH_DIR}" + mkdir -p "${ARCH_DIR}/blobs/sha256" + echo '{"imageLayoutVersion": "1.0.0"}' > "${ARCH_DIR}/oci-layout" + + cp release-payload-multi/blobs/sha256/${LAYER_DIGEST} "${ARCH_DIR}/blobs/sha256/" + cp release-payload-multi/blobs/sha256/${CONFIG_BLOB_NAME} "${ARCH_DIR}/blobs/sha256/" + cp "${ARCH_MANIFEST_FILE}" "${ARCH_DIR}/blobs/sha256/${ARCH_MANIFEST_DIGEST}" + + cat > "${ARCH_DIR}/index.json" </dev/null || true +podman manifest create ${REPO}:${TAG} +for ARCH in amd64 arm64 s390x ppc64le; do + podman manifest add --arch=${ARCH} --os=linux ${REPO}:${TAG} docker://${REPO}:${TAG}-${ARCH} +done + +# Push the manifest list to the registry +podman manifest push --all --remove-signatures ${REPO}:${TAG} docker://${REPO}:${TAG} + +# Save the manifest index for committing (used by generate-release-signature.sh) +skopeo inspect --raw docker://${REPO}:${TAG} > release-payload-multi/index.json + +# Ensure oci-layout marker exists +cat > release-payload-multi/oci-layout <<'EOF' +{"imageLayoutVersion": "1.0.0"} +EOF + +# Clean up staging +rm -rf artifacts/staging/* + +echo "" +echo "Successfully pushed ${REPO}:${TAG}" +echo "" +echo "Next steps:" +echo " make sign-multi (cosign tag-based signature)" +echo " ./generate-release-signature-multi.sh (GPG atomic container signature)" +echo " cp keys/* ../../testdata/keys/" +echo " Commit: release-payload-multi/index.json, ../../testdata/keys/release-pk.asc, ../../testdata/keys/v0.0.1-sha256-*" diff --git a/tests/integration/image-builders/release/generate-release-signature-multi.sh b/tests/integration/image-builders/release/generate-release-signature-multi.sh new file mode 100755 index 000000000..0affedee0 --- /dev/null +++ b/tests/integration/image-builders/release/generate-release-signature-multi.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Generate a throwaway GPG keypair, sign both the single-arch and multi-arch +# test release image digests (atomic container signature JSON), +# and write only the public key plus signatures under keys/. +# No registry or credentials required. +# Requires: gpg, jq. Paths are relative to the integration repo root. +# +# Both images are signed with the same key so that a single release-pk.asc +# validates all signatures. Run this script after building both release images. +# For single-arch only, use generate-release-signature.sh instead. +# +# ./image-builders/release/generate-release-signature-multi.sh + +set -euo pipefail + +if [[ $# -gt 0 ]]; then + echo "Usage: $0" >&2 + echo " (no arguments — signs both releases, writes to keys/)" >&2 + exit 1 +fi + +command -v gpg >/dev/null || { + echo "gpg is required" >&2 + exit 1 +} +command -v jq >/dev/null || { + echo "jq is required" >&2 + exit 1 +} + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +KEYS_DIR="${REPO_ROOT}/keys" +GPG_IDENTITY="robot@test.com" + +TMP_GNUPG="$(mktemp -d)" +cleanup() { + rm -rf "${TMP_GNUPG}" +} +trap cleanup EXIT +export GNUPGHOME="${TMP_GNUPG}" +gpg --batch --passphrase '' --pinentry-mode loopback \ + --quick-generate-key "oc-mirror Release Test <${GPG_IDENTITY}>" rsa4096 default 0 + +mkdir -p "${KEYS_DIR}" +PK_OUT="${KEYS_DIR}/release-pk.asc" + +sign_release() { + local index_json="$1" + local image_ref="$2" + local use_file_digest="${3:-false}" + + if [[ ! -f "${index_json}" ]]; then + echo "Skipping ${image_ref}: ${index_json} not found." >&2 + return 0 + fi + + local digest + if [[ "${use_file_digest}" == "true" ]]; then + # Multi-arch: the manifest list digest IS the sha256 of the index.json blob + digest="sha256:$(sha256sum "${index_json}" | cut -d ' ' -f 1)" + else + digest="$(jq -r ' + ([.manifests[] | select(.platform.os == "linux" and .platform.architecture == "amd64") | .digest][0]) + // .manifests[0].digest + ' "${index_json}")" + if [[ -z "${digest}" || "${digest}" == "null" || "${digest}" != sha256:* ]]; then + echo "Could not read a valid manifest digest from ${index_json}" >&2 + exit 1 + fi + fi + + local digest_hex="${digest#sha256:}" + local tag="${image_ref##*:}" + + local sig_payload + sig_payload="$(jq -nc \ + --arg ref "${image_ref}" \ + --arg dig "${digest}" \ + '{critical:{identity:{"docker-reference":$ref},image:{"docker-manifest-digest":$dig},type:"atomic container signature"},optional:{}}')" + + local sig_file="${KEYS_DIR}/${tag}-sha256-${digest_hex}" + + rm -f "${sig_file}" + printf '%s' "${sig_payload}" | gpg --batch \ + --output "${sig_file}" \ + --local-user "${GPG_IDENTITY}" \ + --digest-algo SHA512 \ + --sign + + if [[ ! -f "${sig_file}" ]]; then + echo "Signing failed — ${sig_file} was not created" >&2 + exit 1 + fi + + echo "Wrote ${sig_file}" +} + +# Single-arch release +sign_release \ + "${REPO_ROOT}/image-builders/release/release-payload/index.json" \ + "quay.io/oc-mirror/release/test-release-index:v0.0.1" + +# Multi-arch release (the manifest list digest is the sha256 of the index.json blob itself) +sign_release \ + "${REPO_ROOT}/image-builders/release/release-payload-multi/index.json" \ + "quay.io/oc-mirror/release/test-release-index-multi:v0.0.1" \ + "true" + +gpg -a --export --output "${PK_OUT}" "${GPG_IDENTITY}" +echo "Wrote ${PK_OUT}" diff --git a/tests/integration/image-builders/release/release-payload-multi/index.json b/tests/integration/image-builders/release/release-payload-multi/index.json new file mode 100644 index 000000000..dd978c586 --- /dev/null +++ b/tests/integration/image-builders/release/release-payload-multi/index.json @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:20d2c230f7dd11aaf428f4700e1f69e9550924dbdf44541b81dbbffb72823f6c","size":478,"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:ba59faf3e9eaee6d1b5b4eebf82eaa59427821e38ba45b4ee8d826fe7e1c34a2","size":478,"platform":{"architecture":"arm64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:e7bdeb0c63b498ea60330a415dcae2a7da28b0cd7366988b8e363c9cb19d1c46","size":478,"platform":{"architecture":"s390x","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:5f1084981de2ef9abcad4099fac1ea692b1896ae795f1890ff0c2b617ae0c61a","size":478,"platform":{"architecture":"ppc64le","os":"linux"}}]} \ No newline at end of file diff --git a/tests/integration/image-builders/release/release-payload-multi/oci-layout b/tests/integration/image-builders/release/release-payload-multi/oci-layout new file mode 100644 index 000000000..60304f2b2 --- /dev/null +++ b/tests/integration/image-builders/release/release-payload-multi/oci-layout @@ -0,0 +1 @@ +{"imageLayoutVersion": "1.0.0"} diff --git a/tests/integration/keys/release-pk.asc b/tests/integration/keys/release-pk.asc new file mode 100644 index 000000000..788d0f864 --- /dev/null +++ b/tests/integration/keys/release-pk.asc @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGpaNDoBEADXPpAbD/WzpFGynxxjA+m1YhZdQvIE/wn41rwrc2JjCuKSgDaj +Ffdzfex0IYyO7iXKb/0lBVFxt0fppnbRwLxnLDKi10jB3ofjuZjsEUPi77DgLMo/ +BsVSfAGmIGS9d6je7cmKCNPJyzXHNg1YcrMYvD15wyRj8XKc/YettZzfhUpLVpUP +qW0c88QryMwniehD2qhQoL/y3aUOgoeKAqPC0YwbUsb/cJuWcW2sQCw0lYAhRkZD +xoNV6ZbF/40JRiazt2Ropj7Q7EntEv+I/PZL90fuA9uOu+vcXE7nbd9YPLBV3gC+ +IyHE/fo/iNU0Fu27HCqE1kPA1PdVnDgs1+ungEOib0YCqtGw1Je5zkaqRDtn3JTp +504kAcmYZ8aFOf3JTvpUy+W3q1BLhQyLuSPi3RcSHIeJF0iyh/iQuUKf1wd2q/MP +/25yeFDF/HDu6izTvdXqQNXjNgqlVWCRtCGhL407Lm3DcrgX/O2zd4SqMAqIn0Bv +2pBu/lGz5R16yp7eVRt5p1iNjrVs6COzBWaTACl2j0uSGmvhx0DWN7htnYvFzypP +OzE51si9dx+cblGW1ZbIPPhAZjQthYD7vVqLXQWltTQyRiBrsLfJ9cepxMBmJHQ1 +wk9GVWkRouweXvaXQvsmRErwEPo2I4ldT77Sqy/WXw4D8RkKTWqz3gh/KQARAQAB +tCdvYy1taXJyb3IgUmVsZWFzZSBUZXN0IDxyb2JvdEB0ZXN0LmNvbT6JAlEEEwEK +ADsWIQTHUZvnODbMUfzWroboWcYtHlC9vAUCalo0OgIbAwULCQgHAgIiAgYVCgkI +CwIEFgIDAQIeBwIXgAAKCRDoWcYtHlC9vPWwD/9EL1F+4e9eU6z4AcRO6mRH1Pmk +0eawjKyXMmexdQVF68v46dPqoR8WwqddGGyLsmplM2VtxXheho8R0vRpddb2MM6F +JaVJ8V95wlO99GA/UCv/7x56p8gtjmuufNoEufxUgKgkS/7Q4ef8Z1rCp6wlL+JY +ElkAxMs9QQqBjHzYE0hHyGGEXnd/XpIIsz+urzQA2OdaNZ5vnIjfRP3tPxvCaTNw +Y5k1wMvEx1x295o2ujSLv4Bf0R6WoNtHfJDwm3uqgETqWRIUBvM0SXAJwVLJSz0B +7yoPB7bWXl7mtXctE3XCr3t29VlfybZkHO6+5jumdRQfgGJqbHshzgMZDeWWAUH0 +VuqOun1k7KBrszSariO+8kbgTZVIs7obiWk5hO0vtaH07uyPow/gcJVC7h+6I0ff +rGo/B0nlpKaTBf8ogTmG8NCS6EEG5gLeCVrf8mRRkXdauBFKh7z2yFtU83CSJo05 +XjtIjiMaJfhe4nz2A8rSBeRjqvSwKJDZn4ob7sOwQIDIDDxZKq/bj8flru0EZzkV +9+vfONPiJ5gZuJNt2Ug3yCLi3Zzq4bmDV3cBhPTytS4XCFHgt852LU9rAADp4h9Z +XYxbAI5agrlkz6bAl3SS3NUnFD+YH22s0DecWotwQ1/OJEL9qKshj6B9QWqd8Akq +IbMmz1SqVng3U1Du+w== +=fmlz +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/integration/keys/v0.0.1-sha256-e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83 b/tests/integration/keys/v0.0.1-sha256-e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83 new file mode 100644 index 0000000000000000000000000000000000000000..d24dfe653e0c7994dc6c269dc31e2f92c046295b GIT binary patch literal 865 zcmV-n1D^b&0h_?f%)rI?BJ!B7T)^HvjHeIyCoyD2nOIjVB^PCuWF{x(C|Ol2Wu~O& zm1LGwg4ikf$=Rtzx<#pJsYR)I$*D?KN`<9~m3o=^`uWMaxtT>p`9=CgsX3{M#i{xw zsl_F_P>yb9UP@|(ZfD7Kw&NDaI*gDaOeu#%alkCTWJLmWgSp#;GRBsYxk@W@*W$mIg`Y#+GJA zNvUQgsRkCthL)D87O9D5X%@yH%StK>Kn_kU$lsEGaEY z1u4oeD9Oyv1N*$Twv)-3g@KWaL03$X<#^!i=N4vX0{>iF*9HqRRwhtz@yitDC*_wo zfc&eMoS)0U#VG(&p>=`(e`?ILwf-NO1I(x86mul(6)vdY+H61Ijotw;#8Bd ztm@52)sw$`@i==ap~JKHm)&oh(#<6g;${e+{Ll6XbvUJ1YfEk8)){wD6(%uJv~+rBkY`zw|vj{7CQF zNzZ`4v%6l%?SG!9x@k@xYj@vl$#4&sjZQwhFHZ9Mqpl~miF;+mT0#4Qn_1TsTcY%m z&mQ&*WqN%-^_+&IU%#KrnJ1??H;P0hJTqJ*K(`@ZGg z|E9(pmrh-2xO@8ZOWF0(yZb)vjon@|UCjH=hd1%AiO)E%M@}kRJ9kmMj`G1g#-fge z*>Oh`VxOLq+!y#V=Far<_c#S6&C7lMvGmvDX`!k6Vm6-ma5i;j-{X7NDs|0{?GSz0 rAH-ZUsfAnHEThl*g2SQ%x;Hf*H?Qf~qa$_g-;9$Aiqhe|TW6>0h_?f%)rI?BJ!B7T)^HvjHeHHCNX41nOIjVB^PCuWF{x(C|Ol2Wu~O& zm1LGwg4ikf$=Rtzx<#pJsYR)I$*D?KN`<9~m3o=^`uWMaxtT>p`9=CgsX3{M#i{xw zsl_F_P>yb9UP@|(RhfaFfu5mKt&UP=Zen^Wl99QId6{V-4Jn!Fsl_EqR!YSgiAJVo zR%sT7=9Wgr#+JzzNv4*@CWe-VDTWq_rp8Hz7D>s5DW(=CsRo9LmTAW3DW;am28oGg zmIkIt1_nk328oG=Ng&HgDhohPO)SaJ%}iEE&d)1J%*;zIQYg+$&r2*RElLF`$}cF% z%+CY+xwf{G$(es>J1myFXLRT^{t2Qr>));#lI;i@6OTg!~YIyl?ThSmA3HkY@QaOF#AH^w}XX3%ib&B zpU{1iD^=;;Zy)2Rb=x9(S+0kF5?-Y)#}U5XdZ+Wu13NoAEqO0&Se2ESl5TGkU2*N6 zu4mO+jebMlsnLA3-=o&$DR;kIn7!_|k@4+E-CtcB4R*XJc5Kp|FQ@uNK_WrLPvxH7 zpC`ZWC$0Zo`p+!W-E{pO|0AO3%G95n+E8}UL!->Pw%I0HJ-@QIzo`Goqz@a6R4Omt ze4x2Pv^D#O+yoAp+Sz}f3WvH1`W4T=kUW2d&q9s-@2lHi_MF#dwJB_>g-o_P=_cOQgzKB1|{_#s7^WV}kh6~H`+1kD(^2Qh5cy?@Un@*owLDBj1 zvYF}kOGETj4!cM`I?eWN+cDnF#eX;s-IsiQcdLozf!j-~CEOp(2-@!PVack~(&`~^ zw*P1ScHO@D;FH>?3=BRw!YhLsr{t>cJ>|8nOxjtOZ%LBKp$#rlsrR>(-ENGNkV^?N zNL-m6!F|txF)U7EC&PcE)K@PzU1P6a8T-4Gt$*s}d3S7;mEQ2%osZ?8$)`VGda+CL iybq5#_@+Ir-uC?1?SpqGOp0R?1WW literal 0 HcmV?d00001 diff --git a/tests/integration/sparse_manifest_helpers_test.go b/tests/integration/sparse_manifest_helpers_test.go new file mode 100644 index 000000000..f14d77a36 --- /dev/null +++ b/tests/integration/sparse_manifest_helpers_test.go @@ -0,0 +1,95 @@ +package integration_test + +import ( + "context" + "fmt" + "net/http" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + + "github.com/openshift/oc-mirror/tests/integration/pkg/registry" +) + +// expectOnlyPlatformBlobsPresent verifies that only the specified platforms have +// reachable blobs in the registry for the given manifest-list image. +// Other platforms may appear in the index but their blobs must be absent (HTTP 404). +func expectOnlyPlatformBlobsPresent(ctx context.Context, reg *registry.Registry, repo, tag string, expectedPlatforms []string) { + GinkgoHelper() + present, missing := getPlatformBlobPresence(ctx, reg, repo, tag) + for _, exp := range expectedPlatforms { + Expect(present).To(ContainElement(exp), + "expected platform %q to have a blob in the registry", exp) + } + for _, got := range present { + matched := false + for _, exp := range expectedPlatforms { + if got == exp || strings.HasPrefix(got, exp+"/") { + matched = true + break + } + } + Expect(matched).To(BeTrue(), + "unexpected platform %q has a blob in the registry (should be filtered out)", got) + } + _ = missing // platforms in index but no blob — expected for sparse manifests +} + +// expectAllPlatformBlobsPresent verifies that every platform entry in the +// manifest list has a reachable blob (no sparse filtering applied). +func expectAllPlatformBlobsPresent(ctx context.Context, reg *registry.Registry, repo, tag string) { + GinkgoHelper() + _, missing := getPlatformBlobPresence(ctx, reg, repo, tag) + Expect(missing).To(BeEmpty(), + "expected all platform blobs to be present, but these are missing: %v", missing) +} + +// getPlatformBlobPresence returns two lists: platforms with blobs present and +// platforms whose blobs are missing (listed in index but not stored). +func getPlatformBlobPresence(ctx context.Context, reg *registry.Registry, repo, tag string) (present, missing []string) { + GinkgoHelper() + + refStr := fmt.Sprintf("%s/%s:%s", reg.Endpoint(), repo, tag) + ref, err := name.ParseReference(refStr, name.Insecure) + Expect(err).NotTo(HaveOccurred()) + + idx, err := remote.Index(ref, + remote.WithContext(ctx), + remote.WithAuth(authn.Anonymous)) + Expect(err).NotTo(HaveOccurred(), "expected a manifest list at %s", refStr) + + manifest, err := idx.IndexManifest() + Expect(err).NotTo(HaveOccurred()) + Expect(manifest.Manifests).NotTo(BeEmpty(), "manifest list has no entries") + + for _, m := range manifest.Manifests { + if m.Platform == nil { + continue + } + plat := m.Platform.OS + "/" + m.Platform.Architecture + if m.Platform.Variant != "" { + plat += "/" + m.Platform.Variant + } + + // Check if the per-platform manifest blob is reachable. + manifestURL := fmt.Sprintf("http://%s/v2/%s/manifests/%s", reg.Endpoint(), repo, m.Digest.String()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, manifestURL, nil) + Expect(err).NotTo(HaveOccurred()) + req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json") + + resp, err := http.DefaultClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + present = append(present, plat) + } else { + missing = append(missing, plat) + } + } + return present, missing +} diff --git a/tests/integration/sparse_manifest_test.go b/tests/integration/sparse_manifest_test.go new file mode 100644 index 000000000..55eb7e0b8 --- /dev/null +++ b/tests/integration/sparse_manifest_test.go @@ -0,0 +1,324 @@ +package integration_test + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/openshift/oc-mirror/tests/integration/pkg/ocmirror" + "github.com/openshift/oc-mirror/tests/integration/pkg/registry" +) + +var _ = Describe("sparse manifest platform filtering", Label("sparse-manifest"), func() { + var workDir string + + BeforeEach(func() { workDir = setupWorkDir() }) + AfterEach(func() { cleanupWorkDir(workDir) }) + + Describe("config validation", func() { + It("should fail when both platform.platforms and platform.architectures are set", func() { + By("running mirrorToMirror with an ISC that uses both platforms and architectures") + result, err := runner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-platforms-and-archs.yaml"), + workDir, testRegistry.Endpoint(), + "--dest-tls-verify=false") + By("verifying oc-mirror exits with an error containing the mutual-exclusivity message") + expectOcMirrorExitCode(result, err, 1, + "platform.platforms and platform.architectures cannot be used together") + }) + }) + + Describe("M2M", func() { + Context("with platforms filter (sparse manifests)", func() { + var sparseRegistry *registry.Registry + + BeforeEach(func() { + sparseRegistryConfigPath := filepath.Join(filepath.Dir(registryConfig), "registry-config-sparse.yaml") + var err error + sparseRegistry, err = registry.Start(ctx, sparseRegistryConfigPath, 0) + Expect(err).NotTo(HaveOccurred()) + Expect(sparseRegistry.WaitReady(ctx, 30*time.Second)).To(Succeed()) + }) + + AfterEach(func() { + Expect(sparseRegistry.Stop()).To(Succeed()) + }) + + It("should mirror only declared arch blobs for each image", func() { + By("mirroring with platforms: [{linux, amd64}, {linux, arm64}] on multi-platform-container and no filter on empty-image") + result, err := runner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-mixed.yaml"), + workDir, sparseRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying only linux/amd64 and linux/arm64 blobs are present for multi-platform-container") + expectOnlyPlatformBlobsPresent(ctx, sparseRegistry, + "rh_ee_aguidi/multi-platform-container", "latest", + []string{"linux/amd64", "linux/arm64"}) + + By("verifying empty-image was mirrored without filtering") + expectRepositoriesExist(*sparseRegistry, []string{"rh_ee_aguidi/empty-image"}) + + By("verifying the release image was mirrored with platform filtering") + expectRepositoriesExist(*sparseRegistry, []string{"openshift/release-images"}) + }) + }) + + Context("with platforms filter — operators, helm, and additionalImages (M2M)", func() { + var sparseRegistry *registry.Registry + + BeforeEach(func() { + sparseRegistryConfigPath := filepath.Join(filepath.Dir(registryConfig), "registry-config-sparse.yaml") + var err error + sparseRegistry, err = registry.Start(ctx, sparseRegistryConfigPath, 0) + Expect(err).NotTo(HaveOccurred()) + Expect(sparseRegistry.WaitReady(ctx, 30*time.Second)).To(Succeed()) + }) + + AfterEach(func() { + Expect(sparseRegistry.Stop()).To(Succeed()) + }) + + It("should mirror only declared arch blobs for each content type", func() { + By("mirroring with platforms filter on additionalImages, operators, and helm") + result, err := runner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-full.yaml"), + workDir, sparseRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying only linux/amd64 and linux/arm64 blobs are present for multi-platform-container") + expectOnlyPlatformBlobsPresent(ctx, sparseRegistry, + "rh_ee_aguidi/multi-platform-container", "latest", + []string{"linux/amd64", "linux/arm64"}) + + By("verifying operator catalog and helm repositories exist") + expectRepositoriesExist(*sparseRegistry, []string{"oc-mirror/oc-mirror-dev", "podinfo"}) + }) + }) + + Context("without platforms filter (backward compat)", func() { + It("should mirror all blobs without sparse filtering", func() { + By("mirroring without platforms filter") + result, err := runner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-no-platforms.yaml"), + workDir, testRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying all platform blobs are present for multi-platform-container") + expectAllPlatformBlobsPresent(ctx, testRegistry, + "rh_ee_aguidi/multi-platform-container", "latest") + + By("verifying the release image was mirrored") + expectRepositoriesExist(*testRegistry, []string{"openshift/release-images"}) + }) + }) + }) + + Describe("M2D + D2M", func() { + Context("with platforms filter (sparse manifests)", func() { + var sparseRegistry *registry.Registry + + BeforeEach(func() { + sparseRegistryConfigPath := filepath.Join(filepath.Dir(registryConfig), "registry-config-sparse.yaml") + var err error + sparseRegistry, err = registry.Start(ctx, sparseRegistryConfigPath, 0) + Expect(err).NotTo(HaveOccurred()) + Expect(sparseRegistry.WaitReady(ctx, 30*time.Second)).To(Succeed()) + }) + + AfterEach(func() { + Expect(sparseRegistry.Stop()).To(Succeed()) + }) + + It("should preserve platform filtering through disk archive", func() { + By("mirroring to disk with platforms filter") + result, err := runner.MirrorToDisk(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-mixed.yaml"), + workDir, "--remove-signatures=true") + expectOcMirrorCommandSuccess(result, err) + + By("mirroring from disk to sparse registry") + result, err = runner.DiskToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-mixed.yaml"), + workDir, sparseRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying only linux/amd64 and linux/arm64 blobs are present for multi-platform-container") + expectOnlyPlatformBlobsPresent(ctx, sparseRegistry, + "rh_ee_aguidi/multi-platform-container", "latest", + []string{"linux/amd64", "linux/arm64"}) + + By("verifying the release image was mirrored") + expectRepositoriesExist(*sparseRegistry, []string{"openshift/release-images"}) + }) + }) + + Context("with platforms filter — operators, helm, and additionalImages (M2D+D2M)", func() { + var sparseRegistry *registry.Registry + + BeforeEach(func() { + sparseRegistryConfigPath := filepath.Join(filepath.Dir(registryConfig), "registry-config-sparse.yaml") + var err error + sparseRegistry, err = registry.Start(ctx, sparseRegistryConfigPath, 0) + Expect(err).NotTo(HaveOccurred()) + Expect(sparseRegistry.WaitReady(ctx, 30*time.Second)).To(Succeed()) + }) + + AfterEach(func() { + Expect(sparseRegistry.Stop()).To(Succeed()) + }) + + It("should preserve platform filtering through disk archive for all content types", func() { + By("mirroring to disk with platforms filter on operators, helm, and additionalImages") + result, err := runner.MirrorToDisk(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-full.yaml"), + workDir, "--remove-signatures=true") + expectOcMirrorCommandSuccess(result, err) + + By("mirroring from disk to sparse registry") + result, err = runner.DiskToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-full.yaml"), + workDir, sparseRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying only linux/amd64 and linux/arm64 blobs are present for multi-platform-container") + expectOnlyPlatformBlobsPresent(ctx, sparseRegistry, + "rh_ee_aguidi/multi-platform-container", "latest", + []string{"linux/amd64", "linux/arm64"}) + + By("verifying operator catalog and helm repositories exist") + expectRepositoriesExist(*sparseRegistry, []string{"oc-mirror/oc-mirror-dev", "podinfo"}) + }) + }) + + Context("without platforms filter (backward compat)", func() { + It("should preserve all blobs through disk archive", func() { + By("mirroring to disk without platforms filter") + result, err := runner.MirrorToDisk(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-no-platforms.yaml"), + workDir, "--remove-signatures=true") + expectOcMirrorCommandSuccess(result, err) + + By("mirroring from disk to registry") + result, err = runner.DiskToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-sparse-no-platforms.yaml"), + workDir, testRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying all platform blobs are present for multi-platform-container") + expectAllPlatformBlobsPresent(ctx, testRegistry, + "rh_ee_aguidi/multi-platform-container", "latest") + + By("verifying the release image was mirrored") + expectRepositoriesExist(*testRegistry, []string{"openshift/release-images"}) + }) + }) + }) +}) + +var _ = Describe("deprecated platform.architectures field", Label("sparse-manifest"), func() { + var workDir string + + BeforeEach(func() { workDir = setupWorkDir() }) + AfterEach(func() { cleanupWorkDir(workDir) }) + + It("should mirror successfully when architectures includes 'multi'", func() { + By("mirroring with deprecated architectures: [multi, amd64]") + result, err := runner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-archs-multi.yaml"), + workDir, testRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying multi-platform-container repository exists") + expectRepositoriesExist(*testRegistry, []string{"rh_ee_aguidi/multi-platform-container"}) + }) +}) + +var _ = Describe("negative", Label("sparse-manifest"), func() { + var workDir string + + BeforeEach(func() { workDir = setupWorkDir() }) + AfterEach(func() { cleanupWorkDir(workDir) }) + + Context("strict registry rejects sparse manifest", func() { + It("should mirror with sparse filtering even to a standard registry (no platforms:none required for push)", func() { + By("mirroring with amd64-only filter to a standard registry") + result, err := runner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-additional-amd64.yaml"), + workDir, testRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + // The in-process distribution registry v3.1.0 does not enforce manifest index + // blob presence by default, so the push succeeds. Strict validation only applies + // when running a real registry with platforms:none disabled — tested in the manual plan. + expectOcMirrorCommandSuccess(result, err) + }) + }) +}) + +var _ = Describe("deprecated platform.architectures field with release channel", Label("sparse-manifest"), Ordered, func() { + var workDir string + var archRunner *ocmirror.Runner + var cincinnatiServer *httptest.Server + + BeforeAll(func() { + workDir = setupWorkDir() + // Mock Cincinnati server: returns different release payloads depending on ?arch= + cincinnatiServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Accept") != "application/json" { + w.WriteHeader(http.StatusNotAcceptable) + return + } + arch := r.URL.Query().Get("arch") + var payload string + switch arch { + case "multi": + // Use digest reference so GenerateReleaseSignatures can find the GPG signature + payload = "quay.io/oc-mirror/release/test-release-index-multi@sha256:e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83" + default: // amd64 and others — use digest reference for GPG signature lookup + payload = "quay.io/oc-mirror/release/test-release-index@sha256:f81792339c8b5934191d18a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b" + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"nodes":[{"version":"4.18.1","payload":"%s"}],"edges":[]}`, payload) + })) + binaryPath := os.Getenv("OC_MIRROR_BINARY") + archRunner = ocmirror.NewRunner(binaryPath). + WithEnv([]string{fmt.Sprintf("UPDATE_URL_OVERRIDE=%s", cincinnatiServer.URL)}) + }) + + AfterAll(func() { + cincinnatiServer.Close() + cleanupWorkDir(workDir) + }) + + It("should make two Cincinnati calls (multi and amd64) and log deprecation warning", func() { + By("mirroring with deprecated architectures [multi, amd64] and a release channel") + result, err := archRunner.MirrorToMirror(ctx, + filepath.Join(iscDir, "sparse_manifest", "isc-archs-multi-release.yaml"), + workDir, testRegistry.Endpoint(), + "--remove-signatures=true", "--dest-tls-verify=false") + expectOcMirrorCommandSuccess(result, err) + + By("verifying the deprecation warning was printed") + Expect(result.Stdout + result.Stderr).To(ContainSubstring( + "platform.architectures is deprecated")) + + By("verifying both release payloads and the additional image are in the registry") + expectRepositoriesExist(*testRegistry, []string{ + "openshift/release-images", + "rh_ee_aguidi/multi-platform-container", + }) + }) +}) diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-amd64.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-amd64.yaml new file mode 100644 index 000000000..5ac909ed9 --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-amd64.yaml @@ -0,0 +1,8 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest + platforms: + - os: linux + architecture: amd64 diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-no-platforms.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-no-platforms.yaml new file mode 100644 index 000000000..ce9843034 --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-additional-no-platforms.yaml @@ -0,0 +1,5 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi-release.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi-release.yaml new file mode 100644 index 000000000..242990690 --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi-release.yaml @@ -0,0 +1,13 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + platform: + architectures: + - multi + - amd64 + channels: + - name: stable-4.18 + minVersion: 4.18.1 + maxVersion: 4.18.1 + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi.yaml new file mode 100644 index 000000000..505690e6b --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-archs-multi.yaml @@ -0,0 +1,9 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + platform: + architectures: + - multi + - amd64 + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-platforms-and-archs.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-platforms-and-archs.yaml new file mode 100644 index 000000000..1da4e7561 --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-platforms-and-archs.yaml @@ -0,0 +1,17 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest + platforms: + - os: linux + architecture: amd64 + platform: + platforms: + - os: linux + architecture: amd64 + architectures: [arm64] + channels: + - name: stable-4.18 + minVersion: 4.18.1 + maxVersion: 4.18.1 diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-full.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-full.yaml new file mode 100644 index 000000000..6e8097ab0 --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-full.yaml @@ -0,0 +1,37 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + platform: + graph: false + release: quay.io/oc-mirror/release/test-release-index-multi:v0.0.1 + platforms: + - os: linux + architecture: amd64 + - os: linux + architecture: arm64 + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest + platforms: + - os: linux + architecture: amd64 + - os: linux + architecture: arm64 + - name: quay.io/rh_ee_aguidi/empty-image:latest + operators: + - catalog: quay.io/oc-mirror/oc-mirror-dev:test-catalog-latest + packages: + - name: foo + - name: bar + platforms: + - os: linux + architecture: amd64 + helm: + repositories: + - name: podinfo + url: https://stefanprodan.github.io/podinfo + charts: + - name: podinfo + version: 5.0.0 + platforms: + - os: linux + architecture: amd64 diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-mixed.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-mixed.yaml new file mode 100644 index 000000000..94a9588ab --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-mixed.yaml @@ -0,0 +1,19 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + platform: + graph: false + release: quay.io/oc-mirror/release/test-release-index-multi:v0.0.1 + platforms: + - os: linux + architecture: amd64 + - os: linux + architecture: arm64 + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest + platforms: + - os: linux + architecture: amd64 + - os: linux + architecture: arm64 + - name: quay.io/rh_ee_aguidi/empty-image:latest diff --git a/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-no-platforms.yaml b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-no-platforms.yaml new file mode 100644 index 000000000..affcb0d29 --- /dev/null +++ b/tests/integration/testdata/imagesetconfigs/sparse_manifest/isc-sparse-no-platforms.yaml @@ -0,0 +1,9 @@ +kind: ImageSetConfiguration +apiVersion: mirror.openshift.io/v2alpha1 +mirror: + platform: + graph: false + release: quay.io/oc-mirror/release/test-release-index:v0.0.1 + additionalImages: + - name: quay.io/rh_ee_aguidi/multi-platform-container:latest + - name: quay.io/rh_ee_aguidi/empty-image:latest diff --git a/tests/integration/testdata/keys/release-pk.asc b/tests/integration/testdata/keys/release-pk.asc index bc453af78..091c1e39a 100644 --- a/tests/integration/testdata/keys/release-pk.asc +++ b/tests/integration/testdata/keys/release-pk.asc @@ -1,29 +1,29 @@ -----BEGIN PGP PUBLIC KEY BLOCK----- -mQINBGnOP5sBEACvf2g92MtrmCMO4rLqegkbOe3wFAD75w2TdcLgdqbrvoVNZPN7 -BJV/AVk02enPSt57EXxoO+p2XFCpmoGQ2Og7qqiJ1Rkoh5fbaKdeVPI3ORXRhMDf -h+aiI1XopdozmQ4b3iGqkrBw3xW6gIUwMc0QSYCf6kyJNLuXhzm5G3xp7PoDiCVv -KQH0/wcY4f+G+bAiq9P8pmLN2i4vNMKo58DDClaLU0WY+epsrh+Dbv7pz0ER/yrp -tZS7hVM16w9Q5Mc1LFLO4tTqWpnNFvjs/C1ITnz/m2cHArG/VySztg0JbJi/Bxw4 -ULz39F0hNNSoGjUWtEG21SvnCnpZZdeAJMPpb6ERZGfNmVQBEmzP+KBofny66DEI -D/tBg6LeEwSrHgsTmawE6SO/w4eSeoMXfIe119bwzj92AuHyFKtBo9LggQHVRc+j -EJX8o0C9h1+ioohkDUZUugKeF3MSEKdW7aF5xveFWi5GroPC4yb0SWu04xTzNWNi -ooAVouI4mv0QgsDi9Hi8lK5KHONrCLTurfwsZkAp7jLWruApwPc5YFMbSTafF3X/ -FCXTaZlMhztxs5MHpLfCDU4o84ye025cVeZZGJc73vVLvF8VJh+PPDbx/pfh8+sZ -Yzz4GZFP4pYywqizOe9PgE//zvKfrWU5nI1aHzddfPFpCrFS/TxirSd3KwARAQAB +mQINBGpaEBIBEADCYhe4nf/NnfNwMkGfwQPrOXqTdN4zt8bCzADIWXZSuyarBEdQ +K5CeYxQmWVtPjCk6jQU4p4OopTS+UmXUXU1pm01ken6jiXmbjLVu2cJItvHxa6B6 +Kt2jbmxzfL/o4FeMgtFGYbsicYSLPy6Np6VAkZzNvxHedgjDLy8+Nag3eelD2JWl +qN8sXL2xzWJu6Dx1EUQyv5eG601nIRmpw/3IZYWg2Urw98YMG00IA+EeQGhzXZQ4 +hskYFF+Q7bRdGL07SbMv6RN/m5p9eQnjGPLz73iAZfd5N3v6T/f2mdLGW2pVyjsF +b7Dkdn/q1+pcreKMJctryYXE1rmHJvP4eH/w2eWh0vl/cKeH87Xq7FjNfpEXXefq +6tAW8XtzCEY6fL8Wgk0DgFbaYWdIqngX+N/XSjWHwgSD3VTNSC+Gb2DIbxYNi4vt +LrSrKifjN2NyokHqfvzoHZBl8+KlAB7OtcipVqUqaLZSeJm2L6/goNCQG5VvqRT6 +ConctExIAc49vYJVeqeeHYIUDqz6AlGV2MnVDFGTwn2CSY/VlWdSInyhqVRnfQ8L +laqgV38p3dj0apWqY3qe1VvpHOCZExYjnfSr3RUO9h2je9F6Gi72o4ZOmNLZCCeR +rborQUXlEwrnHat+ZNJqpiT33nVCnfbLK+kihlG9sKggIg6vHxu0AZb3uwARAQAB tCdvYy1taXJyb3IgUmVsZWFzZSBUZXN0IDxyb2JvdEB0ZXN0LmNvbT6JAlEEEwEK -ADsWIQTJNjGZurFS25ebUF+ldt2a9Y2GMQUCac4/mwIbAwULCQgHAgIiAgYVCgkI -CwIEFgIDAQIeBwIXgAAKCRCldt2a9Y2GMYC2D/9rP7Dv3JkflnTp7K+LJMeohLHA -aR3S1MiAiz8fDGWN4cz/OVmUhMv5q7q3zwP5Ivux18QzH7CPKPBTTxHOUmUBICLm -lkObLW8Lrk3MqoqbeRpM3kUMZLZLPGqNPPRJlr4SK5ak92XZJQZ0xLBblmmPiqNE -6zIIEkdWHc2hAHiTDTctUBO7NLMUXJRxhp+xDW5Tuam/PKjYJ4UIT/2EeoWb+gUx -k6oRDMDUnaXWr92VyhJcEEzEHI7FoJaoXzxH8yvK4RFclhc/1WloriN8iU0IK8QD -zOhmWeCDM5Djxz0HAxEu+DuQ4sQ6qRonzvjWHmKfBwIEvkR7LTqR0qH2DGECYBhZ -G+Yub10CfFLNz+aKw8A6te/ms/oMCbb4/s5uRIG7XyCUKx3Tri59inBt/sALDdIh -uI/TJqQZ35wS7jUMsL87Ee/BZiRZW6cMVC9QHO4Gu6U7VIyHUGbnZnV6wHaI2D8l -D7lQwNLXZL876S4i9KI2IheEfwWBNU9mFp3Giyh1lFdSaoiNVJFVwETF0bWxXtCR -+vFfPbsuQrCz65AxHl+tgA3RnCO4EItJ4eoGnm/uo6tRa8iANKx0nFcISi2Piipz -5ledtBXYN/Be5Ei9ASNCxvKki8o6X+WGDhB2qI0dmUSSAz485Ykz7++hrqfI4+X6 -d4lZiUlY/UKrPHlMsw== -=ZVO9 +ADsWIQSGmkqy5QIlOlUVNKf5ltNto5RokgUCaloQEgIbAwULCQgHAgIiAgYVCgkI +CwIEFgIDAQIeBwIXgAAKCRD5ltNto5RokmKGEACoIT4+zDU2qZPsT+Mh0XUiwpM9 +RkRS2AM4x9Fp5DcN6UsHP51IaATlM1K68TliBKI6RvMtYGhizQ06ZH8wVWtSFUsC +lI/9ujZ9cUpwUKMQAK4zSlIJ1LnqA7drpzlSy9PjK2cckiwgF3HzbKqkmTMYWvON +XlxY+8noDH5eOG4ALgVR4lEsYZFayFEpAmQckVV9eMfMrU1b00x7gd7PS/a6yFlk +JlwPbhvhb5jucZUuH4jV/hVTaUJ1zweLEzRkH66q1d5p+3gIBvMWWMe4X9ZGZnGb +tNiQf2fcwNi9Z8mHqZHOOfCxGRASdAvBuXKqIjWOnkp5h66bEYssuGK88Lywq2Do +OhTVSLJofc1jrXgPm1J0qcMghCpMg3/8OTDUxaKD+oYyggpa/O2E3dRRBsly4s6D +Oo1Mf+stGEP/Od+P3J00puPWfeK78zrKKxDF5JX8du0lgP4wusj/PG6mtDdO560g +7oq2ozay02/ajwdJpVknlnOR9eeZa4piU+hP4oUFQVeVdQ1nG7X9FspPu50Pg8hS +be2TgsG4SxMOfkzT46v1bMZAW64ScaipA8xegGKyUA24VanimgFqYdYuiTiOyoKR +GjWyA5nMEvuZgis+I0RQO4Rq0m7CL/7JIdZsRODk12FVuyce8bCLu+Lreg6s6z+z +7gzfP9SLEHQQ4dlZrg== +=h3in -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/integration/testdata/keys/v0.0.1-sha256-e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83 b/tests/integration/testdata/keys/v0.0.1-sha256-e8a12d3d6d3cd3fca4f1e9afe3e4cebd16fc590b73962be64e083199e8ea6f83 new file mode 100644 index 0000000000000000000000000000000000000000..9c4889ae012d0ec4fde7b63c2863f3334fe5af47 GIT binary patch literal 863 zcmV-l1EBn)0h_?f%)rI?bK2$H#ZxjSF`ho)pTv+AB_LX@lw6cql9`;CqhwXBl$ny6 zSCUy$31X+@CugS?=@zA?r52^;C8sJ`DHWC`R_bNu>*puy=4KWZ8VHt=O*T5rh&AjWTvMUmnc~& z6=x(GnVMOpS|l18r5LA}r5Got7^fvCnxq+~S|+BY8mF2hrzWKsnx!S1S{fvo8(W$g zC8e5~q#9Tl8(LbXTBIhLrCAt*EGwxj0693ZBtJJZSs^(;uOu-uFSSUaI5Ryjv81#p z6{IM?pd>Rt5A5^W+D;~C76wKx23;{lmbO`5o1QYMT7`<5EQbXdD-$TV_+^Unlk!U( zK>pQB&d+7w;uL_XNDLBS=$qL%vFgE&BVmTI`F#0J4`ea~y%%2HyeFhd<>!S(1qJgb z&W^YLS)1%JrGlj{YZ_DW9F`ibigvTBkABEWOl7y+r0>o5kKL>2Q~r^@m*LV2mU6As zFKCyu$xu+MdD!#)<`So?$G0nO6u$6N;%V+i&Dgh_4#oH-eYtb@)Z#TRrv-{bZhTZp z+*HmSSG?#%hGpxl^cPze$nI`V{c%sopqeXj^<0B13zyfe@p?2%Rjj%!`2Q2{-8yXi zo$a$To$}>2@SmHo&1v&BxxJg>ZWZMzZ7lIpE4?yP`jd6fnd|eqyQ&{f%9(8Tz4YL% zg@>=d}{j{ohFKCwPEku{*5eag`p p`SadRo1l=p;&82?@SA%w->=S5RC_I^G-;2PW_e8ReBq@L?*RFWx8wi- literal 0 HcmV?d00001 diff --git a/tests/integration/testdata/keys/v0.0.1-sha256-f81792339c8b5934191d18a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b b/tests/integration/testdata/keys/v0.0.1-sha256-f81792339c8b5934191d18a53b18bc1d584e01a9f37d59c0aa6905b00200aa1b index e70d72a6d5da2a72c365645853aa7072d131d241..0ee72c7c7037bc93771ddb16dd6f377fef4478a5 100644 GIT binary patch delta 624 zcmV-$0+0RK2G<50qXC=1%gn&V_;cFj+{IHeCNZ8q;F-jb6(t~AksS1Ywpm`Ao-(Og zg^HRihXocZ6DXkgWs35X@=F{*KGjRk&t>4^6o9FyC=p=T<~>8ZQNzUe?2gpqiF>$Q zSL|c9t2957AbBFnLW0}=t%PKsVB{SpSI*SlX}|t?D_&K(q;k@}zi58dDnkXo9ZU8F z>^4Y!6en!py|RvB>6?jv^8dnwcW20`bnadHt3+DVb&}lOkMFZI8XXRG`+oZHvYcFz2h?pGm^fP+s1uLN-SiC@k;s^gn(IPZ&p#h*W0N}oD1))&9D z>u!8;FWRg0)zZCJwXC#^>z5k&TPYOv~K?2v`p7UhuAW7`$Wz(an%)zJ*a)+ZScb;B6Kg)%YS+oVoKOnD=s)F zH1*T1|HeNGvrh1T#90T%i>&|9QP-JccgoEw;$>6A>X#RuI{)7vu|iit{JzKei*J8A z9Oh^I`lhwYsf9OS$c$95&Hlh6NfY&j3}or&RPhtw9Vy0g>kQhJ=uRpBfB zo!ogW9;?}JMbyu;-^rwLd!omkYyJyfE-<|mCv6k1Z}r7X<|@l~`(nv!-UqKaE}X0{ zQ{tfLH-FBO`_fGoE{<2-HtlENJ|G!?a^JH0{_}n_T3&5lWXdwbK|lEF*3>0MjVDU> K-fP=;hz|fKR#%k( delta 626 zcmV-&0*(FG2H6H2qXC=1%gn&VxU}rFH@AClwaZi@~K{Oel7zSrvOaF(lq{m+NJ>=;Oe^V$`OB_%7gTDrUY#)gfWM6RJJ8n5XuKvO22Ba?N^_#JOuC#8=+3Ek>w56A& zv0nZyYqBPxS8&HOGnrp0r(gFtQ zDYsiLo_zlK{=1|7N!#9}Urd=R;?%v^x20>R#Ocf(_q?Ybo}^pMGXJmd+=sCV&8KcB z9XD@z*z+?;m7C>%+k^Gf4rlE4cbfC2^;vvhZG+3+Qr}+Nmi5AWO*v*AnHK5y@e!-T zj0LB3`Ly?D=&oDIQ*g>?+FQmXm+7|Me}De+H8XsCw_5AZ-nTndRu*4bd^ziApAMh>Sm5+I2gOZ_P%B4?#b~3k&8?&gIoTXu#8c<3*J8O#9c%wj`JN M2IMe`-